[Data Crawling] Spongebob - 1

spongebob.fandom.com/wiki/Encyclopedia_SpongeBobia

Encyclopedia SpongeBobia

스폰지밥의 대사를 추출하기 위해 Season별 타이틀과 그 대사가 담긴 사이트를 크롤링한다.

import re
import pandas as pd
from urllib.request import urlopen
import glob

pd.read_html로 데이터 추출

o_site = 'https://spongebob.fandom.com/wiki/List_of_transcripts'
season1 = pd.read_html(o_site,header=0)[0]
season1.columns # '#', 'Title', 'Transcript'

season13 까지의 제목과 주소를 담는다. 

for i in range(13):
    total_site = []
    site = pd.read_html(o_site,header=0)[i]
    for title in site.Title:
        temp = {'title':title,
                'addr':str('https://spongebob.fandom.com/wiki/{}/transcript'.format(re.sub(' ','_',title)))}
        total_site.append(temp)
    globals()['season'+str(i+1)] = pd.DataFrame(total_site)

season 별 제목과 내용 추출 후 text로 저장

from bs4 import BeautifulSoup
import requests

for season in range(13):
    season = globals()['season'+str(season+1)]
    for title,addr in zip(season.title,season.addr):
        html = requests.get(addr).text
        soup = BeautifulSoup(html, 'html.parser')
        text = []
        for ea in soup.select('.mw-parser-output > ul'):
            text.append(ea.text)
        with open('{}.txt'.format(re.sub('\?','',str(title).replace(' ','_'))),'w') as f:
            for line in text:
                f.write(line)