(1) 크롤링

• 크롤링(Crawling): 인터넷의 데이터를 활용하기 위해 정보들을 분석하고 활용할 수 있도록 수집하는 행위
• 스크레이핑(Scraping): 크롤링 + 데이터를 추출해서 가공하는 최종 목표

1. Basic English Speaking

import requests

from bs4 import BeautifulSoup as bs

 

site = 'https://basicenglishspeaking.com/daily-english-conversation-topics/'

request = requests.get(site)

print(request)

# print(request.text)

soup = BeautifulSoup(request.text)

 

divs = soup.find('div', {'class':'thrv-columns'})

divs

links = divs.findAll('a')

print(links)

for link in links:

  print(link.text)

subject = []

for link in links:

  subject.append(link.text)

print(subject)

len(subject)

print('총', len(subject),'개의 주제를 찾았습니다')

for i in range(len(subject)):

  print('{0:2d}, {1:s}'.format(i+1, subject[i]))

2. 다음 뉴스기사

# https://v.daum.net/v/20230601093610464

# https://v.daum.net/v/20230601072518640

def daum_news_title(news_id):

  url = 'https://v.daum.net/v/{}'.format(news_id)

  request = requests.get(url)

  soup = bs(request.text)

  title = soup.find('h3',{'class':'tit_view'})

  if title:

    return title.text.strip()

  return '제목없음'

daum_news_title(20230601093610464)

daum_news_title(20230601072518640)

3. 벅스 뮤직 차트

request = requests.get('https://music.bugs.co.kr/chart')

soup = bs(request.text)

titles = soup.findAll('p',{'class':'title'})

artists = soup.findAll('p',{'class':'artist'})

for i,(t, a) in enumerate(zip(titles, artists)):

  title = t.text.strip()

  artist = a.text.strip().split('\n')[0]

  print('{0:3d}위 {1} - {2}'.format(i+1, artist, title))

4. 멜론 차트

• https://www.melon.com/chart/index.htm
• robots.txt: 웹 사이트에 크롤러같은 로봇들의 접근을 제어하기 위한 규약, 권고안이라 꼭 지킬 의무는 없음

request = requests.get('https://www.melon.com/chart/index.htm')

print(request) # <Response [406]>

# User-Agent:

# Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36

header = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'}

request = requests.get('https://www.melon.com/chart/index.htm', headers = header)

print(request)

soup = bs(request.text)

titles = soup.findAll('div',{'class':'rank01'})

artists = soup.findAll('span',{'class':'checkEllipsis'})

for i,(t, a) in enumerate(zip(titles, artists)):

  title = t.text.strip()

  artist = a.text.strip().split('\n')[0]

  print('{0:3d}위 {1} - {2}'.format(i+1, title, artist))

5. 네이버 금융

# https://finance.naver.com/item/main.naver?code=005930

site = 'https://finance.naver.com/item/main.naver?code=005930'

request = requests.get(site)

 

soup = bs(request.text)

 

div_today = soup.find('div',{'class':'today'})

print(div_today)

em = div_today.find('em')

print(em)

price = em.find('span',{'class':'blind'})

print(print(price.text))

def naver_finance(code):

    site = f'https://finance.naver.com/item/main.naver?code={code}'

    request = requests.get(site)

    soup = bs(request.text)

    div_today = soup.find('div', {'class':'today'})

    em = div_today.find('em')

    price = em.find('span', {'class':'blind'}).text # 가격

    wrap_company = soup.find('div', {'class':'wrap_company'})

    name = wrap_company.a.text  # 회사명

    div_description = wrap_company.find('div', {'class':'description'})

    code = div_description.span.text # 코드

    table_no_info = soup.find('table', {'class':'no_info'})

    tds = table_no_info.find_all('td')

    volume = tds[2].find('span', {'class':'blind'}).text # 거래량

    dic = {'price':price, 'name':name, 'code':code, 'volume':volume}

    return dic

data = []

for code in codes:

  dic = naver_finance(code)

  data.append(dic)

print(data)

import pandas as pd

 

df = pd.DataFrame(data)

df

df.to_excel('naver_finance.xlsx')

# 지니 차트 200위 까지 크롤링

# 반복문을 사용하여 크롤링

# 수집한 데이터를 데이터프레임으로 만들고

# 엑셀로 출력

# 과제는 깃허브에 제출

import requests

from bs4 import BeautifulSoup as bs

import pandas as pd

     

data = []

num = 1

for j in range(1,5):

  site = f'https://www.genie.co.kr/chart/top200?ditc=D&ymd=20230601&hh=13&rtm=Y&pg={j}'

  header = {"User-Agent":

  "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}

  request = requests.get(site, headers=header)

  soup = bs(request.text)

  tds = soup.findAll('td', {'class':'info'})

  for i in range(len(tds)):

    title = tds[i].find('a', {'class':'title ellipsis'}).text.strip()

    artist = tds[i].find('a', {'class':'artist ellipsis'}).text.strip()

    print('{0:3d}위 {1} - {2}'.format(num, title, artist))

    dic = {'순위':num, '타이틀':title, '가수':artist}

    data.append(dic)

    num += 1

df = pd.DataFrame(data)

df.to_excel('genie_music.xlsx')

site = 'https://www.genie.co.kr/chart/top200?ditc=D&ymd=20230601&hh=13&rtm=Y&pg=1'

header = {"User-Agent":

"Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}

request = requests.get(site, headers=header)

soup = bs(request.text)

tds = soup.findAll('td', {'class':'info'})

for i in range(len(tds)):

  title = tds[i].find('a', {'class':'title ellipsis'}).text.strip()

  artist = tds[i].find('a', {'class':'artist ellipsis'}).text.strip()

  print('{0:3d}위 {1} - {2}'.format(i+1, title, artist))