【爬蟲】名言名句網站爬取

大致思路是從土味情話網上面爬取標題和內容，然後固定格式輸出。py文件保存到桌面，運行就行了。

import urllib.request import urllib.parse import re import time

url = http://www.yuluju.com/aiqingyulu/list_18
headers = {User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36,
Content-Type: application/x-www-form-urlencoded; charset=UTF-8}

def make_url(a,b):
a = a+"_"+str(b)+".html"
return a

def get_content(c):
request = urllib.request.Request(url = c,headers = headers)
content = urllib.request.urlopen(request).read().decode("gbk")
return content

def find_title(d):
pattern = re.compile(r<h2><a href="/aiqingyulu/d+.html" class="title" target="_blank">(.*?)</a></h2>)
title = pattern.findall(d)
return title

def find_sentence(e):
pattern = re.compile(r<h2><a href="(/aiqingyulu/d+.html)")
sentence = pattern.findall(e)
sentence_new = []
for sentence_url in sentence:
sentence_url = "http://m.yuluju.com"+sentence_url
sentence_new.append(sentence_url)
return sentence_new

def find_love(f):
pattern = re.compile(r<span stylex="font-size:14px;">(.*?)</span></div>)
love = pattern.findall(f)
return love

def main():
start_page = int(input("請輸入開始頁碼："))
end_page = int(input("請輸入結束解碼："))
for page in range(start_page,end_page+1):
new_url = make_url(url,page)
content = get_content(new_url)
title = find_title(content)
for m in range(len(title)):
print("
")
print(title[m])
print(正在下載…………)
time.sleep(2)
sentence_new = find_sentence(content)
sentence_content = get_content(sentence_new[m])
love = find_love(sentence_content)
for i in love:
print(i)
print("
")
time.sleep(2)