python爬取糗事百科

import requests

from lxml import etree
from time import time,sleep

#3.解析**返回資訊

headers =
def response_list_sy(url_list):
print('正在請求:{}'.format(url_list))
response_list = requests.get(url_list,headers=headers)
return response_list.text
#4.獲取資訊 解析資料 獲取詳情頁url 組成列表返回
def parse_list(list_resp):
resp_html = etree.html(list_resp)
li_list = resp_html.xpath('//div/div/div/div/ul/li/div/a/@href')
list_url=
for li in li_list:
detail_url = ''.format(li)
return(list_url)
#5.解析詳情頁url 並解析 標題,時間,好笑度
def list_url_pin(date_url_list):
for u in date_url_list:
url_sp = requests.get(u,headers=headers)
result={}
res_html_list = etree.html(url_sp.text)
result['title'] = res_html_list.xpath('//div/div/div/h1/text()')[0].strip()
result['time1'] = res_html_list.xpath('//div/div/div/div/span/text()')[0].strip()
result['hxd'] = res_html_list.xpath('//div/div/div/div/span/i/text()')[0]
print(result)
if __name__ == '__main__':
s_time = time()
for i in range(10):
#1.請求進入**
url_list = ''.format(i+1)
#2.傳入類
list_resp = response_list_sy(url_list)
date_url_list = parse_list(list_resp)
lp_resp = list_url_pin(date_url_list)
print('單任務時,耗時{}'.format(time()-s_time))

python 爬取糗事百科

step 1 構建乙個提取糗事百科笑話的函式import urllib2 import urllib import re import thread import time import sys reload sys sys.setdefaultencoding utf 8 defgetpage p...

Python爬取糗事百科

一 引入模組 因為urlopen功能比較簡單，所以設定 ip需引入proxyhandler和build opener模組，ip的獲取可以上西祠 查詢 import re from urllib.request import request,build opener,proxyhandler base...

Python 爬取糗事百科

coding utf 8 import urllib2 import urllib import re class qiushi def init self self.page 1 從網頁獲取糗事 def getqiushis self,page url page 偽裝瀏覽器 user agent ...