Python爬蟲 筆趣閣小說爬取

import requests

from lxml import etree

對目錄頁的每個章節的url進行爬取，分析網頁利用xpath定位每個章節的url然後進行爬取，然後重新構造url。

# 目錄每一章節的url

href = html_ele.
xpath
('//dd/a/@href'
)for i in href:
url =
''+ i print
(url)

# 章節標題

href1 = html_ele.
xpath
('//div[@class="box_con"]/div[@class="bookname"]/h1/text()'
) # 章節內容
href2 = html_ele.
xpath
('//div[@class="box_con"]/div[@id="content"]/text()'
)

file_name =

'我有百萬技能點.txt'
with
open
(file_name,
"ab"
)as f:
# 寫入章節標題
f.write
(href1[0]
.encode
('utf=8'))
# 寫入換行符
f.write
('\n'
.encode
('utf-8'))
# 引文爬取到的正文內容是每乙個p標籤的內容，所以要將href2遍 歷然後寫入txt文件
for i in href2:
print
(i) # 寫入正文內容
f.write
(i.encode
('utf=8'))
# 寫入換行符
f.write
('\n'
.encode
('utf-8'
))

import requests

from lxml import etree
url =
''# 設定請求頭
headers =
response = requests.
get(url,headers = headers)
response.encoding=
'gbk'
html_ele = etree.
html
(response.text)
# 目錄每一章節的url
href = html_ele.
xpath
('//dd/a/@href'
)for i in href:
url =
''+ i print
(url)
# 設定請求頭
headers =
response = requests.
get(url,headers = headers)
response.encoding=
'gbk'
html_ele = etree.
html
(response.text)
# 章節標題
href1 = html_ele.
xpath
('//div[@class="box_con"]/div[@class="bookname"]/h1/text()'
) # 章節內容
href2 = html_ele.
xpath
('//div[@class="box_con"]/div[@id="content"]/text()'
) file_name =
'我有百萬技能點.txt'
with
open
(file_name,
"ab"
)as f:
f.write
(href1[0]
.encode
('utf=8'))
f.write
('\n'
.encode
('utf-8'))
for i in href2:
print
(i) f.
write
(i.encode
('utf=8'))
f.write
('\n'
.encode
('utf-8'
))

初級爬蟲爬取筆趣閣小說

import requests from pyquery import pyquery as pq def get content a response requests.get a response.encoding gbk doc pq response.text text doc conten...

用爬蟲爬取筆趣閣小說

時間 2019年3月4日19 16 06 功能 爬取筆趣閣任何 from urllib import request from bs4 import beautifulsoup 此函式用來獲取每章對應的 並儲存 defsecondopenurl url,ch name 請求每章詳細內容 date r...

Python爬蟲練習二 爬取筆趣閣小說

爬取這個 真的很easy！很有成就感 適合爬蟲的初學者！以乙個叫 凡人修仙傳仙劍篇 的 作為目標進行爬取測試。廢話不多說，上 import requests from bs4 import beautifulsoup aimurl 爬蟲目標 url href前面的內容 輸入 獲得網頁的soup de...