單純的從每個欄目去爬取是不顯示的，轉換一下思路，看到搜尋頁面，有時間~，有時間！
注意看url鏈結
字典生成器的語法是時候去複習一下了
import scrapy
from scrapy import request,selector
import time
import datetime
''''''
class blogsspider(scrapy.spider):
name = 'blogs'
allowed_domains = ['zzk.cnblogs.com']
start_urls = ['']
from_time = "2010-01-01"
end_time = "2010-01-01"
keywords = "python"
page =1
url = ""
custom_settings =
}def start_requests(self):
cookie_str = "想辦法自己獲取到"
self.cookies =
yield request(self.url.format(keywords=self.keywords,from_time=self.from_time,end_time=self.end_time,page=self.page),cookies=self.cookies,callback=self.parse)
def parse(self, response):
print("正在爬取",response.url)
count = int(response.css('#countofresults::text').extract_first()) # 獲取是否有資料
if count>0:
for page in range(1,int(count/10)+2):
# 抓取詳細資料
yield request(self.url.format(keywords=self.keywords,from_time=self.from_time,end_time=self.end_time,page=page),cookies=self.cookies,callback=self.parse_detail,dont_filter=true)
time.sleep(2)
# 跳轉下乙個日期
d = datetime.datetime.strptime(self.from_time, '%y-%m-%d')
delta = datetime.timedelta(days=1)
d = d + delta
self.from_time = d.strftime('%y-%m-%d')
self.end_time =self.from_time
yield request(
self.url.format(keywords=self.keywords, from_time=self.from_time, end_time=self.end_time, page=self.page),
cookies=self.cookies, callback=self.parse, dont_filter=true)
db.getcollection('dict').count({})372352條資料
def parse_detail(self,response):
items = response.xpath('//div[@class="searchitem"]')
for item in items:
title = item.xpath('h3[@class="searchitemtitle"]/a//text()').extract()
title = "".join(title)
author = item.xpath(".//span[@class='searchiteminfo-username']/a/text()").extract_first()
public_date = item.xpath(".//span[@class='searchiteminfo-publishdate']/text()").extract_first()
pv = item.xpath(".//span[@class='searchiteminfo-views']/text()").extract_first()
if pv:
pv = pv[3:-1]
url = item.xpath(".//span[@class='searchurl']/text()").extract_first()
#print(title,author,public_date,pv)
yield
Python爬蟲框架
一 u know！二 scrapy入門教程 三 網路爬蟲之scrapy框架詳解 四 scrapy編寫步驟 詳情見二 scrapy入門教程 1 mkdir乙個爬蟲資料夾 例 mkdir home zy pachong pa test 10 28 2 在 home zy pachong pa test ...
Python爬蟲 scrapy框架
開源的,對平台的爬蟲框架 舊版本 需要預先定義所需欄位 class myitem scrapy.item url scrapy.field 再將所需欄位填充 class myspier scrapy.spider defparse self,response return sudo apt inst...
python爬蟲scrapy框架
安裝 pip install scrapy startproject 建立乙個新專案 genspider 根據模板生成乙個新爬蟲 crawl 執行爬蟲 shell 啟動互動式抓取控制台 進入專案目錄 scrapy startproject crawlertest project name cd cr...