爬取雪球網案例

import pymysql

class mysql_conn(object):
# 魔術方法，初始化，析構函式
def __init__(self):
self.db = pymysql.connect("localhost","root","******","xueqiu__text")
# 建立游標物件
self.cursor =self.db.cursor()
# 執行mysql語句
def execute__mysql(self,k):
self.cursor.execute(k)
self.db.commit()
# 魔術方法 使用完物件**資源
def __del__(self):
self.cursor.close()
self.db.close()
if __name__ == '__main__':
# sql = 'insert into zhang(id,age) values (3,2)'
sql = "insert into zhang(uid,title,target,description) values (1,'df','ds','dds')"
mc = mysql_conn()
mc.execute__mysql(sql)

#1

# # next_id:184263
# next_max_id:184275
# tip:null
## #2
# # next_id:184082
# next_max_id:184086
# tip:null##
# #3
# # next_id:183682
# next_max_id:183687
# tip:null

import requests

import json
# 呼叫上面封裝好的執行資料庫新增
from mysql__text import mysql_conn
#1 把url路徑拼接出來
#2 把每個ajax裡面的資訊拿出來
i = 1 #代表一次ajax
max_id = -1
count = 10
while i <=15:
url = ''.format(str(max_id),str(count))
headers = 
response = requests.get(url, headers=headers)
#列印一下看是否獲取到資料
info = response.content
# print(type(info),info)
# bytes 轉換成函式 json.loads物件必須是字串型別，response.text是字串型別
res__dict = json.loads(response.text)
max_id = res__dict['next_max_id']
for j in range(count):
data = json.loads(res__dict['list'][j]['data'])
# print(j)
uid = data['id']
title = data['title']
target = data['target']
description = data['description']
print(description)
# mc = mysql__text()
sql = "insert into zhang(uid,title,target,description) values ('{}','{}','{}','{}')".format(uid,title,target,description)
mc1 = mysql_conn()
mc1.execute__mysql(sql)
# target
# description
# # 取出資料
# res = res__dict['list']
# print(res)
## for i in res:
# result = i['data']
# print(result)
i += 1
count = 15

雪球網爬取

import json import requests import pymysql mysql coon 主要的功能就是,將鏈結資料庫的操作變成只連線一次 class mysql conn object 魔術方法,初始化,建構函式 def init self self.db pymysql.con...

雪球網爬取資料並存入資料庫

from urllib import request import json import pymysql class mysql connect object 初始化的建構函式 def init self self.db pymysql.connect host 127.0.0.1 user ro...

爬蟲day3 （爬取雪球網n頁資料）

爬取雪球網n頁資料 用到 與mysql資料庫的互動 import requests import json import pymysql class mysql conn object 魔術方法,初始化,建構函式 def init self self.db pymysql.connect host ...