python資料批量插入postgreSQL資料庫

1、executemany()方法批量輸入資料到資料庫

import pandas as pd

import psycopg2
from dbutils.pooleddb import pooleddb
import math
'''批量插入 
data：為dataframe資料，size：為批量大小
sql示例： "insert into table(username,password,userid) values(%s,%s,%s)"
'''def
batchinsert
(sql, data, size)
:try
: psycopg_pool = pooleddb(psycopg2, mincached=
5, blocking=
true
, user=
'postgres'
, password=
'postgres'
, database=
'postgres'
, host=
'***.***.xx.xx'
, port=
'5432'
) connection = psycopg_pool.connection(
) cursor = connection.cursor(
)except exception as e:
print
(e)try
: cycles = math.ceil(data.shape[0]
/ size)
for i in
range
(cycles)
: val = data[i * size:
(i +1)
* size]
.values
cursor.executemany(sql, val)
connection.commit(
)except exception as e:
print
(e) connection.rollback(
)finally
: connection.close(
)def
insert
(data, table_name, size)
:##data is a dataframe
try:
table_columns =
str(
list
(pd.dataframe(data.columns)
.astype(
str)[0
]))[
1:-1
].replace(
"'","")
##獲取欄位名 
table_values =
'%s'
for i in
range(0
,len
(data.columns)-1
):table_values = table_values +
','+
'%s'
sql =
"insert into {}({}) values ({})"
.format
(table_name, table_columns, table_values)
batchinsert(sql, data, size)
except exception as e:
print
(e)

from sqlalchemy import create_engine

result = pd.dataframe(data)
engine = create_engine(
'postgresql://user:password@host:port/database'
)pd.io.sql.to_sql(result, table_name, engine, index =
false
, if_exists=
)#增量入庫
pd.io.sql.to_sql(result, table_name, engine, index =
false
, if_exists=
'replace'
)#覆蓋入庫

import psycopg2

from sqlalchemy import create_engine
import pandas as pd
from io import stringio
data1 = pd.dataframe(data)
#dataframe型別轉換為io緩衝區中的str型別
output = stringio(
)data1.to_csv(output, sep=
'\t'
, index=
false
, header=
false
)output1 = output.getvalue() 
conn = psycopg2.connect(host=***
, user=***
, password=***
, database=***
)cur = conn.cursor(
)cur.copy_from(stringio(output1)
, table_name1)
conn.commit(
)cur.close(
)conn.close(
)print
('done'
)

批量插入資料

drop procedure if exists pre 如果存在先刪除 delimiter 以delimiter來標記用 表示儲存過程結束 create procedure pre 建立pre 儲存方法 begin declare i int default 12 declare n int de...

python 往MySQL批量插入資料

在工作用有時候需要批量造測試資料 手工造太麻煩了，可以通過python批量插入表資料 批量插入sql語句 import pymysql,string,random,time defconnet mysql try db pymysql.connect host 192.168.31.103 user...

JDBC批量插入資料

一.直接上源 以匯入課表資料為例 批量插入新申請 匯入預設課表 throws sqlexception catch sqlexception e int number ps.executebatch 把剩餘的資料插入 conn.commit 手動提交事務 try catch sqlexception...