cifar官網給出的python介面的檔案都是用python cpickle工具」pickled」的，可以看見 cifar 官網給出的例程是：
python 2
def
unpickle
(file):
import cpickle
with open(file, 'rb') as fo:
dict = cpickle.load(fo)
return dict
def
unpickle
(file):
import pickle
with open(file, 'rb') as fo:
dict = pickle.load(fo, encoding='bytes')
return dict
import os
import pickle
import numpy as np
import sklearn
import sklearn.linear_model
import lmdb
import caffe
defunpickle
(file):
fo = open(file, 'rb')
dict = pickle.load(fo,encoding ='bytes')
fo.close()
return dict
# 呼叫sklearn對資料進行shuffle操作
defshuffle_data
(data, labels):
data, _, labels, _ = sklearn.cross_validation.train_test_split(
data, labels, test_size=0.0, random_state=42
)return data, labels
defload_data
(train_file):
d = unpickle(train_file)
#dict_keys([b'batch_label', b'filenames', b'data', b'coarse_labels', b'fine_labels'])，每個鍵值前面都有乙個b，不同於 python2
data = d[b'data']
fine_labels = d[b'fine_labels']
length = len(d[b'fine_labels'])
data, labels = shuffle_data(
data,
np.array(fine_labels)
)return (
data.reshape(length, 3, 32, 32),
labels
)if __name__ == '__main__':
# 解壓後的 cifar-100-python 路徑
cifar_python_directory = os.path.abspath(r'f:\software_download\chromedownload\cifar-100-python.tar\cifar-100-python')
print('converting...')
cifar_caffe_directory = os.path.abspath('cifar100_train_lmdb')
ifnot os.path.exists(cifar_caffe_directory):
x, y_f = load_data(os.path.join(cifar_python_directory, 'train'))
xt, yt_f = load_data(os.path.join(cifar_python_directory, 'test'))
print('data is fully loaded,now truly convertung.')
# lmdb操作,將資料寫入資料庫
env = lmdb.open(cifar_caffe_directory, map_size=50000 * 1000 * 5)
txn = env.begin(write=true)
count = 0
for i in range(x.shape[0]):
datum = caffe.io.array_to_datum(x[i], y_f[i])
str_id = ''.format(count)
# txn.put(str_id, datum.serializetostring())
txn.put(str_id.encode('ascii'), datum.serializetostring())
count += 1
if count % 1000 == 0:
print('already handled with {} pictures'.format(count))
txn.commit()
txn = env.begin(write=true)
txn.commit()
env.close()
env = lmdb.open('cifar100_test_lmdb', map_size=10000 * 1000 * 5)
txn = env.begin(write=true)
count = 0
for i in range(xt.shape[0]):
datum = caffe.io.array_to_datum(xt[i], yt_f[i])
str_id = ''.format(count)
# python 3 在 str_id 後多了乙個 .encode('ascii')
txn.put(str_id.encode('ascii'), datum.serializetostring())
count += 1
if count % 1000 == 0:
print('already handled with {} pictures'.format(count))
txn.commit()
txn = env.begin(write=true)
txn.commit()
env.close()
else:
print('conversion was already done. ')
參考部落格：
python3換源 Python3 換源
pip安裝源 介紹2 常用pip源 豆瓣 阿里 3 加速安裝的命令 pip install i 模組名 永久配置安裝源 windows 2 新建 pip 資料夾並在資料夾中新建 pip.ini 配置檔案 3 新增 pip.ini 配置檔案內容 macos linux 1 在使用者根目錄下 下建立 p...
python換源 python3 換源
1.原因 pip是很強大的模組安裝工具，但是由於國外官方pypi經常被牆，導致不可用。所以我們最好是更換pip源，這樣就能解決被牆導致的裝不上庫的問題。2.可用源 網上有很多可用的源 豆瓣 清華 清華大學的pip源，它是官網pypi的映象，每隔5分鐘同步一次，推薦使用。3.使用 3.1 臨時使用 可...
20141010 轉換習題1
練習一 要求 在控制台中人工輸入任意十個數值，最後求十個數值的合。首先要將輸入的數值定義 1 整型定義 以int為例 int a convert.toint console.readline 2 浮點定義以double為例 double a convert.todouble console.read...