39 lines
1.2 KiB
Python
Executable File
39 lines
1.2 KiB
Python
Executable File
import os
|
|
import pickle
|
|
from io import BytesIO
|
|
import argparse
|
|
|
|
import lmdb
|
|
from data.dataset import CARS, ImprovedImageFolder
|
|
from tqdm import tqdm
|
|
|
|
|
|
def content_loader(path):
|
|
with open(path, "rb") as f:
|
|
return f.read()
|
|
|
|
|
|
def dataset_to_lmdb(dataset, lmdb_path):
|
|
env = lmdb.open(lmdb_path, map_size=1099511627776*2, subdir=os.path.isdir(lmdb_path))
|
|
with env.begin(write=True) as txn:
|
|
for i in tqdm(range(len(dataset)), ncols=50):
|
|
txn.put("{}".format(i).encode(), bytearray(dataset[i]))
|
|
txn.put(b"classes_list", pickle.dumps(dataset.classes_list))
|
|
txn.put(b"__len__", pickle.dumps(len(dataset)))
|
|
|
|
|
|
def transform(save_path, dataset_path):
|
|
print(save_path, dataset_path)
|
|
# origin_dataset = CARS("/data/few-shot/STANFORD-CARS/", loader=content_loader)
|
|
origin_dataset = ImprovedImageFolder(dataset_path, loader=content_loader)
|
|
dataset_to_lmdb(origin_dataset, save_path)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser(description="transform dataset to lmdb database")
|
|
parser.add_argument('--save', required=True)
|
|
parser.add_argument('--dataset', required=True)
|
|
args = parser.parse_args()
|
|
transform(args.save, args.dataset)
|
|
|