- Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathPreMyData.py
More file actions
Latest commit
142 lines (121 loc) · 4.96 KB
/
Copy pathPreMyData.py
File metadata and controls
142 lines (121 loc) · 4.96 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
importos
importzipfile
importrandom
importjson
importnumpyasnp
fromPILimportImage
'''
参数配置
'''
train_parameters= {
"class_dim": -1, #分类数
"src_path":"work/maskDetect.zip", #原始数据集路径
"target_path":"data/", #要解压的路径
"train_list_path": "data/train.txt", #train.txt路径
"eval_list_path": "data/eval.txt", #eval.txt路径
"readme_path": "data/readme.json", #readme.json路径
"label_dict":{}, #标签字典
}
defunzip_data(src_path,target_path):
'''
解压原始数据集,将src_path路径下的zip包解压至data目录下
'''
if(notos.path.isdir(target_path+"maskDetect")):
z=zipfile.ZipFile(src_path, 'r')
z.extractall(path=target_path)
z.close()
defget_data_list(target_path,train_list_path,eval_list_path):
'''
生成数据列表
'''
#存放所有类别的信息
class_detail= []
#获取所有类别保存的文件夹名称
data_list_path=target_path+"maskDetect/"
class_dirs=os.listdir(data_list_path) # ['.DS_Store', 'nomaskimages', 'maskimages']
#总的图像数量
all_class_images=0
#存放类别标签
class_label=0
#存放类别数目
class_dim=0
#存储要写进eval.txt和train.txt中的内容
trainer_list=[]
eval_list=[]
#读取每个类别,['maskimages', 'nomaskimages']
forclass_dirinclass_dirs:
ifclass_dir!=".DS_Store":
class_dim+=1
#每个类别的信息
class_detail_list= {}
eval_sum=0
trainer_sum=0
#统计每个类别有多少张图片
class_sum=0
#获取类别路径
path=data_list_path+class_dir# e.g maskDetect/nomaskimages
# 获取所有图片
img_paths=os.listdir(path)
forimg_pathinimg_paths: # 遍历文件夹下的每个图片
name_path=path+'/'+img_path# 每张图片的路径
ifclass_sum%10==0: # 每10张图片取一个做验证数据
eval_sum+=1# test_sum为测试数据的数目
eval_list.append(name_path+"\t%d"%class_label+"\n")
else:
trainer_sum+=1
trainer_list.append(name_path+"\t%d"%class_label+"\n") #trainer_sum测试数据的数目
class_sum+=1#每类图片的数目
all_class_images+=1#所有类图片的数目
# 说明的json文件的class_detail数据
class_detail_list['class_name'] =class_dir#类别名称,如jiangwen
class_detail_list['class_label'] =class_label#类别标签
class_detail_list['class_eval_images'] =eval_sum#该类数据的测试集数目
class_detail_list['class_trainer_images'] =trainer_sum#该类数据的训练集数目
class_detail.append(class_detail_list)
#初始化标签列表
train_parameters['label_dict'][str(class_label)] =class_dir
class_label+=1
#初始化分类数
train_parameters['class_dim'] =class_dim
#乱序
random.shuffle(eval_list)
withopen(eval_list_path, 'a') asf:
foreval_imageineval_list:
f.write(eval_image)
random.shuffle(trainer_list)
withopen(train_list_path, 'a') asf2:
fortrain_imageintrainer_list:
f2.write(train_image)
# 说明的json文件信息
readjson= {}
readjson['all_class_name'] =data_list_path#文件父目录
readjson['all_class_images'] =all_class_images
readjson['class_detail'] =class_detail
jsons=json.dumps(readjson, sort_keys=True, indent=4, separators=(',', ': '))
withopen(train_parameters['readme_path'],'w') asf:
f.write(jsons)
print ('生成数据列表完成!')
if__name__=="__main__":
'''
参数初始化
'''
src_path=train_parameters['src_path']
target_path=train_parameters['target_path']
train_list_path=train_parameters['train_list_path']
eval_list_path=train_parameters['eval_list_path']
'''
解压原始数据到指定路径
'''
# unzip_data(src_path,target_path)
'''
划分训练集与验证集,乱序,生成数据列表
'''
#每次生成数据列表前,首先清空train.txt和eval.txt
withopen(train_list_path, 'w') asf:
f.seek(0)
f.truncate()
withopen(eval_list_path, 'w') asf:
f.seek(0)
f.truncate()
#生成数据列表
get_data_list(target_path,train_list_path,eval_list_path)