pythonBytes/chartPlot.py at master · bourneagain/pythonBytes · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
fromelasticsearchimportElasticsearch
importtime
importpprint
importsys
importsubprocess
importos
JSON_FILE="/Users/sam/cs410/TEAM_BHP_DATA_TOPARSE/srajend2.json"
INDEX_NAME="srajend2_index"
es=Elasticsearch()

defmain():

delete_index(INDEX_NAME)
create_index(INDEX_NAME)
pprint.pprint(es.indices.stats())
#split_file_into_chunks(JSON_FILE)
load_bulk_data_in_chunks()

defdelete_index(_index):
globales
printes.indices.delete(index=_index, ignore=[400, 404])

defcreate_index(_index):
globales
printes.indices.create(index=_index, ignore=400)

deflistFiles(dirpath):
filenames=next(os.walk(dirpath))[2]
yieldfilenames

defcurljson(fil):
# res = es.bulk(index="srajend2_index", body = fil, refresh = True, request_timeout=100)
start_time=time.time()
PWD=os.getcwd()
cmd= ['curl','-s','-XPOST','localhost:9200/_bulk','--data-binary','@'+str(fil)]
printcmd
# process = subprocess.Popen(cmd,stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
try:
res=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,stdin=subprocess.PIPE)
res.communicate()
exceptsubprocess.CalledProcessError:
print"ERROR"
elp_time=time.time() -start_time
returnelp_time


defload_bulk_data_in_chunks():
i=2
iterList=[]
whilei<257:
iterList.append(i)
i=i*2
forchunkiniterList:
print"********* CHUNK ",chunk ," *******************"
PWD=os.getcwd()
timeelp=0
forjsonsinlistFiles("/tmp/JUNKS_"+str(chunk)):
delete_index(INDEX_NAME)
create_index(INDEX_NAME)
p="/tmp/JUNKS_"+str(chunk)
os.chdir(p)
forjsoninjsons:
timeelp+=curljson(json)
print"TABLE,",chunk,",",timeelp
os.chdir(PWD)


defsplit_file_into_chunks(json_file):
cmd= ['/bin/sh','splitJunks.sh',json_file]
printcmd
#process = subprocess.Popen(cmd,stdin=subprocess.PIPE, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
try:
subprocess.check_call(cmd)
exceptsubprocess.CalledProcessError:
print"ERROR!"

print"*** DONE saved in /tmp/JUNKS* *** "


if__name__=="__main__":
main()

#collect the chunks
# split_file_into_chunks(JSON_FILE)