Python-Sample-code/record.py at master · tr-gis/Python-Sample-code · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
fromlxmlimporthtml
importpprint
importrequests
importos
importsubprocess
fromcollectionsimportOrderedDict
productname='MOD13Q1.005'
#'MOD11A1.005'
libspath='/home/gis-admin/libs/'
tiles=['h27v08','h27v07','h26v08','h26v07','h26v06','h26v05','h25v08','h25v07','h25v06','h25v05','h24v07','h24v06','h24v05','h23v05','h23v06', 'h27v06','h24v08','h22v05']

#tiles=['h28v05','h29v05','h28v04']

'''
----------------------------------------------------------------------------------------------------------------------------------
This file contains the following functions :

def create(datetodownload): This funtion creates a dictionary that contains the scrapped contents of the MODIS webpage
 It can be used in two ways:
 1) When the argument is '' then it scraps the MOD13Q1 homepage and returns a dictionary containing
 directory names and last modified date
 ARGUMENTS : datetodownload in quotes ie;'2000.02.18'


def write(filename,date=''): This function writes the information from the above function into a text file and saves it. It stores information for the tiles that are in the list defined at the beginning of this script
 ARGUMENTS : filename in quotes ie;'2000.02.18.txt', date in quotes ie;'2000.02.18'
 by default the date is '' to write infomation about MOD13Q1 homepage

def create_dict(filename): This function reads the contents of the text file created in the above function and puts the content in to a dictionay.
 ARGUMENTS : filename should be in quotes ie;'2000.02.12.txt'


def listfile(date): This function creates a list file with name = datedate+'_MOD13Q1.txt' which is used by the "modis_download.py"
 ARGUMENTS : date should be in quotes ie;'2000.02.12'

def completed(date): This function writes the dates downloaded to completed.txt
 ARGUMENT : date in qoutes ie; '2000.02.18'
----------------------------------------------------------------------------------------------------------------------------------
'''


defcreate(datetodownload):
try:
rec={}
ifdatetodownload=='':
page=requests.get('http://e4ftl01.cr.usgs.gov/MOLT/'+productname+'/')
else :
page=requests.get('http://e4ftl01.cr.usgs.gov/MOLT/'+productname+'/'+datetodownload+'/')
tree=html.fromstring(page.text)
mod_date=tree.xpath('//pre/text()')
t=[xforxinmod_dateifx.count('-')>1]
m=[x.lstrip() forxint ]
p=[x.translate(None,'\n') forxinm ]
mod=[x.split(' ')[0] forxinp]
#print mod
content=tree.xpath('//a/text()')
tmp=[x.translate(None,'/') forxincontent]
prod=[xforxintmpif (x.translate(None,' ')).isalpha()==False]
#print prod
iflen(mod)!=len(prod):
raise"Len Error"
iflen(prod)==0|len(mod)==0:
raiseValueError
foriinrange(0,len(prod)):
rec[prod[i]]=mod[i]
record=OrderedDict(sorted(rec.items(), key=lambdat: t[0]))

#pprint.pprint(record, width=1)

returnrecord
except"Len Error":
print"record.create() :The lengths of lists donot match !! Correct the code"
exceptValueError:
print"record.create():Is the date valid ?"
except:
print"record.create() : Could not create record"


defwrite(filename,date=''):
try:
ifdate=='':
os.chdir(libspath)
os.system("touch "+filename)
fileobject=open(filename,"w+")
b=create('')#creates record of product home page
#pprint.pprint(b, width=1)
fordatainb:
fileobject.write(data)
fileobject.write("\n")
fileobject.close()

else :
os.chdir(libspath)
os.system("touch "+filename)
fileobject=open(filename,"w+")
b=create(date)#creates record of the given date directory in MOD13Q1
#pprint.pprint(b, width=1)
fordatainb:
if'.jpg'notindata:
forxintiles:
ifxindata:
fileobject.write(data)
fileobject.write("\t")
fileobject.write(b[data])
fileobject.write("\n")
fileobject.close()

except :
print"record.write() : Could not write record into "+filename


defcreate_dict(filename):

try:
a={}
os.chdir(libspath)
text=open(filename,"r")
text.seek(0,0)
line=text.readlines()
foriinrange(0,len(line)):
line[i]=line[i].translate(None,'\n')
content=line[i].partition("\t")#splits the line at the tab character
a[content[0]]=content[2]
#pprint.pprint(a,width=1)
text.close()
returna
except:
print"record.create_dict(): Could not read the contents of "+filename


deflistfile(date):
try:
if0!=subprocess.call('ls | grep '+date+'_MOD13Q1.txt',shell=True):#file doesnot exist
os.system("touch "+date+'_MOD13Q1.txt')
fileobject=open(date+'_MOD13Q1.txt',"w+")
b=create(date)#creates record of MOD13Q1 home page
#pprint.pprint(b, width=1)
fordatainb:
if'.jpg'notindata:
forxintiles:
ifxindata:
fileobject.write(data)
fileobject.write("\n")
fileobject.close()


except :
print"record.listfile() : Could not write tiles info into "+date+'_MOD13Q1.txt'


defcompleted(date):
try:
os.chdir(libspath)
os.system("touch completed.txt")
fileobject=open('completed.txt',"a")
fileobject.write(date+' \n')
fileobject.close()

except:
print"record.completed(): Could not write contents to completed.txt"