openmc-dev · drewejohnson · Jan 19, 2020 · Jan 26, 2020 · openmc-dev · drewejohnson
diff --git a/data_formatting_tools.py b/data_formatting_tools.py
@@ -0,0 +1,131 @@
+import collections
+import json
+import os
+import pprint
+
+import pandas as pd
+import pymongo
+#from bson.code import Code
+import xlrd
+from pymongo import MongoClient
+
+
+def find_files_recursive(folder,extension,ignore):
+
+ list_of_files=[]
+
+ for root, dirs, files in os.walk(folder):
+ for file in files:
+ if file.endswith(extension) and not file.endswith(ignore):
+ list_of_files.append(os.path.join(root,file))
+ print('number of files found',len(list_of_files))
+ return list_of_files
+
+def find_files(data_folders,extension):
+ ''' function searches through folders and finds files ending with .txt or .csv
+ input: folder(s) to search through
+ input type: list of strings
+ returns: list of matching filenames
+ '''
+ list_of_filenames =[]
+ for folder in data_folders:
+ print(folder)
+ for file in os.listdir(folder):
+ if file.endswith(extension):
+ filename = os.path.join(folder ,file)
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def convert_csv_files_to_dataframe_objects(filenames ,schema):
+ ''' Loads csv files into a pandas dataframe
+ checks that files match the schema
+ inputs: list of filenames, schema [optional]
+ input types: list of strings, list of strings
+ '''
+ list_of_dataframes =[]
+ for filename in filenames:
+ df = pd.read_csv(filename ,sep=',|\t')
+ for header in df.columns.values:
+ df.rename(columns={header: header.strip('\t .')}, inplace=True)
+ # print(df.columns.values)
+ if set(df.columns.values) == set(schema):
+ print('converting' ,filename, 'to dataframe')
+ list_of_dataframes.append(df)
+ else:
+ print("\x1b[31m\"" + str(filename) + ' does not conform to schema' + "\x1b[0m")
+ return list_of_dataframes
+
+def convert_dataframe_objects_to_json_files(list_of_dataframes, filenames):
+ ''' Converts pandas dataframe into a json file
+ inputs: list of dataframes
+ input types: list of dataframes
+ '''
+ list_of_filenames = []
+ for df, filename in zip(list_of_dataframes, filenames):
+ df.to_json(filename, orient='columns')
+ # df.to_json(filename, orient='columns', index=False) index setting is not currently supported, perhaps with the next release, see http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_json.html?highlight=to_json#pandas.DataFrame.to_json
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def apply_abs_operation_to_field(json_objects,field):
+ for json_object in json_objects:
+ json_object[field]=l = [abs(x) for x in json_object[field]]
+ return json_objects
+
+def read_in_json_files(filenames):
+ # read in json files
+ list_of_json_objects = []
+ for filename in filenames:
+ with open(filename) as f:
+ data = json.load(f)
+ data = remove_df_index_from_json_object(data)
+ list_of_json_objects.append(data)
+ return list_of_json_objects
+
+def convert_list_of_tuples_to_list(list_of_tuples):
+ list_of_single_values=[]
+ for x in range(0,len(list_of_tuples)):
+ list_of_single_values.append(list_of_tuples[str(x)])
+ return list_of_single_values
+
+def remove_df_index_from_json_object(data):
+ newdata={}
+ for key in data.keys():
+ #print(key)
+ list_of_tuples = data[key]
+ newdata[key] = convert_list_of_tuples_to_list(list_of_tuples) 
+ return newdata
+
+# def write_json_files(json_objects, filenames):
+# print('writing json files')
+# list_of_json_objects = []
+# for filename, json_object in zip(filenames, json_objects):
+# with open(filename, 'w') as outfile:
+# json.dump(json_object, outfile, indent=4)
+# return filenames
+
+def save_json_obj_to_file(json_obj,filename,folder=''):
+ print('saving data')
+ with open(os.path.join(folder,filename), 'w') as fout:
+ #json.dump(list_of_data, fout, encoding='utf-8', indent = 4) #python 2
+ json.dump(json_obj, fout, indent = 4)
+
+def save_json_objs_to_files(json_objs,folder=''):
+ for json_obj in json_objs:
+ save_json_obj_to_file(json_obj,json_obj['filename'],folder)
+
+
+def add_meta_data_to_json_objects(json_object, keyname, keyvalue):
+ # add some meta data to json file
+
+ if type(json_object) == list and type(keyname) == list and type(keyvalue) == list:
+ for item_json, item_keyname, item_keyvalue in zip(json_object, keyname, keyvalue):
+ item_json[item_keyname] = item_keyvalue
+ print('added new meta data list', item_keyname, ' = ', item_keyvalue)
+ return json_object
+ else:
+ json_object[keyname] = keyvalue
+ print('added new meta data', keyname, ' = ', keyvalue)
+ return json_object
+
+
diff --git a/database_tools.py b/database_tools.py
@@ -0,0 +1,188 @@
+import pandas as pd
+import os
+import json
+import pprint
+import collections
+import pymongo
+from pymongo import MongoClient
+from bson.code import Code
+import socket
+
+def connect_to_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ defaults are provided for the names
+ '''
+ client = MongoClient('localhost', 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db
+
+def connect_to_docker_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ '''
+ host = socket.gethostbyname(socket.gethostname())
+ client = MongoClient(host, 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db 
+
+def delete_database(client, db_name='my_database'):
+ try:
+ client.drop_database(db_name)
+ print('database deleted')
+ except:
+ print('unable to delete database',db_name)
+
+def upload_json_objects_to_database(data,collection):
+
+ if type(data)==list:
+ for i, item in enumerate(data):
+ try:
+ print('inserting entry into database ',i)
+ collection.insert_many(item)
+ except:
+ print('FAILED inserting entry into database',i)
+ # print('failing json object is ' ,item)
+
+ else:
+ collection.insert_one(data)
+ print('json object commited to database')
+
+#def get_database_fields(collection, query_fields_and_values={},ignore_fields=[]):
+def get_database_fields(collection,ignore_fields=[]):
+ mapper = Code("""
+ function() {
+ for (var key in this) { emit(key, null); }
+ }
+ """)
+
+ reducer = Code("""
+ function(key, stuff) { return null; }
+ """)
+
+ mr = collection.map_reduce(map = mapper,
+ reduce = reducer,
+ #query = query_fields_and_values,
+ out = "my_collection" + "_keys")
+ unique_fields = []
+ for doc in mr.find():
+ if doc["_id"] not in ignore_fields:
+ if doc["_id"] != "_id":
+ unique_fields.append(doc["_id"])
+ return unique_fields
+
+def get_entries_in_field(collection, field, query=None):
+ if query != {}:
+ result = collection.distinct(field,query)
+ else:
+ result = collection.distinct(field)
+ return result
+
+def get_type_of_entries_in_field(collection, field, query=None):
+ return type(collection.find_one({field:{ '$exists': True }})[field]).__name__
+ #return type(collection.find_one()[field]).__name__
+
+def find_all_fields_types_in_database(collection):
+ all_fields = get_database_fields(collection)
+ field_and_type = {}
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ field_and_type[field] = type_of_field_contents
+ return field_and_type
+
+def find_all_fields_of_a_particular_types_in_database(collection,type_required):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents ==type_required:
+ field_of_correct_type.append(field)
+ return field_of_correct_type
+
+def find_all_fields_not_of_a_particular_types_in_database(collection,type_not_desired):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents !=type_not_desired:
+ field_of_correct_type.append(field)
+ return field_of_correct_type 
+
+def get_number_of_matching_entrys(collection, query_fields_and_values):
+ result = collection.find(query_fields_and_values)
+ return result.count()
+
+def get_matching_entrys(collection, query_fields_and_values,limit=10):
+ result = collection.find(query_fields_and_values).limit(limit)
+ return result
+
+def find_metadata_fields_and_their_distinct_values(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field]})
+ return meta_data_fields_and_distinct_entries
+
+
+def find_metadata_fields_and_their_distinct_values_and_types(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field],
+ 'type':type(metadata_fields_and_their_distinct_values[field])})
+ return meta_data_fields_and_distinct_entries
+
+
+if __name__ == '__main__':
+
+
+ collection, client, db = connect_to_database()
+ # collection, client, db = connect_to_docker_database()
+
+ query = {'mass number':5}
+
+ myresults=collection.find(query)
+
+ print('number of results found = ' ,myresults.count())
+
+ print('current reaction ',myresults[0]['reaction']) 
+
+
+
+
+
+ #collection.delete_one(myquery)
+
+
+ #results = get_number_of_matching_entrys(collection, {'uploader':'shimwell'})
+ # results = find_metadata_fields_and_their_distinct_values_and_types(collection)
+ # for result in results:
+ # print(result['type'])
+ # fields = get_database_fields(collection)
+ # for f in fields:
+ # results = get_entries_in_field(collection,f)
+ # type_of_results = get_type_of_entries_in_field(collection,f)
+ # print(f, type_of_results)
+ #print(find_all_fields_types_in_database(collection))
+ # results = collection.find_one({'temperature':{ '$exists': True }})['temperature']
+ # print(results)
+
+ #print(find_all_fields_of_a_particular_types_in_database(collection,'list'))
diff --git a/data_formatting_tools.py b/data_formatting_tools.py
@@ -0,0 +1,131 @@
+import collections
+import json
+import os
+import pprint
+
+import pandas as pd
+import pymongo
+#from bson.code import Code
+import xlrd
+from pymongo import MongoClient
+
+
+def find_files_recursive(folder,extension,ignore):
+
+ list_of_files=[]
+
+ for root, dirs, files in os.walk(folder):
+ for file in files:
+ if file.endswith(extension) and not file.endswith(ignore):
+ list_of_files.append(os.path.join(root,file))
+ print('number of files found',len(list_of_files))
+ return list_of_files
+
+def find_files(data_folders,extension):
+ ''' function searches through folders and finds files ending with .txt or .csv
+ input: folder(s) to search through
+ input type: list of strings
+ returns: list of matching filenames
+ '''
+ list_of_filenames =[]
+ for folder in data_folders:
+ print(folder)
+ for file in os.listdir(folder):
+ if file.endswith(extension):
+ filename = os.path.join(folder ,file)
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def convert_csv_files_to_dataframe_objects(filenames ,schema):
+ ''' Loads csv files into a pandas dataframe
+ checks that files match the schema
+ inputs: list of filenames, schema [optional]
+ input types: list of strings, list of strings
+ '''
+ list_of_dataframes =[]
+ for filename in filenames:
+ df = pd.read_csv(filename ,sep=',|\t')
+ for header in df.columns.values:
+ df.rename(columns={header: header.strip('\t .')}, inplace=True)
+ # print(df.columns.values)
+ if set(df.columns.values) == set(schema):
+ print('converting' ,filename, 'to dataframe')
+ list_of_dataframes.append(df)
+ else:
+ print("\x1b[31m\"" + str(filename) + ' does not conform to schema' + "\x1b[0m")
+ return list_of_dataframes
+
+def convert_dataframe_objects_to_json_files(list_of_dataframes, filenames):
+ ''' Converts pandas dataframe into a json file
+ inputs: list of dataframes
+ input types: list of dataframes
+ '''
+ list_of_filenames = []
+ for df, filename in zip(list_of_dataframes, filenames):
+ df.to_json(filename, orient='columns')
+ # df.to_json(filename, orient='columns', index=False) index setting is not currently supported, perhaps with the next release, see http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_json.html?highlight=to_json#pandas.DataFrame.to_json
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def apply_abs_operation_to_field(json_objects,field):
+ for json_object in json_objects:
+ json_object[field]=l = [abs(x) for x in json_object[field]]
+ return json_objects
+
+def read_in_json_files(filenames):
+ # read in json files
+ list_of_json_objects = []
+ for filename in filenames:
+ with open(filename) as f:
+ data = json.load(f)
+ data = remove_df_index_from_json_object(data)
+ list_of_json_objects.append(data)
+ return list_of_json_objects
+
+def convert_list_of_tuples_to_list(list_of_tuples):
+ list_of_single_values=[]
+ for x in range(0,len(list_of_tuples)):
+ list_of_single_values.append(list_of_tuples[str(x)])
+ return list_of_single_values
+
+def remove_df_index_from_json_object(data):
+ newdata={}
+ for key in data.keys():
+ #print(key)
+ list_of_tuples = data[key]
+ newdata[key] = convert_list_of_tuples_to_list(list_of_tuples) 
+ return newdata
+
+# def write_json_files(json_objects, filenames):
+# print('writing json files')
+# list_of_json_objects = []
+# for filename, json_object in zip(filenames, json_objects):
+# with open(filename, 'w') as outfile:
+# json.dump(json_object, outfile, indent=4)
+# return filenames
+
+def save_json_obj_to_file(json_obj,filename,folder=''):
+ print('saving data')
+ with open(os.path.join(folder,filename), 'w') as fout:
+ #json.dump(list_of_data, fout, encoding='utf-8', indent = 4) #python 2
+ json.dump(json_obj, fout, indent = 4)
+
+def save_json_objs_to_files(json_objs,folder=''):
+ for json_obj in json_objs:
+ save_json_obj_to_file(json_obj,json_obj['filename'],folder)
+
+
+def add_meta_data_to_json_objects(json_object, keyname, keyvalue):
+ # add some meta data to json file
+
+ if type(json_object) == list and type(keyname) == list and type(keyvalue) == list:
+ for item_json, item_keyname, item_keyvalue in zip(json_object, keyname, keyvalue):
+ item_json[item_keyname] = item_keyvalue
+ print('added new meta data list', item_keyname, ' = ', item_keyvalue)
+ return json_object
+ else:
+ json_object[keyname] = keyvalue
+ print('added new meta data', keyname, ' = ', keyvalue)
+ return json_object
+
+
diff --git a/database_tools.py b/database_tools.py
@@ -0,0 +1,188 @@
+import pandas as pd
+import os
+import json
+import pprint
+import collections
+import pymongo
+from pymongo import MongoClient
+from bson.code import Code
+import socket
+
+def connect_to_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ defaults are provided for the names
+ '''
+ client = MongoClient('localhost', 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db
+
+def connect_to_docker_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ '''
+ host = socket.gethostbyname(socket.gethostname())
+ client = MongoClient(host, 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db 
+
+def delete_database(client, db_name='my_database'):
+ try:
+ client.drop_database(db_name)
+ print('database deleted')
+ except:
+ print('unable to delete database',db_name)
+
+def upload_json_objects_to_database(data,collection):
+
+ if type(data)==list:
+ for i, item in enumerate(data):
+ try:
+ print('inserting entry into database ',i)
+ collection.insert_many(item)
+ except:
+ print('FAILED inserting entry into database',i)
+ # print('failing json object is ' ,item)
+
+ else:
+ collection.insert_one(data)
+ print('json object commited to database')
+
+#def get_database_fields(collection, query_fields_and_values={},ignore_fields=[]):
+def get_database_fields(collection,ignore_fields=[]):
+ mapper = Code("""
+ function() {
+ for (var key in this) { emit(key, null); }
+ }
+ """)
+
+ reducer = Code("""
+ function(key, stuff) { return null; }
+ """)
+
+ mr = collection.map_reduce(map = mapper,
+ reduce = reducer,
+ #query = query_fields_and_values,
+ out = "my_collection" + "_keys")
+ unique_fields = []
+ for doc in mr.find():
+ if doc["_id"] not in ignore_fields:
+ if doc["_id"] != "_id":
+ unique_fields.append(doc["_id"])
+ return unique_fields
+
+def get_entries_in_field(collection, field, query=None):
+ if query != {}:
+ result = collection.distinct(field,query)
+ else:
+ result = collection.distinct(field)
+ return result
+
+def get_type_of_entries_in_field(collection, field, query=None):
+ return type(collection.find_one({field:{ '$exists': True }})[field]).__name__
+ #return type(collection.find_one()[field]).__name__
+
+def find_all_fields_types_in_database(collection):
+ all_fields = get_database_fields(collection)
+ field_and_type = {}
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ field_and_type[field] = type_of_field_contents
+ return field_and_type
+
+def find_all_fields_of_a_particular_types_in_database(collection,type_required):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents ==type_required:
+ field_of_correct_type.append(field)
+ return field_of_correct_type
+
+def find_all_fields_not_of_a_particular_types_in_database(collection,type_not_desired):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents !=type_not_desired:
+ field_of_correct_type.append(field)
+ return field_of_correct_type 
+
+def get_number_of_matching_entrys(collection, query_fields_and_values):
+ result = collection.find(query_fields_and_values)
+ return result.count()
+
+def get_matching_entrys(collection, query_fields_and_values,limit=10):
+ result = collection.find(query_fields_and_values).limit(limit)
+ return result
+
+def find_metadata_fields_and_their_distinct_values(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field]})
+ return meta_data_fields_and_distinct_entries
+
+
+def find_metadata_fields_and_their_distinct_values_and_types(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field],
+ 'type':type(metadata_fields_and_their_distinct_values[field])})
+ return meta_data_fields_and_distinct_entries
+
+
+if __name__ == '__main__':
+
+
+ collection, client, db = connect_to_database()
+ # collection, client, db = connect_to_docker_database()
+
+ query = {'mass number':5}
+
+ myresults=collection.find(query)
+
+ print('number of results found = ' ,myresults.count())
+
+ print('current reaction ',myresults[0]['reaction']) 
+
+
+
+
+
+ #collection.delete_one(myquery)
+
+
+ #results = get_number_of_matching_entrys(collection, {'uploader':'shimwell'})
+ # results = find_metadata_fields_and_their_distinct_values_and_types(collection)
+ # for result in results:
+ # print(result['type'])
+ # fields = get_database_fields(collection)
+ # for f in fields:
+ # results = get_entries_in_field(collection,f)
+ # type_of_results = get_type_of_entries_in_field(collection,f)
+ # print(f, type_of_results)
+ #print(find_all_fields_types_in_database(collection))
+ # results = collection.find_one({'temperature':{ '$exists': True }})['temperature']
+ # print(results)
+
+ #print(find_all_fields_of_a_particular_types_in_database(collection,'list'))
diff --git a/data_formatting_tools.py b/data_formatting_tools.py
@@ -0,0 +1,131 @@
+import collections
+import json
+import os
+import pprint
+
+import pandas as pd
+import pymongo
+#from bson.code import Code
+import xlrd
+from pymongo import MongoClient
+
+
+def find_files_recursive(folder,extension,ignore):
+
+ list_of_files=[]
+
+ for root, dirs, files in os.walk(folder):
+ for file in files:
+ if file.endswith(extension) and not file.endswith(ignore):
+ list_of_files.append(os.path.join(root,file))
+ print('number of files found',len(list_of_files))
+ return list_of_files
+
+def find_files(data_folders,extension):
+ ''' function searches through folders and finds files ending with .txt or .csv
+ input: folder(s) to search through
+ input type: list of strings
+ returns: list of matching filenames
+ '''
+ list_of_filenames =[]
+ for folder in data_folders:
+ print(folder)
+ for file in os.listdir(folder):
+ if file.endswith(extension):
+ filename = os.path.join(folder ,file)
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def convert_csv_files_to_dataframe_objects(filenames ,schema):
+ ''' Loads csv files into a pandas dataframe
+ checks that files match the schema
+ inputs: list of filenames, schema [optional]
+ input types: list of strings, list of strings
+ '''
+ list_of_dataframes =[]
+ for filename in filenames:
+ df = pd.read_csv(filename ,sep=',|\t')
+ for header in df.columns.values:
+ df.rename(columns={header: header.strip('\t .')}, inplace=True)
+ # print(df.columns.values)
+ if set(df.columns.values) == set(schema):
+ print('converting' ,filename, 'to dataframe')
+ list_of_dataframes.append(df)
+ else:
+ print("\x1b[31m\"" + str(filename) + ' does not conform to schema' + "\x1b[0m")
+ return list_of_dataframes
+
+def convert_dataframe_objects_to_json_files(list_of_dataframes, filenames):
+ ''' Converts pandas dataframe into a json file
+ inputs: list of dataframes
+ input types: list of dataframes
+ '''
+ list_of_filenames = []
+ for df, filename in zip(list_of_dataframes, filenames):
+ df.to_json(filename, orient='columns')
+ # df.to_json(filename, orient='columns', index=False) index setting is not currently supported, perhaps with the next release, see http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_json.html?highlight=to_json#pandas.DataFrame.to_json
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def apply_abs_operation_to_field(json_objects,field):
+ for json_object in json_objects:
+ json_object[field]=l = [abs(x) for x in json_object[field]]
+ return json_objects
+
+def read_in_json_files(filenames):
+ # read in json files
+ list_of_json_objects = []
+ for filename in filenames:
+ with open(filename) as f:
+ data = json.load(f)
+ data = remove_df_index_from_json_object(data)
+ list_of_json_objects.append(data)
+ return list_of_json_objects
+
+def convert_list_of_tuples_to_list(list_of_tuples):
+ list_of_single_values=[]
+ for x in range(0,len(list_of_tuples)):
+ list_of_single_values.append(list_of_tuples[str(x)])
+ return list_of_single_values
+
+def remove_df_index_from_json_object(data):
+ newdata={}
+ for key in data.keys():
+ #print(key)
+ list_of_tuples = data[key]
+ newdata[key] = convert_list_of_tuples_to_list(list_of_tuples) 
+ return newdata
+
+# def write_json_files(json_objects, filenames):
+# print('writing json files')
+# list_of_json_objects = []
+# for filename, json_object in zip(filenames, json_objects):
+# with open(filename, 'w') as outfile:
+# json.dump(json_object, outfile, indent=4)
+# return filenames
+
+def save_json_obj_to_file(json_obj,filename,folder=''):
+ print('saving data')
+ with open(os.path.join(folder,filename), 'w') as fout:
+ #json.dump(list_of_data, fout, encoding='utf-8', indent = 4) #python 2
+ json.dump(json_obj, fout, indent = 4)
+
+def save_json_objs_to_files(json_objs,folder=''):
+ for json_obj in json_objs:
+ save_json_obj_to_file(json_obj,json_obj['filename'],folder)
+
+
+def add_meta_data_to_json_objects(json_object, keyname, keyvalue):
+ # add some meta data to json file
+
+ if type(json_object) == list and type(keyname) == list and type(keyvalue) == list:
+ for item_json, item_keyname, item_keyvalue in zip(json_object, keyname, keyvalue):
+ item_json[item_keyname] = item_keyvalue
+ print('added new meta data list', item_keyname, ' = ', item_keyvalue)
+ return json_object
+ else:
+ json_object[keyname] = keyvalue
+ print('added new meta data', keyname, ' = ', keyvalue)
+ return json_object
+
+
diff --git a/database_tools.py b/database_tools.py
@@ -0,0 +1,188 @@
+import pandas as pd
+import os
+import json
+import pprint
+import collections
+import pymongo
+from pymongo import MongoClient
+from bson.code import Code
+import socket
+
+def connect_to_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ defaults are provided for the names
+ '''
+ client = MongoClient('localhost', 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db
+
+def connect_to_docker_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ '''
+ host = socket.gethostbyname(socket.gethostname())
+ client = MongoClient(host, 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db 
+
+def delete_database(client, db_name='my_database'):
+ try:
+ client.drop_database(db_name)
+ print('database deleted')
+ except:
+ print('unable to delete database',db_name)
+
+def upload_json_objects_to_database(data,collection):
+
+ if type(data)==list:
+ for i, item in enumerate(data):
+ try:
+ print('inserting entry into database ',i)
+ collection.insert_many(item)
+ except:
+ print('FAILED inserting entry into database',i)
+ # print('failing json object is ' ,item)
+
+ else:
+ collection.insert_one(data)
+ print('json object commited to database')
+
+#def get_database_fields(collection, query_fields_and_values={},ignore_fields=[]):
+def get_database_fields(collection,ignore_fields=[]):
+ mapper = Code("""
+ function() {
+ for (var key in this) { emit(key, null); }
+ }
+ """)
+
+ reducer = Code("""
+ function(key, stuff) { return null; }
+ """)
+
+ mr = collection.map_reduce(map = mapper,
+ reduce = reducer,
+ #query = query_fields_and_values,
+ out = "my_collection" + "_keys")
+ unique_fields = []
+ for doc in mr.find():
+ if doc["_id"] not in ignore_fields:
+ if doc["_id"] != "_id":
+ unique_fields.append(doc["_id"])
+ return unique_fields
+
+def get_entries_in_field(collection, field, query=None):
+ if query != {}:
+ result = collection.distinct(field,query)
+ else:
+ result = collection.distinct(field)
+ return result
+
+def get_type_of_entries_in_field(collection, field, query=None):
+ return type(collection.find_one({field:{ '$exists': True }})[field]).__name__
+ #return type(collection.find_one()[field]).__name__
+
+def find_all_fields_types_in_database(collection):
+ all_fields = get_database_fields(collection)
+ field_and_type = {}
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ field_and_type[field] = type_of_field_contents
+ return field_and_type
+
+def find_all_fields_of_a_particular_types_in_database(collection,type_required):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents ==type_required:
+ field_of_correct_type.append(field)
+ return field_of_correct_type
+
+def find_all_fields_not_of_a_particular_types_in_database(collection,type_not_desired):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents !=type_not_desired:
+ field_of_correct_type.append(field)
+ return field_of_correct_type 
+
+def get_number_of_matching_entrys(collection, query_fields_and_values):
+ result = collection.find(query_fields_and_values)
+ return result.count()
+
+def get_matching_entrys(collection, query_fields_and_values,limit=10):
+ result = collection.find(query_fields_and_values).limit(limit)
+ return result
+
+def find_metadata_fields_and_their_distinct_values(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field]})
+ return meta_data_fields_and_distinct_entries
+
+
+def find_metadata_fields_and_their_distinct_values_and_types(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field],
+ 'type':type(metadata_fields_and_their_distinct_values[field])})
+ return meta_data_fields_and_distinct_entries
+
+
+if __name__ == '__main__':
+
+
+ collection, client, db = connect_to_database()
+ # collection, client, db = connect_to_docker_database()
+
+ query = {'mass number':5}
+
+ myresults=collection.find(query)
+
+ print('number of results found = ' ,myresults.count())
+
+ print('current reaction ',myresults[0]['reaction']) 
+
+
+
+
+
+ #collection.delete_one(myquery)
+
+
+ #results = get_number_of_matching_entrys(collection, {'uploader':'shimwell'})
+ # results = find_metadata_fields_and_their_distinct_values_and_types(collection)
+ # for result in results:
+ # print(result['type'])
+ # fields = get_database_fields(collection)
+ # for f in fields:
+ # results = get_entries_in_field(collection,f)
+ # type_of_results = get_type_of_entries_in_field(collection,f)
+ # print(f, type_of_results)
+ #print(find_all_fields_types_in_database(collection))
+ # results = collection.find_one({'temperature':{ '$exists': True }})['temperature']
+ # print(results)
+
+ #print(find_all_fields_of_a_particular_types_in_database(collection,'list'))
diff --git a/data_formatting_tools.py b/data_formatting_tools.py
@@ -0,0 +1,131 @@
+import collections
+import json
+import os
+import pprint
+
+import pandas as pd
+import pymongo
+#from bson.code import Code
+import xlrd
+from pymongo import MongoClient
+
+
+def find_files_recursive(folder,extension,ignore):
+
+ list_of_files=[]
+
+ for root, dirs, files in os.walk(folder):
+ for file in files:
+ if file.endswith(extension) and not file.endswith(ignore):
+ list_of_files.append(os.path.join(root,file))
+ print('number of files found',len(list_of_files))
+ return list_of_files
+
+def find_files(data_folders,extension):
+ ''' function searches through folders and finds files ending with .txt or .csv
+ input: folder(s) to search through
+ input type: list of strings
+ returns: list of matching filenames
+ '''
+ list_of_filenames =[]
+ for folder in data_folders:
+ print(folder)
+ for file in os.listdir(folder):
+ if file.endswith(extension):
+ filename = os.path.join(folder ,file)
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def convert_csv_files_to_dataframe_objects(filenames ,schema):
+ ''' Loads csv files into a pandas dataframe
+ checks that files match the schema
+ inputs: list of filenames, schema [optional]
+ input types: list of strings, list of strings
+ '''
+ list_of_dataframes =[]
+ for filename in filenames:
+ df = pd.read_csv(filename ,sep=',|\t')
+ for header in df.columns.values:
+ df.rename(columns={header: header.strip('\t .')}, inplace=True)
+ # print(df.columns.values)
+ if set(df.columns.values) == set(schema):
+ print('converting' ,filename, 'to dataframe')
+ list_of_dataframes.append(df)
+ else:
+ print("\x1b[31m\"" + str(filename) + ' does not conform to schema' + "\x1b[0m")
+ return list_of_dataframes
+
+def convert_dataframe_objects_to_json_files(list_of_dataframes, filenames):
+ ''' Converts pandas dataframe into a json file
+ inputs: list of dataframes
+ input types: list of dataframes
+ '''
+ list_of_filenames = []
+ for df, filename in zip(list_of_dataframes, filenames):
+ df.to_json(filename, orient='columns')
+ # df.to_json(filename, orient='columns', index=False) index setting is not currently supported, perhaps with the next release, see http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_json.html?highlight=to_json#pandas.DataFrame.to_json
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def apply_abs_operation_to_field(json_objects,field):
+ for json_object in json_objects:
+ json_object[field]=l = [abs(x) for x in json_object[field]]
+ return json_objects
+
+def read_in_json_files(filenames):
+ # read in json files
+ list_of_json_objects = []
+ for filename in filenames:
+ with open(filename) as f:
+ data = json.load(f)
+ data = remove_df_index_from_json_object(data)
+ list_of_json_objects.append(data)
+ return list_of_json_objects
+
+def convert_list_of_tuples_to_list(list_of_tuples):
+ list_of_single_values=[]
+ for x in range(0,len(list_of_tuples)):
+ list_of_single_values.append(list_of_tuples[str(x)])
+ return list_of_single_values
+
+def remove_df_index_from_json_object(data):
+ newdata={}
+ for key in data.keys():
+ #print(key)
+ list_of_tuples = data[key]
+ newdata[key] = convert_list_of_tuples_to_list(list_of_tuples) 
+ return newdata
+
+# def write_json_files(json_objects, filenames):
+# print('writing json files')
+# list_of_json_objects = []
+# for filename, json_object in zip(filenames, json_objects):
+# with open(filename, 'w') as outfile:
+# json.dump(json_object, outfile, indent=4)
+# return filenames
+
+def save_json_obj_to_file(json_obj,filename,folder=''):
+ print('saving data')
+ with open(os.path.join(folder,filename), 'w') as fout:
+ #json.dump(list_of_data, fout, encoding='utf-8', indent = 4) #python 2
+ json.dump(json_obj, fout, indent = 4)
+
+def save_json_objs_to_files(json_objs,folder=''):
+ for json_obj in json_objs:
+ save_json_obj_to_file(json_obj,json_obj['filename'],folder)
+
+
+def add_meta_data_to_json_objects(json_object, keyname, keyvalue):
+ # add some meta data to json file
+
+ if type(json_object) == list and type(keyname) == list and type(keyvalue) == list:
+ for item_json, item_keyname, item_keyvalue in zip(json_object, keyname, keyvalue):
+ item_json[item_keyname] = item_keyvalue
+ print('added new meta data list', item_keyname, ' = ', item_keyvalue)
+ return json_object
+ else:
+ json_object[keyname] = keyvalue
+ print('added new meta data', keyname, ' = ', keyvalue)
+ return json_object
+
+
diff --git a/database_tools.py b/database_tools.py
@@ -0,0 +1,188 @@
+import pandas as pd
+import os
+import json
+import pprint
+import collections
+import pymongo
+from pymongo import MongoClient
+from bson.code import Code
+import socket
+
+def connect_to_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ defaults are provided for the names
+ '''
+ client = MongoClient('localhost', 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db
+
+def connect_to_docker_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ '''
+ host = socket.gethostbyname(socket.gethostname())
+ client = MongoClient(host, 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db 
+
+def delete_database(client, db_name='my_database'):
+ try:
+ client.drop_database(db_name)
+ print('database deleted')
+ except:
+ print('unable to delete database',db_name)
+
+def upload_json_objects_to_database(data,collection):
+
+ if type(data)==list:
+ for i, item in enumerate(data):
+ try:
+ print('inserting entry into database ',i)
+ collection.insert_many(item)
+ except:
+ print('FAILED inserting entry into database',i)
+ # print('failing json object is ' ,item)
+
+ else:
+ collection.insert_one(data)
+ print('json object commited to database')
+
+#def get_database_fields(collection, query_fields_and_values={},ignore_fields=[]):
+def get_database_fields(collection,ignore_fields=[]):
+ mapper = Code("""
+ function() {
+ for (var key in this) { emit(key, null); }
+ }
+ """)
+
+ reducer = Code("""
+ function(key, stuff) { return null; }
+ """)
+
+ mr = collection.map_reduce(map = mapper,
+ reduce = reducer,
+ #query = query_fields_and_values,
+ out = "my_collection" + "_keys")
+ unique_fields = []
+ for doc in mr.find():
+ if doc["_id"] not in ignore_fields:
+ if doc["_id"] != "_id":
+ unique_fields.append(doc["_id"])
+ return unique_fields
+
+def get_entries_in_field(collection, field, query=None):
+ if query != {}:
+ result = collection.distinct(field,query)
+ else:
+ result = collection.distinct(field)
+ return result
+
+def get_type_of_entries_in_field(collection, field, query=None):
+ return type(collection.find_one({field:{ '$exists': True }})[field]).__name__
+ #return type(collection.find_one()[field]).__name__
+
+def find_all_fields_types_in_database(collection):
+ all_fields = get_database_fields(collection)
+ field_and_type = {}
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ field_and_type[field] = type_of_field_contents
+ return field_and_type
+
+def find_all_fields_of_a_particular_types_in_database(collection,type_required):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents ==type_required:
+ field_of_correct_type.append(field)
+ return field_of_correct_type
+
+def find_all_fields_not_of_a_particular_types_in_database(collection,type_not_desired):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents !=type_not_desired:
+ field_of_correct_type.append(field)
+ return field_of_correct_type 
+
+def get_number_of_matching_entrys(collection, query_fields_and_values):
+ result = collection.find(query_fields_and_values)
+ return result.count()
+
+def get_matching_entrys(collection, query_fields_and_values,limit=10):
+ result = collection.find(query_fields_and_values).limit(limit)
+ return result
+
+def find_metadata_fields_and_their_distinct_values(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field]})
+ return meta_data_fields_and_distinct_entries
+
+
+def find_metadata_fields_and_their_distinct_values_and_types(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field],
+ 'type':type(metadata_fields_and_their_distinct_values[field])})
+ return meta_data_fields_and_distinct_entries
+
+
+if __name__ == '__main__':
+
+
+ collection, client, db = connect_to_database()
+ # collection, client, db = connect_to_docker_database()
+
+ query = {'mass number':5}
+
+ myresults=collection.find(query)
+
+ print('number of results found = ' ,myresults.count())
+
+ print('current reaction ',myresults[0]['reaction']) 
+
+
+
+
+
+ #collection.delete_one(myquery)
+
+
+ #results = get_number_of_matching_entrys(collection, {'uploader':'shimwell'})
+ # results = find_metadata_fields_and_their_distinct_values_and_types(collection)
+ # for result in results:
+ # print(result['type'])
+ # fields = get_database_fields(collection)
+ # for f in fields:
+ # results = get_entries_in_field(collection,f)
+ # type_of_results = get_type_of_entries_in_field(collection,f)
+ # print(f, type_of_results)
+ #print(find_all_fields_types_in_database(collection))
+ # results = collection.find_one({'temperature':{ '$exists': True }})['temperature']
+ # print(results)
+
+ #print(find_all_fields_of_a_particular_types_in_database(collection,'list'))
diff --git a/data_formatting_tools.py b/data_formatting_tools.py
@@ -0,0 +1,131 @@
+import collections
+import json
+import os
+import pprint
+
+import pandas as pd
+import pymongo
+#from bson.code import Code
+import xlrd
+from pymongo import MongoClient
+
+
+def find_files_recursive(folder,extension,ignore):
+
+ list_of_files=[]
+
+ for root, dirs, files in os.walk(folder):
+ for file in files:
+ if file.endswith(extension) and not file.endswith(ignore):
+ list_of_files.append(os.path.join(root,file))
+ print('number of files found',len(list_of_files))
+ return list_of_files
+
+def find_files(data_folders,extension):
+ ''' function searches through folders and finds files ending with .txt or .csv
+ input: folder(s) to search through
+ input type: list of strings
+ returns: list of matching filenames
+ '''
+ list_of_filenames =[]
+ for folder in data_folders:
+ print(folder)
+ for file in os.listdir(folder):
+ if file.endswith(extension):
+ filename = os.path.join(folder ,file)
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def convert_csv_files_to_dataframe_objects(filenames ,schema):
+ ''' Loads csv files into a pandas dataframe
+ checks that files match the schema
+ inputs: list of filenames, schema [optional]
+ input types: list of strings, list of strings
+ '''
+ list_of_dataframes =[]
+ for filename in filenames:
+ df = pd.read_csv(filename ,sep=',|\t')
+ for header in df.columns.values:
+ df.rename(columns={header: header.strip('\t .')}, inplace=True)
+ # print(df.columns.values)
+ if set(df.columns.values) == set(schema):
+ print('converting' ,filename, 'to dataframe')
+ list_of_dataframes.append(df)
+ else:
+ print("\x1b[31m\"" + str(filename) + ' does not conform to schema' + "\x1b[0m")
+ return list_of_dataframes
+
+def convert_dataframe_objects_to_json_files(list_of_dataframes, filenames):
+ ''' Converts pandas dataframe into a json file
+ inputs: list of dataframes
+ input types: list of dataframes
+ '''
+ list_of_filenames = []
+ for df, filename in zip(list_of_dataframes, filenames):
+ df.to_json(filename, orient='columns')
+ # df.to_json(filename, orient='columns', index=False) index setting is not currently supported, perhaps with the next release, see http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_json.html?highlight=to_json#pandas.DataFrame.to_json
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def apply_abs_operation_to_field(json_objects,field):
+ for json_object in json_objects:
+ json_object[field]=l = [abs(x) for x in json_object[field]]
+ return json_objects
+
+def read_in_json_files(filenames):
+ # read in json files
+ list_of_json_objects = []
+ for filename in filenames:
+ with open(filename) as f:
+ data = json.load(f)
+ data = remove_df_index_from_json_object(data)
+ list_of_json_objects.append(data)
+ return list_of_json_objects
+
+def convert_list_of_tuples_to_list(list_of_tuples):
+ list_of_single_values=[]
+ for x in range(0,len(list_of_tuples)):
+ list_of_single_values.append(list_of_tuples[str(x)])
+ return list_of_single_values
+
+def remove_df_index_from_json_object(data):
+ newdata={}
+ for key in data.keys():
+ #print(key)
+ list_of_tuples = data[key]
+ newdata[key] = convert_list_of_tuples_to_list(list_of_tuples) 
+ return newdata
+
+# def write_json_files(json_objects, filenames):
+# print('writing json files')
+# list_of_json_objects = []
+# for filename, json_object in zip(filenames, json_objects):
+# with open(filename, 'w') as outfile:
+# json.dump(json_object, outfile, indent=4)
+# return filenames
+
+def save_json_obj_to_file(json_obj,filename,folder=''):
+ print('saving data')
+ with open(os.path.join(folder,filename), 'w') as fout:
+ #json.dump(list_of_data, fout, encoding='utf-8', indent = 4) #python 2
+ json.dump(json_obj, fout, indent = 4)
+
+def save_json_objs_to_files(json_objs,folder=''):
+ for json_obj in json_objs:
+ save_json_obj_to_file(json_obj,json_obj['filename'],folder)
+
+
+def add_meta_data_to_json_objects(json_object, keyname, keyvalue):
+ # add some meta data to json file
+
+ if type(json_object) == list and type(keyname) == list and type(keyvalue) == list:
+ for item_json, item_keyname, item_keyvalue in zip(json_object, keyname, keyvalue):
+ item_json[item_keyname] = item_keyvalue
+ print('added new meta data list', item_keyname, ' = ', item_keyvalue)
+ return json_object
+ else:
+ json_object[keyname] = keyvalue
+ print('added new meta data', keyname, ' = ', keyvalue)
+ return json_object
+
+
diff --git a/database_tools.py b/database_tools.py
@@ -0,0 +1,188 @@
+import pandas as pd
+import os
+import json
+import pprint
+import collections
+import pymongo
+from pymongo import MongoClient
+from bson.code import Code
+import socket
+
+def connect_to_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ defaults are provided for the names
+ '''
+ client = MongoClient('localhost', 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db
+
+def connect_to_docker_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ '''
+ host = socket.gethostbyname(socket.gethostname())
+ client = MongoClient(host, 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db 
+
+def delete_database(client, db_name='my_database'):
+ try:
+ client.drop_database(db_name)
+ print('database deleted')
+ except:
+ print('unable to delete database',db_name)
+
+def upload_json_objects_to_database(data,collection):
+
+ if type(data)==list:
+ for i, item in enumerate(data):
+ try:
+ print('inserting entry into database ',i)
+ collection.insert_many(item)
+ except:
+ print('FAILED inserting entry into database',i)
+ # print('failing json object is ' ,item)
+
+ else:
+ collection.insert_one(data)
+ print('json object commited to database')
+
+#def get_database_fields(collection, query_fields_and_values={},ignore_fields=[]):
+def get_database_fields(collection,ignore_fields=[]):
+ mapper = Code("""
+ function() {
+ for (var key in this) { emit(key, null); }
+ }
+ """)
+
+ reducer = Code("""
+ function(key, stuff) { return null; }
+ """)
+
+ mr = collection.map_reduce(map = mapper,
+ reduce = reducer,
+ #query = query_fields_and_values,
+ out = "my_collection" + "_keys")
+ unique_fields = []
+ for doc in mr.find():
+ if doc["_id"] not in ignore_fields:
+ if doc["_id"] != "_id":
+ unique_fields.append(doc["_id"])
+ return unique_fields
+
+def get_entries_in_field(collection, field, query=None):
+ if query != {}:
+ result = collection.distinct(field,query)
+ else:
+ result = collection.distinct(field)
+ return result
+
+def get_type_of_entries_in_field(collection, field, query=None):
+ return type(collection.find_one({field:{ '$exists': True }})[field]).__name__
+ #return type(collection.find_one()[field]).__name__
+
+def find_all_fields_types_in_database(collection):
+ all_fields = get_database_fields(collection)
+ field_and_type = {}
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ field_and_type[field] = type_of_field_contents
+ return field_and_type
+
+def find_all_fields_of_a_particular_types_in_database(collection,type_required):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents ==type_required:
+ field_of_correct_type.append(field)
+ return field_of_correct_type
+
+def find_all_fields_not_of_a_particular_types_in_database(collection,type_not_desired):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents !=type_not_desired:
+ field_of_correct_type.append(field)
+ return field_of_correct_type 
+
+def get_number_of_matching_entrys(collection, query_fields_and_values):
+ result = collection.find(query_fields_and_values)
+ return result.count()
+
+def get_matching_entrys(collection, query_fields_and_values,limit=10):
+ result = collection.find(query_fields_and_values).limit(limit)
+ return result
+
+def find_metadata_fields_and_their_distinct_values(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field]})
+ return meta_data_fields_and_distinct_entries
+
+
+def find_metadata_fields_and_their_distinct_values_and_types(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field],
+ 'type':type(metadata_fields_and_their_distinct_values[field])})
+ return meta_data_fields_and_distinct_entries
+
+
+if __name__ == '__main__':
+
+
+ collection, client, db = connect_to_database()
+ # collection, client, db = connect_to_docker_database()
+
+ query = {'mass number':5}
+
+ myresults=collection.find(query)
+
+ print('number of results found = ' ,myresults.count())
+
+ print('current reaction ',myresults[0]['reaction']) 
+
+
+
+
+
+ #collection.delete_one(myquery)
+
+
+ #results = get_number_of_matching_entrys(collection, {'uploader':'shimwell'})
+ # results = find_metadata_fields_and_their_distinct_values_and_types(collection)
+ # for result in results:
+ # print(result['type'])
+ # fields = get_database_fields(collection)
+ # for f in fields:
+ # results = get_entries_in_field(collection,f)
+ # type_of_results = get_type_of_entries_in_field(collection,f)
+ # print(f, type_of_results)
+ #print(find_all_fields_types_in_database(collection))
+ # results = collection.find_one({'temperature':{ '$exists': True }})['temperature']
+ # print(results)
+
+ #print(find_all_fields_of_a_particular_types_in_database(collection,'list'))
diff --git a/data_formatting_tools.py b/data_formatting_tools.py
@@ -0,0 +1,131 @@
+import collections
+import json
+import os
+import pprint
+
+import pandas as pd
+import pymongo
+#from bson.code import Code
+import xlrd
+from pymongo import MongoClient
+
+
+def find_files_recursive(folder,extension,ignore):
+
+ list_of_files=[]
+
+ for root, dirs, files in os.walk(folder):
+ for file in files:
+ if file.endswith(extension) and not file.endswith(ignore):
+ list_of_files.append(os.path.join(root,file))
+ print('number of files found',len(list_of_files))
+ return list_of_files
+
+def find_files(data_folders,extension):
+ ''' function searches through folders and finds files ending with .txt or .csv
+ input: folder(s) to search through
+ input type: list of strings
+ returns: list of matching filenames
+ '''
+ list_of_filenames =[]
+ for folder in data_folders:
+ print(folder)
+ for file in os.listdir(folder):
+ if file.endswith(extension):
+ filename = os.path.join(folder ,file)
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def convert_csv_files_to_dataframe_objects(filenames ,schema):
+ ''' Loads csv files into a pandas dataframe
+ checks that files match the schema
+ inputs: list of filenames, schema [optional]
+ input types: list of strings, list of strings
+ '''
+ list_of_dataframes =[]
+ for filename in filenames:
+ df = pd.read_csv(filename ,sep=',|\t')
+ for header in df.columns.values:
+ df.rename(columns={header: header.strip('\t .')}, inplace=True)
+ # print(df.columns.values)
+ if set(df.columns.values) == set(schema):
+ print('converting' ,filename, 'to dataframe')
+ list_of_dataframes.append(df)
+ else:
+ print("\x1b[31m\"" + str(filename) + ' does not conform to schema' + "\x1b[0m")
+ return list_of_dataframes
+
+def convert_dataframe_objects_to_json_files(list_of_dataframes, filenames):
+ ''' Converts pandas dataframe into a json file
+ inputs: list of dataframes
+ input types: list of dataframes
+ '''
+ list_of_filenames = []
+ for df, filename in zip(list_of_dataframes, filenames):
+ df.to_json(filename, orient='columns')
+ # df.to_json(filename, orient='columns', index=False) index setting is not currently supported, perhaps with the next release, see http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_json.html?highlight=to_json#pandas.DataFrame.to_json
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def apply_abs_operation_to_field(json_objects,field):
+ for json_object in json_objects:
+ json_object[field]=l = [abs(x) for x in json_object[field]]
+ return json_objects
+
+def read_in_json_files(filenames):
+ # read in json files
+ list_of_json_objects = []
+ for filename in filenames:
+ with open(filename) as f:
+ data = json.load(f)
+ data = remove_df_index_from_json_object(data)
+ list_of_json_objects.append(data)
+ return list_of_json_objects
+
+def convert_list_of_tuples_to_list(list_of_tuples):
+ list_of_single_values=[]
+ for x in range(0,len(list_of_tuples)):
+ list_of_single_values.append(list_of_tuples[str(x)])
+ return list_of_single_values
+
+def remove_df_index_from_json_object(data):
+ newdata={}
+ for key in data.keys():
+ #print(key)
+ list_of_tuples = data[key]
+ newdata[key] = convert_list_of_tuples_to_list(list_of_tuples) 
+ return newdata
+
+# def write_json_files(json_objects, filenames):
+# print('writing json files')
+# list_of_json_objects = []
+# for filename, json_object in zip(filenames, json_objects):
+# with open(filename, 'w') as outfile:
+# json.dump(json_object, outfile, indent=4)
+# return filenames
+
+def save_json_obj_to_file(json_obj,filename,folder=''):
+ print('saving data')
+ with open(os.path.join(folder,filename), 'w') as fout:
+ #json.dump(list_of_data, fout, encoding='utf-8', indent = 4) #python 2
+ json.dump(json_obj, fout, indent = 4)
+
+def save_json_objs_to_files(json_objs,folder=''):
+ for json_obj in json_objs:
+ save_json_obj_to_file(json_obj,json_obj['filename'],folder)
+
+
+def add_meta_data_to_json_objects(json_object, keyname, keyvalue):
+ # add some meta data to json file
+
+ if type(json_object) == list and type(keyname) == list and type(keyvalue) == list:
+ for item_json, item_keyname, item_keyvalue in zip(json_object, keyname, keyvalue):
+ item_json[item_keyname] = item_keyvalue
+ print('added new meta data list', item_keyname, ' = ', item_keyvalue)
+ return json_object
+ else:
+ json_object[keyname] = keyvalue
+ print('added new meta data', keyname, ' = ', keyvalue)
+ return json_object
+
+
diff --git a/database_tools.py b/database_tools.py
@@ -0,0 +1,188 @@
+import pandas as pd
+import os
+import json
+import pprint
+import collections
+import pymongo
+from pymongo import MongoClient
+from bson.code import Code
+import socket
+
+def connect_to_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ defaults are provided for the names
+ '''
+ client = MongoClient('localhost', 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db
+
+def connect_to_docker_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ '''
+ host = socket.gethostbyname(socket.gethostname())
+ client = MongoClient(host, 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db 
+
+def delete_database(client, db_name='my_database'):
+ try:
+ client.drop_database(db_name)
+ print('database deleted')
+ except:
+ print('unable to delete database',db_name)
+
+def upload_json_objects_to_database(data,collection):
+
+ if type(data)==list:
+ for i, item in enumerate(data):
+ try:
+ print('inserting entry into database ',i)
+ collection.insert_many(item)
+ except:
+ print('FAILED inserting entry into database',i)
+ # print('failing json object is ' ,item)
+
+ else:
+ collection.insert_one(data)
+ print('json object commited to database')
+
+#def get_database_fields(collection, query_fields_and_values={},ignore_fields=[]):
+def get_database_fields(collection,ignore_fields=[]):
+ mapper = Code("""
+ function() {
+ for (var key in this) { emit(key, null); }
+ }
+ """)
+
+ reducer = Code("""
+ function(key, stuff) { return null; }
+ """)
+
+ mr = collection.map_reduce(map = mapper,
+ reduce = reducer,
+ #query = query_fields_and_values,
+ out = "my_collection" + "_keys")
+ unique_fields = []
+ for doc in mr.find():
+ if doc["_id"] not in ignore_fields:
+ if doc["_id"] != "_id":
+ unique_fields.append(doc["_id"])
+ return unique_fields
+
+def get_entries_in_field(collection, field, query=None):
+ if query != {}:
+ result = collection.distinct(field,query)
+ else:
+ result = collection.distinct(field)
+ return result
+
+def get_type_of_entries_in_field(collection, field, query=None):
+ return type(collection.find_one({field:{ '$exists': True }})[field]).__name__
+ #return type(collection.find_one()[field]).__name__
+
+def find_all_fields_types_in_database(collection):
+ all_fields = get_database_fields(collection)
+ field_and_type = {}
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ field_and_type[field] = type_of_field_contents
+ return field_and_type
+
+def find_all_fields_of_a_particular_types_in_database(collection,type_required):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents ==type_required:
+ field_of_correct_type.append(field)
+ return field_of_correct_type
+
+def find_all_fields_not_of_a_particular_types_in_database(collection,type_not_desired):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents !=type_not_desired:
+ field_of_correct_type.append(field)
+ return field_of_correct_type 
+
+def get_number_of_matching_entrys(collection, query_fields_and_values):
+ result = collection.find(query_fields_and_values)
+ return result.count()
+
+def get_matching_entrys(collection, query_fields_and_values,limit=10):
+ result = collection.find(query_fields_and_values).limit(limit)
+ return result
+
+def find_metadata_fields_and_their_distinct_values(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field]})
+ return meta_data_fields_and_distinct_entries
+
+
+def find_metadata_fields_and_their_distinct_values_and_types(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field],
+ 'type':type(metadata_fields_and_their_distinct_values[field])})
+ return meta_data_fields_and_distinct_entries
+
+
+if __name__ == '__main__':
+
+
+ collection, client, db = connect_to_database()
+ # collection, client, db = connect_to_docker_database()
+
+ query = {'mass number':5}
+
+ myresults=collection.find(query)
+
+ print('number of results found = ' ,myresults.count())
+
+ print('current reaction ',myresults[0]['reaction']) 
+
+
+
+
+
+ #collection.delete_one(myquery)
+
+
+ #results = get_number_of_matching_entrys(collection, {'uploader':'shimwell'})
+ # results = find_metadata_fields_and_their_distinct_values_and_types(collection)
+ # for result in results:
+ # print(result['type'])
+ # fields = get_database_fields(collection)
+ # for f in fields:
+ # results = get_entries_in_field(collection,f)
+ # type_of_results = get_type_of_entries_in_field(collection,f)
+ # print(f, type_of_results)
+ #print(find_all_fields_types_in_database(collection))
+ # results = collection.find_one({'temperature':{ '$exists': True }})['temperature']
+ # print(results)
+
+ #print(find_all_fields_of_a_particular_types_in_database(collection,'list'))
diff --git a/data_formatting_tools.py b/data_formatting_tools.py
@@ -0,0 +1,131 @@
+import collections
+import json
+import os
+import pprint
+
+import pandas as pd
+import pymongo
+#from bson.code import Code
+import xlrd
+from pymongo import MongoClient
+
+
+def find_files_recursive(folder,extension,ignore):
+
+ list_of_files=[]
+
+ for root, dirs, files in os.walk(folder):
+ for file in files:
+ if file.endswith(extension) and not file.endswith(ignore):
+ list_of_files.append(os.path.join(root,file))
+ print('number of files found',len(list_of_files))
+ return list_of_files
+
+def find_files(data_folders,extension):
+ ''' function searches through folders and finds files ending with .txt or .csv
+ input: folder(s) to search through
+ input type: list of strings
+ returns: list of matching filenames
+ '''
+ list_of_filenames =[]
+ for folder in data_folders:
+ print(folder)
+ for file in os.listdir(folder):
+ if file.endswith(extension):
+ filename = os.path.join(folder ,file)
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def convert_csv_files_to_dataframe_objects(filenames ,schema):
+ ''' Loads csv files into a pandas dataframe
+ checks that files match the schema
+ inputs: list of filenames, schema [optional]
+ input types: list of strings, list of strings
+ '''
+ list_of_dataframes =[]
+ for filename in filenames:
+ df = pd.read_csv(filename ,sep=',|\t')
+ for header in df.columns.values:
+ df.rename(columns={header: header.strip('\t .')}, inplace=True)
+ # print(df.columns.values)
+ if set(df.columns.values) == set(schema):
+ print('converting' ,filename, 'to dataframe')
+ list_of_dataframes.append(df)
+ else:
+ print("\x1b[31m\"" + str(filename) + ' does not conform to schema' + "\x1b[0m")
+ return list_of_dataframes
+
+def convert_dataframe_objects_to_json_files(list_of_dataframes, filenames):
+ ''' Converts pandas dataframe into a json file
+ inputs: list of dataframes
+ input types: list of dataframes
+ '''
+ list_of_filenames = []
+ for df, filename in zip(list_of_dataframes, filenames):
+ df.to_json(filename, orient='columns')
+ # df.to_json(filename, orient='columns', index=False) index setting is not currently supported, perhaps with the next release, see http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_json.html?highlight=to_json#pandas.DataFrame.to_json
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def apply_abs_operation_to_field(json_objects,field):
+ for json_object in json_objects:
+ json_object[field]=l = [abs(x) for x in json_object[field]]
+ return json_objects
+
+def read_in_json_files(filenames):
+ # read in json files
+ list_of_json_objects = []
+ for filename in filenames:
+ with open(filename) as f:
+ data = json.load(f)
+ data = remove_df_index_from_json_object(data)
+ list_of_json_objects.append(data)
+ return list_of_json_objects
+
+def convert_list_of_tuples_to_list(list_of_tuples):
+ list_of_single_values=[]
+ for x in range(0,len(list_of_tuples)):
+ list_of_single_values.append(list_of_tuples[str(x)])
+ return list_of_single_values
+
+def remove_df_index_from_json_object(data):
+ newdata={}
+ for key in data.keys():
+ #print(key)
+ list_of_tuples = data[key]
+ newdata[key] = convert_list_of_tuples_to_list(list_of_tuples) 
+ return newdata
+
+# def write_json_files(json_objects, filenames):
+# print('writing json files')
+# list_of_json_objects = []
+# for filename, json_object in zip(filenames, json_objects):
+# with open(filename, 'w') as outfile:
+# json.dump(json_object, outfile, indent=4)
+# return filenames
+
+def save_json_obj_to_file(json_obj,filename,folder=''):
+ print('saving data')
+ with open(os.path.join(folder,filename), 'w') as fout:
+ #json.dump(list_of_data, fout, encoding='utf-8', indent = 4) #python 2
+ json.dump(json_obj, fout, indent = 4)
+
+def save_json_objs_to_files(json_objs,folder=''):
+ for json_obj in json_objs:
+ save_json_obj_to_file(json_obj,json_obj['filename'],folder)
+
+
+def add_meta_data_to_json_objects(json_object, keyname, keyvalue):
+ # add some meta data to json file
+
+ if type(json_object) == list and type(keyname) == list and type(keyvalue) == list:
+ for item_json, item_keyname, item_keyvalue in zip(json_object, keyname, keyvalue):
+ item_json[item_keyname] = item_keyvalue
+ print('added new meta data list', item_keyname, ' = ', item_keyvalue)
+ return json_object
+ else:
+ json_object[keyname] = keyvalue
+ print('added new meta data', keyname, ' = ', keyvalue)
+ return json_object
+
+
diff --git a/database_tools.py b/database_tools.py
@@ -0,0 +1,188 @@
+import pandas as pd
+import os
+import json
+import pprint
+import collections
+import pymongo
+from pymongo import MongoClient
+from bson.code import Code
+import socket
+
+def connect_to_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ defaults are provided for the names
+ '''
+ client = MongoClient('localhost', 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db
+
+def connect_to_docker_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ '''
+ host = socket.gethostbyname(socket.gethostname())
+ client = MongoClient(host, 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db 
+
+def delete_database(client, db_name='my_database'):
+ try:
+ client.drop_database(db_name)
+ print('database deleted')
+ except:
+ print('unable to delete database',db_name)
+
+def upload_json_objects_to_database(data,collection):
+
+ if type(data)==list:
+ for i, item in enumerate(data):
+ try:
+ print('inserting entry into database ',i)
+ collection.insert_many(item)
+ except:
+ print('FAILED inserting entry into database',i)
+ # print('failing json object is ' ,item)
+
+ else:
+ collection.insert_one(data)
+ print('json object commited to database')
+
+#def get_database_fields(collection, query_fields_and_values={},ignore_fields=[]):
+def get_database_fields(collection,ignore_fields=[]):
+ mapper = Code("""
+ function() {
+ for (var key in this) { emit(key, null); }
+ }
+ """)
+
+ reducer = Code("""
+ function(key, stuff) { return null; }
+ """)
+
+ mr = collection.map_reduce(map = mapper,
+ reduce = reducer,
+ #query = query_fields_and_values,
+ out = "my_collection" + "_keys")
+ unique_fields = []
+ for doc in mr.find():
+ if doc["_id"] not in ignore_fields:
+ if doc["_id"] != "_id":
+ unique_fields.append(doc["_id"])
+ return unique_fields
+
+def get_entries_in_field(collection, field, query=None):
+ if query != {}:
+ result = collection.distinct(field,query)
+ else:
+ result = collection.distinct(field)
+ return result
+
+def get_type_of_entries_in_field(collection, field, query=None):
+ return type(collection.find_one({field:{ '$exists': True }})[field]).__name__
+ #return type(collection.find_one()[field]).__name__
+
+def find_all_fields_types_in_database(collection):
+ all_fields = get_database_fields(collection)
+ field_and_type = {}
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ field_and_type[field] = type_of_field_contents
+ return field_and_type
+
+def find_all_fields_of_a_particular_types_in_database(collection,type_required):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents ==type_required:
+ field_of_correct_type.append(field)
+ return field_of_correct_type
+
+def find_all_fields_not_of_a_particular_types_in_database(collection,type_not_desired):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents !=type_not_desired:
+ field_of_correct_type.append(field)
+ return field_of_correct_type 
+
+def get_number_of_matching_entrys(collection, query_fields_and_values):
+ result = collection.find(query_fields_and_values)
+ return result.count()
+
+def get_matching_entrys(collection, query_fields_and_values,limit=10):
+ result = collection.find(query_fields_and_values).limit(limit)
+ return result
+
+def find_metadata_fields_and_their_distinct_values(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field]})
+ return meta_data_fields_and_distinct_entries
+
+
+def find_metadata_fields_and_their_distinct_values_and_types(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field],
+ 'type':type(metadata_fields_and_their_distinct_values[field])})
+ return meta_data_fields_and_distinct_entries
+
+
+if __name__ == '__main__':
+
+
+ collection, client, db = connect_to_database()
+ # collection, client, db = connect_to_docker_database()
+
+ query = {'mass number':5}
+
+ myresults=collection.find(query)
+
+ print('number of results found = ' ,myresults.count())
+
+ print('current reaction ',myresults[0]['reaction']) 
+
+
+
+
+
+ #collection.delete_one(myquery)
+
+
+ #results = get_number_of_matching_entrys(collection, {'uploader':'shimwell'})
+ # results = find_metadata_fields_and_their_distinct_values_and_types(collection)
+ # for result in results:
+ # print(result['type'])
+ # fields = get_database_fields(collection)
+ # for f in fields:
+ # results = get_entries_in_field(collection,f)
+ # type_of_results = get_type_of_entries_in_field(collection,f)
+ # print(f, type_of_results)
+ #print(find_all_fields_types_in_database(collection))
+ # results = collection.find_one({'temperature':{ '$exists': True }})['temperature']
+ # print(results)
+
+ #print(find_all_fields_of_a_particular_types_in_database(collection,'list'))
diff --git a/data_formatting_tools.py b/data_formatting_tools.py
@@ -0,0 +1,131 @@
+import collections
+import json
+import os
+import pprint
+
+import pandas as pd
+import pymongo
+#from bson.code import Code
+import xlrd
+from pymongo import MongoClient
+
+
+def find_files_recursive(folder,extension,ignore):
+
+ list_of_files=[]
+
+ for root, dirs, files in os.walk(folder):
+ for file in files:
+ if file.endswith(extension) and not file.endswith(ignore):
+ list_of_files.append(os.path.join(root,file))
+ print('number of files found',len(list_of_files))
+ return list_of_files
+
+def find_files(data_folders,extension):
+ ''' function searches through folders and finds files ending with .txt or .csv
+ input: folder(s) to search through
+ input type: list of strings
+ returns: list of matching filenames
+ '''
+ list_of_filenames =[]
+ for folder in data_folders:
+ print(folder)
+ for file in os.listdir(folder):
+ if file.endswith(extension):
+ filename = os.path.join(folder ,file)
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def convert_csv_files_to_dataframe_objects(filenames ,schema):
+ ''' Loads csv files into a pandas dataframe
+ checks that files match the schema
+ inputs: list of filenames, schema [optional]
+ input types: list of strings, list of strings
+ '''
+ list_of_dataframes =[]
+ for filename in filenames:
+ df = pd.read_csv(filename ,sep=',|\t')
+ for header in df.columns.values:
+ df.rename(columns={header: header.strip('\t .')}, inplace=True)
+ # print(df.columns.values)
+ if set(df.columns.values) == set(schema):
+ print('converting' ,filename, 'to dataframe')
+ list_of_dataframes.append(df)
+ else:
+ print("\x1b[31m\"" + str(filename) + ' does not conform to schema' + "\x1b[0m")
+ return list_of_dataframes
+
+def convert_dataframe_objects_to_json_files(list_of_dataframes, filenames):
+ ''' Converts pandas dataframe into a json file
+ inputs: list of dataframes
+ input types: list of dataframes
+ '''
+ list_of_filenames = []
+ for df, filename in zip(list_of_dataframes, filenames):
+ df.to_json(filename, orient='columns')
+ # df.to_json(filename, orient='columns', index=False) index setting is not currently supported, perhaps with the next release, see http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.to_json.html?highlight=to_json#pandas.DataFrame.to_json
+ list_of_filenames.append(filename)
+ return list_of_filenames
+
+def apply_abs_operation_to_field(json_objects,field):
+ for json_object in json_objects:
+ json_object[field]=l = [abs(x) for x in json_object[field]]
+ return json_objects
+
+def read_in_json_files(filenames):
+ # read in json files
+ list_of_json_objects = []
+ for filename in filenames:
+ with open(filename) as f:
+ data = json.load(f)
+ data = remove_df_index_from_json_object(data)
+ list_of_json_objects.append(data)
+ return list_of_json_objects
+
+def convert_list_of_tuples_to_list(list_of_tuples):
+ list_of_single_values=[]
+ for x in range(0,len(list_of_tuples)):
+ list_of_single_values.append(list_of_tuples[str(x)])
+ return list_of_single_values
+
+def remove_df_index_from_json_object(data):
+ newdata={}
+ for key in data.keys():
+ #print(key)
+ list_of_tuples = data[key]
+ newdata[key] = convert_list_of_tuples_to_list(list_of_tuples) 
+ return newdata
+
+# def write_json_files(json_objects, filenames):
+# print('writing json files')
+# list_of_json_objects = []
+# for filename, json_object in zip(filenames, json_objects):
+# with open(filename, 'w') as outfile:
+# json.dump(json_object, outfile, indent=4)
+# return filenames
+
+def save_json_obj_to_file(json_obj,filename,folder=''):
+ print('saving data')
+ with open(os.path.join(folder,filename), 'w') as fout:
+ #json.dump(list_of_data, fout, encoding='utf-8', indent = 4) #python 2
+ json.dump(json_obj, fout, indent = 4)
+
+def save_json_objs_to_files(json_objs,folder=''):
+ for json_obj in json_objs:
+ save_json_obj_to_file(json_obj,json_obj['filename'],folder)
+
+
+def add_meta_data_to_json_objects(json_object, keyname, keyvalue):
+ # add some meta data to json file
+
+ if type(json_object) == list and type(keyname) == list and type(keyvalue) == list:
+ for item_json, item_keyname, item_keyvalue in zip(json_object, keyname, keyvalue):
+ item_json[item_keyname] = item_keyvalue
+ print('added new meta data list', item_keyname, ' = ', item_keyvalue)
+ return json_object
+ else:
+ json_object[keyname] = keyvalue
+ print('added new meta data', keyname, ' = ', keyvalue)
+ return json_object
+
+
diff --git a/database_tools.py b/database_tools.py
@@ -0,0 +1,188 @@
+import pandas as pd
+import os
+import json
+import pprint
+import collections
+import pymongo
+from pymongo import MongoClient
+from bson.code import Code
+import socket
+
+def connect_to_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ defaults are provided for the names
+ '''
+ client = MongoClient('localhost', 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db
+
+def connect_to_docker_database(db_name='my_database',collection_name='collection_one'):
+ ''' Creates a local MongoDB database called my_database and connects to it
+ '''
+ host = socket.gethostbyname(socket.gethostname())
+ client = MongoClient(host, 27017)
+ db = client[db_name]
+ collection = db[collection_name]
+ print('connected to database')
+ return collection, client, db 
+
+def delete_database(client, db_name='my_database'):
+ try:
+ client.drop_database(db_name)
+ print('database deleted')
+ except:
+ print('unable to delete database',db_name)
+
+def upload_json_objects_to_database(data,collection):
+
+ if type(data)==list:
+ for i, item in enumerate(data):
+ try:
+ print('inserting entry into database ',i)
+ collection.insert_many(item)
+ except:
+ print('FAILED inserting entry into database',i)
+ # print('failing json object is ' ,item)
+
+ else:
+ collection.insert_one(data)
+ print('json object commited to database')
+
+#def get_database_fields(collection, query_fields_and_values={},ignore_fields=[]):
+def get_database_fields(collection,ignore_fields=[]):
+ mapper = Code("""
+ function() {
+ for (var key in this) { emit(key, null); }
+ }
+ """)
+
+ reducer = Code("""
+ function(key, stuff) { return null; }
+ """)
+
+ mr = collection.map_reduce(map = mapper,
+ reduce = reducer,
+ #query = query_fields_and_values,
+ out = "my_collection" + "_keys")
+ unique_fields = []
+ for doc in mr.find():
+ if doc["_id"] not in ignore_fields:
+ if doc["_id"] != "_id":
+ unique_fields.append(doc["_id"])
+ return unique_fields
+
+def get_entries_in_field(collection, field, query=None):
+ if query != {}:
+ result = collection.distinct(field,query)
+ else:
+ result = collection.distinct(field)
+ return result
+
+def get_type_of_entries_in_field(collection, field, query=None):
+ return type(collection.find_one({field:{ '$exists': True }})[field]).__name__
+ #return type(collection.find_one()[field]).__name__
+
+def find_all_fields_types_in_database(collection):
+ all_fields = get_database_fields(collection)
+ field_and_type = {}
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ field_and_type[field] = type_of_field_contents
+ return field_and_type
+
+def find_all_fields_of_a_particular_types_in_database(collection,type_required):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents ==type_required:
+ field_of_correct_type.append(field)
+ return field_of_correct_type
+
+def find_all_fields_not_of_a_particular_types_in_database(collection,type_not_desired):
+ all_fields = get_database_fields(collection)
+ field_of_correct_type = []
+ for field in all_fields:
+ type_of_field_contents = get_type_of_entries_in_field(collection,field)
+ if type_of_field_contents !=type_not_desired:
+ field_of_correct_type.append(field)
+ return field_of_correct_type 
+
+def get_number_of_matching_entrys(collection, query_fields_and_values):
+ result = collection.find(query_fields_and_values)
+ return result.count()
+
+def get_matching_entrys(collection, query_fields_and_values,limit=10):
+ result = collection.find(query_fields_and_values).limit(limit)
+ return result
+
+def find_metadata_fields_and_their_distinct_values(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field]})
+ return meta_data_fields_and_distinct_entries
+
+
+def find_metadata_fields_and_their_distinct_values_and_types(collection, ignore_fields=[]):
+ meta_data_fields = get_database_fields(collection, ignore_fields)
+ metadata_fields_and_their_distinct_values={}
+ for entry in meta_data_fields:
+ values = get_entries_in_field(collection,entry)
+ # metadata_values.append(values)
+ metadata_fields_and_their_distinct_values[entry]=values
+
+
+
+ meta_data_fields_and_distinct_entries = []
+ for field in meta_data_fields:
+ meta_data_fields_and_distinct_entries.append({'field':[field],
+ 'distinct_values':metadata_fields_and_their_distinct_values[field],
+ 'type':type(metadata_fields_and_their_distinct_values[field])})
+ return meta_data_fields_and_distinct_entries
+
+
+if __name__ == '__main__':
+
+
+ collection, client, db = connect_to_database()
+ # collection, client, db = connect_to_docker_database()
+
+ query = {'mass number':5}
+
+ myresults=collection.find(query)
+
+ print('number of results found = ' ,myresults.count())
+
+ print('current reaction ',myresults[0]['reaction']) 
+
+
+
+
+
+ #collection.delete_one(myquery)
+
+
+ #results = get_number_of_matching_entrys(collection, {'uploader':'shimwell'})
+ # results = find_metadata_fields_and_their_distinct_values_and_types(collection)
+ # for result in results:
+ # print(result['type'])
+ # fields = get_database_fields(collection)
+ # for f in fields:
+ # results = get_entries_in_field(collection,f)
+ # type_of_results = get_type_of_entries_in_field(collection,f)
+ # print(f, type_of_results)
+ #print(find_all_fields_types_in_database(collection))
+ # results = collection.find_one({'temperature':{ '$exists': True }})['temperature']
+ # print(results)
+
+ #print(find_all_fields_of_a_particular_types_in_database(collection,'list'))