From 5875f80beb74102c653d312246d12f6fe09d149a Mon Sep 17 00:00:00 2001 From: dimitri-yatsenko Date: Thu, 7 Jul 2016 20:26:23 -0500 Subject: [PATCH 1/6] added datajoint.set_password and BaseRelation.reverse_engineer --- datajoint/__init__.py | 4 +++- datajoint/admin.py | 6 +++++ datajoint/base_relation.py | 49 ++++++++++++++++++++++++++++++++++++++ datajoint/connection.py | 3 +++ datajoint/declare.py | 6 ++++- datajoint/schema.py | 10 ++++---- 6 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 datajoint/admin.py diff --git a/datajoint/__init__.py b/datajoint/__init__.py index 83a29ae69..a45c4929f 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -22,7 +22,8 @@ 'config', 'conn', 'kill', 'BaseRelation', 'Connection', 'Heading', 'FreeRelation', 'Not', 'schema', 'Manual', 'Lookup', 'Imported', 'Computed', 'Part', - 'AndList', 'OrList', 'ERD', 'U'] + 'AndList', 'OrList', 'ERD', 'U', + 'set_password'] print('DataJoint', __version__, '('+__date__+')') @@ -81,3 +82,4 @@ class DataJointError(Exception): from .schema import Schema as schema from .kill import kill from .erd import ERD +from .admin import set_password diff --git a/datajoint/admin.py b/datajoint/admin.py new file mode 100644 index 000000000..cd85df369 --- /dev/null +++ b/datajoint/admin.py @@ -0,0 +1,6 @@ +from . import conn + + +def set_password(new_password, connection=conn()): + connection.query("SET PASSWORD = PASSWORD('%s')" % new_password) + print('done.') diff --git a/datajoint/base_relation.py b/datajoint/base_relation.py index 1889c62f6..cf7cb805c 100644 --- a/datajoint/base_relation.py +++ b/datajoint/base_relation.py @@ -1,5 +1,6 @@ import collections import itertools +import inspect import numpy as np import logging from . import config, DataJointError @@ -314,6 +315,52 @@ def size_on_disk(self): database=self.database, table=self.table_name), as_dict=True).fetchone() return ret['Data_length'] + ret['Index_length'] + @property + def real_definition(self): + """ + :return: the definition string for the relation using DataJoint DDL. + This does not yet work for aliased foreign keys. + """ + self.connection.dependencies.load() + parents = {r: FreeRelation(self.connection, r).primary_key for r in self.parents()} + in_key = True + definition = '# ' + self.heading.table_info['comment'] + '\n' + attributes_thus_far = set() + for attr in self.heading.attributes.values(): + if in_key and not attr.in_key: + definition += '---\n' + in_key = False + attributes_thus_far.add(attr.name) + do_include = True + for parent, primary_key in list(parents.items()): + if attributes_thus_far.issuperset(primary_key): + parents.pop(parent) + definition += '-> ' + self.lookup_table_name(parent) + '\n' + do_include = False + if do_include: + definition += '%-20s : %-28s # %s\n' % ( + attr.name if attr.default is None else '%s="%s"' % (attr.name, attr.default), + '%s%s' % (attr.type, 'auto_increment' if attr.autoincrement else ''), attr.comment) + return definition + + def lookup_table_name(self, name): + """ + given the name of another table in the form `database`.`table_name`, find its class in the context + :param name: `database`.`table_name` + :return: class name found in the context. + """ + def _lookup(context, name): + for member_name, member in context.items(): + if inspect.isclass(member) and issubclass(member, BaseRelation) and member.full_table_name == name: + return member_name + if inspect.ismodule(member) and member.__name__ != 'datajoint': + candidate_name = _lookup(dict(inspect.getmembers(member)), name) + if candidate_name != name: + return member_name + '.' + candidate_name + return name + + return _lookup(self._context, name) + class FreeRelation(BaseRelation): """ @@ -342,3 +389,5 @@ def table_name(self): :return: the table name in the database """ return self._table_name + + diff --git a/datajoint/connection.py b/datajoint/connection.py index d8a006147..ec5f331f1 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -189,3 +189,6 @@ def transaction(self): raise else: self.commit_transaction() + + + diff --git a/datajoint/declare.py b/datajoint/declare.py index d01bb87a2..c3c3eb387 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -161,7 +161,11 @@ def compile_attribute(line, in_key=False): :returns: (name, sql) -- attribute name and sql code for its declaration """ - match = attribute_parser.parseString(line+'#', parseAll=True) + try: + match = attribute_parser.parseString(line+'#', parseAll=True) + except pp.ParseException: + logger.error('Declaration error in line: ', line) + raise match['comment'] = match['comment'].rstrip('#') if 'default' not in match: match['default'] = '' diff --git a/datajoint/schema.py b/datajoint/schema.py index a2d26de78..4a44282a1 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -2,13 +2,12 @@ import pymysql import logging +import inspect import re from . import conn, DataJointError, config -from datajoint.utils import to_camel_case from .heading import Heading -from .utils import user_choice -from .user_relations import Part, Computed, Imported, Manual, Lookup -import inspect +from .utils import user_choice, to_camel_case +from .user_relations import UserRelation, Part, Computed, Imported, Manual, Lookup logger = logging.getLogger(__name__) @@ -119,6 +118,7 @@ def exists(self): cur = self.connection.query("SHOW DATABASES LIKE '{database}'".format(database=self.database)) return cur.rowcount > 0 + def process_relation_class(self, relation_class, context, assert_declared=False): """ assign schema properties to the relation class and declare the table @@ -142,6 +142,7 @@ def process_relation_class(self, relation_class, context, assert_declared=False) else: instance.insert(contents, skip_duplicates=True) + def __call__(self, cls): """ Binds the passed in class object to a database. This is intended to be used as a decorator. @@ -170,3 +171,4 @@ def jobs(self): :return: jobs relation """ return self.connection.jobs[self.database] + From dfeb4941b240c52f032d14e6f6198cea9fb577f4 Mon Sep 17 00:00:00 2001 From: dimitri-yatsenko Date: Thu, 7 Jul 2016 20:27:26 -0500 Subject: [PATCH 2/6] version 0.3.0 --- datajoint/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datajoint/__init__.py b/datajoint/__init__.py index a45c4929f..bf02a9afc 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -16,8 +16,8 @@ import os __author__ = "Dimitri Yatsenko, Edgar Walker, and Fabian Sinz at Baylor College of Medicine" -__version__ = "0.2.8" -__date__ = "July 1, 2016" +__version__ = "0.3.0" +__date__ = "July 7, 2016" __all__ = ['__author__', '__version__', 'config', 'conn', 'kill', 'BaseRelation', 'Connection', 'Heading', 'FreeRelation', 'Not', 'schema', From 304dd5d01fc7f46d2660c2cb0770b26589f82578 Mon Sep 17 00:00:00 2001 From: dimitri-yatsenko Date: Thu, 14 Jul 2016 19:58:36 -0500 Subject: [PATCH 3/6] added test for BaseRelation.real_definition --- datajoint/base_relation.py | 8 ++++---- datajoint/declare.py | 15 ++++++--------- datajoint/heading.py | 27 ++++++++++++--------------- datajoint/schema.py | 1 - tests/test_declare.py | 9 +++++++++ 5 files changed, 31 insertions(+), 29 deletions(-) diff --git a/datajoint/base_relation.py b/datajoint/base_relation.py index cf7cb805c..01ba8dbb3 100644 --- a/datajoint/base_relation.py +++ b/datajoint/base_relation.py @@ -339,7 +339,7 @@ def real_definition(self): do_include = False if do_include: definition += '%-20s : %-28s # %s\n' % ( - attr.name if attr.default is None else '%s="%s"' % (attr.name, attr.default), + attr.name if attr.default is None else '%s=%s' % (attr.name, attr.default), '%s%s' % (attr.type, 'auto_increment' if attr.autoincrement else ''), attr.comment) return definition @@ -349,12 +349,12 @@ def lookup_table_name(self, name): :param name: `database`.`table_name` :return: class name found in the context. """ - def _lookup(context, name): + def _lookup(context, name, level=0): for member_name, member in context.items(): if inspect.isclass(member) and issubclass(member, BaseRelation) and member.full_table_name == name: return member_name - if inspect.ismodule(member) and member.__name__ != 'datajoint': - candidate_name = _lookup(dict(inspect.getmembers(member)), name) + if level < 5 and inspect.ismodule(member) and member.__name__ != 'datajoint': + candidate_name = _lookup(dict(inspect.getmembers(member)), name, level+1) if candidate_name != name: return member_name + '.' + candidate_name return name diff --git a/datajoint/declare.py b/datajoint/declare.py index c3c3eb387..90fbf9955 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -141,15 +141,12 @@ def declare(full_table_name, definition, context): # compile SQL if not primary_key: raise DataJointError('Table must have a primary key') - sql = 'CREATE TABLE IF NOT EXISTS %s (\n ' % full_table_name - sql += ',\n '.join(attribute_sql) - sql += ',\n PRIMARY KEY (`' + '`,`'.join(primary_key) + '`)' - if foreign_key_sql: - sql += ', \n' + ', \n'.join(foreign_key_sql) - if index_sql: - sql += ', \n' + ', \n'.join(index_sql) - sql += '\n) ENGINE = InnoDB, COMMENT "%s"' % table_comment - return sql + return ('CREATE TABLE IF NOT EXISTS %s (\n' % full_table_name + + ',\n'.join(attribute_sql + + ['PRIMARY KEY (`' + '`,`'.join(primary_key) + '`)'] + + foreign_key_sql + + index_sql) + + '\n) ENGINE=InnoDB, COMMENT "%s"' % table_comment) def compile_attribute(line, in_key=False): diff --git a/datajoint/heading.py b/datajoint/heading.py index fcfcb9fc0..1b50c778d 100644 --- a/datajoint/heading.py +++ b/datajoint/heading.py @@ -19,22 +19,13 @@ def todict(self): @property def sql(self): """ - Convert attribute tuple into its SQL CREATE TABLE clause. + Convert primary key attribute tuple into its SQL CREATE TABLE clause. + Default values are not reflected. :return: SQL code """ - sql_literals = ['CURRENT_TIMESTAMP'] # SQL literals that can be used as default values - if self.nullable: - default = 'DEFAULT NULL' - else: - default = 'NOT NULL' - if self.default: - # enclose value in quotes except special SQL values or already enclosed - quote = self.default.upper() not in sql_literals and self.default[0] not in '"\'' - default += ' DEFAULT ' + ('"%s"' if quote else "%s") % self.default - if any(c in r'\"' for c in self.comment): - raise DataJointError('Illegal characters in attribute comment "%s"' % self.comment) - return '`{name}` {type} {default} COMMENT "{comment}"'.format( - name=self.name, type=self.type, default=default, comment=self.comment) + assert self.in_key and not self.nullable # primary key attributes are never nullable + return '`{name}` {type} NOT NULL COMMENT "{comment}"'.format( + name=self.name, type=self.type, comment=self.comment) class Heading: @@ -99,7 +90,7 @@ def __repr__(self): ret += '---\n' in_key = False ret += '%-20s : %-28s # %s\n' % ( - v.name if v.default is None else '%s="%s"' % (v.name, v.default), + v.name if v.default is None else '%s=%s' % (v.name, v.default), '%s%s' % (v.type, 'auto_increment' if v.autoincrement else ''), v.comment) return ret @@ -178,15 +169,21 @@ def init_from_database(self, conn, database, table_name): # TODO: include types DECIMAL and NUMERIC } + sql_literals = ['CURRENT_TIMESTAMP'] + # additional attribute properties for attr in attributes: attr['nullable'] = (attr['nullable'] == 'YES') attr['in_key'] = (attr['in_key'] == 'PRI') attr['autoincrement'] = bool(re.search(r'auto_increment', attr['Extra'], flags=re.IGNORECASE)) + attr['type'] = re.sub(r'int\(\d+\)', 'int', attr['type'], count=1) # strip size off integers attr['numeric'] = bool(re.match(r'(tiny|small|medium|big)?int|decimal|double|float', attr['type'])) attr['string'] = bool(re.match(r'(var)?char|enum|date|time|timestamp', attr['type'])) attr['is_blob'] = bool(re.match(r'(tiny|medium|long)?blob', attr['type'])) + if attr['string'] and attr['default'] is not None and attr['default'] not in sql_literals: + attr['default'] = '"%s"' % attr['default'] + attr['sql_expression'] = None if not (attr['numeric'] or attr['string'] or attr['is_blob']): raise DataJointError('Unsupported field type {field} in `{database}`.`{table_name}`'.format( diff --git a/datajoint/schema.py b/datajoint/schema.py index 4a44282a1..a121df64b 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -171,4 +171,3 @@ def jobs(self): :return: jobs relation """ return self.connection.jobs[self.database] - diff --git a/tests/test_declare.py b/tests/test_declare.py index b78b02572..74081fd3b 100644 --- a/tests/test_declare.py +++ b/tests/test_declare.py @@ -1,3 +1,4 @@ +import re from nose.tools import assert_true, assert_false, assert_equal, assert_list_equal, raises from . import schema import datajoint as dj @@ -19,6 +20,14 @@ def test_schema_decorator(): assert_true(issubclass(schema.Subject, dj.Manual)) assert_true(not issubclass(schema.Subject, dj.Part)) + @staticmethod + def test_real_definition(): + """real_definition should match original definition""" + rel = schema.Experiment() + s1 = re.sub('r\s+', rel.definition, '') + s2 = re.sub('r\s+', rel.real_definition, '') + assert_equal(s1, s2) + @staticmethod def test_attributes(): # test autoincrement declaration From dbd1a2be8cad61a7f34d236ecdc9897fc330ac05 Mon Sep 17 00:00:00 2001 From: dimitri-yatsenko Date: Thu, 14 Jul 2016 20:07:50 -0500 Subject: [PATCH 4/6] updated test for BaseRelation.real_definition --- datajoint/__init__.py | 2 +- tests/test_declare.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/datajoint/__init__.py b/datajoint/__init__.py index bf02a9afc..1a34177ec 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -17,7 +17,7 @@ __author__ = "Dimitri Yatsenko, Edgar Walker, and Fabian Sinz at Baylor College of Medicine" __version__ = "0.3.0" -__date__ = "July 7, 2016" +__date__ = "July 14, 2016" __all__ = ['__author__', '__version__', 'config', 'conn', 'kill', 'BaseRelation', 'Connection', 'Heading', 'FreeRelation', 'Not', 'schema', diff --git a/tests/test_declare.py b/tests/test_declare.py index 74081fd3b..db4df987c 100644 --- a/tests/test_declare.py +++ b/tests/test_declare.py @@ -1,7 +1,7 @@ -import re from nose.tools import assert_true, assert_false, assert_equal, assert_list_equal, raises from . import schema import datajoint as dj +from datajoint.declare import declare auto = schema.Auto() auto.fill() @@ -24,8 +24,9 @@ def test_schema_decorator(): def test_real_definition(): """real_definition should match original definition""" rel = schema.Experiment() - s1 = re.sub('r\s+', rel.definition, '') - s2 = re.sub('r\s+', rel.real_definition, '') + context = rel._context + s1 = declare(rel.full_table_name, rel.definition, context) + s2 = declare(rel.full_table_name, rel.real_definition, context) assert_equal(s1, s2) @staticmethod From 2692d459179332d1cb8e7c6ff657f497f074a388 Mon Sep 17 00:00:00 2001 From: dimitri-yatsenko Date: Thu, 14 Jul 2016 21:35:01 -0500 Subject: [PATCH 5/6] bugfix in Baseline.real_definition --- datajoint/base_relation.py | 3 ++- datajoint/heading.py | 1 - datajoint/schema.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/datajoint/base_relation.py b/datajoint/base_relation.py index 01ba8dbb3..396fa0341 100644 --- a/datajoint/base_relation.py +++ b/datajoint/base_relation.py @@ -333,10 +333,11 @@ def real_definition(self): attributes_thus_far.add(attr.name) do_include = True for parent, primary_key in list(parents.items()): + if attr.name in primary_key: + do_include = False if attributes_thus_far.issuperset(primary_key): parents.pop(parent) definition += '-> ' + self.lookup_table_name(parent) + '\n' - do_include = False if do_include: definition += '%-20s : %-28s # %s\n' % ( attr.name if attr.default is None else '%s=%s' % (attr.name, attr.default), diff --git a/datajoint/heading.py b/datajoint/heading.py index 1b50c778d..2ef5e7c60 100644 --- a/datajoint/heading.py +++ b/datajoint/heading.py @@ -166,7 +166,6 @@ def init_from_database(self, conn, database, table_name): ('int', True): np.uint32, ('bigint', False): np.int64, ('bigint', True): np.uint64 - # TODO: include types DECIMAL and NUMERIC } sql_literals = ['CURRENT_TIMESTAMP'] diff --git a/datajoint/schema.py b/datajoint/schema.py index a121df64b..3c836ac67 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -1,5 +1,4 @@ import warnings - import pymysql import logging import inspect From 007535ce6f514e32e8998ba516fc6b8e581a5c7e Mon Sep 17 00:00:00 2001 From: dimitri-yatsenko Date: Fri, 15 Jul 2016 18:25:06 -0500 Subject: [PATCH 6/6] renamed `real_definition` to `show_definition()` --- datajoint/base_relation.py | 3 +-- tests/test_declare.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/datajoint/base_relation.py b/datajoint/base_relation.py index 396fa0341..194197938 100644 --- a/datajoint/base_relation.py +++ b/datajoint/base_relation.py @@ -315,8 +315,7 @@ def size_on_disk(self): database=self.database, table=self.table_name), as_dict=True).fetchone() return ret['Data_length'] + ret['Index_length'] - @property - def real_definition(self): + def show_definition(self): """ :return: the definition string for the relation using DataJoint DDL. This does not yet work for aliased foreign keys. diff --git a/tests/test_declare.py b/tests/test_declare.py index db4df987c..c78fa999d 100644 --- a/tests/test_declare.py +++ b/tests/test_declare.py @@ -21,12 +21,12 @@ def test_schema_decorator(): assert_true(not issubclass(schema.Subject, dj.Part)) @staticmethod - def test_real_definition(): + def test_show_definition(): """real_definition should match original definition""" rel = schema.Experiment() context = rel._context s1 = declare(rel.full_table_name, rel.definition, context) - s2 = declare(rel.full_table_name, rel.real_definition, context) + s2 = declare(rel.full_table_name, rel.show_definition(), context) assert_equal(s1, s2) @staticmethod