diff --git a/datajoint/__init__.py b/datajoint/__init__.py index 83a29ae69..1a34177ec 100644 --- a/datajoint/__init__.py +++ b/datajoint/__init__.py @@ -16,13 +16,14 @@ import os __author__ = "Dimitri Yatsenko, Edgar Walker, and Fabian Sinz at Baylor College of Medicine" -__version__ = "0.2.8" -__date__ = "July 1, 2016" +__version__ = "0.3.0" +__date__ = "July 14, 2016" __all__ = ['__author__', '__version__', 'config', 'conn', 'kill', 'BaseRelation', 'Connection', 'Heading', 'FreeRelation', 'Not', 'schema', 'Manual', 'Lookup', 'Imported', 'Computed', 'Part', - 'AndList', 'OrList', 'ERD', 'U'] + 'AndList', 'OrList', 'ERD', 'U', + 'set_password'] print('DataJoint', __version__, '('+__date__+')') @@ -81,3 +82,4 @@ class DataJointError(Exception): from .schema import Schema as schema from .kill import kill from .erd import ERD +from .admin import set_password diff --git a/datajoint/admin.py b/datajoint/admin.py new file mode 100644 index 000000000..cd85df369 --- /dev/null +++ b/datajoint/admin.py @@ -0,0 +1,6 @@ +from . import conn + + +def set_password(new_password, connection=conn()): + connection.query("SET PASSWORD = PASSWORD('%s')" % new_password) + print('done.') diff --git a/datajoint/base_relation.py b/datajoint/base_relation.py index 1889c62f6..194197938 100644 --- a/datajoint/base_relation.py +++ b/datajoint/base_relation.py @@ -1,5 +1,6 @@ import collections import itertools +import inspect import numpy as np import logging from . import config, DataJointError @@ -314,6 +315,52 @@ def size_on_disk(self): database=self.database, table=self.table_name), as_dict=True).fetchone() return ret['Data_length'] + ret['Index_length'] + def show_definition(self): + """ + :return: the definition string for the relation using DataJoint DDL. + This does not yet work for aliased foreign keys. + """ + self.connection.dependencies.load() + parents = {r: FreeRelation(self.connection, r).primary_key for r in self.parents()} + in_key = True + definition = '# ' + self.heading.table_info['comment'] + '\n' + attributes_thus_far = set() + for attr in self.heading.attributes.values(): + if in_key and not attr.in_key: + definition += '---\n' + in_key = False + attributes_thus_far.add(attr.name) + do_include = True + for parent, primary_key in list(parents.items()): + if attr.name in primary_key: + do_include = False + if attributes_thus_far.issuperset(primary_key): + parents.pop(parent) + definition += '-> ' + self.lookup_table_name(parent) + '\n' + if do_include: + definition += '%-20s : %-28s # %s\n' % ( + attr.name if attr.default is None else '%s=%s' % (attr.name, attr.default), + '%s%s' % (attr.type, 'auto_increment' if attr.autoincrement else ''), attr.comment) + return definition + + def lookup_table_name(self, name): + """ + given the name of another table in the form `database`.`table_name`, find its class in the context + :param name: `database`.`table_name` + :return: class name found in the context. + """ + def _lookup(context, name, level=0): + for member_name, member in context.items(): + if inspect.isclass(member) and issubclass(member, BaseRelation) and member.full_table_name == name: + return member_name + if level < 5 and inspect.ismodule(member) and member.__name__ != 'datajoint': + candidate_name = _lookup(dict(inspect.getmembers(member)), name, level+1) + if candidate_name != name: + return member_name + '.' + candidate_name + return name + + return _lookup(self._context, name) + class FreeRelation(BaseRelation): """ @@ -342,3 +389,5 @@ def table_name(self): :return: the table name in the database """ return self._table_name + + diff --git a/datajoint/connection.py b/datajoint/connection.py index d8a006147..ec5f331f1 100644 --- a/datajoint/connection.py +++ b/datajoint/connection.py @@ -189,3 +189,6 @@ def transaction(self): raise else: self.commit_transaction() + + + diff --git a/datajoint/declare.py b/datajoint/declare.py index d01bb87a2..90fbf9955 100644 --- a/datajoint/declare.py +++ b/datajoint/declare.py @@ -141,15 +141,12 @@ def declare(full_table_name, definition, context): # compile SQL if not primary_key: raise DataJointError('Table must have a primary key') - sql = 'CREATE TABLE IF NOT EXISTS %s (\n ' % full_table_name - sql += ',\n '.join(attribute_sql) - sql += ',\n PRIMARY KEY (`' + '`,`'.join(primary_key) + '`)' - if foreign_key_sql: - sql += ', \n' + ', \n'.join(foreign_key_sql) - if index_sql: - sql += ', \n' + ', \n'.join(index_sql) - sql += '\n) ENGINE = InnoDB, COMMENT "%s"' % table_comment - return sql + return ('CREATE TABLE IF NOT EXISTS %s (\n' % full_table_name + + ',\n'.join(attribute_sql + + ['PRIMARY KEY (`' + '`,`'.join(primary_key) + '`)'] + + foreign_key_sql + + index_sql) + + '\n) ENGINE=InnoDB, COMMENT "%s"' % table_comment) def compile_attribute(line, in_key=False): @@ -161,7 +158,11 @@ def compile_attribute(line, in_key=False): :returns: (name, sql) -- attribute name and sql code for its declaration """ - match = attribute_parser.parseString(line+'#', parseAll=True) + try: + match = attribute_parser.parseString(line+'#', parseAll=True) + except pp.ParseException: + logger.error('Declaration error in line: ', line) + raise match['comment'] = match['comment'].rstrip('#') if 'default' not in match: match['default'] = '' diff --git a/datajoint/heading.py b/datajoint/heading.py index fcfcb9fc0..2ef5e7c60 100644 --- a/datajoint/heading.py +++ b/datajoint/heading.py @@ -19,22 +19,13 @@ def todict(self): @property def sql(self): """ - Convert attribute tuple into its SQL CREATE TABLE clause. + Convert primary key attribute tuple into its SQL CREATE TABLE clause. + Default values are not reflected. :return: SQL code """ - sql_literals = ['CURRENT_TIMESTAMP'] # SQL literals that can be used as default values - if self.nullable: - default = 'DEFAULT NULL' - else: - default = 'NOT NULL' - if self.default: - # enclose value in quotes except special SQL values or already enclosed - quote = self.default.upper() not in sql_literals and self.default[0] not in '"\'' - default += ' DEFAULT ' + ('"%s"' if quote else "%s") % self.default - if any(c in r'\"' for c in self.comment): - raise DataJointError('Illegal characters in attribute comment "%s"' % self.comment) - return '`{name}` {type} {default} COMMENT "{comment}"'.format( - name=self.name, type=self.type, default=default, comment=self.comment) + assert self.in_key and not self.nullable # primary key attributes are never nullable + return '`{name}` {type} NOT NULL COMMENT "{comment}"'.format( + name=self.name, type=self.type, comment=self.comment) class Heading: @@ -99,7 +90,7 @@ def __repr__(self): ret += '---\n' in_key = False ret += '%-20s : %-28s # %s\n' % ( - v.name if v.default is None else '%s="%s"' % (v.name, v.default), + v.name if v.default is None else '%s=%s' % (v.name, v.default), '%s%s' % (v.type, 'auto_increment' if v.autoincrement else ''), v.comment) return ret @@ -175,18 +166,23 @@ def init_from_database(self, conn, database, table_name): ('int', True): np.uint32, ('bigint', False): np.int64, ('bigint', True): np.uint64 - # TODO: include types DECIMAL and NUMERIC } + sql_literals = ['CURRENT_TIMESTAMP'] + # additional attribute properties for attr in attributes: attr['nullable'] = (attr['nullable'] == 'YES') attr['in_key'] = (attr['in_key'] == 'PRI') attr['autoincrement'] = bool(re.search(r'auto_increment', attr['Extra'], flags=re.IGNORECASE)) + attr['type'] = re.sub(r'int\(\d+\)', 'int', attr['type'], count=1) # strip size off integers attr['numeric'] = bool(re.match(r'(tiny|small|medium|big)?int|decimal|double|float', attr['type'])) attr['string'] = bool(re.match(r'(var)?char|enum|date|time|timestamp', attr['type'])) attr['is_blob'] = bool(re.match(r'(tiny|medium|long)?blob', attr['type'])) + if attr['string'] and attr['default'] is not None and attr['default'] not in sql_literals: + attr['default'] = '"%s"' % attr['default'] + attr['sql_expression'] = None if not (attr['numeric'] or attr['string'] or attr['is_blob']): raise DataJointError('Unsupported field type {field} in `{database}`.`{table_name}`'.format( diff --git a/datajoint/schema.py b/datajoint/schema.py index a2d26de78..3c836ac67 100644 --- a/datajoint/schema.py +++ b/datajoint/schema.py @@ -1,14 +1,12 @@ import warnings - import pymysql import logging +import inspect import re from . import conn, DataJointError, config -from datajoint.utils import to_camel_case from .heading import Heading -from .utils import user_choice -from .user_relations import Part, Computed, Imported, Manual, Lookup -import inspect +from .utils import user_choice, to_camel_case +from .user_relations import UserRelation, Part, Computed, Imported, Manual, Lookup logger = logging.getLogger(__name__) @@ -119,6 +117,7 @@ def exists(self): cur = self.connection.query("SHOW DATABASES LIKE '{database}'".format(database=self.database)) return cur.rowcount > 0 + def process_relation_class(self, relation_class, context, assert_declared=False): """ assign schema properties to the relation class and declare the table @@ -142,6 +141,7 @@ def process_relation_class(self, relation_class, context, assert_declared=False) else: instance.insert(contents, skip_duplicates=True) + def __call__(self, cls): """ Binds the passed in class object to a database. This is intended to be used as a decorator. diff --git a/tests/test_declare.py b/tests/test_declare.py index b78b02572..c78fa999d 100644 --- a/tests/test_declare.py +++ b/tests/test_declare.py @@ -1,6 +1,7 @@ from nose.tools import assert_true, assert_false, assert_equal, assert_list_equal, raises from . import schema import datajoint as dj +from datajoint.declare import declare auto = schema.Auto() auto.fill() @@ -19,6 +20,15 @@ def test_schema_decorator(): assert_true(issubclass(schema.Subject, dj.Manual)) assert_true(not issubclass(schema.Subject, dj.Part)) + @staticmethod + def test_show_definition(): + """real_definition should match original definition""" + rel = schema.Experiment() + context = rel._context + s1 = declare(rel.full_table_name, rel.definition, context) + s2 = declare(rel.full_table_name, rel.show_definition(), context) + assert_equal(s1, s2) + @staticmethod def test_attributes(): # test autoincrement declaration