Support datetime extended type

CHANGELOG.md

@@ -9,6 +9,63 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Decimal type support (#203).
 - UUID type support (#202).
+- Datetime type support and tarantool.Datetime type (#204).
+
+ Tarantool datetime objects are decoded to `tarantool.Datetime`
+ type. `tarantool.Datetime` may be encoded to Tarantool datetime
+ objects.
+
+ You can create `tarantool.Datetime` objects either from msgpack
+ data or by using the same API as in Tarantool:
+
+ ```python
+ dt1 = tarantool.Datetime(year=2022, month=8, day=31,
+ hour=18, minute=7, sec=54,
+ nsec=308543321)
+
+ dt2 = tarantool.Datetime(timestamp=1661969274)
+
+ dt3 = tarantool.Datetime(timestamp=1661969274, nsec=308543321)
+ ```
+
+ `tarantool.Datetime` exposes `year`, `month`, `day`, `hour`,
+ `minute`, `sec`, `nsec`, `timestamp` and `value` (integer epoch time
+ with nanoseconds precision) properties if you need to convert
+ `tarantool.Datetime` to any other kind of datetime object:
+
+ ```python
+ pdt = pandas.Timestamp(year=dt.year, month=dt.month, day=dt.day,
+ hour=dt.hour, minute=dt.minute, second=dt.sec,
+ microsecond=(dt.nsec // 1000),
+ nanosecond=(dt.nsec % 1000))
+ ```
+
+- Offset in datetime type support (#204).
+
+ Use `tzoffset` parameter to set up offset timezone:
+
+ ```python
+ dt = tarantool.Datetime(year=2022, month=8, day=31,
+ hour=18, minute=7, sec=54,
+ nsec=308543321, tzoffset=180)
+ ```
+
+ You may use `tzoffset` property to get timezone offset of a datetime
+ object.
+
+- Timezone in datetime type support (#204).
+
+ Use `tz` parameter to set up timezone name:
+
+ ```python
+ dt = tarantool.Datetime(year=2022, month=8, day=31,
+ hour=18, minute=7, sec=54,
+ nsec=308543321, tz='Europe/Moscow')
+ ```
+
+ If both `tz` and `tzoffset` is specified, `tz` is used.
+
+ You may use `tz` property to get timezone name of a datetime object.
 
 ### Changed
 - Bump msgpack requirement to 1.0.4 (PR #223).

requirements.txt

@@ -1 +1,3 @@
 msgpack>=1.0.4
+pandas
+pytz

tarantool/__init__.py

@@ -32,6 +32,10 @@
  ENCODING_DEFAULT,
 )
 
+from tarantool.msgpack_ext.types.datetime import (
+ Datetime,
+)
+
 __version__ = "0.9.0"
 
 
@@ -91,7 +95,7 @@ def connectmesh(addrs=({'host': 'localhost', 'port': 3301},), user=None,
 
 __all__ = ['connect', 'Connection', 'connectmesh', 'MeshConnection', 'Schema',
  'Error', 'DatabaseError', 'NetworkError', 'NetworkWarning',
- 'SchemaError', 'dbapi']
+ 'SchemaError', 'dbapi', 'Datetime']
 
 # ConnectionPool is supported only for Python 3.7 or newer.
 if sys.version_info.major >= 3 and sys.version_info.minor >= 7:

tarantool/msgpack_ext/datetime.py

@@ -0,0 +1,9 @@
+from tarantool.msgpack_ext.types.datetime import Datetime
+
+EXT_ID = 4
+
+def encode(obj):
+ return obj.msgpack_encode()
+
+def decode(data):
+ return Datetime(data)

tarantool/msgpack_ext/packer.py

@@ -2,12 +2,16 @@
 from uuid import UUID
 from msgpack import ExtType
 
+from tarantool.msgpack_ext.types.datetime import Datetime
+
 import tarantool.msgpack_ext.decimal as ext_decimal
 import tarantool.msgpack_ext.uuid as ext_uuid
+import tarantool.msgpack_ext.datetime as ext_datetime
 
 encoders = [
- {'type': Decimal, 'ext': ext_decimal},
- {'type': UUID, 'ext': ext_uuid },
+ {'type': Decimal, 'ext': ext_decimal },
+ {'type': UUID, 'ext': ext_uuid },
+ {'type': Datetime, 'ext': ext_datetime},
 ]
 
 def default(obj):

tarantool/msgpack_ext/types/datetime.py

@@ -0,0 +1,264 @@
+from copy import deepcopy
+
+import pandas
+import pytz
+
+import tarantool.msgpack_ext.types.timezones as tt_timezones
+from tarantool.error import MsgpackError
+
+# https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-datetime-type
+#
+# The datetime MessagePack representation looks like this:
+# +---------+----------------+==========+-----------------+
+# | MP_EXT | MP_DATETIME | seconds | nsec; tzoffset; |
+# | = d7/d8 | = 4 | | tzindex; |
+# +---------+----------------+==========+-----------------+
+# MessagePack data contains:
+#
+# * Seconds (8 bytes) as an unencoded 64-bit signed integer stored in the
+# little-endian order.
+# * The optional fields (8 bytes), if any of them have a non-zero value.
+# The fields include nsec (4 bytes), tzoffset (2 bytes), and
+# tzindex (2 bytes) packed in the little-endian order.
+#
+# seconds is seconds since Epoch, where the epoch is the point where the time
+# starts, and is platform dependent. For Unix, the epoch is January 1,
+# 1970, 00:00:00 (UTC). Tarantool uses a double type, see a structure
+# definition in src/lib/core/datetime.h and reasons in
+# https://github.com/tarantool/tarantool/wiki/Datetime-internals#intervals-in-c
+#
+# nsec is nanoseconds, fractional part of seconds. Tarantool uses int32_t, see
+# a definition in src/lib/core/datetime.h.
+#
+# tzoffset is timezone offset in minutes from UTC. Tarantool uses a int16_t type,
+# see a structure definition in src/lib/core/datetime.h.
+#
+# tzindex is Olson timezone id. Tarantool uses a int16_t type, see a structure
+# definition in src/lib/core/datetime.h. If both tzoffset and tzindex are
+# specified, tzindex has the preference and the tzoffset value is ignored.
+
+SECONDS_SIZE_BYTES = 8
+NSEC_SIZE_BYTES = 4
+TZOFFSET_SIZE_BYTES = 2
+TZINDEX_SIZE_BYTES = 2
+
+BYTEORDER = 'little'
+
+NSEC_IN_SEC = 1000000000
+NSEC_IN_MKSEC = 1000
+SEC_IN_MIN = 60
+
+def get_bytes_as_int(data, cursor, size):
+ part = data[cursor:cursor + size]
+ return int.from_bytes(part, BYTEORDER, signed=True), cursor + size
+
+def get_int_as_bytes(data, size):
+ return data.to_bytes(size, byteorder=BYTEORDER, signed=True)
+
+def compute_offset(timestamp):
+ utc_offset = timestamp.tzinfo.utcoffset(timestamp)
+
+ # `None` offset is a valid utcoffset implementation,
+ # but it seems that pytz timezones never return `None`:
+ # https://github.com/pandas-dev/pandas/issues/15986
+ assert utc_offset is not None
+
+ # There is no precision loss since offset is in minutes
+ return int(utc_offset.total_seconds()) // SEC_IN_MIN
+
+def get_python_tzinfo(tz, error_class):
+ if tz in pytz.all_timezones:
+ return pytz.timezone(tz)
+
+ # Checked with timezones/validate_timezones.py
+ tt_tzinfo = tt_timezones.timezoneAbbrevInfo[tz]
+ if (tt_tzinfo['category'] & tt_timezones.TZ_AMBIGUOUS) != 0:
+ raise error_class(f'Failed to create datetime with ambiguous timezone "{tz}"')
+
+ return pytz.FixedOffset(tt_tzinfo['offset'])
+
+def msgpack_decode(data):
+ cursor = 0
+ seconds, cursor = get_bytes_as_int(data, cursor, SECONDS_SIZE_BYTES)
+
+ data_len = len(data)
+ if data_len == (SECONDS_SIZE_BYTES + NSEC_SIZE_BYTES + \
+ TZOFFSET_SIZE_BYTES + TZINDEX_SIZE_BYTES):
+ nsec, cursor = get_bytes_as_int(data, cursor, NSEC_SIZE_BYTES)
+ tzoffset, cursor = get_bytes_as_int(data, cursor, TZOFFSET_SIZE_BYTES)
+ tzindex, cursor = get_bytes_as_int(data, cursor, TZINDEX_SIZE_BYTES)
+ elif data_len == SECONDS_SIZE_BYTES:
+ nsec = 0
+ tzoffset = 0
+ tzindex = 0
+ else:
+ raise MsgpackError(f'Unexpected datetime payload length {data_len}')
+
+ total_nsec = seconds * NSEC_IN_SEC + nsec
+ datetime = pandas.to_datetime(total_nsec, unit='ns')
+
+ if tzindex != 0:
+ if tzindex not in tt_timezones.indexToTimezone:
+ raise MsgpackError(f'Failed to decode datetime with unknown tzindex "{tzindex}"')
+ tz = tt_timezones.indexToTimezone[tzindex]
+ tzinfo = get_python_tzinfo(tz, MsgpackError)
+ return datetime.replace(tzinfo=pytz.UTC).tz_convert(tzinfo), tz
+ elif tzoffset != 0:
+ tzinfo = pytz.FixedOffset(tzoffset)
+ return datetime.replace(tzinfo=pytz.UTC).tz_convert(tzinfo), ''
+ else:
+ return datetime, ''
+
+class Datetime():
+ def __init__(self, data=None, *, timestamp=None, year=None, month=None,
+ day=None, hour=None, minute=None, sec=None, nsec=None,
+ tzoffset=0, tz=''):
+ if data is not None:
+ if not isinstance(data, bytes):
+ raise ValueError('data argument (first positional argument) ' +
+ 'expected to be a "bytes" instance')
+
+ datetime, tz = msgpack_decode(data)
+ self._datetime = datetime
+ self._tz = tz
+ return
+
+ # The logic is same as in Tarantool, refer to datetime API.
+ # https://www.tarantool.io/en/doc/latest/reference/reference_lua/datetime/new/
+ if timestamp is not None:
+ if ((year is not None) or (month is not None) or \
+ (day is not None) or (hour is not None) or \
+ (minute is not None) or (sec is not None)):
+ raise ValueError('Cannot provide both timestamp and year, month, ' +
+ 'day, hour, minute, sec')
+
+ if nsec is not None:
+ if not isinstance(timestamp, int):
+ raise ValueError('timestamp must be int if nsec provided')
+
+ total_nsec = timestamp * NSEC_IN_SEC + nsec
+ datetime = pandas.to_datetime(total_nsec, unit='ns')
+ else:
+ datetime = pandas.to_datetime(timestamp, unit='s')
+ else:
+ if nsec is not None:
+ microsecond = nsec // NSEC_IN_MKSEC
+ nanosecond = nsec % NSEC_IN_MKSEC
+ else:
+ microsecond = 0
+ nanosecond = 0
+
+ datetime = pandas.Timestamp(year=year, month=month, day=day,
+ hour=hour, minute=minute, second=sec,
+ microsecond=microsecond,
+ nanosecond=nanosecond)
+
+ if tz != '':
+ if tz not in tt_timezones.timezoneToIndex:
+ raise ValueError(f'Unknown Tarantool timezone "{tz}"')
+
+ tzinfo = get_python_tzinfo(tz, ValueError)
+ self._datetime = datetime.replace(tzinfo=tzinfo)
+ self._tz = tz
+ elif tzoffset != 0:
+ tzinfo = pytz.FixedOffset(tzoffset)
+ self._datetime = datetime.replace(tzinfo=tzinfo)
+ self._tz = ''
+ else:
+ self._datetime = datetime
+ self._tz = ''
+
+ def __eq__(self, other):
+ if isinstance(other, Datetime):
+ return self._datetime == other._datetime
+ elif isinstance(other, pandas.Timestamp):
+ return self._datetime == other
+ else:
+ return False
+
+ def __str__(self):
+ return self._datetime.__str__()
+
+ def __repr__(self):
+ return f'datetime: {self._datetime.__repr__()}, tz: "{self.tz}"'
+
+ def __copy__(self):
+ cls = self.__class__
+ result = cls.__new__(cls)
+ result.__dict__.update(self.__dict__)
+ return result
+
+ def __deepcopy__(self, memo):
+ cls = self.__class__
+ result = cls.__new__(cls)
+ memo[id(self)] = result
+ for k, v in self.__dict__.items():
+ setattr(result, k, deepcopy(v, memo))
+ return result
+
+ @property
+ def year(self):
+ return self._datetime.year
+
+ @property
+ def month(self):
+ return self._datetime.month
+
+ @property
+ def day(self):
+ return self._datetime.day
+
+ @property
+ def hour(self):
+ return self._datetime.hour
+
+ @property
+ def minute(self):
+ return self._datetime.minute
+
+ @property
+ def sec(self):
+ return self._datetime.second
+
+ @property
+ def nsec(self):
+ # microseconds + nanoseconds
+ return self._datetime.value % NSEC_IN_SEC
+
+ @property
+ def timestamp(self):
+ return self._datetime.timestamp()
+
+ @property
+ def tzoffset(self):
+ if self._datetime.tzinfo is not None:
+ return compute_offset(self._datetime)
+ return 0
+
+ @property
+ def tz(self):
+ return self._tz
+
+ @property
+ def value(self):
+ return self._datetime.value
+
+ def msgpack_encode(self):
+ seconds = self.value // NSEC_IN_SEC
+ nsec = self.nsec
+ tzoffset = self.tzoffset
+
+ tz = self.tz
+ if tz != '':
+ tzindex = tt_timezones.timezoneToIndex[tz]
+ else:
+ tzindex = 0
+
+ buf = get_int_as_bytes(seconds, SECONDS_SIZE_BYTES)
+
+ if (nsec != 0) or (tzoffset != 0) or (tzindex != 0):
+ buf = buf + get_int_as_bytes(nsec, NSEC_SIZE_BYTES)
+ buf = buf + get_int_as_bytes(tzoffset, TZOFFSET_SIZE_BYTES)
+ buf = buf + get_int_as_bytes(tzindex, TZINDEX_SIZE_BYTES)
+
+ return buf

tarantool/msgpack_ext/types/timezones/__init__.py

@@ -0,0 +1,9 @@
+from tarantool.msgpack_ext.types.timezones.timezones import (
+	TZ_AMBIGUOUS,
+	indexToTimezone,
+	timezoneToIndex,
+	timezoneAbbrevInfo,
+)
+
+__all__ = ['TZ_AMBIGUOUS', 'indexToTimezone', 'timezoneToIndex',
+ 'timezoneAbbrevInfo']