Marshal 'BYTES' and 'TIME' column / paramter types

bigquery/google/cloud/bigquery/_helpers.py

@@ -14,6 +14,7 @@
 
 """Shared helper functions for BigQuery API classes."""
 
+import base64
 from collections import OrderedDict
 import datetime
 
@@ -22,6 +23,7 @@
 from google.cloud._helpers import _datetime_to_rfc3339
 from google.cloud._helpers import _microseconds_from_datetime
 from google.cloud._helpers import _RFC3339_NO_FRACTION
+from google.cloud._helpers import _time_from_iso8601_time_naive
 
 
 def _not_null(value, field):
@@ -47,6 +49,17 @@ def _bool_from_json(value, field):
  return value.lower() in ['t', 'true', '1']
 
 
+def _string_from_json(value, _):
+ """NOOP string -> string coercion"""
+ return value
+
+
+def _bytes_from_json(value, field):
+ """Base64-decode value"""
+ if _not_null(value, field):
+ return base64.decodestring(value)
+
+
 def _timestamp_from_json(value, field):
  """Coerce 'value' to a datetime, if set or not nullable."""
  if _not_null(value, field):
@@ -64,9 +77,17 @@ def _datetime_from_json(value, field):
 def _date_from_json(value, field):
  """Coerce 'value' to a datetime date, if set or not nullable"""
  if _not_null(value, field):
+ # value will be a string, in YYYY-MM-DD form.
  return _date_from_iso8601_date(value)
 
 
+def _time_from_json(value, field):
+ """Coerce 'value' to a datetime date, if set or not nullable"""
+ if _not_null(value, field):
+ # value will be a string, in HH:MM:SS form.
+ return _time_from_iso8601_time_naive(value)
+
+
 def _record_from_json(value, field):
  """Coerce 'value' to a mapping, if set or not nullable."""
  if _not_null(value, field):
@@ -82,23 +103,20 @@ def _record_from_json(value, field):
  return record
 
 
-def _string_from_json(value, _):
- """NOOP string -> string coercion"""
- return value
-
-
 _CELLDATA_FROM_JSON = {
  'INTEGER': _int_from_json,
  'INT64': _int_from_json,
  'FLOAT': _float_from_json,
  'FLOAT64': _float_from_json,
  'BOOLEAN': _bool_from_json,
  'BOOL': _bool_from_json,
+ 'STRING': _string_from_json,
+ 'BYTES': _bytes_from_json,
  'TIMESTAMP': _timestamp_from_json,
  'DATETIME': _datetime_from_json,
  'DATE': _date_from_json,
+ 'TIME': _time_from_json,
  'RECORD': _record_from_json,
- 'STRING': _string_from_json,
 }
 
 
@@ -121,6 +139,13 @@ def _bool_to_json(value):
  return value
 
 
+def _bytes_to_json(value):
+ """Coerce 'value' to an JSON-compatible representation."""
+ if isinstance(value, bytes):
+ value = base64.encodestring(value)
+ return value
+
+
 def _timestamp_to_json(value):
  """Coerce 'value' to an JSON-compatible representation."""
  if isinstance(value, datetime.datetime):
@@ -142,16 +167,25 @@ def _date_to_json(value):
  return value
 
 
+def _time_to_json(value):
+ """Coerce 'value' to an JSON-compatible representation."""
+ if isinstance(value, datetime.time):
+ value = value.isoformat()
+ return value
+
+
 _SCALAR_VALUE_TO_JSON = {
  'INTEGER': _int_to_json,
  'INT64': _int_to_json,
  'FLOAT': _float_to_json,
  'FLOAT64': _float_to_json,
  'BOOLEAN': _bool_to_json,
  'BOOL': _bool_to_json,
+ 'BYTES': _bytes_to_json,
  'TIMESTAMP': _timestamp_to_json,
  'DATETIME': _datetime_to_json,
  'DATE': _date_to_json,
+ 'TIME': _time_to_json,
 }
 
 

bigquery/unit_tests/test__helpers.py

@@ -105,6 +105,44 @@ def test_w_value_other(self):
  self.assertFalse(coerced)
 
 
+class Test_string_from_json(unittest.TestCase):
+
+ def _call_fut(self, value, field):
+ from google.cloud.bigquery._helpers import _string_from_json
+ return _string_from_json(value, field)
+
+ def test_w_none_nullable(self):
+ self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
+
+ def test_w_none_required(self):
+ self.assertIsNone(self._call_fut(None, _Field('REQUIRED')))
+
+ def test_w_string_value(self):
+ coerced = self._call_fut('Wonderful!', object())
+ self.assertEqual(coerced, 'Wonderful!')
+
+
+class Test_bytes_from_json(unittest.TestCase):
+
+ def _call_fut(self, value, field):
+ from google.cloud.bigquery._helpers import _bytes_from_json
+ return _bytes_from_json(value, field)
+
+ def test_w_none_nullable(self):
+ self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
+
+ def test_w_none_required(self):
+ with self.assertRaises(TypeError):
+ self._call_fut(None, _Field('REQUIRED'))
+
+ def test_w_base64_encoded_value(self):
+ import base64
+ expected = b'Wonderful!'
+ encoded = base64.encodestring(expected)
+ coerced = self._call_fut(encoded, object())
+ self.assertEqual(coerced, expected)
+
+
 class Test_timestamp_from_json(unittest.TestCase):
 
  def _call_fut(self, value, field):
@@ -177,6 +215,27 @@ def test_w_string_value(self):
  datetime.date(1987, 9, 22))
 
 
+class Test_time_from_json(unittest.TestCase):
+
+ def _call_fut(self, value, field):
+ from google.cloud.bigquery._helpers import _time_from_json
+ return _time_from_json(value, field)
+
+ def test_w_none_nullable(self):
+ self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
+
+ def test_w_none_required(self):
+ with self.assertRaises(TypeError):
+ self._call_fut(None, _Field('REQUIRED'))
+
+ def test_w_string_value(self):
+ import datetime
+ coerced = self._call_fut('12:12:27', object())
+ self.assertEqual(
+ coerced,
+ datetime.time(12, 12, 27))
+
+
 class Test_record_from_json(unittest.TestCase):
 
  def _call_fut(self, value, field):
@@ -238,23 +297,6 @@ def test_w_record_subfield(self):
  self.assertEqual(coerced, expected)
 
 
-class Test_string_from_json(unittest.TestCase):
-
- def _call_fut(self, value, field):
- from google.cloud.bigquery._helpers import _string_from_json
- return _string_from_json(value, field)
-
- def test_w_none_nullable(self):
- self.assertIsNone(self._call_fut(None, _Field('NULLABLE')))
-
- def test_w_none_required(self):
- self.assertIsNone(self._call_fut(None, _Field('RECORD')))
-
- def test_w_string_value(self):
- coerced = self._call_fut('Wonderful!', object())
- self.assertEqual(coerced, 'Wonderful!')
-
-
 class Test_row_from_json(unittest.TestCase):
 
  def _call_fut(self, row, schema):
@@ -471,6 +513,23 @@ def test_w_string(self):
  self.assertEqual(self._call_fut('false'), 'false')
 
 
+class Test_bytes_to_json(unittest.TestCase):
+
+ def _call_fut(self, value):
+ from google.cloud.bigquery._helpers import _bytes_to_json
+ return _bytes_to_json(value)
+
+ def test_w_non_bytes(self):
+ non_bytes = object()
+ self.assertIs(self._call_fut(non_bytes), non_bytes)
+
+ def test_w_bytes(self):
+ import base64
+ source = b'source'
+ expected = base64.encodestring(source)
+ self.assertEqual(self._call_fut(source), expected)
+
+
 class Test_timestamp_to_json(unittest.TestCase):
 
  def _call_fut(self, value):
@@ -522,6 +581,22 @@ def test_w_datetime(self):
  self.assertEqual(self._call_fut(when), '2016-12-03')
 
 
+class Test_time_to_json(unittest.TestCase):
+
+ def _call_fut(self, value):
+ from google.cloud.bigquery._helpers import _time_to_json
+ return _time_to_json(value)
+
+ def test_w_string(self):
+ RFC3339 = '12:13:41'
+ self.assertEqual(self._call_fut(RFC3339), RFC3339)
+
+ def test_w_datetime(self):
+ import datetime
+ when = datetime.time(12, 13, 41)
+ self.assertEqual(self._call_fut(when), '12:13:41')
+
+
 class Test_ConfigurationProperty(unittest.TestCase):
 
  @staticmethod

core/google/cloud/_helpers.py

@@ -251,6 +251,19 @@ def _date_from_iso8601_date(value):
  return datetime.datetime.strptime(value, '%Y-%m-%d').date()
 
 
+def _time_from_iso8601_time_naive(value):
+ """Convert a zoneless ISO8601 time string to naive datetime time
+
+ :type value: str
+ :param value: The time string to convert
+
+ :rtype: :class:`datetime.time`
+ :returns: A datetime time object created from the string
+
+ """
+ return datetime.datetime.strptime(value, '%H:%M:%S').time()
+
+
 def _rfc3339_to_datetime(dt_str):
  """Convert a microsecond-precision timetamp to a native datetime.
 

core/unit_tests/test__helpers.py

@@ -265,6 +265,18 @@ def test_todays_date(self):
  self.assertEqual(self._call_fut(TODAY.strftime("%Y-%m-%d")), TODAY)
 
 
+class Test___time_from_iso8601_time_naive(unittest.TestCase):
+
+ def _call_fut(self, value):
+ from google.cloud._helpers import _time_from_iso8601_time_naive
+ return _time_from_iso8601_time_naive(value)
+
+ def test_todays_date(self):
+ import datetime
+ WHEN = datetime.time(12, 9, 42)
+ self.assertEqual(self._call_fut(("12:09:42")), WHEN)
+
+
 class Test__rfc3339_to_datetime(unittest.TestCase):
 
  def _call_fut(self, dt_str):

system_tests/bigquery.py

@@ -479,12 +479,49 @@ def _job_done(instance):
  # raise an error, and that the job completed (in the `retry()`
  # above).
 
- def test_sync_query_w_nested_arrays_and_structs(self):
+ def test_sync_query_w_standard_sql_types(self):
+ import datetime
+ from google.cloud._helpers import UTC
+ naive = datetime.datetime(2016, 12, 5, 12, 41, 9)
+ stamp = "%s %s" % (naive.date().isoformat(), naive.time().isoformat())
+ zoned = naive.replace(tzinfo=UTC)
  EXAMPLES = [
  {
  'sql': 'SELECT 1',
  'expected': 1,
  },
+ {
+ 'sql': 'SELECT 1.3',
+ 'expected': 1.3,
+ },
+ {
+ 'sql': 'SELECT TRUE',
+ 'expected': True,
+ },
+ {
+ 'sql': 'SELECT "ABC"',
+ 'expected': 'ABC',
+ },
+ {
+ 'sql': 'SELECT CAST("foo" AS BYTES)',
+ 'expected': b'foo',
+ },
+ {
+ 'sql': 'SELECT TIMESTAMP "%s"' % (stamp,),
+ 'expected': zoned,
+ },
+ {
+ 'sql': 'SELECT DATETIME(TIMESTAMP "%s")' % (stamp,),
+ 'expected': naive,
+ },
+ {
+ 'sql': 'SELECT DATE(TIMESTAMP "%s")' % (stamp,),
+ 'expected': naive.date(),
+ },
+ {
+ 'sql': 'SELECT TIME(TIMESTAMP "%s")' % (stamp,),
+ 'expected': naive.time(),
+ },
  {
  'sql': 'SELECT (1, 2)',
  'expected': {'_field_1': 1, '_field_2': 2},