feat: Extended DB API parameter syntax to optionally provide parameter types

docs/dbapi.rst

@@ -4,3 +4,40 @@ DB-API Reference
 .. automodule:: google.cloud.bigquery.dbapi
  :members:
  :show-inheritance:
+
+
+DB-API Query-Parameter Syntax
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The BigQuery DB-API uses the `qmark` `parameter style
+<https://www.python.org/dev/peps/pep-0249/#paramstyle>`_ for
+unnamed/positional parameters and the `pyformat` parameter style for
+named parameters.
+
+An example of a query using unnamed parameters::
+
+ insert into people (name, income) values (?, ?)
+
+and using named parameters::
+
+ insert into people (name, income) values (%(name)s, %(income)s)
+
+Providing explicit type information
+-----------------------------------
+
+BigQuery requires type information for parameters. The The BigQuery
+DB-API can usually determine parameter types for parameters based on
+provided values. Sometimes, however, types can't be determined (for
+example when `None` is passed) or are determined incorrectly (for
+example when passing a floating-point value to a numeric column).
+
+The BigQuery DB-API provides an extended parameter syntax. For named
+parameters, a BigQuery type is provided after the name separated by a
+colon, as in::
+
+ insert into people (name, income) values (%(name:string)s, %(income:numeric)s)
+
+For unnamed parameters, use the named syntax with a type, but now
+name, as in::
+
+ insert into people (name, income) values (%(:string)s, %(:numeric)s)

google/cloud/bigquery/_helpers.py

@@ -275,7 +275,7 @@ def _int_to_json(value):
 
 def _float_to_json(value):
  """Coerce 'value' to an JSON-compatible representation."""
- return value
+ return value if value is None else float(value)
 
 
 def _decimal_to_json(value):

google/cloud/bigquery/dbapi/_helpers.py

@@ -20,15 +20,36 @@
 import numbers
 
 from google.cloud import bigquery
-from google.cloud.bigquery import table
+from google.cloud.bigquery import table, enums
 from google.cloud.bigquery.dbapi import exceptions
 
 
 _NUMERIC_SERVER_MIN = decimal.Decimal("-9.9999999999999999999999999999999999999E+28")
 _NUMERIC_SERVER_MAX = decimal.Decimal("9.9999999999999999999999999999999999999E+28")
 
 
-def scalar_to_query_parameter(value, name=None):
+def _parameter_type(name, value, query_parameter_type=None, value_doc=""):
+ if query_parameter_type:
+ try:
+ parameter_type = getattr(
+ enums.SqlParameterScalarTypes, query_parameter_type.upper()
+ )._type
+ except AttributeError:
+ raise exceptions.ProgrammingError(
+ f"The given parameter type, {query_parameter_type},"
+ f" for {name} is not a valid BigQuery scalar type."
+ )
+ else:
+ parameter_type = bigquery_scalar_type(value)
+ if parameter_type is None:
+ raise exceptions.ProgrammingError(
+ f"Encountered parameter {name} with "
+ f"{value_doc} value {value} of unexpected type."
+ )
+ return parameter_type
+
+
+def scalar_to_query_parameter(value, name=None, query_parameter_type=None):
  """Convert a scalar value into a query parameter.
 
  Args:
@@ -37,6 +58,7 @@ def scalar_to_query_parameter(value, name=None):
 
  name (str):
  (Optional) Name of the query parameter.
+ query_parameter_type (Optional[str]): Given type for the parameter.
 
  Returns:
  google.cloud.bigquery.ScalarQueryParameter:
@@ -47,24 +69,19 @@ def scalar_to_query_parameter(value, name=None):
  google.cloud.bigquery.dbapi.exceptions.ProgrammingError:
  if the type cannot be determined.
  """
- parameter_type = bigquery_scalar_type(value)
-
- if parameter_type is None:
- raise exceptions.ProgrammingError(
- "encountered parameter {} with value {} of unexpected type".format(
- name, value
- )
- )
- return bigquery.ScalarQueryParameter(name, parameter_type, value)
+ return bigquery.ScalarQueryParameter(
+ name, _parameter_type(name, value, query_parameter_type), value
+ )
 
 
-def array_to_query_parameter(value, name=None):
+def array_to_query_parameter(value, name=None, query_parameter_type=None):
  """Convert an array-like value into a query parameter.
 
  Args:
  value (Sequence[Any]): The elements of the array (should not be a
  string-like Sequence).
  name (Optional[str]): Name of the query parameter.
+ query_parameter_type (Optional[str]): Given type for the parameter.
 
  Returns:
  A query parameter corresponding with the type and value of the plain
@@ -80,53 +97,58 @@ def array_to_query_parameter(value, name=None):
  "not string-like.".format(name)
  )
 
- if not value:
+ if query_parameter_type or value:
+ array_type = _parameter_type(
+ name,
+ value[0] if value else None,
+ query_parameter_type,
+ value_doc="array element ",
+ )
+ else:
  raise exceptions.ProgrammingError(
  "Encountered an empty array-like value of parameter {}, cannot "
  "determine array elements type.".format(name)
  )
 
- # Assume that all elements are of the same type, and let the backend handle
- # any type incompatibilities among the array elements
- array_type = bigquery_scalar_type(value[0])
- if array_type is None:
- raise exceptions.ProgrammingError(
- "Encountered unexpected first array element of parameter {}, "
- "cannot determine array elements type.".format(name)
- )
-
  return bigquery.ArrayQueryParameter(name, array_type, value)
 
 
-def to_query_parameters_list(parameters):
+def to_query_parameters_list(parameters, parameter_types):
  """Converts a sequence of parameter values into query parameters.
 
  Args:
  parameters (Sequence[Any]): Sequence of query parameter values.
+ parameter_types:
+ A list of parameter types, one for each parameter.
+ Unknown types are provided as None.
 
  Returns:
  List[google.cloud.bigquery.query._AbstractQueryParameter]:
  A list of query parameters.
  """
  result = []
 
- for valuein parameters:
+ for value, type_ in zip(parameters, parameter_types):
  if isinstance(value, collections_abc.Mapping):
  raise NotImplementedError("STRUCT-like parameter values are not supported.")
  elif array_like(value):
- param = array_to_query_parameter(value)
+ param = array_to_query_parameter(value, None, type_)
  else:
- param = scalar_to_query_parameter(value)
+ param = scalar_to_query_parameter(value, None, type_)
+
  result.append(param)
 
  return result
 
 
-def to_query_parameters_dict(parameters):
+def to_query_parameters_dict(parameters, query_parameter_types):
  """Converts a dictionary of parameter values into query parameters.
 
  Args:
  parameters (Mapping[str, Any]): Dictionary of query parameter values.
+ parameter_types:
+ A dictionary of parameter types. It needn't have a key for each
+ parameter.
 
  Returns:
  List[google.cloud.bigquery.query._AbstractQueryParameter]:
@@ -140,21 +162,38 @@ def to_query_parameters_dict(parameters):
  "STRUCT-like parameter values are not supported "
  "(parameter {}).".format(name)
  )
- elif array_like(value):
- param = array_to_query_parameter(value, name=name)
  else:
- param = scalar_to_query_parameter(value, name=name)
+ query_parameter_type = query_parameter_types.get(name)
+ if array_like(value):
+ param = array_to_query_parameter(
+ value, name=name, query_parameter_type=query_parameter_type
+ )
+ else:
+ param = scalar_to_query_parameter(
+ value, name=name, query_parameter_type=query_parameter_type,
+ )
+
  result.append(param)
 
  return result
 
 
-def to_query_parameters(parameters):
+def to_query_parameters(parameters, parameter_types):
  """Converts DB-API parameter values into query parameters.
 
  Args:
  parameters (Union[Mapping[str, Any], Sequence[Any]]):
  A dictionary or sequence of query parameter values.
+ parameter_types (Union[Mapping[str, str], Sequence[str]]):
+ A dictionary or list of parameter types.
+
+ If parameters is a mapping, then this must be a dictionary
+ of parameter types. It needn't have a key for each
+ parameter.
+
+ If parameters is a sequence, then this must be a list of
+ parameter types, one for each paramater. Unknown types
+ are provided as None.
 
  Returns:
  List[google.cloud.bigquery.query._AbstractQueryParameter]:
@@ -164,9 +203,9 @@ def to_query_parameters(parameters):
  return []
 
  if isinstance(parameters, collections_abc.Mapping):
- return to_query_parameters_dict(parameters)
-
- return to_query_parameters_list(parameters)
+ return to_query_parameters_dict(parameters, parameter_types)
+ else:
+return to_query_parameters_list(parameters, parameter_types)
 
 
 def bigquery_scalar_type(value):

google/cloud/bigquery/dbapi/cursor.py

@@ -18,6 +18,7 @@
 from collections import abc as collections_abc
 import copy
 import logging
+import re
 
 try:
  from google.cloud.bigquery_storage import ArrowSerializationOptions
@@ -161,6 +162,14 @@ def execute(self, operation, parameters=None, job_id=None, job_config=None):
  job_config (google.cloud.bigquery.job.QueryJobConfig):
  (Optional) Extra configuration options for the query job.
  """
+ formatted_operation, parameter_types = _format_operation(operation, parameters)
+ self._execute(
+ formatted_operation, parameters, job_id, job_config, parameter_types
+ )
+
+ def _execute(
+ self, formatted_operation, parameters, job_id, job_config, parameter_types
+ ):
  self._query_data = None
  self._query_job = None
  client = self.connection._client
@@ -169,8 +178,7 @@ def execute(self, operation, parameters=None, job_id=None, job_config=None):
  # query parameters was not one of the standard options. Convert both
  # the query and the parameters to the format expected by the client
  # libraries.
- formatted_operation = _format_operation(operation, parameters=parameters)
- query_parameters = _helpers.to_query_parameters(parameters)
+ query_parameters = _helpers.to_query_parameters(parameters, parameter_types)
 
  if client._default_query_job_config:
  if job_config:
@@ -209,8 +217,19 @@ def executemany(self, operation, seq_of_parameters):
  seq_of_parameters (Union[Sequence[Mapping[str, Any], Sequence[Any]]]):
  Sequence of many sets of parameter values.
  """
- for parameters in seq_of_parameters:
- self.execute(operation, parameters)
+ if seq_of_parameters:
+ # There's no reason to format the line more than once, as
+ # the operation only barely depends on the parameters. So
+ # we just use the first set of parameters. If there are
+ # different numbers or types of parameters, we'll error
+ # anyway.
+ formatted_operation, parameter_types = _format_operation(
+ operation, seq_of_parameters[0]
+ )
+ for parameters in seq_of_parameters:
+ self._execute(
+ formatted_operation, parameters, None, None, parameter_types
+ )
 
  def _try_fetch(self, size=None):
  """Try to start fetching data, if not yet started.
@@ -427,7 +446,7 @@ def _format_operation_dict(operation, parameters):
  raise exceptions.ProgrammingError(exc)
 
 
-def _format_operation(operation, parameters=None):
+def _format_operation(operation, parameters):
  """Formats parameters in operation in way BigQuery expects.
 
  Args:
@@ -445,9 +464,67 @@ def _format_operation(operation, parameters=None):
  ``parameters`` argument.
  """
  if parameters is None or len(parameters) == 0:
- return operation.replace("%%", "%") # Still do percent de-escaping.
+ return operation.replace("%%", "%"), None # Still do percent de-escaping.
+
+ operation, parameter_types = _extract_types(operation)
+ if parameter_types is None:
+ raise exceptions.ProgrammingError(
+ f"Parameters were provided, but {repr(operation)} has no placeholders."
+ )
 
  if isinstance(parameters, collections_abc.Mapping):
- return _format_operation_dict(operation, parameters)
+ return _format_operation_dict(operation, parameters), parameter_types
+
+ return _format_operation_list(operation, parameters), parameter_types
+
+
+def _extract_types(
+ operation, extra_type_sub=re.compile(r"(%*)%(?:\(([^:)]*)(?::(\w+))?\))?s").sub
+):
+ """Remove type information from parameter placeholders.
+
+ For every parameter of the form %(name:type)s, replace with %(name)s and add the
+ item name->type to dict that's returned.
+
+ Returns operation without type information and a dictionary of names and types.
+ """
+ parameter_types = None
+
+ def repl(m):
+ nonlocal parameter_types
+ prefix, name, type_ = m.groups()
+ if len(prefix) % 2:
+ # The prefix has an odd number of %s, the last of which
+ # escapes the % we're looking for, so we don't want to
+ # change anything.
+ return m.group(0)
+
+ try:
+ if name:
+ if not parameter_types:
+ parameter_types = {}
+ if type_:
+ if name in parameter_types:
+ if type_ != parameter_types[name]:
+ raise exceptions.ProgrammingError(
+ f"Conflicting types for {name}: "
+ f"{parameter_types[name]} and {type_}."
+ )
+ else:
+ parameter_types[name] = type_
+ else:
+ if not isinstance(parameter_types, dict):
+ raise TypeError()
+
+ return f"{prefix}%({name})s"
+ else:
+ if parameter_types is None:
+ parameter_types = []
+ parameter_types.append(type_)
+ return f"{prefix}%s"
+ except (AttributeError, TypeError):
+ raise exceptions.ProgrammingError(
+ f"{repr(operation)} mixes named and unamed parameters."
+ )
 
- return _format_operation_list(operation, parameters)
+ return extra_type_sub(repl, operation), parameter_types