feat: support full round-trip persistence for multimodal reference cols (#2511)

Saves multimodal metadata descriptions on .to_gbq().
Fixes #<452681068> 🦕

Author: shuoweil

bigframes/dtypes.py

@@ -772,6 +772,13 @@ def convert_schema_field(
 ) ->typing.Tuple[str, Dtype]:
 is_repeated=field.mode=="REPEATED"
 iffield.field_type=="RECORD":
+iffield.description==OBJ_REF_DESCRIPTION_TAG:
+bf_dtype=OBJ_REF_DTYPE# type: ignore
+ifis_repeated:
+pa_type=pa.list_(bigframes_dtype_to_arrow_dtype(bf_dtype))
+bf_dtype=pd.ArrowDtype(pa_type)
+returnfield.name, bf_dtype
+
 mapped_fields=map(convert_schema_field, field.fields)
 fields= []
 forname, dtypeinmapped_fields:
@@ -815,7 +822,11 @@ def convert_to_schema_field(
  )
 inner_field=convert_to_schema_field(name, inner_type, overrides)
 returngoogle.cloud.bigquery.SchemaField(
-name, inner_field.field_type, mode="REPEATED", fields=inner_field.fields
+name,
+inner_field.field_type,
+mode="REPEATED",
+fields=inner_field.fields,
+description=inner_field.description,
  )
 ifpa.types.is_struct(bigframes_dtype.pyarrow_dtype):
 inner_fields: list[google.cloud.bigquery.SchemaField] = []
@@ -827,6 +838,14 @@ def convert_to_schema_field(
 convert_to_schema_field(field.name, inner_bf_type, overrides)
  )
 
+ifbigframes_dtype==OBJ_REF_DTYPE:
+returngoogle.cloud.bigquery.SchemaField(
+name,
+"RECORD",
+fields=inner_fields,
+description=OBJ_REF_DESCRIPTION_TAG,
+ )
+
 returngoogle.cloud.bigquery.SchemaField(
 name, "RECORD", fields=inner_fields
  )
@@ -971,6 +990,7 @@ def lcd_type_or_throw(dtype1: Dtype, dtype2: Dtype) -> Dtype:
 
 
 TIMEDELTA_DESCRIPTION_TAG="#microseconds"
+OBJ_REF_DESCRIPTION_TAG="bigframes_dtype: OBJ_REF_DTYPE"
 
 
 defcontains_db_dtypes_json_arrow_type(type_):

bigframes/session/bq_caching_executor.py

@@ -334,13 +334,14 @@ def _export_gbq(
 session=array_value.session,
  )
 
-has_timedelta_col=any(
-t==bigframes.dtypes.TIMEDELTA_DTYPEfortinarray_value.schema.dtypes
+has_special_dtype_col=any(
+tin (bigframes.dtypes.TIMEDELTA_DTYPE, bigframes.dtypes.OBJ_REF_DTYPE)
+fortinarray_value.schema.dtypes
  )
 
-ifspec.if_exists!="append"andhas_timedelta_col:
+ifspec.if_exists!="append"andhas_special_dtype_col:
 # Only update schema if this is not modifying an existing table, and the
-# new table contains timedelta columns.
+# new table contains special columns (like timedelta or obj_ref).
 table=self.bqclient.get_table(spec.table)
 table.schema=array_value.schema.to_bigquery()
 self.bqclient.update_table(table, ["schema"])

tests/system/small/test_dataframe_io.py

@@ -1002,6 +1002,28 @@ def test_to_gbq_timedelta_tag_ignored_when_appending(bigquery_client, dataset_id
 asserttable.schema[0].descriptionisNone
 
 
+deftest_to_gbq_obj_ref(session, dataset_id: str, bigquery_client):
+destination_table=f"{dataset_id}.test_to_gbq_obj_ref"
+sql="""
+ SELECT
+ 'gs://cloud-samples-data/vision/ocr/sign.jpg' AS uri_col
+ """
+df=session.read_gbq(sql)
+df["obj_ref_col"] =df["uri_col"].str.to_blob()
+df=df.drop(columns=["uri_col"])
+
+df.to_gbq(destination_table)
+
+table=bigquery_client.get_table(destination_table)
+obj_ref_field=next(fforfintable.schemaiff.name=="obj_ref_col")
+assertobj_ref_field.field_type=="RECORD"
+assertobj_ref_field.description=="bigframes_dtype: OBJ_REF_DTYPE"
+
+reloaded_df=session.read_gbq(destination_table)
+assertreloaded_df["obj_ref_col"].dtype==dtypes.OBJ_REF_DTYPE
+assertlen(reloaded_df) ==1
+
+
 @pytest.mark.parametrize(
  ("index"),
  [True, False],

tests/unit/test_dtypes.py

@@ -71,3 +71,11 @@ def test_infer_literal_type_arrow_scalar(scalar, expected_dtype):
 )
 deftest_contains_db_dtypes_json_arrow_type(type_, expected):
 assertbigframes.dtypes.contains_db_dtypes_json_arrow_type(type_) ==expected
+
+
+deftest_convert_to_schema_field_list_description():
+bf_dtype=bigframes.dtypes.OBJ_REF_DTYPE
+list_bf_dtype=bigframes.dtypes.list_type(bf_dtype)
+field=bigframes.dtypes.convert_to_schema_field("my_list", list_bf_dtype)
+assertfield.description=="bigframes_dtype: OBJ_REF_DTYPE"
+assertfield.mode=="REPEATED"