diff --git a/python/pyiceberg/schema.py b/python/pyiceberg/schema.py index 6620759f86c8..b0fc41b0e177 100644 --- a/python/pyiceberg/schema.py +++ b/python/pyiceberg/schema.py @@ -75,9 +75,7 @@ def __str__(self): return "table {\n" + "\n".join([" " + str(field) for field in self.columns]) + "\n}" def __repr__(self): - return ( - f"Schema(fields={repr(self.columns)}, schema_id={self.schema_id}, identifier_field_ids={self.identifier_field_ids})" - ) + return f"Schema({', '.join(repr(column) for column in self.columns)}, schema_id={self.schema_id}, identifier_field_ids={self.identifier_field_ids})" def __eq__(self, other) -> bool: if not other: diff --git a/python/pyiceberg/table/metadata.py b/python/pyiceberg/table/metadata.py index d3d1fbbfa525..49cd323e5f98 100644 --- a/python/pyiceberg/table/metadata.py +++ b/python/pyiceberg/table/metadata.py @@ -46,6 +46,8 @@ CURRENT_SNAPSHOT_ID = "current_snapshot_id" CURRENT_SCHEMA_ID = "current_schema_id" SCHEMAS = "schemas" +DEFAULT_SPEC_ID = "default_spec_id" +PARTITION_SPEC = "partition_spec" PARTITION_SPECS = "partition_specs" SORT_ORDERS = "sort_orders" REFS = "refs" @@ -261,8 +263,10 @@ def construct_partition_specs(cls, data: Dict[str, Any]) -> Dict[str, Any]: The TableMetadata with the partition_specs set, if not provided """ if not data.get(PARTITION_SPECS): - fields = data["partition_spec"] - data[PARTITION_SPECS] = [PartitionSpec(spec_id=INITIAL_SPEC_ID, fields=fields)] + fields = data[PARTITION_SPEC] + migrated_spec = PartitionSpec(*fields) + data[PARTITION_SPECS] = [migrated_spec] + data[DEFAULT_SPEC_ID] = migrated_spec.spec_id else: check_partition_specs(data) diff --git a/python/pyiceberg/table/partitioning.py b/python/pyiceberg/table/partitioning.py index 30a9d634156a..8dcbf68454b8 100644 --- a/python/pyiceberg/table/partitioning.py +++ b/python/pyiceberg/table/partitioning.py @@ -29,7 +29,7 @@ from pyiceberg.transforms import Transform from pyiceberg.utils.iceberg_base_model import IcebergBaseModel -INITIAL_SPEC_ID = 0 +INITIAL_PARTITION_SPEC_ID = 0 _PARTITION_DATA_ID_START: int = 1000 @@ -82,19 +82,16 @@ class PartitionSpec(IcebergBaseModel): fields(List[PartitionField): list of partition fields to produce partition values """ - spec_id: int = Field(alias="spec-id") - fields: Tuple[PartitionField, ...] = Field(default_factory=tuple) + spec_id: int = Field(alias="spec-id", default=INITIAL_PARTITION_SPEC_ID) + fields: Tuple[PartitionField, ...] = Field(alias="fields", default_factory=tuple) def __init__( self, - spec_id: Optional[int] = None, - fields: Optional[Tuple[PartitionField, ...]] = None, + *fields: PartitionField, **data: Any, ): - if spec_id is not None: - data["spec-id"] = spec_id - if fields is not None: - data["fields"] = fields + if fields: + data["fields"] = tuple(fields) super().__init__(**data) def __eq__(self, other: Any) -> bool: @@ -121,6 +118,10 @@ def __str__(self): result_str += "]" return result_str + def __repr__(self) -> str: + fields = f"{', '.join(repr(column) for column in self.fields)}, " if self.fields else "" + return f"PartitionSpec({fields}spec_id={self.spec_id})" + def is_unpartitioned(self) -> bool: return not self.fields @@ -178,4 +179,4 @@ def assign_fresh_partition_spec_ids(spec: PartitionSpec, old_schema: Schema, fre transform=field.transform, ) ) - return PartitionSpec(INITIAL_SPEC_ID, fields=tuple(partition_fields)) + return PartitionSpec(*partition_fields, spec_id=INITIAL_PARTITION_SPEC_ID) diff --git a/python/pyiceberg/table/sorting.py b/python/pyiceberg/table/sorting.py index 013470fa57aa..4fafcd3b1ff2 100644 --- a/python/pyiceberg/table/sorting.py +++ b/python/pyiceberg/table/sorting.py @@ -104,6 +104,9 @@ def __str__(self): return f"{self.transform}({self.source_id}) {self.direction} {self.null_order}" +INITIAL_SORT_ORDER_ID = 1 + + class SortOrder(IcebergBaseModel): """Describes how the data is sorted within the table @@ -112,20 +115,18 @@ class SortOrder(IcebergBaseModel): The order of the sort fields within the list defines the order in which the sort is applied to the data. Args: - order_id (int): The id of the sort-order. To keep track of historical sorting + order_id (int): An unique id of the sort-order of a table. fields (List[SortField]): The fields how the table is sorted """ - def __init__(self, order_id: Optional[int] = None, *fields: SortField, **data: Any): - if order_id is not None: - data["order-id"] = order_id + order_id: int = Field(alias="order-id", default=INITIAL_SORT_ORDER_ID) + fields: List[SortField] = Field(default_factory=list) + + def __init__(self, *fields: SortField, **data: Any): if fields: data["fields"] = fields super().__init__(**data) - order_id: int = Field(alias="order-id") - fields: List[SortField] = Field(default_factory=list) - @property def is_unsorted(self) -> bool: return len(self.fields) == 0 @@ -137,10 +138,13 @@ def __str__(self) -> str: result_str += "]" return result_str + def __repr__(self): + fields = f"{', '.join(repr(column) for column in self.fields)}, " if self.fields else "" + return f"SortOrder({fields}order_id={self.order_id})" + UNSORTED_SORT_ORDER_ID = 0 UNSORTED_SORT_ORDER = SortOrder(order_id=UNSORTED_SORT_ORDER_ID) -INITIAL_SORT_ORDER_ID = 1 def assign_fresh_sort_order_ids(sort_order: SortOrder, old_schema: Schema, fresh_schema: Schema) -> SortOrder: @@ -164,7 +168,4 @@ def assign_fresh_sort_order_ids(sort_order: SortOrder, old_schema: Schema, fresh ) ) - return SortOrder( - INITIAL_SORT_ORDER_ID, - *fresh_fields, - ) + return SortOrder(*fresh_fields, order_id=INITIAL_SORT_ORDER_ID) diff --git a/python/pyproject.toml b/python/pyproject.toml index aa5ee6cd6c2b..69881b801896 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -51,9 +51,9 @@ pyyaml = "^6.0.0" pydantic = "^1.10.2" fsspec = "2022.8.2" -pyarrow = { version = "^9.0.0", optional = true } +zstandard = "^0.18.0" -zstandard = { version = "^0.18.0", optional = true } +pyarrow = { version = "^9.0.0", optional = true } python-snappy = { version = "^0.6.1", optional = true } @@ -80,7 +80,6 @@ build-backend = "poetry.core.masonry.api" [tool.poetry.extras] pyarrow = ["pyarrow"] snappy = ["python-snappy"] -python-snappy = ["zstandard"] hive = ["thrift"] s3fs = ["s3fs"] diff --git a/python/tests/catalog/test_base.py b/python/tests/catalog/test_base.py index a01b3f5788f4..5aea87c918fb 100644 --- a/python/tests/catalog/test_base.py +++ b/python/tests/catalog/test_base.py @@ -40,9 +40,9 @@ ) from pyiceberg.schema import Schema from pyiceberg.table import Table -from pyiceberg.table.metadata import INITIAL_SPEC_ID -from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec +from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder +from pyiceberg.transforms import IdentityTransform from pyiceberg.typedef import EMPTY_DICT from tests.table.test_metadata import EXAMPLE_TABLE_METADATA_V1 @@ -186,7 +186,7 @@ def update_namespace_properties( TEST_TABLE_NAME = "my_table" TEST_TABLE_SCHEMA = Schema(schema_id=1) TEST_TABLE_LOCATION = "protocol://some/location" -TEST_TABLE_PARTITION_SPEC = PartitionSpec(spec_id=INITIAL_SPEC_ID, fields=()) +TEST_TABLE_PARTITION_SPEC = PartitionSpec(PartitionField(name="x", transform=IdentityTransform(), source_id=1, field_id=1000)) TEST_TABLE_PROPERTIES = {"key1": "value1", "key2": "value2"} NO_SUCH_TABLE_ERROR = "Table does not exist: \\('com', 'organization', 'department', 'my_table'\\)" TABLE_ALREADY_EXISTS_ERROR = "Table already exists: \\('com', 'organization', 'department', 'my_table'\\)" @@ -200,7 +200,7 @@ def given_catalog_has_a_table(catalog: InMemoryCatalog) -> Table: identifier=TEST_TABLE_IDENTIFIER, schema=TEST_TABLE_SCHEMA, location=TEST_TABLE_LOCATION, - partition_spec=TEST_TABLE_PARTITION_SPEC, + partition_spec=UNPARTITIONED_PARTITION_SPEC, properties=TEST_TABLE_PROPERTIES, ) diff --git a/python/tests/catalog/test_hive.py b/python/tests/catalog/test_hive.py index 663fa33fe489..bc45bb457e87 100644 --- a/python/tests/catalog/test_hive.py +++ b/python/tests/catalog/test_hive.py @@ -77,7 +77,8 @@ @pytest.fixture def hive_table(tmp_path_factory, example_table_metadata_v2: Dict[str, Any]) -> HiveTable: metadata_path = str(tmp_path_factory.mktemp("metadata") / f"{uuid.uuid4()}.metadata.json") - ToOutputFile.table_metadata(TableMetadataV2(**example_table_metadata_v2), LocalFileIO().new_output(str(metadata_path)), True) + metadata = TableMetadataV2(**example_table_metadata_v2) + ToOutputFile.table_metadata(metadata, LocalFileIO().new_output(str(metadata_path)), True) return HiveTable( tableName="new_tabl2e", @@ -267,7 +268,7 @@ def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, assert "database/table" in metadata.location - assert metadata == TableMetadataV2( + expected = TableMetadataV2( location=metadata.location, table_uuid=metadata.table_uuid, last_updated_ms=metadata.last_updated_ms, @@ -282,10 +283,10 @@ def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, ) ], current_schema_id=0, - partition_specs=[PartitionSpec(spec_id=0)], - default_spec_id=0, last_partition_id=1000, properties={"owner": "javaberg"}, + partition_specs=[PartitionSpec()], + default_spec_id=0, current_snapshot_id=None, snapshots=[], snapshot_log=[], @@ -297,6 +298,8 @@ def test_create_table(table_schema_simple: Schema, hive_database: HiveDatabase, last_sequence_number=0, ) + assert metadata.dict() == expected.dict() + def test_load_table(hive_table: HiveTable): catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) @@ -307,8 +310,7 @@ def test_load_table(hive_table: HiveTable): catalog._client.__enter__().get_table.assert_called_with(dbname="default", tbl_name="new_tabl2e") - assert table.identifier == ("default", "new_tabl2e") - assert table.metadata == TableMetadataV2( + expected = TableMetadataV2( location="s3://bucket/test/location", table_uuid=uuid.UUID("9c12d441-03fe-4693-9a96-a0705ddf69c1"), last_updated_ms=1602638573590, @@ -329,9 +331,7 @@ def test_load_table(hive_table: HiveTable): ], current_schema_id=1, partition_specs=[ - PartitionSpec( - spec_id=0, fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),) - ) + PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0) ], default_spec_id=0, last_partition_id=1000, @@ -364,7 +364,6 @@ def test_load_table(hive_table: HiveTable): metadata_log=[MetadataLogEntry(metadata_file="s3://bucket/.../v1.json", timestamp_ms=1515100)], sort_orders=[ SortOrder( - 3, SortField( source_id=2, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST ), @@ -374,6 +373,7 @@ def test_load_table(hive_table: HiveTable): direction=SortDirection.DESC, null_order=NullOrder.NULLS_LAST, ), + order_id=3, ) ], default_sort_order_id=3, @@ -397,6 +397,9 @@ def test_load_table(hive_table: HiveTable): last_sequence_number=34, ) + assert table.identifier == ("default", "new_tabl2e") + assert expected == table.metadata + def test_rename_table_from_does_not_exists(): catalog = HiveCatalog(HIVE_CATALOG_NAME, uri=HIVE_METASTORE_FAKE_URL) diff --git a/python/tests/catalog/test_rest.py b/python/tests/catalog/test_rest.py index 5b60cd8f7c3b..e97f8ad50f9c 100644 --- a/python/tests/catalog/test_rest.py +++ b/python/tests/catalog/test_rest.py @@ -344,8 +344,8 @@ def test_load_table_200(rest_mock: Mocker): }, status_code=200, ) - table = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_table(("fokko", "table")) - assert table == Table( + actual = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN).load_table(("fokko", "table")) + expected = Table( identifier=("rest", "fokko", "table"), metadata_location="s3://warehouse/database/table/metadata/00001-5f2f8166-244c-4eae-ac36-384ecdec81fc.gz.metadata.json", metadata=TableMetadataV1( @@ -362,7 +362,6 @@ def test_load_table_200(rest_mock: Mocker): ) ], current_schema_id=0, - partition_specs=[PartitionSpec(spec_id=0, fields=())], default_spec_id=0, last_partition_id=999, properties={"owner": "bryan", "write.metadata.compression-codec": "gzip"}, @@ -422,6 +421,7 @@ def test_load_table_200(rest_mock: Mocker): ), config={"client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", "region": "us-west-2"}, ) + assert actual == expected def test_load_table_404(rest_mock: Mocker): @@ -496,7 +496,6 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): ], "partition-spec": [], "default-spec-id": 0, - "partition-specs": [{"spec-id": 0, "fields": []}], "last-partition-id": 999, "default-sort-order-id": 0, "sort-orders": [{"order-id": 0, "fields": []}], @@ -524,9 +523,9 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): schema=table_schema_simple, location=None, partition_spec=PartitionSpec( - spec_id=1, fields=(PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"),) + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"), spec_id=1 ), - sort_order=SortOrder(1, SortField(source_id=2, transform=IdentityTransform())), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), properties={"owner": "fokko"}, ) assert table == Table( @@ -547,7 +546,6 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema): ) ], current_schema_id=0, - partition_specs=[PartitionSpec(spec_id=0, fields=())], default_spec_id=0, last_partition_id=999, properties={ @@ -595,10 +593,9 @@ def test_create_table_409(rest_mock, table_schema_simple: Schema): schema=table_schema_simple, location=None, partition_spec=PartitionSpec( - spec_id=1, - fields=(PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"),), + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id") ), - sort_order=SortOrder(1, SortField(source_id=2, transform=IdentityTransform())), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), properties={"owner": "fokko"}, ) assert "Table already exists" in str(e.value) diff --git a/python/tests/table/test_init.py b/python/tests/table/test_init.py index 567e1f07a570..d0117c4df15c 100644 --- a/python/tests/table/test_init.py +++ b/python/tests/table/test_init.py @@ -82,13 +82,13 @@ def test_schemas(table): def test_spec(table): assert table.spec() == PartitionSpec( - spec_id=0, fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),) + PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0 ) def test_specs(table): assert table.specs() == { - 0: PartitionSpec(spec_id=0, fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),)) + 0: PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0) } diff --git a/python/tests/table/test_metadata.py b/python/tests/table/test_metadata.py index 4cb6795a4aa1..c88db6a2dad6 100644 --- a/python/tests/table/test_metadata.py +++ b/python/tests/table/test_metadata.py @@ -122,8 +122,20 @@ def test_v1_metadata_parsing_directly(): assert table_metadata.location == "s3://bucket/test/location" assert table_metadata.last_updated_ms == 1602638573874 assert table_metadata.last_column_id == 3 + assert table_metadata.schemas == [ + Schema( + NestedField(field_id=1, name="x", field_type=LongType(), required=True), + NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"), + NestedField(field_id=3, name="z", field_type=LongType(), required=True), + schema_id=0, + identifier_field_ids=[], + ) + ] assert table_metadata.schemas[0].schema_id == 0 assert table_metadata.current_schema_id == 0 + assert table_metadata.partition_specs == [ + PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0) + ] assert table_metadata.default_spec_id == 0 assert table_metadata.last_partition_id == 1000 assert table_metadata.current_snapshot_id is None @@ -163,9 +175,10 @@ def test_updating_metadata(example_table_metadata_v2: Dict[str, Any]): def test_serialize_v1(): - table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1).json() + table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) + table_metadata_json = table_metadata.json() expected = """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" - assert table_metadata == expected + assert table_metadata_json == expected def test_serialize_v2(example_table_metadata_v2: Dict[str, Any]): @@ -189,11 +202,7 @@ def test_migrate_v1_partition_specs(): assert len(table_metadata.partition_specs) == 1 # Spec ID gets added automatically assert table_metadata.partition_specs == [ - PartitionSpec( - spec_id=0, - fields=(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),), - last_assigned_field_id=1000, - ), + PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x")), ] diff --git a/python/tests/table/test_partitioning.py b/python/tests/table/test_partitioning.py index 1fe9dfeb015a..9aa0f7389117 100644 --- a/python/tests/table/test_partitioning.py +++ b/python/tests/table/test_partitioning.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from pyiceberg.table.partitioning import PartitionField, PartitionSpec +from pyiceberg.table.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec from pyiceberg.transforms import BucketTransform, TruncateTransform @@ -34,11 +34,15 @@ def test_partition_field_init(): ) +def test_unpartitioned_partition_spec_repr(): + assert repr(PartitionSpec()) == "PartitionSpec(spec_id=0)" + + def test_partition_spec_init(): bucket_transform: BucketTransform = BucketTransform(4) id_field1 = PartitionField(3, 1001, bucket_transform, "id") - partition_spec1 = PartitionSpec(0, (id_field1,)) + partition_spec1 = PartitionSpec(id_field1) assert partition_spec1.spec_id == 0 assert partition_spec1 == partition_spec1 @@ -47,7 +51,7 @@ def test_partition_spec_init(): assert not partition_spec1.is_unpartitioned() # only differ by PartitionField field_id id_field2 = PartitionField(3, 1002, bucket_transform, "id") - partition_spec2 = PartitionSpec(0, (id_field2,)) + partition_spec2 = PartitionSpec(id_field2) assert partition_spec1 != partition_spec2 assert partition_spec1.compatible_with(partition_spec2) assert partition_spec1.fields_by_source_id(3) == [id_field1] @@ -57,31 +61,28 @@ def test_partition_compatible_with(): bucket_transform: BucketTransform = BucketTransform(4) field1 = PartitionField(3, 100, bucket_transform, "id") field2 = PartitionField(3, 102, bucket_transform, "id") - lhs = PartitionSpec(0, (field1,)) - rhs = PartitionSpec(0, (field1, field2)) + lhs = PartitionSpec( + field1, + ) + rhs = PartitionSpec(field1, field2) assert not lhs.compatible_with(rhs) def test_unpartitioned(): - unpartitioned = PartitionSpec(1, ()) - - assert not unpartitioned.fields - assert unpartitioned.is_unpartitioned() - assert str(unpartitioned) == "[]" + assert len(UNPARTITIONED_PARTITION_SPEC.fields) == 0 + assert UNPARTITIONED_PARTITION_SPEC.is_unpartitioned() + assert str(UNPARTITIONED_PARTITION_SPEC) == "[]" -def test_serialize_unpartition_spec(): - unpartitioned = PartitionSpec(1, ()) - assert unpartitioned.json() == """{"spec-id": 1, "fields": []}""" +def test_serialize_unpartitioned_spec(): + assert UNPARTITIONED_PARTITION_SPEC.json() == """{"spec-id": 0, "fields": []}""" def test_serialize_partition_spec(): partitioned = PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), + PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), spec_id=3, - fields=( - PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), - PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), - ), ) assert ( partitioned.json() diff --git a/python/tests/table/test_sorting.py b/python/tests/table/test_sorting.py index 384a636e0982..00ad34ba8062 100644 --- a/python/tests/table/test_sorting.py +++ b/python/tests/table/test_sorting.py @@ -34,10 +34,10 @@ @pytest.fixture def sort_order() -> SortOrder: return SortOrder( - 22, SortField(source_id=19, transform=IdentityTransform(), null_order=NullOrder.NULLS_FIRST), SortField(source_id=25, transform=BucketTransform(4), direction=SortDirection.DESC), SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC), + order_id=22, ) @@ -61,7 +61,6 @@ def test_sorting_schema(example_table_metadata_v2: Dict[str, Any]): assert table_metadata.sort_orders == [ SortOrder( - 3, SortField(2, IdentityTransform(), SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), SortField( 3, @@ -69,6 +68,7 @@ def test_sorting_schema(example_table_metadata_v2: Dict[str, Any]): direction=SortDirection.DESC, null_order=NullOrder.NULLS_LAST, ), + order_id=3, ) ] @@ -83,10 +83,15 @@ def test_sorting_to_string(sort_order: SortOrder): def test_sorting_to_repr(sort_order: SortOrder): - expected = """SortOrder(order_id=22, fields=[SortField(source_id=19, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), SortField(source_id=25, transform=BucketTransform(num_buckets=4), direction=SortDirection.DESC, null_order=NullOrder.NULLS_LAST), SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST)])""" + expected = """SortOrder(SortField(source_id=19, transform=IdentityTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), SortField(source_id=25, transform=BucketTransform(num_buckets=4), direction=SortDirection.DESC, null_order=NullOrder.NULLS_LAST), SortField(source_id=22, transform=VoidTransform(), direction=SortDirection.ASC, null_order=NullOrder.NULLS_FIRST), order_id=22)""" assert repr(sort_order) == expected +def test_unsorting_to_repr(): + expected = """SortOrder(order_id=0)""" + assert repr(UNSORTED_SORT_ORDER) == expected + + def test_sorting_repr(sort_order: SortOrder): """To make sure that the repr converts back to the original object""" assert sort_order == eval(repr(sort_order)) diff --git a/python/tests/test_schema.py b/python/tests/test_schema.py index f99b315c698e..9fbebb090286 100644 --- a/python/tests/test_schema.py +++ b/python/tests/test_schema.py @@ -49,22 +49,20 @@ def test_schema_str(table_schema_simple: Schema): ) -@pytest.mark.parametrize( - "schema_repr, expected_repr", - [ - ( - schema.Schema(NestedField(1, "foo", StringType()), schema_id=1), - "Schema(fields=(NestedField(field_id=1, name='foo', field_type=StringType(), required=True),), schema_id=1, identifier_field_ids=[])", - ), - ( - schema.Schema(NestedField(1, "foo", StringType()), NestedField(2, "bar", IntegerType(), required=False), schema_id=1), - "Schema(fields=(NestedField(field_id=1, name='foo', field_type=StringType(), required=True), NestedField(field_id=2, name='bar', field_type=IntegerType(), required=False)), schema_id=1, identifier_field_ids=[])", - ), - ], -) -def test_schema_repr(schema_repr: Schema, expected_repr: str): +def test_schema_repr_single_field(): """Test schema representation""" - assert repr(schema_repr) == expected_repr + actual = repr(schema.Schema(NestedField(1, "foo", StringType()), schema_id=1)) + expected = "Schema(NestedField(field_id=1, name='foo', field_type=StringType(), required=True), schema_id=1, identifier_field_ids=[])" + assert expected == actual + + +def test_schema_repr_two_fields(): + """Test schema representation""" + actual = repr( + schema.Schema(NestedField(1, "foo", StringType()), NestedField(2, "bar", IntegerType(), required=False), schema_id=1) + ) + expected = "Schema(NestedField(field_id=1, name='foo', field_type=StringType(), required=True), NestedField(field_id=2, name='bar', field_type=IntegerType(), required=False), schema_id=1, identifier_field_ids=[])" + assert expected == actual def test_schema_raise_on_duplicate_names():