Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Core, Test: Parsing and Writing Tests for V3 Metadata #12025

Open
wants to merge 14 commits into
base: main
Choose a base branch
from
Open

Core, Test: Parsing and Writing Tests for V3 Metadata #12025

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
063b7f5
Add row-lineage fields and next-row-id
HonahX Dec 5, 2024
92c616b
remove unnecessary test
HonahX Dec 5, 2024
74bc724
add unit tests for new fields
HonahX Dec 9, 2024
baf6e0b
add v3 test metadata json
HonahX Dec 9, 2024
4786fbe
add tests for v3 row lineage parsing
HonahX Dec 9, 2024
4d5c87d
Merge branch 'main' into honahx_table_metadata_v3_parse_and_test_2
HonahX Jan 8, 2025
8f667de
Merge branch 'main' into honahx_table_metadata_v3_parse_and_test_2
HonahX Jan 10, 2025
4f50370
Fix Compilation error
HonahX Jan 10, 2025
2551587
remove fields for row lineage
HonahX Jan 10, 2025
0c72574
Merge branch 'main' into v3_metadata_parametrize
HonahX Jan 21, 2025
a5f4568
Reuse field Source
HonahX Jan 21, 2025
8b430ee
add constants for min version
HonahX Jan 21, 2025
ec77a78
fix style issue
HonahX Jan 21, 2025
d3cfffb
remove compatibility check in parser
HonahX Jan 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions core/src/main/java/org/apache/iceberg/TableMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,14 @@ public class TableMetadata implements Serializable {
static final int INITIAL_SPEC_ID = 0;
static final int INITIAL_SORT_ORDER_ID = 1;
static final int INITIAL_SCHEMA_ID = 0;
static final int MIN_FORMAT_VERSION_BRANCHING = 2;
static final int MIN_FORMAT_VERSION_UUID_REQUIRED = 2;
static final int MIN_FORMAT_VERSION_SEQUENCE_NUMBER = 2;
static final int MIN_FORMAT_VERSION_PARTITION_SPECS_REQUIRED = 2;
static final int MIN_FORMAT_VERSION_LAST_PARTITION_ID_REQUIRED = 2;
static final int MIN_FORMAT_VERSION_SORT_ORDER_REQUIRED = 2;
static final int MIN_FORMAT_VERSION_SCHEMAS_REQUIRED = 2;
static final int MIN_FORMAT_VERSION_CURRENT_SCHEMA_ID_REQUIRED = 2;

private static final long ONE_MINUTE = TimeUnit.MINUTES.toMillis(1);

Expand Down
2 changes: 2 additions & 0 deletions core/src/main/java/org/apache/iceberg/TableMetadataParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ private TableMetadataParser() {}
static final String METADATA_LOG = "metadata-log";
static final String STATISTICS = "statistics";
static final String PARTITION_STATISTICS = "partition-statistics";
static final String ROW_LINEAGE = "row-lineage";
static final String NEXT_ROW_ID = "next-row-id";

public static void overwrite(TableMetadata metadata, OutputFile outputFile) {
internalWrite(metadata, outputFile, true);
Expand Down
409 changes: 274 additions & 135 deletions core/src/test/java/org/apache/iceberg/TestTableMetadata.java
View file
Open in desktop

Large diffs are not rendered by default.

0 ...ableMetadataPartitionStatisticsFiles.json → ...leMetadataV2PartitionStatisticsFiles.json
View file
Open in desktop
File renamed without changes.
0 ...sources/TableMetadataStatisticsFiles.json → ...urces/TableMetadataV2StatisticsFiles.json
View file
Open in desktop
File renamed without changes.
146 changes: 146 additions & 0 deletions core/src/test/resources/TableMetadataV3CurrentSchemaNotFound.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
{
"format-version": 3,
"table-uuid": "9faafb13-1ce7-4603-93f8-197afc7394a9",
"location": "s3://bucket/test/location",
"last-sequence-number": 34,
"last-updated-ms": 1733710954170,
"last-column-id": 3,
"current-schema-id": 2,
"schemas": [
{
"type": "struct",
"schema-id": 7,
"fields": [
{
"id": 1,
"name": "x",
"required": true,
"type": "long"
},
{
"id": 2,
"name": "y",
"required": true,
"type": "long",
"doc": "comment"
},
{
"id": 3,
"name": "z",
"required": true,
"type": "long"
}
]
},
{
"type": "struct",
"schema-id": 6,
"fields": [
{
"id": 10,
"name": "x",
"required": true,
"type": "string"
}
]
}
],
"default-spec-id": 5,
"partition-specs": [
{
"spec-id": 5,
"fields": []
}
],
"last-partition-id": 999,
"default-sort-order-id": 3,
"sort-orders": [
{
"order-id": 3,
"fields": [
{
"transform": "identity",
"source-id": 2,
"direction": "asc",
"null-order": "nulls-first"
},
{
"transform": "bucket[4]",
"source-id": 3,
"direction": "desc",
"null-order": "nulls-last"
}
]
}
],
"properties": {
"property": "value"
},
"current-snapshot-id": 1733710954168,
"refs": {
"main": {
"snapshot-id": 1733710954168,
"type": "branch"
},
"previous": {
"snapshot-id": 1733710953138,
"type": "tag"
},
"test": {
"snapshot-id": 1733710953138,
"type": "branch"
}
},
"snapshots": [
{
"snapshot-id": 1733710953138,
"timestamp-ms": 1733710953138,
"manifest-list": "/var/folders/z8/cv70q7pj1l13r_j628cyyks80000gn/T/junit-9477945946492951907/manifests17078459248221091859.tmp"
},
{
"snapshot-id": 1733710954168,
"parent-snapshot-id": 1733710953138,
"timestamp-ms": 1733710954168,
"manifest-list": "/var/folders/z8/cv70q7pj1l13r_j628cyyks80000gn/T/junit-9477945946492951907/manifests17490643917806818981.tmp",
"schema-id": 7
}
],
"statistics": [
{
"snapshot-id": 11,
"statistics-path": "/some/stats/file.puffin",
"file-size-in-bytes": 100,
"file-footer-size-in-bytes": 42,
"blob-metadata": [
{
"type": "some-stats",
"snapshot-id": 11,
"sequence-number": 2,
"fields": [
4
]
}
]
}
],
"partition-statistics": [
{
"snapshot-id": 11,
"statistics-path": "/some/partition/stats/file.parquet",
"file-size-in-bytes": 42
}
],
"row-lineage": false,
"next-row-id": 0,
"snapshot-log": [
{
"timestamp-ms": 1733710953138,
"snapshot-id": 1733710953138
},
{
"timestamp-ms": 1733710954168,
"snapshot-id": 1733710954168
}
],
"metadata-log": []
}
145 changes: 145 additions & 0 deletions core/src/test/resources/TableMetadataV3MissingLastPartitionId.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
{
"format-version": 3,
"table-uuid": "9faafb13-1ce7-4603-93f8-197afc7394a9",
"location": "s3://bucket/test/location",
"last-sequence-number": 34,
"last-updated-ms": 1733710954170,
"last-column-id": 3,
"current-schema-id": 7,
"schemas": [
{
"type": "struct",
"schema-id": 7,
"fields": [
{
"id": 1,
"name": "x",
"required": true,
"type": "long"
},
{
"id": 2,
"name": "y",
"required": true,
"type": "long",
"doc": "comment"
},
{
"id": 3,
"name": "z",
"required": true,
"type": "long"
}
]
},
{
"type": "struct",
"schema-id": 6,
"fields": [
{
"id": 10,
"name": "x",
"required": true,
"type": "string"
}
]
}
],
"default-spec-id": 5,
"partition-specs": [
{
"spec-id": 5,
"fields": []
}
],
"default-sort-order-id": 3,
"sort-orders": [
{
"order-id": 3,
"fields": [
{
"transform": "identity",
"source-id": 2,
"direction": "asc",
"null-order": "nulls-first"
},
{
"transform": "bucket[4]",
"source-id": 3,
"direction": "desc",
"null-order": "nulls-last"
}
]
}
],
"properties": {
"property": "value"
},
"current-snapshot-id": 1733710954168,
"refs": {
"main": {
"snapshot-id": 1733710954168,
"type": "branch"
},
"previous": {
"snapshot-id": 1733710953138,
"type": "tag"
},
"test": {
"snapshot-id": 1733710953138,
"type": "branch"
}
},
"snapshots": [
{
"snapshot-id": 1733710953138,
"timestamp-ms": 1733710953138,
"manifest-list": "/var/folders/z8/cv70q7pj1l13r_j628cyyks80000gn/T/junit-9477945946492951907/manifests17078459248221091859.tmp"
},
{
"snapshot-id": 1733710954168,
"parent-snapshot-id": 1733710953138,
"timestamp-ms": 1733710954168,
"manifest-list": "/var/folders/z8/cv70q7pj1l13r_j628cyyks80000gn/T/junit-9477945946492951907/manifests17490643917806818981.tmp",
"schema-id": 7
}
],
"statistics": [
{
"snapshot-id": 11,
"statistics-path": "/some/stats/file.puffin",
"file-size-in-bytes": 100,
"file-footer-size-in-bytes": 42,
"blob-metadata": [
{
"type": "some-stats",
"snapshot-id": 11,
"sequence-number": 2,
"fields": [
4
]
}
]
}
],
"partition-statistics": [
{
"snapshot-id": 11,
"statistics-path": "/some/partition/stats/file.parquet",
"file-size-in-bytes": 42
}
],
"row-lineage": false,
"next-row-id": 0,
"snapshot-log": [
{
"timestamp-ms": 1733710953138,
"snapshot-id": 1733710953138
},
{
"timestamp-ms": 1733710954168,
"snapshot-id": 1733710954168
}
],
"metadata-log": []
}
Loading