Skip to content

Commit

Permalink
Spark fails to read files if "created_by" metadata field is not in th…
Browse files Browse the repository at this point in the history
…e format is expects. (#262)

* format just like Spark likes it (#261)
  • Loading branch information
aloneguid authored Feb 14, 2023
1 parent 374eab4 commit 2ad390d
Show file tree
Hide file tree
Showing 3 changed files with 10 additions and 3 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/full.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name: 'Full Workflow'

env:
VERSION: 4.5.0
VERSION: 4.5.1
ASM_VERSION: 4.0.0

on:
Expand Down
6 changes: 4 additions & 2 deletions src/Parquet/File/ThriftFooter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@ public ThriftFooter(ParquetSchema schema, long totalRowCount) {
_fileMeta = CreateThriftSchema(schema);
_fileMeta.Num_rows = totalRowCount;


// Looks like Spark is sensitive about this format. See https://github.com/aloneguid/parquet-dotnet/issues/261
#if DEBUG
_fileMeta.Created_by = "Parquet.Net local dev version";
_fileMeta.Created_by = "Parquet.Net version LocalDev (build Local)";
#else
_fileMeta.Created_by = $"Parquet.Net v{Globals.Version}";
_fileMeta.Created_by = $"Parquet.Net version {Globals.Version} (build {Globals.GithubSha})";
#endif
_tree = new ThriftSchemaTree(_fileMeta.Schema);
}
Expand Down
5 changes: 5 additions & 0 deletions src/Parquet/Globals.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,10 @@ public static class Globals {
/// Library version
/// </summary>
public static readonly string Version = "${VERSION}";

/// <summary>
/// Github Actions stamped commit SHA
/// </summary>
public static readonly string GithubSha = "${GITHUB_SHA}";
}
}

0 comments on commit 2ad390d

Please sign in to comment.