Use the Double primative for JSON Schema "number" type (#111)

Problem ======= The JSON Schema stipulates that [a "number" type can be an integer, floating point, or exponential notation](https://json-schema.org/understanding-json-schema/reference/numeric). Currently, the "number" type is treated the same as an integer when instantiating `fromJSONSchema`. In those cases, providing a float value of e.g. `2.5` will fail because it can't be converted into a BigInt. Solution ======== I changed the primitive to Double for "number" types when creating a schema from JSON. Co-authored-by: Wil Wade <[email protected]>
LibertyDSNP · Jan 11, 2024 · 0a42955 · 0a42955
1 parent e0f1ebd
commit 0a42955
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 8 deletions.
diff --git a/lib/jsonSchema.ts b/lib/jsonSchema.ts
@@ -69,8 +69,9 @@ const fromJsonSchemaArray = (fieldValue: SupportedJSONSchema4, optionalFieldList
             }
             return fields.createListField('UTF8', optionalFieldList);
         case 'integer':
-        case 'number':
             return fields.createListField('INT64', optionalFieldList);
+        case 'number':
+            return fields.createListField('DOUBLE', optionalFieldList);
         case 'boolean':
             return fields.createListField('BOOLEAN', optionalFieldList);
         case 'object':
@@ -96,8 +97,9 @@ const fromJsonSchemaField = (jsonSchema: JSONSchema4) => (fieldName: string, fie
             }
             return fields.createStringField(optional);
         case 'integer':
-        case 'number':
             return fields.createIntField(64, optional);
+        case 'number':
+            return fields.createDoubleField(optional);
         case 'boolean':
             return fields.createBooleanField(optional);
         case 'array':

diff --git a/test/jsonSchema.test.ts b/test/jsonSchema.test.ts
@@ -58,6 +58,7 @@ describe("Json Schema Conversion Test File", async function () {
     "properties": {
       "string_field": { "type": "string" },
       "int_field": { "type": "integer" },
+      "number_field": { "type": "number" },
       "array_field": {
         "type": "array",
         "items": { "type": "string" },
@@ -118,6 +119,7 @@ describe("Json Schema Conversion Test File", async function () {
   const row1 = {
     string_field: 'string value',
     int_field: 10n,
+    number_field: 2.5,
     timestamp_array_field: { list: [{ element: new Date("2023-01-01 GMT") }] },
 
     timestamp_field: new Date("2023-01-01 GMT"),

diff --git a/test/test-files/array.schema.result.json b/test/test-files/array.schema.result.json
@@ -9,7 +9,7 @@
           "fields": {
             "element": {
               "optional": true,
-              "type": "INT64",
+              "type": "DOUBLE",
               "encoding": "PLAIN",
               "compression": "UNCOMPRESSED"
             }
@@ -61,7 +61,7 @@
           "fields": {
             "element": {
               "name": "element",
-              "primitiveType": "INT64",
+              "primitiveType": "DOUBLE",
               "path": [
                 "numberArray",
                 "list",
@@ -148,7 +148,7 @@
           "fields": {
             "element": {
               "name": "element",
-              "primitiveType": "INT64",
+              "primitiveType": "DOUBLE",
               "path": [
                 "numberArray",
                 "list",
@@ -179,7 +179,7 @@
       "fields": {
         "element": {
           "name": "element",
-          "primitiveType": "INT64",
+          "primitiveType": "DOUBLE",
           "path": [
             "numberArray",
             "list",
@@ -195,7 +195,7 @@
     },
     {
       "name": "element",
-      "primitiveType": "INT64",
+      "primitiveType": "DOUBLE",
       "path": [
         "numberArray",
         "list",

diff --git a/test/test-files/json-schema-test-file.result.json b/test/test-files/json-schema-test-file.result.json
@@ -4,7 +4,7 @@
     "type_length": null,
     "repetition_type": null,
     "name": "root",
-    "num_children": 7,
+    "num_children": 8,
     "converted_type": null,
     "scale": null,
     "precision": null,
@@ -35,6 +35,18 @@
     "field_id": null,
     "logicalType": null
   },
+  {
+    "type": 5,
+    "type_length": null,
+    "repetition_type": 1,
+    "name": "number_field",
+    "num_children": null,
+    "converted_type": null,
+    "scale": null,
+    "precision": null,
+    "field_id": null,
+    "logicalType": null
+  },
   {
     "type": null,
     "type_length": null,

diff --git a/test/test-files/json-schema-test-file.schema.result.json b/test/test-files/json-schema-test-file.schema.result.json
@@ -12,6 +12,12 @@
       "encoding": "PLAIN",
       "compression": "UNCOMPRESSED"
     },
+    "number_field": {
+      "optional": true,
+      "type": "DOUBLE",
+      "encoding": "PLAIN",
+      "compression": "UNCOMPRESSED"
+    },
     "array_field": {
       "type": "LIST",
       "optional": true,
@@ -157,6 +163,18 @@
       "rLevelMax": 0,
       "dLevelMax": 1
     },
+    "number_field": {
+      "name": "number_field",
+      "primitiveType": "DOUBLE",
+      "path": [
+        "number_field"
+      ],
+      "repetitionType": "OPTIONAL",
+      "encoding": "PLAIN",
+      "compression": "UNCOMPRESSED",
+      "rLevelMax": 0,
+      "dLevelMax": 1
+    },
     "array_field": {
       "name": "array_field",
       "path": [
@@ -501,6 +519,18 @@
       "rLevelMax": 0,
       "dLevelMax": 1
     },
+    {
+      "name": "number_field",
+      "primitiveType": "DOUBLE",
+      "path": [
+        "number_field"
+      ],
+      "repetitionType": "OPTIONAL",
+      "encoding": "PLAIN",
+      "compression": "UNCOMPRESSED",
+      "rLevelMax": 0,
+      "dLevelMax": 1
+    },
     {
       "name": "array_field",
       "path": [