Merge pull request #24 from mhchia/feat/data-commitment-maps-to-file

Generate data commitment in json file
ZKStats · Apr 10, 2024 · e6b49ae · e6b49ae
2 parents c6e2936 + bb15b8c
commit e6b49ae
Show file tree

Hide file tree

Showing 32 changed files with 237 additions and 181 deletions.
diff --git a/README.md b/README.md
@@ -95,13 +95,14 @@ The flow between data providers and users is as follows:
 Data providers should generate commitments for their dataset beforehand. For a dataset (e.g. a table in a SQL database), there should be a commitment for each column. These commitments are used by users later, to verify the zkp proof and be convinced the computation is done with the correct dataset.
 
 ```python
-from zkstats.core import get_data_commitment_maps
+from zkstats.core import generate_data_commitment
 
 data_path = "/path/to/your/data.json"
+data_commitment_path = "/path/to/store/data_commitments.json"
 # possible_scales is a list of possible scales for the data to be encoded. For example, here we use [0, 20) as the possible scales, to make sure
 possible_scales = list(range(20))
-# `commitment_maps` is derived by data providers and shared with users
-commitment_maps = get_data_commitment_maps(data_path, possible_scales)
+# data commitment is generated by data providers and shared with users
+generate_data_commitment(data_path, possible_scales, data_commitment_path)
 ```
 
 When generating a proof, since dataset might contain floating points, data providers need to specify a proper "scale" to encode and decode floating points. Scale is chosen based on the value precision in the dataset and the type of computation. `possible_scales` should cover as many scales as possible and data providers should always use the scales within `possible_scales`, to make sure users can always get the corresponding commitments to verify the proofs.
@@ -187,7 +188,7 @@ res = verifier_verify(
     settings_path,  # path to the settings file
     vk_path,  # path to the verification key
     selected_columns,  # selected columns
-    commitment_maps,  # commitment maps
+    data_commitment_path,  # path to the data commitment
 )
 print("The result is", res)
 ```

diff --git a/examples/computation/computation.ipynb b/examples/computation/computation.ipynb
@@ -25,7 +25,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zkstats.core import prover_gen_settings, setup, prover_gen_proof, verifier_verify, get_data_commitment_maps"
+    "from zkstats.core import prover_gen_settings, setup, prover_gen_proof, verifier_verify, generate_data_commitment"
    ]
   },
   {
@@ -94,7 +94,7 @@
    "source": [
     "scales = [2]\n",
     "selected_columns = ['x', 'y']\n",
-    "commitment_maps = get_data_commitment_maps(data_path, scales)"
+    "generate_data_commitment(data_path, scales, data_commitment_path)"
    ]
   },
   {
@@ -221,7 +221,7 @@
    ],
    "source": [
     "# Verifier verifies\n",
-    "verifier_verify(proof_path, settings_path, vk_path, selected_columns, commitment_maps)"
+    "verifier_verify(proof_path, settings_path, vk_path, selected_columns, data_commitment_path)"
    ]
   },
   {
@@ -248,7 +248,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
+   "version": "3.12.1"
   },
   "orig_nbformat": 4
  },

diff --git a/examples/correlation/correlation.ipynb b/examples/correlation/correlation.ipynb
diff --git a/examples/covariance/covariance.ipynb b/examples/covariance/covariance.ipynb
@@ -25,7 +25,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, get_data_commitment_maps"
+    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, generate_data_commitment"
    ]
   },
   {
@@ -50,7 +50,8 @@
     "# this is private to prover since it contains actual data\n",
     "sel_data_path = os.path.join('prover/sel_data.json')\n",
     "# this is just dummy random value\n",
-    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')"
+    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')\n",
+    "data_commitment_path = os.path.join('shared/data_commitment.json')"
    ]
   },
   {
@@ -82,7 +83,7 @@
     "# note scale = 2, or 3 makes it more precise, but too big.\n",
     "scales = [1]\n",
     "selected_columns = ['x', 'y']\n",
-    "commitment_maps = get_data_commitment_maps(data_path, scales)"
+    "generate_data_commitment(data_path, scales, data_commitment_path)"
    ]
   },
   {
@@ -214,7 +215,7 @@
    ],
    "source": [
     "# Verifier verifies\n",
-    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, commitment_maps)\n",
+    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, data_commitment_path)\n",
     "print(\"Verifier gets result:\", res)"
    ]
   },
@@ -242,7 +243,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
+   "version": "3.12.1"
   },
   "orig_nbformat": 4
  },

diff --git a/examples/geomean/geomean.ipynb b/examples/geomean/geomean.ipynb
@@ -25,7 +25,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, get_data_commitment_maps"
+    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, generate_data_commitment"
    ]
   },
   {
@@ -56,7 +56,8 @@
     "# this is private to prover since it contains actual data\n",
     "sel_data_path = os.path.join('prover/sel_data.json')\n",
     "# this is just dummy random value\n",
-    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')"
+    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')\n",
+    "data_commitment_path = os.path.join('shared/data_commitment.json')"
    ]
   },
   {
@@ -87,7 +88,7 @@
    "source": [
     "scales = [8]\n",
     "selected_columns = ['col_name']\n",
-    "commitment_maps = get_data_commitment_maps(data_path, scales)"
+    "generate_data_commitment(data_path, scales, data_commitment_path)"
    ]
   },
   {
@@ -210,7 +211,7 @@
    ],
    "source": [
     "# Verifier verifies\n",
-    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, commitment_maps)\n",
+    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, data_commitment_path)\n",
     "print(\"Verifier gets result:\", res)"
    ]
   },

diff --git a/examples/harmomean/harmomean.ipynb b/examples/harmomean/harmomean.ipynb
@@ -25,7 +25,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, get_data_commitment_maps"
+    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, generate_data_commitment"
    ]
   },
   {
@@ -50,7 +50,8 @@
     "# this is private to prover since it contains actual data\n",
     "sel_data_path = os.path.join('prover/sel_data.json')\n",
     "# this is just dummy random value\n",
-    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')"
+    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')\n",
+    "data_commitment_path = os.path.join('shared/data_commitment.json')"
    ]
   },
   {
@@ -81,7 +82,7 @@
    "source": [
     "scales = [6]\n",
     "selected_columns = ['col_name']\n",
-    "commitment_maps = get_data_commitment_maps(data_path, scales)"
+    "generate_data_commitment(data_path, scales, data_commitment_path)"
    ]
   },
   {
@@ -217,7 +218,7 @@
    ],
    "source": [
     "# Verifier verifies\n",
-    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, commitment_maps)\n",
+    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, data_commitment_path)\n",
     "print(\"Verifier gets result:\", res)"
    ]
   },

diff --git a/examples/mean+median/mean+median.ipynb b/examples/mean+median/mean+median.ipynb
@@ -25,7 +25,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, get_data_commitment_maps"
+    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, generate_data_commitment"
    ]
   },
   {
@@ -56,7 +56,8 @@
     "# this is private to prover since it contains actual data\n",
     "sel_data_path = os.path.join('prover/sel_data.json')\n",
     "# this is just dummy random value\n",
-    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')"
+    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')\n",
+    "data_commitment_path = os.path.join('shared/data_commitment.json')"
    ]
   },
   {
@@ -95,7 +96,7 @@
    "source": [
     "scales = [5]\n",
     "selected_columns = ['col_1', 'col_2']\n",
-    "commitment_maps = get_data_commitment_maps(data_path, scales)"
+    "generate_data_commitment(data_path, scales, data_commitment_path)"
    ]
   },
   {
@@ -236,7 +237,7 @@
    ],
    "source": [
     "# Verifier verifies\n",
-    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, commitment_maps)\n",
+    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, data_commitment_path)\n",
     "print(\"Verifier gets result:\", res)"
    ]
   },

diff --git a/examples/mean/mean.ipynb b/examples/mean/mean.ipynb
@@ -25,7 +25,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, get_data_commitment_maps"
+    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, generate_data_commitment"
    ]
   },
   {
@@ -56,7 +56,8 @@
     "# this is private to prover since it contains actual data\n",
     "sel_data_path = os.path.join('prover/sel_data.json')\n",
     "# this is just dummy random value\n",
-    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')"
+    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')\n",
+    "data_commitment_path = os.path.join('shared/data_commitment.json')"
    ]
   },
   {
@@ -99,7 +100,7 @@
    "source": [
     "scales = [3]\n",
     "selected_columns = ['col_name']\n",
-    "commitment_maps = get_data_commitment_maps(data_path, scales)"
+    "generate_data_commitment(data_path, scales, data_commitment_path)"
    ]
   },
   {
@@ -245,7 +246,7 @@
    ],
    "source": [
     "# Verifier verifies\n",
-    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, commitment_maps)\n",
+    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, data_commitment_path)\n",
     "print(\"Verifier gets result:\", res)"
    ]
   }

diff --git a/examples/median/median.ipynb b/examples/median/median.ipynb
@@ -25,7 +25,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, get_data_commitment_maps"
+    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, generate_data_commitment"
    ]
   },
   {
@@ -56,7 +56,8 @@
     "# this is private to prover since it contains actual data\n",
     "sel_data_path = os.path.join('prover/sel_data.json')\n",
     "# this is just dummy random value\n",
-    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')"
+    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')\n",
+    "data_commitment_path = os.path.join('shared/data_commitment.json')"
    ]
   },
   {
@@ -85,11 +86,11 @@
    "outputs": [],
    "source": [
     "# use large scale here to get the output really close to the value in dataset, which is important to Median.\n",
-    "# Be careful, since we may need to calculate average of 2 values in case of no middle value in dataset, \n",
+    "# Be careful, since we may need to calculate average of 2 values in case of no middle value in dataset,\n",
     "# this means larger scale can still blow up circuit size, unlike Mode func that scale doesnt affect circuit size much.\n",
     "scales = [7]\n",
     "selected_columns = ['col_name']\n",
-    "commitment_maps = get_data_commitment_maps(data_path, scales)"
+    "generate_data_commitment(data_path, scales, data_commitment_path)"
    ]
   },
   {
@@ -204,7 +205,7 @@
    ],
    "source": [
     "# Verifier verifies\n",
-    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, commitment_maps)\n",
+    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, data_commitment_path)\n",
     "print(\"Verifier gets result:\", res)"
    ]
   },
@@ -232,7 +233,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
+   "version": "3.12.1"
   },
   "orig_nbformat": 4
  },

diff --git a/examples/mode/mode.ipynb b/examples/mode/mode.ipynb
@@ -25,7 +25,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, get_data_commitment_maps"
+    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, generate_data_commitment"
    ]
   },
   {
@@ -56,7 +56,8 @@
     "# this is private to prover since it contains actual data\n",
     "sel_data_path = os.path.join('prover/sel_data.json')\n",
     "# this is just dummy random value\n",
-    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')"
+    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')\n",
+    "data_commitment_path = os.path.join('shared/data_commitment.json')"
    ]
   },
   {
@@ -88,7 +89,7 @@
     "# large scale doesn't blowup circuit size in Mode, so fine.\n",
     "scales = [8]\n",
     "selected_columns = ['col_name']\n",
-    "commitment_maps = get_data_commitment_maps(data_path, scales)"
+    "generate_data_commitment(data_path, scales, data_commitment_path)"
    ]
   },
   {
@@ -213,7 +214,7 @@
    ],
    "source": [
     "# Verifier verifies\n",
-    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, commitment_maps)\n",
+    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, data_commitment_path)\n",
     "print(\"Verifier gets result:\", res)"
    ]
   },

diff --git a/examples/pstdev/pstdev.ipynb b/examples/pstdev/pstdev.ipynb
@@ -25,7 +25,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, get_data_commitment_maps"
+    "from zkstats.core import create_dummy, verifier_define_calculation, prover_gen_settings, setup, prover_gen_proof, verifier_verify, generate_data_commitment"
    ]
   },
   {
@@ -50,7 +50,8 @@
     "# this is private to prover since it contains actual data\n",
     "sel_data_path = os.path.join('prover/sel_data.json')\n",
     "# this is just dummy random value\n",
-    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')"
+    "sel_dummy_data_path = os.path.join('shared/sel_dummy_data.json')\n",
+    "data_commitment_path = os.path.join('shared/data_commitment.json')"
    ]
   },
   {
@@ -80,7 +81,7 @@
    "source": [
     "scales = [3]\n",
     "selected_columns = ['col_name']\n",
-    "commitment_maps = get_data_commitment_maps(data_path, scales)"
+    "generate_data_commitment(data_path, scales, data_commitment_path)"
    ]
   },
   {
@@ -201,7 +202,7 @@
    ],
    "source": [
     "# Verifier verifies\n",
-    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, commitment_maps)\n",
+    "res = verifier_verify(proof_path, settings_path, vk_path, selected_columns, data_commitment_path)\n",
     "print(\"Verifier gets result:\", res)"
    ]
   },