feat: add diff testing (#79)

* feat: add diff testing with python VM on fibonacci

* chore: remove TODO comment

* chore: bump cairo-lang deps in poetry

* chore: bump poetry.lock

* chore: bump bun-types version

* feat: update tests.yml for caching and --no-root

* chore: use modern installer in poetry config tests.yml Github Action

* chore: downgrade GH Action tests.yml to Python 3.9

* chore: downgrade poetry to python 3.9

* chore: trunk fmt

* test: add program parsing to diff testing test for clarity

* test: export diff tests to a dedicated suite

* chore: upgrade to python3.10

* fix: use quote to specify 3.10

* chore: make poetry cache dependend of python version

* chore: remove run config and comments

* fix: export buffer for buffer and memory

* fix: relocate memory in diff testing

* dev: cast ArrayBuffer to Buffer for type correctness

* fix: add offset parameter to export any memory start address

* chore: enforce camelCase

.github/workflows/tests.yml

@@ -15,38 +15,46 @@ jobs:
     name: test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        id: setup-python
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: '3.10'
+
       - name: Load cached Poetry installation
         id: cached-poetry
         uses: actions/cache@v3
         with:
           path: ~/.local
-          key: poetry-${{ runner.os }}
+          key: poetry-${{ steps.setup-python.outputs.python-version }}
+
       - name: Install Poetry
         if: steps.cached-poetry.outputs.cache-hit != 'true'
         uses: snok/install-poetry@v1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
-      - run: poetry config installer.modern-installation false
+
       - name: Load cached venv
         id: cached-poetry-dependencies
         uses: actions/cache@v3
         with:
           path: .venv
-          key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
+          key:
+            venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version
+            }}-${{ hashFiles('**/poetry.lock') }}
+
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-        run: poetry install
+        run: poetry install --no-interaction --no-root
+
       - name: Compile cairo files
         run: make compile
 
       - name: Install bun
         uses: oven-sh/setup-bun@v1
+
       - name: Run tests
         run: bun install && bun test

package.json

@@ -11,7 +11,7 @@
   "author": "Clément Walter <[email protected]>",
   "license": "Apache-2.0",
   "devDependencies": {
-    "bun-types": "^1.0.2",
+    "bun-types": "^1.1.12",
     "typescript": "^5.2.2"
   },
   "prettier": {

pyproject.toml

@@ -6,8 +6,8 @@ authors = ["Gregory Edison <[email protected]>"]
 readme = "README.md"
 
 [tool.poetry.dependencies]
-python = "^3.11"
-cairo-lang = "^0.12.2"
+python = "^3.10"
+cairo-lang = "^0.13.1"
 
 
 [build-system]

src/runners/cairoRunner.test.ts

@@ -1,3 +1,4 @@
+import { $ } from 'bun';
 import { describe, expect, test } from 'bun:test';
 
 import * as fs from 'fs';
@@ -120,12 +121,9 @@ describe('cairoRunner', () => {
     });
 
     /*
-     * TODO: Add differential testing of the content
-     * See this [issue](https://github.com/kkrt-labs/cairo-vm-ts/issues/59) for more details
-
      * NOTE: `fs.access` is only used when checking if a file exists
      * It should be removed if reading the file, to avoid race conditions
-    */
+     */
     test('should export encoded trace', () => {
       const runner = new CairoRunner(FIBONACCI_PROGRAM);
       const config: RunOptions = {
@@ -146,15 +144,15 @@ describe('cairoRunner', () => {
     test('should export encoded memory', () => {
       const runner = new CairoRunner(FIBONACCI_PROGRAM);
       const config: RunOptions = {
-        relocate: false,
+        relocate: true,
         relocateOffset: 1,
       };
       runner.run(config);
-      const memory_filename = 'fibonacci_memory_ts.bin';
-      const memory_path = path.join(tmpDir, memory_filename);
-      runner.exportMemory(memory_path);
+      const memoryFilename = 'fibonacci_memory_ts.bin';
+      const memoryPath = path.join(tmpDir, memoryFilename);
+      runner.exportMemory(memoryPath, config.relocateOffset);
       expect(() =>
-        fs.access(memory_path, (err) => {
+        fs.access(memoryPath, (err) => {
           if (err) throw err;
         })
       ).not.toThrow();
@@ -414,4 +412,47 @@ describe('cairoRunner', () => {
       });
     });
   });
+
+  describe('diff testing', () => {
+    test('should compare memory from TS & Python VMs execution of fibonacci', async () => {
+      const programPath = 'cairo_programs/cairo_0/fibonacci.json';
+      const pyMemoryPath = path.join(tmpDir, 'memory_python.bin');
+      await $`poetry run cairo-run --layout=starknet --program=${programPath} --memory_file ${pyMemoryPath}`;
+
+      const program = parseProgram(fs.readFileSync(programPath, 'utf8'));
+      const runner = new CairoRunner(program);
+      const config: RunOptions = {
+        relocate: true,
+        relocateOffset: 1,
+      };
+      runner.run(config);
+      const tsMemoryPath = path.join(tmpDir, 'memory_ts.bin');
+      runner.exportMemory(tsMemoryPath, config.relocateOffset);
+
+      const tsMemory = fs.readFileSync(tsMemoryPath);
+
+      const pyMemory = fs.readFileSync(pyMemoryPath);
+      expect(pyMemory.equals(tsMemory)).toBeTrue();
+    });
+
+    test('should compare trace from TS & Python VMs execution of fibonacci', async () => {
+      const programPath = 'cairo_programs/cairo_0/fibonacci.json';
+      const pyTracePath = path.join(tmpDir, 'trace_python.bin');
+      await $`poetry run cairo-run --layout=starknet --program=${programPath} --trace_file ${pyTracePath}`;
+
+      const program = parseProgram(fs.readFileSync(programPath, 'utf8'));
+      const runner = new CairoRunner(program);
+      const config: RunOptions = {
+        relocate: true,
+        relocateOffset: 1,
+      };
+      runner.run(config);
+      const tsTracePath = path.join(tmpDir, 'trace_ts.bin');
+      runner.exportTrace(tsTracePath);
+
+      const tsTrace = fs.readFileSync(tsTracePath);
+      const pyTrace = fs.readFileSync(pyTracePath);
+      expect(tsTrace.equals(pyTrace)).toBeTrue();
+    });
+  });
 });

src/runners/cairoRunner.ts

@@ -78,20 +78,24 @@ export class CairoRunner {
       view.setBigUint64(byteOffset + 2 * 8, pc.toBigInt(), true);
     });
 
-    fs.writeFileSync(filename, view, { flag: 'w+' });
+    fs.writeFileSync(filename, Buffer.from(buffer), { flag: 'w+' });
   }
 
   /**
    * Export the relocated memory little-endian encoded to a file
    *
+   * @param offset - Start address of the relocated memory.  Defaults to 0.
+   *
+   *
+   * NOTE: StarkWare verifier expects offset to be 1.
    * @dev DataView must be used to enforce little-endianness
    */
-  exportMemory(filename: string = 'encoded_memory') {
+  exportMemory(filename: string = 'encoded_memory', offset: number = 0) {
     const buffer = new ArrayBuffer(this.vm.relocatedMemory.length * 5 * 8);
     const view = new DataView(buffer);
 
     this.vm.relocatedMemory.forEach(({ address, value }) => {
-      const byteOffset = (address - 1) * 5 * 8;
+      const byteOffset = (address - offset) * 5 * 8;
       const valueAs64BitsWords = value.to64BitsWords();
       view.setBigUint64(byteOffset, BigInt(address), true);
       view.setBigUint64(byteOffset + 8, valueAs64BitsWords[0], true);
@@ -100,7 +104,7 @@ export class CairoRunner {
       view.setBigUint64(byteOffset + 4 * 8, valueAs64BitsWords[3], true);
     });
 
-    fs.writeFileSync(filename, view, { flag: 'w+' });
+    fs.writeFileSync(filename, Buffer.from(buffer), { flag: 'w+' });
   }
 
   getOutput() {