ci: add nix env and gha workflows

First pass at a dev env, which pulls in the usual Rust deps, as well as
go, which is required to build `penumbra-indexer`. The devshell pulls in
the necessary deps, resulting in `cargo build` running the
proxy build for go via `build.rs` just fine.

Not yet supported is `nix build`: will need to figure out a two-pass
approach, because network access during the buildPhase is not permitted,
so the go build via rust fails.

CI cache timing info:

  * First run of cargo-check: 3m13s
  * First run of cargo-fmt: 1m57s

.envrc.example

@@ -0,0 +1 @@
+use flake

.github/workflows/rust.yml

@@ -0,0 +1,80 @@
+name: rust
+on:
+  pull_request:
+
+jobs:
+  test:
+    name: cargo test
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: install nix
+        uses: nixbuild/nix-quick-install-action@v28
+
+      - name: setup nix cache
+        uses: nix-community/cache-nix-action@v5
+        with:
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-
+          backend: buildjet
+
+      - name: Load rust cache
+        uses: astriaorg/[email protected]
+
+      - name: run cargo test
+        run: >-
+          nix develop --command
+          just test
+
+  check:
+    name: cargo check
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: install nix
+        uses: nixbuild/nix-quick-install-action@v28
+
+      - name: setup nix cache
+        uses: nix-community/cache-nix-action@v5
+        with:
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-
+          backend: buildjet
+
+      - name: load rust cache
+        uses: astriaorg/[email protected]
+
+      - name: run cargo check, failing on warnings
+        run: >-
+          nix develop --command
+          just check
+
+  fmt:
+    name: cargo fmt
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: install nix
+        uses: nixbuild/nix-quick-install-action@v28
+
+      - name: setup nix cache
+        uses: nix-community/cache-nix-action@v5
+        with:
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-
+          backend: buildjet
+
+      - name: load rust cache
+        uses: astriaorg/[email protected]
+
+      - name: run cargo fmt, failing on reformatting
+        run: >-
+          nix develop --command
+          just fmt

.gitignore

@@ -5,3 +5,7 @@
 
 # Rust / Cargo
 /target
+
+# Ignore .envrc; copy from .envrc.example to get started
+.envrc
+.direnv/

Cargo.toml

@@ -1,5 +1,9 @@
 [package]
 name = "penumbra-reindexer"
+authors = ["Penumbra Labs <[email protected]"]
+description = "A reindexing tool for Penumbra ABCI event data"
+homepage = "https://penumbra.zone"
+repository = "https://github.com/penumbra-zone/reindexer"
 version = "0.1.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"

build.rs

@@ -13,7 +13,7 @@ fn main() {
         .arg("cometbft.go")
         .current_dir(&go_dir)
         .status()
-        .unwrap();
+        .expect("failed to run go build command; make sure go is installed and on PATH");
     assert!(status.success());
 
     // Link the Go static library

flake.nix

@@ -0,0 +1,96 @@
+{
+  description = "A nix development shell and build environment for penumbra-reindexer";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    flake-utils.url = "github:numtide/flake-utils";
+    fenix = {
+      url = "github:nix-community/fenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.rust-analyzer-src.follows = "";
+    };
+    crane = {
+      url = "github:ipetkov/crane";
+      inputs = { nixpkgs.follows = "nixpkgs"; };
+    };
+  };
+
+  outputs = { self, nixpkgs, flake-utils, crane, ... }:
+    flake-utils.lib.eachDefaultSystem
+      (system:
+        let
+          pkgs = import nixpkgs { inherit system; };
+          # Permit version declarations, but default to unset,
+          # meaning the local working copy will be used.
+          penumbraReindexerRelease = null;
+
+          # Set up for Rust builds.
+          craneLib = crane.mkLib pkgs;
+
+          # Important environment variables so that the build can find the necessary libraries
+          LIBCLANG_PATH="${pkgs.libclang.lib}/lib";
+          ROCKSDB_LIB_DIR="${pkgs.rocksdb.out}/lib";
+        in with pkgs; with pkgs.lib; let
+          # All the Penumbra binaries
+          penumbraReindexer = (craneLib.buildPackage {
+            pname = "penumbra-reindexer";
+            # what
+            src = cleanSourceWith {
+              src = if penumbraReindexerRelease == null then craneLib.path ./. else fetchFromGitHub {
+                owner = "penumbra-zone";
+                repo = "reindexer";
+                rev = "v${penumbraReindexerRelease.version}";
+                sha256 = "${penumbraReindexerRelease.sha256}";
+              };
+              filter = path: type:
+                # Retain non-rust files as build inputs:
+                # * sql: database schema files for indexing
+                # * go, mod, sum: golang files for linking in cometbft
+                (builtins.match ".*\.(sql|go|mod|sum)$" path != null) ||
+                # ... as well as all the normal cargo source files:
+                (craneLib.filterCargoSources path type);
+            };
+            nativeBuildInputs = [ pkg-config ];
+            buildInputs = if stdenv.hostPlatform.isDarwin then 
+              with pkgs.darwin.apple_sdk.frameworks; [clang openssl rocksdb SystemConfiguration CoreServices go]
+            else
+              [clang openssl rocksdb go];
+
+            inherit system LIBCLANG_PATH ROCKSDB_LIB_DIR;
+            cargoExtraArgs = "-p penumbra-reindexer";
+            meta = {
+              description = "A reindexing tool for Penumbra ABCI event data";
+              homepage = "https://penumbra.zone";
+              license = [ licenses.mit licenses.asl20 ];
+            };
+          }).overrideAttrs (_: { doCheck = false; }); # Disable tests to improve build times
+
+        in rec {
+          packages = { inherit penumbraReindexer ; };
+          apps = {
+            penumbra-reindexer.type = "app";
+            penumbra-reindexer.program = "${penumbraReindexer}/bin/penumbra-reindexer";
+          };
+          defaultPackage = symlinkJoin {
+            name = "penumbra-reindexer";
+            paths = [ penumbraReindexer ];
+          };
+          devShells.default = craneLib.devShell {
+            inherit LIBCLANG_PATH ROCKSDB_LIB_DIR;
+            inputsFrom = [ penumbraReindexer ];
+            packages = [
+              cargo-nextest
+              cargo-watch
+              go
+              just
+              nix-prefetch-scripts
+              sqlfluff
+            ];
+            shellHook = ''
+              export LIBCLANG_PATH=${LIBCLANG_PATH}
+              export ROCKSDB_LIB_DIR=${ROCKSDB_LIB_DIR}
+            '';
+          };
+        }
+      );
+}

justfile

@@ -0,0 +1,13 @@
+# Run cargo check, failing on warnings
+check:
+  # The `-D warnings` option causes an error on warnings.
+  RUSTFLAGS="-D warnings" \
+    cargo check --release --all-targets
+
+# Run cargo fmt, failing on warnings
+fmt:
+  cargo fmt --all -- --check
+
+# Run cargo nextest
+test:
+  cargo nextest run

src/storage.rs

@@ -301,7 +301,7 @@ mod test {
 
     const CHAIN_ID: &'static str = "penumbra-test";
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_storage_can_get_version() -> anyhow::Result<()> {
         assert_eq!(
             Storage::new(None, Some(CHAIN_ID))
@@ -314,7 +314,7 @@ mod test {
         Ok(())
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_storage_can_get_chain_id() -> anyhow::Result<()> {
         assert_eq!(
             Storage::new(None, Some(CHAIN_ID))
@@ -327,7 +327,7 @@ mod test {
         Ok(())
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_put_then_get_block() -> anyhow::Result<()> {
         let in_block = Block::test_value();
         let height = in_block.height();
@@ -340,14 +340,14 @@ mod test {
         Ok(())
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_bad_height_returns_no_block() -> anyhow::Result<()> {
         let storage = Storage::new(None, Some(CHAIN_ID)).await?;
         assert!(storage.get_block(100).await?.is_none());
         Ok(())
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_put_twice() -> anyhow::Result<()> {
         let storage = Storage::new(None, Some(CHAIN_ID)).await?;
         let block = Block::test_value();
@@ -356,7 +356,7 @@ mod test {
         Ok(())
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_put_then_get_genesis() -> anyhow::Result<()> {
         let storage = Storage::new(None, Some(CHAIN_ID)).await?;
         let genesis = Genesis::test_value();