From b71542f20185d19d4857c1c4248be5fa0652c67c Mon Sep 17 00:00:00 2001
From: Conor Schaefer <conor@ruin.dev>
Date: Thu, 5 Sep 2024 12:28:09 -0700
Subject: [PATCH] ci: add nix env and gha workflows

First pass at a dev env, which pulls in the usual Rust deps, as well as
go, which is required to build `penumbra-indexer`. The go-linking
functionality isn't well supported yet: within the devShell, `cargo
build` works, but `nix build` does not.

CI cache timing info:

  * First run of cargo-check: 3m13s
  * First run of cargo-fmt: 1m57s
---
 .envrc.example             |  1 +
 .github/workflows/rust.yml | 80 ++++++++++++++++++++++++++++++
 .gitignore                 |  4 ++
 Cargo.toml                 |  4 ++
 build.rs                   |  2 +-
 flake.lock                 | 99 ++++++++++++++++++++++++++++++++++++++
 flake.nix                  | 96 ++++++++++++++++++++++++++++++++++++
 justfile                   | 13 +++++
 src/storage.rs             | 12 ++---
 9 files changed, 304 insertions(+), 7 deletions(-)
 create mode 100644 .envrc.example
 create mode 100644 .github/workflows/rust.yml
 create mode 100644 flake.lock
 create mode 100644 flake.nix
 create mode 100644 justfile

diff --git a/.envrc.example b/.envrc.example
new file mode 100644
index 0000000..3550a30
--- /dev/null
+++ b/.envrc.example
@@ -0,0 +1 @@
+use flake
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
new file mode 100644
index 0000000..1bf29d0
--- /dev/null
+++ b/.github/workflows/rust.yml
@@ -0,0 +1,80 @@
+name: rust
+on:
+  pull_request:
+
+jobs:
+  test:
+    name: cargo test
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: install nix
+        uses: nixbuild/nix-quick-install-action@v28
+
+      - name: setup nix cache
+        uses: nix-community/cache-nix-action@v5
+        with:
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-
+          backend: buildjet
+
+      - name: Load rust cache
+        uses: astriaorg/buildjet-rust-cache@v2.5.1
+
+      - name: run cargo test
+        run: >-
+          nix develop --command
+
+          just test
+  check:
+    name: cargo check
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: install nix
+        uses: nixbuild/nix-quick-install-action@v28
+
+      - name: setup nix cache
+        uses: nix-community/cache-nix-action@v5
+        with:
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-
+          backend: buildjet
+
+      - name: load rust cache
+        uses: astriaorg/buildjet-rust-cache@v2.5.1
+
+      - name: run cargo check, failing on warnings
+        run: >-
+          nix develop --command
+          just check
+
+  fmt:
+    name: cargo fmt
+    runs-on: buildjet-8vcpu-ubuntu-2204
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: install nix
+        uses: nixbuild/nix-quick-install-action@v28
+
+      - name: setup nix cache
+        uses: nix-community/cache-nix-action@v5
+        with:
+          primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }}
+          restore-prefixes-first-match: nix-${{ runner.os }}-
+          backend: buildjet
+
+      - name: load rust cache
+        uses: astriaorg/buildjet-rust-cache@v2.5.1
+
+      - name: run cargo fmt, failing on reformatting
+        run: >-
+          nix develop --command
+          just fmt
diff --git a/.gitignore b/.gitignore
index 2ef8b81..283711f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,7 @@
 
 # Rust / Cargo
 /target
+
+# Ignore .envrc; copy from .envrc.example to get started
+.envrc
+.direnv/
diff --git a/Cargo.toml b/Cargo.toml
index d22f6bf..2d6248d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,9 @@
 [package]
 name = "penumbra-reindexer"
+authors = ["Penumbra Labs <team@penumbralabs.xyz"]
+description = "A reindexing tool for Penumbra ABCI event data"
+homepage = "https://penumbra.zone"
+repository = "https://github.com/penumbra-zone/reindexer"
 version = "0.1.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"
diff --git a/build.rs b/build.rs
index 43d8671..416f98d 100644
--- a/build.rs
+++ b/build.rs
@@ -13,7 +13,7 @@ fn main() {
         .arg("cometbft.go")
         .current_dir(&go_dir)
         .status()
-        .unwrap();
+        .expect("failed to run go build command; make sure go is installed and on PATH");
     assert!(status.success());
 
     // Link the Go static library
diff --git a/flake.lock b/flake.lock
new file mode 100644
index 0000000..4f80759
--- /dev/null
+++ b/flake.lock
@@ -0,0 +1,99 @@
+{
+  "nodes": {
+    "crane": {
+      "locked": {
+        "lastModified": 1725409566,
+        "narHash": "sha256-PrtLmqhM6UtJP7v7IGyzjBFhbG4eOAHT6LPYOFmYfbk=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "7e4586bad4e3f8f97a9271def747cf58c4b68f3c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": []
+      },
+      "locked": {
+        "lastModified": 1725517947,
+        "narHash": "sha256-sB8B3M6CS0Y0rnncsCPz0htg6LoC1RbI2Mq9K88tSOk=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "96072c2af73da16c7db013dbb8c8869000157235",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1710146030,
+        "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1725432240,
+        "narHash": "sha256-+yj+xgsfZaErbfYM3T+QvEE2hU7UuE+Jf0fJCJ8uPS0=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "ad416d066ca1222956472ab7d0555a6946746a80",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "crane": "crane",
+        "fenix": "fenix",
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
diff --git a/flake.nix b/flake.nix
new file mode 100644
index 0000000..c542c58
--- /dev/null
+++ b/flake.nix
@@ -0,0 +1,96 @@
+{
+  description = "A nix development shell and build environment for penumbra-reindexer";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    flake-utils.url = "github:numtide/flake-utils";
+    fenix = {
+      url = "github:nix-community/fenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+      inputs.rust-analyzer-src.follows = "";
+    };
+    crane = {
+      url = "github:ipetkov/crane";
+      inputs = { nixpkgs.follows = "nixpkgs"; };
+    };
+  };
+
+  outputs = { self, nixpkgs, flake-utils, crane, ... }:
+    flake-utils.lib.eachDefaultSystem
+      (system:
+        let
+          pkgs = import nixpkgs { inherit system; };
+          # Permit version declarations, but default to unset,
+          # meaning the local working copy will be used.
+          penumbraReindexerRelease = null;
+
+          # Set up for Rust builds.
+          craneLib = crane.mkLib pkgs;
+
+          # Important environment variables so that the build can find the necessary libraries
+          LIBCLANG_PATH="${pkgs.libclang.lib}/lib";
+          ROCKSDB_LIB_DIR="${pkgs.rocksdb.out}/lib";
+        in with pkgs; with pkgs.lib; let
+          # All the Penumbra binaries
+          penumbraReindexer = (craneLib.buildPackage {
+            pname = "penumbra-reindexer";
+            # what
+            src = cleanSourceWith {
+              src = if penumbraReindexerRelease == null then craneLib.path ./. else fetchFromGitHub {
+                owner = "penumbra-zone";
+                repo = "reindexer";
+                rev = "v${penumbraReindexerRelease.version}";
+                sha256 = "${penumbraReindexerRelease.sha256}";
+              };
+              filter = path: type:
+                # Retain non-rust files as build inputs:
+                # * sql: database schema files for indexing
+                # * go, mod, sum: golang files for linking in cometbft
+                (builtins.match ".*\.(sql|go|mod|sum)$" path != null) ||
+                # ... as well as all the normal cargo source files:
+                (craneLib.filterCargoSources path type);
+            };
+            nativeBuildInputs = [ pkg-config ];
+            buildInputs = if stdenv.hostPlatform.isDarwin then 
+              with pkgs.darwin.apple_sdk.frameworks; [clang openssl rocksdb SystemConfiguration CoreServices go]
+            else
+              [clang openssl rocksdb go];
+
+            inherit system LIBCLANG_PATH ROCKSDB_LIB_DIR;
+            cargoExtraArgs = "-p penumbra-reindexer";
+            meta = {
+              description = "A reindexing tool for Penumbra ABCI event data";
+              homepage = "https://penumbra.zone";
+              license = [ licenses.mit licenses.asl20 ];
+            };
+          }).overrideAttrs (_: { doCheck = false; }); # Disable tests to improve build times
+
+        in rec {
+          packages = { inherit penumbraReindexer ; };
+          apps = {
+            penumbra-reindexer.type = "app";
+            penumbra-reindexer.program = "${penumbraReindexer}/bin/penumbra-reindexer";
+          };
+          defaultPackage = symlinkJoin {
+            name = "penumbra-reindexer";
+            paths = [ penumbraReindexer ];
+          };
+          devShells.default = craneLib.devShell {
+            inherit LIBCLANG_PATH ROCKSDB_LIB_DIR;
+            inputsFrom = [ penumbraReindexer ];
+            packages = [
+              cargo-nextest
+              cargo-watch
+              go
+              just
+              nix-prefetch-scripts
+              sqlfluff
+            ];
+            shellHook = ''
+              export LIBCLANG_PATH=${LIBCLANG_PATH}
+              export ROCKSDB_LIB_DIR=${ROCKSDB_LIB_DIR}
+            '';
+          };
+        }
+      );
+}
diff --git a/justfile b/justfile
new file mode 100644
index 0000000..1f50ccb
--- /dev/null
+++ b/justfile
@@ -0,0 +1,13 @@
+# Run cargo check, failing on warnings
+check:
+  # The `-D warnings` option causes an error on warnings.
+  RUSTFLAGS="-D warnings" \
+    cargo check --release --all-targets
+
+# Run cargo fmt, failing on warnings
+fmt:
+  cargo fmt --all -- --check
+
+# Run cargo nextest
+test:
+  cargo nextest run
diff --git a/src/storage.rs b/src/storage.rs
index a852988..caec1a3 100644
--- a/src/storage.rs
+++ b/src/storage.rs
@@ -301,7 +301,7 @@ mod test {
 
     const CHAIN_ID: &'static str = "penumbra-test";
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_storage_can_get_version() -> anyhow::Result<()> {
         assert_eq!(
             Storage::new(None, Some(CHAIN_ID))
@@ -314,7 +314,7 @@ mod test {
         Ok(())
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_storage_can_get_chain_id() -> anyhow::Result<()> {
         assert_eq!(
             Storage::new(None, Some(CHAIN_ID))
@@ -327,7 +327,7 @@ mod test {
         Ok(())
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_put_then_get_block() -> anyhow::Result<()> {
         let in_block = Block::test_value();
         let height = in_block.height();
@@ -340,14 +340,14 @@ mod test {
         Ok(())
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_bad_height_returns_no_block() -> anyhow::Result<()> {
         let storage = Storage::new(None, Some(CHAIN_ID)).await?;
         assert!(storage.get_block(100).await?.is_none());
         Ok(())
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_put_twice() -> anyhow::Result<()> {
         let storage = Storage::new(None, Some(CHAIN_ID)).await?;
         let block = Block::test_value();
@@ -356,7 +356,7 @@ mod test {
         Ok(())
     }
 
-    #[tokio::test]
+    #[tokio::test(flavor = "multi_thread")]
     async fn test_put_then_get_genesis() -> anyhow::Result<()> {
         let storage = Storage::new(None, Some(CHAIN_ID)).await?;
         let genesis = Genesis::test_value();