[forge] rust bindings for indexer/testnet deployer (#14547)

* [forge] rust bindings for indexer/testnet deployer

* [forge] clean up forge deployer interfaces

* [forge] deploy testnet and indexer in parallel

* [forge] disable genesis blob url for indexer enabled runs

* [forge] install testnet with deployer; clean up dead code

* [forge] do not store config state in deployer manager

* [forge] fixes and log deployer status

* [forge] protect against committing forge.env

* [forge] fix node collection with deployer

* [forge] add debug lines to forge.py and export sourced vars

* [forge] indexer config tuning

* [forge] clean up GHA for forge; add indexer option

* [forge] fix lint

* update gitignore for forge.env

* [forge] add forge.env one time

* Remove ignored files

* [forge] handle statefulset label truncation

.github/workflows/adhoc-forge.yaml

@@ -31,11 +31,10 @@ on:
         required: false
         type: string
         description: The Forge k8s cluster to be used for test
-      FORGE_ENABLE_HAPROXY:
+      FORGE_ENABLE_INDEXER:
         required: false
-        default: false
         type: boolean
-        description: enable haproxy for the forge test
+        description: Whether to use indexer
       FORGE_NUM_VALIDATORS:
         required: false
         type: string
@@ -68,6 +67,7 @@ jobs:
           echo "FORGE_TEST_SUITE: ${{ inputs.FORGE_TEST_SUITE }}"
           echo "FORGE_CLUSTER_NAME: ${{ inputs.FORGE_CLUSTER_NAME }}"
           echo "FORGE_ENABLE_HAPROXY: ${{ inputs.FORGE_ENABLE_HAPROXY }}"
+          echo "FORGE_ENABLE_INDEXER: ${{ inputs.FORGE_ENABLE_INDEXER }}"
           echo "FORGE_NUM_VALIDATORS: ${{ inputs.FORGE_NUM_VALIDATORS }}"
           echo "FORGE_NUM_VALIDATOR_FULLNODES: ${{ inputs.FORGE_NUM_VALIDATOR_FULLNODES }}"
           echo "FORGE_RETAIN_DEBUG_LOGS: ${{ inputs.FORGE_RETAIN_DEBUG_LOGS }}"
@@ -79,6 +79,7 @@ jobs:
       forgeTestSuite: ${{ inputs.FORGE_TEST_SUITE }}
       forgeClusterName: ${{ inputs.FORGE_CLUSTER_NAME }}
       forgeEnableHaproxy: ${{ inputs.FORGE_ENABLE_HAPROXY }}
+      forgeEnableIndexer: ${{ inputs.FORGE_ENABLE_INDEXER }}
       forgeNumValidators: ${{ inputs.FORGE_NUM_VALIDATORS }}
       forgeNumValidatorFullnodes: ${{ inputs.FORGE_NUM_VALIDATOR_FULLNODES }}
 
@@ -94,6 +95,7 @@ jobs:
       FORGE_RUNNER_DURATION_SECS: ${{ fromJSON(needs.determine-forge-run-metadata.outputs.forgeRunnerDurationSecs) }} # fromJSON converts to integer
       FORGE_CLUSTER_NAME: ${{ needs.determine-forge-run-metadata.outputs.forgeClusterName }}
       FORGE_ENABLE_HAPROXY: ${{ needs.determine-forge-run-metadata.outputs.forgeEnableHaproxy }}
+      FORGE_ENABLE_INDEXER: ${{ needs.determine-forge-run-metadata.outputs.forgeEnableIndexer }}
       FORGE_NUM_VALIDATORS: ${{ needs.determine-forge-run-metadata.outputs.forgeNumValidators }}
       FORGE_NUM_VALIDATOR_FULLNODES: ${{ needs.determine-forge-run-metadata.outputs.forgeNumValidatorFullnodes }}
       FORGE_RETAIN_DEBUG_LOGS: ${{ inputs.FORGE_RETAIN_DEBUG_LOGS }}

.github/workflows/workflow-run-forge.yaml

@@ -39,11 +39,6 @@ on:
         type: boolean
         default: false
         description: Whether to post the test results comment to Slack
-      COMMENT_ON_PR:
-        required: false
-        type: boolean
-        default: true
-        description: Whether to post the test results comment to the PR
       TIMEOUT_MINUTES:
         required: false
         type: number
@@ -57,6 +52,10 @@ on:
         required: false
         type: string
         description: Whether to use HAPRoxy
+      FORGE_ENABLE_INDEXER:
+        required: false
+        type: boolean
+        description: Whether to use indexer
       FORGE_ENABLE_PERFORMANCE:
         required: false
         type: string
@@ -104,14 +103,14 @@ env:
   FORGE_RUNNER_DURATION_SECS: ${{ inputs.FORGE_RUNNER_DURATION_SECS }}
   FORGE_NAMESPACE: ${{ inputs.FORGE_NAMESPACE }}
   FORGE_ENABLE_HAPROXY: ${{ inputs.FORGE_ENABLE_HAPROXY }}
+  FORGE_ENABLE_INDEXER: ${{ inputs.FORGE_ENABLE_INDEXER }}
   FORGE_TEST_SUITE: ${{ inputs.FORGE_TEST_SUITE }}
   POST_TO_SLACK: ${{ inputs.POST_TO_SLACK }}
   FORGE_ENABLE_FAILPOINTS: ${{ inputs.FORGE_ENABLE_FAILPOINTS }}
   FORGE_ENABLE_PERFORMANCE: ${{ inputs.FORGE_ENABLE_PERFORMANCE }}
   FORGE_RETAIN_DEBUG_LOGS: ${{ inputs.FORGE_RETAIN_DEBUG_LOGS }}
   COMMENT_HEADER: ${{ inputs.COMMENT_HEADER }}
   VERBOSE: true
-  COMMENT_ON_PR: ${{ inputs.COMMENT_ON_PR }}
   FORGE_NUM_VALIDATORS: ${{ inputs.FORGE_NUM_VALIDATORS }}
   FORGE_NUM_VALIDATOR_FULLNODES: ${{ inputs.FORGE_NUM_VALIDATOR_FULLNODES }}
 
@@ -184,7 +183,7 @@ jobs:
         run: testsuite/run_forge.sh
 
       - name: Post pre-Forge comment
-        if: ${{ !inputs.SKIP_JOB && env.COMMENT_ON_PR == 'true' && github.event.number != null }}
+        if: ${{ !inputs.SKIP_JOB && github.event.number != null }}
         uses: marocchino/sticky-pull-request-comment@39c5b5dc7717447d0cba270cd115037d32d28443 # pin@39c5b5dc7717447d0cba270cd115037d32d2844
         with:
           header: ${{ env.COMMENT_HEADER }}
@@ -199,7 +198,7 @@ jobs:
 
       - name: Post forge result comment
         # Post a Github comment if the run has not been cancelled and if we're running on a PR
-        if: ${{ !inputs.SKIP_JOB  && env.COMMENT_ON_PR == 'true' && github.event.number != null && !cancelled() }}
+        if: ${{ !inputs.SKIP_JOB  && github.event.number != null && !cancelled() }}
         uses: marocchino/sticky-pull-request-comment@39c5b5dc7717447d0cba270cd115037d32d28443 # pin@39c5b5dc7717447d0cba270cd115037d32d2844
         with:
           header: ${{ env.COMMENT_HEADER }}

.gitignore

@@ -80,7 +80,6 @@ target-out-docker
 docker/compose/indexer-grpc/data-service-grpc-server.crt
 docker/compose/indexer-grpc/data-service-grpc-server.key
 
-
 # Doc generation output
 *.md.old
 

testsuite/forge-cli/Cargo.toml

@@ -30,6 +30,7 @@ once_cell = { workspace = true }
 rand = { workspace = true }
 random_word = { workspace = true }
 reqwest = { workspace = true }
+serde_json = { workspace = true }
 serde_yaml = { workspace = true }
 tokio = { workspace = true }
 url = { workspace = true }

testsuite/forge-cli/src/main.rs

@@ -65,6 +65,7 @@ use clap::{Parser, Subcommand};
 use futures::stream::{FuturesUnordered, StreamExt};
 use once_cell::sync::Lazy;
 use rand::{rngs::ThreadRng, seq::SliceRandom, Rng};
+use serde_json::{json, Value};
 use std::{
     env,
     num::NonZeroUsize,
@@ -140,8 +141,8 @@ enum OperatorCommand {
     SetNodeImageTag(SetNodeImageTag),
     /// Clean up an existing cluster
     CleanUp(CleanUp),
-    /// Resize an existing cluster
-    Resize(Resize),
+    /// Create a new cluster for testing purposes
+    Create(Create),
 }
 
 #[derive(Parser, Debug)]
@@ -193,6 +194,11 @@ struct K8sSwarm {
         help = "Retain debug logs and above for all nodes instead of just the first 5 nodes"
     )]
     retain_debug_logs: bool,
+    #[clap(
+        long,
+        help = "If set, spins up an indexer stack alongside the testnet. Same as --enable-indexer"
+    )]
+    enable_indexer: bool,
 }
 
 #[derive(Parser, Debug)]
@@ -217,8 +223,8 @@ struct CleanUp {
 }
 
 #[derive(Parser, Debug)]
-struct Resize {
-    #[clap(long, help = "The kubernetes namespace to resize")]
+struct Create {
+    #[clap(long, help = "The kubernetes namespace to create in")]
     namespace: String,
     #[clap(long, default_value_t = 30)]
     num_validators: usize,
@@ -227,13 +233,13 @@ struct Resize {
     #[clap(
         long,
         help = "Override the image tag used for validators",
-        default_value = "devnet"
+        default_value = "main"
     )]
     validator_image_tag: String,
     #[clap(
         long,
         help = "Override the image tag used for testnet-specific components",
-        default_value = "devnet"
+        default_value = "main"
     )]
     testnet_image_tag: String,
     #[clap(
@@ -248,6 +254,14 @@ struct Resize {
     connect_directly: bool,
     #[clap(long, help = "If set, enables HAProxy for each of the validators")]
     enable_haproxy: bool,
+    #[clap(long, help = "If set, spins up an indexer stack alongside the testnet")]
+    enable_indexer: bool,
+    #[clap(
+        long,
+        help = "Override the image tag used for indexer",
+        requires = "enable_indexer"
+    )]
+    indexer_image_tag: Option<String>,
 }
 
 // common metrics thresholds:
@@ -393,6 +407,7 @@ fn main() -> Result<()> {
                             k8s.reuse,
                             k8s.keep,
                             k8s.enable_haproxy,
+                            k8s.enable_indexer,
                         )
                         .unwrap(),
                         &args.options,
@@ -421,19 +436,51 @@ fn main() -> Result<()> {
                 }
                 Ok(())
             },
-            OperatorCommand::Resize(resize) => {
+            OperatorCommand::Create(create) => {
+                let kube_client = runtime.block_on(create_k8s_client())?;
+                let era = generate_new_era();
+                let indexer_image_tag = create
+                    .indexer_image_tag
+                    .or(Some(create.validator_image_tag.clone()))
+                    .expect("Expected indexer or validator image tag to use");
+                let config: Value = serde_json::from_value(json!({
+                    "profile": DEFAULT_FORGE_DEPLOYER_PROFILE.to_string(),
+                    "era": era.clone(),
+                    "namespace": create.namespace.clone(),
+                    "indexer-grpc-values": {
+                        "indexerGrpcImage": format!("{}:{}", INDEXER_GRPC_DOCKER_IMAGE_REPO, &indexer_image_tag),
+                        "fullnodeConfig": {
+                            "image": format!("{}:{}", VALIDATOR_DOCKER_IMAGE_REPO, &indexer_image_tag),
+                        }
+                    },
+                }))?;
+
                 runtime.block_on(install_testnet_resources(
-                    resize.namespace,
-                    resize.num_validators,
-                    resize.num_fullnodes,
-                    resize.validator_image_tag,
-                    resize.testnet_image_tag,
-                    resize.move_modules_dir,
-                    !resize.connect_directly,
-                    resize.enable_haproxy,
+                    era.clone(),
+                    create.namespace.clone(),
+                    create.num_validators,
+                    create.num_fullnodes,
+                    create.validator_image_tag,
+                    create.testnet_image_tag,
+                    create.move_modules_dir,
+                    false, // since we skip_collecting_running_nodes, we don't connect directly to the nodes to validatet their health
+                    create.enable_haproxy,
+                    create.enable_indexer,
                     None,
                     None,
+                    true,
                 ))?;
+
+                if create.enable_indexer {
+                    let indexer_deployer = ForgeDeployerManager::new(
+                        kube_client.clone(),
+                        create.namespace.clone(),
+                        FORGE_INDEXER_DEPLOYER_DOCKER_IMAGE_REPO.to_string(),
+                        None,
+                    );
+                    runtime.block_on(indexer_deployer.start(config))?;
+                    runtime.block_on(indexer_deployer.wait_completed())?;
+                }
                 Ok(())
             },
         },

testsuite/forge.py

@@ -845,6 +845,8 @@ def run(self, context: ForgeContext) -> ForgeResult:
             MULTIREGION_KUBECONFIG_DIR=MULTIREGION_KUBECONFIG_DIR,
         )
 
+        log.info(f"rendered_forge_test_runner: {rendered}")
+
         with ForgeResult.with_context(context) as forge_result:
             specfile = context.filesystem.mkstemp()
             context.filesystem.write(specfile, rendered.encode())
@@ -1147,6 +1149,7 @@ def create_forge_command(
     forge_namespace_reuse: Optional[str],
     forge_namespace_keep: Optional[str],
     forge_enable_haproxy: Optional[str],
+    forge_enable_indexer: Optional[str],
     cargo_args: Optional[Sequence[str]],
     forge_cli_args: Optional[Sequence[str]],
     test_args: Optional[Sequence[str]],
@@ -1216,6 +1219,8 @@ def create_forge_command(
         forge_args.append("--keep")
     if forge_enable_haproxy == "true":
         forge_args.append("--enable-haproxy")
+    if forge_enable_indexer == "true":
+        forge_args.append("--enable-indexer")
 
     if test_args:
         forge_args.extend(test_args)
@@ -1328,6 +1333,7 @@ def seeded_random_choice(namespace: str, cluster_names: Sequence[str]) -> str:
 @envoption("FORGE_NAMESPACE_KEEP")
 @envoption("FORGE_NAMESPACE_REUSE")
 @envoption("FORGE_ENABLE_HAPROXY")
+@envoption("FORGE_ENABLE_INDEXER")
 @envoption("FORGE_ENABLE_FAILPOINTS")
 @envoption("FORGE_ENABLE_PERFORMANCE")
 @envoption("FORGE_TEST_SUITE")
@@ -1373,6 +1379,7 @@ def test(
     forge_enable_failpoints: Optional[str],
     forge_enable_performance: Optional[str],
     forge_enable_haproxy: Optional[str],
+    forge_enable_indexer: Optional[str],
     forge_test_suite: str,
     forge_runner_duration_secs: str,
     forge_image_tag: Optional[str],
@@ -1598,12 +1605,13 @@ def test(
         forge_namespace_reuse=forge_namespace_reuse,
         forge_namespace_keep=forge_namespace_keep,
         forge_enable_haproxy=forge_enable_haproxy,
+        forge_enable_indexer=forge_enable_indexer,
         cargo_args=cargo_args,
         forge_cli_args=forge_cli_args,
         test_args=test_args,
     )
 
-    log.debug("forge_args: %s", forge_args)
+    log.info("forge_args: %s", forge_args)
 
     # use the github actor username if possible
     forge_username = os.getenv("GITHUB_ACTOR") or "unknown-username"

testsuite/forge/Cargo.toml

@@ -62,6 +62,9 @@ thiserror = { workspace = true }
 tokio = { workspace = true }
 url = { workspace = true }
 
+[dev-dependencies]
+serde_merge = { workspace = true }
+
 [features]
 default = []
 testing = ["aptos-global-constants/testing"]