test: kill prover node and see it recover

spartan/aztec-chaos-scenarios/templates/prover-kill.yaml

@@ -0,0 +1,21 @@
+{{- if .Values.proverKill.enabled }}
+---
+apiVersion: chaos-mesh.org/v1alpha1
+kind: PodChaos
+metadata:
+  name: {{ .Values.global.targetNamespace }}-prover-kill
+  namespace: {{ .Values.global.chaosMeshNamespace }}
+  labels:
+    {{- include "aztec-chaos-scenarios.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/resource-policy": keep
+spec:
+  action: pod-kill
+  mode: fixed-percent
+  value: {{ .Values.proverKill.percent | quote }}
+  selector:
+    namespaces:
+      - {{ .Values.global.targetNamespace }}
+    labelSelectors:
+      app: prover-node
+{{- end }}

spartan/aztec-chaos-scenarios/values.yaml

@@ -69,6 +69,10 @@ proverFailure:
   enabled: false
   duration: 13m
 
+proverKill:
+  enabled: false
+  percent: 100
+
 validatorKill:
   enabled: false
   percent: 30

spartan/aztec-chaos-scenarios/values/prover-kill.yaml

@@ -0,0 +1,6 @@
+global:
+  namespace: "smoke"
+
+proverKill:
+  enabled: true
+  percent: 100

spartan/aztec-network/templates/prover-agent.yaml

@@ -100,6 +100,12 @@ spec:
               value: "1"
             - name: PROVER_AGENT_POLL_INTERVAL_MS
               value: "{{ .Values.proverAgent.pollIntervalMs }}"
+            - name: PROVER_TEST_DELAY_TYPE
+              value: "{{ .Values.proverAgent.testDelayType }}"
+            - name: PROVER_TEST_DELAY_MS
+              value: "{{ .Values.proverAgent.testDelayMs }}"
+            - name: PROVER_TEST_DELAY_FACTOR
+              value: "{{ .Values.proverAgent.testDelayFactor }}"
             - name: PROVER_AGENT_PROOF_TYPES
               value: {{ join "," .Values.proverAgent.proofTypes | quote }}
             - name: OTEL_RESOURCE_ATTRIBUTES

spartan/aztec-network/values.yaml

@@ -234,7 +234,10 @@ proverAgent:
   enabled: true
   replicas: 1
   pollIntervalMs: 1000
-  proofTypes: ["foo", "bar", "baz"]
+  proofTypes: []
+  testDelayType: "fixed"
+  testDelayMs: 100 # each fake proof takes 100ms
+  testDelayFactor: 1
   gke:
     spotEnabled: false
   logLevel: "debug; info: aztec:simulator, json-rpc"

spartan/aztec-network/values/prover-node-chaos.yaml

@@ -0,0 +1,20 @@
+validator:
+  replicas: 1
+  validatorKeys:
+    - 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d
+  validatorAddresses:
+    - 0x70997970C51812dc3A010C7d01b50e0d17dc79C8
+  validator:
+    disabled: false
+
+bootNode:
+  validator:
+    disabled: true
+
+telemetry:
+  enabled: true
+
+proverAgent:
+  testDelayType: "fixed"
+  testDelayMs: 2000
+

yarn-project/bb-prover/src/test/delay_values.ts

@@ -0,0 +1,31 @@
+import { ProvingRequestType } from '@aztec/circuit-types';
+
+export const WITGEN_DELAY_MS: Record<ProvingRequestType, number> = {
+  [ProvingRequestType.BASE_PARITY]: 60,
+  [ProvingRequestType.BLOCK_MERGE_ROLLUP]: 650,
+  [ProvingRequestType.BLOCK_ROOT_ROLLUP]: 60_000,
+  [ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP]: 0,
+  [ProvingRequestType.MERGE_ROLLUP]: 0,
+  [ProvingRequestType.PRIVATE_BASE_ROLLUP]: 400_000,
+  [ProvingRequestType.PRIVATE_KERNEL_EMPTY]: 50,
+  [ProvingRequestType.PUBLIC_BASE_ROLLUP]: 470_000,
+  [ProvingRequestType.ROOT_PARITY]: 100,
+  [ProvingRequestType.ROOT_ROLLUP]: 650,
+  [ProvingRequestType.TUBE_PROOF]: 0,
+  [ProvingRequestType.PUBLIC_VM]: 0,
+};
+
+export const PROOF_DELAY_MS: Record<ProvingRequestType, number> = {
+  [ProvingRequestType.BASE_PARITY]: 3_000,
+  [ProvingRequestType.BLOCK_MERGE_ROLLUP]: 15_000,
+  [ProvingRequestType.BLOCK_ROOT_ROLLUP]: 55_000,
+  [ProvingRequestType.EMPTY_BLOCK_ROOT_ROLLUP]: 0,
+  [ProvingRequestType.MERGE_ROLLUP]: 0,
+  [ProvingRequestType.PRIVATE_BASE_ROLLUP]: 145_000,
+  [ProvingRequestType.PRIVATE_KERNEL_EMPTY]: 10_000,
+  [ProvingRequestType.PUBLIC_BASE_ROLLUP]: 160_000,
+  [ProvingRequestType.ROOT_PARITY]: 30_000,
+  [ProvingRequestType.ROOT_ROLLUP]: 15_000,
+  [ProvingRequestType.TUBE_PROOF]: 30_000,
+  [ProvingRequestType.PUBLIC_VM]: 0,
+};