feat(grafana): add grafana support

README.md

@@ -93,6 +93,20 @@ optimism_package:
       # Prometheus docker image to use
       # Defaults to the latest image
       image: "prom/prometheus:latest"
+    # Default grafana configuration
+    grafana_params:
+      # A list of locators for grafana dashboards to be loaded be the grafana service
+      dashboard_sources: []
+      # Resource management for grafana container
+      # CPU is milicores
+      # RAM is in MB
+      min_cpu: 10
+      max_cpu: 1000
+      min_mem: 128
+      max_mem: 2048
+      # Grafana docker image to use
+      # Defaults to the latest image
+      image: "grafana/grafana:latest"
   # Interop configuration
   interop:
     # Whether or not to enable interop mode

main.star

@@ -7,6 +7,7 @@ op_supervisor_launcher = import_module(
 
 observability = import_module("./src/observability/observability.star")
 prometheus = import_module("./src/observability/prometheus/prometheus_launcher.star")
+grafana = import_module("./src/observability/grafana/grafana_launcher.star")
 
 wait_for_sync = import_module("./src/wait/wait_for_sync.star")
 input_parser = import_module("./src/package_io/input_parser.star")
@@ -131,14 +132,22 @@ def run(plan, args):
             observability_helper,
         )
 
-    if observability_helper.enabled:
+    if observability_helper.enabled and len(observability_helper.metrics_jobs) > 0:
         plan.print("Launching prometheus...")
         prometheus_private_url = prometheus.launch_prometheus(
             plan,
             observability_helper,
             global_node_selectors,
         )
 
+        plan.print("Launching grafana...")
+        grafana.launch_grafana(
+            plan,
+            prometheus_private_url,
+            global_node_selectors,
+            observability_params.grafana_params,
+        )
+
 
 def get_l1_config(all_l1_participants, l1_network_params, l1_network_id):
     env_vars = {}

src/observability/grafana/grafana_launcher.star

@@ -0,0 +1,144 @@
+util = import_module("../../util.star")
+
+ethereum_package_shared_utils = import_module(
+    "github.com/ethpandaops/ethereum-package/src/shared_utils/shared_utils.star"
+)
+
+SERVICE_NAME = "grafana"
+
+HTTP_PORT_ID = "http"
+HTTP_PORT_NUMBER_UINT16 = 3000
+
+TEMPLATES_FILEPATH = "./templates"
+
+DATASOURCE_UID = "grafanacloud-prom"
+DATASOURCE_CONFIG_TEMPLATE_FILEPATH = TEMPLATES_FILEPATH + "/datasource.yml.tmpl"
+DATASOURCE_CONFIG_REL_FILEPATH = "datasources/datasource.yml"
+
+CONFIG_DIRPATH_ON_SERVICE = "/config"
+
+USED_PORTS = {
+    HTTP_PORT_ID: ethereum_package_shared_utils.new_port_spec(
+        HTTP_PORT_NUMBER_UINT16,
+        ethereum_package_shared_utils.TCP_PROTOCOL,
+        ethereum_package_shared_utils.HTTP_APPLICATION_PROTOCOL,
+    )
+}
+
+
+def launch_grafana(
+    plan,
+    prometheus_private_url,
+    global_node_selectors,
+    grafana_params,
+):
+    datasource_config_template = read_file(DATASOURCE_CONFIG_TEMPLATE_FILEPATH)
+
+    grafana_config_artifact_name = upload_grafana_config(
+        plan,
+        datasource_config_template,
+        prometheus_private_url,
+    )
+
+    config = get_config(
+        grafana_config_artifact_name,
+        global_node_selectors,
+        grafana_params,
+    )
+
+    service = plan.add_service(SERVICE_NAME, config)
+
+    service_url = "http://{0}:{1}".format(
+        service.ip_address, service.ports[HTTP_PORT_ID].number
+    )
+
+    provision_dashboards(plan, service_url, grafana_params.dashboard_sources)
+
+    return service_url
+
+
+def upload_grafana_config(
+    plan,
+    datasource_config_template,
+    prometheus_private_url,
+):
+    datasource_data = new_datasource_config_template_data(prometheus_private_url)
+    datasource_template_and_data = ethereum_package_shared_utils.new_template_and_data(
+        datasource_config_template, datasource_data
+    )
+
+    template_and_data_by_rel_dest_filepath = {
+        DATASOURCE_CONFIG_REL_FILEPATH: datasource_template_and_data,
+    }
+
+    grafana_config_artifact_name = plan.render_templates(
+        template_and_data_by_rel_dest_filepath, name="grafana-config"
+    )
+
+    return grafana_config_artifact_name
+
+
+def new_datasource_config_template_data(prometheus_url):
+    return {"PrometheusUID": DATASOURCE_UID, "PrometheusURL": prometheus_url}
+
+
+def get_config(
+    grafana_config_artifact_name,
+    node_selectors,
+    grafana_params,
+):
+    return ServiceConfig(
+        image=grafana_params.image,
+        ports=USED_PORTS,
+        env_vars={
+            "GF_PATHS_PROVISIONING": CONFIG_DIRPATH_ON_SERVICE,
+            "GF_AUTH_ANONYMOUS_ENABLED": "true",
+            "GF_AUTH_ANONYMOUS_ORG_ROLE": "Admin",
+            "GF_AUTH_ANONYMOUS_ORG_NAME": "Main Org.",
+            # "GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH": "/dashboards/default.json",
+        },
+        files={
+            CONFIG_DIRPATH_ON_SERVICE: grafana_config_artifact_name,
+        },
+        min_cpu=grafana_params.min_cpu,
+        max_cpu=grafana_params.max_cpu,
+        min_memory=grafana_params.min_mem,
+        max_memory=grafana_params.max_mem,
+        node_selectors=node_selectors,
+    )
+
+
+def provision_dashboards(plan, service_url, dashboard_sources):
+    if len(dashboard_sources) == 0:
+        return
+
+    def grr_push(dir):
+        return 'grr push "{0}" -e --disable-reporting'.format(dir)
+
+    def grr_push_dashboards(name):
+        return [
+            grr_push("{0}/folders".format(name)),
+            grr_push("{0}/dashboards".format(name)),
+        ]
+
+    grr_commands = [
+        "grr config create-context kurtosis",
+    ]
+
+    files = {}
+    for index, dashboard_src in enumerate(dashboard_sources):
+        dashboard_name = "dashboards-{0}".format(index)
+        dashboard_artifact_name = plan.upload_files(dashboard_src, name=dashboard_name)
+
+        files[dashboard_name] = dashboard_artifact_name
+        grr_commands += grr_push_dashboards(dashboard_name)
+
+    plan.run_sh(
+        description="upload dashboards",
+        image="grafana/grizzly:main-0b88d01",
+        env_vars={
+            "GRAFANA_URL": service_url,
+        },
+        files=files,
+        run=util.join_cmds(grr_commands),
+    )

src/observability/grafana/templates/datasource.yml.tmpl

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    orgId: 1
+    uid: {{ .PrometheusUID }}
+    url: {{ .PrometheusURL }}
+    basicAuth: false
+    isDefault: true
+    editable: true

src/observability/observability.star

@@ -97,6 +97,8 @@ def register_service_metrics_job(
 ):
     labels = {
         "service": service_name,
+        "namespace": "kurtosis",
+        "stack_optimism_io_network": "kurtosis",
     }
     labels.update(additional_labels)
 

src/observability/prometheus/prometheus_launcher.star

@@ -6,9 +6,6 @@ def launch_prometheus(
     observability_helper,
     global_node_selectors,
 ):
-    if len(observability_helper.metrics_jobs) == 0:
-        return None
-
     prometheus_params = observability_helper.params.prometheus_params
 
     prometheus_url = prometheus.run(

src/package_io/input_parser.star

@@ -77,6 +77,16 @@ def input_parser(plan, input_args):
                 min_mem=results["observability"]["prometheus_params"]["min_mem"],
                 max_mem=results["observability"]["prometheus_params"]["max_mem"],
             ),
+            grafana_params=struct(
+                image=results["observability"]["grafana_params"]["image"],
+                dashboard_sources=results["observability"]["grafana_params"][
+                    "dashboard_sources"
+                ],
+                min_cpu=results["observability"]["grafana_params"]["min_cpu"],
+                max_cpu=results["observability"]["grafana_params"]["max_cpu"],
+                min_mem=results["observability"]["grafana_params"]["min_mem"],
+                max_mem=results["observability"]["grafana_params"]["max_mem"],
+            ),
         ),
         interop=struct(
             enabled=results["interop"]["enabled"],
@@ -225,6 +235,11 @@ def parse_network_params(plan, input_args):
         input_args.get("observability", {}).get("prometheus_params", {})
     )
 
+    results["observability"]["grafana_params"] = default_grafana_params()
+    results["observability"]["grafana_params"].update(
+        input_args.get("observability", {}).get("grafana_params", {})
+    )
+
     # configure interop
 
     results["interop"] = default_interop_params()
@@ -383,6 +398,17 @@ def default_prometheus_params():
     }
 
 
+def default_grafana_params():
+    return {
+        "image": "grafana/grafana:latest",
+        "dashboard_sources": [],
+        "min_cpu": 10,
+        "max_cpu": 1000,
+        "min_mem": 128,
+        "max_mem": 2048,
+    }
+
+
 def default_interop_params():
     return {
         "enabled": False,

src/package_io/sanity_check.star

@@ -1,6 +1,7 @@
 OBSERVABILITY_PARAMS = [
     "enabled",
     "prometheus_params",
+    "grafana_params",
 ]
 
 PROMETHEUS_PARAMS = [
@@ -13,6 +14,15 @@ PROMETHEUS_PARAMS = [
     "max_mem",
 ]
 
+GRAFANA_PARAMS = [
+    "image",
+    "dashboard_sources",
+    "min_cpu",
+    "max_cpu",
+    "min_mem",
+    "max_mem",
+]
+
 INTEROP_PARAMS = [
     "enabled",
     "supervisor_params",
@@ -166,6 +176,14 @@ def sanity_check(plan, optimism_config):
                 PROMETHEUS_PARAMS,
             )
 
+        if "grafana_params" in optimism_config["observability"]:
+            validate_params(
+                plan,
+                optimism_config["observability"],
+                "grafana_params",
+                GRAFANA_PARAMS,
+            )
+
     if "interop" in optimism_config:
         validate_params(
             plan,

src/util.star

@@ -66,3 +66,7 @@ def label_from_image(image):
     if len(label) > max_length:
         label = label[-max_length:]
     return label
+
+
+def join_cmds(commands):
+    return " && ".join(commands)