From 7ea6ca25638a7b4939d41e01dc75679560897e80 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Mon, 13 Jan 2025 11:11:02 -0500 Subject: [PATCH 01/16] add grafana_params --- README.md | 14 ++++++++++++++ main.star | 19 +++++++++++++++++++ src/package_io/input_parser.star | 24 ++++++++++++++++++++++++ src/package_io/sanity_check.star | 18 ++++++++++++++++++ 4 files changed, 75 insertions(+) diff --git a/README.md b/README.md index 59966ab5..5b1e343c 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,20 @@ optimism_package: # Prometheus docker image to use # Defaults to the latest image image: "prom/prometheus:latest" + # Default grafana configuration + grafana_params: + # A list of locators for grafana dashboards to be loaded be the grafana service + additional_dashboards: [] + # Resource management for grafana container + # CPU is milicores + # RAM is in MB + min_cpu: 10 + max_cpu: 1000 + min_mem: 128 + max_mem: 2048 + # Grafana docker image to use + # Defaults to the latest image + image: "grafana/grafana:latest" # Interop configuration interop: # Whether or not to enable interop mode diff --git a/main.star b/main.star index 7e847ea7..293120d3 100644 --- a/main.star +++ b/main.star @@ -7,6 +7,7 @@ op_supervisor_launcher = import_module( observability = import_module("./src/observability/observability.star") prometheus = import_module("./src/observability/prometheus/prometheus_launcher.star") +grafana = import_module("github.com/ethpandaops/ethereum-package/src/grafana/grafana_launcher.star") wait_for_sync = import_module("./src/wait/wait_for_sync.star") input_parser = import_module("./src/package_io/input_parser.star") @@ -139,6 +140,24 @@ def run(plan, args): global_node_selectors, ) + # grafana_datasource_config_template = read_file( + # ethereum_package_static_files.GRAFANA_DATASOURCE_CONFIG_TEMPLATE_FILEPATH + # ) + # grafana_dashboards_config_template = read_file( + # ethereum_package_static_files.GRAFANA_DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH + # ) + + plan.print("Launching grafana...") + grafana.launch_grafana( + plan, + # grafana_datasource_config_template, + # grafana_dashboards_config_template, + "", + "", + prometheus_private_url, + global_node_selectors, + observability_params.grafana_params, + ) def get_l1_config(all_l1_participants, l1_network_params, l1_network_id): env_vars = {} diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index fddde4b7..a63b8430 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -77,6 +77,16 @@ def input_parser(plan, input_args): min_mem=results["observability"]["prometheus_params"]["min_mem"], max_mem=results["observability"]["prometheus_params"]["max_mem"], ), + grafana_params=struct( + image=results["observability"]["grafana_params"]["image"], + additional_dashboards=results["observability"][ + "grafana_params" + ]["additional_dashboards"], + min_cpu=results["observability"]["grafana_params"]["min_cpu"], + max_cpu=results["observability"]["grafana_params"]["max_cpu"], + min_mem=results["observability"]["grafana_params"]["min_mem"], + max_mem=results["observability"]["grafana_params"]["max_mem"], + ), ), interop=struct( enabled=results["interop"]["enabled"], @@ -201,6 +211,11 @@ def parse_network_params(plan, input_args): input_args.get("observability", {}).get("prometheus_params", {}) ) + results["observability"]["grafana_params"] = default_grafana_params() + results["observability"]["grafana_params"].update( + input_args.get("observability", {}).get("grafana_params", {}) + ) + # configure interop results["interop"] = default_interop_params() @@ -358,6 +373,15 @@ def default_prometheus_params(): "max_mem": 2048, } +def default_grafana_params(): + return { + "image": "grafana/grafana:latest", + "additional_dashboards": [], + "min_cpu": 10, + "max_cpu": 1000, + "min_mem": 128, + "max_mem": 2048, + } def default_interop_params(): return { diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index 94fa4614..af73d9ea 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -1,6 +1,7 @@ OBSERVABILITY_PARAMS = [ "enabled", "prometheus_params", + "grafana_params", ] PROMETHEUS_PARAMS = [ @@ -13,6 +14,15 @@ PROMETHEUS_PARAMS = [ "max_mem", ] +GRAFANA_PARAMS = [ + "image", + "additional_dashboards", + "min_cpu", + "max_cpu", + "min_mem", + "max_mem", +] + INTEROP_PARAMS = [ "enabled", "supervisor_params", @@ -166,6 +176,14 @@ def sanity_check(plan, optimism_config): PROMETHEUS_PARAMS, ) + if "grafana_params" in optimism_config["observability"]: + validate_params( + plan, + optimism_config["observability"], + "grafana_params", + GRAFANA_PARAMS, + ) + if "interop" in optimism_config: validate_params( plan, From 99e45e0b9e84f2a948ce931a33345614cd103bc3 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Mon, 13 Jan 2025 17:36:02 -0500 Subject: [PATCH 02/16] copy over grafana module to allow customizing dashboards --- main.star | 25 +- .../grafana/grafana_launcher.star | 215 ++++++++++++++++++ .../templates/dashboard-providers.yml.tmpl | 25 ++ .../templates/datasource.yml.tmpl | 11 + 4 files changed, 263 insertions(+), 13 deletions(-) create mode 100644 src/observability/grafana/grafana_launcher.star create mode 100644 static_files/grafana-config/templates/dashboard-providers.yml.tmpl create mode 100644 static_files/grafana-config/templates/datasource.yml.tmpl diff --git a/main.star b/main.star index 293120d3..f12428d2 100644 --- a/main.star +++ b/main.star @@ -7,7 +7,7 @@ op_supervisor_launcher = import_module( observability = import_module("./src/observability/observability.star") prometheus = import_module("./src/observability/prometheus/prometheus_launcher.star") -grafana = import_module("github.com/ethpandaops/ethereum-package/src/grafana/grafana_launcher.star") +grafana = import_module("./src/observability/grafana/grafana_launcher.star") wait_for_sync = import_module("./src/wait/wait_for_sync.star") input_parser = import_module("./src/package_io/input_parser.star") @@ -132,7 +132,7 @@ def run(plan, args): observability_helper, ) - if observability_helper.enabled: + if observability_helper.enabled and len(observability_helper.metrics_jobs) > 0: plan.print("Launching prometheus...") prometheus_private_url = prometheus.launch_prometheus( plan, @@ -140,20 +140,19 @@ def run(plan, args): global_node_selectors, ) - # grafana_datasource_config_template = read_file( - # ethereum_package_static_files.GRAFANA_DATASOURCE_CONFIG_TEMPLATE_FILEPATH - # ) - # grafana_dashboards_config_template = read_file( - # ethereum_package_static_files.GRAFANA_DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH - # ) - plan.print("Launching grafana...") + + grafana_datasource_config_template = read_file( + ethereum_package_static_files.GRAFANA_DATASOURCE_CONFIG_TEMPLATE_FILEPATH + ) + grafana_dashboards_config_template = read_file( + ethereum_package_static_files.GRAFANA_DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH + ) + grafana.launch_grafana( plan, - # grafana_datasource_config_template, - # grafana_dashboards_config_template, - "", - "", + grafana_datasource_config_template, + grafana_dashboards_config_template, prometheus_private_url, global_node_selectors, observability_params.grafana_params, diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star new file mode 100644 index 00000000..ab144714 --- /dev/null +++ b/src/observability/grafana/grafana_launcher.star @@ -0,0 +1,215 @@ +ethereum_package_shared_utils = import_module( + "github.com/ethpandaops/ethereum-package/src/shared_utils/shared_utils.star" +) +ethereum_package_static_files = import_module( + "github.com/ethpandaops/ethereum-package/src/static_files/static_files.star" +) + +SERVICE_NAME = "grafana" + +HTTP_PORT_ID = "http" +HTTP_PORT_NUMBER_UINT16 = 3000 + +DATASOURCE_CONFIG_REL_FILEPATH = "datasources/datasource.yml" + +# this is relative to the files artifact root +DASHBOARD_PROVIDERS_CONFIG_REL_FILEPATH = "dashboards/dashboard-providers.yml" + +CONFIG_DIRPATH_ENV_VAR = "GF_PATHS_PROVISIONING" + +GRAFANA_CONFIG_DIRPATH_ON_SERVICE = "/config" +GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE = "/dashboards" +GRAFANA_DASHBOARDS_FILEPATH_ON_SERVICE = GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE + +GRAFANA_ADDITIONAL_DASHBOARDS_FOLDER_NAME = "grafana-additional-dashboards-{0}" +GRAFANA_ADDITIONAL_DASHBOARDS_MERGED_STORED_PATH_FORMAT = ( + GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE + "/*" +) +GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE = "/additional-dashobards" +GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE_FORMAT = ( + GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE + "/{0}" +) +GRAFANA_ADDITIONAL_DASHBOARDS_SERVICE_PATH_KEY = "ServicePath" +GRANAFA_ADDITIONAL_DASHBOARDS_ARTIFACT_NAME_KEY = "ArtifactName" + +USED_PORTS = { + HTTP_PORT_ID: ethereum_package_shared_utils.new_port_spec( + HTTP_PORT_NUMBER_UINT16, + ethereum_package_shared_utils.TCP_PROTOCOL, + ethereum_package_shared_utils.HTTP_APPLICATION_PROTOCOL, + ) +} + + +def launch_grafana( + plan, + datasource_config_template, + dashboard_providers_config_template, + prometheus_private_url, + global_node_selectors, + grafana_params, +): + ( + grafana_config_artifacts_uuid, + grafana_dashboards_artifacts_uuid, + grafana_additional_dashboards_data, + ) = get_grafana_config_dir_artifact_uuid( + plan, + datasource_config_template, + dashboard_providers_config_template, + prometheus_private_url, + additional_dashboards=grafana_params.additional_dashboards, + ) + + merged_dashboards_artifact_name = merge_dashboards_artifacts( + plan, + grafana_dashboards_artifacts_uuid, + grafana_additional_dashboards_data, + ) + + config = get_config( + grafana_config_artifacts_uuid, + merged_dashboards_artifact_name, + global_node_selectors, + grafana_params, + ) + + plan.add_service(SERVICE_NAME, config) + + +def get_grafana_config_dir_artifact_uuid( + plan, + datasource_config_template, + dashboard_providers_config_template, + prometheus_private_url, + additional_dashboards=[], +): + datasource_data = new_datasource_config_template_data(prometheus_private_url) + datasource_template_and_data = ethereum_package_shared_utils.new_template_and_data( + datasource_config_template, datasource_data + ) + + dashboard_providers_data = new_dashboard_providers_config_template_data( + GRAFANA_DASHBOARDS_FILEPATH_ON_SERVICE + ) + dashboard_providers_template_and_data = ethereum_package_shared_utils.new_template_and_data( + dashboard_providers_config_template, dashboard_providers_data + ) + + template_and_data_by_rel_dest_filepath = {} + template_and_data_by_rel_dest_filepath[ + DATASOURCE_CONFIG_REL_FILEPATH + ] = datasource_template_and_data + template_and_data_by_rel_dest_filepath[ + DASHBOARD_PROVIDERS_CONFIG_REL_FILEPATH + ] = dashboard_providers_template_and_data + + grafana_config_artifacts_name = plan.render_templates( + template_and_data_by_rel_dest_filepath, name="grafana-config" + ) + + grafana_dashboards_artifacts_name = plan.upload_files( + ethereum_package_static_files.GRAFANA_DASHBOARDS_CONFIG_DIRPATH, name="grafana-dashboards" + ) + + grafana_additional_dashboards_data = upload_additional_dashboards( + plan, additional_dashboards + ) + + return ( + grafana_config_artifacts_name, + grafana_dashboards_artifacts_name, + grafana_additional_dashboards_data, + ) + + +def get_config( + grafana_config_artifacts_name, + grafana_dashboards_artifacts_name, + node_selectors, + grafana_params, +): + return ServiceConfig( + image=grafana_params.image, + ports=USED_PORTS, + env_vars={ + CONFIG_DIRPATH_ENV_VAR: GRAFANA_CONFIG_DIRPATH_ON_SERVICE, + "GF_AUTH_ANONYMOUS_ENABLED": "true", + "GF_AUTH_ANONYMOUS_ORG_ROLE": "Admin", + "GF_AUTH_ANONYMOUS_ORG_NAME": "Main Org.", + "GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH": "/dashboards/default.json", + }, + files={ + GRAFANA_CONFIG_DIRPATH_ON_SERVICE: grafana_config_artifacts_name, + GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE: grafana_dashboards_artifacts_name, + }, + min_cpu=grafana_params.min_cpu, + max_cpu=grafana_params.max_cpu, + min_memory=grafana_params.min_mem, + max_memory=grafana_params.max_mem, + node_selectors=node_selectors, + ) + + +def new_datasource_config_template_data(prometheus_url): + return {"PrometheusURL": prometheus_url} + + +def new_dashboard_providers_config_template_data(dashboards_dirpath): + return {"DashboardsDirpath": dashboards_dirpath} + + +def upload_additional_dashboards(plan, additional_dashboards): + data = [] + for index, dashboard_src in enumerate(additional_dashboards): + additional_dashboard_folder_name = ( + GRAFANA_ADDITIONAL_DASHBOARDS_FOLDER_NAME.format(index) + ) + additional_dashboard_service_path = ( + GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE_FORMAT.format( + additional_dashboard_folder_name, + ) + ) + additional_dashboard_artifact_name = plan.upload_files( + dashboard_src, name="additional-grafana-dashboard-{0}".format(index) + ) + data.append( + { + GRAFANA_ADDITIONAL_DASHBOARDS_SERVICE_PATH_KEY: additional_dashboard_service_path, + GRANAFA_ADDITIONAL_DASHBOARDS_ARTIFACT_NAME_KEY: additional_dashboard_artifact_name, + } + ) + return data + + +def merge_dashboards_artifacts( + plan, + grafana_dashboards_artifacts_name, + grafana_additional_dashboards_data=[], +): + if len(grafana_additional_dashboards_data) == 0: + return grafana_dashboards_artifacts_name + + files = { + GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE: grafana_dashboards_artifacts_name, + } + + for additional_dashboard_data in grafana_additional_dashboards_data: + files[ + additional_dashboard_data[GRAFANA_ADDITIONAL_DASHBOARDS_SERVICE_PATH_KEY] + ] = additional_dashboard_data[GRANAFA_ADDITIONAL_DASHBOARDS_ARTIFACT_NAME_KEY] + + result = plan.run_sh( + description="Merging grafana dashboards artifacts", + run="find " + + GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE + + " -type f -exec cp {} " + + GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE + + " \\;", + files=files, + store=[ + GRAFANA_ADDITIONAL_DASHBOARDS_MERGED_STORED_PATH_FORMAT, + ], + ) + + return result.files_artifacts[0] diff --git a/static_files/grafana-config/templates/dashboard-providers.yml.tmpl b/static_files/grafana-config/templates/dashboard-providers.yml.tmpl new file mode 100644 index 00000000..451ae946 --- /dev/null +++ b/static_files/grafana-config/templates/dashboard-providers.yml.tmpl @@ -0,0 +1,25 @@ +apiVersion: 1 + +providers: + # an unique provider name. Required + - name: 'Optimism Kurtosis Module Dashboard Provider' + # Org id. Default to 1 + orgId: 1 + # name of the dashboard folder. + folder: '' + # folder UID. will be automatically generated if not specified + folderUid: '' + # provider type. Default to 'file' + type: file + # disable dashboard deletion + disableDeletion: false + # how often Grafana will scan for changed dashboards + updateIntervalSeconds: 10 + # allow updating provisioned dashboards from the UI + allowUiUpdates: true + editable: true + options: + # path to dashboard files on disk. Required when using the 'file' type + path: {{ .DashboardsDirpath }} + # use folder names from filesystem to create folders in Grafana + foldersFromFilesStructure: true diff --git a/static_files/grafana-config/templates/datasource.yml.tmpl b/static_files/grafana-config/templates/datasource.yml.tmpl new file mode 100644 index 00000000..b873599d --- /dev/null +++ b/static_files/grafana-config/templates/datasource.yml.tmpl @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + orgId: 1 + url: {{ .PrometheusURL }} + basicAuth: false + isDefault: true + editable: true From f4e5d21666be48fa69ba1c6ec403f8cd3481fbda Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Tue, 14 Jan 2025 19:47:28 -0500 Subject: [PATCH 03/16] add standard labels --- src/observability/observability.star | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/observability/observability.star b/src/observability/observability.star index a6c11d25..78f8d6a7 100644 --- a/src/observability/observability.star +++ b/src/observability/observability.star @@ -70,6 +70,7 @@ def new_metrics_job( labels, scrape_interval=DEFAULT_SCRAPE_INTERVAL, ): + return { "Name": job_name, "Endpoint": endpoint, @@ -97,6 +98,8 @@ def register_service_metrics_job( ): labels = { "service": service_name, + "namespace": "kurtosis", + "stack_optimism_io_network": "kurtosis", } labels.update(additional_labels) From 92e649670ffa487550a690b0940bba1a6f254618 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Tue, 14 Jan 2025 19:48:34 -0500 Subject: [PATCH 04/16] move metrics job check --- src/observability/prometheus/prometheus_launcher.star | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/observability/prometheus/prometheus_launcher.star b/src/observability/prometheus/prometheus_launcher.star index 121c8f37..4d95f823 100644 --- a/src/observability/prometheus/prometheus_launcher.star +++ b/src/observability/prometheus/prometheus_launcher.star @@ -6,9 +6,6 @@ def launch_prometheus( observability_helper, global_node_selectors, ): - if len(observability_helper.metrics_jobs) == 0: - return None - prometheus_params = observability_helper.params.prometheus_params prometheus_url = prometheus.run( From f1fc84fee120b0e572aa52e84207b9d383691fd3 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Wed, 15 Jan 2025 16:38:51 -0500 Subject: [PATCH 05/16] load grafana configs in module --- main.star | 8 -------- src/observability/grafana/grafana_launcher.star | 13 +++++++++---- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/main.star b/main.star index f12428d2..4c260d39 100644 --- a/main.star +++ b/main.star @@ -142,17 +142,9 @@ def run(plan, args): plan.print("Launching grafana...") - grafana_datasource_config_template = read_file( - ethereum_package_static_files.GRAFANA_DATASOURCE_CONFIG_TEMPLATE_FILEPATH - ) - grafana_dashboards_config_template = read_file( - ethereum_package_static_files.GRAFANA_DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH - ) grafana.launch_grafana( plan, - grafana_datasource_config_template, - grafana_dashboards_config_template, prometheus_private_url, global_node_selectors, observability_params.grafana_params, diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star index ab144714..038e5892 100644 --- a/src/observability/grafana/grafana_launcher.star +++ b/src/observability/grafana/grafana_launcher.star @@ -43,15 +43,20 @@ USED_PORTS = { def launch_grafana( plan, - datasource_config_template, - dashboard_providers_config_template, prometheus_private_url, global_node_selectors, grafana_params, ): + datasource_config_template = read_file( + ethereum_package_static_files.GRAFANA_DATASOURCE_CONFIG_TEMPLATE_FILEPATH + ) + dashboard_providers_config_template = read_file( + ethereum_package_static_files.GRAFANA_DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH + ) + ( - grafana_config_artifacts_uuid, - grafana_dashboards_artifacts_uuid, + grafana_config_artifact_name, + grafana_dashboards_artifact_name, grafana_additional_dashboards_data, ) = get_grafana_config_dir_artifact_uuid( plan, From c6d918256a431d3bd350f93ceb16e9926d619bd8 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Wed, 15 Jan 2025 16:39:45 -0500 Subject: [PATCH 06/16] clean up upload_grafana_config --- .../grafana/grafana_launcher.star | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star index 038e5892..70431844 100644 --- a/src/observability/grafana/grafana_launcher.star +++ b/src/observability/grafana/grafana_launcher.star @@ -58,7 +58,7 @@ def launch_grafana( grafana_config_artifact_name, grafana_dashboards_artifact_name, grafana_additional_dashboards_data, - ) = get_grafana_config_dir_artifact_uuid( + ) = upload_grafana_config( plan, datasource_config_template, dashboard_providers_config_template, @@ -73,7 +73,7 @@ def launch_grafana( ) config = get_config( - grafana_config_artifacts_uuid, + grafana_config_artifact_name, merged_dashboards_artifact_name, global_node_selectors, grafana_params, @@ -82,7 +82,7 @@ def launch_grafana( plan.add_service(SERVICE_NAME, config) -def get_grafana_config_dir_artifact_uuid( +def upload_grafana_config( plan, datasource_config_template, dashboard_providers_config_template, @@ -101,19 +101,16 @@ def get_grafana_config_dir_artifact_uuid( dashboard_providers_config_template, dashboard_providers_data ) - template_and_data_by_rel_dest_filepath = {} - template_and_data_by_rel_dest_filepath[ - DATASOURCE_CONFIG_REL_FILEPATH - ] = datasource_template_and_data - template_and_data_by_rel_dest_filepath[ - DASHBOARD_PROVIDERS_CONFIG_REL_FILEPATH - ] = dashboard_providers_template_and_data + template_and_data_by_rel_dest_filepath = { + DATASOURCE_CONFIG_REL_FILEPATH: datasource_template_and_data, + DASHBOARD_PROVIDERS_CONFIG_REL_FILEPATH: dashboard_providers_template_and_data, + } - grafana_config_artifacts_name = plan.render_templates( + grafana_config_artifact_name = plan.render_templates( template_and_data_by_rel_dest_filepath, name="grafana-config" ) - grafana_dashboards_artifacts_name = plan.upload_files( + grafana_dashboards_artifact_name = plan.upload_files( ethereum_package_static_files.GRAFANA_DASHBOARDS_CONFIG_DIRPATH, name="grafana-dashboards" ) @@ -122,15 +119,15 @@ def get_grafana_config_dir_artifact_uuid( ) return ( - grafana_config_artifacts_name, - grafana_dashboards_artifacts_name, + grafana_config_artifact_name, + grafana_dashboards_artifact_name, grafana_additional_dashboards_data, ) def get_config( - grafana_config_artifacts_name, - grafana_dashboards_artifacts_name, + grafana_config_artifact_name, + grafana_dashboards_artifact_name, node_selectors, grafana_params, ): @@ -145,8 +142,8 @@ def get_config( "GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH": "/dashboards/default.json", }, files={ - GRAFANA_CONFIG_DIRPATH_ON_SERVICE: grafana_config_artifacts_name, - GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE: grafana_dashboards_artifacts_name, + GRAFANA_CONFIG_DIRPATH_ON_SERVICE: grafana_config_artifact_name, + GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE: grafana_dashboards_artifact_name, }, min_cpu=grafana_params.min_cpu, max_cpu=grafana_params.max_cpu, @@ -189,14 +186,14 @@ def upload_additional_dashboards(plan, additional_dashboards): def merge_dashboards_artifacts( plan, - grafana_dashboards_artifacts_name, + grafana_dashboards_artifact_name, grafana_additional_dashboards_data=[], ): if len(grafana_additional_dashboards_data) == 0: - return grafana_dashboards_artifacts_name + return grafana_dashboards_artifact_name files = { - GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE: grafana_dashboards_artifacts_name, + GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE: grafana_dashboards_artifact_name, } for additional_dashboard_data in grafana_additional_dashboards_data: From 1bf98e468e5f627ce1114ed0acc5b051a8384680 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Wed, 15 Jan 2025 16:40:27 -0500 Subject: [PATCH 07/16] set prometheus datasource UID --- src/observability/grafana/grafana_launcher.star | 10 +++++++--- .../grafana-config/templates/datasource.yml.tmpl | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star index 70431844..2fdd50c2 100644 --- a/src/observability/grafana/grafana_launcher.star +++ b/src/observability/grafana/grafana_launcher.star @@ -10,6 +10,7 @@ SERVICE_NAME = "grafana" HTTP_PORT_ID = "http" HTTP_PORT_NUMBER_UINT16 = 3000 +DATASOURCE_UID = "grafanacloud-prom" DATASOURCE_CONFIG_REL_FILEPATH = "datasources/datasource.yml" # this is relative to the files artifact root @@ -25,7 +26,7 @@ GRAFANA_ADDITIONAL_DASHBOARDS_FOLDER_NAME = "grafana-additional-dashboards-{0}" GRAFANA_ADDITIONAL_DASHBOARDS_MERGED_STORED_PATH_FORMAT = ( GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE + "/*" ) -GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE = "/additional-dashobards" +GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE = "/additional-dashboards" GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE_FORMAT = ( GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE + "/{0}" ) @@ -68,7 +69,7 @@ def launch_grafana( merged_dashboards_artifact_name = merge_dashboards_artifacts( plan, - grafana_dashboards_artifacts_uuid, + grafana_dashboards_artifact_name, grafana_additional_dashboards_data, ) @@ -154,7 +155,10 @@ def get_config( def new_datasource_config_template_data(prometheus_url): - return {"PrometheusURL": prometheus_url} + return { + "PrometheusUID": DATASOURCE_UID, + "PrometheusURL": prometheus_url + } def new_dashboard_providers_config_template_data(dashboards_dirpath): diff --git a/static_files/grafana-config/templates/datasource.yml.tmpl b/static_files/grafana-config/templates/datasource.yml.tmpl index b873599d..a37438b0 100644 --- a/static_files/grafana-config/templates/datasource.yml.tmpl +++ b/static_files/grafana-config/templates/datasource.yml.tmpl @@ -5,6 +5,7 @@ datasources: type: prometheus access: proxy orgId: 1 + uid: {{ .PrometheusUID }} url: {{ .PrometheusURL }} basicAuth: false isDefault: true From 12b0dfb5a93852ee2017bf12b870591ccde5be47 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Wed, 15 Jan 2025 17:26:06 -0500 Subject: [PATCH 08/16] remove inline dashboards and rework dashboard_sources --- README.md | 2 +- .../grafana/grafana_launcher.star | 166 ++++++------------ .../templates/dashboard-providers.yml.tmpl | 0 .../grafana}/templates/datasource.yml.tmpl | 0 src/package_io/input_parser.star | 6 +- src/package_io/sanity_check.star | 2 +- 6 files changed, 54 insertions(+), 122 deletions(-) rename {static_files/grafana-config => src/observability/grafana}/templates/dashboard-providers.yml.tmpl (100%) rename {static_files/grafana-config => src/observability/grafana}/templates/datasource.yml.tmpl (100%) diff --git a/README.md b/README.md index 5b1e343c..ae1b6ec6 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ optimism_package: # Default grafana configuration grafana_params: # A list of locators for grafana dashboards to be loaded be the grafana service - additional_dashboards: [] + dashboard_sources: [] # Resource management for grafana container # CPU is milicores # RAM is in MB diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star index 2fdd50c2..3ac5621e 100644 --- a/src/observability/grafana/grafana_launcher.star +++ b/src/observability/grafana/grafana_launcher.star @@ -1,37 +1,25 @@ ethereum_package_shared_utils = import_module( "github.com/ethpandaops/ethereum-package/src/shared_utils/shared_utils.star" ) -ethereum_package_static_files = import_module( - "github.com/ethpandaops/ethereum-package/src/static_files/static_files.star" -) SERVICE_NAME = "grafana" HTTP_PORT_ID = "http" HTTP_PORT_NUMBER_UINT16 = 3000 +TEMPLATES_FILEPATH = "./templates" + +DATASOURCE_CONFIG_TEMPLATE_FILEPATH = TEMPLATES_FILEPATH + "/datasource.yml.tmpl" +DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH = TEMPLATES_FILEPATH + "/dashboard-providers.yml.tmpl" + DATASOURCE_UID = "grafanacloud-prom" DATASOURCE_CONFIG_REL_FILEPATH = "datasources/datasource.yml" # this is relative to the files artifact root DASHBOARD_PROVIDERS_CONFIG_REL_FILEPATH = "dashboards/dashboard-providers.yml" -CONFIG_DIRPATH_ENV_VAR = "GF_PATHS_PROVISIONING" - -GRAFANA_CONFIG_DIRPATH_ON_SERVICE = "/config" -GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE = "/dashboards" -GRAFANA_DASHBOARDS_FILEPATH_ON_SERVICE = GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE - -GRAFANA_ADDITIONAL_DASHBOARDS_FOLDER_NAME = "grafana-additional-dashboards-{0}" -GRAFANA_ADDITIONAL_DASHBOARDS_MERGED_STORED_PATH_FORMAT = ( - GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE + "/*" -) -GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE = "/additional-dashboards" -GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE_FORMAT = ( - GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE + "/{0}" -) -GRAFANA_ADDITIONAL_DASHBOARDS_SERVICE_PATH_KEY = "ServicePath" -GRANAFA_ADDITIONAL_DASHBOARDS_ARTIFACT_NAME_KEY = "ArtifactName" +CONFIG_DIRPATH_ON_SERVICE = "/config" +DASHBOARDS_DIRPATH_ON_SERVICE = "/dashboards" USED_PORTS = { HTTP_PORT_ID: ethereum_package_shared_utils.new_port_spec( @@ -48,34 +36,23 @@ def launch_grafana( global_node_selectors, grafana_params, ): - datasource_config_template = read_file( - ethereum_package_static_files.GRAFANA_DATASOURCE_CONFIG_TEMPLATE_FILEPATH - ) - dashboard_providers_config_template = read_file( - ethereum_package_static_files.GRAFANA_DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH - ) + datasource_config_template = read_file(DATASOURCE_CONFIG_TEMPLATE_FILEPATH) + dashboard_providers_config_template = read_file(DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH) ( grafana_config_artifact_name, - grafana_dashboards_artifact_name, - grafana_additional_dashboards_data, + grafana_dashboards_artifact_names, ) = upload_grafana_config( plan, datasource_config_template, dashboard_providers_config_template, prometheus_private_url, - additional_dashboards=grafana_params.additional_dashboards, - ) - - merged_dashboards_artifact_name = merge_dashboards_artifacts( - plan, - grafana_dashboards_artifact_name, - grafana_additional_dashboards_data, + dashboard_sources=grafana_params.dashboard_sources, ) config = get_config( grafana_config_artifact_name, - merged_dashboards_artifact_name, + grafana_dashboards_artifact_names, global_node_selectors, grafana_params, ) @@ -88,7 +65,7 @@ def upload_grafana_config( datasource_config_template, dashboard_providers_config_template, prometheus_private_url, - additional_dashboards=[], + dashboard_sources=[], ): datasource_data = new_datasource_config_template_data(prometheus_private_url) datasource_template_and_data = ethereum_package_shared_utils.new_template_and_data( @@ -96,7 +73,7 @@ def upload_grafana_config( ) dashboard_providers_data = new_dashboard_providers_config_template_data( - GRAFANA_DASHBOARDS_FILEPATH_ON_SERVICE + DASHBOARDS_DIRPATH_ON_SERVICE ) dashboard_providers_template_and_data = ethereum_package_shared_utils.new_template_and_data( dashboard_providers_config_template, dashboard_providers_data @@ -111,24 +88,44 @@ def upload_grafana_config( template_and_data_by_rel_dest_filepath, name="grafana-config" ) - grafana_dashboards_artifact_name = plan.upload_files( - ethereum_package_static_files.GRAFANA_DASHBOARDS_CONFIG_DIRPATH, name="grafana-dashboards" - ) - - grafana_additional_dashboards_data = upload_additional_dashboards( - plan, additional_dashboards + grafana_dashboards_artifact_names = upload_dashboards( + plan, dashboard_sources ) return ( grafana_config_artifact_name, - grafana_dashboards_artifact_name, - grafana_additional_dashboards_data, + grafana_dashboards_artifact_names, ) +def new_datasource_config_template_data(prometheus_url): + return { + "PrometheusUID": DATASOURCE_UID, + "PrometheusURL": prometheus_url + } + + +def new_dashboard_providers_config_template_data(dashboards_dirpath): + return {"DashboardsDirpath": dashboards_dirpath} + + +def upload_dashboards(plan, dashboard_sources): + dashboard_artifact_names = [] + + for index, dashboard_src in enumerate(dashboard_sources): + dashboard_name = "grafana-dashboards-{0}".format(index) + + dashboard_artifact_name = plan.upload_files( + dashboard_src, name=dashboard_name + ) + dashboard_artifact_names.append(dashboard_artifact_name) + + return dashboard_artifact_names + + def get_config( grafana_config_artifact_name, - grafana_dashboards_artifact_name, + grafana_dashboards_artifact_names, node_selectors, grafana_params, ): @@ -136,15 +133,17 @@ def get_config( image=grafana_params.image, ports=USED_PORTS, env_vars={ - CONFIG_DIRPATH_ENV_VAR: GRAFANA_CONFIG_DIRPATH_ON_SERVICE, + "GF_PATHS_PROVISIONING": CONFIG_DIRPATH_ON_SERVICE, "GF_AUTH_ANONYMOUS_ENABLED": "true", "GF_AUTH_ANONYMOUS_ORG_ROLE": "Admin", "GF_AUTH_ANONYMOUS_ORG_NAME": "Main Org.", - "GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH": "/dashboards/default.json", + # "GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH": "/dashboards/default.json", }, files={ - GRAFANA_CONFIG_DIRPATH_ON_SERVICE: grafana_config_artifact_name, - GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE: grafana_dashboards_artifact_name, + CONFIG_DIRPATH_ON_SERVICE: grafana_config_artifact_name, + DASHBOARDS_DIRPATH_ON_SERVICE: Directory( + artifact_names=grafana_dashboards_artifact_names, + ), }, min_cpu=grafana_params.min_cpu, max_cpu=grafana_params.max_cpu, @@ -152,70 +151,3 @@ def get_config( max_memory=grafana_params.max_mem, node_selectors=node_selectors, ) - - -def new_datasource_config_template_data(prometheus_url): - return { - "PrometheusUID": DATASOURCE_UID, - "PrometheusURL": prometheus_url - } - - -def new_dashboard_providers_config_template_data(dashboards_dirpath): - return {"DashboardsDirpath": dashboards_dirpath} - - -def upload_additional_dashboards(plan, additional_dashboards): - data = [] - for index, dashboard_src in enumerate(additional_dashboards): - additional_dashboard_folder_name = ( - GRAFANA_ADDITIONAL_DASHBOARDS_FOLDER_NAME.format(index) - ) - additional_dashboard_service_path = ( - GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE_FORMAT.format( - additional_dashboard_folder_name, - ) - ) - additional_dashboard_artifact_name = plan.upload_files( - dashboard_src, name="additional-grafana-dashboard-{0}".format(index) - ) - data.append( - { - GRAFANA_ADDITIONAL_DASHBOARDS_SERVICE_PATH_KEY: additional_dashboard_service_path, - GRANAFA_ADDITIONAL_DASHBOARDS_ARTIFACT_NAME_KEY: additional_dashboard_artifact_name, - } - ) - return data - - -def merge_dashboards_artifacts( - plan, - grafana_dashboards_artifact_name, - grafana_additional_dashboards_data=[], -): - if len(grafana_additional_dashboards_data) == 0: - return grafana_dashboards_artifact_name - - files = { - GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE: grafana_dashboards_artifact_name, - } - - for additional_dashboard_data in grafana_additional_dashboards_data: - files[ - additional_dashboard_data[GRAFANA_ADDITIONAL_DASHBOARDS_SERVICE_PATH_KEY] - ] = additional_dashboard_data[GRANAFA_ADDITIONAL_DASHBOARDS_ARTIFACT_NAME_KEY] - - result = plan.run_sh( - description="Merging grafana dashboards artifacts", - run="find " - + GRAFANA_ADDITIONAL_DASHBOARDS_FILEPATH_ON_SERVICE - + " -type f -exec cp {} " - + GRAFANA_DASHBOARDS_DIRPATH_ON_SERVICE - + " \\;", - files=files, - store=[ - GRAFANA_ADDITIONAL_DASHBOARDS_MERGED_STORED_PATH_FORMAT, - ], - ) - - return result.files_artifacts[0] diff --git a/static_files/grafana-config/templates/dashboard-providers.yml.tmpl b/src/observability/grafana/templates/dashboard-providers.yml.tmpl similarity index 100% rename from static_files/grafana-config/templates/dashboard-providers.yml.tmpl rename to src/observability/grafana/templates/dashboard-providers.yml.tmpl diff --git a/static_files/grafana-config/templates/datasource.yml.tmpl b/src/observability/grafana/templates/datasource.yml.tmpl similarity index 100% rename from static_files/grafana-config/templates/datasource.yml.tmpl rename to src/observability/grafana/templates/datasource.yml.tmpl diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index a63b8430..7838b843 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -79,9 +79,9 @@ def input_parser(plan, input_args): ), grafana_params=struct( image=results["observability"]["grafana_params"]["image"], - additional_dashboards=results["observability"][ + dashboard_sources=results["observability"][ "grafana_params" - ]["additional_dashboards"], + ]["dashboard_sources"], min_cpu=results["observability"]["grafana_params"]["min_cpu"], max_cpu=results["observability"]["grafana_params"]["max_cpu"], min_mem=results["observability"]["grafana_params"]["min_mem"], @@ -376,7 +376,7 @@ def default_prometheus_params(): def default_grafana_params(): return { "image": "grafana/grafana:latest", - "additional_dashboards": [], + "dashboard_sources": [], "min_cpu": 10, "max_cpu": 1000, "min_mem": 128, diff --git a/src/package_io/sanity_check.star b/src/package_io/sanity_check.star index af73d9ea..5d4835f6 100644 --- a/src/package_io/sanity_check.star +++ b/src/package_io/sanity_check.star @@ -16,7 +16,7 @@ PROMETHEUS_PARAMS = [ GRAFANA_PARAMS = [ "image", - "additional_dashboards", + "dashboard_sources", "min_cpu", "max_cpu", "min_mem", From 9142d26597c91665b1fa3267d9b951d5d4947737 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Thu, 16 Jan 2025 16:44:52 -0500 Subject: [PATCH 09/16] switch to API grafana provisioning --- .../grafana/grafana_launcher.star | 76 +++++++++++-------- .../templates/dashboard-providers.yml.tmpl | 25 ------ src/util.star | 3 + 3 files changed, 46 insertions(+), 58 deletions(-) delete mode 100644 src/observability/grafana/templates/dashboard-providers.yml.tmpl diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star index 3ac5621e..2d5c540a 100644 --- a/src/observability/grafana/grafana_launcher.star +++ b/src/observability/grafana/grafana_launcher.star @@ -1,3 +1,5 @@ +util = import_module("../../util.star") + ethereum_package_shared_utils = import_module( "github.com/ethpandaops/ethereum-package/src/shared_utils/shared_utils.star" ) @@ -37,7 +39,6 @@ def launch_grafana( grafana_params, ): datasource_config_template = read_file(DATASOURCE_CONFIG_TEMPLATE_FILEPATH) - dashboard_providers_config_template = read_file(DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH) ( grafana_config_artifact_name, @@ -45,7 +46,6 @@ def launch_grafana( ) = upload_grafana_config( plan, datasource_config_template, - dashboard_providers_config_template, prometheus_private_url, dashboard_sources=grafana_params.dashboard_sources, ) @@ -57,13 +57,20 @@ def launch_grafana( grafana_params, ) - plan.add_service(SERVICE_NAME, config) + service = plan.add_service(SERVICE_NAME, config) + + service_url = "http://{0}:{1}".format( + service.ip_address, service.ports[HTTP_PORT_ID].number + ) + + upload_dashboards(plan, service_url, grafana_dashboards_artifact_names) + + return service_url def upload_grafana_config( plan, datasource_config_template, - dashboard_providers_config_template, prometheus_private_url, dashboard_sources=[], ): @@ -72,16 +79,8 @@ def upload_grafana_config( datasource_config_template, datasource_data ) - dashboard_providers_data = new_dashboard_providers_config_template_data( - DASHBOARDS_DIRPATH_ON_SERVICE - ) - dashboard_providers_template_and_data = ethereum_package_shared_utils.new_template_and_data( - dashboard_providers_config_template, dashboard_providers_data - ) - template_and_data_by_rel_dest_filepath = { DATASOURCE_CONFIG_REL_FILEPATH: datasource_template_and_data, - DASHBOARD_PROVIDERS_CONFIG_REL_FILEPATH: dashboard_providers_template_and_data, } grafana_config_artifact_name = plan.render_templates( @@ -105,24 +104,6 @@ def new_datasource_config_template_data(prometheus_url): } -def new_dashboard_providers_config_template_data(dashboards_dirpath): - return {"DashboardsDirpath": dashboards_dirpath} - - -def upload_dashboards(plan, dashboard_sources): - dashboard_artifact_names = [] - - for index, dashboard_src in enumerate(dashboard_sources): - dashboard_name = "grafana-dashboards-{0}".format(index) - - dashboard_artifact_name = plan.upload_files( - dashboard_src, name=dashboard_name - ) - dashboard_artifact_names.append(dashboard_artifact_name) - - return dashboard_artifact_names - - def get_config( grafana_config_artifact_name, grafana_dashboards_artifact_names, @@ -141,9 +122,6 @@ def get_config( }, files={ CONFIG_DIRPATH_ON_SERVICE: grafana_config_artifact_name, - DASHBOARDS_DIRPATH_ON_SERVICE: Directory( - artifact_names=grafana_dashboards_artifact_names, - ), }, min_cpu=grafana_params.min_cpu, max_cpu=grafana_params.max_cpu, @@ -151,3 +129,35 @@ def get_config( max_memory=grafana_params.max_mem, node_selectors=node_selectors, ) + + +def upload_dashboards(plan, service_url, grafana_dashboards_artifact_names): + def grr_push(dir): + return "grr push \"$DASHBOARDS_DIR/{0}\" -e --disable-reporting".format(dir) + + def grr_push_dashboards(i): + return [ + grr_push("dashboards-{0}/folders".format(i)), + grr_push("dashboards-{0}/dashboards".format(i)), + ] + + grr_commands = [ + "grr config create-context kurtosis", + ] + + for i in range(len(grafana_dashboards_artifact_names)): + grr_commands += grr_push_dashboards(i) + + plan.run_sh( + description="upload dashboards", + image="grafana/grizzly:main-0b88d01", + env_vars={ + "GRAFANA_URL": service_url, + "DASHBOARDS_DIR": DASHBOARDS_DIRPATH_ON_SERVICE, + }, + files={ + "{0}/dashboards-{1}".format(DASHBOARDS_DIRPATH_ON_SERVICE, i): grafana_dashboards_artifact_names[i] + for i in range(len(grafana_dashboards_artifact_names)) + }, + run=util.multiline_cmd(grr_commands), + ) diff --git a/src/observability/grafana/templates/dashboard-providers.yml.tmpl b/src/observability/grafana/templates/dashboard-providers.yml.tmpl deleted file mode 100644 index 451ae946..00000000 --- a/src/observability/grafana/templates/dashboard-providers.yml.tmpl +++ /dev/null @@ -1,25 +0,0 @@ -apiVersion: 1 - -providers: - # an unique provider name. Required - - name: 'Optimism Kurtosis Module Dashboard Provider' - # Org id. Default to 1 - orgId: 1 - # name of the dashboard folder. - folder: '' - # folder UID. will be automatically generated if not specified - folderUid: '' - # provider type. Default to 'file' - type: file - # disable dashboard deletion - disableDeletion: false - # how often Grafana will scan for changed dashboards - updateIntervalSeconds: 10 - # allow updating provisioned dashboards from the UI - allowUiUpdates: true - editable: true - options: - # path to dashboard files on disk. Required when using the 'file' type - path: {{ .DashboardsDirpath }} - # use folder names from filesystem to create folders in Grafana - foldersFromFilesStructure: true diff --git a/src/util.star b/src/util.star index 96415f98..338cd435 100644 --- a/src/util.star +++ b/src/util.star @@ -66,3 +66,6 @@ def label_from_image(image): if len(label) > max_length: label = label[-max_length:] return label + +def multiline_cmd(commands): + return " && ".join(commands) From 36f7a3a302fc42a5cad848cfb4f6493a481a4b19 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Thu, 16 Jan 2025 17:10:32 -0500 Subject: [PATCH 10/16] lint --- main.star | 2 +- .../grafana/grafana_launcher.star | 19 +++++++++---------- src/observability/observability.star | 1 - src/package_io/input_parser.star | 8 +++++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/main.star b/main.star index 4c260d39..efb18835 100644 --- a/main.star +++ b/main.star @@ -142,7 +142,6 @@ def run(plan, args): plan.print("Launching grafana...") - grafana.launch_grafana( plan, prometheus_private_url, @@ -150,6 +149,7 @@ def run(plan, args): observability_params.grafana_params, ) + def get_l1_config(all_l1_participants, l1_network_params, l1_network_id): env_vars = {} env_vars["L1_RPC_KIND"] = "standard" diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star index 2d5c540a..10f03315 100644 --- a/src/observability/grafana/grafana_launcher.star +++ b/src/observability/grafana/grafana_launcher.star @@ -12,7 +12,9 @@ HTTP_PORT_NUMBER_UINT16 = 3000 TEMPLATES_FILEPATH = "./templates" DATASOURCE_CONFIG_TEMPLATE_FILEPATH = TEMPLATES_FILEPATH + "/datasource.yml.tmpl" -DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH = TEMPLATES_FILEPATH + "/dashboard-providers.yml.tmpl" +DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH = ( + TEMPLATES_FILEPATH + "/dashboard-providers.yml.tmpl" +) DATASOURCE_UID = "grafanacloud-prom" DATASOURCE_CONFIG_REL_FILEPATH = "datasources/datasource.yml" @@ -87,9 +89,7 @@ def upload_grafana_config( template_and_data_by_rel_dest_filepath, name="grafana-config" ) - grafana_dashboards_artifact_names = upload_dashboards( - plan, dashboard_sources - ) + grafana_dashboards_artifact_names = upload_dashboards(plan, dashboard_sources) return ( grafana_config_artifact_name, @@ -98,10 +98,7 @@ def upload_grafana_config( def new_datasource_config_template_data(prometheus_url): - return { - "PrometheusUID": DATASOURCE_UID, - "PrometheusURL": prometheus_url - } + return {"PrometheusUID": DATASOURCE_UID, "PrometheusURL": prometheus_url} def get_config( @@ -133,7 +130,7 @@ def get_config( def upload_dashboards(plan, service_url, grafana_dashboards_artifact_names): def grr_push(dir): - return "grr push \"$DASHBOARDS_DIR/{0}\" -e --disable-reporting".format(dir) + return 'grr push "$DASHBOARDS_DIR/{0}" -e --disable-reporting'.format(dir) def grr_push_dashboards(i): return [ @@ -156,7 +153,9 @@ def upload_dashboards(plan, service_url, grafana_dashboards_artifact_names): "DASHBOARDS_DIR": DASHBOARDS_DIRPATH_ON_SERVICE, }, files={ - "{0}/dashboards-{1}".format(DASHBOARDS_DIRPATH_ON_SERVICE, i): grafana_dashboards_artifact_names[i] + "{0}/dashboards-{1}".format( + DASHBOARDS_DIRPATH_ON_SERVICE, i + ): grafana_dashboards_artifact_names[i] for i in range(len(grafana_dashboards_artifact_names)) }, run=util.multiline_cmd(grr_commands), diff --git a/src/observability/observability.star b/src/observability/observability.star index 78f8d6a7..b7e6a45c 100644 --- a/src/observability/observability.star +++ b/src/observability/observability.star @@ -70,7 +70,6 @@ def new_metrics_job( labels, scrape_interval=DEFAULT_SCRAPE_INTERVAL, ): - return { "Name": job_name, "Endpoint": endpoint, diff --git a/src/package_io/input_parser.star b/src/package_io/input_parser.star index 7838b843..e0e83236 100644 --- a/src/package_io/input_parser.star +++ b/src/package_io/input_parser.star @@ -79,9 +79,9 @@ def input_parser(plan, input_args): ), grafana_params=struct( image=results["observability"]["grafana_params"]["image"], - dashboard_sources=results["observability"][ - "grafana_params" - ]["dashboard_sources"], + dashboard_sources=results["observability"]["grafana_params"][ + "dashboard_sources" + ], min_cpu=results["observability"]["grafana_params"]["min_cpu"], max_cpu=results["observability"]["grafana_params"]["max_cpu"], min_mem=results["observability"]["grafana_params"]["min_mem"], @@ -373,6 +373,7 @@ def default_prometheus_params(): "max_mem": 2048, } + def default_grafana_params(): return { "image": "grafana/grafana:latest", @@ -383,6 +384,7 @@ def default_grafana_params(): "max_mem": 2048, } + def default_interop_params(): return { "enabled": False, From 52967308c24d52c21ec5d9dc5c137a12331fda53 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Thu, 16 Jan 2025 17:22:18 -0500 Subject: [PATCH 11/16] refactor provision_dashboards --- .../grafana/grafana_launcher.star | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star index 10f03315..91cf6217 100644 --- a/src/observability/grafana/grafana_launcher.star +++ b/src/observability/grafana/grafana_launcher.star @@ -42,19 +42,14 @@ def launch_grafana( ): datasource_config_template = read_file(DATASOURCE_CONFIG_TEMPLATE_FILEPATH) - ( - grafana_config_artifact_name, - grafana_dashboards_artifact_names, - ) = upload_grafana_config( + grafana_config_artifact_name = upload_grafana_config( plan, datasource_config_template, prometheus_private_url, - dashboard_sources=grafana_params.dashboard_sources, ) config = get_config( grafana_config_artifact_name, - grafana_dashboards_artifact_names, global_node_selectors, grafana_params, ) @@ -65,7 +60,7 @@ def launch_grafana( service.ip_address, service.ports[HTTP_PORT_ID].number ) - upload_dashboards(plan, service_url, grafana_dashboards_artifact_names) + provision_dashboards(plan, service_url, grafana_params.dashboard_sources) return service_url @@ -74,7 +69,6 @@ def upload_grafana_config( plan, datasource_config_template, prometheus_private_url, - dashboard_sources=[], ): datasource_data = new_datasource_config_template_data(prometheus_private_url) datasource_template_and_data = ethereum_package_shared_utils.new_template_and_data( @@ -89,12 +83,7 @@ def upload_grafana_config( template_and_data_by_rel_dest_filepath, name="grafana-config" ) - grafana_dashboards_artifact_names = upload_dashboards(plan, dashboard_sources) - - return ( - grafana_config_artifact_name, - grafana_dashboards_artifact_names, - ) + return grafana_config_artifact_name def new_datasource_config_template_data(prometheus_url): @@ -103,7 +92,6 @@ def new_datasource_config_template_data(prometheus_url): def get_config( grafana_config_artifact_name, - grafana_dashboards_artifact_names, node_selectors, grafana_params, ): @@ -128,7 +116,7 @@ def get_config( ) -def upload_dashboards(plan, service_url, grafana_dashboards_artifact_names): +def provision_dashboards(plan, service_url, dashboard_sources): def grr_push(dir): return 'grr push "$DASHBOARDS_DIR/{0}" -e --disable-reporting'.format(dir) @@ -142,8 +130,14 @@ def upload_dashboards(plan, service_url, grafana_dashboards_artifact_names): "grr config create-context kurtosis", ] - for i in range(len(grafana_dashboards_artifact_names)): - grr_commands += grr_push_dashboards(i) + dashboard_artifact_names = [] + for index, dashboard_src in enumerate(dashboard_sources): + dashboard_name = "grafana-dashboards-{0}".format(index) + + dashboard_artifact_name = plan.upload_files(dashboard_src, name=dashboard_name) + dashboard_artifact_names.append(dashboard_artifact_name) + + grr_commands += grr_push_dashboards(index) plan.run_sh( description="upload dashboards", @@ -155,8 +149,8 @@ def upload_dashboards(plan, service_url, grafana_dashboards_artifact_names): files={ "{0}/dashboards-{1}".format( DASHBOARDS_DIRPATH_ON_SERVICE, i - ): grafana_dashboards_artifact_names[i] - for i in range(len(grafana_dashboards_artifact_names)) + ): dashboard_artifact + for i, dashboard_artifact in enumerate(dashboard_artifact_names) }, run=util.multiline_cmd(grr_commands), ) From 476d066809b690c9dd639257970fb079039cc03c Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Thu, 16 Jan 2025 17:53:26 -0500 Subject: [PATCH 12/16] rename to join_cmds --- src/observability/grafana/grafana_launcher.star | 2 +- src/util.star | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star index 91cf6217..b5c0589c 100644 --- a/src/observability/grafana/grafana_launcher.star +++ b/src/observability/grafana/grafana_launcher.star @@ -152,5 +152,5 @@ def provision_dashboards(plan, service_url, dashboard_sources): ): dashboard_artifact for i, dashboard_artifact in enumerate(dashboard_artifact_names) }, - run=util.multiline_cmd(grr_commands), + run=util.join_cmds(grr_commands), ) diff --git a/src/util.star b/src/util.star index 338cd435..49e9cfb9 100644 --- a/src/util.star +++ b/src/util.star @@ -67,5 +67,5 @@ def label_from_image(image): label = label[-max_length:] return label -def multiline_cmd(commands): +def join_cmds(commands): return " && ".join(commands) From 116927ba0117197eac07a029d7def10a1c09f2d2 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Thu, 16 Jan 2025 17:55:52 -0500 Subject: [PATCH 13/16] construct files in loop --- src/observability/grafana/grafana_launcher.star | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star index b5c0589c..e77fc962 100644 --- a/src/observability/grafana/grafana_launcher.star +++ b/src/observability/grafana/grafana_launcher.star @@ -130,12 +130,14 @@ def provision_dashboards(plan, service_url, dashboard_sources): "grr config create-context kurtosis", ] - dashboard_artifact_names = [] + files = {} for index, dashboard_src in enumerate(dashboard_sources): - dashboard_name = "grafana-dashboards-{0}".format(index) - + dashboard_name = "dashboards-{0}".format(index) dashboard_artifact_name = plan.upload_files(dashboard_src, name=dashboard_name) - dashboard_artifact_names.append(dashboard_artifact_name) + + files[ + "{0}/{1}".format(DASHBOARDS_DIRPATH_ON_SERVICE, dashboard_name) + ] = dashboard_artifact_name grr_commands += grr_push_dashboards(index) @@ -146,11 +148,6 @@ def provision_dashboards(plan, service_url, dashboard_sources): "GRAFANA_URL": service_url, "DASHBOARDS_DIR": DASHBOARDS_DIRPATH_ON_SERVICE, }, - files={ - "{0}/dashboards-{1}".format( - DASHBOARDS_DIRPATH_ON_SERVICE, i - ): dashboard_artifact - for i, dashboard_artifact in enumerate(dashboard_artifact_names) - }, + files=files, run=util.join_cmds(grr_commands), ) From 1148cf3432bc033139c48d9d7d5706c101ab2f12 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Thu, 16 Jan 2025 18:04:58 -0500 Subject: [PATCH 14/16] don't provision dashboards if none have been provided --- src/observability/grafana/grafana_launcher.star | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star index e77fc962..a9465b07 100644 --- a/src/observability/grafana/grafana_launcher.star +++ b/src/observability/grafana/grafana_launcher.star @@ -117,6 +117,9 @@ def get_config( def provision_dashboards(plan, service_url, dashboard_sources): + if len(dashboard_sources) == 0: + return + def grr_push(dir): return 'grr push "$DASHBOARDS_DIR/{0}" -e --disable-reporting'.format(dir) From 9550698af629c8cc242cc701b40cec37e66711d0 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Thu, 16 Jan 2025 18:12:18 -0500 Subject: [PATCH 15/16] cleanup & simplify --- main.star | 1 - .../grafana/grafana_launcher.star | 26 +++++-------------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/main.star b/main.star index efb18835..0b241a0d 100644 --- a/main.star +++ b/main.star @@ -141,7 +141,6 @@ def run(plan, args): ) plan.print("Launching grafana...") - grafana.launch_grafana( plan, prometheus_private_url, diff --git a/src/observability/grafana/grafana_launcher.star b/src/observability/grafana/grafana_launcher.star index a9465b07..73f495b5 100644 --- a/src/observability/grafana/grafana_launcher.star +++ b/src/observability/grafana/grafana_launcher.star @@ -11,19 +11,11 @@ HTTP_PORT_NUMBER_UINT16 = 3000 TEMPLATES_FILEPATH = "./templates" -DATASOURCE_CONFIG_TEMPLATE_FILEPATH = TEMPLATES_FILEPATH + "/datasource.yml.tmpl" -DASHBOARD_PROVIDERS_CONFIG_TEMPLATE_FILEPATH = ( - TEMPLATES_FILEPATH + "/dashboard-providers.yml.tmpl" -) - DATASOURCE_UID = "grafanacloud-prom" +DATASOURCE_CONFIG_TEMPLATE_FILEPATH = TEMPLATES_FILEPATH + "/datasource.yml.tmpl" DATASOURCE_CONFIG_REL_FILEPATH = "datasources/datasource.yml" -# this is relative to the files artifact root -DASHBOARD_PROVIDERS_CONFIG_REL_FILEPATH = "dashboards/dashboard-providers.yml" - CONFIG_DIRPATH_ON_SERVICE = "/config" -DASHBOARDS_DIRPATH_ON_SERVICE = "/dashboards" USED_PORTS = { HTTP_PORT_ID: ethereum_package_shared_utils.new_port_spec( @@ -121,12 +113,12 @@ def provision_dashboards(plan, service_url, dashboard_sources): return def grr_push(dir): - return 'grr push "$DASHBOARDS_DIR/{0}" -e --disable-reporting'.format(dir) + return 'grr push "{0}" -e --disable-reporting'.format(dir) - def grr_push_dashboards(i): + def grr_push_dashboards(name): return [ - grr_push("dashboards-{0}/folders".format(i)), - grr_push("dashboards-{0}/dashboards".format(i)), + grr_push("{0}/folders".format(name)), + grr_push("{0}/dashboards".format(name)), ] grr_commands = [ @@ -138,18 +130,14 @@ def provision_dashboards(plan, service_url, dashboard_sources): dashboard_name = "dashboards-{0}".format(index) dashboard_artifact_name = plan.upload_files(dashboard_src, name=dashboard_name) - files[ - "{0}/{1}".format(DASHBOARDS_DIRPATH_ON_SERVICE, dashboard_name) - ] = dashboard_artifact_name - - grr_commands += grr_push_dashboards(index) + files[dashboard_name] = dashboard_artifact_name + grr_commands += grr_push_dashboards(dashboard_name) plan.run_sh( description="upload dashboards", image="grafana/grizzly:main-0b88d01", env_vars={ "GRAFANA_URL": service_url, - "DASHBOARDS_DIR": DASHBOARDS_DIRPATH_ON_SERVICE, }, files=files, run=util.join_cmds(grr_commands), From 29c2164f5bc0969ebfecb114f4c0f31993550dc9 Mon Sep 17 00:00:00 2001 From: Eugene Dobry Date: Mon, 27 Jan 2025 16:24:59 -0500 Subject: [PATCH 16/16] lint --- src/util.star | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util.star b/src/util.star index 49e9cfb9..e72e248c 100644 --- a/src/util.star +++ b/src/util.star @@ -67,5 +67,6 @@ def label_from_image(image): label = label[-max_length:] return label + def join_cmds(commands): return " && ".join(commands)