loki_out: add stuctured_metadata_map_keys

Add docker-compose to test loki backend

Signed-off-by: Greg Eales <[email protected]>

docker_compose/loki-grafana-structured_metadata_map/README.md

@@ -0,0 +1,12 @@
+### Description
+
+This directory has a docker-compose file and its
+configuration required to run:
+
+1) A fluentbit installation with a dummy input, and Loki output configured for `structured_metadata_map_keys`
+3) A Loki installation 
+4) A grafana installation with a default Loki datasource
+
+To run this, execute:
+
+$ docker-compose up --force-recreate -d

docker_compose/loki-grafana-structured_metadata_map/config/fluent-bit_loki_out-structured_metadata_map.yaml

@@ -0,0 +1,36 @@
+service:
+  log_level: debug
+
+pipeline:
+  inputs:
+    - name: dummy
+      tag: logs
+      dummy: |
+        {
+          "message": "simple log generated",
+          "logger": "my.logger",
+          "level": "INFO",
+          "hostname": "localhost",
+          "my_map_of_attributes_1": {
+            "key_1": "hello, world!",
+            "key_2": "goodbye, world!"
+          },
+          "my_map_of_maps_1": {
+            "root_key": {
+              "sub_key_1": "hello, world!",
+              "sub_key_2": "goodbye, world!"
+            }
+          }
+        }
+
+  outputs:
+    - name: loki
+      match: logs
+      host: loki
+      remove_keys: hostname,my_map_of_attributes_1,my_map_of_maps_1
+      label_keys: $level,$logger
+      labels: service_name=test
+      structured_metadata: $hostname
+      structured_metadata_map_keys: $my_map_of_attributes_1,$my_map_of_maps_1['root_key']
+      line_format: key_value
+      drop_single_key: on

docker_compose/loki-grafana-structured_metadata_map/config/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,20 @@
+# config file version
+apiVersion: 1
+
+# list of datasources that should be deleted from the database
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+# list of datasources to insert/update depending
+# whats available in the database
+datasources:
+  - name: Loki
+    type: loki
+    access: proxy
+    orgId: 1
+    url: http://loki:3100
+    basicAuth: false
+    isDefault: true
+    version: 1
+    editable: false

docker_compose/loki-grafana-structured_metadata_map/config/loki-config.yaml

@@ -0,0 +1,53 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false
+limits_config:
+  allow_structured_metadata: true
+  volume_enabled: true

docker_compose/loki-grafana-structured_metadata_map/docker-compose.yml

@@ -0,0 +1,41 @@
+services:
+  fluentbit:
+    image: fluent/fluent-bit:latest
+    depends_on:
+      - loki
+    container_name: fluentbit
+    command: /fluent-bit/bin/fluent-bit -c /etc/fluent-bit_loki_out-structured_metadata_map.yaml
+    ports:
+      - 2021:2021
+    networks:
+      - loki-network
+    volumes:
+      - ./config/fluent-bit_loki_out-structured_metadata_map.yaml:/etc/fluent-bit_loki_out-structured_metadata_map.yaml
+
+  grafana:
+    image: grafana/grafana:11.4.0
+    depends_on:
+      - loki
+      - fluentbit
+    ports:
+      - 3000:3000
+    volumes:
+      - ./config/grafana/provisioning:/etc/grafana/provisioning
+    networks:
+      - loki-network
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+
+  loki:
+    image: grafana/loki:2.9.2
+    command: -config.file=/etc/loki/loki-config.yaml
+    networks:
+      - loki-network
+    ports:
+      - 3100:3100
+    volumes:
+      - ./config/loki-config.yaml:/etc/loki/loki-config.yaml
+
+networks:
+  loki-network:
+    driver: bridge