OTLP support for metrics

[idrissneumann]

Is your feature request related to a problem? Please describe.

As of today, Quickwit is only able to ingest traces and logs through the OTLP/grpc protocol.

Describe the solution you'd like

I think, until a real TSDB is implemented, it would be a nice quick win to have the metrics routed to a otel-metrics-v0_8 index with this mapping for example:

{
  "doc_mapping": {
    "mode": "dynamic",
    "field_mappings": [
      {
        "name": "timestamp",
        "type": "datetime",
        "fast": true,
        "fast_precision": "seconds",
        "indexed": true,
        "input_formats": [
          "rfc3339",
          "unix_timestamp"
        ],
        "output_format": "unix_timestamp_nanos",
        "stored": true
      },
      {
        "indexed": true,
        "fast": true,
        "name": "name",
        "type": "text",
        "tokenizer": "raw"
      },
      {
        "indexed": true,
        "fast": true,
        "name": "kind",
        "type": "text",
        "tokenizer": "raw"
      },
      {
        "name": "tags",
        "type": "json",
        "fast": true,
        "indexed": true,
        "record": "basic",
        "stored": true,
        "tokenizer": "default"
      },
      {
        "name": "gauge",
        "type": "object",
        "field_mappings": [
          {
            "name": "value",
            "fast": true,
            "indexed": true,
            "type": "f64"
          }
        ]
      },
      {
        "name": "counter",
        "type": "object",
        "field_mappings": [
          {
            "name": "value",
            "fast": true,
            "indexed": true,
            "type": "f64"
          }
        ]
      },
      {
        "name": "aggregated_summary",
        "type": "object",
        "field_mappings": [
          {
            "name": "sum",
            "fast": true,
            "indexed": true,
            "type": "f64"
          },
          {
            "name": "count",
            "fast": true,
            "indexed": true,
            "type": "u64"
          }
        ]
      },
      {
        "name": "aggregated_histogram",
        "type": "object",
        "field_mappings": [
          {
            "name": "sum",
            "fast": true,
            "indexed": true,
            "type": "f64"
          },
          {
            "name": "count",
            "fast": true,
            "indexed": true,
            "type": "u64"
          }
        ]
      }
    ],
    "timestamp_field": "timestamp",
    "max_num_partitions": 200,
    "index_field_presence": true,
    "store_source": false,
    "tokenizers": []
  },
  "index_id": "otel-metrics-v0_8",
  "search_settings": {
    "default_search_fields": [
      "name",
      "kind"
    ]
  },
  "version": "0.8"
}

Describe alternatives you've considered

For now we're scraping the prometheus endpoints using this vector configuration:

And it's working pretty well :)

Additional context

Here's the vector configuration used:

sources:
  prom_app_1:
    type: "prometheus_scrape"
    endpoints:
      - "https://cloud-api.comwork.io/v1/metrics"

transforms:
  remap_prom_app_1:
    inputs: ["prom_app_1"]
    type: "remap"
    source: |
      if is_null(.tags) {
        .tags = {}
      }

      .tags.source = "prom_app_1"

sinks:
  quickwit_app_1:
    type: "http"
    method: "post"
    inputs: ["remap_prom_app_1"]
    encoding:
      codec: "json"
    framing:
      method: "newline_delimited"
    uri: "http://localhost:7280/api/v1/otel-metrics-v0_8/ingest"