[O11y][MSSQL] Add mssql.query field to transaction_logs data stream for debug purposes

packages/microsoft_sqlserver/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.10.1"
+  changes:
+    - description: Add optional mssql.query field to `transaction_logs` data stream for debugging purposes.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12216
 - version: "2.10.0"
   changes:
     - description: Add optional mssql.query field to `performance` data stream for debugging purposes.

packages/microsoft_sqlserver/data_stream/transaction_log/_dev/test/system/test-transaction-config.yml

@@ -3,3 +3,6 @@ vars:
     - "{{Hostname}}"
   username: sa
   password: 1234_asdf
+data_stream:
+  vars:
+    preserve_sql_queries: true

packages/microsoft_sqlserver/data_stream/transaction_log/agent/stream/stream.yml.hbs

@@ -32,3 +32,7 @@ sql_queries:
 processors:
 {{processors}}
 {{/if}}
+tags:
+{{#if preserve_sql_queries}}
+- preserve_sql_queries
+{{/if}}

packages/microsoft_sqlserver/data_stream/transaction_log/elasticsearch/ingest_pipeline/default.yml

@@ -42,6 +42,7 @@ processors:
     ignore_missing: true
 - remove:
     field: mssql.query
+    if: "ctx?.tags == null || !(ctx.tags.contains('preserve_sql_queries'))"
     ignore_missing: true
     ignore_failure: true
 - script:

packages/microsoft_sqlserver/data_stream/transaction_log/fields/fields.yml

@@ -75,3 +75,6 @@
           unit: byte
           metric_type: counter
           description: Total transaction log size in bytes.
+    - name: query
+      type: keyword
+      description: The SQL queries executed.

packages/microsoft_sqlserver/data_stream/transaction_log/manifest.yml

@@ -26,6 +26,14 @@ streams:
         show_user: false
         description: Option to enable fetching transaction_logs metrics from all databases, including both system and user-defined databases. This option overrides any database names provided in the 'Databases' field and instead considers all databases.
         default: false
+      - name: preserve_sql_queries
+        required: true
+        show_user: false
+        title: Preserve SQL Queries
+        description: Preserves SQL queries for debugging purposes.
+        type: bool
+        multi: false
+        default: false
       - name: processors
         type: yaml
         title: Processors

packages/microsoft_sqlserver/data_stream/transaction_log/sample_event.json

@@ -1,52 +1,54 @@
 {
-    "@timestamp": "2022-12-20T07:34:29.687Z",
+    "@timestamp": "2025-01-01T06:34:46.685Z",
     "agent": {
-        "ephemeral_id": "8d528ff8-5e90-4572-89f6-61fb3a6c96f1",
-        "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00",
-        "name": "192.168.1.2",
+        "ephemeral_id": "a4c1f457-cd5c-4b62-aa68-1f5d32acb43b",
+        "id": "7a86be58-ca0e-433a-bd57-44f2cb3ee3a1",
+        "name": "elastic-agent-78750",
         "type": "metricbeat",
-        "version": "8.5.3"
+        "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "microsoft_sqlserver.transaction_log",
-        "namespace": "default",
+        "namespace": "69278",
         "type": "metrics"
     },
     "ecs": {
-        "version": "8.11.0"
+        "version": "8.0.0"
     },
     "elastic_agent": {
-        "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00",
+        "id": "7a86be58-ca0e-433a-bd57-44f2cb3ee3a1",
         "snapshot": false,
-        "version": "8.5.3"
+        "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "microsoft_sqlserver.transaction_log",
-        "duration": 2147044750,
-        "ingested": "2022-12-20T07:34:32Z",
+        "duration": 2065982241,
+        "ingested": "2025-01-01T06:34:49Z",
         "module": "sql"
     },
     "host": {
         "architecture": "x86_64",
-        "containerized": false,
-        "hostname": "192.168.1.2",
-        "id": "627E8AE5-E918-5073-A58E-8A2D9ED96875",
+        "containerized": true,
+        "hostname": "elastic-agent-78750",
+        "id": "8259e024976a406e8a54cdbffeb84fec",
         "ip": [
-            "192.168.1.2"
+            "192.168.241.5",
+            "192.168.253.2"
         ],
         "mac": [
-            "36-F7-DC-28-23-80"
+            "02-42-C0-A8-F1-05",
+            "02-42-C0-A8-FD-02"
         ],
-        "name": "192.168.1.2",
+        "name": "elastic-agent-78750",
         "os": {
-            "build": "21D62",
-            "family": "darwin",
-            "kernel": "21.3.0",
-            "name": "macOS",
-            "platform": "darwin",
-            "type": "macos",
-            "version": "12.2.1"
+            "codename": "focal",
+            "family": "debian",
+            "kernel": "3.10.0-1160.92.1.el7.x86_64",
+            "name": "Ubuntu",
+            "platform": "ubuntu",
+            "type": "linux",
+            "version": "20.04.6 LTS (Focal Fossa)"
         }
     },
     "metricset": {
@@ -55,18 +57,19 @@
     },
     "mssql": {
         "metrics": {
-            "database_id": 1,
-            "database_name": "master",
+            "database_id": 4,
+            "database_name": "msdb",
             "instance_name": "MSSQLSERVER",
-            "log_space_in_bytes_since_last_backup": 602112,
-            "server_name": "obs-int-mssql20",
-            "total_log_size_bytes": 2088960,
-            "used_log_space_bytes": 1024000,
-            "used_log_space_pct": 49.01960754394531
-        }
+            "query_id": "JZwfIFXvA3yOOL15q7PFZK5UE+Y=",
+            "server_name": "7e672355554d"
+        },
+        "query": "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases WHERE name='msdb';"
     },
     "service": {
-        "address": "20.228.135.242",
+        "address": "svc-microsoft_sqlserver",
         "type": "sql"
-    }
-}
+    },
+    "tags": [
+        "preserve_sql_queries"
+    ]
+}

packages/microsoft_sqlserver/docs/README.md

@@ -506,54 +506,56 @@ An example event for `transaction_log` looks as following:
 
 ```json
 {
-    "@timestamp": "2022-12-20T07:34:29.687Z",
+    "@timestamp": "2025-01-01T06:34:46.685Z",
     "agent": {
-        "ephemeral_id": "8d528ff8-5e90-4572-89f6-61fb3a6c96f1",
-        "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00",
-        "name": "192.168.1.2",
+        "ephemeral_id": "a4c1f457-cd5c-4b62-aa68-1f5d32acb43b",
+        "id": "7a86be58-ca0e-433a-bd57-44f2cb3ee3a1",
+        "name": "elastic-agent-78750",
         "type": "metricbeat",
-        "version": "8.5.3"
+        "version": "8.13.0"
     },
     "data_stream": {
         "dataset": "microsoft_sqlserver.transaction_log",
-        "namespace": "default",
+        "namespace": "69278",
         "type": "metrics"
     },
     "ecs": {
-        "version": "8.11.0"
+        "version": "8.0.0"
     },
     "elastic_agent": {
-        "id": "d44a1c4a-95bf-47e9-afb0-453a2ef43c00",
+        "id": "7a86be58-ca0e-433a-bd57-44f2cb3ee3a1",
         "snapshot": false,
-        "version": "8.5.3"
+        "version": "8.13.0"
     },
     "event": {
         "agent_id_status": "verified",
         "dataset": "microsoft_sqlserver.transaction_log",
-        "duration": 2147044750,
-        "ingested": "2022-12-20T07:34:32Z",
+        "duration": 2065982241,
+        "ingested": "2025-01-01T06:34:49Z",
         "module": "sql"
     },
     "host": {
         "architecture": "x86_64",
-        "containerized": false,
-        "hostname": "192.168.1.2",
-        "id": "627E8AE5-E918-5073-A58E-8A2D9ED96875",
+        "containerized": true,
+        "hostname": "elastic-agent-78750",
+        "id": "8259e024976a406e8a54cdbffeb84fec",
         "ip": [
-            "192.168.1.2"
+            "192.168.241.5",
+            "192.168.253.2"
         ],
         "mac": [
-            "36-F7-DC-28-23-80"
+            "02-42-C0-A8-F1-05",
+            "02-42-C0-A8-FD-02"
         ],
-        "name": "192.168.1.2",
+        "name": "elastic-agent-78750",
         "os": {
-            "build": "21D62",
-            "family": "darwin",
-            "kernel": "21.3.0",
-            "name": "macOS",
-            "platform": "darwin",
-            "type": "macos",
-            "version": "12.2.1"
+            "codename": "focal",
+            "family": "debian",
+            "kernel": "3.10.0-1160.92.1.el7.x86_64",
+            "name": "Ubuntu",
+            "platform": "ubuntu",
+            "type": "linux",
+            "version": "20.04.6 LTS (Focal Fossa)"
         }
     },
     "metricset": {
@@ -562,20 +564,21 @@ An example event for `transaction_log` looks as following:
     },
     "mssql": {
         "metrics": {
-            "database_id": 1,
-            "database_name": "master",
+            "database_id": 4,
+            "database_name": "msdb",
             "instance_name": "MSSQLSERVER",
-            "log_space_in_bytes_since_last_backup": 602112,
-            "server_name": "obs-int-mssql20",
-            "total_log_size_bytes": 2088960,
-            "used_log_space_bytes": 1024000,
-            "used_log_space_pct": 49.01960754394531
-        }
+            "query_id": "JZwfIFXvA3yOOL15q7PFZK5UE+Y=",
+            "server_name": "7e672355554d"
+        },
+        "query": "SELECT @@servername AS server_name, @@servicename AS instance_name, name As 'database_name', database_id FROM sys.databases WHERE name='msdb';"
     },
     "service": {
-        "address": "20.228.135.242",
+        "address": "svc-microsoft_sqlserver",
         "type": "sql"
-    }
+    },
+    "tags": [
+        "preserve_sql_queries"
+    ]
 }
 ```
 
@@ -618,5 +621,6 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | mssql.metrics.total_log_size_bytes | Total transaction log size in bytes. | long | byte | counter |
 | mssql.metrics.used_log_space_bytes | The occupied size of the log in bytes. | long | byte | gauge |
 | mssql.metrics.used_log_space_pct | A percentage of the occupied size of the log as a percent of the total log size. | float | percent | gauge |
+| mssql.query | The SQL queries executed. | keyword |  |  |
 | service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword |  |  |
 

packages/microsoft_sqlserver/manifest.yml

@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: microsoft_sqlserver
 title: "Microsoft SQL Server"
-version: "2.10.0"
+version: "2.10.1"
 description: Collect events from Microsoft SQL Server with Elastic Agent
 type: integration
 categories: