[iptables] invoke community_id processor only for supported protocols (…

…#10676) * fix: invoke community_id processor only for supported protocols * feat: update pull request link in changelog.yml * fix: revisit on_failure error message format * fix: handle correctly numeric PROTO values * fix: update README.md * fix: rework iana_number and transport processing * fix: switch to a single rename processor for handling iana_number and transport fields
elastic · Aug 1, 2024 · 18721f6 · 18721f6
1 parent 5652860
commit 18721f6
Show file tree

Hide file tree

Showing 7 changed files with 231 additions and 3 deletions.
diff --git a/packages/iptables/changelog.yml b/packages/iptables/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.16.1"
+  changes:
+    - description: Invoke community_id processor only for supported protocols
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10676
 - version: "1.16.0"
   changes:
     - description: Update package spec to 3.0.3.

diff --git a/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log b/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log
@@ -19,5 +19,8 @@ Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:
 Jan  5 20:17:01 MainFirewall kernel: [source-dest-default-D]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2857 RES=0x00 ACK URGP=0 
 Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2853 RES=0x00 ACK URGP=0 
 Jan  5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2850 RES=0x00 ACK URGP=0 
+Jan  5 20:17:01 firewall kernel: [4276041.728154] IN=eno1 OUT= MAC=0c:c4:7a:0f:51:0c:d4:66:24:80:d8:da:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=120 TOS=0x00 PREC=0x00 TTL=55 ID=0 DF PROTO=ESP SPI=0xcb886522
 Jun 28 04:35:30 Abc-A1 [SOMETHING-1234-A] IN=abc.123 OUT=abc.123 MAC=0a:ea:10:00:f0:06:10:e1:21:31:61:20:01:00:41:00:00:01 SRC=10.251.1.1 DST=10.251.1.1 LEN=32 TOS=00 PREC=0x00 TTL=63 ID=12345 PROTO=UDP SPT=9000 DPT=9000 LEN=12 MARK=0
 Jun 28 04:30:32 Abc-A1 [SOMETHING-1234-A] IN=abc.123 OUT=abc.123 MAC=0a:ea:10:00:f0:06:10:e1:21:31:61:20:01:00:41:00:00:01 SRC=10.251.1.1 DST=10.251.1.1 LEN=84 TOS=00 PREC=0x00 TTL=63 ID=6789 PROTO=ICMP TYPE=8 CODE=0 ID=98765 SEQ=30123 MARK=0
+<4>Jun 27 23:29:32 router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=76 TOS=0x00 PREC=0x00 TTL=243 ID=37763 DF PROTO=1
+<4>Jun 12 20:26:58 router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=77 TOS=0x00 PREC=0x00 TTL=235 ID=24392 PROTO=47
diff --git a/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log-expected.json b/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log-expected.json
@@ -1573,6 +1573,61 @@
                 "preserve_original_event"
             ]
         },
+        {
+            "@timestamp": "2024-01-05T20:17:01.000Z",
+            "destination": {
+                "ip": "192.168.2.25",
+                "mac": "0C-C4-7A-0F-51-0C"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "category": [
+                    "network"
+                ],
+                "created": "2022-01-12T22:56:01.000Z",
+                "kind": "event",
+                "original": "Jan  5 20:17:01 firewall kernel: [4276041.728154] IN=eno1 OUT= MAC=0c:c4:7a:0f:51:0c:d4:66:24:80:d8:da:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=120 TOS=0x00 PREC=0x00 TTL=55 ID=0 DF PROTO=ESP SPI=0xcb886522",
+                "type": [
+                    "connection"
+                ]
+            },
+            "iptables": {
+                "ether_type": 2048,
+                "fragment_flags": [
+                    "DF"
+                ],
+                "id": 0,
+                "input_device": "eno1",
+                "length": 120,
+                "output_device": "",
+                "precedence_bits": 0,
+                "tos": 0,
+                "ttl": 55
+            },
+            "message": "firewall kernel: [4276041.728154] IN=eno1 OUT= MAC=0c:c4:7a:0f:51:0c:d4:66:24:80:d8:da:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=120 TOS=0x00 PREC=0x00 TTL=55 ID=0 DF PROTO=ESP SPI=0xcb886522",
+            "network": {
+                "transport": "esp",
+                "type": "ipv4"
+            },
+            "observer": {
+                "name": "firewall"
+            },
+            "related": {
+                "ip": [
+                    "192.168.110.116",
+                    "192.168.2.25"
+                ]
+            },
+            "source": {
+                "ip": "192.168.110.116",
+                "mac": "D4-66-24-80-D8-DA"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
         {
             "@timestamp": "2023-06-28T04:35:30.000Z",
             "destination": {
@@ -1679,6 +1734,159 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2024-06-27T23:29:32.000Z",
+            "destination": {
+                "ip": "10.251.1.1",
+                "mac": "04-18-D6-F1-2C-20"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "drop",
+                "category": [
+                    "network"
+                ],
+                "created": "2022-01-12T22:56:01.000Z",
+                "kind": "event",
+                "original": "<4>Jun 27 23:29:32 router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=76 TOS=0x00 PREC=0x00 TTL=243 ID=37763 DF PROTO=1",
+                "type": [
+                    "denied",
+                    "connection"
+                ]
+            },
+            "iptables": {
+                "ether_type": 2048,
+                "fragment_flags": [
+                    "DF"
+                ],
+                "id": 37763,
+                "input_device": "eth0",
+                "length": 76,
+                "output_device": "",
+                "precedence_bits": 0,
+                "tos": 0,
+                "ttl": 243,
+                "ubiquiti": {
+                    "input_zone": "wan",
+                    "output_zone": "local",
+                    "rule_number": "default",
+                    "rule_set": "wan-local"
+                }
+            },
+            "log": {
+                "syslog": {
+                    "priority": 4
+                }
+            },
+            "message": "router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=76 TOS=0x00 PREC=0x00 TTL=243 ID=37763 DF PROTO=1",
+            "network": {
+                "community_id": "1:FRJfyWaZVkG3e+uSp7d4BFAySFw=",
+                "iana_number": "1",
+                "type": "ipv4"
+            },
+            "observer": {
+                "egress": {
+                    "zone": "local"
+                },
+                "ingress": {
+                    "zone": "wan"
+                },
+                "name": "router"
+            },
+            "related": {
+                "ip": [
+                    "10.251.1.1"
+                ]
+            },
+            "rule": {
+                "id": "default",
+                "name": "wan-local"
+            },
+            "source": {
+                "ip": "10.251.1.1",
+                "mac": "00-00-5E-00-01-6A"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        },
+        {
+            "@timestamp": "2024-06-12T20:26:58.000Z",
+            "destination": {
+                "ip": "10.251.1.1",
+                "mac": "04-18-D6-F1-2C-20"
+            },
+            "ecs": {
+                "version": "8.11.0"
+            },
+            "event": {
+                "action": "drop",
+                "category": [
+                    "network"
+                ],
+                "created": "2022-01-12T22:56:01.000Z",
+                "kind": "event",
+                "original": "<4>Jun 12 20:26:58 router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=77 TOS=0x00 PREC=0x00 TTL=235 ID=24392 PROTO=47",
+                "type": [
+                    "denied",
+                    "connection"
+                ]
+            },
+            "iptables": {
+                "ether_type": 2048,
+                "id": 24392,
+                "input_device": "eth0",
+                "length": 77,
+                "output_device": "",
+                "precedence_bits": 0,
+                "tos": 0,
+                "ttl": 235,
+                "ubiquiti": {
+                    "input_zone": "wan",
+                    "output_zone": "local",
+                    "rule_number": "default",
+                    "rule_set": "wan-local"
+                }
+            },
+            "log": {
+                "syslog": {
+                    "priority": 4
+                }
+            },
+            "message": "router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=77 TOS=0x00 PREC=0x00 TTL=235 ID=24392 PROTO=47",
+            "network": {
+                "community_id": "1:VuTMLzzBad0b2D5gDo8qiZnYymo=",
+                "iana_number": "47",
+                "type": "ipv4"
+            },
+            "observer": {
+                "egress": {
+                    "zone": "local"
+                },
+                "ingress": {
+                    "zone": "wan"
+                },
+                "name": "router"
+            },
+            "related": {
+                "ip": [
+                    "10.251.1.1"
+                ]
+            },
+            "rule": {
+                "id": "default",
+                "name": "wan-local"
+            },
+            "source": {
+                "ip": "10.251.1.1",
+                "mac": "00-00-5E-00-01-6A"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
 }
diff --git a/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -140,6 +140,12 @@ processors:
       field: observer.hostname
       copy_from: hostname
       if: ctx?.observer?.name == null && ctx?.hostname != null
+  - rename:
+      description: Rename network.transport to network.iana_number if it is a number.
+      if: ctx.network?.iana_number == null && ctx.network?.transport != null && ctx.network.transport.chars().allMatch(Character::isDigit)
+      field: network.transport
+      target_field: network.iana_number
+      ignore_missing: true
   - lowercase:
       field: network.transport
       ignore_missing: true
@@ -188,7 +194,6 @@ processors:
       field: destination.as.organization_name
       target_field: destination.as.organization.name
       ignore_missing: true
-
   - script:
       description: Enrich event with ECS fields.
       lang: painless
@@ -250,6 +255,7 @@ processors:
         }
   - community_id:
       ignore_missing: true
+      ignore_failure: true
       icmp_type: iptables.icmp.type
       icmp_code: iptables.icmp.code
   - script:
@@ -378,4 +384,7 @@ on_failure:
       value: pipeline_error
   - append:
       field: error.message
-      value: '{{{ _ingest.on_failure_message }}}'
+      value: >-
+        Processor '{{ _ingest.on_failure_processor_type }}' in pipeline {{{_ingest.pipeline}}}
+        {{#_ingest.on_failure_processor_tag}} with tag '{{ _ingest.on_failure_processor_tag }}'{{/_ingest.on_failure_processor_tag}}
+        failed with message '{{ _ingest.on_failure_message }}'
diff --git a/packages/iptables/data_stream/log/fields/ecs.yml b/packages/iptables/data_stream/log/fields/ecs.yml
@@ -48,6 +48,8 @@
   name: network.forwarded_ip
 - external: ecs
   name: network.transport
+- external: ecs
+  name: network.iana_number
 - external: ecs
   name: network.type
 - external: ecs

diff --git a/packages/iptables/docs/README.md b/packages/iptables/docs/README.md
@@ -246,6 +246,7 @@ An example event for `log` looks as following:
 | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text |
 | network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword |
 | network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip |
+| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword |
 | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword |
 | network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword |
 | observer.egress.zone | Network zone of outbound traffic as reported by the observer to categorize the destination area of egress traffic, e.g. Internal, External, DMZ, HR, Legal, etc. | keyword |

diff --git a/packages/iptables/manifest.yml b/packages/iptables/manifest.yml
@@ -1,6 +1,6 @@
 name: iptables
 title: Iptables
-version: "1.16.0"
+version: "1.16.1"
 description: Collect logs from Iptables with Elastic Agent.
 type: integration
 icons: