diff --git a/packages/iptables/changelog.yml b/packages/iptables/changelog.yml index ed25e1d4cae9..d75fd2d7bdf5 100644 --- a/packages/iptables/changelog.yml +++ b/packages/iptables/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.16.1" + changes: + - description: Invoke community_id processor only for supported protocols + type: bugfix + link: https://github.com/elastic/integrations/pull/10676 - version: "1.16.0" changes: - description: Update package spec to 3.0.3. diff --git a/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log b/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log index 5865099de8ac..dbdf371f00c0 100644 --- a/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log +++ b/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log @@ -19,5 +19,8 @@ Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10: Jan 5 20:17:01 MainFirewall kernel: [source-dest-default-D]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2857 RES=0x00 ACK URGP=0 Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2853 RES=0x00 ACK URGP=0 Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2850 RES=0x00 ACK URGP=0 +Jan 5 20:17:01 firewall kernel: [4276041.728154] IN=eno1 OUT= MAC=0c:c4:7a:0f:51:0c:d4:66:24:80:d8:da:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=120 TOS=0x00 PREC=0x00 TTL=55 ID=0 DF PROTO=ESP SPI=0xcb886522 Jun 28 04:35:30 Abc-A1 [SOMETHING-1234-A] IN=abc.123 OUT=abc.123 MAC=0a:ea:10:00:f0:06:10:e1:21:31:61:20:01:00:41:00:00:01 SRC=10.251.1.1 DST=10.251.1.1 LEN=32 TOS=00 PREC=0x00 TTL=63 ID=12345 PROTO=UDP SPT=9000 DPT=9000 LEN=12 MARK=0 Jun 28 04:30:32 Abc-A1 [SOMETHING-1234-A] IN=abc.123 OUT=abc.123 MAC=0a:ea:10:00:f0:06:10:e1:21:31:61:20:01:00:41:00:00:01 SRC=10.251.1.1 DST=10.251.1.1 LEN=84 TOS=00 PREC=0x00 TTL=63 ID=6789 PROTO=ICMP TYPE=8 CODE=0 ID=98765 SEQ=30123 MARK=0 +<4>Jun 27 23:29:32 router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=76 TOS=0x00 PREC=0x00 TTL=243 ID=37763 DF PROTO=1 +<4>Jun 12 20:26:58 router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=77 TOS=0x00 PREC=0x00 TTL=235 ID=24392 PROTO=47 diff --git a/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log-expected.json b/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log-expected.json index cdcdfaf3202d..3850f654a27c 100644 --- a/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log-expected.json +++ b/packages/iptables/data_stream/log/_dev/test/pipeline/test-iptables-raw.log-expected.json @@ -1573,6 +1573,61 @@ "preserve_original_event" ] }, + { + "@timestamp": "2024-01-05T20:17:01.000Z", + "destination": { + "ip": "192.168.2.25", + "mac": "0C-C4-7A-0F-51-0C" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "created": "2022-01-12T22:56:01.000Z", + "kind": "event", + "original": "Jan 5 20:17:01 firewall kernel: [4276041.728154] IN=eno1 OUT= MAC=0c:c4:7a:0f:51:0c:d4:66:24:80:d8:da:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=120 TOS=0x00 PREC=0x00 TTL=55 ID=0 DF PROTO=ESP SPI=0xcb886522", + "type": [ + "connection" + ] + }, + "iptables": { + "ether_type": 2048, + "fragment_flags": [ + "DF" + ], + "id": 0, + "input_device": "eno1", + "length": 120, + "output_device": "", + "precedence_bits": 0, + "tos": 0, + "ttl": 55 + }, + "message": "firewall kernel: [4276041.728154] IN=eno1 OUT= MAC=0c:c4:7a:0f:51:0c:d4:66:24:80:d8:da:08:00 SRC=192.168.110.116 DST=192.168.2.25 LEN=120 TOS=0x00 PREC=0x00 TTL=55 ID=0 DF PROTO=ESP SPI=0xcb886522", + "network": { + "transport": "esp", + "type": "ipv4" + }, + "observer": { + "name": "firewall" + }, + "related": { + "ip": [ + "192.168.110.116", + "192.168.2.25" + ] + }, + "source": { + "ip": "192.168.110.116", + "mac": "D4-66-24-80-D8-DA" + }, + "tags": [ + "preserve_original_event" + ] + }, { "@timestamp": "2023-06-28T04:35:30.000Z", "destination": { @@ -1679,6 +1734,159 @@ "tags": [ "preserve_original_event" ] + }, + { + "@timestamp": "2024-06-27T23:29:32.000Z", + "destination": { + "ip": "10.251.1.1", + "mac": "04-18-D6-F1-2C-20" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "drop", + "category": [ + "network" + ], + "created": "2022-01-12T22:56:01.000Z", + "kind": "event", + "original": "<4>Jun 27 23:29:32 router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=76 TOS=0x00 PREC=0x00 TTL=243 ID=37763 DF PROTO=1", + "type": [ + "denied", + "connection" + ] + }, + "iptables": { + "ether_type": 2048, + "fragment_flags": [ + "DF" + ], + "id": 37763, + "input_device": "eth0", + "length": 76, + "output_device": "", + "precedence_bits": 0, + "tos": 0, + "ttl": 243, + "ubiquiti": { + "input_zone": "wan", + "output_zone": "local", + "rule_number": "default", + "rule_set": "wan-local" + } + }, + "log": { + "syslog": { + "priority": 4 + } + }, + "message": "router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=76 TOS=0x00 PREC=0x00 TTL=243 ID=37763 DF PROTO=1", + "network": { + "community_id": "1:FRJfyWaZVkG3e+uSp7d4BFAySFw=", + "iana_number": "1", + "type": "ipv4" + }, + "observer": { + "egress": { + "zone": "local" + }, + "ingress": { + "zone": "wan" + }, + "name": "router" + }, + "related": { + "ip": [ + "10.251.1.1" + ] + }, + "rule": { + "id": "default", + "name": "wan-local" + }, + "source": { + "ip": "10.251.1.1", + "mac": "00-00-5E-00-01-6A" + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2024-06-12T20:26:58.000Z", + "destination": { + "ip": "10.251.1.1", + "mac": "04-18-D6-F1-2C-20" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "action": "drop", + "category": [ + "network" + ], + "created": "2022-01-12T22:56:01.000Z", + "kind": "event", + "original": "<4>Jun 12 20:26:58 router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=77 TOS=0x00 PREC=0x00 TTL=235 ID=24392 PROTO=47", + "type": [ + "denied", + "connection" + ] + }, + "iptables": { + "ether_type": 2048, + "id": 24392, + "input_device": "eth0", + "length": 77, + "output_device": "", + "precedence_bits": 0, + "tos": 0, + "ttl": 235, + "ubiquiti": { + "input_zone": "wan", + "output_zone": "local", + "rule_number": "default", + "rule_set": "wan-local" + } + }, + "log": { + "syslog": { + "priority": 4 + } + }, + "message": "router kernel: [wan-local-default-D]IN=eth0 OUT= MAC=04:18:d6:f1:2c:20:00:00:5e:00:01:6a:08:00 SRC=10.251.1.1 DST=10.251.1.1 LEN=77 TOS=0x00 PREC=0x00 TTL=235 ID=24392 PROTO=47", + "network": { + "community_id": "1:VuTMLzzBad0b2D5gDo8qiZnYymo=", + "iana_number": "47", + "type": "ipv4" + }, + "observer": { + "egress": { + "zone": "local" + }, + "ingress": { + "zone": "wan" + }, + "name": "router" + }, + "related": { + "ip": [ + "10.251.1.1" + ] + }, + "rule": { + "id": "default", + "name": "wan-local" + }, + "source": { + "ip": "10.251.1.1", + "mac": "00-00-5E-00-01-6A" + }, + "tags": [ + "preserve_original_event" + ] } ] } \ No newline at end of file diff --git a/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml index 8ec58c0ba374..8879095e7d99 100644 --- a/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml +++ b/packages/iptables/data_stream/log/elasticsearch/ingest_pipeline/default.yml @@ -140,6 +140,12 @@ processors: field: observer.hostname copy_from: hostname if: ctx?.observer?.name == null && ctx?.hostname != null + - rename: + description: Rename network.transport to network.iana_number if it is a number. + if: ctx.network?.iana_number == null && ctx.network?.transport != null && ctx.network.transport.chars().allMatch(Character::isDigit) + field: network.transport + target_field: network.iana_number + ignore_missing: true - lowercase: field: network.transport ignore_missing: true @@ -188,7 +194,6 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true - - script: description: Enrich event with ECS fields. lang: painless @@ -250,6 +255,7 @@ processors: } - community_id: ignore_missing: true + ignore_failure: true icmp_type: iptables.icmp.type icmp_code: iptables.icmp.code - script: @@ -378,4 +384,7 @@ on_failure: value: pipeline_error - append: field: error.message - value: '{{{ _ingest.on_failure_message }}}' + value: >- + Processor '{{ _ingest.on_failure_processor_type }}' in pipeline {{{_ingest.pipeline}}} + {{#_ingest.on_failure_processor_tag}} with tag '{{ _ingest.on_failure_processor_tag }}'{{/_ingest.on_failure_processor_tag}} + failed with message '{{ _ingest.on_failure_message }}' \ No newline at end of file diff --git a/packages/iptables/data_stream/log/fields/ecs.yml b/packages/iptables/data_stream/log/fields/ecs.yml index 41414cb01d48..541e3d4c9e20 100644 --- a/packages/iptables/data_stream/log/fields/ecs.yml +++ b/packages/iptables/data_stream/log/fields/ecs.yml @@ -48,6 +48,8 @@ name: network.forwarded_ip - external: ecs name: network.transport +- external: ecs + name: network.iana_number - external: ecs name: network.type - external: ecs diff --git a/packages/iptables/docs/README.md b/packages/iptables/docs/README.md index 08719d90d291..26113b96dfc4 100644 --- a/packages/iptables/docs/README.md +++ b/packages/iptables/docs/README.md @@ -246,6 +246,7 @@ An example event for `log` looks as following: | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | | network.community_id | A hash of source and destination IPs and ports, as well as the protocol used in a communication. This is a tool-agnostic standard to identify flows. Learn more at https://github.com/corelight/community-id-spec. | keyword | | network.forwarded_ip | Host IP address when the source IP address is the proxy. | ip | +| network.iana_number | IANA Protocol Number (https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml). Standardized list of protocols. This aligns well with NetFlow and sFlow related logs which use the IANA Protocol Number. | keyword | | network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | | network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | | observer.egress.zone | Network zone of outbound traffic as reported by the observer to categorize the destination area of egress traffic, e.g. Internal, External, DMZ, HR, Legal, etc. | keyword | diff --git a/packages/iptables/manifest.yml b/packages/iptables/manifest.yml index 3e95ded6fbc6..e6613d9433a4 100644 --- a/packages/iptables/manifest.yml +++ b/packages/iptables/manifest.yml @@ -1,6 +1,6 @@ name: iptables title: Iptables -version: "1.16.0" +version: "1.16.1" description: Collect logs from Iptables with Elastic Agent. type: integration icons: