cisagov#553, handle conn.log for zeek v7.1.0 and documentation update

docs/protocols.md

@@ -33,7 +33,7 @@ Malcolm uses [Zeek](https://docs.zeek.org/en/stable/script-reference/proto-analy
 |Open Platform Communications Unified Architecture (OPC UA) Binary|[🔗](https://en.wikipedia.org/wiki/OPC_Unified_Architecture)|[🔗](https://opcfoundation.org/developer-tools/specifications-unified-architecture)||[✓](https://github.com/cisagov/icsnpp-opcua-binary)|
 |Open Shortest Path First (OSPF)|[🔗](https://en.wikipedia.org/wiki/Open_Shortest_Path_First)|[🔗](https://datatracker.ietf.org/wg/ospf/charter/)[🔗](https://datatracker.ietf.org/doc/html/rfc2328)[🔗](https://datatracker.ietf.org/doc/html/rfc5340)||[✓](https://github.com/corelight/zeek-spicy-ospf)|
 |OpenVPN|[🔗](https://en.wikipedia.org/wiki/OpenVPN)|[🔗](https://openvpn.net/community-resources/openvpn-protocol/)[🔗](https://zeek.org/2021/03/16/a-zeek-openvpn-protocol-analyzer/)||[✓](https://github.com/corelight/zeek-spicy-openvpn)|
-|PostgreSQL|[🔗](https://en.wikipedia.org/wiki/PostgreSQL)|[🔗](https://www.postgresql.org/)|[✓](https://github.com/arkime/arkime/blob/master/capture/parsers/postgresql.c)||
+|PostgreSQL|[🔗](https://en.wikipedia.org/wiki/PostgreSQL)|[🔗](https://www.postgresql.org/)|[✓](https://github.com/arkime/arkime/blob/master/capture/parsers/postgresql.c)|[🔗](https://docs.zeek.org/en/master/scripts/base/protocols/postgresql/main.zeek.html)|
 |Process Field Net (PROFINET)|[🔗](https://en.wikipedia.org/wiki/PROFINET)|[🔗](https://us.profinet.com/technology/profinet/)||[✓](https://github.com/amzn/zeek-plugin-profinet/blob/master/scripts/main.zeek)|
 |PROFINET IO CM (Input/Output Context Manager)|[🔗](https://wiki.wireshark.org/PROFINET/IO)|[🔗](https://us.profinet.com/technology/profinet/)[🔗](https://webstore.iec.ch/publication/83418)||[✓](https://github.com/cisagov/icsnpp-profinet-io-cm/blob/main/analyzer/types.zeek)|
 |Remote Authentication Dial-In User Service (RADIUS)|[🔗](https://en.wikipedia.org/wiki/RADIUS)|[🔗](https://tools.ietf.org/html/rfc2865)|[✓](https://github.com/arkime/arkime/blob/master/capture/parsers/radius.c)|[✓](https://docs.zeek.org/en/stable/scripts/base/protocols/radius/main.zeek.html#type-RADIUS::Info)|

logstash/pipelines/zeek/1014_zeek_conn.conf

@@ -14,7 +14,7 @@ filter {
       dissect {
         id => "dissect_zeek_conn_with_all_fields"
         mapping => {
-          "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][community_id]} %{[zeek_cols][ja4l]} %{[zeek_cols][ja4ls]} %{[zeek_cols][ja4t]} %{[zeek_cols][ja4ts]}"
+          "[message]" => "%{[zeek_cols][ts]} %{[zeek_cols][uid]} %{[zeek_cols][orig_h]} %{[zeek_cols][orig_p]} %{[zeek_cols][resp_h]} %{[zeek_cols][resp_p]} %{[zeek_cols][proto]} %{[zeek_cols][service]} %{[zeek_cols][duration]} %{[zeek_cols][orig_bytes]} %{[zeek_cols][resp_bytes]} %{[zeek_cols][conn_state]} %{[zeek_cols][local_orig]} %{[zeek_cols][local_resp]} %{[zeek_cols][missed_bytes]} %{[zeek_cols][history]} %{[zeek_cols][orig_pkts]} %{[zeek_cols][orig_ip_bytes]} %{[zeek_cols][resp_pkts]} %{[zeek_cols][resp_ip_bytes]} %{[zeek_cols][tunnel_parents]} %{[zeek_cols][ip_proto]} %{[zeek_cols][vlan]} %{[zeek_cols][inner_vlan]} %{[zeek_cols][orig_l2_addr]} %{[zeek_cols][resp_l2_addr]} %{[zeek_cols][community_id]} %{[zeek_cols][ja4l]} %{[zeek_cols][ja4ls]} %{[zeek_cols][ja4t]} %{[zeek_cols][ja4ts]}"
         }
       }
       if ("_dissectfailure" in [tags]) {
@@ -24,7 +24,7 @@ filter {
         }
         ruby {
           id => "ruby_zip_zeek_conn"
-          init => "@zeek_conn_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'duration', 'orig_bytes', 'resp_bytes', 'conn_state', 'local_orig', 'local_resp', 'missed_bytes', 'history', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'tunnel_parents', 'vlan', 'inner_vlan', 'orig_l2_addr', 'resp_l2_addr', 'community_id', 'ja4l', 'ja4ls', 'ja4t', 'ja4ts' ]"
+          init => "@zeek_conn_field_names = [ 'ts', 'uid', 'orig_h', 'orig_p', 'resp_h', 'resp_p', 'proto', 'service', 'duration', 'orig_bytes', 'resp_bytes', 'conn_state', 'local_orig', 'local_resp', 'missed_bytes', 'history', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'tunnel_parents', 'ip_proto', 'vlan', 'inner_vlan', 'orig_l2_addr', 'resp_l2_addr', 'community_id', 'ja4l', 'ja4ls', 'ja4t', 'ja4ts' ]"
           code => "event.set('[zeek_cols]', @zeek_conn_field_names.zip(event.get('[message]')).to_h)"
         }
       }
@@ -82,6 +82,8 @@ filter {
       mutate { id => "mutate_add_field_zeek_conn_rootId"
                add_field => { "[rootId]" => "%{[zeek_cols][tunnel_parents][0]}" } }
     }
+    mutate { id => "mutate_rename_conn_ip_proto"
+             rename => { "[zeek_cols][ip_proto]" => "[ipProtocol]" } }
 
   }
 

logstash/pipelines/zeek/1200_zeek_mutate.conf

@@ -141,11 +141,13 @@ filter {
 
   # set user and transport- and application-level protocols if specified
   if ([network][transport]) {
-    translate {
-      id => "translate_zeek_proto"
-      source => "[network][transport]"
-      target => "[ipProtocol]"
-      dictionary_path => "/etc/ip_protocol_name_to_number.yaml"
+    if (![ipProtocol]) {
+      translate {
+        id => "translate_zeek_proto"
+        source => "[network][transport]"
+        target => "[ipProtocol]"
+        dictionary_path => "/etc/ip_protocol_name_to_number.yaml"
+      }
     }
     if ("_jsonparsesuccess" not in [tags]) {
       mutate { id => "mutate_split_zeek_proto"