From 56e6ac7dc9b670ea41e00048fce3d7798aef5302 Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Mon, 15 Jan 2024 16:40:34 +0100 Subject: [PATCH 1/5] Parse lines with client info where the ip-address is set to 'unknown' Note that the value `unknown` is not saved to the ip address field, as this might mess up existing ip-address post-processing. To make the new possibility visible, a new field is added: `postfix_client_ip_unknown`. --- postfix.grok | 19 +++++++++++-------- test/anvil_0008.yaml | 7 +++++++ test/anvil_0009.yaml | 8 ++++++++ test/smtpd_0032.yaml | 7 +++++++ 4 files changed, 33 insertions(+), 8 deletions(-) create mode 100644 test/anvil_0008.yaml create mode 100644 test/anvil_0009.yaml create mode 100644 test/smtpd_0032.yaml diff --git a/postfix.grok b/postfix.grok index 9700c1b..d269b1d 100644 --- a/postfix.grok +++ b/postfix.grok @@ -1,8 +1,16 @@ # Version: 1.0.0 +# helper patterns +GREEDYDATA_NO_COLON [^:]* +GREEDYDATA_NO_SEMICOLON [^;]* +GREEDYDATA_NO_BRACKET [^<>]* +STATUS_WORD [\w-]* +IP_UNKNOWN unknown + # common postfix patterns POSTFIX_QUEUEID ([0-9A-F]{6,}|[0-9a-zA-Z]{12,}|NOQUEUE) -POSTFIX_CLIENT_INFO %{HOSTNAME:postfix_client_hostname}?\[%{IP:postfix_client_ip}\](:%{INT:postfix_client_port})? + +POSTFIX_CLIENT_INFO %{HOSTNAME:postfix_client_hostname}?\[(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\](:%{INT:postfix_client_port})? POSTFIX_RELAY_INFO %{HOSTNAME:postfix_relay_hostname}?\[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})\](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service} POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\.) POSTFIX_ACTION (accept|defer|discard|filter|header-redirect|reject|reject_warning) @@ -25,11 +33,6 @@ POSTFIX_LOSTCONN_REASONS (receiving the initial server greeting|sending message POSTFIX_PROXY_MESSAGE (%{POSTFIX_STATUS_CODE:postfix_proxy_status_code} )?(%{POSTFIX_STATUS_CODE_ENHANCED:postfix_proxy_status_code_enhanced})?.* POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:postfix_cmd_helo} )?(ehlo=(%{INT:postfix_cmd_ehlo_accepted}/)?%{INT:postfix_cmd_ehlo} )?(starttls=(%{INT:postfix_cmd_starttls_accepted}/)?%{INT:postfix_cmd_starttls} )?(auth=(%{INT:postfix_cmd_auth_accepted}/)?%{INT:postfix_cmd_auth} )?(mail=(%{INT:postfix_cmd_mail_accepted}/)?%{INT:postfix_cmd_mail} )?(rcpt=(%{INT:postfix_cmd_rcpt_accepted}/)?%{INT:postfix_cmd_rcpt} )?(data=(%{INT:postfix_cmd_data_accepted}/)?%{INT:postfix_cmd_data} )?(rset=(%{INT:postfix_cmd_rset_accepted}/)?%{INT:postfix_cmd_rset} )?(quit=(%{INT:postfix_cmd_quit_accepted}/)?%{INT:postfix_cmd_quit} )?(unknown=(%{INT:postfix_cmd_unknown_accepted}/)?%{INT:postfix_cmd_unknown} )?commands=(%{INT:postfix_cmd_count_accepted}/)?%{INT:postfix_cmd_count} -# helper patterns -GREEDYDATA_NO_COLON [^:]* -GREEDYDATA_NO_SEMICOLON [^;]* -GREEDYDATA_NO_BRACKET [^<>]* -STATUS_WORD [\w-]* # warning patterns POSTFIX_WARNING_WITH_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?%{POSTFIX_WARNING_LEVEL:postfix_message_level}: (%{POSTFIX_QUEUEID:postfix_queueid}: )?(%{POSTFIX_CLIENT_INFO}: )?%{GREEDYDATA:postfix_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} @@ -87,9 +90,9 @@ POSTFIX_DNSBLOG_LISTING addr %{IP:postfix_client_ip} listed by domain %{HOSTNAME POSTFIX_TLSPROXY_CONN (DIS)?CONNECT( from)? %{POSTFIX_CLIENT_INFO} # anvil patterns -POSTFIX_ANVIL_CONN_RATE statistics: max connection rate %{NUMBER:postfix_anvil_conn_rate}/%{POSTFIX_TIME_UNIT:postfix_anvil_conn_period} for \(%{DATA:postfix_service}:%{IP:postfix_client_ip}\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} +POSTFIX_ANVIL_CONN_RATE statistics: max connection rate %{NUMBER:postfix_anvil_conn_rate}/%{POSTFIX_TIME_UNIT:postfix_anvil_conn_period} for \(%{DATA:postfix_service}:(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} POSTFIX_ANVIL_CONN_CACHE statistics: max cache size %{NUMBER:postfix_anvil_cache_size} at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} -POSTFIX_ANVIL_CONN_COUNT statistics: max connection count %{NUMBER:postfix_anvil_conn_count} for \(%{DATA:postfix_service}:%{IP:postfix_client_ip}\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} +POSTFIX_ANVIL_CONN_COUNT statistics: max connection count %{NUMBER:postfix_anvil_conn_count} for \(%{DATA:postfix_service}:(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} # smtp patterns POSTFIX_SMTP_DELIVERY %{POSTFIX_KEYVALUE} status=%{STATUS_WORD:postfix_status}( \(%{GREEDYDATA:postfix_smtp_response}\))? diff --git a/test/anvil_0008.yaml b/test/anvil_0008.yaml new file mode 100644 index 0000000..e84caf4 --- /dev/null +++ b/test/anvil_0008.yaml @@ -0,0 +1,7 @@ +pattern: ^%{POSTFIX_ANVIL}$ +data: "statistics: max connection count 1 for (smtp:unknown) at Sep 7 07:14:19" +results: + postfix_anvil_conn_count: 1 + postfix_service: smtp + postfix_client_ip_unknown: unknown + postfix_anvil_timestamp: Sep 7 07:14:19 diff --git a/test/anvil_0009.yaml b/test/anvil_0009.yaml new file mode 100644 index 0000000..569c993 --- /dev/null +++ b/test/anvil_0009.yaml @@ -0,0 +1,8 @@ +pattern: ^%{POSTFIX_ANVIL}$ +data: "statistics: max connection rate 1/60s for (smtp:unknown) at Sep 7 07:14:19" +results: + postfix_anvil_conn_rate: 1 + postfix_anvil_conn_period: 60s + postfix_service: smtp + postfix_client_ip_unknown: unknown + postfix_anvil_timestamp: Sep 7 07:14:19 diff --git a/test/smtpd_0032.yaml b/test/smtpd_0032.yaml new file mode 100644 index 0000000..9f35f16 --- /dev/null +++ b/test/smtpd_0032.yaml @@ -0,0 +1,7 @@ +pattern: ^%{POSTFIX_SMTPD}$ +data: "disconnect from unknown[unknown] commands=0/0" +results: + postfix_client_hostname: unknown + postfix_client_ip_unknown: unknown + postfix_command_counter_data: commands=0/0 + From 95f596f3fda499c099e95c67ea2dfac1c23e7ccd Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Mon, 15 Jan 2024 16:46:51 +0100 Subject: [PATCH 2/5] Add support for BDAT and NOOP SMTP stages and command counters --- 50-filter-postfix.conf | 4 ++++ postfix.grok | 4 ++-- test/smtpd_0028.yaml | 1 - test/smtpd_0033.yaml | 7 +++++++ test/smtpd_0034.yaml | 6 ++++++ test/smtpd_0035.yaml | 6 ++++++ 6 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 test/smtpd_0033.yaml create mode 100644 test/smtpd_0034.yaml create mode 100644 test/smtpd_0035.yaml diff --git a/50-filter-postfix.conf b/50-filter-postfix.conf index ee183d9..13f6740 100644 --- a/50-filter-postfix.conf +++ b/50-filter-postfix.conf @@ -237,6 +237,8 @@ filter { "postfix_client_port", "integer", "postfix_cmd_auth", "integer", "postfix_cmd_auth_accepted", "integer", + "postfix_cmd_bdat", "integer", + "postfix_cmd_bdat_accepted", "integer", "postfix_cmd_count", "integer", "postfix_cmd_count_accepted", "integer", "postfix_cmd_data", "integer", @@ -247,6 +249,8 @@ filter { "postfix_cmd_helo_accepted", "integer", "postfix_cmd_mail", "integer", "postfix_cmd_mail_accepted", "integer", + "postfix_cmd_noop", "integer", + "postfix_cmd_noop_accepted", "integer", "postfix_cmd_quit", "integer", "postfix_cmd_quit_accepted", "integer", "postfix_cmd_rcpt", "integer", diff --git a/postfix.grok b/postfix.grok index d269b1d..4ca4dc3 100644 --- a/postfix.grok +++ b/postfix.grok @@ -12,7 +12,7 @@ POSTFIX_QUEUEID ([0-9A-F]{6,}|[0-9a-zA-Z]{12,}|NOQUEUE) POSTFIX_CLIENT_INFO %{HOSTNAME:postfix_client_hostname}?\[(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\](:%{INT:postfix_client_port})? POSTFIX_RELAY_INFO %{HOSTNAME:postfix_relay_hostname}?\[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})\](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service} -POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\.) +POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|BDAT|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\.) POSTFIX_ACTION (accept|defer|discard|filter|header-redirect|reject|reject_warning) POSTFIX_STATUS_CODE \d{3} POSTFIX_STATUS_CODE_ENHANCED \d\.\d+\.\d+ @@ -31,7 +31,7 @@ POSTFIX_DELAYS %{NUMBER:postfix_delay_before_qmgr}/%{NUMBER:postfix_delay_in_qmg POSTFIX_LOSTCONN (Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1) POSTFIX_LOSTCONN_REASONS (receiving the initial server greeting|sending message body|sending end of data -- message may be sent more than once) POSTFIX_PROXY_MESSAGE (%{POSTFIX_STATUS_CODE:postfix_proxy_status_code} )?(%{POSTFIX_STATUS_CODE_ENHANCED:postfix_proxy_status_code_enhanced})?.* -POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:postfix_cmd_helo} )?(ehlo=(%{INT:postfix_cmd_ehlo_accepted}/)?%{INT:postfix_cmd_ehlo} )?(starttls=(%{INT:postfix_cmd_starttls_accepted}/)?%{INT:postfix_cmd_starttls} )?(auth=(%{INT:postfix_cmd_auth_accepted}/)?%{INT:postfix_cmd_auth} )?(mail=(%{INT:postfix_cmd_mail_accepted}/)?%{INT:postfix_cmd_mail} )?(rcpt=(%{INT:postfix_cmd_rcpt_accepted}/)?%{INT:postfix_cmd_rcpt} )?(data=(%{INT:postfix_cmd_data_accepted}/)?%{INT:postfix_cmd_data} )?(rset=(%{INT:postfix_cmd_rset_accepted}/)?%{INT:postfix_cmd_rset} )?(quit=(%{INT:postfix_cmd_quit_accepted}/)?%{INT:postfix_cmd_quit} )?(unknown=(%{INT:postfix_cmd_unknown_accepted}/)?%{INT:postfix_cmd_unknown} )?commands=(%{INT:postfix_cmd_count_accepted}/)?%{INT:postfix_cmd_count} +POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:postfix_cmd_helo} )?(ehlo=(%{INT:postfix_cmd_ehlo_accepted}/)?%{INT:postfix_cmd_ehlo} )?(starttls=(%{INT:postfix_cmd_starttls_accepted}/)?%{INT:postfix_cmd_starttls} )?(auth=(%{INT:postfix_cmd_auth_accepted}/)?%{INT:postfix_cmd_auth} )?(mail=(%{INT:postfix_cmd_mail_accepted}/)?%{INT:postfix_cmd_mail} )?(rcpt=(%{INT:postfix_cmd_rcpt_accepted}/)?%{INT:postfix_cmd_rcpt} )?(bdat=(%{INT:postfix_cmd_bdat_accepted}/)?%{INT:postfix_cmd_bdat} )?(data=(%{INT:postfix_cmd_data_accepted}/)?%{INT:postfix_cmd_data} )?(rset=(%{INT:postfix_cmd_rset_accepted}/)?%{INT:postfix_cmd_rset} )?(noop=(%{INT:postfix_cmd_noop_accepted}/)?%{INT:postfix_cmd_noop} )?(quit=(%{INT:postfix_cmd_quit_accepted}/)?%{INT:postfix_cmd_quit} )?(unknown=(%{INT:postfix_cmd_unknown_accepted}/)?%{INT:postfix_cmd_unknown} )?commands=(%{INT:postfix_cmd_count_accepted}/)?%{INT:postfix_cmd_count} # warning patterns diff --git a/test/smtpd_0028.yaml b/test/smtpd_0028.yaml index 89911a3..fc8a348 100644 --- a/test/smtpd_0028.yaml +++ b/test/smtpd_0028.yaml @@ -4,4 +4,3 @@ results: postfix_client_hostname: unknown postfix_client_ip: 72.10.165.66 postfix_command_counter_data: ehlo=2 starttls=1 mail=1 rcpt=0/1 data=0/1 quit=1 commands=5/7 - diff --git a/test/smtpd_0033.yaml b/test/smtpd_0033.yaml new file mode 100644 index 0000000..a69263e --- /dev/null +++ b/test/smtpd_0033.yaml @@ -0,0 +1,7 @@ +pattern: ^%{POSTFIX_SMTPD}$ +data: "lost connection after BDAT from example.net[158.247.23.50]" +results: + postfix_smtpd_lostconn_data: lost connection + postfix_smtp_stage: BDAT + postfix_client_hostname: example.net + postfix_client_ip: 158.247.23.50 diff --git a/test/smtpd_0034.yaml b/test/smtpd_0034.yaml new file mode 100644 index 0000000..c55e2bc --- /dev/null +++ b/test/smtpd_0034.yaml @@ -0,0 +1,6 @@ +pattern: ^%{POSTFIX_SMTPD}$ +data: "disconnect from example.net[158.247.23.50] ehlo=2 starttls=1 mail=1 rcpt=0/1 bdat=0/1 commands=4/6" +results: + postfix_client_hostname: example.net + postfix_client_ip: 158.247.23.50 + postfix_command_counter_data: ehlo=2 starttls=1 mail=1 rcpt=0/1 bdat=0/1 commands=4/6 diff --git a/test/smtpd_0035.yaml b/test/smtpd_0035.yaml new file mode 100644 index 0000000..5a40658 --- /dev/null +++ b/test/smtpd_0035.yaml @@ -0,0 +1,6 @@ +pattern: ^%{POSTFIX_SMTPD}$ +data: "disconnect from unknown[93.188.162.137] ehlo=2 starttls=1 mail=1 rcpt=0/1 rset=1 noop=1 quit=1 commands=7/8" +results: + postfix_client_hostname: unknown + postfix_client_ip: 93.188.162.137 + postfix_command_counter_data: ehlo=2 starttls=1 mail=1 rcpt=0/1 rset=1 noop=1 quit=1 commands=7/8 From 86e4a89323b859bed6861edb2c9c5e06216990dc Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Mon, 15 Jan 2024 17:14:46 +0100 Subject: [PATCH 3/5] Improve parsing of lost connection reports from postfix/smtp --- postfix.grok | 4 ++-- test/smtp_0006.yaml | 2 ++ test/smtp_0033.yaml | 8 ++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 test/smtp_0033.yaml diff --git a/postfix.grok b/postfix.grok index 4ca4dc3..f049c0c 100644 --- a/postfix.grok +++ b/postfix.grok @@ -28,8 +28,8 @@ POSTFIX_TLSCONN (Anonymous|Trusted|Untrusted|Verified) TLS connection establishe POSTFIX_TLSVERIFICATION certificate verification failed for %{POSTFIX_RELAY_INFO}: %{GREEDYDATA:postfix_tls_error} POSTFIX_DELAYS %{NUMBER:postfix_delay_before_qmgr}/%{NUMBER:postfix_delay_in_qmgr}/%{NUMBER:postfix_delay_conn_setup}/%{NUMBER:postfix_delay_transmission} -POSTFIX_LOSTCONN (Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1) -POSTFIX_LOSTCONN_REASONS (receiving the initial server greeting|sending message body|sending end of data -- message may be sent more than once) +POSTFIX_LOSTCONN (Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1|Address not available|Operation timed out) +POSTFIX_LOSTCONN_REASONS (receiving the initial server greeting|sending message body|sending end of data -- message may be sent more than once|sending %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}) POSTFIX_PROXY_MESSAGE (%{POSTFIX_STATUS_CODE:postfix_proxy_status_code} )?(%{POSTFIX_STATUS_CODE_ENHANCED:postfix_proxy_status_code_enhanced})?.* POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:postfix_cmd_helo} )?(ehlo=(%{INT:postfix_cmd_ehlo_accepted}/)?%{INT:postfix_cmd_ehlo} )?(starttls=(%{INT:postfix_cmd_starttls_accepted}/)?%{INT:postfix_cmd_starttls} )?(auth=(%{INT:postfix_cmd_auth_accepted}/)?%{INT:postfix_cmd_auth} )?(mail=(%{INT:postfix_cmd_mail_accepted}/)?%{INT:postfix_cmd_mail} )?(rcpt=(%{INT:postfix_cmd_rcpt_accepted}/)?%{INT:postfix_cmd_rcpt} )?(bdat=(%{INT:postfix_cmd_bdat_accepted}/)?%{INT:postfix_cmd_bdat} )?(data=(%{INT:postfix_cmd_data_accepted}/)?%{INT:postfix_cmd_data} )?(rset=(%{INT:postfix_cmd_rset_accepted}/)?%{INT:postfix_cmd_rset} )?(noop=(%{INT:postfix_cmd_noop_accepted}/)?%{INT:postfix_cmd_noop} )?(quit=(%{INT:postfix_cmd_quit_accepted}/)?%{INT:postfix_cmd_quit} )?(unknown=(%{INT:postfix_cmd_unknown_accepted}/)?%{INT:postfix_cmd_unknown} )?commands=(%{INT:postfix_cmd_count_accepted}/)?%{INT:postfix_cmd_count} diff --git a/test/smtp_0006.yaml b/test/smtp_0006.yaml index 5912607..84b580f 100644 --- a/test/smtp_0006.yaml +++ b/test/smtp_0006.yaml @@ -2,5 +2,7 @@ pattern: ^%{POSTFIX_SMTP}$ data: "B99FE3D: lost connection with mx3.hotmail.com[65.55.37.72] while receiving the initial server greeting" results: postfix_queueid: B99FE3D + postfix_smtp_lostconn_data: lost connection + postfix_smtp_lostconn_reason: receiving the initial server greeting postfix_relay_hostname: mx3.hotmail.com postfix_relay_ip: 65.55.37.72 diff --git a/test/smtp_0033.yaml b/test/smtp_0033.yaml new file mode 100644 index 0000000..9ee72d0 --- /dev/null +++ b/test/smtp_0033.yaml @@ -0,0 +1,8 @@ +pattern: ^%{POSTFIX_SMTP}$ +data: "745137AA: lost connection with hotmail-com.olc.protection.outlook.com[104.47.55.161] while sending RCPT TO" +results: + postfix_queueid: 745137AA + postfix_smtp_lostconn_data: lost connection + postfix_smtp_lostconn_reason: sending RCPT TO + postfix_relay_hostname: hotmail-com.olc.protection.outlook.com + postfix_relay_ip: 104.47.55.161 From 296336fa1178b08edb97d2cbe068ea673e7b7f0a Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Mon, 15 Jan 2024 17:26:00 +0100 Subject: [PATCH 4/5] Support for parsing the verify cache cleanup results --- 50-filter-postfix.conf | 9 +++++++++ postfix.grok | 5 +++++ test/verify_0001.yaml | 6 ++++++ test/verify_0002.yaml | 6 ++++++ 4 files changed, 26 insertions(+) create mode 100644 test/verify_0001.yaml create mode 100644 test/verify_0002.yaml diff --git a/50-filter-postfix.conf b/50-filter-postfix.conf index 13f6740..712cfe6 100644 --- a/50-filter-postfix.conf +++ b/50-filter-postfix.conf @@ -175,6 +175,13 @@ filter { tag_on_failure => [ "_grok_postfix_script_nomatch" ] add_tag => [ "_grok_postfix_success" ] } + } else if [program] =~ /^postfix.*\/verify$/ { + grok { + patterns_dir => "/etc/logstash/patterns.d" + match => [ "message", "^%{POSTFIX_VERIFY}$" ] + tag_on_failure => [ "_grok_postfix_verify_nomatch" ] + add_tag => [ "_grok_postfix_success" ] + } } else if [program] =~ /^postfix.*/ { mutate { add_tag => [ "_grok_postfix_program_nomatch" ] @@ -270,6 +277,8 @@ filter { "postfix_size", "integer", "postfix_status_code", "integer", "postfix_termination_signal", "integer", + "postfix_verify_cache_dropped", "integer", + "postfix_verify_cache_retained", "integer", # list of float fields "postfix_delay", "float", diff --git a/postfix.grok b/postfix.grok index f049c0c..9995765 100644 --- a/postfix.grok +++ b/postfix.grok @@ -23,6 +23,7 @@ POSTFIX_TIME_UNIT %{NUMBER}[smhd] POSTFIX_KEYVALUE_DATA [\w-]+=[^;]* POSTFIX_KEYVALUE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} POSTFIX_WARNING_LEVEL (warning|fatal|info) +POSTFIX_VERIFY_CLEANUP_TYPE (full|partial) POSTFIX_TLSCONN (Anonymous|Trusted|Untrusted|Verified) TLS connection established (to %{POSTFIX_RELAY_INFO}|from %{POSTFIX_CLIENT_INFO}): %{DATA:postfix_tls_version} with cipher %{DATA:postfix_tls_cipher} \(%{DATA:postfix_tls_cipher_size} bits\) POSTFIX_TLSVERIFICATION certificate verification failed for %{POSTFIX_RELAY_INFO}: %{GREEDYDATA:postfix_tls_error} @@ -117,6 +118,9 @@ POSTFIX_SCACHE_LOOKUPS statistics: (address|domain) lookup hits=%{INT:postfix_sc POSTFIX_SCACHE_SIMULTANEOUS statistics: max simultaneous domains=%{INT:postfix_scache_domains} addresses=%{INT:postfix_scache_addresses} connection=%{INT:postfix_scache_connection} POSTFIX_SCACHE_TIMESTAMP statistics: start interval %{SYSLOGTIMESTAMP:postfix_scache_timestamp} +# verify patterns +POSTFIX_VERIFY_CACHE cache %{DATA} %{POSTFIX_VERIFY_CLEANUP_TYPE:postfix_verify_cleanup_type} cleanup: retained=%{INT:postfix_verify_cache_retained} dropped=%{INT:postfix_verify_cache_dropped} entries + # aggregate all patterns POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE} POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MESSAGEID}|%{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_CLEANUP_PREPEND}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE} @@ -143,3 +147,4 @@ POSTFIX_ERROR %{POSTFIX_ERROR_ANY} POSTFIX_POSTSUPER %{POSTFIX_POSTSUPER_ACTION}|%{POSTFIX_POSTSUPER_SUMMARY} POSTFIX_POSTMAP %{POSTFIX_WARNING} POSTFIX_SCRIPT %{POSTFIX_WARNING} +POSTFIX_VERIFY %{POSTFIX_VERIFY_CACHE} diff --git a/test/verify_0001.yaml b/test/verify_0001.yaml new file mode 100644 index 0000000..0804e17 --- /dev/null +++ b/test/verify_0001.yaml @@ -0,0 +1,6 @@ +pattern: ^%{POSTFIX_VERIFY}$ +data: "cache lmdb:/var/lib/postfix/verify_cache full cleanup: retained=724 dropped=6 entries" +results: + postfix_verify_cleanup_type: full + postfix_verify_cache_retained: 724 + postfix_verify_cache_dropped: 6 diff --git a/test/verify_0002.yaml b/test/verify_0002.yaml new file mode 100644 index 0000000..cf705d1 --- /dev/null +++ b/test/verify_0002.yaml @@ -0,0 +1,6 @@ +pattern: ^%{POSTFIX_VERIFY}$ +data: "cache ??????????????????????????????????? partial cleanup: retained=1 dropped=0 entries" +results: + postfix_verify_cleanup_type: partial + postfix_verify_cache_retained: 1 + postfix_verify_cache_dropped: 0 From 42a53cdba6ceb77c15d1f0c05c92a8a8b74613a3 Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Mon, 15 Jan 2024 17:56:02 +0100 Subject: [PATCH 5/5] Improve handling of various postfix/smtp delivery errors --- postfix.grok | 5 +++-- test/smtp_0034.yaml | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 test/smtp_0034.yaml diff --git a/postfix.grok b/postfix.grok index 9995765..d8bccff 100644 --- a/postfix.grok +++ b/postfix.grok @@ -29,7 +29,7 @@ POSTFIX_TLSCONN (Anonymous|Trusted|Untrusted|Verified) TLS connection establishe POSTFIX_TLSVERIFICATION certificate verification failed for %{POSTFIX_RELAY_INFO}: %{GREEDYDATA:postfix_tls_error} POSTFIX_DELAYS %{NUMBER:postfix_delay_before_qmgr}/%{NUMBER:postfix_delay_in_qmgr}/%{NUMBER:postfix_delay_conn_setup}/%{NUMBER:postfix_delay_transmission} -POSTFIX_LOSTCONN (Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1|Address not available|Operation timed out) +POSTFIX_LOSTCONN (Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1|Address not available|Operation timed out|Address not available|Operation timed out) POSTFIX_LOSTCONN_REASONS (receiving the initial server greeting|sending message body|sending end of data -- message may be sent more than once|sending %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}) POSTFIX_PROXY_MESSAGE (%{POSTFIX_STATUS_CODE:postfix_proxy_status_code} )?(%{POSTFIX_STATUS_CODE_ENHANCED:postfix_proxy_status_code_enhanced})?.* POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:postfix_cmd_helo} )?(ehlo=(%{INT:postfix_cmd_ehlo_accepted}/)?%{INT:postfix_cmd_ehlo} )?(starttls=(%{INT:postfix_cmd_starttls_accepted}/)?%{INT:postfix_cmd_starttls} )?(auth=(%{INT:postfix_cmd_auth_accepted}/)?%{INT:postfix_cmd_auth} )?(mail=(%{INT:postfix_cmd_mail_accepted}/)?%{INT:postfix_cmd_mail} )?(rcpt=(%{INT:postfix_cmd_rcpt_accepted}/)?%{INT:postfix_cmd_rcpt} )?(bdat=(%{INT:postfix_cmd_bdat_accepted}/)?%{INT:postfix_cmd_bdat} )?(data=(%{INT:postfix_cmd_data_accepted}/)?%{INT:postfix_cmd_data} )?(rset=(%{INT:postfix_cmd_rset_accepted}/)?%{INT:postfix_cmd_rset} )?(noop=(%{INT:postfix_cmd_noop_accepted}/)?%{INT:postfix_cmd_noop} )?(quit=(%{INT:postfix_cmd_quit_accepted}/)?%{INT:postfix_cmd_quit} )?(unknown=(%{INT:postfix_cmd_unknown_accepted}/)?%{INT:postfix_cmd_unknown} )?commands=(%{INT:postfix_cmd_count_accepted}/)?%{INT:postfix_cmd_count} @@ -102,6 +102,7 @@ POSTFIX_SMTP_SSLCONNERR SSL_connect error to %{POSTFIX_RELAY_INFO}: %{POSTFIX_LO POSTFIX_SMTP_LOSTCONN %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} with %{POSTFIX_RELAY_INFO}( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})? POSTFIX_SMTP_TIMEOUT %{POSTFIX_QUEUEID:postfix_queueid}: conversation with %{POSTFIX_RELAY_INFO} timed out( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})? POSTFIX_SMTP_RELAYERR %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY_INFO} said: %{GREEDYDATA:postfix_smtp_response} \(in reply to %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} command\) +POSTFIX_SMTP_RELAYREFUSED %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY_INFO} refused to talk to me: %{GREEDYDATA:postfix_smtp_response} POSTFIX_SMTP_SSLAUTHERR %{POSTFIX_QUEUEID:postfix_queueid}: SASL authentication failed; server %{POSTFIX_RELAY_INFO} said: %{GREEDYDATA:postfix_smtp_response} POSTFIX_SMTP_UTF8 host %{POSTFIX_RELAY_INFO} offers SMTPUTF8 support, but not 8BITMIME POSTFIX_SMTP_PIX %{POSTFIX_QUEUEID:postfix_queueid}: enabling PIX workarounds: %{DATA:postfix_pix_workaround} for %{POSTFIX_RELAY_INFO} @@ -129,7 +130,7 @@ POSTFIX_PIPE %{POSTFIX_PIPE_ANY} POSTFIX_POSTSCREEN %{POSTFIX_PS_CONNECT}|%{POSTFIX_PS_ACCESS}|%{POSTFIX_PS_NOQUEUE}|%{POSTFIX_PS_TOOBUSY}|%{POSTFIX_PS_CACHE}|%{POSTFIX_PS_DNSBL}|%{POSTFIX_PS_VIOLATIONS}|%{POSTFIX_WARNING} POSTFIX_DNSBLOG %{POSTFIX_DNSBLOG_LISTING}|%{POSTFIX_WARNING} POSTFIX_ANVIL %{POSTFIX_ANVIL_CONN_RATE}|%{POSTFIX_ANVIL_CONN_CACHE}|%{POSTFIX_ANVIL_CONN_COUNT} -POSTFIX_SMTP %{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_SSLCONNERR}|%{POSTFIX_SMTP_SSLAUTHERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_SMTP_TIMEOUT}|%{POSTFIX_SMTP_RELAYERR}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTP_UTF8}|%{POSTFIX_TLSVERIFICATION}|%{POSTFIX_SMTP_PIX} +POSTFIX_SMTP %{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_SSLCONNERR}|%{POSTFIX_SMTP_SSLAUTHERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_SMTP_TIMEOUT}|%{POSTFIX_SMTP_RELAYERR}|%{POSTFIX_SMTP_RELAYREFUSED}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTP_UTF8}|%{POSTFIX_TLSVERIFICATION}|%{POSTFIX_SMTP_PIX} POSTFIX_DISCARD %{POSTFIX_DISCARD_ANY}|%{POSTFIX_WARNING} POSTFIX_LMTP %{POSTFIX_SMTP} POSTFIX_PICKUP %{POSTFIX_KEYVALUE} diff --git a/test/smtp_0034.yaml b/test/smtp_0034.yaml new file mode 100644 index 0000000..7fb4046 --- /dev/null +++ b/test/smtp_0034.yaml @@ -0,0 +1,7 @@ +pattern: ^%{POSTFIX_SMTP}$ +data: "40197AA: host mg2.egov.bg[213.91.191.86] refused to talk to me: 450 4.3.2 try again later" +results: + postfix_queueid: 40197AA + postfix_relay_hostname: mg2.egov.bg + postfix_relay_ip: 213.91.191.86 + postfix_smtp_response: 450 4.3.2 try again later