From 551c109e4086388aa90ffdfb00ff67d9a569d1f1 Mon Sep 17 00:00:00 2001 From: DaveYesland Date: Wed, 17 Jan 2024 08:50:53 -0800 Subject: [PATCH 1/6] Add check for OpenSSH key block --- pacu/core/secretfinder/regexs.json | 1 + 1 file changed, 1 insertion(+) diff --git a/pacu/core/secretfinder/regexs.json b/pacu/core/secretfinder/regexs.json index 7ffec95a..5b85f759 100644 --- a/pacu/core/secretfinder/regexs.json +++ b/pacu/core/secretfinder/regexs.json @@ -24,6 +24,7 @@ "SSH (DSA) private key": "-----BEGIN DSA PRIVATE KEY-----", "SSH (EC) private key": "-----BEGIN EC PRIVATE KEY-----", "PGP private key block": "-----BEGIN PGP PRIVATE KEY BLOCK-----", + "OpenSSH private key": "-----BEGIN OPENSSH PRIVATE KEY-----", "Twilio API Key": "SK[0-9a-fA-F]{32}", "Twitter Access Token": "[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*[1-9][0-9]+-[0-9a-zA-Z]{40}", "Twitter OAuth": "[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*['|\"][0-9a-zA-Z]{35,44}['|\"]" From 3f30131e499e20bc3b9efd68afdce4fbf355f4b0 Mon Sep 17 00:00:00 2001 From: DaveYesland Date: Wed, 17 Jan 2024 09:38:11 -0800 Subject: [PATCH 2/6] Add github regexs --- pacu/core/secretfinder/regexs.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pacu/core/secretfinder/regexs.json b/pacu/core/secretfinder/regexs.json index 5b85f759..7ab461f0 100644 --- a/pacu/core/secretfinder/regexs.json +++ b/pacu/core/secretfinder/regexs.json @@ -5,7 +5,10 @@ "Bearer_Auth": "bearer [a-zA-Z0-9_\\-\\.=]+", "AWS_Client": "(?:A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}", "AWS_MWS": "amzn\\.mws\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", - "GitHub": "[g|G][i|I][t|T][h|H][u|U][b|B].*['|\"][0-9a-zA-Z]{35,40}['|\"]", + "GitHub Generic": "[g|G][i|I][t|T][h|H][u|U][b|B].*['|\"][0-9a-zA-Z]{35,40}['|\"]", + "GitHub Personal Token": "ghp_[a-zA-Z0-9]{36}", + "GitHub Actions Token": "ghs_[a-zA-Z0-9]{36}", + "GitHub Fine-grained Token": "github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}", "Generic API Key": "[a|A][p|P][i|I][_]?[k|K][e|E][y|Y].*['|\"][0-9a-zA-Z]{32,45}['|\"]", "Generic Secret": "[s|S][e|E][c|C][r|R][e|E][t|T].*['|\"][0-9a-zA-Z]{32,45}['|\"]", "Google API Key": "AIza[0-9A-Za-z\\-_]{35}", From aceca30d6c95734b6d123410808ec721617782c0 Mon Sep 17 00:00:00 2001 From: DaveYesland Date: Wed, 17 Jan 2024 11:02:20 -0800 Subject: [PATCH 3/6] Update regexs --- pacu/core/secretfinder/regexs.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pacu/core/secretfinder/regexs.json b/pacu/core/secretfinder/regexs.json index 7ab461f0..88bb84ec 100644 --- a/pacu/core/secretfinder/regexs.json +++ b/pacu/core/secretfinder/regexs.json @@ -9,8 +9,8 @@ "GitHub Personal Token": "ghp_[a-zA-Z0-9]{36}", "GitHub Actions Token": "ghs_[a-zA-Z0-9]{36}", "GitHub Fine-grained Token": "github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}", - "Generic API Key": "[a|A][p|P][i|I][_]?[k|K][e|E][y|Y].*['|\"][0-9a-zA-Z]{32,45}['|\"]", - "Generic Secret": "[s|S][e|E][c|C][r|R][e|E][t|T].*['|\"][0-9a-zA-Z]{32,45}['|\"]", + "Generic API Key": "[a|A][p|P][i|I].?[k|K][e|E][y|Y].*['|\"][a-zA-Z0-9!@#$%^*()&]{0,60}['|\"]", + "Generic Secret": "['|\"][s|S][e|E][c|C][r|R][e|E][t|T].*['|\"].*[a-zA-Z0-9!@#$%^*()&]{0,60}['|\"]", "Google API Key": "AIza[0-9A-Za-z\\-_]{35}", "Google Cloud Platform API Key": "AIza[0-9A-Za-z\\-_]{35}", "Google Cloud Platform OAuth": "[0-9]+-[0-9A-Za-z_]{32}\\.apps\\.googleusercontent\\.com", @@ -29,6 +29,5 @@ "PGP private key block": "-----BEGIN PGP PRIVATE KEY BLOCK-----", "OpenSSH private key": "-----BEGIN OPENSSH PRIVATE KEY-----", "Twilio API Key": "SK[0-9a-fA-F]{32}", - "Twitter Access Token": "[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*[1-9][0-9]+-[0-9a-zA-Z]{40}", - "Twitter OAuth": "[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*['|\"][0-9a-zA-Z]{35,44}['|\"]" + "Twitter Access Token": "[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*[1-9][0-9]+-[0-9a-zA-Z]{40}" } \ No newline at end of file From e0dbeb9991e1ff0c461d80c166a45b4b305fdcd3 Mon Sep 17 00:00:00 2001 From: DaveYesland Date: Wed, 17 Jan 2024 11:02:38 -0800 Subject: [PATCH 4/6] Create tests for secretfinder --- tests/test_secretfinder_regex_checker.py | 50 ++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/test_secretfinder_regex_checker.py diff --git a/tests/test_secretfinder_regex_checker.py b/tests/test_secretfinder_regex_checker.py new file mode 100644 index 00000000..a4517824 --- /dev/null +++ b/tests/test_secretfinder_regex_checker.py @@ -0,0 +1,50 @@ +import pytest +from pacu.core.secretfinder.utils import regex_checker + +# Test data +TEST_DATA = [ + ("This is just a normal string.", False, None), + ("My AWS client ID is AKIA1234567890ABCDEF", True, "AWS_Client"), + ("GitHub token: ghp_1234567890abcdef1234567890abcdef1234", True, "GitHub Personal Token"), + ("GitHub token: ghs_1234567890abcdefghijklmnopqrstuvwxyzAB", True, "GitHub Actions Token"), + ("GitHub token: github_pat_abcdefghijABCDEFGHIJ12_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567", True, "GitHub Fine-grained Token"), + ("'GitHub': 'abcdefghijklmnopqrstuvwx1234567890123456'", True, "GitHub Generic"), + ("This text contains an IP: 192.168.1.1", True, "IPv4"), + ("Artifactory API key: AKCabcdefghijABCDEFGHIJ1234567890", True, "Artifactory_API"), + ("Artifactory Password: AP1ABCDabcdefghijABCDEFGHIJ12", True, "Artifactory_Password"), + ("Authorization: basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==", True, "Basic_Auth"), + ("Authorization: bearer QWxhZGRpbjpvcGVuIHNlc2FtZQ==", True, "Bearer_Auth"), + ("AWS MWS Key: amzn.mws.4ea38b7b-6c4c-4d0f-a6f2-123456789012", True, "AWS_MWS"), + ("Generic API Key: 'api_key' = 'abcdefghijklmnopqrstuvwxyz12345678901234567890ABCDEF\"", True, "Generic API Key"), + ("Generic Secret: 'secret_key'= '_ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890123456789@$%^&0abcdef\"", True, "Generic Secret"), + ("Google API Key: AIzaSyC123456789012345678901234567890123", True, "Google API Key"), + ("Google Cloud Platform API Key: AIzaSyC123456789012345678901234567890123", True, "Google Cloud Platform API Key"), + ("Google Cloud Platform OAuth: 123456789012-abcd1234abcd1234abcd1234abcd1234.apps.googleusercontent.com", True, "Google Cloud Platform OAuth"), + ("Google Drive API Key: AIzaSyC123456789012345678901234567890123", True, "Google Drive API Key"), + ("Google Drive OAuth: 123456789012-abcd1234abcd1234abcd1234abcd1234.apps.googleusercontent.com", True, "Google Drive OAuth"), + ("Google Service Account: \"type\": \"service_account\"", True, "Google (GCP) Service-account"), + ("HEROKU API Key: hEroKu_1234567-ABC12345-AB12-CD34-12AB-123456789ABA", True, "HEROKU_API"), + ("IPv4 address: 192.168.1.1", True, "IPv4"), + ("MAILGUN API Key: key-abcdefghijklmnopqrstuvwxyz1234567890123456", True, "MAILGUN_API"), + ("MD5 Hash: 5f4dcc3b5aa765d61d8327deb882cf99", True, "MD5"), + ("Slack Token: xoxb-123456789012-ABCDEFGHIJKLMNOPQRSTUVWX", True, "SLACK_TOKEN"), + ("Slack Webhook: https://hooks.slack.com/services/T12345678/B12345678/abcdefghijklmnopqrstuvwxyz123456", True, "SLACK_WEBHOOK"), + ("RSA Private Key: -----BEGIN RSA PRIVATE KEY-----", True, "RSA private key"), + ("DSA Private Key: -----BEGIN DSA PRIVATE KEY-----", True, "SSH (DSA) private key"), + ("EC Private Key: -----BEGIN EC PRIVATE KEY-----", True, "SSH (EC) private key"), + ("PGP Private Key: -----BEGIN PGP PRIVATE KEY BLOCK-----", True, "PGP private key block"), + ("OpenSSH Private Key: -----BEGIN OPENSSH PRIVATE KEY-----", True, "OpenSSH private key"), + ("Twilio API Key: SKAbCdEf123AbCdEf123AbCdEf123AbCdE", True, "Twilio API Key"), + ("Twitter Access Token: twitter_1234567890123456789012345678901234567890-1234567890123456789012345678901234567890", True, "Twitter Access Token"), +] + +@pytest.mark.parametrize("test_input,expected_result,expected_key", TEST_DATA) +def test_regex_checker(test_input, expected_result, expected_key): + result = regex_checker(test_input) + + if expected_result: + assert result + assert expected_key in result + else: + assert not result + From 2e9a5d1e9162f0cd95ce537b0d07dfa3f8dcbd1f Mon Sep 17 00:00:00 2001 From: Dave Yesland <41924355+DaveYesland@users.noreply.github.com> Date: Thu, 18 Jan 2024 08:19:17 -0800 Subject: [PATCH 5/6] Update regexs.json --- pacu/core/secretfinder/regexs.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pacu/core/secretfinder/regexs.json b/pacu/core/secretfinder/regexs.json index 2e43cbf2..b26beb94 100644 --- a/pacu/core/secretfinder/regexs.json +++ b/pacu/core/secretfinder/regexs.json @@ -26,7 +26,7 @@ "RSA private key": "-----BEGIN RSA PRIVATE KEY-----", "SSH (DSA) private key": "-----BEGIN DSA PRIVATE KEY-----", "SSH (EC) private key": "-----BEGIN EC PRIVATE KEY-----", - "SSH (ed25519) private key": "-----BEGIN OPENSSH PRIVATE KEY-----" + "SSH (ed25519) private key": "-----BEGIN OPENSSH PRIVATE KEY-----", "PGP private key block": "-----BEGIN PGP PRIVATE KEY BLOCK-----", "Twilio API Key": "SK[0-9a-fA-F]{32}", "Twitter Access Token": "[t|T][w|W][i|I][t|T][t|T][e|E][r|R].*[1-9][0-9]+-[0-9a-zA-Z]{40}" From 88019d4e96225d262592bfe9eaffa9139d002f5b Mon Sep 17 00:00:00 2001 From: Dave Yesland <41924355+DaveYesland@users.noreply.github.com> Date: Thu, 18 Jan 2024 08:21:35 -0800 Subject: [PATCH 6/6] Update test_secretfinder_regex_checker.py --- tests/test_secretfinder_regex_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_secretfinder_regex_checker.py b/tests/test_secretfinder_regex_checker.py index a4517824..6d61404e 100644 --- a/tests/test_secretfinder_regex_checker.py +++ b/tests/test_secretfinder_regex_checker.py @@ -33,7 +33,7 @@ ("DSA Private Key: -----BEGIN DSA PRIVATE KEY-----", True, "SSH (DSA) private key"), ("EC Private Key: -----BEGIN EC PRIVATE KEY-----", True, "SSH (EC) private key"), ("PGP Private Key: -----BEGIN PGP PRIVATE KEY BLOCK-----", True, "PGP private key block"), - ("OpenSSH Private Key: -----BEGIN OPENSSH PRIVATE KEY-----", True, "OpenSSH private key"), + ("SSH (ed25519) private key: -----BEGIN OPENSSH PRIVATE KEY-----", True, "SSH (ed25519) private key"), ("Twilio API Key: SKAbCdEf123AbCdEf123AbCdEf123AbCdE", True, "Twilio API Key"), ("Twitter Access Token: twitter_1234567890123456789012345678901234567890-1234567890123456789012345678901234567890", True, "Twitter Access Token"), ]