Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Benchmark #83

Closed
wants to merge 2 commits into from
Closed

Benchmark #83

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
67c7288
softreset
babenek Mar 11, 2024
c692f9c
dubfix
babenek Mar 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 43 additions & 23 deletions benchmark/scanner/scanner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,28 +243,55 @@ def check_line_from_meta(self,
file_id = ""
approximate = ""

# todo: use str(line_num) for comparison or integer values in meta
for row in self.meta:
if row["FilePath"] == path:
file_id = row["FileID"]
# by default the cred is false positive
approximate = f"{self.next_id},{file_id},GitHub,{project_id},{path}" \
f",{line_num}:{line_num},F,F,{value_start},{value_end},F,F,,,Info,,0,0,F,F,F,{rule}"
if self._check_line_num(row["LineStart:LineEnd"], line_num):
code = str(project_id) + str(file_id) + str(row["LineStart:LineEnd"])
if code in self.line_checker:
self.result_cnt -= 1
return LineStatus.CHECKED, project_id, file_id
else:
self.line_checker.add(code)

if row["GroundTruth"] == "T":
self.true_cnt += 1
self._increase_result_dict_cnt(row["Category"], True)
return LineStatus.TRUE, project_id, file_id
elif row["GroundTruth"] == "F" or row["GroundTruth"] == "Template":
self.false_cnt += 1
self._increase_result_dict_cnt(row["Category"], False)
return LineStatus.FALSE, project_id, file_id
meta_start_num, meta_end_num = [int(x) for x in row["LineStart:LineEnd"].split(':')]

# check line position
if meta_start_num == meta_end_num:
# single line markup
if line_num != meta_start_num:
# not the line
continue
elif meta_start_num < meta_end_num:
# multiline markup
if not meta_start_num <= line_num <= meta_end_num:
# checked line does not belong to the markup
continue
else:
# wrong markup
print(f"MARKUP ERROR (LineStart:LineEnd): {row}", flush=True)
continue

# check value position if possible. multiline may be without start-end
meta_value_start = int(row["ValueStart"]) if row["ValueStart"] else -1
meta_value_end = int(row["ValueEnd"]) if row["ValueEnd"] else -1

if 0 <= meta_value_start:
# meta start value must be set precisely
if value_start != meta_value_start:
continue

# unique id from meta for single markup (or use just id?)
code = f"{project_id};{file_id};{meta_start_num};{meta_end_num};{meta_value_start};{meta_value_end}"
if code in self.line_checker:
self.result_cnt -= 1
return LineStatus.CHECKED, project_id, file_id
else:
self.line_checker.add(code)
if row["GroundTruth"] == "T":
self.true_cnt += 1
self._increase_result_dict_cnt(row["Category"], True)
return LineStatus.TRUE, project_id, file_id
elif row["GroundTruth"] == "F" or row["GroundTruth"] == "Template":
self.false_cnt += 1
self._increase_result_dict_cnt(row["Category"], False)
return LineStatus.FALSE, project_id, file_id
self.lost_cnt += 1
print(f"LOST: {approximate}", flush=True)
self.next_id += 1
Expand Down Expand Up @@ -338,13 +365,6 @@ def _get_total_true_cnt(self, category: str = None) -> int:
total_true_cnt += 1
return total_true_cnt

@staticmethod
def _check_line_num(line_arrange: str, line_num: int) -> bool:
start_num, end_num = [int(x) for x in line_arrange.split(":")]
if start_num <= line_num <= end_num:
return True
return False

def _increase_result_dict_cnt(self, category: str, cnt_type: bool) -> None:
if category not in self.result_dict:
self.result_dict[category] = TrueFalseCounter()
Expand Down
1 change: 1 addition & 0 deletions meta/23689bd1.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
40745,23c7d6d0,GitHub,23689bd1,data/23689bd1/src/23c7d6d0.ts,141:141,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
40746,23c7d6d0,GitHub,23689bd1,data/23689bd1/src/23c7d6d0.ts,147:147,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
43096,23c7d6d0,GitHub,23689bd1,data/23689bd1/src/23c7d6d0.ts,130:130,T,T,49,113,F,F,Any,,,Secret,3.67,64,F,F,F,Generic Secret
135013,23c7d6d0,GitHub,23689bd1,data/23689bd1/src/23c7d6d0.ts,130:130,F,F,42,48,F,F,,,Info,,0,0,F,F,F,Generic Secret
46520,94864ff4,GitHub,23689bd1,data/23689bd1/test/94864ff4.mjs,297:297,T,F,1,73,F,F,,,,,0,0,F,F,F,Predefined Pattern
46521,94864ff4,GitHub,23689bd1,data/23689bd1/test/94864ff4.mjs,344:344,T,F,1,81,F,F,,,,,0,0,F,F,F,Predefined Pattern
46522,94864ff4,GitHub,23689bd1,data/23689bd1/test/94864ff4.mjs,249:249,T,F,1,73,F,F,,,,,0,0,F,F,F,Predefined Pattern
Expand Down
1 change: 1 addition & 0 deletions meta/255bae6f.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
34186,bbc3a4a5,GitHub,255bae6f,data/255bae6f/src/bbc3a4a5.json,46:46,T,F,17,26,F,F,CharsOnly,,,Token,2.95,9,F,F,F,Generic Token
34200,63061ac6,GitHub,255bae6f,data/255bae6f/src/63061ac6.rb,67:67,T,F,17,27,F,F,Any,,,Token,2.99,10,F,F,F,Generic Token
34578,ced93401,GitHub,255bae6f,data/255bae6f/src/ced93401.rb,10:10,T,F,46,54,F,F,Any,,,Secret,2.50,8,F,F,F,Password
135015,ced93401,GitHub,255bae6f,data/255bae6f/src/ced93401.rb,10:10,T,F,11,19,F,F,Any,,,Secret,0,0,F,F,F,Password
34579,873d5f9d,GitHub,255bae6f,data/255bae6f/src/873d5f9d.rb,16:16,T,F,46,54,F,F,Any,,,Secret,2.50,8,F,F,F,Password
34699,a5f87782,GitHub,255bae6f,data/255bae6f/other/a5f87782.md,240:240,Template,T,98,106,F,F,CharsOnly,,,SeedSalt,2.75,8,F,F,F,Password
34887,8e05158f,GitHub,255bae6f,data/255bae6f/src/8e05158f.rb,79:79,Template,T,75,83,F,F,CharsOnly,,,Token,2.50,8,F,F,F,Generic Token
Expand Down
1 change: 1 addition & 0 deletions meta/2ba83c6a.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9146,6 +9146,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
112479,07d8fbd6,GitHub,2ba83c6a,data/2ba83c6a/test/07d8fbd6.t,63:63,F,F,,,F,F,,,,,0.00,,F,F,F,Generic Secret
112480,07d8fbd6,GitHub,2ba83c6a,data/2ba83c6a/test/07d8fbd6.t,66:66,F,F,,,F,F,,,,,0.00,,F,F,F,Generic Secret
112481,07d8fbd6,GitHub,2ba83c6a,data/2ba83c6a/test/07d8fbd6.t,71:71,T,F,175,207,F,F,,,,,0.00,,F,F,F,Cryptographic Primitives
135016,07d8fbd6,GitHub,2ba83c6a,data/2ba83c6a/test/07d8fbd6.t,71:71,F,F,48,55,F,F,,,,,0,0,F,F,F,Cryptographic Primitives
112482,75550394,GitHub,2ba83c6a,data/2ba83c6a/test/75550394.txt,25:25,F,F,,,F,F,,,,,0.00,,F,F,F,Generic Secret
112483,75550394,GitHub,2ba83c6a,data/2ba83c6a/test/75550394.txt,59:59,F,F,,,F,F,,,,,0.00,,F,F,F,Generic Secret
112484,75550394,GitHub,2ba83c6a,data/2ba83c6a/test/75550394.txt,88:88,F,F,,,F,F,,,,,0.00,,F,F,F,Generic Secret
Expand Down
4 changes: 2 additions & 2 deletions meta/6482c382.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
5177,7d152c3f,GitHub,6482c382,data/6482c382/src/7d152c3f.ts,42:42,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
5187,37cc71c1,GitHub,6482c382,data/6482c382/src/37cc71c1.ts,18:18,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
5347,d2f74f29,GitHub,6482c382,data/6482c382/src/d2f74f29.ts,1551:1551,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
5575,41f75e90,GitHub,6482c382,data/6482c382/src/41f75e90.yml,433:433,T,T,17,30,T,F,Any,,,Secret,2.53,13,F,F,F,Password
5575,41f75e90,GitHub,6482c382,data/6482c382/src/41f75e90.yml,433:433,T,T,31,44,T,F,Any,,,Secret,2.53,13,F,F,F,Password
9298,68a734ce,GitHub,6482c382,data/6482c382/src/68a734ce.template,11:11,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
11493,ce3a851e,GitHub,6482c382,data/6482c382/test/ce3a851e.tsx,89:89,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
12751,324a0f04,GitHub,6482c382,data/6482c382/src/324a0f04.graphql,380:380,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
Expand All @@ -37,7 +37,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
33537,e2425764,GitHub,6482c382,data/6482c382/src/e2425764.yml,22:22,T,F,19,56,F,F,Any,,,Secret,4.52,37,F,F,F,Generic Secret
36999,e2425764,GitHub,6482c382,data/6482c382/src/e2425764.yml,64:64,T,F,19,37,F,F,Any,,,Unknown,3.95,18,F,F,F,Generic Secret
37000,e2425764,GitHub,6482c382,data/6482c382/src/e2425764.yml,65:65,T,F,19,56,F,F,Any,,,Secret,4.52,37,F,F,F,Generic Secret
42664,41f75e90,GitHub,6482c382,data/6482c382/src/41f75e90.yml,456:456,T,T,17,30,T,F,Any,,,Secret,2.53,13,F,F,F,Password
42664,41f75e90,GitHub,6482c382,data/6482c382/src/41f75e90.yml,456:456,T,T,31,44,T,F,Any,,,Secret,2.53,13,F,F,F,Password
47899,71cac11f,GitHub,6482c382,data/6482c382/test/71cac11f.json,13:13,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
74251,e84da162,GitHub,6482c382,data/6482c382/src/e84da162.ts,60:60,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
74252,e84da162,GitHub,6482c382,data/6482c382/src/e84da162.ts,67:67,F,F,,,F,F,,,,,0,0,F,F,F,Generic Secret
Expand Down
9 changes: 6 additions & 3 deletions meta/a8bdca79.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,11 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
28836,a3a7e022,GitHub,a8bdca79,data/a8bdca79/src/a3a7e022.java,186:186,F,F,12,74,F,F,,,,,0,0,F,F,F,Generic Secret
29878,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,213:213,Template,F,17,23,F,F,CharsOnly,,,Secret,2.25,6,F,F,F,Password
29879,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,216:216,Template,F,16,22,F,F,CharsOnly,,,Token,2.25,6,F,F,F,Generic Token
30567,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,148:148,T,F,34,66,F,F,Any,,,SeedSalt,0,1,F,F,F,Cryptographic Primitives
30569,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,113:113,T,F,87,119,F,F,Any,,,SeedSalt,0,1,F,F,F,Cryptographic Primitives
30567,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,148:148,T,F,34,66,F,F,Any,,,SeedSalt,0,1,F,F,F,Authentication Credentials
135268,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,148:148,T,F,88,120,F,F,,,,,0,0,F,F,F,Authentication Credentials
30569,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,113:113,T,F,87,119,F,F,Any,,,SeedSalt,0,1,F,F,F,Authentication Credentials
135266,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,113:113,T,F,37,69,F,F,,,,,0,0,F,F,F,Authentication Credentials
135267,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,113:113,T,F,141,173,F,F,,,,,0,0,F,F,F,Authentication Credentials
32152,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,85:85,T,F,34,66,F,F,CharsOnly,,,SeedSalt,4.05,25,F,F,F,Cryptographic Primitives
32737,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,203:203,F,F,14,42,F,F,Any,,,Unknown,4.59,28,F,F,F,Other
32742,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,29:29,F,F,88,136,F,F,Any,,,Auth,4.84,48,F,F,T,Authentication Credentials
Expand Down Expand Up @@ -44,4 +47,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
108649,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,132:132,F,F,,,F,F,,,,,0.00,,F,F,F,Authentication Credentials
134319,f94e9c95,GitHub,a8bdca79,data/a8bdca79/other/f94e9c95.md,143:143,F,F,,,F,F,,,,,0,0,F,F,F,Authentication Credentials
135005,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,148:148,T,F,138,188,F,F,Any,,,Token,0,50,F,F,F,Generic Token
135006,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,113:113,T,F,191,218,F,F,Any,,,Token,0,27,F,F,F,Generic Token
135006,1a5f2491,GitHub,a8bdca79,data/a8bdca79/test/1a5f2491.java,113:113,T,F,191,218,F,F,Any,,,Token,0,27,F,F,F,Generic Token
2 changes: 1 addition & 1 deletion meta/bd14e9cf.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
63763,cc99cf26,GitHub,bd14e9cf,data/bd14e9cf/src/cc99cf26.sh,1343:1343,F,F,,,F,F,,,,,0,0,F,F,F,Password
64384,cc99cf26,GitHub,bd14e9cf,data/bd14e9cf/src/cc99cf26.sh,1026:1026,F,F,,,F,F,,,,,0,0,F,F,F,Password
74989,9653e9d6,GitHub,bd14e9cf,data/bd14e9cf/src/9653e9d6.sh,27:27,Template,T,28,35,F,F,CharOnly,,,Secret,2.52,7,F,F,F,Password
100325,b16690b7,GitHub,bd14e9cf,data/bd14e9cf/src/b16690b7.sh,146:146,T,F,93,99,F,F,NumberOnly,,,Secret,0.00,0,F,F,F,Password
100325,b16690b7,GitHub,bd14e9cf,data/bd14e9cf/src/b16690b7.sh,146:146,T,F,79,85,F,F,NumberOnly,,,Secret,0.00,0,F,F,F,Password
130048,cc99cf26,GitHub,bd14e9cf,data/bd14e9cf/src/cc99cf26.sh,639:639,F,F,,,F,F,,,,,0.00,,F,F,F,Password
130049,46f3404e,GitHub,bd14e9cf,data/bd14e9cf/src/46f3404e.bat,11:11,F,F,,,F,F,,,,,0.00,,F,F,F,Password
130050,cc99cf26,GitHub,bd14e9cf,data/bd14e9cf/src/cc99cf26.sh,3345:3345,F,F,,,F,F,,,,,0.00,,F,F,F,Password
Expand Down
4 changes: 3 additions & 1 deletion meta/f623c7b3.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,11 @@ Id,FileID,Domain,RepoName,FilePath,LineStart:LineEnd,GroundTruth,WithWords,Value
32631,179cb0ff,GitHub,f623c7b3,data/f623c7b3/test/179cb0ff.py,273:273,T,T,15,26,F,F,Any,,,Secret,2.53,11,F,F,F,Password
33970,2c4cca40,GitHub,f623c7b3,data/f623c7b3/src/2c4cca40.yml,5:5,T,T,19,23,F,F,CharsOnly,,,Secret,2.00,4,F,F,F,Password
34743,179cb0ff,GitHub,f623c7b3,data/f623c7b3/test/179cb0ff.py,290:290,T,F,59,65,F,F,Any,,,Secret,1.72,6,F,F,F,Password
34855,fb4d8904,GitHub,f623c7b3,data/f623c7b3/test/fb4d8904.py,124:124,Template,T,66,74,T,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password
34855,fb4d8904,GitHub,f623c7b3,data/f623c7b3/test/fb4d8904.py,124:124,T,T,66,74,T,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password
135281,fb4d8904,GitHub,f623c7b3,data/f623c7b3/test/fb4d8904.py,124:124,T,F,25,33,F,F,CharsOnly,,,Secret,0,0,F,F,F,Password
34863,fb4d8904,GitHub,f623c7b3,data/f623c7b3/test/fb4d8904.py,122:122,Template,T,15,23,T,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password
34865,fb4d8904,GitHub,f623c7b3,data/f623c7b3/test/fb4d8904.py,139:139,T,T,61,69,T,F,CharsOnly,,,Secret,2.75,8,F,F,F,Password
135282,fb4d8904,GitHub,f623c7b3,data/f623c7b3/test/fb4d8904.py,139:139,T,F,25,33,F,F,CharsOnly,,,Secret,0,0,F,F,F,Password
35352,435497e9,GitHub,f623c7b3,data/f623c7b3/test/435497e9.py,484:484,Template,F,42,46,F,F,CharsOnly,,,Token,2.00,4,F,F,F,Generic Token
35364,435497e9,GitHub,f623c7b3,data/f623c7b3/test/435497e9.py,362:362,Template,F,38,42,F,F,CharsOnly,,,Token,2.00,4,F,F,F,Generic Token
35365,435497e9,GitHub,f623c7b3,data/f623c7b3/test/435497e9.py,374:374,Template,F,35,39,F,F,CharsOnly,,,Token,2.00,4,F,F,F,Generic Token
Expand Down
Loading