Skip to content

Commit

Permalink
corrections and migration to various path (#164)
Browse files Browse the repository at this point in the history 
* correction_and_file_various_path

* correection

* corrections

* missed

* BM upd

* BM upd 2

* markup for missed

* upd
  • Loading branch information
@babenek
babenek authored Aug 29, 2024
1 parent c0e6d9c commit 8566105
Show file tree
Hide file tree
Showing 16 changed files with 52 additions and 39 deletions.
38 changes: 19 additions & 19 deletions .ci/benchmark.txt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
META MD5 67039fe64aba3375bbcf27f16984acc5
DATA MD5 4833f5614e463ecc7989b00a29499240
DATA: 16345157 interested lines. MARKUP: 62644 items
META MD5 a563fa1445f92b235930b2a4b82a379f
DATA MD5 ffb6d9cefac6009b3f35b77e7edf63df
DATA: 16345157 interested lines. MARKUP: 62651 items
FileType FileNumber ValidLines Positives Negatives Templates
--------------- ------------ ------------ ----------- ----------- -----------
194 28318 66 414 85
Expand Down Expand Up @@ -29,7 +29,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.cmd 4 401 2 3
.cnf 8 858 15 36 16
.coffee 1 585 2
.conf 60 4945 53 67 53
.conf 60 4945 55 64 53
.config 20 492 16 38 1
.cpp 15 5688 2 61
.creds 1 10 1 1
Expand Down Expand Up @@ -63,7 +63,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.gd 1 37 1
.gml 3 3075 16
.gni 3 5017 19
.go 1080 566476 692 4117 739
.go 1080 566476 692 4123 738
.golden 5 1168 1 13 29
.gradle 45 3265 4 90 100
.graphql 7 420 13
Expand All @@ -82,7 +82,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.ipynb 1 134 5
.j 1 241 2 2
.j2 30 5530 6 186 10
.java 621 134132 360 1366 171
.java 621 134132 358 1368 171
.jenkinsfile 1 58 2 6
.jinja2 1 64 2
.js 659 536413 535 2489 330
Expand All @@ -91,7 +91,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.jsx 7 857 19
.jwt 1 1 2
.key 83 2737 70 14
.kt 123 20774 67 379 3
.kt 123 20774 65 381 3
.l 1 982 1
.las 1 6656 35
.lasso 1 230 7
Expand Down Expand Up @@ -124,7 +124,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.mqh 1 1023 2
.msg 1 26644 1 1
.mysql 1 36 2
.ndjson 2 5006 69 237 2
.ndjson 2 5006 72 239 2
.nix 4 211 12
.nolint 1 2 1
.odd 1 1281 43
Expand All @@ -134,7 +134,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.patch 4 109405 4 27
.pbxproj 1 941 2
.pem 48 1169 47 8
.php 371 75710 128 1619 79
.php 371 75710 129 1618 79
.pl 16 14727 6 34
.pm 3 744 7
.po 3 2994 15
Expand All @@ -158,7 +158,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.pyx 2 1094 23
.r 4 62 6 3 1
.rake 2 51 2
.rb 860 131838 258 3311 613
.rb 860 131838 259 3311 613
.re 1 31 1
.red 1 159 1
.release 1 13 4
Expand All @@ -171,7 +171,7 @@ FileType FileNumber ValidLines Positives Negatives Templat
.rs 31 9855 2 233 11
.rsc 1 691 1
.rsp 16 7101 19 10 28
.rst 86 33980 70 321 68
.rst 86 33980 69 321 68
.rules 1 6 2
.sample 2 25 3 4 4
.sbt 3 570 5 2
Expand Down Expand Up @@ -222,16 +222,16 @@ FileType FileNumber ValidLines Positives Negatives Templat
.yml 418 36162 545 892 380
.zsh 6 872 12
.zsh-theme 1 97 1
TOTAL: 10259 16345157 12150 50325 5111
TOTAL: 10259 16345157 12152 50333 5110
credsweeper result_cnt : 0, lost_cnt : 0, true_cnt : 0, false_cnt : 0
Rules Positives Negatives Templates Reported TP FP TN FN FPR FNR ACC PRC RCL F1
------------------------------ ----------- ----------- ----------- ---------- ---- ---- ----- ----- -------- -------- -------- ----- -------- ----
API 128 3161 189 0 0 3350 128 0.000000 1.000000 0.963197 0.000000
API 128 3162 189 0 0 3351 128 0.000000 1.000000 0.963208 0.000000
AWS Client ID 167 21 0 0 0 21 167 0.000000 1.000000 0.111702 0.000000
AWS Multi 75 16 0 0 0 16 75 0.000000 1.000000 0.175824 0.000000
AWS S3 Bucket 66 24 0 0 0 24 66 0.000000 1.000000 0.266667 0.000000
Atlassian Old PAT token 27 308 3 0 0 311 27 0.000000 1.000000 0.920118 0.000000
Auth 418 2727 76 0 0 2803 418 0.000000 1.000000 0.870227 0.000000
Auth 420 2730 76 0 0 2806 420 0.000000 1.000000 0.869808 0.000000
Azure Access Token 19 0 0 0 0 0 19 1.000000 0.000000 0.000000
BASE64 Private Key 7 4 0 0 0 4 7 0.000000 1.000000 0.363636 0.000000
BASE64 encoded PEM Private Key 7 0 0 0 0 0 7 1.000000 0.000000 0.000000
Expand All @@ -253,17 +253,17 @@ Grafana Provisioned API Key 22 1 0
JSON Web Token 170 61 0 0 0 61 170 0.000000 1.000000 0.264069 0.000000
Jira / Confluence PAT token 0 4 0 0 0 4 0 0.000000 1.000000
Jira 2FA 15 6 1 0 0 7 15 0.000000 1.000000 0.318182 0.000000
Key 3918 15693 482 0 0 16175 3918 0.000000 1.000000 0.805007 0.000000
Key 3918 15695 482 0 0 16177 3918 0.000000 1.000000 0.805026 0.000000
Nonce 91 49 0 0 0 49 91 0.000000 1.000000 0.350000 0.000000
Other 0 8291 1 0 0 8292 0 0.000000 1.000000
PEM Private Key 1019 1483 0 0 0 1483 1019 0.000000 1.000000 0.592726 0.000000
Password 1843 7527 2711 0 0 10238 1843 0.000000 1.000000 0.847446 0.000000
Password 1843 7529 2711 0 0 10240 1843 0.000000 1.000000 0.847472 0.000000
Salt 45 76 2 0 0 78 45 0.000000 1.000000 0.634146 0.000000
Secret 1297 1575 799 0 0 2374 1297 0.000000 1.000000 0.646690 0.000000
Seed 1 6 0 0 0 6 1 0.000000 1.000000 0.857143 0.000000
Slack Token 4 1 0 0 0 1 4 0.000000 1.000000 0.200000 0.000000
Token 648 4177 438 0 0 4615 648 0.000000 1.000000 0.876876 0.000000
Token 648 4176 438 0 0 4614 648 0.000000 1.000000 0.876853 0.000000
Twilio API Key 0 5 2 0 0 7 0 0.000000 1.000000
URL Credentials 208 145 225 0 0 370 208 0.000000 1.000000 0.640138 0.000000
URL Credentials 208 146 224 0 0 370 208 0.000000 1.000000 0.640138 0.000000
UUID 1069 265 0 0 0 265 1069 0.000000 1.000000 0.198651 0.000000
12150 50325 5111 0 0 0 50325 12150 0.000000 1.000000 0.805522 0.000000
12152 50333 5110 0 0 0 50333 12152 0.000000 1.000000 0.805521 0.000000
3 changes: 3 additions & 0 deletions download_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,9 @@ def gen_random_value(value):
if '-' == i and len(value) in (18, 36) and n in (8, 13, 18, 23):
# UUID separator or something like this
continue
if ':' == i and 2 == n % 3:
# wifi key like 7f:44:52:fe: ...
continue
if hex_upper and i not in "0123456789ABCDEF":
hex_upper = False
if hex_lower and i not in "0123456789abcdef":
Expand Down
1 change: 1 addition & 0 deletions meta/1ce69180.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -629,3 +629,4 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
114065,5e754a57,GitHub,1ce69180,data/1ce69180/src/5e754a57.rb,30,30,T,F,50,80,F,F,,,,,0.0,-1,F,F,F,Auth
114066,fcf0ceac,GitHub,1ce69180,data/1ce69180/src/fcf0ceac.txt,2,2,F,F,,,F,F,,,,,0.00,,F,F,F,Auth
131923,1e5305b3,GitHub,1ce69180,data/1ce69180/src/1e5305b3.rb,36,36,F,F,90,330,F,F,,,,,0.0,-1,F,F,F,JSON Web Token
1479368,560ba91b,GitHub,1ce69180,data/1ce69180/src/560ba91b.rb,14,14,T,F,116,124,F,F,,,,,0.0,0,F,F,F,Auth
2 changes: 2 additions & 0 deletions meta/28728ab4.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -838,3 +838,5 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
1341508,76a6fc48,GitHub,28728ab4,data/28728ab4/other/76a6fc48.md,83,83,F,F,40,83,F,F,,,,,0.0,0,F,F,F,Credential
1341509,d3e2cec1,GitHub,28728ab4,data/28728ab4/src/d3e2cec1.toml,41,41,F,F,40,83,F,F,,,,,0.0,0,F,F,F,Credential
1341513,56575ef0,GitHub,28728ab4,data/28728ab4/test/56575ef0.go,89,89,F,F,17,85,F,F,,,,,0.0,0,F,F,F,API
1479369,2ac10c69,GitHub,28728ab4,data/28728ab4/src/2ac10c69.go,70,70,F,F,,,F,F,,,,,0.0,0,F,F,F,Auth
1479370,bf78cace,GitHub,28728ab4,data/28728ab4/src/bf78cace.go,30,30,F,F,,,F,F,,,,,0.0,0,F,F,F,Auth
2 changes: 1 addition & 1 deletion meta/2df212a2.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
30545,6ac10689,GitHub,2df212a2,data/2df212a2/src/6ac10689.rst,120,120,Template,F,26,30,F,F,Any,,,Secret,2.0,4,F,F,F,Password:Key:Secret
30547,6ac10689,GitHub,2df212a2,data/2df212a2/src/6ac10689.rst,124,124,Template,F,26,30,F,F,Any,,,Secret,2.0,4,F,F,F,Password:Key:Secret
30548,7824b4fc,GitHub,2df212a2,data/2df212a2/test/7824b4fc.py,28,28,Template,F,62,68,F,F,Any,,,Secret,2.58,6,F,F,F,Credential:Key
31125,6f0a498c,GitHub,2df212a2,data/2df212a2/test/6f0a498c.rst,25,25,T,T,14,34,F,F,Any,,AWS Client ID,Unknown,3.68,20,F,F,F,AWS Client ID:Gitlab Feed Token:Key
31125,6f0a498c,GitHub,2df212a2,data/2df212a2/test/6f0a498c.rst,25,25,T,T,14,34,F,F,Any,,AWS Client ID,Unknown,3.68,20,F,F,F,AWS Client ID:Gitlab Feed Token
32689,7824b4fc,GitHub,2df212a2,data/2df212a2/test/7824b4fc.py,68,68,Template,T,35,40,F,F,CharsOnly,,,Token,2.32,5,F,F,F,Token
32690,6ac10689,GitHub,2df212a2,data/2df212a2/src/6ac10689.rst,62,62,F,F,26,39,F,F,,,,,0.0,0,F,F,F,Token
36742,7824b4fc,GitHub,2df212a2,data/2df212a2/test/7824b4fc.py,93,93,F,F,30,35,F,F,,,,,0.0,0,F,F,F,Token
Expand Down
2 changes: 1 addition & 1 deletion meta/35c273e2.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
84944,7849399f,GitHub,35c273e2,data/35c273e2/src/7849399f.java,155,155,F,F,36,40,F,F,,,,,2.0,4,F,F,F,Credential
84945,9c5fb3d1,GitHub,35c273e2,data/35c273e2/src/9c5fb3d1.xml,12,12,F,F,169,189,F,F,,,,,3.17,20,F,F,F,Credential
84946,29edcedd,GitHub,35c273e2,data/35c273e2/test/29edcedd.java,27,27,T,F,45,81,F,F,,,,,3.51,36,F,F,F,Token:UUID
84947,470893ef,GitHub,35c273e2,data/35c273e2/test/470893ef.java,27,27,F,F,26,37,F,F,,,,,2.6,11,F,F,F,Password
84947,470893ef,GitHub,35c273e2,data/35c273e2/test/470893ef.java,27,27,F,F,26,35,F,F,,,,,2.6,11,F,F,F,Password
84948,6e4997a7,GitHub,35c273e2,data/35c273e2/test/6e4997a7.groovy,34,34,T,F,48,84,F,F,,,,,3.29,36,F,F,F,Token:UUID
84949,805bb559,GitHub,35c273e2,data/35c273e2/test/805bb559.java,124,124,T,F,54,67,F,F,,,,,2.74,13,F,F,F,Credential
84950,d1839abf,GitHub,35c273e2,data/35c273e2/test/d1839abf.java,90,90,F,F,29,34,F,F,,,,,1.46,5,F,F,F,Auth
Expand Down
8 changes: 4 additions & 4 deletions meta/48fd3902.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
86772,6ae2e7e4,GitHub,48fd3902,data/48fd3902/test/6ae2e7e4.java,30,30,F,F,42,53,F,F,,,,,2.96,11,F,F,F,Password
86773,f23339e5,GitHub,48fd3902,data/48fd3902/test/f23339e5.java,41,41,F,F,13,23,F,F,,,,,2.59,10,F,F,F,Nonce
86774,73bf4bdd,GitHub,48fd3902,data/48fd3902/test/73bf4bdd.java,15,15,F,F,17,21,F,F,,,,,2.0,4,F,F,F,Key
86775,73bf4bdd,GitHub,48fd3902,data/48fd3902/test/73bf4bdd.java,336,336,T,F,11,70,F,F,,,,,2.99,59,F,F,F,Key
86776,73bf4bdd,GitHub,48fd3902,data/48fd3902/test/73bf4bdd.java,383,383,T,F,11,70,F,F,,,,,2.83,59,F,F,F,Key
86775,73bf4bdd,GitHub,48fd3902,data/48fd3902/test/73bf4bdd.java,336,336,F,F,11,70,F,F,,,,,2.99,59,F,F,F,Key
86776,73bf4bdd,GitHub,48fd3902,data/48fd3902/test/73bf4bdd.java,383,383,F,F,11,70,F,F,,,,,2.83,59,F,F,F,Key
86777,73bf4bdd,GitHub,48fd3902,data/48fd3902/test/73bf4bdd.java,420,420,F,F,23,31,F,F,,,,,2.5,8,F,F,F,Key
86778,7cd2d29d,GitHub,48fd3902,data/48fd3902/test/7cd2d29d.java,812,812,F,F,81,90,F,F,,,,,2.6,9,F,F,F,Credential:Certificate
86779,839267fb,GitHub,48fd3902,data/48fd3902/test/839267fb.kt,11,11,F,F,57,66,F,F,,,,,2.6,9,F,F,F,Credential
Expand All @@ -153,8 +153,8 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
86796,c3d109ca,GitHub,48fd3902,data/48fd3902/test/c3d109ca.java,359,359,F,F,70,82,F,F,,,,,2.79,12,F,F,F,Password
86797,c6175528,GitHub,48fd3902,data/48fd3902/test/c6175528.kt,296,296,F,F,31,50,F,F,,,,,3.17,19,F,F,F,Password
86798,c709b566,GitHub,48fd3902,data/48fd3902/test/c709b566.kt,15,15,F,F,23,27,F,F,,,,,2.0,4,F,F,F,Key
86799,c709b566,GitHub,48fd3902,data/48fd3902/test/c709b566.kt,321,321,T,F,17,76,F,F,,,,,2.99,59,F,F,F,Key
86800,c709b566,GitHub,48fd3902,data/48fd3902/test/c709b566.kt,367,367,T,F,17,76,F,F,,,,,2.83,59,F,F,F,Key
86799,c709b566,GitHub,48fd3902,data/48fd3902/test/c709b566.kt,321,321,F,F,17,76,F,F,,,,,2.99,59,F,F,F,Key
86800,c709b566,GitHub,48fd3902,data/48fd3902/test/c709b566.kt,367,367,F,F,17,76,F,F,,,,,2.83,59,F,F,F,Key
86801,c709b566,GitHub,48fd3902,data/48fd3902/test/c709b566.kt,402,402,F,F,29,37,F,F,,,,,2.5,8,F,F,F,Key
86802,cc3a23ed,GitHub,48fd3902,data/48fd3902/test/cc3a23ed.kt,68,68,F,F,33,46,F,F,,,,,2.9,13,F,F,F,Password
86803,d4ef4fe5,GitHub,48fd3902,data/48fd3902/test/d4ef4fe5.java,119,119,F,F,35,44,F,F,,,,,2.24,9,F,F,F,Credential
Expand Down
2 changes: 1 addition & 1 deletion meta/60f9915d.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ Id,FileID,Domain,RepoName,FilePath,LineStart,LineEnd,GroundTruth,WithWords,Value
82149,4b667fea,GitHub,60f9915d,data/60f9915d/src/4b667fea.go,1176,1176,F,F,22,41,F,F,,,,,0.0,0,F,F,F,Password
82150,4b667fea,GitHub,60f9915d,data/60f9915d/src/4b667fea.go,1316,1316,F,F,22,41,F,F,,,,,0.0,0,F,F,F,Password
82151,4dc56e64,GitHub,60f9915d,data/60f9915d/test/4dc56e64.go,186,186,Template,T,40,46,F,F,CharsOnly,,,Secret,2.25,6,F,F,F,Password
82152,373f5578,GitHub,60f9915d,data/60f9915d/test/373f5578.go,51,51,Template,F,22,25,T,F,CharsOnly,,,Secret,1.58,3,F,F,F,URL Credentials
82152,373f5578,GitHub,60f9915d,data/60f9915d/test/373f5578.go,51,51,F,F,,,T,F,CharsOnly,,,Secret,1.58,3,F,F,F,URL Credentials:Password
100788,9870bda5,GitHub,60f9915d,data/60f9915d/src/9870bda5.sum,174,174,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Other
100789,9870bda5,GitHub,60f9915d,data/60f9915d/src/9870bda5.sum,175,175,F,F,-1,-1,F,F,,,,,0.0,-1,F,F,F,Other
110372,9870bda5,GitHub,60f9915d,data/60f9915d/src/9870bda5.sum,241,241,F,F,,,F,F,,,,,0.00,,F,F,F,Auth
Expand Down
Loading

0 comments on commit 8566105

Please sign in to comment.