Skip to content

Commit

Permalink
Static analysis tool providers from Prospector Project (#866)
Browse files Browse the repository at this point in the history 
* Static analysis tool providers from Prospector Project
Fixes #730
  • Loading branch information
@sourabhsparkala
sourabhsparkala authored Sep 7, 2022
1 parent 38b8bbc commit 0421f12
Show file tree
Hide file tree
Showing 51 changed files with 3,303 additions and 413 deletions.
388 changes: 375 additions & 13 deletions src/main/java/com/sap/oss/phosphor/fosstars/data/AbstractStaticScanToolsDataProvider.java
View file

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions src/main/java/com/sap/oss/phosphor/fosstars/data/DataProviderSelector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import com.sap.oss.phosphor.fosstars.data.github.IsEclipse;
import com.sap.oss.phosphor.fosstars.data.github.LgtmDataProvider;
import com.sap.oss.phosphor.fosstars.data.github.LicenseInfo;
import com.sap.oss.phosphor.fosstars.data.github.MyPyDataProvider;
import com.sap.oss.phosphor.fosstars.data.github.NumberOfCommits;
import com.sap.oss.phosphor.fosstars.data.github.NumberOfContributors;
import com.sap.oss.phosphor.fosstars.data.github.NumberOfDependentProjectOnGitHub;
Expand All @@ -44,6 +45,7 @@
import com.sap.oss.phosphor.fosstars.data.github.OwaspSecurityLibraries;
import com.sap.oss.phosphor.fosstars.data.github.PackageManagement;
import com.sap.oss.phosphor.fosstars.data.github.ProgrammingLanguages;
import com.sap.oss.phosphor.fosstars.data.github.PylintDataProvider;
import com.sap.oss.phosphor.fosstars.data.github.ReadmeInfo;
import com.sap.oss.phosphor.fosstars.data.github.ReleasesFromGitHub;
import com.sap.oss.phosphor.fosstars.data.github.SecurityReviewsFromOpenSSF;
Expand Down Expand Up @@ -239,6 +241,8 @@ public DataProviderSelector(GitHubDataFetcher fetcher, NVD nvd) throws IOExcepti
new VulnerabilitiesFromOwaspDependencyCheck(),
new VulnerabilitiesFromNpmAudit(nvd),
new HasExecutableBinaries(fetcher),
new PylintDataProvider(fetcher),
new MyPyDataProvider(fetcher),
PROJECT_USAGE_PROVIDER,
FUNCTIONALITY_PROVIDER,
HANDLING_UNTRUSTED_DATA_LIKELIHOOD_PROVIDER,
Expand Down
25 changes: 0 additions & 25 deletions src/main/java/com/sap/oss/phosphor/fosstars/data/StaticAnalysisScanTool.java
View file

This file was deleted.

94 changes: 22 additions & 72 deletions src/main/java/com/sap/oss/phosphor/fosstars/data/github/BanditDataProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,31 +10,36 @@
import com.sap.oss.phosphor.fosstars.model.feature.oss.OssFeatures;
import com.sap.oss.phosphor.fosstars.model.subject.oss.GitHubProject;
import com.sap.oss.phosphor.fosstars.model.value.ValueHashSet;
import com.sap.oss.phosphor.fosstars.util.Yaml;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import org.apache.commons.collections4.IteratorUtils;

/**
* The data provider gathers info about how a project uses Bandit for static analysis. In
* particular, it tries to fill out the following features:
* <ul>
* <li>{@link OssFeatures#RUNS_BANDIT_SCANS}</li>
* <li>{@link OssFeatures#USES_BANDIT_SCAN_CHECKS}</li>
* <li>{@link OssFeatures#RUNS_BANDIT_SCANS}</li>
* <li>{@link OssFeatures#USES_BANDIT_SCAN_CHECKS}</li>
* </ul>
*/
public class BanditDataProvider extends AbstractStaticScanToolsDataProvider {

/**
* A step in a GitHub action that triggers analysis with Bandit.
* A Predicate to check the any step in a GitHub action that triggers analysis with Bandit.
*/
private static final Pattern RUN_STEP_BANDIT_REGEX_PATTERN
= Pattern.compile("^.*bandit .*$", Pattern.DOTALL);
private static final Map<String, Predicate<String>> MATCH_BANDIT_PREDICATE = new HashMap<>();

static {
{
MATCH_BANDIT_PREDICATE.put("uses",
step -> Pattern.compile(".*bandit.*$", Pattern.DOTALL).matcher(step).matches());
MATCH_BANDIT_PREDICATE.put("run",
step -> Pattern.compile("^.*bandit .*$", Pattern.DOTALL).matcher(step).matches());
}
}

/**
* Initializes a data provider.
Expand All @@ -51,69 +56,14 @@ protected ValueSet fetchValuesFor(GitHubProject project) throws IOException {

LocalRepository repository = GitHubDataFetcher.localRepositoryFor(project);

Value<Boolean> runsBandit = RUNS_BANDIT_SCANS.value(false);
Value<Boolean> usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(false);

// ideally, we're looking for a GitHub action that runs Bandit scan on pull requests
// but if we just find an action that runs Bandit scans, that's also fine
for (Path configPath : findGitHubActionsIn(repository)) {
try (InputStream content = Files.newInputStream(configPath)) {
Map<String, Object> githubAction = Yaml.readMap(content);
if (triggersScan(githubAction)) {
runsBandit = RUNS_BANDIT_SCANS.value(true);
if (runsOnPullRequests(githubAction)) {
usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(true);
break;
}
}
}
}
Visitor visitor = withVisitor();
browse(repository, MATCH_BANDIT_PREDICATE, Collections.emptyMap(), visitor);

return ValueHashSet.from(runsBandit, usesBanditScanChecks);
}

@Override
public boolean triggersScan(Map<?, ?> githubAction) {
return Optional.ofNullable(githubAction.get("jobs"))
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(jobs -> jobs.values())
.filter(Iterable.class::isInstance)
.map(Iterable.class::cast)
.map(BanditDataProvider::scanJobs)
.orElse(false);
}
Value<Boolean> runsBandit = RUNS_BANDIT_SCANS.value(visitor.runCheck);
Value<Boolean> usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(visitor.usesCheck);

/**
* Checks if any step in a collection of jobs triggers a Bandit scan.
*
* @param jobs The collection of jobs from GitHub action.
* @return True if a step triggers a Bandit scan, false otherwise.
*/
private static boolean scanJobs(Iterable<?> jobs) {
return IteratorUtils.toList(jobs.iterator()).stream()
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(job -> job.get("steps"))
.filter(Iterable.class::isInstance)
.map(Iterable.class::cast)
.anyMatch(BanditDataProvider::hasBanditRunStep);
}

/**
* Checks if a collection of steps from a GitHub action contains a step that triggers a Bandit
* scan.
*
* @param steps The steps to be checked.
* @return True if the steps contain a step that triggers a Bandit scan, false otherwise.
*/
private static boolean hasBanditRunStep(Iterable<?> steps) {
return IteratorUtils.toList(steps.iterator()).stream()
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(step -> step.get("run"))
.filter(String.class::isInstance)
.map(String.class::cast)
.anyMatch(run -> RUN_STEP_BANDIT_REGEX_PATTERN.matcher(run).matches());
return ValueHashSet.from(runsBandit, usesBanditScanChecks);
}
}
}
83 changes: 21 additions & 62 deletions src/main/java/com/sap/oss/phosphor/fosstars/data/github/CodeqlDataProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,31 +10,34 @@
import com.sap.oss.phosphor.fosstars.model.feature.oss.OssFeatures;
import com.sap.oss.phosphor.fosstars.model.subject.oss.GitHubProject;
import com.sap.oss.phosphor.fosstars.model.value.ValueHashSet;
import com.sap.oss.phosphor.fosstars.util.Yaml;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.collections4.IteratorUtils;
import java.util.function.Predicate;

/**
* The data provider gathers info about how a project uses CodeQL for static analysis.
* In particular, it tires to fill out the following features:
* The data provider gathers info about how a project uses CodeQL for static analysis. In
* particular, it tires to fill out the following features:
* <ul>
* <li>{@link OssFeatures#RUNS_CODEQL_SCANS}</li>
* <li>{@link OssFeatures#USES_CODEQL_CHECKS}</li>
* <li>{@link OssFeatures#RUNS_CODEQL_SCANS}</li>
* <li>{@link OssFeatures#USES_CODEQL_CHECKS}</li>
* </ul>
*
* @see LgtmDataProvider
*/
public class CodeqlDataProvider extends AbstractStaticScanToolsDataProvider {

/**
* A step in a GitHub action that triggers analysis with CodeQL.
* A predicate to check if any step in GitHub action that triggers analysis with CodeQL.
*/
private static final String CODEQL_ANALYZE_STEP_TASK = "github/codeql-action/analyze";
private static final Map<String, Predicate<String>> MATCH_CODEQL_ANALYZE_PREDICATE =
new HashMap<>();

static {
MATCH_CODEQL_ANALYZE_PREDICATE.put("uses",
uses -> uses.startsWith("github/codeql-action/analyze"));
}

/**
* Initializes a data provider.
Expand All @@ -51,56 +54,12 @@ protected ValueSet fetchValuesFor(GitHubProject project) throws IOException {

LocalRepository repository = GitHubDataFetcher.localRepositoryFor(project);

Value<Boolean> runsCodeqlScans = RUNS_CODEQL_SCANS.value(false);
Value<Boolean> usesCodeqlChecks = USES_CODEQL_CHECKS.value(false);

// ideally, we're looking for a GitHub action that runs CodeQL scan on pull requests
// but if we just find an action that runs CodeQL scans, that's also fine
for (Path configPath : findGitHubActionsIn(repository)) {
try (InputStream content = Files.newInputStream(configPath)) {
Map<String, Object> githubAction = Yaml.readMap(content);
if (triggersScan(githubAction)) {
runsCodeqlScans = RUNS_CODEQL_SCANS.value(true);
if (runsOnPullRequests(githubAction)) {
usesCodeqlChecks = USES_CODEQL_CHECKS.value(true);
break;
}
}
}
}

return ValueHashSet.from(usesCodeqlChecks, runsCodeqlScans);
}
Visitor visitor = withVisitor();
browse(repository, MATCH_CODEQL_ANALYZE_PREDICATE, Collections.emptyMap(), visitor);

@Override
public boolean triggersScan(Map<?, ?> githubAction) {
return Optional.ofNullable(githubAction.get("jobs"))
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(jobs -> jobs.get("analyze"))
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(jobs -> jobs.get("steps"))
.filter(Iterable.class::isInstance)
.map(Iterable.class::cast)
.map(CodeqlDataProvider::hasCodeqlAnalyzeStep)
.orElse(false);
}
Value<Boolean> runsCodeqlScans = RUNS_CODEQL_SCANS.value(visitor.runCheck);
Value<Boolean> usesCodeqlChecks = USES_CODEQL_CHECKS.value(visitor.usesCheck);

/**
* Checks if a collection of steps from a GitHub action contains a step that triggers
* a CodeQL scan.
*
* @param steps The steps to be checked.
* @return True if the steps contain a step that triggers a CodeQL scan, false otherwise.
*/
private static boolean hasCodeqlAnalyzeStep(Iterable<?> steps) {
return IteratorUtils.toList(steps.iterator()).stream()
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(step -> step.get("uses"))
.filter(String.class::isInstance)
.map(String.class::cast)
.anyMatch(uses -> uses.startsWith(CODEQL_ANALYZE_STEP_TASK));
return ValueHashSet.from(runsCodeqlScans, usesCodeqlChecks);
}
}
}
Loading

0 comments on commit 0421f12

Please sign in to comment.