Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

CLI | Improve Performance In Results Show Command (AST-70015) #928

Merged
merged 9 commits into from
Nov 12, 2024
+261 −127
Merged

CLI | Improve Performance In Results Show Command (AST-70015) #928

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
dc74288
added policy skip when agent is from plugin.
BenAlvo1 Nov 6, 2024
257c169
return nil, error when err is not nil
BenAlvo1 Nov 6, 2024
97ed568
Merge branch 'main' into other/benalvo/optimize-result-show-runtime
AlvoBen Nov 10, 2024
9d2e968
rcreate policy-management service
BenAlvo1 Nov 10, 2024
a85ecdc
added tests
BenAlvo1 Nov 10, 2024
369765a
fix lint
BenAlvo1 Nov 10, 2024
c1378e3
fix lint
BenAlvo1 Nov 10, 2024
b3d5a88
Merge branch 'main' into other/benalvo/optimize-result-show-runtime
OrShamirCM Nov 11, 2024
5236c79
Merge branch 'main' into other/benalvo/optimize-result-show-runtime
AlvoBen Nov 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 72 additions & 91 deletions internal/commands/result.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
"time"

"github.com/MakeNowJust/heredoc"
"github.com/checkmarx/ast-cli/internal/commands/policymanagement"
"github.com/checkmarx/ast-cli/internal/commands/util"
"github.com/checkmarx/ast-cli/internal/commands/util/printer"
errorConstants "github.com/checkmarx/ast-cli/internal/constants/errors"
Expand Down Expand Up @@ -113,62 +112,64 @@ const (
noFileForScorecardResultString = "Issue Found in your GitHub repository"
)

var summaryFormats = []string{
printer.FormatSummaryConsole,
printer.FormatSummary,
printer.FormatSummaryJSON,
printer.FormatPDF,
printer.FormatSummaryMarkdown,
printer.FormatSbom,
printer.FormatGLSast,
printer.FormatGLSca,
}

var filterResultsListFlagUsage = fmt.Sprintf(
"Filter the list of results. Use ';' as the delimiter for arrays. Available filters are: %s",
strings.Join(
[]string{
commonParams.ScanIDQueryParam,
commonParams.LimitQueryParam,
commonParams.OffsetQueryParam,
commonParams.SortQueryParam,
commonParams.IncludeNodesQueryParam,
commonParams.NodeIDsQueryParam,
commonParams.QueryQueryParam,
commonParams.GroupQueryParam,
commonParams.StatusQueryParam,
commonParams.SeverityQueryParam,
commonParams.StateQueryParam,
}, ",",
),
)
var (
summaryFormats = []string{
printer.FormatSummaryConsole,
printer.FormatSummary,
printer.FormatSummaryJSON,
printer.FormatPDF,
printer.FormatSummaryMarkdown,
printer.FormatSbom,
printer.FormatGLSast,
printer.FormatGLSca,
}

// Follows: over 9.0 is critical, 7.0 to 8.9 is high, 4.0 to 6.9 is medium and 3.9 or less is low.
var securities = map[string]string{
infoCx: "1.0",
lowCx: "2.0",
mediumCx: "4.0",
highCx: "7.0",
criticalCx: "9.0",
}
filterResultsListFlagUsage = fmt.Sprintf(
"Filter the list of results. Use ';' as the delimiter for arrays. Available filters are: %s",
strings.Join(
[]string{
commonParams.ScanIDQueryParam,
commonParams.LimitQueryParam,
commonParams.OffsetQueryParam,
commonParams.SortQueryParam,
commonParams.IncludeNodesQueryParam,
commonParams.NodeIDsQueryParam,
commonParams.QueryQueryParam,
commonParams.GroupQueryParam,
commonParams.StatusQueryParam,
commonParams.SeverityQueryParam,
commonParams.StateQueryParam,
}, ",",
),
)

// Match cx severity with sonar severity
var sonarSeverities = map[string]string{
infoCx: infoSonar,
lowCx: lowSonar,
mediumCx: mediumSonar,
highCx: highSonar,
criticalCx: criticalSonar,
}
// Follows: over 9.0 is critical, 7.0 to 8.9 is high, 4.0 to 6.9 is medium and 3.9 or less is low.
securities = map[string]string{
infoCx: "1.0",
lowCx: "2.0",
mediumCx: "4.0",
highCx: "7.0",
criticalCx: "9.0",
}

var containerEngineUnsupportedAgents = []string{
commonParams.JetbrainsAgent, commonParams.VSCodeAgent, commonParams.VisualStudioAgent, commonParams.EclipseAgent,
}
// Match cx severity with sonar severity
sonarSeverities = map[string]string{
infoCx: infoSonar,
lowCx: lowSonar,
mediumCx: mediumSonar,
highCx: highSonar,
criticalCx: criticalSonar,
}

var sscsEngineToOverviewEngineMap = map[string]string{
commonParams.SCSScorecardType: commonParams.SCSScorecardOverviewType,
commonParams.SCSSecretDetectionType: commonParams.SCSSecretDetectionOverviewType,
}
containerEngineUnsupportedAgents = []string{
commonParams.JetbrainsAgent, commonParams.VSCodeAgent, commonParams.VisualStudioAgent, commonParams.EclipseAgent,
}

sscsEngineToOverviewEngineMap = map[string]string{
commonParams.SCSScorecardType: commonParams.SCSScorecardOverviewType,
commonParams.SCSSecretDetectionType: commonParams.SCSSecretDetectionOverviewType,
}
)

func NewResultsCommand(
resultsWrapper wrappers.ResultsWrapper,
Expand Down Expand Up @@ -959,6 +960,9 @@ func runGetResultCommand(
sastRedundancy, _ := cmd.Flags().GetBool(commonParams.SastRedundancyFlag)
agent, _ := cmd.Flags().GetString(commonParams.AgentFlag)
scaHideDevAndTestDep, _ := cmd.Flags().GetBool(commonParams.ScaHideDevAndTestDepFlag)
ignorePolicy, _ := cmd.Flags().GetBool(commonParams.IgnorePolicyFlag)
waitDelay, _ := cmd.Flags().GetInt(commonParams.WaitDelayFlag)
policyTimeout, _ := cmd.Flags().GetInt(commonParams.PolicyTimeoutFlag)

scanID, _ := cmd.Flags().GetString(commonParams.ScanIDFlag)
if scanID == "" {
Expand All @@ -982,42 +986,19 @@ func runGetResultCommand(
return errors.Errorf("%s: CODE: %d, %s", failedGettingScan, errorModel.Code, errorModel.Message)
}

policyResponseModel := &wrappers.PolicyResponseModel{}
policyOverrideFlag, _ := cmd.Flags().GetBool(commonParams.IgnorePolicyFlag)
waitDelay, _ := cmd.Flags().GetInt(commonParams.WaitDelayFlag)
if !policyOverrideFlag {
policyTimeout, _ := cmd.Flags().GetInt(commonParams.PolicyTimeoutFlag)
if policyTimeout < 0 {
return errors.Errorf("--%s should be equal or higher than 0", commonParams.PolicyTimeoutFlag)
}
policyResponseModel, err = policymanagement.HandlePolicyWait(waitDelay, policyTimeout, policyWrapper, scan.ID, scan.ProjectID, cmd)
if err != nil {
return err
}
} else {
logger.PrintIfVerbose("Skipping policy evaluation")
policyResponseModel, err := services.HandlePolicyEvaluation(cmd, policyWrapper, scan, ignorePolicy, agent, waitDelay, policyTimeout)
if err != nil {
return err
}

if sastRedundancy {
resultsParams[commonParams.SastRedundancyFlag] = ""
}

return CreateScanReport(
resultsWrapper,
risksOverviewWrapper,
scsScanOverviewWrapper,
exportWrapper,
policyResponseModel,
resultsPdfReportsWrapper,
scan,
format,
formatPdfToEmail,
formatPdfOptions,
formatSbomOptions,
targetFile,
targetPath,
agent,
resultsParams,
featureFlagsWrapper)
_, err = CreateScanReport(resultsWrapper, risksOverviewWrapper, scsScanOverviewWrapper, exportWrapper,
policyResponseModel, resultsPdfReportsWrapper, scan, format, formatPdfToEmail, formatPdfOptions,
formatSbomOptions, targetFile, targetPath, agent, resultsParams, featureFlagsWrapper)
return err
}
}

Expand Down Expand Up @@ -1124,42 +1105,42 @@ func CreateScanReport(
agent string,
resultsParams map[string]string,
featureFlagsWrapper wrappers.FeatureFlagsWrapper,
) error {
) (*wrappers.ScanResultsCollection, error) {
AlvoBen marked this conversation as resolved.
Show resolved Hide resolved
reportList := strings.Split(reportTypes, ",")
results := &wrappers.ScanResultsCollection{}
setIsSCSEnabled(featureFlagsWrapper)
setIsContainersEnabled(agent, featureFlagsWrapper)
summary, err := convertScanToResultsSummary(scan, resultsWrapper)
if err != nil {
return err
return nil, err
}
scanPending := isScanPending(summary.Status)

err = createDirectory(targetPath)
if err != nil {
return err
return nil, err
}
if !scanPending {
results, err = ReadResults(resultsWrapper, exportWrapper, scan, resultsParams, agent)
if err != nil {
return err
return nil, err
}
}
isSummaryNeeded := verifyFormatsByReportList(reportList, summaryFormats...)
if isSummaryNeeded && !scanPending {
summary, err = summaryReport(summary, policyResponseModel, risksOverviewWrapper, scsScanOverviewWrapper, featureFlagsWrapper, results)
if err != nil {
return err
return nil, err
}
}
for _, reportType := range reportList {
err = createReport(reportType, formatPdfToEmail, formatPdfOptions, formatSbomOptions, targetFile,
targetPath, results, summary, exportWrapper, resultsPdfReportsWrapper, featureFlagsWrapper, agent)
if err != nil {
return err
return nil, err
}
}
return nil
return results, nil
}

func countResult(summary *wrappers.ResultSummary, result *wrappers.ScanResult) {
Expand Down
57 changes: 21 additions & 36 deletions internal/commands/scan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ import (
"github.com/pkg/errors"

"github.com/MakeNowJust/heredoc"
"github.com/checkmarx/ast-cli/internal/commands/policymanagement"
commonParams "github.com/checkmarx/ast-cli/internal/params"
"github.com/checkmarx/ast-cli/internal/wrappers"
"github.com/mssola/user_agent"
Expand Down Expand Up @@ -1682,33 +1681,28 @@ func runCreateScanCommand(
if err != nil {
return err
}
// Handling policy response
policyOverrideFlag, _ := cmd.Flags().GetBool(commonParams.IgnorePolicyFlag)
if !policyOverrideFlag {
policyTimeout, _ := cmd.Flags().GetInt(commonParams.PolicyTimeoutFlag)
if policyTimeout < 0 {
return errors.Errorf("--%s should be equal or higher than 0", commonParams.PolicyTimeoutFlag)
}
policyResponseModel, err = policymanagement.HandlePolicyWait(waitDelay, policyTimeout, policyWrapper, scanResponseModel.ID, scanResponseModel.ProjectID, cmd)
if err != nil {
return err
}
} else {
logger.PrintIfVerbose("Skipping policy evaluation")
}
err = createReportsAfterScan(cmd, scanResponseModel.ID, scansWrapper, exportWrapper, resultsPdfReportsWrapper,
resultsWrapper, risksOverviewWrapper, scsScanOverviewWrapper, policyResponseModel, featureFlagsWrapper)

agent, _ := cmd.Flags().GetString(commonParams.AgentFlag)
ignorePolicy, _ := cmd.Flags().GetBool(commonParams.IgnorePolicyFlag)
policyTimeout, _ := cmd.Flags().GetInt(commonParams.PolicyTimeoutFlag)
policyResponseModel, err = services.HandlePolicyEvaluation(cmd, policyWrapper, scanResponseModel, ignorePolicy, agent, waitDelay, policyTimeout)
if err != nil {
return err
}

err = applyThreshold(cmd, resultsWrapper, exportWrapper, scanResponseModel, thresholdMap, risksOverviewWrapper)
results, reportErr := createReportsAfterScan(cmd, scanResponseModel.ID, scansWrapper, exportWrapper, resultsPdfReportsWrapper,
resultsWrapper, risksOverviewWrapper, scsScanOverviewWrapper, policyResponseModel, featureFlagsWrapper)
if reportErr != nil {
return reportErr
}

err = applyThreshold(cmd, scanResponseModel, thresholdMap, risksOverviewWrapper, results)

if err != nil {
return err
}
} else {
err = createReportsAfterScan(cmd, scanResponseModel.ID, scansWrapper, exportWrapper, resultsPdfReportsWrapper, resultsWrapper,
_, err = createReportsAfterScan(cmd, scanResponseModel.ID, scansWrapper, exportWrapper, resultsPdfReportsWrapper, resultsWrapper,
risksOverviewWrapper, scsScanOverviewWrapper, nil, featureFlagsWrapper)
if err != nil {
return err
Expand Down Expand Up @@ -1907,7 +1901,7 @@ func createReportsAfterScan(
scsScanOverviewWrapper wrappers.ScanOverviewWrapper,
policyResponseModel *wrappers.PolicyResponseModel,
featureFlagsWrapper wrappers.FeatureFlagsWrapper,
) error {
) (*wrappers.ScanResultsCollection, error) {
// Create the required reports
targetFile, _ := cmd.Flags().GetString(commonParams.TargetFlag)
targetPath, _ := cmd.Flags().GetString(commonParams.TargetPathFlag)
Expand All @@ -1920,7 +1914,7 @@ func createReportsAfterScan(

resultsParams, err := getFilters(cmd)
if err != nil {
return err
return nil, err
}

if scaHideDevAndTestDep {
Expand All @@ -1932,10 +1926,10 @@ func createReportsAfterScan(
}
scan, errorModel, scanErr := scansWrapper.GetByID(scanID)
if scanErr != nil {
return errors.Wrapf(scanErr, "%s", failedGetting)
return nil, errors.Wrapf(scanErr, "%s", failedGetting)
}
if errorModel != nil {
return errors.Errorf("%s: CODE: %d, %s", failedGettingScan, errorModel.Code, errorModel.Message)
return nil, errors.Errorf("%s: CODE: %d, %s", failedGettingScan, errorModel.Code, errorModel.Message)
}
return CreateScanReport(
resultsWrapper,
Expand All @@ -1959,24 +1953,22 @@ func createReportsAfterScan(

func applyThreshold(
cmd *cobra.Command,
resultsWrapper wrappers.ResultsWrapper,
exportWrapper wrappers.ExportWrapper,
scanResponseModel *wrappers.ScanResponseModel,
thresholdMap map[string]int,
risksOverviewWrapper wrappers.RisksOverviewWrapper,
results *wrappers.ScanResultsCollection,
) error {
if len(thresholdMap) == 0 {
return nil
}

sastRedundancy, _ := cmd.Flags().GetBool(commonParams.SastRedundancyFlag)
agent, _ := cmd.Flags().GetString(commonParams.AgentFlag)
params := make(map[string]string)
if sastRedundancy {
params[commonParams.SastRedundancyFlag] = ""
}

summaryMap, err := getSummaryThresholdMap(resultsWrapper, exportWrapper, scanResponseModel, params, risksOverviewWrapper, agent)
summaryMap, err := getSummaryThresholdMap(scanResponseModel, risksOverviewWrapper, results)

if err != nil {
return err
Expand Down Expand Up @@ -2061,19 +2053,12 @@ func parseThresholdLimit(limit string) (engineName string, intLimit int, err err
}

func getSummaryThresholdMap(
resultsWrapper wrappers.ResultsWrapper,
exportWrapper wrappers.ExportWrapper,
scan *wrappers.ScanResponseModel,
resultsParams map[string]string,
risksOverviewWrapper wrappers.RisksOverviewWrapper,
agent string,
results *wrappers.ScanResultsCollection,
) (map[string]int, error) {
summaryMap := make(map[string]int)
results, err := ReadResults(resultsWrapper, exportWrapper, scan, resultsParams, agent)

if err != nil {
return nil, err
}
for _, result := range results.Results {
if isExploitable(result.State) {
key := strings.ToLower(fmt.Sprintf("%s-%s", strings.Replace(result.Type, commonParams.KicsType, commonParams.IacType, 1), result.Severity))
Expand Down Expand Up @@ -2176,7 +2161,7 @@ func isScanRunning(
log.Println("Scan Finished with status: ", scanResponseModel.Status)
if scanResponseModel.Status == wrappers.ScanPartial {
_ = printer.Print(cmd.OutOrStdout(), scanResponseModel.StatusDetails, printer.FormatList)
reportErr := createReportsAfterScan(
_, reportErr := createReportsAfterScan(
cmd,
scanResponseModel.ID,
scansWrapper,
Expand Down
Loading
Loading