[sarif] Add Reporting Descriptors & More "Optionality" (#5269)

* [sarif] Add Reporting Descriptors & More "Optionality"
* Added reporting descriptors which allow one to add more meta data to rules, and link findings to a given existing entry.
* Moved the sarif instantiation to the RunBeforeCode object which separates actions from tools deriving from Joern
* Using more "optional" properties where possible on properties which are not required by the sarif schema

* Moved parameter to back

* Test expectations

console/src/main/scala/io/joern/console/BridgeBase.scala

@@ -2,10 +2,12 @@ package io.joern.console
 
 import better.files.*
 import io.shiftleft.codepropertygraph.generated.Languages
+import io.shiftleft.semanticcpg.sarif.SarifConfig
 import org.apache.commons.text.StringEscapeUtils
 import replpp.scripting.ScriptRunner
 
 import java.nio.file.{Files, Path}
+import scala.collection.mutable
 import scala.jdk.CollectionConverters.*
 import scala.util.Try
 
@@ -232,9 +234,6 @@ trait BridgeBase extends InteractiveShell with ScriptExecution with PluginHandli
       builder += s"""openForInputPath("$name")""".stripMargin
     }
     builder ++= config.runBefore
-    builder ++= "import _root_.io.shiftleft.semanticcpg.sarif.SarifConfig"
-      :: "implicit var sarifConfig: SarifConfig = SarifConfig(semanticVersion = version)"
-      :: Nil
     builder.result()
   }
 

joern-cli/src/main/scala/io/joern/joerncli/console/RunBeforeCode.scala

@@ -14,8 +14,10 @@ object RunBeforeCode {
       "import _root_.io.joern.dataflowengineoss.language.*",
       "import _root_.io.shiftleft.semanticcpg.language.*",
       "import scala.jdk.CollectionConverters.*",
+      "import _root_.io.shiftleft.semanticcpg.sarif.SarifConfig",
       "implicit val resolver: ICallResolver = NoResolve",
-      "implicit val finder: NodeExtensionFinder = DefaultNodeExtensionFinder"
+      "implicit val finder: NodeExtensionFinder = DefaultNodeExtensionFinder",
+      "implicit val sarifConfig: SarifConfig = SarifConfig(semanticVersion = Option(version))"
     )
 
   val forInteractiveShell: Seq[String] = {

semanticcpg/src/main/scala/io/shiftleft/semanticcpg/language/SarifExtension.scala

@@ -23,15 +23,20 @@ class SarifExtension(val traversal: Iterator[Finding]) extends AnyVal {
   @Doc(info = "execute this traversal and convert findings to SARIF format")
   def toSarif(implicit config: SarifConfig = SarifConfig()): Sarif = {
 
-    def generateSarif(results: List[SarifSchema.Result], baseUri: Option[URI]): Sarif = {
+    def generateSarif(
+      results: List[SarifSchema.Result],
+      reportingDescriptors: List[SarifSchema.ReportingDescriptor],
+      baseUri: Option[URI]
+    ): Sarif = {
       config.sarifVersion match {
         case SarifVersion.V2_1_0 =>
           val tool = v2_1_0.Schema.ToolComponent(
             name = config.toolName,
             fullName = config.toolFullName,
             organization = config.organization,
             semanticVersion = config.semanticVersion,
-            informationUri = config.toolInformationUri
+            informationUri = config.toolInformationUri,
+            rules = reportingDescriptors
           )
           val projectBaseUri = Map(
             "PROJECT_ROOT" -> v2_1_0.Schema
@@ -47,11 +52,13 @@ class SarifExtension(val traversal: Iterator[Finding]) extends AnyVal {
     }
 
     traversal.l match {
-      case Nil => generateSarif(results = Nil, baseUri = None)
+      case Nil => generateSarif(results = Nil, reportingDescriptors = Nil, baseUri = None)
       case findings @ head :: _ =>
         val baseUri = Cpg(head.graph).metaData.root.headOption.map(java.io.File(_).toURI)
         val results = findings.map(config.resultConverter.convertFindingToResult)
-        generateSarif(results, baseUri)
+        val reportingDescriptors =
+          findings.flatMap(config.resultConverter.convertFindingToReportingDescriptor).distinctBy(_.id)
+        generateSarif(results, reportingDescriptors, baseUri)
     }
 
   }

semanticcpg/src/main/scala/io/shiftleft/semanticcpg/sarif/SarifConfig.scala

@@ -30,10 +30,10 @@ import java.net.URI
   */
 case class SarifConfig(
   toolName: String = "Joern",
-  toolFullName: String = "Joern - The Bug Hunter's Workbench",
-  toolInformationUri: URI = URI("https://joern.io"),
-  organization: String = "Joern.io",
-  semanticVersion: String = "0.0.1",
+  toolFullName: Option[String] = Option("Joern - The Bug Hunter's Workbench"),
+  toolInformationUri: Option[URI] = Option(URI("https://joern.io")),
+  organization: Option[String] = Option("Joern.io"),
+  semanticVersion: Option[String] = None,
   sarifVersion: SarifVersion = SarifVersion.V2_1_0,
   resultConverter: ScanResultToSarifConverter = JoernScanResultToSarifConverter(),
   customSerializers: List[Serializer[?]] = SarifSchema.serializers

semanticcpg/src/main/scala/io/shiftleft/semanticcpg/sarif/SarifSchema.scala

@@ -70,7 +70,7 @@ object SarifSchema {
     /** @return
       *   A message relevant to the code flow.
       */
-    def message: Message
+    def message: Option[Message]
 
     /** @return
       *   An array of one or more unique threadFlow objects, each of which describes the progress of a program through a
@@ -97,6 +97,11 @@ object SarifSchema {
       *   A plain text message string.
       */
     def text: String
+
+    /** @return
+      *   A Markdown message string.
+      */
+    def markdown: Option[String]
   }
 
   /** A physical location relevant to a result. Specifies a reference to a programming artifact together with a range of
@@ -145,6 +150,40 @@ object SarifSchema {
     def snippet: Option[ArtifactContent]
   }
 
+  /** Metadata that describes a specific report produced by the tool, as part of the analysis it provides or its runtime
+    * reporting.
+    */
+  trait ReportingDescriptor private[sarif] {
+
+    /** @return
+      *   A stable, opaque identifier for the report.
+      */
+    def id: String
+
+    /** @return
+      *   A report identifier that is understandable to an end user.
+      */
+    def name: String
+
+    /** @return
+      *   A concise description of the report. Should be a single sentence that is understandable when visible space is
+      *   limited to a single line of text.
+      */
+    def shortDescription: Option[Message]
+
+    /** @return
+      *   A description of the report. Should, as far as possible, provide details sufficient to enable resolution of
+      *   any problem indicated by the result.
+      */
+    def fullDescription: Option[Message]
+
+    /** @return
+      *   A URI where the primary documentation for the report can be found.
+      */
+    def helpUri: Option[URI]
+
+  }
+
   /** A result produced by an analysis tool.
     */
   trait Result private[sarif] {
@@ -247,22 +286,27 @@ object SarifSchema {
       *   The name of the tool component along with its version and any other useful identifying information, such as
       *   its locale.
       */
-    def fullName: String
+    def fullName: Option[String]
 
     /** @return
       *   The organization or company that produced the tool component.
       */
-    def organization: String
+    def organization: Option[String]
 
     /** @return
       *   The tool component version in the format specified by Semantic Versioning 2.0.
       */
-    def semanticVersion: String
+    def semanticVersion: Option[String]
 
     /** @return
       *   The absolute URI at which information about this version of the tool component can be found.
       */
-    def informationUri: URI
+    def informationUri: Option[URI]
+
+    /** @return
+      *   An array of reportingDescriptor objects relevant to the analysis performed by the tool component.
+      */
+    def rules: List[ReportingDescriptor]
   }
 
   /** A value specifying the severity level of the result.
@@ -311,6 +355,19 @@ object SarifSchema {
         }
       )
     ),
+    new CustomSerializer[SarifSchema.CodeFlow](implicit format =>
+      (
+        { case _ =>
+          ???
+        },
+        { case flow: SarifSchema.CodeFlow =>
+          val elementMap = Map.newBuilder[String, Any]
+          flow.message.foreach(x => elementMap.addOne("message" -> x))
+          elementMap.addOne("threadFlows" -> flow.threadFlows)
+          Extraction.decompose(elementMap.result())
+        }
+      )
+    ),
     new CustomSerializer[SarifSchema.Region](implicit format =>
       (
         { case _ =>
@@ -327,6 +384,39 @@ object SarifSchema {
         }
       )
     ),
+    new CustomSerializer[ReportingDescriptor](implicit format =>
+      (
+        { case _ =>
+          ???
+        },
+        { case x: ReportingDescriptor =>
+          val elementMap = Map.newBuilder[String, Any]
+          elementMap.addOne("id"   -> x.id)
+          elementMap.addOne("name" -> x.name)
+          x.shortDescription.foreach(x => elementMap.addOne("shortDescription" -> x))
+          x.fullDescription.foreach(x => elementMap.addOne("fullDescription" -> x))
+          x.helpUri.foreach(x => elementMap.addOne("helpUri" -> x))
+          Extraction.decompose(elementMap.result())
+        }
+      )
+    ),
+    new CustomSerializer[ToolComponent](implicit format =>
+      (
+        { case _ =>
+          ???
+        },
+        { case x: ToolComponent =>
+          val elementMap = Map.newBuilder[String, Any]
+          elementMap.addOne("name" -> x.name)
+          x.fullName.foreach(x => elementMap.addOne("fullName" -> x))
+          x.organization.foreach(x => elementMap.addOne("organization" -> x))
+          x.semanticVersion.foreach(x => elementMap.addOne("semanticVersion" -> x))
+          x.informationUri.foreach(x => elementMap.addOne("informationUri" -> x))
+          elementMap.addOne("rules" -> x.rules)
+          Extraction.decompose(elementMap.result())
+        }
+      )
+    ),
     new CustomSerializer[URI](implicit format =>
       (
         { case _ =>

semanticcpg/src/main/scala/io/shiftleft/semanticcpg/sarif/ScanResultToSarifConverter.scala

@@ -1,12 +1,20 @@
 package io.shiftleft.semanticcpg.sarif
 
 import io.shiftleft.codepropertygraph.generated.nodes.*
-import io.shiftleft.semanticcpg.sarif.SarifSchema.Result
+import io.shiftleft.semanticcpg.sarif.SarifSchema.{ReportingDescriptor, Result}
 
 /** A component that converts a CPG finding to some version of SARIF.
   */
 trait ScanResultToSarifConverter {
 
+  /** Given a finding, will extract any rule data and create a SARIF ReportingDescriptor
+    * @param finding
+    *   the finding to convert.
+    * @return
+    *   a SARIF compliant reporting descriptor object if possible.
+    */
+  def convertFindingToReportingDescriptor(finding: Finding): Option[ReportingDescriptor]
+
   /** Given a finding, will convert it to the SARIF specified result.
     * @param finding
     *   the finding to convert.

semanticcpg/src/main/scala/io/shiftleft/semanticcpg/sarif/v2_1_0/JoernScanResultToSarifConverter.scala

@@ -1,8 +1,8 @@
 package io.shiftleft.semanticcpg.sarif.v2_1_0
 
 import io.shiftleft.codepropertygraph.generated.nodes.*
-import io.shiftleft.semanticcpg.language.*
-import io.shiftleft.semanticcpg.sarif.{ScanResultToSarifConverter, SarifSchema}
+import io.shiftleft.semanticcpg.language.{NodeExtensionFinder, *}
+import io.shiftleft.semanticcpg.sarif.{SarifSchema, ScanResultToSarifConverter}
 
 import java.net.URI
 
@@ -12,28 +12,42 @@ class JoernScanResultToSarifConverter extends ScanResultToSarifConverter {
 
   import JoernScanResultToSarifConverter.*
 
+  override def convertFindingToReportingDescriptor(finding: Finding): Option[SarifSchema.ReportingDescriptor] = {
+    val description = createMessage(finding.description)
+    Option(Schema.ReportingDescriptor(id = finding.name, name = finding.title, fullDescription = Option(description)))
+  }
+
   override def convertFindingToResult(finding: Finding): SarifSchema.Result = {
     val locations        = finding.evidence.lastOption.map(nodeToLocation).toList
     val relatedLocations = finding.evidence.headOption.map(nodeToLocation).toList
+    val codeFlows = evidenceToCodeFlow(finding) match {
+      case codeFlow if codeFlow.threadFlows.isEmpty => Nil
+      case codeFlow                                 => codeFlow :: Nil
+    }
     Schema.Result(
       ruleId = finding.name,
       message = Schema.Message(text = finding.title),
       level = SarifSchema.Level.cvssToLevel(finding.score),
       locations = locations,
       relatedLocations = relatedLocations,
-      codeFlows = evidenceToCodeFlow(finding) :: Nil
+      codeFlows = codeFlows
     )
   }
 
   protected def evidenceToCodeFlow(finding: Finding): Schema.CodeFlow = {
-    Schema.CodeFlow(
-      message = Schema.Message(text = finding.description),
-      threadFlows = Schema.ThreadFlow(
+    Schema.CodeFlow(threadFlows =
+      Schema.ThreadFlow(
         finding.evidence.map(node => Schema.ThreadFlowLocation(location = nodeToLocation(node))).l
       ) :: Nil
     )
   }
 
+  protected def createMessage(text: String): Schema.Message = {
+    val plain    = text.replace("`", "")              // todo: use better markdown stripping
+    val markdown = Option(text).filterNot(_ == plain) // if these are equal, ignore
+    Schema.Message(text = plain, markdown = markdown)
+  }
+
   protected def nodeToLocation(node: StoredNode): Schema.Location = {
     Schema.Location(physicalLocation =
       Schema.PhysicalLocation(
@@ -97,12 +111,12 @@ object JoernScanResultToSarifConverter {
 
     def title: String = getValue(FindingKeys.title)
 
-    def description: String = getValue(FindingKeys.description)
+    def description: String = getValue(FindingKeys.description).trim
 
     def score: Double = getValue(FindingKeys.score).toDoubleOption.getOrElse(-1d)
 
     protected def getValue(key: String, default: String = "<empty>"): String =
-      node.keyValuePairs.find(_.key == key).map(_.value).getOrElse(default)
+      node.keyValuePairs.find(_.key == key).map(_.value).filterNot(_ == "-").getOrElse(default)
 
   }
 }

semanticcpg/src/main/scala/io/shiftleft/semanticcpg/sarif/v2_1_0/Schema.scala

@@ -21,11 +21,11 @@ object Schema {
   final case class ArtifactLocation(uri: Option[URI] = None, uriBaseId: Option[String] = Option("PROJECT_ROOT"))
       extends SarifSchema.ArtifactLocation
 
-  final case class CodeFlow(message: Message, threadFlows: List[ThreadFlow]) extends SarifSchema.CodeFlow
+  final case class CodeFlow(threadFlows: List[ThreadFlow], message: Option[Message] = None) extends SarifSchema.CodeFlow
 
   final case class Location(physicalLocation: PhysicalLocation) extends SarifSchema.Location
 
-  final case class Message(text: String) extends SarifSchema.Message
+  final case class Message(text: String, markdown: Option[String] = None) extends SarifSchema.Message
 
   final case class PhysicalLocation(artifactLocation: ArtifactLocation, region: Region)
       extends SarifSchema.PhysicalLocation
@@ -38,6 +38,14 @@ object Schema {
     snippet: Option[ArtifactContent] = None
   ) extends SarifSchema.Region
 
+  final case class ReportingDescriptor(
+    id: String,
+    name: String,
+    shortDescription: Option[Message] = None,
+    fullDescription: Option[Message] = None,
+    helpUri: Option[URI] = None
+  ) extends SarifSchema.ReportingDescriptor
+
   final case class Result(
     ruleId: String,
     message: Message,
@@ -58,10 +66,11 @@ object Schema {
 
   final case class ToolComponent(
     name: String,
-    fullName: String,
-    organization: String,
-    semanticVersion: String,
-    informationUri: URI
+    fullName: Option[String] = None,
+    organization: Option[String] = None,
+    semanticVersion: Option[String] = None,
+    informationUri: Option[URI] = None,
+    rules: List[SarifSchema.ReportingDescriptor] = Nil
   ) extends SarifSchema.ToolComponent
 
 }