refactor(vcs): VCS configuration to support common and custom attributes

While VCS implementations are already plugins, they are not yet
configurable. VCS implementations require common configurations
(e.g., `revision`, `recursive`) and should support
implementation-specific configurations that are unique to each VCS.

This refactoring encapsulates common configurations as attributes of a
`VersionControlSystemConfiguration` data class, while
implementation-specific configurations are stored generically in an
`options` attribute.

Fixes oss-review-toolkit#8556

Signed-off-by: Wolfgang Klenk <[email protected]>

downloader/src/main/kotlin/VersionControlSystem.kt

@@ -248,7 +248,10 @@ abstract class VersionControlSystem(
 
         for ((index, revision) in revisionCandidates.withIndex()) {
             logger.info { "Trying revision candidate '$revision' (${index + 1} of ${revisionCandidates.size})..." }
-            results += updateWorkingTree(workingTree, revision, pkg.vcsProcessed.path, recursive)
+            results += updateWorkingTree(
+                workingTree,
+                VersionControlSystemConfiguration(revision, pkg.vcsProcessed.path, recursive)
+            )
             if (results.last().isSuccess) break
         }
 
@@ -379,16 +382,9 @@ abstract class VersionControlSystem(
     abstract fun initWorkingTree(targetDir: File, vcs: VcsInfo): WorkingTree
 
     /**
-     * Update the [working tree][workingTree] by checking out the given [revision], optionally limited to the given
-     * [path] and [recursively][recursive] updating any nested working trees. Return a [Result] that encapsulates the
-     * originally requested [revision] on success, or the occurred exception on failure.
+     * Update the [working tree][workingTree] using a VCS configuration [config].
      */
-    abstract fun updateWorkingTree(
-        workingTree: WorkingTree,
-        revision: String,
-        path: String = "",
-        recursive: Boolean = false
-    ): Result<String>
+    abstract fun updateWorkingTree(workingTree: WorkingTree, config: VersionControlSystemConfiguration): Result<String>
 
     /**
      * Check whether the given [revision] is likely to name a fixed revision that does not move. Return a [Result] with

downloader/src/main/kotlin/VersionControlSystemConfiguration.kt

@@ -0,0 +1,51 @@
+/*
+* Copyright (C) 2024 The ORT Project Authors (see <https://github.com/oss-review-toolkit/ort/blob/main/NOTICE>)
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     https://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+* SPDX-License-Identifier: Apache-2.0
+* License-Filename: LICENSE
+*/
+
+package org.ossreviewtoolkit.downloader
+
+import org.ossreviewtoolkit.utils.common.Options
+
+/**
+ * The configuration for a version control system (VCS).
+ * Contains common configuration data for all VCS implementations,
+ * and implementation-specific configuration [options].
+ */
+data class VersionControlSystemConfiguration(
+
+    /**
+     * [revision] to check-out from the VCS.
+     */
+    val revision: String,
+
+    /**
+     * Optional: The check-out can be limited to the given [path].
+     */
+    val path: String = "",
+
+    /**
+     * Optional: Enable check-out of any nested working trees (recursively) if [recursive] is set to true.
+     */
+    val recursive: Boolean = false,
+
+    /**
+     * Custom implementation-specific configuration options for the VCS.
+     * See the documentation of the respective class for available options.
+     */
+    val options: Options? = null
+)

downloader/src/test/kotlin/VersionControlSystemTest.kt

@@ -166,8 +166,6 @@ private class VersionControlSystemTestImpl(
 
     override fun updateWorkingTree(
         workingTree: WorkingTree,
-        revision: String,
-        path: String,
-        recursive: Boolean
+        config: VersionControlSystemConfiguration
     ): Result<String> = Result.failure(UnsupportedOperationException("Unexpected invocation."))
 }

plugins/package-configuration-providers/ort-config/src/main/kotlin/OrtConfigPackageConfigurationProvider.kt

@@ -24,6 +24,7 @@ import java.io.File
 import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.downloader.VersionControlSystem
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.model.Identifier
 import org.ossreviewtoolkit.model.Provenance
 import org.ossreviewtoolkit.model.VcsInfo
@@ -78,7 +79,10 @@ private fun updateOrtConfig(dir: File) {
 
     vcs.apply {
         val workingTree = initWorkingTree(dir, vcsInfo)
-        val revision = updateWorkingTree(workingTree, ORT_CONFIG_REPOSITORY_BRANCH).getOrThrow()
+        val revision = updateWorkingTree(
+            workingTree,
+            VersionControlSystemConfiguration(ORT_CONFIG_REPOSITORY_BRANCH)
+        ).getOrThrow()
         logger.info {
             "Successfully cloned $revision from $ORT_CONFIG_REPOSITORY_URL."
         }

plugins/package-curation-providers/ort-config/src/main/kotlin/OrtConfigPackageCurationProvider.kt

@@ -25,6 +25,7 @@ import java.io.IOException
 import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.downloader.VersionControlSystem
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.model.Identifier
 import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.PackageCuration
@@ -96,7 +97,10 @@ private fun updateOrtConfig(dir: File) {
 
     vcs.apply {
         val workingTree = initWorkingTree(dir, vcsInfo)
-        val revision = updateWorkingTree(workingTree, ORT_CONFIG_REPOSITORY_BRANCH).getOrThrow()
+        val revision = updateWorkingTree(
+            workingTree,
+            VersionControlSystemConfiguration(ORT_CONFIG_REPOSITORY_BRANCH)
+        ).getOrThrow()
         logger.info {
             "Successfully cloned $revision from $ORT_CONFIG_REPOSITORY_URL."
         }

plugins/version-control-systems/git/src/funTest/kotlin/GitFunTest.kt

@@ -26,6 +26,7 @@ import io.kotest.matchers.shouldBe
 
 import java.io.File
 
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.downloader.WorkingTree
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
@@ -62,21 +63,21 @@ class GitFunTest : WordSpec({
     "updateWorkingTree" should {
         "update the working tree to the correct revision" {
             branches.values.forEach { revision ->
-                git.updateWorkingTree(workingTree, revision) shouldBeSuccess revision
+                git.updateWorkingTree(workingTree, VersionControlSystemConfiguration(revision)) shouldBeSuccess revision
                 workingTree.getRevision() shouldBe revision
             }
         }
 
         "update the working tree to the correct tag" {
             tags.forEach { (tag, revision) ->
-                git.updateWorkingTree(workingTree, tag) shouldBeSuccess tag
+                git.updateWorkingTree(workingTree, VersionControlSystemConfiguration(tag)) shouldBeSuccess tag
                 workingTree.getRevision() shouldBe revision
             }
         }
 
         "update the working tree to the correct branch" {
             branches.forEach { (branch, revision) ->
-                git.updateWorkingTree(workingTree, branch) shouldBeSuccess branch
+                git.updateWorkingTree(workingTree, VersionControlSystemConfiguration(branch)) shouldBeSuccess branch
                 workingTree.getRevision() shouldBe revision
             }
         }
@@ -85,10 +86,10 @@ class GitFunTest : WordSpec({
             val branch = "branch1"
             val revision = branches.getValue(branch)
 
-            git.updateWorkingTree(workingTree, branch)
+            git.updateWorkingTree(workingTree, VersionControlSystemConfiguration(branch))
             GitCommand.run("reset", "--hard", "HEAD~1", workingDir = repoDir)
 
-            git.updateWorkingTree(workingTree, branch) shouldBeSuccess branch
+            git.updateWorkingTree(workingTree, VersionControlSystemConfiguration(branch)) shouldBeSuccess branch
             workingTree.getRevision() shouldBe revision
         }
     }

plugins/version-control-systems/git/src/funTest/kotlin/GitWorkingTreeFunTest.kt

@@ -29,6 +29,7 @@ import io.kotest.matchers.shouldBe
 import java.io.File
 
 import org.ossreviewtoolkit.downloader.VersionControlSystem
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.downloader.WorkingTree
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
@@ -47,7 +48,7 @@ class GitWorkingTreeFunTest : StringSpec({
     beforeSpec {
         println("Cloning ${vcsInfo.url} to '$repoDir'...")
         workingTree = git.initWorkingTree(repoDir, vcsInfo)
-        git.updateWorkingTree(workingTree, "main")
+        git.updateWorkingTree(workingTree, VersionControlSystemConfiguration("main"))
     }
 
     "Git detects non-working-trees" {

plugins/version-control-systems/git/src/main/kotlin/Git.kt

@@ -45,6 +45,7 @@ import org.eclipse.jgit.transport.sshd.ServerKeyDatabase
 import org.eclipse.jgit.transport.sshd.SshdSessionFactory
 
 import org.ossreviewtoolkit.downloader.VersionControlSystem
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.downloader.WorkingTree
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
@@ -168,19 +169,17 @@ class Git : VersionControlSystem(GitCommand) {
 
     override fun updateWorkingTree(
         workingTree: WorkingTree,
-        revision: String,
-        path: String,
-        recursive: Boolean
+        config: VersionControlSystemConfiguration
     ): Result<String> =
         (workingTree as GitWorkingTree).useRepo {
             Git(this).use { git ->
                 logger.info { "Updating working tree from ${workingTree.getRemoteUrl()}." }
 
-                updateWorkingTreeWithoutSubmodules(workingTree, git, revision).mapCatching {
+                updateWorkingTreeWithoutSubmodules(workingTree, git, config.revision).mapCatching {
                     // In case this throws the exception gets encapsulated as a failure.
-                    if (recursive) updateSubmodules(workingTree)
+                    if (config.recursive) updateSubmodules(workingTree)
 
-                    revision
+                    config.revision
                 }
             }
         }

plugins/version-control-systems/git/src/main/kotlin/GitRepo.kt

@@ -33,6 +33,7 @@ import org.apache.logging.log4j.kotlin.logger
 import org.eclipse.jgit.lib.SymbolicRef
 
 import org.ossreviewtoolkit.downloader.VersionControlSystem
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.downloader.WorkingTree
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
@@ -190,11 +191,9 @@ class GitRepo : VersionControlSystem(GitRepoCommand) {
 
     override fun updateWorkingTree(
         workingTree: WorkingTree,
-        revision: String,
-        path: String,
-        recursive: Boolean
+        config: VersionControlSystemConfiguration
     ): Result<String> {
-        val manifestRevision = revision.takeUnless { it.isBlank() }
+        val manifestRevision = config.revision.takeUnless { it.isBlank() }
         val manifestPath = workingTree.getInfo().url.parseRepoManifestPath()
 
         val manifestOptions = listOfNotNull(
@@ -211,7 +210,7 @@ class GitRepo : VersionControlSystem(GitRepoCommand) {
             // want to be able to download such projects, so specify "--force-sync" to work around that issue.
             val syncArgs = mutableListOf("sync", "-c", "--force-sync")
 
-            if (recursive) syncArgs += "--fetch-submodules"
+            if (config.recursive) syncArgs += "--fetch-submodules"
 
             runRepo(workingTree.workingDir, *syncArgs.toTypedArray())
 
@@ -225,7 +224,7 @@ class GitRepo : VersionControlSystem(GitRepoCommand) {
                 "Failed to sync the working tree$revisionDetails$pathDetails: ${e.collectMessages()}"
             }
         }.map {
-            revision
+            config.revision
         }
     }
 

plugins/version-control-systems/mercurial/src/main/kotlin/Mercurial.kt

@@ -24,6 +24,7 @@ import java.io.File
 import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.downloader.VersionControlSystem
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.downloader.WorkingTree
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
@@ -93,18 +94,18 @@ class Mercurial : VersionControlSystem(MercurialCommand) {
         return getWorkingTree(targetDir)
     }
 
-    override fun updateWorkingTree(workingTree: WorkingTree, revision: String, path: String, recursive: Boolean) =
+    override fun updateWorkingTree(workingTree: WorkingTree, config: VersionControlSystemConfiguration) =
         runCatching {
             // To safe network bandwidth, only pull exactly the revision we want. Do not use "-u" to update the
             // working tree just yet, as Mercurial would only update if new changesets were pulled. But that might
             // not be the case if the requested revision is already available locally.
-            MercurialCommand.run(workingTree.getRootPath(), "pull", "-r", revision)
+            MercurialCommand.run(workingTree.getRootPath(), "pull", "-r", config.revision)
 
             // TODO: Implement updating of subrepositories.
 
             // Explicitly update the working tree to the desired revision.
-            MercurialCommand.run(workingTree.getRootPath(), "update", revision).isSuccess
+            MercurialCommand.run(workingTree.getRootPath(), "update", config.revision).isSuccess
         }.map {
-            revision
+            config.revision
         }
 }

plugins/version-control-systems/subversion/src/main/kotlin/Subversion.kt

@@ -28,6 +28,7 @@ import java.nio.file.Paths
 import org.apache.logging.log4j.kotlin.logger
 
 import org.ossreviewtoolkit.downloader.VersionControlSystem
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.downloader.WorkingTree
 import org.ossreviewtoolkit.model.VcsInfo
 import org.ossreviewtoolkit.model.VcsType
@@ -126,21 +127,21 @@ class Subversion : VersionControlSystem() {
         return pathRevisions.single()
     }
 
-    override fun updateWorkingTree(workingTree: WorkingTree, revision: String, path: String, recursive: Boolean) =
+    override fun updateWorkingTree(workingTree: WorkingTree, config: VersionControlSystemConfiguration) =
         runCatching {
             // Note that the path should never be part of the URL as that would root the working tree at that path, but
             // the path should be available in the working tree.
-            val (svnUrl, svnRevision) = revision.toLongOrNull()?.let { numericRevision ->
+            val (svnUrl, svnRevision) = config.revision.toLongOrNull()?.let { numericRevision ->
                 val url = workingTree.getRemoteUrl()
 
                 SVNURL.parseURIEncoded(url) to SVNRevision.create(numericRevision)
             } ?: run {
-                val url = listOf(workingTree.getRemoteUrl(), revision).joinToString("/")
+                val url = listOf(workingTree.getRemoteUrl(), config.revision).joinToString("/")
 
                 SVNURL.parseURIEncoded(url) to SVNRevision.HEAD
             }
 
-            clientManager.updateClient.isIgnoreExternals = !recursive
+            clientManager.updateClient.isIgnoreExternals = !config.recursive
 
             logger.info {
                 val printableRevision = svnRevision.name ?: svnRevision.number
@@ -153,16 +154,16 @@ class Subversion : VersionControlSystem() {
                 svnUrl,
                 /* pegRevision = */ SVNRevision.HEAD,
                 /* revision = */ svnRevision,
-                if (path.isEmpty()) SVNDepth.INFINITY else SVNDepth.EMPTY,
+                if (config.path.isEmpty()) SVNDepth.INFINITY else SVNDepth.EMPTY,
                 /* allowUnversionedObstructions = */ false,
                 /* depthIsSticky = */ true
             )
 
             logger.info { "$type working tree '${workingTree.workingDir}' is at revision $workingTreeRevision." }
 
-            if (path.isNotEmpty()) {
-                logger.info { "Deepening path '$path' in $type working tree '${workingTree.workingDir}'." }
-                val pathRevision = deepenWorkingTreePath(workingTree, path, svnRevision)
+            if (config.path.isNotEmpty()) {
+                logger.info { "Deepening path '${config.path}' in $type working tree '${workingTree.workingDir}'." }
+                val pathRevision = deepenWorkingTreePath(workingTree, config.path, svnRevision)
                 check(pathRevision == workingTreeRevision)
             }
 

scanner/src/main/kotlin/provenance/NestedProvenanceResolver.kt

@@ -21,6 +21,7 @@ package org.ossreviewtoolkit.scanner.provenance
 
 import org.apache.logging.log4j.kotlin.logger
 
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.downloader.WorkingTreeCache
 import org.ossreviewtoolkit.model.ArtifactProvenance
 import org.ossreviewtoolkit.model.KnownProvenance
@@ -78,8 +79,10 @@ class DefaultNestedProvenanceResolver(
         return workingTreeCache.use(provenance.vcsInfo) { vcs, workingTree ->
             vcs.updateWorkingTree(
                 workingTree,
-                provenance.resolvedRevision,
-                recursive = true
+                VersionControlSystemConfiguration(
+                    revision = provenance.resolvedRevision,
+                    recursive = true
+                )
             ).onFailure { throw it }
 
             val subRepositories = workingTree.getNested().mapValues { (_, nestedVcs) ->

scanner/src/main/kotlin/provenance/PackageProvenanceResolver.kt

@@ -26,6 +26,7 @@ import okhttp3.Request
 
 import org.apache.logging.log4j.kotlin.logger
 
+import org.ossreviewtoolkit.downloader.VersionControlSystemConfiguration
 import org.ossreviewtoolkit.downloader.WorkingTreeCache
 import org.ossreviewtoolkit.model.ArtifactProvenance
 import org.ossreviewtoolkit.model.KnownProvenance
@@ -228,7 +229,7 @@ class DefaultPackageProvenanceResolver(
 
             revisionCandidates.forEachIndexed { index, revision ->
                 logger.info { "Trying revision candidate '$revision' (${index + 1} of ${revisionCandidates.size})." }
-                val result = vcs.updateWorkingTree(workingTree, revision)
+                val result = vcs.updateWorkingTree(workingTree, VersionControlSystemConfiguration(revision))
 
                 if (pkg.vcsProcessed.path.isNotBlank() &&
                     !workingTree.getRootPath().resolve(pkg.vcsProcessed.path).exists()