Merge pull request #802 from uptane/merge-upstream

Merge upstream

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
         image: uptane/tuf-nginx:latest
 
       db:
-        image: mariadb:10.4
+        image: mariadb:10.11
         env:
           MYSQL_ROOT_PASSWORD: "root"
           MYSQL_DATABASE: "ota_tuf"

.gitlab-ci.yml

@@ -42,7 +42,7 @@ test:
   services:
     - name: uptane/tuf-nginx:latest
       alias: tuf-nginx
-    - name: mariadb:10.4
+    - name: mariadb:10.11
       alias: db
       command:
         - --character-set-server=utf8

reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/PackageSearch.scala

@@ -105,8 +105,7 @@ class PackageSearch()(implicit db: Database) {
         IF(${searchParams.name.isEmpty}, true, name = ${searchParams.name}) AND
         IF(${searchParams.version.isEmpty}, true, version = ${searchParams.version}) AND
         IF(${searchParams.nameContains.isEmpty}, true, LOCATE(${searchParams.nameContains}, name) > 0) AND
-        IF(${searchParams.hardwareIds.isEmpty}, true, JSON_CONTAINS(hardwareids, JSON_QUOTE(${searchParams.hardwareIds.headOption
-          .map(_.value)}))) AND
+        IF(${searchParams.hardwareIds.isEmpty}, true, JSON_OVERLAPS(${searchParams.hardwareIds}, hardwareids) = 1) AND
         IF(${searchParams.hashes.isEmpty}, true, FIND_IN_SET(JSON_UNQUOTE(JSON_EXTRACT(checksum, '$$.hash')), ${searchParams.hashes
           .map(_.value)}) > 0)
       ORDER BY
@@ -115,11 +114,6 @@ class PackageSearch()(implicit db: Database) {
           length
       LIMIT $limit OFFSET $offset""".as[Q]
 
-    // TODO: This needs a newer mariadb version
-    // hardwareId filter should be:
-    //         IF(${searchParams.hardwareIds.isEmpty}, true, JSON_LENGTH(JSON_ARRAY_INTERSECT(hardwareids, ${searchParams.hardwareIds})) > 0)
-    // but JSON_ARRAY_INTERSECT is not supported in our mariadb version
-
     querySqlAction
   }
 
@@ -253,8 +247,7 @@ class PackageSearch()(implicit db: Database) {
         IF(${searchParams.origin.isEmpty}, true, FIND_IN_SET(origin, ${searchParams.origin}) > 0) AND
         IF(${searchParams.nameContains.isEmpty}, true, LOCATE(${searchParams.nameContains}, name) > 0) AND
         IF(${searchParams.version.isEmpty}, true, version = ${searchParams.version}) AND
-        IF(${searchParams.hardwareIds.isEmpty}, true, JSON_CONTAINS(hardwareids, JSON_QUOTE(${searchParams.hardwareIds.headOption
-          .map(_.value)}))) AND
+        IF(${searchParams.hardwareIds.isEmpty}, true, JSON_OVERLAPS(${searchParams.hardwareIds}, hardwareids) = 1) AND
         IF(${searchParams.hashes.isEmpty}, true, FIND_IN_SET(JSON_UNQUOTE(JSON_EXTRACT(checksum, '$$.hash')), ${searchParams.hashes
           .map(_.value)}) > 0)
       GROUP BY name
@@ -264,6 +257,25 @@ class PackageSearch()(implicit db: Database) {
       LIMIT $limit OFFSET $offset""".as[Q]
 
     db.run(q)
+
+  }
+
+   def hardwareIdsWithPackages(repoId: RepoId): Future[Seq[HardwareIdentifier]] = {
+    implicit val getResult: GetResult[HardwareIdentifier] = GetResult.GetString.andThen { str =>
+      refineV[ValidHardwareIdentifier](str)
+        .valueOr(msg =>
+          throw new IllegalArgumentException(s"hardwareid not properly formatted: $str: $msg")
+        )
+    }
+
+    val q =
+      sql"""
+        select distinct hwid from aggregated_items a,
+          json_table(hardwareids, '$$[*]' columns(hwid varchar(255) path '$$')) t1
+          where a.repo_id = ${repoId.show}
+        """.as[HardwareIdentifier]
+
+    db.run(q)
   }
 
 }

reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResource.scala

@@ -25,6 +25,7 @@ import eu.timepit.refined.refineV
 import slick.jdbc.MySQLProfile.api.*
 import com.advancedtelematic.libats.http.RefinedMarshallingSupport.*
 import scala.concurrent.ExecutionContext
+import com.advancedtelematic.libats.codecs.CirceRefined.*
 
 case class PackageSearchParameters(origin: Seq[String],
                                    nameContains: Option[String],
@@ -84,16 +85,28 @@ class RepoTargetsResource(namespaceValidation: NamespaceValidation)(
     )
   }
 
+  private var packageSearch = new PackageSearch()
+
   // format: off
+
   val route =
     (pathPrefix("user_repo")  & NamespaceRepoId(namespaceValidation, repoNamespaceRepo.findFor)) { repoId =>
+      packageSearch = new PackageSearch()
       concat(
+        path("hardwareids-packages") {
+          get {
+            val f = packageSearch.hardwareIdsWithPackages(repoId).map { values =>
+              PaginationResult(values, values.length, 0, values.length)
+            }
+            complete(f)
+          }
+        },
         path("search") {
           (get & PaginationParams & SearchParams & SortByTargetItemsParam) { case (offset, limit, searchParams, sortBy, sortDirection) =>
 
             val f = for {
-              count <- (new PackageSearch()).count(repoId, searchParams)
-              values <- (new PackageSearch()).find(repoId, offset, limit, searchParams, sortBy, sortDirection)
+              count <- packageSearch.count(repoId, searchParams)
+              values <- packageSearch.find(repoId, offset, limit, searchParams, sortBy, sortDirection)
             } yield {
               PaginationResult(values.map(_.transformInto[ClientPackage]), count, offset, limit)
             }
@@ -104,8 +117,8 @@ class RepoTargetsResource(namespaceValidation: NamespaceValidation)(
         path("grouped-search") {
           (get & PaginationParams & SearchParams & SortByAggregatedTargetItemsParam) { case (offset, limit, searchParams, sortBy, sortDirection) =>
             val f = for {
-              count <- (new PackageSearch()).findAggregatedCount(repoId, searchParams)
-              values <- (new PackageSearch()).findAggregated(repoId, offset, limit, searchParams, sortBy, sortDirection)
+              count <- packageSearch.findAggregatedCount(repoId, searchParams)
+              values <- packageSearch.findAggregated(repoId, offset, limit, searchParams, sortBy, sortDirection)
             } yield
               PaginationResult(values.map(_.transformInto[ClientAggregatedPackage]), count, offset, limit)
 

reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/RepoTargetsResourceSpec.scala

@@ -8,21 +8,18 @@ import org.scalatest.OptionValues.*
 import akka.http.scaladsl.model.{HttpEntity, StatusCodes}
 import akka.util.ByteString
 import com.advancedtelematic.libats.data.PaginationResult
-import com.advancedtelematic.tuf.reposerver.util.{
-  RepoResourceDelegationsSpecUtil,
-  ResourceSpec,
-  TufReposerverSpec
-}
+import com.advancedtelematic.tuf.reposerver.util.{RepoResourceDelegationsSpecUtil, ResourceSpec, TufReposerverSpec}
 import com.advancedtelematic.tuf.reposerver.util.NamespaceSpecOps.*
 import de.heikoseeberger.akkahttpcirce.FailFastCirceSupport.*
 import com.advancedtelematic.tuf.reposerver.data.RepoDataType.*
 import com.advancedtelematic.tuf.reposerver.data.RepoDataType.Package.*
 import com.advancedtelematic.libats.data.DataType.HashMethod
 import com.advancedtelematic.libtuf.data.ClientDataType.ClientTargetItem
-import com.advancedtelematic.libtuf.data.TufDataType.TargetFilename
+import com.advancedtelematic.libtuf.data.TufDataType.{HardwareIdentifier, TargetFilename}
 import com.advancedtelematic.libtuf_server.crypto.Sha256Digest
 import eu.timepit.refined.api.Refined
 import io.circe.Json
+import com.advancedtelematic.libats.codecs.CirceRefined.*
 
 class RepoTargetsResourceSpec
     extends TufReposerverSpec
@@ -33,6 +30,10 @@ class RepoTargetsResourceSpec
        The library will endure; it is the universe. As for us, everything has not been written; we are not turning into phantoms. We walk the corridors, searching the shelves and rearranging them, looking for lines of meaning amid leagues of cacophony and incoherence, reading the history of the past and our future, collecting our thoughts and collecting the thoughts of others, and every so often glimpsing mirrors, in which we may recognize creatures of the information.”
       """.stripMargin))
 
+  val testEntity2 = HttpEntity(ByteString("""
+      If honor and wisdom and happiness are not for me, let them be for others. Let heaven exist, though my place be in hell
+      """.stripMargin))
+
   testWithRepo("GET <TBD> returns delegation items ") { implicit ns => implicit repoId =>
     addTargetToRepo(repoId)
 
@@ -378,7 +379,6 @@ class RepoTargetsResourceSpec
     }
   }
 
-  // TODO: Currently only filters by single hardwareid
   testWithRepo("filters by hardwareIds") { implicit ns => implicit repoId =>
     addTargetToRepo(repoId)
 
@@ -414,6 +414,14 @@ class RepoTargetsResourceSpec
       val values = responseAs[PaginationResult[Package]].values
       values shouldBe empty
     }
+
+    Get(
+      apiUriV2(s"user_repo/search?hardwareIds=delegated-hardware-id-001,myid001")
+    ).namespaced ~> routes ~> check {
+      status shouldBe StatusCodes.OK
+      val values = responseAs[PaginationResult[Package]].values
+      values should have size 2
+    }
   }
 
   testWithRepo("grouped-search gets packages aggregated by version") {
@@ -480,6 +488,14 @@ class RepoTargetsResourceSpec
       values should have size 1
     }
 
+    Get(
+      apiUriV2(s"user_repo/grouped-search?hardwareIds=delegated-hardware-id-001,myid001")
+    ).namespaced ~> routes ~> check {
+      status shouldBe StatusCodes.OK
+      val values = responseAs[PaginationResult[AggregatedPackage]].values
+      values should have size 2
+    }
+
     Get(
       apiUriV2(s"user_repo/grouped-search?hardwareIds=somethingelse")
     ).namespaced ~> routes ~> check {
@@ -577,4 +593,31 @@ class RepoTargetsResourceSpec
     }
   }
 
+  testWithRepo("GET hardwareids-packages returns hwids for which there are packages") { implicit ns => implicit repoId =>
+    addTargetToRepo(repoId)
+
+    Put(
+      apiUri("user_repo/targets/mypkg_file?name=library&version=0.0.1&hardwareIds=myid001,myid002"),
+      testEntity
+    ).namespaced ~> routes ~> check {
+      status shouldBe StatusCodes.NoContent
+    }
+
+    Put(
+      apiUri("user_repo/targets/mypkg_file2?name=library&version=0.0.2&hardwareIds=myid002,myid003"),
+      testEntity2
+    ).namespaced ~> routes ~> check {
+      status shouldBe StatusCodes.NoContent
+    }
+
+    Get(apiUriV2(s"user_repo/hardwareids-packages")).namespaced ~> routes ~> check {
+      status shouldBe StatusCodes.OK
+      val result = responseAs[PaginationResult[HardwareIdentifier]]
+      result.total shouldBe 3
+      result.offset shouldBe 0
+      result.limit shouldBe 3
+      result.values.map(_.value) should contain theSameElementsAs List("myid001", "myid002", "myid003")
+    }
+  }
+
 }