From 1fff71fede9b06dadf72c1a8b352b4870460823a Mon Sep 17 00:00:00 2001 From: Thorkild Stray Date: Tue, 12 Nov 2024 09:04:29 +0100 Subject: [PATCH] Expose filtering fields with null values from Raw (#798) * Expose filtering fields with null values from Raw This exposes the query parameter to control whether Raw should return fields where the value is null or not. After several failed tries, it does this through the RawRows constructor, as this is also relevant for clients that do not filter the output (so it can't just be part of RawRowsFilter, sadly). The filter is simply not a query filter, like the other components of RawRowsFilter. * wartignore --------- Co-authored-by: Dmitry Ivankov --- build.sbt | 2 +- .../com/cognite/sdk/scala/v1/Client.scala | 4 +- .../cognite/sdk/scala/v1/resources/raw.scala | 22 +++++-- .../com/cognite/sdk/scala/v1/RawTest.scala | 61 ++++++++++++++++++- 4 files changed, 81 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index f970df031..b25c4967a 100644 --- a/build.sbt +++ b/build.sbt @@ -40,7 +40,7 @@ lazy val commonSettings = Seq( organization := "com.cognite", organizationName := "Cognite", organizationHomepage := Some(url("https://cognite.com")), - version := "2.30." + patchVersion, + version := "2.31." + patchVersion, isSnapshot := patchVersion.endsWith("-SNAPSHOT"), scalaVersion := scala213, // use 2.13 by default // handle cross plugin https://github.com/stringbean/sbt-dependency-lock/issues/13 diff --git a/src/main/scala/com/cognite/sdk/scala/v1/Client.scala b/src/main/scala/com/cognite/sdk/scala/v1/Client.scala index a2e723455..7df409007 100644 --- a/src/main/scala/com/cognite/sdk/scala/v1/Client.scala +++ b/src/main/scala/com/cognite/sdk/scala/v1/Client.scala @@ -237,8 +237,8 @@ class GenericClient[F[_]: Trace]( new RawDatabases[F](requestSession.withResourceType(RAW_METADATA)) def rawTables(database: String): RawTables[F] = new RawTables(requestSession.withResourceType(RAW_METADATA), database) - def rawRows(database: String, table: String): RawRows[F] = - new RawRows(requestSession.withResourceType(RAW_ROWS), database, table) + def rawRows(database: String, table: String, filterNullFields: Boolean = false): RawRows[F] = + new RawRows(requestSession.withResourceType(RAW_ROWS), database, table, filterNullFields) lazy val threeDModels = new ThreeDModels[F](requestSession.withResourceType(THREED)) def threeDRevisions(modelId: Long): ThreeDRevisions[F] = diff --git a/src/main/scala/com/cognite/sdk/scala/v1/resources/raw.scala b/src/main/scala/com/cognite/sdk/scala/v1/resources/raw.scala index 4af2e3582..a78af83ca 100644 --- a/src/main/scala/com/cognite/sdk/scala/v1/resources/raw.scala +++ b/src/main/scala/com/cognite/sdk/scala/v1/resources/raw.scala @@ -106,8 +106,12 @@ object RawTables { implicit val rawTableDecoder: Decoder[RawTable] = deriveDecoder[RawTable] } -class RawRows[F[_]](val requestSession: RequestSession[F], database: String, table: String) - extends WithRequestSession[F] +class RawRows[F[_]]( + val requestSession: RequestSession[F], + database: String, + table: String, + filterNullFields: Boolean = false +) extends WithRequestSession[F] with Readable[RawRow, F] with Create[RawRow, RawRow, F] with DeleteByIds[F, String] @@ -159,7 +163,14 @@ class RawRows[F[_]](val requestSession: RequestSession[F], database: String, tab limit: Option[Int], partition: Option[Partition] ): F[ItemsWithCursor[RawRow]] = - Readable.readWithCursor(requestSession, baseUrl, cursor, limit, None, Constants.rowsBatchSize) + Readable.readWithCursor( + requestSession, + filterFieldsWithNull(baseUrl), + cursor, + limit, + None, + Constants.rowsBatchSize + ) override def deleteByIds(ids: Seq[String]): F[Unit] = RawResource.deleteByIds(requestSession, baseUrl, ids.map(RawRowKey.apply)) @@ -173,7 +184,7 @@ class RawRows[F[_]](val requestSession: RequestSession[F], database: String, tab ): F[ItemsWithCursor[RawRow]] = Readable.readWithCursor( requestSession, - baseUrl.addParams(filterToParams(filter)), + filterFieldsWithNull(baseUrl.addParams(filterToParams(filter))), cursor, limit, None, @@ -230,6 +241,9 @@ class RawRows[F[_]](val requestSession: RequestSession[F], database: String, tab ).collect { case (key, Some(value)) => key -> value } + + def filterFieldsWithNull(url: Uri): Uri = + url.addParam("filterNullFields", filterNullFields.toString) } object RawRows { diff --git a/src/test/scala/com/cognite/sdk/scala/v1/RawTest.scala b/src/test/scala/com/cognite/sdk/scala/v1/RawTest.scala index 340d8f5f6..2cb872cc2 100644 --- a/src/test/scala/com/cognite/sdk/scala/v1/RawTest.scala +++ b/src/test/scala/com/cognite/sdk/scala/v1/RawTest.scala @@ -6,14 +6,17 @@ package com.cognite.sdk.scala.v1 import cats.syntax.either._ import com.cognite.sdk.scala.common.{Items, ReadBehaviours, SdkTestSpec, WritableBehaviors} import fs2.Stream +import io.circe.Json import io.circe.syntax._ import org.scalatest.OptionValues +import sttp.client3.UriContext @SuppressWarnings( Array( "org.wartremover.warts.TraversableOps", "org.wartremover.warts.NonUnitStatements", - "org.wartremover.warts.IterableOps" + "org.wartremover.warts.IterableOps", + "org.wartremover.warts.SizeIs" ) ) class RawTest extends SdkTestSpec with ReadBehaviours with WritableBehaviors with OptionValues { @@ -133,6 +136,62 @@ class RawTest extends SdkTestSpec with ReadBehaviours with WritableBehaviors wit } } + it should "allow controlling filtering of field with null value" in withDatabaseTables { (database, tables) => + val table = tables.head + val rows = client.rawRows(database, table, filterNullFields = true) + + val relevantKeys = List("withNullFields", "normal") + rows.create( + Seq( + RawRow(relevantKeys.head, Map("a" -> "3".asJson, "notthere" -> None.asJson)), + RawRow(relevantKeys.last, Map("a" -> "0".asJson, "abc" -> "".asJson)) + ) + ).unsafeRunSync() + + // We cannot test using a key filter here, as this is translated into a getRowByKey request, which does not + // currently support filtering out null fields. + + val listedRowsWithNullFilter: Map[String, Map[String, Json]] = rows.list() + .compile + .toList + .unsafeRunSync() + .filter(row => relevantKeys.contains(row.key)).map(r => (r.key -> r.columns)) + .toMap + + assert(listedRowsWithNullFilter.size == 2) + val values = listedRowsWithNullFilter.get("withNullFields").toList.head + assert(values.size == 1) + values.get("a").toList.head shouldBe "3".asJson + assert(listedRowsWithNullFilter.get("normal").toList.head.size == 2) + + val filteredWithNullFilter: Map[String, Map[String, Json]] = + rows.filterWithCursor(RawRowFilter(), None, None, None) + .unsafeRunSync() + .items + .filter(row => relevantKeys.contains(row.key)) + .map(r => (r.key -> r.columns)) + .toMap + + assert(filteredWithNullFilter.size == 2) + val filteredValues = filteredWithNullFilter.get("withNullFields").toList.head + assert(filteredValues.size == 1) + filteredValues.get("a").toList.head shouldBe "3".asJson + assert(listedRowsWithNullFilter.get("normal").toList.head.size == 2) + } + + it should "add correct option when filtering out null fields" in withDatabaseTables { (database, tables) => + val table = tables.head + val unfilteredRows = client.rawRows(database, table) + val filteredRows = client.rawRows(database, table, filterNullFields = true) + + val modifiedFilteredUrl = filteredRows.filterFieldsWithNull(uri"http://localhost/testQuery") + assert(modifiedFilteredUrl.params.toMap.get("filterNullFields").contains("true")) + + val modifiedUnfilteredUrl = unfilteredRows.filterFieldsWithNull(uri"http://localhost/testQuery") + assert(modifiedUnfilteredUrl.params.toMap.get("filterNullFields").contains("false")) + } + + it should "allow partition read and filtering of rows" in withDatabaseTables { (database, tables) => val rows = client