Expose filtering fields with null values from Raw (#798)

* Expose filtering fields with null values from Raw

This exposes the query parameter to control whether Raw should return
fields where the value is null or not. After several failed tries, it
does this through the RawRows constructor, as this is also relevant
for clients that do not filter the output (so it can't just be part of
RawRowsFilter, sadly). The filter is simply not a query filter, like
the other components of RawRowsFilter.

* wartignore

---------

Co-authored-by: Dmitry Ivankov <[email protected]>

build.sbt

@@ -40,7 +40,7 @@ lazy val commonSettings = Seq(
   organization := "com.cognite",
   organizationName := "Cognite",
   organizationHomepage := Some(url("https://cognite.com")),
-  version := "2.30." + patchVersion,
+  version := "2.31." + patchVersion,
   isSnapshot := patchVersion.endsWith("-SNAPSHOT"),
   scalaVersion := scala213, // use 2.13 by default
   // handle cross plugin https://github.com/stringbean/sbt-dependency-lock/issues/13

src/main/scala/com/cognite/sdk/scala/v1/Client.scala

@@ -237,8 +237,8 @@ class GenericClient[F[_]: Trace](
     new RawDatabases[F](requestSession.withResourceType(RAW_METADATA))
   def rawTables(database: String): RawTables[F] =
     new RawTables(requestSession.withResourceType(RAW_METADATA), database)
-  def rawRows(database: String, table: String): RawRows[F] =
-    new RawRows(requestSession.withResourceType(RAW_ROWS), database, table)
+  def rawRows(database: String, table: String, filterNullFields: Boolean = false): RawRows[F] =
+    new RawRows(requestSession.withResourceType(RAW_ROWS), database, table, filterNullFields)
 
   lazy val threeDModels = new ThreeDModels[F](requestSession.withResourceType(THREED))
   def threeDRevisions(modelId: Long): ThreeDRevisions[F] =

src/main/scala/com/cognite/sdk/scala/v1/resources/raw.scala

@@ -106,8 +106,12 @@ object RawTables {
   implicit val rawTableDecoder: Decoder[RawTable] = deriveDecoder[RawTable]
 }
 
-class RawRows[F[_]](val requestSession: RequestSession[F], database: String, table: String)
-    extends WithRequestSession[F]
+class RawRows[F[_]](
+    val requestSession: RequestSession[F],
+    database: String,
+    table: String,
+    filterNullFields: Boolean = false
+) extends WithRequestSession[F]
     with Readable[RawRow, F]
     with Create[RawRow, RawRow, F]
     with DeleteByIds[F, String]
@@ -159,7 +163,14 @@ class RawRows[F[_]](val requestSession: RequestSession[F], database: String, tab
       limit: Option[Int],
       partition: Option[Partition]
   ): F[ItemsWithCursor[RawRow]] =
-    Readable.readWithCursor(requestSession, baseUrl, cursor, limit, None, Constants.rowsBatchSize)
+    Readable.readWithCursor(
+      requestSession,
+      filterFieldsWithNull(baseUrl),
+      cursor,
+      limit,
+      None,
+      Constants.rowsBatchSize
+    )
 
   override def deleteByIds(ids: Seq[String]): F[Unit] =
     RawResource.deleteByIds(requestSession, baseUrl, ids.map(RawRowKey.apply))
@@ -173,7 +184,7 @@ class RawRows[F[_]](val requestSession: RequestSession[F], database: String, tab
   ): F[ItemsWithCursor[RawRow]] =
     Readable.readWithCursor(
       requestSession,
-      baseUrl.addParams(filterToParams(filter)),
+      filterFieldsWithNull(baseUrl.addParams(filterToParams(filter))),
       cursor,
       limit,
       None,
@@ -230,6 +241,9 @@ class RawRows[F[_]](val requestSession: RequestSession[F], database: String, tab
     ).collect { case (key, Some(value)) =>
       key -> value
     }
+
+  def filterFieldsWithNull(url: Uri): Uri =
+    url.addParam("filterNullFields", filterNullFields.toString)
 }
 
 object RawRows {

src/test/scala/com/cognite/sdk/scala/v1/RawTest.scala

@@ -6,14 +6,17 @@ package com.cognite.sdk.scala.v1
 import cats.syntax.either._
 import com.cognite.sdk.scala.common.{Items, ReadBehaviours, SdkTestSpec, WritableBehaviors}
 import fs2.Stream
+import io.circe.Json
 import io.circe.syntax._
 import org.scalatest.OptionValues
+import sttp.client3.UriContext
 
 @SuppressWarnings(
   Array(
     "org.wartremover.warts.TraversableOps",
     "org.wartremover.warts.NonUnitStatements",
-    "org.wartremover.warts.IterableOps"
+    "org.wartremover.warts.IterableOps",
+    "org.wartremover.warts.SizeIs"
   )
 )
 class RawTest extends SdkTestSpec with ReadBehaviours with WritableBehaviors with OptionValues {
@@ -133,6 +136,62 @@ class RawTest extends SdkTestSpec with ReadBehaviours with WritableBehaviors wit
     }
   }
 
+  it should "allow controlling filtering of field with null value" in withDatabaseTables { (database, tables) =>
+    val table = tables.head
+    val rows = client.rawRows(database, table, filterNullFields = true)
+
+    val relevantKeys = List("withNullFields", "normal")
+    rows.create(
+      Seq(
+        RawRow(relevantKeys.head, Map("a" -> "3".asJson, "notthere" -> None.asJson)),
+        RawRow(relevantKeys.last, Map("a" -> "0".asJson, "abc" -> "".asJson))
+      )
+    ).unsafeRunSync()
+
+    // We cannot test using a key filter here, as this is translated into a getRowByKey request, which does not
+    // currently support filtering out null fields.
+
+    val listedRowsWithNullFilter: Map[String, Map[String, Json]] = rows.list()
+      .compile
+      .toList
+      .unsafeRunSync()
+      .filter(row => relevantKeys.contains(row.key)).map(r => (r.key -> r.columns))
+      .toMap
+
+    assert(listedRowsWithNullFilter.size == 2)
+    val values = listedRowsWithNullFilter.get("withNullFields").toList.head
+    assert(values.size == 1)
+    values.get("a").toList.head shouldBe "3".asJson
+    assert(listedRowsWithNullFilter.get("normal").toList.head.size == 2)
+
+    val filteredWithNullFilter: Map[String, Map[String, Json]] =
+      rows.filterWithCursor(RawRowFilter(), None, None, None)
+        .unsafeRunSync()
+        .items
+        .filter(row => relevantKeys.contains(row.key))
+        .map(r => (r.key -> r.columns))
+        .toMap
+
+    assert(filteredWithNullFilter.size == 2)
+    val filteredValues = filteredWithNullFilter.get("withNullFields").toList.head
+    assert(filteredValues.size == 1)
+    filteredValues.get("a").toList.head shouldBe "3".asJson
+    assert(listedRowsWithNullFilter.get("normal").toList.head.size == 2)
+  }
+
+  it should "add correct option when filtering out null fields" in withDatabaseTables { (database, tables) =>
+    val table = tables.head
+    val unfilteredRows = client.rawRows(database, table)
+    val filteredRows = client.rawRows(database, table, filterNullFields = true)
+
+    val modifiedFilteredUrl = filteredRows.filterFieldsWithNull(uri"http://localhost/testQuery")
+    assert(modifiedFilteredUrl.params.toMap.get("filterNullFields").contains("true"))
+
+    val modifiedUnfilteredUrl = unfilteredRows.filterFieldsWithNull(uri"http://localhost/testQuery")
+    assert(modifiedUnfilteredUrl.params.toMap.get("filterNullFields").contains("false"))
+  }
+
+
   it should "allow partition read and filtering of rows" in withDatabaseTables {
     (database, tables) =>
       val rows = client