[ComplianceTests] Malformed compliance tests

build.sbt

@@ -767,6 +767,7 @@ lazy val complianceTests = projectMatrix
         else Seq.empty
       ce3 ++ Seq(
         Dependencies.Circe.parser.value,
+        Dependencies.Smithy.utils ,
         Dependencies.Http4s.circe.value,
         Dependencies.Http4s.client.value,
         Dependencies.Weaver.cats.value % Test,

modules/compliance-tests/src/smithy4s/compliancetests/ComplianceTest.scala

@@ -17,9 +17,32 @@
 package smithy4s.compliancetests
 
 import ComplianceTest.ComplianceResult
+import smithy.test.NonEmptyString
+import smithy4s.ShapeId
 
-case class ComplianceTest[F[_]](name: String, run: F[ComplianceResult])
+case class ComplianceTest[F[_]](
+    id: String,
+    endpoint: ShapeId,
+    tags: List[String],
+    run: F[ComplianceResult]
+) {
+  private val showTags =
+    if (tags.isEmpty) "" else tags.mkString(" Tags[", ", ", "]")
+  def show = s"${endpoint.id}: $id $showTags"
+}
 
 object ComplianceTest {
   type ComplianceResult = Either[String, Unit]
+  def apply[F[_]](
+      id: String,
+      endpoint: ShapeId,
+      tags: Option[List[NonEmptyString]],
+      run: F[ComplianceResult]
+  ): ComplianceTest[F] =
+    ComplianceTest(
+      id,
+      endpoint,
+      tags.getOrElse(List.empty).map(_.value),
+      run
+    )
 }

modules/compliance-tests/src/smithy4s/compliancetests/HttpProtocolCompliance.scala

@@ -46,10 +46,28 @@ object HttpProtocolCompliance {
       service
     ).allServerTests()
 
+  def malformedRequestTests[F[_], Alg[_[_, _, _, _, _]]](
+      impl: Router[F],
+      service: Service[Alg]
+  )(implicit ce: CompatEffect[F]): List[ComplianceTest[F]] =
+    new internals.MalformedRequestComplianceTestCase[F, Alg](
+      impl,
+      service
+    ).malformedRequestTests()
+
   def clientAndServerTests[F[_], Alg[_[_, _, _, _, _]]](
       router: Router[F] with ReverseRouter[F],
       service: Service[Alg]
   )(implicit ce: CompatEffect[F]): List[ComplianceTest[F]] =
     clientTests(router, service) ++ serverTests(router, service)
 
+  def allTests[F[_], Alg[_[_, _, _, _, _]]](
+      router: Router[F] with ReverseRouter[F],
+      service: Service[Alg]
+  )(implicit ce: CompatEffect[F]): List[ComplianceTest[F]] =
+    clientAndServerTests(router, service) ++ malformedRequestTests(
+      router,
+      service
+    )
+
 }

modules/compliance-tests/src/smithy4s/compliancetests/internals/Assertions.scala

@@ -51,23 +51,17 @@ private[internals] object assert {
     }
   }
 
-  def neql[A: Eq](expected: A, actual: A): ComplianceResult = {
-    if (expected =!= actual) {
-      success
-    } else {
-      fail(
-        s"This test passed when it was supposed to fail, Actual value: ${pprint
-          .apply(actual)} was equal to ${pprint.apply(expected)}."
-      )
-    }
-  }
-
-  def eql[A: Eq](expected: A, actual: A): ComplianceResult = {
-    if (expected === actual) {
+  def eql[A: Eq](
+      result: A,
+      testCase: A,
+      prefix: String = ""
+  ): ComplianceResult = {
+    if (result === testCase) {
       success
     } else {
       fail(
-        s"Actual value: ${pprint.apply(actual)} was not equal to ${pprint.apply(expected)}."
+        s"$prefix the result value: ${pprint.apply(result)} was not equal to the expected TestCase value ${pprint
+          .apply(testCase)}."
       )
     }
   }
@@ -84,6 +78,19 @@ private[internals] object assert {
     }
   }
 
+  def regexEql(
+      expected: String,
+      actual: String
+  ): ComplianceResult = {
+    if (actual.matches(expected)) {
+      success
+    } else {
+      fail(
+        s"Actual value: ${pprint.apply(actual)} was not equal to ${pprint.apply(expected)}."
+      )
+    }
+  }
+
   private def headersExistenceCheck(
       headers: Headers,
       expected: Either[Option[List[String]], Option[List[String]]]
@@ -101,7 +108,7 @@ private[internals] object assert {
         }.combineAll
     }
   }
-  private def headersCheck(
+  def headersCheck(
       headers: Headers,
       expected: Option[Map[String, String]]
   ) = {

modules/compliance-tests/src/smithy4s/compliancetests/internals/ClientHttpComplianceTestCase.scala

@@ -31,7 +31,6 @@ import smithy4s.Document
 import smithy4s.http.PayloadError
 import smithy4s.Service
 import cats.Eq
-
 import scala.concurrent.duration._
 import smithy4s.http.HttpMediaType
 import org.http4s.MediaType
@@ -56,14 +55,13 @@ private[compliancetests] class ClientHttpComplianceTestCase[
       testCase: HttpRequestTestCase
   ): F[ComplianceResult] = {
 
-    val bodyAssert = testCase.body
-      .map { expectedBody =>
-        request.bodyText.compile.string.map { responseBody =>
-          assert.bodyEql(responseBody, expectedBody, testCase.bodyMediaType)
-
-        }
-      }
-      .getOrElse(assert.success.pure[F])
+    val bodyAssert = request.bodyText.compile.string.map { responseBody =>
+      assert.bodyEql(
+        responseBody,
+        testCase.body.getOrElse(""),
+        testCase.bodyMediaType
+      )
+    }
 
     val expectedUri = baseUri
       .withPath(
@@ -74,14 +72,20 @@ private[compliancetests] class ClientHttpComplianceTestCase[
       )
 
     val pathAssert =
-      assert.eql(expectedUri.path.renderString, request.uri.path.renderString)
+      assert.eql(
+        expectedUri.path.renderString,
+        request.uri.path.renderString,
+        "path test :"
+      )
     val queryAssert = assert.eql(
       expectedUri.query.renderString,
-      request.uri.query.renderString
+      request.uri.query.renderString,
+      "query test :"
     )
     val methodAssert = assert.eql(
       testCase.method.toLowerCase(),
-      request.method.name.toLowerCase()
+      request.method.name.toLowerCase(),
+      "method test :"
     )
     val ioAsserts: List[F[ComplianceResult]] = bodyAssert +:
       List(
@@ -103,7 +107,9 @@ private[compliancetests] class ClientHttpComplianceTestCase[
     val revisedSchema = mapAllTimestampsToEpoch(endpoint.input.awsHintMask)
     val inputFromDocument = Document.Decoder.fromSchema(revisedSchema)
     ComplianceTest[F](
-      name = endpoint.id.toString + "(client|request): " + testCase.id,
+      testCase.id,
+      endpoint.id,
+      testCase.tags,
       run = {
         val input = inputFromDocument
           .decode(testCase.params.getOrElse(Document.obj()))
@@ -152,7 +158,9 @@ private[compliancetests] class ClientHttpComplianceTestCase[
     val dummyInput = DefaultSchemaVisitor(endpoint.input)
 
     ComplianceTest[F](
-      name = endpoint.id.toString + "(client|response): " + testCase.id,
+      testCase.id,
+      endpoint.id,
+      testCase.tags,
       run = {
         val revisedSchema = mapAllTimestampsToEpoch(endpoint.output.awsHintMask)
         implicit val outputEq: Eq[O] =
@@ -211,8 +219,8 @@ private[compliancetests] class ClientHttpComplianceTestCase[
                     .apply(endpoint.wrap(dummyInput))
                   res.map { output =>
                     assert.eql(
-                      expectedOutput,
-                      output
+                      output,
+                      expectedOutput
                     )
                   }
                 }