Work-around for flaky ZIO tests

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ on:
     tags: [v*]
 env:
   # .sbtopts specifies 8g, which is needed to import into IntelliJ, but on GH that exceeds the maximum available memory
-  SBT_JAVA_OPTS: -J-Xms4g -J-Xmx4g
+  SBT_JAVA_OPTS: -J-Xms4g -J-Xmx4g -J-Xlog:gc*::time -J-verbose:gc
 jobs:
   ci:
     # run on 1) push, 2) external PRs, 3) softwaremill-ci PRs

build.sbt

@@ -90,6 +90,8 @@ val commonSettings = commonSmlBuildSettings ++ ossPublishSettings ++ Seq(
 
 val versioningSchemeSettings = Seq(versionScheme := Some("early-semver"))
 
+val _ = ThreadDumpEveryMinute // init
+
 val enableMimaSettings = Seq(
   mimaPreviousArtifacts := {
     // currently only 2.* versions are stable; skipping mima for scala3

project/ThreadDumpEveryMinute.scala

@@ -0,0 +1,20 @@
+import java.lang.management.ManagementFactory
+
+object ThreadDumpEveryMinute {
+  new Thread() {
+    override def run() = {
+      while (true) {
+        println("[XXX] thread dump coming up in 10 minutes ... ")
+        Thread.sleep(1000 * 60 * 10)
+        val threadDump = new StringBuffer(System.lineSeparator)
+        val threadMXBean = ManagementFactory.getThreadMXBean
+        for (threadInfo <- threadMXBean.dumpAllThreads(true, true)) {
+          threadDump.append(threadInfo.toString)
+        }
+        println("--- XXX ---")
+        println(threadDump.toString)
+        println("--- XXX ---")
+      }
+    }
+  }.start()
+}

server/zio-http-server/src/test/scala/sttp/tapir/server/ziohttp/ZioHttpServerTest.scala

@@ -9,6 +9,7 @@ import io.netty.channel.ServerChannel
 import org.scalatest.Assertion
 import org.scalatest.Exceptional
 import org.scalatest.FutureOutcome
+import org.scalatest.concurrent.TimeLimits
 import org.scalatest.matchers.should.Matchers._
 import sttp.capabilities.zio.ZioStreams
 import sttp.client3._
@@ -38,9 +39,7 @@ import zio.http.Middleware
 import zio.http.Path
 import zio.http.Request
 import zio.http.URL
-import zio.http.netty.ChannelFactories
-import zio.http.netty.ChannelType
-import zio.http.netty.EventLoopGroups
+import zio.http.netty.{ChannelFactories, ChannelType, EventLoopGroups, NettyConfig}
 import zio.interop.catz._
 import zio.stream
 import zio.stream.ZPipeline
@@ -52,16 +51,17 @@ import scala.concurrent.Future
 import scala.concurrent.duration.DurationInt
 import zio.stream.ZSink
 
-class ZioHttpServerTest extends TestSuite {
+class ZioHttpServerTest extends TestSuite with TimeLimits {
 
   // zio-http tests often fail with "Cause: java.io.IOException: parsing HTTP/1.1 status line, receiving [DEFAULT], parser state [STATUS_LINE]"
-  // until this is fixed, adding retries to avoid flaky tests
+  // they also sometimes hang, esp. on CI (see #3827)
+  // until this is fixed, adding retries to avoid flaky tests & CI timeouts
   val retries = 5
 
   override def withFixture(test: NoArgAsyncTest): FutureOutcome = withFixture(test, retries)
 
   def withFixture(test: NoArgAsyncTest, count: Int): FutureOutcome = {
-    val outcome = super.withFixture(test)
+    val outcome = failAfter(5.seconds)(super.withFixture(test))
     new FutureOutcome(outcome.toFuture.flatMap {
       case Exceptional(e) =>
         println(s"Test ${test.name} failed, retrying.")
@@ -73,6 +73,7 @@ class ZioHttpServerTest extends TestSuite {
 
   override def tests: Resource[IO, List[Test]] = backendResource.flatMap { backend =>
     implicit val r: Runtime[Any] = Runtime.default
+
     // creating the netty dependencies once, to speed up tests
     Resource
       .scoped[IO, Any, ZEnvironment[EventLoopGroup with ChannelFactory[ServerChannel]]]({

server/zio-http-server/src/test/scala/sttp/tapir/server/ziohttp/ZioHttpTestServerInterpreter.scala

@@ -10,7 +10,11 @@ import sttp.tapir.server.tests.TestServerInterpreter
 import sttp.tapir.tests._
 import zio._
 import zio.http._
+import zio.http.netty.NettyConfig
+import zio.http.netty.server.NettyDriver
 import zio.interop.catz._
+import zio.http.netty.{ChannelFactories, ChannelType, EventLoopGroups, NettyConfig}
+
 import scala.concurrent.duration.FiniteDuration
 
 class ZioHttpTestServerInterpreter(
@@ -31,24 +35,24 @@ class ZioHttpTestServerInterpreter(
       routes: NonEmptyList[Routes[Any, Response]],
       gracefulShutdownTimeout: Option[FiniteDuration] = None
   ): Resource[IO, Port] = {
-    implicit val r: Runtime[Any] = Runtime.default
+    val nettyConfig = ZLayer.succeed(NettyConfig.default)
+    val serverConfig = ZLayer.succeed(
+      Server.Config.default
+        .port(0)
+        .enableRequestStreaming
+        .gracefulShutdownTimeout(gracefulShutdownTimeout.map(Duration.fromScala).getOrElse(50.millis))
+    )
+    val drv = (eventLoopGroup ++ nettyConfig ++ channelFactory ++ serverConfig) >>> zio.test.manual
+
     val effect: ZIO[Scope, Throwable, Port] =
-      (for {
-        driver <- ZIO.service[Driver]
+      for {
+        deps <- drv.build
+        driver = deps.get[Driver]
         result <- driver.start
         _ <- driver.addApp[Any](routes.toList.reduce(_ ++ _), ZEnvironment())
-      } yield result.port)
-        .provideSome[Scope](
-          zio.test.driver,
-          eventLoopGroup,
-          channelFactory,
-          ZLayer.succeed(
-            Server.Config.default
-              .port(0)
-              .enableRequestStreaming
-              .gracefulShutdownTimeout(gracefulShutdownTimeout.map(Duration.fromScala).getOrElse(50.millis))
-          )
-        )
+      } yield result.port
+
+    implicit val r: Runtime[Any] = Runtime.default
     Resource.scoped[IO, Any, Port](effect)
   }
 

server/zio-http-server/src/test/scala/zio/test/package.scala

@@ -11,4 +11,7 @@ package object test {
       ZLayer.succeed(NettyConfig.default),
       NettyDriver.manual
     )
+
+  // work-around for Scala 2.12
+  val manual =  NettyDriver.manual
 }

tests/src/main/scalajvm/sttp/tapir/tests/TestSuite.scala

@@ -15,6 +15,7 @@ trait TestSuite extends AsyncFunSuite with BeforeAndAfterAll {
 
   // we need to register the tests when the class is constructed, as otherwise scalatest skips it
   val (allTests, doRelease) = tests.allocated.unsafeRunSync()
+  println(s"Allocated: ${getClass.getName}")
 
   allTests.foreach { t =>
     if (testNameFilter.forall(filter => t.name.contains(filter))) {
@@ -25,9 +26,11 @@ trait TestSuite extends AsyncFunSuite with BeforeAndAfterAll {
   private val release = doRelease
 
   override protected def afterAll(): Unit = {
+    println(s"Deallocating... ${getClass.getName}")
     // the resources can only be released after all of the tests are run
     release.unsafeRunSync()
     shutdownDispatcher.unsafeRunSync()
+    println(s"Deallocating ${getClass.getName} done")
     super.afterAll()
   }
 }