Merge branch 'develop' into jsaun/leo-service-account

.github/workflows/unit_test.yml

@@ -21,7 +21,7 @@ jobs:
 
     services:
       mysql:
-        image: mysql:8.0.40
+        image: mysql:8.4
         env:
           MYSQL_ROOT_PASSWORD: leonardo-test
           MYSQL_USER: leonardo-test

core/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/samModels.scala

@@ -239,6 +239,9 @@ object WorkspaceAction {
   final case object Delete extends WorkspaceAction {
     val asString = "delete"
   }
+  final case object Compute extends WorkspaceAction {
+    val asString = "compute"
+  }
 
   val allActions = sealerate.values[WorkspaceAction]
   val stringToAction: Map[String, WorkspaceAction] =

docker/run-mysql.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
-# The CloudSQL console simply states "MySQL 8.0" so we may not match the minor version number
-MYSQL_VERSION=8.0.40
+# The CloudSQL console simply states "MySQL 8.4" so we may not match the minor version number
+MYSQL_VERSION=8.4
 start() {
 
     echo "attempting to remove old $CONTAINER container..."

http/src/main/resources/init-resources/shutdown.sh

@@ -37,4 +37,4 @@ else
   DOCKER_COMPOSE='docker-compose'
 fi
 
-$DOCKER_COMPOSE down
+$DOCKER_COMPOSE down

http/src/main/resources/init-resources/startup.sh

@@ -243,7 +243,6 @@ RUNTIME_NAME=${RUNTIME_NAME}
 OWNER_EMAIL=${OWNER_EMAIL}
 PET_SA_EMAIL=${PET_SA_EMAIL}
 WELDER_ENABLED=${WELDER_ENABLED}
-MEM_LIMIT=${MEM_LIMIT}
 SHM_SIZE=${SHM_SIZE}
 END
 
@@ -256,6 +255,9 @@ END
         docker restart $JUPYTER_SERVER_NAME
         docker restart $WELDER_SERVER_NAME
 
+        # update memory size, the memory swap must be updated as well (cannot be < memory)
+        docker update  --memory ${MEM_LIMIT} --memory-swap ${MEM_LIMIT}  $JUPYTER_SERVER_NAME
+
         log 'Copy Jupyter frontend notebook config...'
         $GSUTIL_CMD cp ${JUPYTER_NOTEBOOK_FRONTEND_CONFIG_URI} /var
         JUPYTER_NOTEBOOK_FRONTEND_CONFIG=`basename ${JUPYTER_NOTEBOOK_FRONTEND_CONFIG_URI}`
@@ -277,14 +279,16 @@ RUNTIME_NAME=${RUNTIME_NAME}
 OWNER_EMAIL=${OWNER_EMAIL}
 PET_SA_EMAIL=${PET_SA_EMAIL}
 WELDER_ENABLED=${WELDER_ENABLED}
-MEM_LIMIT=${MEM_LIMIT}
 SHM_SIZE=${SHM_SIZE}
 END
 
         # We do not want to recreate a new container, to make sure we preserve the changes that users made with the startup script
         # We only want to restart the existing container with the latest environment variables
         ${DOCKER_COMPOSE} --env-file=/var/variables.env ${COMPLETE_RSTUDIO_DOCKER_COMPOSE} up -d --no-recreate
 
+        # update memory size, the memory swap must be updated as well (cannot be < memory)
+        docker update  --memory ${MEM_LIMIT} --memory-swap ${MEM_LIMIT} $RSTUDIO_SERVER_NAME
+
         # the docker containers need to be restarted or the R container
         # will fail to start until the appropriate volume/device exists.
         docker restart $RSTUDIO_SERVER_NAME

http/src/main/resources/org/broadinstitute/dsde/workbench/leonardo/liquibase/changesets/20170811_label.xml

@@ -1,6 +1,12 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <databaseChangeLog logicalFilePath="leonardo" xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd">
     <changeSet logicalFilePath="leonardo" author="thibault" id="label">
+        <validCheckSum>8:1d8581dc0977ea88b1f006f6bc00f5b9</validCheckSum>
+        <comment>
+            Mysql 8.4+ does not allow partial keys to be referenced for a foreign key anymore, see https://bugs.mysql.com/bug.php?id=114838.
+            This changeSet has been modified to reflect that; the validCheckSum is the checksum from when the flag did not need to be set because the default was OFF.
+        </comment>
+        <sql> SET restrict_fk_on_non_standard_key = OFF; </sql>
         <createTable tableName="LABEL">
             <column name="clusterId" type="BIGINT">
                 <constraints nullable="false"/>

http/src/main/resources/reference.conf

@@ -70,7 +70,7 @@ dataproc {
   }
 
   # Cached dataproc image used by Terra
-  customDataprocImage = "projects/broad-dsp-gcr-public/global/images/leo-dataproc-image-2-1-11-debian11-2024-12-16-17-22-28"
+  customDataprocImage = "projects/broad-dsp-gcr-public/global/images/leo-dataproc-image-2-1-11-debian11-2025-01-21-15-07-39"
 
   # The ratio of memory allocated to spark. 0.8 = 80%.
   # Hail/Spark users generally allocate 80% of the ram to the JVM.

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/dao/sam/SamUtils.scala

@@ -0,0 +1,80 @@
+package org.broadinstitute.dsde.workbench.leonardo.dao.sam
+
+import akka.http.scaladsl.model.StatusCodes
+import akka.http.scaladsl.model.headers.OAuth2BearerToken
+import cats.effect.Async
+import cats.implicits.{catsSyntaxApplicativeError, toFlatMapOps}
+import cats.mtl.Ask
+import org.broadinstitute.dsde.workbench.leonardo.model.{
+  ForbiddenError,
+  LeoException,
+  RuntimeNotFoundByWorkspaceIdException,
+  RuntimeNotFoundException
+}
+import org.broadinstitute.dsde.workbench.leonardo.{
+  AppContext,
+  CloudContext,
+  RuntimeAction,
+  RuntimeName,
+  SamResourceId,
+  WorkspaceId
+}
+import org.broadinstitute.dsde.workbench.model.{UserInfo, WorkbenchEmail}
+
+trait SamUtils[F[_]] {
+  val samService: SamService[F]
+
+  def checkRuntimeAction(userInfo: UserInfo,
+                         cloudContext: CloudContext,
+                         runtimeName: RuntimeName,
+                         samResourceId: SamResourceId,
+                         action: RuntimeAction,
+                         userEmail: Option[WorkbenchEmail] = None
+  )(implicit F: Async[F], as: Ask[F, AppContext]): F[Unit] =
+    checkRuntimeActionInternal(
+      userInfo.accessToken,
+      userEmail.getOrElse(userInfo.userEmail),
+      samResourceId,
+      action,
+      RuntimeNotFoundException(cloudContext, runtimeName, "Not found in database")
+    )
+
+  def checkRuntimeAction(userInfo: UserInfo,
+                         workspaceId: WorkspaceId,
+                         runtimeName: RuntimeName,
+                         samResourceId: SamResourceId,
+                         action: RuntimeAction
+  )(implicit F: Async[F], as: Ask[F, AppContext]): F[Unit] =
+    checkRuntimeActionInternal(
+      userInfo.accessToken,
+      userInfo.userEmail,
+      samResourceId,
+      action,
+      RuntimeNotFoundByWorkspaceIdException(workspaceId, runtimeName, "Not found in database")
+    )
+
+  private def checkRuntimeActionInternal(userToken: OAuth2BearerToken,
+                                         userEmail: WorkbenchEmail,
+                                         samResourceId: SamResourceId,
+                                         action: RuntimeAction,
+                                         notFoundException: LeoException
+  )(implicit F: Async[F], as: Ask[F, AppContext]): F[Unit] =
+    samService
+      .checkAuthorized(userToken.token, samResourceId, action)
+      .handleErrorWith {
+        // If we've already checked read access and the user doesn't have it, pretend the runtime doesn't exist to avoid leaking its existence
+        case e: SamException if e.statusCode == StatusCodes.Forbidden && action == RuntimeAction.GetRuntimeStatus =>
+          F.raiseError(notFoundException)
+        // Check if the user can read the runtime to determine which error to raise
+        case e: SamException if e.statusCode == StatusCodes.Forbidden =>
+          samService
+            .checkAuthorized(userToken.token, samResourceId, RuntimeAction.GetRuntimeStatus)
+            .attempt
+            .flatMap {
+              // The user can read the runtime, but they don't have the required action. Raise the original Forbidden action from Sam
+              case Right(_) => F.raiseError(ForbiddenError(userEmail))
+              // The user can't read the runtime, pretend it doesn't exist to avoid leaking its existence
+              case Left(_) => F.raiseError(notFoundException)
+            }
+      }
+}

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/GcpDependenciesBuilder.scala

@@ -268,7 +268,8 @@ class GcpDependencyBuilder extends CloudDependenciesBuilder {
       baselineDependencies.proxyResolver,
       baselineDependencies.samDAO,
       baselineDependencies.googleTokenCache,
-      baselineDependencies.samResourceCache
+      baselineDependencies.samResourceCache,
+      baselineDependencies.samService
     )
 
     val diskService = new DiskServiceInterp[IO](

http/src/main/scala/org/broadinstitute/dsde/workbench/leonardo/http/service/ProxyService.scala

@@ -23,6 +23,7 @@ import org.broadinstitute.dsde.workbench.leonardo.SamResourceId._
 import org.broadinstitute.dsde.workbench.leonardo.config.ProxyConfig
 import org.broadinstitute.dsde.workbench.leonardo.dao.HostStatus._
 import org.broadinstitute.dsde.workbench.leonardo.dao.google.GoogleOAuth2Service
+import org.broadinstitute.dsde.workbench.leonardo.dao.sam.{SamService, SamUtils}
 import org.broadinstitute.dsde.workbench.leonardo.dao.{HostStatus, JupyterDAO, Proxy, SamDAO, TerminalName}
 import org.broadinstitute.dsde.workbench.leonardo.db.{appQuery, clusterQuery, DbReference, KubernetesServiceDbQueries}
 import org.broadinstitute.dsde.workbench.leonardo.dns.{KubernetesDnsCache, ProxyResolver, RuntimeDnsCache}
@@ -91,14 +92,16 @@ class ProxyService(
   proxyResolver: ProxyResolver[IO],
   samDAO: SamDAO[IO],
   googleTokenCache: Cache[IO, String, (UserInfo, Instant)],
-  samResourceCache: Cache[IO, SamResourceCacheKey, (Option[String], Option[AppAccessScope])]
+  samResourceCache: Cache[IO, SamResourceCacheKey, (Option[String], Option[AppAccessScope])],
+  val samService: SamService[IO]
 )(implicit
   val system: ActorSystem,
   executionContext: ExecutionContext,
   dbRef: DbReference[IO],
   loggerIO: StructuredLogger[IO],
   metrics: OpenTelemetryMetrics[IO]
-) extends LazyLogging {
+) extends LazyLogging
+    with SamUtils[IO] {
   val httpsConnectionContext = ConnectionContext.httpsClient(sslContext)
   val clientConnectionSettings =
     ClientConnectionSettings(system).withTransport(ClientTransport.withCustomResolver(proxyResolver.resolveAkka))
@@ -267,38 +270,9 @@ class ProxyService(
     for {
       ctx <- ev.ask[AppContext]
 
-      hasWorkspacePermission <- workspaceId match {
-        case Some(wid) =>
-          authProvider
-            .isUserWorkspaceReader(
-              WorkspaceResourceSamResourceId(wid),
-              userInfo
-            )
-        case None => IO.pure(true)
-      }
-
-      _ <- IO.raiseUnless(hasWorkspacePermission)(ForbiddenError(userInfo.userEmail))
-
       samResource <- getCachedRuntimeSamResource(RuntimeCacheKey(cloudContext, runtimeName))
-      // Note both these Sam actions are cached so it should be okay to call hasPermission twice
-      hasViewPermission <- authProvider.hasPermission[RuntimeSamResourceId, RuntimeAction](
-        samResource,
-        RuntimeAction.GetRuntimeStatus,
-        userInfo
-      )
-      _ <-
-        if (!hasViewPermission) {
-          IO.raiseError(RuntimeNotFoundException(cloudContext, runtimeName, ctx.traceId.asString))
-        } else IO.unit
-      hasConnectPermission <- authProvider.hasPermission[RuntimeSamResourceId, RuntimeAction](
-        samResource,
-        RuntimeAction.ConnectToRuntime,
-        userInfo
-      )
-      _ <-
-        if (!hasConnectPermission) {
-          IO.raiseError(ForbiddenError(userInfo.userEmail))
-        } else IO.unit
+      _ <- checkRuntimeAction(userInfo, cloudContext, runtimeName, samResource, RuntimeAction.ConnectToRuntime)
+
       hostStatus <- getRuntimeTargetHost(cloudContext, runtimeName)
       _ <- hostStatus match {
         case HostReady(_, _, _) =>