Merge branch 'master' into looker-filter-usage-mcps

.github/workflows/build-and-test.yml

@@ -67,17 +67,20 @@ jobs:
           timezoneLinux: ${{ matrix.timezone }}
       - name: Check out the repo
         uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/setup-python@v5
-        if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
-        with:
-          python-version: "3.10"
-          cache: pip
       - name: Gradle build (and test) for NOT metadata ingestion
         if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
         run: |

.github/workflows/documentation.yml

@@ -44,7 +44,11 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
-          cache: pip
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
       - name: Install Python dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Build Docs

.github/workflows/metadata-ingestion.yml

@@ -62,16 +62,11 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
       - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/uv
           key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Install package

.github/workflows/metadata-io.yml

@@ -57,17 +57,16 @@ jobs:
       - name: Disk Check
         run: df -h . && docker images
       - uses: acryldata/sane-checkout-action@v3
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:
           distribution: "zulu"
           java-version: 17
       - uses: gradle/actions/setup-gradle@v3
-      - uses: actions/setup-python@v5
-        if: ${{ needs.setup.outputs.ingestion_change == 'true' }}
-        with:
-          python-version: "3.10"
-          cache: "pip"
       - name: Gradle build (and test)
         run: |
           ./gradlew :metadata-io:test

build.gradle

@@ -45,7 +45,9 @@ buildscript {
   ext.elasticsearchVersion = '2.11.1' // ES 7.10, Opensearch 1.x, 2.x
   ext.jacksonVersion = '2.15.3'
   ext.jettyVersion = '11.0.21'
+  // see also datahub-frontend/play.gradle
   ext.playVersion = '2.8.22'
+  ext.playScalaVersion = '2.13'
   ext.log4jVersion = '2.23.1'
   ext.slf4jVersion = '1.7.36'
   ext.logbackClassic = '1.4.14'
@@ -103,7 +105,7 @@ project.ext.spec = [
 ]
 
 project.ext.externalDependency = [
-    'akkaHttp': 'com.typesafe.akka:akka-http-core_2.12:10.2.10',
+    'akkaHttp': "com.typesafe.akka:akka-http-core_$playScalaVersion:10.2.10",
     'antlr4Runtime': 'org.antlr:antlr4-runtime:4.9.3',
     'antlr4': 'org.antlr:antlr4:4.9.3',
     'assertJ': 'org.assertj:assertj-core:3.11.1',
@@ -212,18 +214,18 @@ project.ext.externalDependency = [
     'parquet': 'org.apache.parquet:parquet-avro:1.12.3',
     'parquetHadoop': 'org.apache.parquet:parquet-hadoop:1.13.1',
     'picocli': 'info.picocli:picocli:4.5.0',
-    'playCache': "com.typesafe.play:play-cache_2.12:$playVersion",
-    'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_2.12:$playVersion",
-    'playWs': 'com.typesafe.play:play-ahc-ws-standalone_2.12:2.1.10',
-    'playDocs': "com.typesafe.play:play-docs_2.12:$playVersion",
-    'playGuice': "com.typesafe.play:play-guice_2.12:$playVersion",
-    'playJavaJdbc': "com.typesafe.play:play-java-jdbc_2.12:$playVersion",
-    'playAkkaHttpServer': "com.typesafe.play:play-akka-http-server_2.12:$playVersion",
-    'playServer': "com.typesafe.play:play-server_2.12:$playVersion",
-    'playTest': "com.typesafe.play:play-test_2.12:$playVersion",
-    'playFilters': "com.typesafe.play:filters-helpers_2.12:$playVersion",
-    'pac4j': 'org.pac4j:pac4j-oidc:4.5.8',
-    'playPac4j': 'org.pac4j:play-pac4j_2.12:9.0.2',
+    'playCache': "com.typesafe.play:play-cache_$playScalaVersion:$playVersion",
+    'playCaffeineCache': "com.typesafe.play:play-caffeine-cache_$playScalaVersion:$playVersion",
+    'playWs': "com.typesafe.play:play-ahc-ws-standalone_$playScalaVersion:2.1.10",
+    'playDocs': "com.typesafe.play:play-docs_$playScalaVersion:$playVersion",
+    'playGuice': "com.typesafe.play:play-guice_$playScalaVersion:$playVersion",
+    'playJavaJdbc': "com.typesafe.play:play-java-jdbc_$playScalaVersion:$playVersion",
+    'playAkkaHttpServer': "com.typesafe.play:play-akka-http-server_$playScalaVersion:$playVersion",
+    'playServer': "com.typesafe.play:play-server_$playScalaVersion:$playVersion",
+    'playTest': "com.typesafe.play:play-test_$playScalaVersion:$playVersion",
+    'playFilters': "com.typesafe.play:filters-helpers_$playScalaVersion:$playVersion",
+    'pac4j': 'org.pac4j:pac4j-oidc:6.0.6',
+    'playPac4j': "org.pac4j:play-pac4j_$playScalaVersion:12.0.0-PLAY2.8",
     'postgresql': 'org.postgresql:postgresql:42.3.9',
     'protobuf': 'com.google.protobuf:protobuf-java:3.25.5',
     'grpcProtobuf': 'io.grpc:grpc-protobuf:1.53.0',
@@ -407,6 +409,8 @@ subprojects {
         googleJavaFormat()
         target project.fileTree(project.projectDir) {
           include 'src/**/*.java'
+          include 'app/**/*.java'
+          include 'test/**/*.java'
           exclude 'src/**/resources/'
           exclude 'src/**/generated/'
           exclude 'src/**/mainGeneratedDataTemplate/'

datahub-frontend/app/auth/AuthModule.java

@@ -21,26 +21,28 @@
 import com.linkedin.util.Configuration;
 import config.ConfigurationProvider;
 import controllers.SsoCallbackController;
-import java.nio.charset.StandardCharsets;
-import java.util.Collections;
-
 import io.datahubproject.metadata.context.ActorContext;
 import io.datahubproject.metadata.context.AuthorizationContext;
 import io.datahubproject.metadata.context.EntityRegistryContext;
 import io.datahubproject.metadata.context.OperationContext;
 import io.datahubproject.metadata.context.OperationContextConfig;
 import io.datahubproject.metadata.context.SearchContext;
+import io.datahubproject.metadata.context.ValidationContext;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import javax.annotation.Nonnull;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClients;
 import org.pac4j.core.config.Config;
 import org.pac4j.core.context.session.SessionStore;
+import org.pac4j.core.profile.ProfileManager;
+import org.pac4j.core.util.serializer.JavaSerializer;
 import org.pac4j.play.LogoutController;
 import org.pac4j.play.http.PlayHttpActionAdapter;
 import org.pac4j.play.store.PlayCacheSessionStore;
 import org.pac4j.play.store.PlayCookieSessionStore;
-import org.pac4j.play.store.PlaySessionStore;
 import org.pac4j.play.store.ShiroAesDataEncrypter;
 import org.springframework.context.annotation.AnnotationConfigApplicationContext;
 import play.Environment;
@@ -62,14 +64,16 @@ public class AuthModule extends AbstractModule {
   private static final String PAC4J_SESSIONSTORE_PROVIDER_CONF = "pac4j.sessionStore.provider";
   private static final String ENTITY_CLIENT_RETRY_INTERVAL = "entityClient.retryInterval";
   private static final String ENTITY_CLIENT_NUM_RETRIES = "entityClient.numRetries";
-  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE = "entityClient.restli.get.batchSize";
-  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY = "entityClient.restli.get.batchConcurrency";
+  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE =
+      "entityClient.restli.get.batchSize";
+  private static final String ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY =
+      "entityClient.restli.get.batchConcurrency";
   private static final String GET_SSO_SETTINGS_ENDPOINT = "auth/getSsoSettings";
 
-  private final com.typesafe.config.Config _configs;
+  private final com.typesafe.config.Config configs;
 
   public AuthModule(final Environment environment, final com.typesafe.config.Config configs) {
-    _configs = configs;
+    this.configs = configs;
   }
 
   @Override
@@ -83,31 +87,32 @@ protected void configure() {
      * the response will be rejected by the browser. Default to PlayCacheCookieStore so that
      * datahub-frontend container remains as a stateless service
      */
-    String sessionStoreProvider = _configs.getString(PAC4J_SESSIONSTORE_PROVIDER_CONF);
+    String sessionStoreProvider = configs.getString(PAC4J_SESSIONSTORE_PROVIDER_CONF);
 
     if (sessionStoreProvider.equals("PlayCacheSessionStore")) {
       final PlayCacheSessionStore playCacheSessionStore =
           new PlayCacheSessionStore(getProvider(SyncCacheApi.class));
       bind(SessionStore.class).toInstance(playCacheSessionStore);
-      bind(PlaySessionStore.class).toInstance(playCacheSessionStore);
+      bind(PlayCacheSessionStore.class).toInstance(playCacheSessionStore);
     } else {
       PlayCookieSessionStore playCacheCookieStore;
       try {
         // To generate a valid encryption key from an input value, we first
         // hash the input to generate a fixed-length string. Then, we convert
         // it to hex and slice the first 16 bytes, because AES key length must strictly
         // have a specific length.
-        final String aesKeyBase = _configs.getString(PAC4J_AES_KEY_BASE_CONF);
+        final String aesKeyBase = configs.getString(PAC4J_AES_KEY_BASE_CONF);
         final String aesKeyHash =
             DigestUtils.sha256Hex(aesKeyBase.getBytes(StandardCharsets.UTF_8));
         final String aesEncryptionKey = aesKeyHash.substring(0, 16);
         playCacheCookieStore =
             new PlayCookieSessionStore(new ShiroAesDataEncrypter(aesEncryptionKey.getBytes()));
+        playCacheCookieStore.setSerializer(new JavaSerializer());
       } catch (Exception e) {
         throw new RuntimeException("Failed to instantiate Pac4j cookie session store!", e);
       }
       bind(SessionStore.class).toInstance(playCacheCookieStore);
-      bind(PlaySessionStore.class).toInstance(playCacheCookieStore);
+      bind(PlayCookieSessionStore.class).toInstance(playCacheCookieStore);
     }
 
     try {
@@ -132,9 +137,12 @@ protected void configure() {
 
   @Provides
   @Singleton
-  protected Config provideConfig() {
+  protected Config provideConfig(@Nonnull SessionStore sessionStore) {
     Config config = new Config();
+    config.setSessionStoreFactory(parameters -> sessionStore);
     config.setHttpActionAdapter(new PlayHttpActionAdapter());
+    config.setProfileManagerFactory(ProfileManager::new);
+
     return config;
   }
 
@@ -144,7 +152,7 @@ protected SsoManager provideSsoManager(
       Authentication systemAuthentication, CloseableHttpClient httpClient) {
     SsoManager manager =
         new SsoManager(
-            _configs, systemAuthentication, getSsoSettingsRequestUrl(_configs), httpClient);
+            configs, systemAuthentication, getSsoSettingsRequestUrl(configs), httpClient);
     manager.initializeSsoProvider();
     return manager;
   }
@@ -154,8 +162,8 @@ protected SsoManager provideSsoManager(
   protected Authentication provideSystemAuthentication() {
     // Returns an instance of Authentication used to authenticate system initiated calls to Metadata
     // Service.
-    String systemClientId = _configs.getString(SYSTEM_CLIENT_ID_CONFIG_PATH);
-    String systemSecret = _configs.getString(SYSTEM_CLIENT_SECRET_CONFIG_PATH);
+    String systemClientId = configs.getString(SYSTEM_CLIENT_ID_CONFIG_PATH);
+    String systemSecret = configs.getString(SYSTEM_CLIENT_SECRET_CONFIG_PATH);
     final Actor systemActor =
         new Actor(ActorType.USER, systemClientId); // TODO: Change to service actor once supported.
     return new Authentication(
@@ -168,26 +176,25 @@ protected Authentication provideSystemAuthentication() {
   @Singleton
   @Named("systemOperationContext")
   protected OperationContext provideOperationContext(
-          final Authentication systemAuthentication,
-          final ConfigurationProvider configurationProvider) {
+      final Authentication systemAuthentication,
+      final ConfigurationProvider configurationProvider) {
     ActorContext systemActorContext =
-            ActorContext.builder()
-                    .systemAuth(true)
-                    .authentication(systemAuthentication)
-                    .build();
-    OperationContextConfig systemConfig = OperationContextConfig.builder()
+        ActorContext.builder().systemAuth(true).authentication(systemAuthentication).build();
+    OperationContextConfig systemConfig =
+        OperationContextConfig.builder()
             .viewAuthorizationConfiguration(configurationProvider.getAuthorization().getView())
             .allowSystemAuthentication(true)
             .build();
 
     return OperationContext.builder()
-            .operationContextConfig(systemConfig)
-            .systemActorContext(systemActorContext)
-            // Authorizer.EMPTY is fine since it doesn't actually apply to system auth
-            .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
-            .searchContext(SearchContext.EMPTY)
-            .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
-            .build(systemAuthentication);
+        .operationContextConfig(systemConfig)
+        .systemActorContext(systemActorContext)
+        // Authorizer.EMPTY is fine since it doesn't actually apply to system auth
+        .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
+        .searchContext(SearchContext.EMPTY)
+        .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
+        .validationContext(ValidationContext.builder().alternateValidation(false).build())
+        .build(systemAuthentication);
   }
 
   @Provides
@@ -206,23 +213,23 @@ protected SystemEntityClient provideEntityClient(
 
     return new SystemRestliEntityClient(
         buildRestliClient(),
-        new ExponentialBackoff(_configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
-        _configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
+        new ExponentialBackoff(configs.getInt(ENTITY_CLIENT_RETRY_INTERVAL)),
+        configs.getInt(ENTITY_CLIENT_NUM_RETRIES),
         configurationProvider.getCache().getClient().getEntityClient(),
-        Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)),
-        Math.max(1, _configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY)));
+        Math.max(1, configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_SIZE)),
+        Math.max(1, configs.getInt(ENTITY_CLIENT_RESTLI_GET_BATCH_CONCURRENCY)));
   }
 
   @Provides
   @Singleton
   protected AuthServiceClient provideAuthClient(
       Authentication systemAuthentication, CloseableHttpClient httpClient) {
     // Init a GMS auth client
-    final String metadataServiceHost = getMetadataServiceHost(_configs);
+    final String metadataServiceHost = getMetadataServiceHost(configs);
 
-    final int metadataServicePort = getMetadataServicePort(_configs);
+    final int metadataServicePort = getMetadataServicePort(configs);
 
-    final boolean metadataServiceUseSsl = doesMetadataServiceUseSsl(_configs);
+    final boolean metadataServiceUseSsl = doesMetadataServiceUseSsl(configs);
 
     return new AuthServiceClient(
         metadataServiceHost,
@@ -241,22 +248,22 @@ protected CloseableHttpClient provideHttpClient() {
   private com.linkedin.restli.client.Client buildRestliClient() {
     final String metadataServiceHost =
         utils.ConfigUtil.getString(
-            _configs,
+            configs,
             METADATA_SERVICE_HOST_CONFIG_PATH,
             utils.ConfigUtil.DEFAULT_METADATA_SERVICE_HOST);
     final int metadataServicePort =
         utils.ConfigUtil.getInt(
-            _configs,
+            configs,
             utils.ConfigUtil.METADATA_SERVICE_PORT_CONFIG_PATH,
             utils.ConfigUtil.DEFAULT_METADATA_SERVICE_PORT);
     final boolean metadataServiceUseSsl =
         utils.ConfigUtil.getBoolean(
-            _configs,
+            configs,
             utils.ConfigUtil.METADATA_SERVICE_USE_SSL_CONFIG_PATH,
             ConfigUtil.DEFAULT_METADATA_SERVICE_USE_SSL);
     final String metadataServiceSslProtocol =
         utils.ConfigUtil.getString(
-            _configs,
+            configs,
             utils.ConfigUtil.METADATA_SERVICE_SSL_PROTOCOL_CONFIG_PATH,
             ConfigUtil.DEFAULT_METADATA_SERVICE_SSL_PROTOCOL);
     return DefaultRestliClientFactory.getRestLiClient(

datahub-frontend/app/auth/AuthUtils.java

@@ -75,6 +75,7 @@ public class AuthUtils {
   public static final String RESPONSE_MODE = "responseMode";
   public static final String USE_NONCE = "useNonce";
   public static final String READ_TIMEOUT = "readTimeout";
+  public static final String CONNECT_TIMEOUT = "connectTimeout";
   public static final String EXTRACT_JWT_ACCESS_TOKEN_CLAIMS = "extractJwtAccessTokenClaims";
   // Retained for backwards compatibility
   public static final String PREFERRED_JWS_ALGORITHM = "preferredJwsAlgorithm";