Merge branch 'master' into feat/dremio-connector-source

datahub-project · Oct 28, 2024 · 102b55f · 102b55f
2 parents 9de8be0 + a6e1624
commit 102b55f
Show file tree

Hide file tree

Showing 152 changed files with 50,689 additions and 971 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -70,7 +70,11 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
-          cache: pip
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
       - name: Set up JDK 17
         uses: actions/setup-java@v4
         with:

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -44,7 +44,11 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.10"
-          cache: pip
+      - uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/uv
+          key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
       - name: Install Python dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Build Docs

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -62,16 +62,11 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: "pip"
       - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/uv
           key: ${{ runner.os }}-uv-${{ hashFiles('**/requirements.txt') }}
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "pip"
       - name: Install dependencies
         run: ./metadata-ingestion/scripts/install_deps.sh
       - name: Install package

diff --git a/datahub-frontend/app/auth/AuthModule.java b/datahub-frontend/app/auth/AuthModule.java
@@ -21,6 +21,7 @@
 import com.linkedin.util.Configuration;
 import config.ConfigurationProvider;
 import controllers.SsoCallbackController;
+import io.datahubproject.metadata.context.ValidationContext;
 import java.nio.charset.StandardCharsets;
 import java.util.Collections;
 
@@ -187,6 +188,7 @@ protected OperationContext provideOperationContext(
             .authorizationContext(AuthorizationContext.builder().authorizer(Authorizer.EMPTY).build())
             .searchContext(SearchContext.EMPTY)
             .entityRegistryContext(EntityRegistryContext.builder().build(EmptyEntityRegistry.EMPTY))
+            .validationContext(ValidationContext.builder().alternateValidation(false).build())
             .build(systemAuthentication);
   }
 

diff --git a/...raphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java b/...raphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/ContainerType.java
@@ -52,7 +52,8 @@ public class ContainerType
           Constants.DEPRECATION_ASPECT_NAME,
           Constants.DATA_PRODUCTS_ASPECT_NAME,
           Constants.STRUCTURED_PROPERTIES_ASPECT_NAME,
-          Constants.FORMS_ASPECT_NAME);
+          Constants.FORMS_ASPECT_NAME,
+          Constants.ACCESS_ASPECT_NAME);
 
   private static final Set<String> FACET_FIELDS = ImmutableSet.of("origin", "platform");
   private static final String ENTITY_NAME = "container";

diff --git a/...e/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java b/...e/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java
@@ -2,6 +2,7 @@
 
 import static com.linkedin.metadata.Constants.*;
 
+import com.linkedin.common.Access;
 import com.linkedin.common.DataPlatformInstance;
 import com.linkedin.common.Deprecation;
 import com.linkedin.common.Forms;
@@ -30,6 +31,7 @@
 import com.linkedin.datahub.graphql.types.domain.DomainAssociationMapper;
 import com.linkedin.datahub.graphql.types.form.FormsMapper;
 import com.linkedin.datahub.graphql.types.glossary.mappers.GlossaryTermsMapper;
+import com.linkedin.datahub.graphql.types.rolemetadata.mappers.AccessMapper;
 import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertiesMapper;
 import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper;
 import com.linkedin.domain.Domains;
@@ -105,6 +107,11 @@ public static Container map(
               context, new GlossaryTerms(envelopedTerms.getValue().data()), entityUrn));
     }
 
+    final EnvelopedAspect accessAspect = aspects.get(ACCESS_ASPECT_NAME);
+    if (accessAspect != null) {
+      result.setAccess(AccessMapper.map(new Access(accessAspect.getValue().data()), entityUrn));
+    }
+
     final EnvelopedAspect envelopedInstitutionalMemory =
         aspects.get(Constants.INSTITUTIONAL_MEMORY_ASPECT_NAME);
     if (envelopedInstitutionalMemory != null) {

diff --git a/...ub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java b/...ub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/DatasetType.java
@@ -86,7 +86,7 @@ public class DatasetType
           EMBED_ASPECT_NAME,
           DATA_PRODUCTS_ASPECT_NAME,
           BROWSE_PATHS_V2_ASPECT_NAME,
-          ACCESS_DATASET_ASPECT_NAME,
+          ACCESS_ASPECT_NAME,
           STRUCTURED_PROPERTIES_ASPECT_NAME,
           FORMS_ASPECT_NAME,
           SUB_TYPES_ASPECT_NAME);

diff --git a/...-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/...-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java
@@ -166,7 +166,7 @@ public Dataset apply(
         (dataset, dataMap) ->
             dataset.setBrowsePathV2(BrowsePathsV2Mapper.map(context, new BrowsePathsV2(dataMap))));
     mappingHelper.mapToResult(
-        ACCESS_DATASET_ASPECT_NAME,
+        ACCESS_ASPECT_NAME,
         ((dataset, dataMap) ->
             dataset.setAccess(AccessMapper.map(new Access(dataMap), entityUrn))));
     mappingHelper.mapToResult(

diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql
@@ -2837,6 +2837,11 @@ type Container implements Entity {
     """
     exists: Boolean
 
+    """
+	The Roles and the properties to access the container
+	"""
+	access: Access
+
     """
     Experimental API.
     For fetching extra entities that do not have custom UI code yet

diff --git a/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java b/datahub-upgrade/src/main/java/com/linkedin/datahub/upgrade/config/SystemUpdateConfig.java
@@ -29,6 +29,7 @@
 import io.datahubproject.metadata.context.OperationContextConfig;
 import io.datahubproject.metadata.context.RetrieverContext;
 import io.datahubproject.metadata.context.ServicesRegistryContext;
+import io.datahubproject.metadata.context.ValidationContext;
 import io.datahubproject.metadata.services.RestrictedService;
 import java.util.List;
 import javax.annotation.Nonnull;
@@ -161,7 +162,8 @@ protected OperationContext javaSystemOperationContext(
       @Nonnull final GraphService graphService,
       @Nonnull final SearchService searchService,
       @Qualifier("baseElasticSearchComponents")
-          BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components) {
+          BaseElasticSearchComponentsFactory.BaseElasticSearchComponents components,
+      @Nonnull final ConfigurationProvider configurationProvider) {
 
     EntityServiceAspectRetriever entityServiceAspectRetriever =
         EntityServiceAspectRetriever.builder()
@@ -186,6 +188,10 @@ protected OperationContext javaSystemOperationContext(
                 .aspectRetriever(entityServiceAspectRetriever)
                 .graphRetriever(systemGraphRetriever)
                 .searchRetriever(searchServiceSearchRetriever)
+                .build(),
+            ValidationContext.builder()
+                .alternateValidation(
+                    configurationProvider.getFeatureFlags().isAlternateMCPValidation())
                 .build());
 
     entityServiceAspectRetriever.setSystemOperationContext(systemOperationContext);

diff --git a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx
@@ -8,7 +8,7 @@ import { DocumentationTab } from '../shared/tabs/Documentation/DocumentationTab'
 import { SidebarAboutSection } from '../shared/containers/profile/sidebar/AboutSection/SidebarAboutSection';
 import { SidebarOwnerSection } from '../shared/containers/profile/sidebar/Ownership/sidebar/SidebarOwnerSection';
 import { getDataForEntityType } from '../shared/containers/profile/utils';
-import { useGetContainerQuery } from '../../../graphql/container.generated';
+import { useGetContainerQuery, GetContainerQuery } from '../../../graphql/container.generated';
 import { ContainerEntitiesTab } from './ContainerEntitiesTab';
 import { SidebarTagsSection } from '../shared/containers/profile/sidebar/SidebarTagsSection';
 import { PropertiesTab } from '../shared/tabs/Properties/PropertiesTab';
@@ -17,6 +17,8 @@ import { capitalizeFirstLetterOnly } from '../../shared/textUtil';
 import DataProductSection from '../shared/containers/profile/sidebar/DataProduct/DataProductSection';
 import { getDataProduct } from '../shared/utils';
 import EmbeddedProfile from '../shared/embed/EmbeddedProfile';
+import AccessManagement from '../shared/tabs/Dataset/AccessManagement/AccessManagement';
+import { useAppConfig } from '../../useAppConfig';
 
 /**
  * Definition of the DataHub Container entity.
@@ -65,6 +67,8 @@ export class ContainerEntity implements Entity<Container> {
 
     useEntityQuery = useGetContainerQuery;
 
+    appconfig = useAppConfig;
+
     renderProfile = (urn: string) => (
         <EntityProfile
             urn={urn}
@@ -85,6 +89,23 @@ export class ContainerEntity implements Entity<Container> {
                     name: 'Properties',
                     component: PropertiesTab,
                 },
+                {
+                    name: 'Access Management',
+                    component: AccessManagement,
+                    display: {
+                        visible: (_, container: GetContainerQuery) => {
+                            return (
+                                this.appconfig().config.featureFlags.showAccessManagement &&
+                                !!container?.container?.access
+                            );
+                        },
+                        enabled: (_, container: GetContainerQuery) => {
+                            const accessAspect = container?.container?.access;
+                            const rolesList = accessAspect?.roles;
+                            return !!accessAspect && !!rolesList && rolesList.length > 0;
+                        },
+                    },
+                },
             ]}
             sidebarSections={this.getSidebarSections()}
         />

diff --git a/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx b/datahub-web-react/src/app/entity/shared/tabs/Dataset/AccessManagement/AccessManagement.tsx
@@ -3,8 +3,8 @@ import styled from 'styled-components';
 import { Button, Table } from 'antd';
 import { SpinProps } from 'antd/es/spin';
 import { LoadingOutlined } from '@ant-design/icons';
-import { useBaseEntity } from '../../../EntityContext';
-import { GetDatasetQuery, useGetExternalRolesQuery } from '../../../../../../graphql/dataset.generated';
+import { useEntityData } from '../../../EntityContext';
+import { useGetExternalRolesQuery } from '../../../../../../graphql/dataset.generated';
 import { handleAccessRoles } from './utils';
 import AccessManagerDescription from './AccessManagerDescription';
 
@@ -60,11 +60,12 @@ const AccessButton = styled(Button)`
 `;
 
 export default function AccessManagement() {
-    const baseEntity = useBaseEntity<GetDatasetQuery>();
+    const { entityData } = useEntityData();
+    const entityUrn = (entityData as any)?.urn;
 
     const { data: externalRoles, loading: isLoading } = useGetExternalRolesQuery({
-        variables: { urn: baseEntity?.dataset?.urn as string },
-        skip: !baseEntity?.dataset?.urn,
+        variables: { urn: entityUrn as string },
+        skip: !entityUrn,
     });
 
     const columns = [

diff --git a/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx b/datahub-web-react/src/app/ingest/source/executions/reporting/StructuredReportItemContext.tsx
@@ -17,7 +17,7 @@ const Title = styled.div`
     font-weight: bold;
 `;
 
-const Item = styled.div`
+const Item = styled.pre`
     padding: 6px;
     font-size: 12px;
     border-radius: 2px;

diff --git a/datahub-web-react/src/graphql/container.graphql b/datahub-web-react/src/graphql/container.graphql
@@ -59,6 +59,13 @@ query getContainer($urn: String!) {
         status {
             removed
         }
+        access {
+            roles {
+                role {
+                    urn
+                }
+            }
+        }
         autoRenderAspects: aspects(input: { autoRenderOnly: true }) {
             ...autoRenderAspectFields
         }

diff --git a/docker/build.gradle b/docker/build.gradle
@@ -8,6 +8,8 @@ import com.avast.gradle.dockercompose.tasks.ComposeDownForced
 apply from: "../gradle/versioning/versioning.gradle"
 
 ext {
+    compose_base = "profiles/docker-compose.yml"
+
     backend_profile_modules = [
             ':docker:elasticsearch-setup',
             ':docker:mysql-setup',
@@ -23,7 +25,7 @@ ext {
 
     debug_modules = quickstart_modules - [':metadata-jobs:mce-consumer-job',
                                           ':metadata-jobs:mae-consumer-job']
-    compose_args = ['-f', 'profiles/docker-compose.yml']
+    compose_args = ['-f', compose_base]
     debug_reloadable = [
             'datahub-gms-debug',
             'system-update-debug',
@@ -68,7 +70,7 @@ dockerCompose {
         environment.put 'DATAHUB_VERSION', "v${version}"
         environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
 
-        useComposeFiles = ['profiles/docker-compose.yml']
+        useComposeFiles = [compose_base]
         projectName = 'datahub'
         projectNamePrefix = ''
         buildBeforeUp = false
@@ -86,7 +88,7 @@ dockerCompose {
         environment.put 'DATAHUB_VERSION', "v${version}"
         environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
 
-        useComposeFiles = ['profiles/docker-compose.yml']
+        useComposeFiles = [compose_base]
         projectName = 'datahub'
         projectNamePrefix = ''
         buildBeforeUp = false
@@ -113,7 +115,7 @@ dockerCompose {
         // disabled for spark-lineage smoke-test
         environment.put 'DATAHUB_LOCAL_COMMON_ENV', "${rootProject.project(':metadata-integration:java:spark-lineage-legacy').projectDir}/spark-smoke-test/smoke-gms.env"
 
-        useComposeFiles = ['profiles/docker-compose.yml']
+        useComposeFiles = [compose_base]
         projectName = 'datahub'
         projectNamePrefix = ''
         buildBeforeUp = false
@@ -133,7 +135,7 @@ dockerCompose {
         }
         environment.put 'DATAHUB_TELEMETRY_ENABLED', 'false' // disabled when built locally
 
-        useComposeFiles = ['profiles/docker-compose.yml']
+        useComposeFiles = [compose_base]
         projectName = 'datahub'
         projectNamePrefix = ''
         buildBeforeUp = false
@@ -146,7 +148,7 @@ dockerCompose {
         isRequiredBy(tasks.named('quickstartStorage'))
         composeAdditionalArgs = ['--profile', 'quickstart-storage']
 
-        useComposeFiles = ['profiles/docker-compose.yml']
+        useComposeFiles = [compose_base]
         projectName = 'datahub'
         projectNamePrefix = ''
         buildBeforeUp = false

diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile
@@ -1,4 +1,4 @@
-ARG KAFKA_DOCKER_VERSION=7.4.6
+ARG KAFKA_DOCKER_VERSION=7.7.1
 
 # Defining custom repo urls for use in enterprise environments. Re-used between stages below.
 ARG ALPINE_REPO_URL=http://dl-cdn.alpinelinux.org/alpine

diff --git a/docker/profiles/docker-compose.gms.yml b/docker/profiles/docker-compose.gms.yml
@@ -101,6 +101,7 @@ x-datahub-gms-service: &datahub-gms-service
   environment: &datahub-gms-env
     <<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env]
     ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE: ${ELASTICSEARCH_QUERY_CUSTOM_CONFIG_FILE:-search_config.yaml}
+    ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true}
   healthcheck:
     test: curl -sS --fail http://datahub-gms:${DATAHUB_GMS_PORT:-8080}/health
     start_period: 90s
@@ -182,6 +183,7 @@ x-datahub-mce-consumer-service: &datahub-mce-consumer-service
     - ${DATAHUB_LOCAL_MCE_ENV:-empty2.env}
   environment: &datahub-mce-consumer-env
     <<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env]
+    ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true}
 
 x-datahub-mce-consumer-service-dev: &datahub-mce-consumer-service-dev
   <<: *datahub-mce-consumer-service

diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js
@@ -68,7 +68,7 @@ module.exports = {
     announcementBar: {
           id: "announcement-2",
           content:
-            '<div style="display: flex; justify-content: center; align-items: center;width: 100%;"><!--img src="/img/acryl-logo-white-mark.svg" / --><div style="font-size: .8rem; font-weight: 600; background-color: white; color: #111; padding: 0px 8px; border-radius: 4px; margin-right:12px;">NEW</div><p><span>Join us at Metadata & AI Summit, Oct. 29 & 30!</span></p><a href="http://www.acryldata.io/conference?utm_source=datahub_web&utm_medium=metadata_ai_2024&utm_campaign=home_banner" target="_blank" class="button">Register →</a></div>',
+            '<div style="display: flex; justify-content: center; align-items: center;width: 100%;"><!--img src="/img/acryl-logo-white-mark.svg" / --><div style="font-size: .8rem; font-weight: 600; background-color: white; color: #111; padding: 0px 8px; border-radius: 4px; margin-right:12px;">NEW</div><p>Join us at Metadata & AI Summit, Oct. 29 & 30!</p><a href="http://www.acryldata.io/conference?utm_source=datahub_web&utm_medium=metadata_ai_2024&utm_campaign=home_banner" target="_blank" class="button">Register<span> →</span></a></div>',
           backgroundColor: "#111",
           textColor: "#ffffff",
           isCloseable: false,

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
@@ -101,12 +101,19 @@ module.exports = {
         {
           label: "Automations",
           type: "category",
+          collapsed: false,
           items: [
             {
               label: "Documentation Propagation",
               type: "doc",
               id: "docs/automations/docs-propagation",
             },
+            {
+              label: "BigQuery Metadata Sync",
+              type: "doc",
+              id: "docs/automations/bigquery-metadata-sync",
+              className: "saasOnly",
+            },
             {
               label: "Snowflake Tag Sync",
               type: "doc",