Skip to content

Commit

Permalink
Consolidate resource-lix into datahub-gma (#446)
Browse files Browse the repository at this point in the history
Co-authored-by: Yang Yang <[email protected]>
  • Loading branch information
yangyangv2 and Yang Yang authored Oct 8, 2024
1 parent 6cd913f commit c6be657
Show file tree
Hide file tree
Showing 6 changed files with 155 additions and 176 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import com.linkedin.metadata.query.IndexSortCriterion;
import com.linkedin.metadata.query.ListResultMetadata;
import com.linkedin.metadata.query.MapMetadata;
import com.linkedin.metadata.restli.lix.DummyResourceLix;
import com.linkedin.metadata.restli.lix.RampedResourceImpl;
import com.linkedin.metadata.restli.lix.ResourceLix;
import com.linkedin.parseq.Task;
import com.linkedin.restli.common.EmptyRecord;
Expand Down Expand Up @@ -96,7 +96,7 @@ public abstract class BaseEntityResource<
private final Class<ASSET> _assetClass;
protected final Class<URN> _urnClass;
protected BaseTrackingManager _trackingManager = null;
private ResourceLix _defaultResourceLix = new DummyResourceLix();
private ResourceLix _defaultResourceLix = new RampedResourceImpl();

/**
* This method is to be overriden by specific resource endpoint implementation with real lix impl.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,12 @@
import javax.annotation.Nullable;


public class DummyResourceLix implements ResourceLix {

/**
* Legacy Resource will always choose using the old MG Kernel logic, equivalent to Lix is always 'control'. Legacy
* Resource is only used for the GMSes are on the maintenance mode (using legacy models Snapshot, Aspect Union)
* and are not planning to evolve (e.g. the legacy. AIM ). Usage: assign to 'resourceLix' fields at each GMS entity resource.
*/
public class LegacyResourceImpl implements ResourceLix {
@Override
public boolean testGet(@Nonnull String urn, @Nonnull String entityType) {
return false;
Expand Down Expand Up @@ -80,4 +84,4 @@ public boolean testSearch(@Nullable String urnType) {
public boolean testSearchV2(@Nullable String urnType) {
return false;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package com.linkedin.metadata.restli.lix;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;


/**
* Ramped Resource will always choose using the new MG Kernel logic, equivalent to Lix is always 'treatment'.
* Usage: assign to 'resourceLix' fields at each GMS entity resource.
*/
public class RampedResourceImpl implements ResourceLix {
@Override
public boolean testGet(@Nonnull String urn, @Nonnull String entityType) {
return true;
}

@Override
public boolean testBatchGet(@Nullable String urn, @Nullable String entityType) {
return true;
}

@Override
public boolean testBatchGetWithErrors(@Nullable String urn, @Nullable String type) {
return true;
}

@Override
public boolean testGetSnapshot(@Nullable String urn, @Nullable String entityType) {
return true;
}

@Override
public boolean testBackfillLegacy(@Nullable String urn, @Nullable String entityType) {
return true;
}

@Override
public boolean testBackfillWithUrns(@Nullable String urn, @Nullable String entityType) {
return true;
}

@Override
public boolean testEmitNoChangeMetadataAuditEvent(@Nullable String urn, @Nullable String entityType) {
return true;
}

@Override
public boolean testBackfillWithNewValue(@Nullable String urn, @Nullable String entityType) {
return true;
}

@Override
public boolean testBackfillEntityTables(@Nullable String urn, @Nullable String entityType) {
return true;
}

@Override
public boolean testBackfillRelationshipTables(@Nullable String urn, @Nullable String entityType) {
return true;
}

@Override
public boolean testBackfill(@Nonnull String assetType, @Nonnull String mode) {
return true;
}

@Override
public boolean testFilter(@Nonnull String assetType) {
return true;
}

@Override
public boolean testGetAll(@Nullable String urnType) {
return true;
}

@Override
public boolean testSearch(@Nullable String urnType) {
return true;
}

@Override
public boolean testSearchV2(@Nullable String urnType) {
return true;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.linkedin.metadata.query.IndexSortCriterion;
import com.linkedin.metadata.query.MapMetadata;
import com.linkedin.metadata.query.SortOrder;
import com.linkedin.metadata.restli.lix.LegacyResourceImpl;
import com.linkedin.metadata.restli.lix.ResourceLix;
import com.linkedin.parseq.BaseEngineTest;
import com.linkedin.restli.common.ComplexResourceKey;
Expand Down Expand Up @@ -62,7 +63,6 @@
import java.util.Set;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;

Expand All @@ -82,6 +82,11 @@ class TestResource extends
BaseEntityResource<ComplexResourceKey<EntityKey, EmptyRecord>, EntityValue, FooUrn, EntitySnapshot,
EntityAspectUnion, InternalEntitySnapshot, InternalEntityAspectUnion, EntityAsset> {

@Override
protected ResourceLix getResourceLix() {
return new LegacyResourceImpl();
}

public TestResource() {
super(EntitySnapshot.class, EntityAspectUnion.class, FooUrn.class, InternalEntitySnapshot.class,
InternalEntityAspectUnion.class, EntityAsset.class);
Expand Down Expand Up @@ -160,86 +165,6 @@ class TestInternalResource extends
BaseEntityResource<ComplexResourceKey<EntityKey, EmptyRecord>, EntityValue, FooUrn, EntitySnapshot,
EntityAspectUnion, InternalEntitySnapshot, InternalEntityAspectUnion, EntityAsset> {

@Override
protected ResourceLix getResourceLix() {
return new ResourceLix() {
@Override
public boolean testGet(@Nonnull String urn, @Nonnull String entityType) {
return true;
}

@Override
public boolean testBatchGet(@Nullable String urn, @Nullable String entityType) {
return true;
}

@Override
public boolean testBatchGetWithErrors(@Nullable String urn, @Nullable String type) {
return false;
}

@Override
public boolean testGetSnapshot(@Nullable String urn, @Nullable String entityType) {
return true;
}

@Override
public boolean testBackfillLegacy(@Nullable String urn, @Nullable String entityType) {
return false;
}

@Override
public boolean testBackfillWithUrns(@Nullable String urn, @Nullable String entityType) {
return false;
}

@Override
public boolean testEmitNoChangeMetadataAuditEvent(@Nullable String urn, @Nullable String entityType) {
return false;
}

@Override
public boolean testBackfillWithNewValue(@Nullable String urn, @Nullable String entityType) {
return false;
}

@Override
public boolean testBackfillEntityTables(@Nullable String urn, @Nullable String entityType) {
return false;
}

@Override
public boolean testBackfillRelationshipTables(@Nullable String urn, @Nullable String entityType) {
return false;
}

@Override
public boolean testBackfill(@Nonnull String assetType, @Nonnull String mode) {
return false;
}

@Override
public boolean testFilter(@Nonnull String assetType) {
return true;
}

@Override
public boolean testGetAll(@Nullable String urnType) {
return false;
}

@Override
public boolean testSearch(@Nullable String urnType) {
return false;
}

@Override
public boolean testSearchV2(@Nullable String urnType) {
return false;
}
};
}

public TestInternalResource() {
super(EntitySnapshot.class, EntityAspectUnion.class, FooUrn.class, InternalEntitySnapshot.class,
InternalEntityAspectUnion.class, EntityAsset.class);
Expand Down Expand Up @@ -518,7 +443,8 @@ public void testBatchGetWithErrorsUrnsNotFound() {
.thenReturn(Collections.emptyMap());

BatchResult<ComplexResourceKey<EntityKey, EmptyRecord>, EntityValue> result =
runAndWait(_resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames));
runAndWait(
_resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames));

// convert BatchResult<ComplexResourceKey<EntityKey, EmptyRecord>, EntityValue> to BatchResult<EntityKey, EntityValue>
BatchResult<EntityKey, EntityValue> batchResultMap = convertBatchResult(result);
Expand Down Expand Up @@ -596,7 +522,8 @@ public void testBatchGetWithErrorsSpecificAspectsPartialSuccess() {
.thenReturn(ImmutableMap.of(aspectFooKey1, Optional.of(foo), aspectBarKey2, Optional.of(bar)));

BatchResult<ComplexResourceKey<EntityKey, EmptyRecord>, EntityValue> result =
runAndWait(_resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames));
runAndWait(
_resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames));

// convert BatchResult<ComplexResourceKey<EntityKey, EmptyRecord>, EntityValue> to BatchResult<EntityKey, EntityValue>
BatchResult<EntityKey, EntityValue> batchResultMap = convertBatchResult(result);
Expand Down Expand Up @@ -635,7 +562,8 @@ public void testBatchGetWithErrorsUrnsPartialSuccess() {
.thenReturn(ImmutableMap.of(aspectFooKey1, Optional.of(foo), aspectBarKey2, Optional.empty()));

BatchResult<ComplexResourceKey<EntityKey, EmptyRecord>, EntityValue> result =
runAndWait(_resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames));
runAndWait(
_resource.batchGetWithErrors(ImmutableSet.of(makeResourceKey(urn1), makeResourceKey(urn2)), aspectNames));

// convert BatchResult<ComplexResourceKey<EntityKey, EmptyRecord>, EntityValue> to BatchResult<EntityKey, EntityValue>
BatchResult<EntityKey, EntityValue> batchResultMap = convertBatchResult(result);
Expand Down Expand Up @@ -962,7 +890,8 @@ public void testBackfillWithNewValue() {
AspectBar bar1 = new AspectBar().setValue("bar1");
AspectBar bar2 = new AspectBar().setValue("bar2");
String[] aspects = new String[]{"com.linkedin.testing.AspectFoo", "com.linkedin.testing.AspectBar"};
when(_mockLocalDAO.backfillWithNewValue(_resource.parseAspectsParam(aspects, false), ImmutableSet.of(urn1, urn2)))
when(_mockLocalDAO.backfillWithNewValue(
_resource.parseAspectsParam(aspects, false), ImmutableSet.of(urn1, urn2)))
.thenReturn(
ImmutableMap.of(urn1, ImmutableMap.of(AspectFoo.class, Optional.of(foo1), AspectBar.class, Optional.of(bar1)),
urn2, ImmutableMap.of(AspectBar.class, Optional.of(bar2)))
Expand Down Expand Up @@ -1001,7 +930,8 @@ public void testEmitNoChangeMetadataAuditEvent() {
);

BackfillResult backfillResult =
runAndWait(_resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects,
runAndWait(
_resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects,
IngestionMode.BACKFILL));
assertEquals(backfillResult.getEntities().size(), 2);

Expand Down Expand Up @@ -1034,7 +964,8 @@ public void testEmitNoChangeMetadataAuditEventBootstrap() {
);

BackfillResult backfillResult =
runAndWait(_resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects,
runAndWait(
_resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects,
IngestionMode.BOOTSTRAP));
assertEquals(backfillResult.getEntities().size(), 2);

Expand Down Expand Up @@ -1062,7 +993,8 @@ public void testEmitNoChangeMetadataAuditEventNoResult() {
String[] aspects = new String[]{"com.linkedin.testing.AspectFoo", "com.linkedin.testing.AspectBar"};

BackfillResult backfillResult =
runAndWait(_resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects,
runAndWait(
_resource.emitNoChangeMetadataAuditEvent(new String[]{urn1.toString(), urn2.toString()}, aspects,
IngestionMode.LIVE));
verify(_mockLocalDAO, times(0)).backfill(any(BackfillMode.class), any(Set.class), any(Set.class));
assertFalse(backfillResult.hasEntities());
Expand All @@ -1082,7 +1014,8 @@ public void testBackfillRelationshipTables() {
List<LocalRelationshipUpdates> relationships = Collections.singletonList(updates);

when(_mockLocalDAO.backfillLocalRelationships(fooUrn, AspectFoo.class)).thenReturn(relationships);
BackfillResult backfillResult = runAndWait(_resource.backfillRelationshipTables(new String[]{fooUrn.toString()}, aspects));
BackfillResult backfillResult = runAndWait(
_resource.backfillRelationshipTables(new String[]{fooUrn.toString()}, aspects));

assertTrue(backfillResult.hasRelationships());
assertEquals(backfillResult.getRelationships().size(), 1);
Expand Down Expand Up @@ -1171,7 +1104,8 @@ public void testFilterFromIndexEmptyAspects() {
.build();
when(_mockLocalDAO.listUrns(null, indexSortCriterion, 0, 2)).thenReturn(urnsListResult);
ListResult<EntityValue>
listResultActual = runAndWait(_resource.filter(null, indexSortCriterion, new String[0], new PagingContext(0, 2)));
listResultActual = runAndWait(
_resource.filter(null, indexSortCriterion, new String[0], new PagingContext(0, 2)));
List<EntityValue> actualValues = listResultActual.getValues();
assertEquals(actualValues.size(), 2);
assertEquals(actualValues.get(0), new EntityValue());
Expand Down
Loading

0 comments on commit c6be657

Please sign in to comment.