Skip to content

Commit

Permalink
Merge branch 'cluster' into integ
Browse files Browse the repository at this point in the history
  • Loading branch information
@joshpalis
joshpalis committed Jun 8, 2023
2 parents 0bf30f3 + e8310bf commit 446369d
Show file tree
Hide file tree
Showing 36 changed files with 853 additions and 320 deletions.
16 changes: 2 additions & 14 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -692,7 +692,6 @@ List<String> jacocoExclusions = [
'org.opensearch.ad.stats.ADStat',
'org.opensearch.ad.feature.AbstractRetriever',
'org.opensearch.ad.feature.SearchFeatureDao',
'org.opensearch.ad.ml.SingleStreamModelIdMapper',
'org.opensearch.ad.settings.AbstractSetting',
'org.opensearch.ad.util.ExceptionUtil',
'org.opensearch.ad.util.DiscoveryNodeFilterer.HotDataNodePredicate',
Expand Down Expand Up @@ -723,13 +722,11 @@ List<String> jacocoExclusions = [
'org.opensearch.ad.transport.GetAnomalyDetectorRequest',
'org.opensearch.ad.transport.AnomalyResultTransportAction',
'org.opensearch.ad.transport.DeleteAnomalyDetectorAction',
'org.opensearch.ad.transport.ForwardADTaskRequest',
'org.opensearch.ad.transport.AnomalyResultTransportAction.EntityResultListener',
'org.opensearch.ad.feature.CompositeRetriever',
'org.opensearch.ad.ml.EntityColdStarter',
'org.opensearch.ad.AbstractProfileRunner',
'org.opensearch.ad.Name',
'org.opensearch.ad.NodeStateManager',
'org.opensearch.ad.model.EntityProfile.Builder',
'org.opensearch.ad.transport.AnomalyResultTransportAction.PageListener',
'org.opensearch.ad.feature.CompositeRetriever.Page',
Expand All @@ -739,7 +736,6 @@ List<String> jacocoExclusions = [
'org.opensearch.ad.transport.ADJobRunnerTransportAction*',
'org.opensearch.ad.AnomalyDetectorProfileRunner',
'org.opensearch.ad.EntityProfileRunner',
'org.opensearch.ad.util.BulkUtil',
'org.opensearch.ad.common.exception.InternalFailure',
'org.opensearch.ad.cluster.HourlyCron',
'org.opensearch.ad.cluster.ADClusterEventListener',
Expand All @@ -750,8 +746,6 @@ List<String> jacocoExclusions = [
'org.opensearch.ad.transport.AnomalyResultTransportAction.RCFActionListener',
'org.opensearch.ad.transport.AnomalyResultRequest',
'org.opensearch.ad.transport.AnomalyResultAction',
'org.opensearch.ad.transport.CronNodeResponse',
'org.opensearch.ad.transport.CronResponse',
'org.opensearch.ad.transport.AnomalyResultResponse',
'org.opensearch.ad.common.exception.ClientException',
'org.opensearch.ad.caching.DoorKeeper',
Expand All @@ -765,27 +759,21 @@ List<String> jacocoExclusions = [
'org.opensearch.ad.stats.InternalStatNames',
'org.opensearch.ad.NodeState',
'org.opensearch.ad.transport.DeleteModelRequest',
'org.opensearch.ad.transport.DeleteModelNodeResponse',
'org.opensearch.ad.transport.DeleteModelResponse',
'org.opensearch.ad.transport.StopDetectorTransportAction',
'org.opensearch.ad.transport.StopDetectorRequest',
'org.opensearch.ad.ratelimit.RateLimitedRequestWorker',
'org.opensearch.ad.ratelimit.QueuedRequest',
'org.opensearch.ad.ratelimit.CheckpointReadWorker',
'org.opensearch.ad.ratelimit.ConcurrentWorker',
'org.opensearch.ad.ratelimit.EntityFeatureRequest',
'org.opensearch.ad.ratelimit.RateLimitedRequestWorker.RequestQueue',
'org.opensearch.ad.stats.StatNames',
'org.opensearch.ad.MaintenanceState',
'org.opensearch.ad.AnomalyDetectorExtension.*',
'org.opensearch.ad.EntityProfileRunner',
'org.opensearch.ad.caching.CacheProvider',
'org.opensearch.ad.transport.ADResultBulkTransportAction',
'org.opensearch.ad.transport.ADResultBulkRequest',
'org.opensearch.ad.transport.ADResultBulkAction',
'org.opensearch.ad.ratelimit.ResultWriteRequest',
'org.opensearch.ad.AnomalyDetectorJobRunner.*',
'org.opensearch.ad.util.RestHandlerUtils',
'org.opensearch.ad.transport.SearchAnomalyDetectorInfoTransportAction.*',
'org.opensearch.ad.transport.RCFPollingAction',
'org.opensearch.ad.transport.RCFPollingRequest',
Expand All @@ -803,7 +791,7 @@ jacocoTestCoverageVerification {
excludes = jacocoExclusions
limit {
counter = 'BRANCH'
minimum = 0.50
minimum = 0.40
}
}
rule {
Expand All @@ -812,7 +800,7 @@ jacocoTestCoverageVerification {
limit {
counter = 'LINE'
value = 'COVEREDRATIO'
minimum = 0.60
minimum = 0.50
}
}
}
Expand Down
12 changes: 10 additions & 2 deletions src/main/java/org/opensearch/ad/AnomalyDetectorExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,14 @@ public Collection<Object> createComponents(ExtensionsRunner runner) {

Throttler throttler = new Throttler(getClock());
ClientUtil clientUtil = new ClientUtil(environmentSettings, restClient(), throttler);
IndexUtils indexUtils = new IndexUtils(restClient(), clientUtil, sdkClusterService, indexNameExpressionResolver, javaAsyncClient());
IndexUtils indexUtils = new IndexUtils(
restClient(),
clientUtil,
sdkClusterService,
indexNameExpressionResolver,
javaAsyncClient(),
environmentSettings
);
nodeFilter = new DiscoveryNodeFilterer(sdkClusterService);
AnomalyDetectionIndices anomalyDetectionIndices = new AnomalyDetectionIndices(
sdkRestClient,
Expand Down Expand Up @@ -358,7 +365,8 @@ public PooledObject<LinkedBuffer> wrap(LinkedBuffer obj) {
AnomalyDetectorSettings.MAX_CHECKPOINT_BYTES,
serializeRCFBufferPool,
AnomalyDetectorSettings.SERIALIZATION_BUFFER_BYTES,
1 - AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE
1 - AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE,
environmentSettings
);

Random random = new Random(42);
Expand Down
8 changes: 6 additions & 2 deletions src/main/java/org/opensearch/ad/AnomalyDetectorJobRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -651,7 +651,9 @@ public void onFailure(Exception exception) {
}

});
Response response = acquireLockResponse.orTimeout(15L, TimeUnit.SECONDS).join();
Response response = acquireLockResponse
.orTimeout(AnomalyDetectorSettings.REQUEST_TIMEOUT.get(settings).getMillis(), TimeUnit.MILLISECONDS)
.join();

log.info("Acquired lock for AD job {}", context.getJobId());

Expand Down Expand Up @@ -684,7 +686,9 @@ public void onFailure(Exception exception) {
}

});
Response response = releaseLockResponse.orTimeout(15L, TimeUnit.SECONDS).join();
Response response = releaseLockResponse
.orTimeout(AnomalyDetectorSettings.REQUEST_TIMEOUT.get(settings).getMillis(), TimeUnit.MILLISECONDS)
.join();

boolean lockIsReleased = RestStatus.fromCode(response.getStatusLine().getStatusCode()) == RestStatus.OK ? true : false;
if (lockIsReleased) {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/opensearch/ad/NodeStateManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@
import org.opensearch.ad.transport.BackPressureRouting;
import org.opensearch.ad.util.ClientUtil;
import org.opensearch.ad.util.ExceptionUtil;
import org.opensearch.common.lease.Releasable;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.LoggingDeprecationHandler;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.common.lease.Releasable;
import org.opensearch.core.xcontent.XContentParser;
import org.opensearch.sdk.SDKClient.SDKRestClient;
import org.opensearch.sdk.SDKClusterService;
Expand Down
128 changes: 94 additions & 34 deletions src/main/java/org/opensearch/ad/indices/AnomalyDetectionIndices.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,13 @@
import org.opensearch.ad.model.AnomalyDetectorJob;
import org.opensearch.ad.model.AnomalyResult;
import org.opensearch.ad.rest.handler.AnomalyDetectorFunction;
import org.opensearch.ad.settings.AnomalyDetectorSettings;
import org.opensearch.ad.util.DiscoveryNodeFilterer;
import org.opensearch.client.indices.CreateIndexRequest;
import org.opensearch.client.indices.CreateIndexResponse;
import org.opensearch.client.indices.GetIndexRequest;
import org.opensearch.client.indices.GetMappingsRequest;
import org.opensearch.client.indices.GetMappingsResponse;
import org.opensearch.client.indices.PutMappingRequest;
import org.opensearch.client.indices.rollover.RolloverRequest;
import org.opensearch.client.opensearch.OpenSearchAsyncClient;
Expand All @@ -75,6 +79,7 @@
import org.opensearch.cluster.LocalNodeMasterListener;
import org.opensearch.cluster.metadata.AliasMetadata;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.common.bytes.BytesArray;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.TimeValue;
Expand Down Expand Up @@ -121,6 +126,7 @@ public class AnomalyDetectionIndices implements LocalNodeMasterListener {
private final SDKRestClient adminClient;
private final OpenSearchAsyncClient sdkJavaAsyncClient;
private final ThreadPool threadPool;
private final Settings environmentSettings;

private volatile TimeValue historyRolloverPeriod;
private volatile Long historyMaxDocs;
Expand Down Expand Up @@ -188,6 +194,7 @@ public AnomalyDetectionIndices(
this.sdkJavaAsyncClient = sdkJavaAsyncClient;
this.sdkClusterService = sdkClusterService;
this.threadPool = threadPool;
this.environmentSettings = settings;
// FIXME Implement this
// https://github.com/opensearch-project/opensearch-sdk-java/issues/423
// this.clusterService.addLocalNodeMasterListener(this);
Expand Down Expand Up @@ -301,13 +308,57 @@ public static String getCheckpointMappings() throws IOException {
return Resources.toString(url, Charsets.UTF_8);
}

/**
* Determine if index exists
*
* @param indexName the name of the index
* @return true if index exists
*/
public boolean indexExists(String indexName) {
GetIndexRequest getindexRequest = new GetIndexRequest(indexName);

CompletableFuture<Boolean> existsFuture = new CompletableFuture<>();
sdkRestClient.indices().exists(getindexRequest, ActionListener.wrap(response -> { existsFuture.complete(response); }, exception -> {
existsFuture.completeExceptionally(exception);
}));

Boolean existsResponse = existsFuture
.orTimeout(AnomalyDetectorSettings.REQUEST_TIMEOUT.get(environmentSettings).getMillis(), TimeUnit.MILLISECONDS)
.join();

return existsResponse.booleanValue();
}

/**
* Determine if alias exists
*
* @param aliasName the name of the alias
* @return true if alias exists
*/
public boolean aliasExists(String aliasName) {
GetAliasesRequest getAliasRequest = new GetAliasesRequest(aliasName);

CompletableFuture<Boolean> existsFuture = new CompletableFuture<>();
sdkRestClient
.indices()
.existsAlias(getAliasRequest, ActionListener.wrap(response -> { existsFuture.complete(response); }, exception -> {
existsFuture.completeExceptionally(exception);
}));

Boolean existsResponse = existsFuture
.orTimeout(AnomalyDetectorSettings.REQUEST_TIMEOUT.get(environmentSettings).getMillis(), TimeUnit.MILLISECONDS)
.join();

return existsResponse.booleanValue();
}

/**
* Anomaly detector index exist or not.
*
* @return true if anomaly detector index exists
*/
public boolean doesAnomalyDetectorIndexExist() {
return sdkClusterService.state().getRoutingTable().hasIndex(AnomalyDetector.ANOMALY_DETECTORS_INDEX);
return indexExists(AnomalyDetector.ANOMALY_DETECTORS_INDEX);
}

/**
Expand All @@ -316,7 +367,7 @@ public boolean doesAnomalyDetectorIndexExist() {
* @return true if anomaly detector job index exists
*/
public boolean doesAnomalyDetectorJobIndexExist() {
return sdkClusterService.state().getRoutingTable().hasIndex(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX);
return indexExists(AnomalyDetectorJob.ANOMALY_DETECTOR_JOB_INDEX);
}

/**
Expand All @@ -325,11 +376,11 @@ public boolean doesAnomalyDetectorJobIndexExist() {
* @return true if anomaly result index exists
*/
public boolean doesDefaultAnomalyResultIndexExist() {
return sdkClusterService.state().metadata().hasAlias(CommonName.ANOMALY_RESULT_INDEX_ALIAS);
return aliasExists(CommonName.ANOMALY_RESULT_INDEX_ALIAS);
}

public boolean doesIndexExist(String indexName) {
return sdkClusterService.state().metadata().hasIndex(indexName);
return indexExists(indexName);
}

public <T> void initCustomResultIndexAndExecute(String resultIndex, AnomalyDetectorFunction function, ActionListener<T> listener) {
Expand Down Expand Up @@ -432,8 +483,24 @@ public boolean isValidResultIndexMapping(String resultIndex) {
// failed to populate the field
return false;
}
IndexMetadata indexMetadata = sdkClusterService.state().metadata().index(resultIndex);
Map<String, Object> indexMapping = indexMetadata.mapping().sourceAsMap();

GetMappingsRequest getMappingRequest = new GetMappingsRequest().indices(resultIndex);
CompletableFuture<GetMappingsResponse> getMappingsFuture = new CompletableFuture<>();
sdkRestClient
.indices()
.getMapping(getMappingRequest, ActionListener.wrap(response -> { getMappingsFuture.complete(response); }, exception -> {
getMappingsFuture.completeExceptionally(exception);
}));
GetMappingsResponse getMappingResponse = getMappingsFuture
.orTimeout(AnomalyDetectorSettings.REQUEST_TIMEOUT.get(environmentSettings).getMillis(), TimeUnit.MILLISECONDS)
.join();

Map<String, MappingMetadata> resultIndexMappings = getMappingResponse.mappings();
if (resultIndexMappings.size() == 0) {
return false;
}
Map<String, Object> indexMapping = resultIndexMappings.get(resultIndex).sourceAsMap();

String propertyName = CommonName.PROPERTIES;
if (!indexMapping.containsKey(propertyName) || !(indexMapping.get(propertyName) instanceof LinkedHashMap)) {
return false;
Expand Down Expand Up @@ -468,7 +535,7 @@ public boolean isValidResultIndexMapping(String resultIndex) {
* @return true if anomaly state index exists
*/
public boolean doesDetectorStateIndexExist() {
return sdkClusterService.state().getRoutingTable().hasIndex(CommonName.DETECTION_STATE_INDEX);
return indexExists(CommonName.DETECTION_STATE_INDEX);
}

/**
Expand All @@ -477,27 +544,7 @@ public boolean doesDetectorStateIndexExist() {
* @return true if checkpoint index exists
*/
public boolean doesCheckpointIndexExist() {
return sdkClusterService.state().getRoutingTable().hasIndex(CommonName.CHECKPOINT_INDEX_NAME);
}

/**
* Index exists or not
* @param sdkClusterService Cluster service
* @param name Index name
* @return true if the index exists
*/
public static boolean doesIndexExists(SDKClusterService sdkClusterService, String name) {
return sdkClusterService.state().getRoutingTable().hasIndex(name);
}

/**
* Alias exists or not
* @param sdkClusterService Cluster service
* @param alias Alias name
* @return true if the alias exists
*/
public static boolean doesAliasExists(SDKClusterService sdkClusterService, String alias) {
return sdkClusterService.state().metadata().hasAlias(alias);
return indexExists(CommonName.CHECKPOINT_INDEX_NAME);
}

private ActionListener<CreateIndexResponse> markMappingUpToDate(ADIndex index, ActionListener<CreateIndexResponse> followingListener) {
Expand Down Expand Up @@ -976,9 +1023,9 @@ private void markMappingUpdated(ADIndex adIndex) {
private void shouldUpdateIndex(ADIndex index, ActionListener<Boolean> thenDo) {
boolean exists = false;
if (index.isAlias()) {
exists = AnomalyDetectionIndices.doesAliasExists(sdkClusterService, index.getIndexName());
exists = aliasExists(index.getIndexName());
} else {
exists = AnomalyDetectionIndices.doesIndexExists(sdkClusterService, index.getIndexName());
exists = indexExists(index.getIndexName());
}
if (false == exists) {
thenDo.onResponse(Boolean.FALSE);
Expand Down Expand Up @@ -1012,14 +1059,25 @@ private void shouldUpdateIndex(ADIndex index, ActionListener<Boolean> thenDo) {

@SuppressWarnings("unchecked")
private void shouldUpdateConcreteIndex(String concreteIndex, Integer newVersion, ActionListener<Boolean> thenDo) {
IndexMetadata indexMeataData = sdkClusterService.state().getMetadata().indices().get(concreteIndex);
if (indexMeataData == null) {
GetMappingsRequest getMappingsRequest = new GetMappingsRequest().indices(concreteIndex);
CompletableFuture<GetMappingsResponse> getMappingsFuture = new CompletableFuture<>();
sdkRestClient
.indices()
.getMapping(getMappingsRequest, ActionListener.wrap(response -> { getMappingsFuture.complete(response); }, exception -> {
getMappingsFuture.completeExceptionally(exception);
}));
GetMappingsResponse getMappingResponse = getMappingsFuture
.orTimeout(AnomalyDetectorSettings.REQUEST_TIMEOUT.get(environmentSettings).getMillis(), TimeUnit.MILLISECONDS)
.join();

Map<String, MappingMetadata> concreteIndexMappings = getMappingResponse.mappings();
if (concreteIndexMappings.size() == 0) {
thenDo.onResponse(Boolean.FALSE);
return;
}
Integer oldVersion = CommonValue.NO_SCHEMA_VERSION;

Map<String, Object> indexMapping = indexMeataData.mapping().getSourceAsMap();
Map<String, Object> indexMapping = concreteIndexMappings.get(concreteIndex).sourceAsMap();
Object meta = indexMapping.get(META);
if (meta != null && meta instanceof Map) {
Map<String, Object> metaMapping = (Map<String, Object>) meta;
Expand Down Expand Up @@ -1098,7 +1156,9 @@ private void updateJobIndexSettingIfNecessary(IndexState jobIndexState, ActionLi

GetIndicesSettingsResponse settingResponse;
try {
settingResponse = getIndicesSettingsResponse.orTimeout(10L, TimeUnit.SECONDS).get();
settingResponse = getIndicesSettingsResponse
.orTimeout(AnomalyDetectorSettings.REQUEST_TIMEOUT.get(environmentSettings).getMillis(), TimeUnit.MILLISECONDS)
.get();
// auto expand setting is a range string like "1-all"
org.opensearch.client.opensearch.indices.IndexState indexState = settingResponse.get(ADIndex.JOB.getIndexName());
String autoExpandReplica = indexState.settings().autoExpandReplicas();
Expand Down
Loading

0 comments on commit 446369d

Please sign in to comment.