From 0c0808f383059ad68ae30b12831060033aceece9 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 1 Nov 2024 17:51:56 +0000 Subject: [PATCH] Add API Consistency Tests with ML-Common and Set Up Daily GitHub Action Trigger (#937) * Added the rest api spec comparison test Signed-off-by: Junwei Dai * Added github Action to run api consistency test daily Signed-off-by: Junwei Dai * fix formating issue Signed-off-by: Junwei Dai * add change log Signed-off-by: Junwei Dai * fix *import Signed-off-by: Junwei Dai * addrssed all comment Signed-off-by: Junwei Dai --------- Signed-off-by: Junwei Dai Co-authored-by: Junwei Dai (cherry picked from commit 50e45ac0899338ca35f1504922f3e05d1303c9fa) Signed-off-by: github-actions[bot] --- .github/workflows/test-api-consistency.yml | 26 +++++++++++++++++++ CHANGELOG.md | 1 + .../flowframework/util/ApiSpecFetcher.java | 3 +-- .../workflow/CreateConnectorStepTests.java | 17 ++++++++++++ .../RegisterLocalCustomModelStepTests.java | 17 ++++++++++++ ...RegisterLocalPretrainedModelStepTests.java | 17 ++++++++++++ ...sterLocalSparseEncodingModelStepTests.java | 17 ++++++++++++ .../workflow/RegisterModelGroupStepTests.java | 16 ++++++++++++ .../RegisterRemoteModelStepTests.java | 17 ++++++++++++ 9 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/test-api-consistency.yml diff --git a/.github/workflows/test-api-consistency.yml b/.github/workflows/test-api-consistency.yml new file mode 100644 index 000000000..4c71b4d3b --- /dev/null +++ b/.github/workflows/test-api-consistency.yml @@ -0,0 +1,26 @@ +name: Daily API Consistency Test + +on: + schedule: + - cron: '0 8 * * *' # Runs daily at 8 AM UTC + workflow_dispatch: + +jobs: + API-consistency-test: + runs-on: ubuntu-latest + strategy: + matrix: + java: [21] + + steps: + - name: Checkout Flow Framework + uses: actions/checkout@v3 + + - name: Setup Java ${{ matrix.java }} + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: ${{ matrix.java }} + + - name: Run API Consistency Tests + run: ./gradlew test --tests "org.opensearch.flowframework.workflow.*" diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ee5340d4..3193e2c4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) ### Features - Add ApiSpecFetcher for Fetching and Comparing API Specifications ([#651](https://github.com/opensearch-project/flow-framework/issues/651)) - Add optional config field to tool step ([#899](https://github.com/opensearch-project/flow-framework/pull/899)) +- Add API Consistency Tests with ML-Common and Set Up Daily GitHub Action Trigger([#908](https://github.com/opensearch-project/flow-framework/issues/908)) ### Enhancements - Incrementally remove resources from workflow state during deprovisioning ([#898](https://github.com/opensearch-project/flow-framework/pull/898)) diff --git a/src/main/java/org/opensearch/flowframework/util/ApiSpecFetcher.java b/src/main/java/org/opensearch/flowframework/util/ApiSpecFetcher.java index 80be71b65..12630b6c3 100644 --- a/src/main/java/org/opensearch/flowframework/util/ApiSpecFetcher.java +++ b/src/main/java/org/opensearch/flowframework/util/ApiSpecFetcher.java @@ -14,7 +14,6 @@ import org.opensearch.flowframework.exception.ApiSpecParseException; import org.opensearch.rest.RestRequest; -import java.util.HashSet; import java.util.List; import io.swagger.v3.oas.models.OpenAPI; @@ -81,7 +80,7 @@ public static boolean compareRequiredFields(List requiredEnumParams, Str List requiredApiParams = schema.getRequired(); if (requiredApiParams != null && !requiredApiParams.isEmpty()) { - return new HashSet<>(requiredEnumParams).equals(new HashSet<>(requiredApiParams)); + return requiredApiParams.stream().allMatch(requiredEnumParams::contains); } } return false; diff --git a/src/test/java/org/opensearch/flowframework/workflow/CreateConnectorStepTests.java b/src/test/java/org/opensearch/flowframework/workflow/CreateConnectorStepTests.java index dd9eb369d..3ed8d15ec 100644 --- a/src/test/java/org/opensearch/flowframework/workflow/CreateConnectorStepTests.java +++ b/src/test/java/org/opensearch/flowframework/workflow/CreateConnectorStepTests.java @@ -14,20 +14,24 @@ import org.opensearch.flowframework.common.CommonValue; import org.opensearch.flowframework.exception.FlowFrameworkException; import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler; +import org.opensearch.flowframework.util.ApiSpecFetcher; import org.opensearch.ml.client.MachineLearningNodeClient; import org.opensearch.ml.common.connector.ConnectorAction; import org.opensearch.ml.common.transport.connector.MLCreateConnectorInput; import org.opensearch.ml.common.transport.connector.MLCreateConnectorResponse; +import org.opensearch.rest.RestRequest; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI; import static org.opensearch.flowframework.common.WorkflowResources.CONNECTOR_ID; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; @@ -134,4 +138,17 @@ public void testCreateConnectorFailure() throws IOException { assertEquals("Failed to create connector", ex.getCause().getMessage()); } + public void testApiSpecCreateConnectorInputParamComparison() throws Exception { + List requiredEnumParams = WorkflowStepFactory.WorkflowSteps.CREATE_CONNECTOR.inputs(); + + boolean isMatch = ApiSpecFetcher.compareRequiredFields( + requiredEnumParams, + ML_COMMONS_API_SPEC_YAML_URI, + "/_plugins/_ml/connectors/_create", + RestRequest.Method.POST + ); + + assertTrue(isMatch); + } + } diff --git a/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalCustomModelStepTests.java b/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalCustomModelStepTests.java index 6a6809d07..d42a1ae21 100644 --- a/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalCustomModelStepTests.java +++ b/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalCustomModelStepTests.java @@ -20,11 +20,13 @@ import org.opensearch.flowframework.exception.FlowFrameworkException; import org.opensearch.flowframework.exception.WorkflowStepException; import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler; +import org.opensearch.flowframework.util.ApiSpecFetcher; import org.opensearch.ml.client.MachineLearningNodeClient; import org.opensearch.ml.common.MLTask; import org.opensearch.ml.common.MLTaskState; import org.opensearch.ml.common.transport.register.MLRegisterModelInput; import org.opensearch.ml.common.transport.register.MLRegisterModelResponse; +import org.opensearch.rest.RestRequest; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ScalingExecutorBuilder; import org.opensearch.threadpool.TestThreadPool; @@ -33,6 +35,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -43,6 +46,7 @@ import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD; import static org.opensearch.flowframework.common.CommonValue.FLOW_FRAMEWORK_THREAD_POOL_PREFIX; +import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI; import static org.opensearch.flowframework.common.CommonValue.PROVISION_WORKFLOW_THREAD_POOL; import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS; import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_THREAD_POOL; @@ -398,4 +402,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage()); assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus()); } + + public void testApiSpecRegisterLocalCustomModelInputParamComparison() throws Exception { + List requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_LOCAL_CUSTOM_MODEL.inputs(); + + boolean isMatch = ApiSpecFetcher.compareRequiredFields( + requiredEnumParams, + ML_COMMONS_API_SPEC_YAML_URI, + "/_plugins/_ml/models/_register", + RestRequest.Method.POST + ); + + assertTrue(isMatch); + } } diff --git a/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalPretrainedModelStepTests.java b/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalPretrainedModelStepTests.java index 162b97dba..a0ef430b9 100644 --- a/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalPretrainedModelStepTests.java +++ b/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalPretrainedModelStepTests.java @@ -20,11 +20,13 @@ import org.opensearch.flowframework.exception.FlowFrameworkException; import org.opensearch.flowframework.exception.WorkflowStepException; import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler; +import org.opensearch.flowframework.util.ApiSpecFetcher; import org.opensearch.ml.client.MachineLearningNodeClient; import org.opensearch.ml.common.MLTask; import org.opensearch.ml.common.MLTaskState; import org.opensearch.ml.common.transport.register.MLRegisterModelInput; import org.opensearch.ml.common.transport.register.MLRegisterModelResponse; +import org.opensearch.rest.RestRequest; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ScalingExecutorBuilder; import org.opensearch.threadpool.TestThreadPool; @@ -33,6 +35,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -42,6 +45,7 @@ import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD; import static org.opensearch.flowframework.common.CommonValue.FLOW_FRAMEWORK_THREAD_POOL_PREFIX; +import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI; import static org.opensearch.flowframework.common.CommonValue.PROVISION_WORKFLOW_THREAD_POOL; import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS; import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_THREAD_POOL; @@ -303,4 +307,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage()); assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus()); } + + public void testApiSpecRegisterLocalPretrainedModelInputParamComparison() throws Exception { + List requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_LOCAL_PRETRAINED_MODEL.inputs(); + + boolean isMatch = ApiSpecFetcher.compareRequiredFields( + requiredEnumParams, + ML_COMMONS_API_SPEC_YAML_URI, + "/_plugins/_ml/models/_register", + RestRequest.Method.POST + ); + + assertTrue(isMatch); + } } diff --git a/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalSparseEncodingModelStepTests.java b/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalSparseEncodingModelStepTests.java index 79d7bb883..e3157b20b 100644 --- a/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalSparseEncodingModelStepTests.java +++ b/src/test/java/org/opensearch/flowframework/workflow/RegisterLocalSparseEncodingModelStepTests.java @@ -20,11 +20,13 @@ import org.opensearch.flowframework.exception.FlowFrameworkException; import org.opensearch.flowframework.exception.WorkflowStepException; import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler; +import org.opensearch.flowframework.util.ApiSpecFetcher; import org.opensearch.ml.client.MachineLearningNodeClient; import org.opensearch.ml.common.MLTask; import org.opensearch.ml.common.MLTaskState; import org.opensearch.ml.common.transport.register.MLRegisterModelInput; import org.opensearch.ml.common.transport.register.MLRegisterModelResponse; +import org.opensearch.rest.RestRequest; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ScalingExecutorBuilder; import org.opensearch.threadpool.TestThreadPool; @@ -33,6 +35,7 @@ import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -42,6 +45,7 @@ import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD; import static org.opensearch.flowframework.common.CommonValue.FLOW_FRAMEWORK_THREAD_POOL_PREFIX; +import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI; import static org.opensearch.flowframework.common.CommonValue.PROVISION_WORKFLOW_THREAD_POOL; import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS; import static org.opensearch.flowframework.common.CommonValue.WORKFLOW_THREAD_POOL; @@ -310,4 +314,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage()); assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus()); } + + public void testApiSpecRegisterLocalSparseEncodingModelInputParamComparison() throws Exception { + List requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_LOCAL_SPARSE_ENCODING_MODEL.inputs(); + + boolean isMatch = ApiSpecFetcher.compareRequiredFields( + requiredEnumParams, + ML_COMMONS_API_SPEC_YAML_URI, + "/_plugins/_ml/models/_register", + RestRequest.Method.POST + ); + + assertTrue(isMatch); + } } diff --git a/src/test/java/org/opensearch/flowframework/workflow/RegisterModelGroupStepTests.java b/src/test/java/org/opensearch/flowframework/workflow/RegisterModelGroupStepTests.java index 7f7adf44b..05eeb8500 100644 --- a/src/test/java/org/opensearch/flowframework/workflow/RegisterModelGroupStepTests.java +++ b/src/test/java/org/opensearch/flowframework/workflow/RegisterModelGroupStepTests.java @@ -14,11 +14,13 @@ import org.opensearch.flowframework.exception.FlowFrameworkException; import org.opensearch.flowframework.exception.WorkflowStepException; import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler; +import org.opensearch.flowframework.util.ApiSpecFetcher; import org.opensearch.ml.client.MachineLearningNodeClient; import org.opensearch.ml.common.AccessMode; import org.opensearch.ml.common.MLTaskState; import org.opensearch.ml.common.transport.model_group.MLRegisterModelGroupInput; import org.opensearch.ml.common.transport.model_group.MLRegisterModelGroupResponse; +import org.opensearch.rest.RestRequest; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; @@ -31,6 +33,7 @@ import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI; import static org.opensearch.flowframework.common.CommonValue.MODEL_GROUP_STATUS; import static org.opensearch.flowframework.common.WorkflowResources.MODEL_GROUP_ID; import static org.mockito.ArgumentMatchers.any; @@ -204,4 +207,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup assertEquals("Failed to parse value [no] as only [true] or [false] are allowed.", w.getMessage()); assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus()); } + + public void testApiSpecRegisterModelGroupInputParamComparison() throws Exception { + List requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_MODEL_GROUP.inputs(); + + boolean isMatch = ApiSpecFetcher.compareRequiredFields( + requiredEnumParams, + ML_COMMONS_API_SPEC_YAML_URI, + "/_plugins/_ml/model_groups/_register", + RestRequest.Method.POST + ); + + assertTrue(isMatch); + } } diff --git a/src/test/java/org/opensearch/flowframework/workflow/RegisterRemoteModelStepTests.java b/src/test/java/org/opensearch/flowframework/workflow/RegisterRemoteModelStepTests.java index 0e2ab91e9..362601264 100644 --- a/src/test/java/org/opensearch/flowframework/workflow/RegisterRemoteModelStepTests.java +++ b/src/test/java/org/opensearch/flowframework/workflow/RegisterRemoteModelStepTests.java @@ -17,15 +17,18 @@ import org.opensearch.flowframework.exception.FlowFrameworkException; import org.opensearch.flowframework.exception.WorkflowStepException; import org.opensearch.flowframework.indices.FlowFrameworkIndicesHandler; +import org.opensearch.flowframework.util.ApiSpecFetcher; import org.opensearch.ml.client.MachineLearningNodeClient; import org.opensearch.ml.common.MLTaskState; import org.opensearch.ml.common.transport.register.MLRegisterModelInput; import org.opensearch.ml.common.transport.register.MLRegisterModelResponse; +import org.opensearch.rest.RestRequest; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.transport.RemoteTransportException; import java.io.IOException; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicInteger; @@ -35,6 +38,7 @@ import static org.opensearch.flowframework.common.CommonValue.DEPLOY_FIELD; import static org.opensearch.flowframework.common.CommonValue.INTERFACE_FIELD; +import static org.opensearch.flowframework.common.CommonValue.ML_COMMONS_API_SPEC_YAML_URI; import static org.opensearch.flowframework.common.CommonValue.REGISTER_MODEL_STATUS; import static org.opensearch.flowframework.common.WorkflowResources.CONNECTOR_ID; import static org.opensearch.flowframework.common.WorkflowResources.MODEL_ID; @@ -416,4 +420,17 @@ public void testBoolParseFail() throws IOException, ExecutionException, Interrup assertEquals("Failed to parse value [yes] as only [true] or [false] are allowed.", w.getMessage()); assertEquals(RestStatus.BAD_REQUEST, w.getRestStatus()); } + + public void testApiSpecRegisterRemoteModelInputParamComparison() throws Exception { + List requiredEnumParams = WorkflowStepFactory.WorkflowSteps.REGISTER_REMOTE_MODEL.inputs(); + + boolean isMatch = ApiSpecFetcher.compareRequiredFields( + requiredEnumParams, + ML_COMMONS_API_SPEC_YAML_URI, + "/_plugins/_ml/model_groups/_register", + RestRequest.Method.POST + ); + + assertTrue(isMatch); + } }