From f1b5d08d126c0d534cd6304e9ae0cf519b717c2c Mon Sep 17 00:00:00 2001 From: Igor Abdrakhimov Date: Wed, 5 Jun 2024 12:53:11 -0700 Subject: [PATCH] Add discovery --- .github/workflows/ci.yml | 31 ++++---- .../workflows/ci_run_basic_connect_cfg.json | 4 + .../workflows/ci_run_cognito_connect_cfg.json | 4 + .../ci_run_custom_authorizer_connect_cfg.json | 4 + .../ci_run_fleet_provisioning_cfg.json | 4 + .../ci_run_greengrass_discovery_cfg.json | 7 +- .../workflows/ci_run_greengrass_ipc_cfg.json | 1 - .github/workflows/ci_run_jobs_cfg.json | 4 + .../ci_run_mqtt5_custom_authorizer_cfg.json | 4 + ...qtt5_custom_authorizer_websockets_cfg.json | 4 + .../ci_run_mqtt5_fleet_provisioning_cfg.json | 4 + .github/workflows/ci_run_mqtt5_jobs_cfg.json | 4 + .../ci_run_mqtt5_pkcs11_connect_cfg.json | 4 + .../workflows/ci_run_pkcs11_connect_cfg.json | 4 + .../workflows/ci_run_pkcs12_connect_cfg.json | 4 + .../ci_run_websocket_connect_cfg.json | 4 + .../ci_run_windows_cert_connect_cfg.json | 4 + .../workflows/ci_run_x509_connect_cfg.json | 4 + samples/basic_discovery.py | 6 +- test/greengrass/basic_discovery/copy_files.sh | 3 + .../basic_discovery/gdk-config.json | 22 ++++++ .../basic_discovery/gg-e2e-tests/pom.xml | 63 ++++++++++++++++ .../greengrass/features/component.feature | 74 +++++++++++++++++++ .../basic_discovery/hello_world_subscriber.py | 55 ++++++++++++++ test/greengrass/basic_discovery/recipe.yaml | 45 +++++++++++ utils/run_in_ci.py | 3 - 26 files changed, 344 insertions(+), 26 deletions(-) create mode 100644 test/greengrass/basic_discovery/copy_files.sh create mode 100644 test/greengrass/basic_discovery/gdk-config.json create mode 100644 test/greengrass/basic_discovery/gg-e2e-tests/pom.xml create mode 100644 test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature create mode 100644 test/greengrass/basic_discovery/hello_world_subscriber.py create mode 100644 test/greengrass/basic_discovery/recipe.yaml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe94509c..c98181bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -430,21 +430,22 @@ jobs: with: role-to-assume: ${{ env.CI_GREENGRASS_INSTALLER_ROLE }} aws-region: ${{ env.AWS_DEFAULT_REGION }} - # - name: Build and run Greengrass basic discovery sample - # working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery - # run: | - # gdk component build - # gdk test-e2e build - # gdk test-e2e run - # - name: Show logs - # working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery - # # Print logs unconditionally to provide more details on Greengrass run even if the test failed. - # if: always() - # run: | - # echo "=== greengrass.log" - # cat testResults/gg*/greengrass.log - # echo "=== software.amazon.awssdk.sdk-gg-test-discovery.log" - # cat testResults/gg*/software.amazon.awssdk.sdk-gg-test-discovery.log + - name: Build and run Greengrass basic discovery sample + working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery + run: | + export PYTHONPATH=${{ github.workspace }}/aws-iot-device-sdk-python-v2/samples + gdk component build + gdk test-e2e build + gdk test-e2e run + - name: Show logs + working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/basic_discovery + # Print logs unconditionally to provide more details on Greengrass run even if the test failed. + if: always() + run: | + echo "=== greengrass.log" + cat testResults/gg*/greengrass.log + echo "=== software.amazon.awssdk.sdk-gg-test-discovery.log" + cat testResults/gg*/software.amazon.awssdk.sdk-gg-test-discovery.log - name: Build and run Greengrass IPC sample working-directory: ./aws-iot-device-sdk-python-v2/test/greengrass/ipc run: | diff --git a/.github/workflows/ci_run_basic_connect_cfg.json b/.github/workflows/ci_run_basic_connect_cfg.json index fae564c2..8b3beccc 100644 --- a/.github/workflows/ci_run_basic_connect_cfg.json +++ b/.github/workflows/ci_run_basic_connect_cfg.json @@ -17,6 +17,10 @@ "name": "--key", "secret": "ci/PubSub/key", "filename": "tmp_key.pem" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_cognito_connect_cfg.json b/.github/workflows/ci_run_cognito_connect_cfg.json index 1f6b6cd1..06294750 100644 --- a/.github/workflows/ci_run_cognito_connect_cfg.json +++ b/.github/workflows/ci_run_cognito_connect_cfg.json @@ -15,6 +15,10 @@ { "name": "--cognito_identity", "secret": "ci/Cognito/identity_id" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_custom_authorizer_connect_cfg.json b/.github/workflows/ci_run_custom_authorizer_connect_cfg.json index cbd9afa9..3921e89f 100644 --- a/.github/workflows/ci_run_custom_authorizer_connect_cfg.json +++ b/.github/workflows/ci_run_custom_authorizer_connect_cfg.json @@ -15,6 +15,10 @@ { "name": "--custom_auth_password", "secret": "ci/CustomAuthorizer/password" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_fleet_provisioning_cfg.json b/.github/workflows/ci_run_fleet_provisioning_cfg.json index 0370102f..7068e745 100644 --- a/.github/workflows/ci_run_fleet_provisioning_cfg.json +++ b/.github/workflows/ci_run_fleet_provisioning_cfg.json @@ -25,6 +25,10 @@ { "name": "--template_parameters", "data": "{\"SerialNumber\":\"$INPUT_UUID\"}" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_greengrass_discovery_cfg.json b/.github/workflows/ci_run_greengrass_discovery_cfg.json index b89c1240..34c389ce 100644 --- a/.github/workflows/ci_run_greengrass_discovery_cfg.json +++ b/.github/workflows/ci_run_greengrass_discovery_cfg.json @@ -1,6 +1,6 @@ { - "language": "CPP", - "runnable_file": "basic-discovery", + "language": "Python", + "runnable_file": "basic_discovery.py", "runnable_region": "us-east-1", "runnable_main_class": "", "arguments": [ @@ -30,6 +30,5 @@ "name": "--mode", "data": "publish" } - ], - "stdin_file": "messages.txt" + ] } diff --git a/.github/workflows/ci_run_greengrass_ipc_cfg.json b/.github/workflows/ci_run_greengrass_ipc_cfg.json index 7680faba..767b98df 100644 --- a/.github/workflows/ci_run_greengrass_ipc_cfg.json +++ b/.github/workflows/ci_run_greengrass_ipc_cfg.json @@ -16,6 +16,5 @@ "name": "--is_ci", "data": "true" } - ] } diff --git a/.github/workflows/ci_run_jobs_cfg.json b/.github/workflows/ci_run_jobs_cfg.json index bb7300d2..8df922b2 100644 --- a/.github/workflows/ci_run_jobs_cfg.json +++ b/.github/workflows/ci_run_jobs_cfg.json @@ -21,6 +21,10 @@ { "name": "--thing_name", "data": "CI_Jobs_Thing" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json b/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json index f3608180..1325b5a6 100644 --- a/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json +++ b/.github/workflows/ci_run_mqtt5_custom_authorizer_cfg.json @@ -15,6 +15,10 @@ { "name": "--custom_auth_password", "secret": "ci/CustomAuthorizer/password" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json b/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json index c77cbc12..90bd08e8 100644 --- a/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json +++ b/.github/workflows/ci_run_mqtt5_custom_authorizer_websockets_cfg.json @@ -19,6 +19,10 @@ { "name": "--use_websockets", "data": "true" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_mqtt5_fleet_provisioning_cfg.json b/.github/workflows/ci_run_mqtt5_fleet_provisioning_cfg.json index 58b03e7f..9919f29f 100644 --- a/.github/workflows/ci_run_mqtt5_fleet_provisioning_cfg.json +++ b/.github/workflows/ci_run_mqtt5_fleet_provisioning_cfg.json @@ -25,6 +25,10 @@ { "name": "--template_parameters", "data": "{\"SerialNumber\":\"$INPUT_UUID\"}" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_mqtt5_jobs_cfg.json b/.github/workflows/ci_run_mqtt5_jobs_cfg.json index 118af85a..6c0249e0 100644 --- a/.github/workflows/ci_run_mqtt5_jobs_cfg.json +++ b/.github/workflows/ci_run_mqtt5_jobs_cfg.json @@ -21,6 +21,10 @@ { "name": "--thing_name", "data": "CI_Jobs_Thing" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json b/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json index a8aaa3d9..4470facc 100644 --- a/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json +++ b/.github/workflows/ci_run_mqtt5_pkcs11_connect_cfg.json @@ -34,6 +34,10 @@ { "name": "--key_label", "data": "my-key" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_pkcs11_connect_cfg.json b/.github/workflows/ci_run_pkcs11_connect_cfg.json index dfcf9cdc..4b1a2660 100644 --- a/.github/workflows/ci_run_pkcs11_connect_cfg.json +++ b/.github/workflows/ci_run_pkcs11_connect_cfg.json @@ -34,6 +34,10 @@ { "name": "--key_label", "data": "my-key" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_pkcs12_connect_cfg.json b/.github/workflows/ci_run_pkcs12_connect_cfg.json index 3b633187..f2f96224 100644 --- a/.github/workflows/ci_run_pkcs12_connect_cfg.json +++ b/.github/workflows/ci_run_pkcs12_connect_cfg.json @@ -15,6 +15,10 @@ { "name": "--pkcs12_password", "secret": "ci/PubSub/key_pkcs12_password" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_websocket_connect_cfg.json b/.github/workflows/ci_run_websocket_connect_cfg.json index d21980cd..378123f2 100644 --- a/.github/workflows/ci_run_websocket_connect_cfg.json +++ b/.github/workflows/ci_run_websocket_connect_cfg.json @@ -11,6 +11,10 @@ { "name": "--signing_region", "data": "us-east-1" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_windows_cert_connect_cfg.json b/.github/workflows/ci_run_windows_cert_connect_cfg.json index 2e16ae15..e441e832 100644 --- a/.github/workflows/ci_run_windows_cert_connect_cfg.json +++ b/.github/workflows/ci_run_windows_cert_connect_cfg.json @@ -15,6 +15,10 @@ "windows_cert_key": "ci/PubSub/key", "windows_cert_key_path": "tmp_key.pem", "windows_cert_pfx_key_path": "tmp_pfx.pem" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/.github/workflows/ci_run_x509_connect_cfg.json b/.github/workflows/ci_run_x509_connect_cfg.json index ac64d384..24f43488 100644 --- a/.github/workflows/ci_run_x509_connect_cfg.json +++ b/.github/workflows/ci_run_x509_connect_cfg.json @@ -33,6 +33,10 @@ { "name": "--x509_thing_name", "data": "CI_PubSub_Thing" + }, + { + "name": "--is_ci", + "data": "true" } ] } diff --git a/samples/basic_discovery.py b/samples/basic_discovery.py index 63a69d72..8998a977 100644 --- a/samples/basic_discovery.py +++ b/samples/basic_discovery.py @@ -102,9 +102,9 @@ def on_publish(topic, payload, dup, qos, retain, **kwargs): message['message'] = cmdData.input_message message['sequence'] = loop_count messageJson = json.dumps(message) - pub_future, _ = mqtt_connection.publish(cmdData.input_topic, messageJson, QoS.AT_MOST_ONCE) - pub_future.result() - print('Published topic {}: {}\n'.format(cmdData.input_topic, messageJson)) + pub_future, _ = mqtt_connection.publish(cmdData.input_topic, messageJson, QoS.AT_LEAST_ONCE) + publish_completion_data = pub_future.result() + print('Published topic {}: {} (puback reason: {})\n'.format(cmdData.input_topic, messageJson, repr(publish_completion_data.puback.reason_code))) loop_count += 1 time.sleep(1) diff --git a/test/greengrass/basic_discovery/copy_files.sh b/test/greengrass/basic_discovery/copy_files.sh new file mode 100644 index 00000000..ed8e31c3 --- /dev/null +++ b/test/greengrass/basic_discovery/copy_files.sh @@ -0,0 +1,3 @@ +cp ../../../samples/basic_discovery.py . +cp ../../../utils/run_in_ci.py . +cp ../../../.github/workflows/ci_run_greengrass_discovery_cfg.json . diff --git a/test/greengrass/basic_discovery/gdk-config.json b/test/greengrass/basic_discovery/gdk-config.json new file mode 100644 index 00000000..468e794e --- /dev/null +++ b/test/greengrass/basic_discovery/gdk-config.json @@ -0,0 +1,22 @@ +{ + "component": { + "software.amazon.awssdk.sdk-gg-test-discovery": { + "author": "iot-device-sdk", + "version": "NEXT_PATCH", + "build": { + "build_system": "custom", + "custom_build_command": ["bash", "copy_files.sh"] + }, + "publish": { + "bucket": "iot-sdk-ci-bucket-us-east-1", + "region": "us-east-1" + } + } + }, + "gdk_version": "1.3.0", + "test-e2e": { + "gtf_options": { + "tags": "testgg" + } + } +} diff --git a/test/greengrass/basic_discovery/gg-e2e-tests/pom.xml b/test/greengrass/basic_discovery/gg-e2e-tests/pom.xml new file mode 100644 index 00000000..460832fc --- /dev/null +++ b/test/greengrass/basic_discovery/gg-e2e-tests/pom.xml @@ -0,0 +1,63 @@ + + + 4.0.0 + + com.aws.greengrass + uat-features + jar + 1.0.0 + OTF + + + 1.2.0-SNAPSHOT + 1.8 + 1.8 + + + + + greengrass-common + greengrass common + + https://d2jrmugq4soldf.cloudfront.net/snapshots + + + + + + + com.aws.greengrass + aws-greengrass-testing-standalone + ${otf.version} + compile + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.2 + + + package + + shade + + + + + + + com.aws.greengrass.testing.launcher.TestLauncher + + + + + + + + + diff --git a/test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature b/test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature new file mode 100644 index 00000000..9f4c0453 --- /dev/null +++ b/test/greengrass/basic_discovery/gg-e2e-tests/src/main/resources/greengrass/features/component.feature @@ -0,0 +1,74 @@ +Feature: Testing features of Greengrassv2 basic discovery sample + + @testgg + Scenario: As a developer, I can create a component and deploy it on my device + Given my device is registered as a Thing + And my device is running Greengrass + When I create a Greengrass deployment with components + | aws.greengrass.clientdevices.Auth | LATEST | + | aws.greengrass.clientdevices.mqtt.Moquette | LATEST | + | aws.greengrass.clientdevices.mqtt.Bridge | LATEST | + | aws.greengrass.clientdevices.IPDetector | LATEST | + | software.amazon.awssdk.sdk-gg-test-discovery | file:recipe.yaml | + When I update my Greengrass deployment configuration, setting the component aws.greengrass.clientdevices.Auth configuration to: + """ + { + "MERGE": { + "deviceGroups": { + "formatVersion": "2021-03-05", + "definitions": { + "MyDeviceGroup": { + "selectionRule": "thingName: CI_Greengrass_Discovery_Thing", + "policyName": "MyRestrictivePolicy" + } + }, + "policies": { + "MyRestrictivePolicy": { + "AllowConnect": { + "statementDescription": "Allow client devices to connect.", + "operations": [ + "mqtt:connect" + ], + "resources": [ + "*" + ] + }, + "AllowPublish": { + "statementDescription": "Allow client devices to publish on topic.", + "operations": [ + "mqtt:publish" + ], + "resources": [ + "*clients/*/hello/world/*" + ] + } + } + } + } + } + } + """ + When I update my Greengrass deployment configuration, setting the component aws.greengrass.clientdevices.mqtt.Bridge configuration to: + """ + { + "MERGE": { + "mqttTopicMapping": { + "HelloWorldCoreMapping": { + "topic": "clients/+/hello/world/+", + "source": "LocalMqtt", + "target": "IotCore" + }, + "HelloWorldPubsubMapping": { + "topic": "clients/+/hello/world/+", + "source": "LocalMqtt", + "target": "Pubsub" + } + } + } + } + """ + And I deploy the Greengrass deployment configuration + Then the Greengrass deployment is COMPLETED on the device after 190 seconds + And the software.amazon.awssdk.sdk-gg-test-discovery log on the device contains the line "Successfully subscribed to topic" within 180 seconds + And the software.amazon.awssdk.sdk-gg-test-discovery log on the device contains the line "Received new message" within 240 seconds + And the software.amazon.awssdk.sdk-gg-test-discovery log on the device contains the line "disassociated CI_Greengrass_Discovery_Thing" within 260 seconds diff --git a/test/greengrass/basic_discovery/hello_world_subscriber.py b/test/greengrass/basic_discovery/hello_world_subscriber.py new file mode 100644 index 00000000..44fd99b7 --- /dev/null +++ b/test/greengrass/basic_discovery/hello_world_subscriber.py @@ -0,0 +1,55 @@ +import argparse +import sys +import time +import traceback +import uuid + +from awsiot.greengrasscoreipc.clientv2 import GreengrassCoreIPCClientV2 + + +def on_message(event): + try: + print('Topic: {}'.format(event.binary_message.context.topic)) + message = str(event.binary_message.message, 'utf-8') + print('Received new message: {}'.format(message)) + except: + traceback.print_exc() + + +def main(): + argument_parser = argparse.ArgumentParser( + description="Run Greengrass subscriber component") + argument_parser.add_argument( + "--input_uuid", required=False, help="UUID for unique topic name. UUID will be generated if this option is omit") + parsed_commands = argument_parser.parse_args() + + input_uuid = parsed_commands.input_uuid if parsed_commands.input_uuid else str(uuid.uuid4()) + + try: + ipc_client = GreengrassCoreIPCClientV2() + + client_device_hello_world_topic = 'clients/+/hello/world/{}'.format(input_uuid) + + # SubscribeToTopic returns a tuple with the response and the operation. + _, operation = ipc_client.subscribe_to_topic( + topic=client_device_hello_world_topic, on_stream_event=on_message) + print('Successfully subscribed to topic: {}'.format(client_device_hello_world_topic)) + + # Keep the main thread alive, or the process will exit. + try: + while True: + time.sleep(10) + except InterruptedError: + print('Subscribe interrupted.') + + operation.close() + except Exception: + print('Exception occurred when using IPC.', file=sys.stderr) + traceback.print_exc() + exit(1) + + print("Subscriber done") + + +if __name__ == "__main__": + main() diff --git a/test/greengrass/basic_discovery/recipe.yaml b/test/greengrass/basic_discovery/recipe.yaml new file mode 100644 index 00000000..d39af2b8 --- /dev/null +++ b/test/greengrass/basic_discovery/recipe.yaml @@ -0,0 +1,45 @@ +--- +RecipeFormatVersion: "2020-01-25" +ComponentName: software.amazon.awssdk.sdk-gg-test-discovery +ComponentVersion: "1.0.0" +ComponentDescription: "This is test for the Greengrass basic discovery sample" +ComponentPublisher: "iot-device-sdk" +ComponentConfiguration: + DefaultConfiguration: + accessControl: + aws.greengrass.ipc.pubsub: + software.amazon.awssdk.sdk-gg-test-discovery:pubsub:1: + policyDescription: "Allows access to subscribe to a Greengrass IPC test topic" + operations: + - aws.greengrass#SubscribeToTopic + - aws.greengrass#PublishToTopic + resources: + - "clients/*/hello/world/*" +Manifests: + - Platform: + os: all + Artifacts: + - URI: "file:hello_world_subscriber.py" + - URI: "file:run_in_ci.py" + - URI: "file:ci_run_greengrass_discovery_cfg.json" + - URI: "file:basic_discovery.py" + Lifecycle: + Install: | + echo "GG core:" {iot:thingName} + aws greengrassv2 batch-associate-client-device-with-core-device --core-device-thing-name {iot:thingName} --entries thingName=CI_Greengrass_Discovery_Thing + aws greengrassv2 list-client-devices-associated-with-core-device --core-device-thing-name {iot:thingName} + Run: | + UUID=$(python3 -c "import uuid; print (uuid.uuid4())") + echo "Starting subscriber" + python3 -u {artifacts:path}/hello_world_subscriber.py --input_uuid ${UUID} & + sleep 10 + echo "Starting discovery" + python3 {artifacts:path}/run_in_ci.py --runnable_dir {artifacts:path} --input_uuid ${UUID} --file {artifacts:path}/ci_run_greengrass_discovery_cfg.json + aws greengrassv2 batch-disassociate-client-device-from-core-device --core-device-thing-name {iot:thingName} --entries thingName=CI_Greengrass_Discovery_Thing + echo "Run: disassociated CI_Greengrass_Discovery_Thing" + Shutdown: | + echo "Shutdown: disassociating CI_Greengrass_Discovery_Thing" + aws greengrassv2 batch-disassociate-client-device-from-core-device --core-device-thing-name {iot:thingName} --entries thingName=CI_Greengrass_Discovery_Thing + Recover: | + echo "Recover: disassociating CI_Greengrass_Discovery_Thing" + aws greengrassv2 batch-disassociate-client-device-from-core-device --core-device-thing-name {iot:thingName} --entries thingName=CI_Greengrass_Discovery_Thing diff --git a/utils/run_in_ci.py b/utils/run_in_ci.py index d2eea2ea..d8d284a6 100644 --- a/utils/run_in_ci.py +++ b/utils/run_in_ci.py @@ -327,9 +327,6 @@ def launch_runnable(runnable_dir): exit_code = runnable_return.returncode elif (config_json['language'] == "Python"): - config_json_arguments_list.append("--is_ci") - config_json_arguments_list.append("True") - runnable_file = os.path.join(runnable_dir, config_json['runnable_file']) runnable_return = subprocess.run( args=[sys.executable, runnable_file] + config_json_arguments_list, input=subprocess_stdin, timeout=runnable_timeout)