From 408eda38c976cba3d12d2b2709de4f66c433897a Mon Sep 17 00:00:00 2001 From: ilijabojanovic Date: Mon, 17 Jun 2024 16:09:24 +0200 Subject: [PATCH 01/63] Test 5.4.0 images --- deployments/tyk/docker-compose.yml | 6 +++--- deployments/tyk2/docker-compose.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deployments/tyk/docker-compose.yml b/deployments/tyk/docker-compose.yml index 4781dc0f..078808a0 100755 --- a/deployments/tyk/docker-compose.yml +++ b/deployments/tyk/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.3.2} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc2} ports: - 3000:3000 networks: @@ -22,7 +22,7 @@ services: - tyk-redis - tyk-mongo tyk-gateway: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.3.2} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0-rc1} ports: - 8080:8080 - 8086:8086 @@ -55,7 +55,7 @@ services: - tyk-redis - tyk-dashboard tyk-gateway-2: - image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.3.2} + image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.4.0-rc1} ports: - 8081:8080 networks: diff --git a/deployments/tyk2/docker-compose.yml b/deployments/tyk2/docker-compose.yml index 259a3720..118ae26a 100644 --- a/deployments/tyk2/docker-compose.yml +++ b/deployments/tyk2/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk2-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.3.2} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc2} ports: - 3002:3000 networks: @@ -20,7 +20,7 @@ services: - tyk2-redis - tyk2-mongo tyk2-gateway: - image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.3.2} + image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.4.0-rc1} ports: - 8085:8080 networks: @@ -50,7 +50,7 @@ services: - tyk2-gateway tyk2-mongo: image: mongo:4.0 - command: [ "mongod", "--smallfiles" ] + command: ['mongod', '--smallfiles'] volumes: - tyk2-mongo-data:/data/db networks: From 228af37b038c416bd072ae2911edbee8f25482b0 Mon Sep 17 00:00:00 2001 From: ilijabojanovic Date: Wed, 19 Jun 2024 21:02:15 +0200 Subject: [PATCH 02/63] Test latest images --- deployments/tyk/docker-compose.yml | 6 +++--- deployments/tyk2/docker-compose.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deployments/tyk/docker-compose.yml b/deployments/tyk/docker-compose.yml index 078808a0..d056ff70 100755 --- a/deployments/tyk/docker-compose.yml +++ b/deployments/tyk/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc2} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc3} ports: - 3000:3000 networks: @@ -22,7 +22,7 @@ services: - tyk-redis - tyk-mongo tyk-gateway: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0-rc1} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0-rc3} ports: - 8080:8080 - 8086:8086 @@ -55,7 +55,7 @@ services: - tyk-redis - tyk-dashboard tyk-gateway-2: - image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.4.0-rc1} + image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.4.0-rc3} ports: - 8081:8080 networks: diff --git a/deployments/tyk2/docker-compose.yml b/deployments/tyk2/docker-compose.yml index 118ae26a..571dc336 100644 --- a/deployments/tyk2/docker-compose.yml +++ b/deployments/tyk2/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk2-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc2} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc3} ports: - 3002:3000 networks: @@ -20,7 +20,7 @@ services: - tyk2-redis - tyk2-mongo tyk2-gateway: - image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.4.0-rc1} + image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.4.0-rc3} ports: - 8085:8080 networks: From e93758c7da667ad83554d72ab5e97621704ed243 Mon Sep 17 00:00:00 2001 From: ilijabojanovic Date: Sun, 30 Jun 2024 16:25:50 +0200 Subject: [PATCH 03/63] chore: Update Tyk images to v5.4.0-rc4 --- deployments/tyk/docker-compose.yml | 6 +++--- deployments/tyk2/docker-compose.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deployments/tyk/docker-compose.yml b/deployments/tyk/docker-compose.yml index d056ff70..c12f5861 100755 --- a/deployments/tyk/docker-compose.yml +++ b/deployments/tyk/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc3} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc4} ports: - 3000:3000 networks: @@ -22,7 +22,7 @@ services: - tyk-redis - tyk-mongo tyk-gateway: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0-rc3} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0-rc4} ports: - 8080:8080 - 8086:8086 @@ -55,7 +55,7 @@ services: - tyk-redis - tyk-dashboard tyk-gateway-2: - image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.4.0-rc3} + image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.4.0-rc4} ports: - 8081:8080 networks: diff --git a/deployments/tyk2/docker-compose.yml b/deployments/tyk2/docker-compose.yml index 571dc336..ea84744b 100644 --- a/deployments/tyk2/docker-compose.yml +++ b/deployments/tyk2/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk2-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc3} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc4} ports: - 3002:3000 networks: @@ -20,7 +20,7 @@ services: - tyk2-redis - tyk2-mongo tyk2-gateway: - image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.4.0-rc3} + image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.4.0-rc4} ports: - 8085:8080 networks: From 999377d9a5828ba70877de1c801ca6408172a998 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 10:31:53 +0200 Subject: [PATCH 04/63] use streaming alpha images --- deployments/load-balancer-nginx/docker-compose.yml | 4 ++-- deployments/mdcb/docker-compose.yml | 2 +- deployments/portal/docker-compose.yml | 2 +- deployments/sso/docker-compose.yml | 2 +- deployments/tyk/docker-compose.yml | 6 +++--- deployments/tyk2/docker-compose.yml | 4 ++-- scripts/add-gateway.sh | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/deployments/load-balancer-nginx/docker-compose.yml b/deployments/load-balancer-nginx/docker-compose.yml index b32e02cd..47af0ae0 100755 --- a/deployments/load-balancer-nginx/docker-compose.yml +++ b/deployments/load-balancer-nginx/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-gateway-3: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0-alpha2} networks: - tyk environment: @@ -28,7 +28,7 @@ services: - tyk-redis - tyk-dashboard tyk-gateway-4: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0-alpha2} networks: - tyk environment: diff --git a/deployments/mdcb/docker-compose.yml b/deployments/mdcb/docker-compose.yml index 5d298073..4e2ca5f6 100644 --- a/deployments/mdcb/docker-compose.yml +++ b/deployments/mdcb/docker-compose.yml @@ -15,7 +15,7 @@ services: - tyk-redis - tyk-mongo tyk-worker-gateway: - image: tykio/tyk-gateway:${GATEWAY_WORKER_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${GATEWAY_WORKER_VERSION:-v5.5.0-alpha2} ports: - 8090:8080 networks: diff --git a/deployments/portal/docker-compose.yml b/deployments/portal/docker-compose.yml index cc1b77e6..43494ee4 100644 --- a/deployments/portal/docker-compose.yml +++ b/deployments/portal/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-portal: - image: tykio/portal:${PORTAL_VERSION:-v1.9.0} + image: tykio/portal:${PORTAL_VERSION:-v1.10.0-alpha2} networks: - tyk env_file: diff --git a/deployments/sso/docker-compose.yml b/deployments/sso/docker-compose.yml index 2fd936c7..0bd19c8d 100644 --- a/deployments/sso/docker-compose.yml +++ b/deployments/sso/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-dashboard-sso: - image: tykio/tyk-dashboard:${DASHBOARD_SSO_VERSION:-v5.4.0} + image: tykio/tyk-dashboard:${DASHBOARD_SSO_VERSION:-s5.5.0-alpha1} ports: - 3001:3000 networks: diff --git a/deployments/tyk/docker-compose.yml b/deployments/tyk/docker-compose.yml index 6fe660c8..ae7a96da 100755 --- a/deployments/tyk/docker-compose.yml +++ b/deployments/tyk/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-s5.5.0-alpha1} ports: - 3000:3000 networks: @@ -22,7 +22,7 @@ services: - tyk-redis - tyk-mongo tyk-gateway: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0-alpha2} ports: - 8080:8080 - 8086:8086 @@ -55,7 +55,7 @@ services: - tyk-redis - tyk-dashboard tyk-gateway-2: - image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.5.0-alpha2} ports: - 8081:8080 networks: diff --git a/deployments/tyk2/docker-compose.yml b/deployments/tyk2/docker-compose.yml index dbe7974b..159ebf3a 100644 --- a/deployments/tyk2/docker-compose.yml +++ b/deployments/tyk2/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk2-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-s5.5.0-alpha1} ports: - 3002:3000 networks: @@ -20,7 +20,7 @@ services: - tyk2-redis - tyk2-mongo tyk2-gateway: - image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.4.0} + image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.5.0-alpha2} ports: - 8085:8080 networks: diff --git a/scripts/add-gateway.sh b/scripts/add-gateway.sh index f75e6452..a734934b 100755 --- a/scripts/add-gateway.sh +++ b/scripts/add-gateway.sh @@ -14,7 +14,7 @@ if [ "$1" == "" ]; then -v $(pwd)/deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json \ -v $(pwd)/deployments/tyk/volumes/databases/GeoLite2-Country.mmdb:/opt/tyk-gateway/databases/GeoLite2-Country.mmdb \ --network tyk-demo_tyk \ - tykio/tyk-gateway:v5.4.0 + tykio/tyk-gateway:v5.5.0-alpha2 else docker run \ --name $1 \ @@ -27,5 +27,5 @@ else -v $(pwd)/deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json \ -v $(pwd)/deployments/tyk/volumes/databases/GeoLite2-Country.mmdb:/opt/tyk-gateway/databases/GeoLite2-Country.mmdb \ --network tyk-demo_tyk \ - tykio/tyk-gateway:v5.4.0 + tykio/tyk-gateway:v5.5.0-alpha2 fi From b9229230e9a3a82af7b041e4871e8d926166a2c0 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 10:32:16 +0200 Subject: [PATCH 05/63] enable steaming via labs flag --- .../volumes/tyk-dashboard/tyk_analytics.conf | 241 +++++------ .../tyk/volumes/tyk-gateway/tyk-2.conf | 365 +++++++++-------- deployments/tyk/volumes/tyk-gateway/tyk.conf | 387 +++++++++--------- 3 files changed, 504 insertions(+), 489 deletions(-) diff --git a/deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf b/deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf index 436ee598..ed2a2d74 100755 --- a/deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf +++ b/deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf @@ -1,124 +1,129 @@ { - "listen_port": 3000, - "tyk_api_config": { - "Host": "http://tyk-gateway", - "Port": "8080", - "Secret": "28d220fd77974a4facfb07dc1e49c2aa" - }, - "enable_ownership": true, - "mongo_url": "mongodb://tyk-mongo:27017/tyk_analytics", - "page_size": 10, - "admin_secret": "ff8289874f5d45de945a2ea5c02580fe", - "shared_node_secret": "352d20ee67be67f6340b4c0605b044b7", - "redis_port": 6379, - "redis_host": "tyk-redis", - "redis_password": "", - "enable_cluster": false, - "force_api_defaults": false, - "notify_on_change": true, - "license_key": "", - "redis_database": 0, - "redis_hosts": null, - "hash_keys": true, - "enable_multi_org_users": true, - "email_backend": { - "enable_email_notifications": true, - "code": "smtp", - "settings": { - "SMTPAddress": "mailserver:2500", - "SMTPPassword": "examplepassword", - "SMTPUsername": "email@example.com" + "listen_port": 3000, + "tyk_api_config": { + "Host": "http://tyk-gateway", + "Port": "8080", + "Secret": "28d220fd77974a4facfb07dc1e49c2aa" }, - "default_from_email": "jeff@wheresmyrug.com", - "default_from_name": "Jeffrey (The Dude) Lebowski", - "dashboard_hostname": "tyk-dashboard.localhost:3000" - }, - "hide_listen_path": false, - "sentry_code": "", - "sentry_js_code": "", - "use_sentry": false, - "enable_master_keys": false, - "enable_duplicate_slugs": true, - "show_org_id": true, - "host_config": { - "enable_host_names": true, - "disable_org_slug_prefix": true, - "hostname": "", - "override_hostname": "tyk-gateway.localhost:8080", - "portal_domains": {}, - "portal_root_path": "/portal", - "generate_secure_paths": false, - "use_strict_hostmatch": false - }, - "http_server_options": { - "use_ssl": false, - "certificates": [], - "min_version": 0 - }, - "ui": { - "languages": { - "Chinese": "cn", - "English": "en", - "Korean": "ko" + "enable_ownership": true, + "mongo_url": "mongodb://tyk-mongo:27017/tyk_analytics", + "page_size": 10, + "admin_secret": "ff8289874f5d45de945a2ea5c02580fe", + "shared_node_secret": "352d20ee67be67f6340b4c0605b044b7", + "redis_port": 6379, + "redis_host": "tyk-redis", + "redis_password": "", + "enable_cluster": false, + "force_api_defaults": false, + "notify_on_change": true, + "license_key": "", + "redis_database": 0, + "redis_hosts": null, + "hash_keys": true, + "enable_multi_org_users": true, + "email_backend": { + "enable_email_notifications": true, + "code": "smtp", + "settings": { + "SMTPAddress": "mailserver:2500", + "SMTPPassword": "examplepassword", + "SMTPUsername": "email@example.com" + }, + "default_from_email": "jeff@wheresmyrug.com", + "default_from_name": "Jeffrey (The Dude) Lebowski", + "dashboard_hostname": "tyk-dashboard.localhost:3000" }, - "hide_help": true, - "default_lang": "en", - "login_page": {}, - "nav": { - "dont_show_admin_sockets": false, - "hide_activity_by_api_section": false, - "hide_geo": false, - "hide_licenses_section": false, - "hide_logs": false, - "hide_tib_section": false + "hide_listen_path": false, + "sentry_code": "", + "sentry_js_code": "", + "use_sentry": false, + "enable_master_keys": false, + "enable_duplicate_slugs": true, + "show_org_id": true, + "host_config": { + "enable_host_names": true, + "disable_org_slug_prefix": true, + "hostname": "", + "override_hostname": "tyk-gateway.localhost:8080", + "portal_domains": {}, + "portal_root_path": "/portal", + "generate_secure_paths": false, + "use_strict_hostmatch": false }, - "uptime": {}, - "portal_section": null, - "designer": {}, - "dont_show_admin_sockets": false, - "dont_allow_license_management": false, - "dont_allow_license_management_view": false - }, - "home_dir": "/opt/tyk-dashboard", - "tagging_options": { - "tag_all_apis_by_org": false - }, - "use_sharded_analytics": true, - "enable_aggregate_lookups": true, - "enable_analytics_cache": false, - "aggregate_lookup_cutoff": "26/05/2016", - "maintenance_mode": false, - "allow_explicit_policy_id": true, - "private_key_path": "certs/private-key.pem", - "node_schema_path": "", - "oauth_redirect_uri_separator": ";", - "statsd_connection_string": "graphite:8125", - "statsd_prefix": "", - "enable_hashed_keys_listing": true, - "enable_update_key_by_hash": true, - "enable_delete_key_by_hash": true, - "sso_enable_user_lookup": true, - "sso_custom_login_url": "", - "security": { - "open_policy": { - "enabled": true, - "enable_api": true - } - }, - "audit": { - "enabled": true, - "format": "", - "path": "/opt/tyk-dashboard/audit/audit.log", - "detailed_recording": true - }, - "storage": { - "main": { - "type": "postgres", - "connection_string": "user=postgres password=qtpNQY8UKPH3YrDk database=tyk_analytics host=tyk-postgres port=5432" + "http_server_options": { + "use_ssl": false, + "certificates": [], + "min_version": 0 + }, + "ui": { + "languages": { + "Chinese": "cn", + "English": "en", + "Korean": "ko" + }, + "hide_help": true, + "default_lang": "en", + "login_page": {}, + "nav": { + "dont_show_admin_sockets": false, + "hide_activity_by_api_section": false, + "hide_geo": false, + "hide_licenses_section": false, + "hide_logs": false, + "hide_tib_section": false + }, + "uptime": {}, + "portal_section": null, + "designer": {}, + "dont_show_admin_sockets": false, + "dont_allow_license_management": false, + "dont_allow_license_management_view": false + }, + "home_dir": "/opt/tyk-dashboard", + "tagging_options": { + "tag_all_apis_by_org": false + }, + "use_sharded_analytics": true, + "enable_aggregate_lookups": true, + "enable_analytics_cache": false, + "aggregate_lookup_cutoff": "26/05/2016", + "maintenance_mode": false, + "allow_explicit_policy_id": true, + "private_key_path": "certs/private-key.pem", + "node_schema_path": "", + "oauth_redirect_uri_separator": ";", + "statsd_connection_string": "graphite:8125", + "statsd_prefix": "", + "enable_hashed_keys_listing": true, + "enable_update_key_by_hash": true, + "enable_delete_key_by_hash": true, + "sso_enable_user_lookup": true, + "sso_custom_login_url": "", + "security": { + "open_policy": { + "enabled": true, + "enable_api": true + } + }, + "audit": { + "enabled": true, + "format": "", + "path": "/opt/tyk-dashboard/audit/audit.log", + "detailed_recording": true + }, + "storage": { + "main": { + "type": "postgres", + "connection_string": "user=postgres password=qtpNQY8UKPH3YrDk database=tyk_analytics host=tyk-postgres port=5432" + }, + "analytics": { + "type": "mongo", + "connection_string": "mongodb://tyk-mongo:27017/tyk_analytics" + } }, - "analytics": { - "type": "mongo", - "connection_string": "mongodb://tyk-mongo:27017/tyk_analytics" + "labs": { + "streaming": { + "enabled": true + } } - } -} +} \ No newline at end of file diff --git a/deployments/tyk/volumes/tyk-gateway/tyk-2.conf b/deployments/tyk/volumes/tyk-gateway/tyk-2.conf index 88d2ac71..5f3fd67f 100755 --- a/deployments/tyk/volumes/tyk-gateway/tyk-2.conf +++ b/deployments/tyk/volumes/tyk-gateway/tyk-2.conf @@ -1,185 +1,190 @@ { - "listen_address": "", - "listen_port": 8080, - "secret": "28d220fd77974a4facfb07dc1e49c2aa", - "node_secret": "352d20ee67be67f6340b4c0605b044b7", - "template_path": "./templates", - "tyk_js_path": "./js/tyk.js", - "middleware_path": "./middleware", - "policies": { - "policy_source": "service", - "policy_connection_string": "http://tyk-dashboard:3000", - "policy_record_name": "tyk_policies", - "allow_explicit_policy_id": true - }, - "use_db_app_configs": true, - "db_app_conf_options": { - "connection_string": "http://tyk-dashboard:3000", - "node_is_segmented": true, - "tags": [ - "tyk-gateway-2" - ] - }, - "disable_dashboard_zeroconf": false, - "app_path": "./test_apps/", - "storage": { - "type": "redis", - "host": "tyk-redis", - "port": 6379, - "hosts": null, - "username": "", - "password": "", - "database": 0, - "optimisation_max_idle": 3000, - "optimisation_max_active": 5000, - "enable_cluster": false - }, - "enable_separate_cache_store": false, - "enable_analytics": true, - "analytics_config": { - "type": "mongo", - "ignored_ips": [], - "enable_detailed_recording": true, - "enable_geo_ip": false, - "geo_ip_db_path": "./GeoLite2-City.mmdb", - "normalise_urls": { - "enabled": true, - "normalise_uuids": true, - "normalise_numbers": true, - "custom_patterns": [] - } - }, - "health_check": { - "enable_health_checks": false, - "health_check_value_timeouts": 0 - }, - "optimisations_use_async_session_write": true, - "allow_master_keys": false, - "hash_keys": true, - "hash_key_function": "murmur64", - "suppress_redis_signal_reload": false, - "suppress_default_org_store": false, - "use_redis_log": true, - "sentry_code": "", - "use_sentry": false, - "use_syslog": false, - "use_graylog": false, - "use_logstash": false, - "graylog_network_addr": "", - "logstash_network_addr": "", - "syslog_transport": "", - "logstash_transport": "", - "syslog_network_addr": "", - "enforce_org_data_age": true, - "enforce_org_data_detail_logging": false, - "enforce_org_quotas": true, - "experimental_process_org_off_thread": false, - "enable_non_transactional_rate_limiter": true, - "enable_sentinel_rate_limiter": false, - "Monitor": { - "enable_trigger_monitors": false, - "configuration": { - "method": "", - "target_path": "", - "template_path": "", - "header_map": null, - "event_timeout": 0 + "listen_address": "", + "listen_port": 8080, + "secret": "28d220fd77974a4facfb07dc1e49c2aa", + "node_secret": "352d20ee67be67f6340b4c0605b044b7", + "template_path": "./templates", + "tyk_js_path": "./js/tyk.js", + "middleware_path": "./middleware", + "policies": { + "policy_source": "service", + "policy_connection_string": "http://tyk-dashboard:3000", + "policy_record_name": "tyk_policies", + "allow_explicit_policy_id": true }, - "global_trigger_limit": 0, - "monitor_user_keys": false, - "monitor_org_keys": false - }, - "oauth_refresh_token_expire": 0, - "oauth_token_expire": 0, - "oauth_redirect_uri_separator": ";", - "slave_options": { - "use_rpc": false, - "connection_string": "", - "rpc_key": "", - "api_key": "", - "enable_rpc_cache": false, - "bind_to_slugs": false, - "disable_keyspace_sync": false, - "group_id": "" - }, - "disable_virtual_path_blobs": false, - "local_session_cache": { - "disable_cached_session_state": true, - "cached_session_timeout": 0, - "cached_session_eviction": 0 - }, - "http_server_options": { - "override_defaults": false, - "read_timeout": 0, - "write_timeout": 0, - "use_ssl": true, - "use_ssl_le": false, - "enable_websockets": true, - "certificates": [ - { - "domain_name": "tyk-gateway-2.localhost", - "cert_file": "certs/tls-certificate.pem", - "key_file": "certs/tls-private-key.pem" - } - ], - "server_name": "tyk-gateway-2.localhost", - "min_version": 771, - "flush_interval": 0, - "ssl_insecure_skip_verify": true - }, - "service_discovery": { - "default_cache_timeout": 0 - }, - "close_connections": false, - "auth_override": { - "force_auth_provider": false, - "auth_provider": { - "name": "", - "storage_engine": "", - "meta": null + "use_db_app_configs": true, + "db_app_conf_options": { + "connection_string": "http://tyk-dashboard:3000", + "node_is_segmented": true, + "tags": [ + "tyk-gateway-2" + ] }, - "force_session_provider": false, - "session_provider": { - "name": "", - "storage_engine": "", - "meta": null - } - }, - "uptime_tests": { - "disable": false, - "config": { - "failure_trigger_sample_size": 1, - "time_wait": 30, - "checker_pool_size": 50, - "enable_uptime_analytics": true - } - }, - "hostname": "", - "enable_api_segregation": false, - "control_api_hostname": "", - "enable_custom_domains": true, - "enable_jsvm": true, - "enable_coprocess": false, - "hide_generator_header": false, - "event_handlers": { - "events": {} - }, - "event_trigers_defunct": {}, - "pid_file_location": "./tyk-gateway.pid", - "allow_insecure_configs": false, - "public_key_path": "certs/public-key.pem", - "close_idle_connections": false, - "allow_remote_config": true, - "enable_bundle_downloader": false, - "max_idle_connections_per_host": 500, - "tracing": { - "enabled": false, - "name": "zipkin", - "options": { - "reporter": { - "url": "http://zipkin:9411/api/v2/spans" - } + "disable_dashboard_zeroconf": false, + "app_path": "./test_apps/", + "storage": { + "type": "redis", + "host": "tyk-redis", + "port": 6379, + "hosts": null, + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 3000, + "optimisation_max_active": 5000, + "enable_cluster": false + }, + "enable_separate_cache_store": false, + "enable_analytics": true, + "analytics_config": { + "type": "mongo", + "ignored_ips": [], + "enable_detailed_recording": true, + "enable_geo_ip": false, + "geo_ip_db_path": "./GeoLite2-City.mmdb", + "normalise_urls": { + "enabled": true, + "normalise_uuids": true, + "normalise_numbers": true, + "custom_patterns": [] + } + }, + "health_check": { + "enable_health_checks": false, + "health_check_value_timeouts": 0 + }, + "optimisations_use_async_session_write": true, + "allow_master_keys": false, + "hash_keys": true, + "hash_key_function": "murmur64", + "suppress_redis_signal_reload": false, + "suppress_default_org_store": false, + "use_redis_log": true, + "sentry_code": "", + "use_sentry": false, + "use_syslog": false, + "use_graylog": false, + "use_logstash": false, + "graylog_network_addr": "", + "logstash_network_addr": "", + "syslog_transport": "", + "logstash_transport": "", + "syslog_network_addr": "", + "enforce_org_data_age": true, + "enforce_org_data_detail_logging": false, + "enforce_org_quotas": true, + "experimental_process_org_off_thread": false, + "enable_non_transactional_rate_limiter": true, + "enable_sentinel_rate_limiter": false, + "Monitor": { + "enable_trigger_monitors": false, + "configuration": { + "method": "", + "target_path": "", + "template_path": "", + "header_map": null, + "event_timeout": 0 + }, + "global_trigger_limit": 0, + "monitor_user_keys": false, + "monitor_org_keys": false + }, + "oauth_refresh_token_expire": 0, + "oauth_token_expire": 0, + "oauth_redirect_uri_separator": ";", + "slave_options": { + "use_rpc": false, + "connection_string": "", + "rpc_key": "", + "api_key": "", + "enable_rpc_cache": false, + "bind_to_slugs": false, + "disable_keyspace_sync": false, + "group_id": "" + }, + "disable_virtual_path_blobs": false, + "local_session_cache": { + "disable_cached_session_state": true, + "cached_session_timeout": 0, + "cached_session_eviction": 0 + }, + "http_server_options": { + "override_defaults": false, + "read_timeout": 0, + "write_timeout": 0, + "use_ssl": true, + "use_ssl_le": false, + "enable_websockets": true, + "certificates": [ + { + "domain_name": "tyk-gateway-2.localhost", + "cert_file": "certs/tls-certificate.pem", + "key_file": "certs/tls-private-key.pem" + } + ], + "server_name": "tyk-gateway-2.localhost", + "min_version": 771, + "flush_interval": 0, + "ssl_insecure_skip_verify": true + }, + "service_discovery": { + "default_cache_timeout": 0 + }, + "close_connections": false, + "auth_override": { + "force_auth_provider": false, + "auth_provider": { + "name": "", + "storage_engine": "", + "meta": null + }, + "force_session_provider": false, + "session_provider": { + "name": "", + "storage_engine": "", + "meta": null + } + }, + "uptime_tests": { + "disable": false, + "config": { + "failure_trigger_sample_size": 1, + "time_wait": 30, + "checker_pool_size": 50, + "enable_uptime_analytics": true + } + }, + "hostname": "", + "enable_api_segregation": false, + "control_api_hostname": "", + "enable_custom_domains": true, + "enable_jsvm": true, + "enable_coprocess": false, + "hide_generator_header": false, + "event_handlers": { + "events": {} + }, + "event_trigers_defunct": {}, + "pid_file_location": "./tyk-gateway.pid", + "allow_insecure_configs": false, + "public_key_path": "certs/public-key.pem", + "close_idle_connections": false, + "allow_remote_config": true, + "enable_bundle_downloader": false, + "max_idle_connections_per_host": 500, + "tracing": { + "enabled": false, + "name": "zipkin", + "options": { + "reporter": { + "url": "http://zipkin:9411/api/v2/spans" + } + } + }, + "statsd_connection_string": "graphite:8125", + "labs": { + "streaming": { + "enabled": true + } } - }, - "statsd_connection_string": "graphite:8125" } \ No newline at end of file diff --git a/deployments/tyk/volumes/tyk-gateway/tyk.conf b/deployments/tyk/volumes/tyk-gateway/tyk.conf index 45c3e2a8..88d019da 100755 --- a/deployments/tyk/volumes/tyk-gateway/tyk.conf +++ b/deployments/tyk/volumes/tyk-gateway/tyk.conf @@ -1,193 +1,198 @@ { - "listen_address": "", - "listen_port": 8080, - "secret": "28d220fd77974a4facfb07dc1e49c2aa", - "node_secret": "352d20ee67be67f6340b4c0605b044b7", - "template_path": "./templates", - "tyk_js_path": "./js/tyk.js", - "middleware_path": "./middleware", - "policies": { - "policy_source": "service", - "policy_connection_string": "http://tyk-dashboard:3000", - "policy_record_name": "tyk_policies", - "allow_explicit_policy_id": true - }, - "use_db_app_configs": true, - "db_app_conf_options": { - "connection_string": "http://tyk-dashboard:3000", - "node_is_segmented": false, - "tags": [] - }, - "disable_ports_whitelist": true, - "disable_dashboard_zeroconf": false, - "app_path": "./test_apps/", - "storage": { - "type": "redis", - "host": "tyk-redis", - "port": 6379, - "hosts": null, - "username": "", - "password": "", - "database": 0, - "optimisation_max_idle": 3000, - "optimisation_max_active": 5000, - "enable_cluster": false - }, - "enable_separate_cache_store": false, - "enable_analytics": true, - "analytics_config": { - "type": "mongo", - "ignored_ips": [], - "enable_detailed_recording": false, - "enable_geo_ip": false, - "geo_ip_db_path": "./GeoLite2-City.mmdb", - "normalise_urls": { - "enabled": true, - "normalise_uuids": true, - "normalise_numbers": true, - "custom_patterns": [] - } - }, - "health_check": { - "enable_health_checks": false, - "health_check_value_timeouts": 0 - }, - "optimisations_use_async_session_write": true, - "allow_master_keys": false, - "hash_keys": true, - "hash_key_function": "murmur64", - "suppress_redis_signal_reload": false, - "suppress_default_org_store": false, - "use_redis_log": true, - "sentry_code": "", - "use_sentry": false, - "use_syslog": false, - "use_graylog": false, - "use_logstash": false, - "graylog_network_addr": "", - "logstash_network_addr": "", - "syslog_transport": "", - "logstash_transport": "", - "syslog_network_addr": "", - "enforce_org_data_age": true, - "enforce_org_data_detail_logging": false, - "enforce_org_quotas": true, - "experimental_process_org_off_thread": false, - "enable_redis_rolling_limiter": false, - "enable_non_transactional_rate_limiter": true, - "enable_sentinel_rate_limiter": false, - "drl_threshold": 5, - "Monitor": { - "enable_trigger_monitors": false, - "configuration": { - "method": "", - "target_path": "", - "template_path": "", - "header_map": null, - "event_timeout": 0 - }, - "global_trigger_limit": 0, - "monitor_user_keys": false, - "monitor_org_keys": false - }, - "oauth_refresh_token_expire": 0, - "oauth_token_expire": 0, - "oauth_redirect_uri_separator": ";", - "slave_options": { - "use_rpc": false, - "connection_string": "", - "rpc_key": "", - "api_key": "", - "enable_rpc_cache": false, - "bind_to_slugs": false, - "disable_keyspace_sync": false, - "group_id": "" - }, - "disable_virtual_path_blobs": false, - "local_session_cache": { - "disable_cached_session_state": true, - "cached_session_timeout": 0, - "cached_session_eviction": 0 - }, - "http_server_options": { - "override_defaults": false, - "read_timeout": 0, - "write_timeout": 0, - "use_ssl": false, - "use_ssl_le": false, - "enable_websockets": true, - "certificates": [], - "server_name": "", - "min_version": 0, - "flush_interval": 0 - }, - "service_discovery": { - "default_cache_timeout": 0 - }, - "close_connections": false, - "auth_override": { - "force_auth_provider": false, - "auth_provider": { - "name": "", - "storage_engine": "", - "meta": null - }, - "force_session_provider": false, - "session_provider": { - "name": "", - "storage_engine": "", - "meta": null - } - }, - "uptime_tests": { - "disable": false, - "config": { - "failure_trigger_sample_size": 1, - "time_wait": 30, - "checker_pool_size": 50, - "enable_uptime_analytics": true - } - }, - "hostname": "", - "enable_api_segregation": false, - "control_api_hostname": "", - "enable_custom_domains": true, - "enable_jsvm": true, - "coprocess_options": { - "enable_coprocess": true - }, - "hide_generator_header": false, - "event_handlers": { - "events": {} - }, - "event_trigers_defunct": {}, - "pid_file_location": "./tyk-gateway.pid", - "allow_insecure_configs": false, - "public_key_path": "certs/public-key.pem", - "close_idle_connections": false, - "allow_remote_config": true, - "enable_bundle_downloader": true, - "bundle_base_url": "", - "max_idle_connections_per_host": 500, - "tracing": { - "enabled": false, - "name": "zipkin", - "options": { - "reporter": { - "url": "http://zipkin:9411/api/v2/spans" - } + "listen_address": "", + "listen_port": 8080, + "secret": "28d220fd77974a4facfb07dc1e49c2aa", + "node_secret": "352d20ee67be67f6340b4c0605b044b7", + "template_path": "./templates", + "tyk_js_path": "./js/tyk.js", + "middleware_path": "./middleware", + "policies": { + "policy_source": "service", + "policy_connection_string": "http://tyk-dashboard:3000", + "policy_record_name": "tyk_policies", + "allow_explicit_policy_id": true + }, + "use_db_app_configs": true, + "db_app_conf_options": { + "connection_string": "http://tyk-dashboard:3000", + "node_is_segmented": false, + "tags": [] + }, + "disable_ports_whitelist": true, + "disable_dashboard_zeroconf": false, + "app_path": "./test_apps/", + "storage": { + "type": "redis", + "host": "tyk-redis", + "port": 6379, + "hosts": null, + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 3000, + "optimisation_max_active": 5000, + "enable_cluster": false + }, + "enable_separate_cache_store": false, + "enable_analytics": true, + "analytics_config": { + "type": "mongo", + "ignored_ips": [], + "enable_detailed_recording": false, + "enable_geo_ip": false, + "geo_ip_db_path": "./GeoLite2-City.mmdb", + "normalise_urls": { + "enabled": true, + "normalise_uuids": true, + "normalise_numbers": true, + "custom_patterns": [] + } + }, + "health_check": { + "enable_health_checks": false, + "health_check_value_timeouts": 0 + }, + "optimisations_use_async_session_write": true, + "allow_master_keys": false, + "hash_keys": true, + "hash_key_function": "murmur64", + "suppress_redis_signal_reload": false, + "suppress_default_org_store": false, + "use_redis_log": true, + "sentry_code": "", + "use_sentry": false, + "use_syslog": false, + "use_graylog": false, + "use_logstash": false, + "graylog_network_addr": "", + "logstash_network_addr": "", + "syslog_transport": "", + "logstash_transport": "", + "syslog_network_addr": "", + "enforce_org_data_age": true, + "enforce_org_data_detail_logging": false, + "enforce_org_quotas": true, + "experimental_process_org_off_thread": false, + "enable_redis_rolling_limiter": false, + "enable_non_transactional_rate_limiter": true, + "enable_sentinel_rate_limiter": false, + "drl_threshold": 5, + "Monitor": { + "enable_trigger_monitors": false, + "configuration": { + "method": "", + "target_path": "", + "template_path": "", + "header_map": null, + "event_timeout": 0 + }, + "global_trigger_limit": 0, + "monitor_user_keys": false, + "monitor_org_keys": false + }, + "oauth_refresh_token_expire": 0, + "oauth_token_expire": 0, + "oauth_redirect_uri_separator": ";", + "slave_options": { + "use_rpc": false, + "connection_string": "", + "rpc_key": "", + "api_key": "", + "enable_rpc_cache": false, + "bind_to_slugs": false, + "disable_keyspace_sync": false, + "group_id": "" + }, + "disable_virtual_path_blobs": false, + "local_session_cache": { + "disable_cached_session_state": true, + "cached_session_timeout": 0, + "cached_session_eviction": 0 + }, + "http_server_options": { + "override_defaults": false, + "read_timeout": 0, + "write_timeout": 0, + "use_ssl": false, + "use_ssl_le": false, + "enable_websockets": true, + "certificates": [], + "server_name": "", + "min_version": 0, + "flush_interval": 0 + }, + "service_discovery": { + "default_cache_timeout": 0 + }, + "close_connections": false, + "auth_override": { + "force_auth_provider": false, + "auth_provider": { + "name": "", + "storage_engine": "", + "meta": null + }, + "force_session_provider": false, + "session_provider": { + "name": "", + "storage_engine": "", + "meta": null + } + }, + "uptime_tests": { + "disable": false, + "config": { + "failure_trigger_sample_size": 1, + "time_wait": 30, + "checker_pool_size": 50, + "enable_uptime_analytics": true + } + }, + "hostname": "", + "enable_api_segregation": false, + "control_api_hostname": "", + "enable_custom_domains": true, + "enable_jsvm": true, + "coprocess_options": { + "enable_coprocess": true + }, + "hide_generator_header": false, + "event_handlers": { + "events": {} + }, + "event_trigers_defunct": {}, + "pid_file_location": "./tyk-gateway.pid", + "allow_insecure_configs": false, + "public_key_path": "certs/public-key.pem", + "close_idle_connections": false, + "allow_remote_config": true, + "enable_bundle_downloader": true, + "bundle_base_url": "", + "max_idle_connections_per_host": 500, + "tracing": { + "enabled": false, + "name": "zipkin", + "options": { + "reporter": { + "url": "http://zipkin:9411/api/v2/spans" + } + } + }, + "enable_hashed_keys_listing": true, + "statsd_connection_string": "graphite:8125", + "secrets": { + "target_url": "http://httpbin/", + "listen_path": "/secret-path/", + "header": "secret-header-value" + }, + "opentelemetry": { + "enabled": false, + "exporter": "grpc", + "endpoint": "collector-gateway:4317" + }, + "labs": { + "streaming": { + "enabled": true + } } - }, - "enable_hashed_keys_listing": true, - "statsd_connection_string": "graphite:8125", - "secrets": { - "target_url": "http://httpbin/", - "listen_path": "/secret-path/", - "header": "secret-header-value" - }, - "opentelemetry": { - "enabled": false, - "exporter": "grpc", - "endpoint": "collector-gateway:4317" - } -} +} \ No newline at end of file From 901b9fb7dfd739e322066fe858e4bbdd8b9cc534 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 10:45:20 +0200 Subject: [PATCH 06/63] Add HTTP to Websocket example --- ...-oas-a6c6eb76a5344d1a43a8d321f85be323.json | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 deployments/tyk/data/tyk-dashboard/1/apis/api-oas-a6c6eb76a5344d1a43a8d321f85be323.json diff --git a/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-a6c6eb76a5344d1a43a8d321f85be323.json b/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-a6c6eb76a5344d1a43a8d321f85be323.json new file mode 100644 index 00000000..edd26281 --- /dev/null +++ b/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-a6c6eb76a5344d1a43a8d321f85be323.json @@ -0,0 +1,62 @@ +{ + "components": {}, + "info": { + "title": "Streams - Websocket", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": {}, + "servers": [ + { + "url": "http://tyk-gateway.localhost:8080/streams-websocket/" + } + ], + "x-tyk-api-gateway": { + "info": { + "dbId": "669a260ec913c400011a2c1e", + "id": "a6c6eb76a5344d1a43a8d321f85be323", + "name": "Streams - Websocket", + "orgId": "5e9d9544a1dcd60001d0ed20", + "state": { + "active": true + } + }, + "middleware": { + "global": { + "contextVariables": { + "enabled": true + }, + "trafficLogs": { + "enabled": true + } + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/streams-websocket/" + } + }, + "upstream": { + "url": "http://httpbin/" + } + }, + "x-tyk-streaming": { + "streams": { + "Websocket": { + "input": { + "http_server": { + "path": "/post" + }, + "label": "example_generator_input" + }, + "output": { + "http_server": { + "ws_path": "/subscribe" + }, + "label": "example_websocket_output" + } + } + } + } +} From d1a0793f5ccdcf184665cd4565a242d0ebc3e02d Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 10:59:10 +0200 Subject: [PATCH 07/63] update name and listen path --- .../1/apis/api-oas-a6c6eb76a5344d1a43a8d321f85be323.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-a6c6eb76a5344d1a43a8d321f85be323.json b/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-a6c6eb76a5344d1a43a8d321f85be323.json index edd26281..8af36ebf 100644 --- a/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-a6c6eb76a5344d1a43a8d321f85be323.json +++ b/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-a6c6eb76a5344d1a43a8d321f85be323.json @@ -8,14 +8,14 @@ "paths": {}, "servers": [ { - "url": "http://tyk-gateway.localhost:8080/streams-websocket/" + "url": "http://tyk-gateway.localhost:8080/streams-http-to-websocket/" } ], "x-tyk-api-gateway": { "info": { "dbId": "669a260ec913c400011a2c1e", "id": "a6c6eb76a5344d1a43a8d321f85be323", - "name": "Streams - Websocket", + "name": "Streams - HTTP to Websocket", "orgId": "5e9d9544a1dcd60001d0ed20", "state": { "active": true @@ -34,7 +34,7 @@ "server": { "listenPath": { "strip": true, - "value": "/streams-websocket/" + "value": "/streams-http-to-websocket/" } }, "upstream": { From e39321c5f08fe9ecb331106444b37d059bc34cea Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 16:50:10 +0200 Subject: [PATCH 08/63] add scripted examples for http to ws --- deployments/tyk/scripts/examples/streams/http-to-ws-1.sh | 7 +++++++ deployments/tyk/scripts/examples/streams/http-to-ws-2.sh | 6 ++++++ 2 files changed, 13 insertions(+) create mode 100755 deployments/tyk/scripts/examples/streams/http-to-ws-1.sh create mode 100755 deployments/tyk/scripts/examples/streams/http-to-ws-2.sh diff --git a/deployments/tyk/scripts/examples/streams/http-to-ws-1.sh b/deployments/tyk/scripts/examples/streams/http-to-ws-1.sh new file mode 100755 index 00000000..1903d988 --- /dev/null +++ b/deployments/tyk/scripts/examples/streams/http-to-ws-1.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +echo "Streams - HTTP to Websocket Example" +echo "Part 1: Listening to a Websocket" +echo "Once connected, run part two of this example in a different terminal. The messages sent will appear here." +echo "Connecting to the websocket..." +wscat -c ws://tyk-gateway.localhost:8080/streams-http-to-websocket/subscribe diff --git a/deployments/tyk/scripts/examples/streams/http-to-ws-2.sh b/deployments/tyk/scripts/examples/streams/http-to-ws-2.sh new file mode 100755 index 00000000..2c66c2be --- /dev/null +++ b/deployments/tyk/scripts/examples/streams/http-to-ws-2.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +echo "Streams - HTTP to Websocket Example" +echo "Part 2: Sending a message" +echo "Each time this script is run, the message \"Hello, Tyk Streams!\" will be received through the websocket in part 1. Check the other terminal to see the messages." +curl -d '{"message":"Hello, Tyk Streams!"}' http://tyk-gateway.localhost:8080/streams-http-to-websocket/post From f8e102d3512e67d4adf67ec7018cfeda6cd183ff Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 16:50:34 +0200 Subject: [PATCH 09/63] add partial example to postman for http to ws --- .../tyk/tyk_demo_tyk.postman_collection.json | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/deployments/tyk/tyk_demo_tyk.postman_collection.json b/deployments/tyk/tyk_demo_tyk.postman_collection.json index 08c4a526..9cf1f92f 100644 --- a/deployments/tyk/tyk_demo_tyk.postman_collection.json +++ b/deployments/tyk/tyk_demo_tyk.postman_collection.json @@ -5848,6 +5848,61 @@ "response": [] } ] + }, + { + "name": "Streams", + "item": [ + { + "name": "HTTP to Websocket", + "item": [ + { + "name": "Post message to websocket", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript", + "packages": {} + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\"message\":\"Hello, Tyk Streams!\"}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://tyk-gateway.localhost:8080/streams-http-to-websocket/post", + "protocol": "http", + "host": [ + "tyk-gateway", + "localhost" + ], + "port": "8080", + "path": [ + "streams-http-to-websocket", + "post" + ] + }, + "description": "This example uses the gateway to proxy messages between HTTP and WS protocols.\n\nThe HTTP request sends a message payload that the gateway then relays to clients subscribing via a websocket.\n\nTo see this in action, run the two scripts in the `deployments/tyk/scripts/examples/streams` directory:\n\n- `http-to-ws-1.sh`: Listens to the websocket for messages.\n \n- `http-to-ws-2.sh`: Sends messages to the websocket via HTTP.\n \n\nThis example request is exact the same as the 2nd script, so can be used for the same purpose." + }, + "response": [] + } + ] + } + ] } ] }, From a24a463c7cf722e744e597fe35c30a5d0610a7f3 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 16:53:33 +0200 Subject: [PATCH 10/63] improve docs --- deployments/tyk/tyk_demo_tyk.postman_collection.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/tyk/tyk_demo_tyk.postman_collection.json b/deployments/tyk/tyk_demo_tyk.postman_collection.json index 9cf1f92f..eafeef54 100644 --- a/deployments/tyk/tyk_demo_tyk.postman_collection.json +++ b/deployments/tyk/tyk_demo_tyk.postman_collection.json @@ -5896,7 +5896,7 @@ "post" ] }, - "description": "This example uses the gateway to proxy messages between HTTP and WS protocols.\n\nThe HTTP request sends a message payload that the gateway then relays to clients subscribing via a websocket.\n\nTo see this in action, run the two scripts in the `deployments/tyk/scripts/examples/streams` directory:\n\n- `http-to-ws-1.sh`: Listens to the websocket for messages.\n \n- `http-to-ws-2.sh`: Sends messages to the websocket via HTTP.\n \n\nThis example request is exact the same as the 2nd script, so can be used for the same purpose." + "description": "This example uses the gateway to proxy messages between HTTP and WS protocols.\n\nThe HTTP request sends a message payload that the gateway then relays to clients subscribing via a websocket.\n\nTo see this in action, run the two scripts in the `deployments/tyk/scripts/examples/streams` directory:\n\n- `http-to-ws-1.sh`: Listens to the websocket for messages.\n \n- `http-to-ws-2.sh`: Sends messages to the websocket via HTTP.\n \n\nThis example request is exactly the same as the 2nd script, so can be used for the same purpose. Requests sent from here will appear in the terminal window running the 1st script." }, "response": [] } From cc701061a43346b96e5baceff74a179eb33647d7 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 16:55:44 +0200 Subject: [PATCH 11/63] fix hostname for use in tests --- deployments/tyk/tyk_demo_tyk.postman_collection.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/deployments/tyk/tyk_demo_tyk.postman_collection.json b/deployments/tyk/tyk_demo_tyk.postman_collection.json index eafeef54..63a5f8d9 100644 --- a/deployments/tyk/tyk_demo_tyk.postman_collection.json +++ b/deployments/tyk/tyk_demo_tyk.postman_collection.json @@ -5884,13 +5884,11 @@ } }, "url": { - "raw": "http://tyk-gateway.localhost:8080/streams-http-to-websocket/post", + "raw": "http://{{tyk-gateway.host}}/streams-http-to-websocket/post", "protocol": "http", "host": [ - "tyk-gateway", - "localhost" + "{{tyk-gateway.host}}" ], - "port": "8080", "path": [ "streams-http-to-websocket", "post" From 1e118e79b07b07a4073a86028a447ac2dea77cfe Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 17:33:45 +0200 Subject: [PATCH 12/63] move scripts to separate directory --- .../scripts/examples/streams/{ => http-to-ws}/http-to-ws-1.sh | 0 .../scripts/examples/streams/{ => http-to-ws}/http-to-ws-2.sh | 0 deployments/tyk/tyk_demo_tyk.postman_collection.json | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename deployments/tyk/scripts/examples/streams/{ => http-to-ws}/http-to-ws-1.sh (100%) rename deployments/tyk/scripts/examples/streams/{ => http-to-ws}/http-to-ws-2.sh (100%) diff --git a/deployments/tyk/scripts/examples/streams/http-to-ws-1.sh b/deployments/tyk/scripts/examples/streams/http-to-ws/http-to-ws-1.sh similarity index 100% rename from deployments/tyk/scripts/examples/streams/http-to-ws-1.sh rename to deployments/tyk/scripts/examples/streams/http-to-ws/http-to-ws-1.sh diff --git a/deployments/tyk/scripts/examples/streams/http-to-ws-2.sh b/deployments/tyk/scripts/examples/streams/http-to-ws/http-to-ws-2.sh similarity index 100% rename from deployments/tyk/scripts/examples/streams/http-to-ws-2.sh rename to deployments/tyk/scripts/examples/streams/http-to-ws/http-to-ws-2.sh diff --git a/deployments/tyk/tyk_demo_tyk.postman_collection.json b/deployments/tyk/tyk_demo_tyk.postman_collection.json index 63a5f8d9..09092c30 100644 --- a/deployments/tyk/tyk_demo_tyk.postman_collection.json +++ b/deployments/tyk/tyk_demo_tyk.postman_collection.json @@ -5894,7 +5894,7 @@ "post" ] }, - "description": "This example uses the gateway to proxy messages between HTTP and WS protocols.\n\nThe HTTP request sends a message payload that the gateway then relays to clients subscribing via a websocket.\n\nTo see this in action, run the two scripts in the `deployments/tyk/scripts/examples/streams` directory:\n\n- `http-to-ws-1.sh`: Listens to the websocket for messages.\n \n- `http-to-ws-2.sh`: Sends messages to the websocket via HTTP.\n \n\nThis example request is exactly the same as the 2nd script, so can be used for the same purpose. Requests sent from here will appear in the terminal window running the 1st script." + "description": "This example uses the gateway to proxy messages between HTTP and WS protocols.\n\nThe HTTP request sends a message payload that the gateway then relays to clients subscribing via a websocket.\n\nTo see this in action, run the two scripts in the `deployments/tyk/scripts/examples/streams/http-to-ws` directory:\n\n- `http-to-ws-1.sh`: Listens to the websocket for messages.\n \n- `http-to-ws-2.sh`: Sends messages to the websocket via HTTP.\n \n\nThis example request is exactly the same as the 2nd script, so can be used for the same purpose. Requests sent from here will appear in the terminal window running the 1st script." }, "response": [] } From 7afe612eca0c4a205db4619eb5182ded427e61d6 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 17:44:58 +0200 Subject: [PATCH 13/63] Streams example for HTTP to Redis PubSub --- ...-oas-3901e49949cf4c2369ab30ac5dd0e91e.json | 70 +++++++++++++++++++ .../http-to-redis-pubsub-1.sh | 7 ++ .../http-to-redis-pubsub-2.sh | 6 ++ 3 files changed, 83 insertions(+) create mode 100644 deployments/tyk/data/tyk-dashboard/1/apis/api-oas-3901e49949cf4c2369ab30ac5dd0e91e.json create mode 100755 deployments/tyk/scripts/examples/streams/http-to-redis-pubsub/http-to-redis-pubsub-1.sh create mode 100755 deployments/tyk/scripts/examples/streams/http-to-redis-pubsub/http-to-redis-pubsub-2.sh diff --git a/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-3901e49949cf4c2369ab30ac5dd0e91e.json b/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-3901e49949cf4c2369ab30ac5dd0e91e.json new file mode 100644 index 00000000..d8a7b0a2 --- /dev/null +++ b/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-3901e49949cf4c2369ab30ac5dd0e91e.json @@ -0,0 +1,70 @@ +{ + "components": {}, + "info": { + "title": "Streams - HTTP to Redis PubSub", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": {}, + "servers": [ + { + "url": "http://tyk-gateway.localhost:8080/streams-http-to-redis-pubsub/" + } + ], + "x-tyk-api-gateway": { + "info": { + "dbId": "669a81e7c913c400011a2c73", + "id": "3901e49949cf4c2369ab30ac5dd0e91e", + "name": "Streams - HTTP to Redis PubSub", + "orgId": "5e9d9544a1dcd60001d0ed20", + "state": { + "active": true + } + }, + "middleware": { + "global": { + "contextVariables": { + "enabled": true + }, + "trafficLogs": { + "enabled": true + } + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/streams-http-to-redis-pubsub/" + } + }, + "upstream": { + "url": "http://httpbin/" + } + }, + "x-tyk-streaming": { + "streams": { + "HTTP to Redis PubSub": { + "input": { + "http_server": { + "path": "/post" + }, + "label": "example_generator_input" + }, + "output": { + "label": "push_to_redis_pubsub", + "redis_pubsub": { + "batching": { + "byte_size": 0, + "check": "", + "count": 0, + "period": "" + }, + "channel": "streams", + "max_in_flight": 64, + "url": "redis://tyk-redis:6379" + } + } + } + } + } +} diff --git a/deployments/tyk/scripts/examples/streams/http-to-redis-pubsub/http-to-redis-pubsub-1.sh b/deployments/tyk/scripts/examples/streams/http-to-redis-pubsub/http-to-redis-pubsub-1.sh new file mode 100755 index 00000000..e03cfab1 --- /dev/null +++ b/deployments/tyk/scripts/examples/streams/http-to-redis-pubsub/http-to-redis-pubsub-1.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +echo "Streams - HTTP to Redis Pub/Sub" +echo "Part 1: Listening to a Redis Channel" +echo "Once connected, run part two of this example in a different terminal. The messages sent will appear here." +echo "Connecting to the Redis channel..." +docker exec tyk-demo-tyk-redis-1 redis-cli SUBSCRIBE streams diff --git a/deployments/tyk/scripts/examples/streams/http-to-redis-pubsub/http-to-redis-pubsub-2.sh b/deployments/tyk/scripts/examples/streams/http-to-redis-pubsub/http-to-redis-pubsub-2.sh new file mode 100755 index 00000000..250ba94a --- /dev/null +++ b/deployments/tyk/scripts/examples/streams/http-to-redis-pubsub/http-to-redis-pubsub-2.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +echo "Streams - HTTP to Redis Pub/Sub" +echo "Part 2: Sending a message" +echo "Each time this script is run, the message \"Hello, Tyk Streams!\" will be received through the Redis channel in part 1. Check the other terminal to see the messages." +curl -d 'Hello, Tyk Streams!' http://tyk-gateway.localhost:8080/streams-http-to-redis-pubsub/post From 981b9c9d823af23e5be653e75c5b14e99c67f70e Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 17:47:28 +0200 Subject: [PATCH 14/63] add readme for example scripts --- deployments/tyk/scripts/examples/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 deployments/tyk/scripts/examples/README.md diff --git a/deployments/tyk/scripts/examples/README.md b/deployments/tyk/scripts/examples/README.md new file mode 100644 index 00000000..b2da7d23 --- /dev/null +++ b/deployments/tyk/scripts/examples/README.md @@ -0,0 +1 @@ +Some example can't be adequately demonstrated from Postman, so they are demonstrated here instead using Bash scripts. \ No newline at end of file From dea3985ea656ee626fb649120f3d77e9539db195 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Fri, 19 Jul 2024 17:48:50 +0200 Subject: [PATCH 15/63] add partial HTTP to Redis Pub/Sub example to Postman --- .../tyk/tyk_demo_tyk.postman_collection.json | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/deployments/tyk/tyk_demo_tyk.postman_collection.json b/deployments/tyk/tyk_demo_tyk.postman_collection.json index 09092c30..506adf15 100644 --- a/deployments/tyk/tyk_demo_tyk.postman_collection.json +++ b/deployments/tyk/tyk_demo_tyk.postman_collection.json @@ -5899,6 +5899,54 @@ "response": [] } ] + }, + { + "name": "HTTP to Redis Pub/Sub", + "item": [ + { + "name": "Post message to Redis Pub/Sub", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript", + "packages": {} + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "Hello, Tyk Streams!", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "http://{{tyk-gateway.host}}/streams-http-to-redis-pubsub/post", + "protocol": "http", + "host": [ + "{{tyk-gateway.host}}" + ], + "path": [ + "streams-http-to-redis-pubsub", + "post" + ] + }, + "description": "This example shows how Tyk can proxy messages between HTTP and Redis pub/sub channels.\n\nMessages posted in this example will be sent to the `streams` channel of the Tyk Demo Redis instance.\n\nTo see it in action, run the scripts found in deployments/tyk/scripts/examples/streams/http-to-redis-pubsub:\n\n- `http-to-redis-pubsub-1.sh`: Subscribes to the Redis `streams` channel and waits for messages.\n \n- `http-to-redis-pubsub-2.sh`: Sends an HTTP POST requests to the gateway that is published into the `streams` channel.\n \n\nYou can also send the message via this example request - it does the same thing as the 2nd script above." + }, + "response": [] + } + ] } ] } From e934e00ec5840561d72405120e414c391e984e6e Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 24 Jul 2024 17:15:41 +0200 Subject: [PATCH 16/63] add kafka to deployment --- deployments/tyk/bootstrap.sh | 5 +++++ deployments/tyk/docker-compose.yml | 28 +++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index d50700f3..36df7d6a 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -210,6 +210,11 @@ bootstrap_progress log_message "Wait for services to be available after restart" wait_for_liveness +# Kafka + +log_message "Creating Kafka topic" +docker exec tyk-demo-kafka-1 sh -c "/opt/kafka/bin/kafka-topics.sh --create --topic quickstart-events --bootstrap-server localhost:9092" + # Go plugins build_go_plugin "example-go-plugin.so" "example" diff --git a/deployments/tyk/docker-compose.yml b/deployments/tyk/docker-compose.yml index ae7a96da..57578b24 100755 --- a/deployments/tyk/docker-compose.yml +++ b/deployments/tyk/docker-compose.yml @@ -142,10 +142,36 @@ services: - 4040:4040 environment: - NGROK_AUTHTOKEN=${NGROK_AUTHTOKEN-} - + kafka: + image: apache/kafka:3.7.1 + networks: + - tyk + ports: + - "9092:9092" + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_LISTENERS: 'PLAINTEXT://:9092,CONTROLLER://:9093' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:9093' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk' + volumes: + - tyk-kafka-data:/opt/kafka/data + command: > + bash -c " + /opt/kafka/bin/kafka-storage.sh format -t $${CLUSTER_ID} -c /opt/kafka/config/kraft/server.properties && + /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kraft/server.properties + " + volumes: tyk-redis-data: tyk-mongo-data: + tyk-kafka-data: networks: tyk: From 2de9be723b85855f9cdd40307af18ea44ad022b0 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 24 Jul 2024 17:16:24 +0200 Subject: [PATCH 17/63] WIP api def for HTTP to kafka --- ...-oas-742ed355325d4bde71c6986543d35c39.json | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 deployments/tyk/data/tyk-dashboard/1/apis/api-oas-742ed355325d4bde71c6986543d35c39.json diff --git a/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-742ed355325d4bde71c6986543d35c39.json b/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-742ed355325d4bde71c6986543d35c39.json new file mode 100644 index 00000000..06d42949 --- /dev/null +++ b/deployments/tyk/data/tyk-dashboard/1/apis/api-oas-742ed355325d4bde71c6986543d35c39.json @@ -0,0 +1,80 @@ +{ + "components": {}, + "info": { + "title": "Streams - HTTP to Kafka", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": {}, + "servers": [ + { + "url": "http://tyk-gateway.localhost:8080/streams-http-to-kafka/" + } + ], + "x-tyk-api-gateway": { + "info": { + "dbId": "66a1194158bbbd000191b178", + "id": "742ed355325d4bde71c6986543d35c39", + "name": "Streams - HTTP to Kafka", + "orgId": "5e9d9544a1dcd60001d0ed20", + "state": { + "active": true + } + }, + "middleware": { + "global": { + "contextVariables": { + "enabled": true + }, + "trafficLogs": { + "enabled": true + } + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/streams-http-to-kafka/" + } + }, + "upstream": { + "url": "http://httpbin/" + } + }, + "x-tyk-streaming": { + "streams": { + "HTTP to Kafka": { + "input": { + "http_server": { + "path": "/post" + }, + "label": "http_input" + }, + "output": { + "kafka": { + "addresses": [ + "kafka:9092" + ], + "batching": { + "byte_size": 0, + "check": "", + "count": 0, + "period": "" + }, + "compression": "none", + "key": "", + "max_in_flight": 64, + "metadata": { + "exclude_prefixes": [] + }, + "partitioner": "fnv1a_hash", + "static_headers": {}, + "target_version": "3.7.1", + "topic": "quickstart-events" + }, + "label": "kafka_output" + } + } + } + } +} From d8e112957b4fe2e3254e1977abf8adebfb2ac66c Mon Sep 17 00:00:00 2001 From: ilijabojanovic Date: Fri, 26 Jul 2024 11:52:49 +0200 Subject: [PATCH 18/63] Test 5.3.3 images --- deployments/tyk/docker-compose.yml | 6 +++--- deployments/tyk2/docker-compose.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deployments/tyk/docker-compose.yml b/deployments/tyk/docker-compose.yml index c12f5861..10b98dbd 100755 --- a/deployments/tyk/docker-compose.yml +++ b/deployments/tyk/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc4} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.3.3-rc1} ports: - 3000:3000 networks: @@ -22,7 +22,7 @@ services: - tyk-redis - tyk-mongo tyk-gateway: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.4.0-rc4} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.3.3-rc1} ports: - 8080:8080 - 8086:8086 @@ -55,7 +55,7 @@ services: - tyk-redis - tyk-dashboard tyk-gateway-2: - image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.4.0-rc4} + image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.3.3-rc1} ports: - 8081:8080 networks: diff --git a/deployments/tyk2/docker-compose.yml b/deployments/tyk2/docker-compose.yml index ea84744b..d2b93303 100644 --- a/deployments/tyk2/docker-compose.yml +++ b/deployments/tyk2/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk2-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.4.0-rc4} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.3.3-rc1} ports: - 3002:3000 networks: @@ -20,7 +20,7 @@ services: - tyk2-redis - tyk2-mongo tyk2-gateway: - image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.4.0-rc4} + image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.3.3-rc1} ports: - 8085:8080 networks: From c0db9efeb9f62e4b9ee0e84c8b1919d0cc307e21 Mon Sep 17 00:00:00 2001 From: ilijabojanovic Date: Fri, 26 Jul 2024 12:13:02 +0200 Subject: [PATCH 19/63] Test 5.5.0 images --- deployments/tyk/docker-compose.yml | 6 +++--- deployments/tyk2/docker-compose.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/deployments/tyk/docker-compose.yml b/deployments/tyk/docker-compose.yml index 10b98dbd..bc8106b0 100755 --- a/deployments/tyk/docker-compose.yml +++ b/deployments/tyk/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.3.3-rc1} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.5.0-rc1} ports: - 3000:3000 networks: @@ -22,7 +22,7 @@ services: - tyk-redis - tyk-mongo tyk-gateway: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.3.3-rc1} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0-rc2} ports: - 8080:8080 - 8086:8086 @@ -55,7 +55,7 @@ services: - tyk-redis - tyk-dashboard tyk-gateway-2: - image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.3.3-rc1} + image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.5.0-rc2} ports: - 8081:8080 networks: diff --git a/deployments/tyk2/docker-compose.yml b/deployments/tyk2/docker-compose.yml index d2b93303..eced7e7e 100644 --- a/deployments/tyk2/docker-compose.yml +++ b/deployments/tyk2/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk2-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.3.3-rc1} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.5.0-rc1} ports: - 3002:3000 networks: @@ -20,7 +20,7 @@ services: - tyk2-redis - tyk2-mongo tyk2-gateway: - image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.3.3-rc1} + image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.5.0-rc2} ports: - 8085:8080 networks: From 24d1aedac357fd7b734ed85d92c8a4c5802e47b9 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 10:36:21 +0200 Subject: [PATCH 20/63] ignore errors if certs don't exist --- deployments/tyk/bootstrap.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 36df7d6a..7ada65b5 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -88,8 +88,8 @@ done log_message "OpenSSL version used for generating certs: $(docker exec $OPENSSL_CONTAINER_NAME openssl version)" log_message "Removing any pre-existing certs" -rm deployments/tyk/volumes/tyk-dashboard/certs/*.pem 1> /dev/null 2>> logs/bootstrap.log -rm deployments/tyk/volumes/tyk-gateway/certs/*.pem 1> /dev/null 2>> logs/bootstrap.log +rm deployments/tyk/volumes/tyk-dashboard/certs/*.pem > /dev/null 2>&1 +rm deployments/tyk/volumes/tyk-gateway/certs/*.pem > /dev/null 2>&1 log_ok bootstrap_progress From 662544004b36644f818196ce3515a9dd89a0549e Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 10:55:02 +0200 Subject: [PATCH 21/63] debug message --- deployments/tyk/bootstrap.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 7ada65b5..9d21220e 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -195,6 +195,8 @@ fi log_ok bootstrap_progress +log_message "DEBUG: gateway cert directory $(docker exec tyk-demo-tyk-gateway-1 ls /opt/tyk-gateway/certs)" + log_message "Recreating containers to ensure new certificates are loaded (tyk-gateway, tyk-gateway-2, tyk-dashboard)" eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 tyk-dashboard # if there are gateways from other deployments connecting to this deployment From 31d789df6ef864216cc270bc59622cad27e35219 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 11:23:39 +0200 Subject: [PATCH 22/63] debug message --- deployments/tyk/bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 9d21220e..c609417e 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -196,6 +196,7 @@ log_ok bootstrap_progress log_message "DEBUG: gateway cert directory $(docker exec tyk-demo-tyk-gateway-1 ls /opt/tyk-gateway/certs)" +log_message "DEBUG: dashboard cert directory $(docker exec tyk-demo-tyk-dashboard-1 ls /opt/tyk-dashboard/certs)" log_message "Recreating containers to ensure new certificates are loaded (tyk-gateway, tyk-gateway-2, tyk-dashboard)" eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 tyk-dashboard From d729f0bedb1025dffc80df5b3d9cd4dc266e5851 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 11:28:40 +0200 Subject: [PATCH 23/63] remove debug log --- deployments/tyk/bootstrap.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index c609417e..7ada65b5 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -195,9 +195,6 @@ fi log_ok bootstrap_progress -log_message "DEBUG: gateway cert directory $(docker exec tyk-demo-tyk-gateway-1 ls /opt/tyk-gateway/certs)" -log_message "DEBUG: dashboard cert directory $(docker exec tyk-demo-tyk-dashboard-1 ls /opt/tyk-dashboard/certs)" - log_message "Recreating containers to ensure new certificates are loaded (tyk-gateway, tyk-gateway-2, tyk-dashboard)" eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 tyk-dashboard # if there are gateways from other deployments connecting to this deployment From 963ab502f202b5a44e7563992c3499d3871f1161 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 11:30:26 +0200 Subject: [PATCH 24/63] debug container reload --- deployments/tyk/bootstrap.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 7ada65b5..77c4be8e 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -195,8 +195,8 @@ fi log_ok bootstrap_progress -log_message "Recreating containers to ensure new certificates are loaded (tyk-gateway, tyk-gateway-2, tyk-dashboard)" -eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 tyk-dashboard +log_message "Recreating containers to ensure new certificates are loaded (tyk-dashboard, tyk-gateway, tyk-gateway-2)" +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 # if there are gateways from other deployments connecting to this deployment # (such as MDCB), then they must be recreated to. The MDCB deployment already # handles recreation. @@ -423,6 +423,8 @@ for data_group_path in deployments/tyk/data/tyk-dashboard/*; do fi done + eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 + # OAuth - Clients log_message "Creating OAuth Clients" for file in $data_group_path/oauth/clients/*; do From 7867cbdf2b187a13a776a77213a7ed48e7717e48 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 11:48:54 +0200 Subject: [PATCH 25/63] remove debug reload --- deployments/tyk/bootstrap.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 77c4be8e..d29229d4 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -423,8 +423,6 @@ for data_group_path in deployments/tyk/data/tyk-dashboard/*; do fi done - eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 - # OAuth - Clients log_message "Creating OAuth Clients" for file in $data_group_path/oauth/clients/*; do From 721d2b09c538b389d8d88ca409f6d4c594268e4e Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 13:11:52 +0200 Subject: [PATCH 26/63] add payload signature error check --- deployments/tyk/bootstrap.sh | 53 ++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index d29229d4..9247b50d 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -195,15 +195,52 @@ fi log_ok bootstrap_progress -log_message "Recreating containers to ensure new certificates are loaded (tyk-dashboard, tyk-gateway, tyk-gateway-2)" +# eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 +# # if there are gateways from other deployments connecting to this deployment +# # (such as MDCB), then they must be recreated to. The MDCB deployment already +# # handles recreation. +# if [ "$?" != "0" ]; then +# echo "ERROR: Could not recreate containers" +# exit 1 +# fi + + + +log_message "Recreating containers to load new certificates" eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 -# if there are gateways from other deployments connecting to this deployment -# (such as MDCB), then they must be recreated to. The MDCB deployment already -# handles recreation. -if [ "$?" != "0" ]; then - echo "ERROR: Could not recreate containers" - exit 1 -fi +# pause to allow logs to capture any payload signature errors +sleep 2 +log_ok + +log_message "Validating that secure messaging is functioning on gateway containers" +gateway_container_names=("tyk-demo-tyk-gateway-1" "tyk-demo-tyk-gateway-2-1") +attempts=0 +max_attempts=3 +phrase="Payload signature is invalid!" +while true; do + attempts=$((attempts + 1)) + if [ "$attempts" -gt "$max_attempts" ]; then + echo "Gateways unable to recover from payload signature error" + exit 1 + fi + + all_clear=true + for container in "${gateway_container_names[@]}"; do + if docker logs "$container" 2>&1 | grep -q "$phrase"; then + log_message " Attempt $attempts: Payload signature error detected in the logs of container '$container'." + eval $(generate_docker_compose_command) up -d --no-deps --force-recreate $container + all_clear=false + fi + done + + if [ "$all_clear" = true ]; then + log_message " Payload signature error is not present in any container logs after $attempts attempts." + break + fi + + sleep 3 # Wait for 3 seconds before checking again +done + log_ok bootstrap_progress From 809db9e8d4def7c162018eeedb57ea97b663b7f7 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 13:12:41 +0200 Subject: [PATCH 27/63] tidy --- deployments/tyk/bootstrap.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 9247b50d..574d8c32 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -195,17 +195,6 @@ fi log_ok bootstrap_progress -# eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 -# # if there are gateways from other deployments connecting to this deployment -# # (such as MDCB), then they must be recreated to. The MDCB deployment already -# # handles recreation. -# if [ "$?" != "0" ]; then -# echo "ERROR: Could not recreate containers" -# exit 1 -# fi - - - log_message "Recreating containers to load new certificates" eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 # pause to allow logs to capture any payload signature errors @@ -240,7 +229,6 @@ while true; do sleep 3 # Wait for 3 seconds before checking again done - log_ok bootstrap_progress From b0ae48857b82f592f48a7206f534023a26381d65 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 13:28:55 +0200 Subject: [PATCH 28/63] debugging --- deployments/tyk/bootstrap.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 574d8c32..1fc5ac56 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -201,8 +201,8 @@ eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-das sleep 2 log_ok -log_message "Validating that secure messaging is functioning on gateway containers" -gateway_container_names=("tyk-demo-tyk-gateway-1" "tyk-demo-tyk-gateway-2-1") +echo "Validating that secure messaging is functioning on gateway containers" +gateway_service_names=("tyk-gateway" "tyk-gateway-2") attempts=0 max_attempts=3 phrase="Payload signature is invalid!" @@ -214,16 +214,16 @@ while true; do fi all_clear=true - for container in "${gateway_container_names[@]}"; do - if docker logs "$container" 2>&1 | grep -q "$phrase"; then - log_message " Attempt $attempts: Payload signature error detected in the logs of container '$container'." - eval $(generate_docker_compose_command) up -d --no-deps --force-recreate $container + for gateway_service in "${gateway_service_names[@]}"; do + if $(generate_docker_compose_command) logs "$gateway_service" 2>&1 | grep -q "$phrase"; then + echo " Attempt $attempts: Payload signature error detected in the logs of service '$gateway_service'." + $(generate_docker_compose_command) up -d --no-deps --force-recreate $gateway_service all_clear=false fi done if [ "$all_clear" = true ]; then - log_message " Payload signature error is not present in any container logs after $attempts attempts." + echo " Payload signature error is not present in any container logs after $attempts attempts." break fi From a095a2227d0fb4eeb0a9d9afc2803f40ad55653f Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 13:32:26 +0200 Subject: [PATCH 29/63] debug --- deployments/tyk/bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 1fc5ac56..18492d12 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -196,6 +196,7 @@ log_ok bootstrap_progress log_message "Recreating containers to load new certificates" +sleep 2 eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 # pause to allow logs to capture any payload signature errors sleep 2 From c7a28570f9e1a5fc16f25e20b124b4b9198b0133 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 14:34:08 +0200 Subject: [PATCH 30/63] debug --- deployments/tyk/bootstrap.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 18492d12..0ba0c885 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -197,9 +197,8 @@ bootstrap_progress log_message "Recreating containers to load new certificates" sleep 2 -eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 -# pause to allow logs to capture any payload signature errors -sleep 2 +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway +# attempt hot reloads to test payloads log_ok echo "Validating that secure messaging is functioning on gateway containers" @@ -208,23 +207,28 @@ attempts=0 max_attempts=3 phrase="Payload signature is invalid!" while true; do + hot_reload "http://tyk-gateway.localhost:8080" "28d220fd77974a4facfb07dc1e49c2aa" + hot_reload "https://tyk-gateway-2.localhost:8081" "28d220fd77974a4facfb07dc1e49c2aa" + # pause to allow logs to capture any payload signature errors caused by hot reload command + sleep 2 + attempts=$((attempts + 1)) if [ "$attempts" -gt "$max_attempts" ]; then - echo "Gateways unable to recover from payload signature error" + echo "ERROR: Unable to clear '$phrase' from logs" exit 1 fi all_clear=true for gateway_service in "${gateway_service_names[@]}"; do if $(generate_docker_compose_command) logs "$gateway_service" 2>&1 | grep -q "$phrase"; then - echo " Attempt $attempts: Payload signature error detected in the logs of service '$gateway_service'." + echo " Attempt $attempts: '$phrase' detected in the logs of service '$gateway_service' - recreating" $(generate_docker_compose_command) up -d --no-deps --force-recreate $gateway_service all_clear=false fi done if [ "$all_clear" = true ]; then - echo " Payload signature error is not present in any container logs after $attempts attempts." + echo " '$phrase' is not present in any container logs after $attempts attempts." break fi From ede2e866a82a9602464aeac255e24f0557c89919 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 14:46:07 +0200 Subject: [PATCH 31/63] debug --- deployments/tyk/bootstrap.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 0ba0c885..c95afe2e 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -111,6 +111,7 @@ while true; do sleep 2 fi done +sleep 1 log_message "Generating private key for secure messaging and signing" docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl genrsa -out /tmp/private-key.pem 2048" >>logs/bootstrap.log @@ -130,6 +131,7 @@ while true; do sleep 2 fi done +sleep 1 log_message "Generating public key for secure messaging and signing" docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl rsa -in /tmp/private-key.pem -pubout -out /tmp/public-key.pem" >>logs/bootstrap.log @@ -149,6 +151,7 @@ while true; do sleep 2 fi done +sleep 1 log_message "Copying private-key.pem to dashboard volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/private-key.pem deployments/tyk/volumes/tyk-dashboard/certs >>logs/bootstrap.log @@ -158,6 +161,7 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress +sleep 1 log_message "Copying public-key.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/public-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -167,6 +171,7 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress +sleep 1 log_message "Copying tls-certificate.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-certificate.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -176,6 +181,7 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress +sleep 1 log_message "Copying tls-private-key.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-private-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -185,6 +191,7 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress +sleep 1 log_message "Removing temporary OpenSSL container $OPENSSL_CONTAINER_NAME" docker rm -f $OPENSSL_CONTAINER_NAME @@ -194,10 +201,11 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress +sleep 1 log_message "Recreating containers to load new certificates" sleep 2 -eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 # attempt hot reloads to test payloads log_ok @@ -210,7 +218,7 @@ while true; do hot_reload "http://tyk-gateway.localhost:8080" "28d220fd77974a4facfb07dc1e49c2aa" hot_reload "https://tyk-gateway-2.localhost:8081" "28d220fd77974a4facfb07dc1e49c2aa" # pause to allow logs to capture any payload signature errors caused by hot reload command - sleep 2 + sleep 5 attempts=$((attempts + 1)) if [ "$attempts" -gt "$max_attempts" ]; then @@ -228,11 +236,11 @@ while true; do done if [ "$all_clear" = true ]; then - echo " '$phrase' is not present in any container logs after $attempts attempts." + echo " '$phrase' is not present in any container logs" break fi - sleep 3 # Wait for 3 seconds before checking again + sleep 5 # Wait for 3 seconds before checking again done log_ok bootstrap_progress From eac3c7959f58841acb7c13ff7af8b7fdf58fc927 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 14:52:44 +0200 Subject: [PATCH 32/63] resolve issue with keys/certificates not loading correctly --- deployments/tyk/bootstrap.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index c95afe2e..5e9a8d01 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -93,6 +93,7 @@ rm deployments/tyk/volumes/tyk-gateway/certs/*.pem > /dev/null 2>&1 log_ok bootstrap_progress +# sleeps have been added after the cert commands in this section to prevent race conditions log_message "Generating self-signed certificate for TLS connections to tyk-gateway-2.localhost" docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl req -x509 -newkey rsa:4096 -subj \"/CN=tyk-gateway-2.localhost\" -keyout /tmp/tls-private-key.pem -out /tmp/tls-certificate.pem -days 365 -nodes" >>logs/bootstrap.log if [ "$?" -ne "0" ]; then @@ -204,9 +205,7 @@ bootstrap_progress sleep 1 log_message "Recreating containers to load new certificates" -sleep 2 eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 -# attempt hot reloads to test payloads log_ok echo "Validating that secure messaging is functioning on gateway containers" @@ -215,10 +214,11 @@ attempts=0 max_attempts=3 phrase="Payload signature is invalid!" while true; do + # attempt hot reloads to test payload delivery process hot_reload "http://tyk-gateway.localhost:8080" "28d220fd77974a4facfb07dc1e49c2aa" hot_reload "https://tyk-gateway-2.localhost:8081" "28d220fd77974a4facfb07dc1e49c2aa" # pause to allow logs to capture any payload signature errors caused by hot reload command - sleep 5 + sleep 2 attempts=$((attempts + 1)) if [ "$attempts" -gt "$max_attempts" ]; then @@ -239,8 +239,6 @@ while true; do echo " '$phrase' is not present in any container logs" break fi - - sleep 5 # Wait for 3 seconds before checking again done log_ok bootstrap_progress From 7c3787408ec11f6c8833cd4317c3954527dc74d5 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 14:54:22 +0200 Subject: [PATCH 33/63] send normal messages to log file --- deployments/tyk/bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 5e9a8d01..d5953dc6 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -208,7 +208,7 @@ log_message "Recreating containers to load new certificates" eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 log_ok -echo "Validating that secure messaging is functioning on gateway containers" +log_message "Validating that secure messaging is functioning on gateway containers" gateway_service_names=("tyk-gateway" "tyk-gateway-2") attempts=0 max_attempts=3 @@ -229,14 +229,14 @@ while true; do all_clear=true for gateway_service in "${gateway_service_names[@]}"; do if $(generate_docker_compose_command) logs "$gateway_service" 2>&1 | grep -q "$phrase"; then - echo " Attempt $attempts: '$phrase' detected in the logs of service '$gateway_service' - recreating" + log_message " Attempt $attempts: '$phrase' detected in the logs of service '$gateway_service' - recreating" $(generate_docker_compose_command) up -d --no-deps --force-recreate $gateway_service all_clear=false fi done if [ "$all_clear" = true ]; then - echo " '$phrase' is not present in any container logs" + log_message " '$phrase' is not present in any container logs" break fi done From 2df7d906808e4e6ca278b4c579c69f762d6866f9 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 14:58:17 +0200 Subject: [PATCH 34/63] add comments --- deployments/tyk/bootstrap.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index d5953dc6..1a04415f 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -228,10 +228,14 @@ while true; do all_clear=true for gateway_service in "${gateway_service_names[@]}"; do + # check service log for presence of error message if $(generate_docker_compose_command) logs "$gateway_service" 2>&1 | grep -q "$phrase"; then log_message " Attempt $attempts: '$phrase' detected in the logs of service '$gateway_service' - recreating" + # payload issue is resolved by recreating the service $(generate_docker_compose_command) up -d --no-deps --force-recreate $gateway_service all_clear=false + # allow restarted service to initialise + sleep 2 fi done From cf9a2d39d668f0e4b3bb6cfa27a574846ff0dcd5 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 15:15:08 +0200 Subject: [PATCH 35/63] test sleep --- deployments/tyk/bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 1a04415f..a00aeb7a 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -204,6 +204,7 @@ log_ok bootstrap_progress sleep 1 +sleep 5 log_message "Recreating containers to load new certificates" eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 log_ok From 879b2af1fc86036b28c8fe95032b459a056ab49b Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 31 Jul 2024 17:52:56 +0200 Subject: [PATCH 36/63] wip --- deployments/tyk/bootstrap.sh | 61 ++++++++++-------------------------- scripts/common.sh | 27 ++++++++++++++++ 2 files changed, 43 insertions(+), 45 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index a00aeb7a..92837025 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -100,19 +100,9 @@ if [ "$?" -ne "0" ]; then echo "ERROR: Could not generate self-signed certificate" exit 1 fi -while true; do - docker exec $OPENSSL_CONTAINER_NAME sh -c "[ -s /tmp/tls-certificate.pem ]" - if [ $? -eq 0 ]; then - log_ok - bootstrap_progress - break; - else - log_message " Waiting for /tmp/tls-certificate.pem to be ready" - bootstrap_progress - sleep 2 - fi -done -sleep 1 +log_ok +bootstrap_progress +wait_for_file "/tmp/tls-certificate.pem" "$OPENSSL_CONTAINER_NAME" log_message "Generating private key for secure messaging and signing" docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl genrsa -out /tmp/private-key.pem 2048" >>logs/bootstrap.log @@ -120,19 +110,9 @@ if [ "$?" -ne "0" ]; then echo "ERROR: Could not generate private key" exit 1 fi -while true; do - docker exec $OPENSSL_CONTAINER_NAME sh -c "[ -s /tmp/private-key.pem ]" - if [ $? -eq 0 ]; then - log_ok - bootstrap_progress - break; - else - log_message " Waiting for /tmp/private-key.pem to be ready" - bootstrap_progress - sleep 2 - fi -done -sleep 1 +log_ok +bootstrap_progress +wait_for_file "/tmp/private-key.pem" "$OPENSSL_CONTAINER_NAME" log_message "Generating public key for secure messaging and signing" docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl rsa -in /tmp/private-key.pem -pubout -out /tmp/public-key.pem" >>logs/bootstrap.log @@ -140,19 +120,9 @@ if [ "$?" -ne "0" ]; then echo "ERROR: Could not generate public key" exit 1 fi -while true; do - docker exec $OPENSSL_CONTAINER_NAME sh -c "[ -s /tmp/public-key.pem ]" - if [ $? -eq 0 ]; then - log_ok - bootstrap_progress - break; - else - log_message " Waiting for /tmp/public-key.pem to be ready" - bootstrap_progress - sleep 2 - fi -done -sleep 1 +log_ok +bootstrap_progress +wait_for_file "/tmp/public-key.pem" "$OPENSSL_CONTAINER_NAME" log_message "Copying private-key.pem to dashboard volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/private-key.pem deployments/tyk/volumes/tyk-dashboard/certs >>logs/bootstrap.log @@ -162,7 +132,7 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress -sleep 1 +wait_for_file "/opt/tyk-dashboard/certs/private-key.pem" "tyk-demo-tyk-dashboard-1" log_message "Copying public-key.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/public-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -172,7 +142,8 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress -sleep 1 +wait_for_file "/opt/tyk-gateway/certs/public-key.pem" "tyk-demo-tyk-gateway-1" +wait_for_file "/opt/tyk-gateway/certs/public-key.pem" "tyk-demo-tyk-gateway-2-1" log_message "Copying tls-certificate.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-certificate.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -182,7 +153,8 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress -sleep 1 +wait_for_file "/opt/tyk-gateway/certs/tls-certificate.pem" "tyk-demo-tyk-gateway-1" +wait_for_file "/opt/tyk-gateway/certs/tls-certificate.pem" "tyk-demo-tyk-gateway-2-1" log_message "Copying tls-private-key.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-private-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -192,7 +164,8 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress -sleep 1 +wait_for_file "/opt/tyk-gateway/certs/tls-private-key.pem" "tyk-demo-tyk-gateway-1" +wait_for_file "/opt/tyk-gateway/certs/tls-private-key.pem" "tyk-demo-tyk-gateway-2-1" log_message "Removing temporary OpenSSL container $OPENSSL_CONTAINER_NAME" docker rm -f $OPENSSL_CONTAINER_NAME @@ -202,9 +175,7 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress -sleep 1 -sleep 5 log_message "Recreating containers to load new certificates" eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 log_ok diff --git a/scripts/common.sh b/scripts/common.sh index 7b94750e..59452f14 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -159,6 +159,33 @@ wait_for_response () { done } +# TODO: make function here for this, and then check all certs etc exist in bootstrap +wait_for_file () { + local file_path="$1" + local container_name="$2" + local try_max=10 + local try_count=0 + log_message "Waiting for $file_path to be present in $container_name" + while true; do + ((try_count++)) + if [ "$try_count" -gt "$try_max" ]; then + echo "ERROR: Maximum retry count reached for file $file_path in container $container_name" + exit 1 + fi + + docker exec $container_name sh -c "[ -s $file_path ]" + if [ $? -eq 0 ]; then + log_ok + bootstrap_progress + return 0 + else + log_message " File not present, waiting... $try_count/$try_max" + bootstrap_progress + sleep 2 + fi + done +} + hot_reload () { gateway_host="$1" gateway_secret="$2" From fd1904541c1554fa049a689487cbaea3712cfc34 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 12:14:36 +0200 Subject: [PATCH 37/63] udpate to rc images --- deployments/load-balancer-nginx/docker-compose.yml | 4 ++-- deployments/mdcb/docker-compose.yml | 2 +- deployments/sso/docker-compose.yml | 2 +- deployments/tyk/docker-compose.yml | 6 +++--- deployments/tyk2/docker-compose.yml | 4 ++-- scripts/add-gateway.sh | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/deployments/load-balancer-nginx/docker-compose.yml b/deployments/load-balancer-nginx/docker-compose.yml index 47af0ae0..1d5c584b 100755 --- a/deployments/load-balancer-nginx/docker-compose.yml +++ b/deployments/load-balancer-nginx/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-gateway-3: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0-alpha2} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0-rc2} networks: - tyk environment: @@ -28,7 +28,7 @@ services: - tyk-redis - tyk-dashboard tyk-gateway-4: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0-alpha2} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0-rc2} networks: - tyk environment: diff --git a/deployments/mdcb/docker-compose.yml b/deployments/mdcb/docker-compose.yml index 4e2ca5f6..04a008b3 100644 --- a/deployments/mdcb/docker-compose.yml +++ b/deployments/mdcb/docker-compose.yml @@ -15,7 +15,7 @@ services: - tyk-redis - tyk-mongo tyk-worker-gateway: - image: tykio/tyk-gateway:${GATEWAY_WORKER_VERSION:-v5.5.0-alpha2} + image: tykio/tyk-gateway:${GATEWAY_WORKER_VERSION:-v5.5.0-rc2} ports: - 8090:8080 networks: diff --git a/deployments/sso/docker-compose.yml b/deployments/sso/docker-compose.yml index 0bd19c8d..d4965dcd 100644 --- a/deployments/sso/docker-compose.yml +++ b/deployments/sso/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-dashboard-sso: - image: tykio/tyk-dashboard:${DASHBOARD_SSO_VERSION:-s5.5.0-alpha1} + image: tykio/tyk-dashboard:${DASHBOARD_SSO_VERSION:-v5.5.0-rc1} ports: - 3001:3000 networks: diff --git a/deployments/tyk/docker-compose.yml b/deployments/tyk/docker-compose.yml index 57578b24..2417f6f6 100755 --- a/deployments/tyk/docker-compose.yml +++ b/deployments/tyk/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-s5.5.0-alpha1} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.5.0-rc1} ports: - 3000:3000 networks: @@ -22,7 +22,7 @@ services: - tyk-redis - tyk-mongo tyk-gateway: - image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0-alpha2} + image: tykio/tyk-gateway:${GATEWAY_VERSION:-v5.5.0-rc2} ports: - 8080:8080 - 8086:8086 @@ -55,7 +55,7 @@ services: - tyk-redis - tyk-dashboard tyk-gateway-2: - image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.5.0-alpha2} + image: tykio/tyk-gateway:${GATEWAY2_VERSION:-v5.5.0-rc2} ports: - 8081:8080 networks: diff --git a/deployments/tyk2/docker-compose.yml b/deployments/tyk2/docker-compose.yml index 159ebf3a..f5993065 100644 --- a/deployments/tyk2/docker-compose.yml +++ b/deployments/tyk2/docker-compose.yml @@ -1,7 +1,7 @@ --- services: tyk2-dashboard: - image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-s5.5.0-alpha1} + image: tykio/tyk-dashboard:${DASHBOARD_VERSION:-v5.5.0-rc1} ports: - 3002:3000 networks: @@ -20,7 +20,7 @@ services: - tyk2-redis - tyk2-mongo tyk2-gateway: - image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.5.0-alpha2} + image: tykio/tyk-gateway:${TYK2_GATEWAY_VERSION:-v5.5.0-rc2} ports: - 8085:8080 networks: diff --git a/scripts/add-gateway.sh b/scripts/add-gateway.sh index a734934b..e3417604 100755 --- a/scripts/add-gateway.sh +++ b/scripts/add-gateway.sh @@ -14,7 +14,7 @@ if [ "$1" == "" ]; then -v $(pwd)/deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json \ -v $(pwd)/deployments/tyk/volumes/databases/GeoLite2-Country.mmdb:/opt/tyk-gateway/databases/GeoLite2-Country.mmdb \ --network tyk-demo_tyk \ - tykio/tyk-gateway:v5.5.0-alpha2 + tykio/tyk-gateway:v5.5.0-rc2 else docker run \ --name $1 \ @@ -27,5 +27,5 @@ else -v $(pwd)/deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json \ -v $(pwd)/deployments/tyk/volumes/databases/GeoLite2-Country.mmdb:/opt/tyk-gateway/databases/GeoLite2-Country.mmdb \ --network tyk-demo_tyk \ - tykio/tyk-gateway:v5.5.0-alpha2 + tykio/tyk-gateway:v5.5.0-rc2 fi From a11e5fb3750a10cade4d7839a9035ec1674a4f43 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 12:15:03 +0200 Subject: [PATCH 38/63] wip debug cert issue --- deployments/tyk/bootstrap.sh | 21 ++++++++++++++------- scripts/common.sh | 24 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 92837025..8a4711d5 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -132,7 +132,9 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress -wait_for_file "/opt/tyk-dashboard/certs/private-key.pem" "tyk-demo-tyk-dashboard-1" +wait_for_file_local "deployments/tyk/volumes/tyk-dashboard/certs/private-key.pem" +# sleep 2 +# wait_for_file "/opt/tyk-dashboard/certs/private-key.pem" "tyk-demo-tyk-dashboard-1" log_message "Copying public-key.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/public-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -142,8 +144,10 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress -wait_for_file "/opt/tyk-gateway/certs/public-key.pem" "tyk-demo-tyk-gateway-1" -wait_for_file "/opt/tyk-gateway/certs/public-key.pem" "tyk-demo-tyk-gateway-2-1" +wait_for_file_local "deployments/tyk/volumes/tyk-gateway/certs/public-key.pem" +# sleep 2 +# wait_for_file "/opt/tyk-gateway/certs/public-key.pem" "tyk-demo-tyk-gateway-1" +# wait_for_file "/opt/tyk-gateway/certs/public-key.pem" "tyk-demo-tyk-gateway-2-1" log_message "Copying tls-certificate.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-certificate.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -153,8 +157,7 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress -wait_for_file "/opt/tyk-gateway/certs/tls-certificate.pem" "tyk-demo-tyk-gateway-1" -wait_for_file "/opt/tyk-gateway/certs/tls-certificate.pem" "tyk-demo-tyk-gateway-2-1" +wait_for_file_local "deployments/tyk/volumes/tyk-gateway/certs/tls-certificate.pem" log_message "Copying tls-private-key.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-private-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -164,10 +167,14 @@ if [ "$?" != "0" ]; then fi log_ok bootstrap_progress -wait_for_file "/opt/tyk-gateway/certs/tls-private-key.pem" "tyk-demo-tyk-gateway-1" -wait_for_file "/opt/tyk-gateway/certs/tls-private-key.pem" "tyk-demo-tyk-gateway-2-1" +wait_for_file_local "deployments/tyk/volumes/tyk-gateway/certs/tls-private-key.pem" +# sleep 2 +# wait_for_file "/opt/tyk-gateway/certs/tls-private-key.pem" "tyk-demo-tyk-gateway-1" +# wait_for_file "/opt/tyk-gateway/certs/tls-private-key.pem" "tyk-demo-tyk-gateway-2-1" log_message "Removing temporary OpenSSL container $OPENSSL_CONTAINER_NAME" +# pause before removing container, to allow file copies to complete +sleep 2 docker rm -f $OPENSSL_CONTAINER_NAME if [ "$?" != "0" ]; then echo "ERROR: Could not remove temporary OpenSSL container $OPENSSL_CONTAINER_NAME" diff --git a/scripts/common.sh b/scripts/common.sh index 59452f14..46e50e18 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -186,6 +186,30 @@ wait_for_file () { done } +wait_for_file_local() { + local file_path="$1" + local try_max=10 + local try_count=0 + log_message "Waiting for $file_path to be present" + while true; do + ((try_count++)) + if [ "$try_count" -gt "$try_max" ]; then + echo "ERROR: Maximum retry count reached for file $file_path" + exit 1 + fi + + if [ -s $file_path ]; then + log_ok + bootstrap_progress + return 0 + else + log_message " File not present, waiting... $try_count/$try_max" + bootstrap_progress + sleep 2 + fi + done +} + hot_reload () { gateway_host="$1" gateway_secret="$2" From 9ef60c359d815fc4939626f39040083562f8ecca Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 12:15:15 +0200 Subject: [PATCH 39/63] remove unneeded sleep --- deployments/tyk/bootstrap.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 8a4711d5..b0aa11c4 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -173,8 +173,6 @@ wait_for_file_local "deployments/tyk/volumes/tyk-gateway/certs/tls-private-key.p # wait_for_file "/opt/tyk-gateway/certs/tls-private-key.pem" "tyk-demo-tyk-gateway-2-1" log_message "Removing temporary OpenSSL container $OPENSSL_CONTAINER_NAME" -# pause before removing container, to allow file copies to complete -sleep 2 docker rm -f $OPENSSL_CONTAINER_NAME if [ "$?" != "0" ]; then echo "ERROR: Could not remove temporary OpenSSL container $OPENSSL_CONTAINER_NAME" From 5c88f22f8bacfeb18bafda1c8a14baa0cf26dbee Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 12:34:19 +0200 Subject: [PATCH 40/63] remove trailing spaces --- deployments/tyk/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployments/tyk/docker-compose.yml b/deployments/tyk/docker-compose.yml index 2417f6f6..1b581e81 100755 --- a/deployments/tyk/docker-compose.yml +++ b/deployments/tyk/docker-compose.yml @@ -167,7 +167,7 @@ services: /opt/kafka/bin/kafka-storage.sh format -t $${CLUSTER_ID} -c /opt/kafka/config/kraft/server.properties && /opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/kraft/server.properties " - + volumes: tyk-redis-data: tyk-mongo-data: From a5bfe7fe15b45a54b490b8d5df36e112ed8a236b Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 13:41:16 +0200 Subject: [PATCH 41/63] restart and pause before validating logs --- deployments/tyk/bootstrap.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index b0aa11c4..6c3a369c 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -185,7 +185,12 @@ log_message "Recreating containers to load new certificates" eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 log_ok +log_message "Wait for services to be available after restart" +wait_for_liveness + log_message "Validating that secure messaging is functioning on gateway containers" +# pause before checking logs +sleep 2 gateway_service_names=("tyk-gateway" "tyk-gateway-2") attempts=0 max_attempts=3 @@ -224,8 +229,7 @@ done log_ok bootstrap_progress -log_message "Wait for services to be available after restart" -wait_for_liveness + # Kafka From 349f0a4516064820ed1a2f52e085f409e52dd54b Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 14:05:42 +0200 Subject: [PATCH 42/63] debug wip --- deployments/tyk/bootstrap.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 6c3a369c..e342be12 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -188,6 +188,15 @@ log_ok log_message "Wait for services to be available after restart" wait_for_liveness + + +sleep 3 +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard +sleep 3 +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 +sleep 3 + + log_message "Validating that secure messaging is functioning on gateway containers" # pause before checking logs sleep 2 @@ -197,10 +206,10 @@ max_attempts=3 phrase="Payload signature is invalid!" while true; do # attempt hot reloads to test payload delivery process - hot_reload "http://tyk-gateway.localhost:8080" "28d220fd77974a4facfb07dc1e49c2aa" - hot_reload "https://tyk-gateway-2.localhost:8081" "28d220fd77974a4facfb07dc1e49c2aa" + # hot_reload "http://tyk-gateway.localhost:8080" "28d220fd77974a4facfb07dc1e49c2aa" + # hot_reload "https://tyk-gateway-2.localhost:8081" "28d220fd77974a4facfb07dc1e49c2aa" # pause to allow logs to capture any payload signature errors caused by hot reload command - sleep 2 + # sleep 2 attempts=$((attempts + 1)) if [ "$attempts" -gt "$max_attempts" ]; then From 4bc9ad5e943ecac6f9bf1396f44842599e7ccf0b Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 14:23:52 +0200 Subject: [PATCH 43/63] debug print certs --- deployments/tyk/bootstrap.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index e342be12..635da73c 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -133,8 +133,8 @@ fi log_ok bootstrap_progress wait_for_file_local "deployments/tyk/volumes/tyk-dashboard/certs/private-key.pem" -# sleep 2 -# wait_for_file "/opt/tyk-dashboard/certs/private-key.pem" "tyk-demo-tyk-dashboard-1" +echo "dash private:" +cat deployments/tyk/volumes/tyk-dashboard/certs/private-key.pem log_message "Copying public-key.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/public-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -145,9 +145,8 @@ fi log_ok bootstrap_progress wait_for_file_local "deployments/tyk/volumes/tyk-gateway/certs/public-key.pem" -# sleep 2 -# wait_for_file "/opt/tyk-gateway/certs/public-key.pem" "tyk-demo-tyk-gateway-1" -# wait_for_file "/opt/tyk-gateway/certs/public-key.pem" "tyk-demo-tyk-gateway-2-1" +echo "gateway public:" +cat deployments/tyk/volumes/tyk-gateway/certs/public-key.pem log_message "Copying tls-certificate.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-certificate.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -158,6 +157,8 @@ fi log_ok bootstrap_progress wait_for_file_local "deployments/tyk/volumes/tyk-gateway/certs/tls-certificate.pem" +echo "gateway tls:" +cat deployments/tyk/volumes/tyk-gateway/certs/tls-certificate.pem log_message "Copying tls-private-key.pem to gateway volume mount" docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-private-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log @@ -168,9 +169,8 @@ fi log_ok bootstrap_progress wait_for_file_local "deployments/tyk/volumes/tyk-gateway/certs/tls-private-key.pem" -# sleep 2 -# wait_for_file "/opt/tyk-gateway/certs/tls-private-key.pem" "tyk-demo-tyk-gateway-1" -# wait_for_file "/opt/tyk-gateway/certs/tls-private-key.pem" "tyk-demo-tyk-gateway-2-1" +echo "gateway tls private:" +cat deployments/tyk/volumes/tyk-gateway/certs/tls-private-key.pem log_message "Removing temporary OpenSSL container $OPENSSL_CONTAINER_NAME" docker rm -f $OPENSSL_CONTAINER_NAME @@ -190,11 +190,11 @@ wait_for_liveness -sleep 3 -eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard -sleep 3 -eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 -sleep 3 +# sleep 3 +# eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard +# sleep 3 +# eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 +# sleep 3 log_message "Validating that secure messaging is functioning on gateway containers" From 28c7b9c2c1fc77d999ea783889a8783cc4015c0e Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 14:36:19 +0200 Subject: [PATCH 44/63] debug sleep --- deployments/tyk/bootstrap.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 635da73c..75168169 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -182,7 +182,11 @@ log_ok bootstrap_progress log_message "Recreating containers to load new certificates" -eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard tyk-gateway tyk-gateway-2 +sleep 3 +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard +sleep 3 +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 +sleep 3 log_ok log_message "Wait for services to be available after restart" From cf16d11fe1c0a737ab68e9383a49efd203186afc Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 17:36:30 +0200 Subject: [PATCH 45/63] remove unneeded certs dir --- .../tyk/volumes/tyk-dashboard/certs/certificates-go-here | 2 -- deployments/tyk/volumes/tyk-gateway/certs/certificates-go-here | 2 -- 2 files changed, 4 deletions(-) delete mode 100644 deployments/tyk/volumes/tyk-dashboard/certs/certificates-go-here delete mode 100644 deployments/tyk/volumes/tyk-gateway/certs/certificates-go-here diff --git a/deployments/tyk/volumes/tyk-dashboard/certs/certificates-go-here b/deployments/tyk/volumes/tyk-dashboard/certs/certificates-go-here deleted file mode 100644 index 0adc19d6..00000000 --- a/deployments/tyk/volumes/tyk-dashboard/certs/certificates-go-here +++ /dev/null @@ -1,2 +0,0 @@ -this directory is mapped to the container -certs generated by the bootstrap script are stored here \ No newline at end of file diff --git a/deployments/tyk/volumes/tyk-gateway/certs/certificates-go-here b/deployments/tyk/volumes/tyk-gateway/certs/certificates-go-here deleted file mode 100644 index 0adc19d6..00000000 --- a/deployments/tyk/volumes/tyk-gateway/certs/certificates-go-here +++ /dev/null @@ -1,2 +0,0 @@ -this directory is mapped to the container -certs generated by the bootstrap script are stored here \ No newline at end of file From 3cf7dc0da968a05398a8ef91bbd357701ba34589 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 17:36:47 +0200 Subject: [PATCH 46/63] change certs to use shared volume --- deployments/sso/docker-compose.yml | 2 +- deployments/tyk/docker-compose.yml | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/deployments/sso/docker-compose.yml b/deployments/sso/docker-compose.yml index d4965dcd..d3a505b6 100644 --- a/deployments/sso/docker-compose.yml +++ b/deployments/sso/docker-compose.yml @@ -8,7 +8,7 @@ services: - tyk volumes: - ./deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf:/opt/tyk-dashboard/tyk_analytics.conf - - ./deployments/tyk/volumes/tyk-dashboard/private-key.pem:/opt/tyk-dashboard/private-key.pem + - ./deployments/tyk/volumes/tyk-dashboard/certs/private-key.pem:/opt/tyk-dashboard/certs/private-key.pem environment: - TYK_DB_LICENSEKEY=${DASHBOARD_LICENCE:?Please set DASHBOARD_LICENCE in .env} - TYK_DB_SSOCUSTOMLOGINURL=http://localhost:3010/auth/tyk-dashboard/openid-connect diff --git a/deployments/tyk/docker-compose.yml b/deployments/tyk/docker-compose.yml index 1b581e81..f2cbf6b2 100755 --- a/deployments/tyk/docker-compose.yml +++ b/deployments/tyk/docker-compose.yml @@ -8,7 +8,7 @@ services: - tyk volumes: - ./deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf:/opt/tyk-dashboard/tyk_analytics.conf - - ./deployments/tyk/volumes/tyk-dashboard/certs:/opt/tyk-dashboard/certs + - tyk-dashboard-certs:/opt/tyk-dashboard/certs - ./deployments/tyk/volumes/tyk-dashboard/catalogue.html:/opt/tyk-dashboard/portal/templates/catalogue.html - ./deployments/tyk/volumes/tyk-dashboard/navigation.html:/opt/tyk-dashboard/portal/templates/navigation.html - ./deployments/tyk/volumes/tyk-dashboard/audit:/opt/tyk-dashboard/audit @@ -46,7 +46,7 @@ services: - .env volumes: - ./deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware - ./deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins - ./deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json @@ -68,7 +68,7 @@ services: - .env volumes: - ./deployments/tyk/volumes/tyk-gateway/tyk-2.conf:/opt/tyk-gateway/tyk.conf - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware depends_on: - tyk-redis @@ -172,6 +172,8 @@ volumes: tyk-redis-data: tyk-mongo-data: tyk-kafka-data: + tyk-gateway-certs: + tyk-dashboard-certs: networks: tyk: From 0071307e2dfa1545398c1e7c445ea3ede6febf40 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 17:44:07 +0200 Subject: [PATCH 47/63] refactor cert generation to use shared volume mapping --- deployments/tyk/bootstrap.sh | 138 +++++++---------------------------- 1 file changed, 26 insertions(+), 112 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 75168169..80c98bcd 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -67,16 +67,19 @@ fi bootstrap_progress log_message "Creating temporary container $OPENSSL_CONTAINER_NAME for OpenSSL usage" -docker run -d --name $OPENSSL_CONTAINER_NAME alpine:3.20.1 tail -f /dev/null > /dev/null 2>&1 +docker run -d --name $OPENSSL_CONTAINER_NAME \ + -v tyk-demo_tyk-gateway-certs:/tyk-gateway-certs \ + -v tyk-demo_tyk-dashboard-certs:/tyk-dashboard-certs \ + alpine:3.20.1 tail -f /dev/null >/dev/null 2>&1 log_ok bootstrap_progress log_message "Install OpenSSL into container $OPENSSL_CONTAINER_NAME" -docker exec -d $OPENSSL_CONTAINER_NAME apk add --no-cache openssl +docker exec $OPENSSL_CONTAINER_NAME apk add --no-cache openssl >/dev/null 2>>logs/bootstrap.log # Wait for the installation to complete while true; do # Check if OpenSSL is installed by trying to get its version - if docker exec $OPENSSL_CONTAINER_NAME openssl version > /dev/null 2>&1; then + if docker exec $OPENSSL_CONTAINER_NAME openssl version >/dev/null 2>&1; then log_message " OpenSSL has been successfully installed" break else @@ -87,93 +90,53 @@ done log_message "OpenSSL version used for generating certs: $(docker exec $OPENSSL_CONTAINER_NAME openssl version)" -log_message "Removing any pre-existing certs" -rm deployments/tyk/volumes/tyk-dashboard/certs/*.pem > /dev/null 2>&1 -rm deployments/tyk/volumes/tyk-gateway/certs/*.pem > /dev/null 2>&1 -log_ok -bootstrap_progress - -# sleeps have been added after the cert commands in this section to prevent race conditions log_message "Generating self-signed certificate for TLS connections to tyk-gateway-2.localhost" -docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl req -x509 -newkey rsa:4096 -subj \"/CN=tyk-gateway-2.localhost\" -keyout /tmp/tls-private-key.pem -out /tmp/tls-certificate.pem -days 365 -nodes" >>logs/bootstrap.log +docker exec $OPENSSL_CONTAINER_NAME sh -c "openssl req -x509 -newkey rsa:4096 -subj \"/CN=tyk-gateway-2.localhost\" -keyout /tyk-gateway-certs/tls-private-key.pem -out /tyk-gateway-certs/tls-certificate.pem -days 365 -nodes" >/dev/null 2>&1 if [ "$?" -ne "0" ]; then echo "ERROR: Could not generate self-signed certificate" exit 1 fi log_ok bootstrap_progress -wait_for_file "/tmp/tls-certificate.pem" "$OPENSSL_CONTAINER_NAME" +wait_for_file "/tyk-gateway-certs/tls-certificate.pem" "$OPENSSL_CONTAINER_NAME" +wait_for_file "/tyk-gateway-certs/tls-private-key.pem" "$OPENSSL_CONTAINER_NAME" log_message "Generating private key for secure messaging and signing" -docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl genrsa -out /tmp/private-key.pem 2048" >>logs/bootstrap.log +docker exec $OPENSSL_CONTAINER_NAME sh -c "openssl genrsa -out /tyk-dashboard-certs/private-key.pem 2048" >/dev/null 2>>logs/bootstrap.log if [ "$?" -ne "0" ]; then echo "ERROR: Could not generate private key" exit 1 fi log_ok bootstrap_progress -wait_for_file "/tmp/private-key.pem" "$OPENSSL_CONTAINER_NAME" +wait_for_file "/tyk-dashboard-certs/private-key.pem" "$OPENSSL_CONTAINER_NAME" log_message "Generating public key for secure messaging and signing" -docker exec -d $OPENSSL_CONTAINER_NAME sh -c "openssl rsa -in /tmp/private-key.pem -pubout -out /tmp/public-key.pem" >>logs/bootstrap.log +docker exec $OPENSSL_CONTAINER_NAME sh -c "openssl rsa -in /tyk-dashboard-certs/private-key.pem -pubout -out /tyk-gateway-certs/public-key.pem" >/dev/null 2>>logs/bootstrap.log if [ "$?" -ne "0" ]; then echo "ERROR: Could not generate public key" exit 1 fi log_ok bootstrap_progress -wait_for_file "/tmp/public-key.pem" "$OPENSSL_CONTAINER_NAME" - -log_message "Copying private-key.pem to dashboard volume mount" -docker cp $OPENSSL_CONTAINER_NAME:/tmp/private-key.pem deployments/tyk/volumes/tyk-dashboard/certs >>logs/bootstrap.log -if [ "$?" != "0" ]; then - echo "ERROR: Could not copy private-key.pem to dashboard volume mount" - exit 1 -fi -log_ok -bootstrap_progress -wait_for_file_local "deployments/tyk/volumes/tyk-dashboard/certs/private-key.pem" -echo "dash private:" -cat deployments/tyk/volumes/tyk-dashboard/certs/private-key.pem - -log_message "Copying public-key.pem to gateway volume mount" -docker cp $OPENSSL_CONTAINER_NAME:/tmp/public-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log -if [ "$?" != "0" ]; then - echo "ERROR: Could not copy public-key.pem to gateway volume mount" - exit 1 -fi -log_ok -bootstrap_progress -wait_for_file_local "deployments/tyk/volumes/tyk-gateway/certs/public-key.pem" -echo "gateway public:" -cat deployments/tyk/volumes/tyk-gateway/certs/public-key.pem +wait_for_file "/tyk-gateway-certs/public-key.pem" "$OPENSSL_CONTAINER_NAME" -log_message "Copying tls-certificate.pem to gateway volume mount" -docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-certificate.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log +log_message "Setting read permissions on certificate volumes" +docker exec $OPENSSL_CONTAINER_NAME chmod -R a+r /tyk-gateway-certs >/dev/null 2>>logs/bootstrap.log if [ "$?" != "0" ]; then - echo "ERROR: Could not copy tls-certificate.pem to gateway volume mount" + echo "ERROR: Could not set read permissions on /tyk-gateway-certs volume" exit 1 fi -log_ok -bootstrap_progress -wait_for_file_local "deployments/tyk/volumes/tyk-gateway/certs/tls-certificate.pem" -echo "gateway tls:" -cat deployments/tyk/volumes/tyk-gateway/certs/tls-certificate.pem - -log_message "Copying tls-private-key.pem to gateway volume mount" -docker cp $OPENSSL_CONTAINER_NAME:/tmp/tls-private-key.pem deployments/tyk/volumes/tyk-gateway/certs >>logs/bootstrap.log +docker exec $OPENSSL_CONTAINER_NAME chmod -R a+r /tyk-dashboard-certs >/dev/null 2>>logs/bootstrap.log if [ "$?" != "0" ]; then - echo "ERROR: Could not copy tls-private-key.pem to gateway volume mount" + echo "ERROR: Could not set read permissions on /tyk-dashboard-certs volume" exit 1 fi log_ok bootstrap_progress -wait_for_file_local "deployments/tyk/volumes/tyk-gateway/certs/tls-private-key.pem" -echo "gateway tls private:" -cat deployments/tyk/volumes/tyk-gateway/certs/tls-private-key.pem log_message "Removing temporary OpenSSL container $OPENSSL_CONTAINER_NAME" -docker rm -f $OPENSSL_CONTAINER_NAME +docker rm -f $OPENSSL_CONTAINER_NAME >/dev/null 2>>logs/bootstrap.log if [ "$?" != "0" ]; then echo "ERROR: Could not remove temporary OpenSSL container $OPENSSL_CONTAINER_NAME" exit 1 @@ -182,72 +145,23 @@ log_ok bootstrap_progress log_message "Recreating containers to load new certificates" -sleep 3 eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard -sleep 3 eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 -sleep 3 log_ok log_message "Wait for services to be available after restart" wait_for_liveness - - -# sleep 3 -# eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-dashboard -# sleep 3 -# eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway tyk-gateway-2 -# sleep 3 - - -log_message "Validating that secure messaging is functioning on gateway containers" -# pause before checking logs -sleep 2 -gateway_service_names=("tyk-gateway" "tyk-gateway-2") -attempts=0 -max_attempts=3 -phrase="Payload signature is invalid!" -while true; do - # attempt hot reloads to test payload delivery process - # hot_reload "http://tyk-gateway.localhost:8080" "28d220fd77974a4facfb07dc1e49c2aa" - # hot_reload "https://tyk-gateway-2.localhost:8081" "28d220fd77974a4facfb07dc1e49c2aa" - # pause to allow logs to capture any payload signature errors caused by hot reload command - # sleep 2 - - attempts=$((attempts + 1)) - if [ "$attempts" -gt "$max_attempts" ]; then - echo "ERROR: Unable to clear '$phrase' from logs" - exit 1 - fi - - all_clear=true - for gateway_service in "${gateway_service_names[@]}"; do - # check service log for presence of error message - if $(generate_docker_compose_command) logs "$gateway_service" 2>&1 | grep -q "$phrase"; then - log_message " Attempt $attempts: '$phrase' detected in the logs of service '$gateway_service' - recreating" - # payload issue is resolved by recreating the service - $(generate_docker_compose_command) up -d --no-deps --force-recreate $gateway_service - all_clear=false - # allow restarted service to initialise - sleep 2 - fi - done - - if [ "$all_clear" = true ]; then - log_message " '$phrase' is not present in any container logs" - break - fi -done -log_ok -bootstrap_progress - - - # Kafka log_message "Creating Kafka topic" -docker exec tyk-demo-kafka-1 sh -c "/opt/kafka/bin/kafka-topics.sh --create --topic quickstart-events --bootstrap-server localhost:9092" +docker exec tyk-demo-kafka-1 sh -c "/opt/kafka/bin/kafka-topics.sh --create --topic quickstart-events --bootstrap-server localhost:9092" >/dev/null 2>>logs/bootstrap.log +if [ "$?" -ne "0" ]; then + echo "ERROR: Could not create kafka topic" + exit 1 +fi +log_ok +bootstrap_progress # Go plugins From bad81c44777a91c16aa77eac47a785f9c42e5a80 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 17:52:12 +0200 Subject: [PATCH 48/63] target specific newman release --- scripts/test-all.sh | 2 +- scripts/test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/test-all.sh b/scripts/test-all.sh index ef22aab9..ce9d04bc 100755 --- a/scripts/test-all.sh +++ b/scripts/test-all.sh @@ -111,7 +111,7 @@ do --network tyk-demo_tyk \ -v $(pwd)/$postman_collection_path:/etc/postman/tyk_demo.postman_collection.json \ -v $(pwd)/test.postman_environment.json:/etc/postman/test.postman_environment.json \ - postman/newman:alpine \ + postman/newman:6.1.3-alpine \ run "/etc/postman/tyk_demo.postman_collection.json" \ --environment /etc/postman/test.postman_environment.json \ --insecure \ diff --git a/scripts/test.sh b/scripts/test.sh index bb68200c..e41324ef 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -32,7 +32,7 @@ while IFS= read -r deployment; do --network tyk-demo_tyk \ -v $collection_path:/etc/postman/tyk_demo.postman_collection.json \ -v $(pwd)/test.postman_environment.json:/etc/postman/test.postman_environment.json \ - postman/newman:alpine \ + postman/newman:6.1.3-alpine \ run "/etc/postman/tyk_demo.postman_collection.json" \ --environment /etc/postman/test.postman_environment.json \ --insecure From 64438455518f4458b8f550d8407621333021c717 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 18:16:52 +0200 Subject: [PATCH 49/63] use new shared certs volume --- deployments/load-balancer-nginx/docker-compose.yml | 4 ++-- deployments/mdcb/docker-compose.yml | 2 +- deployments/sso/docker-compose.yml | 2 +- deployments/tyk2/docker-compose.yml | 4 ++-- scripts/add-gateway.sh | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/deployments/load-balancer-nginx/docker-compose.yml b/deployments/load-balancer-nginx/docker-compose.yml index 1d5c584b..3e88a321 100755 --- a/deployments/load-balancer-nginx/docker-compose.yml +++ b/deployments/load-balancer-nginx/docker-compose.yml @@ -19,7 +19,7 @@ services: - .env volumes: - ./deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware - ./deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins - ./deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json @@ -46,7 +46,7 @@ services: - .env volumes: - ./deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware - ./deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins - ./deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json diff --git a/deployments/mdcb/docker-compose.yml b/deployments/mdcb/docker-compose.yml index 04a008b3..a8a604b1 100644 --- a/deployments/mdcb/docker-compose.yml +++ b/deployments/mdcb/docker-compose.yml @@ -28,7 +28,7 @@ services: - TYK_GW_OPENTELEMETRY_ENDPOINT=${OPENTELEMETRY_ENDPOINT:-false} volumes: - ./deployments/mdcb/volumes/tyk-worker-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware - ./deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins depends_on: diff --git a/deployments/sso/docker-compose.yml b/deployments/sso/docker-compose.yml index d3a505b6..e2b08314 100644 --- a/deployments/sso/docker-compose.yml +++ b/deployments/sso/docker-compose.yml @@ -8,7 +8,7 @@ services: - tyk volumes: - ./deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf:/opt/tyk-dashboard/tyk_analytics.conf - - ./deployments/tyk/volumes/tyk-dashboard/certs/private-key.pem:/opt/tyk-dashboard/certs/private-key.pem + - tyk-dashboard-certs:/opt/tyk-dashboard/certs environment: - TYK_DB_LICENSEKEY=${DASHBOARD_LICENCE:?Please set DASHBOARD_LICENCE in .env} - TYK_DB_SSOCUSTOMLOGINURL=http://localhost:3010/auth/tyk-dashboard/openid-connect diff --git a/deployments/tyk2/docker-compose.yml b/deployments/tyk2/docker-compose.yml index f5993065..d47658f5 100644 --- a/deployments/tyk2/docker-compose.yml +++ b/deployments/tyk2/docker-compose.yml @@ -8,7 +8,7 @@ services: - tyk volumes: - ./deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf:/opt/tyk-dashboard/tyk_analytics.conf - - ./deployments/tyk/volumes/tyk-dashboard/certs:/opt/tyk-dashboard/certs + - tyk-dashboard-certs:/opt/tyk-dashboard/certs environment: - TYK_DB_LICENSEKEY=${DASHBOARD_LICENCE:?Please set DASHBOARD_LICENCE in .env} - TYK_DB_MONGOURL=mongodb://tyk2-mongo:27017/tyk_analytics @@ -28,7 +28,7 @@ services: volumes: - ./deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf - ./deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware - - ./deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs + - tyk-gateway-certs:/opt/tyk-gateway/certs environment: - TYK_GW_POLICIES_POLICYCONNECTIONSTRING=http://tyk2-dashboard:3000 - TYK_GW_DBAPPCONFOPTIONS_CONNECTIONSTRING=http://tyk2-dashboard:3000 diff --git a/scripts/add-gateway.sh b/scripts/add-gateway.sh index e3417604..029ab583 100755 --- a/scripts/add-gateway.sh +++ b/scripts/add-gateway.sh @@ -8,7 +8,7 @@ if [ "$1" == "" ]; then -d \ -P \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf \ - -v $(pwd)/deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs \ + -v tyk-demo_tyk-gateway-certs:/opt/tyk-gateway/certs \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json \ @@ -21,7 +21,7 @@ else -d \ -P \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf \ - -v $(pwd)/deployments/tyk/volumes/tyk-gateway/certs:/opt/tyk-gateway/certs \ + -v tyk-demo_tyk-gateway-certs:/opt/tyk-gateway/certs \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/middleware:/opt/tyk-gateway/middleware \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/plugins:/opt/tyk-gateway/plugins \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/templates/error_401.json:/opt/tyk-gateway/templates/error_401.json \ From 81ec23614795ee121eb4600e49d697934099f7c0 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 18:20:37 +0200 Subject: [PATCH 50/63] restart tyk2 services to load newly created certs --- deployments/tyk2/bootstrap.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deployments/tyk2/bootstrap.sh b/deployments/tyk2/bootstrap.sh index 32a3eddb..ed789552 100755 --- a/deployments/tyk2/bootstrap.sh +++ b/deployments/tyk2/bootstrap.sh @@ -8,6 +8,10 @@ bootstrap_progress dashboard_base_url="http://localhost:3002" gateway2_base_url="http://localhost:8085" +log_message "Restarting Tyk 2 services to use newly created certificates" +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk2-dashboard tyk2-gateway +log_ok + log_message "Waiting for Tyk 2 Dashboard to respond ok" dashboard_admin_api_credentials=$(cat deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf | jq -r .admin_secret) wait_for_response "$dashboard_base_url/admin/organisations" "200" "admin-auth: $dashboard_admin_api_credentials" From 1716f8e644ee424ab3bc16fe8dcdf4f976d337da Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 18:38:09 +0200 Subject: [PATCH 51/63] map exposed port --- scripts/add-gateway.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/add-gateway.sh b/scripts/add-gateway.sh index 029ab583..58b7690d 100755 --- a/scripts/add-gateway.sh +++ b/scripts/add-gateway.sh @@ -6,6 +6,7 @@ if [ "$1" == "" ]; then docker run \ -d \ + --expose 8080 \ -P \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf \ -v tyk-demo_tyk-gateway-certs:/opt/tyk-gateway/certs \ @@ -19,6 +20,7 @@ else docker run \ --name $1 \ -d \ + --expose 8080 \ -P \ -v $(pwd)/deployments/tyk/volumes/tyk-gateway/tyk.conf:/opt/tyk-gateway/tyk.conf \ -v tyk-demo_tyk-gateway-certs:/opt/tyk-gateway/certs \ From 3c12db9fc9503bf71ccf64bb87b9dec0109d54cf Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 18:44:54 +0200 Subject: [PATCH 52/63] restart gateways to load certs --- deployments/load-balancer-nginx/bootstrap.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/deployments/load-balancer-nginx/bootstrap.sh b/deployments/load-balancer-nginx/bootstrap.sh index 943ae946..607f5dd5 100755 --- a/deployments/load-balancer-nginx/bootstrap.sh +++ b/deployments/load-balancer-nginx/bootstrap.sh @@ -6,13 +6,13 @@ deployment="Load Balancer" log_start_deployment bootstrap_progress -# log_message "Restart Gateways to load latest certificates" -# docker restart tyk-demo-tyk-gateway-3-1 tyk-demo-tyk-gateway-4-1 1>/dev/null 2>>logs/bootstrap.log -# if [ "$?" != 0 ]; then -# echo "Error when restart Gateways to load latest certificates" -# exit 1 -# fi -# log_ok +log_message "Restart Gateways to load latest certificates" +eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk-gateway-3 tyk-gateway-4 1>/dev/null 2>>logs/bootstrap.log +if [ "$?" != 0 ]; then + echo "Error when restart Gateways to load latest certificates" + exit 1 +fi +log_ok log_message "Restart nginx to reset load balancer" docker restart tyk-demo-nginx-1 1>/dev/null 2>>logs/bootstrap.log From 1a61d343b69004031d583a3f0fa1479bfaf98acb Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 21:00:46 +0200 Subject: [PATCH 53/63] resolve postman decprecation message --- deployments/tyk/tyk_demo_tyk.postman_collection.json | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/deployments/tyk/tyk_demo_tyk.postman_collection.json b/deployments/tyk/tyk_demo_tyk.postman_collection.json index 506adf15..720eb1e0 100644 --- a/deployments/tyk/tyk_demo_tyk.postman_collection.json +++ b/deployments/tyk/tyk_demo_tyk.postman_collection.json @@ -4085,7 +4085,7 @@ "get" ] }, - "description": "This example is a placeholder for the real dynamic client MTLS example, which uses a bash script and `curl` to demonstrate. Run the script from the Tyk Demo repo root as so:\n\n``` bash\n./deployments/tyk/scripts/examples/dynamic-client-mtls.sh\n\n ```\n\nThe `curl` request in the script references the certificate and private key, which the gateway then uses the identify the bearer token (used in the `Authorization` header here). The authorization decision is then based on the access rights defined in the bearer token. The client only needs to provide the certificate, rather than the `Authorization` header, as this request shows." + "description": "This example is just a placeholder for the real dynamic client MTLS example - it is not an example of dynamic client TLS, but it does reference the token , which uses a bash script and `curl` to demonstrate. Run the script from the Tyk Demo repo root as so:\n\n``` bash\n./deployments/tyk/scripts/examples/dynamic-client-mtls.sh\n\n ```\n\nThe `curl` request in the script references the certificate and private key, which the gateway then uses the identify the bearer token (used in the `Authorization` header here). The authorization decision is then based on the access rights defined in the bearer token. The client only needs to provide the certificate, rather than the `Authorization` header, as this request shows." }, "response": [] } @@ -8687,7 +8687,7 @@ "listen": "test", "script": { "exec": [ - "var requestBody = JSON.parse(request.data);", + "var requestBody = JSON.parse(pm.request.body);", "", "pm.test(\"Status code is 200\", function () {", " pm.response.to.have.status(200);", @@ -8724,7 +8724,8 @@ " pm", ");" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } }, { @@ -8738,7 +8739,8 @@ "pm.variables.set(\"api-id\", \"fa0b0268b4124faa70e151a879d3e92c\")", "pm.variables.set(\"policy-id\", \"615d2e528bf3980001c7c6c2\")" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], From 64f6ccfe970c2f1603bb2e78a0274be8630f7385 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 21:05:51 +0200 Subject: [PATCH 54/63] resolve postman deprecation issue --- deployments/tyk/tyk_demo_tyk.postman_collection.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/deployments/tyk/tyk_demo_tyk.postman_collection.json b/deployments/tyk/tyk_demo_tyk.postman_collection.json index 720eb1e0..50d72810 100644 --- a/deployments/tyk/tyk_demo_tyk.postman_collection.json +++ b/deployments/tyk/tyk_demo_tyk.postman_collection.json @@ -6665,10 +6665,11 @@ "if (pm.response.code != 200) {", " console.warn(\"Unable to get Ngrok data, skipping geolocation tests\")", " // if we can't get the Ngrok data then we need to skip the geolocation request, and move to the 'safe' post-request placeholder instead", - " postman.setNextRequest(\"Geolocation - post-request placeholder\")", + " pm.execution.setNextRequest(\"Geolocation - post-request placeholder\")", "}" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], From 0ac5152277926029588ef84b05bf555aedd02862 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 21:24:42 +0200 Subject: [PATCH 55/63] make status check func able to take URL param --- scripts/common.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/scripts/common.sh b/scripts/common.sh index 46e50e18..9e551881 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -980,21 +980,20 @@ wait_for_api_loaded () { } wait_for_liveness () { - - attempt_count=0 - pass="pass" + local status_endpoint="${1:-http:\/\/tyk-gateway.localhost:8080/hello}" + local attempt_count=0 + local pass="pass" log_message "Waiting for Gateway, Dashboard and Redis to be up and running" - while true - do + while true; do attempt_count=$((attempt_count+1)) #Check Gateway, Redis and Dashboard status - local hello=$(curl http://tyk-gateway.localhost:8080/hello -s) - local gw_status=$(echo "$hello" | jq -r '.status') - local dash_status=$(echo "$hello" | jq -r '.details.dashboard.status') - local redis_status=$(echo "$hello" | jq -r '.details.redis.status') + local status_response=$(curl $status_endpoint -s) + local gw_status=$(echo "$status_response" | jq -r '.status') + local dash_status=$(echo "$status_response" | jq -r '.details.dashboard.status') + local redis_status=$(echo "$status_response" | jq -r '.details.redis.status') if [[ "$gw_status" = "pass" ]] && [[ "$dash_status" = "pass" ]] && [[ "$redis_status" = "pass" ]] then @@ -1005,7 +1004,6 @@ wait_for_liveness () { fi sleep 2 - done } From c052451811fe12b4fa4f8e06c93f2e72c3c3ece3 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 21:24:53 +0200 Subject: [PATCH 56/63] use status check function --- deployments/tyk2/bootstrap.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/tyk2/bootstrap.sh b/deployments/tyk2/bootstrap.sh index ed789552..0f5be784 100755 --- a/deployments/tyk2/bootstrap.sh +++ b/deployments/tyk2/bootstrap.sh @@ -12,9 +12,9 @@ log_message "Restarting Tyk 2 services to use newly created certificates" eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk2-dashboard tyk2-gateway log_ok -log_message "Waiting for Tyk 2 Dashboard to respond ok" +wait_for_liveness "http:\/\/localhost:8085/hello" + dashboard_admin_api_credentials=$(cat deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf | jq -r .admin_secret) -wait_for_response "$dashboard_base_url/admin/organisations" "200" "admin-auth: $dashboard_admin_api_credentials" log_message "Importing organisation" log_json_result "$(curl $dashboard_base_url/admin/organisations/import -s \ From 6795a7808fe6119437afb2ee097615c19db7ee0a Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 5 Aug 2024 21:29:08 +0200 Subject: [PATCH 57/63] fix status url override logic --- deployments/tyk2/bootstrap.sh | 2 +- scripts/common.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployments/tyk2/bootstrap.sh b/deployments/tyk2/bootstrap.sh index 0f5be784..a41bcdb7 100755 --- a/deployments/tyk2/bootstrap.sh +++ b/deployments/tyk2/bootstrap.sh @@ -12,7 +12,7 @@ log_message "Restarting Tyk 2 services to use newly created certificates" eval $(generate_docker_compose_command) up -d --no-deps --force-recreate tyk2-dashboard tyk2-gateway log_ok -wait_for_liveness "http:\/\/localhost:8085/hello" +wait_for_liveness "http://localhost:8085/hello" dashboard_admin_api_credentials=$(cat deployments/tyk/volumes/tyk-dashboard/tyk_analytics.conf | jq -r .admin_secret) diff --git a/scripts/common.sh b/scripts/common.sh index 9e551881..1293df5f 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -980,7 +980,7 @@ wait_for_api_loaded () { } wait_for_liveness () { - local status_endpoint="${1:-http:\/\/tyk-gateway.localhost:8080/hello}" + local status_endpoint="${1:-http://tyk-gateway.localhost:8080/hello}" local attempt_count=0 local pass="pass" From 32d3d7d35b16945d10bfbe92aacc9ce1b35f960f Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 14 Aug 2024 16:11:31 +0200 Subject: [PATCH 58/63] remove unneeded wait for liveness --- deployments/tyk/bootstrap.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/deployments/tyk/bootstrap.sh b/deployments/tyk/bootstrap.sh index 39d633bf..1a51afb5 100755 --- a/deployments/tyk/bootstrap.sh +++ b/deployments/tyk/bootstrap.sh @@ -53,9 +53,6 @@ bootstrap_progress # Certificates -log_message "Wait for services to be ready before beginning to bootstrap" -wait_for_liveness - log_message "Checking for existing OpenSSL container" OPENSSL_CONTAINER_NAME="tyk-demo-openssl" if [ "$(docker ps -a --format '{{.Names}}' | grep -w "$OPENSSL_CONTAINER_NAME" | wc -l)" -gt 0 ]; then From 99a50fd53965034c113c4209c7435eb363fe2d0b Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 14 Aug 2024 16:22:05 +0200 Subject: [PATCH 59/63] use term 'creating' instead of 'importing' as import endpoint is no longer used --- scripts/common.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/common.sh b/scripts/common.sh index 3891dd4d..41366aaa 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -682,12 +682,12 @@ create_api () { api_id=$(jq -r '.["x-tyk-api-gateway"].info.id' $api_data_path) # import endpoint differs between classic and OAS APIs api_endpoint="$api_endpoint/oas" - log_message " Importing OAS API: $api_name" + log_message " Creating OAS API: $api_name" else # Tyk API api_name=$(jq -r '.api_definition.name' $api_data_path) api_id=$(jq -r '.api_definition.api_id' $api_data_path) - log_message " Importing Classic API: $api_name" + log_message " Creating Classic API: $api_name" fi log_message " Id: $api_id" @@ -731,7 +731,7 @@ create_policy () { check_variables - log_message " Importing Policy: $policy_name" + log_message " Creating Policy: $policy_name" api_response="$(curl $dashboard_base_url/api/portal/policies -s \ -H "authorization: $dashboard_api_key" \ From 22fa085b480d1f615c5c8f0df174f37ed92a9ec6 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 14 Aug 2024 16:22:56 +0200 Subject: [PATCH 60/63] Tyk Sync 1.5.1 --- deployments/cicd/Dockerfile | 11 +++++++++++ deployments/cicd/data/jenkins/Jenkinsfile | 2 +- deployments/cicd/docker-compose.yml | 3 ++- deployments/cicd/scripts/dump-tyk.sh | 2 +- 4 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 deployments/cicd/Dockerfile diff --git a/deployments/cicd/Dockerfile b/deployments/cicd/Dockerfile new file mode 100644 index 00000000..3a0f4955 --- /dev/null +++ b/deployments/cicd/Dockerfile @@ -0,0 +1,11 @@ +FROM jenkins/jenkins:2.319.2 + +USER root + +RUN apt-get update && \ + curl -O https://packagecloud.io/install/repositories/tyk/tyk-sync/script.deb.sh && \ + chmod +x script.deb.sh && \ + ./script.deb.sh && \ + apt-get install -y tyk-sync=1.5.1 + +USER jenkins \ No newline at end of file diff --git a/deployments/cicd/data/jenkins/Jenkinsfile b/deployments/cicd/data/jenkins/Jenkinsfile index a5cfba9f..fcd50eb5 100644 --- a/deployments/cicd/data/jenkins/Jenkinsfile +++ b/deployments/cicd/data/jenkins/Jenkinsfile @@ -14,7 +14,7 @@ pipeline { script { if (fileExists('.tyk.json')) { echo "Deploying from ${env.BRANCH_NAME}" - sh "./tyk-sync sync -d http://tyk2-dashboard:3000 -s ${env.tyk2_dashboard_credentials} -p ." + sh "/opt/tyk-sync/tyk-sync sync -d http://tyk2-dashboard:3000 -s ${env.tyk2_dashboard_credentials} -p ." } else { echo 'No files to deploy' } diff --git a/deployments/cicd/docker-compose.yml b/deployments/cicd/docker-compose.yml index 0d296a08..047ec8ce 100644 --- a/deployments/cicd/docker-compose.yml +++ b/deployments/cicd/docker-compose.yml @@ -1,7 +1,8 @@ --- services: jenkins: - image: jenkins/jenkins:2.319.2-lts-alpine + build: ./deployments/cicd + image: tyk-demo-jenkins-tyk-sync:v1.5.1 ports: - 8070:8080 networks: diff --git a/deployments/cicd/scripts/dump-tyk.sh b/deployments/cicd/scripts/dump-tyk.sh index f61da76c..4a381dc2 100755 --- a/deployments/cicd/scripts/dump-tyk.sh +++ b/deployments/cicd/scripts/dump-tyk.sh @@ -7,5 +7,5 @@ gitea_tyk_data_repo_path=$(cat .context-data/gitea-tyk-data-repo-path) docker run --rm \ --network tyk-demo_tyk \ -v $gitea_tyk_data_repo_path:/opt/tyk-sync/data \ - tykio/tyk-sync:v1.4.3 \ + tykio/tyk-sync:v1.5.1 \ dump -d http://tyk-dashboard:3000 -s $dashboard_user_api_credentials -t data From cf48a214eaf62cfceaec544695e351d283feca91 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 14 Aug 2024 16:30:41 +0200 Subject: [PATCH 61/63] calc elapsed time for up script --- up.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/up.sh b/up.sh index f19a5c6e..0c234ea2 100755 --- a/up.sh +++ b/up.sh @@ -2,6 +2,8 @@ source scripts/common.sh +up_start_time=$(date +%s) + # persistence of log files is disabled by default, meaning the files are recreated between each bootstrap to prevent them from growing too large # to enable persistence, use argument "persist-log" when running this script persist_log=false @@ -165,6 +167,16 @@ for deployment in "${deployments_to_create[@]}"; do fi done +up_end_time=$(date +%s) +up_elapsed_time=$((up_end_time - up_start_time)) +up_minutes=$((up_elapsed_time / 60)) +up_seconds=$((up_elapsed_time % 60)) +if [ $up_minutes -gt 0 ]; then + log_message "Elapsed time: $up_minutes minutes $up_seconds seconds" +else + log_message "Elapsed time: $up_seconds seconds" +fi + # Confirm initialisation process is complete printf "\nTyk Demo initialisation process completed" printf "\n-----------------------------------------\n\n" From 7979dd385716f25ca71dd55495280e37819751c7 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 14 Aug 2024 16:51:13 +0200 Subject: [PATCH 62/63] log plugin cache used/max --- scripts/common.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/common.sh b/scripts/common.sh index 41366aaa..ccbdc863 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -366,8 +366,10 @@ build_go_plugin () { cp $go_plugin_directory/*.so $go_plugin_cache_version_directory # limit the number of plugin caches to prevent uncontrolled growth - local PLUGIN_CACHE_MAX_SIZE=3 - local plugin_cache_count=$(find "$go_plugin_cache_directory" -maxdepth 1 -type d -not -path "$go_plugin_cache_directory" | wc -l) + local PLUGIN_CACHE_MAX_SIZE=$(grep -E '^PLUGIN_CACHE_MAX_SIZE=[0-9]+' .env | cut -d '=' -f2) + PLUGIN_CACHE_MAX_SIZE=${PLUGIN_CACHE_MAX_SIZE:-3} + local plugin_cache_count=$(find "$go_plugin_cache_directory" -maxdepth 1 -type d -not -path "$go_plugin_cache_directory" | wc -l | xargs) + log_message " Plugin cache used/max: $plugin_cache_count/$PLUGIN_CACHE_MAX_SIZE" if [ "$plugin_cache_count" -gt "$PLUGIN_CACHE_MAX_SIZE" ]; then oldest_plugin_cache_path=$(find "$go_plugin_cache_directory" -type d -not -path "$go_plugin_cache_directory" -exec ls -ld -ltr {} + | head -n 1 | awk '{print $9}') if [ -n "$oldest_plugin_cache_path" ]; then From 3e0d6e13c1ebc54a940a8442a20dc43f323356c1 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Wed, 14 Aug 2024 16:55:29 +0200 Subject: [PATCH 63/63] remove unneeded var --- scripts/common.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/common.sh b/scripts/common.sh index ccbdc863..feb1636c 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -934,7 +934,6 @@ wait_for_api_loaded () { wait_for_liveness () { local status_endpoint="${1:-http://tyk-gateway.localhost:8080/hello}" local attempt_count=0 - local pass="pass" log_message "Waiting for Gateway, Dashboard and Redis to be up and running"