FI-2429: Migrate to HL7 Validator Wrapper

.env

@@ -1,5 +1,5 @@
 JS_HOST=""
-BULK_DATA_VALIDATOR_URL=http://validator_service:4567
+BULK_DATA_FHIR_RESOURCE_VALIDATOR_URL=http://hl7_validator_service:3500
 REDIS_URL=redis://redis:6379/0
 
 # Full path to a custom JWKS json file, will use lib/bulk_data_test_kit/bulk_data_jwks.json if left blank

.env.development

@@ -1,4 +1,3 @@
-BULK_DATA_VALIDATOR_URL=http://localhost/validatorapi
-VALIDATOR_URL=http://localhost/validatorapi
+BULK_DATA_FHIR_RESOURCE_VALIDATOR_URL=http://localhost/hl7validatorapi
 REDIS_URL=redis://localhost:6379/0
 INFERNO_HOST=http://localhost:4567

.env.production

@@ -1,4 +1,3 @@
 REDIS_URL=redis://redis:6379/0
-BULK_DATA_VALIDATOR_URL=http://validator_service:4567
-VALIDATOR_URL=http://validator_service:4567
+BULK_DATA_FHIR_RESOURCE_VALIDATOR_URL=http://hl7_validator_service:3500
 INFERNO_HOST=http://localhost

Gemfile.lock

@@ -161,7 +161,7 @@ GEM
     i18n (1.14.4)
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
-    inferno_core (0.4.33)
+    inferno_core (0.4.34)
       activesupport (~> 6.1.7.5)
       base62-rb (= 0.3.1)
       blueprinter (= 0.25.2)
@@ -200,11 +200,11 @@ GEM
       base64
     kramdown (2.4.0)
       rexml
-    method_source (1.0.0)
+    method_source (1.1.0)
     mime-types (3.4.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2024.0305)
-    mini_portile2 (2.8.5)
+    mini_portile2 (2.8.6)
     minitest (5.22.3)
     msgpack (1.7.2)
     multi_json (1.15.0)
@@ -218,14 +218,14 @@ GEM
     ndjson (1.0.0)
     netrc (0.11.0)
     nio4r (2.7.1)
-    nokogiri (1.16.3)
+    nokogiri (1.16.4)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    nokogiri (1.16.3-arm64-darwin)
+    nokogiri (1.16.4-arm64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.3-x86_64-darwin)
+    nokogiri (1.16.4-x86_64-darwin)
       racc (~> 1.4)
-    nokogiri (1.16.3-x86_64-linux)
+    nokogiri (1.16.4-x86_64-linux)
       racc (~> 1.4)
     oauth2 (1.4.11)
       faraday (>= 0.17.3, < 3.0)
@@ -337,4 +337,4 @@ DEPENDENCIES
   webmock (~> 3.11)
 
 BUNDLED WITH
-   2.3.14
+   2.5.7

config/nginx.background.conf

@@ -53,22 +53,38 @@ http {
     # the server will close connections after this time
     keepalive_timeout 600;
 
-    location /validator {
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header Host $http_host;
-      proxy_set_header X-Forwarded-Proto $scheme;
-      proxy_set_header X-Forwarded-Port $server_port;
-      proxy_redirect off;
-      proxy_set_header Connection '';
-      proxy_http_version 1.1;
-      chunked_transfer_encoding off;
-      proxy_buffering off;
-      proxy_cache off;
+#    location /validator {
+#      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#      proxy_set_header Host $http_host;
+#      proxy_set_header X-Forwarded-Proto $scheme;
+#      proxy_set_header X-Forwarded-Port $server_port;
+#      proxy_redirect off;
+#      proxy_set_header Connection '';
+#      proxy_http_version 1.1;
+#      chunked_transfer_encoding off;
+#      proxy_buffering off;
+#      proxy_cache off;
+#
+#      proxy_pass http://fhir_validator_app;
+#    }
 
-      proxy_pass http://fhir_validator_app;
-    }
+#    location /validatorapi/ {
+#      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#      proxy_set_header Host $http_host;
+#      proxy_set_header X-Forwarded-Proto $scheme;
+#      proxy_set_header X-Forwarded-Port $server_port;
+#      proxy_redirect off;
+#      proxy_set_header Connection '';
+#      proxy_http_version 1.1;
+#      chunked_transfer_encoding off;
+#      proxy_buffering off;
+#      proxy_cache off;
+#
+#      proxy_pass http://validator_service:4567/;
+#    }
+#  }
 
-    location /validatorapi/ {
+    location /hl7validatorapi/ {
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header Host $http_host;
       proxy_set_header X-Forwarded-Proto $scheme;
@@ -79,8 +95,9 @@ http {
       chunked_transfer_encoding off;
       proxy_buffering off;
       proxy_cache off;
+      proxy_read_timeout 600s;
 
-      proxy_pass http://validator_service:4567/;
+      proxy_pass http://hl7_validator_service:3500/;
     }
   }
 }

config/nginx.conf

@@ -68,7 +68,37 @@ http {
       proxy_pass http://inferno:4567;
     }
 
-    location /validator {
+#    location /validator {
+#      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#      proxy_set_header Host $http_host;
+#      proxy_set_header X-Forwarded-Proto $scheme;
+#      proxy_set_header X-Forwarded-Port $server_port;
+#      proxy_redirect off;
+#      proxy_set_header Connection '';
+#      proxy_http_version 1.1;
+#      chunked_transfer_encoding off;
+#      proxy_buffering off;
+#      proxy_cache off;
+#
+#      proxy_pass http://fhir_validator_app;
+#    }
+
+#    location /validatorapi/ {
+#      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+#      proxy_set_header Host $http_host;
+#      proxy_set_header X-Forwarded-Proto $scheme;
+#      proxy_set_header X-Forwarded-Port $server_port;
+#      proxy_redirect off;
+#      proxy_set_header Connection '';
+#      proxy_http_version 1.1;
+#      chunked_transfer_encoding off;
+#      proxy_buffering off;
+#      proxy_cache off;
+#
+#      proxy_pass http://validator_service:4567/;
+#    }
+
+    location /hl7validatorapi/ {
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
       proxy_set_header Host $http_host;
       proxy_set_header X-Forwarded-Proto $scheme;
@@ -79,23 +109,9 @@ http {
       chunked_transfer_encoding off;
       proxy_buffering off;
       proxy_cache off;
+      proxy_read_timeout 600s;
 
-      proxy_pass http://fhir_validator_app;
-    }
-
-    location /validatorapi/ {
-      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-      proxy_set_header Host $http_host;
-      proxy_set_header X-Forwarded-Proto $scheme;
-      proxy_set_header X-Forwarded-Port $server_port;
-      proxy_redirect off;
-      proxy_set_header Connection '';
-      proxy_http_version 1.1;
-      chunked_transfer_encoding off;
-      proxy_buffering off;
-      proxy_cache off;
-
-      proxy_pass http://validator_service:4567/;
+      proxy_pass http://hl7_validator_service:3500/;
     }
   }
 }

docker-compose.background.yml

@@ -1,17 +1,28 @@
 version: '3'
 services:
-  validator_service:
-    image: infernocommunity/fhir-validator-service
-    # Update this path to match your directory structure
+  hl7_validator_service:
+    image: infernocommunity/inferno-resource-validator
+    environment:
+      # Defines how long validator sessions last if unused, in minutes:
+      # Negative values mean sessions never expire, 0 means sessions immediately expire
+      SESSION_CACHE_DURATION: -1
     volumes:
       - ./lib/bulk_data_test_kit/igs:/home/igs
-  fhir_validator_app:
-    image: infernocommunity/fhir-validator-app
-    depends_on:
-      - validator_service
-    environment:
-      EXTERNAL_VALIDATOR_URL: http://localhost/validatorapi
-      VALIDATOR_BASE_PATH: /validator
+      # To let the service share your local FHIR package cache,
+      # uncomment the below line
+      # - ~/.fhir:/home/ktor/.fhir
+  # validator_service:
+  #   image: infernocommunity/fhir-validator-service
+  #   # Update this path to match your directory structure
+  #   volumes:
+  #     - ./lib/bulk_data_test_kit/igs:/home/igs
+  # fhir_validator_app:
+  #   image: infernocommunity/fhir-validator-app
+  #   depends_on:
+  #     - validator_service
+  #   environment:
+  #     EXTERNAL_VALIDATOR_URL: http://localhost/validatorapi
+  #     VALIDATOR_BASE_PATH: /validator
   nginx:
     image: nginx
     volumes:
@@ -20,7 +31,8 @@ services:
       - "80:80"
     command: [nginx, '-g', 'daemon off;']
     depends_on:
-      - fhir_validator_app
+      - hl7_validator_service
+    #  - fhir_validator_app
   redis:
     image: redis
     ports:

docker-compose.yml

@@ -6,7 +6,7 @@ services:
     volumes:
       - ./data:/opt/inferno/data
     depends_on:
-      - validator_service
+      - hl7_validator_service
   worker:
     build:
       context: ./
@@ -15,14 +15,18 @@ services:
     command: bundle exec sidekiq -r ./worker.rb
     depends_on:
       - redis
-  validator_service:
+  hl7_validator_service:
     extends:
       file: docker-compose.background.yml
-      service: validator_service
-  fhir_validator_app:
-    extends:
-      file: docker-compose.background.yml
-      service: fhir_validator_app
+      service: hl7_validator_service
+  # validator_service:
+  #   extends:
+  #     file: docker-compose.background.yml
+  #     service: validator_service
+  # fhir_validator_app:
+  #   extends:
+  #     file: docker-compose.background.yml
+  #     service: fhir_validator_app
   nginx:
     extends:
       file: docker-compose.background.yml

lib/bulk_data_test_kit/v1.0.1/bulk_data_test_suite.rb

@@ -26,13 +26,14 @@ class BulkDataTestSuite < Inferno::TestSuite
       ]
 
       VALIDATION_MESSAGE_FILTERS = [
-        /Observation\.effective\.ofType\(Period\): .*vs-1:/ # Invalid invariant in FHIR v4.0.1
+        /Observation\.effective\.ofType\(Period\): .*vs-1:/, # Invalid invariant in FHIR v4.0.1
+        /\A\S+: \S+: URL value '.*' does not resolve/
       ].freeze
 
       VERSION_SPECIFIC_MESSAGE_FILTERS = [].freeze
 
-      validator do
-        url ENV.fetch('BULK_DATA_VALIDATOR_URL', 'http://validator_service:4567')
+      fhir_resource_validator do
+        url ENV.fetch('BULK_DATA_FHIR_RESOURCE_VALIDATOR_URL', 'http://hl7_validator_service:3500')
 
         message_filters = VALIDATION_MESSAGE_FILTERS + VERSION_SPECIFIC_MESSAGE_FILTERS
 
@@ -42,13 +43,17 @@ class BulkDataTestSuite < Inferno::TestSuite
         $capped_errors = false
 
         exclude_message do |message|
-          if message.type != 'error'
-            $num_messages += 1
-          else
-            $num_errors += 1
+          matches_filter = message_filters.any? { |filter| filter.match? message.message }
+
+          unless matches_filter
+            if message.type != 'error'
+              $num_messages += 1
+            else
+              $num_errors += 1
+            end
           end
 
-          message_filters.any? { |filter| filter.match? message.message } ||
+          matches_filter ||
             (message.type != 'error' && $num_messages > 50 && !message.message.include?('Inferno is only showing the first')) ||
             (message.type == 'error' && $num_errors > 20 && !message.message.include?('Inferno is only showing the first'))
         end

lib/bulk_data_test_kit/v2.0.0/bulk_data_test_suite.rb

@@ -26,13 +26,14 @@ class BulkDataTestSuite < Inferno::TestSuite
       ]
 
       VALIDATION_MESSAGE_FILTERS = [
-        /Observation\.effective\.ofType\(Period\): .*vs-1:/ # Invalid invariant in FHIR v4.0.1
+        /Observation\.effective\.ofType\(Period\): .*vs-1:/, # Invalid invariant in FHIR v4.0.1
+        /\A\S+: \S+: URL value '.*' does not resolve/
       ].freeze
 
       VERSION_SPECIFIC_MESSAGE_FILTERS = [].freeze
 
-      validator do
-        url ENV.fetch('BULK_DATA_VALIDATOR_URL', 'http://validator_service:4567')
+      fhir_resource_validator do
+        url ENV.fetch('BULK_DATA_FHIR_RESOURCE_VALIDATOR_URL', 'http://hl7_validator_service:3500')
 
         message_filters = VALIDATION_MESSAGE_FILTERS + VERSION_SPECIFIC_MESSAGE_FILTERS
 
@@ -42,13 +43,17 @@ class BulkDataTestSuite < Inferno::TestSuite
         $capped_errors = false
 
         exclude_message do |message|
-          if message.type != 'error'
-            $num_messages += 1
-          else
-            $num_errors += 1
+          matches_filter = message_filters.any? { |filter| filter.match? message.message }
+
+          unless matches_filter
+            if message.type != 'error'
+              $num_messages += 1
+            else
+              $num_errors += 1
+            end
           end
 
-          message_filters.any? { |filter| filter.match? message.message } ||
+           matches_filter ||
             (message.type != 'error' && $num_messages > 50 && !message.message.include?('Inferno is only showing the first')) ||
             (message.type == 'error' && $num_errors > 20 && !message.message.include?('Inferno is only showing the first'))
         end