From 53494a76b5ae4501628b9ae474a5e402a75dd93b Mon Sep 17 00:00:00 2001 From: Jonathan Moraes Date: Tue, 27 Aug 2024 12:37:14 -0300 Subject: [PATCH] Include test folder and adjusting Dockerfile and compose file Signed-off-by: Jonathan Moraes --- proxy/fluentd/docker/Dockerfile | 6 +- proxy/fluentd/test/helper.rb | 8 + .../test/plugin/test_parser_openlineage.rb | 141 ++++++++++++++++++ 3 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 proxy/fluentd/test/helper.rb create mode 100644 proxy/fluentd/test/plugin/test_parser_openlineage.rb diff --git a/proxy/fluentd/docker/Dockerfile b/proxy/fluentd/docker/Dockerfile index bfe30c8a0c..7bc98b98a6 100644 --- a/proxy/fluentd/docker/Dockerfile +++ b/proxy/fluentd/docker/Dockerfile @@ -5,7 +5,7 @@ USER root RUN buildDeps="sudo make gcc g++ libc-dev" \ && apt-get update \ - && apt-get install -y --no-install-recommends "$buildDeps" \ + && apt-get install -y --no-install-recommends $buildDeps \ && apt-get install -y ruby-full \ && apt-get install -y git \ && gem install ffi -v '1.15.5' --source 'https://rubygems.org/' \ @@ -16,11 +16,11 @@ RUN buildDeps="sudo make gcc g++ libc-dev" \ && SUDO_FORCE_REMOVE=yes \ apt-get purge -y --auto-remove \ -o APT::AutoRemove::RecommendsImportant=false \ - "$buildDeps" \ + $buildDeps \ && rm -rf /var/lib/apt/lists/* \ && rm -rf /tmp/* /var/tmp/* /usr/lib/ruby/gems/*/cache/*.gem COPY proxy/fluentd/docker/entrypoint.sh /bin/entrypoint.sh -COPY proxy/fluentd/spec/ /etc/spec +COPY /spec /etc/spec USER fluent \ No newline at end of file diff --git a/proxy/fluentd/test/helper.rb b/proxy/fluentd/test/helper.rb new file mode 100644 index 0000000000..edbcd1bc04 --- /dev/null +++ b/proxy/fluentd/test/helper.rb @@ -0,0 +1,8 @@ +$LOAD_PATH.unshift(File.expand_path("../../", __FILE__)) +require "test-unit" +require "fluent/test" +require "fluent/test/driver/parser" +require "fluent/test/helpers" + +Test::Unit::TestCase.include(Fluent::Test::Helpers) +Test::Unit::TestCase.extend(Fluent::Test::Helpers) \ No newline at end of file diff --git a/proxy/fluentd/test/plugin/test_parser_openlineage.rb b/proxy/fluentd/test/plugin/test_parser_openlineage.rb new file mode 100644 index 0000000000..30c7dcc99c --- /dev/null +++ b/proxy/fluentd/test/plugin/test_parser_openlineage.rb @@ -0,0 +1,141 @@ +require "helper" +require "fluent/plugin/parser_openlineage.rb" + +class OpenlineageParserTest < Test::Unit::TestCase + setup do + Fluent::Test.setup + @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::OpenlineageParser) + @parser.configure( + 'spec_directory' => '../../spec/' + ) + end + + test "test event with no runId" do + ol_event = File.read("events/event_no_run_id.json") + + err = assert_raise Fluent::ParserError do + @parser.instance.parse(ol_event) + end + assert_match(/Openlineage validation failed: (.+) path "\/run": "runId" is a required property/, err.message) + end + + test "test invalid json" do + assert_raise Fluent::ParserError do + @parser.instance.parse('{"run": Not a JSON}') + end + end + + test "event full test" do + ol_event = File.read("events/event_full.json") + @parser.instance.parse(ol_event) { | time, json | + assert_equal("ea041791-68bc-4ae1-bd89-4c8106a157e4", json['run']['runId']) + assert_equal(2000, json['outputs'][0]['outputFacets']['outputStatistics']['rowCount']) + } + end + + test "event simple test" do + ol_event = File.read("events/event_simple.json") + @parser.instance.parse(ol_event) { | time, json | + assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId']) + } + end + + test "invalid spec_directory test" do + assert_raise Fluent::ParserError do + @parser.configure( + 'spec_directory' => './non-existent-spec/' + ) + end + end + + test "valid spec_directory without slash" do + @parser.configure( + 'spec_directory' => '../../spec' + ) + ol_event = File.read("events/event_simple.json") + @parser.instance.parse(ol_event) { | time, json | + assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId']) + } + end + + test "run facet validation" do + ol_event = File.read("events/event_invalid_run_facet.json") + err = assert_raise Fluent::ParserError do + @parser.instance.parse(ol_event) + end + assert_match(/Openlineage validation failed: (.+) path "\/run\/facets/, err.message) + end + + test "run facet validation turned off" do + ol_event = File.read("events/event_invalid_run_facet.json") + @parser.configure( + 'spec_directory' => '../../spec/', + 'validate_run_facets' => false, + ) + @parser.instance.parse(ol_event) { | time, json | + assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId']) + } + end + + test "job facet validation" do + ol_event = File.read("events/event_invalid_job_facet.json") + err = assert_raise Fluent::ParserError do + @parser.instance.parse(ol_event) + end + assert_match(/Openlineage validation failed: (.+) path "\/job\/facets\/ownership/, err.message) + end + + test "job facet validation turned off" do + ol_event = File.read("events/event_invalid_job_facet.json") + @parser.configure( + 'spec_directory' => '../../spec/', + 'validate_job_facets' => false, + ) + @parser.instance.parse(ol_event) { | time, json | + assert_equal("41fb5137-f0fd-4ee5-ba5c-56f8571d1bd7", json['run']['runId']) + } + end + + test "dataset facet validation" do + ol_event = File.read("events/event_invalid_dataset_facet.json") + @parser.configure( + 'spec_directory' => '../../spec/', + 'validate_dataset_facets' => true, + ) + err = assert_raise Fluent::ParserError do + @parser.instance.parse(ol_event) + end + assert_match(/Openlineage validation failed: (.+) path "\/outputs\/0\/facets\/ownership\/owners/, err.message) + end + + test "input dataset facet validation" do + ol_event = File.read("events/event_invalid_input_dataset_facet.json") + @parser.configure( + 'json_parser' => 'yajl', + 'spec_directory' => '../../spec/', + 'validate_input_dataset_facets' => true, + ) + err = assert_raise Fluent::ParserError do + @parser.instance.parse(ol_event) + end + assert_match(/Openlineage validation failed: (.+) "columnMetrics" is a required property/, err.message) + end + + test "output dataset facet validation" do + ol_event = File.read("events/event_invalid_output_dataset_facet.json") + @parser.configure( + 'spec_directory' => '../../spec/', + 'validate_output_dataset_facets' => true, + ) + err = assert_raise Fluent::ParserError do + @parser.instance.parse(ol_event) + end + assert_match(/Openlineage validation failed: (.+) path \"\/outputs\/0\/outputFacets\/outputStatistics\/rowCount/, err.message) + end + + private + + def create_driver(conf) + Fluent::Test::Driver::Parser.new(Fluent::Plugin::OpenlineageParser).configure(conf) + end +end \ No newline at end of file