From c5afd7816645da8ae6b5eaf756942f6a15d44d9f Mon Sep 17 00:00:00 2001 From: afthab vp Date: Tue, 7 May 2024 21:56:59 +0530 Subject: [PATCH] fix: skip cursor query builder for cursor empty --- server/lib/reverse_etl/utils/batch_query.rb | 2 +- .../lib/reverse_etl/utils/batch_query_spec.rb | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/server/lib/reverse_etl/utils/batch_query.rb b/server/lib/reverse_etl/utils/batch_query.rb index da65dc60..dbf15899 100644 --- a/server/lib/reverse_etl/utils/batch_query.rb +++ b/server/lib/reverse_etl/utils/batch_query.rb @@ -14,7 +14,7 @@ def self.execute_in_batches(params) params[:sync_config].limit = params[:batch_size] params[:sync_config].offset = current_offset - if initial_sync_config.cursor_field + if initial_sync_config.cursor_field.present? query_with_cursor = CursorQueryBuilder.build_cursor_query(initial_sync_config, last_cursor_field_value) params[:sync_config] = build_cursor_sync_config(params[:sync_config], query_with_cursor) end diff --git a/server/spec/lib/reverse_etl/utils/batch_query_spec.rb b/server/spec/lib/reverse_etl/utils/batch_query_spec.rb index fa480dea..9a369058 100644 --- a/server/spec/lib/reverse_etl/utils/batch_query_spec.rb +++ b/server/spec/lib/reverse_etl/utils/batch_query_spec.rb @@ -42,6 +42,44 @@ module Utils # rubocop:disable Metrics/ModuleLength expect(results.last.size).to eq(100) end end + context "when both cursor_field is empty" do + let(:existing_query) { "SELECT * FROM table" } + let(:source) { create(:connector, connector_type: "source", connector_name: "Snowflake") } + let(:destination) { create(:connector, connector_type: "destination") } + let!(:catalog) { create(:catalog, connector: destination) } + let(:model) { create(:model, connector: source, query: existing_query) } + let(:sync) do + create(:sync, model:, source:, destination:, cursor_field: "") + end + let(:record) do + Multiwoven::Integrations::Protocol::RecordMessage.new(data: { "id" => 1, "email" => "test1@mail.com", + "first_name" => "John", + "Last Name" => "Doe" }, + emitted_at: DateTime.now.to_i).to_multiwoven_message + end + let(:record1) do + Multiwoven::Integrations::Protocol::RecordMessage.new(data: { "id" => 1, "email" => "test1@mail.com", + "first_name" => "John", + "Last Name" => "Doe" }, + emitted_at: DateTime.now.to_i).to_multiwoven_message + end + + it "executes batches and not call CursorQueryBuilder for cursor_field is empty" do + params = { + offset: 0, + limit: 100, + batch_size: 100, + sync_config: sync.to_protocol, + client: + } + allow(client).to receive(:read).and_return(*Array.new(1, [record]), []) + expect(CursorQueryBuilder).not_to receive(:build_cursor_query).with(sync.to_protocol, "") + results = [] + BatchQuery.execute_in_batches(params) do |result, _current_offset, _last_cursor_field_value| + results << result + end + end + end context "when both cursor_field is present" do let(:existing_query) { "SELECT * FROM table" } let(:source) { create(:connector, connector_type: "source", connector_name: "Snowflake") }