diff --git a/fe/fe-core/src/test/java/com/starrocks/qe/TPCDSCoordTest.java b/fe/fe-core/src/test/java/com/starrocks/qe/TPCDSCoordTest.java index df1e167a33f34..314752ec62fa3 100644 --- a/fe/fe-core/src/test/java/com/starrocks/qe/TPCDSCoordTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/qe/TPCDSCoordTest.java @@ -16,11 +16,14 @@ package com.starrocks.qe; import com.starrocks.common.FeConstants; +import com.starrocks.planner.OlapTableSink; import com.starrocks.planner.PlanFragmentId; import com.starrocks.planner.RuntimeFilterDescription; import com.starrocks.sql.plan.ExecPlan; import com.starrocks.sql.plan.TPCDSPlanTest; import com.starrocks.sql.plan.TPCDSPlanTestBase; +import com.starrocks.system.SystemInfoService; +import com.starrocks.thrift.TNetworkAddress; import com.starrocks.thrift.TUniqueId; import com.starrocks.utframe.UtFrameUtils; import org.junit.After; @@ -32,6 +35,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -155,4 +159,58 @@ public void testSubQueryExtractedFromQ5() throws Exception { } Assert.assertTrue(rfExists); } + + @Test + public void testOlapTableSinkAsGRFCoordinator() throws Exception { + FeConstants.runningUnitTest = true; + ConnectContext ctx = starRocksAssert.getCtx(); + ctx.setExecutionId(new TUniqueId(0x33, 0x0)); + ConnectContext.threadLocalInfo.set(ctx); + ctx.getSessionVariable().setParallelExecInstanceNum(8); + ctx.getSessionVariable().setEnablePipelineEngine(true); + setTPCDSFactor(1); + + // make sure global runtime filter been push-downed to two fragments. + String sql = "insert into item \n" + + "select item.*\n" + + "from\n" + + " item inner join[shuffle] store_sales on store_sales.ss_item_sk = item.i_item_sk \n" + + " inner join [shuffle] date_dim dt on dt.d_date_sk = store_sales.ss_sold_date_sk\n" + + "where \n" + + " item.i_manufact_id = 128\n" + + " and dt.d_moy=11"; + String plan = UtFrameUtils.getVerboseFragmentPlan(ctx, sql); + String[] ss = plan.split("\\n"); + List filterLines = Stream.of(ss).filter(s -> s.contains("filter_id =")).collect(Collectors.toList()); + Assert.assertFalse(filterLines.isEmpty()); + Assert.assertTrue(filterLines.stream().anyMatch(ln -> ln.contains("remote = true"))); + ExecPlan execPlan = UtFrameUtils.getPlanAndFragment(ctx, sql).second; + Coordinator coord = new Coordinator(ctx, execPlan.getFragments(), execPlan.getScanNodes(), + execPlan.getDescTbl().toThrift()); + coord.prepareExec(); + + PlanFragmentId rootFragmentId = coord.getFragments().get(0).getFragmentId(); + CoordinatorPreprocessor.FragmentExecParams rootExecFragment = + coord.getFragmentExecParamsMap().get(rootFragmentId); + CoordinatorPreprocessor.FInstanceExecParam rootFInstance = rootExecFragment.instanceExecParams.get(0); + Assert.assertTrue(rootExecFragment.fragment.getSink() instanceof OlapTableSink); + Assert.assertTrue(rootFInstance.isRuntimeFilterCoordinator()); + Assert.assertFalse(rootFInstance.runtimeFilterParams.getRuntime_filter_builder_number().isEmpty()); + + Set grfCoordinators = + coord.getFragmentExecParamsMap().values().stream().flatMap(execFragment -> { + Map buildRfFilters = + execFragment.fragment.getBuildRuntimeFilters(); + if (buildRfFilters == null || buildRfFilters.isEmpty()) { + return Stream.empty(); + } else { + return buildRfFilters.values().stream() + .filter(RuntimeFilterDescription::isHasRemoteTargets) + .flatMap(rf -> rf.toThrift().getRuntime_filter_merge_nodes().stream()); + } + }).collect(Collectors.toSet()); + + Assert.assertEquals(grfCoordinators.size(), 1); + Assert.assertTrue(grfCoordinators.contains(SystemInfoService.toBrpcHost(rootFInstance.getHost()))); + } } diff --git a/test/sql/test_global_runtime_filter_olap_table_sink/R/test_global_runtime_filter_olap_table_sink b/test/sql/test_global_runtime_filter_olap_table_sink/R/test_global_runtime_filter_olap_table_sink deleted file mode 100644 index 1940232a6a374..0000000000000 --- a/test/sql/test_global_runtime_filter_olap_table_sink/R/test_global_runtime_filter_olap_table_sink +++ /dev/null @@ -1,208 +0,0 @@ --- name: test_global_runtime_filter_olap_table_sink -DROP TABLE if exists t0; --- result: --- !result -CREATE TABLE if not exists t0 -( -c0 BIGINT NULL, -dt DATE NULL, -c1 VARCHAR(64) NULL -) ENGINE=OLAP -DUPLICATE KEY(`c0`, `dt`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`c0`, `dt`) BUCKETS 64 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); --- result: --- !result -DROP TABLE if exists t1; --- result: --- !result -CREATE TABLE if not exists t1 -( -c0 BIGINT NOT NULL, -dt DATE NOT NULL, -c1 VARCHAR(64) NOT NULL -) ENGINE=OLAP -DUPLICATE KEY(`c0`, `dt`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`c0`, `dt`) BUCKETS 64 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); --- result: --- !result -DROP TABLE if exists profile_table; --- result: --- !result -CREATE TABLE if not exists profile_table -( -query_no BIGINT NOT NULL, -line_no BIGINT NOT NULL, -line VARCHAR(4096) NOT NULL -) ENGINE=OLAP -DUPLICATE KEY(`query_no`, `line_no`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`query_no`, `line_no`) BUCKETS 2 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); --- result: --- !result -DROP TABLE if exists result_table; --- result: --- !result -CREATE TABLE if not exists result_table -( -result BIGINT NOT NULL -) ENGINE=OLAP -DUPLICATE KEY(`result`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`result`) BUCKETS 1 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); --- result: --- !result -DROP TABLE if exists seed0; --- result: --- !result -CREATE TABLE if not exists seed0 -( -c INT NOT NULL -) ENGINE=OLAP -DUPLICATE KEY(`c`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`c`) BUCKETS 64 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); --- result: --- !result -INSERT INTO seed0 - (c) - select row_number() over() as r from (SELECT __tbl_7.__col_4 AS __col_8, __tbl_7.__col_5 AS __col_9, __tbl_7.__col_6 AS __col_10, __tbl_8.__col_7 AS __col_11 from (SELECT __tbl_4.__col_1 AS __col_4, __tbl_4.__col_2 AS __col_5, __tbl_5.__col_3 AS __col_6 from (SELECT __tbl_1.v AS __col_1, __tbl_2.__col_0 AS __col_2 from (select v from (values(0)) t(v)) AS __tbl_1, (SELECT __tbl_0.column_0 AS __col_0 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99),(100),(101)) __tbl_0) AS __tbl_2) AS __tbl_4, (SELECT __tbl_3.column_0 AS __col_3 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99)) __tbl_3) AS __tbl_5) AS __tbl_7, (SELECT __tbl_6.column_0 AS __col_7 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99)) __tbl_6) AS __tbl_8) as __tbl_9; --- result: --- !result -DROP TABLE if exists seed1; --- result: --- !result -CREATE TABLE if not exists seed1 -( -c INT NOT NULL -) ENGINE=OLAP -DUPLICATE KEY(`c`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`c`) BUCKETS 64 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); --- result: --- !result -INSERT INTO seed1 - (c) - select row_number() over() as r from (SELECT __tbl_20.__col_20 AS __col_25, __tbl_20.__col_21 AS __col_26, __tbl_20.__col_22 AS __col_27, __tbl_20.__col_23 AS __col_28, __tbl_21.__col_24 AS __col_29 from (SELECT __tbl_17.__col_16 AS __col_20, __tbl_17.__col_17 AS __col_21, __tbl_17.__col_18 AS __col_22, __tbl_18.__col_19 AS __col_23 from (SELECT __tbl_14.__col_13 AS __col_16, __tbl_14.__col_14 AS __col_17, __tbl_15.__col_15 AS __col_18 from (SELECT __tbl_11.v AS __col_13, __tbl_12.__col_12 AS __col_14 from (select v from (values(0)) t(v)) AS __tbl_11, (SELECT __tbl_10.column_0 AS __col_12 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9)) __tbl_10) AS __tbl_12) AS __tbl_14, (SELECT __tbl_13.column_0 AS __col_15 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99)) __tbl_13) AS __tbl_15) AS __tbl_17, (SELECT __tbl_16.column_0 AS __col_19 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99)) __tbl_16) AS __tbl_18) AS __tbl_20, (SELECT __tbl_19.column_0 AS __col_24 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99)) __tbl_19) AS __tbl_21) as __tbl_22; --- result: --- !result -INSERT INTO t0 - (c0, dt, c1) - select cast((rand()+rand())*1024000/2 as int) as __col_30, to_date(days_add('2023-01-31', cast((rand()+rand())*180/2 as int))) as __col_31, cast((rand()+rand())*1024000/2 as int) as __col_32 from (SELECT c FROM seed0) __tbl_23; --- result: --- !result -INSERT INTO t1 - (c0, dt, c1) - select cast((rand()+rand())*10240000/2 as int) as __col_33, to_date(days_add('2023-01-31', cast((rand()+rand())*365/2 as int))) as __col_34, cast((rand()+rand())*10240000/2 as int) as __col_35 from (SELECT c FROM seed1) __tbl_24; --- result: --- !result -set enable_profile='true'; --- result: --- !result -show variables like 'enable_profile'; --- result: -enable_profile true --- !result -set pipeline_dop='1'; --- result: --- !result -show variables like 'pipeline_dop'; --- result: -pipeline_dop 1 --- !result -set runtime_filter_scan_wait_time = 10000; --- result: --- !result -INSERT INTO result_table - (result) - select (sum(murmur_hash3_32(ifnull(c0,0))+murmur_hash3_32(ifnull(dt,0))+murmur_hash3_32(ifnull(c1,0)))) as fingerprint from (select t1.c0,t1.dt,t1.c1 from t1 join[shuffle] t0 on t0.c1 = t1.c1) as t; --- result: --- !result -INSERT INTO profile_table - (query_no, line_no, line) - select 0 as query_no, row_number() over() as line_no, profile.line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()),"\n")) profile(line); --- result: --- !result -INSERT INTO result_table - (result) - select (sum(murmur_hash3_32(ifnull(c0,0))+murmur_hash3_32(ifnull(dt,0))+murmur_hash3_32(ifnull(c1,0)))) as fingerprint from (select t1.c0,t1.dt,t1.c1 from t1 join[broadcast] t0 on t0.c1 = t1.c1) as t; --- result: --- !result -INSERT INTO profile_table - (query_no, line_no, line) - select 1 as query_no, row_number() over() as line_no, profile.line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()),"\n")) profile(line); --- result: --- !result -with olap_scan as ( -select line from profile_table where query_no=0 and line_no >= (select line_no from profile_table where query_no=0 and line like "%OLAP_SCAN (plan_node_id=0)%") order by line_no limit 50 -), -JoinRuntimeFilterInputRows as ( -select regexp_extract(line, ".*- JoinRuntimeFilterInputRows: (\\d+(?:\\.\\d+[KMB])?).*",1) as value from olap_scan where line like "%- JoinRuntimeFilterInputRows%" -), -JoinRuntimeFilterOutputRows as ( -select regexp_extract(line, ".*- JoinRuntimeFilterOutputRows: (\\d+(?:\\.\\d+[KMB])?).*",1) as value from olap_scan where line like "%- JoinRuntimeFilterOutputRows%" -), -rfInputRows as ( -select case unit when "B" then value0*1000000000 when "M" then value0*1000000 when "K" then value0*1000 else value1 end as value from (select right(value,1) as unit, cast(substr(value, 1, length(value)-1) as double) value0, cast(value as double) value1 from JoinRuntimeFilterInputRows) t -), -rfOutputRows as ( -select case unit when "B" then value0*1000000000 when "M" then value0*1000000 when "K" then value0*1000 else value1 end as value from (select right(value,1) as unit, cast(substr(value, 1, length(value)-1) as double) value0, cast(value as double) value1 from JoinRuntimeFilterOutputRows) t -) -select assert_true(rfOutputRows.value/rfInputRows.value < 0.5) from rfInputRows,rfOutputRows; --- result: -1 --- !result -with olap_scan as ( -select line from profile_table where query_no=1 and line_no >= (select line_no from profile_table where query_no=1 and line like "%OLAP_SCAN (plan_node_id=0)%") order by line_no limit 50 -), -JoinRuntimeFilterInputRows as ( -select regexp_extract(line, ".*- JoinRuntimeFilterInputRows: (\\d+(?:\\.\\d+[KMB])?).*",1) as value from olap_scan where line like "%- JoinRuntimeFilterInputRows%" -), -JoinRuntimeFilterOutputRows as ( -select regexp_extract(line, ".*- JoinRuntimeFilterOutputRows: (\\d+(?:\\.\\d+[KMB])?).*",1) as value from olap_scan where line like "%- JoinRuntimeFilterOutputRows%" -), -rfInputRows as ( -select case unit when "B" then value0*1000000000 when "M" then value0*1000000 when "K" then value0*1000 else value1 end as value from (select right(value,1) as unit, cast(substr(value, 1, length(value)-1) as double) value0, cast(value as double) value1 from JoinRuntimeFilterInputRows) t -), -rfOutputRows as ( -select case unit when "B" then value0*1000000000 when "M" then value0*1000000 when "K" then value0*1000 else value1 end as value from (select right(value,1) as unit, cast(substr(value, 1, length(value)-1) as double) value0, cast(value as double) value1 from JoinRuntimeFilterOutputRows) t -) -select assert_true(rfOutputRows.value/rfInputRows.value < 0.5) from rfInputRows,rfOutputRows; --- result: -1 --- !result -select assert_true(count(distinct result)=1) from result_table; --- result: -1 --- !result \ No newline at end of file diff --git a/test/sql/test_global_runtime_filter_olap_table_sink/T/test_global_runtime_filter_olap_table_sink b/test/sql/test_global_runtime_filter_olap_table_sink/T/test_global_runtime_filter_olap_table_sink deleted file mode 100644 index 1bf18b2b856a3..0000000000000 --- a/test/sql/test_global_runtime_filter_olap_table_sink/T/test_global_runtime_filter_olap_table_sink +++ /dev/null @@ -1,153 +0,0 @@ --- name: test_global_runtime_filter_olap_table_sink -DROP TABLE if exists t0; - -CREATE TABLE if not exists t0 -( -c0 BIGINT NULL, -dt DATE NULL, -c1 VARCHAR(64) NULL -) ENGINE=OLAP -DUPLICATE KEY(`c0`, `dt`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`c0`, `dt`) BUCKETS 64 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); -DROP TABLE if exists t1; - -CREATE TABLE if not exists t1 -( -c0 BIGINT NOT NULL, -dt DATE NOT NULL, -c1 VARCHAR(64) NOT NULL -) ENGINE=OLAP -DUPLICATE KEY(`c0`, `dt`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`c0`, `dt`) BUCKETS 64 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); -DROP TABLE if exists profile_table; - -CREATE TABLE if not exists profile_table -( -query_no BIGINT NOT NULL, -line_no BIGINT NOT NULL, -line VARCHAR(4096) NOT NULL -) ENGINE=OLAP -DUPLICATE KEY(`query_no`, `line_no`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`query_no`, `line_no`) BUCKETS 2 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); -DROP TABLE if exists result_table; - -CREATE TABLE if not exists result_table -( -result BIGINT NOT NULL -) ENGINE=OLAP -DUPLICATE KEY(`result`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`result`) BUCKETS 1 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); -DROP TABLE if exists seed0; - -CREATE TABLE if not exists seed0 -( -c INT NOT NULL -) ENGINE=OLAP -DUPLICATE KEY(`c`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`c`) BUCKETS 64 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); -INSERT INTO seed0 - (c) - select row_number() over() as r from (SELECT __tbl_7.__col_4 AS __col_8, __tbl_7.__col_5 AS __col_9, __tbl_7.__col_6 AS __col_10, __tbl_8.__col_7 AS __col_11 from (SELECT __tbl_4.__col_1 AS __col_4, __tbl_4.__col_2 AS __col_5, __tbl_5.__col_3 AS __col_6 from (SELECT __tbl_1.v AS __col_1, __tbl_2.__col_0 AS __col_2 from (select v from (values(0)) t(v)) AS __tbl_1, (SELECT __tbl_0.column_0 AS __col_0 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99),(100),(101)) __tbl_0) AS __tbl_2) AS __tbl_4, (SELECT __tbl_3.column_0 AS __col_3 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99)) __tbl_3) AS __tbl_5) AS __tbl_7, (SELECT __tbl_6.column_0 AS __col_7 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99)) __tbl_6) AS __tbl_8) as __tbl_9; -DROP TABLE if exists seed1; - -CREATE TABLE if not exists seed1 -( -c INT NOT NULL -) ENGINE=OLAP -DUPLICATE KEY(`c`) -COMMENT "OLAP" -DISTRIBUTED BY HASH(`c`) BUCKETS 64 -PROPERTIES( -"replication_num" = "1", -"in_memory" = "false", -"storage_format" = "default" -); -INSERT INTO seed1 - (c) - select row_number() over() as r from (SELECT __tbl_20.__col_20 AS __col_25, __tbl_20.__col_21 AS __col_26, __tbl_20.__col_22 AS __col_27, __tbl_20.__col_23 AS __col_28, __tbl_21.__col_24 AS __col_29 from (SELECT __tbl_17.__col_16 AS __col_20, __tbl_17.__col_17 AS __col_21, __tbl_17.__col_18 AS __col_22, __tbl_18.__col_19 AS __col_23 from (SELECT __tbl_14.__col_13 AS __col_16, __tbl_14.__col_14 AS __col_17, __tbl_15.__col_15 AS __col_18 from (SELECT __tbl_11.v AS __col_13, __tbl_12.__col_12 AS __col_14 from (select v from (values(0)) t(v)) AS __tbl_11, (SELECT __tbl_10.column_0 AS __col_12 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9)) __tbl_10) AS __tbl_12) AS __tbl_14, (SELECT __tbl_13.column_0 AS __col_15 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99)) __tbl_13) AS __tbl_15) AS __tbl_17, (SELECT __tbl_16.column_0 AS __col_19 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99)) __tbl_16) AS __tbl_18) AS __tbl_20, (SELECT __tbl_19.column_0 AS __col_24 from (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9),(10),(11),(12),(13),(14),(15),(16),(17),(18),(19),(20),(21),(22),(23),(24),(25),(26),(27),(28),(29),(30),(31),(32),(33),(34),(35),(36),(37),(38),(39),(40),(41),(42),(43),(44),(45),(46),(47),(48),(49),(50),(51),(52),(53),(54),(55),(56),(57),(58),(59),(60),(61),(62),(63),(64),(65),(66),(67),(68),(69),(70),(71),(72),(73),(74),(75),(76),(77),(78),(79),(80),(81),(82),(83),(84),(85),(86),(87),(88),(89),(90),(91),(92),(93),(94),(95),(96),(97),(98),(99)) __tbl_19) AS __tbl_21) as __tbl_22; -INSERT INTO t0 - (c0, dt, c1) - select cast((rand()+rand())*1024000/2 as int) as __col_30, to_date(days_add('2023-01-31', cast((rand()+rand())*180/2 as int))) as __col_31, cast((rand()+rand())*1024000/2 as int) as __col_32 from (SELECT c FROM seed0) __tbl_23; -INSERT INTO t1 - (c0, dt, c1) - select cast((rand()+rand())*10240000/2 as int) as __col_33, to_date(days_add('2023-01-31', cast((rand()+rand())*365/2 as int))) as __col_34, cast((rand()+rand())*10240000/2 as int) as __col_35 from (SELECT c FROM seed1) __tbl_24; -set enable_profile='true'; -show variables like 'enable_profile'; -set pipeline_dop='1'; -show variables like 'pipeline_dop'; -set runtime_filter_scan_wait_time = 10000; -INSERT INTO result_table - (result) - select (sum(murmur_hash3_32(ifnull(c0,0))+murmur_hash3_32(ifnull(dt,0))+murmur_hash3_32(ifnull(c1,0)))) as fingerprint from (select t1.c0,t1.dt,t1.c1 from t1 join[shuffle] t0 on t0.c1 = t1.c1) as t; -INSERT INTO profile_table - (query_no, line_no, line) - select 0 as query_no, row_number() over() as line_no, profile.line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()),"\n")) profile(line); -INSERT INTO result_table - (result) - select (sum(murmur_hash3_32(ifnull(c0,0))+murmur_hash3_32(ifnull(dt,0))+murmur_hash3_32(ifnull(c1,0)))) as fingerprint from (select t1.c0,t1.dt,t1.c1 from t1 join[broadcast] t0 on t0.c1 = t1.c1) as t; -INSERT INTO profile_table - (query_no, line_no, line) - select 1 as query_no, row_number() over() as line_no, profile.line from (values(1))t(v) join unnest(split(get_query_profile(last_query_id()),"\n")) profile(line); -with olap_scan as ( -select line from profile_table where query_no=0 and line_no >= (select line_no from profile_table where query_no=0 and line like "%OLAP_SCAN (plan_node_id=0)%") order by line_no limit 50 -), -JoinRuntimeFilterInputRows as ( -select regexp_extract(line, ".*- JoinRuntimeFilterInputRows: (\\d+(?:\\.\\d+[KMB])?).*",1) as value from olap_scan where line like "%- JoinRuntimeFilterInputRows%" -), -JoinRuntimeFilterOutputRows as ( -select regexp_extract(line, ".*- JoinRuntimeFilterOutputRows: (\\d+(?:\\.\\d+[KMB])?).*",1) as value from olap_scan where line like "%- JoinRuntimeFilterOutputRows%" -), -rfInputRows as ( -select case unit when "B" then value0*1000000000 when "M" then value0*1000000 when "K" then value0*1000 else value1 end as value from (select right(value,1) as unit, cast(substr(value, 1, length(value)-1) as double) value0, cast(value as double) value1 from JoinRuntimeFilterInputRows) t -), -rfOutputRows as ( -select case unit when "B" then value0*1000000000 when "M" then value0*1000000 when "K" then value0*1000 else value1 end as value from (select right(value,1) as unit, cast(substr(value, 1, length(value)-1) as double) value0, cast(value as double) value1 from JoinRuntimeFilterOutputRows) t -) -select assert_true(rfOutputRows.value/rfInputRows.value < 0.5) from rfInputRows,rfOutputRows; -with olap_scan as ( -select line from profile_table where query_no=1 and line_no >= (select line_no from profile_table where query_no=1 and line like "%OLAP_SCAN (plan_node_id=0)%") order by line_no limit 50 -), -JoinRuntimeFilterInputRows as ( -select regexp_extract(line, ".*- JoinRuntimeFilterInputRows: (\\d+(?:\\.\\d+[KMB])?).*",1) as value from olap_scan where line like "%- JoinRuntimeFilterInputRows%" -), -JoinRuntimeFilterOutputRows as ( -select regexp_extract(line, ".*- JoinRuntimeFilterOutputRows: (\\d+(?:\\.\\d+[KMB])?).*",1) as value from olap_scan where line like "%- JoinRuntimeFilterOutputRows%" -), -rfInputRows as ( -select case unit when "B" then value0*1000000000 when "M" then value0*1000000 when "K" then value0*1000 else value1 end as value from (select right(value,1) as unit, cast(substr(value, 1, length(value)-1) as double) value0, cast(value as double) value1 from JoinRuntimeFilterInputRows) t -), -rfOutputRows as ( -select case unit when "B" then value0*1000000000 when "M" then value0*1000000 when "K" then value0*1000 else value1 end as value from (select right(value,1) as unit, cast(substr(value, 1, length(value)-1) as double) value0, cast(value as double) value1 from JoinRuntimeFilterOutputRows) t -) -select assert_true(rfOutputRows.value/rfInputRows.value < 0.5) from rfInputRows,rfOutputRows; -select assert_true(count(distinct result)=1) from result_table;