From 4ab1e25d4e5d05c559f5d6ae4047c15f044fb29c Mon Sep 17 00:00:00 2001 From: dmitrybugakov Date: Fri, 22 Sep 2023 16:12:08 +0200 Subject: [PATCH] [#442]: Support LowCardinality series datatype --- .../housepower/data/DataTypeFactory.java | 18 +-- .../type/complex/DataTypeLowCardinality.java | 115 ++++++++++++++++++ .../housepower/settings/SettingKey.java | 6 + .../jdbc/LowCardinalityTypeTest.java | 114 +++++++++++++++++ 4 files changed, 239 insertions(+), 14 deletions(-) create mode 100644 clickhouse-native-jdbc/src/main/java/com/github/housepower/data/type/complex/DataTypeLowCardinality.java create mode 100644 clickhouse-native-jdbc/src/test/java/com/github/housepower/jdbc/LowCardinalityTypeTest.java diff --git a/clickhouse-native-jdbc/src/main/java/com/github/housepower/data/DataTypeFactory.java b/clickhouse-native-jdbc/src/main/java/com/github/housepower/data/DataTypeFactory.java index bc9cfcfb..4f3d6f1d 100644 --- a/clickhouse-native-jdbc/src/main/java/com/github/housepower/data/DataTypeFactory.java +++ b/clickhouse-native-jdbc/src/main/java/com/github/housepower/data/DataTypeFactory.java @@ -34,19 +34,7 @@ import com.github.housepower.data.type.DataTypeUInt64; import com.github.housepower.data.type.DataTypeUInt8; import com.github.housepower.data.type.DataTypeUUID; -import com.github.housepower.data.type.complex.DataTypeArray; -import com.github.housepower.data.type.complex.DataTypeCreator; -import com.github.housepower.data.type.complex.DataTypeDateTime; -import com.github.housepower.data.type.complex.DataTypeDateTime64; -import com.github.housepower.data.type.complex.DataTypeDecimal; -import com.github.housepower.data.type.complex.DataTypeEnum16; -import com.github.housepower.data.type.complex.DataTypeEnum8; -import com.github.housepower.data.type.complex.DataTypeFixedString; -import com.github.housepower.data.type.complex.DataTypeMap; -import com.github.housepower.data.type.complex.DataTypeNothing; -import com.github.housepower.data.type.complex.DataTypeNullable; -import com.github.housepower.data.type.complex.DataTypeString; -import com.github.housepower.data.type.complex.DataTypeTuple; +import com.github.housepower.data.type.complex.*; import com.github.housepower.misc.LRUCache; import com.github.housepower.misc.SQLLexer; import com.github.housepower.misc.Validate; @@ -89,6 +77,8 @@ public class DataTypeFactory { return DataTypeDateTime64.creator.createDataType(lexer, serverContext); } else if (dataTypeName.equalsIgnoreCase("Nullable")) { return DataTypeNullable.creator.createDataType(lexer, serverContext); + } else if (dataTypeName.equalsIgnoreCase("LowCardinality")) { + return DataTypeLowCardinality.creator.createDataType(lexer, serverContext); } else if (dataTypeName.equalsIgnoreCase("FixedString") || dataTypeName.equals("Binary")) { return DataTypeFixedString.creator.createDataType(lexer, serverContext); } else if (dataTypeName.equalsIgnoreCase("Decimal")) { @@ -98,7 +88,7 @@ public class DataTypeFactory { } else if (dataTypeName.equalsIgnoreCase("Nothing")) { return DataTypeNothing.CREATOR.createDataType(lexer, serverContext); } else if (dataTypeName.equalsIgnoreCase("Map")) { - return DataTypeMap.creator.createDataType(lexer, serverContext); + return DataTypeMap.creator.createDataType(lexer, serverContext); } else { IDataType dataType = dataTypes.get(dataTypeName.toLowerCase(Locale.ROOT)); Validate.isTrue(dataType != null, "Unknown data type: " + dataTypeName); diff --git a/clickhouse-native-jdbc/src/main/java/com/github/housepower/data/type/complex/DataTypeLowCardinality.java b/clickhouse-native-jdbc/src/main/java/com/github/housepower/data/type/complex/DataTypeLowCardinality.java new file mode 100644 index 00000000..f40bd50a --- /dev/null +++ b/clickhouse-native-jdbc/src/main/java/com/github/housepower/data/type/complex/DataTypeLowCardinality.java @@ -0,0 +1,115 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.housepower.data.type.complex; + +import com.github.housepower.data.DataTypeFactory; +import com.github.housepower.data.IDataType; +import com.github.housepower.misc.SQLLexer; +import com.github.housepower.misc.Validate; +import com.github.housepower.serde.BinaryDeserializer; +import com.github.housepower.serde.BinarySerializer; + +import java.io.IOException; +import java.sql.SQLException; + +public class DataTypeLowCardinality implements IDataType { + + public static DataTypeCreator creator = (lexer, serverContext) -> { + Validate.isTrue(lexer.character() == '('); + IDataType nestedType = DataTypeFactory.get(lexer, serverContext); + Validate.isTrue(lexer.character() == ')'); + return new DataTypeLowCardinality( + "LowCardinality(" + nestedType.name() + ")", nestedType); + }; + + private final String name; + private final IDataType nestedDataType; + + public DataTypeLowCardinality(String name, IDataType nestedDataType) { + this.name = name; + this.nestedDataType = nestedDataType; + } + + @Override + public String name() { + return this.name; + } + + @Override + public int sqlTypeId() { + return this.nestedDataType.sqlTypeId(); + } + + @Override + public Object defaultValue() { + return this.nestedDataType.defaultValue(); + } + + @Override + public Class javaType() { + return this.nestedDataType.javaType(); + } + + @Override + public Class jdbcJavaType() { + return this.nestedDataType.jdbcJavaType(); + } + + @Override + public boolean nullable() { + return this.nestedDataType.nullable(); + } + + @Override + public int getPrecision() { + return this.nestedDataType.getPrecision(); + } + + @Override + public int getScale() { + return this.nestedDataType.getScale(); + } + + @Override + public Object deserializeText(SQLLexer lexer) throws SQLException { + return this.nestedDataType.deserializeText(lexer); + } + + @Override + public void serializeBinary(Object data, BinarySerializer serializer) throws SQLException, IOException { + this.nestedDataType.serializeBinary(data, serializer); + } + + @Override + public void serializeBinaryBulk(Object[] data, BinarySerializer serializer) throws SQLException, IOException { + this.nestedDataType.serializeBinaryBulk(data, serializer); + } + + @Override + public Object deserializeBinary(BinaryDeserializer deserializer) throws SQLException, IOException { + return this.nestedDataType.deserializeBinary(deserializer); + } + + @Override + public Object[] deserializeBinaryBulk(int rows, BinaryDeserializer deserializer) throws SQLException, IOException { + Object[] data = this.nestedDataType.deserializeBinaryBulk(rows, deserializer); + return data; + } + + @Override + public boolean isSigned() { + return this.nestedDataType.isSigned(); + } +} diff --git a/clickhouse-native-jdbc/src/main/java/com/github/housepower/settings/SettingKey.java b/clickhouse-native-jdbc/src/main/java/com/github/housepower/settings/SettingKey.java index 47ed5720..1b1d921e 100644 --- a/clickhouse-native-jdbc/src/main/java/com/github/housepower/settings/SettingKey.java +++ b/clickhouse-native-jdbc/src/main/java/com/github/housepower/settings/SettingKey.java @@ -791,6 +791,12 @@ public class SettingKey implements Serializable { .withDescription("Enable conditional computations") .build(); + public static SettingKey allow_suspicious_low_cardinality_types = SettingKey.builder() + .withName("allow_suspicious_low_cardinality_types") + .withType(SettingType.Int32) + .withDescription("Permits the use of LowCardinality with data types that might negatively impact performance") + .build(); + public static SettingKey allow_experimental_bigint_types = SettingKey.builder() .withName("allow_experimental_bigint_types") .withType(SettingType.Int64) diff --git a/clickhouse-native-jdbc/src/test/java/com/github/housepower/jdbc/LowCardinalityTypeTest.java b/clickhouse-native-jdbc/src/test/java/com/github/housepower/jdbc/LowCardinalityTypeTest.java new file mode 100644 index 00000000..b36aa805 --- /dev/null +++ b/clickhouse-native-jdbc/src/test/java/com/github/housepower/jdbc/LowCardinalityTypeTest.java @@ -0,0 +1,114 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.github.housepower.jdbc; + +import com.github.housepower.misc.BytesHelper; +import org.junit.jupiter.api.Test; + +import java.sql.*; + +import static org.junit.jupiter.api.Assertions.*; + +//Refer to [[https://github.com/housepower/ClickHouse-Native-JDBC/issues/442]] for more details. +public class LowCardinalityTypeTest extends AbstractITest implements BytesHelper { + @Test + public void testAllLowCardinalityTypes() throws Exception { + withStatement(statement -> { + statement.execute("DROP TABLE IF EXISTS low_cardinality_test"); + + StringBuilder createTableSQL = new StringBuilder(); + createTableSQL.append("CREATE TABLE IF NOT EXISTS low_cardinality_test (") + .append("value_string LowCardinality(String), ") + .append("fixed_string LowCardinality(FixedString(10)), ") + .append("date_value LowCardinality(Nullable(Date)), ") + .append("datetime_value LowCardinality(Nullable(DateTime)), ") + .append("number_value LowCardinality(Nullable(Int32))) Engine=Memory()"); + + statement.execute(createTableSQL.toString()); + + String sql = "INSERT INTO low_cardinality_test " + + "(value_string, fixed_string, date_value, datetime_value, number_value) values(?, ?, ?, ?, ?);"; + + try (PreparedStatement pstmt = statement.getConnection().prepareStatement(sql)) { + for (int i = 0; i < 300; i++) { + pstmt.setString(1, "test"); + pstmt.setString(2, "abcdefghij"); + if (i % 50 == 0) { + pstmt.setNull(3, Types.DATE); + pstmt.setNull(4, Types.TIMESTAMP); + pstmt.setNull(5, Types.INTEGER); + } else { + pstmt.setDate(3, new java.sql.Date(System.currentTimeMillis())); + pstmt.setTimestamp(4, new java.sql.Timestamp(System.currentTimeMillis())); + pstmt.setInt(5, i); + } + pstmt.addBatch(); + } + pstmt.executeBatch(); + } + + DatabaseMetaData metaData = statement.getConnection().getMetaData(); + ResultSet columns = metaData.getColumns(null, "default", "low_cardinality_test", "%"); + while (columns.next()) { + String columnName = columns.getString("COLUMN_NAME"); + String columnType = columns.getString("TYPE_NAME"); + switch (columnName) { + case "value_string": + assertEquals(columnType, "LowCardinality(String)"); + break; + case "fixed_string": + assertEquals(columnType, "LowCardinality(FixedString(10))"); + break; + case "date_value": + assertEquals(columnType, "LowCardinality(Nullable(Date))"); + break; + case "datetime_value": + assertEquals(columnType, "LowCardinality(Nullable(DateTime))"); + break; + case "number_value": + assertEquals(columnType, "LowCardinality(Nullable(Int32))"); + break; + } + } + + ResultSet rs = statement.executeQuery("SELECT * FROM low_cardinality_test;"); + int size = 0; + while (rs.next()) { + String valueStr = rs.getString("value_string"); + String fixedStr = rs.getString("fixed_string"); + Date dateValue = rs.getDate("date_value"); + Timestamp datetimeValue = rs.getTimestamp("datetime_value"); + Integer numberValue = (Integer) rs.getObject("number_value"); + + assertEquals("test", valueStr); + assertEquals("abcdefghij", fixedStr); + if (size % 50 == 0) { + assertNull(dateValue); + assertNull(datetimeValue); + assertNull(numberValue); + } else { + assertNotNull(dateValue); + assertNotNull(datetimeValue); + assertTrue(numberValue >= 0 && numberValue < 300); + } + + size++; + } + assertEquals(300, size); + + statement.execute("DROP TABLE IF EXISTS low_cardinality_test"); + }, "allow_suspicious_low_cardinality_types", "1"); + } +}