Merge branch 'awslabs:master' into master

porscheofficial · Nov 11, 2024 · 61480c7 · 61480c7
2 parents 17da3a9 + f6d3620
commit 61480c7
Show file tree

Hide file tree

Showing 15 changed files with 82 additions and 27 deletions.
diff --git a/athena-clickhouse/pom.xml b/athena-clickhouse/pom.xml
@@ -22,7 +22,7 @@
         <dependency>
             <groupId>com.clickhouse</groupId>
             <artifactId>clickhouse-jdbc</artifactId>
-            <version>0.6.5</version>
+            <version>0.7.1-patch1</version>
             <classifier>all</classifier>
         </dependency>
         <dependency>

diff --git a/...a-hive/src/main/java/com/amazonaws/athena/connectors/cloudera/HiveQueryStringBuilder.java b/...a-hive/src/main/java/com/amazonaws/athena/connectors/cloudera/HiveQueryStringBuilder.java
@@ -22,7 +22,10 @@
 import com.amazonaws.athena.connector.lambda.domain.Split;
 import com.amazonaws.athena.connectors.jdbc.manager.FederationExpressionParser;
 import com.amazonaws.athena.connectors.jdbc.manager.JdbcSplitQueryBuilder;
+import com.amazonaws.athena.connectors.jdbc.manager.TypeAndValue;
 import com.google.common.base.Strings;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.pojo.ArrowType;
 
 import java.util.Collections;
 import java.util.List;
@@ -56,4 +59,21 @@ protected List<String> getPartitionWhereClauses(Split split)
         }
         return Collections.emptyList();
     }
+
+    protected String toPredicate(String columnName, String operator, Object value, ArrowType type,
+                                 List<TypeAndValue> accumulator)
+    {
+        Types.MinorType minorType = Types.getMinorTypeForArrowType(type);
+        if (minorType.equals(Types.MinorType.DATEMILLI) && isOperatorEquals(operator)) {
+            accumulator.add(new TypeAndValue(type, value));
+            return quote(columnName) + " " + operator + " cast(? as timestamp)";
+        }
+        // Default to parent's behavior
+        return super.toPredicate(columnName, operator, value, type, accumulator);
+    }
+
+    private boolean isOperatorEquals(String operator)
+    {
+        return operator.equals("=");
+    }
 }
diff --git a/...ra-hive/src/test/java/com/amazonaws/athena/connectors/cloudera/HiveRecordHandlerTest.java b/...ra-hive/src/test/java/com/amazonaws/athena/connectors/cloudera/HiveRecordHandlerTest.java
@@ -51,6 +51,10 @@
 import java.sql.Date;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 
@@ -136,4 +140,37 @@ public void buildSplitSql()
         Mockito.verify(preparedStatement, Mockito.times(1))
                 .setDate(1, expectedDate);
     }
+
+    @Test
+    public void buildSplitSqlTimestamp()
+            throws SQLException
+    {
+        TableName tableName = new TableName("testSchema", "testTable");
+
+        SchemaBuilder schemaBuilder = SchemaBuilder.newBuilder();
+        schemaBuilder.addField(FieldBuilder.newBuilder("testCol1", Types.MinorType.DATEMILLI.getType()).build());
+        schemaBuilder.addField(FieldBuilder.newBuilder("partition", Types.MinorType.VARCHAR.getType()).build());
+        Schema schema = schemaBuilder.build();
+
+        Split split = Mockito.mock(Split.class);
+        Mockito.when(split.getProperties()).thenReturn(Collections.singletonMap("partition", "p0"));
+        Mockito.when(split.getProperty(Mockito.eq("partition"))).thenReturn("p0");
+
+        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+        LocalDateTime timestamp = LocalDateTime.parse("2024-10-03 12:34:56", formatter);
+        ValueSet valueSet2 = getSingleValueSet(timestamp);
+        Constraints constraints = Mockito.mock(Constraints.class);
+        Mockito.when(constraints.getSummary()).thenReturn(new ImmutableMap.Builder<String, ValueSet>()
+                .put("testCol1", valueSet2)
+                .build());
+        PreparedStatement expectedPreparedStatement = Mockito.mock(PreparedStatement.class);
+        Mockito.when(this.connection.prepareStatement(nullable(String.class))).thenReturn(expectedPreparedStatement);
+        PreparedStatement preparedStatement = this.hiveRecordHandler.buildSplitSql(this.connection, "testCatalogName", tableName, schema, constraints, split);
+        Assert.assertEquals(expectedPreparedStatement, preparedStatement);
+        LocalDateTime timestampExp = LocalDateTime.parse("2024-10-03 12:34:56", formatter);
+        Timestamp expectedTimestamp = new Timestamp(timestamp.toInstant(ZoneOffset.UTC).toEpochMilli());
+        Assert.assertEquals(expectedPreparedStatement, preparedStatement);
+        Mockito.verify(preparedStatement, Mockito.times(1))
+                .setTimestamp(1, expectedTimestamp);
+    }
 }
diff --git a/athena-cloudwatch/pom.xml b/athena-cloudwatch/pom.xml
@@ -31,7 +31,7 @@
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>cloudwatchlogs</artifactId>
-            <version>2.29.1</version>
+            <version>2.29.9</version>
             <exclusions>
                 <!-- replaced with jcl-over-slf4j -->
                 <exclusion>

diff --git a/...src/main/java/com/amazonaws/athena/connectors/elasticsearch/ElasticsearchSchemaUtils.java b/...src/main/java/com/amazonaws/athena/connectors/elasticsearch/ElasticsearchSchemaUtils.java
@@ -162,6 +162,8 @@ private static FieldType toFieldType(Map<String, Object> mapping)
             case "byte":
                 minorType = Types.MinorType.TINYINT;
                 break;
+            case "float":
+            case "half_float":
             case "double":
                 minorType = Types.MinorType.FLOAT8;
                 break;
@@ -170,10 +172,6 @@ private static FieldType toFieldType(Map<String, Object> mapping)
                 minorType = Types.MinorType.BIGINT;
                 metadata.put("scaling_factor", mapping.get("scaling_factor").toString());
                 break;
-            case "float":
-            case "half_float":
-                minorType = Types.MinorType.FLOAT4;
-                break;
             case "date":
             case "date_nanos":
                 minorType = Types.MinorType.DATEMILLI;

diff --git a/.../java/com/amazonaws/athena/connectors/elasticsearch/ElasticsearchMetadataHandlerTest.java b/.../java/com/amazonaws/athena/connectors/elasticsearch/ElasticsearchMetadataHandlerTest.java
@@ -283,8 +283,8 @@ public void doGetTable()
                 .addField(new Field("myscaled",
                         new FieldType(true, Types.MinorType.BIGINT.getType(), null,
                                 ImmutableMap.of("scaling_factor", "10.0")), null))
-                .addField("myfloat", Types.MinorType.FLOAT4.getType())
-                .addField("myhalf", Types.MinorType.FLOAT4.getType())
+                .addField("myfloat", Types.MinorType.FLOAT8.getType())
+                .addField("myhalf", Types.MinorType.FLOAT8.getType())
                 .addField("mydatemilli", Types.MinorType.DATEMILLI.getType())
                 .addField("mydatenano", Types.MinorType.DATEMILLI.getType())
                 .addField("myboolean", Types.MinorType.BIT.getType())
@@ -328,10 +328,10 @@ public void doGetTable()
                 "          \"type\" : \"double\"\n" +                   // type: double (FLOAT8)
                 "        },\n" +
                 "        \"myfloat\" : {\n" +                           // myfloat:
-                "          \"type\" : \"float\"\n" +                    // type: float (FLOAT4)
+                "          \"type\" : \"float\"\n" +                    // type: float (FLOAT8)
                 "        },\n" +
                 "        \"myhalf\" : {\n" +                            // myhalf:
-                "          \"type\" : \"half_float\"\n" +               // type: half_float (FLOAT4)
+                "          \"type\" : \"half_float\"\n" +               // type: half_float (FLOAT8)
                 "        },\n" +
                 "        \"myinteger\" : {\n" +                         // myinteger:
                 "          \"type\" : \"integer\"\n" +                  // type: integer (INT)

diff --git a/...test/java/com/amazonaws/athena/connectors/elasticsearch/ElasticsearchSchemaUtilsTest.java b/...test/java/com/amazonaws/athena/connectors/elasticsearch/ElasticsearchSchemaUtilsTest.java
@@ -69,8 +69,8 @@ public void setUp()
                 .addField(new Field("myscaled",
                         new FieldType(true, Types.MinorType.BIGINT.getType(), null,
                                 ImmutableMap.of("scaling_factor", "10.0")), null))
-                .addField("myfloat", Types.MinorType.FLOAT4.getType())
-                .addField("myhalf", Types.MinorType.FLOAT4.getType())
+                .addField("myfloat", Types.MinorType.FLOAT8.getType())
+                .addField("myhalf", Types.MinorType.FLOAT8.getType())
                 .addField("mydatemilli", Types.MinorType.DATEMILLI.getType())
                 .addField("mydatenano", Types.MinorType.DATEMILLI.getType())
                 .addField("myboolean", Types.MinorType.BIT.getType())
@@ -111,10 +111,10 @@ public void setUp()
                         "    \"type\" : \"double\"\n" +                   // type: double (FLOAT8)
                         "  },\n" +
                         "  \"myfloat\" : {\n" +                           // myfloat:
-                        "    \"type\" : \"float\"\n" +                    // type: float (FLOAT4)
+                        "    \"type\" : \"float\"\n" +                    // type: float (FLOAT8)
                         "  },\n" +
                         "  \"myhalf\" : {\n" +                            // myhalf:
-                        "    \"type\" : \"half_float\"\n" +               // type: half_float (FLOAT4)
+                        "    \"type\" : \"half_float\"\n" +               // type: half_float (FLOAT8)
                         "  },\n" +
                         "  \"myinteger\" : {\n" +                         // myinteger:
                         "    \"type\" : \"integer\"\n" +                  // type: integer (INT)

diff --git a/athena-google-bigquery/pom.xml b/athena-google-bigquery/pom.xml
@@ -66,7 +66,7 @@
         <dependency>
             <groupId>nl.jqno.equalsverifier</groupId>
             <artifactId>equalsverifier</artifactId>
-            <version>3.17.1</version>
+            <version>3.17.3</version>
             <scope>test</scope>
         </dependency>
         <dependency>

diff --git a/...dbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcSplitQueryBuilder.java b/...dbc/src/main/java/com/amazonaws/athena/connectors/jdbc/manager/JdbcSplitQueryBuilder.java
@@ -336,8 +336,8 @@ else if (singleValues.size() > 1) {
         return "(" + Joiner.on(" OR ").join(disjuncts) + ")";
     }
 
-    private String toPredicate(String columnName, String operator, Object value, ArrowType type,
-            List<TypeAndValue> accumulator)
+    protected String toPredicate(String columnName, String operator, Object value, ArrowType type,
+                                 List<TypeAndValue> accumulator)
     {
         accumulator.add(new TypeAndValue(type, value));
         return quote(columnName) + " " + operator + " ?";

diff --git a/athena-neptune/pom.xml b/athena-neptune/pom.xml
@@ -10,7 +10,7 @@
     <version>2022.47.1</version>
     <properties>
         <!-- make sure gremlin driver version stays within the Neptune supported range -->
-        <gremlinDriverVersion>3.7.2</gremlinDriverVersion>
+        <gremlinDriverVersion>3.7.3</gremlinDriverVersion>
         <neptune.sigv4.signer.version>2.4.0</neptune.sigv4.signer.version>
     </properties>
     <dependencies>
@@ -124,7 +124,7 @@
         <dependency>
             <groupId>org.eclipse.rdf4j</groupId>
             <artifactId>rdf4j-repository-sparql</artifactId>
-            <version>5.0.2</version>
+            <version>5.0.3</version>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>

diff --git a/athena-oracle/pom.xml b/athena-oracle/pom.xml
@@ -30,7 +30,7 @@
         <dependency>
             <groupId>com.oracle.database.jdbc</groupId>
             <artifactId>ojdbc8</artifactId>
-            <version>23.5.0.24.07</version>
+            <version>23.6.0.24.10</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/software.amazon.awssdk/rds -->
         <dependency>

diff --git a/athena-redis/pom.xml b/athena-redis/pom.xml
@@ -25,7 +25,7 @@
         <dependency>
             <groupId>io.lettuce</groupId>
             <artifactId>lettuce-core</artifactId>
-            <version>6.4.0.RELEASE</version>
+            <version>6.5.0.RELEASE</version>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>

diff --git a/...hana/src/main/java/com/amazonaws/athena/connectors/saphana/SaphanaQueryStringBuilder.java b/...hana/src/main/java/com/amazonaws/athena/connectors/saphana/SaphanaQueryStringBuilder.java
@@ -350,7 +350,7 @@ else if (singleValues.size() > 1) {
         return "(" + Joiner.on(" OR ").join(disjuncts) + ")";
     }
 
-    private String toPredicate(String columnName, String operator, Object value, ArrowType type,
+    protected String toPredicate(String columnName, String operator, Object value, ArrowType type,
                                List<TypeAndValue> accumulator)
     {
         accumulator.add(new TypeAndValue(type, value));

diff --git a/athena-snowflake/pom.xml b/athena-snowflake/pom.xml
@@ -30,7 +30,7 @@
         <dependency>
             <groupId>net.snowflake</groupId>
             <artifactId>snowflake-jdbc</artifactId>
-            <version>3.19.1</version>
+            <version>3.20.0</version>
         </dependency>
         <!-- https://mvnrepository.com/artifact/software.amazon.awssdk/rds -->
         <dependency>

diff --git a/pom.xml b/pom.xml
@@ -14,7 +14,7 @@
         <maven.compiler.release>11</maven.compiler.release>
         <maven.compiler.plugin.version>3.13.0</maven.compiler.plugin.version>
         <!--- to meet engine version 1.12.533-->
-        <aws-sdk-v2.version>2.29.1</aws-sdk-v2.version>
+        <aws-sdk-v2.version>2.29.9</aws-sdk-v2.version>
         <aws.lambda-java-core.version>1.2.2</aws.lambda-java-core.version>
         <aws.lambda-java-log4j2.version>1.6.0</aws.lambda-java-log4j2.version>
         <aws-cdk.version>1.204.0</aws-cdk.version>
@@ -27,8 +27,8 @@
         <assertj.version>3.26.3</assertj.version>
         <testng.version>7.10.2</testng.version>
          <!--- to meet engine version 2.12.6 -->
-        <fasterxml.jackson.version>2.18.0</fasterxml.jackson.version>
-        <surefire.failsafe.version>3.5.1</surefire.failsafe.version>
+        <fasterxml.jackson.version>2.18.1</fasterxml.jackson.version>
+        <surefire.failsafe.version>3.5.2</surefire.failsafe.version>
         <log4j2Version>2.24.1</log4j2Version>
         <apache.arrow.version>13.0.0</apache.arrow.version>
         <guava.version>33.3.1-jre</guava.version>
@@ -44,7 +44,7 @@
         <mvn.checkstyle.version>3.6.0</mvn.checkstyle.version>
         <mvn.shade.plugin.version>3.6.0</mvn.shade.plugin.version>
         <mvn.source.plugin.version>3.3.1</mvn.source.plugin.version>
-        <mvn.javadoc.plugin.version>3.10.1</mvn.javadoc.plugin.version>
+        <mvn.javadoc.plugin.version>3.11.1</mvn.javadoc.plugin.version>
         <mvn.jar.plugin.version>3.4.2</mvn.jar.plugin.version>
         <doclint>none</doclint>
     </properties>