diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1abf3cc93a2..7087522eee63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -695,6 +695,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.TRINO_AWS_SECRET_ACCESS_KEY }} AWS_REGION: ${{ vars.TRINO_AWS_REGION }} S3_BUCKET: ${{ vars.TRINO_S3_BUCKET }} + S3_TABLES_BUCKET: ${{ vars.TRINO_S3_TABLES_BUCKET }} GCP_CREDENTIALS_KEY: ${{ secrets.GCP_CREDENTIALS_KEY }} GCP_STORAGE_BUCKET: ${{ vars.GCP_STORAGE_BUCKET }} ABFS_CONTAINER: ${{ vars.AZURE_ABFS_HIERARCHICAL_CONTAINER }} diff --git a/plugin/trino-iceberg/README.md b/plugin/trino-iceberg/README.md new file mode 100644 index 000000000000..0a4d5ad7d93f --- /dev/null +++ b/plugin/trino-iceberg/README.md @@ -0,0 +1,30 @@ +# Iceberg Connector Developer Notes + +Steps to create TPCH tables on S3 Tables: +1. Set `AWS_REGION`, `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables. +2. Replace placeholders in the following command and run it: +```sh +./spark-sql \ +--packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,software.amazon.awssdk:bundle:2.20.10,software.amazon.s3tables:s3-tables-catalog-for-iceberg-runtime:0.1.3,org.apache.kyuubi:kyuubi-spark-connector-tpch_2.12:1.8.0 \ +--conf spark.sql.catalog.s3tablesbucket=org.apache.iceberg.spark.SparkCatalog \ +--conf spark.sql.catalog.s3tablesbucket.catalog-impl=software.amazon.s3tables.iceberg.S3TablesCatalog \ +--conf spark.sql.catalog.s3tablesbucket.warehouse=arn:aws:s3tables:{region}:{account-id}:bucket/{bucket-name} \ +--conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ +--conf spark.sql.catalog.tpch=org.apache.kyuubi.spark.connector.tpch.TPCHCatalog +``` + +3. Run the following command to create TPCH tables: +```sql +CREATE TABLE s3tablesbucket.tpch.nation AS SELECT + n_nationkey AS nationkey, + n_name AS name, + n_regionkey AS regionkey, + n_comment AS comment +FROM tpch.tiny.nation; + +CREATE TABLE s3tablesbucket.tpch.region AS SELECT + r_regionkey AS regionkey, + r_name AS name, + r_comment AS comment +FROM tpch.tiny.region; +``` diff --git a/plugin/trino-iceberg/pom.xml b/plugin/trino-iceberg/pom.xml index ce70b35219c8..620a5f22fc94 100644 --- a/plugin/trino-iceberg/pom.xml +++ b/plugin/trino-iceberg/pom.xml @@ -130,6 +130,11 @@ trino-filesystem-manager + + io.trino + trino-filesystem-s3 + + io.trino trino-hive @@ -218,6 +223,11 @@ iceberg-api + + org.apache.iceberg + iceberg-aws + + org.apache.iceberg iceberg-core @@ -359,12 +369,6 @@ runtime - - io.trino - trino-filesystem-s3 - runtime - - jakarta.servlet jakarta.servlet-api diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java index c7370d65d810..d761e03f0e60 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/TrinoCatalog.java @@ -25,6 +25,7 @@ import io.trino.spi.connector.RelationCommentMetadata; import io.trino.spi.connector.SchemaTableName; import io.trino.spi.security.TrinoPrincipal; +import jakarta.annotation.Nullable; import org.apache.iceberg.BaseTable; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; @@ -159,6 +160,7 @@ Transaction newCreateOrReplaceTableTransaction( void updateViewColumnComment(ConnectorSession session, SchemaTableName schemaViewName, String columnName, Optional comment); + @Nullable String defaultTableLocation(ConnectorSession session, SchemaTableName schemaTableName); void setTablePrincipal(ConnectorSession session, SchemaTableName schemaTableName, TrinoPrincipal principal); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/AwsProperties.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/AwsProperties.java new file mode 100644 index 000000000000..c6b34e58c127 --- /dev/null +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/AwsProperties.java @@ -0,0 +1,21 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg.catalog.rest; + +import java.util.Map; + +public interface AwsProperties +{ + Map get(); +} diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogConfig.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogConfig.java index 0390d9ab38f4..e02889c70a9e 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogConfig.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogConfig.java @@ -47,6 +47,8 @@ public enum SessionType private Security security = Security.NONE; private SessionType sessionType = SessionType.NONE; private boolean vendedCredentialsEnabled; + private boolean viewEndpointsEnabled = true; + private boolean sigv4Enabled; private boolean caseInsensitiveNameMatching; private Duration caseInsensitiveNameMatchingCacheTtl = new Duration(1, MINUTES); @@ -146,6 +148,32 @@ public IcebergRestCatalogConfig setVendedCredentialsEnabled(boolean vendedCreden return this; } + public boolean isViewEndpointsEnabled() + { + return viewEndpointsEnabled; + } + + @Config("iceberg.rest-catalog.view-endpoints-enabled") + @ConfigDescription("Enable view endpoints") + public IcebergRestCatalogConfig setViewEndpointsEnabled(boolean viewEndpointsEnabled) + { + this.viewEndpointsEnabled = viewEndpointsEnabled; + return this; + } + + public boolean isSigv4Enabled() + { + return sigv4Enabled; + } + + @Config("iceberg.rest-catalog.sigv4-enabled") + @ConfigDescription("Enable AWSSignature Version 4 (SigV4)") + public IcebergRestCatalogConfig setSigv4Enabled(boolean sigv4Enabled) + { + this.sigv4Enabled = sigv4Enabled; + return this; + } + public boolean isCaseInsensitiveNameMatching() { return caseInsensitiveNameMatching; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogModule.java index 4753b5c8cb12..384f77bbbdc1 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogModule.java @@ -18,6 +18,7 @@ import io.airlift.configuration.AbstractConfigurationAwareModule; import io.trino.plugin.iceberg.IcebergConfig; import io.trino.plugin.iceberg.IcebergFileSystemFactory; +import io.trino.plugin.iceberg.IcebergSecurityConfig; import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory; import io.trino.plugin.iceberg.catalog.rest.IcebergRestCatalogConfig.Security; import io.trino.spi.TrinoException; @@ -25,6 +26,7 @@ import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; import static io.airlift.configuration.ConditionalModule.conditionalModule; import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.trino.plugin.iceberg.IcebergSecurityConfig.IcebergSecurity.READ_ONLY; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; public class IcebergRestCatalogModule @@ -39,6 +41,15 @@ protected void setup(Binder binder) config -> config.getSecurity() == Security.OAUTH2, new OAuth2SecurityModule(), new NoneSecurityModule())); + install(conditionalModule( + IcebergRestCatalogConfig.class, + IcebergRestCatalogConfig::isSigv4Enabled, + internalBinder -> { + configBinder(internalBinder).bindConfig(IcebergRestCatalogSigv4Config.class); + configBinder(internalBinder).bindConfigDefaults(IcebergSecurityConfig.class, config -> config.setSecuritySystem(READ_ONLY)); + internalBinder.bind(AwsProperties.class).to(SigV4AwsProperties.class).in(Scopes.SINGLETON); + }, + internalBinder -> internalBinder.bind(AwsProperties.class).to(NoneAwsProperties.class))); binder.bind(TrinoCatalogFactory.class).to(TrinoIcebergRestCatalogFactory.class).in(Scopes.SINGLETON); newOptionalBinder(binder, IcebergFileSystemFactory.class).setBinding().to(IcebergRestCatalogFileSystemFactory.class).in(Scopes.SINGLETON); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogSigv4Config.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogSigv4Config.java new file mode 100644 index 000000000000..0dfc181edbf9 --- /dev/null +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/IcebergRestCatalogSigv4Config.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg.catalog.rest; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import jakarta.validation.constraints.NotNull; +import org.apache.iceberg.aws.AwsProperties; + +public class IcebergRestCatalogSigv4Config +{ + private String signingName = AwsProperties.REST_SIGNING_NAME_DEFAULT; + + @NotNull + public String getSigningName() + { + return signingName; + } + + @Config("iceberg.rest-catalog.signing-name") + @ConfigDescription("The service name to be used by the SigV4 protocol for signing requests") + + public IcebergRestCatalogSigv4Config setSigningName(String signingName) + { + this.signingName = signingName; + return this; + } +} diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/NoneAwsProperties.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/NoneAwsProperties.java new file mode 100644 index 000000000000..6d1156e5b322 --- /dev/null +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/NoneAwsProperties.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg.catalog.rest; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; + +public class NoneAwsProperties + implements AwsProperties +{ + @Override + public Map get() + { + return ImmutableMap.of(); + } +} diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/SigV4AwsProperties.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/SigV4AwsProperties.java new file mode 100644 index 000000000000..40324ec7daac --- /dev/null +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/SigV4AwsProperties.java @@ -0,0 +1,45 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg.catalog.rest; + +import com.google.common.collect.ImmutableMap; +import com.google.inject.Inject; +import io.trino.filesystem.s3.S3FileSystemConfig; + +import java.util.Map; + +public class SigV4AwsProperties + implements AwsProperties +{ + private final Map properties; + + @Inject + public SigV4AwsProperties(IcebergRestCatalogSigv4Config sigv4Config, S3FileSystemConfig s3Config) + { + this.properties = ImmutableMap.builder() + .put("rest.sigv4-enabled", "true") + .put("rest.signing-name", sigv4Config.getSigningName()) + .put("rest.access-key-id", s3Config.getAwsAccessKey()) + .put("rest.secret-access-key", s3Config.getAwsSecretKey()) + .put("rest.signing-region", s3Config.getRegion()) + .put("rest-metrics-reporting-enabled", "false") + .buildOrThrow(); + } + + @Override + public Map get() + { + return properties; + } +} diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java index c3f47dffc2f0..a4cb154aa758 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoIcebergRestCatalogFactory.java @@ -57,7 +57,9 @@ public class TrinoIcebergRestCatalogFactory private final boolean nestedNamespaceEnabled; private final SessionType sessionType; private final boolean vendedCredentialsEnabled; + private final boolean viewEndpointsEnabled; private final SecurityProperties securityProperties; + private final AwsProperties awsProperties; private final boolean uniqueTableLocation; private final TypeManager typeManager; private final boolean caseInsensitiveNameMatching; @@ -73,6 +75,7 @@ public TrinoIcebergRestCatalogFactory( CatalogName catalogName, IcebergRestCatalogConfig restConfig, SecurityProperties securityProperties, + AwsProperties awsProperties, IcebergConfig icebergConfig, TypeManager typeManager, NodeVersion nodeVersion) @@ -87,7 +90,9 @@ public TrinoIcebergRestCatalogFactory( this.nestedNamespaceEnabled = restConfig.isNestedNamespaceEnabled(); this.sessionType = restConfig.getSessionType(); this.vendedCredentialsEnabled = restConfig.isVendedCredentialsEnabled(); + this.viewEndpointsEnabled = restConfig.isViewEndpointsEnabled(); this.securityProperties = requireNonNull(securityProperties, "securityProperties is null"); + this.awsProperties = requireNonNull(awsProperties, "awsProperties is null"); requireNonNull(icebergConfig, "icebergConfig is null"); this.uniqueTableLocation = icebergConfig.isUniqueTableLocation(); this.typeManager = requireNonNull(typeManager, "typeManager is null"); @@ -112,9 +117,10 @@ public synchronized TrinoCatalog create(ConnectorIdentity identity) properties.put(CatalogProperties.URI, serverUri.toString()); warehouse.ifPresent(location -> properties.put(CatalogProperties.WAREHOUSE_LOCATION, location)); prefix.ifPresent(prefix -> properties.put("prefix", prefix)); - properties.put("view-endpoints-supported", "true"); + properties.put("view-endpoints-supported", Boolean.toString(viewEndpointsEnabled)); properties.put("trino-version", trinoVersion); properties.putAll(securityProperties.get()); + properties.putAll(awsProperties.get()); if (vendedCredentialsEnabled) { properties.put("header.X-Iceberg-Access-Delegation", "vended-credentials"); diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java index d72f75d3f334..2360a2f5409b 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/rest/TrinoRestCatalog.java @@ -83,7 +83,6 @@ import java.util.function.UnaryOperator; import java.util.stream.Stream; -import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.collect.ImmutableList.toImmutableList; import static io.trino.cache.CacheUtils.uncheckedCacheGet; import static io.trino.filesystem.Locations.appendPath; @@ -477,7 +476,10 @@ public String defaultTableLocation(ConnectorSession session, SchemaTableName sch Map properties = loadNamespaceMetadata(session, schemaTableName.getSchemaName()); String databaseLocation = (String) properties.get(IcebergSchemaProperties.LOCATION_PROPERTY); - checkArgument(databaseLocation != null, "location must be set for %s", schemaTableName.getSchemaName()); + if (databaseLocation == null) { + // REST catalog spec doesn't ensure that a location property is present in the namespace metadata + return null; + } return appendPath(databaseLocation, tableName); } diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogConfig.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogConfig.java index 010e2878f194..c18967c72ca9 100644 --- a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogConfig.java +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogConfig.java @@ -37,6 +37,8 @@ public void testDefaults() .setSessionType(IcebergRestCatalogConfig.SessionType.NONE) .setSecurity(IcebergRestCatalogConfig.Security.NONE) .setVendedCredentialsEnabled(false) + .setViewEndpointsEnabled(true) + .setSigv4Enabled(false) .setCaseInsensitiveNameMatching(false) .setCaseInsensitiveNameMatchingCacheTtl(new Duration(1, MINUTES))); } @@ -52,6 +54,8 @@ public void testExplicitPropertyMappings() .put("iceberg.rest-catalog.security", "OAUTH2") .put("iceberg.rest-catalog.session", "USER") .put("iceberg.rest-catalog.vended-credentials-enabled", "true") + .put("iceberg.rest-catalog.view-endpoints-enabled", "false") + .put("iceberg.rest-catalog.sigv4-enabled", "true") .put("iceberg.rest-catalog.case-insensitive-name-matching", "true") .put("iceberg.rest-catalog.case-insensitive-name-matching.cache-ttl", "3m") .buildOrThrow(); @@ -64,6 +68,8 @@ public void testExplicitPropertyMappings() .setSessionType(IcebergRestCatalogConfig.SessionType.USER) .setSecurity(IcebergRestCatalogConfig.Security.OAUTH2) .setVendedCredentialsEnabled(true) + .setViewEndpointsEnabled(false) + .setSigv4Enabled(true) .setCaseInsensitiveNameMatching(true) .setCaseInsensitiveNameMatchingCacheTtl(new Duration(3, MINUTES)); diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogSigv4Config.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogSigv4Config.java new file mode 100644 index 000000000000..24fb67629cbf --- /dev/null +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergRestCatalogSigv4Config.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg.catalog.rest; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +final class TestIcebergRestCatalogSigv4Config +{ + @Test + void testDefaults() + { + assertRecordedDefaults(recordDefaults(IcebergRestCatalogSigv4Config.class) + .setSigningName("execute-api")); + } + + @Test + void testExplicitPropertyMappings() + { + Map properties = ImmutableMap.builder() + .put("iceberg.rest-catalog.signing-name", "glue") + .buildOrThrow(); + + IcebergRestCatalogSigv4Config expected = new IcebergRestCatalogSigv4Config() + .setSigningName("glue"); + + assertFullMapping(properties, expected); + } +} diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java new file mode 100644 index 000000000000..cd692aab96ac --- /dev/null +++ b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/rest/TestIcebergS3TablesConnectorSmokeTest.java @@ -0,0 +1,540 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.iceberg.catalog.rest; + +import io.trino.filesystem.Location; +import io.trino.plugin.iceberg.BaseIcebergConnectorSmokeTest; +import io.trino.plugin.iceberg.IcebergConfig; +import io.trino.plugin.iceberg.IcebergQueryRunner; +import io.trino.testing.QueryRunner; +import io.trino.testing.TestingConnectorBehavior; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import static io.trino.testing.SystemEnvironmentUtils.requireEnv; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +@TestInstance(PER_CLASS) +final class TestIcebergS3TablesConnectorSmokeTest + extends BaseIcebergConnectorSmokeTest +{ + public static final String S3_TABLES_BUCKET = requireEnv("S3_TABLES_BUCKET"); + public static final String AWS_ACCESS_KEY_ID = requireEnv("AWS_ACCESS_KEY_ID"); + public static final String AWS_SECRET_ACCESS_KEY = requireEnv("AWS_SECRET_ACCESS_KEY"); + public static final String AWS_REGION = requireEnv("AWS_REGION"); + + public TestIcebergS3TablesConnectorSmokeTest() + { + super(new IcebergConfig().getFileFormat().toIceberg()); + } + + @Override + protected boolean hasBehavior(TestingConnectorBehavior connectorBehavior) + { + return switch (connectorBehavior) { + case SUPPORTS_CREATE_MATERIALIZED_VIEW, + SUPPORTS_RENAME_MATERIALIZED_VIEW, + SUPPORTS_RENAME_SCHEMA -> false; + default -> super.hasBehavior(connectorBehavior); + }; + } + + @Override + protected QueryRunner createQueryRunner() + throws Exception + { + return IcebergQueryRunner.builder("tpch") + .addIcebergProperty("iceberg.security", "read_only") + .addIcebergProperty("iceberg.file-format", format.name()) + .addIcebergProperty("iceberg.register-table-procedure.enabled", "true") + .addIcebergProperty("iceberg.writer-sort-buffer-size", "1MB") + .addIcebergProperty("iceberg.allowed-extra-properties", "write.metadata.delete-after-commit.enabled,write.metadata.previous-versions-max") + .addIcebergProperty("iceberg.catalog.type", "rest") + .addIcebergProperty("iceberg.rest-catalog.uri", "https://glue.%s.amazonaws.com/iceberg".formatted(AWS_REGION)) + .addIcebergProperty("iceberg.rest-catalog.warehouse", "s3tablescatalog/" + S3_TABLES_BUCKET) + .addIcebergProperty("iceberg.rest-catalog.view-endpoints-enabled", "false") + .addIcebergProperty("iceberg.rest-catalog.sigv4-enabled", "true") + .addIcebergProperty("iceberg.rest-catalog.signing-name", "glue") + .addIcebergProperty("fs.hadoop.enabled", "false") + .addIcebergProperty("fs.native-s3.enabled", "true") + .addIcebergProperty("s3.region", AWS_REGION) + .addIcebergProperty("s3.aws-access-key", AWS_ACCESS_KEY_ID) + .addIcebergProperty("s3.aws-secret-key", AWS_SECRET_ACCESS_KEY) + .disableSchemaInitializer() + .build(); + } + + @Override + protected void dropTableFromMetastore(String tableName) + { + throw new UnsupportedOperationException(); + } + + @Override + protected String getMetadataLocation(String tableName) + { + throw new UnsupportedOperationException(); + } + + @Override + protected String schemaPath() + { + return "dummy"; + } + + @Override + protected boolean locationExists(String location) + { + throw new UnsupportedOperationException(); + } + + @Override + protected boolean isFileSorted(Location path, String sortColumnName) + { + throw new UnsupportedOperationException(); + } + + @Override + protected void deleteDirectory(String location) + { + throw new UnsupportedOperationException(); + } + + @Test + @Override // Override because the location pattern differs + public void testShowCreateTable() + { + assertThat((String) computeScalar("SHOW CREATE TABLE region")) + .matches("CREATE TABLE iceberg.tpch.region \\(\n" + + " regionkey bigint,\n" + + " name varchar,\n" + + " comment varchar\n" + + "\\)\n" + + "WITH \\(\n" + + " format = 'PARQUET',\n" + + " format_version = 2,\n" + + " location = 's3://.*--table-s3'\n" + + "\\)"); + } + + @Test + @Override + public void testView() + { + assertThatThrownBy(super::testView) + .hasStackTraceContaining("Access Denied"); + } + + @Test + @Override + public void testCommentView() + { + assertThatThrownBy(super::testCommentView) + .hasStackTraceContaining("Access Denied"); + } + + @Test + @Override + public void testCommentViewColumn() + { + assertThatThrownBy(super::testCommentViewColumn) + .hasStackTraceContaining("Access Denied"); + } + + @Test + @Override + public void testMaterializedView() + { + assertThatThrownBy(super::testMaterializedView) + .hasStackTraceContaining("Access Denied"); + } + + @Test + @Override + public void testRenameSchema() + { + assertThatThrownBy(super::testRenameSchema) + .hasStackTraceContaining("Access Denied"); + } + + @Test + @Override + public void testRenameTable() + { + assertThatThrownBy(super::testRenameTable) + .hasStackTraceContaining("Access Denied"); + } + + @Test + @Override + public void testRenameTableAcrossSchemas() + { + assertThatThrownBy(super::testRenameTableAcrossSchemas) + .hasStackTraceContaining("Access Denied"); + } + + @Test + @Override + public void testCreateTable() + { + assertThatThrownBy(super::testCreateTable) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testCreateTableAsSelect() + { + assertThatThrownBy(super::testCreateTableAsSelect) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testUpdate() + { + assertThatThrownBy(super::testUpdate) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testInsert() + { + assertThatThrownBy(super::testInsert) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testHiddenPathColumn() + { + assertThatThrownBy(super::testHiddenPathColumn) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testRowLevelDelete() + { + assertThatThrownBy(super::testRowLevelDelete) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testDeleteAllDataFromTable() + { + assertThatThrownBy(super::testDeleteAllDataFromTable) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testDeleteRowsConcurrently() + { + assertThatThrownBy(super::testDeleteRowsConcurrently) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testCreateOrReplaceTable() + { + assertThatThrownBy(super::testCreateOrReplaceTable) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testCreateOrReplaceTableChangeColumnNamesAndTypes() + { + assertThatThrownBy(super::testCreateOrReplaceTableChangeColumnNamesAndTypes) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testRegisterTableWithTableLocation() + { + assertThatThrownBy(super::testRegisterTableWithTableLocation) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testRegisterTableWithComments() + { + assertThatThrownBy(super::testRegisterTableWithComments) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testRowLevelUpdate() + { + assertThatThrownBy(super::testRowLevelUpdate) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testMerge() + { + assertThatThrownBy(super::testMerge) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testCreateSchema() + { + assertThatThrownBy(super::testCreateSchema) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testCreateSchemaWithNonLowercaseOwnerName() + { + assertThatThrownBy(super::testCreateSchemaWithNonLowercaseOwnerName) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testRegisterTableWithShowCreateTable() + { + assertThatThrownBy(super::testRegisterTableWithShowCreateTable) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testRegisterTableWithReInsert() + { + assertThatThrownBy(super::testRegisterTableWithReInsert) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testRegisterTableWithDroppedTable() + { + assertThatThrownBy(super::testRegisterTableWithDroppedTable) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testRegisterTableWithDifferentTableName() + { + assertThatThrownBy(super::testRegisterTableWithDifferentTableName) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testRegisterTableWithMetadataFile() + { + assertThatThrownBy(super::testRegisterTableWithMetadataFile) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testCreateTableWithTrailingSpaceInLocation() + { + assertThatThrownBy(super::testCreateTableWithTrailingSpaceInLocation) + .hasStackTraceContaining("Access Denied"); + } + + @Test + @Override + public void testRegisterTableWithTrailingSpaceInLocation() + { + assertThatThrownBy(super::testRegisterTableWithTrailingSpaceInLocation) + .hasStackTraceContaining("Access Denied"); + } + + @Test + @Override + public void testUnregisterTable() + { + assertThatThrownBy(super::testUnregisterTable) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testUnregisterBrokenTable() + { + assertThatThrownBy(super::testUnregisterBrokenTable) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testUnregisterTableNotExistingTable() + { + assertThatThrownBy(super::testUnregisterTableNotExistingTable) + .hasStackTraceContaining("Table .* not found"); + } + + @Test + @Override + public void testUnregisterTableNotExistingSchema() + { + assertThatThrownBy(super::testUnregisterTableNotExistingSchema) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testRepeatUnregisterTable() + { + assertThatThrownBy(super::testRepeatUnregisterTable) + .hasStackTraceContaining("Table .* not found"); + } + + @Test + @Override + public void testUnregisterTableAccessControl() + { + assertThatThrownBy(super::testUnregisterTableAccessControl) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testCreateTableWithNonExistingSchemaVerifyLocation() + { + assertThatThrownBy(super::testCreateTableWithNonExistingSchemaVerifyLocation) + .hasStackTraceContaining("Access Denied"); + } + + @Test + @Override + public void testSortedNationTable() + { + assertThatThrownBy(super::testSortedNationTable) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testFileSortingWithLargerTable() + { + assertThatThrownBy(super::testFileSortingWithLargerTable) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testDropTableWithMissingMetadataFile() + { + assertThatThrownBy(super::testDropTableWithMissingMetadataFile) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testDropTableWithMissingSnapshotFile() + { + assertThatThrownBy(super::testDropTableWithMissingSnapshotFile) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testDropTableWithMissingManifestListFile() + { + assertThatThrownBy(super::testDropTableWithMissingManifestListFile) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testDropTableWithMissingDataFile() + { + assertThatThrownBy(super::testDropTableWithMissingDataFile) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testDropTableWithNonExistentTableLocation() + { + assertThatThrownBy(super::testDropTableWithNonExistentTableLocation) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testMetadataTables() + { + assertThatThrownBy(super::testMetadataTables) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testPartitionFilterRequired() + { + assertThatThrownBy(super::testPartitionFilterRequired) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testTableChangesFunction() + { + assertThatThrownBy(super::testTableChangesFunction) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testRowLevelDeletesWithTableChangesFunction() + { + assertThatThrownBy(super::testRowLevelDeletesWithTableChangesFunction) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testCreateOrReplaceWithTableChangesFunction() + { + assertThatThrownBy(super::testCreateOrReplaceWithTableChangesFunction) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testTruncateTable() + { + assertThatThrownBy(super::testTruncateTable) + .hasMessageContaining("Access Denied"); + } + + @Test + @Override + public void testMetadataDeleteAfterCommitEnabled() + { + assertThatThrownBy(super::testMetadataDeleteAfterCommitEnabled) + .hasStackTraceContaining("Access Denied"); + } +} diff --git a/pom.xml b/pom.xml index a68ea1203a3a..b5d8aa6c50f3 100644 --- a/pom.xml +++ b/pom.xml @@ -1923,6 +1923,12 @@ + + org.apache.iceberg + iceberg-aws + ${dep.iceberg.version} + + org.apache.iceberg iceberg-core