From 00c4af62b0074b30da30bfc084c6bdfd825958f6 Mon Sep 17 00:00:00 2001 From: mosiac Date: Tue, 1 Apr 2025 12:57:41 +0100 Subject: [PATCH 1/2] Refactor CredentialsController, add RemoteConnection plugin This PR moves to a more dynamic approach to resolving remote credentials, where the whole Request object can be used to determine the endpoint it should be proxied to, the remote credentials to use and STS mechanism. RemoteS3ConnectionProvider returns remote credentials, optional STS role and optional RemoteS3Facade configs (used to provide and endpoint but also change the path-style if needed). It takes as arguments the ParsedRequest, Identity and SigningMetadata. The request flow with this change goes: Client request to aws-proxy -> CredentialsProvider to get emulated secret key and Identity -> SecurityFacade -> S3RequestRewriter -> RemoteS3ConnectionProvider to get remote credentials, role and remote configs -> (Optional) Assume Remote Role -> Sing request to remote -> Send request to remote --- .../proxy/glue/rest/TrinoGlueResource.java | 2 +- .../io/trino/aws/proxy/glue/TestGlueBase.java | 4 +- .../aws/proxy/glue/TestGlueInS3Proxy.java | 4 +- .../glue/TestingCredentialsProvider.java | 6 +- .../proxy/spi/credentials/Credentials.java | 61 -------- .../spi/credentials/CredentialsProvider.java | 3 +- .../spi/credentials/EmulatedAssumedRole.java | 2 +- .../spi/credentials/IdentityCredential.java | 42 +++++ .../plugin/TrinoAwsProxyServerBinding.java | 7 + .../RemoteS3ConnectionProviderConfig.java | 38 +++++ .../proxy/spi/remote/RemoteS3Connection.java | 35 +++++ .../remote/RemoteS3ConnectionProvider.java | 28 ++++ .../proxy/spi/remote/RemoteSessionRole.java | 3 +- .../aws/proxy/spi/rest/S3RequestRewriter.java | 10 +- .../proxy/spi/signing/SigningController.java | 19 ++- .../proxy/spi/signing/SigningMetadata.java | 20 ++- .../TrinoAwsProxyPluginValidatorModule.java | 13 +- .../server/TrinoAwsProxyServerModule.java | 16 ++ .../server/credentials/CredentialsModule.java | 2 - .../file/FileBasedCredentialsModule.java | 4 +- .../file/FileBasedCredentialsProvider.java | 12 +- .../http/HttpCredentialsModule.java | 4 +- .../http/HttpCredentialsProvider.java | 18 +-- .../RemoteS3ConnectionController.java} | 66 +++++--- .../aws/proxy/server/rest/ParamProvider.java | 36 ++++- .../server/rest/S3PresignController.java | 13 +- .../aws/proxy/server/rest/SecurityFilter.java | 19 ++- .../proxy/server/rest/TrinoS3ProxyClient.java | 146 ++++++++++-------- .../proxy/server/rest/TrinoS3Resource.java | 57 ++++--- .../proxy/server/rest/TrinoStsResource.java | 2 +- .../signing/InternalSigningController.java | 41 ++--- .../server/AbstractTestPresignedRequests.java | 6 +- .../proxy/server/AbstractTestStsRequests.java | 9 +- .../trino/aws/proxy/server/LocalServer.java | 4 +- .../proxy/server/TestGenericRestRequests.java | 22 ++- .../aws/proxy/server/TestHttpChunked.java | 16 +- .../aws/proxy/server/TestLogsResource.java | 9 +- .../proxy/server/TestPresignedRequests.java | 4 +- .../TestPresignedRequestsWithRewrite.java | 4 +- .../TestProxiedAssumedRoleRequests.java | 11 +- ...dEmulatedAndRemoteAssumedRoleRequests.java | 24 ++- .../TestRemoteSessionProxiedRequests.java | 28 +++- .../server/TestStsRequestsWithEmptyPath.java | 5 +- .../proxy/server/TestStsRequestsWithPath.java | 5 +- .../DelegatingCredentialsProvider.java | 4 +- .../server/credentials/TestAssumingRoles.java | 10 +- .../TestFileBasedCredentialsProvider.java | 40 +++-- .../http/TestHttpCredentialsProvider.java | 9 +- .../server/signing/TestSigningController.java | 24 ++- .../server/testing/RequestRewriteUtil.java | 10 +- .../TestingCredentialsRolesProvider.java | 87 +++++++---- ...TestingHttpCredentialsProviderServlet.java | 12 +- .../TestingRemoteCredentialsProvider.java | 52 ------- .../server/testing/TestingS3ClientModule.java | 6 +- .../testing/TestingS3PresignController.java | 7 +- .../TestingS3RequestRewriteController.java | 24 +-- .../testing/TestingS3RequestRewriter.java | 9 +- .../testing/TestingTrinoAwsProxyServer.java | 22 ++- .../TestingTrinoAwsProxyServerModule.java | 14 -- .../aws/proxy/server/testing/TestingUtil.java | 9 +- .../containers/MetastoreContainer.java | 9 +- .../testing/containers/PySparkContainer.java | 18 ++- .../testing/containers/S3Container.java | 8 +- .../src/test/resources/credentials.json | 16 -- 64 files changed, 749 insertions(+), 521 deletions(-) delete mode 100644 trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/Credentials.java create mode 100644 trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/IdentityCredential.java create mode 100644 trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/plugin/config/RemoteS3ConnectionProviderConfig.java create mode 100644 trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3Connection.java create mode 100644 trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3ConnectionProvider.java rename trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/{credentials/CredentialsController.java => remote/RemoteS3ConnectionController.java} (65%) delete mode 100644 trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingRemoteCredentialsProvider.java delete mode 100644 trino-aws-proxy/src/test/resources/credentials.json diff --git a/trino-aws-proxy-glue/src/main/java/io/trino/aws/proxy/glue/rest/TrinoGlueResource.java b/trino-aws-proxy-glue/src/main/java/io/trino/aws/proxy/glue/rest/TrinoGlueResource.java index b1b5e266..1c86f178 100644 --- a/trino-aws-proxy-glue/src/main/java/io/trino/aws/proxy/glue/rest/TrinoGlueResource.java +++ b/trino-aws-proxy-glue/src/main/java/io/trino/aws/proxy/glue/rest/TrinoGlueResource.java @@ -59,7 +59,7 @@ public TrinoGlueResource(ObjectMapper objectMapper, GlueRequestHandler requestHa @Produces(MediaType.APPLICATION_JSON) public Response gluePost(@Context Request request, @Context SigningMetadata signingMetadata, @Context RequestLoggingSession requestLoggingSession) { - requestLoggingSession.logProperty("request.glue.emulated.key", signingMetadata.credentials().emulated().secretKey()); + requestLoggingSession.logProperty("request.glue.emulated.key", signingMetadata.credential().secretKey()); String target = request.requestHeaders().unmodifiedHeaders().getFirst("x-amz-target") .orElseThrow(() -> InvalidInputException.builder().statusCode(BAD_REQUEST.getStatusCode()).build()); diff --git a/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestGlueBase.java b/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestGlueBase.java index 1aa01b2f..101151d2 100644 --- a/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestGlueBase.java +++ b/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestGlueBase.java @@ -13,7 +13,7 @@ */ package io.trino.aws.proxy.glue; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import jakarta.annotation.PreDestroy; import jakarta.ws.rs.core.UriBuilder; import org.junit.jupiter.api.Test; @@ -64,7 +64,7 @@ public abstract class TestGlueBase protected final GlueClient glueClient; protected final T context; - protected TestGlueBase(TrinoGlueConfig config, Credentials testingCredentials, T context) + protected TestGlueBase(TrinoGlueConfig config, IdentityCredential testingCredentials, T context) { this.context = requireNonNull(context, "context is null"); diff --git a/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestGlueInS3Proxy.java b/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestGlueInS3Proxy.java index c9db4a34..94d34b34 100644 --- a/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestGlueInS3Proxy.java +++ b/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestGlueInS3Proxy.java @@ -19,7 +19,7 @@ import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; import io.trino.aws.proxy.server.testing.harness.BuilderFilter; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import java.net.URI; @@ -51,7 +51,7 @@ public record Context(URI baseUrl) } @Inject - public TestGlueInS3Proxy(TestingHttpServer httpServer, TrinoGlueConfig config, @ForTesting Credentials testingCredentials) + public TestGlueInS3Proxy(TestingHttpServer httpServer, TrinoGlueConfig config, @ForTesting IdentityCredential testingCredentials) { super(config, testingCredentials, new Context(httpServer.getBaseUrl())); } diff --git a/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestingCredentialsProvider.java b/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestingCredentialsProvider.java index 00d74792..7a7b7788 100644 --- a/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestingCredentialsProvider.java +++ b/trino-aws-proxy-glue/src/test/java/io/trino/aws/proxy/glue/TestingCredentialsProvider.java @@ -14,8 +14,8 @@ package io.trino.aws.proxy.glue; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import java.util.Optional; import java.util.UUID; @@ -23,10 +23,10 @@ public class TestingCredentialsProvider implements CredentialsProvider { - public static final Credentials CREDENTIALS = Credentials.build(new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString())); + public static final IdentityCredential CREDENTIALS = new IdentityCredential(new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString())); @Override - public Optional credentials(String emulatedAccessKey, Optional session) + public Optional credentials(String emulatedAccessKey, Optional session) { return Optional.of(CREDENTIALS); } diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/Credentials.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/Credentials.java deleted file mode 100644 index da5e5c95..00000000 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/Credentials.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.aws.proxy.spi.credentials; - -import io.trino.aws.proxy.spi.remote.RemoteSessionRole; - -import java.util.Optional; - -import static java.util.Objects.requireNonNull; - -public record Credentials(Credential emulated, Optional remote, Optional remoteSessionRole, Optional identity) -{ - public Credentials - { - requireNonNull(emulated, "emulated is null"); - requireNonNull(remote, "remote is null"); - requireNonNull(remoteSessionRole, "remoteSessionRole is null"); - requireNonNull(identity, "identity is null"); - } - - public Credential requiredRemoteCredential() - { - return remote.orElseThrow(() -> new IllegalStateException("Credentials are emulated only and cannot be used for remote access")); - } - - public static Credentials build(Credential emulated) - { - return new Credentials(emulated, Optional.empty(), Optional.empty(), Optional.empty()); - } - - public static Credentials build(Credential emulated, Credential remote) - { - return new Credentials(emulated, Optional.of(remote), Optional.empty(), Optional.empty()); - } - - public static Credentials build(Credential emulated, Optional remote) - { - return new Credentials(emulated, remote, Optional.empty(), Optional.empty()); - } - - public static Credentials build(Credential emulated, Credential remote, RemoteSessionRole remoteSessionRole) - { - return new Credentials(emulated, Optional.of(remote), Optional.of(remoteSessionRole), Optional.empty()); - } - - public static Credentials build(Credential emulated, Credential remote, Identity identity) - { - return new Credentials(emulated, Optional.of(remote), Optional.empty(), Optional.of(identity)); - } -} diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java index 5e0ec06f..787efb8f 100644 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java @@ -15,6 +15,7 @@ import java.util.Optional; +// TODO: Add back file-based provider public interface CredentialsProvider { CredentialsProvider NOOP = (_, _) -> Optional.empty(); @@ -24,5 +25,5 @@ public interface CredentialsProvider * Your implementation should have a centralized credentials mechanism, likely * some type of database along with a way of registering credentials, etc. */ - Optional credentials(String emulatedAccessKey, Optional session); + Optional credentials(String emulatedAccessKey, Optional session); } diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/EmulatedAssumedRole.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/EmulatedAssumedRole.java index ba003236..f6b8fc0e 100644 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/EmulatedAssumedRole.java +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/EmulatedAssumedRole.java @@ -21,7 +21,7 @@ public record EmulatedAssumedRole(Credential emulatedCredential, String arn, Str { public EmulatedAssumedRole { - requireNonNull(emulatedCredential, "emulatedCredential is null"); + requireNonNull(emulatedCredential, "credential is null"); requireNonNull(arn, "arn is null"); requireNonNull(roleId, "roleId is null"); requireNonNull(expiration, "expiration is null"); diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/IdentityCredential.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/IdentityCredential.java new file mode 100644 index 00000000..4b0dec14 --- /dev/null +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/IdentityCredential.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.spi.credentials; + +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public record IdentityCredential(Credential emulated, Optional identity) +{ + public IdentityCredential + { + requireNonNull(emulated, "emulated is null"); + requireNonNull(identity, "identity is null"); + } + + public IdentityCredential(Credential emulatedCredential, Identity identity) + { + this(emulatedCredential, Optional.of(identity)); + } + + public IdentityCredential(Credential emulatedCredential) + { + this(emulatedCredential, Optional.empty()); + } + + public static IdentityCredential of(Credential emulatedCredential) + { + return new IdentityCredential(emulatedCredential, Optional.empty()); + } +} diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/plugin/TrinoAwsProxyServerBinding.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/plugin/TrinoAwsProxyServerBinding.java index af36474c..53f2f491 100644 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/plugin/TrinoAwsProxyServerBinding.java +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/plugin/TrinoAwsProxyServerBinding.java @@ -25,8 +25,10 @@ import io.trino.aws.proxy.spi.plugin.config.CredentialsProviderConfig; import io.trino.aws.proxy.spi.plugin.config.PluginIdentifierConfig; import io.trino.aws.proxy.spi.plugin.config.RemoteS3Config; +import io.trino.aws.proxy.spi.plugin.config.RemoteS3ConnectionProviderConfig; import io.trino.aws.proxy.spi.plugin.config.S3RequestRewriterConfig; import io.trino.aws.proxy.spi.plugin.config.S3SecurityFacadeProviderConfig; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; import io.trino.aws.proxy.spi.remote.RemoteS3Facade; import io.trino.aws.proxy.spi.rest.S3RequestRewriter; import io.trino.aws.proxy.spi.security.S3SecurityFacadeProvider; @@ -49,6 +51,11 @@ static Module assumedRoleProviderModule(String identifier, Class implementationClass, Module module) + { + return optionalPluginModule(RemoteS3ConnectionProviderConfig.class, identifier, RemoteS3ConnectionProvider.class, implementationClass, module); + } + static Module s3SecurityFacadeProviderModule(String identifier, Class implementationClass, Module module) { return optionalPluginModule(S3SecurityFacadeProviderConfig.class, identifier, S3SecurityFacadeProvider.class, implementationClass, module); diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/plugin/config/RemoteS3ConnectionProviderConfig.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/plugin/config/RemoteS3ConnectionProviderConfig.java new file mode 100644 index 00000000..b76334e9 --- /dev/null +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/plugin/config/RemoteS3ConnectionProviderConfig.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.spi.plugin.config; + +import io.airlift.configuration.Config; +import jakarta.validation.constraints.NotNull; + +import java.util.Optional; + +public class RemoteS3ConnectionProviderConfig + implements PluginIdentifierConfig +{ + private Optional identifier = Optional.empty(); + + @NotNull + @Override + public Optional getPluginIdentifier() + { + return identifier; + } + + @Config("remote-s3-connection-provider.type") + public void setPluginIdentifier(String identifier) + { + this.identifier = Optional.ofNullable(identifier); + } +} diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3Connection.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3Connection.java new file mode 100644 index 00000000..3aeeb7aa --- /dev/null +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3Connection.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.spi.remote; + +import com.google.common.collect.ImmutableMap; +import io.trino.aws.proxy.spi.credentials.Credential; + +import java.util.Map; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public record RemoteS3Connection( + Credential remoteCredential, + Optional remoteSessionRole, + Optional> remoteS3FacadeConfiguration) +{ + public RemoteS3Connection + { + requireNonNull(remoteCredential, "remoteCredential is null"); + requireNonNull(remoteSessionRole, "remoteSessionRole is null"); + remoteS3FacadeConfiguration = requireNonNull(remoteS3FacadeConfiguration, "remoteS3FacadeConfiguration is null").map(ImmutableMap::copyOf); + } +} diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3ConnectionProvider.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3ConnectionProvider.java new file mode 100644 index 00000000..705a5dc0 --- /dev/null +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3ConnectionProvider.java @@ -0,0 +1,28 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.spi.remote; + +import io.trino.aws.proxy.spi.credentials.Identity; +import io.trino.aws.proxy.spi.rest.ParsedS3Request; +import io.trino.aws.proxy.spi.signing.SigningMetadata; + +import java.util.Optional; + +// TODO: This should have a config implementation (hard-coding a single set of Remote Credentials) and an HTTP implementation +public interface RemoteS3ConnectionProvider +{ + RemoteS3ConnectionProvider NOOP = (_, _, _) -> Optional.empty(); + + Optional remoteConnection(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request); +} diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteSessionRole.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteSessionRole.java index 1503b979..232c583d 100644 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteSessionRole.java +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteSessionRole.java @@ -13,11 +13,12 @@ */ package io.trino.aws.proxy.spi.remote; +import java.net.URI; import java.util.Optional; import static java.util.Objects.requireNonNull; -public record RemoteSessionRole(String region, String roleArn, Optional externalId) +public record RemoteSessionRole(String region, String roleArn, Optional externalId, Optional stsEndpoint) { public RemoteSessionRole { diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/rest/S3RequestRewriter.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/rest/S3RequestRewriter.java index 477731e6..5ce179c4 100644 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/rest/S3RequestRewriter.java +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/rest/S3RequestRewriter.java @@ -13,7 +13,8 @@ */ package io.trino.aws.proxy.spi.rest; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.Identity; +import io.trino.aws.proxy.spi.signing.SigningMetadata; import java.util.Optional; @@ -21,15 +22,16 @@ public interface S3RequestRewriter { - S3RequestRewriter NOOP = (_, _) -> Optional.empty(); + S3RequestRewriter NOOP = (_, _, _) -> Optional.empty(); record S3RewriteResult(String finalRequestBucket, String finalRequestKey) { - public S3RewriteResult { + public S3RewriteResult + { requireNonNull(finalRequestBucket, "finalRequestBucket is null"); requireNonNull(finalRequestKey, "finalRequestKey is null"); } } - Optional rewrite(Credentials credentials, ParsedS3Request request); + Optional rewrite(Optional identity, SigningMetadata signingMetadata, ParsedS3Request request); } diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/signing/SigningController.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/signing/SigningController.java index 3ab0d13e..20acc12e 100644 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/signing/SigningController.java +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/signing/SigningController.java @@ -13,15 +13,15 @@ */ package io.trino.aws.proxy.spi.signing; -import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.Identity; import io.trino.aws.proxy.spi.rest.Request; import io.trino.aws.proxy.spi.util.MultiMap; import java.net.URI; import java.time.Instant; import java.util.Optional; -import java.util.function.Function; + +import static java.util.Objects.requireNonNull; public interface SigningController { @@ -30,7 +30,6 @@ SigningContext signRequest( String region, Instant requestDate, Optional signatureExpiry, - Function credentialsSupplier, URI requestURI, MultiMap requestHeaders, MultiMap queryParameters, @@ -41,10 +40,18 @@ SigningContext presignRequest( String region, Instant requestDate, Optional signatureExpiry, - Function credentialsSupplier, URI requestURI, MultiMap queryParameters, String httpMethod); - SigningMetadata validateAndParseAuthorization(Request request, SigningServiceType signingServiceType); + SigningIdentity validateAndParseAuthorization(Request request, SigningServiceType signingServiceType); + + record SigningIdentity(SigningMetadata signingMetadata, Optional identity) + { + public SigningIdentity + { + requireNonNull(signingMetadata, "signingMetadata is null"); + requireNonNull(identity, "identity is null"); + } + } } diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/signing/SigningMetadata.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/signing/SigningMetadata.java index 72541b87..c08ab1c6 100644 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/signing/SigningMetadata.java +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/signing/SigningMetadata.java @@ -13,24 +13,36 @@ */ package io.trino.aws.proxy.spi.signing; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.Credential; import java.util.Optional; import static java.util.Objects.requireNonNull; -public record SigningMetadata(SigningServiceType signingServiceType, Credentials credentials, Optional signingContext) +// TODO: This can be in two states - with or without a signing context. The only time its ok for it not to have a signing context is when we pass it to `isValidAuthorization`. +// We should refactor this class always have a SigningContext and pass something else to `SigningController`. +public record SigningMetadata(SigningServiceType signingServiceType, Credential credential, Optional signingContext) { public SigningMetadata { requireNonNull(signingServiceType, "signingService is null"); - requireNonNull(credentials, "credentials is null"); + requireNonNull(credential, "credential is null"); requireNonNull(signingContext, "signingContext is null"); } + public SigningMetadata(SigningServiceType signingServiceType, Credential credential) + { + this(signingServiceType, credential, Optional.empty()); + } + public SigningMetadata withSigningContext(SigningContext signingContext) { - return new SigningMetadata(signingServiceType, credentials, Optional.of(signingContext)); + return new SigningMetadata(signingServiceType, credential, Optional.of(signingContext)); + } + + public SigningMetadata withCredential(Credential credential) + { + return new SigningMetadata(signingServiceType, credential, signingContext); } public SigningContext requiredSigningContext() diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyPluginValidatorModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyPluginValidatorModule.java index 5ef0651b..47d8e824 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyPluginValidatorModule.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyPluginValidatorModule.java @@ -19,6 +19,8 @@ import io.trino.aws.proxy.spi.credentials.CredentialsProvider; import io.trino.aws.proxy.spi.plugin.config.AssumedRoleProviderConfig; import io.trino.aws.proxy.spi.plugin.config.CredentialsProviderConfig; +import io.trino.aws.proxy.spi.plugin.config.RemoteS3ConnectionProviderConfig; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; import static com.google.common.base.Preconditions.checkArgument; @@ -32,7 +34,9 @@ public TrinoAwsProxyPluginValidator( CredentialsProviderConfig credentialsProviderConfig, CredentialsProvider credentialsProvider, AssumedRoleProviderConfig assumedRoleProviderConfig, - AssumedRoleProvider assumedRoleProvider) + AssumedRoleProvider assumedRoleProvider, + RemoteS3ConnectionProvider remoteS3ConnectionProvider, + RemoteS3ConnectionProviderConfig remoteS3ConnectionProviderConfig) { boolean credentialsProviderIsNoop = credentialsProvider.equals(CredentialsProvider.NOOP); boolean credentialsProviderIsConfigured = credentialsProviderConfig.getPluginIdentifier().isPresent(); @@ -47,6 +51,13 @@ public TrinoAwsProxyPluginValidator( "%s of type \"%s\" is not registered", AssumedRoleProvider.class.getSimpleName(), assumedRoleProviderConfig.getPluginIdentifier().orElse("")); + + boolean remoteS3ConnectionProviderIsNoop = remoteS3ConnectionProvider.equals(RemoteS3ConnectionProvider.NOOP); + boolean remoteS3ConnectionProviderIsConfigured = remoteS3ConnectionProviderConfig.getPluginIdentifier().isPresent(); + checkArgument(!(remoteS3ConnectionProviderIsNoop && remoteS3ConnectionProviderIsConfigured), + "%s of type \"%s\" is not registered", + RemoteS3ConnectionProvider.class.getSimpleName(), + remoteS3ConnectionProviderConfig.getPluginIdentifier().orElse("")); } } diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java index 2a5aaa29..643c9492 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java @@ -33,7 +33,9 @@ import io.trino.aws.proxy.server.credentials.file.FileBasedCredentialsModule; import io.trino.aws.proxy.server.credentials.http.HttpCredentialsModule; import io.trino.aws.proxy.server.remote.DefaultRemoteS3Module; +import io.trino.aws.proxy.server.remote.RemoteS3ConnectionController; import io.trino.aws.proxy.server.rest.LimitStreamController; +import io.trino.aws.proxy.server.rest.ResourceSecurityDynamicFeature; import io.trino.aws.proxy.server.rest.RestModule; import io.trino.aws.proxy.server.rest.S3PresignController; import io.trino.aws.proxy.server.rest.ThrowableMapper; @@ -49,8 +51,10 @@ import io.trino.aws.proxy.server.signing.SigningModule; import io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerPlugin; import io.trino.aws.proxy.spi.plugin.config.RemoteS3Config; +import io.trino.aws.proxy.spi.plugin.config.RemoteS3ConnectionProviderConfig; import io.trino.aws.proxy.spi.plugin.config.S3RequestRewriterConfig; import io.trino.aws.proxy.spi.plugin.config.S3SecurityFacadeProviderConfig; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; import io.trino.aws.proxy.spi.remote.RemoteS3Facade; import io.trino.aws.proxy.spi.remote.RemoteUriFacade; import io.trino.aws.proxy.spi.rest.S3RequestRewriter; @@ -69,6 +73,7 @@ import static io.airlift.http.client.HttpClientBinder.httpClientBinder; import static io.airlift.http.server.HttpServerBinder.httpServerBinder; import static io.airlift.jaxrs.JaxrsBinder.jaxrsBinder; +import static org.weakref.jmx.guice.ExportBinder.newExporter; public class TrinoAwsProxyServerModule extends AbstractConfigurationAwareModule @@ -105,11 +110,18 @@ protected void setup(Binder binder) // TODO config, etc. httpClientBinder(binder).bindHttpClient("ProxyClient", ForProxyClient.class); binder.bind(TrinoS3ProxyClient.class).in(Scopes.SINGLETON); + binder.bind(RemoteS3ConnectionController.class).in(Scopes.SINGLETON); HttpServerBinder httpServerBinder = httpServerBinder(binder); httpServerBinder.enableLegacyUriCompliance(); httpServerBinder.enableCaseSensitiveHeaderCache(); + configBinder(binder).bindConfig(RemoteS3ConnectionProviderConfig.class); + newOptionalBinder(binder, RemoteS3ConnectionProvider.class).setDefault().toProvider(() -> { + log.info("Using default %s NOOP implementation", RemoteS3ConnectionProvider.class.getSimpleName()); + return RemoteS3ConnectionProvider.NOOP; + }); + // S3SecurityFacadeProvider binder configBinder(binder).bindConfig(S3SecurityFacadeProviderConfig.class); newOptionalBinder(binder, S3SecurityFacadeProvider.class).setDefault().toProvider(() -> { @@ -144,6 +156,10 @@ protected void setup(Binder binder) install(new TrinoAwsProxyPluginValidatorModule()); addNullCollectionModule(binder); + + newExporter(binder).export(RemoteS3ConnectionController.class).withGeneratedName(); + newExporter(binder).export(ResourceSecurityDynamicFeature.class).withGeneratedName(); + newExporter(binder).export(TrinoS3ProxyClient.class).withGeneratedName(); } @Provides diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/CredentialsModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/CredentialsModule.java index eeb299af..4bda97bc 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/CredentialsModule.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/CredentialsModule.java @@ -37,8 +37,6 @@ public class CredentialsModule @Override protected void setup(Binder binder) { - binder.bind(CredentialsController.class).in(Scopes.SINGLETON); - // CredentialsProvider binder configBinder(binder).bindConfig(CredentialsProviderConfig.class); newOptionalBinder(binder, CredentialsProvider.class).setDefault().toProvider(() -> { diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/file/FileBasedCredentialsModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/file/FileBasedCredentialsModule.java index 1b9beffb..abbc5e61 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/file/FileBasedCredentialsModule.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/file/FileBasedCredentialsModule.java @@ -15,7 +15,7 @@ import com.google.inject.Binder; import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import static io.airlift.configuration.ConfigBinder.configBinder; import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; @@ -36,7 +36,7 @@ protected void setup(Binder binder) innerBinder -> { configBinder(innerBinder).bindConfig(FileBasedCredentialsProviderConfig.class); innerBinder.bind(FileBasedCredentialsProvider.class); - jsonCodecBinder(innerBinder).bindListJsonCodec(Credentials.class); + jsonCodecBinder(innerBinder).bindListJsonCodec(IdentityCredential.class); })); } } diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/file/FileBasedCredentialsProvider.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/file/FileBasedCredentialsProvider.java index 0bc9a195..23b0cfb4 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/file/FileBasedCredentialsProvider.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/file/FileBasedCredentialsProvider.java @@ -16,8 +16,8 @@ import com.google.common.io.Files; import com.google.inject.Inject; import io.airlift.json.JsonCodec; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import java.io.File; import java.io.IOException; @@ -33,19 +33,19 @@ public class FileBasedCredentialsProvider implements CredentialsProvider { - private final Map credentialsStore; + private final Map credentialsStore; @Inject - public FileBasedCredentialsProvider(FileBasedCredentialsProviderConfig config, JsonCodec> jsonCodec) + public FileBasedCredentialsProvider(FileBasedCredentialsProviderConfig config, JsonCodec> jsonCodec) { requireNonNull(config, "Config is null"); requireNonNull(jsonCodec, "jsonCodec is null"); this.credentialsStore = buildCredentialsMap(config.getCredentialsFile(), jsonCodec); } - private Map buildCredentialsMap(File credentialsFile, JsonCodec> jsonCodec) + private Map buildCredentialsMap(File credentialsFile, JsonCodec> jsonCodec) { - List credentialsList; + List credentialsList; try { credentialsList = jsonCodec.fromJson(Files.toByteArray(credentialsFile)); } @@ -57,7 +57,7 @@ private Map buildCredentialsMap(File credentialsFile, JsonC } @Override - public Optional credentials(String emulatedAccessKey, Optional session) + public Optional credentials(String emulatedAccessKey, Optional session) { return Optional.ofNullable(credentialsStore.get(emulatedAccessKey)); } diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/http/HttpCredentialsModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/http/HttpCredentialsModule.java index 8c0d29a5..c98fa88f 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/http/HttpCredentialsModule.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/http/HttpCredentialsModule.java @@ -15,7 +15,7 @@ import com.google.inject.Binder; import io.airlift.configuration.AbstractConfigurationAwareModule; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import static io.airlift.configuration.ConfigBinder.configBinder; import static io.airlift.http.client.HttpClientBinder.httpClientBinder; @@ -38,7 +38,7 @@ protected void setup(Binder binder) innerBinder -> { configBinder(innerBinder).bindConfig(HttpCredentialsProviderConfig.class); httpClientBinder(innerBinder).bindHttpClient(HTTP_CREDENTIALS_PROVIDER_HTTP_CLIENT_NAME, ForHttpCredentialsProvider.class); - jsonCodecBinder(innerBinder).bindJsonCodec(Credentials.class); + jsonCodecBinder(innerBinder).bindJsonCodec(IdentityCredential.class); })); } } diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/http/HttpCredentialsProvider.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/http/HttpCredentialsProvider.java index cf3a7c19..aee4a9cb 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/http/HttpCredentialsProvider.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/http/HttpCredentialsProvider.java @@ -24,8 +24,8 @@ import io.airlift.http.client.HttpStatus; import io.airlift.http.client.Request; import io.airlift.json.JsonCodec; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import jakarta.ws.rs.core.UriBuilder; import java.net.URI; @@ -49,21 +49,21 @@ private record CredentialsKey(String emulatedAccessKey, Optional session } private final HttpClient httpClient; - private final JsonCodec jsonCodec; + private final JsonCodec jsonCodec; private final URI httpCredentialsProviderEndpoint; private final Map httpHeaders; - private final Optional>> credentialsCache; - private final Function> credentialsFetcher; + private final Optional>> credentialsCache; + private final Function> credentialsFetcher; @Inject - public HttpCredentialsProvider(@ForHttpCredentialsProvider HttpClient httpClient, HttpCredentialsProviderConfig config, JsonCodec jsonCodec) + public HttpCredentialsProvider(@ForHttpCredentialsProvider HttpClient httpClient, HttpCredentialsProviderConfig config, JsonCodec jsonCodec) { this.httpClient = requireNonNull(httpClient, "httpClient is null"); this.jsonCodec = requireNonNull(jsonCodec, "jsonCodec is null"); this.httpCredentialsProviderEndpoint = config.getEndpoint(); this.httpHeaders = ImmutableMap.copyOf(config.getHttpHeaders()); if (config.getCacheSize() > 0 && config.getCacheTtl().toMillis() > 0) { - LoadingCache> cache = Caffeine.newBuilder() + LoadingCache> cache = Caffeine.newBuilder() .maximumSize(config.getCacheSize()) .expireAfterWrite(config.getCacheTtl().toJavaTime()) .build(this::fetchCredentials); @@ -77,7 +77,7 @@ public HttpCredentialsProvider(@ForHttpCredentialsProvider HttpClient httpClient } @Override - public Optional credentials(String emulatedAccessKey, Optional session) + public Optional credentials(String emulatedAccessKey, Optional session) { return credentialsFetcher.apply(new CredentialsKey(emulatedAccessKey, session)); } @@ -91,14 +91,14 @@ void resetCache() }); } - private Optional fetchCredentials(CredentialsKey credentialsKey) + private Optional fetchCredentials(CredentialsKey credentialsKey) { UriBuilder uriBuilder = UriBuilder.fromUri(httpCredentialsProviderEndpoint).path(credentialsKey.emulatedAccessKey()); credentialsKey.session().ifPresent(sessionToken -> uriBuilder.queryParam("sessionToken", sessionToken)); Request.Builder requestBuilder = prepareGet() .addHeaders(Multimaps.forMap(httpHeaders)) .setUri(uriBuilder.build()); - JsonResponse response = httpClient.execute(requestBuilder.build(), createFullJsonResponseHandler(jsonCodec)); + JsonResponse response = httpClient.execute(requestBuilder.build(), createFullJsonResponseHandler(jsonCodec)); if (response.getStatusCode() == HttpStatus.NOT_FOUND.code() || !response.hasValue()) { return Optional.empty(); } diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/CredentialsController.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3ConnectionController.java similarity index 65% rename from trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/CredentialsController.java rename to trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3ConnectionController.java index d44477fd..a459618f 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/credentials/CredentialsController.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/RemoteS3ConnectionController.java @@ -11,15 +11,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package io.trino.aws.proxy.server.credentials; +package io.trino.aws.proxy.server.remote; import com.google.inject.Inject; +import com.google.inject.Injector; +import io.airlift.bootstrap.Bootstrap; import io.airlift.log.Logger; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; -import io.trino.aws.proxy.spi.credentials.CredentialsProvider; +import io.trino.aws.proxy.spi.credentials.Identity; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; +import io.trino.aws.proxy.spi.remote.RemoteS3Facade; import io.trino.aws.proxy.spi.remote.RemoteSessionRole; -import io.trino.aws.proxy.spi.remote.RemoteUriFacade; +import io.trino.aws.proxy.spi.rest.ParsedS3Request; +import io.trino.aws.proxy.spi.signing.SigningMetadata; import jakarta.annotation.PreDestroy; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentials; @@ -31,22 +35,24 @@ import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; import java.io.Closeable; +import java.net.URI; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.BiFunction; import java.util.function.Function; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; import static java.util.concurrent.CompletableFuture.completedFuture; -public class CredentialsController +public class RemoteS3ConnectionController { - private static final Logger log = Logger.get(CredentialsController.class); + private static final Logger log = Logger.get(RemoteS3ConnectionController.class); - private final RemoteUriFacade remoteUriFacade; - private final CredentialsProvider credentialsProvider; + private final RemoteS3Facade defaultS3Facade; + private final RemoteS3ConnectionProvider remoteS3ConnectionProvider; private final Map remoteSessions = new ConcurrentHashMap<>(); private final class Session @@ -77,12 +83,11 @@ public void close() stsClient.close(); } - private Optional withUsage(Credentials credentials, Function> credentialsConsumer) + private T withUsage(Function credentialsConsumer) { incrementUsage(); try { - Credentials remoteSessionCredentials = Credentials.build(credentials.emulated(), currentCredential()); - return credentialsConsumer.apply(remoteSessionCredentials); + return credentialsConsumer.apply(currentCredential()); } finally { decrementUsage(); @@ -117,10 +122,10 @@ private Credential currentCredential() } @Inject - public CredentialsController(RemoteUriFacade remoteUriFacade, CredentialsProvider credentialsProvider) + public RemoteS3ConnectionController(RemoteS3Facade defaultS3Facade, RemoteS3ConnectionProvider remoteS3ConnectionProvider) { - this.remoteUriFacade = requireNonNull(remoteUriFacade, "remoteUriFacade is null"); - this.credentialsProvider = requireNonNull(credentialsProvider, "credentialsProvider is null"); + this.defaultS3Facade = requireNonNull(defaultS3Facade, "remoteUriFacade is null"); + this.remoteS3ConnectionProvider = requireNonNull(remoteS3ConnectionProvider, "remoteS3ConnectionProvider is null"); } @PreDestroy @@ -130,20 +135,29 @@ public void shutdown() } @SuppressWarnings("resource") - public Optional withCredentials(String emulatedAccessKey, Optional emulatedSessionToken, Function> credentialsConsumer) + public Optional withRemoteConnection(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request, + BiFunction credentialsConsumer) { - Optional retrievedCredentials = credentialsProvider.credentials(emulatedAccessKey, emulatedSessionToken); - retrievedCredentials.ifPresentOrElse(_ -> log.debug("Credentials found. EmulatedAccessKey: %s", emulatedAccessKey), - () -> log.debug("Credentials not found. EmulatedAccessKey: %s", emulatedAccessKey)); - return retrievedCredentials.flatMap(credentials -> credentials.remoteSessionRole() - .flatMap(remoteSessionRole -> internalRemoteSession(remoteSessionRole, credentials).withUsage(credentials, credentialsConsumer)) - .or(() -> credentialsConsumer.apply(credentials))); + return remoteS3ConnectionProvider.remoteConnection(signingMetadata, identity, request) + .flatMap(remoteConnection -> { + RemoteS3Facade contextRemoteS3Facade = remoteConnection.remoteS3FacadeConfiguration().map(config -> { + // TODO: This should respect the plugin installed for the RemoteS3Facade somehow + Injector subInjector = new Bootstrap(new DefaultRemoteS3Module()).doNotInitializeLogging().quiet().setRequiredConfigurationProperties(config).initialize(); + return subInjector.getInstance(RemoteS3Facade.class); + }).orElse(defaultS3Facade); + + return remoteConnection.remoteSessionRole() + .map(remoteSessionRole -> + internalRemoteSession(remoteSessionRole, remoteConnection.remoteCredential()) + .withUsage(credentials -> credentialsConsumer.apply(credentials, contextRemoteS3Facade))) + .or(() -> Optional.of(credentialsConsumer.apply(remoteConnection.remoteCredential(), contextRemoteS3Facade))); + }); } - private Session internalRemoteSession(RemoteSessionRole remoteSessionRole, Credentials credentials) + private Session internalRemoteSession(RemoteSessionRole remoteSessionRole, Credential remoteCredential) { - String emulatedAccessKey = credentials.emulated().accessKey(); - return remoteSessions.computeIfAbsent(emulatedAccessKey, _ -> internalStartRemoteSession(remoteSessionRole, credentials.requiredRemoteCredential(), emulatedAccessKey)); + String remoteAccessKey = remoteCredential.accessKey(); + return remoteSessions.computeIfAbsent(remoteAccessKey, _ -> internalStartRemoteSession(remoteSessionRole, remoteCredential, remoteAccessKey)); } private Session internalStartRemoteSession(RemoteSessionRole remoteSessionRole, Credential remoteCredential, String sessionName) @@ -152,10 +166,12 @@ private Session internalStartRemoteSession(RemoteSessionRole remoteSessionRole, .map(session -> (AwsCredentials) AwsSessionCredentials.create(remoteCredential.accessKey(), remoteCredential.secretKey(), session)) .orElseGet(() -> AwsBasicCredentials.create(remoteCredential.accessKey(), remoteCredential.secretKey())); + URI stsEndpoint = remoteSessionRole.stsEndpoint().orElseGet(() -> defaultS3Facade.remoteUri(remoteSessionRole.region())); + StsClient stsClient = StsClient.builder() .region(Region.of(remoteSessionRole.region())) .credentialsProvider(StaticCredentialsProvider.create(awsCredentials)) - .endpointProvider(_ -> completedFuture(Endpoint.builder().url(remoteUriFacade.remoteUri(remoteSessionRole.region())).build())) + .endpointProvider(_ -> completedFuture(Endpoint.builder().url(stsEndpoint).build())) .build(); StsAssumeRoleCredentialsProvider credentialsProvider = StsAssumeRoleCredentialsProvider.builder() diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/ParamProvider.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/ParamProvider.java index da195481..24731f18 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/ParamProvider.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/ParamProvider.java @@ -13,27 +13,35 @@ */ package io.trino.aws.proxy.server.rest; +import com.google.common.reflect.TypeToken; +import io.trino.aws.proxy.spi.credentials.Identity; import io.trino.aws.proxy.spi.rest.Request; import io.trino.aws.proxy.spi.signing.SigningMetadata; import org.glassfish.jersey.server.ContainerRequest; import org.glassfish.jersey.server.model.Parameter; import org.glassfish.jersey.server.spi.internal.ValueParamProvider; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.List; +import java.util.Optional; import java.util.function.Function; +import java.util.stream.Stream; -import static io.trino.aws.proxy.server.rest.SecurityFilter.unwrap; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.trino.aws.proxy.server.rest.SecurityFilter.unwrapType; import static org.glassfish.jersey.server.spi.internal.ValueParamProvider.Priority.HIGH; public class ParamProvider implements ValueParamProvider { + private static final List> SUPPORTED_TYPES = + Stream.of(Request.class, SigningMetadata.class, Identity.class, RequestLoggingSession.class).map(TypeToken::of).collect(toImmutableList()); + @Override public Function getValueProvider(Parameter parameter) { - if (Request.class.isAssignableFrom(parameter.getRawType()) || SigningMetadata.class.isAssignableFrom(parameter.getRawType()) || RequestLoggingSession.class.isAssignableFrom(parameter.getRawType())) { - return containerRequest -> unwrap(containerRequest, parameter.getRawType()); - } - return null; + return getValueProvider(parameter.getType()); } @Override @@ -41,4 +49,22 @@ public PriorityType getPriority() { return HIGH; } + + private Function getValueProvider(Type type) + { + if (type instanceof ParameterizedType parameterizedType) { + if (parameterizedType.getRawType().equals(Optional.class)) { + var innerValueProvider = getValueProvider(parameterizedType.getActualTypeArguments()[0]); + if (innerValueProvider != null) { + return containerRequest -> Optional.ofNullable(innerValueProvider.apply(containerRequest)); + } + return null; + } + } + + if (SUPPORTED_TYPES.stream().anyMatch(supportedType -> supportedType.isSubtypeOf(type))) { + return containerRequest -> unwrapType(containerRequest, type); + } + return null; + } } diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/S3PresignController.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/S3PresignController.java index 6c4cf46f..65ef1fc3 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/S3PresignController.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/S3PresignController.java @@ -16,7 +16,7 @@ import com.google.inject.Inject; import io.trino.aws.proxy.server.TrinoAwsProxyConfig; import io.trino.aws.proxy.server.security.S3SecurityController; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.Identity; import io.trino.aws.proxy.spi.rest.ParsedS3Request; import io.trino.aws.proxy.spi.security.SecurityResponse.Failure; import io.trino.aws.proxy.spi.security.SecurityResponse.Success; @@ -49,23 +49,24 @@ public S3PresignController(SigningController signingController, TrinoAwsProxyCon presignUrlDuration = trinoAwsProxyConfig.getPresignedUrlsDuration().toJavaTime(); } - public Map buildPresignedRemoteUrls(SigningMetadata signingMetadata, ParsedS3Request request, Instant targetRequestTimestamp, URI remoteUri) + public Map buildPresignedRemoteUrls(Optional identity, SigningMetadata signingMetadata, ParsedS3Request request, Instant targetRequestTimestamp, + URI remoteUri) { Optional signatureExpiry = Optional.of(Instant.now().plusMillis(presignUrlDuration.toMillis())); return Stream.of("GET", "PUT", "POST", "DELETE") - .flatMap(httpMethod -> buildPresignedRemoteUrl(httpMethod, signingMetadata, request, targetRequestTimestamp, remoteUri, signatureExpiry)) + .flatMap(httpMethod -> buildPresignedRemoteUrl(httpMethod, signingMetadata, identity, request, targetRequestTimestamp, remoteUri, signatureExpiry)) .collect(toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); } - private Stream> buildPresignedRemoteUrl(String httpMethod, SigningMetadata signingMetadata, ParsedS3Request request, Instant targetRequestTimestamp, URI remoteUri, Optional signatureExpiry) + private Stream> buildPresignedRemoteUrl(String httpMethod, SigningMetadata signingMetadata, Optional identity, ParsedS3Request request, + Instant targetRequestTimestamp, URI remoteUri, Optional signatureExpiry) { SigningContext signingContext = signingController.presignRequest( signingMetadata, request.requestAuthorization().region(), targetRequestTimestamp, signatureExpiry, - Credentials::requiredRemoteCredential, remoteUri, request.queryParameters(), httpMethod); @@ -84,7 +85,7 @@ private Stream> buildPresignedRemoteUrl(String httpMethod request.rawQuery(), request.requestContent()); - return switch (s3SecurityController.apply(checkRequest, signingMetadata.credentials().identity())) { + return switch (s3SecurityController.apply(checkRequest, identity)) { case Success _ -> Stream.of(Map.entry(httpMethod, signingContext.signingUri())); case Failure _ -> Stream.empty(); }; diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/SecurityFilter.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/SecurityFilter.java index 32fb54f6..2411a56e 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/SecurityFilter.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/SecurityFilter.java @@ -15,6 +15,7 @@ import com.google.common.base.Throwables; import io.airlift.log.Logger; +import io.trino.aws.proxy.spi.credentials.Identity; import io.trino.aws.proxy.spi.rest.Request; import io.trino.aws.proxy.spi.signing.SigningController; import io.trino.aws.proxy.spi.signing.SigningMetadata; @@ -30,6 +31,7 @@ import java.io.Closeable; import java.io.IOException; import java.io.OutputStream; +import java.lang.reflect.Type; import java.util.Optional; import static jakarta.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR; @@ -68,10 +70,9 @@ public void filter(ContainerRequestContext requestContext) RequestLoggingSession requestLoggingSession = requestLoggerController.newRequestSession(request, signingServiceType); containerRequest.setProperty(RequestLoggingSession.class.getName(), requestLoggingSession); - SigningMetadata signingMetadata; + SigningController.SigningIdentity signingIdentity; try { - signingMetadata = signingController.validateAndParseAuthorization(request, signingServiceType); - containerRequest.setProperty(SigningMetadata.class.getName(), signingMetadata); + signingIdentity = signingController.validateAndParseAuthorization(request, signingServiceType); } catch (Exception e) { requestLoggingSession.logException(e); @@ -83,6 +84,9 @@ public void filter(ContainerRequestContext requestContext) default -> throw new RuntimeException(e); } } + + containerRequest.setProperty(SigningMetadata.class.getName(), signingIdentity.signingMetadata()); + signingIdentity.identity().ifPresent(identity -> containerRequest.setProperty(Identity.class.getName(), identity)); } else { log.warn("%s is not a ContainerRequest", requestContext.getRequest().getClass().getName()); @@ -151,9 +155,14 @@ public void close() }; } + static Object unwrapType(ContainerRequest containerRequest, Type type) + { + return containerRequest.getProperty(type.getTypeName()); + } + @SuppressWarnings("unchecked") - static T unwrap(ContainerRequest containerRequest, Class type) + static T unwrap(ContainerRequest containerRequest, Class clazz) { - return (T) containerRequest.getProperty(type.getName()); + return (T) containerRequest.getProperty(clazz.getTypeName()); } } diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoS3ProxyClient.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoS3ProxyClient.java index 87cb64aa..c2c50380 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoS3ProxyClient.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoS3ProxyClient.java @@ -20,9 +20,9 @@ import io.airlift.http.client.Request; import io.airlift.log.Logger; import io.trino.aws.proxy.server.TrinoAwsProxyConfig; +import io.trino.aws.proxy.server.remote.RemoteS3ConnectionController; import io.trino.aws.proxy.server.security.S3SecurityController; -import io.trino.aws.proxy.spi.credentials.Credentials; -import io.trino.aws.proxy.spi.remote.RemoteS3Facade; +import io.trino.aws.proxy.spi.credentials.Identity; import io.trino.aws.proxy.spi.rest.ParsedS3Request; import io.trino.aws.proxy.spi.rest.RequestContent; import io.trino.aws.proxy.spi.rest.S3RequestRewriter; @@ -67,37 +67,39 @@ public class TrinoS3ProxyClient private final HttpClient httpClient; private final SigningController signingController; - private final RemoteS3Facade remoteS3Facade; private final S3SecurityController s3SecurityController; private final S3PresignController s3PresignController; private final LimitStreamController limitStreamController; private final S3RequestRewriter s3RequestRewriter; + private final RemoteS3ConnectionController remoteS3ConnectionController; private final ExecutorService executorService = Executors.newVirtualThreadPerTaskExecutor(); private final boolean generatePresignedUrlsOnHead; @Retention(RUNTIME) @Target({FIELD, PARAMETER, METHOD}) @BindingAnnotation - public @interface ForProxyClient {} + public @interface ForProxyClient + { + } @Inject public TrinoS3ProxyClient( @ForProxyClient HttpClient httpClient, SigningController signingController, - RemoteS3Facade remoteS3Facade, S3SecurityController s3SecurityController, TrinoAwsProxyConfig trinoAwsProxyConfig, S3PresignController s3PresignController, LimitStreamController limitStreamController, - S3RequestRewriter s3RequestRewriter) + S3RequestRewriter s3RequestRewriter, + RemoteS3ConnectionController credentialsProvider) { this.httpClient = requireNonNull(httpClient, "httpClient is null"); this.signingController = requireNonNull(signingController, "signingController is null"); - this.remoteS3Facade = requireNonNull(remoteS3Facade, "objectStore is null"); this.s3SecurityController = requireNonNull(s3SecurityController, "securityController is null"); this.s3PresignController = requireNonNull(s3PresignController, "presignController is null"); this.limitStreamController = requireNonNull(limitStreamController, "quotaStreamController is null"); this.s3RequestRewriter = requireNonNull(s3RequestRewriter, "s3RequestRewriter is null"); + this.remoteS3ConnectionController = requireNonNull(credentialsProvider, "credentialsController is null"); generatePresignedUrlsOnHead = trinoAwsProxyConfig.isGeneratePresignedUrlsOnHead(); } @@ -110,89 +112,98 @@ public void shutDown() } } - public void proxyRequest(SigningMetadata signingMetadata, ParsedS3Request request, AsyncResponse asyncResponse, RequestLoggingSession requestLoggingSession) + public void proxyRequest(Optional identity, SigningMetadata signingMetadata, ParsedS3Request request, AsyncResponse asyncResponse, + RequestLoggingSession requestLoggingSession) { - SecurityResponse securityResponse = s3SecurityController.apply(request, signingMetadata.credentials().identity()); + SecurityResponse securityResponse = s3SecurityController.apply(request, identity); if (securityResponse instanceof Failure(var error)) { - log.debug("SecurityController check failed. AccessKey: %s, Request: %s, SecurityResponse: %s", signingMetadata.credentials().emulated().accessKey(), request, securityResponse); - requestLoggingSession.logError("request.security.fail.credentials", signingMetadata.credentials().emulated()); + log.debug("SecurityController check failed. AccessKey: %s, Request: %s, SecurityResponse: %s", signingMetadata.credential().accessKey(), request, securityResponse); + requestLoggingSession.logError("request.security.fail.credentials", signingMetadata.credential()); requestLoggingSession.logError("request.security.fail.request", request); requestLoggingSession.logError("request.security.fail.error", error); throw new WebApplicationException(Response.Status.UNAUTHORIZED); } - Optional rewriteResult = s3RequestRewriter.rewrite(signingMetadata.credentials(), request); + Optional rewriteResult = s3RequestRewriter.rewrite(identity, signingMetadata, request); String targetBucket = rewriteResult.map(S3RewriteResult::finalRequestBucket).orElse(request.bucketName()); String targetKey = rewriteResult .map(S3RewriteResult::finalRequestKey) .map(SdkHttpUtils::urlEncodeIgnoreSlashes) .orElse(request.rawPath()); - URI remoteUri = remoteS3Facade.buildEndpoint(uriBuilder(request.queryParameters()), targetKey, targetBucket, request.requestAuthorization().region()); - Request.Builder remoteRequestBuilder = new Request.Builder() - .setMethod(request.httpVerb()) - .setUri(remoteUri) - .setFollowRedirects(true); + RemoteRequestWithPresignedURIs remoteRequest = remoteS3ConnectionController.withRemoteConnection(signingMetadata, identity, request, (remoteCredential, remoteS3Facade) -> { + URI remoteUri = remoteS3Facade.buildEndpoint(uriBuilder(request.queryParameters()), targetKey, targetBucket, request.requestAuthorization().region()); - if (remoteUri.getHost() == null) { - log.debug("RemoteURI missing host. AccessKey: %s, Request: %s", signingMetadata.credentials().emulated().accessKey(), request); - throw new WebApplicationException(Response.Status.BAD_REQUEST); - } + Request.Builder remoteRequestBuilder = new Request.Builder() + .setMethod(request.httpVerb()) + .setUri(remoteUri) + .setFollowRedirects(true); - ImmutableMultiMap.Builder remoteRequestHeadersBuilder = ImmutableMultiMap.builder(false); - Instant targetRequestTimestamp = Instant.now(); - request.requestHeaders().passthroughHeaders().forEach(remoteRequestHeadersBuilder::addAll); - remoteRequestHeadersBuilder.putOrReplaceSingle("Host", buildRemoteHost(remoteUri)); + if (remoteUri.getHost() == null) { + log.debug("RemoteURI missing host. AccessKey: %s, Request: %s", signingMetadata.credential().accessKey(), request); + throw new WebApplicationException(Response.Status.BAD_REQUEST); + } - // Use now for the remote request - remoteRequestHeadersBuilder.putOrReplaceSingle("X-Amz-Date", AwsTimestamp.toRequestFormat(targetRequestTimestamp)); + ImmutableMultiMap.Builder remoteRequestHeadersBuilder = ImmutableMultiMap.builder(false); + Instant targetRequestTimestamp = Instant.now(); + request.requestHeaders().passthroughHeaders().forEach(remoteRequestHeadersBuilder::addAll); + remoteRequestHeadersBuilder.putOrReplaceSingle("Host", buildRemoteHost(remoteUri)); - signingMetadata.credentials() - .requiredRemoteCredential() - .session() - .ifPresent(sessionToken -> remoteRequestHeadersBuilder.putOrReplaceSingle("x-amz-security-token", sessionToken)); + // Use now for the remote request + remoteRequestHeadersBuilder.putOrReplaceSingle("X-Amz-Date", AwsTimestamp.toRequestFormat(targetRequestTimestamp)); - request.requestContent().contentLength().ifPresent(length -> remoteRequestHeadersBuilder.putOrReplaceSingle("content-length", Integer.toString(length))); + request.requestContent().contentLength().ifPresent(length -> remoteRequestHeadersBuilder.putOrReplaceSingle("content-length", Integer.toString(length))); + // All SigV4 requests require an x-amz-content-sha256 + remoteRequestHeadersBuilder.putOrReplaceSingle("x-amz-content-sha256", "UNSIGNED-PAYLOAD"); - contentInputStream(request.requestContent(), signingMetadata).ifPresent(inputStream -> remoteRequestBuilder.setBodyGenerator(streamingBodyGenerator(inputStream))); - // All SigV4 requests require an x-amz-content-sha256 - remoteRequestHeadersBuilder.putOrReplaceSingle("x-amz-content-sha256", "UNSIGNED-PAYLOAD"); + SigningMetadata remoteSigningMetadata = signingMetadata.withCredential(remoteCredential); - Map presignedUrls; - if (generatePresignedUrlsOnHead && request.httpVerb().equalsIgnoreCase("HEAD")) { - presignedUrls = s3PresignController.buildPresignedRemoteUrls(signingMetadata, request, targetRequestTimestamp, remoteUri); - } - else { - presignedUrls = ImmutableMap.of(); - } + Map presignedUrls; + if (generatePresignedUrlsOnHead && request.httpVerb().equalsIgnoreCase("HEAD")) { + presignedUrls = s3PresignController.buildPresignedRemoteUrls(identity, remoteSigningMetadata, request, targetRequestTimestamp, remoteUri); + } + else { + presignedUrls = ImmutableMap.of(); + } - // set the new signed request auth header - MultiMap remoteRequestHeaders = remoteRequestHeadersBuilder.build(); - String signature = signingController.signRequest( - signingMetadata, - request.requestAuthorization().region(), - targetRequestTimestamp, - Optional.empty(), - Credentials::requiredRemoteCredential, - remoteUri, - remoteRequestHeaders, - request.queryParameters(), - request.httpVerb()).signingAuthorization().authorization(); - - // remoteRequestHeaders now has correct values, copy to the remote request - remoteRequestHeaders.forEachEntry(remoteRequestBuilder::addHeader); - remoteRequestBuilder.addHeader("Authorization", signature); - - Request remoteRequest = remoteRequestBuilder.build(); + remoteCredential + .session() + .ifPresent(sessionToken -> remoteRequestHeadersBuilder.putOrReplaceSingle("x-amz-security-token", sessionToken)); + + contentInputStream(request.requestContent(), remoteSigningMetadata).ifPresent(inputStream -> remoteRequestBuilder.setBodyGenerator(streamingBodyGenerator(inputStream))); + + // set the new signed request auth header + MultiMap remoteRequestHeaders = remoteRequestHeadersBuilder.build(); + String signature = signingController.signRequest( + remoteSigningMetadata, + request.requestAuthorization().region(), + targetRequestTimestamp, + Optional.empty(), + remoteUri, + remoteRequestHeaders, + request.queryParameters(), + request.httpVerb()).signingAuthorization().authorization(); + + // remoteRequestHeaders now has correct values, copy to the remote request + remoteRequestHeaders.forEachEntry(remoteRequestBuilder::addHeader); + remoteRequestBuilder.addHeader("Authorization", signature); + + return new RemoteRequestWithPresignedURIs(remoteRequestBuilder.build(), presignedUrls); + }).orElseThrow(() -> { + requestLoggingSession.logError("request.remote.fail.resolution", "Failed to resolve remote"); + return new WebApplicationException(Response.Status.NOT_FOUND); + }); executorService.submit(() -> { - StreamingResponseHandler responseHandler = new StreamingResponseHandler(asyncResponse, presignedUrls, requestLoggingSession, limitStreamController); + StreamingResponseHandler responseHandler = new StreamingResponseHandler(asyncResponse, remoteRequest.presignedUrls(), requestLoggingSession, limitStreamController); try { - httpClient.execute(remoteRequest, responseHandler); + httpClient.execute(remoteRequest.remoteRequest(), responseHandler); } catch (Throwable e) { - responseHandler.handleException(remoteRequest, new RuntimeException(e)); + // TODO: if responseHandler is null this will throw an NPE inside a catch clause, so the request doesn't terminate properly; fix; also we should have a timeout + // for request processing + responseHandler.handleException(remoteRequest.remoteRequest(), new RuntimeException(e)); } }); } @@ -230,4 +241,13 @@ private static UriBuilder uriBuilder(MultiMap queryParameters) queryParameters.forEachEntry(uriBuilder::queryParam); return uriBuilder; } + + private record RemoteRequestWithPresignedURIs(Request remoteRequest, Map presignedUrls) + { + private RemoteRequestWithPresignedURIs + { + requireNonNull(remoteRequest, "remoteRequest is null"); + requireNonNull(presignedUrls, "presignedUrls is null"); + } + } } diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoS3Resource.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoS3Resource.java index 002c0971..e4556a03 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoS3Resource.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoS3Resource.java @@ -16,6 +16,7 @@ import com.google.inject.Inject; import io.trino.aws.proxy.server.TrinoAwsProxyConfig; import io.trino.aws.proxy.server.rest.ResourceSecurity.S3; +import io.trino.aws.proxy.spi.credentials.Identity; import io.trino.aws.proxy.spi.rest.ParsedS3Request; import io.trino.aws.proxy.spi.rest.Request; import io.trino.aws.proxy.spi.signing.SigningMetadata; @@ -53,80 +54,90 @@ public TrinoS3Resource(TrinoS3ProxyClient proxyClient, TrinoAwsProxyConfig trino } @GET - public void s3Get(@Context Request request, @Context SigningMetadata signingMetadata, @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) + public void s3Get(@Context Request request, @Context Optional identity, @Context SigningMetadata signingMetadata, + @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) { - handler(request, signingMetadata, requestLoggingSession, asyncResponse); + handler(request, identity, signingMetadata, requestLoggingSession, asyncResponse); } @GET @Path("{path:.*}") - public void s3GetWithPath(@Context Request request, @Context SigningMetadata signingMetadata, @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) + public void s3GetWithPath(@Context Request request, @Context Optional identity, @Context SigningMetadata signingMetadata, + @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) { - handler(request, signingMetadata, requestLoggingSession, asyncResponse); + handler(request, identity, signingMetadata, requestLoggingSession, asyncResponse); } @HEAD - public void s3Head(@Context Request request, @Context SigningMetadata signingMetadata, @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) + public void s3Head(@Context Request request, @Context Optional identity, @Context SigningMetadata signingMetadata, + @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) { - handler(request, signingMetadata, requestLoggingSession, asyncResponse); + handler(request, identity, signingMetadata, requestLoggingSession, asyncResponse); } @HEAD @Path("{path:.*}") - public void s3HeadWithPath(@Context Request request, @Context SigningMetadata signingMetadata, @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) + public void s3HeadWithPath(@Context Request request, @Context Optional identity, @Context SigningMetadata signingMetadata, + @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) { - handler(request, signingMetadata, requestLoggingSession, asyncResponse); + handler(request, identity, signingMetadata, requestLoggingSession, asyncResponse); } @PUT - public void s3Put(@Context Request request, @Context SigningMetadata signingMetadata, @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) + public void s3Put(@Context Request request, @Context Optional identity, @Context SigningMetadata signingMetadata, + @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) { - handler(request, signingMetadata, requestLoggingSession, asyncResponse); + handler(request, identity, signingMetadata, requestLoggingSession, asyncResponse); } @PUT @Path("{path:.*}") - public void s3PutWithPath(@Context Request request, @Context SigningMetadata signingMetadata, @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) + public void s3PutWithPath(@Context Request request, @Context Optional identity, @Context SigningMetadata signingMetadata, + @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) { - handler(request, signingMetadata, requestLoggingSession, asyncResponse); + handler(request, identity, signingMetadata, requestLoggingSession, asyncResponse); } @POST - public void s3Post(@Context Request request, @Context SigningMetadata signingMetadata, @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) + public void s3Post(@Context Request request, @Context Optional identity, @Context SigningMetadata signingMetadata, + @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) { - handler(request, signingMetadata, requestLoggingSession, asyncResponse); + handler(request, identity, signingMetadata, requestLoggingSession, asyncResponse); } @POST @Path("{path:.*}") - public void s3PostWithPath(@Context Request request, @Context SigningMetadata signingMetadata, @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) + public void s3PostWithPath(@Context Request request, @Context Optional identity, @Context SigningMetadata signingMetadata, + @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) { - handler(request, signingMetadata, requestLoggingSession, asyncResponse); + handler(request, identity, signingMetadata, requestLoggingSession, asyncResponse); } @DELETE - public void s3Delete(@Context Request request, @Context SigningMetadata signingMetadata, @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) + public void s3Delete(@Context Request request, @Context Optional identity, @Context SigningMetadata signingMetadata, + @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) { - handler(request, signingMetadata, requestLoggingSession, asyncResponse); + handler(request, identity, signingMetadata, requestLoggingSession, asyncResponse); } @DELETE @Path("{path:.*}") - public void s3DeleteWithPath(@Context Request request, @Context SigningMetadata signingMetadata, @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) + public void s3DeleteWithPath(@Context Request request, @Context Optional identity, @Context SigningMetadata signingMetadata, + @Context RequestLoggingSession requestLoggingSession, @Suspended AsyncResponse asyncResponse) { - handler(request, signingMetadata, requestLoggingSession, asyncResponse); + handler(request, identity, signingMetadata, requestLoggingSession, asyncResponse); } - private void handler(Request request, SigningMetadata signingMetadata, RequestLoggingSession requestLoggingSession, AsyncResponse asyncResponse) + private void handler(Request request, Optional identity, SigningMetadata signingMetadata, RequestLoggingSession requestLoggingSession, AsyncResponse asyncResponse) { try { ParsedS3Request parsedS3Request = parseRequest(request); requestLoggingSession.logProperty("request.parsed.bucket", parsedS3Request.bucketName()); requestLoggingSession.logProperty("request.parsed.key", parsedS3Request.keyInBucket()); - requestLoggingSession.logProperty("request.emulated.key", signingMetadata.credentials().emulated().secretKey()); + requestLoggingSession.logProperty("request.emulated.key", signingMetadata.credential().secretKey()); - proxyClient.proxyRequest(signingMetadata, parsedS3Request, asyncResponse, requestLoggingSession); + proxyClient.proxyRequest(identity, signingMetadata, parsedS3Request, asyncResponse, requestLoggingSession); } catch (Throwable e) { requestLoggingSession.logException(e); diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoStsResource.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoStsResource.java index f83b0ac1..bbafc12e 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoStsResource.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/rest/TrinoStsResource.java @@ -85,7 +85,7 @@ private Response assumeRole(String region, SigningMetadata signingMetadata, Map< Optional externalId = Optional.ofNullable(arguments.get("ExternalId")); Optional durationSeconds = Optional.ofNullable(arguments.get("DurationSeconds")).map(TrinoStsResource::mapToInt); - EmulatedAssumedRole assumedRole = assumedRoleProvider.assumeEmulatedRole(signingMetadata.credentials().emulated(), region, roleArn, externalId, roleSessionName, durationSeconds) + EmulatedAssumedRole assumedRole = assumedRoleProvider.assumeEmulatedRole(signingMetadata.credential(), region, roleArn, externalId, roleSessionName, durationSeconds) .orElseThrow(() -> { log.debug("Assume role failed. Arguments: %s", arguments); requestLoggingSession.logError("request.assume-role.failure", arguments); diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/signing/InternalSigningController.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/signing/InternalSigningController.java index 753da79a..b837b292 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/signing/InternalSigningController.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/signing/InternalSigningController.java @@ -17,10 +17,9 @@ import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import io.airlift.log.Logger; -import io.trino.aws.proxy.server.credentials.CredentialsController; import io.trino.aws.proxy.server.rest.RequestLoggerController; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.CredentialsProvider; import io.trino.aws.proxy.spi.rest.Request; import io.trino.aws.proxy.spi.rest.RequestContent; import io.trino.aws.proxy.spi.signing.SigningContext; @@ -36,7 +35,6 @@ import java.time.Instant; import java.util.Optional; import java.util.Set; -import java.util.function.Function; import static java.util.Objects.requireNonNull; @@ -47,14 +45,14 @@ public class InternalSigningController private final Duration maxClockDrift; private final RequestLoggerController requestLoggerController; - private final CredentialsController credentialsController; + private final CredentialsProvider credentialsProvider; private static final Set LOWERCASE_HEADERS = ImmutableSet.of("content-type"); @Inject - public InternalSigningController(CredentialsController credentialsController, SigningControllerConfig signingControllerConfig, RequestLoggerController requestLoggerController) + public InternalSigningController(CredentialsProvider credentialsProvider, SigningControllerConfig signingControllerConfig, RequestLoggerController requestLoggerController) { - this.credentialsController = requireNonNull(credentialsController, "credentialsController is null"); + this.credentialsProvider = requireNonNull(credentialsProvider, "signingCredentialProvider is null"); this.requestLoggerController = requireNonNull(requestLoggerController, "requestLoggerController is null"); maxClockDrift = signingControllerConfig.getMaxClockDrift().toJavaTime(); @@ -66,7 +64,6 @@ public SigningContext signRequest( String region, Instant requestDate, Optional signatureExpiry, - Function credentialsSupplier, URI requestURI, MultiMap requestHeaders, MultiMap queryParameters, @@ -78,7 +75,6 @@ public SigningContext signRequest( requestDate, signatureExpiry, RequestContent.EMPTY, - credentialsSupplier, requestURI, SigningHeaders.build(requestHeaders), queryParameters, @@ -91,7 +87,6 @@ public SigningContext presignRequest( String region, Instant requestDate, Optional signatureExpiry, - Function credentialsSupplier, URI requestURI, MultiMap queryParameters, String httpMethod) @@ -102,7 +97,6 @@ public SigningContext presignRequest( requestDate, signatureExpiry, RequestContent.EMPTY, - credentialsSupplier, requestURI, SigningHeaders.EMPTY, queryParameters, @@ -110,20 +104,21 @@ public SigningContext presignRequest( } @Override - public SigningMetadata validateAndParseAuthorization(Request request, SigningServiceType signingServiceType) + public SigningIdentity validateAndParseAuthorization(Request request, SigningServiceType signingServiceType) { if (!request.requestAuthorization().isValid()) { log.debug("Invalid requestAuthorization. Request: %s, SigningServiceType: %s", request, signingServiceType); throw new WebApplicationException(Response.Status.UNAUTHORIZED); } - return credentialsController.withCredentials(request.requestAuthorization().accessKey(), request.requestAuthorization().securityToken(), credentials -> { - SigningMetadata metadata = new SigningMetadata(signingServiceType, credentials, Optional.empty()); - return isValidAuthorization(metadata, request, Credentials::emulated); - }).orElseThrow(() -> { - log.debug("ValidateAndParseAuthorization failed. Request: %s, SigningServiceType: %s", request, signingServiceType); - return new WebApplicationException(Response.Status.UNAUTHORIZED); - }); + return credentialsProvider.credentials(request.requestAuthorization().accessKey(), request.requestAuthorization().securityToken()) + .flatMap(identityCredential -> + isValidAuthorization(new SigningMetadata(signingServiceType, identityCredential.emulated(), Optional.empty()), request) + .map(signingMetadata -> new SigningIdentity(signingMetadata, identityCredential.identity()))) + .orElseThrow(() -> { + log.debug("ValidateAndParseAuthorization failed. Request: %s, SigningServiceType: %s", request, signingServiceType); + return new WebApplicationException(Response.Status.UNAUTHORIZED); + }); } private SigningContext internalSignRequest( @@ -132,13 +127,12 @@ private SigningContext internalSignRequest( Instant requestDate, Optional signatureExpiry, RequestContent requestContent, - Function credentialsSupplier, URI requestURI, SigningHeaders signingHeaders, MultiMap queryParameters, String httpMethod) { - Credential credential = credentialsSupplier.apply(metadata.credentials()); + Credential credential = metadata.credential(); return signatureExpiry.map(expiry -> Signer.presign( metadata.signingServiceType(), @@ -168,8 +162,7 @@ private SigningContext internalSignRequest( @SuppressWarnings("resource") private Optional isValidAuthorization( SigningMetadata metadata, - Request request, - Function credentialsSupplier) + Request request) { SigningHeaders signingHeaders = SigningHeaders.build(request.requestHeaders().unmodifiedHeaders(), request.requestAuthorization().lowercaseSignedHeaders()); SigningContext signingContext = internalSignRequest( @@ -178,7 +171,6 @@ private Optional isValidAuthorization( request.requestDate(), request.requestAuthorization().expiry(), request.requestContent(), - credentialsSupplier, request.requestUri(), signingHeaders, request.requestQueryParameters(), @@ -190,7 +182,8 @@ private Optional isValidAuthorization( } requestLoggerController.currentRequestSession(request.requestId()) - .logError("request.security.authorization.mismatch", ImmutableMap.of("request", request.requestAuthorization(), "generated", signingContext.signingAuthorization())); + .logError("request.security.authorization.mismatch", ImmutableMap.of("request", request.requestAuthorization(), "generated", + signingContext.signingAuthorization())); return Optional.empty(); } } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/AbstractTestPresignedRequests.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/AbstractTestPresignedRequests.java index 9e784bd9..90637940 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/AbstractTestPresignedRequests.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/AbstractTestPresignedRequests.java @@ -28,7 +28,7 @@ import io.trino.aws.proxy.server.testing.TestingS3RequestRewriteController; import io.trino.aws.proxy.server.testing.TestingUtil; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import org.junit.jupiter.api.Test; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @@ -85,7 +85,7 @@ public abstract class AbstractTestPresignedRequests private final HttpClient httpClient; private final S3Client internalClient; private final S3Client storageClient; - private final Credentials testingCredentials; + private final IdentityCredential testingCredentials; private final URI s3ProxyUrl; private final XmlMapper xmlMapper; private final TestingS3RequestRewriteController requestRewriteController; @@ -96,7 +96,7 @@ protected AbstractTestPresignedRequests( HttpClient httpClient, S3Client internalClient, S3Client storageClient, - Credentials testingCredentials, + IdentityCredential testingCredentials, TestingHttpServer httpServer, TrinoAwsProxyConfig s3ProxyConfig, XmlMapper xmlMapper, diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/AbstractTestStsRequests.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/AbstractTestStsRequests.java index 81da46b6..60b59407 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/AbstractTestStsRequests.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/AbstractTestStsRequests.java @@ -15,8 +15,8 @@ import io.airlift.http.server.testing.TestingHttpServer; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import org.junit.jupiter.api.Test; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.endpoints.Endpoint; @@ -37,7 +37,7 @@ public abstract class AbstractTestStsRequests private final StsClient stsClient; private final CredentialsProvider credentialsProvider; - public AbstractTestStsRequests(Credentials testingCredentials, TestingHttpServer httpServer, CredentialsProvider credentialsProvider, TrinoAwsProxyConfig s3ProxyConfig) + public AbstractTestStsRequests(IdentityCredential testingCredentials, TestingHttpServer httpServer, CredentialsProvider credentialsProvider, TrinoAwsProxyConfig s3ProxyConfig) { this.credentialsProvider = requireNonNull(credentialsProvider, "credentialsProvider is null"); @@ -59,8 +59,9 @@ public void testAssumeRole() software.amazon.awssdk.services.sts.model.Credentials awsCredentials = assumeRoleResponse.credentials(); - Optional credentials = credentialsProvider.credentials(awsCredentials.accessKeyId(), Optional.of(awsCredentials.sessionToken())); + Optional credentials = credentialsProvider.credentials(awsCredentials.accessKeyId(), Optional.of(awsCredentials.sessionToken())); assertThat(credentials).isNotEmpty(); - assertThat(credentials.map(Credentials::emulated)).contains(new Credential(awsCredentials.accessKeyId(), awsCredentials.secretAccessKey(), Optional.of(awsCredentials.sessionToken()))); + assertThat(credentials.map(IdentityCredential::emulated)).contains(new Credential(awsCredentials.accessKeyId(), awsCredentials.secretAccessKey(), + Optional.of(awsCredentials.sessionToken()))); } } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/LocalServer.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/LocalServer.java index c0357308..d067df57 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/LocalServer.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/LocalServer.java @@ -18,7 +18,7 @@ import io.airlift.log.Logger; import io.trino.aws.proxy.server.testing.TestingTrinoAwsProxyServer; import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; public final class LocalServer { @@ -38,7 +38,7 @@ public static void main(String[] args) log.info("======== TESTING SERVER STARTED ========"); TestingHttpServer httpServer = trinoS3ProxyServer.getInjector().getInstance(TestingHttpServer.class); - Credentials testingCredentials = trinoS3ProxyServer.getInjector().getInstance(Key.get(Credentials.class, ForTesting.class)); + IdentityCredential testingCredentials = trinoS3ProxyServer.getInjector().getInstance(Key.get(IdentityCredential.class, ForTesting.class)); TrinoAwsProxyConfig s3ProxyConfig = trinoS3ProxyServer.getInjector().getInstance(TrinoAwsProxyConfig.class); log.info(""); diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestGenericRestRequests.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestGenericRestRequests.java index deed0d8c..417436d2 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestGenericRestRequests.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestGenericRestRequests.java @@ -20,21 +20,19 @@ import io.airlift.http.client.StatusResponseHandler.StatusResponse; import io.airlift.http.server.testing.TestingHttpServer; import io.airlift.units.Duration; -import io.trino.aws.proxy.server.credentials.CredentialsController; import io.trino.aws.proxy.server.rest.RequestLoggerConfig; import io.trino.aws.proxy.server.rest.RequestLoggerController; import io.trino.aws.proxy.server.signing.InternalSigningController; import io.trino.aws.proxy.server.signing.SigningControllerConfig; import io.trino.aws.proxy.server.signing.TestingChunkSigningSession; import io.trino.aws.proxy.server.testing.TestingCredentialsRolesProvider; -import io.trino.aws.proxy.server.testing.TestingRemoteS3Facade; import io.trino.aws.proxy.server.testing.TestingTrinoAwsProxyServer; import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithTestingHttpClient; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import io.trino.aws.proxy.spi.signing.RequestAuthorization; import io.trino.aws.proxy.spi.signing.SigningMetadata; import io.trino.aws.proxy.spi.signing.SigningServiceType; @@ -79,7 +77,7 @@ public class TestGenericRestRequests private final TestingCredentialsRolesProvider credentialsRolesProvider; private final InternalSigningController signingController; private final HttpClient httpClient; - private final Credentials testingCredentials; + private final IdentityCredential testingCredentials; private final S3Client storageClient; private static final String TEST_CONTENT_TYPE = "text/plain;charset=utf-8"; @@ -107,14 +105,14 @@ public TestGenericRestRequests( TestingHttpServer httpServer, TestingCredentialsRolesProvider credentialsRolesProvider, @ForTesting HttpClient httpClient, - @ForTesting Credentials testingCredentials, + @ForTesting IdentityCredential testingCredentials, @ForS3Container S3Client storageClient, TrinoAwsProxyConfig trinoAwsProxyConfig) { baseUri = httpServer.getBaseUrl().resolve(trinoAwsProxyConfig.getS3Path()); this.credentialsRolesProvider = requireNonNull(credentialsRolesProvider, "credentialsRolesProvider is null"); this.signingController = new InternalSigningController( - new CredentialsController(new TestingRemoteS3Facade(), credentialsRolesProvider), + credentialsRolesProvider, new SigningControllerConfig().setMaxClockDrift(new Duration(10, TimeUnit.SECONDS)), new RequestLoggerController(new RequestLoggerConfig())); this.httpClient = requireNonNull(httpClient, "httpClient is null"); @@ -136,7 +134,7 @@ public void testAwsChunkedUploadValid() storageClient.createBucket(r -> r.bucket(bucket).build()); Credential validCredential = new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - credentialsRolesProvider.addCredentials(Credentials.build(validCredential, testingCredentials.requiredRemoteCredential())); + credentialsRolesProvider.addCredentials(new IdentityCredential(validCredential)); // Upload in 2 chunks assertThat(doAwsChunkedUpload(bucket, "aws-chunked-2-partitions", LOREM_IPSUM, 2, validCredential).getStatusCode()).isEqualTo(200); @@ -167,9 +165,9 @@ public void testAwsChunkedUploadInvalidContent() storageClient.createBucket(r -> r.bucket(bucket).build()); Credential validCredential = new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - credentialsRolesProvider.addCredentials(Credentials.build(validCredential, testingCredentials.requiredRemoteCredential())); + credentialsRolesProvider.addCredentials(new IdentityCredential(validCredential)); Credential validCredentialTwo = new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - credentialsRolesProvider.addCredentials(Credentials.build(validCredentialTwo, testingCredentials.requiredRemoteCredential())); + credentialsRolesProvider.addCredentials(new IdentityCredential(validCredentialTwo)); Credential unknownCredential = new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()); // Credential is not known to the credential controller @@ -300,7 +298,7 @@ private void testAwsChunkedIllegalChunks(String bucket, String key, String rawCo { Instant requestDate = Instant.now(); Credential validCredential = new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()); - credentialsRolesProvider.addCredentials(Credentials.build(validCredential, testingCredentials.requiredRemoteCredential())); + credentialsRolesProvider.addCredentials(new IdentityCredential(validCredential)); ImmutableMultiMap.Builder requestHeaderBuilder = ImmutableMultiMap.builder(false); requestHeaderBuilder @@ -379,8 +377,8 @@ private StatusResponse doPutObject(String bucket, String key, String content, St private RequestAuthorization signRequest(Credential signingCredential, URI uri, Instant requestDate, String method, MultiMap headers) { - return signingController.signRequest(new SigningMetadata(SigningServiceType.S3, Credentials.build(signingCredential, testingCredentials.requiredRemoteCredential()), Optional.empty()), - "us-east-1", requestDate, Optional.empty(), Credentials::emulated, uri, headers, ImmutableMultiMap.empty(), method).signingAuthorization(); + return signingController.signRequest(new SigningMetadata(SigningServiceType.S3, signingCredential, Optional.empty()), + "us-east-1", requestDate, Optional.empty(), uri, headers, ImmutableMultiMap.empty(), method).signingAuthorization(); } private static Function getMutatorToBreakSignatureForChunk(int chunkNumber) diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestHttpChunked.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestHttpChunked.java index 4d12735f..f295c440 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestHttpChunked.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestHttpChunked.java @@ -18,19 +18,17 @@ import io.airlift.http.client.Request; import io.airlift.http.server.testing.TestingHttpServer; import io.airlift.units.Duration; -import io.trino.aws.proxy.server.credentials.CredentialsController; import io.trino.aws.proxy.server.rest.RequestLoggerConfig; import io.trino.aws.proxy.server.rest.RequestLoggerController; import io.trino.aws.proxy.server.signing.InternalSigningController; import io.trino.aws.proxy.server.signing.SigningControllerConfig; import io.trino.aws.proxy.server.signing.TestingChunkSigningSession; import io.trino.aws.proxy.server.testing.TestingCredentialsRolesProvider; -import io.trino.aws.proxy.server.testing.TestingRemoteS3Facade; import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import io.trino.aws.proxy.spi.signing.RequestAuthorization; import io.trino.aws.proxy.spi.signing.SigningMetadata; import io.trino.aws.proxy.spi.signing.SigningServiceType; @@ -76,7 +74,7 @@ public class TestHttpChunked private final URI baseUri; private final TestingCredentialsRolesProvider credentialsRolesProvider; private final HttpClient httpClient; - private final Credentials testingCredentials; + private final IdentityCredential testingCredentials; private final S3Client storageClient; private static final String TEST_CONTENT_TYPE = "text/plain;charset=utf-8"; @@ -85,7 +83,7 @@ public class TestHttpChunked @BeforeEach public void setupCredentials() { - credentialsRolesProvider.addCredentials(Credentials.build(VALID_CREDENTIAL, testingCredentials.requiredRemoteCredential())); + credentialsRolesProvider.addCredentials(new IdentityCredential(VALID_CREDENTIAL)); } @Inject @@ -93,7 +91,7 @@ public TestHttpChunked( TestingHttpServer httpServer, TestingCredentialsRolesProvider credentialsRolesProvider, @ForTesting HttpClient httpClient, - @ForTesting Credentials testingCredentials, + @ForTesting IdentityCredential testingCredentials, @ForS3Container S3Client storageClient, TrinoAwsProxyConfig trinoAwsProxyConfig) { @@ -279,11 +277,11 @@ private int doCustomHttpChunkedUpload(String bucket, String key, int chunkCount, URI requestUri = UriBuilder.fromUri(baseUri).path(bucket).path(key).build(); InternalSigningController signingController = new InternalSigningController( - new CredentialsController(new TestingRemoteS3Facade(), credentialsRolesProvider), + credentialsRolesProvider, new SigningControllerConfig().setMaxClockDrift(new Duration(10, TimeUnit.SECONDS)), new RequestLoggerController(new RequestLoggerConfig())); - RequestAuthorization requestAuthorization = signingController.signRequest(new SigningMetadata(SigningServiceType.S3, Credentials.build(VALID_CREDENTIAL, testingCredentials.requiredRemoteCredential()), Optional.empty()), - "us-east-1", requestDate, Optional.empty(), Credentials::emulated, requestUri, requestHeaderBuilder.build(), ImmutableMultiMap.empty(), "PUT").signingAuthorization(); + RequestAuthorization requestAuthorization = signingController.signRequest(new SigningMetadata(SigningServiceType.S3, VALID_CREDENTIAL, Optional.empty()), + "us-east-1", requestDate, Optional.empty(), requestUri, requestHeaderBuilder.build(), ImmutableMultiMap.empty(), "PUT").signingAuthorization(); requestHeaderBuilder.add("Authorization", requestAuthorization.authorization()); Request.Builder requestBuilder = preparePut() diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestLogsResource.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestLogsResource.java index ec73a69c..ce4ed079 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestLogsResource.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestLogsResource.java @@ -26,7 +26,7 @@ import io.trino.aws.proxy.server.rest.RequestLoggingSession; import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import io.trino.aws.proxy.spi.rest.Request; import io.trino.aws.proxy.spi.rest.RequestContent; import io.trino.aws.proxy.spi.rest.RequestHeaders; @@ -68,7 +68,12 @@ public class TestLogsResource private final CloudWatchLogsClient cloudWatchClient; @Inject - public TestLogsResource(TestingHttpServer httpServer, RequestLoggerController loggerController, TrinoAwsProxyConfig config, @ForTesting Credentials testingCredentials, ObjectMapper objectMapper) + public TestLogsResource( + TestingHttpServer httpServer, + RequestLoggerController loggerController, + TrinoAwsProxyConfig config, + @ForTesting IdentityCredential testingCredentials, + ObjectMapper objectMapper) { this.loggerController = requireNonNull(loggerController, "loggerController is null"); this.objectMapper = requireNonNull(objectMapper, "objectMapper is null"); diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestPresignedRequests.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestPresignedRequests.java index f7dfaf23..2c88ff27 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestPresignedRequests.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestPresignedRequests.java @@ -23,7 +23,7 @@ import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithConfiguredBuckets; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithTestingHttpClient; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import software.amazon.awssdk.services.s3.S3Client; @TrinoAwsProxyTest(filters = {WithConfiguredBuckets.class, WithTestingHttpClient.class, TestProxiedRequests.Filter.class}) @@ -35,7 +35,7 @@ public TestPresignedRequests( @ForTesting HttpClient httpClient, S3Client internalClient, @ForS3Container S3Client storageClient, - @ForTesting Credentials testingCredentials, + @ForTesting IdentityCredential testingCredentials, TestingHttpServer httpServer, TrinoAwsProxyConfig s3ProxyConfig, XmlMapper xmlMapper, diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestPresignedRequestsWithRewrite.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestPresignedRequestsWithRewrite.java index c5d9e74d..0985737d 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestPresignedRequestsWithRewrite.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestPresignedRequestsWithRewrite.java @@ -25,7 +25,7 @@ import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithConfiguredBuckets; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithTestingHttpClient; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.GetObjectRequest; @@ -53,7 +53,7 @@ public TestPresignedRequestsWithRewrite( @ForTesting HttpClient httpClient, S3Client internalClient, @ForS3Container S3Client storageClient, - @ForTesting Credentials testingCredentials, + @ForTesting IdentityCredential testingCredentials, TestingHttpServer httpServer, TrinoAwsProxyConfig s3ProxyConfig, XmlMapper xmlMapper, diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedAssumedRoleRequests.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedAssumedRoleRequests.java index 82867800..215a671c 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedAssumedRoleRequests.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedAssumedRoleRequests.java @@ -21,7 +21,7 @@ import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithConfiguredBuckets; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.Credential; import org.junit.jupiter.api.AfterAll; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; @@ -47,13 +47,14 @@ public class TestProxiedAssumedRoleRequests @Inject public TestProxiedAssumedRoleRequests( TestingHttpServer httpServer, - @ForTesting Credentials testingCredentials, + @ForTesting Credential testingCredential, TestingCredentialsRolesProvider credentialsController, @ForS3Container S3Client storageClient, TrinoAwsProxyConfig trinoAwsProxyConfig, TestingS3RequestRewriteController requestRewriteController) { - this(buildClient(httpServer, testingCredentials, trinoAwsProxyConfig.getS3Path(), trinoAwsProxyConfig.getStsPath()), credentialsController, storageClient, requestRewriteController); + this(buildClient(httpServer, testingCredential, trinoAwsProxyConfig.getS3Path(), trinoAwsProxyConfig.getStsPath()), credentialsController, storageClient, + requestRewriteController); } protected TestProxiedAssumedRoleRequests( @@ -74,13 +75,13 @@ public void validateCount() credentialsController.resetAssumedRoles(); } - protected static S3Client buildClient(TestingHttpServer httpServer, Credentials credentials, String s3Path, String stsPath) + protected static S3Client buildClient(TestingHttpServer httpServer, Credential credential, String s3Path, String stsPath) { URI baseUrl = httpServer.getBaseUrl(); URI localProxyServerUri = baseUrl.resolve(s3Path); URI localStsServerUri = baseUrl.resolve(stsPath); - AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(credentials.emulated().accessKey(), credentials.emulated().secretKey()); + AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(credential.accessKey(), credential.secretKey()); StsClient stsClient = StsClient.builder() .region(Region.US_EAST_1) diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedEmulatedAndRemoteAssumedRoleRequests.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedEmulatedAndRemoteAssumedRoleRequests.java index 89610761..6cd878fe 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedEmulatedAndRemoteAssumedRoleRequests.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestProxiedEmulatedAndRemoteAssumedRoleRequests.java @@ -17,23 +17,39 @@ import io.airlift.http.server.testing.TestingHttpServer; import io.trino.aws.proxy.server.testing.TestingCredentialsRolesProvider; import io.trino.aws.proxy.server.testing.TestingS3RequestRewriteController; -import io.trino.aws.proxy.server.testing.TestingTrinoAwsProxyServerModule.ForTestingRemoteCredentials; +import io.trino.aws.proxy.server.testing.containers.S3Container; import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.Credential; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; +import io.trino.aws.proxy.spi.remote.RemoteSessionRole; import software.amazon.awssdk.services.s3.S3Client; +import java.util.Optional; +import java.util.UUID; + +import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_IDENTITY_CREDENTIAL; + public class TestProxiedEmulatedAndRemoteAssumedRoleRequests extends TestProxiedAssumedRoleRequests { + private static final Credential CREDENTIAL = new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + @Inject public TestProxiedEmulatedAndRemoteAssumedRoleRequests( TestingHttpServer httpServer, TestingCredentialsRolesProvider credentialsController, @ForS3Container S3Client storageClient, - @ForTestingRemoteCredentials Credentials remoteCredentials, TrinoAwsProxyConfig trinoAwsProxyConfig, + S3Container s3Container, TestingS3RequestRewriteController requestRewriteController) { - super(buildClient(httpServer, remoteCredentials, trinoAwsProxyConfig.getS3Path(), trinoAwsProxyConfig.getStsPath()), credentialsController, storageClient, requestRewriteController); + super(buildClient(httpServer, CREDENTIAL, trinoAwsProxyConfig.getS3Path(), trinoAwsProxyConfig.getStsPath()), credentialsController, storageClient, + requestRewriteController); + + Credential policyUserCredential = s3Container.policyUserCredential(); + RemoteSessionRole remoteSessionRole = new RemoteSessionRole("us-east-1", "minio-doesnt-care", Optional.empty(), Optional.empty()); + IdentityCredential identityCredential = new IdentityCredential(CREDENTIAL, TESTING_IDENTITY_CREDENTIAL.identity()); + credentialsController.addCredentials(identityCredential, new RemoteS3Connection(policyUserCredential, Optional.of(remoteSessionRole), Optional.empty())); } } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestRemoteSessionProxiedRequests.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestRemoteSessionProxiedRequests.java index 831175d5..20d918e0 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestRemoteSessionProxiedRequests.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestRemoteSessionProxiedRequests.java @@ -15,17 +15,23 @@ import com.google.inject.Inject; import io.airlift.http.server.testing.TestingHttpServer; +import io.trino.aws.proxy.server.testing.TestingCredentialsRolesProvider; import io.trino.aws.proxy.server.testing.TestingS3RequestRewriteController; -import io.trino.aws.proxy.server.testing.TestingTrinoAwsProxyServerModule.ForTestingRemoteCredentials; +import io.trino.aws.proxy.server.testing.containers.S3Container; import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTestCommonModules.WithConfiguredBuckets; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.Credential; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; +import io.trino.aws.proxy.spi.remote.RemoteSessionRole; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.services.s3.S3Client; import java.util.Optional; +import java.util.UUID; +import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_IDENTITY_CREDENTIAL; import static io.trino.aws.proxy.server.testing.TestingUtil.clientBuilder; @TrinoAwsProxyTest(filters = WithConfiguredBuckets.class) @@ -33,16 +39,22 @@ public class TestRemoteSessionProxiedRequests extends AbstractTestProxiedRequests { @Inject - public TestRemoteSessionProxiedRequests(@ForS3Container S3Client storageClient, @ForTestingRemoteCredentials Credentials remoteCredentials, TestingHttpServer httpServer, TrinoAwsProxyConfig trinoAwsProxyConfig, TestingS3RequestRewriteController requestRewriteController) + public TestRemoteSessionProxiedRequests(@ForS3Container S3Client storageClient, S3Container s3Container, TestingCredentialsRolesProvider testingCredentialsRolesProvider, + TestingHttpServer httpServer, TrinoAwsProxyConfig trinoAwsProxyConfig, TestingS3RequestRewriteController requestRewriteController) { - super(buildInternalClient(remoteCredentials, httpServer, trinoAwsProxyConfig.getS3Path()), storageClient, requestRewriteController); + super(buildClient(httpServer, trinoAwsProxyConfig, s3Container, testingCredentialsRolesProvider), storageClient, requestRewriteController); } - private static S3Client buildInternalClient(Credentials credentials, TestingHttpServer httpServer, String s3Path) + private static S3Client buildClient(TestingHttpServer httpServer, TrinoAwsProxyConfig trinoAwsProxyConfig, S3Container s3Container, + TestingCredentialsRolesProvider testingCredentialsRolesProvider) { - AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(credentials.emulated().accessKey(), credentials.emulated().secretKey()); - - return clientBuilder(httpServer.getBaseUrl(), Optional.of(s3Path)) + Credential policyUserCredential = s3Container.policyUserCredential(); + RemoteSessionRole remoteSessionRole = new RemoteSessionRole("us-east-1", "minio-doesnt-care", Optional.empty(), Optional.empty()); + IdentityCredential identityCredential = new IdentityCredential(new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()), + TESTING_IDENTITY_CREDENTIAL.identity()); + testingCredentialsRolesProvider.addCredentials(identityCredential, new RemoteS3Connection(policyUserCredential, Optional.of(remoteSessionRole), Optional.empty())); + AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(identityCredential.emulated().accessKey(), identityCredential.emulated().secretKey()); + return clientBuilder(httpServer.getBaseUrl(), Optional.of(trinoAwsProxyConfig.getS3Path())) .credentialsProvider(() -> awsBasicCredentials) .build(); } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestStsRequestsWithEmptyPath.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestStsRequestsWithEmptyPath.java index 4395f6e9..a65fce9c 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestStsRequestsWithEmptyPath.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestStsRequestsWithEmptyPath.java @@ -19,8 +19,8 @@ import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; import io.trino.aws.proxy.server.testing.harness.BuilderFilter; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; @TrinoAwsProxyTest(filters = TestStsRequestsWithEmptyPath.Filter.class) public class TestStsRequestsWithEmptyPath @@ -37,7 +37,8 @@ public TestingTrinoAwsProxyServer.Builder filter(TestingTrinoAwsProxyServer.Buil } @Inject - public TestStsRequestsWithEmptyPath(@ForTesting Credentials testingCredentials, TestingHttpServer httpServer, CredentialsProvider credentialsProvider, TrinoAwsProxyConfig s3ProxyConfig) + public TestStsRequestsWithEmptyPath(@ForTesting IdentityCredential testingCredentials, TestingHttpServer httpServer, CredentialsProvider credentialsProvider, + TrinoAwsProxyConfig s3ProxyConfig) { super(testingCredentials, httpServer, credentialsProvider, s3ProxyConfig); } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestStsRequestsWithPath.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestStsRequestsWithPath.java index b4983d3f..193b13c8 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestStsRequestsWithPath.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/TestStsRequestsWithPath.java @@ -19,8 +19,8 @@ import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; import io.trino.aws.proxy.server.testing.harness.BuilderFilter; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; @TrinoAwsProxyTest(filters = TestStsRequestsWithPath.Filter.class) public class TestStsRequestsWithPath @@ -37,7 +37,8 @@ public TestingTrinoAwsProxyServer.Builder filter(TestingTrinoAwsProxyServer.Buil } @Inject - public TestStsRequestsWithPath(@ForTesting Credentials testingCredentials, TestingHttpServer httpServer, CredentialsProvider credentialsProvider, TrinoAwsProxyConfig s3ProxyConfig) + public TestStsRequestsWithPath(@ForTesting IdentityCredential testingCredentials, TestingHttpServer httpServer, CredentialsProvider credentialsProvider, + TrinoAwsProxyConfig s3ProxyConfig) { super(testingCredentials, httpServer, credentialsProvider, s3ProxyConfig); } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/DelegatingCredentialsProvider.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/DelegatingCredentialsProvider.java index 4dc12d1d..7d4c66af 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/DelegatingCredentialsProvider.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/DelegatingCredentialsProvider.java @@ -13,8 +13,8 @@ */ package io.trino.aws.proxy.server.credentials; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; @@ -32,7 +32,7 @@ public void setDelegate(CredentialsProvider delegate) } @Override - public Optional credentials(String emulatedAccessKey, Optional session) + public Optional credentials(String emulatedAccessKey, Optional session) { return delegate.get().credentials(emulatedAccessKey, session); } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/TestAssumingRoles.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/TestAssumingRoles.java index 825d4100..5f3d8434 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/TestAssumingRoles.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/TestAssumingRoles.java @@ -19,8 +19,8 @@ import io.trino.aws.proxy.server.testing.TestingCredentialsRolesProvider; import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.EmulatedAssumedRole; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -44,10 +44,11 @@ public class TestAssumingRoles private final TestingCredentialsRolesProvider credentialsController; private final URI localS3URI; - private final Credentials testingCredentials; + private final IdentityCredential testingCredentials; @Inject - public TestAssumingRoles(TestingCredentialsRolesProvider credentialsController, TestingHttpServer httpServer, @ForTesting Credentials testingCredentials, TrinoAwsProxyConfig trinoS3ProxyConfig) + public TestAssumingRoles(TestingCredentialsRolesProvider credentialsController, TestingHttpServer httpServer, @ForTesting IdentityCredential testingCredentials, + TrinoAwsProxyConfig trinoS3ProxyConfig) { this.credentialsController = requireNonNull(credentialsController, "credentialsController is null"); this.testingCredentials = requireNonNull(testingCredentials, "testingCredentials is null"); @@ -63,7 +64,8 @@ public void reset() @Test public void testStsSession() { - EmulatedAssumedRole emulatedAssumedRole = credentialsController.assumeEmulatedRole(testingCredentials.emulated(), "us-east-1", ARN, Optional.empty(), Optional.empty(), Optional.empty()) + EmulatedAssumedRole emulatedAssumedRole = credentialsController.assumeEmulatedRole(testingCredentials.emulated(), "us-east-1", ARN, Optional.empty(), + Optional.empty(), Optional.empty()) .orElseThrow(() -> new RuntimeException("Failed to assume role")); try (S3Client client = clientBuilder(localS3URI) diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/file/TestFileBasedCredentialsProvider.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/file/TestFileBasedCredentialsProvider.java index fe9a00b4..a0d9e1f9 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/file/TestFileBasedCredentialsProvider.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/file/TestFileBasedCredentialsProvider.java @@ -14,21 +14,22 @@ package io.trino.aws.proxy.server.credentials.file; import com.google.common.collect.ImmutableList; -import com.google.common.io.Resources; import com.google.inject.Inject; import io.trino.aws.proxy.server.testing.TestingIdentity; import io.trino.aws.proxy.server.testing.TestingTrinoAwsProxyServer; import io.trino.aws.proxy.server.testing.harness.BuilderFilter; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import org.junit.jupiter.api.Test; import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; import java.util.Optional; -import static io.trino.aws.proxy.server.credentials.file.FileBasedCredentialsModule.FILE_BASED_CREDENTIALS_IDENTIFIER; import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.bindIdentityType; import static java.util.Objects.requireNonNull; import static org.assertj.core.api.Assertions.assertThat; @@ -44,12 +45,34 @@ public static class Filter @Override public TestingTrinoAwsProxyServer.Builder filter(TestingTrinoAwsProxyServer.Builder builder) { - File configFile = new File(Resources.getResource("credentials.json").getFile()); + File configFile; + try { + configFile = File.createTempFile("credentials-provider", ".json"); + configFile.deleteOnExit(); + String jsonContent = """ + [ + { + "emulated": { + "accessKey": "test-emulated-access-key", + "secretKey": "test-emulated-secret" + }, + "identity": { + "user": "test-username", + "id": "test-id" + } + } + ] + """; + Files.writeString(configFile.toPath(), jsonContent); + } + catch (IOException exception) { + throw new UncheckedIOException(exception); + } return builder.withoutTestingCredentialsRoleProviders() .addModule(new FileBasedCredentialsModule()) .addModule(binder -> bindIdentityType(binder, TestingIdentity.class)) - .withProperty("credentials-provider.type", FILE_BASED_CREDENTIALS_IDENTIFIER) + .withProperty("credentials-provider.type", "file") .withProperty("credentials-provider.credentials-file-path", configFile.getAbsolutePath()); } } @@ -64,16 +87,15 @@ public TestFileBasedCredentialsProvider(CredentialsProvider credentialsProvider) public void testValidCredentials() { Credential emulated = new Credential("test-emulated-access-key", "test-emulated-secret"); - Credential remote = new Credential("test-remote-access-key", "test-remote-secret"); - Credentials expected = new Credentials(emulated, Optional.of(remote), Optional.empty(), Optional.of(new TestingIdentity("test-username", ImmutableList.of(), "xyzpdq"))); - Optional actual = credentialsProvider.credentials("test-emulated-access-key", Optional.empty()); + IdentityCredential expected = new IdentityCredential(emulated, new TestingIdentity("test-username", ImmutableList.of(), "test-id")); + Optional actual = credentialsProvider.credentials("test-emulated-access-key", Optional.empty()); assertThat(actual).contains(expected); } @Test public void testInvalidCredentials() { - Optional actual = credentialsProvider.credentials("non-existent-key", Optional.empty()); + Optional actual = credentialsProvider.credentials("non-existent-key", Optional.empty()); assertThat(actual).isEmpty(); } } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/http/TestHttpCredentialsProvider.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/http/TestHttpCredentialsProvider.java index b8121c9d..39d00f87 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/http/TestHttpCredentialsProvider.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/credentials/http/TestHttpCredentialsProvider.java @@ -23,8 +23,8 @@ import io.trino.aws.proxy.server.testing.harness.BuilderFilter; import io.trino.aws.proxy.server.testing.harness.TrinoAwsProxyTest; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -34,8 +34,6 @@ import static io.trino.aws.proxy.server.credentials.http.HttpCredentialsModule.HTTP_CREDENTIALS_PROVIDER_IDENTIFIER; import static io.trino.aws.proxy.server.testing.TestingHttpCredentialsProviderServlet.DUMMY_EMULATED_ACCESS_KEY; import static io.trino.aws.proxy.server.testing.TestingHttpCredentialsProviderServlet.DUMMY_EMULATED_SECRET_KEY; -import static io.trino.aws.proxy.server.testing.TestingHttpCredentialsProviderServlet.DUMMY_REMOTE_ACCESS_KEY; -import static io.trino.aws.proxy.server.testing.TestingHttpCredentialsProviderServlet.DUMMY_REMOTE_SECRET_KEY; import static io.trino.aws.proxy.server.testing.TestingUtil.createTestingHttpServer; import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.bindIdentityType; import static java.util.Objects.requireNonNull; @@ -139,10 +137,9 @@ private void testValidCredentials(Optional emulatedAccessToken) private void testValidCredentials(Optional emulatedAccessToken, int expectedRequestCount) { Credential expectedEmulated = new Credential(DUMMY_EMULATED_ACCESS_KEY, DUMMY_EMULATED_SECRET_KEY, emulatedAccessToken); - Credential expectedRemote = new Credential(DUMMY_REMOTE_ACCESS_KEY, DUMMY_REMOTE_SECRET_KEY); - Credentials expected = new Credentials(expectedEmulated, Optional.of(expectedRemote), Optional.empty(), Optional.of(new TestingIdentity("test-username", ImmutableList.of(), "xyzpdq"))); - Optional actual = credentialsProvider.credentials(DUMMY_EMULATED_ACCESS_KEY, emulatedAccessToken); + IdentityCredential expected = new IdentityCredential(expectedEmulated, new TestingIdentity("test-username", ImmutableList.of(), "xyzpdq")); + Optional actual = credentialsProvider.credentials(DUMMY_EMULATED_ACCESS_KEY, emulatedAccessToken); assertThat(actual).contains(expected); assertThat(httpCredentialsServlet.getRequestCount()).isEqualTo(expectedRequestCount); } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/signing/TestSigningController.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/signing/TestSigningController.java index ea9333f7..d923c9b6 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/signing/TestSigningController.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/signing/TestSigningController.java @@ -14,13 +14,11 @@ package io.trino.aws.proxy.server.signing; import io.airlift.units.Duration; -import io.trino.aws.proxy.server.credentials.CredentialsController; import io.trino.aws.proxy.server.rest.RequestLoggerConfig; import io.trino.aws.proxy.server.rest.RequestLoggerController; -import io.trino.aws.proxy.server.testing.TestingRemoteS3Facade; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import io.trino.aws.proxy.spi.rest.Request; import io.trino.aws.proxy.spi.rest.RequestContent; import io.trino.aws.proxy.spi.rest.RequestHeaders; @@ -45,10 +43,10 @@ public class TestSigningController { - private static final Credentials CREDENTIALS = Credentials.build(new Credential("THIS_IS_AN_ACCESS_KEY", "THIS_IS_A_SECRET_KEY")); - private static final CredentialsProvider CREDENTIALS_PROVIDER = (_, _) -> Optional.of(CREDENTIALS); - private static final CredentialsController CREDENTIALS_CONTROLLER = new CredentialsController(new TestingRemoteS3Facade(), CREDENTIALS_PROVIDER); - private static final SigningController LARGE_DRIFT_SIGNING_CONTROLLER = new InternalSigningController(CREDENTIALS_CONTROLLER, new SigningControllerConfig().setMaxClockDrift(new Duration(99999, TimeUnit.DAYS)), new RequestLoggerController(new RequestLoggerConfig())); + private static final Credential CREDENTIAL = new Credential("THIS_IS_AN_ACCESS_KEY", "THIS_IS_A_SECRET_KEY"); + private static final CredentialsProvider CREDENTIALS_PROVIDER = (_, _) -> Optional.of(new IdentityCredential(CREDENTIAL)); + private static final SigningController LARGE_DRIFT_SIGNING_CONTROLLER = new InternalSigningController(CREDENTIALS_PROVIDER, + new SigningControllerConfig().setMaxClockDrift(new Duration(99999, TimeUnit.DAYS)), new RequestLoggerController(new RequestLoggerConfig())); @Test public void testRootLs() @@ -63,11 +61,10 @@ public void testRootLs() requestHeadersBuilder.putOrReplaceSingle("Host", "localhost:10064"); String signature = LARGE_DRIFT_SIGNING_CONTROLLER.signRequest( - new SigningMetadata(SigningServiceType.S3, CREDENTIALS, Optional.empty()), + new SigningMetadata(SigningServiceType.S3, CREDENTIAL, Optional.empty()), "us-east-1", parsedXAmzDate, Optional.empty(), - Credentials::emulated, URI.create("http://localhost:10064/"), requestHeadersBuilder.build(), ImmutableMultiMap.empty(), @@ -95,11 +92,10 @@ public void testBucketLs() queryParametersBuilder.putOrReplaceSingle("encoding-type", "url"); String signature = LARGE_DRIFT_SIGNING_CONTROLLER.signRequest( - new SigningMetadata(SigningServiceType.S3, CREDENTIALS, Optional.empty()), + new SigningMetadata(SigningServiceType.S3, CREDENTIAL, Optional.empty()), "us-east-1", parsedXAmzDate, Optional.empty(), - Credentials::emulated, URI.create("http://localhost:10064/mybucket"), requestHeadersBuilder.build(), queryParametersBuilder.build(), @@ -178,7 +174,8 @@ private static void tryValidateRequestOfAgeAndExpiry(Instant requestDate, Instan private static void tryValidateRequestOfAgeAndExpiry(Instant requestDate, Optional requestExpiry, Duration maxClockDrift) { RequestLoggerController requestLoggerController = new RequestLoggerController(new RequestLoggerConfig()); - SigningController requestSigningController = new InternalSigningController(CREDENTIALS_CONTROLLER, new SigningControllerConfig().setMaxClockDrift(maxClockDrift), requestLoggerController); + SigningController requestSigningController = new InternalSigningController(CREDENTIALS_PROVIDER, new SigningControllerConfig().setMaxClockDrift(maxClockDrift), + requestLoggerController); URI requestUri = URI.create("http://dummy-url"); MultiMap requestHeaderValues = ImmutableMultiMap.builder(false).putOrReplaceSingle("Host", "http://127.0.0.1:8888").build(); @@ -186,11 +183,10 @@ private static void tryValidateRequestOfAgeAndExpiry(Instant requestDate, Option MultiMap requestQueryParams = ImmutableMultiMap.empty(); RequestAuthorization authorization = requestSigningController.signRequest( - new SigningMetadata(SigningServiceType.S3, CREDENTIALS, Optional.empty()), + new SigningMetadata(SigningServiceType.S3, CREDENTIAL, Optional.empty()), "some-region", requestDate, requestExpiry, - Credentials::emulated, requestUri, requestHeaderValues, requestQueryParams, diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/RequestRewriteUtil.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/RequestRewriteUtil.java index 245dbc21..0376c0a5 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/RequestRewriteUtil.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/RequestRewriteUtil.java @@ -15,11 +15,10 @@ import com.google.inject.Inject; import com.google.inject.Scopes; -import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; import io.trino.aws.proxy.server.testing.harness.BuilderFilter; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import software.amazon.awssdk.services.s3.S3Client; import java.util.List; @@ -43,11 +42,10 @@ public static class SetupRequestRewrites @Inject public SetupRequestRewrites( TestingCredentialsRolesProvider credentialsRolesProvider, - @ForTesting Credentials testingCredentials, @ForS3Container List configuredBuckets, @ForS3Container S3Client storageClient) { - credentialsRolesProvider.addCredentials(Credentials.build(CREDENTIAL_TO_REDIRECT, testingCredentials.requiredRemoteCredential())); + credentialsRolesProvider.addCredentials(new IdentityCredential(CREDENTIAL_TO_REDIRECT)); configuredBuckets.forEach(bucket -> storageClient.createBucket(r -> r.bucket(getTargetName(bucket)))); storageClient.createBucket(r -> r.bucket(TEST_CREDENTIAL_REDIRECT_BUCKET)); } @@ -79,10 +77,10 @@ public int getCallCount() } @Override - public Optional testRewrite(Credentials credentials, String bucketName, String keyName) + public Optional testRewrite(String accessKey, String bucketName, String keyName) { callCount.incrementAndGet(); - boolean redirectForTestCredential = credentials.emulated().accessKey().equalsIgnoreCase(CREDENTIAL_TO_REDIRECT.accessKey()); + boolean redirectForTestCredential = accessKey.equalsIgnoreCase(CREDENTIAL_TO_REDIRECT.accessKey()); if (redirectForTestCredential) { return Optional.of(new S3RewriteResult(TEST_CREDENTIAL_REDIRECT_BUCKET, keyName.isEmpty() ? "" : TEST_CREDENTIAL_REDIRECT_KEY)); } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingCredentialsRolesProvider.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingCredentialsRolesProvider.java index e21baf2a..960ec7ad 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingCredentialsRolesProvider.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingCredentialsRolesProvider.java @@ -15,9 +15,14 @@ import io.trino.aws.proxy.spi.credentials.AssumedRoleProvider; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; import io.trino.aws.proxy.spi.credentials.CredentialsProvider; import io.trino.aws.proxy.spi.credentials.EmulatedAssumedRole; +import io.trino.aws.proxy.spi.credentials.Identity; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; +import io.trino.aws.proxy.spi.rest.ParsedS3Request; +import io.trino.aws.proxy.spi.signing.SigningMetadata; import java.time.Instant; import java.util.Map; @@ -36,10 +41,13 @@ * store credentials and assumed roles in a database, etc. */ public class TestingCredentialsRolesProvider - implements CredentialsProvider, AssumedRoleProvider + implements CredentialsProvider, AssumedRoleProvider, RemoteS3ConnectionProvider { - private final Map credentials = new ConcurrentHashMap<>(); + private final Map credentials = new ConcurrentHashMap<>(); private final Map assumedRoleSessions = new ConcurrentHashMap<>(); + private final Map remoteConnections = new ConcurrentHashMap<>(); + private RemoteS3Connection defaultRemoteS3Connection; + private final AtomicInteger assumedRoleCount = new AtomicInteger(); private record Session(Credential sessionCredential, String originalEmulatedAccessKey, Instant expiration) @@ -53,29 +61,43 @@ private record Session(Credential sessionCredential, String originalEmulatedAcce } @Override - public Optional credentials(String emulatedAccessKey, Optional maybeSessionToken) + public Optional remoteConnection(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request) { - if (maybeSessionToken.isPresent()) { - return maybeSessionToken.flatMap(sessionToken -> { - Session session = assumedRoleSessions.get(sessionToken); - boolean isValid = (session != null) && session.expiration.isAfter(Instant.now()); - if (!isValid) { - assumedRoleSessions.remove(sessionToken); - return Optional.empty(); - } - - assumedRoleCount.incrementAndGet(); - - checkState(emulatedAccessKey.equals(session.sessionCredential.accessKey()), "emulatedAccessKey and session accessKey mismatch"); - - Credentials originalCredentials = requireNonNull(credentials.get(session.originalEmulatedAccessKey), "original credentials missing for: " + session.originalEmulatedAccessKey); - return Optional.of(originalCredentials.remoteSessionRole() - .map(remoteSessionRole -> Credentials.build(session.sessionCredential, originalCredentials.requiredRemoteCredential(), remoteSessionRole)) - .orElseGet(() -> Credentials.build(session.sessionCredential, originalCredentials.requiredRemoteCredential()))); - }); - } + return signingMetadata.credential().session().flatMap(sessionToken -> { + Session session = assumedRoleSessions.get(sessionToken); + + boolean isValid = (session != null) && session.expiration.isAfter(Instant.now()); + if (!isValid) { + assumedRoleSessions.remove(sessionToken); + return Optional.empty(); + } + + assumedRoleCount.incrementAndGet(); + + String emulatedAccessKey = signingMetadata.credential().accessKey(); + checkState(emulatedAccessKey.equals(session.sessionCredential().accessKey()), "emulatedAccessKey and session accessKey mismatch"); + + return Optional.ofNullable(remoteConnections.get(session.originalEmulatedAccessKey())); + }).or(() -> Optional.ofNullable(remoteConnections.get(signingMetadata.credential().accessKey()))).or(() -> Optional.ofNullable(defaultRemoteS3Connection)); + } + + @Override + public Optional credentials(String emulatedAccessKey, Optional maybeSessionToken) + { + return maybeSessionToken.flatMap(sessionToken -> { + Session session = assumedRoleSessions.get(sessionToken); + boolean isValid = (session != null) && session.expiration.isAfter(Instant.now()); + if (!isValid) { + assumedRoleSessions.remove(sessionToken); + return Optional.empty(); + } + + assumedRoleCount.incrementAndGet(); + + checkState(emulatedAccessKey.equals(session.sessionCredential.accessKey()), "emulatedAccessKey and session accessKey mismatch"); - return Optional.ofNullable(credentials.get(emulatedAccessKey)); + return Optional.of(new IdentityCredential(session.sessionCredential(), credentials.get(session.originalEmulatedAccessKey()).identity())); + }).or(() -> Optional.ofNullable(credentials.get(emulatedAccessKey))); } @Override @@ -89,7 +111,7 @@ public Optional assumeEmulatedRole( { String originalEmulatedAccessKey = emulatedCredential.accessKey(); return Optional.ofNullable(credentials.get(originalEmulatedAccessKey)) - .map(internal -> { + .map(_ -> { String sessionToken = UUID.randomUUID().toString(); Session session = new Session(new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString(), Optional.of(sessionToken)), originalEmulatedAccessKey, Instant.now().plusSeconds(TimeUnit.HOURS.toSeconds(1))); @@ -104,9 +126,15 @@ public int assumedRoleCount() return assumedRoleCount.get(); } - public void addCredentials(Credentials credentials) + public void addCredentials(IdentityCredential credential) { - this.credentials.put(credentials.emulated().accessKey(), credentials); + this.credentials.put(credential.emulated().accessKey(), credential); + } + + public void addCredentials(IdentityCredential credential, RemoteS3Connection remoteS3Connection) + { + addCredentials(credential); + this.remoteConnections.put(credential.emulated().accessKey(), remoteS3Connection); } public void resetAssumedRoles() @@ -114,4 +142,9 @@ public void resetAssumedRoles() assumedRoleCount.set(0); assumedRoleSessions.clear(); } + + public void setDefaultRemoteConnection(RemoteS3Connection remoteS3Connection) + { + this.defaultRemoteS3Connection = remoteS3Connection; + } } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingHttpCredentialsProviderServlet.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingHttpCredentialsProviderServlet.java index df8f7db9..6839af8d 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingHttpCredentialsProviderServlet.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingHttpCredentialsProviderServlet.java @@ -15,9 +15,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airlift.json.ObjectMapperProvider; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import jakarta.servlet.http.HttpServlet; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; @@ -27,6 +26,7 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; +import static io.airlift.json.JsonCodec.jsonCodec; import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; public class TestingHttpCredentialsProviderServlet @@ -34,8 +34,6 @@ public class TestingHttpCredentialsProviderServlet { public static final String DUMMY_EMULATED_ACCESS_KEY = "test-emulated-access-key"; public static final String DUMMY_EMULATED_SECRET_KEY = "test-emulated-secret-key"; - public static final String DUMMY_REMOTE_ACCESS_KEY = "test-remote-access-key"; - public static final String DUMMY_REMOTE_SECRET_KEY = "test-remote-secret-key"; private final Map expectedHeaders; private final AtomicInteger requestCounter; @@ -73,9 +71,9 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) switch (emulatedAccessKey) { case DUMMY_EMULATED_ACCESS_KEY -> { Credential emulated = new Credential(DUMMY_EMULATED_ACCESS_KEY, DUMMY_EMULATED_SECRET_KEY, sessionToken); - Credential remote = new Credential(DUMMY_REMOTE_ACCESS_KEY, DUMMY_REMOTE_SECRET_KEY); - Credentials credentials = new Credentials(emulated, Optional.of(remote), Optional.empty(), Optional.of(new TestingIdentity("test-username", ImmutableList.of(), "xyzpdq"))); - String jsonCredentials = new ObjectMapperProvider().get().writeValueAsString(credentials); +// Credential remote = new Credential(DUMMY_REMOTE_ACCESS_KEY, DUMMY_REMOTE_SECRET_KEY); + IdentityCredential credentials = new IdentityCredential(emulated, new TestingIdentity("test-username", ImmutableList.of(), "xyzpdq")); + String jsonCredentials = jsonCodec(IdentityCredential.class).toJson(credentials); response.setContentType(APPLICATION_JSON); response.getWriter().print(jsonCredentials); } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingRemoteCredentialsProvider.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingRemoteCredentialsProvider.java deleted file mode 100644 index c7a24217..00000000 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingRemoteCredentialsProvider.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.aws.proxy.server.testing; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import io.trino.aws.proxy.server.testing.containers.S3Container; -import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; -import io.trino.aws.proxy.spi.remote.RemoteSessionRole; - -import java.util.Optional; -import java.util.UUID; - -import static java.util.Objects.requireNonNull; - -public class TestingRemoteCredentialsProvider - implements Provider -{ - private final S3Container s3MockContainer; - private final TestingCredentialsRolesProvider credentialsController; - - @Inject - public TestingRemoteCredentialsProvider(S3Container s3MockContainer, TestingCredentialsRolesProvider credentialsController) - { - this.s3MockContainer = requireNonNull(s3MockContainer, "s3MockContainer is null"); - this.credentialsController = requireNonNull(credentialsController, "credentialsController is null"); - } - - @Override - public Credentials get() - { - Credential policyUserCredential = s3MockContainer.policyUserCredential(); - - RemoteSessionRole remoteSessionRole = new RemoteSessionRole("us-east-1", "minio-doesnt-care", Optional.empty()); - Credentials remoteCredentials = Credentials.build(new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()), policyUserCredential, remoteSessionRole); - credentialsController.addCredentials(remoteCredentials); - - return remoteCredentials; - } -} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3ClientModule.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3ClientModule.java index 6c7f3158..c35b7266 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3ClientModule.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3ClientModule.java @@ -20,7 +20,7 @@ import io.trino.aws.proxy.server.TrinoAwsProxyConfig; import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import jakarta.ws.rs.core.UriBuilder; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.services.s3.S3Client; @@ -44,7 +44,7 @@ public class TestingS3ClientModule public @interface ForVirtualHostProxy {} @Provides - public S3Client getPathStyleAddressingClient(TestingHttpServer httpServer, @ForTesting Credentials credentials, TrinoAwsProxyConfig config) + public S3Client getPathStyleAddressingClient(TestingHttpServer httpServer, @ForTesting IdentityCredential credentials, TrinoAwsProxyConfig config) { Credential emulatedCredentials = credentials.emulated(); AwsBasicCredentials awsBasicCredentials = AwsBasicCredentials.create(emulatedCredentials.accessKey(), emulatedCredentials.secretKey()); @@ -56,7 +56,7 @@ public S3Client getPathStyleAddressingClient(TestingHttpServer httpServer, @ForT @Provides @ForVirtualHostProxy - public S3Client getVirtualHostAddressingClient(TestingHttpServer httpServer, @ForTesting Credentials credentials, TrinoAwsProxyConfig config) + public S3Client getVirtualHostAddressingClient(TestingHttpServer httpServer, @ForTesting IdentityCredential credentials, TrinoAwsProxyConfig config) { checkArgument(config.getS3HostName().isPresent(), "virtual host addressing proxy client requested but S3 hostname is not set"); String hostname = config.getS3HostName().orElseThrow(); diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3PresignController.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3PresignController.java index b0ebbe90..3349bd62 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3PresignController.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3PresignController.java @@ -18,6 +18,7 @@ import io.trino.aws.proxy.server.rest.S3PresignController; import io.trino.aws.proxy.server.security.S3SecurityController; import io.trino.aws.proxy.server.testing.containers.TestContainerUtil; +import io.trino.aws.proxy.spi.credentials.Identity; import io.trino.aws.proxy.spi.rest.ParsedS3Request; import io.trino.aws.proxy.spi.signing.SigningController; import io.trino.aws.proxy.spi.signing.SigningMetadata; @@ -25,6 +26,7 @@ import java.net.URI; import java.time.Instant; import java.util.Map; +import java.util.Optional; public class TestingS3PresignController extends S3PresignController @@ -38,12 +40,13 @@ public TestingS3PresignController(SigningController signingController, TrinoAwsP } @Override - public Map buildPresignedRemoteUrls(SigningMetadata signingMetadata, ParsedS3Request request, Instant targetRequestTimestamp, URI remoteUri) + public Map buildPresignedRemoteUrls(Optional identity, SigningMetadata signingMetadata, ParsedS3Request request, Instant targetRequestTimestamp, + URI remoteUri) { if (rewriteUrisForContainers) { remoteUri = URI.create(TestContainerUtil.asHostUrl(remoteUri.toString())); } - return super.buildPresignedRemoteUrls(signingMetadata, request, targetRequestTimestamp, remoteUri); + return super.buildPresignedRemoteUrls(identity, signingMetadata, request, targetRequestTimestamp, remoteUri); } public void setRewriteUrisForContainers(boolean doRewrites) diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3RequestRewriteController.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3RequestRewriteController.java index 0c79816e..5b22a7c1 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3RequestRewriteController.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3RequestRewriteController.java @@ -15,7 +15,7 @@ import com.google.inject.Inject; import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.Credential; import io.trino.aws.proxy.spi.rest.S3RequestRewriter.S3RewriteResult; import static java.util.Objects.requireNonNull; @@ -23,37 +23,37 @@ public class TestingS3RequestRewriteController { private final TestingS3RequestRewriter s3RequestRewriter; - private final Credentials defaultCredentials; + private final Credential defaultCredential; @Inject - public TestingS3RequestRewriteController(TestingS3RequestRewriter rewriter, @ForTesting Credentials defaultCredentials) + public TestingS3RequestRewriteController(TestingS3RequestRewriter rewriter, @ForTesting Credential defaultCredential) { this.s3RequestRewriter = requireNonNull(rewriter, "rewriter is null"); - this.defaultCredentials = requireNonNull(defaultCredentials, "defaultCredentials is null"); + this.defaultCredential = requireNonNull(defaultCredential, "defaultCredentials is null"); } - private S3RewriteResult rewriteOrNoop(Credentials credentials, String bucket, String key) + private S3RewriteResult rewriteOrNoop(String accessKey, String bucket, String key) { - return s3RequestRewriter.testRewrite(credentials, bucket, key).orElseGet(() -> new S3RewriteResult(bucket, key)); + return s3RequestRewriter.testRewrite(accessKey, bucket, key).orElseGet(() -> new S3RewriteResult(bucket, key)); } - public String getTargetBucket(Credentials credentials, String bucket, String key) + public String getTargetBucket(String accessKey, String bucket, String key) { - return rewriteOrNoop(credentials, bucket, key).finalRequestBucket(); + return rewriteOrNoop(accessKey, bucket, key).finalRequestBucket(); } public String getTargetBucket(String bucket, String key) { - return getTargetBucket(defaultCredentials, bucket, key); + return getTargetBucket(defaultCredential.accessKey(), bucket, key); } - public String getTargetKey(Credentials credentials, String bucket, String key) + public String getTargetKey(String accessKey, String bucket, String key) { - return rewriteOrNoop(credentials, bucket, key).finalRequestKey(); + return rewriteOrNoop(accessKey, bucket, key).finalRequestKey(); } public String getTargetKey(String bucket, String key) { - return getTargetKey(defaultCredentials, bucket, key); + return getTargetKey(defaultCredential.accessKey(), bucket, key); } } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3RequestRewriter.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3RequestRewriter.java index 441a2f91..74e365b9 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3RequestRewriter.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingS3RequestRewriter.java @@ -13,9 +13,10 @@ */ package io.trino.aws.proxy.server.testing; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.Identity; import io.trino.aws.proxy.spi.rest.ParsedS3Request; import io.trino.aws.proxy.spi.rest.S3RequestRewriter; +import io.trino.aws.proxy.spi.signing.SigningMetadata; import java.util.Optional; @@ -25,11 +26,11 @@ public interface TestingS3RequestRewriter { TestingS3RequestRewriter NOOP = (_, _, _) -> Optional.empty(); - Optional testRewrite(Credentials credentials, String bucketName, String keyName); + Optional testRewrite(String accessKey, String bucketName, String keyName); @Override - default Optional rewrite(Credentials credentials, ParsedS3Request request) + default Optional rewrite(Optional identity, SigningMetadata signingMetadata, ParsedS3Request request) { - return testRewrite(credentials, request.bucketName(), request.keyInBucket()); + return testRewrite(signingMetadata.credential().accessKey(), request.bucketName(), request.keyInBucket()); } } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServer.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServer.java index 87ffb23a..fdd828b7 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServer.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServer.java @@ -30,7 +30,6 @@ import io.airlift.log.Level; import io.airlift.log.Logging; import io.airlift.node.testing.TestingNodeModule; -import io.trino.aws.proxy.server.testing.TestingTrinoAwsProxyServerModule.ForTestingRemoteCredentials; import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; import io.trino.aws.proxy.server.testing.containers.MetastoreContainer; import io.trino.aws.proxy.server.testing.containers.OpaContainer; @@ -40,18 +39,23 @@ import io.trino.aws.proxy.server.testing.containers.PySparkContainer.PySparkV4Container; import io.trino.aws.proxy.server.testing.containers.S3Container; import io.trino.aws.proxy.server.testing.containers.S3Container.ForS3Container; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.Credential; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; import io.trino.aws.proxy.spi.remote.RemoteS3Facade; import java.io.Closeable; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Optional; import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; -import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_CREDENTIALS; +import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_IDENTITY_CREDENTIAL; +import static io.trino.aws.proxy.server.testing.TestingUtil.TESTING_REMOTE_CREDENTIAL; import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.assumedRoleProviderModule; import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.credentialsProviderModule; +import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.remoteS3ConnectionProviderModule; import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.remoteS3Module; public final class TestingTrinoAwsProxyServer @@ -112,7 +116,9 @@ public Builder withS3Container() modules.add(binder -> { binder.bind(S3Container.class).asEagerSingleton(); - binder.bind(Credentials.class).annotatedWith(ForTesting.class).toInstance(TESTING_CREDENTIALS); + binder.bind(IdentityCredential.class).annotatedWith(ForTesting.class).toInstance(TESTING_IDENTITY_CREDENTIAL); + binder.bind(Credential.class).annotatedWith(ForTesting.class).toInstance(TESTING_IDENTITY_CREDENTIAL.emulated()); + binder.bind(Credential.class).annotatedWith(ForS3Container.class).toInstance(TESTING_REMOTE_CREDENTIAL); newOptionalBinder(binder, Key.get(new TypeLiteral>() {}, ForS3Container.class)).setDefault().toInstance(ImmutableList.of()); newOptionalBinder(binder, Key.get(RemoteS3Facade.class, ForTesting.class)) @@ -223,8 +229,9 @@ public TestingTrinoAwsProxyServer buildAndStart() withProperty("credentials-provider.type", "testing"); addModule(assumedRoleProviderModule("testing", TestingCredentialsRolesProvider.class, (binder) -> binder.bind(TestingCredentialsRolesProvider.class).in(Scopes.SINGLETON))); withProperty("assumed-role-provider.type", "testing"); - - modules.add(binder -> binder.bind(Credentials.class).annotatedWith(ForTestingRemoteCredentials.class).toProvider(TestingRemoteCredentialsProvider.class)); + addModule(remoteS3ConnectionProviderModule("testing", TestingCredentialsRolesProvider.class, + binder -> binder.bind(TestingCredentialsInitializer.class).in(Scopes.SINGLETON))); + withProperty("remote-s3-connection-provider.type", "testing"); } return start(modules.build(), properties.buildKeepingLast()); @@ -236,7 +243,8 @@ static class TestingCredentialsInitializer @Inject TestingCredentialsInitializer(TestingCredentialsRolesProvider credentialsController) { - credentialsController.addCredentials(TESTING_CREDENTIALS); + credentialsController.addCredentials(TESTING_IDENTITY_CREDENTIAL); + credentialsController.setDefaultRemoteConnection(new RemoteS3Connection(TESTING_REMOTE_CREDENTIAL, Optional.empty(), Optional.empty())); } } diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServerModule.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServerModule.java index f3a60226..3855a257 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServerModule.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingTrinoAwsProxyServerModule.java @@ -14,28 +14,14 @@ package io.trino.aws.proxy.server.testing; import com.google.inject.Binder; -import com.google.inject.BindingAnnotation; import com.google.inject.Scopes; import io.trino.aws.proxy.server.TrinoAwsProxyServerModule; import io.trino.aws.proxy.server.rest.S3PresignController; import io.trino.aws.proxy.server.security.S3SecurityController; -import java.lang.annotation.Retention; -import java.lang.annotation.Target; - -import static java.lang.annotation.ElementType.FIELD; -import static java.lang.annotation.ElementType.METHOD; -import static java.lang.annotation.ElementType.PARAMETER; -import static java.lang.annotation.RetentionPolicy.RUNTIME; - public class TestingTrinoAwsProxyServerModule extends TrinoAwsProxyServerModule { - @Retention(RUNTIME) - @Target({FIELD, PARAMETER, METHOD}) - @BindingAnnotation - public @interface ForTestingRemoteCredentials {} - @Override protected void installS3SecurityController(Binder binder) { diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingUtil.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingUtil.java index bfa23817..aa99f69d 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingUtil.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/TestingUtil.java @@ -22,7 +22,7 @@ import io.airlift.node.NodeConfig; import io.airlift.node.NodeInfo; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import jakarta.servlet.Servlet; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; @@ -61,10 +61,11 @@ public final class TestingUtil { private TestingUtil() {} - public static final Credentials TESTING_CREDENTIALS = Credentials.build( + public static final IdentityCredential TESTING_IDENTITY_CREDENTIAL = new IdentityCredential( new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()), - new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()), - new TestingIdentity(UUID.randomUUID().toString(), List.of(), UUID.randomUUID().toString())); + Optional.of(new TestingIdentity(UUID.randomUUID().toString(), List.of(), UUID.randomUUID().toString()))); + + public static final Credential TESTING_REMOTE_CREDENTIAL = new Credential(UUID.randomUUID().toString(), UUID.randomUUID().toString()); // Domain name with a wildcard CNAME pointing to localhost - needed to test Virtual Host style addressing public static final String LOCALHOST_DOMAIN = "local.gate0.net"; diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/MetastoreContainer.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/MetastoreContainer.java index 49e519e2..f89b349e 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/MetastoreContainer.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/MetastoreContainer.java @@ -18,8 +18,7 @@ import io.airlift.http.server.testing.TestingHttpServer; import io.airlift.log.Logger; import io.trino.aws.proxy.server.testing.TestingUtil; -import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.Credential; import jakarta.annotation.PreDestroy; import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; @@ -47,7 +46,7 @@ public class MetastoreContainer @SuppressWarnings("resource") @Inject - public MetastoreContainer(PostgresContainer postgresContainer, TestingHttpServer httpServer, @ForTesting Credentials testingCredentials, S3Container s3Container) + public MetastoreContainer(PostgresContainer postgresContainer, TestingHttpServer httpServer, @S3Container.ForS3Container Credential remoteCredential, S3Container s3Container) throws IOException { String s3Endpoint = asHostUrl(s3Container.endpoint().toString()); @@ -58,8 +57,8 @@ public MetastoreContainer(PostgresContainer postgresContainer, TestingHttpServer String hiveSiteXml = Resources.toString(Resources.getResource("hive-site.xml"), StandardCharsets.UTF_8) .replace("$ENDPOINT$", s3Endpoint) - .replace("$ACCESS_KEY$", testingCredentials.requiredRemoteCredential().accessKey()) - .replace("$SECRET_KEY$", testingCredentials.requiredRemoteCredential().secretKey()); + .replace("$ACCESS_KEY$", remoteCredential.accessKey()) + .replace("$SECRET_KEY$", remoteCredential.secretKey()); // need to disable SSL to postgres otherwise HMS throws an exception and quits String serviceOpts = "-Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver -Djavax.jdo.option.ConnectionURL=%s -Djavax.jdo.option.ConnectionUserName=%s -Djavax.jdo.option.ConnectionPassword=%s" diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/PySparkContainer.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/PySparkContainer.java index 1e25b564..e016b72e 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/PySparkContainer.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/PySparkContainer.java @@ -18,7 +18,7 @@ import io.airlift.log.Logger; import io.trino.aws.proxy.server.TrinoAwsProxyConfig; import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; -import io.trino.aws.proxy.spi.credentials.Credentials; +import io.trino.aws.proxy.spi.credentials.IdentityCredential; import jakarta.annotation.PreDestroy; import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; @@ -49,7 +49,12 @@ public static class PySparkV3Container extends PySparkContainer { @Inject - public PySparkV3Container(MetastoreContainer metastoreContainer, S3Container s3Container, TestingHttpServer httpServer, @ForTesting Credentials testingCredentials, TrinoAwsProxyConfig trinoS3ProxyConfig) + public PySparkV3Container( + MetastoreContainer metastoreContainer, + S3Container s3Container, + TestingHttpServer httpServer, + @ForTesting IdentityCredential testingCredentials, + TrinoAwsProxyConfig trinoS3ProxyConfig) { super(metastoreContainer, s3Container, httpServer, testingCredentials, trinoS3ProxyConfig, Version.VERSION_3); } @@ -59,7 +64,12 @@ public static class PySparkV4Container extends PySparkContainer { @Inject - public PySparkV4Container(MetastoreContainer metastoreContainer, S3Container s3Container, TestingHttpServer httpServer, @ForTesting Credentials testingCredentials, TrinoAwsProxyConfig trinoS3ProxyConfig) + public PySparkV4Container( + MetastoreContainer metastoreContainer, + S3Container s3Container, + TestingHttpServer httpServer, + @ForTesting IdentityCredential testingCredentials, + TrinoAwsProxyConfig trinoS3ProxyConfig) { super(metastoreContainer, s3Container, httpServer, testingCredentials, trinoS3ProxyConfig, Version.VERSION_4); } @@ -76,7 +86,7 @@ private PySparkContainer( MetastoreContainer metastoreContainer, S3Container s3Container, TestingHttpServer httpServer, - Credentials testingCredentials, + IdentityCredential testingCredentials, TrinoAwsProxyConfig trinoS3ProxyConfig, Version version) { diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/S3Container.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/S3Container.java index e5d23a6f..ded75656 100644 --- a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/S3Container.java +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/testing/containers/S3Container.java @@ -18,9 +18,7 @@ import com.google.inject.Inject; import com.google.inject.Provider; import io.airlift.log.Logger; -import io.trino.aws.proxy.server.testing.TestingUtil.ForTesting; import io.trino.aws.proxy.spi.credentials.Credential; -import io.trino.aws.proxy.spi.credentials.Credentials; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import org.testcontainers.containers.Container; @@ -93,8 +91,6 @@ public class S3Container private final Credential credential; private final Credential policyUserCredential; - private volatile Credentials sessionCredentials; - @Override public S3Client get() { @@ -107,10 +103,10 @@ public S3Client get() public @interface ForS3Container {} @Inject - public S3Container(@ForS3Container List initialBuckets, @ForTesting Credentials credentials) + public S3Container(@ForS3Container List initialBuckets, @ForS3Container Credential remoteCredentials) { this.initialBuckets = requireNonNull(initialBuckets, "initialBuckets is null"); - this.credential = requireNonNull(credentials, "credentials is null").requiredRemoteCredential(); + this.credential = requireNonNull(remoteCredentials, "credentials is null"); Transferable config = Transferable.of(CONFIG_TEMPLATE.formatted(credential.accessKey(), credential.secretKey())); Transferable policyFile = Transferable.of(POLICY); diff --git a/trino-aws-proxy/src/test/resources/credentials.json b/trino-aws-proxy/src/test/resources/credentials.json deleted file mode 100644 index 2f4c4fe4..00000000 --- a/trino-aws-proxy/src/test/resources/credentials.json +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "emulated": { - "accessKey": "test-emulated-access-key", - "secretKey": "test-emulated-secret" - }, - "remote": { - "accessKey": "test-remote-access-key", - "secretKey": "test-remote-secret" - }, - "identity": { - "user": "test-username", - "id": "xyzpdq" - } - } -] \ No newline at end of file From 801e9a1f7243ce4db260196f2777eea9c1da31a0 Mon Sep 17 00:00:00 2001 From: mosiac Date: Tue, 1 Apr 2025 18:16:22 +0100 Subject: [PATCH 2/2] Add RemoteS3ConnectionProvider plugin implementations --- .../spi/credentials/CredentialsProvider.java | 1 - .../proxy/spi/remote/RemoteS3Connection.java | 5 + .../server/TrinoAwsProxyServerModule.java | 6 + ...onfigRemoteS3ConnectionProviderConfig.java | 54 ++++++++ ...onfigRemoteS3ConnectionProviderModule.java | 46 ++++++ .../FileBasedRemoteS3ConnectionModule.java | 42 ++++++ .../FileBasedRemoteS3ConnectionProvider.java | 76 ++++++++++ ...BasedRemoteS3ConnectionProviderConfig.java | 39 ++++++ .../ForHttpRemoteS3ConnectionProvider.java | 31 +++++ .../http/HttpRemoteS3ConnectionProvider.java | 108 +++++++++++++++ .../HttpRemoteS3ConnectionProviderConfig.java | 90 ++++++++++++ ...emoteS3ConnectionProviderConfigModule.java | 43 ++++++ .../server/remote/provider/http/README.md | 131 ++++++++++++++++++ .../remote/provider/http/RequestQuery.java | 57 ++++++++ ...tHttpRemoteS3ConnectionProviderConfig.java | 56 ++++++++ 15 files changed, 784 insertions(+), 1 deletion(-) create mode 100644 trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/config/ConfigRemoteS3ConnectionProviderConfig.java create mode 100644 trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/config/ConfigRemoteS3ConnectionProviderModule.java create mode 100644 trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionModule.java create mode 100644 trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProvider.java create mode 100644 trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProviderConfig.java create mode 100644 trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/ForHttpRemoteS3ConnectionProvider.java create mode 100644 trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProvider.java create mode 100644 trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderConfig.java create mode 100644 trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderConfigModule.java create mode 100644 trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/README.md create mode 100644 trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/RequestQuery.java create mode 100644 trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProviderConfig.java diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java index 787efb8f..237cb357 100644 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java @@ -15,7 +15,6 @@ import java.util.Optional; -// TODO: Add back file-based provider public interface CredentialsProvider { CredentialsProvider NOOP = (_, _) -> Optional.empty(); diff --git a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3Connection.java b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3Connection.java index 3aeeb7aa..ee6f0637 100644 --- a/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3Connection.java +++ b/trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3Connection.java @@ -32,4 +32,9 @@ public record RemoteS3Connection( requireNonNull(remoteSessionRole, "remoteSessionRole is null"); remoteS3FacadeConfiguration = requireNonNull(remoteS3FacadeConfiguration, "remoteS3FacadeConfiguration is null").map(ImmutableMap::copyOf); } + + public RemoteS3Connection(Credential remoteCredential) + { + this(remoteCredential, Optional.empty(), Optional.empty()); + } } diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java index 643c9492..4f9a5bc1 100644 --- a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java @@ -34,6 +34,9 @@ import io.trino.aws.proxy.server.credentials.http.HttpCredentialsModule; import io.trino.aws.proxy.server.remote.DefaultRemoteS3Module; import io.trino.aws.proxy.server.remote.RemoteS3ConnectionController; +import io.trino.aws.proxy.server.remote.provider.config.ConfigRemoteS3ConnectionProviderModule; +import io.trino.aws.proxy.server.remote.provider.file.FileBasedRemoteS3ConnectionModule; +import io.trino.aws.proxy.server.remote.provider.http.HttpRemoteS3ConnectionProviderConfigModule; import io.trino.aws.proxy.server.rest.LimitStreamController; import io.trino.aws.proxy.server.rest.ResourceSecurityDynamicFeature; import io.trino.aws.proxy.server.rest.RestModule; @@ -140,6 +143,9 @@ protected void setup(Binder binder) install(new FileBasedCredentialsModule()); install(new OpaS3SecurityModule()); install(new HttpCredentialsModule()); + install(new FileBasedRemoteS3ConnectionModule()); + install(new ConfigRemoteS3ConnectionProviderModule()); + install(new HttpRemoteS3ConnectionProviderConfigModule()); // RemoteS3 binder newOptionalBinder(binder, RemoteS3Facade.class); diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/config/ConfigRemoteS3ConnectionProviderConfig.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/config/ConfigRemoteS3ConnectionProviderConfig.java new file mode 100644 index 00000000..0df0d248 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/config/ConfigRemoteS3ConnectionProviderConfig.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.config; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigSecuritySensitive; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +public class ConfigRemoteS3ConnectionProviderConfig +{ + private String accessKey; + private String secretKey; + + @NotNull + @NotEmpty + public String getAccessKey() + { + return accessKey; + } + + @Config("remote-s3-connection-provider.access-key") + public ConfigRemoteS3ConnectionProviderConfig setAccessKey(String accessKey) + { + this.accessKey = accessKey; + return this; + } + + @NotNull + @NotEmpty + public String getSecretKey() + { + return secretKey; + } + + @ConfigSecuritySensitive + @Config("remote-s3-connection-provider.access-key") + public ConfigRemoteS3ConnectionProviderConfig setSecretKey(String secretKey) + { + this.secretKey = secretKey; + return this; + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/config/ConfigRemoteS3ConnectionProviderModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/config/ConfigRemoteS3ConnectionProviderModule.java new file mode 100644 index 00000000..b3df5454 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/config/ConfigRemoteS3ConnectionProviderModule.java @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.config; + +import com.google.inject.Binder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.aws.proxy.spi.credentials.Credential; +import io.trino.aws.proxy.spi.plugin.config.RemoteS3ConnectionProviderConfig; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; + +import java.util.Optional; + +import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; +import static io.airlift.configuration.ConditionalModule.conditionalModule; + +public class ConfigRemoteS3ConnectionProviderModule + extends AbstractConfigurationAwareModule +{ + public static final String CONFIG_REMOTE_S3_CONNECTION_PROVIDER = "config"; + + @Override + protected void setup(Binder binder) + { + install(conditionalModule( + RemoteS3ConnectionProviderConfig.class, + config -> config.getPluginIdentifier().map(CONFIG_REMOTE_S3_CONNECTION_PROVIDER::equals).orElse(false), + innerBinder -> { + ConfigRemoteS3ConnectionProviderConfig config = buildConfigObject(ConfigRemoteS3ConnectionProviderConfig.class); + newOptionalBinder(innerBinder, RemoteS3ConnectionProvider.class) + .setBinding() + .toInstance((_, _, _) -> Optional.of(new RemoteS3Connection(new Credential(config.getAccessKey(), config.getSecretKey())))); + })); + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionModule.java new file mode 100644 index 00000000..3d3321d9 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionModule.java @@ -0,0 +1,42 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.file; + +import com.google.inject.Binder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; + +import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.remoteS3ConnectionProviderModule; + +public class FileBasedRemoteS3ConnectionModule + extends AbstractConfigurationAwareModule +{ + // set as config value for "remote-s3-connection-provider.type" + public static final String FILE_BASED_REMOTE_S3_CONNECTION_PROVIDER = "file"; + + @Override + protected void setup(Binder binder) + { + install(remoteS3ConnectionProviderModule( + FILE_BASED_REMOTE_S3_CONNECTION_PROVIDER, + FileBasedRemoteS3ConnectionProvider.class, + innerBinder -> { + configBinder(innerBinder).bindConfig(FileBasedRemoteS3ConnectionProviderConfig.class); + innerBinder.bind(FileBasedRemoteS3ConnectionProvider.class); + jsonCodecBinder(innerBinder).bindMapJsonCodec(String.class, RemoteS3Connection.class); + })); + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProvider.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProvider.java new file mode 100644 index 00000000..282dac6a --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProvider.java @@ -0,0 +1,76 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.file; + +import com.google.common.io.Files; +import com.google.inject.Inject; +import io.airlift.json.JsonCodec; +import io.trino.aws.proxy.spi.credentials.Identity; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; +import io.trino.aws.proxy.spi.rest.ParsedS3Request; +import io.trino.aws.proxy.spi.signing.SigningMetadata; + +import java.util.Map; +import java.util.Optional; + +/** + *

File-based RemoteS3ConnectionProvider that reads a JSON file containing a mapping from emulated access key to + * RemoteS3Connection.

+ * 
{@code
+ * {
+ *   "emulated-access-key-1": {
+ *     "remoteCredential": {
+ *       "accessKey": "remote-access-key",
+ *       "secretKey": "remote-secret-key"
+ *     },
+ *     "remoteSessionRole": {
+ *       "region": "us-east-1",
+ *       "roleArn": "arn:aws:iam::123456789012:role/role-name",
+ *       "externalId": "external-id",
+ *       "stsEndpoint": "https://sts.us-east-1.amazonaws.com"
+ *     },
+ *     "remoteS3FacadeConfiguration": {
+ *       "remoteS3.https": true,
+ *       "remoteS3.domain": "s3.amazonaws.com",
+ *       "remoteS3.port": 443,
+ *       "remoteS3.virtual-host-style": false,
+ *       "remoteS3.hostname.template": "${domain}"
+ *     }
+ *   }
+ * }
+ * }
+ */ +public class FileBasedRemoteS3ConnectionProvider + implements RemoteS3ConnectionProvider +{ + private final Map remoteS3Connections; + + @Inject + public FileBasedRemoteS3ConnectionProvider(FileBasedRemoteS3ConnectionProviderConfig config, JsonCodec> jsonCodec) + { + try { + this.remoteS3Connections = jsonCodec.fromJson(Files.toByteArray(config.getConnectionsFile())); + } + catch (Exception e) { + throw new RuntimeException("Failed to read remote S3 connections file", e); + } + } + + @Override + public Optional remoteConnection(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request) + { + return Optional.ofNullable(remoteS3Connections.get(signingMetadata.credential().accessKey())); + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProviderConfig.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProviderConfig.java new file mode 100644 index 00000000..635e509f --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProviderConfig.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.file; + +import io.airlift.configuration.Config; +import io.airlift.configuration.validation.FileExists; +import jakarta.validation.constraints.NotNull; + +import java.io.File; + +public class FileBasedRemoteS3ConnectionProviderConfig +{ + private File connectionsFile; + + @NotNull + @FileExists + public File getConnectionsFile() + { + return connectionsFile; + } + + @Config("remote-s3.connections-file-path") + public FileBasedRemoteS3ConnectionProviderConfig setConnectionsFile(File connectionsFile) + { + this.connectionsFile = connectionsFile; + return this; + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/ForHttpRemoteS3ConnectionProvider.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/ForHttpRemoteS3ConnectionProvider.java new file mode 100644 index 00000000..24d97f00 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/ForHttpRemoteS3ConnectionProvider.java @@ -0,0 +1,31 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@BindingAnnotation +@Target({FIELD, PARAMETER, METHOD}) +@Retention(RUNTIME) +public @interface ForHttpRemoteS3ConnectionProvider +{ +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProvider.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProvider.java new file mode 100644 index 00000000..472e810b --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProvider.java @@ -0,0 +1,108 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.benmanes.caffeine.cache.Cache; +import io.airlift.http.client.FullJsonResponseHandler.JsonResponse; +import io.airlift.http.client.HttpClient; +import io.airlift.http.client.HttpStatus; +import io.airlift.json.JsonCodec; +import io.trino.aws.proxy.spi.credentials.Identity; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; +import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider; +import io.trino.aws.proxy.spi.rest.ParsedS3Request; +import io.trino.aws.proxy.spi.signing.SigningMetadata; +import jakarta.ws.rs.core.UriBuilder; + +import java.net.URI; +import java.util.EnumSet; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static com.github.benmanes.caffeine.cache.Caffeine.newBuilder; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static io.airlift.http.client.FullJsonResponseHandler.createFullJsonResponseHandler; +import static io.airlift.http.client.Request.Builder.prepareGet; +import static java.util.Objects.requireNonNull; + +public class HttpRemoteS3ConnectionProvider + implements RemoteS3ConnectionProvider +{ + private final HttpClient httpClient; + private final URI endpoint; + private final EnumSet requestQueryFields; + private final JsonCodec responseCodec; + private final ObjectMapper objectMapper; + private final Optional>, Optional>> cache; + + public HttpRemoteS3ConnectionProvider( + @ForHttpRemoteS3ConnectionProvider HttpClient httpClient, + HttpRemoteS3ConnectionProviderConfig config, + JsonCodec responseCodec, + ObjectMapper objectMapper) + { + this.httpClient = requireNonNull(httpClient, "httpClient is null"); + this.responseCodec = requireNonNull(responseCodec, "responseCodec is null"); + this.endpoint = config.getEndpoint(); + this.requestQueryFields = config.getRequestFields(); + this.objectMapper = requireNonNull(objectMapper, "objectMapper is null"); + if (config.getCacheSize() > 0) { + this.cache = Optional.of(newBuilder() + .maximumSize(config.getCacheSize()) + .expireAfterWrite(config.getCacheTtl().toJavaTime()) + .build(this::requestRemoteConnection)); + } + else { + this.cache = Optional.empty(); + } + } + + @Override + public Optional remoteConnection(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request) + { + Set> requestQueries = buildRequestQueries(signingMetadata, identity, request); + return cache.map(actualCache -> actualCache.get(requestQueries, this::requestRemoteConnection)) + .orElseGet(() -> requestRemoteConnection(requestQueries)); + } + + private Set> buildRequestQueries(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request) + { + return requestQueryFields.stream() + .map(requestField -> Map.entry(requestField.toString(), requestField.getValue(signingMetadata, identity, request, objectMapper))) + .collect(toImmutableSet()); + } + + private Optional requestRemoteConnection(Set> requestQueries) + { + UriBuilder uriBuilder = UriBuilder.fromUri(endpoint); + requestQueries.forEach(query -> uriBuilder.queryParam(query.getKey().toLowerCase(Locale.ROOT), query.getValue())); + JsonResponse response = httpClient.execute( + prepareGet().setUri(uriBuilder.build()).build(), + createFullJsonResponseHandler(responseCodec)); + HttpStatus statusCode = HttpStatus.fromStatusCode(response.getStatusCode()); + if (statusCode.family() != HttpStatus.Family.SUCCESSFUL) { + if (statusCode == HttpStatus.NOT_FOUND) { + return Optional.empty(); + } + throw new RuntimeException("Failed to get remote S3 connection with HTTP plugin. Response code: " + statusCode + "; body: \n" + response.getResponseBody()); + } + if (!response.hasValue()) { + throw new RuntimeException("Failed to get remote S3 connection with HTTP plugin. Response code: " + statusCode + "; no body"); + } + return Optional.of(response.getValue()); + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderConfig.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderConfig.java new file mode 100644 index 00000000..5dc99914 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderConfig.java @@ -0,0 +1,90 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import io.airlift.configuration.Config; +import io.airlift.units.Duration; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; + +import java.net.URI; +import java.util.EnumSet; +import java.util.List; + +public class HttpRemoteS3ConnectionProviderConfig +{ + private URI endpoint; + private EnumSet requestFields = EnumSet.allOf(RequestQuery.class); + private long cacheSize; + private Duration cacheTtl = Duration.valueOf("1s"); + + @NotNull + public URI getEndpoint() + { + return endpoint; + } + + @Config("remote-s3-connection-provider.http.endpoint") + public HttpRemoteS3ConnectionProviderConfig setEndpoint(URI endpoint) + { + this.endpoint = endpoint; + return this; + } + + @NotNull + @NotEmpty + public EnumSet getRequestFields() + { + return requestFields; + } + + @Config("remote-s3-connection-provider.http.request-fields") + public HttpRemoteS3ConnectionProviderConfig setRequestFields(List requestFields) + { + if (requestFields.isEmpty()) { + this.requestFields = EnumSet.noneOf(RequestQuery.class); + } + else { + this.requestFields = EnumSet.copyOf(requestFields); + } + return this; + } + + @Min(0) + public long getCacheSize() + { + return cacheSize; + } + + @Config("remote-s3-connection-provider.http.cache-size") + public HttpRemoteS3ConnectionProviderConfig setCacheSize(long cacheSize) + { + this.cacheSize = cacheSize; + return this; + } + + @NotNull + public Duration getCacheTtl() + { + return cacheTtl; + } + + @Config("remote-s3-connection-provider.http.cache-ttl") + public HttpRemoteS3ConnectionProviderConfig setCacheTtl(Duration cacheTtl) + { + this.cacheTtl = cacheTtl; + return this; + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderConfigModule.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderConfigModule.java new file mode 100644 index 00000000..443ae31c --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/HttpRemoteS3ConnectionProviderConfigModule.java @@ -0,0 +1,43 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.google.inject.Binder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.aws.proxy.spi.remote.RemoteS3Connection; + +import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.airlift.http.client.HttpClientBinder.httpClientBinder; +import static io.airlift.json.JsonCodecBinder.jsonCodecBinder; +import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.remoteS3ConnectionProviderModule; + +public class HttpRemoteS3ConnectionProviderConfigModule + extends AbstractConfigurationAwareModule +{ + public static final String HTTP_REMOTE_S3_CONNECTION_PROVIDER = "http"; + + @Override + protected void setup(Binder binder) + { + install(remoteS3ConnectionProviderModule( + HTTP_REMOTE_S3_CONNECTION_PROVIDER, + HttpRemoteS3ConnectionProvider.class, + innerBinder -> { + httpClientBinder(innerBinder).bindHttpClient("remote-s3-connection-provider.http", ForHttpRemoteS3ConnectionProvider.class); + configBinder(innerBinder).bindConfig(HttpRemoteS3ConnectionProviderConfig.class); + innerBinder.bind(HttpRemoteS3ConnectionProvider.class); + jsonCodecBinder(innerBinder).bindJsonCodec(RemoteS3Connection.class); + })); + } +} diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/README.md b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/README.md new file mode 100644 index 00000000..9a4f1817 --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/README.md @@ -0,0 +1,131 @@ +# HttpRemoteS3ConnectionProvider Plugin + +## Overview + +The `HttpRemoteS3ConnectionProvider` plugin provides a way to retrieve remote S3 connection details via HTTP requests. This plugin is configurable and supports caching of the +connection details. + +## Configuration + +The following table lists the configuration properties available for the `HttpRemoteS3ConnectionProvider`: + +| Property | Description | Default Value | +|-----------------------------------------------------|-----------------------------------------------------------------|---------------| +| `remote-s3-connection-provider.http.endpoint` | The HTTP endpoint to retrieve the remote S3 connection details. | None | +| `remote-s3-connection-provider.http.request-fields` | The fields to include in the HTTP request query parameters. | All fields | +| `remote-s3-connection-provider.http.cache-size` | The maximum size of the cache for remote S3 connections. | 0 | +| `remote-s3-connection-provider.http.cache-ttl` | The time-to-live for cache entries. | 1s | + +## Example Configuration + +Here is an example configuration for the `HttpRemoteS3ConnectionProvider`: + +```properties +remote-s3-connection-provider.type=http +remote-s3-connection-provider.http.endpoint=https://example.com/api/v1 +remote-s3-connection-provider.http.request-fields=BUCKET,EMULATED_ACCESS_KEY +remote-s3-connection-provider.http.cache-size=100 +remote-s3-connection-provider.http.cache-ttl=5m +``` + +## RequestQuery + +The `RequestQuery` enum defines the fields that can be included in the HTTP request query parameters. Each field is associated with a `FieldSelector` that determines how the value +for the field is obtained. The available fields are: + +- `BUCKET`: The bucket name from the `ParsedS3Request`. +- `KEY`: The key in the bucket from the `ParsedS3Request`. +- `EMULATED_ACCESS_KEY`: The access key from the `SigningMetadata`. +- `IDENTITY`: The identity in JSON format, if available. + +## OpenAPI Specification + +The following OpenAPI specification defines the API for retrieving remote S3 connection details: + +```yaml +openapi: 3.0.3 +info: + title: Remote S3 Connection Service + description: API for retrieving remote S3 connection details + version: 1.0.0 +servers: + - url: http://localhost:8080 +paths: + /: + get: + summary: Get Remote S3 Connection + description: Retrieve the remote S3 connection details based on query parameters + parameters: + - in: query + name: bucket + schema: + type: string + required: true + description: The bucket name + - in: query + name: key + schema: + type: string + required: true + description: The key in the bucket + - in: query + name: emulatedAccessKey + schema: + type: string + required: true + description: The emulated access key + - in: query + name: identity + schema: + type: string + required: false + description: The identity in JSON format + responses: + '200': + description: Successful response with remote S3 connection details + content: + application/json: + schema: + $ref: '#/components/schemas/RemoteS3Connection' + '404': + description: Remote S3 connection not found + '500': + description: Internal server error +components: + schemas: + RemoteS3Connection: + type: object + properties: + remoteCredential: + $ref: '#/components/schemas/Credential' + remoteSessionRole: + $ref: '#/components/schemas/RemoteSessionRole' + remoteS3FacadeConfiguration: + type: object + additionalProperties: + type: string + Credential: + type: object + properties: + accessKey: + type: string + secretKey: + type: string + session: + type: string + nullable: true + RemoteSessionRole: + type: object + properties: + region: + type: string + roleArn: + type: string + externalId: + type: string + nullable: true + stsEndpoint: + type: string + format: uri + nullable: true +``` diff --git a/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/RequestQuery.java b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/RequestQuery.java new file mode 100644 index 00000000..c2595d2a --- /dev/null +++ b/trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/RequestQuery.java @@ -0,0 +1,57 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.trino.aws.proxy.spi.credentials.Identity; +import io.trino.aws.proxy.spi.rest.ParsedS3Request; +import io.trino.aws.proxy.spi.signing.SigningMetadata; + +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public enum RequestQuery +{ + BUCKET((_, _, request, _) -> request.bucketName()), + KEY((_, _, request, _) -> request.keyInBucket()), + EMULATED_ACCESS_KEY((signingMetadata, _, _, _) -> signingMetadata.credential().accessKey()), + IDENTITY((_, identity, _, objectMapper) -> identity.map(value -> { + try { + return objectMapper.writeValueAsString(value); + } + catch (JsonProcessingException exception) { + throw new RuntimeException(exception); + } + }).orElse("")); + + private final FieldSelector selector; + + RequestQuery(FieldSelector selector) + { + this.selector = requireNonNull(selector, "selector is null"); + } + + public String getValue(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request, ObjectMapper objectMapper) + { + return selector.apply(signingMetadata, identity, request, objectMapper); + } + + @FunctionalInterface + private interface FieldSelector + { + String apply(SigningMetadata signingMetadata, Optional identity, ParsedS3Request request, ObjectMapper objectMapper); + } +} diff --git a/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProviderConfig.java b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProviderConfig.java new file mode 100644 index 00000000..ea531b8c --- /dev/null +++ b/trino-aws-proxy/src/test/java/io/trino/aws/proxy/server/remote/provider/http/TestHttpRemoteS3ConnectionProviderConfig.java @@ -0,0 +1,56 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.aws.proxy.server.remote.provider.http; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import io.airlift.units.Duration; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.util.EnumSet; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; + +public class TestHttpRemoteS3ConnectionProviderConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(HttpRemoteS3ConnectionProviderConfig.class) + .setEndpoint(null) + .setRequestFields(EnumSet.allOf(RequestQuery.class).stream().collect(toImmutableList())) + .setCacheSize(0) + .setCacheTtl(Duration.valueOf("1s"))); + } + + @Test + public void testExplicitPropertyMappings() + { + assertFullMapping( + ImmutableMap.of( + "remote-s3-connection-provider.http.endpoint", "https://example.com/get_connection", + "remote-s3-connection-provider.http.request-fields", "bucket,emulated-access-key", + "remote-s3-connection-provider.http.cache-size", "1000", + "remote-s3-connection-provider.http.cache-ttl", "5m"), + new HttpRemoteS3ConnectionProviderConfig() + .setEndpoint(URI.create("https://example.com/get_connection")) + .setRequestFields(ImmutableList.of(RequestQuery.BUCKET, RequestQuery.EMULATED_ACCESS_KEY)) + .setCacheSize(1000) + .setCacheTtl(Duration.valueOf("5m"))); + } +}