Skip to content

Commit a59a2f1

Browse files
mosiac1mosiac1
committed
Add RemoteS3ConnectionProvider plugin implementations
Added generic implementations for the RemoteS3ConnectionProvider interface: static, file and http. Added a README for the file and http ones as those are a bit more complex. Added more tests to dynamic RemoteS3Facade creation
1 parent f2353a5 commit a59a2f1

30 files changed

+1688
-39
lines changed

‎trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/credentials/CredentialsProvider.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515

1616
import java.util.Optional;
1717

18-
// TODO: Add back file-based provider
1918
public interface CredentialsProvider
2019
{
2120
CredentialsProvider NOOP = (_, _) -> Optional.empty();

‎trino-aws-proxy-spi/src/main/java/io/trino/aws/proxy/spi/remote/RemoteS3ConnectionProvider.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,5 @@ public interface RemoteS3ConnectionProvider
2424
{
2525
RemoteS3ConnectionProvider NOOP = (_, _, _) -> Optional.empty();
2626

27-
Optional<RemoteS3Connection> remoteConnection(SigningMetadata signingMetadata, Optional<Identity> identity, ParsedS3Request request);
27+
Optional<? extends RemoteS3Connection> remoteConnection(SigningMetadata signingMetadata, Optional<Identity> identity, ParsedS3Request request);
2828
}

‎trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/TrinoAwsProxyServerModule.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@
3434
import io.trino.aws.proxy.server.credentials.http.HttpCredentialsModule;
3535
import io.trino.aws.proxy.server.remote.DefaultRemoteS3Module;
3636
import io.trino.aws.proxy.server.remote.RemoteS3ConnectionController;
37+
import io.trino.aws.proxy.server.remote.provider.file.FileBasedRemoteS3ConnectionModule;
38+
import io.trino.aws.proxy.server.remote.provider.http.HttpRemoteS3ConnectionProviderModule;
39+
import io.trino.aws.proxy.server.remote.provider.preset.StaticRemoteS3ConnectionProviderModule;
3740
import io.trino.aws.proxy.server.rest.LimitStreamController;
3841
import io.trino.aws.proxy.server.rest.ResourceSecurityDynamicFeature;
3942
import io.trino.aws.proxy.server.rest.RestModule;
@@ -134,6 +137,9 @@ protected void setup(Binder binder)
134137
install(new FileBasedCredentialsModule());
135138
install(new OpaS3SecurityModule());
136139
install(new HttpCredentialsModule());
140+
install(new FileBasedRemoteS3ConnectionModule());
141+
install(new StaticRemoteS3ConnectionProviderModule());
142+
install(new HttpRemoteS3ConnectionProviderModule());
137143

138144
configBinder(binder).bindConfig(RemoteS3Config.class);
139145
// RemoteS3 provided implementation

‎trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/SerializableRemoteS3Connection.java

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.aws.proxy.server.remote.provider;
15+
16+
import com.fasterxml.jackson.annotation.JsonCreator;
17+
import com.fasterxml.jackson.annotation.JsonProperty;
18+
import com.google.inject.ConfigurationException;
19+
import io.airlift.configuration.ConfigurationFactory;
20+
import io.trino.aws.proxy.server.remote.DefaultRemoteS3Config;
21+
import io.trino.aws.proxy.server.remote.PathStyleRemoteS3Facade;
22+
import io.trino.aws.proxy.server.remote.VirtualHostStyleRemoteS3Facade;
23+
import io.trino.aws.proxy.spi.credentials.Credential;
24+
import io.trino.aws.proxy.spi.remote.RemoteS3Connection;
25+
import io.trino.aws.proxy.spi.remote.RemoteS3Facade;
26+
import io.trino.aws.proxy.spi.remote.RemoteSessionRole;
27+
28+
import java.util.Map;
29+
import java.util.Optional;
30+
import java.util.Set;
31+
32+
import static com.google.common.collect.Sets.difference;
33+
import static java.lang.String.format;
34+
import static java.util.Objects.requireNonNull;
35+
36+
public record SerializableRemoteS3Connection(
37+
Credential remoteCredential,
38+
Optional<RemoteSessionRole> remoteSessionRole,
39+
Optional<RemoteS3Facade> remoteS3Facade)
40+
implements RemoteS3Connection
41+
{
42+
public SerializableRemoteS3Connection
43+
{
44+
requireNonNull(remoteCredential, "remoteCredential is null");
45+
requireNonNull(remoteSessionRole, "remoteSessionRole is null");
46+
requireNonNull(remoteS3Facade, "remoteS3Facade is null");
47+
}
48+
49+
@JsonCreator
50+
public static SerializableRemoteS3Connection fromConfig(
51+
@JsonProperty("remoteCredential") Credential remoteCredential,
52+
@JsonProperty("remoteSessionRole") Optional<RemoteSessionRole> remoteSessionRole,
53+
@JsonProperty("remoteS3FacadeConfiguration") Optional<Map<String, String>> remoteS3FacadeConfiguration)
54+
{
55+
Optional<RemoteS3Facade> facade = remoteS3FacadeConfiguration.map(config -> {
56+
ConfigurationFactory configurationFactory = new ConfigurationFactory(config);
57+
DefaultRemoteS3Config parsedConfig;
58+
try {
59+
parsedConfig = configurationFactory.build(DefaultRemoteS3Config.class);
60+
}
61+
catch (ConfigurationException e) {
62+
throw new IllegalArgumentException("Failed create RemoteS3Facade from RemoteS3FacadeConfiguration", e);
63+
}
64+
Set<String> unusedProperties = difference(configurationFactory.getProperties().keySet(), configurationFactory.getUsedProperties());
65+
if (!unusedProperties.isEmpty()) {
66+
throw new IllegalArgumentException(format("Failed to create RemoteS3Facade from RemoteS3FacadeConfiguration. Unused properties when instantiating " +
67+
"DefaultRemoteS3Config: %s", unusedProperties));
68+
}
69+
return parsedConfig.getVirtualHostStyle() ? new VirtualHostStyleRemoteS3Facade(parsedConfig) : new PathStyleRemoteS3Facade(parsedConfig);
70+
});
71+
return new SerializableRemoteS3Connection(remoteCredential, remoteSessionRole, facade);
72+
}
73+
}

‎trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionModule.java

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.aws.proxy.server.remote.provider.file;
15+
16+
import com.google.inject.Binder;
17+
import io.airlift.configuration.AbstractConfigurationAwareModule;
18+
import io.trino.aws.proxy.server.remote.provider.SerializableRemoteS3Connection;
19+
20+
import static io.airlift.configuration.ConfigBinder.configBinder;
21+
import static io.airlift.json.JsonCodecBinder.jsonCodecBinder;
22+
import static io.trino.aws.proxy.spi.plugin.TrinoAwsProxyServerBinding.remoteS3ConnectionProviderModule;
23+
24+
public class FileBasedRemoteS3ConnectionModule
25+
extends AbstractConfigurationAwareModule
26+
{
27+
// set as config value for "remote-s3-connection-provider.type"
28+
public static final String FILE_BASED_REMOTE_S3_CONNECTION_PROVIDER = "file";
29+
30+
@Override
31+
protected void setup(Binder binder)
32+
{
33+
install(remoteS3ConnectionProviderModule(
34+
FILE_BASED_REMOTE_S3_CONNECTION_PROVIDER,
35+
FileBasedRemoteS3ConnectionProvider.class,
36+
innerBinder -> {
37+
configBinder(innerBinder).bindConfig(FileBasedRemoteS3ConnectionProviderConfig.class);
38+
innerBinder.bind(FileBasedRemoteS3ConnectionProvider.class);
39+
jsonCodecBinder(innerBinder).bindMapJsonCodec(String.class, SerializableRemoteS3Connection.class);
40+
}));
41+
}
42+
}

‎trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProvider.java

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.aws.proxy.server.remote.provider.file;
15+
16+
import com.google.common.io.Files;
17+
import com.google.inject.Inject;
18+
import io.airlift.json.JsonCodec;
19+
import io.trino.aws.proxy.server.remote.provider.SerializableRemoteS3Connection;
20+
import io.trino.aws.proxy.spi.credentials.Identity;
21+
import io.trino.aws.proxy.spi.remote.RemoteS3Connection;
22+
import io.trino.aws.proxy.spi.remote.RemoteS3ConnectionProvider;
23+
import io.trino.aws.proxy.spi.rest.ParsedS3Request;
24+
import io.trino.aws.proxy.spi.signing.SigningMetadata;
25+
26+
import java.util.Map;
27+
import java.util.Optional;
28+
29+
/**
30+
* <p>File-based RemoteS3ConnectionProvider that reads a JSON file containing a mapping from emulated access key to
31+
* RemoteS3Connection.</p>
32+
* <pre>{@code
33+
* {
34+
* "emulated-access-key-1": {
35+
* "remoteCredential": {
36+
* "accessKey": "remote-access-key",
37+
* "secretKey": "remote-secret-key"
38+
* },
39+
* "remoteSessionRole": {
40+
* "region": "us-east-1",
41+
* "roleArn": "arn:aws:iam::123456789012:role/role-name",
42+
* "externalId": "external-id",
43+
* "stsEndpoint": "https://sts.us-east-1.amazonaws.com"
44+
* },
45+
* "remoteS3FacadeConfiguration": {
46+
* "remoteS3.https": true,
47+
* "remoteS3.domain": "s3.amazonaws.com",
48+
* "remoteS3.port": 443,
49+
* "remoteS3.virtual-host-style": false,
50+
* "remoteS3.hostname.template": "${domain}"
51+
* }
52+
* }
53+
* }
54+
* }</pre>
55+
*/
56+
public class FileBasedRemoteS3ConnectionProvider
57+
implements RemoteS3ConnectionProvider
58+
{
59+
private final Map<String, SerializableRemoteS3Connection> remoteS3Connections;
60+
61+
@Inject
62+
public FileBasedRemoteS3ConnectionProvider(FileBasedRemoteS3ConnectionProviderConfig config, JsonCodec<Map<String, SerializableRemoteS3Connection>> jsonCodec)
63+
{
64+
try {
65+
this.remoteS3Connections = jsonCodec.fromJson(Files.toByteArray(config.getConnectionsFile()));
66+
}
67+
catch (Exception e) {
68+
throw new RuntimeException("Failed to read remote S3 connections file", e);
69+
}
70+
}
71+
72+
@Override
73+
public Optional<RemoteS3Connection> remoteConnection(SigningMetadata signingMetadata, Optional<Identity> identity, ParsedS3Request request)
74+
{
75+
return Optional.ofNullable(remoteS3Connections.get(signingMetadata.credential().accessKey()));
76+
}
77+
}

‎trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/FileBasedRemoteS3ConnectionProviderConfig.java

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.aws.proxy.server.remote.provider.file;
15+
16+
import io.airlift.configuration.Config;
17+
import io.airlift.configuration.validation.FileExists;
18+
import jakarta.validation.constraints.NotNull;
19+
20+
import java.io.File;
21+
22+
public class FileBasedRemoteS3ConnectionProviderConfig
23+
{
24+
private File connectionsFile;
25+
26+
@NotNull
27+
@FileExists
28+
public File getConnectionsFile()
29+
{
30+
return connectionsFile;
31+
}
32+
33+
@Config("remote-s3-connection-provider.connections-file-path")
34+
public FileBasedRemoteS3ConnectionProviderConfig setConnectionsFile(File connectionsFile)
35+
{
36+
this.connectionsFile = connectionsFile;
37+
return this;
38+
}
39+
}

‎trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/file/README.md

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# FileBasedRemoteS3ConnectionProvider Plugin
2+
3+
## Overview
4+
5+
The `FileBasedRemoteS3ConnectionProvider` plugin reads remote S3 connection details from a JSON file. This plugin is configured via a file path and supports a flexible JSON mapping
6+
of access keys to connection details.
7+
8+
## Configuration
9+
10+
The following property is available for the `FileBasedRemoteS3ConnectionProvider`:
11+
12+
| Property | Description | Default Value |
13+
|-----------------------------------|-----------------------------------------------------------------|---------------|
14+
| `remote-s3.connections-file-path` | The path to the JSON file containing the S3 connection mapping. | None |
15+
16+
## Example Configuration
17+
18+
Below is an example configuration for the `FileBasedRemoteS3ConnectionProvider`:
19+
20+
```properties
21+
remote-s3-connection-provider.type=file
22+
remote-s3.connections-file-path=/path/to/your/connections.json
23+
```
24+
25+
## JSON File Format
26+
27+
The JSON file should map an emulated access key to its corresponding S3 connection details. For example:
28+
29+
```json
30+
{
31+
"emulated-access-key-1": {
32+
"remoteCredential": {
33+
"accessKey": "remote-access-key",
34+
"secretKey": "remote-secret-key"
35+
},
36+
"remoteSessionRole": {
37+
"region": "us-east-1",
38+
"roleArn": "arn:aws:iam::123456789012:role/role-name",
39+
"externalId": "external-id",
40+
"stsEndpoint": "https://sts.us-east-1.amazonaws.com"
41+
},
42+
"remoteS3FacadeConfiguration": {
43+
"remoteS3.https": true,
44+
"remoteS3.domain": "s3.amazonaws.com",
45+
"remoteS3.port": 443,
46+
"remoteS3.virtual-host-style": false,
47+
"remoteS3.hostname.template": "${domain}"
48+
}
49+
}
50+
}
51+
```
52+
53+
// ...existing content if any...

‎trino-aws-proxy/src/main/java/io/trino/aws/proxy/server/remote/provider/http/ForHttpRemoteS3ConnectionProvider.java

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Licensed under the Apache License, Version 2.0 (the "License");
3+
* you may not use this file except in compliance with the License.
4+
* You may obtain a copy of the License at
5+
*
6+
* http://www.apache.org/licenses/LICENSE-2.0
7+
*
8+
* Unless required by applicable law or agreed to in writing, software
9+
* distributed under the License is distributed on an "AS IS" BASIS,
10+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
* See the License for the specific language governing permissions and
12+
* limitations under the License.
13+
*/
14+
package io.trino.aws.proxy.server.remote.provider.http;
15+
16+
import com.google.inject.BindingAnnotation;
17+
18+
import java.lang.annotation.Retention;
19+
import java.lang.annotation.Target;
20+
21+
import static java.lang.annotation.ElementType.FIELD;
22+
import static java.lang.annotation.ElementType.METHOD;
23+
import static java.lang.annotation.ElementType.PARAMETER;
24+
import static java.lang.annotation.RetentionPolicy.RUNTIME;
25+
26+
@BindingAnnotation
27+
@Target({FIELD, PARAMETER, METHOD})
28+
@Retention(RUNTIME)
29+
public @interface ForHttpRemoteS3ConnectionProvider
30+
{
31+
}

0 commit comments

Comments
 (0)