From d375447bb17c9c5e0762f0e32b606b4c8fc1de68 Mon Sep 17 00:00:00 2001 From: Adnan Al <98621989+achdmbp@users.noreply.github.com> Date: Wed, 27 Aug 2025 22:12:37 +0000 Subject: [PATCH 1/2] feat: Add IRSA and session token support to AWS Storage Service This enhancement addresses the AWS credential limitations in OpenVSX by adding support for multiple authentication methods: 1. Static credentials with session token (temporary credentials) 2. Static credentials without session token (permanent credentials) 3. IRSA credentials (IAM Roles for Service Accounts) 4. Default credential provider chain (fallback) Key improvements: - Enables secure Kubernetes deployments using IRSA - Supports temporary credentials from AWS STS - Maintains backward compatibility with existing configurations - Follows AWS security best practices - Eliminates need for long-lived static credentials in containers The service automatically detects available credential types and uses appropriate AWS SDK credential providers based on configuration. Updated documentation includes examples for all authentication methods and deployment scenarios. Fixes: https://github.com/eclipse/openvsx/issues/1316 Signed-off-by: Adnan Al <98621989+achdmbp@users.noreply.github.com> --- README.md | 38 +- server/build.gradle | 21 +- .../openvsx/storage/AwsStorageService.java | 53 +- .../AwsStorageServiceIntegrationTest.java | 588 ++++++++++++++++++ .../storage/AwsStorageServiceTest.java | 256 ++++++++ 5 files changed, 946 insertions(+), 10 deletions(-) create mode 100644 server/src/test/java/org/eclipse/openvsx/storage/AwsStorageServiceIntegrationTest.java create mode 100644 server/src/test/java/org/eclipse/openvsx/storage/AwsStorageServiceTest.java diff --git a/README.md b/README.md index df3fbfe09..d4b4b9a8a 100644 --- a/README.md +++ b/README.md @@ -154,10 +154,46 @@ If you would like to test file storage via Amazon S3, follow these steps: } ] ``` + +#### Authentication Methods + +OpenVSX supports multiple AWS authentication methods with the following precedence: + +1. **Static credentials with session token** (temporary credentials) +2. **Static credentials without session token** (permanent credentials) +3. **IAM role-based credentials** (using AWS Web Identity Token authentication) +4. **Default credential provider chain** (fallback for other AWS credential sources) + +#### Option 1: Static Credentials (Traditional) + * Follow the steps for [programmatic access](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) to create your access key id and secret access key -* Configure the following environment variables on your server environment +* Configure the following environment variables: * `AWS_ACCESS_KEY_ID` with your access key id * `AWS_SECRET_ACCESS_KEY` with your secret access key + * `AWS_SESSION_TOKEN` with your session token (optional, for temporary credentials) + +#### Option 2: IAM Role with Web Identity Token (Recommended for containerized deployments) + +For deployments using IAM roles with web identity token authentication (such as IRSA in Kubernetes, ECS tasks with task roles, or other container orchestration platforms): + +* Create an IAM role with S3 permissions and appropriate trust policy +* Configure your deployment environment to provide the following environment variables: + * `AWS_ROLE_ARN` - The ARN of the IAM role to assume + * `AWS_WEB_IDENTITY_TOKEN_FILE` - Path to the web identity token file +* No static credentials needed! + +#### Option 3: Default Credential Provider Chain + +OpenVSX will automatically detect credentials from: +* Environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`) +* AWS credentials file (`~/.aws/credentials`) +* AWS config file (`~/.aws/config`) +* IAM instance profile (for EC2 instances) +* Container credentials (for ECS tasks) + +#### Common Configuration + +Regardless of authentication method, configure these environment variables: * `AWS_REGION` with your bucket region name * `AWS_SERVICE_ENDPOINT` with the url of your S3 provider if not using AWS (for AWS do not set) * `AWS_BUCKET` with your bucket name diff --git a/server/build.gradle b/server/build.gradle index 58cdeabf3..1b5c8704e 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -29,7 +29,7 @@ def versions = [ azure: '12.23.0', aws: '2.29.29', junit: '5.9.2', - testcontainers: '1.15.2', + testcontainers: '1.19.3', jackson: '2.15.2', woodstox: '6.4.0', jobrunr: '7.5.0', @@ -128,10 +128,12 @@ dependencies { } testImplementation "org.springframework.security:spring-security-test" testImplementation "org.testcontainers:elasticsearch:${versions.testcontainers}" + testImplementation "org.testcontainers:localstack:${versions.testcontainers}" + testImplementation "org.testcontainers:junit-jupiter:${versions.testcontainers}" testImplementation "org.junit.jupiter:junit-jupiter-api:${versions.junit}" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${versions.junit}" testRuntimeOnly "org.testcontainers:postgresql:${versions.testcontainers}" - + gatling "io.gatling:gatling-core:${versions.gatling}" gatling "io.gatling:gatling-app:${versions.gatling}" } @@ -198,8 +200,23 @@ task unitTests(type: Test) { exclude 'org/eclipse/openvsx/IntegrationTest.class' exclude 'org/eclipse/openvsx/cache/CacheServiceTest.class' exclude 'org/eclipse/openvsx/repositories/RepositoryServiceSmokeTest.class' + exclude 'org/eclipse/openvsx/storage/AwsStorageServiceIntegrationTest.class' } +task s3IntegrationTests(type: Test) { + description = 'Runs S3 integration tests using LocalStack (requires Docker/Podman).' + group = 'verification' + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + useJUnitPlatform() + include 'org/eclipse/openvsx/storage/AwsStorageServiceIntegrationTest.class' + + // Set system properties for test configuration + systemProperty 'spring.profiles.active', 's3-integration' +} + + + jacocoTestReport { reports { xml.required = true diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java index e1aaf541e..7ef52b22c 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java @@ -23,6 +23,9 @@ import org.springframework.data.util.Pair; import org.springframework.stereotype.Component; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.awscore.defaultsmode.DefaultsMode; import software.amazon.awssdk.regions.Region; @@ -60,6 +63,9 @@ public class AwsStorageService implements IStorageService { @Value("${ovsx.storage.aws.secret-access-key:}") String secretAccessKey; + @Value("${ovsx.storage.aws.session-token:}") + String sessionToken; + @Value("${ovsx.storage.aws.region:}") String region; @@ -81,12 +87,11 @@ public AwsStorageService(FileCacheDurationConfig fileCacheDurationConfig, FilesC protected S3Client getS3Client() { if (s3Client == null) { - var credentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey); var s3ClientBuilder = S3Client.builder() .defaultsMode(DefaultsMode.STANDARD) .forcePathStyle(pathStyleAccess) - .credentialsProvider(StaticCredentialsProvider.create(credentials)) - .region(Region.of(region)); + .region(Region.of(region)) + .credentialsProvider(getCredentialsProvider()); if(StringUtils.isNotEmpty(serviceEndpoint)) { var endpointParams = S3EndpointParams.builder() @@ -107,10 +112,9 @@ protected S3Client getS3Client() { } private S3Presigner getS3Presigner() { - var credentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey); var builder = S3Presigner.builder() - .credentialsProvider(StaticCredentialsProvider.create(credentials)) - .region(Region.of(region)); + .region(Region.of(region)) + .credentialsProvider(getCredentialsProvider()); if(StringUtils.isNotEmpty(serviceEndpoint)) { var endpointParams = S3EndpointParams.builder() @@ -128,9 +132,44 @@ private S3Presigner getS3Presigner() { return builder.build(); } + private AwsCredentialsProvider getCredentialsProvider() { + // Use static credentials if provided, otherwise DefaultCredentialsProvider handles everything + if (hasStaticCredentials()) { + var credentials = hasSessionToken() + ? AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken) + : AwsBasicCredentials.create(accessKeyId, secretAccessKey); + return StaticCredentialsProvider.create(credentials); + } + return DefaultCredentialsProvider.create(); + } + + + private boolean hasStaticCredentials() { + return !StringUtils.isEmpty(accessKeyId) && !StringUtils.isEmpty(secretAccessKey); + } + + private boolean hasSessionToken() { + return !StringUtils.isEmpty(sessionToken); + } @Override public boolean isEnabled() { - return !StringUtils.isEmpty(accessKeyId); + // Require region and bucket to be configured + if (StringUtils.isEmpty(region) || StringUtils.isEmpty(bucket)) { + return false; + } + + // If any credential fields are provided, validate them properly + boolean hasAccessKey = !StringUtils.isEmpty(accessKeyId); + boolean hasSecretKey = !StringUtils.isEmpty(secretAccessKey); + boolean hasSessionToken = !StringUtils.isEmpty(sessionToken); + + if (hasAccessKey || hasSecretKey || hasSessionToken) { + // If any credential is provided, both access key and secret key must be present + return hasAccessKey && hasSecretKey; + } + + // No static credentials provided - allow AWS default credential provider chain + return true; } @Override diff --git a/server/src/test/java/org/eclipse/openvsx/storage/AwsStorageServiceIntegrationTest.java b/server/src/test/java/org/eclipse/openvsx/storage/AwsStorageServiceIntegrationTest.java new file mode 100644 index 000000000..68e48b3dc --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/storage/AwsStorageServiceIntegrationTest.java @@ -0,0 +1,588 @@ +/** ****************************************************************************** + * Copyright (c) 2025 Adnan Al and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.storage; + +import org.eclipse.openvsx.cache.FilesCacheKeyGenerator; +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.entities.ExtensionVersion; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.entities.Namespace; +import org.eclipse.openvsx.util.TempFile; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.util.ReflectionTestUtils; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.time.Duration; +import java.util.List; + +import org.springframework.data.util.Pair; + +import static org.junit.jupiter.api.Assertions.*; + +@Testcontainers +@ExtendWith(SpringExtension.class) +@ContextConfiguration(classes = AwsStorageServiceIntegrationTest.TestConfig.class) +class AwsStorageServiceIntegrationTest { + + private static final String TEST_BUCKET = "openvsx-test-bucket"; + private static final String TEST_REGION = "us-east-1"; + + @Container + static LocalStackContainer localstack = new LocalStackContainer(DockerImageName.parse("localstack/localstack:4.7")) + .withServices(LocalStackContainer.Service.S3, LocalStackContainer.Service.IAM) + .withReuse(true); + + @Autowired + private AwsStorageService awsStorageService; + + private AwsStorageService storageService; + private S3Client testS3Client; + + private Namespace namespace; + private Extension extension; + private ExtensionVersion extVersion; + private FileResource resource; + + @DynamicPropertySource + static void configureProperties(DynamicPropertyRegistry registry) { + registry.add("ovsx.storage.aws.service-endpoint", () -> localstack.getEndpointOverride(LocalStackContainer.Service.S3).toString()); + registry.add("ovsx.storage.aws.access-key-id", () -> "test"); + registry.add("ovsx.storage.aws.secret-access-key", () -> "test"); + registry.add("ovsx.storage.aws.region", () -> TEST_REGION); + registry.add("ovsx.storage.aws.bucket", () -> TEST_BUCKET); + registry.add("ovsx.storage.aws.path-style-access", () -> "true"); + } + + @BeforeEach + void setUp() { + System.setProperty("aws.accessKeyId", "test"); + System.setProperty("aws.secretAccessKey", "test"); + System.setProperty("aws.region", TEST_REGION); + + storageService = awsStorageService; + + ReflectionTestUtils.setField(storageService, "region", TEST_REGION); + ReflectionTestUtils.setField(storageService, "bucket", TEST_BUCKET); + ReflectionTestUtils.setField(storageService, "serviceEndpoint", localstack.getEndpointOverride(LocalStackContainer.Service.S3).toString()); + ReflectionTestUtils.setField(storageService, "pathStyleAccess", true); + + testS3Client = S3Client.builder() + .endpointOverride(localstack.getEndpointOverride(LocalStackContainer.Service.S3)) + .credentialsProvider(StaticCredentialsProvider.create( + AwsBasicCredentials.create("test", "test"))) + .region(Region.of(TEST_REGION)) + .forcePathStyle(true) + .build(); + + testS3Client.createBucket(CreateBucketRequest.builder() + .bucket(TEST_BUCKET) + .build()); + + setupTestEntities(); + } + + @AfterEach + void tearDown() { + System.clearProperty("aws.accessKeyId"); + System.clearProperty("aws.secretAccessKey"); + System.clearProperty("aws.region"); + } + + private void setupTestEntities() { + namespace = new Namespace(); + namespace.setName("testnamespace"); + namespace.setLogoName("logo.png"); + + extension = new Extension(); + extension.setName("test-extension"); + extension.setNamespace(namespace); + + extVersion = new ExtensionVersion(); + extVersion.setVersion("1.0.0"); + extVersion.setTargetPlatform("universal"); + extVersion.setExtension(extension); + + resource = new FileResource(); + resource.setName("extension.vsix"); + resource.setExtension(extVersion); + } + + @Test + void testServiceIsEnabled() { + assertTrue(storageService.isEnabled()); + } + + @Test + void testUploadAndDownloadFile() throws IOException { + var tempFile = new TempFile("test_", ".vsix"); + var testContent = "This is test extension content"; + Files.write(tempFile.getPath(), testContent.getBytes(), StandardOpenOption.CREATE); + tempFile.setResource(resource); + + assertDoesNotThrow(() -> storageService.uploadFile(tempFile)); + + var objectKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", resource); + assertTrue(objectExists(objectKey)); + + var downloadedFile = storageService.downloadFile(resource); + assertNotNull(downloadedFile); + + var downloadedContent = Files.readString(downloadedFile.getPath()); + assertEquals(testContent, downloadedContent); + + tempFile.close(); + downloadedFile.close(); + } + + @Test + void testUploadAndDownloadNamespaceLogo() throws IOException { + var logoFile = new TempFile("logo_", ".png"); + var logoContent = "fake-png-content"; + Files.write(logoFile.getPath(), logoContent.getBytes(), StandardOpenOption.CREATE); + logoFile.setNamespace(namespace); + + assertDoesNotThrow(() -> storageService.uploadNamespaceLogo(logoFile)); + + var objectKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", namespace); + assertTrue(objectExists(objectKey)); + + var location = storageService.getNamespaceLogoLocation(namespace); + assertNotNull(location); + assertTrue(location.toString().contains(TEST_BUCKET)); + + logoFile.close(); + } + + @Test + void testRemoveFile() throws IOException { + var tempFile = new TempFile("test_", ".vsix"); + Files.write(tempFile.getPath(), "test content".getBytes(), StandardOpenOption.CREATE); + tempFile.setResource(resource); + storageService.uploadFile(tempFile); + + var objectKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", resource); + assertTrue(objectExists(objectKey)); + + assertDoesNotThrow(() -> storageService.removeFile(resource)); + assertFalse(objectExists(objectKey)); + + tempFile.close(); + } + + @Test + void testRemoveNamespaceLogo() throws IOException { + var logoFile = new TempFile("logo_", ".png"); + Files.write(logoFile.getPath(), "logo content".getBytes(), StandardOpenOption.CREATE); + logoFile.setNamespace(namespace); + storageService.uploadNamespaceLogo(logoFile); + + var objectKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", namespace); + assertTrue(objectExists(objectKey)); + + assertDoesNotThrow(() -> storageService.removeNamespaceLogo(namespace)); + assertFalse(objectExists(objectKey)); + + logoFile.close(); + } + + @Test + void testGetPresignedUrl() throws IOException { + var tempFile = new TempFile("test_", ".vsix"); + Files.write(tempFile.getPath(), "test content".getBytes(), StandardOpenOption.CREATE); + tempFile.setResource(resource); + storageService.uploadFile(tempFile); + + var location = storageService.getLocation(resource); + assertNotNull(location); + + var locationStr = location.toString(); + assertTrue(locationStr.contains(TEST_BUCKET)); + assertTrue(locationStr.contains("testnamespace/test-extension/1.0.0/extension.vsix")); + assertTrue(locationStr.contains("X-Amz-Algorithm")); + + tempFile.close(); + } + + @Test + void testObjectKeyGeneration() { + var objectKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", resource); + assertEquals("testnamespace/test-extension/1.0.0/extension.vsix", objectKey); + + extVersion.setTargetPlatform("linux-x64"); + objectKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", resource); + assertEquals("testnamespace/test-extension/linux-x64/1.0.0/extension.vsix", objectKey); + + var logoKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", namespace); + assertEquals("testnamespace/logo/logo.png", logoKey); + } + + @Test + void testCopyFiles() throws IOException { + var tempFile = new TempFile("test_", ".vsix"); + Files.write(tempFile.getPath(), "test content".getBytes(), StandardOpenOption.CREATE); + tempFile.setResource(resource); + storageService.uploadFile(tempFile); + + var targetExtVersion = new ExtensionVersion(); + targetExtVersion.setVersion("2.0.0"); + targetExtVersion.setTargetPlatform("universal"); + targetExtVersion.setExtension(extension); + + var targetResource = new FileResource(); + targetResource.setName("extension.vsix"); + targetResource.setExtension(targetExtVersion); + + var pairs = List.of(Pair.of(resource, targetResource)); + assertDoesNotThrow(() -> storageService.copyFiles(pairs)); + + var sourceKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", resource); + var targetKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", targetResource); + + assertTrue(objectExists(sourceKey)); + assertTrue(objectExists(targetKey)); + + tempFile.close(); + } + + @Test + void testCopyNamespaceLogo() throws IOException { + var logoFile = new TempFile("logo_", ".png"); + Files.write(logoFile.getPath(), "logo content".getBytes(), StandardOpenOption.CREATE); + logoFile.setNamespace(namespace); + storageService.uploadNamespaceLogo(logoFile); + + var targetNamespace = new Namespace(); + targetNamespace.setName("targetnamespace"); + targetNamespace.setLogoName("logo.png"); + + assertDoesNotThrow(() -> storageService.copyNamespaceLogo(namespace, targetNamespace)); + + var sourceKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", namespace); + var targetKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", targetNamespace); + + assertTrue(objectExists(sourceKey)); + assertTrue(objectExists(targetKey)); + + logoFile.close(); + } + + @Test + void testListObjects() throws IOException { + var initialListRequest = ListObjectsV2Request.builder() + .bucket(TEST_BUCKET) + .prefix("testnamespace/test-extension/") + .build(); + var initialResponse = testS3Client.listObjectsV2(initialListRequest); + int initialCount = initialResponse.contents().size(); + + for (int i = 1; i <= 3; i++) { + var tempFile = new TempFile("test_" + i + "_", ".vsix"); + Files.write(tempFile.getPath(), ("content " + i).getBytes(), StandardOpenOption.CREATE); + + var testResource = new FileResource(); + testResource.setName("extension" + i + ".vsix"); + testResource.setExtension(extVersion); + tempFile.setResource(testResource); + + storageService.uploadFile(tempFile); + tempFile.close(); + } + + var listRequest = ListObjectsV2Request.builder() + .bucket(TEST_BUCKET) + .prefix("testnamespace/test-extension/") + .build(); + + var response = testS3Client.listObjectsV2(listRequest); + assertEquals(initialCount + 3, response.contents().size()); + } + + @Test + void testErrorHandling() { + var nonExistentResource = new FileResource(); + nonExistentResource.setName("nonexistent.vsix"); + nonExistentResource.setExtension(extVersion); + + assertThrows(Exception.class, () -> storageService.downloadFile(nonExistentResource)); + } + + @Test + void testStaticCredentialsAuthentication() { + assertTrue(storageService.isEnabled()); + + assertDoesNotThrow(() -> { + var s3Client = ReflectionTestUtils.invokeMethod(storageService, "getS3Client"); + assertNotNull(s3Client); + }); + } + + @Test + void testSessionTokenAuthentication() throws IOException { + var sessionTokenService = createStorageServiceWithCredentials("ASIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "AQoDYXdzEJr..."); + + assertTrue(sessionTokenService.isEnabled()); + + Boolean hasStaticCreds = ReflectionTestUtils.invokeMethod(sessionTokenService, "hasStaticCredentials"); + Boolean hasSessionToken = ReflectionTestUtils.invokeMethod(sessionTokenService, "hasSessionToken"); + assertTrue(hasStaticCreds); + assertTrue(hasSessionToken); + + testBasicFileOperation(sessionTokenService, "session-token"); + } + + @Test + void testDefaultCredentialProviderAuthentication() throws IOException { + var defaultService = createStorageServiceWithCredentials("", "", ""); + + Boolean hasStaticCreds = ReflectionTestUtils.invokeMethod(defaultService, "hasStaticCredentials"); + Boolean hasSessionToken = ReflectionTestUtils.invokeMethod(defaultService, "hasSessionToken"); + assertFalse(hasStaticCreds); + assertFalse(hasSessionToken); + + assertDoesNotThrow(() -> { + var s3Client = ReflectionTestUtils.invokeMethod(defaultService, "getS3Client"); + assertNotNull(s3Client); + }); + + testBasicFileOperation(defaultService, "default-provider"); + } + + @Test + void testAuthenticationMethodDetection() { + var permanentCredsService = createStorageServiceWithCredentials("AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", null); + assertTrue((Boolean) ReflectionTestUtils.invokeMethod(permanentCredsService, "hasStaticCredentials")); + assertFalse((Boolean) ReflectionTestUtils.invokeMethod(permanentCredsService, "hasSessionToken")); + + var temporaryCredsService = createStorageServiceWithCredentials("ASIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "AQoDYXdzEJr..."); + assertTrue((Boolean) ReflectionTestUtils.invokeMethod(temporaryCredsService, "hasStaticCredentials")); + assertTrue((Boolean) ReflectionTestUtils.invokeMethod(temporaryCredsService, "hasSessionToken")); + + var defaultProviderService = createStorageServiceWithCredentials("", "", ""); + assertFalse((Boolean) ReflectionTestUtils.invokeMethod(defaultProviderService, "hasStaticCredentials")); + assertFalse((Boolean) ReflectionTestUtils.invokeMethod(defaultProviderService, "hasSessionToken")); + } + + @Test + void testCredentialPriorityOrder() { + var storageService = createStorageServiceWithCredentials("AKIAIOSFODNN7EXAMPLE", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", null); + + Boolean hasStaticCreds = ReflectionTestUtils.invokeMethod(storageService, "hasStaticCredentials"); + assertTrue(hasStaticCreds); + + ReflectionTestUtils.setField(storageService, "sessionToken", "test-session-token"); + + Boolean hasSessionToken = ReflectionTestUtils.invokeMethod(storageService, "hasSessionToken"); + assertTrue(hasSessionToken); + assertTrue(hasStaticCreds); + } + + @Test + void testInvalidCredentials() { + var storageService = createStorageServiceWithCredentials("", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", null); + assertFalse(storageService.isEnabled()); + + storageService = createStorageServiceWithCredentials("AKIAIOSFODNN7EXAMPLE", "", null); + assertFalse(storageService.isEnabled()); + } + + @Test + void testS3ClientCreationWithDifferentCredentials() { + final var storageService1 = createStorageServiceWithCredentials("test-access-key", "test-secret-key", null); + + assertDoesNotThrow(() -> { + var s3Client = ReflectionTestUtils.invokeMethod(storageService1, "getS3Client"); + assertNotNull(s3Client); + }); + + final var storageService2 = createStorageServiceWithCredentials("test-access-key", "test-secret-key", "test-session-token"); + + assertDoesNotThrow(() -> { + var s3Client = ReflectionTestUtils.invokeMethod(storageService2, "getS3Client"); + assertNotNull(s3Client); + }); + } + + @Test + void testPresignerCreationWithDifferentCredentials() { + final var storageService1 = createStorageServiceWithCredentials("test-access-key", "test-secret-key", null); + + assertDoesNotThrow(() -> { + var presigner = ReflectionTestUtils.invokeMethod(storageService1, "getS3Presigner"); + assertNotNull(presigner); + }); + + final var storageService2 = createStorageServiceWithCredentials("test-access-key", "test-secret-key", "test-session-token"); + + assertDoesNotThrow(() -> { + var presigner = ReflectionTestUtils.invokeMethod(storageService2, "getS3Presigner"); + assertNotNull(presigner); + }); + + final var storageService3 = createStorageServiceWithCredentials("", "", ""); + + assertDoesNotThrow(() -> { + var presigner = ReflectionTestUtils.invokeMethod(storageService3, "getS3Presigner"); + assertNotNull(presigner); + }); + } + + @Test + void testOperationsFailWithRevokedPermissions() throws IOException { + var restrictedService = awsStorageService; + + ReflectionTestUtils.setField(restrictedService, "region", TEST_REGION); + ReflectionTestUtils.setField(restrictedService, "bucket", "non-existent-bucket"); + ReflectionTestUtils.setField(restrictedService, "serviceEndpoint", "http://localhost:99999"); + ReflectionTestUtils.setField(restrictedService, "pathStyleAccess", true); + + ReflectionTestUtils.setField(restrictedService, "s3Client", null); + + var tempFile = new TempFile("test_", ".vsix"); + Files.write(tempFile.getPath(), "test content".getBytes(), StandardOpenOption.CREATE); + tempFile.setResource(resource); + + assertThrows(Exception.class, () -> restrictedService.uploadFile(tempFile)); + assertThrows(Exception.class, () -> restrictedService.downloadFile(resource)); + assertThrows(Exception.class, () -> restrictedService.removeFile(resource)); + + tempFile.close(); + } + + @Test + void testValidAuthenticationButInsufficientBucketPermissions() throws IOException { + var restrictedBucketService = awsStorageService; + + ReflectionTestUtils.setField(restrictedBucketService, "region", TEST_REGION); + ReflectionTestUtils.setField(restrictedBucketService, "bucket", "unauthorized-bucket-no-access"); + ReflectionTestUtils.setField(restrictedBucketService, "serviceEndpoint", localstack.getEndpointOverride(LocalStackContainer.Service.S3).toString()); + ReflectionTestUtils.setField(restrictedBucketService, "pathStyleAccess", true); + + ReflectionTestUtils.setField(restrictedBucketService, "s3Client", null); + + var tempFile = new TempFile("test_", ".vsix"); + Files.write(tempFile.getPath(), "test content".getBytes(), StandardOpenOption.CREATE); + tempFile.setResource(resource); + + assertThrows(Exception.class, () -> restrictedBucketService.uploadFile(tempFile)); + assertThrows(Exception.class, () -> restrictedBucketService.downloadFile(resource)); + assertThrows(Exception.class, () -> restrictedBucketService.removeFile(resource)); + + var logoFile = new TempFile("logo_", ".png"); + Files.write(logoFile.getPath(), "logo content".getBytes(), StandardOpenOption.CREATE); + logoFile.setNamespace(namespace); + + assertThrows(Exception.class, () -> restrictedBucketService.uploadNamespaceLogo(logoFile)); + assertThrows(Exception.class, () -> restrictedBucketService.removeNamespaceLogo(namespace)); + + tempFile.close(); + logoFile.close(); + } + + private AwsStorageService createStorageServiceWithCredentials(String accessKeyId, String secretAccessKey, String sessionToken) { + ReflectionTestUtils.setField(awsStorageService, "accessKeyId", accessKeyId); + ReflectionTestUtils.setField(awsStorageService, "secretAccessKey", secretAccessKey); + ReflectionTestUtils.setField(awsStorageService, "sessionToken", sessionToken); + ReflectionTestUtils.setField(awsStorageService, "region", TEST_REGION); + ReflectionTestUtils.setField(awsStorageService, "bucket", TEST_BUCKET); + ReflectionTestUtils.setField(awsStorageService, "serviceEndpoint", localstack.getEndpointOverride(LocalStackContainer.Service.S3).toString()); + ReflectionTestUtils.setField(awsStorageService, "pathStyleAccess", true); + + return awsStorageService; + } + + private void testBasicFileOperation(AwsStorageService storageService, String testPrefix) throws IOException { + var tempFile = new TempFile(testPrefix + "_", ".vsix"); + var testContent = "Authentication test content for " + testPrefix; + Files.write(tempFile.getPath(), testContent.getBytes(), StandardOpenOption.CREATE); + + var testResource = new FileResource(); + testResource.setName(testPrefix + "-extension.vsix"); + testResource.setExtension(extVersion); + tempFile.setResource(testResource); + + assertDoesNotThrow(() -> storageService.uploadFile(tempFile)); + + assertDoesNotThrow(() -> { + var downloadedFile = storageService.downloadFile(testResource); + assertNotNull(downloadedFile); + var downloadedContent = Files.readString(downloadedFile.getPath()); + assertEquals(testContent, downloadedContent); + downloadedFile.close(); + }); + + assertDoesNotThrow(() -> storageService.removeFile(testResource)); + + tempFile.close(); + } + + private boolean objectExists(String objectKey) { + try { + testS3Client.headObject(HeadObjectRequest.builder() + .bucket(TEST_BUCKET) + .key(objectKey) + .build()); + return true; + } catch (NoSuchKeyException e) { + return false; + } + } + + @TestConfiguration + static class TestConfig { + + @Bean + public FileCacheDurationConfig fileCacheDurationConfig() { + var config = new FileCacheDurationConfig(); + ReflectionTestUtils.setField(config, "cacheDuration", Duration.ofDays(7)); + return config; + } + + @Bean + public FilesCacheKeyGenerator filesCacheKeyGenerator() { + return new FilesCacheKeyGenerator(); + } + + @Bean + public AwsStorageService awsStorageService(FileCacheDurationConfig fileCacheDurationConfig, + FilesCacheKeyGenerator filesCacheKeyGenerator) { + return new AwsStorageService(fileCacheDurationConfig, filesCacheKeyGenerator); + } + } +} diff --git a/server/src/test/java/org/eclipse/openvsx/storage/AwsStorageServiceTest.java b/server/src/test/java/org/eclipse/openvsx/storage/AwsStorageServiceTest.java new file mode 100644 index 000000000..966663550 --- /dev/null +++ b/server/src/test/java/org/eclipse/openvsx/storage/AwsStorageServiceTest.java @@ -0,0 +1,256 @@ +/** ****************************************************************************** + * Copyright (c) 2025 Adnan Al and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + * ****************************************************************************** */ +package org.eclipse.openvsx.storage; + +import org.eclipse.openvsx.cache.FilesCacheKeyGenerator; +import org.eclipse.openvsx.entities.Extension; +import org.eclipse.openvsx.entities.ExtensionVersion; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.entities.Namespace; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit integration tests for AWS Storage Service that don't require Docker/LocalStack. + * These tests focus on configuration, authentication logic, and object key generation. + */ +class AwsStorageServiceTest { + + private AwsStorageService storageService; + private Namespace namespace; + private Extension extension; + private ExtensionVersion extVersion; + private FileResource resource; + + @BeforeEach + void setUp() { + var fileCacheDurationConfig = new FileCacheDurationConfig(); + var filesCacheKeyGenerator = new FilesCacheKeyGenerator(); + storageService = new AwsStorageService(fileCacheDurationConfig, filesCacheKeyGenerator); + + // Set up test entities + setupTestEntities(); + } + + private void setupTestEntities() { + namespace = new Namespace(); + namespace.setName("testnamespace"); + namespace.setLogoName("logo.png"); + + extension = new Extension(); + extension.setName("test-extension"); + extension.setNamespace(namespace); + + extVersion = new ExtensionVersion(); + extVersion.setVersion("1.0.0"); + extVersion.setTargetPlatform("universal"); + extVersion.setExtension(extension); + + resource = new FileResource(); + resource.setName("extension.vsix"); + resource.setExtension(extVersion); + } + + @Test + void testServiceNotEnabledWithoutCredentials() { + // Test with no credentials + ReflectionTestUtils.setField(storageService, "accessKeyId", ""); + ReflectionTestUtils.setField(storageService, "secretAccessKey", ""); + + assertFalse(storageService.isEnabled(), "Service should not be enabled without credentials"); + } + + @Test + void testServiceEnabledWithStaticCredentials() { + // Test with static credentials + ReflectionTestUtils.setField(storageService, "accessKeyId", "AKIAIOSFODNN7EXAMPLE"); + ReflectionTestUtils.setField(storageService, "secretAccessKey", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + ReflectionTestUtils.setField(storageService, "region", "us-east-1"); + ReflectionTestUtils.setField(storageService, "bucket", "test-bucket"); + + assertTrue(storageService.isEnabled(), "Service should be enabled with static credentials"); + } + + @Test + void testHasStaticCredentials() { + // Test with both access key and secret key + ReflectionTestUtils.setField(storageService, "accessKeyId", "AKIAIOSFODNN7EXAMPLE"); + ReflectionTestUtils.setField(storageService, "secretAccessKey", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + + Boolean hasCredentials = ReflectionTestUtils.invokeMethod(storageService, "hasStaticCredentials"); + assertTrue(hasCredentials, "Should detect static credentials when both access key and secret key are present"); + + // Test with missing access key + ReflectionTestUtils.setField(storageService, "accessKeyId", ""); + hasCredentials = ReflectionTestUtils.invokeMethod(storageService, "hasStaticCredentials"); + assertFalse(hasCredentials, "Should not detect static credentials when access key is missing"); + + // Test with missing secret key + ReflectionTestUtils.setField(storageService, "accessKeyId", "AKIAIOSFODNN7EXAMPLE"); + ReflectionTestUtils.setField(storageService, "secretAccessKey", ""); + hasCredentials = ReflectionTestUtils.invokeMethod(storageService, "hasStaticCredentials"); + assertFalse(hasCredentials, "Should not detect static credentials when secret key is missing"); + } + + @Test + void testHasSessionToken() { + // Test with session token + ReflectionTestUtils.setField(storageService, "sessionToken", "AQoDYXdzEJr..."); + + Boolean hasToken = ReflectionTestUtils.invokeMethod(storageService, "hasSessionToken"); + assertTrue(hasToken, "Should detect session token when present"); + + // Test without session token + ReflectionTestUtils.setField(storageService, "sessionToken", ""); + hasToken = ReflectionTestUtils.invokeMethod(storageService, "hasSessionToken"); + assertFalse(hasToken, "Should not detect session token when empty"); + + // Test with null session token + ReflectionTestUtils.setField(storageService, "sessionToken", null); + hasToken = ReflectionTestUtils.invokeMethod(storageService, "hasSessionToken"); + assertFalse(hasToken, "Should not detect session token when null"); + } + + @Test + void testCredentialPriorityLogic() { + // Test 1: Static credentials without session token (permanent credentials) + ReflectionTestUtils.setField(storageService, "accessKeyId", "AKIAIOSFODNN7EXAMPLE"); + ReflectionTestUtils.setField(storageService, "secretAccessKey", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + ReflectionTestUtils.setField(storageService, "sessionToken", ""); + + Boolean hasStaticCreds1 = ReflectionTestUtils.invokeMethod(storageService, "hasStaticCredentials"); + Boolean hasSessionToken1 = ReflectionTestUtils.invokeMethod(storageService, "hasSessionToken"); + assertTrue(hasStaticCreds1); + assertFalse(hasSessionToken1); + + // Test 2: Static credentials with session token (temporary credentials) + ReflectionTestUtils.setField(storageService, "sessionToken", "AQoDYXdzEJr..."); + + Boolean hasStaticCreds2 = ReflectionTestUtils.invokeMethod(storageService, "hasStaticCredentials"); + Boolean hasSessionToken2 = ReflectionTestUtils.invokeMethod(storageService, "hasSessionToken"); + assertTrue(hasStaticCreds2); + assertTrue(hasSessionToken2); + + // Test 3: No static credentials (should fall back to web identity or default provider) + ReflectionTestUtils.setField(storageService, "accessKeyId", ""); + ReflectionTestUtils.setField(storageService, "secretAccessKey", ""); + ReflectionTestUtils.setField(storageService, "sessionToken", ""); + + Boolean hasStaticCreds3 = ReflectionTestUtils.invokeMethod(storageService, "hasStaticCredentials"); + Boolean hasSessionToken3 = ReflectionTestUtils.invokeMethod(storageService, "hasSessionToken"); + assertFalse(hasStaticCreds3); + assertFalse(hasSessionToken3); + } + + @Test + void testObjectKeyGeneration() { + // Test regular file object key + var objectKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", resource); + assertEquals("testnamespace/test-extension/1.0.0/extension.vsix", objectKey); + + // Test with non-universal target platform + extVersion.setTargetPlatform("linux-x64"); + objectKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", resource); + assertEquals("testnamespace/test-extension/linux-x64/1.0.0/extension.vsix", objectKey); + + // Test namespace logo object key + var logoKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", namespace); + assertEquals("testnamespace/logo/logo.png", logoKey); + } + + @Test + void testObjectKeyGenerationWithComplexNames() { + // Test with complex namespace and extension names + namespace.setName("my-complex-namespace"); + extension.setName("my.complex.extension-name"); + extVersion.setVersion("2.1.0-beta.1"); + resource.setName("complex/path/extension.vsix"); + + var objectKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", resource); + assertEquals("my-complex-namespace/my.complex.extension-name/2.1.0-beta.1/complex/path/extension.vsix", objectKey); + } + + @Test + void testObjectKeyGenerationWithDifferentTargetPlatforms() { + String[] platforms = {"universal", "win32-x64", "linux-x64", "darwin-x64", "darwin-arm64"}; + + for (String platform : platforms) { + extVersion.setTargetPlatform(platform); + var objectKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", resource); + + if ("universal".equals(platform)) { + assertEquals("testnamespace/test-extension/1.0.0/extension.vsix", objectKey); + } else { + assertEquals("testnamespace/test-extension/" + platform + "/1.0.0/extension.vsix", objectKey); + } + } + } + + @Test + void testNamespaceLogoObjectKeyGeneration() { + // Test different logo file types + String[] logoNames = {"logo.png", "logo.jpg", "logo.svg", "namespace-logo.webp"}; + + for (String logoName : logoNames) { + namespace.setLogoName(logoName); + var logoKey = (String) ReflectionTestUtils.invokeMethod(storageService, "getObjectKey", namespace); + assertEquals("testnamespace/logo/" + logoName, logoKey); + } + } + + @Test + void testConfigurationProperties() { + // Test that configuration properties are properly set + ReflectionTestUtils.setField(storageService, "accessKeyId", "test-access-key"); + ReflectionTestUtils.setField(storageService, "secretAccessKey", "test-secret-key"); + ReflectionTestUtils.setField(storageService, "sessionToken", "test-session-token"); + ReflectionTestUtils.setField(storageService, "region", "us-west-2"); + ReflectionTestUtils.setField(storageService, "bucket", "my-test-bucket"); + ReflectionTestUtils.setField(storageService, "serviceEndpoint", "https://s3.us-west-2.amazonaws.com"); + ReflectionTestUtils.setField(storageService, "pathStyleAccess", true); + + // Verify properties are set correctly + assertEquals("test-access-key", ReflectionTestUtils.getField(storageService, "accessKeyId")); + assertEquals("test-secret-key", ReflectionTestUtils.getField(storageService, "secretAccessKey")); + assertEquals("test-session-token", ReflectionTestUtils.getField(storageService, "sessionToken")); + assertEquals("us-west-2", ReflectionTestUtils.getField(storageService, "region")); + assertEquals("my-test-bucket", ReflectionTestUtils.getField(storageService, "bucket")); + assertEquals("https://s3.us-west-2.amazonaws.com", ReflectionTestUtils.getField(storageService, "serviceEndpoint")); + assertTrue((Boolean) ReflectionTestUtils.getField(storageService, "pathStyleAccess")); + } + + @Test + void testAuthenticationMethodDetection() { + // Test static credentials detection + ReflectionTestUtils.setField(storageService, "accessKeyId", "AKIAIOSFODNN7EXAMPLE"); + ReflectionTestUtils.setField(storageService, "secretAccessKey", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"); + ReflectionTestUtils.setField(storageService, "sessionToken", ""); + + assertTrue((Boolean) ReflectionTestUtils.invokeMethod(storageService, "hasStaticCredentials")); + assertFalse((Boolean) ReflectionTestUtils.invokeMethod(storageService, "hasSessionToken")); + + // Test session token detection + ReflectionTestUtils.setField(storageService, "sessionToken", "test-session-token"); + + assertTrue((Boolean) ReflectionTestUtils.invokeMethod(storageService, "hasStaticCredentials")); + assertTrue((Boolean) ReflectionTestUtils.invokeMethod(storageService, "hasSessionToken")); + + // Test no credentials + ReflectionTestUtils.setField(storageService, "accessKeyId", ""); + ReflectionTestUtils.setField(storageService, "secretAccessKey", ""); + ReflectionTestUtils.setField(storageService, "sessionToken", ""); + + assertFalse((Boolean) ReflectionTestUtils.invokeMethod(storageService, "hasStaticCredentials")); + assertFalse((Boolean) ReflectionTestUtils.invokeMethod(storageService, "hasSessionToken")); + } +} From a85c5011576cac7d8c4b6893d5bbd3826689fe3f Mon Sep 17 00:00:00 2001 From: Adnan Al <98621989+achdmbp@users.noreply.github.com> Date: Sun, 12 Oct 2025 03:29:58 +0000 Subject: [PATCH 2/2] Add AWS STS SDK dependency for IRSA support - Enables Web Identity Token authentication for S3 storage - Fixes 'sts service module must be on the class path' error - Required for IAM Roles for Service Accounts (IRSA) integration --- server/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/server/build.gradle b/server/build.gradle index 1b5c8704e..723796e5e 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -101,6 +101,7 @@ dependencies { implementation "com.google.cloud:google-cloud-storage:${versions.gcloud}" implementation "com.azure:azure-storage-blob:${versions.azure}" implementation "software.amazon.awssdk:s3:${versions.aws}" + implementation "software.amazon.awssdk:sts:${versions.aws}" implementation "org.springdoc:springdoc-openapi-starter-webmvc-ui:${versions.springdoc}" implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"