Lcra/change workspace files storage to seaweed prod 14290

api/build.gradle.kts

@@ -11,10 +11,6 @@ plugins {
   id("org.jetbrains.kotlinx.kover")
 }
 
-val testNgVersion = "7.8.0"
-val testContainersRedisVersion = "1.6.4"
-val testContainersPostgreSQLVersion = "1.19.7"
-
 dependencies {
   implementation(projects.cosmotechMetaApi)
   implementation(projects.cosmotechConnectorApi)

api/kubernetes/helm-chart/templates/_helpers.tpl

@@ -108,13 +108,6 @@ E.g:
 {{- end }}
 {{- end }}
 
-{{/*
-Location of the persistence data
-*/}}
-{{- define "cosmotech-api.blobPersistencePath" -}}
-"/var/lib/cosmotech-api/data"
-{{- end }}
-
 {{- define "cosmotech-api.custom-rootca-path" -}}
 /mnt/cosmotech/certificates
 {{- end }}
@@ -197,8 +190,6 @@ csm:
       {{- else }}
       image-pull-secrets: []
       {{- end }}
-    blobPersistence:
-      path: {{ include "cosmotech-api.blobPersistencePath" . }}
     identityProvider:
       tls:
         bundle: {{ include "cosmotech-api.custom-rootca-bundle" . }}

api/kubernetes/helm-chart/templates/deployment.yaml

@@ -46,11 +46,6 @@ spec:
         - name: helm-config
           secret:
             secretName: {{ include "cosmotech-api.fullname" . }}
-        {{if .Values.persistence.enabled}}
-        - name: blob-storage
-          persistentVolumeClaim:
-            claimName: {{ include "cosmotech-api.fullname" . }}-blob-storage
-        {{end}}
         {{if .Values.api.tlsTruststore.enabled}}
         - name: custom-rootca
           secret:
@@ -90,11 +85,6 @@ spec:
             - mountPath: /config
               name: helm-config
               readOnly: true
-            {{if .Values.persistence.enabled}}
-            - mountPath: {{ include "cosmotech-api.blobPersistencePath" . }}
-              name: blob-storage
-              readOnly: false
-            {{end}}
             {{if .Values.api.tlsTruststore.enabled }}
             - mountPath: {{ include "cosmotech-api.custom-rootca-path" . | quote }}
               name: custom-rootca

api/kubernetes/helm-chart/values.yaml

@@ -192,6 +192,11 @@ config:
           requests:
             # File storage minimal claim is 100Gi for Premium classes
             storage: 100Gi
+      s3:
+        endpointUrl: "http://s3-server:9000"
+        bucketName: "changeme"
+        accessKeyId: "changeme"
+        secretAccessKey: "changeme"
       authorization:
         allowed-tenants: []
       identityProvider:

api/src/integrationTest/kotlin/com/cosmotech/api/home/ControllerTestBase.kt

@@ -34,7 +34,9 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders
 import org.springframework.web.context.WebApplicationContext
 import org.testcontainers.containers.PostgreSQLContainer
 import org.testcontainers.containers.PostgreSQLContainer.POSTGRESQL_PORT
+import org.testcontainers.containers.localstack.LocalStackContainer
 import org.testcontainers.junit.jupiter.Testcontainers
+import org.testcontainers.utility.DockerImageName
 import org.testcontainers.utility.MountableFile
 
 @Testcontainers
@@ -99,7 +101,8 @@ abstract class ControllerTestBase : AbstractTestcontainersRedisTestBase() {
     private const val READER_USER_CREDENTIALS = "readusertest"
     private const val WRITER_USER_CREDENTIALS = "writeusertest"
     private const val DEFAULT_REDIS_PORT = 6379
-    private const val REDIS_STACK_LASTEST_TAG_WITH_GRAPH = "6.2.6-v18"
+    private const val REDIS_STACK_LATEST_TAG_WITH_GRAPH = "6.2.6-v18"
+    private const val LOCALSTACK_FULL_IMAGE_NAME = "localstack/localstack:3.5.0"
 
     var postgres: PostgreSQLContainer<*> =
         PostgreSQLContainer("postgres:alpine3.19")
@@ -108,18 +111,25 @@ abstract class ControllerTestBase : AbstractTestcontainersRedisTestBase() {
 
     var redisStackServer =
         RedisStackContainer(
-            RedisStackContainer.DEFAULT_IMAGE_NAME.withTag(REDIS_STACK_LASTEST_TAG_WITH_GRAPH))
+            RedisStackContainer.DEFAULT_IMAGE_NAME.withTag(REDIS_STACK_LATEST_TAG_WITH_GRAPH))
+
+    val localStackServer =
+        LocalStackContainer(DockerImageName.parse(LOCALSTACK_FULL_IMAGE_NAME))
+            .withServices(LocalStackContainer.Service.S3)
 
     init {
       redisStackServer.start()
       postgres.start()
+      localStackServer.start()
+      localStackServer.execInContainer("awslocal", "s3", "mb", "s3://test-bucket")
     }
 
     @JvmStatic
     @DynamicPropertySource
     fun connectionProperties(registry: DynamicPropertyRegistry) {
       initPostgresConfiguration(registry)
       initRedisConfiguration(registry)
+      initS3Configuration(registry)
     }
 
     private fun initRedisConfiguration(registry: DynamicPropertyRegistry) {
@@ -133,6 +143,13 @@ abstract class ControllerTestBase : AbstractTestcontainersRedisTestBase() {
       registry.add("spring.data.redis.port") { DEFAULT_REDIS_PORT }
     }
 
+    private fun initS3Configuration(registry: DynamicPropertyRegistry) {
+      registry.add("spring.cloud.aws.s3.endpoint") { localStackServer.endpoint }
+      registry.add("spring.cloud.aws.credentials.access-key") { localStackServer.accessKey }
+      registry.add("spring.cloud.aws.credentials.secret-key") { localStackServer.secretKey }
+      registry.add("spring.cloud.aws.s3.region") { localStackServer.region }
+    }
+
     private fun initPostgresConfiguration(registry: DynamicPropertyRegistry) {
       registry.add("csm.platform.internalResultServices.storage.host") { postgres.host }
       registry.add("csm.platform.internalResultServices.storage.port") {

api/src/integrationTest/resources/application-test.yml

@@ -12,6 +12,20 @@ spring:
       client-type: jedis
   main:
     banner-mode: "off"
+  cloud:
+    aws:
+      credentials:
+        access-key: "s3_username"
+        secret-key: "s3_password"
+      s3:
+        # We don't need/have a region for our local S3 service but the AWS SDK requires one to be configured
+        region: "dummy"
+        # Enable path-style / disable DNS-style
+        # By default, and for AWS S3, the client crafts its URL with the bucket as sub-domain of the endpoint
+        # which is not how our current S3 implementation works as it expects the bucket in the path
+        # '<bucket_name>.<endpoint>/<object_key>' DNS vs Path '<endpoint>/<bucket_name>/<object_key>'
+        path-style-access-enabled: true
+        endpoint: "http://localhost:9000"
 
 csm:
   platform:
@@ -105,5 +119,11 @@ csm:
         default-page-size: 5
     rbac:
       enabled: true
+    s3:
+      endpointUrl: "http://localhost:9000"
+      bucketName: "test-bucket"
+      accessKeyId: "s3_username"
+      secretAccessKey: "s3_password"
+      region: "dummy"
 
 

api/src/main/kotlin/com/cosmotech/api/exceptions/AwsExceptionHandling.kt

@@ -0,0 +1,31 @@
+// Copyright (c) Cosmo Tech.
+// Licensed under the MIT license.
+package com.cosmotech.api.exceptions
+
+import java.net.URI
+import org.springframework.core.Ordered
+import org.springframework.core.annotation.Order
+import org.springframework.http.HttpStatus
+import org.springframework.http.ProblemDetail
+import org.springframework.web.bind.annotation.ExceptionHandler
+import org.springframework.web.bind.annotation.RestControllerAdvice
+import software.amazon.awssdk.awscore.exception.AwsServiceException
+
+@RestControllerAdvice
+@Order(Ordered.HIGHEST_PRECEDENCE)
+internal class AwsExceptionHandling : CsmExceptionHandling() {
+  private val httpStatusCodeTypePrefix = "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/"
+
+  @ExceptionHandler
+  fun handleBlobStorageException(exception: AwsServiceException): ProblemDetail {
+    val status = HttpStatus.INTERNAL_SERVER_ERROR
+
+    val problemDetail = ProblemDetail.forStatus(status)
+    problemDetail.type = URI.create(httpStatusCodeTypePrefix + status.value())
+
+    if (exception.message != null) {
+      problemDetail.detail = exception.message
+    }
+    return problemDetail
+  }
+}

api/src/main/resources/application.yml

@@ -34,6 +34,19 @@ spring:
       enabled: false
     kubernetes:
       enabled: false
+    aws:
+      credentials:
+        access-key: ${csm.platform.s3.accessKeyId}
+        secret-key: ${csm.platform.s3.secretAccessKey}
+      s3:
+        # We don't need/have a region for our local S3 service but the AWS SDK requires one to be configured
+        region: ${csm.platform.s3.region}
+        # Enable path-style / disable DNS-style
+        # By default, and for AWS S3, the client crafts its URL with the bucket as sub-domain of the endpoint
+        # which is not how our current S3 implementation works as it expects the bucket in the path
+        # '<bucket_name>.<endpoint>/<object_key>' DNS vs Path '<endpoint>/<bucket_name>/<object_key>'
+        path-style-access-enabled: true
+        endpoint: ${csm.platform.s3.endpointUrl}
 
 management:
   endpoints:
@@ -119,8 +132,11 @@ csm:
         sender:
           username: "eventbus_sender_username"
           password: "eventbus_sender_password"
-    blobPersistence:
-      path: /tmp/cosmotech-api-data
+    s3:
+      endpointUrl: "http://localhost:9000"
+      bucketName: "cosmotech-api"
+      accessKeyId: "s3_username"
+      secretAccessKey: "s3_password"
     argo:
       base-uri: "https://localhost:2746"
       image-pull-secrets: []

build.gradle.kts

@@ -77,8 +77,10 @@ val orgJsonVersion = "20240303"
 val jacksonModuleKotlinVersion = "2.18.3"
 val testNgVersion = "7.8.0"
 val testContainersRedisVersion = "1.6.4"
-val testContainersPostgreSQLVersion = "1.19.7"
+val testContainersPostgreSQLVersion = "1.20.6"
+val testContainersLocalStackVersion = "1.20.6"
 val commonCompressVersion = "1.27.1"
+val awsSpringVersion = "3.1.1"
 
 // Checks
 val detektVersion = "1.23.7"
@@ -312,6 +314,9 @@ subprojects {
 
     implementation("org.json:json:$orgJsonVersion")
 
+    implementation(platform("io.awspring.cloud:spring-cloud-aws-dependencies:$awsSpringVersion"))
+    implementation("io.awspring.cloud:spring-cloud-aws-starter-s3:$awsSpringVersion")
+
     testImplementation(kotlin("test"))
     testImplementation(platform("org.junit:junit-bom:$jUnitBomVersion"))
     testImplementation("org.junit.jupiter:junit-jupiter")
@@ -322,6 +327,7 @@ subprojects {
     testImplementation(
         "com.redis.testcontainers:testcontainers-redis-junit:$testContainersRedisVersion")
     testImplementation("org.testcontainers:postgresql:$testContainersPostgreSQLVersion")
+    testImplementation("org.testcontainers:localstack:$testContainersLocalStackVersion")
     testImplementation("org.springframework.boot:spring-boot-starter-test")
 
     integrationTestImplementation("org.springframework.boot:spring-boot-starter-test") {

config/application-dev.sample.yml

@@ -72,6 +72,12 @@ csm:
       tokenUrl: "[fill-this-value]" # eg. https://kubernetes.cosmotech.com/keycloak/realms/brewery/protocol/openid-connect/token
     metrics:
       enabled: false
+    s3:
+      endpointUrl: "http://localhost:9000"
+      bucketName: "cosmotech-api"
+      accessKeyId: "s3_username"
+      secretAccessKey: "s3_password"
+      region: "dummy"
     argo:
       base-uri: "http://localhost:2746"
       workflows:

connector/src/integrationTest/resources/application-connector-test.yml

@@ -12,6 +12,20 @@ spring:
       client-type: jedis
   main:
     banner-mode: "off"
+  cloud:
+    aws:
+      credentials:
+        access-key: "s3_username"
+        secret-key: "s3_password"
+      s3:
+        # We don't need/have a region for our local S3 service but the AWS SDK requires one to be configured
+        region: "dummy"
+        # Enable path-style / disable DNS-style
+        # By default, and for AWS S3, the client crafts its URL with the bucket as sub-domain of the endpoint
+        # which is not how our current S3 implementation works as it expects the bucket in the path
+        # '<bucket_name>.<endpoint>/<object_key>' DNS vs Path '<endpoint>/<bucket_name>/<object_key>'
+        path-style-access-enabled: true
+        endpoint: "http://localhost:9000"
 
 csm:
   platform:
@@ -95,5 +109,9 @@ csm:
         default-page-size: 5
     rbac:
       enabled: true
-    blobPersistence:
-      path: /tmp/cosmotech-api-connector-test-data
+    s3:
+      endpointUrl: "http://localhost:9000"
+      bucketName: "test-bucket"
+      accessKeyId: "s3_username"
+      secretAccessKey: "s3_password"
+      region: "dummy"

dataset/src/integrationTest/resources/application-dataset-test.yml

@@ -12,6 +12,20 @@ spring:
       client-type: jedis
   main:
     banner-mode: "off"
+  cloud:
+    aws:
+      credentials:
+        access-key: "s3_username"
+        secret-key: "s3_password"
+      s3:
+        # We don't need/have a region for our local S3 service but the AWS SDK requires one to be configured
+        region: "dummy"
+        # Enable path-style / disable DNS-style
+        # By default, and for AWS S3, the client crafts its URL with the bucket as sub-domain of the endpoint
+        # which is not how our current S3 implementation works as it expects the bucket in the path
+        # '<bucket_name>.<endpoint>/<object_key>' DNS vs Path '<endpoint>/<bucket_name>/<object_key>'
+        path-style-access-enabled: true
+        endpoint: "http://localhost:9000"
 
 csm:
   platform:
@@ -100,5 +114,9 @@ csm:
         default-page-size: 5
     rbac:
       enabled: true
-    blobPersistence:
-      path: /tmp/cosmotech-api-dataset-test-data
+    s3:
+      endpointUrl: "http://localhost:9000"
+      bucketName: "test-bucket"
+      accessKeyId: "s3_username"
+      secretAccessKey: "s3_password"
+      region: "dummy"

organization/src/integrationTest/resources/application-organization-test.yml

@@ -12,6 +12,20 @@ spring:
       client-type: jedis
   main:
     banner-mode: "off"
+  cloud:
+    aws:
+      credentials:
+        access-key: "s3_username"
+        secret-key: "s3_password"
+      s3:
+        # We don't need/have a region for our local S3 service but the AWS SDK requires one to be configured
+        region: "dummy"
+        # Enable path-style / disable DNS-style
+        # By default, and for AWS S3, the client crafts its URL with the bucket as sub-domain of the endpoint
+        # which is not how our current S3 implementation works as it expects the bucket in the path
+        # '<bucket_name>.<endpoint>/<object_key>' DNS vs Path '<endpoint>/<bucket_name>/<object_key>'
+        path-style-access-enabled: true
+        endpoint: "http://localhost:9000"
 
 csm:
   platform:
@@ -95,5 +109,9 @@ csm:
         default-page-size: 5
     rbac:
       enabled: true
-    blobPersistence:
-      path: /tmp/cosmotech-api-organization-test-data
+    s3:
+      endpointUrl: "http://localhost:9000"
+      bucketName: "test-bucket"
+      accessKeyId: "s3_username"
+      secretAccessKey: "s3_password"
+      region: "dummy"