HELLODATA-1881 - create virtual folders and groups

hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/HDSidecarSftpGo.java

@@ -2,6 +2,7 @@
 
 import ch.bedag.dap.hellodata.commons.nats.annotation.EnableJetStream;
 import ch.bedag.dap.hellodata.commons.sidecars.context.HelloDataContextConfig;
+import ch.bedag.dap.hellodata.sidecars.sftpgo.config.S3ConnectionsConfig;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import org.springframework.boot.SpringApplication;
@@ -20,7 +21,7 @@
 @EnableDiscoveryClient
 @ConfigurationPropertiesScan
 @ComponentScan("ch.bedag.dap.hellodata")
-@EnableConfigurationProperties({HelloDataContextConfig.class})
+@EnableConfigurationProperties({HelloDataContextConfig.class, S3ConnectionsConfig.class})
 public class HDSidecarSftpGo {
 
     public static void main(String[] args) {

hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/config/S3ConnectionsConfig.java

@@ -0,0 +1,33 @@
+package ch.bedag.dap.hellodata.sidecars.sftpgo.config;
+
+import lombok.Data;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.validation.annotation.Validated;
+
+import java.util.List;
+
+@Log4j2
+@Data
+@Validated
+@ConfigurationProperties("hello-data")
+public class S3ConnectionsConfig {
+
+    private List<S3Connection> s3Connections;
+
+    public S3Connection getS3Connection(String contextKey) {
+        return s3Connections.stream().filter(c -> c.contextKey.equals(contextKey))
+                .findFirst().orElseThrow(() -> new RuntimeException(String.format("No s3 config for data domain: %s", contextKey)));
+    }
+
+    @Data
+    public static class S3Connection {
+        private String contextKey;
+        private String endpoint;
+        private String bucket;
+        private String accessKey;
+        private String accessSecret;
+        private boolean forcePathStyle;
+    }
+
+}

hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/config/WebClientConfig.java

@@ -1,41 +1,41 @@
 package ch.bedag.dap.hellodata.sidecars.sftpgo.config;
 
 import ch.bedag.dap.hellodata.sidecars.sftpgo.client.invoker.ApiClient;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import lombok.extern.log4j.Log4j2;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.http.client.reactive.ReactorClientHttpConnector;
 import org.springframework.web.reactive.function.client.ExchangeFilterFunction;
 import org.springframework.web.reactive.function.client.WebClient;
 import reactor.core.publisher.Mono;
+import reactor.netty.http.client.HttpClient;
 
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-
+@Log4j2
 @Configuration
 public class WebClientConfig {
 
     @Value("${hello-data.sftpgo.base-url}")
     private String baseUrl;
 
+    private static ExchangeFilterFunction logRequest() {
+        return ExchangeFilterFunction.ofRequestProcessor(clientRequest -> {
+            log.info("Request: {} {}", clientRequest.method(), clientRequest.url());
+            clientRequest.headers().forEach((name, values) -> values.forEach(value -> log.info("{}={}", name, value)));
+            return Mono.just(clientRequest);
+        });
+    }
+
     @Bean
-    public ApiClient apiClient(ObjectMapper objectMapper) {
+    public ApiClient apiClient() {
         WebClient webClient = WebClient.builder()
                 .baseUrl(baseUrl)
-                .filter(logRequest())
+                .clientConnector(new ReactorClientHttpConnector(
+                        HttpClient.create().wiretap(true)
+                ))
                 .build();
-        DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
         ApiClient apiClient = new ApiClient(webClient);
         apiClient.setBasePath(baseUrl + apiClient.getBasePath());
         return apiClient;
     }
-
-    private ExchangeFilterFunction logRequest() {
-        return ExchangeFilterFunction.ofRequestProcessor(clientRequest -> {
-            System.out.println("Request: " + clientRequest.method() + " " + clientRequest.url());
-            clientRequest.headers()
-                    .forEach((name, values) -> values.forEach(value -> System.out.println(name + ": " + value)));
-            return Mono.just(clientRequest);
-        });
-    }
 }

hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/service/user/SftpGoCreateUserConsumer.java → hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/listener/SftpGoCreateUserConsumer.java

@@ -1,4 +1,4 @@
-package ch.bedag.dap.hellodata.sidecars.sftpgo.service.user;
+package ch.bedag.dap.hellodata.sidecars.sftpgo.listener;
 
 import ch.bedag.dap.hellodata.commons.nats.annotation.JetStreamSubscribe;
 import ch.bedag.dap.hellodata.commons.sidecars.resources.v1.user.data.SubsystemUserUpdate;

hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/service/user/SftpGoDisableUserConsumer.java → hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/listener/SftpGoDisableUserConsumer.java

@@ -1,4 +1,4 @@
-package ch.bedag.dap.hellodata.sidecars.sftpgo.service.user;
+package ch.bedag.dap.hellodata.sidecars.sftpgo.listener;
 
 import ch.bedag.dap.hellodata.commons.nats.annotation.JetStreamSubscribe;
 import ch.bedag.dap.hellodata.commons.sidecars.resources.v1.user.data.SubsystemUserUpdate;

hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/service/user/SftpGoEnableUserConsumer.java → hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/listener/SftpGoEnableUserConsumer.java

@@ -1,4 +1,4 @@
-package ch.bedag.dap.hellodata.sidecars.sftpgo.service.user;
+package ch.bedag.dap.hellodata.sidecars.sftpgo.listener;
 
 import ch.bedag.dap.hellodata.commons.nats.annotation.JetStreamSubscribe;
 import ch.bedag.dap.hellodata.commons.sidecars.resources.v1.user.data.SubsystemUserUpdate;

hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/listener/SftpGoPublishedAppInfoResourcesConsumer.java

@@ -0,0 +1,33 @@
+package ch.bedag.dap.hellodata.sidecars.sftpgo.listener;
+
+import ch.bedag.dap.hellodata.commons.nats.annotation.JetStreamSubscribe;
+import ch.bedag.dap.hellodata.commons.sidecars.context.HdBusinessContextInfo;
+import ch.bedag.dap.hellodata.commons.sidecars.context.HdContextType;
+import ch.bedag.dap.hellodata.commons.sidecars.resources.v1.appinfo.AppInfoResource;
+import ch.bedag.dap.hellodata.sidecars.sftpgo.service.SftpGoService;
+import lombok.AllArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Service;
+
+import static ch.bedag.dap.hellodata.commons.sidecars.events.HDEvent.PUBLISH_APP_INFO_RESOURCES;
+
+@Log4j2
+@Service
+@AllArgsConstructor
+public class SftpGoPublishedAppInfoResourcesConsumer {
+
+    private final SftpGoService sftpGoService;
+
+    @SuppressWarnings("unused")
+    @JetStreamSubscribe(event = PUBLISH_APP_INFO_RESOURCES)
+    public void subscribe(AppInfoResource appInfoResource) {
+        HdBusinessContextInfo businessContextInfo = appInfoResource.getBusinessContextInfo();
+        HdBusinessContextInfo subContext = businessContextInfo.getSubContext();
+        if (subContext != null && subContext.getType().equalsIgnoreCase(HdContextType.DATA_DOMAIN.getTypeName())) {
+            log.info("------- Received appInfo resource {}, for the following context config {}", appInfoResource, businessContextInfo);
+            String dataDomainKey = subContext.getKey();
+            log.info("--> Creating missing groups with virtual folders for the data domain: {} ", dataDomainKey);
+            sftpGoService.createGroup(dataDomainKey, subContext.getName());
+        }
+    }
+}

hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/listener/SftpGoSyncUsersConsumer.java

@@ -0,0 +1,39 @@
+package ch.bedag.dap.hellodata.sidecars.sftpgo.listener;
+
+
+import ch.bedag.dap.hellodata.commons.nats.annotation.JetStreamSubscribe;
+import ch.bedag.dap.hellodata.commons.sidecars.resources.v1.user.data.UserContextRoleUpdate;
+import ch.bedag.dap.hellodata.commons.sidecars.resources.v1.user.data.UsersContextRoleUpdate;
+import lombok.AllArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.apache.commons.lang3.time.StopWatch;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+import static ch.bedag.dap.hellodata.commons.sidecars.events.HDEvent.SYNC_USERS;
+
+@Log4j2
+@Service
+@AllArgsConstructor
+public class SftpGoSyncUsersConsumer {
+
+    private final SftpGoUpdateUserContextRoleConsumer sftpGoUpdateUserContextRoleConsumer;
+
+    @SuppressWarnings("unused")
+    @JetStreamSubscribe(event = SYNC_USERS, timeoutMinutes = 15L)
+    public void subscribe(UsersContextRoleUpdate usersContextRoleUpdate) {
+        StopWatch stopWatch = new StopWatch();
+        stopWatch.start();
+        log.info("[SYNC_USERS] Started users synchronization");
+        List<UserContextRoleUpdate> userContextRoleUpdates = usersContextRoleUpdate.getUserContextRoleUpdates();
+        for (UserContextRoleUpdate userContextRoleUpdate : userContextRoleUpdates) {
+            try {
+                sftpGoUpdateUserContextRoleConsumer.processContextRoleUpdate(userContextRoleUpdate);
+            } catch (Exception e) {
+                log.error("Could not synchronize user {}", userContextRoleUpdate.getEmail(), e);
+            }
+        }
+        log.info("[SYNC_USERS] Finished users synchronization. Operation took {}", stopWatch.formatTime());
+    }
+}

hello-data-sidecars/hello-data-sidecar-sftpgo/src/main/java/ch/bedag/dap/hellodata/sidecars/sftpgo/listener/SftpGoUpdateUserContextRoleConsumer.java

@@ -0,0 +1,31 @@
+package ch.bedag.dap.hellodata.sidecars.sftpgo.listener;
+
+import ch.bedag.dap.hellodata.commons.nats.annotation.JetStreamSubscribe;
+import ch.bedag.dap.hellodata.commons.sidecars.resources.v1.user.data.UserContextRoleUpdate;
+import ch.bedag.dap.hellodata.sidecars.sftpgo.service.SftpGoService;
+import ch.bedag.dap.hellodata.sidecars.sftpgo.service.resource.SftpGoUserResourceProviderService;
+import lombok.AllArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Service;
+
+import java.util.List;
+
+import static ch.bedag.dap.hellodata.commons.sidecars.events.HDEvent.UPDATE_USER_CONTEXT_ROLE;
+
+@Log4j2
+@Service
+@AllArgsConstructor
+public class SftpGoUpdateUserContextRoleConsumer {
+
+    private final SftpGoService sftpgoService;
+    private final SftpGoUserResourceProviderService sftpGoUserResourceProviderService;
+
+    @SuppressWarnings("unused")
+    @JetStreamSubscribe(event = UPDATE_USER_CONTEXT_ROLE)
+    public void processContextRoleUpdate(UserContextRoleUpdate userContextRoleUpdate) {
+        log.info("-=-=-=-= RECEIVED USER CONTEXT ROLES UPDATE: payload: {}", userContextRoleUpdate);
+        List<UserContextRoleUpdate.ContextRole> contextRoles = userContextRoleUpdate.getContextRoles().stream()
+                .filter(contextRole -> contextRole.getParentContextKey() != null).toList();
+    }
+
+}