diff --git a/src/main/java/kr/co/mcmp/softwarecatalog/application/service/ApplicationHistoryService.java b/src/main/java/kr/co/mcmp/softwarecatalog/application/service/ApplicationHistoryService.java index 9a62f68..579043b 100644 --- a/src/main/java/kr/co/mcmp/softwarecatalog/application/service/ApplicationHistoryService.java +++ b/src/main/java/kr/co/mcmp/softwarecatalog/application/service/ApplicationHistoryService.java @@ -46,6 +46,16 @@ public interface ApplicationHistoryService { void createApplicationStatusForVm(DeploymentHistory history, String vmId, String publicIp, Integer servicePort, String status, User user); + /** + * VM별 DeploymentHistory를 생성합니다. (다중 VM 배포용) + * + * @param request 배포 요청 정보 + * @param vmId VM ID + * @param user 사용자 + * @return VM별 배포 이력 + */ + DeploymentHistory createDeploymentHistoryForVm(DeploymentRequest request, String vmId, User user); + /** * 특정 VM에 기존 설치가 있는지 확인합니다. * diff --git a/src/main/java/kr/co/mcmp/softwarecatalog/application/service/impl/ApplicationHistoryServiceImpl.java b/src/main/java/kr/co/mcmp/softwarecatalog/application/service/impl/ApplicationHistoryServiceImpl.java index dcd9d35..a65c639 100644 --- a/src/main/java/kr/co/mcmp/softwarecatalog/application/service/impl/ApplicationHistoryServiceImpl.java +++ b/src/main/java/kr/co/mcmp/softwarecatalog/application/service/impl/ApplicationHistoryServiceImpl.java @@ -135,6 +135,49 @@ public void createApplicationStatusForVm(DeploymentHistory history, String vmId, log.info("Created ApplicationStatus for VM: {} with status: {}", vmId, status); } + /** + * VM별 DeploymentHistory를 생성합니다. (다중 VM 배포용) + */ + @Override + public DeploymentHistory createDeploymentHistoryForVm(DeploymentRequest request, String vmId, User user) { + SoftwareCatalogDTO catalog = catalogService.getCatalog(request.getCatalogId()); + + if (request.getDeploymentType() == DeploymentType.VM) { + VmAccessInfo vmInfo = cbtumblebugRestApi.getVmInfo(request.getNamespace(), request.getMciId(), vmId); + String[] parts = vmInfo.getConnectionName().split("-"); + + return DeploymentHistory.builder() + .catalog(catalog.toEntity()) + .deploymentType(DeploymentType.VM) + .cloudProvider(parts.length > 0 ? parts[0] : "") + .cloudRegion(vmInfo.getRegion().getRegion()) + .namespace(request.getNamespace()) + .mciId(request.getMciId()) + .vmId(vmId) + .clusterName(request.getClusterName()) + .publicIp(vmInfo.getPublicIP()) + .actionType(ActionType.INSTALL) + .status("IN_PROGRESS") + .servicePort(request.getServicePort()) + .executedAt(LocalDateTime.now()) + .executedBy(user) + .build(); + } else if (request.getDeploymentType() == DeploymentType.K8S) { + return DeploymentHistory.builder() + .catalog(catalog.toEntity()) + .deploymentType(DeploymentType.K8S) + .namespace(request.getNamespace()) + .clusterName(request.getClusterName()) + .actionType(ActionType.INSTALL) + .status("IN_PROGRESS") + .executedAt(LocalDateTime.now()) + .executedBy(user) + .build(); + } + + throw new RuntimeException("Unsupported deployment type: " + request.getDeploymentType()); + } + @Override public boolean hasExistingInstallation(String namespace, String mciId, String vmId, Long catalogId) { try { diff --git a/src/main/java/kr/co/mcmp/softwarecatalog/application/service/impl/DockerDeploymentService.java b/src/main/java/kr/co/mcmp/softwarecatalog/application/service/impl/DockerDeploymentService.java index 6169acf..e306638 100644 --- a/src/main/java/kr/co/mcmp/softwarecatalog/application/service/impl/DockerDeploymentService.java +++ b/src/main/java/kr/co/mcmp/softwarecatalog/application/service/impl/DockerDeploymentService.java @@ -1,5 +1,6 @@ package kr.co.mcmp.softwarecatalog.application.service.impl; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -7,6 +8,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.springframework.stereotype.Service; @@ -21,6 +23,7 @@ import kr.co.mcmp.softwarecatalog.application.dto.DeploymentRequest; import kr.co.mcmp.softwarecatalog.application.exception.ApplicationException; import kr.co.mcmp.softwarecatalog.application.model.DeploymentHistory; +import kr.co.mcmp.softwarecatalog.application.repository.DeploymentHistoryRepository; import kr.co.mcmp.softwarecatalog.application.service.ApplicationHistoryService; import kr.co.mcmp.softwarecatalog.application.service.DeploymentService; import kr.co.mcmp.softwarecatalog.application.config.NexusConfig; @@ -45,6 +48,7 @@ public class DockerDeploymentService implements DeploymentService { private final DockerOperationService dockerOperationService; private final CbtumblebugRestApi cbtumblebugRestApi; private final ApplicationHistoryService applicationHistoryService; + private final DeploymentHistoryRepository deploymentHistoryRepository; private final UserService userService; private final NexusConfig nexusConfig; @@ -78,20 +82,142 @@ public DeploymentHistory deployApplication(DeploymentRequest request) { } User user = userService.findUserByUsername(request.getUsername()).orElse(null); - DeploymentHistory history = applicationHistoryService.createDeploymentHistory(request, user); - + + // 다중 VM 배포의 경우 VM별 DeploymentHistory 생성 + if (request.getVmIds() != null && request.getVmIds().size() > 1) { + return deployToMultipleVms(request, catalog, user); + } else { + // 단일 VM 배포의 경우 기존 로직 사용 + DeploymentHistory history = applicationHistoryService.createDeploymentHistory(request, user); + + try { + deployToVms(request, catalog, history, user); + } catch (Exception e) { + log.error("Deployment failed", e); + history.setStatus("FAILED"); + applicationHistoryService.updateApplicationStatus(history, "FAILED", user); + applicationHistoryService.addDeploymentLog(history, LogType.ERROR, "Deployment failed: " + e.getMessage()); + } + + return history; + } + } + + /** + * 다중 VM 배포 - VM별 DeploymentHistory 생성 + */ + private DeploymentHistory deployToMultipleVms(DeploymentRequest request, SoftwareCatalogDTO catalog, User user) { + List vmIds = request.getVmIds(); + DeploymentHistory firstHistory = null; + + // 기존 설치 확인 + List alreadyInstalledVms = checkExistingInstallations(request, catalog, vmIds); + if (!alreadyInstalledVms.isEmpty()) { + log.info("Found existing installations on VMs: {}", alreadyInstalledVms); + } + + // 클러스터 설정 생성 (클러스터링 모드인 경우에만) + final Map clusterConfig = request.getVmDeploymentMode() == VmDeploymentMode.CLUSTERING ? buildClusterConfig(request, catalog, vmIds) : null; + + // VM별 배포 작업 생성 + List> deploymentFutures = new ArrayList<>(); + Map vmHistories = new HashMap<>(); + + for (int i = 0; i < vmIds.size(); i++) { + final String vmId = vmIds.get(i); + final int vmIndex = i; + + // 기존 설치가 있는 VM은 건너뛰기 + if (alreadyInstalledVms.contains(vmId)) { + log.info("Skipping deployment for VM {} due to existing installation", vmId); + continue; + } + + // VM별 DeploymentHistory 생성 + DeploymentHistory vmHistory = applicationHistoryService.createDeploymentHistoryForVm(request, vmId, user); + deploymentHistoryRepository.save(vmHistory); + vmHistories.put(vmId, vmHistory); + + if (firstHistory == null) { + firstHistory = vmHistory; + } + + CompletableFuture future = CompletableFuture.supplyAsync(() -> { + return deployToSingleVmAsync(request, catalog, vmHistory, user, vmId, vmIndex, vmIds, clusterConfig); + }, asyncExecutor); + + deploymentFutures.add(future); + } + + // 모든 배포 작업 완료 대기 try { - // 통합 배포 로직 - VM 개수와 배포 모드에 따라 처리 - deployToVms(request, catalog, history, user); + CompletableFuture allFutures = CompletableFuture.allOf( + deploymentFutures.toArray(new CompletableFuture[0]) + ); + + allFutures.get(30, TimeUnit.MINUTES); // 30분 타임아웃 + + // 배포 결과 처리 + List successfulVms = new ArrayList<>(); + List failedVms = new ArrayList<>(); + + for (int i = 0; i < deploymentFutures.size(); i++) { + try { + DeploymentResult result = deploymentFutures.get(i).get(); + String vmId = vmIds.get(i); + DeploymentHistory vmHistory = vmHistories.get(vmId); + + if (vmHistory == null) continue; + + VmAccessInfo vmAccessInfo = cbtumblebugRestApi.getVmInfo(request.getNamespace(), request.getMciId(), vmId); + Integer servicePort = request.getServicePort(); + + if (result.isSuccess()) { + successfulVms.add(vmId); + vmHistory.setStatus("SUCCESS"); + vmHistory.setUpdatedAt(LocalDateTime.now()); + deploymentHistoryRepository.save(vmHistory); + + String logMessage = createMessage(request, MessageType.SUCCESS, vmId, "deployment completed"); + applicationHistoryService.addDeploymentLog(vmHistory, LogType.INFO, logMessage); + + // 성공한 VM의 ApplicationStatus 생성 + applicationHistoryService.createApplicationStatusForVm( + vmHistory, vmId, vmAccessInfo.getPublicIP(), servicePort, "SUCCESS", user); + } else { + failedVms.add(vmId + " (" + result.getErrorMessage() + ")"); + vmHistory.setStatus("FAILED"); + vmHistory.setUpdatedAt(LocalDateTime.now()); + deploymentHistoryRepository.save(vmHistory); + + String errorMessage = createMessage(request, MessageType.ERROR, vmId, result.getErrorMessage()); + applicationHistoryService.addDeploymentLog(vmHistory, LogType.ERROR, errorMessage); + + // 실패한 VM의 ApplicationStatus 생성 + applicationHistoryService.createApplicationStatusForVm( + vmHistory, vmId, vmAccessInfo.getPublicIP(), servicePort, "FAILED", user); + } + } catch (Exception e) { + log.error("Failed to get deployment result", e); + failedVms.add("Unknown VM (future failed)"); + } + } + + // 배포 결과 처리 + processDeploymentResults(firstHistory, user, successfulVms, failedVms, request); } catch (Exception e) { - log.error("Deployment failed", e); - history.setStatus("FAILED"); - applicationHistoryService.updateApplicationStatus(history, "FAILED", user); - applicationHistoryService.addDeploymentLog(history, LogType.ERROR, "Deployment failed: " + e.getMessage()); + log.error("Deployment timeout or error", e); + // 모든 VM History를 FAILED로 설정 + for (DeploymentHistory vmHistory : vmHistories.values()) { + vmHistory.setStatus("FAILED"); + vmHistory.setUpdatedAt(LocalDateTime.now()); + deploymentHistoryRepository.save(vmHistory); + applicationHistoryService.addDeploymentLog(vmHistory, LogType.ERROR, "Deployment timeout or error: " + e.getMessage()); + } } - - return history; + + return firstHistory; } /** @@ -332,8 +458,8 @@ private ContainerConfig createClusterContainerConfig(SoftwareCatalogDTO catalog, // 애플리케이션별 특화 포트 설정 if ("elasticsearch".equals(appType)) { - int internalPort = 9300 + nodeIndex; - portBindings += "," + internalPort + ":9300"; + // Elasticsearch 클러스터링: 모든 노드가 9300 포트 사용 + portBindings += ",9300:9300"; } return new ContainerConfig(nodeName, portBindings); diff --git a/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/ContainerStatsCollector.java b/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/ContainerStatsCollector.java index 53b8814..5e2dfdc 100644 --- a/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/ContainerStatsCollector.java +++ b/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/ContainerStatsCollector.java @@ -7,7 +7,6 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.stereotype.Component; @@ -33,8 +32,26 @@ public class ContainerStatsCollector { public String getContainerId(DockerClient dockerClient, String containerName) { try { List containers = dockerClient.listContainersCmd().withShowAll(true).exec(); + log.info("Searching for container with name pattern: {}", containerName); + log.info("Available containers: {}", + containers.stream() + .map(container -> Arrays.toString(container.getNames())) + .collect(java.util.stream.Collectors.toList())); + return containers.stream() - .filter(container -> Arrays.asList(container.getNames()).contains("/" + containerName)) + .filter(container -> { + String[] names = container.getNames(); + if (names != null) { + for (String name : names) { + // 부분 문자열 매칭으로 변경 (정확한 이름 매칭에서) + if (name.contains(containerName)) { + log.debug("Found matching container: {}", name); + return true; + } + } + } + return false; + }) .findFirst() .map(Container::getId) .orElse(null); @@ -45,12 +62,16 @@ public String getContainerId(DockerClient dockerClient, String containerName) { } public ContainerHealthInfo collectContainerStats(DockerClient dockerClient, String containerId) { + log.info("Collecting container stats for containerId: {}", containerId); try { InspectContainerResponse containerInfo = dockerClient.inspectContainerCmd(containerId).exec(); + log.debug("Container info retrieved successfully"); + Statistics stats = collectStatistics(dockerClient, containerId); + log.debug("Statistics collected: {}", stats != null ? "SUCCESS" : "NULL"); + Integer servicePort = getServicePort(containerInfo); String ipAddress = getContainerIpAddress(containerInfo); - // List portAccessibilities = servicePorts.stream().map(port -> isPortAccessible(ipAddress, port)).collect(Collectors.toList()); Boolean isPortAccessible = servicePort != null && ipAddress != null && isPortAccessible(ipAddress, servicePort); Boolean isHealthCheck = isContainerHealthy(containerInfo); @@ -59,6 +80,9 @@ public ContainerHealthInfo collectContainerStats(DockerClient dockerClient, Stri Double networkIn = calculateNetworkIn(stats); Double networkOut = calculateNetworkOut(stats); + log.info("Container stats - CPU: {}%, Memory: {}%, Network In: {}, Network Out: {}", + cpuUsage, memoryUsage, networkIn, networkOut); + return ContainerHealthInfo.builder() .status(mapContainerStatus(containerInfo.getState().getStatus())) .servicePorts(servicePort) @@ -70,7 +94,7 @@ public ContainerHealthInfo collectContainerStats(DockerClient dockerClient, Stri .networkOut(networkOut) .build(); } catch (Exception e) { - log.error("Error collecting container stats for {}", containerId, e); + log.error("Error collecting container stats for containerId: {}", containerId, e); return ContainerHealthInfo.builder() .status("ERROR") .build(); @@ -90,12 +114,23 @@ private String getContainerIpAddress(InspectContainerResponse containerInfo) { } private Statistics collectStatistics(DockerClient dockerClient, String containerId) { + log.debug("Collecting statistics for containerId: {}", containerId); try (StatsCallback statsCallback = new StatsCallback()) { dockerClient.statsCmd(containerId).exec(statsCallback); - statsCallback.awaitCompletion(5, TimeUnit.SECONDS); - return statsCallback.getStats(); + boolean completed = statsCallback.awaitCompletion(5, TimeUnit.SECONDS); + log.debug("Statistics collection completed: {}", completed); + + Statistics stats = statsCallback.getStats(); + if (stats == null) { + log.warn("Statistics is null for containerId: {}", containerId); + } else { + log.debug("Statistics collected successfully - CPU: {}, Memory: {}", + stats.getCpuStats() != null ? "Available" : "NULL", + stats.getMemoryStats() != null ? "Available" : "NULL"); + } + return stats; } catch (Exception e) { - log.error("Error collecting statistics for {}", containerId, e); + log.error("Error collecting statistics for containerId: {}", containerId, e); return null; } } @@ -153,7 +188,7 @@ private Integer getServicePort(InspectContainerResponse containerInfo) { return bindings.entrySet().stream() .flatMap(entry -> { - ExposedPort exposedPort = entry.getKey(); + // ExposedPort exposedPort = entry.getKey(); Ports.Binding[] bindingsArray = entry.getValue(); return bindingsArray != null ? Arrays.stream(bindingsArray) : Stream.empty(); }) diff --git a/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/DockerClientFactory.java b/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/DockerClientFactory.java index ee4bc44..3de16f1 100644 --- a/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/DockerClientFactory.java +++ b/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/DockerClientFactory.java @@ -19,6 +19,8 @@ public class DockerClientFactory { public DockerClient getDockerClient(String host) { String dockerHost = "tcp://" + host + ":2375"; + log.info("Creating Docker client for host: {}", dockerHost); + DockerClientConfig config = DefaultDockerClientConfig.createDefaultConfigBuilder() .withDockerHost(dockerHost) .build(); @@ -28,7 +30,10 @@ public DockerClient getDockerClient(String host) { .connectionTimeout(Duration.ofSeconds(30)) .responseTimeout(Duration.ofSeconds(45)) .build(); - return DockerClientImpl.getInstance(config, httpClient); + + DockerClient client = DockerClientImpl.getInstance(config, httpClient); + log.info("Docker client created successfully for host: {}", dockerHost); + return client; } } \ No newline at end of file diff --git a/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/DockerMonitoringService.java b/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/DockerMonitoringService.java index 9619554..8bf731a 100644 --- a/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/DockerMonitoringService.java +++ b/src/main/java/kr/co/mcmp/softwarecatalog/docker/service/DockerMonitoringService.java @@ -11,7 +11,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; -import kr.co.mcmp.ape.cbtumblebug.api.CbtumblebugRestApi; import kr.co.mcmp.softwarecatalog.SoftwareCatalog; import kr.co.mcmp.softwarecatalog.application.constants.DeploymentType; import kr.co.mcmp.softwarecatalog.application.model.ApplicationStatus; @@ -19,7 +18,6 @@ import kr.co.mcmp.softwarecatalog.application.repository.ApplicationStatusRepository; import kr.co.mcmp.softwarecatalog.application.repository.DeploymentHistoryRepository; import kr.co.mcmp.softwarecatalog.docker.model.ContainerHealthInfo; -import kr.co.mcmp.softwarecatalog.docker.service.DockerLogCollector; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -32,9 +30,7 @@ public class DockerMonitoringService { private final DeploymentHistoryRepository deploymentHistoryRepository; private final DockerClientFactory dockerClientFactory; private final ContainerStatsCollector containerStatsCollector; - private final ContainerLogCollector containerLogCollector; private final DockerLogCollector dockerLogCollector; - private final CbtumblebugRestApi cbtumblebugRestApi; @Value("${docker.monitoring.interval:60000}") private long monitoringInterval; @@ -72,30 +68,47 @@ private List getActiveDeployments() { } private void updateContainerHealth(DeploymentHistory deployment) { - ApplicationStatus status = applicationStatusRepository.findTopByCatalogIdOrderByCheckedAtDesc(deployment.getCatalog().getId()).orElse(new ApplicationStatus()); + log.info("Updating container health for deployment: {} (VM: {})", deployment.getId(), deployment.getVmId()); + + // VM별 ApplicationStatus를 찾기 위해 catalogId와 vmId로 검색 + ApplicationStatus status = applicationStatusRepository.findByCatalogIdAndVmId( + deployment.getCatalog().getId(), deployment.getVmId()).orElse(new ApplicationStatus()); try (var dockerClient = dockerClientFactory.getDockerClient(deployment.getPublicIp())) { + log.info("Docker client connected successfully to: {}", deployment.getPublicIp()); + String catalogName = deployment.getCatalog().getName().toLowerCase().replaceAll("\\s+", "-"); + log.info("Looking for container with name pattern: {}", catalogName); + String containerId = containerStatsCollector.getContainerId(dockerClient, catalogName); - log.debug("containerId: {}", containerId); + log.info("Found containerId: {}", containerId); if (containerId == null) { + log.warn("Container not found for catalog: {} on VM: {}", catalogName, deployment.getVmId()); status.setStatus("NOT_FOUND"); + status.setCheckedAt(LocalDateTime.now()); applicationStatusRepository.save(status); return; } ContainerHealthInfo healthInfo = containerStatsCollector.collectContainerStats(dockerClient, containerId); + log.info("Health info collected - Status: {}, CPU: {}%, Memory: {}%", + healthInfo.getStatus(), healthInfo.getCpuUsage(), healthInfo.getMemoryUsage()); + updateApplicationStatus(status, deployment, healthInfo); + if (isThresholdExceeded(deployment.getCatalog(), healthInfo)) { + log.warn("Resource thresholds exceeded for deployment: {}", deployment.getId()); // UnifiedLog를 사용하여 에러 로그 수집 dockerLogCollector.collectAndSaveLogs(deployment.getId(), deployment.getVmId(), containerId); } applicationStatusRepository.save(status); + log.info("ApplicationStatus updated successfully for deployment: {}", deployment.getId()); } catch (Exception e) { - log.error("Failed to update container health for deployment: {}", deployment.getId(), e); + log.error("Failed to update container health for deployment: {} (VM: {})", deployment.getId(), deployment.getVmId(), e); status.setStatus("ERROR"); + status.setCheckedAt(LocalDateTime.now()); applicationStatusRepository.save(status); } }