Skip to content

Commit

Permalink
Merge pull request 'Implement kubernetes-additional-manifests' (#6) f…
Browse files Browse the repository at this point in the history 
…rom feature/add_manifests into develop
  • Loading branch information
@fmichielssen
fmichielssen committed Sep 22, 2020
2 parents 0e53e9a + e84b538 commit 55e24ab
Show file tree
Hide file tree
Showing 4 changed files with 265 additions and 3 deletions.
49 changes: 48 additions & 1 deletion src/main/java/eu/openanalytics/containerproxy/backend/kubernetes/KubernetesBackend.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
*/
package eu.openanalytics.containerproxy.backend.kubernetes;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
Expand Down Expand Up @@ -52,6 +53,7 @@
import io.fabric8.kubernetes.api.model.ContainerPortBuilder;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.EnvVarSourceBuilder;
import io.fabric8.kubernetes.api.model.HasMetadata;
import io.fabric8.kubernetes.api.model.LocalObjectReference;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.PodBuilder;
Expand All @@ -71,9 +73,9 @@
import io.fabric8.kubernetes.client.ConfigBuilder;
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.NamespacedKubernetesClient;
import io.fabric8.kubernetes.client.dsl.LogWatch;
import io.fabric8.kubernetes.client.internal.readiness.Readiness;
import io.fabric8.kubernetes.client.utils.Serialization;

public class KubernetesBackend extends AbstractContainerBackend {

Expand Down Expand Up @@ -237,6 +239,10 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
Pod patchedPod = podPatcher.patchWithDebug(startupPod, proxy.getSpec().getKubernetesPodPatchAsJsonpatch());
final String effectiveKubeNamespace = patchedPod.getMetadata().getNamespace(); // use the namespace of the patched Pod, in case the patch changes the namespace.
container.getParameters().put(PARAM_NAMESPACE, effectiveKubeNamespace);

// create additional manifests -> use the effective (i.e. patched) namespace if no namespace is provided
createAdditionalManifstes(proxy, effectiveKubeNamespace);

Pod startedPod = kubeClient.pods().inNamespace(effectiveKubeNamespace).create(patchedPod);

// Workaround: waitUntilReady appears to be buggy.
Expand Down Expand Up @@ -287,9 +293,45 @@ protected Container startContainer(ContainerSpec spec, Proxy proxy) throws Excep
proxy.getTargets().put(mapping, target);
}


return container;
}


/**
* Creates the extra manifests/resources defined in the ProxySpec.
*
* The resource will only be created if it does not already exist.
*/
private void createAdditionalManifstes(Proxy proxy, String namespace) {
for (HasMetadata fullObject: getAdditionManifestsAsObjects(proxy, namespace)) {
if (kubeClient.resource(fullObject).fromServer().get() == null) {
kubeClient.resource(fullObject).createOrReplace();
}
}
}

/**
* Converts the additional manifests of the spec into HasMetadat objects.
* When the resource has no namespace definition, the provided namespace
* parameter will be used.
*/
private List<HasMetadata> getAdditionManifestsAsObjects(Proxy proxy, String namespace) {
ArrayList<HasMetadata> result = new ArrayList<HasMetadata>();
for (String manifest : proxy.getSpec().getKubernetesAdditionalManifests()) {
HasMetadata object = Serialization.unmarshal(new ByteArrayInputStream(manifest.getBytes())); // used to determine whether the manifest has specified a namespace

HasMetadata fullObject = kubeClient.load(new ByteArrayInputStream(manifest.getBytes())).get().get(0);
if (object.getMetadata().getNamespace() == null) {
// the load method (in some cases) automatically sets a namepsace when no namespace is provided
// therefore we overwrite this namespace with the namsepace of the pod.
fullObject.getMetadata().setNamespace(namespace);
}
result.add(fullObject);
}
return result;
}

private boolean isServiceReady(Service service) {
if (service == null) {
return false;
Expand Down Expand Up @@ -334,6 +376,11 @@ protected void doStopProxy(Proxy proxy) throws Exception {
if (pod != null) kubeClient.pods().inNamespace(kubeNamespace).delete(pod);
Service service = Service.class.cast(container.getParameters().get(PARAM_SERVICE));
if (service != null) kubeClient.services().inNamespace(kubeNamespace).delete(service);

// delete additional manifests
for (HasMetadata fullObject: getAdditionManifestsAsObjects(proxy, kubeNamespace)) {
kubeClient.resource(fullObject).delete();
}
}
}

Expand Down
14 changes: 14 additions & 0 deletions src/main/java/eu/openanalytics/containerproxy/model/spec/ProxySpec.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import javax.json.JsonPatch;
import javax.json.JsonValue;
Expand All @@ -50,6 +51,7 @@ public class ProxySpec {
private Map<String, String> settings = new HashMap<>();

private JsonPatch kubernetesPodPatches;
private List<String> kubernetesAdditionalManifests = new ArrayList<>();

public ProxySpec() {
settings = new HashMap<>();
Expand Down Expand Up @@ -154,6 +156,14 @@ public void setKubernetesPodPatches(String kubernetesPodPatches) throws JsonPars
private void setKubernetesPodPatches(JsonPatch kubernetesPodPatches) {
this.kubernetesPodPatches = kubernetesPodPatches;
}

public void setKubernetesAdditionalManifests(List<String> manifests) {
this.kubernetesAdditionalManifests = manifests;
}

public List<String> getKubernetesAdditionalManifests() {
return kubernetesAdditionalManifests;
}

public void copy(ProxySpec target) {
target.setId(id);
Expand Down Expand Up @@ -195,6 +205,10 @@ public void copy(ProxySpec target) {
target.setKubernetesPodPatches(kubernetesPodPatches);
}

if (kubernetesAdditionalManifests != null) {
target.setKubernetesAdditionalManifests(kubernetesAdditionalManifests.stream().collect(Collectors.toList()));
}

}

}
170 changes: 169 additions & 1 deletion src/test/java/eu/openanalytics/containerproxy/test/proxy/TestIntegrationOnKube.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;

import java.io.ByteArrayInputStream;
import java.net.URI;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -64,11 +65,15 @@
import io.fabric8.kubernetes.api.model.ContainerStatus;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.NamespaceBuilder;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaimList;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.PodList;
import io.fabric8.kubernetes.api.model.Quantity;
import io.fabric8.kubernetes.api.model.ResourceRequirements;
import io.fabric8.kubernetes.api.model.Secret;
import io.fabric8.kubernetes.api.model.SecretBuilder;
import io.fabric8.kubernetes.api.model.SecretList;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.ServiceAccountBuilder;
import io.fabric8.kubernetes.api.model.ServiceList;
Expand Down Expand Up @@ -474,7 +479,7 @@ public void launchProxyWithPodPatches() throws Exception {
}

/**
* Test whethet the merging of properties works properly
* Test whether the merging of properties works properly
*/
@Test
public void launchProxyWithPatchesWithMerging() throws Exception {
Expand Down Expand Up @@ -524,6 +529,169 @@ public void launchProxyWithPatchesWithMerging() throws Exception {
assertEquals(0, proxyService.getProxies(null, true).size());
}

/**
* Tests the creation and deleting of additional manifests.
* The first manifest contains a namespace definition.
* The second manifest does not contain a namespace definition, but in the end should have the same namespace as the pod.
*/
@Test
public void launchProxyWithAdditionalManifests() throws Exception {
final String overridenNamespace = "it-b9fa0a24-overriden";
try {
System.out.println(client);
client.namespaces().create(new NamespaceBuilder()
.withNewMetadata()
.withName(overridenNamespace)
.endMetadata()
.build());

String specId = environment.getProperty("proxy.specs[8].id");

ProxySpec baseSpec = proxyService.findProxySpec(s -> s.getId().equals(specId), true);
ProxySpec spec = proxyService.resolveProxySpec(baseSpec, null, null);
Proxy proxy = proxyService.startProxy(spec, true);
String containerId = proxy.getContainers().get(0).getId();

PodList podList = client.pods().inNamespace(overridenNamespace).list();
assertEquals(1, podList.getItems().size());
Pod pod = podList.getItems().get(0);
assertEquals("Running", pod.getStatus().getPhase());
assertEquals(overridenNamespace, pod.getMetadata().getNamespace());
assertEquals("sp-pod-" + containerId, pod.getMetadata().getName());
assertEquals(1, pod.getStatus().getContainerStatuses().size());
ContainerStatus container = pod.getStatus().getContainerStatuses().get(0);
assertEquals(true, container.getReady());
assertEquals("openanalytics/shinyproxy-demo:latest", container.getImage());

PersistentVolumeClaimList claimList = client.persistentVolumeClaims().inNamespace(overridenNamespace).list();
assertEquals(1, claimList.getItems().size());
PersistentVolumeClaim claim = claimList.getItems().get(0);
assertEquals(overridenNamespace, claim.getMetadata().getNamespace());
assertEquals("manifests-pvc", claim.getMetadata().getName());

// secret has no namespace defined -> should be created in the namespace defined by the pod patches
SecretList sercetList = client.secrets().inNamespace(overridenNamespace).list();
assertEquals(2, sercetList.getItems().size());
for (Secret secret : sercetList.getItems()) {
if (secret.getMetadata().getName().startsWith("default-token")) {
continue;
}
assertEquals(overridenNamespace, secret.getMetadata().getNamespace());
assertEquals("manifests-secret", secret.getMetadata().getName());
}

proxyService.stopProxy(proxy, false, true);

// Give Kube the time to clean
Thread.sleep(2000);

// all pods should be deleted
podList = client.pods().inNamespace(session.getNamespace()).list();
assertEquals(0, podList.getItems().size());
// all additional manifests should be deleted
assertEquals(1, client.secrets().inNamespace(overridenNamespace).list().getItems().size());
assertTrue(client.secrets().inNamespace(overridenNamespace).list()
.getItems().get(0).getMetadata().getName().startsWith("default-token"));
assertEquals(0, client.persistentVolumeClaims().inNamespace(overridenNamespace).list().getItems().size());

assertEquals(0, proxyService.getProxies(null, true).size());
} finally {
// just to be sure both the namespace and service account are cleaned up
client.namespaces().withName(overridenNamespace).delete();
}
}

/**
* Tests the creation and deleting of additional manifests.
* The first manifest contains a namespace definition.
* The second manifest does not contain a namespace definition, but in the end should have the same namespace as the pod.
*
* This is exactly the same test as the previous one, except that the PVC already exists (and should not be re-created).
*/
@Test
public void launchProxyWithAdditionalManifestsOfWhichOneAlreadyExists() throws Exception {
final String overridenNamespace = "it-b9fa0a24-overriden";
try {
System.out.println(client);
client.namespaces().create(new NamespaceBuilder()
.withNewMetadata()
.withName(overridenNamespace)
.endMetadata()
.build());

// create the PVC
String pvcSpec =
"apiVersion: v1\n" +
"kind: PersistentVolumeClaim\n" +
"metadata:\n" +
" name: manifests-pvc\n" +
" namespace: it-b9fa0a24-overriden\n" +
"spec:\n" +
" storageClassName: standard\n" +
" accessModes:\n" +
" - ReadWriteOnce\n" +
" resources:\n" +
" requests:\n" +
" storage: 5Gi";

client.load(new ByteArrayInputStream(pvcSpec.getBytes())).createOrReplace();

String specId = environment.getProperty("proxy.specs[8].id");

ProxySpec baseSpec = proxyService.findProxySpec(s -> s.getId().equals(specId), true);
ProxySpec spec = proxyService.resolveProxySpec(baseSpec, null, null);
Proxy proxy = proxyService.startProxy(spec, true);
String containerId = proxy.getContainers().get(0).getId();

PodList podList = client.pods().inNamespace(overridenNamespace).list();
assertEquals(1, podList.getItems().size());
Pod pod = podList.getItems().get(0);
assertEquals("Running", pod.getStatus().getPhase());
assertEquals(overridenNamespace, pod.getMetadata().getNamespace());
assertEquals("sp-pod-" + containerId, pod.getMetadata().getName());
assertEquals(1, pod.getStatus().getContainerStatuses().size());
ContainerStatus container = pod.getStatus().getContainerStatuses().get(0);
assertEquals(true, container.getReady());
assertEquals("openanalytics/shinyproxy-demo:latest", container.getImage());

PersistentVolumeClaimList claimList = client.persistentVolumeClaims().inNamespace(overridenNamespace).list();
assertEquals(1, claimList.getItems().size());
PersistentVolumeClaim claim = claimList.getItems().get(0);
assertEquals(overridenNamespace, claim.getMetadata().getNamespace());
assertEquals("manifests-pvc", claim.getMetadata().getName());

// secret has no namespace defined -> should be created in the namespace defined by the pod patches
SecretList sercetList = client.secrets().inNamespace(overridenNamespace).list();
assertEquals(2, sercetList.getItems().size());
for (Secret secret : sercetList.getItems()) {
if (secret.getMetadata().getName().startsWith("default-token")) {
continue;
}
assertEquals(overridenNamespace, secret.getMetadata().getNamespace());
assertEquals("manifests-secret", secret.getMetadata().getName());
}

proxyService.stopProxy(proxy, false, true);

// Give Kube the time to clean
Thread.sleep(2000);

// all pods should be deleted
podList = client.pods().inNamespace(session.getNamespace()).list();
assertEquals(0, podList.getItems().size());
// all additional manifests should be deleted
assertEquals(1, client.secrets().inNamespace(overridenNamespace).list().getItems().size());
assertTrue(client.secrets().inNamespace(overridenNamespace).list()
.getItems().get(0).getMetadata().getName().startsWith("default-token"));
assertEquals(0, client.persistentVolumeClaims().inNamespace(overridenNamespace).list().getItems().size());

assertEquals(0, proxyService.getProxies(null, true).size());
} finally {
// just to be sure both the namespace and service account are cleaned up
client.namespaces().withName(overridenNamespace).delete();
}
}

public static class TestConfiguration {
@Bean
@Primary
Expand Down
35 changes: 34 additions & 1 deletion src/test/resources/application-test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,42 @@ proxy:
value:
name: ADDED_VAR
value: VALUE
- id: 02_hello
- id: 01_hello_manifests
container-specs:
- image: "openanalytics/shinyproxy-demo"
cmd: ["R", "-e", "shinyproxy::run_01_hello()"]
port-mapping:
default: 3838
kubernetes-pod-patches: |
- op: replace
path: /metadata/namespace
value: it-b9fa0a24-overriden
kubernetes-additional-manifests:
- |
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: manifests-pvc
namespace: it-b9fa0a24-overriden
spec:
storageClassName: standard
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 5Gi
- |
apiVersion: v1
kind: Secret
metadata:
name: manifests-secret
type: Opaque
data:
password: cGFzc3dvcmQ=
- id: 02_hello
container-specs:
- image: "openanalytics/shinyproxy-demo"
cmd: ["R", "-e", "shinyproxy::run_01_hello()"]
port-mapping:
default: 3838

0 comments on commit 55e24ab

Please sign in to comment.