Merge pull request kubearmor#553 from nyrahul/rel-issue

fix release issue due to k3s tests
rootxrishabh · Dec 29, 2021 · 57dd6eb · 57dd6eb
2 parents ebeb87c + cb79673
commit 57dd6eb
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 66 deletions.
diff --git a/.github/workflows/latest-release.yml b/.github/workflows/latest-release.yml
@@ -10,6 +10,7 @@ jobs:
   build:
     name: Create KubeArmor Release - 18.04
     runs-on: ubuntu-18.04
+    timeout-minutes: 20
     steps:
       - name: Checkout KubeArmor code
         uses: actions/checkout@v2
@@ -19,42 +20,18 @@ jobs:
 
       - name: Setup Enviroment
         run: |        
-          # create a single-node K3s cluster
-          curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE="644" INSTALL_K3S_EXEC="--flannel-backend=none --cluster-cidr=192.168.0.0/16 --disable-network-policy --disable=traefik" sh -
+          ./contribution/k3s/install_k3s.sh
 
-          #setting a variable environment
-          mkdir ~/.kube/
-          cp /etc/rancher/k3s/k3s.yaml ~/.kube/config 
-
-          # install Calico-Operator
-          kubectl create -f https://docs.projectcalico.org/manifests/tigera-operator.yaml
-
-          # install Calico-Manifest
-          kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml
-            
       - name: Run KubeArmor 
         run: |
           kubectl apply -f KubeArmor/build/kubearmor-test-k3s.yaml
-          kubectl get pods --all-namespaces 
-
           while [[ $(kubectl get pods -l kubearmor-app=kubearmor -n kube-system -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do echo "waiting for pod" && sleep 1; done
-
-          sleep 60
+          kubectl get pods -A
 
       - name: Run KubeArmor tests
         run: |
           ./tests/test-scenarios-github.sh
 
-      - name: Archive log artifacts
-        if: ${{ failure() }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kubearmor.logs
-          path: |
-            /tmp/kubearmor.test
-            /tmp/kubearmor.log
-            /tmp/kubearmor.msg
-      
       - name: Login to Docker Hub
         uses: docker/login-action@v1
         with:

diff --git a/KubeArmor/build/kubearmor-test-k3s.yaml b/KubeArmor/build/kubearmor-test-k3s.yaml
@@ -55,11 +55,12 @@ spec:
         ports:
         - containerPort: 32767
         volumeMounts:
-        - name: containerd-sock-path # containerd (read-only)
-          mountPath: /var/run/containerd/containerd.sock
-          readOnly: true
-        - name: containerd-storage-path # containerd storage (read-only)
-          mountPath: /run/containerd
+        - name: docker-sock-path # docker (read-only)
+          mountPath: /var/run/docker.sock
+          readOnly: true 
+        - name: docker-storage-path # docker storage (read-only)
+          mountPath: /var/lib/docker
+          readOnly: true 
         - name: usr-src-path # BPF (read-only)
           mountPath: /usr/src
           readOnly: true
@@ -90,14 +91,14 @@ spec:
         terminationMessagePath: /dev/termination-log
       terminationGracePeriodSeconds: 30
       volumes:
-      - name: containerd-sock-path # containerd
+      - name: docker-sock-path # docker
         hostPath:
-          path: /run/k3s/containerd/containerd.sock
+          path: /var/run/docker.sock
           type: Socket
-      - name: containerd-storage-path # containerd
+      - name: docker-storage-path # docker
         hostPath:
-          path: /run/k3s/containerd
-          type: DirectoryOrCreate
+          path: /var/lib/docker
+          type: Directory
       - name: usr-src-path # BPF
         hostPath:
           path: /usr/src

diff --git a/contribution/k3s/install_k3s.sh b/contribution/k3s/install_k3s.sh
@@ -3,10 +3,17 @@
 # Copyright 2021 Authors of KubeArmor
 
 # create a single-node K3s cluster
-curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE="644" INSTALL_K3S_EXEC="--flannel-backend=none --cluster-cidr=192.168.0.0/16 --disable-network-policy --disable=traefik" sh -
+curl -sfL https://get.k3s.io | K3S_KUBECONFIG_MODE="644" INSTALL_K3S_EXEC="--disable=traefik --docker" sh -
 
-# install Calico-Operator
-kubectl create -f https://docs.projectcalico.org/manifests/tigera-operator.yaml
+KUBEDIR=~/.kube
+KUBECONFIG=$KUBEDIR/config
+
+[[ ! -d $KUBEDIR ]] && mkdir ~/.kube/
+if [ -f $KUBECONFIG ]; then
+	KUBECONFIGBKP=$KUBEDIR/config.backup
+	echo "Found $KUBECONFIG already in place ... backing it up to $KUBECONFIGBKP"
+	cp $KUBECONFIG $KUBECONFIGBKP
+fi
+
+cp /etc/rancher/k3s/k3s.yaml $KUBEDIR/config 
 
-# install Calico-Manifest
-kubectl apply -f https://docs.projectcalico.org/manifests/calico.yaml
diff --git a/tests/test-scenarios-github.sh b/tests/test-scenarios-github.sh
@@ -109,12 +109,6 @@ function start_and_wait_for_kubearmor_initialization() {
         exit 1
     fi
 
-    PROXY=$(ps -ef | grep "kubectl proxy" | wc -l)
-    if [ $PROXY != 2 ]; then
-        FAIL "Proxy is not running"
-        exit 1
-    fi
-
     cd $ARMOR_HOME
 
     echo "Options: -logPath=$ARMOR_LOG ${ARMOR_OPTIONS[@]}"
@@ -123,19 +117,32 @@ function start_and_wait_for_kubearmor_initialization() {
         SKIP_NATIVE_HOST_POLICY=1
     fi
 
-    echo "Github Actions - Environment"
-    make clean; make build-test
-    sudo -E ./kubearmor -test.coverprofile=.coverprofile -logPath=$ARMOR_LOG ${ARMOR_OPTIONS[@]} > $ARMOR_MSG &
+    ka_podname=`kubectl get pods -n kube-system -l kubearmor-app=kubearmor -o custom-columns=":metadata.name" --no-headers`
+    if [ "$ka_podname" != "" ]; then
+        CAT_LOG="kubectl exec -n kube-system $ka_podname -- cat $ARMOR_LOG"
+    else # start kubearmor as local process
+        PROXY=$(ps -ef | grep "kubectl proxy" | wc -l)
+        if [ $PROXY != 2 ]; then
+           FAIL "Proxy is not running"
+           exit 1
+        fi
 
-    for (( ; ; ))
-    do
-        grep "Initialized KubeArmor" $ARMOR_MSG &> /dev/null
-        [[ $? -eq 0 ]] && break
-        sleep 1
-    done
+        echo "Github Actions - Environment"
+        make clean; make build-test
+        sudo -E ./kubearmor -test.coverprofile=.coverprofile -logPath=$ARMOR_LOG ${ARMOR_OPTIONS[@]} > $ARMOR_MSG &
+        CAT_LOG="cat $ARMOR_LOG"
+
+        for (( ; ; ))
+        do
+           grep "Initialized KubeArmor" $ARMOR_MSG &> /dev/null
+           [[ $? -eq 0 ]] && break
+           sleep 1
+        done
+    fi
 }
 
 function stop_and_wait_for_kubearmor_termination() {
+    [[ "$ka_podname" != "" ]] && echo "kubearmor not started by this script, hence not stopping" && return
     ps -e | grep kubearmor | awk '{print $1}' | xargs -I {} sudo kill {}
 
     for (( ; ; ))
@@ -189,7 +196,7 @@ function should_not_find_any_log() {
 
     sleep 5
 
-    audit_log=$(grep -E "$1.*policyName.*\"$2\".*MatchedPolicy.*\"$6\".*$3.*resource.*$4.*$5" $ARMOR_LOG | tail -n 1 | grep -v Passed)
+    audit_log=$($CAT_LOG | grep -E "$1.*policyName.*\"$2\".*MatchedPolicy.*\"$6\".*$3.*resource.*$4.*$5" | tail -n 1 | grep -v Passed)
     if [ $? == 0 ]; then
         echo $audit_log
         FAIL "Found the log from logs"
@@ -205,7 +212,7 @@ function should_find_passed_log() {
 
     sleep 5
 
-    audit_log=$(grep -E "$1.*policyName.*\"$2\".*MatchedPolicy.*$3.*resource.*$4.*$5" $ARMOR_LOG | tail -n 1 | grep Passed)
+    audit_log=$($CAT_LOG | grep -E "$1.*policyName.*\"$2\".*MatchedPolicy.*$3.*resource.*$4.*$5" | tail -n 1 | grep Passed)
     if [ $? != 0 ]; then
         audit_log="<No Log>"
         FAIL "Failed to find the log from logs"
@@ -227,9 +234,9 @@ function should_find_blocked_log() {
     fi
 
     if [[ $6 -eq 0 ]]; then
-        audit_log=$(grep -E "$1.*policyName.*\"$2\".*$match_type.*$3.*resource.*$4.*$5" $ARMOR_LOG | tail -n 1 | grep -v Passed)
+        audit_log=$($CAT_LOG | grep -E "$1.*policyName.*\"$2\".*$match_type.*$3.*resource.*$4.*$5" | tail -n 1 | grep -v Passed)
     else
-        audit_log=$(grep -E "$1.*policyName.*\"NativePolicy\".*$match_type.*$3.*resource.*$4.*$5" $ARMOR_LOG | tail -n 1 | grep -v Passed)
+        audit_log=$($CAT_LOG | grep -E "$1.*policyName.*\"NativePolicy\".*$match_type.*$3.*resource.*$4.*$5" | tail -n 1 | grep -v Passed)
     fi
     if [ $? != 0 ]; then
         audit_log="<No Log>"
@@ -246,7 +253,7 @@ function should_not_find_any_host_log() {
 
     sleep 5
 
-    audit_log=$(grep -E "$HOST_NAME.*policyName.*\"$1\".*MatchedHostPolicy.*\"$5\".*$2.*resource.*$3.*$4" $ARMOR_LOG | tail -n 1 | grep -v Passed)
+    audit_log=$($CAT_LOG | grep -E "$HOST_NAME.*policyName.*\"$1\".*MatchedHostPolicy.*\"$5\".*$2.*resource.*$3.*$4" | tail -n 1 | grep -v Passed)
     if [ $? == 0 ]; then
         echo $audit_log
         FAIL "Found the log from logs"
@@ -262,7 +269,7 @@ function should_find_passed_host_log() {
 
     sleep 5
 
-    audit_log=$(grep -E "$HOST_NAME.*policyName.*\"$1\".*MatchedHostPolicy.*$2.*resource.*$3.*$4" $ARMOR_LOG | tail -n 1 | grep Passed)
+    audit_log=$($CAT_LOG | grep -E "$HOST_NAME.*policyName.*\"$1\".*MatchedHostPolicy.*$2.*resource.*$3.*$4" | tail -n 1 | grep Passed)
     if [ $? != 0 ]; then
         audit_log="<No Log>"
         FAIL "Failed to find the log from logs"
@@ -284,9 +291,9 @@ function should_find_blocked_host_log() {
     fi
 
     if [[ $5 -eq 0 ]]; then
-        audit_log=$(grep -E "$HOST_NAME.*policyName.*\"$1\".*$match_type.*$2.*resource.*$3.*$4" $ARMOR_LOG | tail -n 1 | grep -v Passed)
+        audit_log=$($CAT_LOG | grep -E "$HOST_NAME.*policyName.*\"$1\".*$match_type.*$2.*resource.*$3.*$4" | tail -n 1 | grep -v Passed)
     else
-        audit_log=$(grep -E "$HOST_NAME.*policyName.*\"NativePolicy\".*$match_type.*$2.*resource.*$3.*$4" $ARMOR_LOG | tail -n 1 | grep -v Passed)
+        audit_log=$($CAT_LOG | grep -E "$HOST_NAME.*policyName.*\"NativePolicy\".*$match_type.*$2.*resource.*$3.*$4" | tail -n 1 | grep -v Passed)
     fi
     if [ $? != 0 ]; then
         audit_log="<No Log>"
@@ -555,8 +562,6 @@ if [ ! -f kubearmor ]; then
     DBG "Built KubeArmor"
 fi
 
-sleep 1
-
 INFO "Starting KubeArmor"
 start_and_wait_for_kubearmor_initialization
 INFO "Started KubeArmor"