Release-0.5.3

.gitignore

@@ -20,3 +20,4 @@ credentials.env
 credentials.prod.env
 cmd/dotmesh-server/pkg/dynamic-provisioning/dm-provisioner
 LAST_IMAGE_BUILD
+tests/config.sh

.gitlab-ci.yml

@@ -20,6 +20,7 @@ stages:
     - test
     - deploy
     - manual_deploy
+    - notify
 
 before_script:
     - mkdir -p gopath
@@ -32,8 +33,6 @@ before_script:
     - git checkout master && git pull --ff-only origin master
     - git checkout $CI_COMMIT_REF_NAME && git reset --hard $CI_COMMIT_SHA
     - while [ -f /dotmesh-test-cleanup.lock ]; do echo "waiting for test cleanup..."; sleep 1; done
-    - docker login -u $QUAY_USER -p $QUAY_PASSWORD quay.io
-    - docker login -u $OVH_REGISTRY_USERNAME -p $OVH_REGISTRY_PASSWORD registry.containers.ovh.net
 
 build_client_linux:
     stage: build
@@ -78,6 +77,7 @@ build_server:
         - fast
     script:
         # push to the OVH registry.
+        - docker login -u $OVH_REGISTRY_USERNAME -p $OVH_REGISTRY_PASSWORD registry.containers.ovh.net
         - (cd cmd/dotmesh-server && ./rebuild.sh) # also pushes
         - echo "pushed $CI_DOCKER_SERVER_IMAGE"
         - echo "pushed $CI_DOCKER_PROVISIONER_IMAGE"
@@ -86,6 +86,7 @@ build_server:
         # push to quay.io, by overriding various variables (although notably
         # not CI_DOCKER_DIND_PROVISIONER_IMAGE because that's test-only
         # tooling).
+        - docker login -u $QUAY_USER -p $QUAY_PASSWORD quay.io
         - (cd cmd/dotmesh-server && export CI_DOCKER_SERVER_IMAGE=$RELEASE_DOCKER_REGISTRY/dotmesh-server:$CI_COMMIT_SHA; export CI_DOCKER_PROVISIONER_IMAGE=$RELEASE_DOCKER_REGISTRY/dotmesh-dynamic-provisioner:$CI_COMMIT_SHA; export CI_DOCKER_OPERATOR_IMAGE=$RELEASE_DOCKER_REGISTRY/dotmesh-operator:$CI_COMMIT_SHA; ./rebuild.sh; echo "pushed $CI_DOCKER_SERVER_IMAGE"; echo "pushed $CI_DOCKER_PROVISIONER_IMAGE"; echo "pushed $CI_DOCKER_OPERATOR_IMAGE")
     artifacts:
         paths:
@@ -136,6 +137,8 @@ macos_docker_stable:
         - docker version
         - which docker
         - docker container prune -f
+    only:
+        - schedules
 
 macos_docker_edge:
     stage: test
@@ -155,6 +158,8 @@ macos_docker_edge:
         - docker version
         - which docker
         - docker container prune -f
+    only:
+        - schedules
 
 linux_smoke:
     stage: test
@@ -173,6 +178,8 @@ linux_smoke:
         - echo "DOCKER VERSION"
         - docker version
         - which docker
+    only:
+        - schedules
 
 versioner_unit_tests:
     stage: test
@@ -219,6 +226,23 @@ linux_single_node:
     except:
         - schedules
 
+linux_recover_from_unmounted_dot_on_master:
+    stage: test
+    dependencies:
+        - build_server
+        - build_client_linux
+    tags:
+        - ubuntu
+        - fast
+    script:
+        - ./test.sh -run TestRecoverFromUnmountedDotOnMaster
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
+    except:
+        - schedules
+
 linux_two_nodes_same_cluster:
     stage: test
     dependencies:
@@ -253,8 +277,9 @@ linux_kubernetes_volumes:
         when: always
     except:
         - schedules
+    allow_failure: true
 
-linux_kubernetes_operator:
+linux_kubernetes_operator_dynamic_provisioning:
     stage: test
     dependencies:
         - build_server
@@ -264,7 +289,25 @@ linux_kubernetes_operator:
         - fast
     script:
         - pwd
-        - ./test.sh -run TestKubernetesOperator
+        - ./test.sh -run TestKubernetesOperator/DynamicProvisioning
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
+    except:
+        - schedules
+
+linux_kubernetes_operator_PVC:
+    stage: test
+    dependencies:
+        - build_server
+        - build_client_linux
+    tags:
+        - ubuntu
+        - fast
+    script:
+        - pwd
+        - ./test.sh -run TestKubernetesOperator/PVC*
     artifacts:
         paths:
         - extracted_logs
@@ -485,3 +528,11 @@ mark_release_as_stable:
   when: manual
   except:
     - schedules
+
+notify_smoke_test_failure:
+  stage: notify
+  script:
+    - "curl -X POST -H 'Content-type: application/json' --data \"{\\\"text\\\":\\\"ALERT, Smoke tests failed $CI_PROJECT_URL/pipelines/$CI_PIPELINE_ID\\\"}\" $SLACK_WEBHOOK_URL"
+  when: on_failure
+  only:
+    - schedules

cmd/dm/pkg/commands/cluster.go

@@ -877,7 +877,7 @@ func clusterCommonSetup(clusterUrl, adminPassword, adminKey, pkiPath string) err
 			return err
 		}
 	}
-	err = config.AddRemote("local", "admin", getHostFromEnv(), adminKey)
+	err = config.AddRemote("local", "admin", getHostFromEnv(), 0, adminKey)
 	if err != nil {
 		return err
 	}
@@ -917,7 +917,7 @@ func clusterCommonSetup(clusterUrl, adminPassword, adminKey, pkiPath string) err
 				return false
 			}
 			var response bool
-			response, err = dm.Ping()
+			response, err = dm.PingLocal()
 			if err != nil {
 				e()
 				return false

cmd/dm/pkg/commands/main.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"text/tabwriter"
 
@@ -123,7 +124,7 @@ func NewCmdRemote(out io.Writer) *cobra.Command {
 		},
 	}
 	cmd.AddCommand(&cobra.Command{
-		Use:   "add <remote-name> <user@cluster-hostname>",
+		Use:   "add <remote-name> <user@cluster-hostname>[:<port-number>]",
 		Short: "Add a remote",
 		Long:  "Online help: https://docs.dotmesh.com/references/cli/#add-a-new-remote-dm-remote-add-name-user-hostname",
 
@@ -143,10 +144,28 @@ func NewCmdRemote(out io.Writer) *cobra.Command {
 				}
 				user := shrapnel[0]
 				hostname := shrapnel[1]
+				port := 0
+				var err error
+				if strings.Contains(hostname, ":") {
+					shrapnel = strings.SplitN(hostname, ":", 2)
+					if len(shrapnel) != 2 {
+						return fmt.Errorf(
+							"Please specify user@cluster-hostname:port, got %s", shrapnel,
+						)
+					}
+					hostname = shrapnel[0]
+					port, err = strconv.Atoi(shrapnel[1])
+					if err != nil {
+						return fmt.Errorf(
+							"Please specify a port as an integer, e.g user@cluster-hostname:port, got %s", shrapnel[1],
+						)
+					}
+				}
 				dm, err := remotes.NewDotmeshAPI(configPath)
 				if err != nil {
 					return err
 				}
+				fmt.Printf("dm client %#v\n", dm)
 				// allow this to be used be a script
 				apiKey := os.Getenv("DOTMESH_PASSWORD")
 				if apiKey == "" {
@@ -161,14 +180,19 @@ func NewCmdRemote(out io.Writer) *cobra.Command {
 				client := &remotes.JsonRpcClient{
 					User:     user,
 					Hostname: hostname,
+					Port:     port,
 					ApiKey:   apiKey,
 				}
 				var result bool
 				err = client.CallRemote(context.Background(), "DotmeshRPC.Ping", nil, &result)
+
+				_, err = remotes.Ping(client)
+
 				if err != nil {
 					return err
 				}
-				err = dm.Configuration.AddRemote(remote, user, hostname, string(apiKey))
+
+				err = dm.Configuration.AddRemote(remote, user, hostname, port, string(apiKey))
 				if err != nil {
 					return err
 				}

cmd/dm/pkg/remotes/api.go

@@ -15,7 +15,7 @@ import (
 )
 
 const DEFAULT_BRANCH string = "master"
-const RPC_TIMEOUT time.Duration = 2 * time.Minute
+const RPC_TIMEOUT time.Duration = 2 * time.Second
 
 type VersionInfo struct {
 	InstalledVersion    string `json:"installed_version"`
@@ -72,15 +72,21 @@ func (dm *DotmeshAPI) CallRemote(
 	return dm.client.CallRemote(ctx, method, args, response)
 }
 
-func (dm *DotmeshAPI) Ping() (bool, error) {
+func Ping(client *JsonRpcClient) (bool, error) {
 	var response bool
-	err := dm.client.CallRemote(context.Background(), "DotmeshRPC.Ping", struct{}{}, &response)
+	ctx, cancel := context.WithTimeout(context.Background(), RPC_TIMEOUT)
+	defer cancel()
+	err := client.CallRemote(ctx, "DotmeshRPC.Ping", struct{}{}, &response)
 	if err != nil {
 		return false, err
 	}
 	return response, nil
 }
 
+func (dm *DotmeshAPI) PingLocal() (bool, error) {
+	return Ping(dm.client)
+}
+
 func (dm *DotmeshAPI) BackupEtcd() (string, error) {
 	var response string
 	err := dm.client.CallRemote(context.Background(), "DotmeshRPC.DumpEtcd",

cmd/dm/pkg/remotes/remotes.go

@@ -43,6 +43,7 @@ import (
 type Remote struct {
 	User                 string
 	Hostname             string
+	Port                 int `json:",omitempty"`
 	ApiKey               string
 	CurrentVolume        string
 	CurrentBranches      map[string]string
@@ -283,14 +284,15 @@ func (c *Configuration) RemoteExists(remote string) bool {
 	return ok
 }
 
-func (c *Configuration) AddRemote(remote, user, hostname, apiKey string) error {
+func (c *Configuration) AddRemote(remote, user, hostname string, port int, apiKey string) error {
 	_, ok := c.Remotes[remote]
 	if ok {
 		return fmt.Errorf("Remote exists '%s'", remote)
 	}
 	c.Remotes[remote] = &Remote{
 		User:     user,
 		Hostname: hostname,
+		Port:     port,
 		ApiKey:   apiKey,
 	}
 	return c.save()
@@ -318,6 +320,7 @@ func (c *Configuration) ClusterFromRemote(remote string) (*JsonRpcClient, error)
 	return &JsonRpcClient{
 		User:     remoteCreds.User,
 		Hostname: remoteCreds.Hostname,
+		Port:     remoteCreds.Port,
 		ApiKey:   remoteCreds.ApiKey,
 	}, nil
 }
@@ -329,6 +332,7 @@ func (c *Configuration) ClusterFromCurrentRemote() (*JsonRpcClient, error) {
 type JsonRpcClient struct {
 	User     string
 	Hostname string
+	Port     int
 	ApiKey   string
 }
 
@@ -378,22 +382,25 @@ func (j *JsonRpcClient) CallRemote(
 				"add'. Try 'dm cluster init' if you don't have a cluster yet.",
 		)
 	}
-
+	addressToUse := Address{}
 	addressesToTry := []Address{}
-
-	if j.Hostname == "saas.dotmesh.io" || j.Hostname == "dothub.com" {
-		scheme := "https"
-		port := 443
-		addressesToTry = append(addressesToTry, Address{scheme, j.Hostname, port})
+	var err error
+	if j.Port == 0 {
+		if j.Hostname == "saas.dotmesh.io" || j.Hostname == "dothub.com" {
+			scheme := "https"
+			port := 443
+			addressesToTry = append(addressesToTry, Address{scheme, j.Hostname, port})
+		} else {
+			scheme := "http"
+			addressesToTry = append(addressesToTry, Address{scheme, j.Hostname, 32607})
+			addressesToTry = append(addressesToTry, Address{scheme, j.Hostname, 6969})
+		}
+		addressToUse, err = j.tryAddresses(ctx, addressesToTry)
+		if err != nil {
+			return err
+		}
 	} else {
-		scheme := "http"
-		addressesToTry = append(addressesToTry, Address{scheme, j.Hostname, 32607})
-		addressesToTry = append(addressesToTry, Address{scheme, j.Hostname, 6969})
-	}
-
-	addressToUse, err := j.tryAddresses(ctx, addressesToTry)
-	if err != nil {
-		return err
+		addressToUse = Address{"http", j.Hostname, j.Port}
 	}
 
 	return j.reallyCallRemote(ctx, method, args, result, addressToUse)

cmd/dotmesh-server/pkg/dind-flexvolume/flexvolume.go

@@ -125,9 +125,15 @@ func (d *FlexVolumeDriver) mount(targetMountDir, jsonOptions string) (map[string
 func (d *FlexVolumeDriver) unmount(targetMountDir string) (map[string]interface{}, error) {
 	err := System("umount", targetMountDir)
 	if err != nil {
-		logger.Printf("MOUNT: unmount err for %s: %v", targetMountDir, err)
+		logger.Printf("UNMOUNT: unmount err for %s: %v", targetMountDir, err)
 		return nil, err
 	}
+	err = os.Remove(targetMountDir)
+	if err != nil {
+		logger.Printf("UNMOUNT: remove mountpoint err for %s: %v", targetMountDir, err)
+		return nil, err
+	}
+
 	return nil, nil
 }