Release 0.3.2

Merge branch 'master' into release-0.3

.gitlab-ci.yml

@@ -3,7 +3,9 @@ variables:
     SERVER_NAME: dotmesh-server
     PROVISIONER_NAME: dotmesh-dynamic-provisioner
 
-    CI_DOCKER_REGISTRY: quay.io/dotmesh
+    CI_DOCKER_REGISTRY: registry.containers.ovh.net/dotmesh
+    RELEASE_DOCKER_REGISTRY: quay.io/dotmesh
+
     CI_DOCKER_TAG: $CI_COMMIT_SHA
 
     CI_DOCKER_SERVER_IMAGE: $CI_DOCKER_REGISTRY/dotmesh-server:$CI_COMMIT_SHA
@@ -29,6 +31,7 @@ before_script:
     - git checkout $CI_COMMIT_REF_NAME && git reset --hard $CI_COMMIT_SHA
     - while [ -f /dotmesh-test-cleanup.lock ]; do echo "waiting for test cleanup..."; sleep 1; done
     - docker login -u $QUAY_USER -p $QUAY_PASSWORD quay.io
+    - docker login -u $OVH_REGISTRY_USERNAME -p $OVH_REGISTRY_PASSWORD registry.containers.ovh.net
 
 build_client_linux:
     stage: build
@@ -75,6 +78,7 @@ build_server:
         - (cd cmd/dotmesh-server && ./rebuild.sh) # also pushes
         - echo "pushed $CI_DOCKER_SERVER_IMAGE"
         - echo "pushed $CI_DOCKER_PROVISIONER_IMAGE"
+        - (cd cmd/dotmesh-server && export CI_DOCKER_SERVER_IMAGE=$RELEASE_DOCKER_REGISTRY/dotmesh-server:$CI_COMMIT_SHA; export CI_DOCKER_PROVISIONER_IMAGE=$RELEASE_DOCKER_REGISTRY/dotmesh-dynamic-provisioner:$CI_COMMIT_SHA; ./rebuild.sh; echo "pushed $CI_DOCKER_SERVER_IMAGE"; echo "pushed $CI_DOCKER_PROVISIONER_IMAGE")
     artifacts:
         paths:
             - cmd/dotmesh-server/target
@@ -167,6 +171,10 @@ linux_default_dot:
         - fast
     script:
         - ./test.sh -run TestDefaultDot
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
 
 linux_single_node:
     stage: test
@@ -178,6 +186,10 @@ linux_single_node:
         - fast
     script:
         - ./test.sh -run TestSingleNode
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
 
 linux_two_nodes_same_cluster:
     stage: test
@@ -189,6 +201,10 @@ linux_two_nodes_same_cluster:
         - fast
     script:
         - ./test.sh -run TestTwoNodesSameCluster
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
 
 linux_kubernetes:
     stage: test
@@ -201,6 +217,10 @@ linux_kubernetes:
     script:
         - pwd
         - ./test.sh -run TestKubernetes
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
 
 linux_deletion_simple:
     stage: test
@@ -212,6 +232,10 @@ linux_deletion_simple:
         - fast
     script:
         - ./test.sh -run TestDeletionSimple
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
 
 linux_deletion_complex:
     stage: test
@@ -223,6 +247,10 @@ linux_deletion_complex:
         - fast
     script:
         - ./test.sh -run TestDeletionComplex
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
     allow_failure: true
 
 linux_two_single_node_clusters:
@@ -235,6 +263,10 @@ linux_two_single_node_clusters:
         - fast
     script:
         - ./test.sh -run TestTwoSingleNodeClusters
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
 
 linux_three_single_node_clusters:
     stage: test
@@ -246,6 +278,10 @@ linux_three_single_node_clusters:
         - fast
     script:
         - ./test.sh -run TestThreeSingleNodeClusters
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
 
 linux_two_double_node_clusters:
     stage: test
@@ -257,18 +293,26 @@ linux_two_double_node_clusters:
         - fast
     script:
         - ./test.sh -run TestTwoDoubleNodeClusters
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
 
 linux_stress:
-   stage: test
-   dependencies:
-       - build_server
-       - build_client_linux
-   tags:
-       - ubuntu
-       - fast
-   script:
-       - ./test.sh -timeout 30m -run TestStress
-   when: manual
+    stage: test
+    dependencies:
+        - build_server
+        - build_client_linux
+    tags:
+        - ubuntu
+        - fast
+    script:
+        - ./test.sh -timeout 30m -run TestStress
+    artifacts:
+        paths:
+        - extracted_logs
+        when: always
+    when: manual
 
 trigger_e2e_test:
     stage: deploy
@@ -307,7 +351,7 @@ deploy_release_build:
         - ubuntu
         - fast
     script:
-        - VERSION=`cd cmd/versioner; go run versioner.go`
+        - VERSION=`cd cmd/versioner; go run versioner.go`; echo VERSION=$VERSION
         - "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no [email protected] mkdir -p /pool/releases/$VERSION"
         - "rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz binaries/{Linux,Darwin} yaml [email protected]:/pool/releases/$VERSION"
     only:

cmd/dm/pkg/commands/cluster.go

@@ -259,7 +259,7 @@ func clusterUpgrade(cmd *cobra.Command, args []string, out io.Writer) error {
 	fmt.Printf("Upgrading local Dotmesh server to version %s (docker image %s)\n", clientVersion, dotmeshDockerImage)
 
 	if !offline {
-		fmt.Printf("Pulling dotmesh-server docker image... ")
+		fmt.Printf("Pulling dotmesh-server docker image...\n")
 		resp, err := exec.Command(
 			"docker", "pull", dotmeshDockerImage,
 		).CombinedOutput()
@@ -343,7 +343,7 @@ func clusterCommonPreflight() error {
 
 	containers := []string{"dotmesh-etcd", "dotmesh-server", "dotmesh-server-inner"}
 
-	fmt.Printf("Checking dotmesh isn't running... ")
+	fmt.Printf("Checking dotmesh isn't running...\n")
 	// - Are all the containers running?
 	//   If yes, exit: We're good already.
 	anyContainersMissing := false
@@ -354,10 +354,10 @@ func clusterCommonPreflight() error {
 			return err
 		}
 		if ret == 0 {
-			fmt.Printf("%s✓ ", c)
+			fmt.Printf("  * %s: is running\n", c)
 			anyContainersRunning = true
 		} else {
-			fmt.Printf("%s❌ ", c)
+			fmt.Printf("  * %s: is not running\n", c)
 			anyContainersMissing = true
 		}
 	}
@@ -384,7 +384,7 @@ func clusterCommonPreflight() error {
 	}
 
 	if !offline {
-		fmt.Printf("Pulling dotmesh-server docker image... ")
+		fmt.Printf("Pulling dotmesh-server docker image...\n")
 		resp, err := exec.Command(
 			"docker", "pull", dotmeshDockerImage,
 		).CombinedOutput()

cmd/dotmesh-server/pkg/main/controller.go

@@ -183,40 +183,40 @@ func (s *InMemoryState) getOne(ctx context.Context, fs string) (DotmeshVolume, e
 		return DotmeshVolume{}, err
 	}
 
-	log.Printf("[getOne] starting for %v", fs)
+	quietLogger(fmt.Sprintf("[getOne] starting for %v", fs))
 
 	if tlf, clone, err := s.registry.LookupFilesystemById(fs); err == nil {
 		authorized, err := tlf.Authorize(ctx)
 		if err != nil {
 			return DotmeshVolume{}, err
 		}
 		if !authorized {
-			log.Printf(
+			quietLogger(fmt.Sprintf(
 				"[getOne] notauth for %v", fs,
-			)
+			))
 			return DotmeshVolume{}, PermissionDenied{}
 		}
 		// if not exists, 0 is fine
 		s.globalDirtyCacheLock.Lock()
-		log.Printf(
+		quietLogger(fmt.Sprintf(
 			"[getOne] looking up %s with master %s in %s",
 			fs, master, *s.globalDirtyCache,
-		)
+		))
 		dirty, ok := (*s.globalDirtyCache)[fs]
 		var dirtyBytes int64
 		var sizeBytes int64
 		if ok {
 			dirtyBytes = dirty.DirtyBytes
 			sizeBytes = dirty.SizeBytes
-			log.Printf(
+			quietLogger(fmt.Sprintf(
 				"[getOne] got dirtyInfo %d,%d for %s with master %s in %s",
 				sizeBytes, dirtyBytes, fs, master, *s.globalDirtyCache,
-			)
+			))
 		} else {
-			log.Printf(
+			quietLogger(fmt.Sprintf(
 				"[getOne] %s was not in %s",
 				fs, *s.globalDirtyCache,
-			)
+			))
 		}
 		s.globalDirtyCacheLock.Unlock()
 		// if not exists, 0 is fine
@@ -269,9 +269,9 @@ func (s *InMemoryState) getOne(ctx context.Context, fs string) (DotmeshVolume, e
 			d.ServerStatuses[server.Id] = status
 			s.globalStateCacheLock.Unlock()
 		}
-		log.Printf(
+		quietLogger(fmt.Sprintf(
 			"[getOne] here is your volume: %s", d,
-		)
+		))
 		return d, nil
 	} else {
 		return DotmeshVolume{}, fmt.Errorf("Unable to find filesystem name for id %s", fs)

cmd/dotmesh-server/pkg/main/http.go

@@ -46,38 +46,61 @@ func (state *InMemoryState) runServer() {
 		log.Printf("Error while registering services %s", err)
 	}
 
-	tracer := opentracing.GlobalTracer()
-
 	router := mux.NewRouter()
-	router.Handle("/rpc",
-		middleware.FromHTTPRequest(tracer, "rpc")(NewAuthHandler(r)),
-	)
+
+	// only use the zipkin middleware if we have a TRACE_ADDR
+	if os.Getenv("TRACE_ADDR") != "" {
+		tracer := opentracing.GlobalTracer()
+
+		router.Handle("/rpc",
+			middleware.FromHTTPRequest(tracer, "rpc")(NewAuthHandler(r)),
+		)
+
+		router.Handle(
+			"/filesystems/{filesystem}/{fromSnap}/{toSnap}",
+			middleware.FromHTTPRequest(tracer, "zfs-sender")(
+				NewAuthHandler(state.NewZFSSendingServer()),
+			),
+		).Methods("GET")
+
+		router.Handle(
+			"/filesystems/{filesystem}/{fromSnap}/{toSnap}",
+			middleware.FromHTTPRequest(tracer, "zfs-receiver")(
+				NewAuthHandler(state.NewZFSReceivingServer()),
+			),
+		).Methods("POST")
+
+	} else {
+		router.Handle("/rpc", NewAuthHandler(r))
+
+		router.Handle(
+			"/filesystems/{filesystem}/{fromSnap}/{toSnap}",
+			NewAuthHandler(state.NewZFSSendingServer()),
+		).Methods("GET")
+
+		router.Handle(
+			"/filesystems/{filesystem}/{fromSnap}/{toSnap}",
+			NewAuthHandler(state.NewZFSReceivingServer()),
+		).Methods("POST")
+
+	}
 
 	router.HandleFunc("/status",
 		func(w http.ResponseWriter, r *http.Request) {
 			fmt.Fprintf(w, "OK")
 		},
 	)
 
-	router.Handle(
-		"/filesystems/{filesystem}/{fromSnap}/{toSnap}",
-		middleware.FromHTTPRequest(tracer, "zfs-sender")(
-			NewAuthHandler(state.NewZFSSendingServer()),
-		),
-	).Methods("GET")
-
-	router.Handle(
-		"/filesystems/{filesystem}/{fromSnap}/{toSnap}",
-		middleware.FromHTTPRequest(tracer, "zfs-receiver")(
-			NewAuthHandler(state.NewZFSReceivingServer()),
-		),
-	).Methods("POST")
+	if os.Getenv("PRINT_HTTP_LOGS") != "" {
+		loggingRouter := handlers.LoggingHandler(getLogfile("requests"), router)
+		err = http.ListenAndServe(fmt.Sprintf(":%s", SERVER_PORT), loggingRouter)
+	} else {
+		err = http.ListenAndServe(fmt.Sprintf(":%s", SERVER_PORT), router)
+	}
 
-	loggedRouter := handlers.LoggingHandler(getLogfile("requests"), router)
-	err = http.ListenAndServe(":32607", loggedRouter)
 	if err != nil {
-		out(fmt.Sprintf("Unable to listen on port 32607: '%s'\n", err))
-		log.Fatalf("Unable to listen on port 32607: '%s'", err)
+		out(fmt.Sprintf("Unable to listen on port %s: '%s'\n", SERVER_PORT, err))
+		log.Fatalf("Unable to listen on port %s: '%s'", SERVER_PORT, err)
 	}
 }
 

cmd/dotmesh-server/pkg/main/main.go

@@ -25,6 +25,8 @@ const ZPOOL = "zpool"
 const META_KEY_PREFIX = "io.dotmesh:meta-"
 const ETCD_PREFIX = "/dotmesh.io"
 const CONTAINER_MOUNT_PREFIX = "/var/dotmesh"
+const SERVER_PORT = "32607"
+const SERVER_PORT_OLD = "6969"
 
 var LOG_TO_STDOUT bool
 var POOL string

cmd/dotmesh-server/pkg/main/rpc.go

@@ -1272,7 +1272,7 @@ func (d *DotmeshRPC) AllDotsAndBranches(
 	args *struct{},
 	result *VolumesAndBranches,
 ) error {
-	log.Printf("[AllDotsAndBranches] starting...")
+	quietLogger("[AllDotsAndBranches] starting...")
 
 	vac := VolumesAndBranches{}
 
@@ -1340,7 +1340,7 @@ func (d *DotmeshRPC) AllDotsAndBranches(
 		vac.Dots = append(vac.Dots, tlf)
 	}
 	*result = vac
-	log.Printf("[AllDotsAndBranches] finished!")
+	quietLogger("[AllDotsAndBranches] finished!")
 	return nil
 }