[cloud] Implement port forwarding using mutagen (#461)

## Summary

Implement port forwarding using mutagen. 

This implements the following:

* `devbox cloud port-forward <port> | :<port> | <port1>:<port2>`
* `devbox cloud port-forward list`
* `devbox cloud port-forward terminate` 

Improvements:

* port forward fails if port is in use
* Automatically find a port if user uses ":<port>" argument
* Better output after forwarding

## How was it tested?

* Started cloud vm, installed and started apache
* Started a few port forwards
* Went to browser and confirmed it was working.
* Tested `list` command, `terminate` command and `list` again.

Author: mikeland73

internal/boxcli/cloud.go

@@ -53,27 +53,74 @@ func cloudShellCmd() *cobra.Command {
 
 func cloudPortForwardCmd() *cobra.Command {
 	command := &cobra.Command{
-		Use:    "port-forward <local-port>:<remote-port>",
-		Short:  "Port forwards a local port to a remote devbox cloud port",
+		Use:   "port-forward <local-port>:<remote-port> | <port> | :<remote-port> | terminate",
+		Short: "Port forwards a local port to a remote devbox cloud port",
+		Long: "Port forwards a local port to a remote devbox cloud port. If a " +
+			"single port is specified, it is used for local and remote. If no local" +
+			" port is specified, we find a suitable local port. Use 'terminate' to " +
+			"terminate all port forwards.",
 		Hidden: true,
 		Args:   cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ports := strings.Split(args[0], ":")
+			ports := []string{}
+			if strings.ContainsRune(args[0], ':') {
+				ports = strings.Split(args[0], ":")
+			} else {
+				ports = append(ports, args[0], args[0])
+			}
+
 			if len(ports) != 2 {
 				return usererr.New("Invalid port format. Expected <local-port>:<remote-port>")
 			}
-			err := cloud.PortForward(ports[0], ports[1])
+			localPort, err := cloud.PortForward(ports[0], ports[1])
 			if err != nil {
 				return errors.WithStack(err)
 			}
-			cmd.PrintErrf("Port forwarding %s:%s\n", ports[0], ports[1])
+			cmd.PrintErrf(
+				"Port forwarding %s:%s\nTo view in browser, visit http://localhost:%[1]s\n",
+				localPort,
+				ports[1],
+			)
 			return nil
 		},
 	}
-
+	command.AddCommand(cloudPortForwardList())
+	command.AddCommand(cloudPortForwardTerminateAllCmd())
 	return command
 }
 
+func cloudPortForwardTerminateAllCmd() *cobra.Command {
+	return &cobra.Command{
+		Use:    "terminate",
+		Short:  "Terminates all port forwards managed by devbox",
+		Hidden: true,
+		Args:   cobra.ExactArgs(0),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return cloud.PortForwardTerminateAll()
+		},
+	}
+}
+
+func cloudPortForwardList() *cobra.Command {
+	return &cobra.Command{
+		Use:     "list",
+		Aliases: []string{"ls"},
+		Short:   "Lists all port forwards managed by devbox",
+		Hidden:  true,
+		Args:    cobra.ExactArgs(0),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			l, err := cloud.PortForwardList()
+			if err != nil {
+				return errors.WithStack(err)
+			}
+			for _, p := range l {
+				cmd.Println(p)
+			}
+			return nil
+		},
+	}
+}
+
 func runCloudShellCmd(cmd *cobra.Command, flags *cloudShellCmdFlags) error {
 	box, err := devbox.Open(flags.config.path, cmd.ErrOrStderr())
 	if err != nil {

internal/cloud/cloud.go

@@ -103,13 +103,20 @@ func Shell(w io.Writer, projectDir string, githubUsername string) error {
 	return shell(username, vmHostname, projectDir)
 }
 
-func PortForward(local, remote string) error {
+func PortForward(local, remote string) (string, error) {
 	vmHostname := vmHostnameFromSSHControlPath()
 	if vmHostname == "" {
-		return usererr.New("No VM found. Please run `devbox cloud shell` first.")
+		return "", usererr.New("No VM found. Please run `devbox cloud shell` first.")
 	}
-	portMap := fmt.Sprintf("%s:%s:%s", local, vmHostname, remote)
-	return exec.Command("ssh", "-N", vmHostname, "-L", portMap).Run()
+	return mutagenbox.ForwardCreate(vmHostname, local, remote)
+}
+
+func PortForwardTerminateAll() error {
+	return mutagenbox.ForwardTerminateAll()
+}
+
+func PortForwardList() ([]string, error) {
+	return mutagenbox.ForwardList()
 }
 
 func getGithubUsername() string {

internal/cloud/mutagen/forward.go

@@ -0,0 +1,47 @@
+package mutagen
+
+import (
+	"encoding/json"
+
+	"github.com/pkg/errors"
+)
+
+type Forward struct {
+	Source struct {
+		Connected bool   `json:"connected"`
+		Endpoint  string `json:"endpoint"`
+	} `json:"source"`
+	Destination struct {
+		Endpoint string `json:"endpoint"`
+	} `json:"destination"`
+	LastError string `json:"lastError"`
+}
+
+// ForwardCreate creates a new port forward using mutagen.
+// local looks like tcp:127.0.0.1:<port>
+// remote looks like <host>:<ssh-port>:tcp::<port> (ssh-port is usually 22)
+func ForwardCreate(env map[string]string, local, remote string, labels map[string]string) error {
+	args := []string{"forward", "create", local, remote}
+	return execMutagenEnv(append(args, labelFlag(labels)...), env)
+}
+
+func ForwardTerminate(env map[string]string, labels map[string]string) error {
+	args := []string{"forward", "terminate"}
+	return execMutagenEnv(append(args, labelSelectorFlag(labels)...), env)
+}
+
+func ForwardList(env map[string]string, labels map[string]string) ([]Forward, error) {
+	args := []string{"forward", "list", "--template", "{{json .}}"}
+	out, err := execMutagenOut(append(args, labelSelectorFlag(labels)...), env)
+	if err != nil {
+		return nil, err
+	}
+
+	list := []Forward{}
+
+	if err := json.Unmarshal(out, &list); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return list, errors.WithStack(json.Unmarshal(out, &list))
+}

internal/cloud/mutagen/wrapper.go

@@ -1,13 +1,15 @@
 package mutagen
 
 import (
+	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"strings"
 
+	"github.com/pkg/errors"
+
 	"go.jetpack.io/devbox/internal/debug"
 )
 
@@ -55,7 +57,7 @@ func Create(spec *SessionSpec) error {
 		}
 	}
 
-	return execMutagen(args, spec.EnvVars)
+	return execMutagenEnv(args, spec.EnvVars)
 }
 
 func List(envVars map[string]string, names ...string) ([]Session, error) {
@@ -92,25 +94,25 @@ func List(envVars map[string]string, names ...string) ([]Session, error) {
 func Pause(names ...string) error {
 	args := []string{"sync", "pause"}
 	args = append(args, names...)
-	return execMutagen(args, nil /*envVars*/)
+	return execMutagen(args)
 }
 
 func Resume(envVars map[string]string, names ...string) error {
 	args := []string{"sync", "resume"}
 	args = append(args, names...)
-	return execMutagen(args, envVars)
+	return execMutagenEnv(args, envVars)
 }
 
 func Flush(names ...string) error {
 	args := []string{"sync", "flush"}
 	args = append(args, names...)
-	return execMutagen(args, nil /*envVars*/)
+	return execMutagen(args)
 }
 
 func Reset(envVars map[string]string, names ...string) error {
 	args := []string{"sync", "reset"}
 	args = append(args, names...)
-	return execMutagen(args, envVars)
+	return execMutagenEnv(args, envVars)
 }
 
 func Terminate(env map[string]string, labels map[string]string, names ...string) error {
@@ -121,27 +123,45 @@ func Terminate(env map[string]string, labels map[string]string, names ...string)
 	}
 
 	args = append(args, names...)
-	return execMutagen(args, env)
+	return execMutagenEnv(args, env)
+}
+
+func execMutagen(args []string) error {
+	return execMutagenEnv(args, nil)
+}
+
+func execMutagenEnv(args []string, envVars map[string]string) error {
+	_, err := execMutagenOut(args, envVars)
+	return err
 }
 
-func execMutagen(args []string, envVars map[string]string) error {
+func execMutagenOut(args []string, envVars map[string]string) ([]byte, error) {
 	binPath := ensureMutagen()
 	cmd := exec.Command(binPath, args...)
 	cmd.Env = envAsKeyValueStrings(envVars)
 
+	var stdout bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
 	debugPrintExecCmd(cmd)
-	out, err := cmd.CombinedOutput()
 
-	if err != nil {
-		debug.Log("execMutagen error: %s, out: %s", err, string(out))
+	if err := cmd.Run(); err != nil {
+		debug.Log(
+			"execMutagen error: %s, stdout: %s, stderr: %s",
+			err,
+			stdout.String(),
+			stderr.String(),
+		)
 		if e := (&exec.ExitError{}); errors.As(err, &e) {
-			return errors.New(strings.TrimSpace(string(out)))
+			return nil, errors.New(strings.TrimSpace(stderr.String()))
 		}
-		return err
+		return nil, err
 	}
 
 	debug.Log("execMutagen worked for cmd: %s", cmd)
-	return nil
+	return stdout.Bytes(), nil
 }
 
 // debugPrintExecCmd prints the command to be run, along with MUTAGEN env-vars
@@ -201,3 +221,25 @@ func ensureMutagen() string {
 	}
 	return installPath
 }
+
+func labelFlag(labels map[string]string) []string {
+	if len(labels) == 0 {
+		return []string{}
+	}
+	labelSlice := []string{}
+	for k, v := range labels {
+		labelSlice = append(labelSlice, fmt.Sprintf("%s=%s", k, v))
+	}
+	return []string{"--label", strings.Join(labelSlice, ",")}
+}
+
+func labelSelectorFlag(labels map[string]string) []string {
+	if len(labels) == 0 {
+		return []string{}
+	}
+	labelSlice := []string{}
+	for k, v := range labels {
+		labelSlice = append(labelSlice, fmt.Sprintf("%s=%s", k, v))
+	}
+	return []string{"--label-selector", strings.Join(labelSlice, ",")}
+}

internal/cloud/mutagenbox/forward.go

@@ -0,0 +1,93 @@
+package mutagenbox
+
+import (
+	"fmt"
+	"net"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/samber/lo"
+	"go.jetpack.io/devbox/internal/boxcli/usererr"
+	"go.jetpack.io/devbox/internal/cloud/mutagen"
+)
+
+func ForwardCreate(host, localPort, remotePort string) (string, error) {
+	var err error
+	if localPort == "" {
+		localPort, err = getFreePort()
+		if err != nil {
+			return "", err
+		}
+	}
+
+	if !isPortAvailable(localPort) {
+		return "", usererr.New("Port %s is not available", localPort)
+	}
+
+	local := "tcp:127.0.0.1:" + localPort
+	remote := host + ":22:tcp::" + remotePort
+	labels := map[string]string{"devbox": "true"}
+	env, err := DefaultEnv()
+	if err != nil {
+		return "", err
+	}
+	return localPort, mutagen.ForwardCreate(env, local, remote, labels)
+}
+
+func ForwardTerminateAll() error {
+	env, err := DefaultEnv()
+	if err != nil {
+		return err
+	}
+	return mutagen.ForwardTerminate(env, map[string]string{"devbox": "true"})
+}
+
+func ForwardList() ([]string, error) {
+	env, err := DefaultEnv()
+	if err != nil {
+		return nil, err
+	}
+	forwards, err := mutagen.ForwardList(env, map[string]string{"devbox": "true"})
+	if err != nil {
+		return nil, err
+	}
+
+	result := []string{}
+	for _, item := range forwards {
+		srcParts := strings.Split(item.Source.Endpoint, ":")
+		destParts := strings.Split(item.Destination.Endpoint, ":")
+		result = append(result, fmt.Sprintf(
+			"%s:%s connected: %t %s",
+			srcParts[len(srcParts)-1],
+			destParts[len(destParts)-1],
+			item.Source.Connected,
+			lo.Ternary(item.LastError != "", "Error: "+item.LastError, ""),
+		))
+	}
+
+	return result, nil
+
+}
+
+func isPortAvailable(port string) bool {
+	ln, err := net.Listen("tcp", net.JoinHostPort("localhost", port))
+	if err != nil {
+		return false
+	}
+	_ = ln.Close()
+	return true
+}
+
+func getFreePort() (string, error) {
+	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	l, err := net.ListenTCP("tcp", addr)
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+	defer l.Close()
+	return fmt.Sprintf("%d", l.Addr().(*net.TCPAddr).Port), nil
+}