diff --git a/README.md b/README.md index 569a43d..57e081d 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ source allow it) and retention policy. All the documentation is in the [doc/](doc/) directory. You should start by the [tutorial](doc/tutorial.md) and then jump to advanced topics ([file format](doc/file-format.md), [custom sources](doc/custom-sources.md), -TODO: custom destinations) or the documentation specific to each source -or destination. +[custom destinations](doc/custom-destinations.md)) or the documentation +specific to each source or destination. ## Supported Sources @@ -50,7 +50,7 @@ storage `uback` is in a preliminary stage, quite lacking feature-wide, but fairly stable with a good test suite. Here is a rough sketch of the roadmap : -### 0.1 (released) +### 0.1 * [x] Core features: * [x] Backups & Incremental Backups @@ -63,14 +63,16 @@ stable with a good test suite. Here is a rough sketch of the roadmap : * [x] Documentation * [x] CI/Release Management -### 0.2 (next) +### 0.2 (released) * [x] Custom sources -* [ ] Custom destinations +* [x] Custom destinations -### 0.3 +### 0.3 (next) -* [ ] Proxy support +* [ ] switch to [age](https://age-encryption.org/) for encryption. This +will be the first (and hopefully the last) breaking change for the file +format and keys format. ### 0.4 @@ -79,6 +81,10 @@ stable with a good test suite. Here is a rough sketch of the roadmap : ### 0.5 +* [ ] Proxy support + +### 0.6 + * [ ] remove mariabackup footguns * [ ] add the option to use a dockerized mariabackup in the restoration process to have an exect version match diff --git a/destinations/command.go b/destinations/command.go new file mode 100644 index 0000000..e6f094c --- /dev/null +++ b/destinations/command.go @@ -0,0 +1,143 @@ +package destinations + +import ( + "github.com/sloonz/uback/lib" + + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + + "github.com/gobuffalo/flect" + "github.com/sirupsen/logrus" +) + +var ( + ErrCommandMissing = errors.New("command destination: missing command") + commandLog = logrus.WithFields(logrus.Fields{ + "destination": "fs", + }) +) + +type commandDestination struct { + options *uback.Options + command string + env []string +} + +func newCommandDestination(options *uback.Options) (uback.Destination, error) { + command := options.String["Command"] + if command == "" { + return nil, ErrCommandMissing + } + + env := os.Environ() + for k, v := range options.String { + env = append(env, fmt.Sprintf("UBACK_OPT_%s=%s", flect.New(k).Underscore().ToUpper().String(), v)) + } + for k, v := range options.StrSlice { + jsonVal, err := json.Marshal(v) + if err != nil { + return nil, err + } + env = append(env, fmt.Sprintf("UBACK_SOPT_%s=%s", flect.New(k).Underscore().ToUpper().String(), string(jsonVal))) + } + + buf := bytes.NewBuffer(nil) + cmd := exec.Command(command, "destination", "validate-options") + cmd.Stdout = buf + cmd.Stderr = os.Stderr + cmd.Env = env + err := cmd.Run() + if err != nil { + return nil, err + } + + return &commandDestination{options: options, command: command, env: env}, nil +} + +func (d *commandDestination) ListBackups() ([]uback.Backup, error) { + var res []uback.Backup + + buf := bytes.NewBuffer(nil) + cmd := exec.Command(d.command, "destination", "list-backups") + cmd.Stdout = buf + cmd.Stderr = os.Stderr + cmd.Env = d.env + err := cmd.Run() + if err != nil { + return nil, err + } + + for { + entry, err := buf.ReadString('\n') + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + + if strings.HasPrefix(entry, ".") || strings.HasPrefix(entry, "_") { + continue + } + + backup, err := uback.ParseBackupFilename(entry, false) + if err != nil { + commandLog.WithFields(logrus.Fields{ + "entry": entry, + }) + logrus.Warnf("invalid backup file: %v", err) + continue + } + + res = append(res, backup) + } + + return res, nil +} + +func (d *commandDestination) RemoveBackup(backup uback.Backup) error { + cmd := exec.Command(d.command, "destination", "remove-backup", backup.FullName()) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + cmd.Env = d.env + return cmd.Run() +} + +func (d *commandDestination) SendBackup(backup uback.Backup, data io.Reader) error { + cmd := exec.Command(d.command, "destination", "send-backup", backup.FullName()) + cmd.Stdin = data + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + cmd.Env = d.env + return cmd.Run() +} + +func (d *commandDestination) ReceiveBackup(backup uback.Backup) (io.ReadCloser, error) { + pr, pw := io.Pipe() + cmd := exec.Command(d.command, "destination", "receive-backup", backup.FullName()) + cmd.Stdout = pw + cmd.Stderr = os.Stderr + cmd.Env = d.env + + commandLog.Printf("running: %v", cmd.String()) + err := cmd.Start() + if err != nil { + return nil, err + } + + go func() { + pw.CloseWithError(cmd.Wait()) + }() + + return pr, nil +} diff --git a/destinations/new.go b/destinations/new.go index 940213e..a221323 100644 --- a/destinations/new.go +++ b/destinations/new.go @@ -12,6 +12,8 @@ func New(options *uback.Options) (uback.Destination, error) { return newFSDestination(options) case "object-storage": return newObjectStorageDestination(options) + case "command": + return newCommandDestination(options) default: return nil, fmt.Errorf("invalid destination type %v", options.String["Type"]) } diff --git a/doc/custom-destinations.md b/doc/custom-destinations.md new file mode 100644 index 0000000..c8cf665 --- /dev/null +++ b/doc/custom-destinations.md @@ -0,0 +1,85 @@ +# Custom Destinations + +Custom destinations are destintations that are not build-in into `uback`, +but implemented by an external command, typicall a script, under the +control of the user. + +## Custom Destinations for Users + +When creating a backup, you give `command` as the destination type, +and set the `Command` option to the custom source command. It can be +either a full path or relative path if the custom destination command +in in the PATH. + +```shell +$ uback backup $SOURCE_OPTIONS id=test,type=custom,command=uback-fs-dest +``` + +## Custom Destinations for Implementors + +The first argument passed to the command is the kind of command: `source` +for a source, and `destination` for a destination. In the context of +this document, the first argument will always be `destination`. + +`uback` passes the requested operation and arguments as the next arguments +on the command line, and options passed to the destination as environment +variable. For example, the `SnapshotsPath` will be translated into the +`UBACK_OPT_SNAPSHOTS_PATH` environment variable. If the option is a +slice option (for example `@AdditionalArguments`), it will be passed into +`UBACK_SOPT_ADDITIONAL_ARGUMENTS`, and the value will be a JSON-serialized +string array (the `S` in `SOPT` stands for "slice"). + +A custom destination command should follow the usual external process +conventions : use stdout for normal output that will be consumed by +`uback`, use stdin for normal input that will be given by `uback`, +use stderr to print free-format messages destined to the end user. An +exit code of 0 indicates a successful operation, whereas a non-zero one +indicates a failure. + +A destination must implement those operations, which will be described in +the next sections : + +* validate-options +* list-backups +* remove-backup +* send-backup +* receive-backup + +## Operations + +### validate-options + +This operation takes no argument ; it is called before any other operation +and should be used to validate options. + +### list-backups + +This operation takes no argument, and should print all backups currently +present on the destination, one per line. You can either +give a backup name (`20210102T000000.000-full`) or a filename +(`20210102T000000.000-full.ubkp`). + +For convenience purposes, lines starting with a `.` or `_` are ignored. + +### remove-backup + +This operation takes one argument, the full +name of a backup (`20210102T000000.000-full` or +`20210102T000000.000-from-20210101T000000.000`), and should remove the +backup on the destination. + +### send-backup + +This operation takes one argument, the full name of a backup. `uback` +will provide the backup to the command standard input ; the command +should store it. + +### receive-backup + +This operation takes one argument, the full name of a backup. The command +should output the backup on its standard output. + +## Example + +As an example, you can look at the [uback-fs-test](../tests/uback-fs-test) +test script, which reimplement the fs destination as a bash script. diff --git a/doc/custom-sources.md b/doc/custom-sources.md index 9acc94e..704bae8 100644 --- a/doc/custom-sources.md +++ b/doc/custom-sources.md @@ -6,9 +6,9 @@ of the user. ## Custom Sources for Users -When creating a backup, you give `command` as the source type, and set the -`@SourceCommand` option to the custom source command. It can be either -a full path or relative path if the custom source command in in the PATH. +When creating a backup, you give `command` as the source type, and set +the `Command` option to the custom source command. It can be either a +full path or relative path if the custom source command in in the PATH. ```shell $ uback backup type=command,@source-command=uback-tar-src,path=/etc,snapshots-path=/var/lib/uback/custom-tar-snapshots/ $DEST_OPTIONS diff --git a/tests/dest-command.bats b/tests/dest-command.bats new file mode 100644 index 0000000..bfbd343 --- /dev/null +++ b/tests/dest-command.bats @@ -0,0 +1,42 @@ +load "helpers/bats-support/load" +load "helpers/bats-assert/load" + +UBACK="$BATS_TEST_DIRNAME/../uback" +TEST_TMPDIR="$BATS_RUN_TMPDIR/dest-command" + +@test "command destination" { + export PATH="$PATH:$BATS_TEST_DIRNAME" + mkdir -p "$TEST_TMPDIR/backups" + $UBACK key gen "$TEST_TMPDIR/backup.key" "$TEST_TMPDIR/backup.pub" + source=type=tar,path="$TEST_TMPDIR/source",key-file="$TEST_TMPDIR/backup.pub",state-file="$TEST_TMPDIR/state.json",snapshots-path="$TEST_TMPDIR/snapshots",full-interval=weekly + dest=id=test,type=command,command=uback-fs-dest,path="$TEST_TMPDIR/backups",@retention-policy=daily=3,key-file="$TEST_TMPDIR/backup.key" + + mkdir -p "$TEST_TMPDIR/restore" + mkdir -p "$TEST_TMPDIR/source" + echo "hello" > "$TEST_TMPDIR/source/a" + + # Full 1 + assert_equal "$($UBACK list backups "$dest" | wc -l)" 0 + $UBACK backup -n -f "$source" "$dest" + assert_equal "$($UBACK list backups "$dest" | wc -l)" 1 + sleep 0.01 + + # Full 2 + $UBACK backup -n -f "$source" "$dest" + assert_equal "$($UBACK list backups "$dest" | wc -l)" 2 + sleep 0.01 + + # Incremental + echo "world" > "$TEST_TMPDIR/source/b" + $UBACK backup -n "$source" "$dest" + assert_equal "$($UBACK list backups "$dest" | wc -l)" 3 + + # Prune (remove full 1) + $UBACK prune backups "$dest" + assert_equal "$($UBACK list backups "$dest" | wc -l)" 2 + + # Restore full 2 + incremental + $UBACK restore -d "$TEST_TMPDIR/restore" "$dest" + assert_equal "$(cat "$TEST_TMPDIR"/restore/*/a)" "hello" + assert_equal "$(cat "$TEST_TMPDIR"/restore/*/b)" "world" +} diff --git a/tests/uback-fs-dest b/tests/uback-fs-dest new file mode 100755 index 0000000..097e66a --- /dev/null +++ b/tests/uback-fs-dest @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e + +if [ "$1" != "destination" ] ; then + echo "Invalid kind" >&2 + exit 1 +fi + +case "$2" in + validate-options) + if [ "$UBACK_OPT_PATH" = "" ] ; then + echo "Missing option: Path" >&2 + exit 1 + fi + mkdir -p -- "$UBACK_OPT_PATH" + ;; + list-backups) + ls -- "$UBACK_OPT_PATH" + ;; + remove-backup) + rm -f -- "$UBACK_OPT_PATH/$3.ubkp" + ;; + send-backup) + cat > "$UBACK_OPT_PATH/_tmp-$3.ubkp" + mv "$UBACK_OPT_PATH/_tmp-$3.ubkp" "$UBACK_OPT_PATH/$3.ubkp" + ;; + receive-backup) + cat "$UBACK_OPT_PATH/$3.ubkp" + ;; + *) + echo "Invalid operation: $2" >&2 + exit 1 +esac + +exit 0