Add cayleyimport and cayleyexport CLIs (#934)

cmd/cayleyexport/cayleyexport.go

@@ -0,0 +1,95 @@
+package main
+
+import (
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/cayleygraph/cayley/clog"
+
+	// Load all supported quad formats.
+	"github.com/cayleygraph/quad"
+	_ "github.com/cayleygraph/quad/jsonld"
+	_ "github.com/cayleygraph/quad/nquads"
+
+	"github.com/spf13/cobra"
+)
+
+const defaultFormat = "jsonld"
+
+// NewCmd creates the command
+func NewCmd() *cobra.Command {
+	var quiet bool
+	var uri, formatName, out string
+
+	var cmd = &cobra.Command{
+		Use:   "cayleyexport",
+		Short: "Export data from Cayley. If no file is provided, cayleyexport writes to stdout.",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if quiet {
+				clog.SetV(500)
+			}
+			var format *quad.Format
+			var w io.Writer
+			if formatName != "" {
+				format = quad.FormatByName(formatName)
+			}
+			if out == "" {
+				w = cmd.OutOrStdout()
+			} else {
+				if formatName == "" {
+					format = formatByFileName(out)
+					if format == nil {
+						clog.Warningf("File has unknown extension %v. Defaulting to %v", out, defaultFormat)
+					}
+				}
+				file, err := os.Create(out)
+				if err != nil {
+					return err
+				}
+				w = file
+				defer file.Close()
+			}
+			if format == nil {
+				format = quad.FormatByName(defaultFormat)
+			}
+			req, err := http.NewRequest(http.MethodGet, uri+"/api/v2/read", nil)
+			req.Header.Set("Accept", format.Mime[0])
+			if err != nil {
+				return err
+			}
+			client := &http.Client{}
+			resp, err := client.Do(req)
+			if err != nil {
+				return err
+			}
+			defer resp.Body.Close()
+			_, err = io.Copy(w, resp.Body)
+			if err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVarP(&uri, "uri", "", "http://127.0.0.1:64210", "Cayley URI connection string")
+	cmd.Flags().StringVarP(&formatName, "format", "", "", "format of the provided data (if can not be detected defaults to JSON-LD)")
+	cmd.Flags().StringVarP(&out, "out", "o", "", "output file; if not specified, stdout is used")
+	cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "hide all log output")
+
+	return cmd
+}
+
+func main() {
+	cmd := NewCmd()
+	if err := cmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}
+
+func formatByFileName(fileName string) *quad.Format {
+	ext := filepath.Ext(fileName)
+	return quad.FormatByExt(ext)
+}

cmd/cayleyexport/cayleyexport_test.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/cayleygraph/cayley/graph"
+	"github.com/cayleygraph/cayley/graph/memstore"
+	chttp "github.com/cayleygraph/cayley/internal/http"
+	"github.com/cayleygraph/quad"
+	"github.com/cayleygraph/quad/jsonld"
+	"github.com/phayes/freeport"
+	"github.com/stretchr/testify/require"
+)
+
+var testData = []quad.Quad{
+	{
+		Subject:   quad.IRI("http://example.com/alice"),
+		Predicate: quad.IRI("http://example.com/likes"),
+		Object:    quad.IRI("http://example.com/bob"),
+		Label:     nil,
+	},
+}
+
+func serializeTestData() string {
+	buffer := bytes.NewBuffer(nil)
+	writer := jsonld.NewWriter(buffer)
+	writer.WriteQuads(testData)
+	writer.Close()
+	return buffer.String()
+}
+
+func serve(addr string) {
+	qs := memstore.New(testData...)
+	qw, err := graph.NewQuadWriter("single", qs, graph.Options{})
+	if err != nil {
+		panic(err)
+	}
+	h := &graph.Handle{QuadStore: qs, QuadWriter: qw}
+	chttp.SetupRoutes(h, &chttp.Config{})
+	err = http.ListenAndServe(addr, nil)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func TestCayleyExport(t *testing.T) {
+	port, err := freeport.GetFreePort()
+	require.NoError(t, err)
+	addr := fmt.Sprintf("127.0.0.1:%d", port)
+	uri := fmt.Sprintf("http://%s", addr)
+	go serve(addr)
+	time.Sleep(3)
+	cmd := NewCmd()
+	b := bytes.NewBufferString("")
+	cmd.SetOut(b)
+	cmd.SetArgs([]string{
+		"--uri",
+		uri,
+	})
+	err = cmd.Execute()
+	require.NoError(t, err)
+	data := serializeTestData()
+	require.NotEmpty(t, data)
+	require.Equal(t, data, b.String())
+}

cmd/cayleyimport/cayleyimport.go

@@ -0,0 +1,120 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/cayleygraph/cayley/clog"
+
+	// Load all supported quad formats.
+	"github.com/cayleygraph/quad"
+	_ "github.com/cayleygraph/quad/jsonld"
+	_ "github.com/cayleygraph/quad/nquads"
+
+	"github.com/spf13/cobra"
+)
+
+const defaultFormat = "jsonld"
+
+// NewCmd creates the command
+func NewCmd() *cobra.Command {
+	var quiet bool
+	var uri, formatName string
+
+	var cmd = &cobra.Command{
+		Use:   "cayleyimport <file>",
+		Short: "Import data into Cayley. If no file is provided, cayleyimport reads from stdin.",
+		Args:  cobra.MaximumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if quiet {
+				clog.SetV(500)
+			}
+			var format *quad.Format
+			var reader io.Reader
+			if formatName != "" {
+				format = quad.FormatByName(formatName)
+			}
+			if len(args) == 0 {
+				in := cmd.InOrStdin()
+				if !hasIn(in) {
+					return errors.New("Either provide file to read from or pipe data")
+				}
+				reader = in
+			} else {
+				fileName := args[0]
+				if formatName == "" {
+					format = formatByFileName(fileName)
+					if format == nil {
+						clog.Warningf("File has unknown extension %v. Defaulting to %v", fileName, defaultFormat)
+					}
+				}
+				file, err := os.Open(fileName)
+				if err != nil {
+					return err
+				}
+				defer file.Close()
+				reader = file
+			}
+			if format == nil {
+				format = quad.FormatByName(defaultFormat)
+			}
+			r, err := http.Post(uri+"/api/v2/write", format.Mime[0], reader)
+			if err != nil {
+				return err
+			}
+			defer r.Body.Close()
+			body, err := ioutil.ReadAll(r.Body)
+			if err != nil {
+				return err
+			}
+			if r.StatusCode == http.StatusOK {
+				var response struct {
+					Result string `json:"result"`
+					Count  string `json:"count"`
+					Error  string `json:"error"`
+				}
+				json.Unmarshal(body, &response)
+				if response.Error != "" {
+					return errors.New(response.Error)
+				}
+				if !quiet {
+					fmt.Println(response.Result)
+				}
+			} else if r.StatusCode == http.StatusNotFound {
+				return errors.New("Database instance does not support write")
+			}
+			return nil
+		},
+	}
+
+	cmd.Flags().StringVarP(&uri, "uri", "", "http://127.0.0.1:64210", "Cayley URI connection string")
+	cmd.Flags().StringVarP(&formatName, "format", "", "", "format of the provided data (if can not be detected defaults to JSON-LD)")
+	cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "hide all log output")
+	return cmd
+}
+
+func main() {
+	cmd := NewCmd()
+	if err := cmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}
+
+func hasIn(in io.Reader) bool {
+	if in == os.Stdin {
+		stat, _ := os.Stdin.Stat()
+		return (stat.Mode() & os.ModeCharDevice) == 0
+	}
+	return true
+}
+
+func formatByFileName(fileName string) *quad.Format {
+	ext := filepath.Ext(fileName)
+	return quad.FormatByExt(ext)
+}

cmd/cayleyimport/cayleyimport_test.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"path"
+	"testing"
+	"time"
+
+	"github.com/cayleygraph/cayley/graph"
+	"github.com/cayleygraph/cayley/graph/memstore"
+	chttp "github.com/cayleygraph/cayley/internal/http"
+	"github.com/phayes/freeport"
+	"github.com/stretchr/testify/require"
+)
+
+func serve(addr string) {
+	qs := memstore.New()
+	qw, err := graph.NewQuadWriter("single", qs, graph.Options{})
+	if err != nil {
+		panic(err)
+	}
+	h := &graph.Handle{QuadStore: qs, QuadWriter: qw}
+	chttp.SetupRoutes(h, &chttp.Config{})
+	err = http.ListenAndServe(addr, nil)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func TestCayleyImport(t *testing.T) {
+	port, err := freeport.GetFreePort()
+	require.NoError(t, err)
+	addr := fmt.Sprintf("127.0.0.1:%d", port)
+	uri := fmt.Sprintf("http://%s", addr)
+	go serve(addr)
+	time.Sleep(3)
+	cmd := NewCmd()
+	b := bytes.NewBufferString("")
+	cmd.SetOut(b)
+	fileName := path.Join("..", "..", "data", "people.jsonld")
+	cmd.SetArgs([]string{
+		fileName,
+		"--uri",
+		uri,
+	})
+	err = cmd.Execute()
+	require.NoError(t, err)
+	require.Empty(t, b.String())
+}

docs/cayleyexport.md

@@ -0,0 +1,41 @@
+# `cayleyexport`
+
+```
+cayleyexport <file>
+```
+
+## Synopsis
+
+The `cayleyexport` tool exports content from a Cayley deployment.
+
+See the [`cayleyimport`](cayleyimport.md) document for more information regarding [`cayleyimport`](cayleyimport.md), which provides the inverse “importing” capability.
+
+Run `cayleyexport` from the system command line, not the Cayley shell.
+
+## Arguments
+
+## Options
+
+### `--help`
+
+Returns information on the options and use of **cayleyexport**.
+
+### `--quiet`
+
+Runs **cayleyexport** in a quiet mode that attempts to limit the amount of output.
+
+### `--uri=<connectionString>`
+
+Specify a resolvable URI connection string (enclose in quotes) to connect to the Cayley deployment.
+
+```
+--uri "http://host[:port]"
+```
+
+### `--format=<format>`
+
+Format to use for the exported data (if can not be detected defaults to JSON-LD)
+
+### `--out=<filename>`
+
+Specifies the location and name of a file to export the data to. If you do not specify a file, **cayleyexport** writes data to the standard output (e.g. “stdout”).