Merge pull request #22 from fuzzitdev/local_job

Add Local Sanity option via docker

Author: yevgenypats

client/auth.go

@@ -9,7 +9,6 @@ import (
 	"io/ioutil"
 	"log"
 	"net/url"
-	"os"
 	"os/user"
 	"path"
 	"time"
@@ -19,33 +18,7 @@ import (
 	"google.golang.org/api/option"
 )
 
-func (c *fuzzitClient) ReAuthenticate(force bool) error {
-	usr, err := user.Current()
-	if err != nil {
-		log.Fatal(err)
-	}
-	cacheFile := path.Join(usr.HomeDir, ".fuzzit.cache")
-
-	if !force {
-		file, err := os.Open(cacheFile)
-		if err != nil {
-			return err
-		}
-
-		err = json.NewDecoder(file).Decode(c)
-		file.Close()
-		if err != nil {
-			// try to prevent being stuck forever if cache file gets corrupted
-			os.Remove(cacheFile)    // if a file
-			os.RemoveAll(cacheFile) // if a directory
-			return err
-		}
-	}
-
-	if c.ApiKey == "" {
-		return errors.New("API Key is no configured, please run fuzzit auth [api_key]")
-	}
-
+func (c *fuzzitClient) refreshToken() error {
 	if c.IdToken == "" || (time.Now().Unix()-c.LastRefresh) > 60*45 {
 		createCustomTokenEndpoint := fmt.Sprintf("%s/createCustomToken?api_key=%s", FuzzitEndpoint, url.QueryEscape(c.ApiKey))
 		r, err := c.httpClient.Get(createCustomTokenEndpoint)
@@ -54,7 +27,7 @@ func (c *fuzzitClient) ReAuthenticate(force bool) error {
 		}
 		defer r.Body.Close()
 		if r.StatusCode != 200 {
-			return errors.New("API Key is not valid")
+			return errors.New("API Key is not valid. Try running fuzzit auth <API_KEY> again")
 		}
 
 		err = json.NewDecoder(r.Body).Decode(c)
@@ -81,6 +54,12 @@ func (c *fuzzitClient) ReAuthenticate(force bool) error {
 		if err != nil {
 			return err
 		}
+
+		usr, err := user.Current()
+		if err != nil {
+			log.Fatal(err)
+		}
+		cacheFile := path.Join(usr.HomeDir, ".fuzzit.cache")
 		err = ioutil.WriteFile(cacheFile, cBytes, 0644)
 		if err != nil {
 			return err
@@ -97,7 +76,6 @@ func (c *fuzzitClient) ReAuthenticate(force bool) error {
 	tokenSource := oauth2.StaticTokenSource(&token)
 	ctx := context.Background()
 
-	// some known issue with go afaik
 	firestoreClient, err := firestore.NewClient(ctx, "fuzzit-b5fbf", option.WithTokenSource(tokenSource))
 	c.firestoreClient = firestoreClient
 

client/client.go

@@ -2,7 +2,12 @@ package client
 
 import (
 	"cloud.google.com/go/firestore"
+	"encoding/json"
+	"log"
 	"net/http"
+	"os"
+	"os/user"
+	"path"
 	"time"
 )
 
@@ -15,6 +20,7 @@ type Target struct {
 type Job struct {
 	TargetId     string `firestore:"target_id"`
 	Args         string `firestore:"args"`
+	Local        bool
 	Type         string `firestore:"type"`
 	Host         string `firestore:"host"`
 	Revision     string `firestore:"revision"`
@@ -49,21 +55,48 @@ type fuzzitClient struct {
 	httpClient      *http.Client
 }
 
-func NewFuzzitClient(apiKey string) *fuzzitClient {
+func NewFuzzitClient(apiKey string) (*fuzzitClient, error) {
 	c := &fuzzitClient{}
 	c.httpClient = &http.Client{Timeout: 60 * time.Second}
 	c.ApiKey = apiKey
-
-	return c
+	err := c.refreshToken()
+	if err != nil {
+		return nil, err
+	}
+	return c, nil
 }
 
 func LoadFuzzitFromCache() (*fuzzitClient, error) {
 	c := &fuzzitClient{}
 	c.httpClient = &http.Client{Timeout: 60 * time.Second}
-	err := c.ReAuthenticate(false)
+
+	usr, err := user.Current()
+	if err != nil {
+		log.Fatal(err)
+	}
+	cacheFile := path.Join(usr.HomeDir, ".fuzzit.cache")
+
+	if _, err := os.Stat(cacheFile); os.IsNotExist(err) {
+		return c, nil
+	}
+
+	file, err := os.Open(cacheFile)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.NewDecoder(file).Decode(c)
+	file.Close()
 	if err != nil {
+		// try to prevent being stuck forever if cache file gets corrupted
+		os.Remove(cacheFile)    // if a file
+		os.RemoveAll(cacheFile) // if a directory
 		return nil, err
 	}
 
+	//if c.ApiKey == "" {
+	//	return errors.New("API Key is not configured (will have access only to public repositories)")
+	//}
+
 	return c, nil
 }

client/commands.go

@@ -1,23 +1,88 @@
 package client
 
 import (
+	"cloud.google.com/go/firestore"
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/google/uuid"
+	"github.com/mholt/archiver"
+	"google.golang.org/api/iterator"
+	"io"
 	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
 	"strings"
-
-	"cloud.google.com/go/firestore"
-	"github.com/google/uuid"
-	"github.com/mholt/archiver"
-	"google.golang.org/api/iterator"
+	//"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	//"github.com/docker/docker/pkg/stdcopy"
 )
 
+func (c *fuzzitClient) archiveFiles(files []string) (string, error) {
+	fuzzerPath := files[0]
+	filename := filepath.Base(fuzzerPath)
+	if !strings.HasSuffix(filename, ".tar.gz") {
+		tmpDir, err := ioutil.TempDir("", "fuzzit")
+		if err != nil {
+			return "", err
+		}
+		dstPath := filepath.Join(tmpDir, "fuzzer")
+		_, err = copyFile(dstPath, fuzzerPath)
+		if err != nil {
+			return "", err
+		}
+
+		prefix, err := uuid.NewRandom()
+		if err != nil {
+			return "", err
+		}
+		filesToArchive := append([]string{dstPath}, files[1:]...)
+
+		tmpfile := filepath.Join(os.TempDir(), prefix.String()+".tar.gz")
+		z := archiver.NewTarGz()
+		err = z.Archive(filesToArchive, tmpfile)
+		if err != nil {
+			return "", err
+		}
+		fuzzerPath = tmpfile
+	}
+
+	return fuzzerPath, nil
+}
+
+func (c *fuzzitClient) DownloadSeed(dst string, target string) error {
+	storagePath := fmt.Sprintf("orgs/%s/targets/%s/seed", c.Org, target)
+	err := c.downloadFile(dst, storagePath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *fuzzitClient) DownloadCorpus(dst string, target string) error {
+	storagePath := fmt.Sprintf("orgs/%s/targets/%s/corpus", c.Org, target)
+	err := c.downloadFile(dst, storagePath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *fuzzitClient) DownloadFuzzer(dst string, target string, job string) error {
+	storagePath := fmt.Sprintf("orgs/%s/targets/%s/jobs/%s/fuzzer", c.Org, target, job)
+	err := c.downloadFile(dst, storagePath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func (c *fuzzitClient) GetResource(resource string) error {
-	err := c.ReAuthenticate(false)
+	err := c.refreshToken()
 	if err != nil {
 		return err
 	}
@@ -74,9 +139,14 @@ func (c *fuzzitClient) GetResource(resource string) error {
 }
 
 func (c *fuzzitClient) CreateTarget(targetName string, seedPath string) (*firestore.DocumentRef, error) {
+	err := c.refreshToken()
+	if err != nil {
+		return nil, err
+	}
+
 	ctx := context.Background()
 	docRef := c.firestoreClient.Doc("orgs/" + c.Org + "/targets/" + targetName)
-	_, err := docRef.Get(ctx)
+	_, err = docRef.Get(ctx)
 	if err == nil {
 		return nil, fmt.Errorf("target %s already exist", targetName)
 	}
@@ -97,9 +167,116 @@ func (c *fuzzitClient) CreateTarget(targetName string, seedPath string) (*firest
 	return docRef, nil
 }
 
-func (c *fuzzitClient) CreateJob(jobConfig Job, files []string) (*firestore.DocumentRef, error) {
+func (c *fuzzitClient) CreateLocalJob(jobConfig Job, files []string) error {
 	ctx := context.Background()
+	cli, err := client.NewClientWithOpts(client.FromEnv)
+	if err != nil {
+		return err
+	}
+	cli.NegotiateAPIVersion(ctx)
 
+	fuzzerPath, err := c.archiveFiles(files)
+	if err != nil {
+		return err
+	}
+
+	fuzzer, err := os.Open(fuzzerPath)
+	if err != nil {
+		return err
+	}
+
+	corpusPath := fmt.Sprintf("orgs/%s/targets/%s/corpus.tar.gz", c.Org, jobConfig.TargetId)
+	log.Print(corpusPath)
+	corpusLink, err := c.getStorageLink(corpusPath, "read")
+	if err != nil {
+		return err
+	}
+
+	seedPath := fmt.Sprintf("orgs/%s/targets/%s/seed", c.Org, jobConfig.TargetId)
+	seedLink, err := c.getStorageLink(seedPath, "read")
+	if err != nil {
+		return err
+	}
+
+	log.Println("Pulling container")
+	reader, err := cli.ImagePull(ctx, "docker.io/fuzzitdev/fuzzit:stretch-llvm9", types.ImagePullOptions{})
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(os.Stdout, reader)
+	if err != nil {
+		return err
+	}
+
+	log.Println("Creating container")
+	createdContainer, err := cli.ContainerCreate(ctx, &container.Config{
+		Env: []string{
+			"CORPUS_LINK=" + corpusLink,
+			"SEED_LINK=" + seedLink,
+			"ASAN_OPTIONS=" + jobConfig.AsanOptions,
+			"UBSAN_OPTIONS=" + jobConfig.UbsanOptions,
+			"ARGS=" + jobConfig.Args},
+		Image:       "docker.io/fuzzitdev/fuzzit:stretch-llvm9",
+		AttachStdin: true,
+	}, nil, nil, "")
+	if err != nil {
+		return err
+	}
+
+	log.Println("Uploading fuzzer to container")
+	err = cli.CopyToContainer(ctx, createdContainer.ID, "/app/", fuzzer, types.CopyToContainerOptions{
+		AllowOverwriteDirWithFile: true,
+	})
+	if err != nil {
+		return err
+	}
+
+	log.Println("Starting the container")
+	err = cli.ContainerStart(ctx, createdContainer.ID, types.ContainerStartOptions{})
+	if err != nil {
+		return err
+	}
+
+	out, err := cli.ContainerLogs(ctx, createdContainer.ID, types.ContainerLogsOptions{
+		ShowStdout: true,
+		ShowStderr: true,
+		Follow:     true,
+	})
+	if err != nil {
+		return err
+	}
+
+	stdcopy.StdCopy(os.Stdout, os.Stderr, out)
+
+	log.Println("Waiting for container")
+	statusCh, errCh := cli.ContainerWait(ctx, createdContainer.ID, container.WaitConditionNotRunning)
+	select {
+	case err := <-errCh:
+		if err != nil {
+			cli.ContainerRemove(ctx, createdContainer.ID, types.ContainerRemoveOptions{})
+			return err
+		}
+	case status := <-statusCh:
+		if status.StatusCode != 0 {
+			cli.ContainerRemove(ctx, createdContainer.ID, types.ContainerRemoveOptions{})
+			return fmt.Errorf("fuzzer exited with %d", status.StatusCode)
+		}
+	}
+
+	err = cli.ContainerRemove(ctx, createdContainer.ID, types.ContainerRemoveOptions{})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *fuzzitClient) CreateJob(jobConfig Job, files []string) (*firestore.DocumentRef, error) {
+	err := c.refreshToken()
+	if err != nil {
+		return nil, err
+	}
+	ctx := context.Background()
 	collectionRef := c.firestoreClient.Collection("orgs/" + c.Org + "/targets/" + jobConfig.TargetId + "/jobs")
 	fullJob := job{}
 	fullJob.Job = jobConfig
@@ -141,7 +318,7 @@ func (c *fuzzitClient) CreateJob(jobConfig Job, files []string) (*firestore.Docu
 
 	storagePath := fmt.Sprintf("orgs/%s/targets/%s/jobs/%s/fuzzer", c.Org, jobConfig.TargetId, jobRef.ID)
 	log.Println("Uploading fuzzer...")
-	err := c.uploadFile(fuzzerPath, storagePath, "application/gzip", "fuzzer.tar.gz")
+	err = c.uploadFile(fuzzerPath, storagePath, "application/gzip", "fuzzer.tar.gz")
 	if err != nil {
 		return nil, err
 	}