preliminary support for issue: #27

Author: Yevgeny Pats

client/agent.go

@@ -135,6 +135,20 @@ func (c *FuzzitClient) RunFuzzer(job Job, jobId string, updateDB bool) error {
 		if err := c.DownloadAndExtractFuzzer(".", c.currentJob.TargetId, jobId); err != nil {
 			return err
 		}
+
+		log.Println("downloading additional corpus")
+		if err := os.Mkdir("additional-corpus", 0644); err != nil {
+			return err
+		}
+		if err := c.downloadAndExtract(
+			"additional-corpus",
+			fmt.Sprintf("orgs/%s/targets/%s/jobs/%s/corpus.tar.gz", c.Org, c.currentJob.TargetId, c.jobId)); err != nil {
+			if err.Error() == "404 Not Found" {
+				log.Println("no additional-corpus. skipping...")
+			} else {
+				return err
+			}
+		}
 	}
 
 	var err error

client/client.go

@@ -11,7 +11,7 @@ import (
 )
 
 const FuzzitEndpoint = "https://app.fuzzit.dev"
-const Version = "v2.4.69"
+const Version = "v2.4.70"
 
 type Target struct {
 	Name         string `firestore:"target_name"`

client/commands.go

@@ -294,7 +294,7 @@ chmod a+x fuzzit
 	return nil
 }
 
-func (c *FuzzitClient) CreateJob(jobConfig Job, files []string) (*firestore.DocumentRef, error) {
+func (c *FuzzitClient) CreateJob(jobConfig Job, additionalCorpus string, files []string) (*firestore.DocumentRef, error) {
 	err := c.refreshToken()
 	if err != nil {
 		return nil, err
@@ -343,6 +343,17 @@ func (c *FuzzitClient) CreateJob(jobConfig Job, files []string) (*firestore.Docu
 		return nil, err
 	}
 
+	if additionalCorpus != "" {
+		log.Println("Uploading additional corpus...")
+		err = c.uploadFile(
+			additionalCorpus,
+			fmt.Sprintf("orgs/%s/targets/%s/jobs/%s/additional-corpus", c.Org, jobConfig.TargetId, jobRef.ID),
+			filepath.Base(additionalCorpus))
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	log.Println("Starting job")
 	_, err = jobRef.Set(ctx, jobConfig)
 	if err != nil {

client/gofuzz.go

@@ -67,26 +67,6 @@ func (c *FuzzitClient) loadCurrentCrashes() (map[string]bool, error) {
 	return uniqueCrashes, err
 }
 
-// this merges corpus into workdir
-func (c *FuzzitClient) mergeCorpus() error {
-	err := filepath.Walk("corpus", func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			log.Printf("prevent panic by handling failure accessing a path %q: %v\n", path, err)
-			return err
-		}
-		if !info.IsDir() {
-			fileName := info.Name()
-			err = os.Rename("corpus/"+fileName, "workdir/corpus/"+fileName)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-
-	return err
-}
-
 func (c *FuzzitClient) runGoFuzzFuzzing() error {
 	ctx := context.Background()
 
@@ -141,8 +121,10 @@ func (c *FuzzitClient) runGoFuzzFuzzing() error {
 		}
 	}
 
-	err = c.mergeCorpus()
-	if err != nil {
+	if err := mergeDirectories("workdir/corpus", "corpus"); err != nil {
+		log.Fatal(err)
+	}
+	if err := mergeDirectories("workdir/corpus", "additional-corpus"); err != nil {
 		log.Fatal(err)
 	}
 
@@ -267,7 +249,7 @@ func (c *FuzzitClient) runGoFuzz() error {
 	if c.currentJob.Type == "fuzzing" {
 		err = c.runGoFuzzFuzzing()
 	} else {
-		return fmt.Errorf("JQF currently only supports fuzzing")
+		return fmt.Errorf("go-fuzz currently only supports fuzzing")
 	}
 
 	return err

client/utils.go

@@ -171,3 +171,22 @@ func DownloadFile(filepath string, url string) error {
 	_, err = io.Copy(out, resp.Body)
 	return err
 }
+
+func mergeDirectories(dst string, src string) error {
+	err := filepath.Walk(src, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			log.Printf("prevent panic by handling failure accessing a path %q: %v\n", path, err)
+			return err
+		}
+		if !info.IsDir() {
+			fileName := info.Name()
+			err = os.Rename(filepath.Join(src, fileName), filepath.Join(dst, fileName))
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+
+	return err
+}

cmd/job.go

@@ -91,6 +91,11 @@ var jobCmd = &cobra.Command{
 			log.Fatal(err)
 		}
 
+		additionalCorpus, err := cmd.Flags().GetString("additional-corpus")
+		if err != nil {
+			log.Fatal(err)
+		}
+
 		log.Println("Creating job...")
 
 		target := args[0]
@@ -111,7 +116,7 @@ var jobCmd = &cobra.Command{
 				return
 			}
 		} else {
-			_, err = gFuzzitClient.CreateJob(newJob, args[1:])
+			_, err = gFuzzitClient.CreateJob(newJob, additionalCorpus, args[1:])
 		}
 
 		if err != nil {
@@ -138,13 +143,14 @@ func init() {
 	branch := client.GetValueFromEnv("TRAVIS_BRANCH", "CIRCLE_BRANCH", "GITHUB_REF")
 
 	jobCmd.Flags().StringVar(&newJob.Type, "type", "fuzzing", "fuzzing/regression/local-regression")
-	jobCmd.Flags().StringVar(&newJob.Engine, "engine", "libfuzzer", "libfuzzer/jqf")
+	jobCmd.Flags().StringVar(&newJob.Engine, "engine", "libfuzzer", "libfuzzer/jqf/go-fuzz")
 	jobCmd.Flags().StringVar(&newJob.CPUs, "cpus", "1", "number of cpus to use (only relevant for fuzzing job)")
 	jobCmd.Flags().StringVar(&newJob.Memory, "memory", "2048Mi", "number of cpus to use (only relevant for fuzzing job)")
 	jobCmd.Flags().MarkHidden("memory")
 	jobCmd.Flags().MarkHidden("cpus")
-	jobCmd.Flags().StringVar(&newJob.Revision, "revision", revision, "Revision tag of fuzzer (populates automatically from git,travis,circleci)")
-	jobCmd.Flags().StringVar(&newJob.Branch, "branch", branch, "Branch of the fuzzer (populates automatically from git,travis,circleci)")
+	jobCmd.Flags().StringVar(&newJob.Revision, "revision", revision, "revision tag of fuzzer (populates automatically from git,travis,circleci)")
+	jobCmd.Flags().StringVar(&newJob.Branch, "branch", branch, "branch of the fuzzer (populates automatically from git,travis,circleci)")
+	jobCmd.Flags().String("additional-corpus", "", "path to additional corpus for this job (should be a flat zip/tar.gz containing the test cases)")
 	jobCmd.Flags().StringVar(&newJob.Host, "host", "", "docker image to use when running the fuzzer. Options: stretch-llvm8/stretch-llvm9/bionic-swift51")
 	jobCmd.Flags().StringArrayVarP(&newJob.EnvironmentVariables, "environment", "e", nil,
 		"Additional environment variables for the fuzzer. For example ASAN_OPTINOS, UBSAN_OPTIONS or any other")