Factor out config so client can reuse it (#64)

cmd/client/client.go

@@ -22,6 +22,10 @@ import (
 
 	"github.com/golang/glog"
 	"github.com/transparency-dev/distributor/client"
+	"github.com/transparency-dev/distributor/config"
+	f_log "github.com/transparency-dev/formats/log"
+	"golang.org/x/exp/maps"
+	"golang.org/x/mod/sumdb/note"
 )
 
 var (
@@ -32,18 +36,46 @@ var (
 func main() {
 	flag.Parse()
 
+	ls := getLogsOrDie()
+	ws := getWitnessesOrDie()
+
 	d := client.NewRestDistributor(*baseURL, http.DefaultClient)
 
 	logs, err := d.GetLogs()
 	if err != nil {
 		glog.Exitf("Failed to enumerate logs: %v", err)
 	}
 	for _, l := range logs {
+		log, ok := ls[string(l)]
+		if !ok {
+			glog.Warningf("Saw unknown logID %q from distributor", l)
+			continue
+		}
 		cp, err := d.GetCheckpointN(l, *n)
 		if err != nil {
-			glog.Warningf("Could not get checkpoint.%d for log %s: %v", *n, l, err)
+			glog.Warningf("Could not get checkpoint.%d for log %q (%s): %v", *n, log.Origin, l, err)
+			continue
+		}
+		if _, _, _, err := f_log.ParseCheckpoint(cp, log.Origin, log.Verifier, maps.Values(ws)...); err != nil {
+			glog.Warningf("Failed to open checkpoint: %v", err)
 			continue
 		}
 		fmt.Printf("Checkpoint.%d for log %s:\n\n%s\n\n", *n, l, cp)
 	}
 }
+
+func getLogsOrDie() map[string]config.LogInfo {
+	r, err := config.DefaultLogs()
+	if err != nil {
+		glog.Exitf("Failed to get log config: %v", err)
+	}
+	return r
+}
+
+func getWitnessesOrDie() map[uint32]note.Verifier {
+	r, err := config.DefaultWitnesses()
+	if err != nil {
+		glog.Exitf("Failed to get witness config: %v", err)
+	}
+	return r
+}

cmd/internal/distributor/distributor.go

@@ -25,6 +25,7 @@ import (
 	"sort"
 
 	"github.com/golang/glog"
+	"github.com/transparency-dev/distributor/config"
 	"github.com/transparency-dev/distributor/internal/checkpoints"
 	"github.com/transparency-dev/formats/log"
 	"golang.org/x/mod/sumdb/note"
@@ -35,19 +36,12 @@ import (
 // maxSigs is the maximum number of sigs that can be requested.
 const maxSigs = 100
 
-// LogInfo contains the information that the distributor needs to know about
-// a log, other than its ID.
-type LogInfo struct {
-	Origin   string
-	Verifier note.Verifier
-}
-
 // NewDistributor returns a distributor that will accept checkpoints from
 // the given witnesses, for the given logs, and persist its state in the
 // database provided. Callers must call Init() on the returned distributor.
 // `ws` is a map from witness ID (verifier key name) to the note verifier.
 // `ls` is a map from log ID (github.com/transparency-dev/formats/log.ID) to log info.
-func NewDistributor(ws map[string]note.Verifier, ls map[string]LogInfo, db *sql.DB) (*Distributor, error) {
+func NewDistributor(ws map[string]note.Verifier, ls map[string]config.LogInfo, db *sql.DB) (*Distributor, error) {
 	d := &Distributor{
 		ws: ws,
 		ls: ls,
@@ -59,7 +53,7 @@ func NewDistributor(ws map[string]note.Verifier, ls map[string]LogInfo, db *sql.
 // Distributor persists witnessed checkpoints and allows querying of them.
 type Distributor struct {
 	ws map[string]note.Verifier
-	ls map[string]LogInfo
+	ls map[string]config.LogInfo
 	db *sql.DB
 }
 

cmd/internal/distributor/distributor_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/ory/dockertest/v3"
 	"github.com/transparency-dev/distributor/cmd/internal/distributor"
+	"github.com/transparency-dev/distributor/config"
 	docktest "github.com/transparency-dev/distributor/internal/testonly/docker"
 	"github.com/transparency-dev/formats/log"
 	"golang.org/x/mod/sumdb/note"
@@ -39,14 +40,14 @@ import (
 
 var (
 	logFoo = fakeLog{
-		LogInfo: distributor.LogInfo{
+		LogInfo: config.LogInfo{
 			Origin:   "from foo",
 			Verifier: verifierOrDie("FooLog+3d42aea6+Aby03a35YY+FNI4dfRSvLtq1jQE5UjxIW5CXfK0hiIac"),
 		},
 		signer: signerOrDie("PRIVATE+KEY+FooLog+3d42aea6+AdLOqvyC6Q/86GltHux+trlUT3fRKyCtnc/1VMrmLIdo"),
 	}
 	logBar = fakeLog{
-		LogInfo: distributor.LogInfo{
+		LogInfo: config.LogInfo{
 			Origin:   "from bar",
 			Verifier: verifierOrDie("BarLog+74e9e60a+AQXax81tHt0hpLWhLfnmZ677jAQ7+PLWenJqNrj83CeC"),
 		},
@@ -174,24 +175,24 @@ func TestGetLogs(t *testing.T) {
 	ws := map[string]note.Verifier{}
 	testCases := []struct {
 		desc string
-		logs map[string]distributor.LogInfo
+		logs map[string]config.LogInfo
 		want []string
 	}{
 		{
 			desc: "No logs",
-			logs: map[string]distributor.LogInfo{},
+			logs: map[string]config.LogInfo{},
 			want: []string{},
 		},
 		{
 			desc: "One log",
-			logs: map[string]distributor.LogInfo{
+			logs: map[string]config.LogInfo{
 				"FooLog": logFoo.LogInfo,
 			},
 			want: []string{"FooLog"},
 		},
 		{
 			desc: "Two logs",
-			logs: map[string]distributor.LogInfo{
+			logs: map[string]config.LogInfo{
 				"FooLog": logFoo.LogInfo,
 				"BarLog": logBar.LogInfo,
 			},
@@ -226,7 +227,7 @@ func TestDistributeLogAndWitnessMustMatchCheckpoint(t *testing.T) {
 		"Aardvark": witAardvark.verifier,
 		"Badger":   witBadger.verifier,
 	}
-	ls := map[string]distributor.LogInfo{
+	ls := map[string]config.LogInfo{
 		"FooLog": logFoo.LogInfo,
 		"BarLog": logBar.LogInfo,
 	}
@@ -324,7 +325,7 @@ func TestDistributeEvolution(t *testing.T) {
 		"Aardvark": witAardvark.verifier,
 		"Badger":   witBadger.verifier,
 	}
-	ls := map[string]distributor.LogInfo{
+	ls := map[string]config.LogInfo{
 		"FooLog": logFoo.LogInfo,
 		"BarLog": logBar.LogInfo,
 	}
@@ -433,7 +434,7 @@ func TestGetCheckpointWitness(t *testing.T) {
 		"Aardvark": witAardvark.verifier,
 		"Badger":   witBadger.verifier,
 	}
-	ls := map[string]distributor.LogInfo{
+	ls := map[string]config.LogInfo{
 		"FooLog": logFoo.LogInfo,
 		"BarLog": logBar.LogInfo,
 	}
@@ -503,7 +504,7 @@ func TestFiltersUnknownSignatures(t *testing.T) {
 	ws := map[string]note.Verifier{
 		"Aardvark": witAardvark.verifier,
 	}
-	ls := map[string]distributor.LogInfo{
+	ls := map[string]config.LogInfo{
 		"FooLog": logFoo.LogInfo,
 	}
 
@@ -557,7 +558,7 @@ func TestGetCheckpointN(t *testing.T) {
 		"Badger":    witBadger.verifier,
 		"Chameleon": witChameleon.verifier,
 	}
-	ls := map[string]distributor.LogInfo{
+	ls := map[string]config.LogInfo{
 		"FooLog": logFoo.LogInfo,
 		"BarLog": logBar.LogInfo,
 	}
@@ -711,7 +712,7 @@ func TestGetCheckpointNHistoric(t *testing.T) {
 		"Badger":    witBadger.verifier,
 		"Chameleon": witChameleon.verifier,
 	}
-	ls := map[string]distributor.LogInfo{
+	ls := map[string]config.LogInfo{
 		"FooLog": logFoo.LogInfo,
 	}
 	type witnessAndSize struct {
@@ -850,7 +851,7 @@ func signerOrDie(skey string) note.Signer {
 }
 
 type fakeLog struct {
-	distributor.LogInfo
+	config.LogInfo
 	signer note.Signer
 }
 

cmd/main.go

@@ -30,11 +30,9 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/transparency-dev/distributor/cmd/internal/distributor"
 	ihttp "github.com/transparency-dev/distributor/cmd/internal/http"
-	"github.com/transparency-dev/formats/log"
-	f_note "github.com/transparency-dev/formats/note"
+	"github.com/transparency-dev/distributor/config"
 	"golang.org/x/mod/sumdb/note"
 	"golang.org/x/sync/errgroup"
-	"gopkg.in/yaml.v3"
 
 	_ "embed"
 
@@ -48,14 +46,6 @@ var (
 	mysqlURI    = flag.String("mysql_uri", "", "URI for MySQL DB")
 
 	witnessConfigFile = flag.String("witness_config_file", "", "Path to a file containing the public keys of allowed witnesses")
-
-	// configLogs is the config for the logs this distributor will accept.
-	//go:embed logs.yaml
-	configLogs []byte
-
-	// defaultWitnesses is the witness config that will be used if witness_config_file flag is not provided.
-	//go:embed witnesses.yaml
-	defaultWitnesses []byte
 )
 
 func main() {
@@ -151,80 +141,37 @@ func getCloudSqlOrDie() *sql.DB {
 	return dbPool
 }
 
-func getLogsOrDie() map[string]distributor.LogInfo {
-	logsCfg := logsConfig{}
-	if err := yaml.Unmarshal(configLogs, &logsCfg); err != nil {
+func getLogsOrDie() map[string]config.LogInfo {
+	r, err := config.DefaultLogs()
+	if err != nil {
 		glog.Exitf("Failed to unmarshal log config: %v", err)
 	}
-	ls := make(map[string]distributor.LogInfo, len(logsCfg.Logs))
-	for _, l := range logsCfg.Logs {
-		lSigV, err := f_note.NewVerifier(l.PublicKey)
-		if err != nil {
-			glog.Exitf("Invalid log public key: %v", err)
-		}
-		l.ID = log.ID(l.Origin)
-		ls[l.ID] = distributor.LogInfo{
-			Origin:   l.Origin,
-			Verifier: lSigV,
-		}
-		glog.Infof("Added log %q (%s)", l.Origin, l.ID)
+	for id, l := range r {
+		glog.Infof("Added log %q (%s)", l.Origin, id)
 	}
-	return ls
+	return r
 }
 
 func getWitnessesOrDie() map[string]note.Verifier {
-	var configWitnesses []byte
-	if len(*witnessConfigFile) == 0 {
-		glog.Info("Flag witness_config_file not specified; default witness list will be used")
-		configWitnesses = defaultWitnesses
-	} else {
-		var err error
-		configWitnesses, err = os.ReadFile(*witnessConfigFile)
+	cfg := config.WitnessesYAML
+	if *witnessConfigFile != "" {
+		c, err := os.ReadFile(*witnessConfigFile)
 		if err != nil {
 			glog.Exitf("Failed to read witness_config_file (%q): %v", *witnessConfigFile, err)
 		}
-		glog.Info("Witness list read from %v", *witnessConfigFile)
+		glog.Infof("Witness list read from %v", *witnessConfigFile)
+		cfg = c
+	} else {
+		glog.Info("Flag witness_config_file not specified; default witness list will be used")
 	}
-
-	witCfg := witnessConfig{}
-	if err := yaml.Unmarshal(configWitnesses, &witCfg); err != nil {
+	w, err := config.ParseWitnessesConfig(cfg)
+	if err != nil {
 		glog.Exitf("Failed to unmarshal witness config: %v", err)
 	}
-	ws := make(map[string]note.Verifier, len(witCfg.Witnesses))
-	for _, w := range witCfg.Witnesses {
-		wSigV, err := f_note.NewVerifierForCosignatureV1(w)
-		if err != nil {
-			glog.Exitf("Invalid witness public key: %v", err)
-		}
-		ws[wSigV.Name()] = wSigV
-	}
-
-	return ws
-}
 
-// logsConfig contains all of the metadata for the logs.
-type logsConfig struct {
-	// Log defines the log checkpoints are being distributed for.
-	Logs []logConfig `yaml:"Logs"`
-}
-
-// Log describes a verifiable log in a config file.
-type logConfig struct {
-	// ID is used to refer to the log in directory paths.
-	// This field should not be manually set in configs, instead it will be
-	// derived automatically by logfmt.ID.
-	ID string `yaml:"ID"`
-	// PublicKey used to verify checkpoints from this log.
-	PublicKey string `yaml:"PublicKey"`
-	// Origin is the expected first line of checkpoints from the log.
-	Origin string `yaml:"Origin"`
-	// URL is the URL of the root of the log.
-	// This is optional if direct log communication is not required.
-	URL string `yaml:"URL"`
-}
-
-// witnessConfig contains all of the witness public keys.
-type witnessConfig struct {
-	// Witnesses lists the public keys that will be accepted as witnesses.
-	Witnesses []string `yaml:"Witnesses"`
+	r := make(map[string]note.Verifier, len(w))
+	for _, v := range w {
+		r[v.Name()] = v
+	}
+	return r
 }