From 1e9ca9e277430f29f93d72105264a9b0f1311433 Mon Sep 17 00:00:00 2001 From: p4u Date: Fri, 27 Oct 2023 10:53:18 +0200 Subject: [PATCH] new mode "census" and import/export db methdos A new mode, next to gateway, miner and seed, is introduced: census. The census mode performs only the census API operation. This is useful to separate the census API from the rest, since it is the only one having to preserve a persistent storage. In addition, a new import/export API endpoints are added to backup the whole census database. Finally small tweaks and minor improvements (specially with error checking). Signed-off-by: p4u --- api/accounts.go | 14 +- api/api.go | 34 +++-- api/censusdb/censusdb.go | 129 +++++++++++++++++- api/censusdb/censusdb_test.go | 168 ++++++++++++++++++++++++ api/censuses.go | 83 ++++++++++-- api/chain.go | 44 +++---- api/elections.go | 20 +-- api/siks.go | 6 +- api/vote.go | 6 +- api/wallet.go | 8 +- cmd/node/main.go | 182 ++++++++++++++++++++------ config/config.go | 4 + httprouter/apirest/apirest.go | 38 ++++-- service/offchaindata.go | 3 +- types/consts.go | 2 + vochain/offchaindatahandler/census.go | 2 +- vocone/vocone.go | 10 +- 17 files changed, 616 insertions(+), 137 deletions(-) create mode 100644 api/censusdb/censusdb_test.go diff --git a/api/accounts.go b/api/accounts.go index 468b52a1d..b6ce6e269 100644 --- a/api/accounts.go +++ b/api/accounts.go @@ -27,7 +27,7 @@ const ( ) func (a *API) enableAccountHandlers() error { - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/accounts/{address}", "GET", apirest.MethodAccessTypePublic, @@ -35,7 +35,7 @@ func (a *API) enableAccountHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/accounts", "POST", apirest.MethodAccessTypePublic, @@ -43,7 +43,7 @@ func (a *API) enableAccountHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/accounts/{organizationID}/elections/count", "GET", apirest.MethodAccessTypePublic, @@ -51,7 +51,7 @@ func (a *API) enableAccountHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/accounts/{organizationID}/elections/status/{status}/page/{page}", "GET", apirest.MethodAccessTypePublic, @@ -59,7 +59,7 @@ func (a *API) enableAccountHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/accounts/{organizationID}/elections/page/{page}", "GET", apirest.MethodAccessTypePublic, @@ -67,7 +67,7 @@ func (a *API) enableAccountHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/accounts/{accountID}/transfers/page/{page}", "GET", apirest.MethodAccessTypePublic, @@ -75,7 +75,7 @@ func (a *API) enableAccountHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/accounts/{accountID}/fees/page/{page}", "GET", apirest.MethodAccessTypePublic, diff --git a/api/api.go b/api/api.go index 26659f279..82ee5d0cb 100644 --- a/api/api.go +++ b/api/api.go @@ -64,9 +64,9 @@ type API struct { PrivateCalls uint64 PublicCalls uint64 BaseRoute string + Endpoint *apirest.API router *httprouter.HTTProuter - endpoint *apirest.API indexer *indexer.Indexer vocapp *vochain.BaseApplication storage data.Storage @@ -95,7 +95,7 @@ func NewAPI(router *httprouter.HTTProuter, baseRoute, dataDir, dbType string) (* router: router, } var err error - api.endpoint, err = apirest.NewAPI(router, baseRoute) + api.Endpoint, err = apirest.NewAPI(router, baseRoute) if err != nil { return nil, err } @@ -120,7 +120,7 @@ func (a *API) Attach(vocdoniAPP *vochain.BaseApplication, vocdoniInfo *vochainin // RouterHandler returns the API router handler which can be used to register new custom endpoints. func (a *API) RouterHandler() *apirest.API { - return a.endpoint + return a.Endpoint } // EnableHandlers enables the list of handlers. Attach must be called before. @@ -131,34 +131,48 @@ func (a *API) EnableHandlers(handlers ...string) error { if a.vocapp == nil || a.indexer == nil { return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h) } - a.enableVoteHandlers() + if err := a.enableVoteHandlers(); err != nil { + return err + } case ElectionHandler: if a.indexer == nil || a.vocinfo == nil { return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h) } - a.enableElectionHandlers() + if err := a.enableElectionHandlers(); err != nil { + return err + } case ChainHandler: if a.indexer == nil { return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h) } - a.enableChainHandlers() + if err := a.enableChainHandlers(); err != nil { + return err + } case WalletHandler: if a.vocapp == nil { return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h) } - a.enableWalletHandlers() + if err := a.enableWalletHandlers(); err != nil { + return err + } case AccountHandler: if a.vocapp == nil { return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h) } - a.enableAccountHandlers() + if err := a.enableAccountHandlers(); err != nil { + return err + } case CensusHandler: - a.enableCensusHandlers() + if err := a.enableCensusHandlers(); err != nil { + return err + } if a.censusdb == nil { return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h) } case SIKHandler: - a.enableSIKHandlers() + if err := a.enableSIKHandlers(); err != nil { + return err + } if a.censusdb == nil { return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h) } diff --git a/api/censusdb/censusdb.go b/api/censusdb/censusdb.go index 51a13c5f2..639ac8f7a 100644 --- a/api/censusdb/censusdb.go +++ b/api/censusdb/censusdb.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "github.com/google/uuid" "go.vocdoni.io/dvote/censustree" @@ -14,6 +15,7 @@ import ( "go.vocdoni.io/dvote/data/ipfs" "go.vocdoni.io/dvote/db" "go.vocdoni.io/dvote/log" + "go.vocdoni.io/dvote/types" "go.vocdoni.io/proto/build/go/models" ) @@ -59,11 +61,13 @@ func (cr *CensusRef) SetTree(tree *censustree.Tree) { // for import/export operations. type CensusDump struct { Type models.Census_Type `json:"type"` - RootHash []byte `json:"rootHash"` + RootHash types.HexBytes `json:"rootHash"` Data []byte `json:"data"` // MaxLevels is required to load the census with the original size because // it could be different according to the election (and census) type. - MaxLevels int `json:"maxLevels"` + MaxLevels int `json:"maxLevels"` + CensusID types.HexBytes `json:"censusID,omitempty"` + Token *uuid.UUID `json:"token,omitempty"` } // CensusDB is a safe and persistent database of census trees. It allows @@ -131,7 +135,19 @@ func (c *CensusDB) Load(censusID []byte, authToken *uuid.UUID) (*CensusRef, erro if ref.URI != "" { ref.tree.Publish() } - log.Debugf("loaded tree %x of type %s", censusID, models.Census_Type_name[ref.CensusType]) + root, err := ref.Tree().Root() + if err != nil { + return nil, err + } + size, err := ref.Tree().Size() + if err != nil { + return nil, err + } + log.Debugw("loaded census tree", + "id", hex.EncodeToString(censusID), + "type", models.Census_Type_name[ref.CensusType], + "size", size, + "root", hex.EncodeToString(root)) return ref, nil } @@ -169,8 +185,8 @@ func BuildExportDump(root, data []byte, typ models.Census_Type, maxLevels int) ( return exportData, nil } -// ImportAsPublic imports a census from a dump and makes it public. -func (c *CensusDB) ImportAsPublic(data []byte) error { +// ImportTreeAsPublic imports a census from a dump and makes it public. +func (c *CensusDB) ImportTreeAsPublic(data []byte) error { cdata := CensusDump{} if err := json.Unmarshal(data, &cdata); err != nil { return fmt.Errorf("could not unmarshal census: %w", err) @@ -236,7 +252,7 @@ func (c *CensusDB) getCensusRefFromDB(censusID []byte) (*CensusRef, error) { )) if err != nil { if errors.Is(err, db.ErrKeyNotFound) { - return nil, ErrCensusNotFound + return nil, fmt.Errorf("%w: %x", ErrCensusNotFound, censusID) } return nil, err } @@ -249,3 +265,104 @@ func (c *CensusDB) getCensusRefFromDB(censusID []byte) (*CensusRef, error) { func censusName(censusID []byte) string { return fmt.Sprintf("%s%x", censusDBprefix, censusID) } + +// ExportCensusDB will create a memory buffer, iterate over all the censuses in the database, load each one, +// create a dump of its data, and finally write this information into a JSON array inside the buffer. +func (c *CensusDB) ExportCensusDB(buffer io.Writer) error { + var censusList []CensusDump + + // Iterate through all census entries in the DB + err := c.db.Iterate([]byte(censusDBreferencePrefix), func(key []byte, data []byte) bool { + censusID := make([]byte, len(key)) + copy(censusID, key) + dec := gob.NewDecoder(bytes.NewReader(data)) + ref := CensusRef{} + if err := dec.Decode(&ref); err != nil { + log.Errorf("error decoding census reference: %s", err) + return true + } + // Load the census tree + var err error + ref.tree, err = censustree.New(censustree.Options{ + Name: censusName(censusID), + ParentDB: c.db, + MaxLevels: ref.MaxLevels, + CensusType: models.Census_Type(ref.CensusType), + }) + if err != nil { + log.Errorf("error loading census tree: %s", err) + return true + } + // Gather the information needed for the dump. + root, err := ref.Tree().Root() + if err != nil { + log.Errorf("Error getting tree root: %s", err) + return false + } + + // Assuming the tree's Dump method returns the data you want to store + treeData, err := ref.Tree().Dump() + if err != nil { + log.Errorf("Error dumping tree data: %s", err) + return false + } + + dump := CensusDump{ + Type: models.Census_Type(ref.CensusType), + RootHash: root, + Data: treeData, + MaxLevels: ref.MaxLevels, + CensusID: censusID, + Token: ref.AuthToken, + } + + censusList = append(censusList, dump) + return true // continue iteration + }) + if err != nil { + return fmt.Errorf("error during database iteration: %w", err) + } + + // Convert the accumulated censusList to JSON and write it to the buffer + err = json.NewEncoder(buffer).Encode(censusList) + if err != nil { + return fmt.Errorf("error encoding data to JSON: %w", err) + } + return nil +} + +// ImportCensusDB takes a buffer with JSON data (as created by DumpCensusDB), decodes it, +// and imports the data back into the database, effectively restoring the censuses. +func (c *CensusDB) ImportCensusDB(buffer io.Reader) error { + var censusList []CensusDump + // Decode the JSON back to the list of census dumps + err := json.NewDecoder(buffer).Decode(&censusList) + if err != nil { + return fmt.Errorf("error decoding JSON data: %w", err) + } + + // Iterate through the decoded list and import each census + for _, dump := range censusList { + // Check if the census already exists + if c.Exists(dump.CensusID) { + log.Warnw("census already exists, skiping", "root", dump.RootHash.String(), + "type", dump.Type.String(), "censusID", dump.CensusID.String()) + continue + } + + // Create a new census reference + ref, err := c.New(dump.CensusID, dump.Type, "", dump.Token, dump.MaxLevels) + if err != nil { + return err + } + + // Import the tree data from the dump. + err = ref.Tree().ImportDump(dump.Data) + if err != nil { + return err + } + log.Infow("importing census", "id", dump.CensusID.String(), "token", dump.Token.String(), "size", len(dump.Data)) + } + + return nil +} diff --git a/api/censusdb/censusdb_test.go b/api/censusdb/censusdb_test.go new file mode 100644 index 000000000..5a7e99a6a --- /dev/null +++ b/api/censusdb/censusdb_test.go @@ -0,0 +1,168 @@ +package censusdb + +import ( + "bytes" + "testing" + "time" + + qt "github.com/frankban/quicktest" + "github.com/google/uuid" // This is a helper library for cleaner test assertions. + "go.vocdoni.io/dvote/db" + "go.vocdoni.io/dvote/db/metadb" + "go.vocdoni.io/proto/build/go/models" +) + +func newDatabase(t *testing.T) db.Database { + // For the sake of this test, we are creating an in-memory database. + // This should be replaced with your actual database initialization. + return metadb.NewTest(t) +} + +func TestNewCensusDB(t *testing.T) { + db := newDatabase(t) + censusDB := NewCensusDB(db) + qt.Assert(t, censusDB, qt.IsNotNil) + qt.Assert(t, censusDB.db, qt.IsNotNil) +} + +func TestCensusDBNew(t *testing.T) { + censusDB := NewCensusDB(newDatabase(t)) + censusID := []byte("testCensus") + censusType := models.Census_ARBO_BLAKE2B + uri := "testURI" + authToken, _ := uuid.NewRandom() + maxLevels := 32 + + censusRef, err := censusDB.New(censusID, censusType, uri, &authToken, maxLevels) + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, censusRef, qt.IsNotNil) + qt.Assert(t, censusRef.Tree(), qt.IsNotNil) +} + +func TestCensusDBExists(t *testing.T) { + censusDB := NewCensusDB(newDatabase(t)) + censusID := []byte("testCensus") + + existsBefore := censusDB.Exists(censusID) + qt.Assert(t, existsBefore, qt.IsFalse) + + // Create a new census to test existence + _, err := censusDB.New(censusID, models.Census_ARBO_BLAKE2B, "", nil, 32) + qt.Assert(t, err, qt.IsNil) + + existsAfter := censusDB.Exists(censusID) + qt.Assert(t, existsAfter, qt.IsTrue) +} + +func TestCensusDBLoad(t *testing.T) { + censusDB := NewCensusDB(newDatabase(t)) + censusID := []byte("testCensus") + authToken, _ := uuid.NewRandom() + + // Attempting to load a non-existent census should return an error + _, err := censusDB.Load(censusID, &authToken) + qt.Assert(t, err, qt.IsNotNil) + + // Create a new census for loading + _, err = censusDB.New(censusID, models.Census_ARBO_BLAKE2B, "", &authToken, 32) + qt.Assert(t, err, qt.IsNil) + + // Load the census with the correct auth token + censusRef, err := censusDB.Load(censusID, &authToken) + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, censusRef, qt.IsNotNil) +} + +func TestCensusDBDel(t *testing.T) { + censusDB := NewCensusDB(newDatabase(t)) + censusID := []byte("testCensus") + + // Create a new census for deletion + _, err := censusDB.New(censusID, models.Census_ARBO_BLAKE2B, "", nil, 32) + qt.Assert(t, err, qt.IsNil) + + // Delete the census + err = censusDB.Del(censusID) + qt.Assert(t, err, qt.IsNil) + + time.Sleep(1 * time.Second) // Wait because deletion is async + + // Ensure it's deleted + existsAfter := censusDB.Exists(censusID) + qt.Assert(t, existsAfter, qt.IsFalse) +} + +func TestImportCensusDB(t *testing.T) { + censusDB := NewCensusDB(newDatabase(t)) + token1 := uuid.New() + censusID1 := []byte("testCensus1") + token2 := uuid.New() + censusID2 := []byte("testCensus2") + + // Create census2 + censusRef, err := censusDB.New(censusID1, models.Census_ARBO_BLAKE2B, "", &token1, 32) + qt.Assert(t, err, qt.IsNil) + + // Populate the census with mock data + mockData1 := map[string][]byte{ + "key1": []byte("value1"), + "key2": []byte("value2"), + } + + for k, v := range mockData1 { + err = censusRef.Tree().Add([]byte(k), v) + qt.Assert(t, err, qt.IsNil) + } + + // Create census2 + censusRef, err = censusDB.New(censusID2, models.Census_ARBO_BLAKE2B, "", &token2, 32) + qt.Assert(t, err, qt.IsNil) + + // Populate the census with mock data + mockData2 := map[string][]byte{ + "key3": []byte("value3"), + "key4": []byte("value4"), + } + + for k, v := range mockData2 { + err = censusRef.Tree().Add([]byte(k), v) + qt.Assert(t, err, qt.IsNil) + } + + // Ensure the data is in the database + _, err = censusDB.Load(censusID1, &token1) + qt.Assert(t, err, qt.IsNil) + _, err = censusDB.Load(censusID2, &token2) + qt.Assert(t, err, qt.IsNil) + + // Dump the census data to a byte buffer + var buf bytes.Buffer + err = censusDB.ExportCensusDB(&buf) + qt.Assert(t, err, qt.IsNil) + + // Create a new database instance to simulate importing into a fresh database + newDB := NewCensusDB(newDatabase(t)) + + // Import the data from the buffer + err = newDB.ImportCensusDB(&buf) + qt.Assert(t, err, qt.IsNil) + + // Verify the data in the new database + newCensusRef, err := newDB.Load(censusID1, &token1) + qt.Assert(t, err, qt.IsNil) + + for k, v := range mockData1 { + value, err := newCensusRef.Tree().Get([]byte(k)) + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, v, qt.DeepEquals, value) + } + + newCensusRef, err = newDB.Load(censusID2, &token2) + qt.Assert(t, err, qt.IsNil) + + for k, v := range mockData2 { + value, err := newCensusRef.Tree().Get([]byte(k)) + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, v, qt.DeepEquals, value) + } +} diff --git a/api/censuses.go b/api/censuses.go index 30f2ed4a1..24cb053a6 100644 --- a/api/censuses.go +++ b/api/censuses.go @@ -12,6 +12,7 @@ import ( "go.vocdoni.io/dvote/api/censusdb" "go.vocdoni.io/dvote/censustree" "go.vocdoni.io/dvote/crypto/zk" + "go.vocdoni.io/dvote/crypto/zk/circuit" "go.vocdoni.io/dvote/data/compressor" "go.vocdoni.io/dvote/httprouter" "go.vocdoni.io/dvote/httprouter/apirest" @@ -35,7 +36,7 @@ const ( ) func (a *API) enableCensusHandlers() error { - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{type}", "POST", apirest.MethodAccessTypePublic, @@ -43,7 +44,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/participants", "POST", apirest.MethodAccessTypePublic, @@ -51,7 +52,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/type", "GET", apirest.MethodAccessTypePublic, @@ -59,7 +60,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/root", "GET", apirest.MethodAccessTypePublic, @@ -67,7 +68,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/export", "GET", apirest.MethodAccessTypePublic, @@ -75,7 +76,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/import", "POST", apirest.MethodAccessTypePublic, @@ -83,7 +84,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/weight", "GET", apirest.MethodAccessTypePublic, @@ -91,7 +92,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/size", "GET", apirest.MethodAccessTypePublic, @@ -99,7 +100,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/publish", "POST", apirest.MethodAccessTypePublic, @@ -107,7 +108,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/publish/{root}", "POST", apirest.MethodAccessTypePublic, @@ -115,7 +116,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}", "DELETE", apirest.MethodAccessTypePublic, @@ -123,7 +124,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/proof/{key}", "GET", apirest.MethodAccessTypePublic, @@ -131,7 +132,7 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/censuses/{censusID}/verify", "POST", apirest.MethodAccessTypePublic, @@ -139,6 +140,22 @@ func (a *API) enableCensusHandlers() error { ); err != nil { return err } + if err := a.Endpoint.RegisterMethod( + "/censuses/export", + "GET", + apirest.MethodAccessTypeAdmin, + a.censusExportDBHandler, + ); err != nil { + return err + } + if err := a.Endpoint.RegisterMethod( + "/censuses/import", + "POST", + apirest.MethodAccessTypeAdmin, + a.censusImportDBHandler, + ); err != nil { + return err + } return nil } @@ -164,7 +181,12 @@ func (a *API) censusCreateHandler(msg *apirest.APIdata, ctx *httprouter.HTTPCont return ErrCensusTypeUnknown } - maxLevels := a.vocapp.TransactionHandler.ZkCircuit.Config.Levels + // get census max levels from vochain app if available + maxLevels := circuit.CircuitsConfigurations[circuit.DefaultCircuitConfigurationTag].Levels + if a.vocapp != nil { + maxLevels = a.vocapp.TransactionHandler.ZkCircuit.Config.Levels + } + censusID := util.RandomBytes(32) _, err = a.censusdb.New(censusID, censusType, "", &token, maxLevels) if err != nil { @@ -800,3 +822,36 @@ func (a *API) censusVerifyHandler(msg *apirest.APIdata, ctx *httprouter.HTTPCont } return ctx.Send(data, apirest.HTTPstatusOK) } + +// censusExportDBHandler +// +// @Summary Export census database +// @Description Export the whole census database to a JSON file. Requires Admin Bearer token. +// @Tags Censuses +// @Accept json +// @Produce json +// @Success 200 {object} object{valid=bool} +// @Router /censuses/export [get] +func (a *API) censusExportDBHandler(msg *apirest.APIdata, ctx *httprouter.HTTPContext) error { + buf := bytes.Buffer{} + if err := a.censusdb.ExportCensusDB(&buf); err != nil { + return err + } + return ctx.Send(buf.Bytes(), apirest.HTTPstatusOK) +} + +// censusImportHandler +// +// @Summary Import census database +// @Description Import the whole census database from a JSON file. +// @Tags Censuses +// @Accept json +// @Produce json +// @Success 200 {object} object{valid=bool} +// @Router /censuses/import [post] +func (a *API) censusImportDBHandler(msg *apirest.APIdata, ctx *httprouter.HTTPContext) error { + if err := a.censusdb.ImportCensusDB(bytes.NewReader(msg.Data)); err != nil { + return err + } + return ctx.Send(nil, apirest.HTTPstatusOK) +} diff --git a/api/chain.go b/api/chain.go index 06e46e93d..14626bccb 100644 --- a/api/chain.go +++ b/api/chain.go @@ -26,7 +26,7 @@ const ( ) func (a *API) enableChainHandlers() error { - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/organizations/page/{page}", "GET", apirest.MethodAccessTypePublic, @@ -34,7 +34,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/organizations/count", "GET", apirest.MethodAccessTypePublic, @@ -42,7 +42,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/info", "GET", apirest.MethodAccessTypePublic, @@ -50,7 +50,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/info/circuit", "GET", apirest.MethodAccessTypePublic, @@ -58,7 +58,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/info/electionPriceFactors", "GET", apirest.MethodAccessTypePublic, @@ -66,7 +66,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/dateToBlock/{timestamp}", "GET", apirest.MethodAccessTypePublic, @@ -74,7 +74,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/blockToDate/{height}", "GET", apirest.MethodAccessTypePublic, @@ -82,7 +82,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/transactions/cost", "GET", apirest.MethodAccessTypePublic, @@ -90,7 +90,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/transactions/reference/{hash}", "GET", apirest.MethodAccessTypePublic, @@ -98,7 +98,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/transactions/reference/index/{index}", "GET", apirest.MethodAccessTypePublic, @@ -106,7 +106,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/blocks/{height}/transactions/page/{page}", "GET", apirest.MethodAccessTypePublic, @@ -114,7 +114,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/transactions/{height}/{index}", "GET", apirest.MethodAccessTypePublic, @@ -122,7 +122,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/transactions", "POST", apirest.MethodAccessTypePublic, @@ -130,7 +130,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/transactions/page/{page}", "GET", apirest.MethodAccessTypePublic, @@ -138,7 +138,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/validators", "GET", apirest.MethodAccessTypePublic, @@ -146,7 +146,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/blocks/{height}", "GET", apirest.MethodAccessTypePublic, @@ -154,7 +154,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/blocks/hash/{hash}", "GET", apirest.MethodAccessTypePublic, @@ -162,7 +162,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/organizations/filter/page/{page}", "POST", apirest.MethodAccessTypePublic, @@ -170,7 +170,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/transactions/count", "GET", apirest.MethodAccessTypePublic, @@ -178,7 +178,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/fees/page/{page}", "GET", apirest.MethodAccessTypePublic, @@ -186,7 +186,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/fees/reference/{reference}/page/{page}", "GET", apirest.MethodAccessTypePublic, @@ -194,7 +194,7 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/chain/fees/type/{type}/page/{page}", "GET", apirest.MethodAccessTypePublic, diff --git a/api/elections.go b/api/elections.go index 032fa3037..9a4e9226e 100644 --- a/api/elections.go +++ b/api/elections.go @@ -30,7 +30,7 @@ const ( ) func (a *API) enableElectionHandlers() error { - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/elections/page/{page}", "GET", apirest.MethodAccessTypePublic, @@ -38,7 +38,7 @@ func (a *API) enableElectionHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/elections/{electionID}", "GET", apirest.MethodAccessTypePublic, @@ -46,7 +46,7 @@ func (a *API) enableElectionHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/elections/{electionID}/keys", "GET", apirest.MethodAccessTypePublic, @@ -54,7 +54,7 @@ func (a *API) enableElectionHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/elections/{electionID}/votes/count", "GET", apirest.MethodAccessTypePublic, @@ -62,7 +62,7 @@ func (a *API) enableElectionHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/elections/{electionID}/votes/page/{page}", "GET", apirest.MethodAccessTypePublic, @@ -70,7 +70,7 @@ func (a *API) enableElectionHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/elections/{electionID}/scrutiny", "GET", apirest.MethodAccessTypePublic, @@ -78,7 +78,7 @@ func (a *API) enableElectionHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/elections", "POST", apirest.MethodAccessTypePublic, @@ -86,7 +86,7 @@ func (a *API) enableElectionHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/elections/price", "POST", apirest.MethodAccessTypePublic, @@ -94,7 +94,7 @@ func (a *API) enableElectionHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/files/cid", "POST", apirest.MethodAccessTypePublic, @@ -103,7 +103,7 @@ func (a *API) enableElectionHandlers() error { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/elections/filter/page/{page}", "POST", apirest.MethodAccessTypePublic, diff --git a/api/siks.go b/api/siks.go index 55e4c5b1e..6624cc4d8 100644 --- a/api/siks.go +++ b/api/siks.go @@ -17,7 +17,7 @@ const ( ) func (a *API) enableSIKHandlers() error { - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/siks/{address}", "GET", apirest.MethodAccessTypePublic, @@ -25,7 +25,7 @@ func (a *API) enableSIKHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/siks/roots", "GET", apirest.MethodAccessTypePublic, @@ -33,7 +33,7 @@ func (a *API) enableSIKHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/siks/proof/{address}", "GET", apirest.MethodAccessTypePublic, diff --git a/api/vote.go b/api/vote.go index ed03eaebf..5ac6c2a10 100644 --- a/api/vote.go +++ b/api/vote.go @@ -16,7 +16,7 @@ import ( const VoteHandler = "votes" func (a *API) enableVoteHandlers() error { - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/votes", "POST", apirest.MethodAccessTypePublic, @@ -24,7 +24,7 @@ func (a *API) enableVoteHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/votes/{voteID}", "GET", apirest.MethodAccessTypePublic, @@ -32,7 +32,7 @@ func (a *API) enableVoteHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/votes/verify/{electionID}/{voteID}", "GET", apirest.MethodAccessTypePublic, diff --git a/api/wallet.go b/api/wallet.go index 509e07908..bec76a900 100644 --- a/api/wallet.go +++ b/api/wallet.go @@ -30,7 +30,7 @@ const ( ) func (a *API) enableWalletHandlers() error { - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/wallet/add/{privateKey}", "POST", apirest.MethodAccessTypePublic, @@ -38,7 +38,7 @@ func (a *API) enableWalletHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/wallet/bootstrap", "GET", apirest.MethodAccessTypePublic, @@ -46,7 +46,7 @@ func (a *API) enableWalletHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/wallet/transfer/{dstAddress}/{amount}", "GET", apirest.MethodAccessTypePublic, @@ -54,7 +54,7 @@ func (a *API) enableWalletHandlers() error { ); err != nil { return err } - if err := a.endpoint.RegisterMethod( + if err := a.Endpoint.RegisterMethod( "/wallet/election", "POST", apirest.MethodAccessTypePublic, diff --git a/cmd/node/main.go b/cmd/node/main.go index f6aa45be2..24f94f014 100644 --- a/cmd/node/main.go +++ b/cmd/node/main.go @@ -16,14 +16,17 @@ import ( "syscall" "time" + "github.com/google/uuid" flag "github.com/spf13/pflag" "github.com/spf13/viper" urlapi "go.vocdoni.io/dvote/api" + "go.vocdoni.io/dvote/api/censusdb" "go.vocdoni.io/dvote/api/faucet" "go.vocdoni.io/dvote/config" "go.vocdoni.io/dvote/crypto/ethereum" "go.vocdoni.io/dvote/crypto/zk/circuit" "go.vocdoni.io/dvote/db" + "go.vocdoni.io/dvote/db/metadb" "go.vocdoni.io/dvote/httprouter" "go.vocdoni.io/dvote/internal" "go.vocdoni.io/dvote/log" @@ -54,7 +57,6 @@ func deprecatedFlagsFunc(_ *flag.FlagSet, name string) flag.NormalizedName { // newConfig creates a new config object and loads the stored configuration file func newConfig() (*config.Config, config.Error) { - var err error var cfgError config.Error // create base config globalCfg := config.NewConfig() @@ -92,7 +94,7 @@ func newConfig() (*config.Config, config.Error) { globalCfg.SaveConfig = *flag.Bool("saveConfig", false, "overwrite an existing config file with the provided CLI flags") globalCfg.Mode = *flag.StringP("mode", "m", types.ModeGateway, - "global operation mode. Available options: [gateway, miner, seed]") + "global operation mode. Available options: [gateway, miner, seed, census]") globalCfg.SigningKey = *flag.StringP("signingKey", "k", "", "signing private Key as hex string (auto-generated if empty)") @@ -103,6 +105,8 @@ func newConfig() (*config.Config, config.Error) { "API endpoint http port") globalCfg.EnableAPI = *flag.Bool("enableAPI", true, "enable HTTP API endpoints") + globalCfg.AdminToken = *flag.String("adminToken", "", + "bearer token for admin API endpoints (leave empty to autogenerate)") globalCfg.TLS.Domain = *flag.String("tlsDomain", "", "enable TLS-secure domain with LetsEncrypt (listenPort=443 is required)") globalCfg.EnableFaucetWithAmount = *flag.Uint64("enableFaucetWithAmount", 0, @@ -157,13 +161,6 @@ func newConfig() (*config.Config, config.Error) { // parse flags flag.CommandLine.SortFlags = false flag.CommandLine.SetNormalizeFunc(deprecatedFlagsFunc) - flag.Bool("enableRPC", false, "deprecated") - _ = flag.CommandLine.MarkDeprecated("enableRPC", - "JSON-RPC endpoint support was deprecated and removed, this flag is ignored") - flag.String("vochainRPCListen", "", "deprecated") - _ = flag.CommandLine.MarkDeprecated("vochainRPCListen", - "JSON-RPC endpoint support was deprecated and removed, this flag is ignored") - flag.Parse() // setting up viper @@ -206,48 +203,114 @@ func newConfig() (*config.Config, config.Error) { viper.AddConfigPath(globalCfg.DataDir) // binding flags to viper - viper.BindPFlag("mode", flag.Lookup("mode")) - viper.BindPFlag("logLevel", flag.Lookup("logLevel")) - viper.BindPFlag("logErrorFile", flag.Lookup("logErrorFile")) - viper.BindPFlag("logOutput", flag.Lookup("logOutput")) - viper.BindPFlag("saveConfig", flag.Lookup("saveConfig")) - viper.BindPFlag("signingKey", flag.Lookup("signingKey")) - viper.BindPFlag("listenHost", flag.Lookup("listenHost")) - viper.BindPFlag("listenPort", flag.Lookup("listenPort")) - viper.BindPFlag("enableAPI", flag.Lookup("enableAPI")) - viper.BindPFlag("enableFaucetWithAmount", flag.Lookup("enableFaucetWithAmount")) + if err = viper.BindPFlag("mode", flag.Lookup("mode")); err != nil { + log.Fatalf("failed to bind mode flag to viper: %v", err) + } + if err = viper.BindPFlag("logLevel", flag.Lookup("logLevel")); err != nil { + log.Fatalf("failed to bind logLevel flag to viper: %v", err) + } + if err = viper.BindPFlag("logErrorFile", flag.Lookup("logErrorFile")); err != nil { + log.Fatalf("failed to bind logErrorFile flag to viper: %v", err) + } + if err = viper.BindPFlag("logOutput", flag.Lookup("logOutput")); err != nil { + log.Fatalf("failed to bind logOutput flag to viper: %v", err) + } + if err = viper.BindPFlag("saveConfig", flag.Lookup("saveConfig")); err != nil { + log.Fatalf("failed to bind saveConfig flag to viper: %v", err) + } + if err = viper.BindPFlag("signingKey", flag.Lookup("signingKey")); err != nil { + log.Fatalf("failed to bind signingKey flag to viper: %v", err) + } + if err = viper.BindPFlag("listenHost", flag.Lookup("listenHost")); err != nil { + log.Fatalf("failed to bind listenHost flag to viper: %v", err) + } + if err = viper.BindPFlag("listenPort", flag.Lookup("listenPort")); err != nil { + log.Fatalf("failed to bind listenPort flag to viper: %v", err) + } + if err = viper.BindPFlag("enableAPI", flag.Lookup("enableAPI")); err != nil { + log.Fatalf("failed to bind enableAPI flag to viper: %v", err) + } + if err = viper.BindPFlag("adminToken", flag.Lookup("adminToken")); err != nil { + log.Fatalf("failed to bind adminToken flag to viper: %v", err) + } + if err = viper.BindPFlag("enableFaucetWithAmount", flag.Lookup("enableFaucetWithAmount")); err != nil { + log.Fatalf("failed to bind enableFaucetWithAmount flag to viper: %v", err) + } viper.Set("TLS.DirCert", globalCfg.DataDir+"/tls") - viper.BindPFlag("TLS.Domain", flag.Lookup("tlsDomain")) + if err = viper.BindPFlag("TLS.Domain", flag.Lookup("tlsDomain")); err != nil { + log.Fatalf("failed to bind TLS.Domain flag to viper: %v", err) + } // ipfs viper.Set("ipfs.ConfigPath", globalCfg.DataDir+"/ipfs") - viper.BindPFlag("ipfs.ConnectKey", flag.Lookup("ipfsConnectKey")) - viper.BindPFlag("ipfs.ConnectPeers", flag.Lookup("ipfsConnectPeers")) + if err = viper.BindPFlag("ipfs.ConnectKey", flag.Lookup("ipfsConnectKey")); err != nil { + log.Fatalf("failed to bind ipfsConnectKey flag to viper: %v", err) + } + if err = viper.BindPFlag("ipfs.ConnectPeers", flag.Lookup("ipfsConnectPeers")); err != nil { + log.Fatalf("failed to bind ipfsConnectPeers flag to viper: %v", err) + } // vochain viper.Set("vochain.DataDir", globalCfg.DataDir+"/vochain") viper.Set("vochain.Dev", globalCfg.Dev) - viper.BindPFlag("vochain.P2PListen", flag.Lookup("vochainP2PListen")) - viper.BindPFlag("vochain.PublicAddr", flag.Lookup("vochainPublicAddr")) - viper.BindPFlag("vochain.LogLevel", flag.Lookup("vochainLogLevel")) - viper.BindPFlag("vochain.Peers", flag.Lookup("vochainPeers")) - viper.BindPFlag("vochain.Seeds", flag.Lookup("vochainSeeds")) - viper.BindPFlag("vochain.CreateGenesis", flag.Lookup("vochainCreateGenesis")) - viper.BindPFlag("vochain.Genesis", flag.Lookup("vochainGenesis")) - viper.BindPFlag("vochain.MinerKey", flag.Lookup("vochainMinerKey")) - viper.BindPFlag("vochain.NodeKey", flag.Lookup("vochainNodeKey")) - viper.BindPFlag("vochain.NoWaitSync", flag.Lookup("vochainNoWaitSync")) - viper.BindPFlag("vochain.MempoolSize", flag.Lookup("vochainMempoolSize")) - viper.BindPFlag("vochain.MinerTargetBlockTimeSeconds", flag.Lookup("vochainBlockTime")) - viper.BindPFlag("vochain.SkipPreviousOffchainData", flag.Lookup("skipPreviousOffchainData")) + + if err = viper.BindPFlag("vochain.P2PListen", flag.Lookup("vochainP2PListen")); err != nil { + log.Fatalf("failed to bind vochainP2PListen flag to viper: %v", err) + } + if err = viper.BindPFlag("vochain.PublicAddr", flag.Lookup("vochainPublicAddr")); err != nil { + log.Fatalf("failed to bind vochainPublicAddr flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.LogLevel", flag.Lookup("vochainLogLevel")); err != nil { + log.Fatalf("failed to bind vochainLogLevel flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.Peers", flag.Lookup("vochainPeers")); err != nil { + log.Fatalf("failed to bind vochainPeers flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.Seeds", flag.Lookup("vochainSeeds")); err != nil { + log.Fatalf("failed to bind vochainSeeds flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.CreateGenesis", flag.Lookup("vochainCreateGenesis")); err != nil { + log.Fatalf("failed to bind vochainCreateGenesis flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.Genesis", flag.Lookup("vochainGenesis")); err != nil { + log.Fatalf("failed to bind vochainGenesis flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.MinerKey", flag.Lookup("vochainMinerKey")); err != nil { + log.Fatalf("failed to bind vochainMinerKey flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.NodeKey", flag.Lookup("vochainNodeKey")); err != nil { + log.Fatalf("failed to bind vochainNodeKey flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.NoWaitSync", flag.Lookup("vochainNoWaitSync")); err != nil { + log.Fatalf("failed to bind vochainNoWaitSync flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.MempoolSize", flag.Lookup("vochainMempoolSize")); err != nil { + log.Fatalf("failed to bind vochainMempoolSize flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.MinerTargetBlockTimeSeconds", flag.Lookup("vochainBlockTime")); err != nil { + log.Fatalf("failed to bind vochainBlockTime flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.SkipPreviousOffchainData", flag.Lookup("skipPreviousOffchainData")); err != nil { + log.Fatalf("failed to bind skipPreviousOffchainData flag to viper: %v", err) + } viper.Set("vochain.ProcessArchiveDataDir", globalCfg.DataDir+"/archive") - viper.BindPFlag("vochain.ProcessArchive", flag.Lookup("processArchive")) - viper.BindPFlag("vochain.ProcessArchiveKey", flag.Lookup("processArchiveKey")) - viper.BindPFlag("vochain.OffChainDataDownload", flag.Lookup("offChainDataDownload")) + if err := viper.BindPFlag("vochain.ProcessArchive", flag.Lookup("processArchive")); err != nil { + log.Fatalf("failed to bind processArchive flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.ProcessArchiveKey", flag.Lookup("processArchiveKey")); err != nil { + log.Fatalf("failed to bind processArchiveKey flag to viper: %v", err) + } + if err := viper.BindPFlag("vochain.OffChainDataDownload", flag.Lookup("offChainDataDownload")); err != nil { + log.Fatalf("failed to bind offChainDataDownload flag to viper: %v", err) + } // metrics - viper.BindPFlag("metrics.Enabled", flag.Lookup("metricsEnabled")) - viper.BindPFlag("metrics.RefreshInterval", flag.Lookup("metricsRefreshInterval")) + if err := viper.BindPFlag("metrics.Enabled", flag.Lookup("metricsEnabled")); err != nil { + log.Fatalf("failed to bind metricsEnabled flag to viper: %v", err) + } + if err := viper.BindPFlag("metrics.RefreshInterval", flag.Lookup("metricsRefreshInterval")); err != nil { + log.Fatalf("failed to bind metricsRefreshInterval flag to viper: %v", err) + } // check if config file exists _, err = os.Stat(globalCfg.DataDir + "/vocdoni.yml") @@ -277,6 +340,7 @@ func newConfig() (*config.Config, config.Error) { } } } + err = viper.Unmarshal(&globalCfg) if err != nil { cfgError = config.Error{ @@ -304,6 +368,11 @@ func newConfig() (*config.Config, config.Error) { globalCfg.Vochain.MinerKey = globalCfg.SigningKey } + if globalCfg.AdminToken == "" { + globalCfg.AdminToken = uuid.New().String() + fmt.Println("created new admin API token", globalCfg.AdminToken) + } + if globalCfg.SaveConfig { viper.Set("saveConfig", false) if err := viper.WriteConfig(); err != nil { @@ -428,7 +497,7 @@ func main() { } // HTTP(s) router for Gateway or Prometheus metrics - if globalCfg.Mode == types.ModeGateway || globalCfg.Metrics.Enabled { + if globalCfg.Mode == types.ModeGateway || globalCfg.Metrics.Enabled || globalCfg.Mode == types.ModeCensus { // Initialize the HTTP router srv.Router = new(httprouter.HTTProuter) srv.Router.TLSdomain = globalCfg.TLS.Domain @@ -444,7 +513,7 @@ func main() { } // Storage service for Gateway - if globalCfg.Mode == types.ModeGateway { + if globalCfg.Mode == types.ModeGateway || globalCfg.Mode == types.ModeCensus { srv.Storage, err = srv.IPFS(globalCfg.Ipfs) if err != nil { log.Fatal(err) @@ -556,6 +625,7 @@ func main() { srv.Storage, srv.CensusDB, ) + uAPI.Endpoint.SetAdminToken(globalCfg.AdminToken) if err := uAPI.EnableHandlers( urlapi.ElectionHandler, urlapi.VoteHandler, @@ -582,7 +652,33 @@ func main() { } } - log.Info("startup complete") + if globalCfg.Mode == types.ModeCensus { + log.Info("enabling API") + uAPI, err := urlapi.NewAPI(srv.Router, "/v2", globalCfg.DataDir, globalCfg.Vochain.DBType) + if err != nil { + log.Fatal(err) + } + db, err := metadb.New(globalCfg.Vochain.DBType, filepath.Join(globalCfg.DataDir, "censusdb")) + if err != nil { + log.Fatal(err) + } + censusDB := censusdb.NewCensusDB(db) + uAPI.Attach( + nil, + nil, + nil, + srv.Storage, + censusDB, + ) + uAPI.Endpoint.SetAdminToken(globalCfg.AdminToken) + if err := uAPI.EnableHandlers( + urlapi.CensusHandler, + ); err != nil { + log.Fatal(err) + } + } + + log.Infof("startup complete at %s", time.Now().Format(time.RFC850)) // close if interrupt received c := make(chan os.Signal, 1) diff --git a/config/config.go b/config/config.go index 51307e15f..216f61f46 100644 --- a/config/config.go +++ b/config/config.go @@ -41,6 +41,8 @@ type Config struct { } // EnableAPI enables the HTTP API REST service EnableAPI bool + // AdminAPIToken is the token used to authenticate admin API requests + AdminToken string // EnableFaucet enables the faucet API service for the given amounts EnableFaucetWithAmount uint64 } @@ -54,6 +56,8 @@ func (c *Config) ValidMode() bool { case types.ModeSeed: + case types.ModeCensus: + default: return false } diff --git a/httprouter/apirest/apirest.go b/httprouter/apirest/apirest.go index 5010f12ef..6fa28e674 100644 --- a/httprouter/apirest/apirest.go +++ b/httprouter/apirest/apirest.go @@ -2,6 +2,7 @@ package apirest import ( "encoding/json" + "errors" "fmt" "io" "net/http" @@ -24,8 +25,9 @@ const ( // MethodAccessTypeAdmin for admin requests MethodAccessTypeAdmin = "admin" - namespace = "bearerStd" - bearerPrefix = "Bearer " + namespace = "bearerStd" + bearerPrefix = "Bearer " + maxRequestBodyLog = 1024 // maximum request body size to log ) // HTTPstatus* equal http.Status*, simple sugar to avoid importing http everywhere @@ -175,24 +177,39 @@ func (a *API) AuthorizeRequest(data any, accessType httprouter.AuthAccessType) ( } } -// ProcessData is a function for the RouterNamespace interface. +// ProcessData processes the HTTP request and returns structured data. // The body of the http requests and the bearer auth token are readed. func (a *API) ProcessData(req *http.Request) (any, error) { + // Read and handle the request body reqBody, err := io.ReadAll(req.Body) if err != nil { - return nil, fmt.Errorf("HTTP connection closed: (%v)", err) + return nil, fmt.Errorf("error reading request body: %v", err) } + // Log the request body if it exists if len(reqBody) > 0 { - req := string(reqBody) - if len(req) > 1024 { - req = req[:1024] + "..." // truncate + displayReq := string(reqBody) + if len(displayReq) > maxRequestBodyLog { + displayReq = displayReq[:maxRequestBodyLog] + "..." // truncate for display } - log.Debugf("request: %s", req) + // This assumes you have a configured logging method. Replace with your logger instance. + log.Debugf("request: %s", displayReq) } + // Get and validate the Authorization header + token := "" + authHeader := req.Header.Get("Authorization") + if authHeader != "" { + if !strings.HasPrefix(authHeader, bearerPrefix) { + return nil, errors.New("authorization header is not a Bearer token") + } + // Extract the token + token = strings.TrimPrefix(authHeader, bearerPrefix) + } + // Prepare the structured data msg := &APIdata{ Data: reqBody, - AuthToken: strings.TrimPrefix(req.Header.Get("Authorization"), bearerPrefix), + AuthToken: token, } + // If verbose logging is enabled, log the verbose authentication information if a.verboseAuthLog && msg.AuthToken != "" { fmt.Printf("[BearerAPI/%d/%s] %s {%s}\n", time.Now().Unix(), msg.AuthToken, req.URL.RequestURI(), reqBody) } @@ -203,8 +220,7 @@ func (a *API) ProcessData(req *http.Request) (any, error) { // The pattern URL can contain variable names by using braces, such as /send/{name}/hello // The pattern can also contain wildcard at the end of the path, such as /send/{name}/hello/* // The accessType can be of type private, public or admin. -func (a *API) RegisterMethod(pattern, HTTPmethod string, - accessType string, handler APIhandler) error { +func (a *API) RegisterMethod(pattern, HTTPmethod string, accessType string, handler APIhandler) error { if pattern[0] != '/' { panic("pattern must start with /") } diff --git a/service/offchaindata.go b/service/offchaindata.go index f2f60fd37..942a61b88 100644 --- a/service/offchaindata.go +++ b/service/offchaindata.go @@ -6,7 +6,6 @@ import ( "go.vocdoni.io/dvote/api/censusdb" "go.vocdoni.io/dvote/data/downloader" - "go.vocdoni.io/dvote/db" "go.vocdoni.io/dvote/db/metadb" "go.vocdoni.io/dvote/log" "go.vocdoni.io/dvote/vochain/offchaindatahandler" @@ -21,7 +20,7 @@ func (vs *VocdoniService) OffChainDataHandler() error { go vs.DataDownloader.PrintLogInfo(time.Second * 120) } if vs.CensusDB == nil { - db, err := metadb.New(db.TypePebble, filepath.Join(vs.Config.DataDir, "censusdb")) + db, err := metadb.New(vs.Config.DBType, filepath.Join(vs.Config.DataDir, "censusdb")) if err != nil { return err } diff --git a/types/consts.go b/types/consts.go index 0a83dfdf5..b52ab69d7 100644 --- a/types/consts.go +++ b/types/consts.go @@ -24,6 +24,8 @@ const ( ModeSeed = "seed" // ModeGateway starts the vocdoninode as a gateway ModeGateway = "gateway" + // ModeCensus starts the vocdoninode as a census only service + ModeCensus = "census" // ProcessIDsize is the size of a process id ProcessIDsize = 32 diff --git a/vochain/offchaindatahandler/census.go b/vochain/offchaindatahandler/census.go index 73b403142..2d4f405a2 100644 --- a/vochain/offchaindatahandler/census.go +++ b/vochain/offchaindatahandler/census.go @@ -14,7 +14,7 @@ func (d *OffChainDataHandler) importExternalCensus(uri string, data []byte) { log.Warnf("cannot import empty census from %s", uri) return } - if err := d.census.ImportAsPublic(data); err != nil { + if err := d.census.ImportTreeAsPublic(data); err != nil { if !errors.Is(err, censusdb.ErrCensusAlreadyExists) { log.Warnf("cannot import census from %s: %v", uri, err) } diff --git a/vocone/vocone.go b/vocone/vocone.go index 351809b52..4354b2955 100644 --- a/vocone/vocone.go +++ b/vocone/vocone.go @@ -17,6 +17,7 @@ import ( tmcoretypes "github.com/cometbft/cometbft/rpc/core/types" tmtypes "github.com/cometbft/cometbft/types" "github.com/ethereum/go-ethereum/common" + "github.com/google/uuid" "go.vocdoni.io/dvote/api" "go.vocdoni.io/dvote/config" "go.vocdoni.io/dvote/crypto/ethereum" @@ -65,6 +66,7 @@ func NewVocone(dataDir string, keymanager *ethereum.SignKeys, disableIPFS bool, vc := &Vocone{} vc.Config = &config.VochainCfg{} vc.Config.DataDir = dataDir + vc.Config.DBType = db.TypePebble vc.App, err = vochain.NewBaseApplication(db.TypePebble, dataDir) if err != nil { return nil, err @@ -115,7 +117,9 @@ func NewVocone(dataDir string, keymanager *ethereum.SignKeys, disableIPFS bool, } // Create the data downloader and offchain data handler - vc.OffChainDataHandler() + if err := vc.OffChainDataHandler(); err != nil { + return nil, err + } } return vc, err @@ -138,6 +142,10 @@ func (vc *Vocone) EnableAPI(host string, port int, URLpath string) (*api.API, er vc.Storage, vc.CensusDB, ) + adminToken := uuid.New() + log.Warnw("new admin token generated", "token", adminToken.String()) + uAPI.Endpoint.SetAdminToken(adminToken.String()) + return uAPI, uAPI.EnableHandlers( api.ElectionHandler, api.VoteHandler,