Skip to content

Commit

Permalink
new mode "census" and import/export db methdos
Browse files Browse the repository at this point in the history 
A new mode, next to gateway, miner and seed, is introduced: census.
The census mode performs only the census API operation.
This is useful to separate the census API from the rest, since it
is the only one having to preserve a persistent storage.

In addition, a new import/export API endpoints are added to backup
the whole census database.

Finally small tweaks and minor improvements (specially with error checking).

Signed-off-by: p4u <[email protected]>
  • Loading branch information
@p4u
p4u committed Oct 27, 2023
1 parent da422da commit 1e9ca9e
Show file tree
Hide file tree
Showing 17 changed files with 616 additions and 137 deletions.
14 changes: 7 additions & 7 deletions api/accounts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,55 +27,55 @@ const (
)

func (a *API) enableAccountHandlers() error {
if err := a.endpoint.RegisterMethod(
if err := a.Endpoint.RegisterMethod(
"/accounts/{address}",
"GET",
apirest.MethodAccessTypePublic,
a.accountHandler,
); err != nil {
return err
}
if err := a.endpoint.RegisterMethod(
if err := a.Endpoint.RegisterMethod(
"/accounts",
"POST",
apirest.MethodAccessTypePublic,
a.accountSetHandler,
); err != nil {
return err
}
if err := a.endpoint.RegisterMethod(
if err := a.Endpoint.RegisterMethod(
"/accounts/{organizationID}/elections/count",
"GET",
apirest.MethodAccessTypePublic,
a.electionCountHandler,
); err != nil {
return err
}
if err := a.endpoint.RegisterMethod(
if err := a.Endpoint.RegisterMethod(
"/accounts/{organizationID}/elections/status/{status}/page/{page}",
"GET",
apirest.MethodAccessTypePublic,
a.electionListHandler,
); err != nil {
return err
}
if err := a.endpoint.RegisterMethod(
if err := a.Endpoint.RegisterMethod(
"/accounts/{organizationID}/elections/page/{page}",
"GET",
apirest.MethodAccessTypePublic,
a.electionListHandler,
); err != nil {
return err
}
if err := a.endpoint.RegisterMethod(
if err := a.Endpoint.RegisterMethod(
"/accounts/{accountID}/transfers/page/{page}",
"GET",
apirest.MethodAccessTypePublic,
a.tokenTransfersHandler,
); err != nil {
return err
}
if err := a.endpoint.RegisterMethod(
if err := a.Endpoint.RegisterMethod(
"/accounts/{accountID}/fees/page/{page}",
"GET",
apirest.MethodAccessTypePublic,
Expand Down
34 changes: 24 additions & 10 deletions api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,9 @@ type API struct {
PrivateCalls uint64
PublicCalls uint64
BaseRoute string
Endpoint *apirest.API

router *httprouter.HTTProuter
endpoint *apirest.API
indexer *indexer.Indexer
vocapp *vochain.BaseApplication
storage data.Storage
Expand Down Expand Up @@ -95,7 +95,7 @@ func NewAPI(router *httprouter.HTTProuter, baseRoute, dataDir, dbType string) (*
router: router,
}
var err error
api.endpoint, err = apirest.NewAPI(router, baseRoute)
api.Endpoint, err = apirest.NewAPI(router, baseRoute)
if err != nil {
return nil, err
}
Expand All @@ -120,7 +120,7 @@ func (a *API) Attach(vocdoniAPP *vochain.BaseApplication, vocdoniInfo *vochainin

// RouterHandler returns the API router handler which can be used to register new custom endpoints.
func (a *API) RouterHandler() *apirest.API {
return a.endpoint
return a.Endpoint
}

// EnableHandlers enables the list of handlers. Attach must be called before.
Expand All @@ -131,34 +131,48 @@ func (a *API) EnableHandlers(handlers ...string) error {
if a.vocapp == nil || a.indexer == nil {
return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h)
}
a.enableVoteHandlers()
if err := a.enableVoteHandlers(); err != nil {
return err
}
case ElectionHandler:
if a.indexer == nil || a.vocinfo == nil {
return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h)
}
a.enableElectionHandlers()
if err := a.enableElectionHandlers(); err != nil {
return err
}
case ChainHandler:
if a.indexer == nil {
return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h)
}
a.enableChainHandlers()
if err := a.enableChainHandlers(); err != nil {
return err
}
case WalletHandler:
if a.vocapp == nil {
return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h)
}
a.enableWalletHandlers()
if err := a.enableWalletHandlers(); err != nil {
return err
}
case AccountHandler:
if a.vocapp == nil {
return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h)
}
a.enableAccountHandlers()
if err := a.enableAccountHandlers(); err != nil {
return err
}
case CensusHandler:
a.enableCensusHandlers()
if err := a.enableCensusHandlers(); err != nil {
return err
}
if a.censusdb == nil {
return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h)
}
case SIKHandler:
a.enableSIKHandlers()
if err := a.enableSIKHandlers(); err != nil {
return err
}
if a.censusdb == nil {
return fmt.Errorf("%w %s", ErrMissingModulesForHandler, h)
}
Expand Down
129 changes: 123 additions & 6 deletions api/censusdb/censusdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@ import (
"encoding/json"
"errors"
"fmt"
"io"

"github.com/google/uuid"
"go.vocdoni.io/dvote/censustree"
"go.vocdoni.io/dvote/data/compressor"
"go.vocdoni.io/dvote/data/ipfs"
"go.vocdoni.io/dvote/db"
"go.vocdoni.io/dvote/log"
"go.vocdoni.io/dvote/types"
"go.vocdoni.io/proto/build/go/models"
)

Expand Down Expand Up @@ -59,11 +61,13 @@ func (cr *CensusRef) SetTree(tree *censustree.Tree) {
// for import/export operations.
type CensusDump struct {
Type models.Census_Type `json:"type"`
RootHash []byte `json:"rootHash"`
RootHash types.HexBytes `json:"rootHash"`
Data []byte `json:"data"`
// MaxLevels is required to load the census with the original size because
// it could be different according to the election (and census) type.
MaxLevels int `json:"maxLevels"`
MaxLevels int `json:"maxLevels"`
CensusID types.HexBytes `json:"censusID,omitempty"`
Token *uuid.UUID `json:"token,omitempty"`
}

// CensusDB is a safe and persistent database of census trees. It allows
Expand Down Expand Up @@ -131,7 +135,19 @@ func (c *CensusDB) Load(censusID []byte, authToken *uuid.UUID) (*CensusRef, erro
if ref.URI != "" {
ref.tree.Publish()
}
log.Debugf("loaded tree %x of type %s", censusID, models.Census_Type_name[ref.CensusType])
root, err := ref.Tree().Root()
if err != nil {
return nil, err
}
size, err := ref.Tree().Size()
if err != nil {
return nil, err
}
log.Debugw("loaded census tree",
"id", hex.EncodeToString(censusID),
"type", models.Census_Type_name[ref.CensusType],
"size", size,
"root", hex.EncodeToString(root))
return ref, nil
}

Expand Down Expand Up @@ -169,8 +185,8 @@ func BuildExportDump(root, data []byte, typ models.Census_Type, maxLevels int) (
return exportData, nil
}

// ImportAsPublic imports a census from a dump and makes it public.
func (c *CensusDB) ImportAsPublic(data []byte) error {
// ImportTreeAsPublic imports a census from a dump and makes it public.
func (c *CensusDB) ImportTreeAsPublic(data []byte) error {
cdata := CensusDump{}
if err := json.Unmarshal(data, &cdata); err != nil {
return fmt.Errorf("could not unmarshal census: %w", err)
Expand Down Expand Up @@ -236,7 +252,7 @@ func (c *CensusDB) getCensusRefFromDB(censusID []byte) (*CensusRef, error) {
))
if err != nil {
if errors.Is(err, db.ErrKeyNotFound) {
return nil, ErrCensusNotFound
return nil, fmt.Errorf("%w: %x", ErrCensusNotFound, censusID)
}
return nil, err
}
Expand All @@ -249,3 +265,104 @@ func (c *CensusDB) getCensusRefFromDB(censusID []byte) (*CensusRef, error) {
func censusName(censusID []byte) string {
return fmt.Sprintf("%s%x", censusDBprefix, censusID)
}

// ExportCensusDB will create a memory buffer, iterate over all the censuses in the database, load each one,
// create a dump of its data, and finally write this information into a JSON array inside the buffer.
func (c *CensusDB) ExportCensusDB(buffer io.Writer) error {
var censusList []CensusDump

// Iterate through all census entries in the DB
err := c.db.Iterate([]byte(censusDBreferencePrefix), func(key []byte, data []byte) bool {
censusID := make([]byte, len(key))
copy(censusID, key)
dec := gob.NewDecoder(bytes.NewReader(data))
ref := CensusRef{}
if err := dec.Decode(&ref); err != nil {
log.Errorf("error decoding census reference: %s", err)
return true
}
// Load the census tree
var err error
ref.tree, err = censustree.New(censustree.Options{
Name: censusName(censusID),
ParentDB: c.db,
MaxLevels: ref.MaxLevels,
CensusType: models.Census_Type(ref.CensusType),
})
if err != nil {
log.Errorf("error loading census tree: %s", err)
return true
}
// Gather the information needed for the dump.
root, err := ref.Tree().Root()
if err != nil {
log.Errorf("Error getting tree root: %s", err)
return false
}

// Assuming the tree's Dump method returns the data you want to store
treeData, err := ref.Tree().Dump()
if err != nil {
log.Errorf("Error dumping tree data: %s", err)
return false
}

dump := CensusDump{
Type: models.Census_Type(ref.CensusType),
RootHash: root,
Data: treeData,
MaxLevels: ref.MaxLevels,
CensusID: censusID,
Token: ref.AuthToken,
}

censusList = append(censusList, dump)
return true // continue iteration
})
if err != nil {
return fmt.Errorf("error during database iteration: %w", err)
}

// Convert the accumulated censusList to JSON and write it to the buffer
err = json.NewEncoder(buffer).Encode(censusList)
if err != nil {
return fmt.Errorf("error encoding data to JSON: %w", err)
}
return nil
}

// ImportCensusDB takes a buffer with JSON data (as created by DumpCensusDB), decodes it,
// and imports the data back into the database, effectively restoring the censuses.
func (c *CensusDB) ImportCensusDB(buffer io.Reader) error {
var censusList []CensusDump
// Decode the JSON back to the list of census dumps
err := json.NewDecoder(buffer).Decode(&censusList)
if err != nil {
return fmt.Errorf("error decoding JSON data: %w", err)
}

// Iterate through the decoded list and import each census
for _, dump := range censusList {
// Check if the census already exists
if c.Exists(dump.CensusID) {
log.Warnw("census already exists, skiping", "root", dump.RootHash.String(),
"type", dump.Type.String(), "censusID", dump.CensusID.String())
continue
}

// Create a new census reference
ref, err := c.New(dump.CensusID, dump.Type, "", dump.Token, dump.MaxLevels)
if err != nil {
return err
}

// Import the tree data from the dump.
err = ref.Tree().ImportDump(dump.Data)
if err != nil {
return err
}
log.Infow("importing census", "id", dump.CensusID.String(), "token", dump.Token.String(), "size", len(dump.Data))
}

return nil
}
Loading

0 comments on commit 1e9ca9e

Please sign in to comment.