From 8bee1f97888988beccf019b5f077f079a8ef3b41 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 8 Jul 2024 14:13:19 +0200 Subject: [PATCH 01/11] api: fetch blocks and transactions from indexer rather than app BlockStore these endpoints now fetch blocks from indexer, and include txCount * /chain/blocks * /chain/blocks/{height} * /chain/blocks/hash/{hash} this endpoint now fetches the full tx from indexer * /chain/transactions/{height}/{index} refactor: * api: rename chainBlockHandler -> chainBlockByHeightHandler --- api/api_types.go | 7 +-- api/chain.go | 115 ++++++++++++++++++++--------------------------- test/api_test.go | 35 +++++++++++++++ 3 files changed, 88 insertions(+), 69 deletions(-) diff --git a/api/api_types.go b/api/api_types.go index b1169db8d..bb6f06573 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -292,9 +292,9 @@ type TransfersList struct { } type GenericTransactionWithInfo struct { - TxContent json.RawMessage `json:"tx"` - TxInfo indexertypes.Transaction `json:"txInfo"` - Signature types.HexBytes `json:"signature"` + TxContent json.RawMessage `json:"tx"` + TxInfo *indexertypes.Transaction `json:"txInfo"` + Signature types.HexBytes `json:"signature"` } type ChainInfo struct { @@ -446,6 +446,7 @@ func CensusTypeToOrigin(ctype CensusTypeDescription) (models.CensusOrigin, []byt type Block struct { comettypes.Block `json:",inline"` Hash types.HexBytes `json:"hash" ` + TxCount int64 `json:"txCount"` } // BlockList is used to return a paginated list to the client diff --git a/api/chain.go b/api/chain.go index 47b86fbaa..927611194 100644 --- a/api/chain.go +++ b/api/chain.go @@ -13,12 +13,9 @@ import ( "go.vocdoni.io/dvote/crypto/zk/circuit" "go.vocdoni.io/dvote/httprouter" "go.vocdoni.io/dvote/httprouter/apirest" - "go.vocdoni.io/dvote/types" "go.vocdoni.io/dvote/util" - "go.vocdoni.io/dvote/vochain" "go.vocdoni.io/dvote/vochain/genesis" "go.vocdoni.io/dvote/vochain/indexer" - "go.vocdoni.io/dvote/vochain/indexer/indexertypes" "go.vocdoni.io/dvote/vochain/state" ) @@ -167,7 +164,7 @@ func (a *API) enableChainHandlers() error { "/chain/blocks/{height}", "GET", apirest.MethodAccessTypePublic, - a.chainBlockHandler, + a.chainBlockByHeightHandler, ); err != nil { return err } @@ -690,14 +687,6 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er if err != nil { return err } - stx, err := a.vocapp.GetTx(uint32(height), int32(index)) - if err != nil { - if errors.Is(err, vochain.ErrTransactionNotFound) { - return ErrTransactionNotFound - } - return ErrVochainGetTxFailed.WithErr(err) - } - ref, err := a.indexer.GetTransactionByHeightAndIndex(height, index) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { @@ -706,9 +695,9 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ErrVochainGetTxFailed.WithErr(err) } tx := &GenericTransactionWithInfo{ - TxContent: []byte(protoFormat(stx.Tx)), - Signature: stx.Signature, - TxInfo: *ref, + TxContent: []byte(protoFormat(ref.RawTx)), + TxInfo: ref, + Signature: ref.Signature, } data, err := json.Marshal(tx) if err != nil { @@ -911,7 +900,7 @@ func (a *API) chainValidatorsHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCon return ctx.Send(data, apirest.HTTPstatusOK) } -// chainBlockHandler +// chainBlockByHeightHandler // // @Summary Get block (by height) // @Description Returns the full block information at the given height @@ -921,23 +910,36 @@ func (a *API) chainValidatorsHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCon // @Param height path int true "Block height" // @Success 200 {object} api.Block // @Router /chain/blocks/{height} [get] -func (a *API) chainBlockHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { +func (a *API) chainBlockByHeightHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { height, err := strconv.ParseUint(ctx.URLParam(ParamHeight), 10, 64) if err != nil { return err } - tmblock := a.vocapp.GetBlockByHeight(int64(height)) - if tmblock == nil { - return ErrBlockNotFound + idxblock, err := a.indexer.BlockByHeight(int64(height)) + if err != nil { + if errors.Is(err, indexer.ErrBlockNotFound) { + return ErrBlockNotFound + } + return ErrBlockNotFound.WithErr(err) + } + txcount, err := a.indexer.CountTransactionsByHeight(int64(height)) + if err != nil { + return ErrIndexerQueryFailed.WithErr(err) } block := &Block{ Block: comettypes.Block{ - Header: tmblock.Header, - Data: tmblock.Data, - Evidence: tmblock.Evidence, - LastCommit: tmblock.LastCommit, + Header: comettypes.Header{ + ChainID: idxblock.ChainID, + Height: idxblock.Height, + Time: idxblock.Time, + ProposerAddress: []byte(idxblock.ProposerAddress), + LastBlockID: comettypes.BlockID{ + Hash: []byte(idxblock.LastBlockHash), + }, + }, }, - Hash: types.HexBytes(tmblock.Hash()), + Hash: idxblock.Hash, + TxCount: txcount, } data, err := json.Marshal(block) if err != nil { @@ -961,18 +963,31 @@ func (a *API) chainBlockByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo if err != nil { return err } - tmblock := a.vocapp.GetBlockByHash(hash) - if tmblock == nil { - return ErrBlockNotFound + idxblock, err := a.indexer.BlockByHash(hash) + if err != nil { + if errors.Is(err, indexer.ErrBlockNotFound) { + return ErrBlockNotFound + } + return ErrBlockNotFound.WithErr(err) + } + txcount, err := a.indexer.CountTransactionsByHeight(idxblock.Height) + if err != nil { + return ErrIndexerQueryFailed.WithErr(err) } block := &Block{ Block: comettypes.Block{ - Header: tmblock.Header, - Data: tmblock.Data, - Evidence: tmblock.Evidence, - LastCommit: tmblock.LastCommit, + Header: comettypes.Header{ + ChainID: idxblock.ChainID, + Height: idxblock.Height, + Time: idxblock.Time, + ProposerAddress: []byte(idxblock.ProposerAddress), + LastBlockID: comettypes.BlockID{ + Hash: []byte(idxblock.LastBlockHash), + }, + }, }, - Hash: types.HexBytes(tmblock.Hash()), + Hash: idxblock.Hash, + TxCount: txcount, } data, err := json.Marshal(block) if err != nil { @@ -1015,39 +1030,7 @@ func (a *API) chainBlockListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCont // // Errors returned are always of type APIerror. func (a *API) sendBlockList(ctx *httprouter.HTTPContext, params *BlockParams) error { - // TODO: replace this by a.indexer.BlockList when it's available - blockList := func(limit, offset int, _, _, _ string) ([]*indexertypes.Block, uint64, error) { - if offset < 0 { - return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) - } - if limit <= 0 { - return nil, 0, fmt.Errorf("invalid value: limit cannot be %d", limit) - } - height := a.vocapp.Height() - total := uint64(height) - uint64(a.vocapp.Node.BlockStore().Base()) - start := height - uint32(params.Page*params.Limit) - end := start - uint32(params.Limit) - list := []*indexertypes.Block{} - for h := start; h > end; h-- { - tmblock := a.vocapp.GetBlockByHeight(int64(h)) - if tmblock == nil { - break - } - list = append(list, &indexertypes.Block{ - ChainID: tmblock.ChainID, - Height: tmblock.Height, - Time: tmblock.Time, - Hash: types.HexBytes(tmblock.Hash()), - ProposerAddress: tmblock.ProposerAddress.Bytes(), - LastBlockHash: tmblock.LastBlockID.Hash.Bytes(), - TxCount: int64(len(tmblock.Txs)), - }) - } - - return list, uint64(total), nil - } - - blocks, total, err := blockList( + blocks, total, err := a.indexer.BlockList( params.Limit, params.Page*params.Limit, params.ChainID, diff --git a/test/api_test.go b/test/api_test.go index 80e4f3699..267895636 100644 --- a/test/api_test.go +++ b/test/api_test.go @@ -461,6 +461,41 @@ func TestAPIAccountTokentxs(t *testing.T) { qt.Assert(t, gotAcct1.Balance, qt.Equals, initBalance+amountAcc2toAcct1-amountAcc1toAcct2-uint64(txBasePrice)) } +func TestAPIBlocks(t *testing.T) { + server := testcommon.APIserver{} + server.Start(t, + api.ChainHandler, + api.CensusHandler, + api.VoteHandler, + api.AccountHandler, + api.ElectionHandler, + api.WalletHandler, + ) + token1 := uuid.New() + c := testutil.NewTestHTTPclient(t, server.ListenAddr, &token1) + + // Block 1 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, c, 1) + + // create a new account + initBalance := uint64(80) + _ = createAccount(t, c, server, initBalance) + + // Block 2 + server.VochainAPP.AdvanceTestBlock() + waitUntilHeight(t, c, 2) + + // check the txCount + resp, code := c.Request("GET", nil, "chain", "blocks", "1") + qt.Assert(t, code, qt.Equals, 200, qt.Commentf("response: %s", resp)) + + block := api.Block{} + err := json.Unmarshal(resp, &block) + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, block.TxCount, qt.Equals, int64(1)) +} + func runAPIElectionCostWithParams(t *testing.T, electionParams electionprice.ElectionParameters, startBlock uint32, initialBalance, From 8651ab3a26634f2a36d3c00fc4fda9e0a64d147c Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Wed, 21 Aug 2024 10:52:15 +0200 Subject: [PATCH 02/11] api: return just comettypes.Header in /chain/blocks endpoints --- api/api_types.go | 6 +++--- api/chain.go | 32 ++++++++++++++------------------ 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/api/api_types.go b/api/api_types.go index bb6f06573..36294b293 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -444,9 +444,9 @@ func CensusTypeToOrigin(ctype CensusTypeDescription) (models.CensusOrigin, []byt } type Block struct { - comettypes.Block `json:",inline"` - Hash types.HexBytes `json:"hash" ` - TxCount int64 `json:"txCount"` + comettypes.Header `json:"header"` + Hash types.HexBytes `json:"hash" ` + TxCount int64 `json:"txCount"` } // BlockList is used to return a paginated list to the client diff --git a/api/chain.go b/api/chain.go index 927611194..d61ae9ad3 100644 --- a/api/chain.go +++ b/api/chain.go @@ -927,15 +927,13 @@ func (a *API) chainBlockByHeightHandler(_ *apirest.APIdata, ctx *httprouter.HTTP return ErrIndexerQueryFailed.WithErr(err) } block := &Block{ - Block: comettypes.Block{ - Header: comettypes.Header{ - ChainID: idxblock.ChainID, - Height: idxblock.Height, - Time: idxblock.Time, - ProposerAddress: []byte(idxblock.ProposerAddress), - LastBlockID: comettypes.BlockID{ - Hash: []byte(idxblock.LastBlockHash), - }, + Header: comettypes.Header{ + ChainID: idxblock.ChainID, + Height: idxblock.Height, + Time: idxblock.Time, + ProposerAddress: []byte(idxblock.ProposerAddress), + LastBlockID: comettypes.BlockID{ + Hash: []byte(idxblock.LastBlockHash), }, }, Hash: idxblock.Hash, @@ -975,15 +973,13 @@ func (a *API) chainBlockByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo return ErrIndexerQueryFailed.WithErr(err) } block := &Block{ - Block: comettypes.Block{ - Header: comettypes.Header{ - ChainID: idxblock.ChainID, - Height: idxblock.Height, - Time: idxblock.Time, - ProposerAddress: []byte(idxblock.ProposerAddress), - LastBlockID: comettypes.BlockID{ - Hash: []byte(idxblock.LastBlockHash), - }, + Header: comettypes.Header{ + ChainID: idxblock.ChainID, + Height: idxblock.Height, + Time: idxblock.Time, + ProposerAddress: []byte(idxblock.ProposerAddress), + LastBlockID: comettypes.BlockID{ + Hash: []byte(idxblock.LastBlockHash), }, }, Hash: idxblock.Hash, From 4ec03dab47a47fc8b3fdb0cc9a498f50a4c895b6 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Wed, 21 Aug 2024 10:21:49 +0200 Subject: [PATCH 03/11] indexer: add ReindexBlocks and make indexerdb.CreateBlock an UPSERT --- vochain/indexer/db/blocks.sql.go | 6 ++++++ vochain/indexer/indexer.go | 25 +++++++++++++++++++++++++ vochain/indexer/queries/blocks.sql | 8 +++++++- 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index 01e6ba54a..dcb19bf6c 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -17,6 +17,12 @@ INSERT INTO blocks( ) VALUES ( ?, ?, ?, ?, ?, ? ) +ON CONFLICT(height) DO UPDATE +SET chain_id = sqlc.arg(chain_id), + time = sqlc.arg(time), + hash = sqlc.arg(hash), + proposer_address = sqlc.arg(proposer_address), + last_block_hash = sqlc.arg(last_block_hash) ` type CreateBlockParams struct { diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index d66ce1f07..21821c10b 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -402,6 +402,31 @@ func (idx *Indexer) AfterSyncBootstrap(inTest bool) { log.Infof("live results recovery computation finished, took %s", time.Since(startTime)) } +func (idx *Indexer) ReindexBlocks() { + queries := idx.blockTxQueries() + + for i := idx.App.Node.BlockStore().Base(); i <= idx.App.Node.BlockStore().Height(); i++ { + if b := idx.App.GetBlockByHeight(int64(i)); b != nil { + idxBlock, err := idx.readOnlyQuery.GetBlockByHeight(context.TODO(), i) + if err == nil && idxBlock.Time != b.Time { + log.Errorf("while reindexing blocks, block %d timestamp in db (%s) differs from blockstore (%s), leaving untouched", i, idxBlock.Time, b.Time) + continue + } + // if we got here, the block doesn't exist + if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + ChainID: b.ChainID, + Height: b.Height, + Time: b.Time, + Hash: nonNullBytes(b.Hash()), + ProposerAddress: nonNullBytes(b.ProposerAddress), + LastBlockHash: nonNullBytes(b.LastBlockID.Hash), + }); err != nil { + log.Errorw(err, "cannot index new block") + } + } + } +} + // Commit is called by the APP when a block is confirmed and included into the chain func (idx *Indexer) Commit(height uint32) error { idx.blockMu.Lock() diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index 38380967b..f52ef3b4b 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -3,7 +3,13 @@ INSERT INTO blocks( chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( ?, ?, ?, ?, ?, ? -); +) +ON CONFLICT(height) DO UPDATE +SET chain_id = sqlc.arg(chain_id), + time = sqlc.arg(time), + hash = sqlc.arg(hash), + proposer_address = sqlc.arg(proposer_address), + last_block_hash = sqlc.arg(last_block_hash); -- name: GetBlockByHeight :one SELECT * FROM blocks From 0ecd3fa5e20203c8d301b9870bbabee02294334e Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Wed, 21 Aug 2024 12:04:26 +0200 Subject: [PATCH 04/11] indexer: drop concept of transaction id, hash is now the primary key * api: remove endpoint /chain/transactions/reference/index/{index} --- api/chain.go | 38 ------------ vochain/indexer/bench_test.go | 4 +- vochain/indexer/db/blocks.sql.go | 10 ++-- vochain/indexer/db/db.go | 10 ---- vochain/indexer/db/models.go | 1 - vochain/indexer/db/transactions.sql.go | 35 ++--------- vochain/indexer/indexer_test.go | 5 +- vochain/indexer/indexertypes/types.go | 3 - .../0015_recreate_table_transactions.sql | 60 +++++++++++++++++++ vochain/indexer/queries/blocks.sql | 10 ++-- vochain/indexer/queries/transactions.sql | 7 +-- vochain/indexer/transaction.go | 13 ---- 12 files changed, 79 insertions(+), 117 deletions(-) create mode 100644 vochain/indexer/migrations/0015_recreate_table_transactions.sql diff --git a/api/chain.go b/api/chain.go index d61ae9ad3..f8247c6a2 100644 --- a/api/chain.go +++ b/api/chain.go @@ -104,14 +104,6 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } - if err := a.Endpoint.RegisterMethod( - "/chain/transactions/reference/index/{index}", - "GET", - apirest.MethodAccessTypePublic, - a.chainTxRefByIndexHandler, - ); err != nil { - return err - } if err := a.Endpoint.RegisterMethod( "/chain/blocks/{height}/transactions/page/{page}", "GET", @@ -706,36 +698,6 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ctx.Send(data, apirest.HTTPstatusOK) } -// chainTxRefByIndexHandler -// -// @Summary Transaction metadata (by db index) -// @Description Get transaction by its internal index. This is not the transaction hash, and neither the block height and block index. The transaction index is an incremental counter for each transaction. You could use the transaction `block` and `index` to retrieve full info using [transaction by block and index](transaction-by-block-index). -// @Tags Chain -// @Accept json -// @Produce json -// @Param index path int true "Index of the transaction" -// @Success 200 {object} indexertypes.Transaction -// @Success 204 "See [errors](vocdoni-api#errors) section" -// @Router /chain/transactions/reference/index/{index} [get] -func (a *API) chainTxRefByIndexHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { - index, err := strconv.ParseUint(ctx.URLParam("index"), 10, 64) - if err != nil { - return err - } - ref, err := a.indexer.GetTxMetadataByID(index) - if err != nil { - if errors.Is(err, indexer.ErrTransactionNotFound) { - return ErrTransactionNotFound - } - return ErrVochainGetTxFailed.WithErr(err) - } - data, err := json.Marshal(ref) - if err != nil { - return err - } - return ctx.Send(data, apirest.HTTPstatusOK) -} - // chainTxListHandler // // @Summary List transactions diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index f73645bd9..7e289857a 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -152,10 +152,10 @@ func BenchmarkFetchTx(b *testing.B) { startTime := time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTxMetadataByID(uint64((i * numTxs) + j + 1)) + _, err = idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) qt.Assert(b, err, qt.IsNil) } - log.Infof("fetched %d transactions (out of %d total) by index, took %s", + log.Infof("fetched %d transactions (out of %d total) by height+index, took %s", numTxs, (i+1)*numTxs, time.Since(startTime)) startTime = time.Now() for j := 0; j < numTxs; j++ { diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index dcb19bf6c..c45e0bf6a 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -18,11 +18,11 @@ INSERT INTO blocks( ?, ?, ?, ?, ?, ? ) ON CONFLICT(height) DO UPDATE -SET chain_id = sqlc.arg(chain_id), - time = sqlc.arg(time), - hash = sqlc.arg(hash), - proposer_address = sqlc.arg(proposer_address), - last_block_hash = sqlc.arg(last_block_hash) +SET chain_id = excluded.chain_id, + time = excluded.time, + hash = excluded.hash, + proposer_address = excluded.proposer_address, + last_block_hash = excluded.last_block_hash ` type CreateBlockParams struct { diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index 786977651..47bf88d46 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -87,9 +87,6 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getTokenTransferStmt, err = db.PrepareContext(ctx, getTokenTransfer); err != nil { return nil, fmt.Errorf("error preparing query GetTokenTransfer: %w", err) } - if q.getTransactionStmt, err = db.PrepareContext(ctx, getTransaction); err != nil { - return nil, fmt.Errorf("error preparing query GetTransaction: %w", err) - } if q.getTransactionByHashStmt, err = db.PrepareContext(ctx, getTransactionByHash); err != nil { return nil, fmt.Errorf("error preparing query GetTransactionByHash: %w", err) } @@ -251,11 +248,6 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getTokenTransferStmt: %w", cerr) } } - if q.getTransactionStmt != nil { - if cerr := q.getTransactionStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getTransactionStmt: %w", cerr) - } - } if q.getTransactionByHashStmt != nil { if cerr := q.getTransactionByHashStmt.Close(); cerr != nil { err = fmt.Errorf("error closing getTransactionByHashStmt: %w", cerr) @@ -401,7 +393,6 @@ type Queries struct { getProcessIDsByFinalResultsStmt *sql.Stmt getProcessStatusStmt *sql.Stmt getTokenTransferStmt *sql.Stmt - getTransactionStmt *sql.Stmt getTransactionByHashStmt *sql.Stmt getTransactionByHeightAndIndexStmt *sql.Stmt getVoteStmt *sql.Stmt @@ -446,7 +437,6 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, getProcessStatusStmt: q.getProcessStatusStmt, getTokenTransferStmt: q.getTokenTransferStmt, - getTransactionStmt: q.getTransactionStmt, getTransactionByHashStmt: q.getTransactionByHashStmt, getTransactionByHeightAndIndexStmt: q.getTransactionByHeightAndIndexStmt, getVoteStmt: q.getVoteStmt, diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index db1778b1c..56baeb18f 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -60,7 +60,6 @@ type TokenTransfer struct { } type Transaction struct { - ID int64 Hash types.Hash BlockHeight int64 BlockIndex int64 diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index a38c1a81a..cacb7e90f 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -63,29 +63,8 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa ) } -const getTransaction = `-- name: GetTransaction :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions -WHERE id = ? -LIMIT 1 -` - -func (q *Queries) GetTransaction(ctx context.Context, id int64) (Transaction, error) { - row := q.queryRow(ctx, q.getTransactionStmt, getTransaction, id) - var i Transaction - err := row.Scan( - &i.ID, - &i.Hash, - &i.BlockHeight, - &i.BlockIndex, - &i.Type, - &i.RawTx, - &i.Signature, - ) - return i, err -} - const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions +SELECT hash, block_height, block_index, type, raw_tx, signature FROM transactions WHERE hash = ? LIMIT 1 ` @@ -94,7 +73,6 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr row := q.queryRow(ctx, q.getTransactionByHashStmt, getTransactionByHash, hash) var i Transaction err := row.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, @@ -106,7 +84,7 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr } const getTransactionByHeightAndIndex = `-- name: GetTransactionByHeightAndIndex :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions +SELECT hash, block_height, block_index, type, raw_tx, signature FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` @@ -120,7 +98,6 @@ func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTra row := q.queryRow(ctx, q.getTransactionByHeightAndIndexStmt, getTransactionByHeightAndIndex, arg.BlockHeight, arg.BlockIndex) var i Transaction err := row.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, @@ -133,16 +110,16 @@ func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTra const searchTransactions = `-- name: SearchTransactions :many WITH results AS ( - SELECT id, hash, block_height, block_index, type, raw_tx, signature + SELECT hash, block_height, block_index, type, raw_tx, signature FROM transactions WHERE ( (?3 = 0 OR block_height = ?3) AND (?4 = '' OR LOWER(type) = LOWER(?4)) ) ) -SELECT id, hash, block_height, block_index, type, raw_tx, signature, COUNT(*) OVER() AS total_count +SELECT hash, block_height, block_index, type, raw_tx, signature, COUNT(*) OVER() AS total_count FROM results -ORDER BY id DESC +ORDER BY block_height DESC, block_index DESC LIMIT ?2 OFFSET ?1 ` @@ -155,7 +132,6 @@ type SearchTransactionsParams struct { } type SearchTransactionsRow struct { - ID int64 Hash []byte BlockHeight int64 BlockIndex int64 @@ -180,7 +156,6 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions for rows.Next() { var i SearchTransactionsRow if err := rows.Scan( - &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index b780f3b87..d7bb830f0 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1406,7 +1406,7 @@ func TestTxIndexer(t *testing.T) { for i := 0; i < totalBlocks; i++ { for j := 0; j < txsPerBlock; j++ { - ref, err := idx.GetTxMetadataByID(uint64(i*txsPerBlock + j + 1)) + ref, err := idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ref.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, ref.TxBlockIndex, qt.Equals, int32(j)) @@ -1424,8 +1424,6 @@ func TestTxIndexer(t *testing.T) { txs, _, err := idx.SearchTransactions(15, 0, 0, "") qt.Assert(t, err, qt.IsNil) for i, tx := range txs { - // Index is between 1 and totalCount. - qt.Assert(t, tx.Index, qt.Equals, uint64(totalTxs-i)) // BlockIndex and TxBlockIndex start at 0, so subtract 1. qt.Assert(t, tx.BlockHeight, qt.Equals, uint32(totalTxs-i-1)/txsPerBlock) qt.Assert(t, tx.TxBlockIndex, qt.Equals, int32(totalTxs-i-1)%txsPerBlock) @@ -1435,7 +1433,6 @@ func TestTxIndexer(t *testing.T) { txs, _, err = idx.SearchTransactions(1, 5, 0, "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, txs, qt.HasLen, 1) - qt.Assert(t, txs[0].Index, qt.Equals, uint64(95)) } func TestCensusUpdate(t *testing.T) { diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index 1f6270596..0476ae488 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -178,7 +178,6 @@ type TxPackage struct { // TransactionMetadata contains tx information for the TransactionList api type TransactionMetadata struct { - Index uint64 `json:"transactionNumber" format:"int64" example:"944"` Hash types.HexBytes `json:"transactionHash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` BlockHeight uint32 `json:"blockHeight" format:"int32" example:"64924"` TxBlockIndex int32 `json:"transactionIndex" format:"int32" example:"0"` @@ -187,7 +186,6 @@ type TransactionMetadata struct { func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata { return &TransactionMetadata{ - Index: uint64(dbtx.ID), Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), @@ -197,7 +195,6 @@ func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata func TransactionMetadataFromDBRow(dbtx *indexerdb.SearchTransactionsRow) *TransactionMetadata { return &TransactionMetadata{ - Index: uint64(dbtx.ID), Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), diff --git a/vochain/indexer/migrations/0015_recreate_table_transactions.sql b/vochain/indexer/migrations/0015_recreate_table_transactions.sql new file mode 100644 index 000000000..68e609751 --- /dev/null +++ b/vochain/indexer/migrations/0015_recreate_table_transactions.sql @@ -0,0 +1,60 @@ +-- +goose Up +PRAGMA foreign_keys = OFF; + +-- Create a new table with hash as primary key +CREATE TABLE transactions_new ( + hash BLOB NOT NULL PRIMARY KEY, + block_height INTEGER NOT NULL, + block_index INTEGER NOT NULL, + type TEXT NOT NULL +); + +-- Copy data from the old table to the new table +INSERT INTO transactions_new (hash, block_height, block_index, type) +SELECT hash, block_height, block_index, type +FROM transactions; + +-- Drop the old table +DROP TABLE transactions; + +-- Rename the new table to the old table name +ALTER TABLE transactions_new RENAME TO transactions; + +-- Recreate necessary indexes +CREATE INDEX transactions_block_height_index +ON transactions(block_height, block_index); + +-- Add new columns +ALTER TABLE transactions ADD COLUMN raw_tx BLOB NOT NULL DEFAULT x''; +ALTER TABLE transactions ADD COLUMN signature BLOB NOT NULL DEFAULT x''; + +PRAGMA foreign_keys = ON; + +-- +goose Down +PRAGMA foreign_keys = OFF; + +-- Recreate the old table structure +CREATE TABLE transactions ( + id INTEGER NOT NULL PRIMARY KEY, + hash BLOB NOT NULL, + block_height INTEGER NOT NULL, + block_index INTEGER NOT NULL, + type TEXT NOT NULL +); + +-- Copy data back from the new table to the old table +INSERT INTO transactions (hash, block_height, block_index, type) +SELECT hash, block_height, block_index, type +FROM transactions_new; + +-- Drop the new table +DROP TABLE transactions_new; + +-- Recreate the old indexes +CREATE INDEX transactions_hash +ON transactions(hash); + +CREATE INDEX transactions_block_height_index +ON transactions(block_height, block_index); + +PRAGMA foreign_keys = ON; diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index f52ef3b4b..a00af5ca7 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -5,11 +5,11 @@ INSERT INTO blocks( ?, ?, ?, ?, ?, ? ) ON CONFLICT(height) DO UPDATE -SET chain_id = sqlc.arg(chain_id), - time = sqlc.arg(time), - hash = sqlc.arg(hash), - proposer_address = sqlc.arg(proposer_address), - last_block_hash = sqlc.arg(last_block_hash); +SET chain_id = excluded.chain_id, + time = excluded.time, + hash = excluded.hash, + proposer_address = excluded.proposer_address, + last_block_hash = excluded.last_block_hash; -- name: GetBlockByHeight :one SELECT * FROM blocks diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index 98307ea4c..ae58aedbe 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -5,11 +5,6 @@ INSERT INTO transactions ( ?, ?, ?, ?, ?, ? ); --- name: GetTransaction :one -SELECT * FROM transactions -WHERE id = ? -LIMIT 1; - -- name: GetTransactionByHash :one SELECT * FROM transactions WHERE hash = ? @@ -38,6 +33,6 @@ WITH results AS ( ) SELECT *, COUNT(*) OVER() AS total_count FROM results -ORDER BY id DESC +ORDER BY block_height DESC, block_index DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 4b8fe7b83..232700d3e 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -28,18 +28,6 @@ func (idx *Indexer) CountTransactionsByHeight(height int64) (int64, error) { return idx.readOnlyQuery.CountTransactionsByHeight(context.TODO(), height) } -// GetTxMetadataByID fetches the tx metadata for the given tx height -func (idx *Indexer) GetTxMetadataByID(id uint64) (*indexertypes.TransactionMetadata, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTransaction(context.TODO(), int64(id)) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, ErrTransactionNotFound - } - return nil, fmt.Errorf("tx with id %d not found: %v", id, err) - } - return indexertypes.TransactionMetadataFromDB(&sqlTxRef), nil -} - // GetTxMetadataByHash fetches the tx metadata for the given tx hash func (idx *Indexer) GetTxMetadataByHash(hash types.HexBytes) (*indexertypes.TransactionMetadata, error) { sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash) @@ -89,7 +77,6 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx list := []*indexertypes.TransactionMetadata{} for _, row := range results { list = append(list, &indexertypes.TransactionMetadata{ - Index: uint64(row.ID), Hash: row.Hash, BlockHeight: uint32(row.BlockHeight), TxBlockIndex: int32(row.BlockIndex), From a2e8c2462ddeb6d4cd0b740a9c3576c45e4b3cf1 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 26 Aug 2024 18:22:05 +0200 Subject: [PATCH 05/11] Revert "indexer: drop concept of transaction id, hash is now the primary key" This reverts commit 0ecd3fa5e20203c8d301b9870bbabee02294334e. --- api/chain.go | 38 ++++++++++++ vochain/indexer/bench_test.go | 4 +- vochain/indexer/db/blocks.sql.go | 10 ++-- vochain/indexer/db/db.go | 10 ++++ vochain/indexer/db/models.go | 1 + vochain/indexer/db/transactions.sql.go | 35 +++++++++-- vochain/indexer/indexer_test.go | 5 +- vochain/indexer/indexertypes/types.go | 3 + .../0015_recreate_table_transactions.sql | 60 ------------------- vochain/indexer/queries/blocks.sql | 10 ++-- vochain/indexer/queries/transactions.sql | 7 ++- vochain/indexer/transaction.go | 13 ++++ 12 files changed, 117 insertions(+), 79 deletions(-) delete mode 100644 vochain/indexer/migrations/0015_recreate_table_transactions.sql diff --git a/api/chain.go b/api/chain.go index f8247c6a2..d61ae9ad3 100644 --- a/api/chain.go +++ b/api/chain.go @@ -104,6 +104,14 @@ func (a *API) enableChainHandlers() error { ); err != nil { return err } + if err := a.Endpoint.RegisterMethod( + "/chain/transactions/reference/index/{index}", + "GET", + apirest.MethodAccessTypePublic, + a.chainTxRefByIndexHandler, + ); err != nil { + return err + } if err := a.Endpoint.RegisterMethod( "/chain/blocks/{height}/transactions/page/{page}", "GET", @@ -698,6 +706,36 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ctx.Send(data, apirest.HTTPstatusOK) } +// chainTxRefByIndexHandler +// +// @Summary Transaction metadata (by db index) +// @Description Get transaction by its internal index. This is not the transaction hash, and neither the block height and block index. The transaction index is an incremental counter for each transaction. You could use the transaction `block` and `index` to retrieve full info using [transaction by block and index](transaction-by-block-index). +// @Tags Chain +// @Accept json +// @Produce json +// @Param index path int true "Index of the transaction" +// @Success 200 {object} indexertypes.Transaction +// @Success 204 "See [errors](vocdoni-api#errors) section" +// @Router /chain/transactions/reference/index/{index} [get] +func (a *API) chainTxRefByIndexHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) error { + index, err := strconv.ParseUint(ctx.URLParam("index"), 10, 64) + if err != nil { + return err + } + ref, err := a.indexer.GetTxMetadataByID(index) + if err != nil { + if errors.Is(err, indexer.ErrTransactionNotFound) { + return ErrTransactionNotFound + } + return ErrVochainGetTxFailed.WithErr(err) + } + data, err := json.Marshal(ref) + if err != nil { + return err + } + return ctx.Send(data, apirest.HTTPstatusOK) +} + // chainTxListHandler // // @Summary List transactions diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index 7e289857a..f73645bd9 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -152,10 +152,10 @@ func BenchmarkFetchTx(b *testing.B) { startTime := time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) + _, err = idx.GetTxMetadataByID(uint64((i * numTxs) + j + 1)) qt.Assert(b, err, qt.IsNil) } - log.Infof("fetched %d transactions (out of %d total) by height+index, took %s", + log.Infof("fetched %d transactions (out of %d total) by index, took %s", numTxs, (i+1)*numTxs, time.Since(startTime)) startTime = time.Now() for j := 0; j < numTxs; j++ { diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index c45e0bf6a..dcb19bf6c 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -18,11 +18,11 @@ INSERT INTO blocks( ?, ?, ?, ?, ?, ? ) ON CONFLICT(height) DO UPDATE -SET chain_id = excluded.chain_id, - time = excluded.time, - hash = excluded.hash, - proposer_address = excluded.proposer_address, - last_block_hash = excluded.last_block_hash +SET chain_id = sqlc.arg(chain_id), + time = sqlc.arg(time), + hash = sqlc.arg(hash), + proposer_address = sqlc.arg(proposer_address), + last_block_hash = sqlc.arg(last_block_hash) ` type CreateBlockParams struct { diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index 47bf88d46..786977651 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -87,6 +87,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getTokenTransferStmt, err = db.PrepareContext(ctx, getTokenTransfer); err != nil { return nil, fmt.Errorf("error preparing query GetTokenTransfer: %w", err) } + if q.getTransactionStmt, err = db.PrepareContext(ctx, getTransaction); err != nil { + return nil, fmt.Errorf("error preparing query GetTransaction: %w", err) + } if q.getTransactionByHashStmt, err = db.PrepareContext(ctx, getTransactionByHash); err != nil { return nil, fmt.Errorf("error preparing query GetTransactionByHash: %w", err) } @@ -248,6 +251,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getTokenTransferStmt: %w", cerr) } } + if q.getTransactionStmt != nil { + if cerr := q.getTransactionStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getTransactionStmt: %w", cerr) + } + } if q.getTransactionByHashStmt != nil { if cerr := q.getTransactionByHashStmt.Close(); cerr != nil { err = fmt.Errorf("error closing getTransactionByHashStmt: %w", cerr) @@ -393,6 +401,7 @@ type Queries struct { getProcessIDsByFinalResultsStmt *sql.Stmt getProcessStatusStmt *sql.Stmt getTokenTransferStmt *sql.Stmt + getTransactionStmt *sql.Stmt getTransactionByHashStmt *sql.Stmt getTransactionByHeightAndIndexStmt *sql.Stmt getVoteStmt *sql.Stmt @@ -437,6 +446,7 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, getProcessStatusStmt: q.getProcessStatusStmt, getTokenTransferStmt: q.getTokenTransferStmt, + getTransactionStmt: q.getTransactionStmt, getTransactionByHashStmt: q.getTransactionByHashStmt, getTransactionByHeightAndIndexStmt: q.getTransactionByHeightAndIndexStmt, getVoteStmt: q.getVoteStmt, diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index 56baeb18f..db1778b1c 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -60,6 +60,7 @@ type TokenTransfer struct { } type Transaction struct { + ID int64 Hash types.Hash BlockHeight int64 BlockIndex int64 diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index cacb7e90f..a38c1a81a 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -63,8 +63,29 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa ) } +const getTransaction = `-- name: GetTransaction :one +SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions +WHERE id = ? +LIMIT 1 +` + +func (q *Queries) GetTransaction(ctx context.Context, id int64) (Transaction, error) { + row := q.queryRow(ctx, q.getTransactionStmt, getTransaction, id) + var i Transaction + err := row.Scan( + &i.ID, + &i.Hash, + &i.BlockHeight, + &i.BlockIndex, + &i.Type, + &i.RawTx, + &i.Signature, + ) + return i, err +} + const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT hash, block_height, block_index, type, raw_tx, signature FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions WHERE hash = ? LIMIT 1 ` @@ -73,6 +94,7 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr row := q.queryRow(ctx, q.getTransactionByHashStmt, getTransactionByHash, hash) var i Transaction err := row.Scan( + &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, @@ -84,7 +106,7 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr } const getTransactionByHeightAndIndex = `-- name: GetTransactionByHeightAndIndex :one -SELECT hash, block_height, block_index, type, raw_tx, signature FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` @@ -98,6 +120,7 @@ func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTra row := q.queryRow(ctx, q.getTransactionByHeightAndIndexStmt, getTransactionByHeightAndIndex, arg.BlockHeight, arg.BlockIndex) var i Transaction err := row.Scan( + &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, @@ -110,16 +133,16 @@ func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTra const searchTransactions = `-- name: SearchTransactions :many WITH results AS ( - SELECT hash, block_height, block_index, type, raw_tx, signature + SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions WHERE ( (?3 = 0 OR block_height = ?3) AND (?4 = '' OR LOWER(type) = LOWER(?4)) ) ) -SELECT hash, block_height, block_index, type, raw_tx, signature, COUNT(*) OVER() AS total_count +SELECT id, hash, block_height, block_index, type, raw_tx, signature, COUNT(*) OVER() AS total_count FROM results -ORDER BY block_height DESC, block_index DESC +ORDER BY id DESC LIMIT ?2 OFFSET ?1 ` @@ -132,6 +155,7 @@ type SearchTransactionsParams struct { } type SearchTransactionsRow struct { + ID int64 Hash []byte BlockHeight int64 BlockIndex int64 @@ -156,6 +180,7 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions for rows.Next() { var i SearchTransactionsRow if err := rows.Scan( + &i.ID, &i.Hash, &i.BlockHeight, &i.BlockIndex, diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index d7bb830f0..b780f3b87 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1406,7 +1406,7 @@ func TestTxIndexer(t *testing.T) { for i := 0; i < totalBlocks; i++ { for j := 0; j < txsPerBlock; j++ { - ref, err := idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) + ref, err := idx.GetTxMetadataByID(uint64(i*txsPerBlock + j + 1)) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ref.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, ref.TxBlockIndex, qt.Equals, int32(j)) @@ -1424,6 +1424,8 @@ func TestTxIndexer(t *testing.T) { txs, _, err := idx.SearchTransactions(15, 0, 0, "") qt.Assert(t, err, qt.IsNil) for i, tx := range txs { + // Index is between 1 and totalCount. + qt.Assert(t, tx.Index, qt.Equals, uint64(totalTxs-i)) // BlockIndex and TxBlockIndex start at 0, so subtract 1. qt.Assert(t, tx.BlockHeight, qt.Equals, uint32(totalTxs-i-1)/txsPerBlock) qt.Assert(t, tx.TxBlockIndex, qt.Equals, int32(totalTxs-i-1)%txsPerBlock) @@ -1433,6 +1435,7 @@ func TestTxIndexer(t *testing.T) { txs, _, err = idx.SearchTransactions(1, 5, 0, "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, txs, qt.HasLen, 1) + qt.Assert(t, txs[0].Index, qt.Equals, uint64(95)) } func TestCensusUpdate(t *testing.T) { diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index 0476ae488..1f6270596 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -178,6 +178,7 @@ type TxPackage struct { // TransactionMetadata contains tx information for the TransactionList api type TransactionMetadata struct { + Index uint64 `json:"transactionNumber" format:"int64" example:"944"` Hash types.HexBytes `json:"transactionHash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` BlockHeight uint32 `json:"blockHeight" format:"int32" example:"64924"` TxBlockIndex int32 `json:"transactionIndex" format:"int32" example:"0"` @@ -186,6 +187,7 @@ type TransactionMetadata struct { func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata { return &TransactionMetadata{ + Index: uint64(dbtx.ID), Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), @@ -195,6 +197,7 @@ func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata func TransactionMetadataFromDBRow(dbtx *indexerdb.SearchTransactionsRow) *TransactionMetadata { return &TransactionMetadata{ + Index: uint64(dbtx.ID), Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), diff --git a/vochain/indexer/migrations/0015_recreate_table_transactions.sql b/vochain/indexer/migrations/0015_recreate_table_transactions.sql deleted file mode 100644 index 68e609751..000000000 --- a/vochain/indexer/migrations/0015_recreate_table_transactions.sql +++ /dev/null @@ -1,60 +0,0 @@ --- +goose Up -PRAGMA foreign_keys = OFF; - --- Create a new table with hash as primary key -CREATE TABLE transactions_new ( - hash BLOB NOT NULL PRIMARY KEY, - block_height INTEGER NOT NULL, - block_index INTEGER NOT NULL, - type TEXT NOT NULL -); - --- Copy data from the old table to the new table -INSERT INTO transactions_new (hash, block_height, block_index, type) -SELECT hash, block_height, block_index, type -FROM transactions; - --- Drop the old table -DROP TABLE transactions; - --- Rename the new table to the old table name -ALTER TABLE transactions_new RENAME TO transactions; - --- Recreate necessary indexes -CREATE INDEX transactions_block_height_index -ON transactions(block_height, block_index); - --- Add new columns -ALTER TABLE transactions ADD COLUMN raw_tx BLOB NOT NULL DEFAULT x''; -ALTER TABLE transactions ADD COLUMN signature BLOB NOT NULL DEFAULT x''; - -PRAGMA foreign_keys = ON; - --- +goose Down -PRAGMA foreign_keys = OFF; - --- Recreate the old table structure -CREATE TABLE transactions ( - id INTEGER NOT NULL PRIMARY KEY, - hash BLOB NOT NULL, - block_height INTEGER NOT NULL, - block_index INTEGER NOT NULL, - type TEXT NOT NULL -); - --- Copy data back from the new table to the old table -INSERT INTO transactions (hash, block_height, block_index, type) -SELECT hash, block_height, block_index, type -FROM transactions_new; - --- Drop the new table -DROP TABLE transactions_new; - --- Recreate the old indexes -CREATE INDEX transactions_hash -ON transactions(hash); - -CREATE INDEX transactions_block_height_index -ON transactions(block_height, block_index); - -PRAGMA foreign_keys = ON; diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index a00af5ca7..f52ef3b4b 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -5,11 +5,11 @@ INSERT INTO blocks( ?, ?, ?, ?, ?, ? ) ON CONFLICT(height) DO UPDATE -SET chain_id = excluded.chain_id, - time = excluded.time, - hash = excluded.hash, - proposer_address = excluded.proposer_address, - last_block_hash = excluded.last_block_hash; +SET chain_id = sqlc.arg(chain_id), + time = sqlc.arg(time), + hash = sqlc.arg(hash), + proposer_address = sqlc.arg(proposer_address), + last_block_hash = sqlc.arg(last_block_hash); -- name: GetBlockByHeight :one SELECT * FROM blocks diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index ae58aedbe..98307ea4c 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -5,6 +5,11 @@ INSERT INTO transactions ( ?, ?, ?, ?, ?, ? ); +-- name: GetTransaction :one +SELECT * FROM transactions +WHERE id = ? +LIMIT 1; + -- name: GetTransactionByHash :one SELECT * FROM transactions WHERE hash = ? @@ -33,6 +38,6 @@ WITH results AS ( ) SELECT *, COUNT(*) OVER() AS total_count FROM results -ORDER BY block_height DESC, block_index DESC +ORDER BY id DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 232700d3e..4b8fe7b83 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -28,6 +28,18 @@ func (idx *Indexer) CountTransactionsByHeight(height int64) (int64, error) { return idx.readOnlyQuery.CountTransactionsByHeight(context.TODO(), height) } +// GetTxMetadataByID fetches the tx metadata for the given tx height +func (idx *Indexer) GetTxMetadataByID(id uint64) (*indexertypes.TransactionMetadata, error) { + sqlTxRef, err := idx.readOnlyQuery.GetTransaction(context.TODO(), int64(id)) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrTransactionNotFound + } + return nil, fmt.Errorf("tx with id %d not found: %v", id, err) + } + return indexertypes.TransactionMetadataFromDB(&sqlTxRef), nil +} + // GetTxMetadataByHash fetches the tx metadata for the given tx hash func (idx *Indexer) GetTxMetadataByHash(hash types.HexBytes) (*indexertypes.TransactionMetadata, error) { sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash) @@ -77,6 +89,7 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx list := []*indexertypes.TransactionMetadata{} for _, row := range results { list = append(list, &indexertypes.TransactionMetadata{ + Index: uint64(row.ID), Hash: row.Hash, BlockHeight: uint32(row.BlockHeight), TxBlockIndex: int32(row.BlockIndex), From e97908b504d7978433580268606ebb3b2919f4fc Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 26 Aug 2024 18:22:38 +0200 Subject: [PATCH 06/11] Revert "indexer: add ReindexBlocks and make indexerdb.CreateBlock an UPSERT" This reverts commit 4ec03dab47a47fc8b3fdb0cc9a498f50a4c895b6. --- vochain/indexer/db/blocks.sql.go | 6 ------ vochain/indexer/indexer.go | 25 ------------------------- vochain/indexer/queries/blocks.sql | 8 +------- 3 files changed, 1 insertion(+), 38 deletions(-) diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index dcb19bf6c..01e6ba54a 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -17,12 +17,6 @@ INSERT INTO blocks( ) VALUES ( ?, ?, ?, ?, ?, ? ) -ON CONFLICT(height) DO UPDATE -SET chain_id = sqlc.arg(chain_id), - time = sqlc.arg(time), - hash = sqlc.arg(hash), - proposer_address = sqlc.arg(proposer_address), - last_block_hash = sqlc.arg(last_block_hash) ` type CreateBlockParams struct { diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index 21821c10b..d66ce1f07 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -402,31 +402,6 @@ func (idx *Indexer) AfterSyncBootstrap(inTest bool) { log.Infof("live results recovery computation finished, took %s", time.Since(startTime)) } -func (idx *Indexer) ReindexBlocks() { - queries := idx.blockTxQueries() - - for i := idx.App.Node.BlockStore().Base(); i <= idx.App.Node.BlockStore().Height(); i++ { - if b := idx.App.GetBlockByHeight(int64(i)); b != nil { - idxBlock, err := idx.readOnlyQuery.GetBlockByHeight(context.TODO(), i) - if err == nil && idxBlock.Time != b.Time { - log.Errorf("while reindexing blocks, block %d timestamp in db (%s) differs from blockstore (%s), leaving untouched", i, idxBlock.Time, b.Time) - continue - } - // if we got here, the block doesn't exist - if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ - ChainID: b.ChainID, - Height: b.Height, - Time: b.Time, - Hash: nonNullBytes(b.Hash()), - ProposerAddress: nonNullBytes(b.ProposerAddress), - LastBlockHash: nonNullBytes(b.LastBlockID.Hash), - }); err != nil { - log.Errorw(err, "cannot index new block") - } - } - } -} - // Commit is called by the APP when a block is confirmed and included into the chain func (idx *Indexer) Commit(height uint32) error { idx.blockMu.Lock() diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index f52ef3b4b..38380967b 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -3,13 +3,7 @@ INSERT INTO blocks( chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( ?, ?, ?, ?, ?, ? -) -ON CONFLICT(height) DO UPDATE -SET chain_id = sqlc.arg(chain_id), - time = sqlc.arg(time), - hash = sqlc.arg(hash), - proposer_address = sqlc.arg(proposer_address), - last_block_hash = sqlc.arg(last_block_hash); +); -- name: GetBlockByHeight :one SELECT * FROM blocks From 017ec2f0424cfc49209eb5692c7958e87d588b12 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 26 Aug 2024 18:11:32 +0200 Subject: [PATCH 07/11] vochaintx: add tx.TxSubType method --- vochain/transaction/vochaintx/vochaintx.go | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/vochain/transaction/vochaintx/vochaintx.go b/vochain/transaction/vochaintx/vochaintx.go index f413e5e3b..0c70886b4 100644 --- a/vochain/transaction/vochaintx/vochaintx.go +++ b/vochain/transaction/vochaintx/vochaintx.go @@ -50,6 +50,29 @@ func (tx *Tx) Unmarshal(content []byte, chainID string) error { return nil } +// TxSubType returns the content of the "txtype" field inside the tx.Tx. +// +// The function determines the type of the transaction using Protocol Buffers reflection. +// If the field doesn't exist, it returns the empty string "". +func (tx *Tx) TxSubType() string { + txReflectDescriptor := tx.Tx.ProtoReflect().Descriptor().Oneofs().Get(0) + if txReflectDescriptor == nil { + return "" + } + whichOneTxModelType := tx.Tx.ProtoReflect().WhichOneof(txReflectDescriptor) + if whichOneTxModelType == nil { + return "" + } + // Get the value of the selected field in the oneof + fieldValue := tx.Tx.ProtoReflect().Get(whichOneTxModelType) + // Now, fieldValue is a protoreflect.Value, retrieve the txtype field + txtypeFieldDescriptor := fieldValue.Message().Descriptor().Fields().ByName("txtype") + if txtypeFieldDescriptor == nil { + return "" + } + return fieldValue.Message().Get(txtypeFieldDescriptor).String() +} + // TxKey computes the checksum of the tx func TxKey(tx []byte) [32]byte { return comettypes.Tx(tx).Key() From 9a6ec7057e7a291e02f31448c918d3f15dc9f159 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 26 Aug 2024 18:15:14 +0200 Subject: [PATCH 08/11] indexer: OnNewTx now indexes SubType and Account --- vochain/indexer/transaction.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 4b8fe7b83..e4ab8e6e9 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" + "go.vocdoni.io/dvote/crypto/ethereum" "go.vocdoni.io/dvote/log" "go.vocdoni.io/dvote/types" indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" @@ -112,6 +113,14 @@ func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) return } + account, err := ethereum.AddrFromSignature(tx.SignedBody, tx.Signature) + if err != nil { + if len(tx.Signature) > 0 { + log.Warnf("indexing signed tx with empty account field, can't recover signer from signature: %s", err) + } + // otherwise ignore, since some txs are not signed by design, for example zk ones + } + queries := idx.blockTxQueries() if _, err := queries.CreateTransaction(context.TODO(), indexerdb.CreateTransactionParams{ Hash: tx.TxID[:], @@ -120,6 +129,8 @@ func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) Type: tx.TxModelType, RawTx: rawtx, Signature: tx.Signature, + SubType: tx.TxSubType(), + Account: account, }); err != nil { log.Errorw(err, "cannot index new transaction") } From 34005a12cfabbd64efc3e0edc1db11c40cc86e45 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 26 Aug 2024 18:36:23 +0200 Subject: [PATCH 09/11] api: endpoint /chain/transactions/{height}/{index} now includes account field --- api/api_types.go | 1 + api/chain.go | 1 + 2 files changed, 2 insertions(+) diff --git a/api/api_types.go b/api/api_types.go index 36294b293..3b978a3cb 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -295,6 +295,7 @@ type GenericTransactionWithInfo struct { TxContent json.RawMessage `json:"tx"` TxInfo *indexertypes.Transaction `json:"txInfo"` Signature types.HexBytes `json:"signature"` + Account types.HexBytes `json:"account"` } type ChainInfo struct { diff --git a/api/chain.go b/api/chain.go index d61ae9ad3..8ef819d72 100644 --- a/api/chain.go +++ b/api/chain.go @@ -698,6 +698,7 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er TxContent: []byte(protoFormat(ref.RawTx)), TxInfo: ref, Signature: ref.Signature, + Account: ref.Account, } data, err := json.Marshal(tx) if err != nil { From 440d9ace1dfa3af73adff5abcef577885087879a Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 26 Aug 2024 18:57:56 +0200 Subject: [PATCH 10/11] fixup! indexer: OnNewTx now indexes SubType and Account --- vochain/indexer/transaction.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index e4ab8e6e9..efe8331e3 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -113,10 +113,10 @@ func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) return } - account, err := ethereum.AddrFromSignature(tx.SignedBody, tx.Signature) + signer, err := ethereum.AddrFromSignature(tx.SignedBody, tx.Signature) if err != nil { if len(tx.Signature) > 0 { - log.Warnf("indexing signed tx with empty account field, can't recover signer from signature: %s", err) + log.Warnf("indexing signed tx with empty signer field, can't recover signer from signature: %s", err) } // otherwise ignore, since some txs are not signed by design, for example zk ones } @@ -129,8 +129,8 @@ func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) Type: tx.TxModelType, RawTx: rawtx, Signature: tx.Signature, - SubType: tx.TxSubType(), - Account: account, + Subtype: tx.TxSubType(), + Signer: signer.Bytes(), }); err != nil { log.Errorw(err, "cannot index new transaction") } From 5730d0cb13b33491a299cca0158cc95d3493e227 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 26 Aug 2024 20:23:40 +0200 Subject: [PATCH 11/11] many fixups (dirty commit, needs squashing) signer field, nonNullBytes, indexer: add subtype and signer todo: add new params to api /chain/transactions --- api/api_types.go | 2 +- api/chain.go | 4 ++- vochain/indexer/db/models.go | 2 ++ vochain/indexer/db/transactions.sql.go | 34 +++++++++++++++---- vochain/indexer/indexertypes/types.go | 5 +++ ...015_alter_columns_table_transactions_2.sql | 7 ++++ vochain/indexer/queries/transactions.sql | 6 ++-- vochain/indexer/transaction.go | 33 +++++++++--------- vochain/transaction/vochaintx/vochaintx.go | 5 ++- 9 files changed, 69 insertions(+), 29 deletions(-) create mode 100644 vochain/indexer/migrations/0015_alter_columns_table_transactions_2.sql diff --git a/api/api_types.go b/api/api_types.go index 3b978a3cb..1b8c89a21 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -295,7 +295,7 @@ type GenericTransactionWithInfo struct { TxContent json.RawMessage `json:"tx"` TxInfo *indexertypes.Transaction `json:"txInfo"` Signature types.HexBytes `json:"signature"` - Account types.HexBytes `json:"account"` + Signer types.HexBytes `json:"signer"` } type ChainInfo struct { diff --git a/api/chain.go b/api/chain.go index 8ef819d72..1b4d5ca14 100644 --- a/api/chain.go +++ b/api/chain.go @@ -698,7 +698,7 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er TxContent: []byte(protoFormat(ref.RawTx)), TxInfo: ref, Signature: ref.Signature, - Account: ref.Account, + Signer: ref.Signer, } data, err := json.Marshal(tx) if err != nil { @@ -849,6 +849,8 @@ func (a *API) transactionList(params *TransactionParams) (*TransactionsList, err params.Page*params.Limit, params.Height, params.Type, + "", // TODO(gui): support new params + "", // ) if err != nil { return nil, ErrIndexerQueryFailed.WithErr(err) diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index db1778b1c..fab3d35ad 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -67,4 +67,6 @@ type Transaction struct { Type string RawTx []byte Signature []byte + Subtype string + Signer []byte } diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index a38c1a81a..c8628aebc 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -37,9 +37,9 @@ func (q *Queries) CountTransactionsByHeight(ctx context.Context, blockHeight int const createTransaction = `-- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type, raw_tx, signature + hash, block_height, block_index, type, subtype, raw_tx, signature, signer ) VALUES ( - ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ? ) ` @@ -48,8 +48,10 @@ type CreateTransactionParams struct { BlockHeight int64 BlockIndex int64 Type string + Subtype string RawTx []byte Signature []byte + Signer []byte } func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionParams) (sql.Result, error) { @@ -58,13 +60,15 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa arg.BlockHeight, arg.BlockIndex, arg.Type, + arg.Subtype, arg.RawTx, arg.Signature, + arg.Signer, ) } const getTransaction = `-- name: GetTransaction :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer FROM transactions WHERE id = ? LIMIT 1 ` @@ -80,12 +84,14 @@ func (q *Queries) GetTransaction(ctx context.Context, id int64) (Transaction, er &i.Type, &i.RawTx, &i.Signature, + &i.Subtype, + &i.Signer, ) return i, err } const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer FROM transactions WHERE hash = ? LIMIT 1 ` @@ -101,12 +107,14 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr &i.Type, &i.RawTx, &i.Signature, + &i.Subtype, + &i.Signer, ) return i, err } const getTransactionByHeightAndIndex = `-- name: GetTransactionByHeightAndIndex :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` @@ -127,20 +135,24 @@ func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTra &i.Type, &i.RawTx, &i.Signature, + &i.Subtype, + &i.Signer, ) return i, err } const searchTransactions = `-- name: SearchTransactions :many WITH results AS ( - SELECT id, hash, block_height, block_index, type, raw_tx, signature + SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer FROM transactions WHERE ( (?3 = 0 OR block_height = ?3) AND (?4 = '' OR LOWER(type) = LOWER(?4)) + AND (?5 = '' OR LOWER(subtype) = LOWER(?5)) + AND (?6 = '' OR LOWER(HEX(signer)) = LOWER(?6)) ) ) -SELECT id, hash, block_height, block_index, type, raw_tx, signature, COUNT(*) OVER() AS total_count +SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer, COUNT(*) OVER() AS total_count FROM results ORDER BY id DESC LIMIT ?2 @@ -152,6 +164,8 @@ type SearchTransactionsParams struct { Limit int64 BlockHeight interface{} TxType interface{} + TxSubtype interface{} + TxSigner interface{} } type SearchTransactionsRow struct { @@ -162,6 +176,8 @@ type SearchTransactionsRow struct { Type string RawTx []byte Signature []byte + Subtype string + Signer []byte TotalCount int64 } @@ -171,6 +187,8 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions arg.Limit, arg.BlockHeight, arg.TxType, + arg.TxSubtype, + arg.TxSigner, ) if err != nil { return nil, err @@ -187,6 +205,8 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions &i.Type, &i.RawTx, &i.Signature, + &i.Subtype, + &i.Signer, &i.TotalCount, ); err != nil { return nil, err diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index 1f6270596..d46257da6 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -183,6 +183,7 @@ type TransactionMetadata struct { BlockHeight uint32 `json:"blockHeight" format:"int32" example:"64924"` TxBlockIndex int32 `json:"transactionIndex" format:"int32" example:"0"` TxType string `json:"transactionType" enums:"vote,newProcess,admin,setProcess,registerKey,mintTokens,sendTokens,setTransactionCosts,setAccount,collectFaucet,setKeykeeper" example:"Vote"` + TxSubtype string `json:"transactionSubtype" enums:"newProcess,setProcessCensus,setProcessDuration" example:"setProcessCensus"` } func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata { @@ -192,6 +193,7 @@ func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), TxType: dbtx.Type, + TxSubtype: dbtx.Subtype, } } @@ -202,6 +204,7 @@ func TransactionMetadataFromDBRow(dbtx *indexerdb.SearchTransactionsRow) *Transa BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), TxType: dbtx.Type, + TxSubtype: dbtx.Subtype, } } @@ -210,6 +213,7 @@ type Transaction struct { *TransactionMetadata RawTx types.HexBytes `json:"-"` Signature types.HexBytes `json:"-"` + Signer types.HexBytes `json:"-"` } // TransactionFromDB converts an indexerdb.Transaction into a Transaction @@ -218,6 +222,7 @@ func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { TransactionMetadata: TransactionMetadataFromDB(dbtx), RawTx: dbtx.RawTx, Signature: dbtx.Signature, + Signer: dbtx.Signer, } } diff --git a/vochain/indexer/migrations/0015_alter_columns_table_transactions_2.sql b/vochain/indexer/migrations/0015_alter_columns_table_transactions_2.sql new file mode 100644 index 000000000..b114f56bd --- /dev/null +++ b/vochain/indexer/migrations/0015_alter_columns_table_transactions_2.sql @@ -0,0 +1,7 @@ +-- +goose Up +ALTER TABLE transactions ADD COLUMN subtype TEXT NOT NULL DEFAULT ''; +ALTER TABLE transactions ADD COLUMN signer BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE transactions DROP COLUMN signer; +ALTER TABLE transactions DROP COLUMN subtype; diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index 98307ea4c..4466a5387 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -1,8 +1,8 @@ -- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type, raw_tx, signature + hash, block_height, block_index, type, subtype, raw_tx, signature, signer ) VALUES ( - ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ? ); -- name: GetTransaction :one @@ -34,6 +34,8 @@ WITH results AS ( WHERE ( (sqlc.arg(block_height) = 0 OR block_height = sqlc.arg(block_height)) AND (sqlc.arg(tx_type) = '' OR LOWER(type) = LOWER(sqlc.arg(tx_type))) + AND (sqlc.arg(tx_subtype) = '' OR LOWER(subtype) = LOWER(sqlc.arg(tx_subtype))) + AND (sqlc.arg(tx_signer) = '' OR LOWER(HEX(signer)) = LOWER(sqlc.arg(tx_signer))) ) ) SELECT *, COUNT(*) OVER() AS total_count diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index efe8331e3..1cd73fdde 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "strings" "go.vocdoni.io/dvote/crypto/ethereum" "go.vocdoni.io/dvote/log" @@ -69,9 +70,9 @@ func (idx *Indexer) GetTransactionByHeightAndIndex(blockHeight, blockIndex int64 } // SearchTransactions returns the list of transactions indexed. -// height and txType are optional, if declared as zero-value will be ignored. +// blockHeight, txType, txSubtype and txSigner are optional, if declared as zero-value will be ignored. // The first one returned is the newest, so they are in descending order. -func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txType string) ([]*indexertypes.TransactionMetadata, uint64, error) { +func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txType, txSubtype, txSigner string) ([]*indexertypes.TransactionMetadata, uint64, error) { if offset < 0 { return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) } @@ -83,19 +84,15 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx Offset: int64(offset), BlockHeight: blockHeight, TxType: txType, + TxSubtype: txSubtype, + TxSigner: txSigner, }) if err != nil { return nil, 0, err } list := []*indexertypes.TransactionMetadata{} for _, row := range results { - list = append(list, &indexertypes.TransactionMetadata{ - Index: uint64(row.ID), - Hash: row.Hash, - BlockHeight: uint32(row.BlockHeight), - TxBlockIndex: int32(row.BlockIndex), - TxType: row.Type, - }) + list = append(list, indexertypes.TransactionMetadataFromDBRow(&row)) } if len(results) == 0 { return list, 0, nil @@ -113,12 +110,14 @@ func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) return } - signer, err := ethereum.AddrFromSignature(tx.SignedBody, tx.Signature) - if err != nil { - if len(tx.Signature) > 0 { - log.Warnf("indexing signed tx with empty signer field, can't recover signer from signature: %s", err) + signer := []byte{} + if len(tx.Signature) > 0 { // not all txs are signed, for example zk ones + addr, err := ethereum.AddrFromSignature(tx.SignedBody, tx.Signature) + if err != nil { + log.Errorw(err, "indexer cannot recover signer from signature") + return } - // otherwise ignore, since some txs are not signed by design, for example zk ones + signer = addr.Bytes() } queries := idx.blockTxQueries() @@ -127,10 +126,10 @@ func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) BlockHeight: int64(blockHeight), BlockIndex: int64(txIndex), Type: tx.TxModelType, + Subtype: strings.ToLower(tx.TxSubType()), RawTx: rawtx, - Signature: tx.Signature, - Subtype: tx.TxSubType(), - Signer: signer.Bytes(), + Signature: nonNullBytes(tx.Signature), + Signer: nonNullBytes(signer), }); err != nil { log.Errorw(err, "cannot index new transaction") } diff --git a/vochain/transaction/vochaintx/vochaintx.go b/vochain/transaction/vochaintx/vochaintx.go index 0c70886b4..e3e30c185 100644 --- a/vochain/transaction/vochaintx/vochaintx.go +++ b/vochain/transaction/vochaintx/vochaintx.go @@ -70,7 +70,10 @@ func (tx *Tx) TxSubType() string { if txtypeFieldDescriptor == nil { return "" } - return fieldValue.Message().Get(txtypeFieldDescriptor).String() + // Get the integer value of txtype as protoreflect.EnumNumber + enumNumber := fieldValue.Message().Get(txtypeFieldDescriptor).Enum() + // Convert the EnumNumber to a string using the EnumType descriptor + return string(txtypeFieldDescriptor.Enum().Values().ByNumber(enumNumber).Name()) } // TxKey computes the checksum of the tx