From 823c84149b9a6f7eff9621b53367f4ba54f8ceea Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 10 Jun 2024 11:18:11 +0200 Subject: [PATCH] indexer: index more details about blocks and transactions * indexerdb: add block details chain_id, proposer_address and last_block_hash * indexerdb: rename method GetBlock -> GetBlockByHeight * indexer: new method BlockByHeight (and indexerdb.GetBlockByHash) * indexer: new method SearchBlocks (and indexerdb.SearchBlocks) transactions: * indexerdb: add raw_tx, subtype, signature and signer in transactions table * indexer: new methods CountTransactionsByHeight and SearchTransactions * indexer: rename GetTransaction* methods -> GetTxMetadata* indexer migrations: * 0014_alter_columns_table_blocks.sql * 0015_alter_columns_table_transactions.sql * vochaintx: add tx.TxSubtype method --- api/api_types.go | 4 +- api/chain.go | 4 +- vochain/indexer/bench_test.go | 13 +- vochain/indexer/block.go | 62 +++++- vochain/indexer/db/blocks.sql.go | 141 ++++++++++++- vochain/indexer/db/db.go | 198 ++++++++++-------- vochain/indexer/db/models.go | 13 +- vochain/indexer/db/transactions.sql.go | 62 +++++- vochain/indexer/indexer.go | 19 +- vochain/indexer/indexer_test.go | 9 +- vochain/indexer/indexertypes/block.go | 26 +++ vochain/indexer/indexertypes/types.go | 39 +++- .../0014_alter_columns_table_blocks.sql | 13 ++ .../0015_alter_columns_table_transactions.sql | 11 + vochain/indexer/queries/blocks.sql | 35 +++- vochain/indexer/queries/transactions.sql | 12 +- vochain/indexer/transaction.go | 68 ++++-- vochain/transaction/vochaintx/vochaintx.go | 26 +++ 18 files changed, 592 insertions(+), 163 deletions(-) create mode 100644 vochain/indexer/migrations/0014_alter_columns_table_blocks.sql create mode 100644 vochain/indexer/migrations/0015_alter_columns_table_transactions.sql diff --git a/api/api_types.go b/api/api_types.go index efb390503..b1169db8d 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -275,8 +275,8 @@ type TransactionReference struct { // TransactionsList is used to return a paginated list to the client type TransactionsList struct { - Transactions []*indexertypes.Transaction `json:"transactions"` - Pagination *Pagination `json:"pagination"` + Transactions []*indexertypes.TransactionMetadata `json:"transactions"` + Pagination *Pagination `json:"pagination"` } // FeesList is used to return a paginated list to the client diff --git a/api/chain.go b/api/chain.go index 143c73c91..5bf39e6ff 100644 --- a/api/chain.go +++ b/api/chain.go @@ -646,7 +646,7 @@ func (a *API) chainTxRefByHashHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo if err != nil { return err } - ref, err := a.indexer.GetTxHashReference(hash) + ref, err := a.indexer.GetTxMetadataByHash(hash) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound @@ -690,7 +690,7 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er return ErrVochainGetTxFailed.WithErr(err) } - ref, err := a.indexer.GetTxReferenceByBlockHeightAndBlockIndex(height, index) + ref, err := a.indexer.GetTransactionByHeightAndIndex(height, index) if err != nil { if errors.Is(err, indexer.ErrTransactionNotFound) { return ErrTransactionNotFound diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index 9e8cb1210..7e289857a 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -85,6 +85,7 @@ func BenchmarkIndexer(b *testing.B) { tx := &vochaintx.Tx{ TxID: rnd.Random32(), TxModelType: "vote", + Tx: &models.Tx{Payload: &models.Tx_Vote{}}, } idx.OnNewTx(tx, height, txBlockIndex) curTxs = append(curTxs, tx) @@ -112,7 +113,7 @@ func BenchmarkIndexer(b *testing.B) { qt.Check(b, bytes.Equal(voteRef.Meta.TxHash, tx.TxID[:]), qt.IsTrue) } - txRef, err := idx.GetTxHashReference(tx.TxID[:]) + txRef, err := idx.GetTxMetadataByHash(tx.TxID[:]) qt.Check(b, err, qt.IsNil) if err == nil { qt.Check(b, txRef.BlockHeight, qt.Equals, vote.Height) @@ -138,7 +139,11 @@ func BenchmarkFetchTx(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { for j := 0; j < numTxs; j++ { - idx.OnNewTx(&vochaintx.Tx{TxID: util.Random32()}, uint32(i), int32(j)) + idx.OnNewTx(&vochaintx.Tx{ + TxID: util.Random32(), + TxModelType: "vote", + Tx: &models.Tx{Payload: &models.Tx_Vote{}}, + }, uint32(i), int32(j)) } err := idx.Commit(uint32(i)) qt.Assert(b, err, qt.IsNil) @@ -147,14 +152,14 @@ func BenchmarkFetchTx(b *testing.B) { startTime := time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTxReferenceByBlockHeightAndBlockIndex(int64(i), int64(j)) + _, err = idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) qt.Assert(b, err, qt.IsNil) } log.Infof("fetched %d transactions (out of %d total) by height+index, took %s", numTxs, (i+1)*numTxs, time.Since(startTime)) startTime = time.Now() for j := 0; j < numTxs; j++ { - _, err = idx.GetTxHashReference([]byte(fmt.Sprintf("hash%d%d", i, j))) + _, err = idx.GetTxMetadataByHash([]byte(fmt.Sprintf("hash%d%d", i, j))) qt.Assert(b, err, qt.IsNil) } log.Infof("fetched %d transactions (out of %d total) by hash, took %s", diff --git a/vochain/indexer/block.go b/vochain/indexer/block.go index 8c9d76a22..5d2118ef4 100644 --- a/vochain/indexer/block.go +++ b/vochain/indexer/block.go @@ -6,6 +6,9 @@ import ( "errors" "fmt" "time" + + indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" + "go.vocdoni.io/dvote/vochain/indexer/indexertypes" ) // ErrBlockNotFound is returned if the block is not found in the indexer database. @@ -13,12 +16,63 @@ var ErrBlockNotFound = fmt.Errorf("block not found") // BlockTimestamp returns the timestamp of the block at the given height func (idx *Indexer) BlockTimestamp(height int64) (time.Time, error) { - block, err := idx.readOnlyQuery.GetBlock(context.TODO(), height) + block, err := idx.BlockByHeight(height) if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return time.Time{}, ErrBlockNotFound - } return time.Time{}, err } return block.Time, nil } + +// BlockByHeight returns the available information of the block at the given height +func (idx *Indexer) BlockByHeight(height int64) (*indexertypes.Block, error) { + block, err := idx.readOnlyQuery.GetBlockByHeight(context.TODO(), height) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrBlockNotFound + } + return nil, err + } + return indexertypes.BlockFromDB(&block), nil +} + +// BlockByHash returns the available information of the block with the given hash +func (idx *Indexer) BlockByHash(hash []byte) (*indexertypes.Block, error) { + block, err := idx.readOnlyQuery.GetBlockByHash(context.TODO(), hash) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrBlockNotFound + } + return nil, err + } + return indexertypes.BlockFromDB(&block), nil +} + +// BlockList returns the list of blocks indexed. +// chainID, hash, proposerAddress are optional, if declared as zero-value will be ignored. +// The first one returned is the newest, so they are in descending order. +func (idx *Indexer) BlockList(limit, offset int, chainID, hash, proposerAddress string) ([]*indexertypes.Block, uint64, error) { + if offset < 0 { + return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) + } + if limit <= 0 { + return nil, 0, fmt.Errorf("invalid value: limit cannot be %d", limit) + } + results, err := idx.readOnlyQuery.SearchBlocks(context.TODO(), indexerdb.SearchBlocksParams{ + Limit: int64(limit), + Offset: int64(offset), + ChainID: chainID, + HashSubstr: hash, + ProposerAddress: proposerAddress, + }) + if err != nil { + return nil, 0, err + } + list := []*indexertypes.Block{} + for _, row := range results { + list = append(list, indexertypes.BlockFromDBRow(&row)) + } + if len(results) == 0 { + return list, 0, nil + } + return list, uint64(results[0].TotalCount), nil +} diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index ffa3fa896..e634a10b4 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -13,31 +13,150 @@ import ( const createBlock = `-- name: CreateBlock :execresult INSERT INTO blocks( - height, time, data_hash + chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( - ?, ?, ? + ?, ?, ?, ?, ?, ? ) ` type CreateBlockParams struct { - Height int64 - Time time.Time - DataHash []byte + ChainID string + Height int64 + Time time.Time + Hash []byte + ProposerAddress []byte + LastBlockHash []byte } func (q *Queries) CreateBlock(ctx context.Context, arg CreateBlockParams) (sql.Result, error) { - return q.exec(ctx, q.createBlockStmt, createBlock, arg.Height, arg.Time, arg.DataHash) + return q.exec(ctx, q.createBlockStmt, createBlock, + arg.ChainID, + arg.Height, + arg.Time, + arg.Hash, + arg.ProposerAddress, + arg.LastBlockHash, + ) } -const getBlock = `-- name: GetBlock :one -SELECT height, time, data_hash FROM blocks +const getBlockByHash = `-- name: GetBlockByHash :one +SELECT height, time, chain_id, hash, proposer_address, last_block_hash FROM blocks +WHERE hash = ? +LIMIT 1 +` + +func (q *Queries) GetBlockByHash(ctx context.Context, hash []byte) (Block, error) { + row := q.queryRow(ctx, q.getBlockByHashStmt, getBlockByHash, hash) + var i Block + err := row.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + ) + return i, err +} + +const getBlockByHeight = `-- name: GetBlockByHeight :one +SELECT height, time, chain_id, hash, proposer_address, last_block_hash FROM blocks WHERE height = ? LIMIT 1 ` -func (q *Queries) GetBlock(ctx context.Context, height int64) (Block, error) { - row := q.queryRow(ctx, q.getBlockStmt, getBlock, height) +func (q *Queries) GetBlockByHeight(ctx context.Context, height int64) (Block, error) { + row := q.queryRow(ctx, q.getBlockByHeightStmt, getBlockByHeight, height) var i Block - err := row.Scan(&i.Height, &i.Time, &i.DataHash) + err := row.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + ) return i, err } + +const searchBlocks = `-- name: SearchBlocks :many +SELECT + b.height, b.time, b.chain_id, b.hash, b.proposer_address, b.last_block_hash, + COUNT(t.block_index) AS tx_count, + COUNT(*) OVER() AS total_count +FROM blocks AS b +LEFT JOIN transactions AS t + ON b.height = t.block_height +WHERE ( + (?1 = '' OR b.chain_id = ?1) + AND LENGTH(?2) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?2 = '' + OR (LENGTH(?2) = 64 AND LOWER(HEX(b.hash)) = LOWER(?2)) + OR (LENGTH(?2) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(?2)) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (?3 = '' OR LOWER(HEX(b.proposer_address)) = LOWER(?3)) +) +GROUP BY b.height +ORDER BY b.height DESC +LIMIT ?5 +OFFSET ?4 +` + +type SearchBlocksParams struct { + ChainID interface{} + HashSubstr interface{} + ProposerAddress interface{} + Offset int64 + Limit int64 +} + +type SearchBlocksRow struct { + Height int64 + Time time.Time + ChainID string + Hash []byte + ProposerAddress []byte + LastBlockHash []byte + TxCount int64 + TotalCount int64 +} + +func (q *Queries) SearchBlocks(ctx context.Context, arg SearchBlocksParams) ([]SearchBlocksRow, error) { + rows, err := q.query(ctx, q.searchBlocksStmt, searchBlocks, + arg.ChainID, + arg.HashSubstr, + arg.ProposerAddress, + arg.Offset, + arg.Limit, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SearchBlocksRow + for rows.Next() { + var i SearchBlocksRow + if err := rows.Scan( + &i.Height, + &i.Time, + &i.ChainID, + &i.Hash, + &i.ProposerAddress, + &i.LastBlockHash, + &i.TxCount, + &i.TotalCount, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index dd22f9d0f..47bf88d46 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -36,6 +36,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.countTransactionsStmt, err = db.PrepareContext(ctx, countTransactions); err != nil { return nil, fmt.Errorf("error preparing query CountTransactions: %w", err) } + if q.countTransactionsByHeightStmt, err = db.PrepareContext(ctx, countTransactionsByHeight); err != nil { + return nil, fmt.Errorf("error preparing query CountTransactionsByHeight: %w", err) + } if q.countVotesStmt, err = db.PrepareContext(ctx, countVotes); err != nil { return nil, fmt.Errorf("error preparing query CountVotes: %w", err) } @@ -60,8 +63,11 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.createVoteStmt, err = db.PrepareContext(ctx, createVote); err != nil { return nil, fmt.Errorf("error preparing query CreateVote: %w", err) } - if q.getBlockStmt, err = db.PrepareContext(ctx, getBlock); err != nil { - return nil, fmt.Errorf("error preparing query GetBlock: %w", err) + if q.getBlockByHashStmt, err = db.PrepareContext(ctx, getBlockByHash); err != nil { + return nil, fmt.Errorf("error preparing query GetBlockByHash: %w", err) + } + if q.getBlockByHeightStmt, err = db.PrepareContext(ctx, getBlockByHeight); err != nil { + return nil, fmt.Errorf("error preparing query GetBlockByHeight: %w", err) } if q.getEntityCountStmt, err = db.PrepareContext(ctx, getEntityCount); err != nil { return nil, fmt.Errorf("error preparing query GetEntityCount: %w", err) @@ -84,8 +90,8 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getTransactionByHashStmt, err = db.PrepareContext(ctx, getTransactionByHash); err != nil { return nil, fmt.Errorf("error preparing query GetTransactionByHash: %w", err) } - if q.getTxReferenceByBlockHeightAndBlockIndexStmt, err = db.PrepareContext(ctx, getTxReferenceByBlockHeightAndBlockIndex); err != nil { - return nil, fmt.Errorf("error preparing query GetTxReferenceByBlockHeightAndBlockIndex: %w", err) + if q.getTransactionByHeightAndIndexStmt, err = db.PrepareContext(ctx, getTransactionByHeightAndIndex); err != nil { + return nil, fmt.Errorf("error preparing query GetTransactionByHeightAndIndex: %w", err) } if q.getVoteStmt, err = db.PrepareContext(ctx, getVote); err != nil { return nil, fmt.Errorf("error preparing query GetVote: %w", err) @@ -93,6 +99,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.searchAccountsStmt, err = db.PrepareContext(ctx, searchAccounts); err != nil { return nil, fmt.Errorf("error preparing query SearchAccounts: %w", err) } + if q.searchBlocksStmt, err = db.PrepareContext(ctx, searchBlocks); err != nil { + return nil, fmt.Errorf("error preparing query SearchBlocks: %w", err) + } if q.searchEntitiesStmt, err = db.PrepareContext(ctx, searchEntities); err != nil { return nil, fmt.Errorf("error preparing query SearchEntities: %w", err) } @@ -154,6 +163,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing countTransactionsStmt: %w", cerr) } } + if q.countTransactionsByHeightStmt != nil { + if cerr := q.countTransactionsByHeightStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing countTransactionsByHeightStmt: %w", cerr) + } + } if q.countVotesStmt != nil { if cerr := q.countVotesStmt.Close(); cerr != nil { err = fmt.Errorf("error closing countVotesStmt: %w", cerr) @@ -194,9 +208,14 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing createVoteStmt: %w", cerr) } } - if q.getBlockStmt != nil { - if cerr := q.getBlockStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getBlockStmt: %w", cerr) + if q.getBlockByHashStmt != nil { + if cerr := q.getBlockByHashStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getBlockByHashStmt: %w", cerr) + } + } + if q.getBlockByHeightStmt != nil { + if cerr := q.getBlockByHeightStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getBlockByHeightStmt: %w", cerr) } } if q.getEntityCountStmt != nil { @@ -234,9 +253,9 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getTransactionByHashStmt: %w", cerr) } } - if q.getTxReferenceByBlockHeightAndBlockIndexStmt != nil { - if cerr := q.getTxReferenceByBlockHeightAndBlockIndexStmt.Close(); cerr != nil { - err = fmt.Errorf("error closing getTxReferenceByBlockHeightAndBlockIndexStmt: %w", cerr) + if q.getTransactionByHeightAndIndexStmt != nil { + if cerr := q.getTransactionByHeightAndIndexStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing getTransactionByHeightAndIndexStmt: %w", cerr) } } if q.getVoteStmt != nil { @@ -249,6 +268,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing searchAccountsStmt: %w", cerr) } } + if q.searchBlocksStmt != nil { + if cerr := q.searchBlocksStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing searchBlocksStmt: %w", cerr) + } + } if q.searchEntitiesStmt != nil { if cerr := q.searchEntitiesStmt.Close(); cerr != nil { err = fmt.Errorf("error closing searchEntitiesStmt: %w", cerr) @@ -346,83 +370,89 @@ func (q *Queries) queryRow(ctx context.Context, stmt *sql.Stmt, query string, ar } type Queries struct { - db DBTX - tx *sql.Tx - computeProcessVoteCountStmt *sql.Stmt - countAccountsStmt *sql.Stmt - countTokenTransfersByAccountStmt *sql.Stmt - countTransactionsStmt *sql.Stmt - countVotesStmt *sql.Stmt - createAccountStmt *sql.Stmt - createBlockStmt *sql.Stmt - createProcessStmt *sql.Stmt - createTokenFeeStmt *sql.Stmt - createTokenTransferStmt *sql.Stmt - createTransactionStmt *sql.Stmt - createVoteStmt *sql.Stmt - getBlockStmt *sql.Stmt - getEntityCountStmt *sql.Stmt - getProcessStmt *sql.Stmt - getProcessCountStmt *sql.Stmt - getProcessIDsByFinalResultsStmt *sql.Stmt - getProcessStatusStmt *sql.Stmt - getTokenTransferStmt *sql.Stmt - getTransactionByHashStmt *sql.Stmt - getTxReferenceByBlockHeightAndBlockIndexStmt *sql.Stmt - getVoteStmt *sql.Stmt - searchAccountsStmt *sql.Stmt - searchEntitiesStmt *sql.Stmt - searchProcessesStmt *sql.Stmt - searchTokenFeesStmt *sql.Stmt - searchTokenTransfersStmt *sql.Stmt - searchTransactionsStmt *sql.Stmt - searchVotesStmt *sql.Stmt - setProcessResultsCancelledStmt *sql.Stmt - setProcessResultsReadyStmt *sql.Stmt - updateProcessEndDateStmt *sql.Stmt - updateProcessFromStateStmt *sql.Stmt - updateProcessResultByIDStmt *sql.Stmt - updateProcessResultsStmt *sql.Stmt + db DBTX + tx *sql.Tx + computeProcessVoteCountStmt *sql.Stmt + countAccountsStmt *sql.Stmt + countTokenTransfersByAccountStmt *sql.Stmt + countTransactionsStmt *sql.Stmt + countTransactionsByHeightStmt *sql.Stmt + countVotesStmt *sql.Stmt + createAccountStmt *sql.Stmt + createBlockStmt *sql.Stmt + createProcessStmt *sql.Stmt + createTokenFeeStmt *sql.Stmt + createTokenTransferStmt *sql.Stmt + createTransactionStmt *sql.Stmt + createVoteStmt *sql.Stmt + getBlockByHashStmt *sql.Stmt + getBlockByHeightStmt *sql.Stmt + getEntityCountStmt *sql.Stmt + getProcessStmt *sql.Stmt + getProcessCountStmt *sql.Stmt + getProcessIDsByFinalResultsStmt *sql.Stmt + getProcessStatusStmt *sql.Stmt + getTokenTransferStmt *sql.Stmt + getTransactionByHashStmt *sql.Stmt + getTransactionByHeightAndIndexStmt *sql.Stmt + getVoteStmt *sql.Stmt + searchAccountsStmt *sql.Stmt + searchBlocksStmt *sql.Stmt + searchEntitiesStmt *sql.Stmt + searchProcessesStmt *sql.Stmt + searchTokenFeesStmt *sql.Stmt + searchTokenTransfersStmt *sql.Stmt + searchTransactionsStmt *sql.Stmt + searchVotesStmt *sql.Stmt + setProcessResultsCancelledStmt *sql.Stmt + setProcessResultsReadyStmt *sql.Stmt + updateProcessEndDateStmt *sql.Stmt + updateProcessFromStateStmt *sql.Stmt + updateProcessResultByIDStmt *sql.Stmt + updateProcessResultsStmt *sql.Stmt } func (q *Queries) WithTx(tx *sql.Tx) *Queries { return &Queries{ - db: tx, - tx: tx, - computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, - countAccountsStmt: q.countAccountsStmt, - countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, - countTransactionsStmt: q.countTransactionsStmt, - countVotesStmt: q.countVotesStmt, - createAccountStmt: q.createAccountStmt, - createBlockStmt: q.createBlockStmt, - createProcessStmt: q.createProcessStmt, - createTokenFeeStmt: q.createTokenFeeStmt, - createTokenTransferStmt: q.createTokenTransferStmt, - createTransactionStmt: q.createTransactionStmt, - createVoteStmt: q.createVoteStmt, - getBlockStmt: q.getBlockStmt, - getEntityCountStmt: q.getEntityCountStmt, - getProcessStmt: q.getProcessStmt, - getProcessCountStmt: q.getProcessCountStmt, - getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, - getProcessStatusStmt: q.getProcessStatusStmt, - getTokenTransferStmt: q.getTokenTransferStmt, - getTransactionByHashStmt: q.getTransactionByHashStmt, - getTxReferenceByBlockHeightAndBlockIndexStmt: q.getTxReferenceByBlockHeightAndBlockIndexStmt, - getVoteStmt: q.getVoteStmt, - searchAccountsStmt: q.searchAccountsStmt, - searchEntitiesStmt: q.searchEntitiesStmt, - searchProcessesStmt: q.searchProcessesStmt, - searchTokenFeesStmt: q.searchTokenFeesStmt, - searchTokenTransfersStmt: q.searchTokenTransfersStmt, - searchTransactionsStmt: q.searchTransactionsStmt, - searchVotesStmt: q.searchVotesStmt, - setProcessResultsCancelledStmt: q.setProcessResultsCancelledStmt, - setProcessResultsReadyStmt: q.setProcessResultsReadyStmt, - updateProcessEndDateStmt: q.updateProcessEndDateStmt, - updateProcessFromStateStmt: q.updateProcessFromStateStmt, - updateProcessResultByIDStmt: q.updateProcessResultByIDStmt, - updateProcessResultsStmt: q.updateProcessResultsStmt, + db: tx, + tx: tx, + computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, + countAccountsStmt: q.countAccountsStmt, + countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, + countTransactionsStmt: q.countTransactionsStmt, + countTransactionsByHeightStmt: q.countTransactionsByHeightStmt, + countVotesStmt: q.countVotesStmt, + createAccountStmt: q.createAccountStmt, + createBlockStmt: q.createBlockStmt, + createProcessStmt: q.createProcessStmt, + createTokenFeeStmt: q.createTokenFeeStmt, + createTokenTransferStmt: q.createTokenTransferStmt, + createTransactionStmt: q.createTransactionStmt, + createVoteStmt: q.createVoteStmt, + getBlockByHashStmt: q.getBlockByHashStmt, + getBlockByHeightStmt: q.getBlockByHeightStmt, + getEntityCountStmt: q.getEntityCountStmt, + getProcessStmt: q.getProcessStmt, + getProcessCountStmt: q.getProcessCountStmt, + getProcessIDsByFinalResultsStmt: q.getProcessIDsByFinalResultsStmt, + getProcessStatusStmt: q.getProcessStatusStmt, + getTokenTransferStmt: q.getTokenTransferStmt, + getTransactionByHashStmt: q.getTransactionByHashStmt, + getTransactionByHeightAndIndexStmt: q.getTransactionByHeightAndIndexStmt, + getVoteStmt: q.getVoteStmt, + searchAccountsStmt: q.searchAccountsStmt, + searchBlocksStmt: q.searchBlocksStmt, + searchEntitiesStmt: q.searchEntitiesStmt, + searchProcessesStmt: q.searchProcessesStmt, + searchTokenFeesStmt: q.searchTokenFeesStmt, + searchTokenTransfersStmt: q.searchTokenTransfersStmt, + searchTransactionsStmt: q.searchTransactionsStmt, + searchVotesStmt: q.searchVotesStmt, + setProcessResultsCancelledStmt: q.setProcessResultsCancelledStmt, + setProcessResultsReadyStmt: q.setProcessResultsReadyStmt, + updateProcessEndDateStmt: q.updateProcessEndDateStmt, + updateProcessFromStateStmt: q.updateProcessFromStateStmt, + updateProcessResultByIDStmt: q.updateProcessResultByIDStmt, + updateProcessResultsStmt: q.updateProcessResultsStmt, } } diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index b64fc72c8..d69facf2f 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -11,9 +11,12 @@ import ( ) type Block struct { - Height int64 - Time time.Time - DataHash []byte + Height int64 + Time time.Time + ChainID string + Hash []byte + ProposerAddress []byte + LastBlockHash []byte } type Process struct { @@ -61,4 +64,8 @@ type Transaction struct { BlockHeight int64 BlockIndex int64 Type string + Subtype string + RawTx []byte + Signature []byte + Signer []byte } diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index a477ebad0..e6c5efa2d 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -23,11 +23,23 @@ func (q *Queries) CountTransactions(ctx context.Context) (int64, error) { return count, err } +const countTransactionsByHeight = `-- name: CountTransactionsByHeight :one +SELECT COUNT(*) FROM transactions +WHERE block_height = ? +` + +func (q *Queries) CountTransactionsByHeight(ctx context.Context, blockHeight int64) (int64, error) { + row := q.queryRow(ctx, q.countTransactionsByHeightStmt, countTransactionsByHeight, blockHeight) + var count int64 + err := row.Scan(&count) + return count, err +} + const createTransaction = `-- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type + hash, block_height, block_index, type, subtype, raw_tx, signature, signer ) VALUES ( - ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ? ) ` @@ -36,6 +48,10 @@ type CreateTransactionParams struct { BlockHeight int64 BlockIndex int64 Type string + Subtype string + RawTx []byte + Signature []byte + Signer []byte } func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionParams) (sql.Result, error) { @@ -44,11 +60,15 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa arg.BlockHeight, arg.BlockIndex, arg.Type, + arg.Subtype, + arg.RawTx, + arg.Signature, + arg.Signer, ) } const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT hash, block_height, block_index, type FROM transactions +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer FROM transactions WHERE hash = ? LIMIT 1 ` @@ -61,43 +81,53 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, + &i.RawTx, + &i.Signature, + &i.Signer, ) return i, err } -const getTxReferenceByBlockHeightAndBlockIndex = `-- name: GetTxReferenceByBlockHeightAndBlockIndex :one -SELECT hash, block_height, block_index, type FROM transactions +const getTransactionByHeightAndIndex = `-- name: GetTransactionByHeightAndIndex :one +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` -type GetTxReferenceByBlockHeightAndBlockIndexParams struct { +type GetTransactionByHeightAndIndexParams struct { BlockHeight int64 BlockIndex int64 } -func (q *Queries) GetTxReferenceByBlockHeightAndBlockIndex(ctx context.Context, arg GetTxReferenceByBlockHeightAndBlockIndexParams) (Transaction, error) { - row := q.queryRow(ctx, q.getTxReferenceByBlockHeightAndBlockIndexStmt, getTxReferenceByBlockHeightAndBlockIndex, arg.BlockHeight, arg.BlockIndex) +func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTransactionByHeightAndIndexParams) (Transaction, error) { + row := q.queryRow(ctx, q.getTransactionByHeightAndIndexStmt, getTransactionByHeightAndIndex, arg.BlockHeight, arg.BlockIndex) var i Transaction err := row.Scan( &i.Hash, &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, + &i.RawTx, + &i.Signature, + &i.Signer, ) return i, err } const searchTransactions = `-- name: SearchTransactions :many WITH results AS ( - SELECT hash, block_height, block_index, type + SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer FROM transactions WHERE ( (?3 = 0 OR block_height = ?3) AND (?4 = '' OR LOWER(type) = LOWER(?4)) + AND (?5 = '' OR LOWER(subtype) = LOWER(?5)) + AND (?6 = '' OR LOWER(HEX(signer)) = LOWER(?6)) ) ) -SELECT hash, block_height, block_index, type, COUNT(*) OVER() AS total_count +SELECT hash, block_height, block_index, type, subtype, raw_tx, signature, signer, COUNT(*) OVER() AS total_count FROM results ORDER BY block_height DESC, block_index DESC LIMIT ?2 @@ -109,6 +139,8 @@ type SearchTransactionsParams struct { Limit int64 BlockHeight interface{} TxType interface{} + TxSubtype interface{} + TxSigner interface{} } type SearchTransactionsRow struct { @@ -116,6 +148,10 @@ type SearchTransactionsRow struct { BlockHeight int64 BlockIndex int64 Type string + Subtype string + RawTx []byte + Signature []byte + Signer []byte TotalCount int64 } @@ -125,6 +161,8 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions arg.Limit, arg.BlockHeight, arg.TxType, + arg.TxSubtype, + arg.TxSigner, ) if err != nil { return nil, err @@ -138,6 +176,10 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions &i.BlockHeight, &i.BlockIndex, &i.Type, + &i.Subtype, + &i.RawTx, + &i.Signature, + &i.Signer, &i.TotalCount, ); err != nil { return nil, err diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index 769b76720..d66ce1f07 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -415,14 +415,17 @@ func (idx *Indexer) Commit(height uint32) error { ctx := context.TODO() // index the new block - bb := idx.App.GetBlockByHeight(int64(height)) - if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ - Height: bb.Height, - Time: bb.Time, - DataHash: nonNullBytes(bb.DataHash), - // TODO: ProposerAddress, Hash, AppHash, ChainID, LastBlockHash?, ValidatorSignatures? - }); err != nil { - log.Errorw(err, "cannot index new block") + if b := idx.App.GetBlockByHeight(int64(height)); b != nil { + if _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + ChainID: b.ChainID, + Height: b.Height, + Time: b.Time, + Hash: nonNullBytes(b.Hash()), + ProposerAddress: nonNullBytes(b.ProposerAddress), + LastBlockHash: nonNullBytes(b.LastBlockID.Hash), + }); err != nil { + log.Errorw(err, "cannot index new block") + } } for _, pidStr := range updateProcs { diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index 76c306910..8f08bd398 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -1392,6 +1392,7 @@ func TestTxIndexer(t *testing.T) { idx.OnNewTx(&vochaintx.Tx{ TxID: getTxID(i, j), TxModelType: "setAccount", + Tx: &models.Tx{Payload: &models.Tx_SetAccount{}}, }, uint32(i), int32(j)) } } @@ -1404,7 +1405,7 @@ func TestTxIndexer(t *testing.T) { for i := 0; i < totalBlocks; i++ { for j := 0; j < txsPerBlock; j++ { - ref, err := idx.GetTxReferenceByBlockHeightAndBlockIndex(int64(i), int64(j)) + ref, err := idx.GetTransactionByHeightAndIndex(int64(i), int64(j)) qt.Assert(t, err, qt.IsNil) qt.Assert(t, ref.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, ref.TxBlockIndex, qt.Equals, int32(j)) @@ -1412,14 +1413,14 @@ func TestTxIndexer(t *testing.T) { h := make([]byte, 32) id := getTxID(i, j) copy(h, id[:]) - hashRef, err := idx.GetTxHashReference(h) + hashRef, err := idx.GetTxMetadataByHash(h) qt.Assert(t, err, qt.IsNil) qt.Assert(t, hashRef.BlockHeight, qt.Equals, uint32(i)) qt.Assert(t, hashRef.TxBlockIndex, qt.Equals, int32(j)) } } - txs, _, err := idx.SearchTransactions(15, 0, 0, "") + txs, _, err := idx.SearchTransactions(15, 0, 0, "", "", "") qt.Assert(t, err, qt.IsNil) for i, tx := range txs { // BlockIndex and TxBlockIndex start at 0, so subtract 1. @@ -1428,7 +1429,7 @@ func TestTxIndexer(t *testing.T) { qt.Assert(t, tx.TxType, qt.Equals, "setAccount") } - txs, _, err = idx.SearchTransactions(1, 5, 0, "") + txs, _, err = idx.SearchTransactions(1, 5, 0, "", "", "") qt.Assert(t, err, qt.IsNil) qt.Assert(t, txs, qt.HasLen, 1) } diff --git a/vochain/indexer/indexertypes/block.go b/vochain/indexer/indexertypes/block.go index 4954dbc22..4b711bdbc 100644 --- a/vochain/indexer/indexertypes/block.go +++ b/vochain/indexer/indexertypes/block.go @@ -4,6 +4,7 @@ import ( "time" "go.vocdoni.io/dvote/types" + indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" ) // Block represents a block handled by the Vochain. @@ -18,3 +19,28 @@ type Block struct { LastBlockHash types.HexBytes `json:"lastBlockHash"` TxCount int64 `json:"txCount"` } + +// BlockFromDB converts the indexerdb.Block into a Block +func BlockFromDB(dbblock *indexerdb.Block) *Block { + return &Block{ + ChainID: dbblock.ChainID, + Height: dbblock.Height, + Time: dbblock.Time, + Hash: nonEmptyBytes(dbblock.Hash), + ProposerAddress: nonEmptyBytes(dbblock.ProposerAddress), + LastBlockHash: nonEmptyBytes(dbblock.LastBlockHash), + } +} + +// BlockFromDBRow converts the indexerdb.SearchBlocksRow into a Block +func BlockFromDBRow(row *indexerdb.SearchBlocksRow) *Block { + return &Block{ + ChainID: row.ChainID, + Height: row.Height, + Time: row.Time, + Hash: nonEmptyBytes(row.Hash), + ProposerAddress: nonEmptyBytes(row.ProposerAddress), + LastBlockHash: nonEmptyBytes(row.LastBlockHash), + TxCount: row.TxCount, + } +} diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index bc0e56485..7e54e25a7 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -176,20 +176,51 @@ type TxPackage struct { Signature types.HexBytes `json:"signature"` } -// Transaction holds the db reference for a single transaction -type Transaction struct { +// TransactionMetadata contains tx information for the TransactionList api +type TransactionMetadata struct { Hash types.HexBytes `json:"hash" swaggertype:"string" example:"75e8f822f5dd13973ac5158d600f0a2a5fea4bfefce9712ab5195bf17884cfad"` BlockHeight uint32 `json:"height" format:"int32" example:"64924"` TxBlockIndex int32 `json:"index" format:"int32" example:"0"` TxType string `json:"type" enums:"vote,newProcess,admin,setProcess,registerKey,mintTokens,sendTokens,setTransactionCosts,setAccount,collectFaucet,setKeykeeper" example:"Vote"` + TxSubtype string `json:"subtype" example:"set_process_census"` + Signer types.HexBytes `json:"signer" swaggertype:"string" example:"0e45513942cf95330fc5e9020851b8bdd9b9c9df"` } -func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { - return &Transaction{ +func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata { + return &TransactionMetadata{ + Hash: dbtx.Hash, + BlockHeight: uint32(dbtx.BlockHeight), + TxBlockIndex: int32(dbtx.BlockIndex), + TxType: dbtx.Type, + TxSubtype: dbtx.Subtype, + Signer: dbtx.Signer, + } +} + +func TransactionMetadataFromDBRow(dbtx *indexerdb.SearchTransactionsRow) *TransactionMetadata { + return &TransactionMetadata{ Hash: dbtx.Hash, BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), TxType: dbtx.Type, + TxSubtype: dbtx.Subtype, + Signer: dbtx.Signer, + } +} + +// Transaction holds a single transaction +type Transaction struct { + *TransactionMetadata + RawTx types.HexBytes `json:"-"` + Signature types.HexBytes `json:"signature,omitempty"` +} + +// TransactionFromDB converts an indexerdb.Transaction into a Transaction +func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { + return &Transaction{ + TransactionMetadata: TransactionMetadataFromDB(dbtx), + RawTx: dbtx.RawTx, + Signature: dbtx.Signature, } } diff --git a/vochain/indexer/migrations/0014_alter_columns_table_blocks.sql b/vochain/indexer/migrations/0014_alter_columns_table_blocks.sql new file mode 100644 index 000000000..c83c32c06 --- /dev/null +++ b/vochain/indexer/migrations/0014_alter_columns_table_blocks.sql @@ -0,0 +1,13 @@ +-- +goose Up +ALTER TABLE blocks DROP COLUMN data_hash; +ALTER TABLE blocks ADD COLUMN chain_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE blocks ADD COLUMN hash BLOB NOT NULL DEFAULT x''; +ALTER TABLE blocks ADD COLUMN proposer_address BLOB NOT NULL DEFAULT x''; +ALTER TABLE blocks ADD COLUMN last_block_hash BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE blocks ADD COLUMN data_hash BLOB NOT NULL; +ALTER TABLE blocks DROP COLUMN chain_id; +ALTER TABLE blocks DROP COLUMN hash; +ALTER TABLE blocks DROP COLUMN proposer_address; +ALTER TABLE blocks DROP COLUMN last_block_hash; diff --git a/vochain/indexer/migrations/0015_alter_columns_table_transactions.sql b/vochain/indexer/migrations/0015_alter_columns_table_transactions.sql new file mode 100644 index 000000000..81dcb9dec --- /dev/null +++ b/vochain/indexer/migrations/0015_alter_columns_table_transactions.sql @@ -0,0 +1,11 @@ +-- +goose Up +ALTER TABLE transactions ADD COLUMN subtype TEXT NOT NULL DEFAULT ''; +ALTER TABLE transactions ADD COLUMN raw_tx BLOB NOT NULL DEFAULT x''; +ALTER TABLE transactions ADD COLUMN signature BLOB NOT NULL DEFAULT x''; +ALTER TABLE transactions ADD COLUMN signer BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE transactions DROP COLUMN signer; +ALTER TABLE transactions DROP COLUMN signature; +ALTER TABLE transactions DROP COLUMN raw_tx; +ALTER TABLE transactions DROP COLUMN subtype; diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index 577e875b5..656de4e21 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -1,11 +1,40 @@ -- name: CreateBlock :execresult INSERT INTO blocks( - height, time, data_hash + chain_id, height, time, hash, proposer_address, last_block_hash ) VALUES ( - ?, ?, ? + ?, ?, ?, ?, ?, ? ); --- name: GetBlock :one +-- name: GetBlockByHeight :one SELECT * FROM blocks WHERE height = ? LIMIT 1; + +-- name: GetBlockByHash :one +SELECT * FROM blocks +WHERE hash = ? +LIMIT 1; + +-- name: SearchBlocks :many +SELECT + b.*, + COUNT(t.block_index) AS tx_count, + COUNT(*) OVER() AS total_count +FROM blocks AS b +LEFT JOIN transactions AS t + ON b.height = t.block_height +WHERE ( + (sqlc.arg(chain_id) = '' OR b.chain_id = sqlc.arg(chain_id)) + AND LENGTH(sqlc.arg(hash_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(hash_substr) = '' + OR (LENGTH(sqlc.arg(hash_substr)) = 64 AND LOWER(HEX(b.hash)) = LOWER(sqlc.arg(hash_substr))) + OR (LENGTH(sqlc.arg(hash_substr)) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(sqlc.arg(hash_substr))) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (sqlc.arg(proposer_address) = '' OR LOWER(HEX(b.proposer_address)) = LOWER(sqlc.arg(proposer_address))) +) +GROUP BY b.height +ORDER BY b.height DESC +LIMIT sqlc.arg(limit) +OFFSET sqlc.arg(offset); diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index eb8b2b617..fe32c46f6 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -1,8 +1,8 @@ -- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type + hash, block_height, block_index, type, subtype, raw_tx, signature, signer ) VALUES ( - ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ? ); -- name: GetTransactionByHash :one @@ -13,7 +13,11 @@ LIMIT 1; -- name: CountTransactions :one SELECT COUNT(*) FROM transactions; --- name: GetTxReferenceByBlockHeightAndBlockIndex :one +-- name: CountTransactionsByHeight :one +SELECT COUNT(*) FROM transactions +WHERE block_height = ?; + +-- name: GetTransactionByHeightAndIndex :one SELECT * FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1; @@ -25,6 +29,8 @@ WITH results AS ( WHERE ( (sqlc.arg(block_height) = 0 OR block_height = sqlc.arg(block_height)) AND (sqlc.arg(tx_type) = '' OR LOWER(type) = LOWER(sqlc.arg(tx_type))) + AND (sqlc.arg(tx_subtype) = '' OR LOWER(subtype) = LOWER(sqlc.arg(tx_subtype))) + AND (sqlc.arg(tx_signer) = '' OR LOWER(HEX(signer)) = LOWER(sqlc.arg(tx_signer))) ) ) SELECT *, COUNT(*) OVER() AS total_count diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index 16e8cad81..09ce738ab 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -5,12 +5,15 @@ import ( "database/sql" "errors" "fmt" + "strings" + "go.vocdoni.io/dvote/crypto/ethereum" "go.vocdoni.io/dvote/log" "go.vocdoni.io/dvote/types" indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" "go.vocdoni.io/dvote/vochain/indexer/indexertypes" "go.vocdoni.io/dvote/vochain/transaction/vochaintx" + "google.golang.org/protobuf/proto" ) // ErrTransactionNotFound is returned if the transaction is not found. @@ -22,37 +25,42 @@ func (idx *Indexer) CountTotalTransactions() (uint64, error) { return uint64(count), err } -// GetTxReferenceByBlockHeightAndBlockIndex fetches the txReference for the given tx height and block tx index -func (idx *Indexer) GetTxReferenceByBlockHeightAndBlockIndex(blockHeight, blockIndex int64) (*indexertypes.Transaction, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTxReferenceByBlockHeightAndBlockIndex(context.TODO(), indexerdb.GetTxReferenceByBlockHeightAndBlockIndexParams{ - BlockHeight: blockHeight, - BlockIndex: blockIndex, - }) +// CountTransactionsByHeight returns the number of transactions indexed for a given height +func (idx *Indexer) CountTransactionsByHeight(height int64) (int64, error) { + return idx.readOnlyQuery.CountTransactionsByHeight(context.TODO(), height) +} + +// GetTxMetadataByHash fetches the tx metadata for the given tx hash +func (idx *Indexer) GetTxMetadataByHash(hash types.HexBytes) (*indexertypes.TransactionMetadata, error) { + sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrTransactionNotFound } - return nil, fmt.Errorf("tx at block %d and index %d not found: %v", blockHeight, blockIndex, err) + return nil, fmt.Errorf("tx hash %x not found: %v", hash, err) } - return indexertypes.TransactionFromDB(&sqlTxRef), nil + return indexertypes.TransactionMetadataFromDB(&sqlTxRef), nil } -// GetTxHashReference fetches the txReference for the given tx hash -func (idx *Indexer) GetTxHashReference(hash types.HexBytes) (*indexertypes.Transaction, error) { - sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHash(context.TODO(), hash) +// GetTransactionByHeightAndIndex fetches the full tx for the given tx height and block tx index +func (idx *Indexer) GetTransactionByHeightAndIndex(blockHeight, blockIndex int64) (*indexertypes.Transaction, error) { + sqlTxRef, err := idx.readOnlyQuery.GetTransactionByHeightAndIndex(context.TODO(), indexerdb.GetTransactionByHeightAndIndexParams{ + BlockHeight: blockHeight, + BlockIndex: blockIndex, + }) if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, ErrTransactionNotFound } - return nil, fmt.Errorf("tx hash %x not found: %v", hash, err) + return nil, fmt.Errorf("tx at block %d and index %d not found: %v", blockHeight, blockIndex, err) } return indexertypes.TransactionFromDB(&sqlTxRef), nil } // SearchTransactions returns the list of transactions indexed. -// height and txType are optional, if declared as zero-value will be ignored. +// blockHeight, txType, txSubtype and txSigner are optional, if declared as zero-value will be ignored. // The first one returned is the newest, so they are in descending order. -func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txType string) ([]*indexertypes.Transaction, uint64, error) { +func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txType, txSubtype, txSigner string) ([]*indexertypes.TransactionMetadata, uint64, error) { if offset < 0 { return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) } @@ -64,18 +72,15 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx Offset: int64(offset), BlockHeight: blockHeight, TxType: txType, + TxSubtype: txSubtype, + TxSigner: txSigner, }) if err != nil { return nil, 0, err } - list := []*indexertypes.Transaction{} + list := []*indexertypes.TransactionMetadata{} for _, row := range results { - list = append(list, &indexertypes.Transaction{ - Hash: row.Hash, - BlockHeight: uint32(row.BlockHeight), - TxBlockIndex: int32(row.BlockIndex), - TxType: row.Type, - }) + list = append(list, indexertypes.TransactionMetadataFromDBRow(&row)) } if len(results) == 0 { return list, 0, nil @@ -86,12 +91,33 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) { idx.blockMu.Lock() defer idx.blockMu.Unlock() + + rawtx, err := proto.Marshal(tx.Tx) + if err != nil { + log.Errorw(err, "indexer cannot marshal new transaction") + return + } + + signer := []byte{} + if len(tx.Signature) > 0 { // not all txs are signed, for example zk ones + addr, err := ethereum.AddrFromSignature(tx.SignedBody, tx.Signature) + if err != nil { + log.Errorw(err, "indexer cannot recover signer from signature") + return + } + signer = addr.Bytes() + } + queries := idx.blockTxQueries() if _, err := queries.CreateTransaction(context.TODO(), indexerdb.CreateTransactionParams{ Hash: tx.TxID[:], BlockHeight: int64(blockHeight), BlockIndex: int64(txIndex), Type: tx.TxModelType, + Subtype: strings.ToLower(tx.TxSubtype()), + RawTx: rawtx, + Signature: nonNullBytes(tx.Signature), + Signer: nonNullBytes(signer), }); err != nil { log.Errorw(err, "cannot index new transaction") } diff --git a/vochain/transaction/vochaintx/vochaintx.go b/vochain/transaction/vochaintx/vochaintx.go index f413e5e3b..66bc950fe 100644 --- a/vochain/transaction/vochaintx/vochaintx.go +++ b/vochain/transaction/vochaintx/vochaintx.go @@ -50,6 +50,32 @@ func (tx *Tx) Unmarshal(content []byte, chainID string) error { return nil } +// TxSubtype returns the content of the "txtype" field inside the tx.Tx. +// +// The function determines the type of the transaction using Protocol Buffers reflection. +// If the field doesn't exist, it returns the empty string "". +func (tx *Tx) TxSubtype() string { + txReflectDescriptor := tx.Tx.ProtoReflect().Descriptor().Oneofs().Get(0) + if txReflectDescriptor == nil { + return "" + } + whichOneTxModelType := tx.Tx.ProtoReflect().WhichOneof(txReflectDescriptor) + if whichOneTxModelType == nil { + return "" + } + // Get the value of the selected field in the oneof + fieldValue := tx.Tx.ProtoReflect().Get(whichOneTxModelType) + // Now, fieldValue is a protoreflect.Value, retrieve the txtype field + txtypeFieldDescriptor := fieldValue.Message().Descriptor().Fields().ByName("txtype") + if txtypeFieldDescriptor == nil { + return "" + } + // Get the integer value of txtype as protoreflect.EnumNumber + enumNumber := fieldValue.Message().Get(txtypeFieldDescriptor).Enum() + // Convert the EnumNumber to a string using the EnumType descriptor + return string(txtypeFieldDescriptor.Enum().Values().ByNumber(enumNumber).Name()) +} + // TxKey computes the checksum of the tx func TxKey(tx []byte) [32]byte { return comettypes.Tx(tx).Key()