From 420aee28c0846d3eaf87613289ff7196caa4c11d Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Fri, 6 Sep 2024 11:29:41 +0200 Subject: [PATCH 1/3] indexer: hotfix 0013_recreate_table_transactions turns out, some transactions in stage were indexed twice (same hash but different ID), this likely happened during last chain upgrade. handle this situation gracefully by simply replacing the duplicate entries. --- vochain/indexer/migrations/0013_recreate_table_transactions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vochain/indexer/migrations/0013_recreate_table_transactions.sql b/vochain/indexer/migrations/0013_recreate_table_transactions.sql index bf091d7c9..636f0e2bc 100644 --- a/vochain/indexer/migrations/0013_recreate_table_transactions.sql +++ b/vochain/indexer/migrations/0013_recreate_table_transactions.sql @@ -10,7 +10,7 @@ CREATE TABLE transactions_new ( ); -- Copy data from the old table to the new table -INSERT INTO transactions_new (hash, block_height, block_index, type) +INSERT OR REPLACE INTO transactions_new (hash, block_height, block_index, type) SELECT hash, block_height, block_index, type FROM transactions; From c3cf08ada328710fb6d0b597de700ece301cfa10 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 9 Sep 2024 10:50:25 +0200 Subject: [PATCH 2/3] indexer: optimize idx.BlockList performance turns out, a unified query with `COUNT(*) OVER() AS total_count` is 10x slower than two separate queries `SELECT *` and `SELECT COUNT(*)` also, optimize even further (~1000x) for the most common query: when listing all blocks without filters, don't even count, just return last height the benchmark code used to test is included --- vochain/indexer/bench_test.go | 83 ++++++++++++++++++++++++++++++ vochain/indexer/block.go | 28 ++++++---- vochain/indexer/db/blocks.sql.go | 47 +++++++++++++++-- vochain/indexer/db/db.go | 20 +++++++ vochain/indexer/indexer.go | 2 +- vochain/indexer/queries/blocks.sql | 23 ++++++++- 6 files changed, 186 insertions(+), 17 deletions(-) diff --git a/vochain/indexer/bench_test.go b/vochain/indexer/bench_test.go index 7e289857a..a2c60cf1d 100644 --- a/vochain/indexer/bench_test.go +++ b/vochain/indexer/bench_test.go @@ -2,6 +2,7 @@ package indexer import ( "bytes" + "context" "fmt" "math/big" "sync" @@ -14,6 +15,7 @@ import ( "go.vocdoni.io/dvote/test/testcommon/testutil" "go.vocdoni.io/dvote/util" "go.vocdoni.io/dvote/vochain" + indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" "go.vocdoni.io/dvote/vochain/state" "go.vocdoni.io/dvote/vochain/transaction/vochaintx" "go.vocdoni.io/proto/build/go/models" @@ -197,3 +199,84 @@ func BenchmarkNewProcess(b *testing.B) { log.Infof("indexed %d new processes, took %s", numProcesses, time.Since(startTime)) } + +func BenchmarkBlockList(b *testing.B) { + app := vochain.TestBaseApplication(b) + + idx, err := New(app, Options{DataDir: b.TempDir()}) + qt.Assert(b, err, qt.IsNil) + + count := 100000 + + createDummyBlocks(b, idx, count) + + b.ReportAllocs() + b.ResetTimer() + + benchmarkBlockList := func(b *testing.B, + limit int, offset int, chainID string, hash string, proposerAddress string, + ) { + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + blocks, total, err := idx.BlockList(limit, offset, chainID, hash, proposerAddress) + qt.Assert(b, err, qt.IsNil) + qt.Assert(b, blocks, qt.HasLen, limit, qt.Commentf("%+v", blocks)) + qt.Assert(b, blocks[0].TxCount, qt.Equals, int64(0)) + qt.Assert(b, total, qt.Equals, uint64(count)) + } + } + + // Run sub-benchmarks with different limits and filters + b.Run("BlockListLimit1", func(b *testing.B) { + benchmarkBlockList(b, 1, 0, "", "", "") + }) + + b.Run("BlockListLimit10", func(b *testing.B) { + benchmarkBlockList(b, 10, 0, "", "", "") + }) + + b.Run("BlockListLimit100", func(b *testing.B) { + benchmarkBlockList(b, 100, 0, "", "", "") + }) + + b.Run("BlockListOffset", func(b *testing.B) { + benchmarkBlockList(b, 10, count/2, "", "", "") + }) + + b.Run("BlockListWithChainID", func(b *testing.B) { + benchmarkBlockList(b, 10, 0, "test", "", "") + }) + + b.Run("BlockListWithHashSubstr", func(b *testing.B) { + benchmarkBlockList(b, 10, 0, "", "cafe", "") + }) + b.Run("BlockListWithHashExact", func(b *testing.B) { + benchmarkBlockList(b, 10, 0, "", "cafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe", "") + }) +} + +func createDummyBlocks(b *testing.B, idx *Indexer, n int) { + idx.blockMu.Lock() + defer idx.blockMu.Unlock() + + queries := idx.blockTxQueries() + for h := 1; h <= n; h++ { + _, err := queries.CreateBlock(context.TODO(), indexerdb.CreateBlockParams{ + ChainID: "test", + Height: int64(h), + Time: time.Now(), + Hash: nonNullBytes([]byte{ + 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, + 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, + }), + ProposerAddress: nonNullBytes([]byte{0xfe, 0xde}), + LastBlockHash: nonNullBytes([]byte{0xca, 0xfe}), + }, + ) + qt.Assert(b, err, qt.IsNil) + } + err := idx.blockTx.Commit() + qt.Assert(b, err, qt.IsNil) +} diff --git a/vochain/indexer/block.go b/vochain/indexer/block.go index c6021e530..b6528e890 100644 --- a/vochain/indexer/block.go +++ b/vochain/indexer/block.go @@ -71,22 +71,30 @@ func (idx *Indexer) BlockList(limit, offset int, chainID, hash, proposerAddress for _, row := range results { list = append(list, indexertypes.BlockFromDBRow(&row)) } - if len(results) == 0 { - return list, 0, nil + count, err := idx.CountBlocks(chainID, hash, proposerAddress) + if err != nil { + return nil, 0, err } - return list, uint64(results[0].TotalCount), nil + return list, count, nil } // CountBlocks returns how many blocks are indexed. -func (idx *Indexer) CountBlocks() (uint64, error) { - results, err := idx.readOnlyQuery.SearchBlocks(context.TODO(), indexerdb.SearchBlocksParams{ - Limit: 1, +// If all args passed are empty ("") it will return the last block height, as an optimization. +func (idx *Indexer) CountBlocks(chainID, hash, proposerAddress string) (uint64, error) { + if chainID == "" && hash == "" && proposerAddress == "" { + count, err := idx.readOnlyQuery.LastBlockHeight(context.TODO()) + if err != nil { + return 0, err + } + return uint64(count), nil + } + count, err := idx.readOnlyQuery.CountBlocks(context.TODO(), indexerdb.CountBlocksParams{ + ChainID: chainID, + HashSubstr: hash, + ProposerAddress: proposerAddress, }) if err != nil { return 0, err } - if len(results) == 0 { - return 0, nil - } - return uint64(results[0].TotalCount), nil + return uint64(count), nil } diff --git a/vochain/indexer/db/blocks.sql.go b/vochain/indexer/db/blocks.sql.go index 669f59602..a57f31c6b 100644 --- a/vochain/indexer/db/blocks.sql.go +++ b/vochain/indexer/db/blocks.sql.go @@ -11,6 +11,35 @@ import ( "time" ) +const countBlocks = `-- name: CountBlocks :one +SELECT COUNT(*) +FROM blocks AS b +WHERE ( + (?1 = '' OR b.chain_id = ?1) + AND LENGTH(?2) <= 64 -- if passed arg is longer, then just abort the query + AND ( + ?2 = '' + OR (LENGTH(?2) = 64 AND LOWER(HEX(b.hash)) = LOWER(?2)) + OR (LENGTH(?2) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(?2)) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (?3 = '' OR LOWER(HEX(b.proposer_address)) = LOWER(?3)) +) +` + +type CountBlocksParams struct { + ChainID interface{} + HashSubstr interface{} + ProposerAddress interface{} +} + +func (q *Queries) CountBlocks(ctx context.Context, arg CountBlocksParams) (int64, error) { + row := q.queryRow(ctx, q.countBlocksStmt, countBlocks, arg.ChainID, arg.HashSubstr, arg.ProposerAddress) + var count int64 + err := row.Scan(&count) + return count, err +} + const createBlock = `-- name: CreateBlock :execresult INSERT INTO blocks( chain_id, height, time, hash, proposer_address, last_block_hash @@ -85,11 +114,23 @@ func (q *Queries) GetBlockByHeight(ctx context.Context, height int64) (Block, er return i, err } +const lastBlockHeight = `-- name: LastBlockHeight :one +SELECT height FROM blocks +ORDER BY height DESC +LIMIT 1 +` + +func (q *Queries) LastBlockHeight(ctx context.Context) (int64, error) { + row := q.queryRow(ctx, q.lastBlockHeightStmt, lastBlockHeight) + var height int64 + err := row.Scan(&height) + return height, err +} + const searchBlocks = `-- name: SearchBlocks :many SELECT b.height, b.time, b.chain_id, b.hash, b.proposer_address, b.last_block_hash, - COUNT(t.block_index) AS tx_count, - COUNT(*) OVER() AS total_count + COUNT(t.block_index) AS tx_count FROM blocks AS b LEFT JOIN transactions AS t ON b.height = t.block_height @@ -126,7 +167,6 @@ type SearchBlocksRow struct { ProposerAddress []byte LastBlockHash []byte TxCount int64 - TotalCount int64 } func (q *Queries) SearchBlocks(ctx context.Context, arg SearchBlocksParams) ([]SearchBlocksRow, error) { @@ -152,7 +192,6 @@ func (q *Queries) SearchBlocks(ctx context.Context, arg SearchBlocksParams) ([]S &i.ProposerAddress, &i.LastBlockHash, &i.TxCount, - &i.TotalCount, ); err != nil { return nil, err } diff --git a/vochain/indexer/db/db.go b/vochain/indexer/db/db.go index 47bf88d46..064c63538 100644 --- a/vochain/indexer/db/db.go +++ b/vochain/indexer/db/db.go @@ -30,6 +30,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.countAccountsStmt, err = db.PrepareContext(ctx, countAccounts); err != nil { return nil, fmt.Errorf("error preparing query CountAccounts: %w", err) } + if q.countBlocksStmt, err = db.PrepareContext(ctx, countBlocks); err != nil { + return nil, fmt.Errorf("error preparing query CountBlocks: %w", err) + } if q.countTokenTransfersByAccountStmt, err = db.PrepareContext(ctx, countTokenTransfersByAccount); err != nil { return nil, fmt.Errorf("error preparing query CountTokenTransfersByAccount: %w", err) } @@ -96,6 +99,9 @@ func Prepare(ctx context.Context, db DBTX) (*Queries, error) { if q.getVoteStmt, err = db.PrepareContext(ctx, getVote); err != nil { return nil, fmt.Errorf("error preparing query GetVote: %w", err) } + if q.lastBlockHeightStmt, err = db.PrepareContext(ctx, lastBlockHeight); err != nil { + return nil, fmt.Errorf("error preparing query LastBlockHeight: %w", err) + } if q.searchAccountsStmt, err = db.PrepareContext(ctx, searchAccounts); err != nil { return nil, fmt.Errorf("error preparing query SearchAccounts: %w", err) } @@ -153,6 +159,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing countAccountsStmt: %w", cerr) } } + if q.countBlocksStmt != nil { + if cerr := q.countBlocksStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing countBlocksStmt: %w", cerr) + } + } if q.countTokenTransfersByAccountStmt != nil { if cerr := q.countTokenTransfersByAccountStmt.Close(); cerr != nil { err = fmt.Errorf("error closing countTokenTransfersByAccountStmt: %w", cerr) @@ -263,6 +274,11 @@ func (q *Queries) Close() error { err = fmt.Errorf("error closing getVoteStmt: %w", cerr) } } + if q.lastBlockHeightStmt != nil { + if cerr := q.lastBlockHeightStmt.Close(); cerr != nil { + err = fmt.Errorf("error closing lastBlockHeightStmt: %w", cerr) + } + } if q.searchAccountsStmt != nil { if cerr := q.searchAccountsStmt.Close(); cerr != nil { err = fmt.Errorf("error closing searchAccountsStmt: %w", cerr) @@ -374,6 +390,7 @@ type Queries struct { tx *sql.Tx computeProcessVoteCountStmt *sql.Stmt countAccountsStmt *sql.Stmt + countBlocksStmt *sql.Stmt countTokenTransfersByAccountStmt *sql.Stmt countTransactionsStmt *sql.Stmt countTransactionsByHeightStmt *sql.Stmt @@ -396,6 +413,7 @@ type Queries struct { getTransactionByHashStmt *sql.Stmt getTransactionByHeightAndIndexStmt *sql.Stmt getVoteStmt *sql.Stmt + lastBlockHeightStmt *sql.Stmt searchAccountsStmt *sql.Stmt searchBlocksStmt *sql.Stmt searchEntitiesStmt *sql.Stmt @@ -418,6 +436,7 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { tx: tx, computeProcessVoteCountStmt: q.computeProcessVoteCountStmt, countAccountsStmt: q.countAccountsStmt, + countBlocksStmt: q.countBlocksStmt, countTokenTransfersByAccountStmt: q.countTokenTransfersByAccountStmt, countTransactionsStmt: q.countTransactionsStmt, countTransactionsByHeightStmt: q.countTransactionsByHeightStmt, @@ -440,6 +459,7 @@ func (q *Queries) WithTx(tx *sql.Tx) *Queries { getTransactionByHashStmt: q.getTransactionByHashStmt, getTransactionByHeightAndIndexStmt: q.getTransactionByHeightAndIndexStmt, getVoteStmt: q.getVoteStmt, + lastBlockHeightStmt: q.lastBlockHeightStmt, searchAccountsStmt: q.searchAccountsStmt, searchBlocksStmt: q.searchBlocksStmt, searchEntitiesStmt: q.searchEntitiesStmt, diff --git a/vochain/indexer/indexer.go b/vochain/indexer/indexer.go index b764c8812..4d6710560 100644 --- a/vochain/indexer/indexer.go +++ b/vochain/indexer/indexer.go @@ -444,7 +444,7 @@ func (idx *Indexer) ReindexBlocks(inTest bool) { return } - idxBlockCount, err := idx.CountBlocks() + idxBlockCount, err := idx.CountBlocks("", "", "") if err != nil { log.Warnf("indexer CountBlocks returned error: %s", err) } diff --git a/vochain/indexer/queries/blocks.sql b/vochain/indexer/queries/blocks.sql index d15332cdf..98533e60c 100644 --- a/vochain/indexer/queries/blocks.sql +++ b/vochain/indexer/queries/blocks.sql @@ -21,11 +21,15 @@ SELECT * FROM blocks WHERE hash = ? LIMIT 1; +-- name: LastBlockHeight :one +SELECT height FROM blocks +ORDER BY height DESC +LIMIT 1; + -- name: SearchBlocks :many SELECT b.*, - COUNT(t.block_index) AS tx_count, - COUNT(*) OVER() AS total_count + COUNT(t.block_index) AS tx_count FROM blocks AS b LEFT JOIN transactions AS t ON b.height = t.block_height @@ -44,3 +48,18 @@ GROUP BY b.height ORDER BY b.height DESC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset); + +-- name: CountBlocks :one +SELECT COUNT(*) +FROM blocks AS b +WHERE ( + (sqlc.arg(chain_id) = '' OR b.chain_id = sqlc.arg(chain_id)) + AND LENGTH(sqlc.arg(hash_substr)) <= 64 -- if passed arg is longer, then just abort the query + AND ( + sqlc.arg(hash_substr) = '' + OR (LENGTH(sqlc.arg(hash_substr)) = 64 AND LOWER(HEX(b.hash)) = LOWER(sqlc.arg(hash_substr))) + OR (LENGTH(sqlc.arg(hash_substr)) < 64 AND INSTR(LOWER(HEX(b.hash)), LOWER(sqlc.arg(hash_substr))) > 0) + -- TODO: consider keeping an hash_hex column for faster searches + ) + AND (sqlc.arg(proposer_address) = '' OR LOWER(HEX(b.proposer_address)) = LOWER(sqlc.arg(proposer_address))) +); From 39404633e6ed10e77bd953c98206d1c7465ed23a Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 9 Sep 2024 13:04:12 +0200 Subject: [PATCH 3/3] api: hotfix /elections 'organization not found' error due to how the indexer is organized, if an organization hasn't created yet any election, EntityExists returns false although the account does exist. --- api/elections.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/elections.go b/api/elections.go index 9e921f616..5548a48b6 100644 --- a/api/elections.go +++ b/api/elections.go @@ -292,7 +292,7 @@ func (a *API) electionListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContex // // Errors returned are always of type APIerror. func (a *API) electionList(params *ElectionParams) (*ElectionsList, error) { - if params.OrganizationID != "" && !a.indexer.EntityExists(params.OrganizationID) { + if params.OrganizationID != "" && !a.indexer.AccountExists(params.OrganizationID) { return nil, ErrOrgNotFound }