From 5730d0cb13b33491a299cca0158cc95d3493e227 Mon Sep 17 00:00:00 2001 From: Gui Iribarren Date: Mon, 26 Aug 2024 20:23:40 +0200 Subject: [PATCH] many fixups (dirty commit, needs squashing) signer field, nonNullBytes, indexer: add subtype and signer todo: add new params to api /chain/transactions --- api/api_types.go | 2 +- api/chain.go | 4 ++- vochain/indexer/db/models.go | 2 ++ vochain/indexer/db/transactions.sql.go | 34 +++++++++++++++---- vochain/indexer/indexertypes/types.go | 5 +++ ...015_alter_columns_table_transactions_2.sql | 7 ++++ vochain/indexer/queries/transactions.sql | 6 ++-- vochain/indexer/transaction.go | 33 +++++++++--------- vochain/transaction/vochaintx/vochaintx.go | 5 ++- 9 files changed, 69 insertions(+), 29 deletions(-) create mode 100644 vochain/indexer/migrations/0015_alter_columns_table_transactions_2.sql diff --git a/api/api_types.go b/api/api_types.go index 3b978a3cb..1b8c89a21 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -295,7 +295,7 @@ type GenericTransactionWithInfo struct { TxContent json.RawMessage `json:"tx"` TxInfo *indexertypes.Transaction `json:"txInfo"` Signature types.HexBytes `json:"signature"` - Account types.HexBytes `json:"account"` + Signer types.HexBytes `json:"signer"` } type ChainInfo struct { diff --git a/api/chain.go b/api/chain.go index 8ef819d72..1b4d5ca14 100644 --- a/api/chain.go +++ b/api/chain.go @@ -698,7 +698,7 @@ func (a *API) chainTxHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContext) er TxContent: []byte(protoFormat(ref.RawTx)), TxInfo: ref, Signature: ref.Signature, - Account: ref.Account, + Signer: ref.Signer, } data, err := json.Marshal(tx) if err != nil { @@ -849,6 +849,8 @@ func (a *API) transactionList(params *TransactionParams) (*TransactionsList, err params.Page*params.Limit, params.Height, params.Type, + "", // TODO(gui): support new params + "", // ) if err != nil { return nil, ErrIndexerQueryFailed.WithErr(err) diff --git a/vochain/indexer/db/models.go b/vochain/indexer/db/models.go index db1778b1c..fab3d35ad 100644 --- a/vochain/indexer/db/models.go +++ b/vochain/indexer/db/models.go @@ -67,4 +67,6 @@ type Transaction struct { Type string RawTx []byte Signature []byte + Subtype string + Signer []byte } diff --git a/vochain/indexer/db/transactions.sql.go b/vochain/indexer/db/transactions.sql.go index a38c1a81a..c8628aebc 100644 --- a/vochain/indexer/db/transactions.sql.go +++ b/vochain/indexer/db/transactions.sql.go @@ -37,9 +37,9 @@ func (q *Queries) CountTransactionsByHeight(ctx context.Context, blockHeight int const createTransaction = `-- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type, raw_tx, signature + hash, block_height, block_index, type, subtype, raw_tx, signature, signer ) VALUES ( - ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ? ) ` @@ -48,8 +48,10 @@ type CreateTransactionParams struct { BlockHeight int64 BlockIndex int64 Type string + Subtype string RawTx []byte Signature []byte + Signer []byte } func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionParams) (sql.Result, error) { @@ -58,13 +60,15 @@ func (q *Queries) CreateTransaction(ctx context.Context, arg CreateTransactionPa arg.BlockHeight, arg.BlockIndex, arg.Type, + arg.Subtype, arg.RawTx, arg.Signature, + arg.Signer, ) } const getTransaction = `-- name: GetTransaction :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer FROM transactions WHERE id = ? LIMIT 1 ` @@ -80,12 +84,14 @@ func (q *Queries) GetTransaction(ctx context.Context, id int64) (Transaction, er &i.Type, &i.RawTx, &i.Signature, + &i.Subtype, + &i.Signer, ) return i, err } const getTransactionByHash = `-- name: GetTransactionByHash :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer FROM transactions WHERE hash = ? LIMIT 1 ` @@ -101,12 +107,14 @@ func (q *Queries) GetTransactionByHash(ctx context.Context, hash types.Hash) (Tr &i.Type, &i.RawTx, &i.Signature, + &i.Subtype, + &i.Signer, ) return i, err } const getTransactionByHeightAndIndex = `-- name: GetTransactionByHeightAndIndex :one -SELECT id, hash, block_height, block_index, type, raw_tx, signature FROM transactions +SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer FROM transactions WHERE block_height = ? AND block_index = ? LIMIT 1 ` @@ -127,20 +135,24 @@ func (q *Queries) GetTransactionByHeightAndIndex(ctx context.Context, arg GetTra &i.Type, &i.RawTx, &i.Signature, + &i.Subtype, + &i.Signer, ) return i, err } const searchTransactions = `-- name: SearchTransactions :many WITH results AS ( - SELECT id, hash, block_height, block_index, type, raw_tx, signature + SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer FROM transactions WHERE ( (?3 = 0 OR block_height = ?3) AND (?4 = '' OR LOWER(type) = LOWER(?4)) + AND (?5 = '' OR LOWER(subtype) = LOWER(?5)) + AND (?6 = '' OR LOWER(HEX(signer)) = LOWER(?6)) ) ) -SELECT id, hash, block_height, block_index, type, raw_tx, signature, COUNT(*) OVER() AS total_count +SELECT id, hash, block_height, block_index, type, raw_tx, signature, subtype, signer, COUNT(*) OVER() AS total_count FROM results ORDER BY id DESC LIMIT ?2 @@ -152,6 +164,8 @@ type SearchTransactionsParams struct { Limit int64 BlockHeight interface{} TxType interface{} + TxSubtype interface{} + TxSigner interface{} } type SearchTransactionsRow struct { @@ -162,6 +176,8 @@ type SearchTransactionsRow struct { Type string RawTx []byte Signature []byte + Subtype string + Signer []byte TotalCount int64 } @@ -171,6 +187,8 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions arg.Limit, arg.BlockHeight, arg.TxType, + arg.TxSubtype, + arg.TxSigner, ) if err != nil { return nil, err @@ -187,6 +205,8 @@ func (q *Queries) SearchTransactions(ctx context.Context, arg SearchTransactions &i.Type, &i.RawTx, &i.Signature, + &i.Subtype, + &i.Signer, &i.TotalCount, ); err != nil { return nil, err diff --git a/vochain/indexer/indexertypes/types.go b/vochain/indexer/indexertypes/types.go index 1f6270596..d46257da6 100644 --- a/vochain/indexer/indexertypes/types.go +++ b/vochain/indexer/indexertypes/types.go @@ -183,6 +183,7 @@ type TransactionMetadata struct { BlockHeight uint32 `json:"blockHeight" format:"int32" example:"64924"` TxBlockIndex int32 `json:"transactionIndex" format:"int32" example:"0"` TxType string `json:"transactionType" enums:"vote,newProcess,admin,setProcess,registerKey,mintTokens,sendTokens,setTransactionCosts,setAccount,collectFaucet,setKeykeeper" example:"Vote"` + TxSubtype string `json:"transactionSubtype" enums:"newProcess,setProcessCensus,setProcessDuration" example:"setProcessCensus"` } func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata { @@ -192,6 +193,7 @@ func TransactionMetadataFromDB(dbtx *indexerdb.Transaction) *TransactionMetadata BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), TxType: dbtx.Type, + TxSubtype: dbtx.Subtype, } } @@ -202,6 +204,7 @@ func TransactionMetadataFromDBRow(dbtx *indexerdb.SearchTransactionsRow) *Transa BlockHeight: uint32(dbtx.BlockHeight), TxBlockIndex: int32(dbtx.BlockIndex), TxType: dbtx.Type, + TxSubtype: dbtx.Subtype, } } @@ -210,6 +213,7 @@ type Transaction struct { *TransactionMetadata RawTx types.HexBytes `json:"-"` Signature types.HexBytes `json:"-"` + Signer types.HexBytes `json:"-"` } // TransactionFromDB converts an indexerdb.Transaction into a Transaction @@ -218,6 +222,7 @@ func TransactionFromDB(dbtx *indexerdb.Transaction) *Transaction { TransactionMetadata: TransactionMetadataFromDB(dbtx), RawTx: dbtx.RawTx, Signature: dbtx.Signature, + Signer: dbtx.Signer, } } diff --git a/vochain/indexer/migrations/0015_alter_columns_table_transactions_2.sql b/vochain/indexer/migrations/0015_alter_columns_table_transactions_2.sql new file mode 100644 index 000000000..b114f56bd --- /dev/null +++ b/vochain/indexer/migrations/0015_alter_columns_table_transactions_2.sql @@ -0,0 +1,7 @@ +-- +goose Up +ALTER TABLE transactions ADD COLUMN subtype TEXT NOT NULL DEFAULT ''; +ALTER TABLE transactions ADD COLUMN signer BLOB NOT NULL DEFAULT x''; + +-- +goose Down +ALTER TABLE transactions DROP COLUMN signer; +ALTER TABLE transactions DROP COLUMN subtype; diff --git a/vochain/indexer/queries/transactions.sql b/vochain/indexer/queries/transactions.sql index 98307ea4c..4466a5387 100644 --- a/vochain/indexer/queries/transactions.sql +++ b/vochain/indexer/queries/transactions.sql @@ -1,8 +1,8 @@ -- name: CreateTransaction :execresult INSERT INTO transactions ( - hash, block_height, block_index, type, raw_tx, signature + hash, block_height, block_index, type, subtype, raw_tx, signature, signer ) VALUES ( - ?, ?, ?, ?, ?, ? + ?, ?, ?, ?, ?, ?, ?, ? ); -- name: GetTransaction :one @@ -34,6 +34,8 @@ WITH results AS ( WHERE ( (sqlc.arg(block_height) = 0 OR block_height = sqlc.arg(block_height)) AND (sqlc.arg(tx_type) = '' OR LOWER(type) = LOWER(sqlc.arg(tx_type))) + AND (sqlc.arg(tx_subtype) = '' OR LOWER(subtype) = LOWER(sqlc.arg(tx_subtype))) + AND (sqlc.arg(tx_signer) = '' OR LOWER(HEX(signer)) = LOWER(sqlc.arg(tx_signer))) ) ) SELECT *, COUNT(*) OVER() AS total_count diff --git a/vochain/indexer/transaction.go b/vochain/indexer/transaction.go index efe8331e3..1cd73fdde 100644 --- a/vochain/indexer/transaction.go +++ b/vochain/indexer/transaction.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + "strings" "go.vocdoni.io/dvote/crypto/ethereum" "go.vocdoni.io/dvote/log" @@ -69,9 +70,9 @@ func (idx *Indexer) GetTransactionByHeightAndIndex(blockHeight, blockIndex int64 } // SearchTransactions returns the list of transactions indexed. -// height and txType are optional, if declared as zero-value will be ignored. +// blockHeight, txType, txSubtype and txSigner are optional, if declared as zero-value will be ignored. // The first one returned is the newest, so they are in descending order. -func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txType string) ([]*indexertypes.TransactionMetadata, uint64, error) { +func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, txType, txSubtype, txSigner string) ([]*indexertypes.TransactionMetadata, uint64, error) { if offset < 0 { return nil, 0, fmt.Errorf("invalid value: offset cannot be %d", offset) } @@ -83,19 +84,15 @@ func (idx *Indexer) SearchTransactions(limit, offset int, blockHeight uint64, tx Offset: int64(offset), BlockHeight: blockHeight, TxType: txType, + TxSubtype: txSubtype, + TxSigner: txSigner, }) if err != nil { return nil, 0, err } list := []*indexertypes.TransactionMetadata{} for _, row := range results { - list = append(list, &indexertypes.TransactionMetadata{ - Index: uint64(row.ID), - Hash: row.Hash, - BlockHeight: uint32(row.BlockHeight), - TxBlockIndex: int32(row.BlockIndex), - TxType: row.Type, - }) + list = append(list, indexertypes.TransactionMetadataFromDBRow(&row)) } if len(results) == 0 { return list, 0, nil @@ -113,12 +110,14 @@ func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) return } - signer, err := ethereum.AddrFromSignature(tx.SignedBody, tx.Signature) - if err != nil { - if len(tx.Signature) > 0 { - log.Warnf("indexing signed tx with empty signer field, can't recover signer from signature: %s", err) + signer := []byte{} + if len(tx.Signature) > 0 { // not all txs are signed, for example zk ones + addr, err := ethereum.AddrFromSignature(tx.SignedBody, tx.Signature) + if err != nil { + log.Errorw(err, "indexer cannot recover signer from signature") + return } - // otherwise ignore, since some txs are not signed by design, for example zk ones + signer = addr.Bytes() } queries := idx.blockTxQueries() @@ -127,10 +126,10 @@ func (idx *Indexer) OnNewTx(tx *vochaintx.Tx, blockHeight uint32, txIndex int32) BlockHeight: int64(blockHeight), BlockIndex: int64(txIndex), Type: tx.TxModelType, + Subtype: strings.ToLower(tx.TxSubType()), RawTx: rawtx, - Signature: tx.Signature, - Subtype: tx.TxSubType(), - Signer: signer.Bytes(), + Signature: nonNullBytes(tx.Signature), + Signer: nonNullBytes(signer), }); err != nil { log.Errorw(err, "cannot index new transaction") } diff --git a/vochain/transaction/vochaintx/vochaintx.go b/vochain/transaction/vochaintx/vochaintx.go index 0c70886b4..e3e30c185 100644 --- a/vochain/transaction/vochaintx/vochaintx.go +++ b/vochain/transaction/vochaintx/vochaintx.go @@ -70,7 +70,10 @@ func (tx *Tx) TxSubType() string { if txtypeFieldDescriptor == nil { return "" } - return fieldValue.Message().Get(txtypeFieldDescriptor).String() + // Get the integer value of txtype as protoreflect.EnumNumber + enumNumber := fieldValue.Message().Get(txtypeFieldDescriptor).Enum() + // Convert the EnumNumber to a string using the EnumType descriptor + return string(txtypeFieldDescriptor.Enum().Values().ByNumber(enumNumber).Name()) } // TxKey computes the checksum of the tx