diff --git a/api/accounts.go b/api/accounts.go index e832da44a..c0ca91bc7 100644 --- a/api/accounts.go +++ b/api/accounts.go @@ -361,32 +361,32 @@ func (a *API) electionListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPContex var pids [][]byte switch ctx.URLParam("status") { case "ready": - pids, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "READY", false) + pids, _, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "READY", false) if err != nil { return ErrCantFetchElectionList.WithErr(err) } case "paused": - pids, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "PAUSED", false) + pids, _, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "PAUSED", false) if err != nil { return ErrCantFetchElectionList.WithErr(err) } case "canceled": - pids, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "CANCELED", false) + pids, _, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "CANCELED", false) if err != nil { return ErrCantFetchElectionList.WithErr(err) } case "ended", "results": - pids, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "RESULTS", false) + pids, _, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "RESULTS", false) if err != nil { return ErrCantFetchElectionList.WithErr(err) } - pids2, err := a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "ENDED", false) + pids2, _, err := a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "ENDED", false) if err != nil { return ErrCantFetchElectionList.WithErr(err) } pids = append(pids, pids2...) case "": - pids, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "", false) + pids, _, err = a.indexer.ProcessList(organizationID, page, MaxPageSize, "", 0, 0, "", false) if err != nil { return ErrCantFetchElectionList.WithErr(err) } diff --git a/api/api_types.go b/api/api_types.go index b5e1c2550..ba72947d8 100644 --- a/api/api_types.go +++ b/api/api_types.go @@ -41,6 +41,7 @@ type ElectionSummary struct { // return empty object if the list does not contains any result type ElectionsList struct { Elections []ElectionSummary `json:"elections"` + Total uint64 `json:"total"` } // ElectionResults is the struct used to wrap the results of an election diff --git a/api/elections.go b/api/elections.go index d2408d305..3dab8817c 100644 --- a/api/elections.go +++ b/api/elections.go @@ -181,12 +181,14 @@ func (a *API) electionFullListHandler(_ *apirest.APIdata, ctx *httprouter.HTTPCo } func (a *API) electionFullListByPage(ctx *httprouter.HTTPContext, page int) error { - elections, err := a.indexer.ProcessList(nil, page*MaxPageSize, MaxPageSize, "", 0, 0, "", false) + elections, total, err := a.indexer.ProcessList(nil, page*MaxPageSize, MaxPageSize, "", 0, 0, "", false) if err != nil { return ErrCantFetchElectionList.WithErr(err) } - list := ElectionsList{} + list := ElectionsList{ + Total: total, + } for _, eid := range elections { e, err := a.indexer.ProcessInfo(eid) if err != nil { @@ -681,7 +683,8 @@ func (a *API) electionFilterPaginatedHandler(msg *apirest.APIdata, ctx *httprout withResults := false body.WithResults = &withResults } - elections, err := a.indexer.ProcessList( + // TODO: use returned total + elections, _, err := a.indexer.ProcessList( body.OrganizationID, page, MaxPageSize, diff --git a/vochain/indexer/db/processes.sql.go b/vochain/indexer/db/processes.sql.go index 6b313659c..ddd6ea6a2 100644 --- a/vochain/indexer/db/processes.sql.go +++ b/vochain/indexer/db/processes.sql.go @@ -262,52 +262,63 @@ func (q *Queries) SearchEntities(ctx context.Context, arg SearchEntitiesParams) } const searchProcesses = `-- name: SearchProcesses :many -SELECT id FROM processes -WHERE (LENGTH(?1) = 0 OR entity_id = ?1) - AND (?2 = 0 OR namespace = ?2) - AND (?3 = 0 OR status = ?3) - AND (?4 = 0 OR source_network_id = ?4) - -- TODO(mvdan): consider keeping an id_hex column for faster searches - AND (?5 = '' OR (INSTR(LOWER(HEX(id)), ?5) > 0)) - AND (?6 = FALSE OR have_results) +WITH filtered_processes AS ( + SELECT id, entity_id, start_date, end_date, vote_count, chain_id, have_results, final_results, results_votes, results_weight, results_block_height, census_root, max_census_size, census_uri, metadata, census_origin, status, namespace, envelope, mode, vote_opts, private_keys, public_keys, question_index, creation_time, source_block_height, source_network_id, manually_ended, + COUNT(*) OVER() AS total_process_count + FROM processes + WHERE (LENGTH(?3) = 0 OR entity_id = ?3) + AND (?4 = 0 OR namespace = ?4) + AND (?5 = 0 OR status = ?5) + AND (?6 = 0 OR source_network_id = ?6) + -- TODO: consider keeping an id_hex column for faster searches + AND (?7 = '' OR (INSTR(LOWER(HEX(id)), ?7) > 0)) + AND (?8 = FALSE OR have_results) +) +SELECT id, total_process_count +FROM filtered_processes ORDER BY creation_time DESC, id ASC -LIMIT ?8 -OFFSET ?7 +LIMIT ?2 +OFFSET ?1 ` type SearchProcessesParams struct { + Offset int64 + Limit int64 EntityID interface{} Namespace interface{} Status interface{} SourceNetworkID interface{} IDSubstr interface{} WithResults interface{} - Offset int64 - Limit int64 } -func (q *Queries) SearchProcesses(ctx context.Context, arg SearchProcessesParams) ([]types.ProcessID, error) { +type SearchProcessesRow struct { + ID []byte + TotalProcessCount int64 +} + +func (q *Queries) SearchProcesses(ctx context.Context, arg SearchProcessesParams) ([]SearchProcessesRow, error) { rows, err := q.query(ctx, q.searchProcessesStmt, searchProcesses, + arg.Offset, + arg.Limit, arg.EntityID, arg.Namespace, arg.Status, arg.SourceNetworkID, arg.IDSubstr, arg.WithResults, - arg.Offset, - arg.Limit, ) if err != nil { return nil, err } defer rows.Close() - var items []types.ProcessID + var items []SearchProcessesRow for rows.Next() { - var id types.ProcessID - if err := rows.Scan(&id); err != nil { + var i SearchProcessesRow + if err := rows.Scan(&i.ID, &i.TotalProcessCount); err != nil { return nil, err } - items = append(items, id) + items = append(items, i) } if err := rows.Close(); err != nil { return nil, err diff --git a/vochain/indexer/indexer_test.go b/vochain/indexer/indexer_test.go index 901769c0e..cca961a14 100644 --- a/vochain/indexer/indexer_test.go +++ b/vochain/indexer/indexer_test.go @@ -324,10 +324,11 @@ func testProcessList(t *testing.T, procsCount int) { procs := make(map[string]bool) last := 0 for len(procs) < procsCount { - list, err := idx.ProcessList(eidProcsCount, last, 10, "", 0, 0, "", false) + list, total, err := idx.ProcessList(eidProcsCount, last, 10, "", 0, 0, "", false) if err != nil { t.Fatal(err) } + qt.Assert(t, total, qt.Equals, uint64(procsCount)) if len(list) < 1 { t.Log("list is empty") break @@ -342,8 +343,9 @@ func testProcessList(t *testing.T, procsCount int) { } qt.Assert(t, procs, qt.HasLen, procsCount) - _, err := idx.ProcessList(nil, 0, 64, "", 0, 0, "", false) + _, total, err := idx.ProcessList(nil, 0, 64, "", 0, 0, "", false) qt.Assert(t, err, qt.IsNil) + qt.Assert(t, total, qt.Equals, uint64(10+procsCount)) qt.Assert(t, idx.CountTotalProcesses(), qt.Equals, uint64(10+procsCount)) countEntityProcs := func(eid []byte) int64 { @@ -356,6 +358,11 @@ func testProcessList(t *testing.T, procsCount int) { qt.Assert(t, countEntityProcs(eidOneProcess), qt.Equals, int64(1)) qt.Assert(t, countEntityProcs(eidProcsCount), qt.Equals, int64(procsCount)) qt.Assert(t, countEntityProcs([]byte("not an entity id that exists")), qt.Equals, int64(-1)) + + // Past the end (from=10000) should return an empty list + emptyList, _, err := idx.ProcessList(nil, 10000, 64, "", 0, 0, "", false) + qt.Assert(t, err, qt.IsNil) + qt.Assert(t, emptyList, qt.DeepEquals, [][]byte{}) } func TestProcessSearch(t *testing.T) { @@ -443,7 +450,7 @@ func TestProcessSearch(t *testing.T) { app.AdvanceTestBlock() // Exact process search - list, err := idx.ProcessList(eidTest, 0, 10, pidExact, 0, 0, "", false) + list, _, err := idx.ProcessList(eidTest, 0, 10, pidExact, 0, 0, "", false) if err != nil { t.Fatal(err) } @@ -452,7 +459,7 @@ func TestProcessSearch(t *testing.T) { } // Exact process search, with it being encrypted. // This once caused a sqlite bug due to a mistake in the SQL query. - list, err = idx.ProcessList(eidTest, 0, 10, pidExactEncrypted, 0, 0, "", false) + list, _, err = idx.ProcessList(eidTest, 0, 10, pidExactEncrypted, 0, 0, "", false) if err != nil { t.Fatal(err) } @@ -460,7 +467,7 @@ func TestProcessSearch(t *testing.T) { t.Fatalf("expected 1 process, got %d", len(list)) } // Search for nonexistent process - list, err = idx.ProcessList(eidTest, 0, 10, + list, _, err = idx.ProcessList(eidTest, 0, 10, "4011d50537fa164b6fef261141797bbe4014526f", 0, 0, "", false) if err != nil { t.Fatal(err) @@ -469,7 +476,7 @@ func TestProcessSearch(t *testing.T) { t.Fatalf("expected 0 processes, got %d", len(list)) } // Search containing part of all manually-defined processes - list, err = idx.ProcessList(eidTest, 0, 10, + list, _, err = idx.ProcessList(eidTest, 0, 10, "011d50537fa164b6fef261141797bbe4014526e", 0, 0, "", false) if err != nil { t.Fatal(err) @@ -478,7 +485,7 @@ func TestProcessSearch(t *testing.T) { t.Fatalf("expected %d processes, got %d", len(processIds), len(list)) } - list, err = idx.ProcessList(eidTest, 0, 100, + list, _, err = idx.ProcessList(eidTest, 0, 100, "0c6ca22d2c175a1fbdd15d7595ae532bb1094b5", 0, 0, "ENDED", false) if err != nil { t.Fatal(err) @@ -489,7 +496,7 @@ func TestProcessSearch(t *testing.T) { // Search with an exact Entity ID, but starting with a null byte. // This can trip up sqlite, as it assumes TEXT strings are NUL-terminated. - list, err = idx.ProcessList([]byte("\x00foobar"), 0, 100, "", 0, 0, "", false) + list, _, err = idx.ProcessList([]byte("\x00foobar"), 0, 100, "", 0, 0, "", false) if err != nil { t.Fatal(err) } @@ -498,12 +505,12 @@ func TestProcessSearch(t *testing.T) { } // list all processes, with a max of 10 - list, err = idx.ProcessList(nil, 0, 10, "", 0, 0, "", false) + list, _, err = idx.ProcessList(nil, 0, 10, "", 0, 0, "", false) qt.Assert(t, err, qt.IsNil) qt.Assert(t, list, qt.HasLen, 10) // list all processes, with a max of 1000 - list, err = idx.ProcessList(nil, 0, 1000, "", 0, 0, "", false) + list, _, err = idx.ProcessList(nil, 0, 1000, "", 0, 0, "", false) qt.Assert(t, err, qt.IsNil) qt.Assert(t, list, qt.HasLen, 21) } @@ -552,25 +559,25 @@ func TestProcessListWithNamespaceAndStatus(t *testing.T) { app.AdvanceTestBlock() // Get the process list for namespace 123 - list, err := idx.ProcessList(eid20, 0, 100, "", 123, 0, "", false) + list, _, err := idx.ProcessList(eid20, 0, 100, "", 123, 0, "", false) qt.Assert(t, err, qt.IsNil) // Check there are exactly 10 qt.Assert(t, len(list), qt.CmpEquals(), 10) // Get the process list for all namespaces - list, err = idx.ProcessList(nil, 0, 100, "", 0, 0, "", false) + list, _, err = idx.ProcessList(nil, 0, 100, "", 0, 0, "", false) qt.Assert(t, err, qt.IsNil) // Check there are exactly 10 + 10 qt.Assert(t, len(list), qt.CmpEquals(), 20) // Get the process list for namespace 10 - list, err = idx.ProcessList(nil, 0, 100, "", 10, 0, "", false) + list, _, err = idx.ProcessList(nil, 0, 100, "", 10, 0, "", false) qt.Assert(t, err, qt.IsNil) // Check there is exactly 1 qt.Assert(t, len(list), qt.CmpEquals(), 1) // Get the process list for namespace 10 - list, err = idx.ProcessList(nil, 0, 100, "", 0, 0, "READY", false) + list, _, err = idx.ProcessList(nil, 0, 100, "", 0, 0, "READY", false) qt.Assert(t, err, qt.IsNil) // Check there is exactly 1 qt.Assert(t, len(list), qt.CmpEquals(), 10) diff --git a/vochain/indexer/process.go b/vochain/indexer/process.go index a20b0aa64..6b4358146 100644 --- a/vochain/indexer/process.go +++ b/vochain/indexer/process.go @@ -48,9 +48,9 @@ func (idx *Indexer) ProcessInfo(pid []byte) (*indexertypes.Process, error) { // declared as zero-values will be ignored. SearchTerm is a partial or full PID. // Status is one of READY, CANCELED, ENDED, PAUSED, RESULTS func (idx *Indexer) ProcessList(entityID []byte, from, max int, searchTerm string, namespace uint32, - srcNetworkId int32, status string, withResults bool) ([][]byte, error) { + srcNetworkId int32, status string, withResults bool) ([][]byte, uint64, error) { if from < 0 { - return nil, fmt.Errorf("processList: invalid value: from is invalid value %d", from) + return nil, 0, fmt.Errorf("processList: invalid value: from is invalid value %d", from) } // For filtering on Status we use a badgerhold match function. // If status is not defined, then the match function will return always true. @@ -58,15 +58,14 @@ func (idx *Indexer) ProcessList(entityID []byte, from, max int, searchTerm strin statusfound := false if status != "" { if statusnum, statusfound = models.ProcessStatus_value[status]; !statusfound { - return nil, fmt.Errorf("processList: status %s is unknown", status) + return nil, 0, fmt.Errorf("processList: status %s is unknown", status) } } // Filter match function for source network Id if _, ok := models.SourceNetworkId_name[srcNetworkId]; !ok { - return nil, fmt.Errorf("sourceNetworkId is unknown %d", srcNetworkId) + return nil, 0, fmt.Errorf("sourceNetworkId is unknown %d", srcNetworkId) } - - procs, err := idx.readOnlyQuery.SearchProcesses(context.TODO(), indexerdb.SearchProcessesParams{ + results, err := idx.readOnlyQuery.SearchProcesses(context.TODO(), indexerdb.SearchProcessesParams{ EntityID: nonNullBytes(entityID), // so that LENGTH never returns NULL Namespace: int64(namespace), Status: int64(statusnum), @@ -77,9 +76,16 @@ func (idx *Indexer) ProcessList(entityID []byte, from, max int, searchTerm strin WithResults: withResults, }) if err != nil { - return nil, err + return nil, 0, err + } + if len(results) == 0 { + return [][]byte{}, 0, nil + } + procs := [][]byte{} + for _, row := range results { + procs = append(procs, row.ID) } - return procs, nil + return procs, uint64(results[0].TotalProcessCount), nil } // CountTotalProcesses returns the total number of processes indexed. diff --git a/vochain/indexer/queries/processes.sql b/vochain/indexer/queries/processes.sql index 8c62c1045..c30e00ad3 100644 --- a/vochain/indexer/queries/processes.sql +++ b/vochain/indexer/queries/processes.sql @@ -31,14 +31,20 @@ WHERE id = ? LIMIT 1; -- name: SearchProcesses :many -SELECT id FROM processes -WHERE (LENGTH(sqlc.arg(entity_id)) = 0 OR entity_id = sqlc.arg(entity_id)) - AND (sqlc.arg(namespace) = 0 OR namespace = sqlc.arg(namespace)) - AND (sqlc.arg(status) = 0 OR status = sqlc.arg(status)) - AND (sqlc.arg(source_network_id) = 0 OR source_network_id = sqlc.arg(source_network_id)) - -- TODO(mvdan): consider keeping an id_hex column for faster searches - AND (sqlc.arg(id_substr) = '' OR (INSTR(LOWER(HEX(id)), sqlc.arg(id_substr)) > 0)) - AND (sqlc.arg(with_results) = FALSE OR have_results) +WITH filtered_processes AS ( + SELECT *, + COUNT(*) OVER() AS total_process_count + FROM processes + WHERE (LENGTH(sqlc.arg(entity_id)) = 0 OR entity_id = sqlc.arg(entity_id)) + AND (sqlc.arg(namespace) = 0 OR namespace = sqlc.arg(namespace)) + AND (sqlc.arg(status) = 0 OR status = sqlc.arg(status)) + AND (sqlc.arg(source_network_id) = 0 OR source_network_id = sqlc.arg(source_network_id)) + -- TODO: consider keeping an id_hex column for faster searches + AND (sqlc.arg(id_substr) = '' OR (INSTR(LOWER(HEX(id)), sqlc.arg(id_substr)) > 0)) + AND (sqlc.arg(with_results) = FALSE OR have_results) +) +SELECT id, total_process_count +FROM filtered_processes ORDER BY creation_time DESC, id ASC LIMIT sqlc.arg(limit) OFFSET sqlc.arg(offset)