From 4c9bc7b44ff7062e77e940cdedc9475b642d8d0b Mon Sep 17 00:00:00 2001 From: iuwqyir Date: Tue, 14 Jan 2025 01:44:04 +0200 Subject: [PATCH] automated clickhouse migrations --- cmd/orchestrator.go | 6 + db/README.md | 14 ++ .../main/20250114110057_init.down.sql | 4 + .../main/20250114110057_init.up.sql | 142 ++++++++++++++++++ .../orchestrator/20250114110046_init.down.sql | 2 + .../orchestrator/20250114110046_init.up.sql | 22 +++ .../staging/20250114110054_init.down.sql | 1 + .../staging/20250114110054_init.up.sql | 5 +- db/migrations.go | 139 +++++++++++++++++ go.mod | 9 +- go.sum | 25 ++- ...clickhouse_create_block_failures_table.sql | 12 -- .../tools/clickhouse_create_blocks_table.sql | 31 ---- .../tools/clickhouse_create_cursors_table.sql | 7 - .../tools/clickhouse_create_logs_table.sql | 28 ---- .../tools/clickhouse_create_traces_table.sql | 34 ----- .../clickhouse_create_transactions_table.sql | 42 ------ 17 files changed, 359 insertions(+), 164 deletions(-) create mode 100644 db/README.md create mode 100644 db/ch_migrations/main/20250114110057_init.down.sql create mode 100644 db/ch_migrations/main/20250114110057_init.up.sql create mode 100644 db/ch_migrations/orchestrator/20250114110046_init.down.sql create mode 100644 db/ch_migrations/orchestrator/20250114110046_init.up.sql create mode 100644 db/ch_migrations/staging/20250114110054_init.down.sql rename internal/tools/clickhouse_create_staging_table.sql => db/ch_migrations/staging/20250114110054_init.up.sql (72%) create mode 100644 db/migrations.go delete mode 100644 internal/tools/clickhouse_create_block_failures_table.sql delete mode 100644 internal/tools/clickhouse_create_blocks_table.sql delete mode 100644 internal/tools/clickhouse_create_cursors_table.sql delete mode 100644 internal/tools/clickhouse_create_logs_table.sql delete mode 100644 internal/tools/clickhouse_create_traces_table.sql delete mode 100644 internal/tools/clickhouse_create_transactions_table.sql diff --git a/cmd/orchestrator.go b/cmd/orchestrator.go index 84665df..ae1c8e9 100644 --- a/cmd/orchestrator.go +++ b/cmd/orchestrator.go @@ -6,6 +6,7 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog/log" "github.com/spf13/cobra" + "github.com/thirdweb-dev/indexer/db" "github.com/thirdweb-dev/indexer/internal/orchestrator" "github.com/thirdweb-dev/indexer/internal/rpc" ) @@ -28,6 +29,11 @@ func RunOrchestrator(cmd *cobra.Command, args []string) { log.Fatal().Err(err).Msg("Failed to initialize RPC") } + err = db.RunMigrations() + if err != nil { + log.Fatal().Err(err).Msg("Failed to run migrations") + } + orchestrator, err := orchestrator.NewOrchestrator(rpc) if err != nil { log.Fatal().Err(err).Msg("Failed to create orchestrator") diff --git a/db/README.md b/db/README.md new file mode 100644 index 0000000..9a73696 --- /dev/null +++ b/db/README.md @@ -0,0 +1,14 @@ +# Insight DB migrations + +## Clickhouse + +Migrations are managed using [golang-migrate](https://github.com/golang-migrate/migrate) and the migration files are located in the `ch_migrations` directory. + +Each storage type (orchestrator, staging and main) has its own migrations. + +To add a new migration, run +``` +migrate create -ext sql -dir db/ch_migrations/ +``` + +Migrations are run when the indexer is started. diff --git a/db/ch_migrations/main/20250114110057_init.down.sql b/db/ch_migrations/main/20250114110057_init.down.sql new file mode 100644 index 0000000..7cd5f69 --- /dev/null +++ b/db/ch_migrations/main/20250114110057_init.down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS blocks; +DROP TABLE IF EXISTS logs; +DROP TABLE IF EXISTS transactions; +DROP TABLE IF EXISTS traces; diff --git a/db/ch_migrations/main/20250114110057_init.up.sql b/db/ch_migrations/main/20250114110057_init.up.sql new file mode 100644 index 0000000..f3847a4 --- /dev/null +++ b/db/ch_migrations/main/20250114110057_init.up.sql @@ -0,0 +1,142 @@ +-- create blocks table +CREATE TABLE IF NOT EXISTS blocks ( + `chain_id` UInt256, + `number` UInt256, + `timestamp` UInt64 CODEC(Delta, ZSTD), + `hash` FixedString(66), + `parent_hash` FixedString(66), + `sha3_uncles` FixedString(66), + `nonce` FixedString(18), + `mix_hash` FixedString(66), + `miner` FixedString(42), + `state_root` FixedString(66), + `transactions_root` FixedString(66), + `receipts_root` FixedString(66), + `logs_bloom` String, + `size` UInt64, + `extra_data` String, + `difficulty` UInt256, + `total_difficulty` UInt256, + `transaction_count` UInt64, + `gas_limit` UInt256, + `gas_used` UInt256, + `withdrawals_root` Nullable(FixedString(66)), + `base_fee_per_gas` Nullable(UInt64), + `insert_timestamp` DateTime DEFAULT now(), + `is_deleted` UInt8 DEFAULT 0, + INDEX idx_timestamp timestamp TYPE minmax GRANULARITY 1, + INDEX idx_hash hash TYPE bloom_filter GRANULARITY 1, +) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted) +ORDER BY (chain_id, number) +PARTITION BY chain_id +SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; + +-- create logs table +CREATE TABLE IF NOT EXISTS logs ( + `chain_id` UInt256, + `block_number` UInt256, + `block_hash` FixedString(66), + `block_timestamp` UInt64 CODEC(Delta, ZSTD), + `transaction_hash` FixedString(66), + `transaction_index` UInt64, + `log_index` UInt64, + `address` FixedString(42), + `data` String, + `topic_0` String, + `topic_1` Nullable(String), + `topic_2` Nullable(String), + `topic_3` Nullable(String), + `insert_timestamp` DateTime DEFAULT now(), + `is_deleted` UInt8 DEFAULT 0, + INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1, + INDEX idx_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 1, + INDEX idx_block_hash block_hash TYPE bloom_filter GRANULARITY 1, + INDEX idx_address address TYPE bloom_filter GRANULARITY 1, + INDEX idx_topic0 topic_0 TYPE bloom_filter GRANULARITY 1, + INDEX idx_topic1 topic_1 TYPE bloom_filter GRANULARITY 1, + INDEX idx_topic2 topic_2 TYPE bloom_filter GRANULARITY 1, + INDEX idx_topic3 topic_3 TYPE bloom_filter GRANULARITY 1, +) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted) +ORDER BY (chain_id, block_number, transaction_hash, log_index) +PARTITION BY chain_id +SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; + +-- create transactions table +CREATE TABLE IF NOT EXISTS transactions ( + `chain_id` UInt256, + `hash` FixedString(66), + `nonce` UInt64, + `block_hash` FixedString(66), + `block_number` UInt256, + `block_timestamp` UInt64 CODEC(Delta, ZSTD), + `transaction_index` UInt64, + `from_address` FixedString(42), + `to_address` FixedString(42), + `value` UInt256, + `gas` UInt64, + `gas_price` UInt256, + `data` String, + `function_selector` FixedString(10), + `max_fee_per_gas` UInt128, + `max_priority_fee_per_gas` UInt128, + `transaction_type` UInt8, + `r` UInt256, + `s` UInt256, + `v` UInt256, + `access_list` Nullable(String), + `contract_address` Nullable(FixedString(42)), + `gas_used` Nullable(UInt64), + `cumulative_gas_used` Nullable(UInt64), + `effective_gas_price` Nullable(UInt256), + `blob_gas_used` Nullable(UInt64), + `blob_gas_price` Nullable(UInt256), + `logs_bloom` Nullable(String), + `status` Nullable(UInt64), + `is_deleted` UInt8 DEFAULT 0, + `insert_timestamp` DateTime DEFAULT now(), + INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1, + INDEX idx_block_hash block_hash TYPE bloom_filter GRANULARITY 1, + INDEX idx_hash hash TYPE bloom_filter GRANULARITY 1, + INDEX idx_from_address from_address TYPE bloom_filter GRANULARITY 1, + INDEX idx_to_address to_address TYPE bloom_filter GRANULARITY 1, + INDEX idx_function_selector function_selector TYPE bloom_filter GRANULARITY 1, +) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted) +ORDER BY (chain_id, block_number, hash) +PARTITION BY chain_id +SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; + +-- create traces table +CREATE TABLE IF NOT EXISTS traces ( + `chain_id` UInt256, + `block_number` UInt256, + `block_hash` FixedString(66), + `block_timestamp` UInt64 CODEC(Delta, ZSTD), + `transaction_hash` FixedString(66), + `transaction_index` UInt64, + `subtraces` Int64, + `trace_address` Array(Int64), + `type` LowCardinality(String), + `call_type` LowCardinality(String), + `error` Nullable(String), + `from_address` FixedString(42), + `to_address` FixedString(42), + `gas` UInt64, + `gas_used` UInt64, + `input` String, + `output` Nullable(String), + `value` UInt256, + `author` Nullable(FixedString(42)), + `reward_type` LowCardinality(Nullable(String)), + `refund_address` Nullable(FixedString(42)), + `is_deleted` UInt8 DEFAULT 0, + `insert_timestamp` DateTime DEFAULT now(), + INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1, + INDEX idx_block_hash block_hash TYPE bloom_filter GRANULARITY 1, + INDEX idx_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 1, + INDEX idx_from_address from_address TYPE bloom_filter GRANULARITY 1, + INDEX idx_to_address to_address TYPE bloom_filter GRANULARITY 1, + INDEX idx_type type TYPE bloom_filter GRANULARITY 1, +) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted) +ORDER BY (chain_id, block_number, transaction_hash, trace_address) +PARTITION BY chain_id +SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; diff --git a/db/ch_migrations/orchestrator/20250114110046_init.down.sql b/db/ch_migrations/orchestrator/20250114110046_init.down.sql new file mode 100644 index 0000000..5a8c346 --- /dev/null +++ b/db/ch_migrations/orchestrator/20250114110046_init.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS block_failures; +DROP TABLE IF EXISTS cursors; diff --git a/db/ch_migrations/orchestrator/20250114110046_init.up.sql b/db/ch_migrations/orchestrator/20250114110046_init.up.sql new file mode 100644 index 0000000..841671b --- /dev/null +++ b/db/ch_migrations/orchestrator/20250114110046_init.up.sql @@ -0,0 +1,22 @@ +-- create block failures table +CREATE TABLE IF NOT EXISTS block_failures ( + `chain_id` UInt256, + `block_number` UInt256, + `last_error_timestamp` UInt64 CODEC(Delta, ZSTD), + `count` UInt16, + `reason` String, + `insert_timestamp` DateTime DEFAULT now(), + `is_deleted` UInt8 DEFAULT 0, + INDEX idx_block_number block_number TYPE minmax GRANULARITY 1, +) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted) +ORDER BY (chain_id, block_number) +SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; + +-- create cursors table +CREATE TABLE IF NOT EXISTS cursors ( + `chain_id` UInt256, + `cursor_type` String, + `cursor_value` String, + `insert_timestamp` DateTime DEFAULT now(), +) ENGINE = ReplacingMergeTree(insert_timestamp) +ORDER BY (chain_id, cursor_type); diff --git a/db/ch_migrations/staging/20250114110054_init.down.sql b/db/ch_migrations/staging/20250114110054_init.down.sql new file mode 100644 index 0000000..1070b1d --- /dev/null +++ b/db/ch_migrations/staging/20250114110054_init.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS block_data; diff --git a/internal/tools/clickhouse_create_staging_table.sql b/db/ch_migrations/staging/20250114110054_init.up.sql similarity index 72% rename from internal/tools/clickhouse_create_staging_table.sql rename to db/ch_migrations/staging/20250114110054_init.up.sql index fdcfe0c..a218042 100644 --- a/internal/tools/clickhouse_create_staging_table.sql +++ b/db/ch_migrations/staging/20250114110054_init.up.sql @@ -1,4 +1,5 @@ -CREATE TABLE block_data ( +-- create staging table +CREATE TABLE IF NOT EXISTS block_data ( `chain_id` UInt256, `block_number` UInt256, `data` String, @@ -8,4 +9,4 @@ CREATE TABLE block_data ( ) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted) ORDER BY (chain_id, block_number) PARTITION BY chain_id -SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; \ No newline at end of file +SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; diff --git a/db/migrations.go b/db/migrations.go new file mode 100644 index 0000000..abab717 --- /dev/null +++ b/db/migrations.go @@ -0,0 +1,139 @@ +package db + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/clickhouse" + _ "github.com/golang-migrate/migrate/v4/source/file" + "github.com/rs/zerolog/log" + config "github.com/thirdweb-dev/indexer/configs" +) + +func RunMigrations() error { + storageConfigs := []struct { + Type string + Config *config.ClickhouseConfig + }{ + {Type: "orchestrator", Config: config.Cfg.Storage.Orchestrator.Clickhouse}, + {Type: "staging", Config: config.Cfg.Storage.Staging.Clickhouse}, + {Type: "main", Config: config.Cfg.Storage.Main.Clickhouse}, + } + + groupedConfigs := make(map[string][]struct { + Type string + Config *config.ClickhouseConfig + }) + for _, cfg := range storageConfigs { + key := fmt.Sprintf("%s:%d:%s", cfg.Config.Host, cfg.Config.Port, cfg.Config.Database) + groupedConfigs[key] = append(groupedConfigs[key], cfg) + } + + removeTmpMigrations() // just in case + for _, cfgs := range groupedConfigs { + var types []string + for _, cfg := range cfgs { + copyMigrationsToTmp([]string{"db/ch_migrations/" + cfg.Type}) + types = append(types, cfg.Type) + } + log.Info().Msgf("Running Clickhouse migrations for %s", strings.Join(types, ", ")) + runClickhouseMigrations(cfgs[0].Config) + removeTmpMigrations() + log.Info().Msgf("Clickhouse migration completed for %s", strings.Join(types, ", ")) + } + + log.Info().Msg("All Clickhouse migrations completed") + + return nil +} + +func runClickhouseMigrations(cfg *config.ClickhouseConfig) error { + if cfg.Host == "" { + return nil + } + + secureParam := "&secure=true" + if cfg.DisableTLS { + secureParam = "&secure=false" + } + + url := fmt.Sprintf("clickhouse://%s:%d/%s?username=%s&password=%s%s&x-multi-statement=true&x-migrations-table-engine=MergeTree", + cfg.Host, + cfg.Port, + cfg.Database, + cfg.Username, + cfg.Password, + secureParam, + ) + + m, err := migrate.New("file://db/ch_migrations/tmp", url) + if err != nil { + return err + } + m.Up() + + m.Close() + + return nil +} + +func copyMigrationsToTmp(sources []string) error { + destination := "db/ch_migrations/tmp" + err := os.MkdirAll(destination, os.ModePerm) + if err != nil { + return err + } + + for _, source := range sources { + filepath.Walk(source, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + // skip directories + if info.IsDir() { + return nil + } + // determine destination path + relPath, err := filepath.Rel(source, path) + if err != nil { + return err + } + destPath := filepath.Join(destination, relPath) + if err := os.MkdirAll(filepath.Dir(destPath), os.ModePerm); err != nil { + return err + } + + return copyFile(path, destPath) + }) + } + return nil +} + +func copyFile(src string, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + dstFile, err := os.Create(dst) + if err != nil { + return err + } + defer dstFile.Close() + + _, err = io.Copy(dstFile, srcFile) + return err +} + +func removeTmpMigrations() error { + err := os.RemoveAll("db/ch_migrations/tmp") + if err != nil { + return fmt.Errorf("error removing directory: %w", err) + } + return nil +} diff --git a/go.mod b/go.mod index e174c4b..4091c3c 100644 --- a/go.mod +++ b/go.mod @@ -7,8 +7,10 @@ require ( github.com/ethereum/go-ethereum v1.14.8 github.com/gin-gonic/gin v1.10.0 github.com/go-redis/redis/v8 v8.11.5 + github.com/golang-migrate/migrate/v4 v4.18.1 github.com/gorilla/schema v1.4.1 github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/lib/pq v1.10.9 github.com/prometheus/client_golang v1.20.4 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.1 @@ -57,6 +59,8 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.3.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -100,8 +104,9 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect github.com/urfave/cli/v2 v2.27.4 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect - go.opentelemetry.io/otel v1.26.0 // indirect - go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.opentelemetry.io/otel v1.29.0 // indirect + go.opentelemetry.io/otel/trace v1.29.0 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect golang.org/x/crypto v0.27.0 // indirect diff --git a/go.sum b/go.sum index 00e34e6..792eb44 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-migrate/migrate/v4 v4.18.1 h1:JML/k+t4tpHCpQTCAD62Nu43NUFzHY4CV3uAuvHGC+Y= +github.com/golang-migrate/migrate/v4 v4.18.1/go.mod h1:HAX6m3sQgcdO81tdjn5exv20+3Kb13cmGli1hrD6hks= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= @@ -132,6 +134,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -139,8 +143,13 @@ github.com/gorilla/schema v1.4.1 h1:jUg5hUjCSDZpNGLuXQOgIWGdlgrIdYvgQ0wZtdK1M3E= github.com/gorilla/schema v1.4.1/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -183,6 +192,8 @@ github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -240,8 +251,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -323,10 +334,12 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= +go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= +go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= +go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= diff --git a/internal/tools/clickhouse_create_block_failures_table.sql b/internal/tools/clickhouse_create_block_failures_table.sql deleted file mode 100644 index 1333d72..0000000 --- a/internal/tools/clickhouse_create_block_failures_table.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE TABLE block_failures ( - `chain_id` UInt256, - `block_number` UInt256, - `last_error_timestamp` UInt64 CODEC(Delta, ZSTD), - `count` UInt16, - `reason` String, - `insert_timestamp` DateTime DEFAULT now(), - `is_deleted` UInt8 DEFAULT 0, - INDEX idx_block_number block_number TYPE minmax GRANULARITY 1, -) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted) -ORDER BY (chain_id, block_number) -SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; \ No newline at end of file diff --git a/internal/tools/clickhouse_create_blocks_table.sql b/internal/tools/clickhouse_create_blocks_table.sql deleted file mode 100644 index 7a03300..0000000 --- a/internal/tools/clickhouse_create_blocks_table.sql +++ /dev/null @@ -1,31 +0,0 @@ -CREATE TABLE blocks ( - `chain_id` UInt256, - `number` UInt256, - `timestamp` UInt64 CODEC(Delta, ZSTD), - `hash` FixedString(66), - `parent_hash` FixedString(66), - `sha3_uncles` FixedString(66), - `nonce` FixedString(18), - `mix_hash` FixedString(66), - `miner` FixedString(42), - `state_root` FixedString(66), - `transactions_root` FixedString(66), - `receipts_root` FixedString(66), - `logs_bloom` String, - `size` UInt64, - `extra_data` String, - `difficulty` UInt256, - `total_difficulty` UInt256, - `transaction_count` UInt64, - `gas_limit` UInt256, - `gas_used` UInt256, - `withdrawals_root` Nullable(FixedString(66)), - `base_fee_per_gas` Nullable(UInt64), - `insert_timestamp` DateTime DEFAULT now(), - `is_deleted` UInt8 DEFAULT 0, - INDEX idx_timestamp timestamp TYPE minmax GRANULARITY 1, - INDEX idx_hash hash TYPE bloom_filter GRANULARITY 1, -) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted) -ORDER BY (chain_id, number) -PARTITION BY chain_id -SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; \ No newline at end of file diff --git a/internal/tools/clickhouse_create_cursors_table.sql b/internal/tools/clickhouse_create_cursors_table.sql deleted file mode 100644 index a33a7d6..0000000 --- a/internal/tools/clickhouse_create_cursors_table.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE TABLE cursors ( - `chain_id` UInt256, - `cursor_type` String, - `cursor_value` String, - `insert_timestamp` DateTime DEFAULT now(), -) ENGINE = ReplacingMergeTree(insert_timestamp) -ORDER BY (chain_id, cursor_type); diff --git a/internal/tools/clickhouse_create_logs_table.sql b/internal/tools/clickhouse_create_logs_table.sql deleted file mode 100644 index 81cbd6e..0000000 --- a/internal/tools/clickhouse_create_logs_table.sql +++ /dev/null @@ -1,28 +0,0 @@ -CREATE TABLE logs ( - `chain_id` UInt256, - `block_number` UInt256, - `block_hash` FixedString(66), - `block_timestamp` UInt64 CODEC(Delta, ZSTD), - `transaction_hash` FixedString(66), - `transaction_index` UInt64, - `log_index` UInt64, - `address` FixedString(42), - `data` String, - `topic_0` String, - `topic_1` Nullable(String), - `topic_2` Nullable(String), - `topic_3` Nullable(String), - `insert_timestamp` DateTime DEFAULT now(), - `is_deleted` UInt8 DEFAULT 0, - INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1, - INDEX idx_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 1, - INDEX idx_block_hash block_hash TYPE bloom_filter GRANULARITY 1, - INDEX idx_address address TYPE bloom_filter GRANULARITY 1, - INDEX idx_topic0 topic_0 TYPE bloom_filter GRANULARITY 1, - INDEX idx_topic1 topic_1 TYPE bloom_filter GRANULARITY 1, - INDEX idx_topic2 topic_2 TYPE bloom_filter GRANULARITY 1, - INDEX idx_topic3 topic_3 TYPE bloom_filter GRANULARITY 1, -) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted) -ORDER BY (chain_id, block_number, transaction_hash, log_index) -PARTITION BY chain_id -SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; \ No newline at end of file diff --git a/internal/tools/clickhouse_create_traces_table.sql b/internal/tools/clickhouse_create_traces_table.sql deleted file mode 100644 index ea0fce9..0000000 --- a/internal/tools/clickhouse_create_traces_table.sql +++ /dev/null @@ -1,34 +0,0 @@ -CREATE TABLE traces ( - `chain_id` UInt256, - `block_number` UInt256, - `block_hash` FixedString(66), - `block_timestamp` UInt64 CODEC(Delta, ZSTD), - `transaction_hash` FixedString(66), - `transaction_index` UInt64, - `subtraces` Int64, - `trace_address` Array(Int64), - `type` LowCardinality(String), - `call_type` LowCardinality(String), - `error` Nullable(String), - `from_address` FixedString(42), - `to_address` FixedString(42), - `gas` UInt64, - `gas_used` UInt64, - `input` String, - `output` Nullable(String), - `value` UInt256, - `author` Nullable(FixedString(42)), - `reward_type` LowCardinality(Nullable(String)), - `refund_address` Nullable(FixedString(42)), - `is_deleted` UInt8 DEFAULT 0, - `insert_timestamp` DateTime DEFAULT now(), - INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1, - INDEX idx_block_hash block_hash TYPE bloom_filter GRANULARITY 1, - INDEX idx_transaction_hash transaction_hash TYPE bloom_filter GRANULARITY 1, - INDEX idx_from_address from_address TYPE bloom_filter GRANULARITY 1, - INDEX idx_to_address to_address TYPE bloom_filter GRANULARITY 1, - INDEX idx_type type TYPE bloom_filter GRANULARITY 1, -) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted) -ORDER BY (chain_id, block_number, transaction_hash, trace_address) -PARTITION BY chain_id -SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; \ No newline at end of file diff --git a/internal/tools/clickhouse_create_transactions_table.sql b/internal/tools/clickhouse_create_transactions_table.sql deleted file mode 100644 index eaaaadf..0000000 --- a/internal/tools/clickhouse_create_transactions_table.sql +++ /dev/null @@ -1,42 +0,0 @@ -CREATE TABLE transactions ( - `chain_id` UInt256, - `hash` FixedString(66), - `nonce` UInt64, - `block_hash` FixedString(66), - `block_number` UInt256, - `block_timestamp` UInt64 CODEC(Delta, ZSTD), - `transaction_index` UInt64, - `from_address` FixedString(42), - `to_address` FixedString(42), - `value` UInt256, - `gas` UInt64, - `gas_price` UInt256, - `data` String, - `function_selector` FixedString(10), - `max_fee_per_gas` UInt128, - `max_priority_fee_per_gas` UInt128, - `transaction_type` UInt8, - `r` UInt256, - `s` UInt256, - `v` UInt256, - `access_list` Nullable(String), - `contract_address` Nullable(FixedString(42)), - `gas_used` Nullable(UInt64), - `cumulative_gas_used` Nullable(UInt64), - `effective_gas_price` Nullable(UInt256), - `blob_gas_used` Nullable(UInt64), - `blob_gas_price` Nullable(UInt256), - `logs_bloom` Nullable(String), - `status` Nullable(UInt64), - `is_deleted` UInt8 DEFAULT 0, - `insert_timestamp` DateTime DEFAULT now(), - INDEX idx_block_timestamp block_timestamp TYPE minmax GRANULARITY 1, - INDEX idx_block_hash block_hash TYPE bloom_filter GRANULARITY 1, - INDEX idx_hash hash TYPE bloom_filter GRANULARITY 1, - INDEX idx_from_address from_address TYPE bloom_filter GRANULARITY 1, - INDEX idx_to_address to_address TYPE bloom_filter GRANULARITY 1, - INDEX idx_function_selector function_selector TYPE bloom_filter GRANULARITY 1, -) ENGINE = ReplacingMergeTree(insert_timestamp, is_deleted) -ORDER BY (chain_id, block_number, hash) -PARTITION BY chain_id -SETTINGS allow_experimental_replacing_merge_with_cleanup = 1; \ No newline at end of file