diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index e6f3c5ea7a2..21da07f92e8 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -140,9 +140,6 @@ api: # blocks storage. [store_gateway: ] -# The purger_config configures the purger which takes care of delete requests. -[purger: ] - tenant_federation: # If enabled on all Cortex services, queries can be federated across multiple # tenants. The tenant IDs involved need to be specified separated by a `|` @@ -604,8 +601,8 @@ instance_limits: The `ingester_config` configures the Cortex ingester. ```yaml -# Configures the Write-Ahead Log (WAL) for the Cortex chunks storage. This -# config is ignored when running the Cortex blocks storage. +# Configures the Write-Ahead Log (WAL) for the removed Cortex chunks storage. +# This config is now always ignored. walconfig: # Enable writing of ingested data into WAL. # CLI flag: -ingester.wal-enabled @@ -2832,9 +2829,9 @@ chunk_tables_provisioning: The `storage_config` configures where Cortex stores the data (chunks storage engine). ```yaml -# The storage engine to use: chunks (deprecated) or blocks. +# The storage engine to use: blocks is the only supported option today. # CLI flag: -store.engine -[engine: | default = "chunks"] +[engine: | default = "blocks"] aws: dynamodb: @@ -3375,93 +3372,6 @@ index_queries_cache_config: # The CLI flags prefix for this block config is: store.index-cache-read [fifocache: ] -delete_store: - # Store for keeping delete request - # CLI flag: -deletes.store - [store: | default = ""] - - # Name of the table which stores delete requests - # CLI flag: -deletes.requests-table-name - [requests_table_name: | default = "delete_requests"] - - table_provisioning: - # Enables on demand throughput provisioning for the storage provider (if - # supported). Applies only to tables which are not autoscaled. Supported by - # DynamoDB - # CLI flag: -deletes.table.enable-ondemand-throughput-mode - [enable_ondemand_throughput_mode: | default = false] - - # Table default write throughput. Supported by DynamoDB - # CLI flag: -deletes.table.write-throughput - [provisioned_write_throughput: | default = 1] - - # Table default read throughput. Supported by DynamoDB - # CLI flag: -deletes.table.read-throughput - [provisioned_read_throughput: | default = 300] - - write_scale: - # Should we enable autoscale for the table. - # CLI flag: -deletes.table.write-throughput.scale.enabled - [enabled: | default = false] - - # AWS AutoScaling role ARN - # CLI flag: -deletes.table.write-throughput.scale.role-arn - [role_arn: | default = ""] - - # DynamoDB minimum provision capacity. - # CLI flag: -deletes.table.write-throughput.scale.min-capacity - [min_capacity: | default = 3000] - - # DynamoDB maximum provision capacity. - # CLI flag: -deletes.table.write-throughput.scale.max-capacity - [max_capacity: | default = 6000] - - # DynamoDB minimum seconds between each autoscale up. - # CLI flag: -deletes.table.write-throughput.scale.out-cooldown - [out_cooldown: | default = 1800] - - # DynamoDB minimum seconds between each autoscale down. - # CLI flag: -deletes.table.write-throughput.scale.in-cooldown - [in_cooldown: | default = 1800] - - # DynamoDB target ratio of consumed capacity to provisioned capacity. - # CLI flag: -deletes.table.write-throughput.scale.target-value - [target: | default = 80] - - read_scale: - # Should we enable autoscale for the table. - # CLI flag: -deletes.table.read-throughput.scale.enabled - [enabled: | default = false] - - # AWS AutoScaling role ARN - # CLI flag: -deletes.table.read-throughput.scale.role-arn - [role_arn: | default = ""] - - # DynamoDB minimum provision capacity. - # CLI flag: -deletes.table.read-throughput.scale.min-capacity - [min_capacity: | default = 3000] - - # DynamoDB maximum provision capacity. - # CLI flag: -deletes.table.read-throughput.scale.max-capacity - [max_capacity: | default = 6000] - - # DynamoDB minimum seconds between each autoscale up. - # CLI flag: -deletes.table.read-throughput.scale.out-cooldown - [out_cooldown: | default = 1800] - - # DynamoDB minimum seconds between each autoscale down. - # CLI flag: -deletes.table.read-throughput.scale.in-cooldown - [in_cooldown: | default = 1800] - - # DynamoDB target ratio of consumed capacity to provisioned capacity. - # CLI flag: -deletes.table.read-throughput.scale.target-value - [target: | default = 80] - - # Tag (of the form key=value) to be added to the tables. Supported by - # DynamoDB - # CLI flag: -deletes.table.tags - [tags: | default = ] - grpc_store: # Hostname or IP of the gRPC store instance. # CLI flag: -grpc-store.server-address @@ -3473,16 +3383,17 @@ grpc_store: The `flusher_config` configures the WAL flusher target, used to manually run one-time flushes when scaling down ingesters. ```yaml -# Directory to read WAL from (chunks storage engine only). +# Has no effect: directory to read WAL from (chunks storage engine only). # CLI flag: -flusher.wal-dir [wal_dir: | default = "wal"] -# Number of concurrent goroutines flushing to storage (chunks storage engine -# only). +# Has no effect: number of concurrent goroutines flushing to storage (chunks +# storage engine only). # CLI flag: -flusher.concurrent-flushes [concurrent_flushes: | default = 50] -# Timeout for individual flush operations (chunks storage engine only). +# Has no effect: timeout for individual flush operations (chunks storage engine +# only). # CLI flag: -flusher.flush-op-timeout [flush_op_timeout: | default = 2m] @@ -5492,31 +5403,6 @@ sharding_ring: [sharding_strategy: | default = "default"] ``` -### `purger_config` - -The `purger_config` configures the purger which takes care of delete requests. - -```yaml -# Enable purger to allow deletion of series. Be aware that Delete series feature -# is still experimental -# CLI flag: -purger.enable -[enable: | default = false] - -# Number of workers executing delete plans in parallel -# CLI flag: -purger.num-workers -[num_workers: | default = 2] - -# Name of the object store to use for storing delete plans -# CLI flag: -purger.object-store-type -[object_store_type: | default = ""] - -# Allow cancellation of delete request until duration after they are created. -# Data would be deleted only after delete requests have been older than this -# duration. Ideally this should be set to at least 24h. -# CLI flag: -purger.delete-request-cancel-period -[delete_request_cancel_period: | default = 24h] -``` - ### `s3_sse_config` The `s3_sse_config` configures the S3 server-side encryption. The supported CLI flags `` used to reference this config block are: diff --git a/docs/configuration/single-process-config-blocks-local.yaml b/docs/configuration/single-process-config-blocks-local.yaml new file mode 100644 index 00000000000..d79a2a61092 --- /dev/null +++ b/docs/configuration/single-process-config-blocks-local.yaml @@ -0,0 +1,95 @@ + +# Configuration for running Cortex in single-process mode. +# This should not be used in production. It is only for getting started +# and development. + +# Disable the requirement that every request to Cortex has a +# X-Scope-OrgID header. `fake` will be substituted in instead. +auth_enabled: false + +server: + http_listen_port: 9009 + + # Configure the server to allow messages up to 100MB. + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + grpc_server_max_concurrent_streams: 1000 + +distributor: + shard_by_all_labels: true + pool: + health_check_ingesters: true + +ingester_client: + grpc_client_config: + # Configure the client to allow messages up to 100MB. + max_recv_msg_size: 104857600 + max_send_msg_size: 104857600 + grpc_compression: gzip + +ingester: + lifecycler: + # The address to advertise for this ingester. Will be autodiscovered by + # looking up address on eth0 or en0; can be specified if this fails. + # address: 127.0.0.1 + + # We want to start immediately and flush on shutdown. + join_after: 0 + min_ready_duration: 0s + final_sleep: 0s + num_tokens: 512 + + # Use an in memory ring store, so we don't need to launch a Consul. + ring: + kvstore: + store: inmemory + replication_factor: 1 + +storage: + engine: blocks + +blocks_storage: + tsdb: + dir: /tmp/cortex/tsdb + + bucket_store: + sync_dir: /tmp/cortex/tsdb-sync + + # You can choose between local storage and Amazon S3, Google GCS and Azure storage. Each option requires additional configuration + # as shown below. All options can be configured via flags as well which might be handy for secret inputs. + backend: filesystem # s3, gcs, azure or filesystem are valid options +# s3: +# bucket_name: cortex +# endpoint: s3.dualstack.us-east-1.amazonaws.com + # Configure your S3 credentials below. + # secret_access_key: "TODO" + # access_key_id: "TODO" +# gcs: +# bucket_name: cortex +# service_account: # if empty or omitted Cortex will use your default service account as per Google's fallback logic +# azure: +# account_name: +# account_key: +# container_name: +# endpoint_suffix: +# max_retries: # Number of retries for recoverable errors (defaults to 20) + filesystem: + dir: ./data/tsdb + +compactor: + data_dir: /tmp/cortex/compactor + sharding_ring: + kvstore: + store: inmemory + +frontend_worker: + match_max_concurrent: true + +ruler: + enable_api: true + enable_sharding: false + +ruler_storage: + backend: local + local: + directory: /tmp/cortex/rules diff --git a/go.mod b/go.mod index f840197a935..f7943664045 100644 --- a/go.mod +++ b/go.mod @@ -95,3 +95,6 @@ replace github.com/thanos-io/thanos v0.22.0 => github.com/thanos-io/thanos v0.19 // Replace memberlist with Grafana's fork which includes some fixes that haven't been merged upstream yet replace github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b + +// This commit is now only accessible via SHA if you're not using the Go modules proxy. +replace github.com/efficientgo/tools/core => github.com/efficientgo/tools/core v0.0.0-20210829154005-c7bad8450208 diff --git a/go.sum b/go.sum index a09f4460dc6..db39a229159 100644 --- a/go.sum +++ b/go.sum @@ -548,7 +548,6 @@ github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/efficientgo/e2e v0.11.2-0.20211027134903-67d538984a47 h1:k0qDUhOU0KJqKztQYJL1qMBR9nCOntuIRWYwA56Z634= github.com/efficientgo/e2e v0.11.2-0.20211027134903-67d538984a47/go.mod h1:vDnF4AAEZmO0mvyFIATeDJPFaSRM7ywaOnKd61zaSoE= -github.com/efficientgo/tools/core v0.0.0-20210129205121-421d0828c9a6/go.mod h1:OmVcnJopJL8d3X3sSXTiypGoUSgFq1aDGmlrdi9dn/M= github.com/efficientgo/tools/core v0.0.0-20210829154005-c7bad8450208 h1:jIALuFymwBqVsF32JhgzVsbCB6QsWvXqhetn8QgyrZ4= github.com/efficientgo/tools/core v0.0.0-20210829154005-c7bad8450208/go.mod h1:OmVcnJopJL8d3X3sSXTiypGoUSgFq1aDGmlrdi9dn/M= github.com/efficientgo/tools/extkingpin v0.0.0-20210609125236-d73259166f20 h1:kM/ALyvAnTrwSB+nlKqoKaDnZbInp1YImZvW+gtHwc8= diff --git a/integration/api_endpoints_test.go b/integration/api_endpoints_test.go index 0b65c5b7ed0..7ec73a46ada 100644 --- a/integration/api_endpoints_test.go +++ b/integration/api_endpoints_test.go @@ -1,3 +1,4 @@ +//go:build requires_docker // +build requires_docker package integration @@ -22,7 +23,7 @@ func TestIndexAPIEndpoint(t *testing.T) { defer s.Close() // Start Cortex in single binary mode, reading the config from file. - require.NoError(t, copyFileToSharedDir(s, "docs/chunks-storage/single-process-config.yaml", cortexConfigFile)) + require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex1)) @@ -44,7 +45,7 @@ func TestConfigAPIEndpoint(t *testing.T) { defer s.Close() // Start Cortex in single binary mode, reading the config from file. - require.NoError(t, copyFileToSharedDir(s, "docs/chunks-storage/single-process-config.yaml", cortexConfigFile)) + require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex1)) diff --git a/integration/asserts.go b/integration/asserts.go index ba4206045a1..2958133ae9a 100644 --- a/integration/asserts.go +++ b/integration/asserts.go @@ -30,16 +30,19 @@ const ( var ( // Service-specific metrics prefixes which shouldn't be used by any other service. serviceMetricsPrefixes = map[ServiceType][]string{ - Distributor: {}, - Ingester: {"!cortex_ingester_client", "cortex_ingester"}, // The metrics prefix cortex_ingester_client may be used by other components so we ignore it. - Querier: {"cortex_querier"}, + Distributor: {}, + // The metrics prefix cortex_ingester_client may be used by other components so we ignore it. + Ingester: {"!cortex_ingester_client", "cortex_ingester"}, + // The metrics prefixes cortex_querier_storegateway and cortex_querier_blocks may be used by other components so we ignore them. + Querier: {"!cortex_querier_storegateway", "!cortex_querier_blocks", "cortex_querier"}, QueryFrontend: {"cortex_frontend", "cortex_query_frontend"}, QueryScheduler: {"cortex_query_scheduler"}, TableManager: {}, AlertManager: {"cortex_alertmanager"}, Ruler: {}, - StoreGateway: {"!cortex_storegateway_client", "cortex_storegateway"}, // The metrics prefix cortex_storegateway_client may be used by other components so we ignore it. - Purger: {"cortex_purger"}, + // The metrics prefix cortex_storegateway_client may be used by other components so we ignore it. + StoreGateway: {"!cortex_storegateway_client", "cortex_storegateway"}, + Purger: {"cortex_purger"}, } // Blacklisted metrics prefixes across any Cortex service. diff --git a/integration/backward_compatibility_test.go b/integration/backward_compatibility_test.go index 96da22c4282..79396269dae 100644 --- a/integration/backward_compatibility_test.go +++ b/integration/backward_compatibility_test.go @@ -1,3 +1,4 @@ +//go:build requires_docker // +build requires_docker package integration @@ -20,12 +21,6 @@ var ( // If you change the image tag, remember to update it in the preloading done // by GitHub Actions too (see .github/workflows/test-build-deploy.yml). previousVersionImages = map[string]func(map[string]string) map[string]string{ - "quay.io/cortexproject/cortex:v1.0.0": preCortex14Flags, - "quay.io/cortexproject/cortex:v1.1.0": preCortex14Flags, - "quay.io/cortexproject/cortex:v1.2.0": preCortex14Flags, - "quay.io/cortexproject/cortex:v1.3.0": preCortex14Flags, - "quay.io/cortexproject/cortex:v1.4.0": preCortex16Flags, - "quay.io/cortexproject/cortex:v1.5.0": preCortex16Flags, "quay.io/cortexproject/cortex:v1.6.0": preCortex110Flags, "quay.io/cortexproject/cortex:v1.7.0": preCortex110Flags, "quay.io/cortexproject/cortex:v1.8.0": preCortex110Flags, @@ -34,48 +29,37 @@ var ( } ) -func preCortex14Flags(flags map[string]string) map[string]string { +func preCortex110Flags(flags map[string]string) map[string]string { return e2e.MergeFlagsWithoutRemovingEmpty(flags, map[string]string{ - // Blocks storage CLI flags removed the "experimental" prefix in 1.4. - "-store-gateway.sharding-enabled": "", - "-store-gateway.sharding-ring.store": "", - "-store-gateway.sharding-ring.consul.hostname": "", - "-store-gateway.sharding-ring.replication-factor": "", - // Query-scheduler has been introduced in 1.6.0 - "-frontend.scheduler-dns-lookup-period": "", // Store-gateway "wait ring stability" has been introduced in 1.10.0 "-store-gateway.sharding-ring.wait-stability-min-duration": "", "-store-gateway.sharding-ring.wait-stability-max-duration": "", }) } -func preCortex16Flags(flags map[string]string) map[string]string { - return e2e.MergeFlagsWithoutRemovingEmpty(flags, map[string]string{ - // Query-scheduler has been introduced in 1.6.0 - "-frontend.scheduler-dns-lookup-period": "", - // Store-gateway "wait ring stability" has been introduced in 1.10.0 - "-store-gateway.sharding-ring.wait-stability-min-duration": "", - "-store-gateway.sharding-ring.wait-stability-max-duration": "", - }) -} +func TestBackwardCompatibilityWithBlocksStorage(t *testing.T) { + for previousImage, flagsFn := range previousVersionImages { + t.Run(fmt.Sprintf("Backward compatibility upgrading from %s", previousImage), func(t *testing.T) { + flags := blocksStorageFlagsWithFlushOnShutdown() + if flagsFn != nil { + flags = flagsFn(flags) + } -func preCortex110Flags(flags map[string]string) map[string]string { - return e2e.MergeFlagsWithoutRemovingEmpty(flags, map[string]string{ - // Store-gateway "wait ring stability" has been introduced in 1.10.0 - "-store-gateway.sharding-ring.wait-stability-min-duration": "", - "-store-gateway.sharding-ring.wait-stability-max-duration": "", - }) + runBackwardCompatibilityTestWithBlocksStorage(t, previousImage, flags) + }) + } } -func TestBackwardCompatibilityWithChunksStorage(t *testing.T) { +func TestBackwardCompatibilityWithChunksStorageAsSecondStore(t *testing.T) { for previousImage, flagsFn := range previousVersionImages { - t.Run(fmt.Sprintf("Backward compatibility upgrading from %s", previousImage), func(t *testing.T) { + t.Run(fmt.Sprintf("Backward compatibility with data from %s", previousImage), func(t *testing.T) { flags := ChunksStorageFlags() + flags["-ingester.max-transfer-retries"] = "0" if flagsFn != nil { flags = flagsFn(flags) } - runBackwardCompatibilityTestWithChunksStorage(t, previousImage, flags) + runBackwardCompatibilityTestWithChunksStorageAsSecondStore(t, previousImage, flags) }) } } @@ -83,7 +67,7 @@ func TestBackwardCompatibilityWithChunksStorage(t *testing.T) { func TestNewDistributorsCanPushToOldIngestersWithReplication(t *testing.T) { for previousImage, flagsFn := range previousVersionImages { t.Run(fmt.Sprintf("Backward compatibility upgrading from %s", previousImage), func(t *testing.T) { - flags := ChunksStorageFlags() + flags := blocksStorageFlagsWithFlushOnShutdown() if flagsFn != nil { flags = flagsFn(flags) } @@ -93,30 +77,27 @@ func TestNewDistributorsCanPushToOldIngestersWithReplication(t *testing.T) { } } -func runBackwardCompatibilityTestWithChunksStorage(t *testing.T, previousImage string, flagsForOldImage map[string]string) { +func blocksStorageFlagsWithFlushOnShutdown() map[string]string { + return mergeFlags(BlocksStorageFlags(), map[string]string{ + "-blocks-storage.tsdb.flush-blocks-on-shutdown": "true", + }) +} + +func runBackwardCompatibilityTestWithBlocksStorage(t *testing.T, previousImage string, flagsForOldImage map[string]string) { s, err := e2e.NewScenario(networkName) require.NoError(t, err) defer s.Close() // Start dependencies. - dynamo := e2edb.NewDynamoDB() + minio := e2edb.NewMinio(9000, flagsForOldImage["-blocks-storage.s3.bucket-name"]) consul := e2edb.NewConsul() - require.NoError(t, s.StartAndWaitReady(dynamo, consul)) - - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - - // Start Cortex table-manager (running on current version since the backward compatibility - // test is about testing a rolling update of other services). - tableManager := e2ecortex.NewTableManager("table-manager", ChunksStorageFlags(), "") - require.NoError(t, s.StartAndWaitReady(tableManager)) + require.NoError(t, s.StartAndWaitReady(minio, consul)) - // Wait until the first table-manager sync has completed, so that we're - // sure the tables have been created. - require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds")) + flagsForNewImage := blocksStorageFlagsWithFlushOnShutdown() // Start other Cortex components (ingester running on previous version). ingester1 := e2ecortex.NewIngester("ingester-1", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flagsForOldImage, previousImage) - distributor := e2ecortex.NewDistributor("distributor", "consul", consul.NetworkHTTPEndpoint(), ChunksStorageFlags(), "") + distributor := e2ecortex.NewDistributor("distributor", "consul", consul.NetworkHTTPEndpoint(), flagsForNewImage, "") require.NoError(t, s.StartAndWaitReady(distributor, ingester1)) // Wait until the distributor has updated the ring. @@ -133,53 +114,107 @@ func runBackwardCompatibilityTestWithChunksStorage(t *testing.T, previousImage s require.NoError(t, err) require.Equal(t, 200, res.StatusCode) - ingester2 := e2ecortex.NewIngester("ingester-2", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), mergeFlags(ChunksStorageFlags(), map[string]string{ + ingester2 := e2ecortex.NewIngester("ingester-2", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), mergeFlags(flagsForNewImage, map[string]string{ "-ingester.join-after": "10s", }), "") - // Start ingester-2 on new version, to ensure the transfer is backward compatible. require.NoError(t, s.Start(ingester2)) // Stop ingester-1. This function will return once the ingester-1 is successfully - // stopped, which means the transfer to ingester-2 is completed. + // stopped, which means it has uploaded all its data to the object store. require.NoError(t, s.Stop(ingester1)) checkQueries(t, consul, expectedVector, previousImage, flagsForOldImage, - ChunksStorageFlags(), + flagsForNewImage, now, s, 1, ) } -// Check for issues like https://github.com/cortexproject/cortex/issues/2356 -func runNewDistributorsCanPushToOldIngestersWithReplication(t *testing.T, previousImage string, flagsForPreviousImage map[string]string) { +func runBackwardCompatibilityTestWithChunksStorageAsSecondStore(t *testing.T, previousImage string, flagsForOldImage map[string]string) { s, err := e2e.NewScenario(networkName) require.NoError(t, err) defer s.Close() + flagsForNewImage := mergeFlags(blocksStorageFlagsWithFlushOnShutdown(), ChunksStorageFlags()) + flagsForNewImage = mergeFlags(flagsForNewImage, map[string]string{ + "-querier.second-store-engine": chunksStorageEngine, + "-store.engine": blocksStorageEngine, + }) + // Start dependencies. dynamo := e2edb.NewDynamoDB() + minio := e2edb.NewMinio(9000, flagsForNewImage["-blocks-storage.s3.bucket-name"]) consul := e2edb.NewConsul() - require.NoError(t, s.StartAndWaitReady(dynamo, consul)) - - flagsForNewImage := mergeFlags(ChunksStorageFlags(), map[string]string{ - "-distributor.replication-factor": "3", - }) + require.NoError(t, s.StartAndWaitReady(dynamo, minio, consul)) require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - // Start Cortex table-manager (running on current version since the backward compatibility - // test is about testing a rolling update of other services). - tableManager := e2ecortex.NewTableManager("table-manager", ChunksStorageFlags(), "") + // Start Cortex table-manager (on old version since it's gone in current) + tableManager := e2ecortex.NewTableManager("table-manager", flagsForOldImage, previousImage) require.NoError(t, s.StartAndWaitReady(tableManager)) // Wait until the first table-manager sync has completed, so that we're // sure the tables have been created. require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds")) + // Start other Cortex components (ingester running on previous version). + ingester1 := e2ecortex.NewIngester("ingester-1", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flagsForOldImage, previousImage) + distributor := e2ecortex.NewDistributor("distributor", "consul", consul.NetworkHTTPEndpoint(), flagsForOldImage, previousImage) + require.NoError(t, s.StartAndWaitReady(distributor, ingester1)) + + // Wait until the distributor has updated the ring. + require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total")) + + // Push some series to Cortex. + now := time.Now() + series, expectedVector := generateSeries("series_1", now) + + c, err := e2ecortex.NewClient(distributor.HTTPEndpoint(), "", "", "", "user-1") + require.NoError(t, err) + + res, err := c.Push(series) + require.NoError(t, err) + require.Equal(t, 200, res.StatusCode) + + // Stop ingester-1. This function will return once the ingester-1 is successfully + // stopped, which has been configured to flush all data to the backing store. + require.NoError(t, s.Stop(ingester1)) + + ingester2 := e2ecortex.NewIngester("ingester-2", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), mergeFlags(flagsForNewImage, map[string]string{ + "-ingester.join-after": "10s", + }), "") + require.NoError(t, s.Start(ingester2)) + + checkQueries(t, consul, + expectedVector, + "", + flagsForNewImage, + flagsForNewImage, + now, + s, + 1, + ) +} + +// Check for issues like https://github.com/cortexproject/cortex/issues/2356 +func runNewDistributorsCanPushToOldIngestersWithReplication(t *testing.T, previousImage string, flagsForPreviousImage map[string]string) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + // Start dependencies. + minio := e2edb.NewMinio(9000, flagsForPreviousImage["-blocks-storage.s3.bucket-name"]) + consul := e2edb.NewConsul() + require.NoError(t, s.StartAndWaitReady(minio, consul)) + + flagsForNewImage := mergeFlags(blocksStorageFlagsWithFlushOnShutdown(), map[string]string{ + "-distributor.replication-factor": "3", + }) + // Start other Cortex components (ingester running on previous version). ingester1 := e2ecortex.NewIngester("ingester-1", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flagsForPreviousImage, previousImage) ingester2 := e2ecortex.NewIngester("ingester-2", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flagsForPreviousImage, previousImage) @@ -227,18 +262,24 @@ func checkQueries( queryFrontendFlags map[string]string querierImage string querierFlags map[string]string + storeGatewayImage string + storeGatewayFlags map[string]string }{ - "old query-frontend, new querier": { + "old query-frontend, new querier, old store-gateway": { queryFrontendImage: previousImage, queryFrontendFlags: flagsForOldImage, querierImage: "", querierFlags: flagsForNewImage, + storeGatewayImage: previousImage, + storeGatewayFlags: flagsForOldImage, }, - "new query-frontend, old querier": { + "new query-frontend, old querier, new store-gateway": { queryFrontendImage: "", queryFrontendFlags: flagsForNewImage, querierImage: previousImage, querierFlags: flagsForOldImage, + storeGatewayImage: "", + storeGatewayFlags: flagsForNewImage, }, } @@ -261,9 +302,18 @@ func checkQueries( require.NoError(t, s.Stop(querier)) }() + // Start store gateway. + storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), c.storeGatewayFlags, c.storeGatewayImage) + + require.NoError(t, s.Start(storeGateway)) + defer func() { + require.NoError(t, s.Stop(storeGateway)) + }() + // Wait until querier and query-frontend are ready, and the querier has updated the ring. - require.NoError(t, s.WaitReady(querier, queryFrontend)) - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(float64(numIngesters*512)), "cortex_ring_tokens_total")) + require.NoError(t, s.WaitReady(querier, queryFrontend, storeGateway)) + expectedTokens := float64((numIngesters + 1) * 512) // Ingesters and Store Gateway. + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(expectedTokens), "cortex_ring_tokens_total")) // Query the series. for _, endpoint := range []string{queryFrontend.HTTPEndpoint(), querier.HTTPEndpoint()} { diff --git a/integration/configs.go b/integration/configs.go index 5d2f8c78c36..635c075ca76 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -1,3 +1,4 @@ +//go:build requires_docker // +build requires_docker package integration @@ -27,6 +28,7 @@ const ( cortexConfigFile = "config.yaml" cortexSchemaConfigFile = "schema.yaml" blocksStorageEngine = "blocks" + chunksStorageEngine = "chunks" clientCertFile = "certs/client.crt" clientKeyFile = "certs/client.key" caCertFile = "certs/root.crt" @@ -233,32 +235,13 @@ blocks_storage: ChunksStorageFlags = func() map[string]string { return map[string]string{ + "-store.engine": chunksStorageEngine, "-dynamodb.url": fmt.Sprintf("dynamodb://u:p@%s-dynamodb.:8000", networkName), "-table-manager.poll-interval": "1m", "-schema-config-file": filepath.Join(e2e.ContainerSharedDir, cortexSchemaConfigFile), "-table-manager.retention-period": "168h", } } - - ChunksStorageConfig = buildConfigFromTemplate(` -storage: - aws: - dynamodb: - dynamodb_url: {{.DynamoDBURL}} - -table_manager: - poll_interval: 1m - retention_period: 168h - -schema: -{{.SchemaConfig}} -`, struct { - DynamoDBURL string - SchemaConfig string - }{ - DynamoDBURL: fmt.Sprintf("dynamodb://u:p@%s-dynamodb.:8000", networkName), - SchemaConfig: indentConfig(cortexSchemaConfigYaml, 2), - }) ) func buildConfigFromTemplate(tmpl string, data interface{}) string { @@ -275,23 +258,6 @@ func buildConfigFromTemplate(tmpl string, data interface{}) string { return w.String() } -func indentConfig(config string, indentation int) string { - output := strings.Builder{} - - for _, line := range strings.Split(config, "\n") { - if line == "" { - output.WriteString("\n") - continue - } - - output.WriteString(strings.Repeat(" ", indentation)) - output.WriteString(line) - output.WriteString("\n") - } - - return output.String() -} - func buildSchemaConfigWith(configs []storeConfig) string { configYamls := "" for _, config := range configs { diff --git a/integration/e2ecortex/services.go b/integration/e2ecortex/services.go index 1ed6c53e2d0..ebda7a27349 100644 --- a/integration/e2ecortex/services.go +++ b/integration/e2ecortex/services.go @@ -424,9 +424,14 @@ func NewRuler(name string, consulAddress string, flags map[string]string, image e2e.NewCommandWithoutEntrypoint("cortex", e2e.BuildArgs(e2e.MergeFlags(map[string]string{ "-target": "ruler", "-log.level": "warn", - // Configure the ingesters ring backend - "-ring.store": "consul", - "-consul.hostname": consulAddress, + // Configure the ring backend + "-ring.store": "consul", + "-store-gateway.sharding-ring.store": "consul", + "-consul.hostname": consulAddress, + "-store-gateway.sharding-ring.consul.hostname": consulAddress, + // Store-gateway ring backend. + "-store-gateway.sharding-enabled": "true", + "-store-gateway.sharding-ring.replication-factor": "1", }, flags))...), e2e.NewHTTPReadinessProbe(httpPort, "/ready", 200, 299), httpPort, diff --git a/integration/integration_memberlist_single_binary_test.go b/integration/integration_memberlist_single_binary_test.go index df1a3cf8a63..6146f477c4c 100644 --- a/integration/integration_memberlist_single_binary_test.go +++ b/integration/integration_memberlist_single_binary_test.go @@ -43,11 +43,10 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string) defer s.Close() // Start dependencies - dynamo := e2edb.NewDynamoDB() + minio := e2edb.NewMinio(9000, bucketName) // Look ma, no Consul! - require.NoError(t, s.StartAndWaitReady(dynamo)) + require.NoError(t, s.StartAndWaitReady(minio)) - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) var cortex1, cortex2, cortex3 *e2ecortex.CortexService if tlsEnabled { var ( @@ -136,7 +135,7 @@ func newSingleBinary(name string, servername string, join string, testFlags map[ serv := e2ecortex.NewSingleBinary( name, mergeFlags( - ChunksStorageFlags(), + BlocksStorageFlags(), flags, testFlags, getTLSFlagsWithPrefix("memberlist", servername, servername == ""), @@ -160,9 +159,8 @@ func TestSingleBinaryWithMemberlistScaling(t *testing.T) { require.NoError(t, err) defer s.Close() - dynamo := e2edb.NewDynamoDB() - require.NoError(t, s.StartAndWaitReady(dynamo)) - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) + minio := e2edb.NewMinio(9000, bucketName) + require.NoError(t, s.StartAndWaitReady(minio)) // Scale up instances. These numbers seem enough to reliably reproduce some unwanted // consequences of slow propagation, such as missing tombstones. diff --git a/integration/querier_remote_read_test.go b/integration/querier_remote_read_test.go index 217044f3eee..20be210df3b 100644 --- a/integration/querier_remote_read_test.go +++ b/integration/querier_remote_read_test.go @@ -29,20 +29,12 @@ func TestQuerierRemoteRead(t *testing.T) { defer s.Close() require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - flags := mergeFlags(ChunksStorageFlags(), map[string]string{}) + flags := mergeFlags(BlocksStorageFlags(), map[string]string{}) // Start dependencies. - dynamo := e2edb.NewDynamoDB() - + minio := e2edb.NewMinio(9000, bucketName) consul := e2edb.NewConsul() - require.NoError(t, s.StartAndWaitReady(consul, dynamo)) - - tableManager := e2ecortex.NewTableManager("table-manager", ChunksStorageFlags(), "") - require.NoError(t, s.StartAndWaitReady(tableManager)) - - // Wait until the first table-manager sync has completed, so that we're - // sure the tables have been created. - require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds")) + require.NoError(t, s.StartAndWaitReady(consul, minio)) // Start Cortex components for the write path. distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") @@ -63,11 +55,13 @@ func TestQuerierRemoteRead(t *testing.T) { require.NoError(t, err) require.Equal(t, 200, res.StatusCode) - querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), ChunksStorageFlags(), "") + storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + require.NoError(t, s.StartAndWaitReady(storeGateway)) + querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), BlocksStorageFlags(), "") require.NoError(t, s.StartAndWaitReady(querier)) // Wait until the querier has updated the ring. - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total")) + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*512), "cortex_ring_tokens_total")) matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "series_1") require.NoError(t, err) diff --git a/integration/querier_streaming_mixed_ingester_test.go b/integration/querier_streaming_mixed_ingester_test.go deleted file mode 100644 index 910356b5460..00000000000 --- a/integration/querier_streaming_mixed_ingester_test.go +++ /dev/null @@ -1,164 +0,0 @@ -// +build requires_docker - -package integration - -import ( - "context" - "flag" - "fmt" - "strings" - "testing" - "time" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/require" - "github.com/weaveworks/common/user" - - "github.com/cortexproject/cortex/integration/e2e" - e2edb "github.com/cortexproject/cortex/integration/e2e/db" - "github.com/cortexproject/cortex/integration/e2ecortex" - "github.com/cortexproject/cortex/pkg/cortexpb" - ingester_client "github.com/cortexproject/cortex/pkg/ingester/client" -) - -func TestQuerierWithStreamingBlocksAndChunksIngesters(t *testing.T) { - for _, streamChunks := range []bool{false, true} { - t.Run(fmt.Sprintf("%v", streamChunks), func(t *testing.T) { - testQuerierWithStreamingBlocksAndChunksIngesters(t, streamChunks) - }) - } -} - -func testQuerierWithStreamingBlocksAndChunksIngesters(t *testing.T, streamChunks bool) { - s, err := e2e.NewScenario(networkName) - require.NoError(t, err) - defer s.Close() - - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - chunksFlags := ChunksStorageFlags() - blockFlags := mergeFlags(BlocksStorageFlags(), map[string]string{ - "-blocks-storage.tsdb.block-ranges-period": "1h", - "-blocks-storage.tsdb.head-compaction-interval": "1m", - "-store-gateway.sharding-enabled": "false", - "-querier.ingester-streaming": "true", - }) - blockFlags["-ingester.stream-chunks-when-using-blocks"] = fmt.Sprintf("%v", streamChunks) - - // Start dependencies. - consul := e2edb.NewConsul() - minio := e2edb.NewMinio(9000, blockFlags["-blocks-storage.s3.bucket-name"]) - require.NoError(t, s.StartAndWaitReady(consul, minio)) - - // Start Cortex components. - ingesterBlocks := e2ecortex.NewIngester("ingester-blocks", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), blockFlags, "") - ingesterChunks := e2ecortex.NewIngester("ingester-chunks", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), chunksFlags, "") - storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), blockFlags, "") - require.NoError(t, s.StartAndWaitReady(ingesterBlocks, ingesterChunks, storeGateway)) - - // Sharding is disabled, pass gateway address. - querierFlags := mergeFlags(blockFlags, map[string]string{ - "-querier.store-gateway-addresses": strings.Join([]string{storeGateway.NetworkGRPCEndpoint()}, ","), - "-distributor.shard-by-all-labels": "true", - }) - querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), querierFlags, "") - require.NoError(t, s.StartAndWaitReady(querier)) - - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(1024), "cortex_ring_tokens_total")) - - s1 := []cortexpb.Sample{ - {Value: 1, TimestampMs: 1000}, - {Value: 2, TimestampMs: 2000}, - {Value: 3, TimestampMs: 3000}, - {Value: 4, TimestampMs: 4000}, - {Value: 5, TimestampMs: 5000}, - } - - s2 := []cortexpb.Sample{ - {Value: 1, TimestampMs: 1000}, - {Value: 2.5, TimestampMs: 2500}, - {Value: 3, TimestampMs: 3000}, - {Value: 5.5, TimestampMs: 5500}, - } - - clientConfig := ingester_client.Config{} - clientConfig.RegisterFlags(flag.NewFlagSet("unused", flag.ContinueOnError)) // registers default values - - // Push data to chunks ingester. - { - ingesterChunksClient, err := ingester_client.MakeIngesterClient(ingesterChunks.GRPCEndpoint(), clientConfig) - require.NoError(t, err) - defer ingesterChunksClient.Close() - - _, err = ingesterChunksClient.Push(user.InjectOrgID(context.Background(), "user"), &cortexpb.WriteRequest{ - Timeseries: []cortexpb.PreallocTimeseries{ - {TimeSeries: &cortexpb.TimeSeries{Labels: []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "s"}, {Name: "l", Value: "1"}}, Samples: s1}}, - {TimeSeries: &cortexpb.TimeSeries{Labels: []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "s"}, {Name: "l", Value: "2"}}, Samples: s1}}}, - Source: cortexpb.API, - }) - require.NoError(t, err) - } - - // Push data to blocks ingester. - { - ingesterBlocksClient, err := ingester_client.MakeIngesterClient(ingesterBlocks.GRPCEndpoint(), clientConfig) - require.NoError(t, err) - defer ingesterBlocksClient.Close() - - _, err = ingesterBlocksClient.Push(user.InjectOrgID(context.Background(), "user"), &cortexpb.WriteRequest{ - Timeseries: []cortexpb.PreallocTimeseries{ - {TimeSeries: &cortexpb.TimeSeries{Labels: []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "s"}, {Name: "l", Value: "2"}}, Samples: s2}}, - {TimeSeries: &cortexpb.TimeSeries{Labels: []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "s"}, {Name: "l", Value: "3"}}, Samples: s1}}}, - Source: cortexpb.API, - }) - require.NoError(t, err) - } - - c, err := e2ecortex.NewClient("", querier.HTTPEndpoint(), "", "", "user") - require.NoError(t, err) - - // Query back the series (1 only in the storage, 1 only in the ingesters, 1 on both). - result, err := c.Query("s[1m]", time.Unix(10, 0)) - require.NoError(t, err) - - s1Values := []model.SamplePair{ - {Value: 1, Timestamp: 1000}, - {Value: 2, Timestamp: 2000}, - {Value: 3, Timestamp: 3000}, - {Value: 4, Timestamp: 4000}, - {Value: 5, Timestamp: 5000}, - } - - s1AndS2ValuesMerged := []model.SamplePair{ - {Value: 1, Timestamp: 1000}, - {Value: 2, Timestamp: 2000}, - {Value: 2.5, Timestamp: 2500}, - {Value: 3, Timestamp: 3000}, - {Value: 4, Timestamp: 4000}, - {Value: 5, Timestamp: 5000}, - {Value: 5.5, Timestamp: 5500}, - } - - expectedMatrix := model.Matrix{ - // From chunks ingester only. - &model.SampleStream{ - Metric: model.Metric{labels.MetricName: "s", "l": "1"}, - Values: s1Values, - }, - - // From blocks ingester only. - &model.SampleStream{ - Metric: model.Metric{labels.MetricName: "s", "l": "3"}, - Values: s1Values, - }, - - // Merged from both ingesters. - &model.SampleStream{ - Metric: model.Metric{labels.MetricName: "s", "l": "2"}, - Values: s1AndS2ValuesMerged, - }, - } - - require.Equal(t, model.ValMatrix, result.Type()) - require.ElementsMatch(t, expectedMatrix, result.(model.Matrix)) -} diff --git a/integration/querier_test.go b/integration/querier_test.go index 4a1166ee824..4e8d8b5eec4 100644 --- a/integration/querier_test.go +++ b/integration/querier_test.go @@ -853,20 +853,13 @@ func TestHashCollisionHandling(t *testing.T) { defer s.Close() require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - flags := ChunksStorageFlags() + flags := BlocksStorageFlags() // Start dependencies. - dynamo := e2edb.NewDynamoDB() + minio := e2edb.NewMinio(9000, bucketName) consul := e2edb.NewConsul() - require.NoError(t, s.StartAndWaitReady(consul, dynamo)) - - tableManager := e2ecortex.NewTableManager("table-manager", ChunksStorageFlags(), "") - require.NoError(t, s.StartAndWaitReady(tableManager)) - - // Wait until the first table-manager sync has completed, so that we're - // sure the tables have been created. - require.NoError(t, tableManager.WaitSumMetrics(e2e.Greater(0), "cortex_table_manager_sync_success_timestamp_seconds")) + require.NoError(t, s.StartAndWaitReady(consul, minio)) // Start Cortex components for the write path. distributor := e2ecortex.NewDistributor("distributor", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") @@ -922,11 +915,13 @@ func TestHashCollisionHandling(t *testing.T) { require.NoError(t, err) require.Equal(t, 200, res.StatusCode) + storeGateway := e2ecortex.NewStoreGateway("store-gateway", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") + require.NoError(t, s.StartAndWaitReady(storeGateway)) querier := e2ecortex.NewQuerier("querier", e2ecortex.RingStoreConsul, consul.NetworkHTTPEndpoint(), flags, "") require.NoError(t, s.StartAndWaitReady(querier)) // Wait until the querier has updated the ring. - require.NoError(t, querier.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total")) + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(2*512), "cortex_ring_tokens_total")) // Query the series. c, err = e2ecortex.NewClient("", querier.HTTPEndpoint(), "", "", "user-0") diff --git a/integration/ruler_test.go b/integration/ruler_test.go index 15c7d4b4ba4..ac8ab0ea40d 100644 --- a/integration/ruler_test.go +++ b/integration/ruler_test.go @@ -56,13 +56,11 @@ func TestRulerAPI(t *testing.T) { // Start dependencies. consul := e2edb.NewConsul() - dynamo := e2edb.NewDynamoDB() - minio := e2edb.NewMinio(9000, rulestoreBucketName) - require.NoError(t, s.StartAndWaitReady(consul, minio, dynamo)) + minio := e2edb.NewMinio(9000, rulestoreBucketName, bucketName) + require.NoError(t, s.StartAndWaitReady(consul, minio)) // Start Cortex components. - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(ChunksStorageFlags(), RulerFlags(testCfg.legacyRuleStore)), "") + ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(BlocksStorageFlags(), RulerFlags(testCfg.legacyRuleStore)), "") require.NoError(t, s.StartAndWaitReady(ruler)) // Create a client with the ruler address configured @@ -159,7 +157,7 @@ func TestRulerAPISingleBinary(t *testing.T) { } // Start Cortex components. - require.NoError(t, copyFileToSharedDir(s, "docs/chunks-storage/single-process-config.yaml", cortexConfigFile)) + require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) require.NoError(t, writeFileToSharedDir(s, filepath.Join("ruler_configs", user, namespace), []byte(cortexRulerUserConfigYaml))) cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex", cortexConfigFile, configOverrides, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex)) @@ -218,7 +216,7 @@ func TestRulerEvaluationDelay(t *testing.T) { } // Start Cortex components. - require.NoError(t, copyFileToSharedDir(s, "docs/chunks-storage/single-process-config.yaml", cortexConfigFile)) + require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) require.NoError(t, writeFileToSharedDir(s, filepath.Join("ruler_configs", user, namespace), []byte(cortexRulerEvalStaleNanConfigYaml))) cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex", cortexConfigFile, configOverrides, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex)) @@ -404,9 +402,8 @@ func TestRulerAlertmanager(t *testing.T) { // Start dependencies. consul := e2edb.NewConsul() - dynamo := e2edb.NewDynamoDB() - minio := e2edb.NewMinio(9000, rulestoreBucketName) - require.NoError(t, s.StartAndWaitReady(consul, minio, dynamo)) + minio := e2edb.NewMinio(9000, rulestoreBucketName, bucketName) + require.NoError(t, s.StartAndWaitReady(consul, minio)) // Have at least one alertmanager configuration. require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs/user-1.yaml", []byte(cortexAlertmanagerUserConfigYaml))) @@ -426,8 +423,7 @@ func TestRulerAlertmanager(t *testing.T) { } // Start Ruler. - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(ChunksStorageFlags(), RulerFlags(false), configOverrides), "") + ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(BlocksStorageFlags(), RulerFlags(false), configOverrides), "") require.NoError(t, s.StartAndWaitReady(ruler)) // Create a client with the ruler address configured @@ -454,9 +450,8 @@ func TestRulerAlertmanagerTLS(t *testing.T) { // Start dependencies. consul := e2edb.NewConsul() - dynamo := e2edb.NewDynamoDB() - minio := e2edb.NewMinio(9000, rulestoreBucketName) - require.NoError(t, s.StartAndWaitReady(consul, minio, dynamo)) + minio := e2edb.NewMinio(9000, rulestoreBucketName, bucketName) + require.NoError(t, s.StartAndWaitReady(consul, minio)) // set the ca cert := ca.New("Ruler/Alertmanager Test") @@ -506,8 +501,7 @@ func TestRulerAlertmanagerTLS(t *testing.T) { ) // Start Ruler. - require.NoError(t, writeFileToSharedDir(s, cortexSchemaConfigFile, []byte(cortexSchemaConfigYaml))) - ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(ChunksStorageFlags(), RulerFlags(false), configOverrides), "") + ruler := e2ecortex.NewRuler("ruler", consul.NetworkHTTPEndpoint(), mergeFlags(BlocksStorageFlags(), RulerFlags(false), configOverrides), "") require.NoError(t, s.StartAndWaitReady(ruler)) // Create a client with the ruler address configured diff --git a/pkg/api/api.go b/pkg/api/api.go index ced7685448f..4d9e2f5cabd 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -6,13 +6,11 @@ import ( "net/http" "path" "strings" - "time" "github.com/NYTimes/gziphandler" "github.com/felixge/fgprof" "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/storage" "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/server" @@ -263,22 +261,6 @@ func (a *API) RegisterIngester(i Ingester, pushConfig distributor.Config) { a.RegisterRoute("/push", push.Handler(pushConfig.MaxRecvMsgSize, a.sourceIPs, i.Push), true, "POST") // For testing and debugging. } -// RegisterChunksPurger registers the endpoints associated with the Purger/DeleteStore. They do not exactly -// match the Prometheus API but mirror it closely enough to justify their routing under the Prometheus -// component/ -func (a *API) RegisterChunksPurger(store *purger.DeleteStore, deleteRequestCancelPeriod time.Duration) { - deleteRequestHandler := purger.NewDeleteRequestHandler(store, deleteRequestCancelPeriod, prometheus.DefaultRegisterer) - - a.RegisterRoute(path.Join(a.cfg.PrometheusHTTPPrefix, "/api/v1/admin/tsdb/delete_series"), http.HandlerFunc(deleteRequestHandler.AddDeleteRequestHandler), true, "PUT", "POST") - a.RegisterRoute(path.Join(a.cfg.PrometheusHTTPPrefix, "/api/v1/admin/tsdb/delete_series"), http.HandlerFunc(deleteRequestHandler.GetAllDeleteRequestsHandler), true, "GET") - a.RegisterRoute(path.Join(a.cfg.PrometheusHTTPPrefix, "/api/v1/admin/tsdb/cancel_delete_request"), http.HandlerFunc(deleteRequestHandler.CancelDeleteRequestHandler), true, "PUT", "POST") - - // Legacy Routes - a.RegisterRoute(path.Join(a.cfg.LegacyHTTPPrefix, "/api/v1/admin/tsdb/delete_series"), http.HandlerFunc(deleteRequestHandler.AddDeleteRequestHandler), true, "PUT", "POST") - a.RegisterRoute(path.Join(a.cfg.LegacyHTTPPrefix, "/api/v1/admin/tsdb/delete_series"), http.HandlerFunc(deleteRequestHandler.GetAllDeleteRequestsHandler), true, "GET") - a.RegisterRoute(path.Join(a.cfg.LegacyHTTPPrefix, "/api/v1/admin/tsdb/cancel_delete_request"), http.HandlerFunc(deleteRequestHandler.CancelDeleteRequestHandler), true, "PUT", "POST") -} - func (a *API) RegisterTenantDeletion(api *purger.TenantDeletionAPI) { a.RegisterRoute("/purger/delete_tenant", http.HandlerFunc(api.DeleteTenant), true, "POST") a.RegisterRoute("/purger/delete_tenant_status", http.HandlerFunc(api.DeleteTenantStatus), true, "GET") diff --git a/pkg/api/handlers.go b/pkg/api/handlers.go index b775573e838..67cdadfcd94 100644 --- a/pkg/api/handlers.go +++ b/pkg/api/handlers.go @@ -160,7 +160,7 @@ func NewQuerierHandler( exemplarQueryable storage.ExemplarQueryable, engine *promql.Engine, distributor Distributor, - tombstonesLoader *purger.TombstonesLoader, + tombstonesLoader purger.TombstonesLoader, reg prometheus.Registerer, logger log.Logger, ) http.Handler { diff --git a/pkg/api/middlewares.go b/pkg/api/middlewares.go index 7e0e88e8030..b60326e51c1 100644 --- a/pkg/api/middlewares.go +++ b/pkg/api/middlewares.go @@ -11,7 +11,7 @@ import ( ) // middleware for setting cache gen header to let consumer of response know all previous responses could be invalid due to delete operation -func getHTTPCacheGenNumberHeaderSetterMiddleware(cacheGenNumbersLoader *purger.TombstonesLoader) middleware.Interface { +func getHTTPCacheGenNumberHeaderSetterMiddleware(cacheGenNumbersLoader purger.TombstonesLoader) middleware.Interface { return middleware.Func(func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { tenantIDs, err := tenant.TenantIDs(r.Context()) diff --git a/pkg/chunk/purger/delete_plan.pb.go b/pkg/chunk/purger/delete_plan.pb.go deleted file mode 100644 index 5646b2b4eb6..00000000000 --- a/pkg/chunk/purger/delete_plan.pb.go +++ /dev/null @@ -1,1353 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: delete_plan.proto - -package purger - -import ( - fmt "fmt" - _ "github.com/cortexproject/cortex/pkg/cortexpb" - github_com_cortexproject_cortex_pkg_cortexpb "github.com/cortexproject/cortex/pkg/cortexpb" - _ "github.com/gogo/protobuf/gogoproto" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" - reflect "reflect" - strings "strings" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -// DeletePlan holds all the chunks that are supposed to be deleted within an interval(usually a day) -// This Proto file is used just for storing Delete Plans in proto format. -type DeletePlan struct { - PlanInterval *Interval `protobuf:"bytes,1,opt,name=plan_interval,json=planInterval,proto3" json:"plan_interval,omitempty"` - ChunksGroup []ChunksGroup `protobuf:"bytes,2,rep,name=chunks_group,json=chunksGroup,proto3" json:"chunks_group"` -} - -func (m *DeletePlan) Reset() { *m = DeletePlan{} } -func (*DeletePlan) ProtoMessage() {} -func (*DeletePlan) Descriptor() ([]byte, []int) { - return fileDescriptor_c38868cf63b27372, []int{0} -} -func (m *DeletePlan) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *DeletePlan) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_DeletePlan.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *DeletePlan) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeletePlan.Merge(m, src) -} -func (m *DeletePlan) XXX_Size() int { - return m.Size() -} -func (m *DeletePlan) XXX_DiscardUnknown() { - xxx_messageInfo_DeletePlan.DiscardUnknown(m) -} - -var xxx_messageInfo_DeletePlan proto.InternalMessageInfo - -func (m *DeletePlan) GetPlanInterval() *Interval { - if m != nil { - return m.PlanInterval - } - return nil -} - -func (m *DeletePlan) GetChunksGroup() []ChunksGroup { - if m != nil { - return m.ChunksGroup - } - return nil -} - -// ChunksGroup holds ChunkDetails and Labels for a group of chunks which have same series ID -type ChunksGroup struct { - Labels []github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter `protobuf:"bytes,1,rep,name=labels,proto3,customtype=github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter" json:"labels"` - Chunks []ChunkDetails `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks"` -} - -func (m *ChunksGroup) Reset() { *m = ChunksGroup{} } -func (*ChunksGroup) ProtoMessage() {} -func (*ChunksGroup) Descriptor() ([]byte, []int) { - return fileDescriptor_c38868cf63b27372, []int{1} -} -func (m *ChunksGroup) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ChunksGroup) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ChunksGroup.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *ChunksGroup) XXX_Merge(src proto.Message) { - xxx_messageInfo_ChunksGroup.Merge(m, src) -} -func (m *ChunksGroup) XXX_Size() int { - return m.Size() -} -func (m *ChunksGroup) XXX_DiscardUnknown() { - xxx_messageInfo_ChunksGroup.DiscardUnknown(m) -} - -var xxx_messageInfo_ChunksGroup proto.InternalMessageInfo - -func (m *ChunksGroup) GetChunks() []ChunkDetails { - if m != nil { - return m.Chunks - } - return nil -} - -type ChunkDetails struct { - ID string `protobuf:"bytes,1,opt,name=ID,proto3" json:"ID,omitempty"` - PartiallyDeletedInterval *Interval `protobuf:"bytes,2,opt,name=partially_deleted_interval,json=partiallyDeletedInterval,proto3" json:"partially_deleted_interval,omitempty"` -} - -func (m *ChunkDetails) Reset() { *m = ChunkDetails{} } -func (*ChunkDetails) ProtoMessage() {} -func (*ChunkDetails) Descriptor() ([]byte, []int) { - return fileDescriptor_c38868cf63b27372, []int{2} -} -func (m *ChunkDetails) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ChunkDetails) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ChunkDetails.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *ChunkDetails) XXX_Merge(src proto.Message) { - xxx_messageInfo_ChunkDetails.Merge(m, src) -} -func (m *ChunkDetails) XXX_Size() int { - return m.Size() -} -func (m *ChunkDetails) XXX_DiscardUnknown() { - xxx_messageInfo_ChunkDetails.DiscardUnknown(m) -} - -var xxx_messageInfo_ChunkDetails proto.InternalMessageInfo - -func (m *ChunkDetails) GetID() string { - if m != nil { - return m.ID - } - return "" -} - -func (m *ChunkDetails) GetPartiallyDeletedInterval() *Interval { - if m != nil { - return m.PartiallyDeletedInterval - } - return nil -} - -type Interval struct { - StartTimestampMs int64 `protobuf:"varint,1,opt,name=start_timestamp_ms,json=startTimestampMs,proto3" json:"start_timestamp_ms,omitempty"` - EndTimestampMs int64 `protobuf:"varint,2,opt,name=end_timestamp_ms,json=endTimestampMs,proto3" json:"end_timestamp_ms,omitempty"` -} - -func (m *Interval) Reset() { *m = Interval{} } -func (*Interval) ProtoMessage() {} -func (*Interval) Descriptor() ([]byte, []int) { - return fileDescriptor_c38868cf63b27372, []int{3} -} -func (m *Interval) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Interval) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Interval.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Interval) XXX_Merge(src proto.Message) { - xxx_messageInfo_Interval.Merge(m, src) -} -func (m *Interval) XXX_Size() int { - return m.Size() -} -func (m *Interval) XXX_DiscardUnknown() { - xxx_messageInfo_Interval.DiscardUnknown(m) -} - -var xxx_messageInfo_Interval proto.InternalMessageInfo - -func (m *Interval) GetStartTimestampMs() int64 { - if m != nil { - return m.StartTimestampMs - } - return 0 -} - -func (m *Interval) GetEndTimestampMs() int64 { - if m != nil { - return m.EndTimestampMs - } - return 0 -} - -func init() { - proto.RegisterType((*DeletePlan)(nil), "purgeplan.DeletePlan") - proto.RegisterType((*ChunksGroup)(nil), "purgeplan.ChunksGroup") - proto.RegisterType((*ChunkDetails)(nil), "purgeplan.ChunkDetails") - proto.RegisterType((*Interval)(nil), "purgeplan.Interval") -} - -func init() { proto.RegisterFile("delete_plan.proto", fileDescriptor_c38868cf63b27372) } - -var fileDescriptor_c38868cf63b27372 = []byte{ - // 446 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x52, 0x41, 0x8b, 0xd4, 0x30, - 0x18, 0x6d, 0xba, 0x52, 0xdc, 0x74, 0x5c, 0xd6, 0x2c, 0x68, 0x99, 0x43, 0x76, 0xe9, 0x69, 0x0e, - 0xda, 0x81, 0x15, 0x41, 0x41, 0x90, 0x1d, 0x0b, 0x32, 0xa0, 0xb0, 0x16, 0x4f, 0x5e, 0x4a, 0xda, - 0xc6, 0x6e, 0xdd, 0xb4, 0x89, 0x69, 0x2a, 0x7a, 0xf3, 0xe6, 0xd5, 0x9f, 0xe1, 0x0f, 0xf0, 0x47, - 0xec, 0x71, 0x8e, 0x8b, 0x87, 0xc1, 0xe9, 0x5c, 0x3c, 0xce, 0x4f, 0x90, 0xa6, 0xed, 0x4c, 0x15, - 0x3c, 0x78, 0xcb, 0xfb, 0xde, 0x7b, 0xc9, 0xcb, 0x4b, 0xe0, 0xed, 0x84, 0x32, 0xaa, 0x68, 0x28, - 0x18, 0x29, 0x3c, 0x21, 0xb9, 0xe2, 0x68, 0x5f, 0x54, 0x32, 0xa5, 0xcd, 0x60, 0x7c, 0x3f, 0xcd, - 0xd4, 0x45, 0x15, 0x79, 0x31, 0xcf, 0xa7, 0x29, 0x4f, 0xf9, 0x54, 0x2b, 0xa2, 0xea, 0xad, 0x46, - 0x1a, 0xe8, 0x55, 0xeb, 0x1c, 0x3f, 0x1e, 0xc8, 0x63, 0x2e, 0x15, 0xfd, 0x28, 0x24, 0x7f, 0x47, - 0x63, 0xd5, 0xa1, 0xa9, 0xb8, 0x4c, 0x7b, 0x22, 0xea, 0x16, 0xad, 0xd5, 0xfd, 0x02, 0x20, 0xf4, - 0x75, 0x94, 0x73, 0x46, 0x0a, 0xf4, 0x08, 0xde, 0x6a, 0x02, 0x84, 0x59, 0xa1, 0xa8, 0xfc, 0x40, - 0x98, 0x03, 0x4e, 0xc0, 0xc4, 0x3e, 0x3d, 0xf2, 0xb6, 0xd9, 0xbc, 0x79, 0x47, 0x05, 0xa3, 0x06, - 0xf6, 0x08, 0x3d, 0x85, 0xa3, 0xf8, 0xa2, 0x2a, 0x2e, 0xcb, 0x30, 0x95, 0xbc, 0x12, 0x8e, 0x79, - 0xb2, 0x37, 0xb1, 0x4f, 0xef, 0x0c, 0x8c, 0xcf, 0x34, 0xfd, 0xbc, 0x61, 0x67, 0x37, 0xae, 0x96, - 0xc7, 0x46, 0x60, 0xc7, 0xbb, 0x91, 0xfb, 0x1d, 0x40, 0x7b, 0x20, 0x41, 0x05, 0xb4, 0x18, 0x89, - 0x28, 0x2b, 0x1d, 0xa0, 0xb7, 0x3a, 0xf2, 0xfa, 0x1b, 0x78, 0x2f, 0x9a, 0xf9, 0x39, 0xc9, 0xe4, - 0xec, 0xac, 0xd9, 0xe7, 0xc7, 0xf2, 0xf8, 0xbf, 0x1a, 0x68, 0xfd, 0x67, 0x09, 0x11, 0x8a, 0xca, - 0xa0, 0x3b, 0x05, 0x3d, 0x84, 0x56, 0x1b, 0xa7, 0x8b, 0x7e, 0xf7, 0xef, 0xe8, 0x3e, 0x55, 0x24, - 0x63, 0x65, 0x97, 0xbd, 0x13, 0xbb, 0xef, 0xe1, 0x68, 0xc8, 0xa2, 0x03, 0x68, 0xce, 0x7d, 0x5d, - 0xdb, 0x7e, 0x60, 0xce, 0x7d, 0xf4, 0x0a, 0x8e, 0x05, 0x91, 0x2a, 0x23, 0x8c, 0x7d, 0x0a, 0xdb, - 0x47, 0x4f, 0x76, 0xf5, 0x9a, 0xff, 0xae, 0xd7, 0xd9, 0xda, 0xda, 0xf7, 0x49, 0x7a, 0xc6, 0x8d, - 0xe0, 0xcd, 0x6d, 0xed, 0xf7, 0x20, 0x2a, 0x15, 0x91, 0x2a, 0x54, 0x59, 0x4e, 0x4b, 0x45, 0x72, - 0x11, 0xe6, 0xa5, 0x3e, 0x7e, 0x2f, 0x38, 0xd4, 0xcc, 0xeb, 0x9e, 0x78, 0x59, 0xa2, 0x09, 0x3c, - 0xa4, 0x45, 0xf2, 0xa7, 0xd6, 0xd4, 0xda, 0x03, 0x5a, 0x24, 0x03, 0xe5, 0xec, 0xc9, 0x62, 0x85, - 0x8d, 0xeb, 0x15, 0x36, 0x36, 0x2b, 0x0c, 0x3e, 0xd7, 0x18, 0x7c, 0xab, 0x31, 0xb8, 0xaa, 0x31, - 0x58, 0xd4, 0x18, 0xfc, 0xac, 0x31, 0xf8, 0x55, 0x63, 0x63, 0x53, 0x63, 0xf0, 0x75, 0x8d, 0x8d, - 0xc5, 0x1a, 0x1b, 0xd7, 0x6b, 0x6c, 0xbc, 0xb1, 0xf4, 0x3d, 0x64, 0x64, 0xe9, 0xcf, 0xf5, 0xe0, - 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xf5, 0x46, 0x96, 0xf6, 0xe6, 0x02, 0x00, 0x00, -} - -func (this *DeletePlan) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*DeletePlan) - if !ok { - that2, ok := that.(DeletePlan) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if !this.PlanInterval.Equal(that1.PlanInterval) { - return false - } - if len(this.ChunksGroup) != len(that1.ChunksGroup) { - return false - } - for i := range this.ChunksGroup { - if !this.ChunksGroup[i].Equal(&that1.ChunksGroup[i]) { - return false - } - } - return true -} -func (this *ChunksGroup) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*ChunksGroup) - if !ok { - that2, ok := that.(ChunksGroup) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if len(this.Labels) != len(that1.Labels) { - return false - } - for i := range this.Labels { - if !this.Labels[i].Equal(that1.Labels[i]) { - return false - } - } - if len(this.Chunks) != len(that1.Chunks) { - return false - } - for i := range this.Chunks { - if !this.Chunks[i].Equal(&that1.Chunks[i]) { - return false - } - } - return true -} -func (this *ChunkDetails) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*ChunkDetails) - if !ok { - that2, ok := that.(ChunkDetails) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.ID != that1.ID { - return false - } - if !this.PartiallyDeletedInterval.Equal(that1.PartiallyDeletedInterval) { - return false - } - return true -} -func (this *Interval) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*Interval) - if !ok { - that2, ok := that.(Interval) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.StartTimestampMs != that1.StartTimestampMs { - return false - } - if this.EndTimestampMs != that1.EndTimestampMs { - return false - } - return true -} -func (this *DeletePlan) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&purger.DeletePlan{") - if this.PlanInterval != nil { - s = append(s, "PlanInterval: "+fmt.Sprintf("%#v", this.PlanInterval)+",\n") - } - if this.ChunksGroup != nil { - vs := make([]*ChunksGroup, len(this.ChunksGroup)) - for i := range vs { - vs[i] = &this.ChunksGroup[i] - } - s = append(s, "ChunksGroup: "+fmt.Sprintf("%#v", vs)+",\n") - } - s = append(s, "}") - return strings.Join(s, "") -} -func (this *ChunksGroup) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&purger.ChunksGroup{") - s = append(s, "Labels: "+fmt.Sprintf("%#v", this.Labels)+",\n") - if this.Chunks != nil { - vs := make([]*ChunkDetails, len(this.Chunks)) - for i := range vs { - vs[i] = &this.Chunks[i] - } - s = append(s, "Chunks: "+fmt.Sprintf("%#v", vs)+",\n") - } - s = append(s, "}") - return strings.Join(s, "") -} -func (this *ChunkDetails) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&purger.ChunkDetails{") - s = append(s, "ID: "+fmt.Sprintf("%#v", this.ID)+",\n") - if this.PartiallyDeletedInterval != nil { - s = append(s, "PartiallyDeletedInterval: "+fmt.Sprintf("%#v", this.PartiallyDeletedInterval)+",\n") - } - s = append(s, "}") - return strings.Join(s, "") -} -func (this *Interval) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 6) - s = append(s, "&purger.Interval{") - s = append(s, "StartTimestampMs: "+fmt.Sprintf("%#v", this.StartTimestampMs)+",\n") - s = append(s, "EndTimestampMs: "+fmt.Sprintf("%#v", this.EndTimestampMs)+",\n") - s = append(s, "}") - return strings.Join(s, "") -} -func valueToGoStringDeletePlan(v interface{}, typ string) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) -} -func (m *DeletePlan) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *DeletePlan) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *DeletePlan) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.ChunksGroup) > 0 { - for iNdEx := len(m.ChunksGroup) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.ChunksGroup[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintDeletePlan(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - } - if m.PlanInterval != nil { - { - size, err := m.PlanInterval.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintDeletePlan(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *ChunksGroup) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ChunksGroup) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ChunksGroup) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Chunks) > 0 { - for iNdEx := len(m.Chunks) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Chunks[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintDeletePlan(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - } - if len(m.Labels) > 0 { - for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- { - { - size := m.Labels[iNdEx].Size() - i -= size - if _, err := m.Labels[iNdEx].MarshalTo(dAtA[i:]); err != nil { - return 0, err - } - i = encodeVarintDeletePlan(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } - } - return len(dAtA) - i, nil -} - -func (m *ChunkDetails) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ChunkDetails) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ChunkDetails) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.PartiallyDeletedInterval != nil { - { - size, err := m.PartiallyDeletedInterval.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintDeletePlan(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - if len(m.ID) > 0 { - i -= len(m.ID) - copy(dAtA[i:], m.ID) - i = encodeVarintDeletePlan(dAtA, i, uint64(len(m.ID))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func (m *Interval) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Interval) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Interval) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.EndTimestampMs != 0 { - i = encodeVarintDeletePlan(dAtA, i, uint64(m.EndTimestampMs)) - i-- - dAtA[i] = 0x10 - } - if m.StartTimestampMs != 0 { - i = encodeVarintDeletePlan(dAtA, i, uint64(m.StartTimestampMs)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - -func encodeVarintDeletePlan(dAtA []byte, offset int, v uint64) int { - offset -= sovDeletePlan(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *DeletePlan) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.PlanInterval != nil { - l = m.PlanInterval.Size() - n += 1 + l + sovDeletePlan(uint64(l)) - } - if len(m.ChunksGroup) > 0 { - for _, e := range m.ChunksGroup { - l = e.Size() - n += 1 + l + sovDeletePlan(uint64(l)) - } - } - return n -} - -func (m *ChunksGroup) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if len(m.Labels) > 0 { - for _, e := range m.Labels { - l = e.Size() - n += 1 + l + sovDeletePlan(uint64(l)) - } - } - if len(m.Chunks) > 0 { - for _, e := range m.Chunks { - l = e.Size() - n += 1 + l + sovDeletePlan(uint64(l)) - } - } - return n -} - -func (m *ChunkDetails) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.ID) - if l > 0 { - n += 1 + l + sovDeletePlan(uint64(l)) - } - if m.PartiallyDeletedInterval != nil { - l = m.PartiallyDeletedInterval.Size() - n += 1 + l + sovDeletePlan(uint64(l)) - } - return n -} - -func (m *Interval) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.StartTimestampMs != 0 { - n += 1 + sovDeletePlan(uint64(m.StartTimestampMs)) - } - if m.EndTimestampMs != 0 { - n += 1 + sovDeletePlan(uint64(m.EndTimestampMs)) - } - return n -} - -func sovDeletePlan(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozDeletePlan(x uint64) (n int) { - return sovDeletePlan(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (this *DeletePlan) String() string { - if this == nil { - return "nil" - } - repeatedStringForChunksGroup := "[]ChunksGroup{" - for _, f := range this.ChunksGroup { - repeatedStringForChunksGroup += strings.Replace(strings.Replace(f.String(), "ChunksGroup", "ChunksGroup", 1), `&`, ``, 1) + "," - } - repeatedStringForChunksGroup += "}" - s := strings.Join([]string{`&DeletePlan{`, - `PlanInterval:` + strings.Replace(this.PlanInterval.String(), "Interval", "Interval", 1) + `,`, - `ChunksGroup:` + repeatedStringForChunksGroup + `,`, - `}`, - }, "") - return s -} -func (this *ChunksGroup) String() string { - if this == nil { - return "nil" - } - repeatedStringForChunks := "[]ChunkDetails{" - for _, f := range this.Chunks { - repeatedStringForChunks += strings.Replace(strings.Replace(f.String(), "ChunkDetails", "ChunkDetails", 1), `&`, ``, 1) + "," - } - repeatedStringForChunks += "}" - s := strings.Join([]string{`&ChunksGroup{`, - `Labels:` + fmt.Sprintf("%v", this.Labels) + `,`, - `Chunks:` + repeatedStringForChunks + `,`, - `}`, - }, "") - return s -} -func (this *ChunkDetails) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&ChunkDetails{`, - `ID:` + fmt.Sprintf("%v", this.ID) + `,`, - `PartiallyDeletedInterval:` + strings.Replace(this.PartiallyDeletedInterval.String(), "Interval", "Interval", 1) + `,`, - `}`, - }, "") - return s -} -func (this *Interval) String() string { - if this == nil { - return "nil" - } - s := strings.Join([]string{`&Interval{`, - `StartTimestampMs:` + fmt.Sprintf("%v", this.StartTimestampMs) + `,`, - `EndTimestampMs:` + fmt.Sprintf("%v", this.EndTimestampMs) + `,`, - `}`, - }, "") - return s -} -func valueToStringDeletePlan(v interface{}) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("*%v", pv) -} -func (m *DeletePlan) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: DeletePlan: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: DeletePlan: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PlanInterval", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.PlanInterval == nil { - m.PlanInterval = &Interval{} - } - if err := m.PlanInterval.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ChunksGroup", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ChunksGroup = append(m.ChunksGroup, ChunksGroup{}) - if err := m.ChunksGroup[len(m.ChunksGroup)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipDeletePlan(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *ChunksGroup) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ChunksGroup: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ChunksGroup: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Labels = append(m.Labels, github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter{}) - if err := m.Labels[len(m.Labels)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Chunks", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Chunks = append(m.Chunks, ChunkDetails{}) - if err := m.Chunks[len(m.Chunks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipDeletePlan(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *ChunkDetails) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ChunkDetails: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ChunkDetails: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.ID = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PartiallyDeletedInterval", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthDeletePlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthDeletePlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.PartiallyDeletedInterval == nil { - m.PartiallyDeletedInterval = &Interval{} - } - if err := m.PartiallyDeletedInterval.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipDeletePlan(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Interval) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Interval: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Interval: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field StartTimestampMs", wireType) - } - m.StartTimestampMs = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.StartTimestampMs |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field EndTimestampMs", wireType) - } - m.EndTimestampMs = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.EndTimestampMs |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipDeletePlan(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthDeletePlan - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipDeletePlan(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthDeletePlan - } - iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthDeletePlan - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowDeletePlan - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipDeletePlan(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthDeletePlan - } - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") -} - -var ( - ErrInvalidLengthDeletePlan = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowDeletePlan = fmt.Errorf("proto: integer overflow") -) diff --git a/pkg/chunk/purger/delete_plan.proto b/pkg/chunk/purger/delete_plan.proto deleted file mode 100644 index 834fc087498..00000000000 --- a/pkg/chunk/purger/delete_plan.proto +++ /dev/null @@ -1,34 +0,0 @@ -syntax = "proto3"; - -package purgeplan; - -option go_package = "purger"; - -import "github.com/gogo/protobuf/gogoproto/gogo.proto"; -import "github.com/cortexproject/cortex/pkg/cortexpb/cortex.proto"; - -option (gogoproto.marshaler_all) = true; -option (gogoproto.unmarshaler_all) = true; - -// DeletePlan holds all the chunks that are supposed to be deleted within an interval(usually a day) -// This Proto file is used just for storing Delete Plans in proto format. -message DeletePlan { - Interval plan_interval = 1; - repeated ChunksGroup chunks_group = 2 [(gogoproto.nullable) = false]; -} - -// ChunksGroup holds ChunkDetails and Labels for a group of chunks which have same series ID -message ChunksGroup { - repeated cortexpb.LabelPair labels = 1 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter"]; - repeated ChunkDetails chunks = 2 [(gogoproto.nullable) = false]; -} - -message ChunkDetails { - string ID = 1; - Interval partially_deleted_interval = 2; -} - -message Interval { - int64 start_timestamp_ms = 1; - int64 end_timestamp_ms = 2; -} diff --git a/pkg/chunk/purger/delete_requests_store.go b/pkg/chunk/purger/delete_requests_store.go deleted file mode 100644 index f3ec1edbc1f..00000000000 --- a/pkg/chunk/purger/delete_requests_store.go +++ /dev/null @@ -1,394 +0,0 @@ -package purger - -import ( - "context" - "encoding/binary" - "encoding/hex" - "errors" - "flag" - "fmt" - "hash/fnv" - "strconv" - "strings" - "time" - - "github.com/cortexproject/cortex/pkg/chunk" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" -) - -type ( - DeleteRequestStatus string - CacheKind string - indexType string -) - -const ( - StatusReceived DeleteRequestStatus = "received" - StatusBuildingPlan DeleteRequestStatus = "buildingPlan" - StatusDeleting DeleteRequestStatus = "deleting" - StatusProcessed DeleteRequestStatus = "processed" - - separator = "\000" // separator for series selectors in delete requests - - // CacheKindStore is for cache gen number for store cache - CacheKindStore CacheKind = "store" - // CacheKindResults is for cache gen number for results cache - CacheKindResults CacheKind = "results" - - deleteRequestID indexType = "1" - deleteRequestDetails indexType = "2" - cacheGenNum indexType = "3" -) - -var ( - pendingDeleteRequestStatuses = []DeleteRequestStatus{StatusReceived, StatusBuildingPlan, StatusDeleting} - - ErrDeleteRequestNotFound = errors.New("could not find matching delete request") -) - -// DeleteRequest holds all the details about a delete request. -type DeleteRequest struct { - RequestID string `json:"request_id"` - UserID string `json:"-"` - StartTime model.Time `json:"start_time"` - EndTime model.Time `json:"end_time"` - Selectors []string `json:"selectors"` - Status DeleteRequestStatus `json:"status"` - Matchers [][]*labels.Matcher `json:"-"` - CreatedAt model.Time `json:"created_at"` -} - -// cacheGenNumbers holds store and results cache gen numbers for a user. -type cacheGenNumbers struct { - store, results string -} - -// DeleteStore provides all the methods required to manage lifecycle of delete request and things related to it. -type DeleteStore struct { - cfg DeleteStoreConfig - indexClient chunk.IndexClient -} - -// DeleteStoreConfig holds configuration for delete store. -type DeleteStoreConfig struct { - Store string `yaml:"store"` - RequestsTableName string `yaml:"requests_table_name"` - ProvisionConfig TableProvisioningConfig `yaml:"table_provisioning"` -} - -// RegisterFlags adds the flags required to configure this flag set. -func (cfg *DeleteStoreConfig) RegisterFlags(f *flag.FlagSet) { - cfg.ProvisionConfig.RegisterFlags("deletes.table", f) - f.StringVar(&cfg.Store, "deletes.store", "", "Store for keeping delete request") - f.StringVar(&cfg.RequestsTableName, "deletes.requests-table-name", "delete_requests", "Name of the table which stores delete requests") -} - -// NewDeleteStore creates a store for managing delete requests. -func NewDeleteStore(cfg DeleteStoreConfig, indexClient chunk.IndexClient) (*DeleteStore, error) { - ds := DeleteStore{ - cfg: cfg, - indexClient: indexClient, - } - - return &ds, nil -} - -// Add creates entries for a new delete request. -func (ds *DeleteStore) AddDeleteRequest(ctx context.Context, userID string, startTime, endTime model.Time, selectors []string) error { - return ds.addDeleteRequest(ctx, userID, model.Now(), startTime, endTime, selectors) - -} - -// addDeleteRequest is also used for tests to create delete requests with different createdAt time. -func (ds *DeleteStore) addDeleteRequest(ctx context.Context, userID string, createdAt, startTime, endTime model.Time, selectors []string) error { - requestID := generateUniqueID(userID, selectors) - - for { - _, err := ds.GetDeleteRequest(ctx, userID, string(requestID)) - if err != nil { - if err == ErrDeleteRequestNotFound { - break - } - return err - } - - // we have a collision here, lets recreate a new requestID and check for collision - time.Sleep(time.Millisecond) - requestID = generateUniqueID(userID, selectors) - } - - // userID, requestID - userIDAndRequestID := fmt.Sprintf("%s:%s", userID, requestID) - - // Add an entry with userID, requestID as range key and status as value to make it easy to manage and lookup status - // We don't want to set anything in hash key here since we would want to find delete requests by just status - writeBatch := ds.indexClient.NewWriteBatch() - writeBatch.Add(ds.cfg.RequestsTableName, string(deleteRequestID), []byte(userIDAndRequestID), []byte(StatusReceived)) - - // Add another entry with additional details like creation time, time range of delete request and selectors in value - rangeValue := fmt.Sprintf("%x:%x:%x", int64(createdAt), int64(startTime), int64(endTime)) - writeBatch.Add(ds.cfg.RequestsTableName, fmt.Sprintf("%s:%s", deleteRequestDetails, userIDAndRequestID), - []byte(rangeValue), []byte(strings.Join(selectors, separator))) - - // we update only cache gen number because only query responses are changing at this stage. - // we still have to query data from store for doing query time filtering and we don't want to invalidate its results now. - writeBatch.Add(ds.cfg.RequestsTableName, fmt.Sprintf("%s:%s:%s", cacheGenNum, userID, CacheKindResults), - []byte{}, []byte(strconv.FormatInt(time.Now().Unix(), 10))) - - return ds.indexClient.BatchWrite(ctx, writeBatch) -} - -// GetDeleteRequestsByStatus returns all delete requests for given status. -func (ds *DeleteStore) GetDeleteRequestsByStatus(ctx context.Context, status DeleteRequestStatus) ([]DeleteRequest, error) { - return ds.queryDeleteRequests(ctx, chunk.IndexQuery{ - TableName: ds.cfg.RequestsTableName, - HashValue: string(deleteRequestID), - ValueEqual: []byte(status), - }) -} - -// GetDeleteRequestsForUserByStatus returns all delete requests for a user with given status. -func (ds *DeleteStore) GetDeleteRequestsForUserByStatus(ctx context.Context, userID string, status DeleteRequestStatus) ([]DeleteRequest, error) { - return ds.queryDeleteRequests(ctx, chunk.IndexQuery{ - TableName: ds.cfg.RequestsTableName, - HashValue: string(deleteRequestID), - RangeValuePrefix: []byte(userID), - ValueEqual: []byte(status), - }) -} - -// GetAllDeleteRequestsForUser returns all delete requests for a user. -func (ds *DeleteStore) GetAllDeleteRequestsForUser(ctx context.Context, userID string) ([]DeleteRequest, error) { - return ds.queryDeleteRequests(ctx, chunk.IndexQuery{ - TableName: ds.cfg.RequestsTableName, - HashValue: string(deleteRequestID), - RangeValuePrefix: []byte(userID), - }) -} - -// UpdateStatus updates status of a delete request. -func (ds *DeleteStore) UpdateStatus(ctx context.Context, userID, requestID string, newStatus DeleteRequestStatus) error { - userIDAndRequestID := fmt.Sprintf("%s:%s", userID, requestID) - - writeBatch := ds.indexClient.NewWriteBatch() - writeBatch.Add(ds.cfg.RequestsTableName, string(deleteRequestID), []byte(userIDAndRequestID), []byte(newStatus)) - - if newStatus == StatusProcessed { - // we have deleted data from store so invalidate cache only for store since we don't have to do runtime filtering anymore. - // we don't have to change cache gen number because we were anyways doing runtime filtering - writeBatch.Add(ds.cfg.RequestsTableName, fmt.Sprintf("%s:%s:%s", cacheGenNum, userID, CacheKindStore), []byte{}, []byte(strconv.FormatInt(time.Now().Unix(), 10))) - } - - return ds.indexClient.BatchWrite(ctx, writeBatch) -} - -// GetDeleteRequest returns delete request with given requestID. -func (ds *DeleteStore) GetDeleteRequest(ctx context.Context, userID, requestID string) (*DeleteRequest, error) { - userIDAndRequestID := fmt.Sprintf("%s:%s", userID, requestID) - - deleteRequests, err := ds.queryDeleteRequests(ctx, chunk.IndexQuery{ - TableName: ds.cfg.RequestsTableName, - HashValue: string(deleteRequestID), - RangeValuePrefix: []byte(userIDAndRequestID), - }) - - if err != nil { - return nil, err - } - - if len(deleteRequests) == 0 { - return nil, ErrDeleteRequestNotFound - } - - return &deleteRequests[0], nil -} - -// GetPendingDeleteRequestsForUser returns all delete requests for a user which are not processed. -func (ds *DeleteStore) GetPendingDeleteRequestsForUser(ctx context.Context, userID string) ([]DeleteRequest, error) { - pendingDeleteRequests := []DeleteRequest{} - for _, status := range pendingDeleteRequestStatuses { - deleteRequests, err := ds.GetDeleteRequestsForUserByStatus(ctx, userID, status) - if err != nil { - return nil, err - } - - pendingDeleteRequests = append(pendingDeleteRequests, deleteRequests...) - } - - return pendingDeleteRequests, nil -} - -func (ds *DeleteStore) queryDeleteRequests(ctx context.Context, deleteQuery chunk.IndexQuery) ([]DeleteRequest, error) { - deleteRequests := []DeleteRequest{} - // No need to lock inside the callback since we run a single index query. - err := ds.indexClient.QueryPages(ctx, []chunk.IndexQuery{deleteQuery}, func(query chunk.IndexQuery, batch chunk.ReadBatch) (shouldContinue bool) { - itr := batch.Iterator() - for itr.Next() { - userID, requestID := splitUserIDAndRequestID(string(itr.RangeValue())) - - deleteRequests = append(deleteRequests, DeleteRequest{ - UserID: userID, - RequestID: requestID, - Status: DeleteRequestStatus(itr.Value()), - }) - } - return true - }) - if err != nil { - return nil, err - } - - for i, deleteRequest := range deleteRequests { - deleteRequestQuery := []chunk.IndexQuery{ - { - TableName: ds.cfg.RequestsTableName, - HashValue: fmt.Sprintf("%s:%s:%s", deleteRequestDetails, deleteRequest.UserID, deleteRequest.RequestID), - }, - } - - var parseError error - err := ds.indexClient.QueryPages(ctx, deleteRequestQuery, func(query chunk.IndexQuery, batch chunk.ReadBatch) (shouldContinue bool) { - itr := batch.Iterator() - itr.Next() - - deleteRequest, err = parseDeleteRequestTimestamps(itr.RangeValue(), deleteRequest) - if err != nil { - parseError = err - return false - } - - deleteRequest.Selectors = strings.Split(string(itr.Value()), separator) - deleteRequests[i] = deleteRequest - - return true - }) - - if err != nil { - return nil, err - } - - if parseError != nil { - return nil, parseError - } - } - - return deleteRequests, nil -} - -// getCacheGenerationNumbers returns cache gen numbers for a user. -func (ds *DeleteStore) getCacheGenerationNumbers(ctx context.Context, userID string) (*cacheGenNumbers, error) { - storeCacheGen, err := ds.queryCacheGenerationNumber(ctx, userID, CacheKindStore) - if err != nil { - return nil, err - } - - resultsCacheGen, err := ds.queryCacheGenerationNumber(ctx, userID, CacheKindResults) - if err != nil { - return nil, err - } - - return &cacheGenNumbers{storeCacheGen, resultsCacheGen}, nil -} - -func (ds *DeleteStore) queryCacheGenerationNumber(ctx context.Context, userID string, kind CacheKind) (string, error) { - query := chunk.IndexQuery{TableName: ds.cfg.RequestsTableName, HashValue: fmt.Sprintf("%s:%s:%s", cacheGenNum, userID, kind)} - - genNumber := "" - err := ds.indexClient.QueryPages(ctx, []chunk.IndexQuery{query}, func(query chunk.IndexQuery, batch chunk.ReadBatch) (shouldContinue bool) { - itr := batch.Iterator() - for itr.Next() { - genNumber = string(itr.Value()) - break - } - return false - }) - - if err != nil { - return "", err - } - - return genNumber, nil -} - -// RemoveDeleteRequest removes a delete request and increments cache gen number -func (ds *DeleteStore) RemoveDeleteRequest(ctx context.Context, userID, requestID string, createdAt, startTime, endTime model.Time) error { - userIDAndRequestID := fmt.Sprintf("%s:%s", userID, requestID) - - writeBatch := ds.indexClient.NewWriteBatch() - writeBatch.Delete(ds.cfg.RequestsTableName, string(deleteRequestID), []byte(userIDAndRequestID)) - - // Add another entry with additional details like creation time, time range of delete request and selectors in value - rangeValue := fmt.Sprintf("%x:%x:%x", int64(createdAt), int64(startTime), int64(endTime)) - writeBatch.Delete(ds.cfg.RequestsTableName, fmt.Sprintf("%s:%s", deleteRequestDetails, userIDAndRequestID), - []byte(rangeValue)) - - // we need to invalidate results cache since removal of delete request would cause query results to change - writeBatch.Add(ds.cfg.RequestsTableName, fmt.Sprintf("%s:%s:%s", cacheGenNum, userID, CacheKindResults), - []byte{}, []byte(strconv.FormatInt(time.Now().Unix(), 10))) - - return ds.indexClient.BatchWrite(ctx, writeBatch) -} - -func parseDeleteRequestTimestamps(rangeValue []byte, deleteRequest DeleteRequest) (DeleteRequest, error) { - hexParts := strings.Split(string(rangeValue), ":") - if len(hexParts) != 3 { - return deleteRequest, errors.New("invalid key in parsing delete request lookup response") - } - - createdAt, err := strconv.ParseInt(hexParts[0], 16, 64) - if err != nil { - return deleteRequest, err - } - - from, err := strconv.ParseInt(hexParts[1], 16, 64) - if err != nil { - return deleteRequest, err - - } - through, err := strconv.ParseInt(hexParts[2], 16, 64) - if err != nil { - return deleteRequest, err - - } - - deleteRequest.CreatedAt = model.Time(createdAt) - deleteRequest.StartTime = model.Time(from) - deleteRequest.EndTime = model.Time(through) - - return deleteRequest, nil -} - -// An id is useful in managing delete requests -func generateUniqueID(orgID string, selectors []string) []byte { - uniqueID := fnv.New32() - _, _ = uniqueID.Write([]byte(orgID)) - - timeNow := make([]byte, 8) - binary.LittleEndian.PutUint64(timeNow, uint64(time.Now().UnixNano())) - _, _ = uniqueID.Write(timeNow) - - for _, selector := range selectors { - _, _ = uniqueID.Write([]byte(selector)) - } - - return encodeUniqueID(uniqueID.Sum32()) -} - -func encodeUniqueID(t uint32) []byte { - throughBytes := make([]byte, 4) - binary.BigEndian.PutUint32(throughBytes, t) - encodedThroughBytes := make([]byte, 8) - hex.Encode(encodedThroughBytes, throughBytes) - return encodedThroughBytes -} - -func splitUserIDAndRequestID(rangeValue string) (userID, requestID string) { - lastIndex := strings.LastIndex(rangeValue, ":") - - userID = rangeValue[:lastIndex] - requestID = rangeValue[lastIndex+1:] - - return -} diff --git a/pkg/chunk/purger/purger.go b/pkg/chunk/purger/purger.go deleted file mode 100644 index 7c37c29300b..00000000000 --- a/pkg/chunk/purger/purger.go +++ /dev/null @@ -1,828 +0,0 @@ -package purger - -import ( - "bytes" - "context" - "flag" - "fmt" - "io/ioutil" - "sync" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/gogo/protobuf/proto" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/promql" - "github.com/prometheus/prometheus/promql/parser" - "github.com/weaveworks/common/user" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/cortexpb" - util_log "github.com/cortexproject/cortex/pkg/util/log" - "github.com/cortexproject/cortex/pkg/util/services" -) - -const ( - millisecondPerDay = int64(24 * time.Hour / time.Millisecond) - statusSuccess = "success" - statusFail = "fail" - loadRequestsInterval = time.Hour - retryFailedRequestsInterval = 15 * time.Minute -) - -type purgerMetrics struct { - deleteRequestsProcessedTotal *prometheus.CounterVec - deleteRequestsChunksSelectedTotal *prometheus.CounterVec - deleteRequestsProcessingFailures *prometheus.CounterVec - loadPendingRequestsAttempsTotal *prometheus.CounterVec - oldestPendingDeleteRequestAgeSeconds prometheus.Gauge - pendingDeleteRequestsCount prometheus.Gauge -} - -func newPurgerMetrics(r prometheus.Registerer) *purgerMetrics { - m := purgerMetrics{} - - m.deleteRequestsProcessedTotal = promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "purger_delete_requests_processed_total", - Help: "Number of delete requests processed per user", - }, []string{"user"}) - m.deleteRequestsChunksSelectedTotal = promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "purger_delete_requests_chunks_selected_total", - Help: "Number of chunks selected while building delete plans per user", - }, []string{"user"}) - m.deleteRequestsProcessingFailures = promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "purger_delete_requests_processing_failures_total", - Help: "Number of delete requests processing failures per user", - }, []string{"user"}) - m.loadPendingRequestsAttempsTotal = promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "purger_load_pending_requests_attempts_total", - Help: "Number of attempts that were made to load pending requests with status", - }, []string{"status"}) - m.oldestPendingDeleteRequestAgeSeconds = promauto.With(r).NewGauge(prometheus.GaugeOpts{ - Namespace: "cortex", - Name: "purger_oldest_pending_delete_request_age_seconds", - Help: "Age of oldest pending delete request in seconds, since they are over their cancellation period", - }) - m.pendingDeleteRequestsCount = promauto.With(r).NewGauge(prometheus.GaugeOpts{ - Namespace: "cortex", - Name: "purger_pending_delete_requests_count", - Help: "Count of delete requests which are over their cancellation period and have not finished processing yet", - }) - - return &m -} - -type deleteRequestWithLogger struct { - DeleteRequest - logger log.Logger // logger is initialized with userID and requestID to add context to every log generated using this -} - -// Config holds config for chunks Purger -type Config struct { - Enable bool `yaml:"enable"` - NumWorkers int `yaml:"num_workers"` - ObjectStoreType string `yaml:"object_store_type"` - DeleteRequestCancelPeriod time.Duration `yaml:"delete_request_cancel_period"` -} - -// RegisterFlags registers CLI flags for Config -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.BoolVar(&cfg.Enable, "purger.enable", false, "Enable purger to allow deletion of series. Be aware that Delete series feature is still experimental") - f.IntVar(&cfg.NumWorkers, "purger.num-workers", 2, "Number of workers executing delete plans in parallel") - f.StringVar(&cfg.ObjectStoreType, "purger.object-store-type", "", "Name of the object store to use for storing delete plans") - f.DurationVar(&cfg.DeleteRequestCancelPeriod, "purger.delete-request-cancel-period", 24*time.Hour, "Allow cancellation of delete request until duration after they are created. Data would be deleted only after delete requests have been older than this duration. Ideally this should be set to at least 24h.") -} - -type workerJob struct { - planNo int - userID string - deleteRequestID string - logger log.Logger -} - -// Purger does the purging of data which is requested to be deleted. Purger only works for chunks. -type Purger struct { - services.Service - - cfg Config - deleteStore *DeleteStore - chunkStore chunk.Store - objectClient chunk.ObjectClient - metrics *purgerMetrics - - executePlansChan chan deleteRequestWithLogger - workerJobChan chan workerJob - - // we would only allow processing of singe delete request at a time since delete requests touching same chunks could change the chunk IDs of partially deleted chunks - // and break the purge plan for other requests - inProcessRequests *inProcessRequestsCollection - - // We do not want to limit pulling new delete requests to a fixed interval which otherwise would limit number of delete requests we process per user. - // While loading delete requests if we find more requests from user pending to be processed, we just set their id in usersWithPendingRequests and - // when a user's delete request gets processed we just check this map to see whether we want to load more requests without waiting for next ticker to load new batch. - usersWithPendingRequests map[string]struct{} - usersWithPendingRequestsMtx sync.Mutex - pullNewRequestsChan chan struct{} - - pendingPlansCount map[string]int // per request pending plan count - pendingPlansCountMtx sync.Mutex - - wg sync.WaitGroup -} - -// NewPurger creates a new Purger -func NewPurger(cfg Config, deleteStore *DeleteStore, chunkStore chunk.Store, storageClient chunk.ObjectClient, registerer prometheus.Registerer) (*Purger, error) { - util_log.WarnExperimentalUse("Delete series API") - - purger := Purger{ - cfg: cfg, - deleteStore: deleteStore, - chunkStore: chunkStore, - objectClient: storageClient, - metrics: newPurgerMetrics(registerer), - pullNewRequestsChan: make(chan struct{}, 1), - executePlansChan: make(chan deleteRequestWithLogger, 50), - workerJobChan: make(chan workerJob, 50), - inProcessRequests: newInProcessRequestsCollection(), - usersWithPendingRequests: map[string]struct{}{}, - pendingPlansCount: map[string]int{}, - } - - purger.Service = services.NewBasicService(purger.init, purger.loop, purger.stop) - return &purger, nil -} - -// init starts workers, scheduler and then loads in process delete requests -func (p *Purger) init(ctx context.Context) error { - for i := 0; i < p.cfg.NumWorkers; i++ { - p.wg.Add(1) - go p.worker() - } - - p.wg.Add(1) - go p.jobScheduler(ctx) - - return p.loadInprocessDeleteRequests() -} - -func (p *Purger) loop(ctx context.Context) error { - loadRequests := func() { - status := statusSuccess - - err := p.pullDeleteRequestsToPlanDeletes() - if err != nil { - status = statusFail - level.Error(util_log.Logger).Log("msg", "error pulling delete requests for building plans", "err", err) - } - - p.metrics.loadPendingRequestsAttempsTotal.WithLabelValues(status).Inc() - } - - // load requests on startup instead of waiting for first ticker - loadRequests() - - loadRequestsTicker := time.NewTicker(loadRequestsInterval) - defer loadRequestsTicker.Stop() - - retryFailedRequestsTicker := time.NewTicker(retryFailedRequestsInterval) - defer retryFailedRequestsTicker.Stop() - - for { - select { - case <-loadRequestsTicker.C: - loadRequests() - case <-p.pullNewRequestsChan: - loadRequests() - case <-retryFailedRequestsTicker.C: - p.retryFailedRequests() - case <-ctx.Done(): - return nil - } - } -} - -// Stop waits until all background tasks stop. -func (p *Purger) stop(_ error) error { - p.wg.Wait() - return nil -} - -func (p *Purger) retryFailedRequests() { - userIDsWithFailedRequest := p.inProcessRequests.listUsersWithFailedRequest() - - for _, userID := range userIDsWithFailedRequest { - deleteRequest := p.inProcessRequests.get(userID) - if deleteRequest == nil { - level.Error(util_log.Logger).Log("msg", "expected an in-process delete request", "user", userID) - continue - } - - p.inProcessRequests.unsetFailedRequestForUser(userID) - err := p.resumeStalledRequest(*deleteRequest) - if err != nil { - reqWithLogger := makeDeleteRequestWithLogger(*deleteRequest, util_log.Logger) - level.Error(reqWithLogger.logger).Log("msg", "failed to resume failed request", "err", err) - } - } -} - -func (p *Purger) workerJobCleanup(job workerJob) { - err := p.removeDeletePlan(context.Background(), job.userID, job.deleteRequestID, job.planNo) - if err != nil { - level.Error(job.logger).Log("msg", "error removing delete plan", - "plan_no", job.planNo, "err", err) - return - } - - p.pendingPlansCountMtx.Lock() - p.pendingPlansCount[job.deleteRequestID]-- - - if p.pendingPlansCount[job.deleteRequestID] == 0 { - level.Info(job.logger).Log("msg", "finished execution of all plans, cleaning up and updating status of request") - - err := p.deleteStore.UpdateStatus(context.Background(), job.userID, job.deleteRequestID, StatusProcessed) - if err != nil { - level.Error(job.logger).Log("msg", "error updating delete request status to process", "err", err) - } - - p.metrics.deleteRequestsProcessedTotal.WithLabelValues(job.userID).Inc() - delete(p.pendingPlansCount, job.deleteRequestID) - p.pendingPlansCountMtx.Unlock() - - p.inProcessRequests.remove(job.userID) - - // request loading of more delete request if - // - user has more pending requests and - // - we do not have a pending request to load more requests - p.usersWithPendingRequestsMtx.Lock() - defer p.usersWithPendingRequestsMtx.Unlock() - if _, ok := p.usersWithPendingRequests[job.userID]; ok { - delete(p.usersWithPendingRequests, job.userID) - select { - case p.pullNewRequestsChan <- struct{}{}: - // sent - default: - // already sent - } - } else if len(p.usersWithPendingRequests) == 0 { - // there are no pending requests from any of the users, set the oldest pending request and number of pending requests to 0 - p.metrics.oldestPendingDeleteRequestAgeSeconds.Set(0) - p.metrics.pendingDeleteRequestsCount.Set(0) - } - } else { - p.pendingPlansCountMtx.Unlock() - } -} - -// we send all the delete plans to workerJobChan -func (p *Purger) jobScheduler(ctx context.Context) { - defer p.wg.Done() - - for { - select { - case req := <-p.executePlansChan: - numPlans := numPlans(req.StartTime, req.EndTime) - level.Info(req.logger).Log("msg", "sending jobs to workers for purging data", "num_jobs", numPlans) - - p.pendingPlansCountMtx.Lock() - p.pendingPlansCount[req.RequestID] = numPlans - p.pendingPlansCountMtx.Unlock() - - for i := 0; i < numPlans; i++ { - p.workerJobChan <- workerJob{planNo: i, userID: req.UserID, - deleteRequestID: req.RequestID, logger: req.logger} - } - case <-ctx.Done(): - close(p.workerJobChan) - return - } - } -} - -func (p *Purger) worker() { - defer p.wg.Done() - - for job := range p.workerJobChan { - err := p.executePlan(job.userID, job.deleteRequestID, job.planNo, job.logger) - if err != nil { - p.metrics.deleteRequestsProcessingFailures.WithLabelValues(job.userID).Inc() - level.Error(job.logger).Log("msg", "error executing delete plan", - "plan_no", job.planNo, "err", err) - continue - } - - p.workerJobCleanup(job) - } -} - -func (p *Purger) executePlan(userID, requestID string, planNo int, logger log.Logger) (err error) { - logger = log.With(logger, "plan_no", planNo) - - defer func() { - if err != nil { - p.inProcessRequests.setFailedRequestForUser(userID) - } - }() - - plan, err := p.getDeletePlan(context.Background(), userID, requestID, planNo) - if err != nil { - if err == chunk.ErrStorageObjectNotFound { - level.Info(logger).Log("msg", "plan not found, must have been executed already") - // this means plan was already executed and got removed. Do nothing. - return nil - } - return err - } - - level.Info(logger).Log("msg", "executing plan") - - ctx := user.InjectOrgID(context.Background(), userID) - - for i := range plan.ChunksGroup { - level.Debug(logger).Log("msg", "deleting chunks", "labels", plan.ChunksGroup[i].Labels) - - for _, chunkDetails := range plan.ChunksGroup[i].Chunks { - chunkRef, err := chunk.ParseExternalKey(userID, chunkDetails.ID) - if err != nil { - return err - } - - var partiallyDeletedInterval *model.Interval = nil - if chunkDetails.PartiallyDeletedInterval != nil { - partiallyDeletedInterval = &model.Interval{ - Start: model.Time(chunkDetails.PartiallyDeletedInterval.StartTimestampMs), - End: model.Time(chunkDetails.PartiallyDeletedInterval.EndTimestampMs), - } - } - - err = p.chunkStore.DeleteChunk(ctx, chunkRef.From, chunkRef.Through, chunkRef.UserID, - chunkDetails.ID, cortexpb.FromLabelAdaptersToLabels(plan.ChunksGroup[i].Labels), partiallyDeletedInterval) - if err != nil { - if isMissingChunkErr(err) { - level.Error(logger).Log("msg", "chunk not found for deletion. We may have already deleted it", - "chunk_id", chunkDetails.ID) - continue - } - return err - } - } - - level.Debug(logger).Log("msg", "deleting series", "labels", plan.ChunksGroup[i].Labels) - - // this is mostly required to clean up series ids from series store - err := p.chunkStore.DeleteSeriesIDs(ctx, model.Time(plan.PlanInterval.StartTimestampMs), model.Time(plan.PlanInterval.EndTimestampMs), - userID, cortexpb.FromLabelAdaptersToLabels(plan.ChunksGroup[i].Labels)) - if err != nil { - return err - } - } - - level.Info(logger).Log("msg", "finished execution of plan") - - return -} - -// we need to load all in process delete requests on startup to finish them first -func (p *Purger) loadInprocessDeleteRequests() error { - inprocessRequests, err := p.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusBuildingPlan) - if err != nil { - return err - } - - requestsWithDeletingStatus, err := p.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusDeleting) - if err != nil { - return err - } - - inprocessRequests = append(inprocessRequests, requestsWithDeletingStatus...) - - for i := range inprocessRequests { - deleteRequest := inprocessRequests[i] - p.inProcessRequests.set(deleteRequest.UserID, &deleteRequest) - req := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - - level.Info(req.logger).Log("msg", "resuming in process delete requests", "status", deleteRequest.Status) - err = p.resumeStalledRequest(deleteRequest) - if err != nil { - level.Error(req.logger).Log("msg", "failed to resume stalled request", "err", err) - } - - } - - return nil -} - -func (p *Purger) resumeStalledRequest(deleteRequest DeleteRequest) error { - req := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - - if deleteRequest.Status == StatusBuildingPlan { - err := p.buildDeletePlan(req) - if err != nil { - p.metrics.deleteRequestsProcessingFailures.WithLabelValues(deleteRequest.UserID).Inc() - return errors.Wrap(err, "failed to build delete plan") - } - - deleteRequest.Status = StatusDeleting - } - - if deleteRequest.Status == StatusDeleting { - level.Info(req.logger).Log("msg", "sending delete request for execution") - p.executePlansChan <- req - } - - return nil -} - -// pullDeleteRequestsToPlanDeletes pulls delete requests which do not have their delete plans built yet and sends them for building delete plans -// after pulling delete requests for building plans, it updates its status to StatusBuildingPlan status to avoid picking this up again next time -func (p *Purger) pullDeleteRequestsToPlanDeletes() error { - deleteRequests, err := p.deleteStore.GetDeleteRequestsByStatus(context.Background(), StatusReceived) - if err != nil { - return err - } - - pendingDeleteRequestsCount := p.inProcessRequests.len() - now := model.Now() - oldestPendingRequestCreatedAt := model.Time(0) - - // requests which are still being processed are also considered pending - if pendingDeleteRequestsCount != 0 { - oldestInProcessRequest := p.inProcessRequests.getOldest() - if oldestInProcessRequest != nil { - oldestPendingRequestCreatedAt = oldestInProcessRequest.CreatedAt - } - } - - for i := range deleteRequests { - deleteRequest := deleteRequests[i] - - // adding an extra minute here to avoid a race between cancellation of request and picking of the request for processing - if deleteRequest.CreatedAt.Add(p.cfg.DeleteRequestCancelPeriod).Add(time.Minute).After(model.Now()) { - continue - } - - pendingDeleteRequestsCount++ - if oldestPendingRequestCreatedAt == 0 || deleteRequest.CreatedAt.Before(oldestPendingRequestCreatedAt) { - oldestPendingRequestCreatedAt = deleteRequest.CreatedAt - } - - if inprocessDeleteRequest := p.inProcessRequests.get(deleteRequest.UserID); inprocessDeleteRequest != nil { - p.usersWithPendingRequestsMtx.Lock() - p.usersWithPendingRequests[deleteRequest.UserID] = struct{}{} - p.usersWithPendingRequestsMtx.Unlock() - - level.Debug(util_log.Logger).Log("msg", "skipping delete request processing for now since another request from same user is already in process", - "inprocess_request_id", inprocessDeleteRequest.RequestID, - "skipped_request_id", deleteRequest.RequestID, "user_id", deleteRequest.UserID) - continue - } - - err = p.deleteStore.UpdateStatus(context.Background(), deleteRequest.UserID, deleteRequest.RequestID, StatusBuildingPlan) - if err != nil { - return err - } - - deleteRequest.Status = StatusBuildingPlan - p.inProcessRequests.set(deleteRequest.UserID, &deleteRequest) - req := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - - level.Info(req.logger).Log("msg", "building plan for a new delete request") - - err := p.buildDeletePlan(req) - if err != nil { - p.metrics.deleteRequestsProcessingFailures.WithLabelValues(deleteRequest.UserID).Inc() - - // We do not want to remove this delete request from inProcessRequests to make sure - // we do not move multiple deleting requests in deletion process. - // None of the other delete requests from the user would be considered for processing until then. - level.Error(req.logger).Log("msg", "error building delete plan", "err", err) - return err - } - - level.Info(req.logger).Log("msg", "sending delete request for execution") - p.executePlansChan <- req - } - - // track age of oldest delete request since they are over their cancellation period - oldestPendingRequestAge := time.Duration(0) - if oldestPendingRequestCreatedAt != 0 { - oldestPendingRequestAge = now.Sub(oldestPendingRequestCreatedAt.Add(p.cfg.DeleteRequestCancelPeriod)) - } - p.metrics.oldestPendingDeleteRequestAgeSeconds.Set(float64(oldestPendingRequestAge / time.Second)) - p.metrics.pendingDeleteRequestsCount.Set(float64(pendingDeleteRequestsCount)) - - return nil -} - -// buildDeletePlan builds per day delete plan for given delete requests. -// A days plan will include chunk ids and labels of all the chunks which are supposed to be deleted. -// Chunks are grouped together by labels to avoid storing labels repetitively. -// After building delete plans it updates status of delete request to StatusDeleting and sends it for execution -func (p *Purger) buildDeletePlan(req deleteRequestWithLogger) (err error) { - ctx := context.Background() - ctx = user.InjectOrgID(ctx, req.UserID) - - defer func() { - if err != nil { - p.inProcessRequests.setFailedRequestForUser(req.UserID) - } else { - req.Status = StatusDeleting - p.inProcessRequests.set(req.UserID, &req.DeleteRequest) - } - }() - - perDayTimeRange := splitByDay(req.StartTime, req.EndTime) - level.Info(req.logger).Log("msg", "building delete plan", "num_plans", len(perDayTimeRange)) - - plans := make([][]byte, len(perDayTimeRange)) - includedChunkIDs := map[string]struct{}{} - - for i, planRange := range perDayTimeRange { - chunksGroups := []ChunksGroup{} - - for _, selector := range req.Selectors { - matchers, err := parser.ParseMetricSelector(selector) - if err != nil { - return err - } - - chunks, err := p.chunkStore.Get(ctx, req.UserID, planRange.Start, planRange.End, matchers...) - if err != nil { - return err - } - - var cg []ChunksGroup - cg, includedChunkIDs = groupChunks(chunks, req.StartTime, req.EndTime, includedChunkIDs) - - if len(cg) != 0 { - chunksGroups = append(chunksGroups, cg...) - } - } - - plan := DeletePlan{ - PlanInterval: &Interval{ - StartTimestampMs: int64(planRange.Start), - EndTimestampMs: int64(planRange.End), - }, - ChunksGroup: chunksGroups, - } - - pb, err := proto.Marshal(&plan) - if err != nil { - return err - } - - plans[i] = pb - } - - err = p.putDeletePlans(ctx, req.UserID, req.RequestID, plans) - if err != nil { - return - } - - err = p.deleteStore.UpdateStatus(ctx, req.UserID, req.RequestID, StatusDeleting) - if err != nil { - return - } - - p.metrics.deleteRequestsChunksSelectedTotal.WithLabelValues(req.UserID).Add(float64(len(includedChunkIDs))) - - level.Info(req.logger).Log("msg", "built delete plans", "num_plans", len(perDayTimeRange)) - - return -} - -func (p *Purger) putDeletePlans(ctx context.Context, userID, requestID string, plans [][]byte) error { - for i, plan := range plans { - objectKey := buildObjectKeyForPlan(userID, requestID, i) - - err := p.objectClient.PutObject(ctx, objectKey, bytes.NewReader(plan)) - if err != nil { - return err - } - } - - return nil -} - -func (p *Purger) getDeletePlan(ctx context.Context, userID, requestID string, planNo int) (*DeletePlan, error) { - objectKey := buildObjectKeyForPlan(userID, requestID, planNo) - - readCloser, err := p.objectClient.GetObject(ctx, objectKey) - if err != nil { - return nil, err - } - - defer readCloser.Close() - - buf, err := ioutil.ReadAll(readCloser) - if err != nil { - return nil, err - } - - var plan DeletePlan - err = proto.Unmarshal(buf, &plan) - if err != nil { - return nil, err - } - - return &plan, nil -} - -func (p *Purger) removeDeletePlan(ctx context.Context, userID, requestID string, planNo int) error { - objectKey := buildObjectKeyForPlan(userID, requestID, planNo) - return p.objectClient.DeleteObject(ctx, objectKey) -} - -// returns interval per plan -func splitByDay(start, end model.Time) []model.Interval { - numOfDays := numPlans(start, end) - - perDayTimeRange := make([]model.Interval, numOfDays) - startOfNextDay := model.Time(((int64(start) / millisecondPerDay) + 1) * millisecondPerDay) - perDayTimeRange[0] = model.Interval{Start: start, End: startOfNextDay - 1} - - for i := 1; i < numOfDays; i++ { - interval := model.Interval{Start: startOfNextDay} - startOfNextDay += model.Time(millisecondPerDay) - interval.End = startOfNextDay - 1 - perDayTimeRange[i] = interval - } - - perDayTimeRange[numOfDays-1].End = end - - return perDayTimeRange -} - -func numPlans(start, end model.Time) int { - // rounding down start to start of the day - if start%model.Time(millisecondPerDay) != 0 { - start = model.Time((int64(start) / millisecondPerDay) * millisecondPerDay) - } - - // rounding up end to end of the day - if end%model.Time(millisecondPerDay) != 0 { - end = model.Time((int64(end)/millisecondPerDay)*millisecondPerDay + millisecondPerDay) - } - - return int(int64(end-start) / millisecondPerDay) -} - -// groups chunks together by unique label sets i.e all the chunks with same labels would be stored in a group -// chunk details are stored in groups for each unique label set to avoid storing them repetitively for each chunk -func groupChunks(chunks []chunk.Chunk, deleteFrom, deleteThrough model.Time, includedChunkIDs map[string]struct{}) ([]ChunksGroup, map[string]struct{}) { - metricToChunks := make(map[string]ChunksGroup) - - for _, chk := range chunks { - chunkID := chk.ExternalKey() - - if _, ok := includedChunkIDs[chunkID]; ok { - continue - } - // chunk.Metric are assumed to be sorted which should give same value from String() for same series. - // If they stop being sorted then in the worst case we would lose the benefit of grouping chunks to avoid storing labels repetitively. - metricString := chk.Metric.String() - group, ok := metricToChunks[metricString] - if !ok { - group = ChunksGroup{Labels: cortexpb.FromLabelsToLabelAdapters(chk.Metric)} - } - - chunkDetails := ChunkDetails{ID: chunkID} - - if deleteFrom > chk.From || deleteThrough < chk.Through { - partiallyDeletedInterval := Interval{StartTimestampMs: int64(chk.From), EndTimestampMs: int64(chk.Through)} - - if deleteFrom > chk.From { - partiallyDeletedInterval.StartTimestampMs = int64(deleteFrom) - } - - if deleteThrough < chk.Through { - partiallyDeletedInterval.EndTimestampMs = int64(deleteThrough) - } - chunkDetails.PartiallyDeletedInterval = &partiallyDeletedInterval - } - - group.Chunks = append(group.Chunks, chunkDetails) - includedChunkIDs[chunkID] = struct{}{} - metricToChunks[metricString] = group - } - - chunksGroups := make([]ChunksGroup, 0, len(metricToChunks)) - - for _, group := range metricToChunks { - chunksGroups = append(chunksGroups, group) - } - - return chunksGroups, includedChunkIDs -} - -func isMissingChunkErr(err error) bool { - if err == chunk.ErrStorageObjectNotFound { - return true - } - if promqlStorageErr, ok := err.(promql.ErrStorage); ok && promqlStorageErr.Err == chunk.ErrStorageObjectNotFound { - return true - } - - return false -} - -func buildObjectKeyForPlan(userID, requestID string, planNo int) string { - return fmt.Sprintf("%s:%s/%d", userID, requestID, planNo) -} - -func makeDeleteRequestWithLogger(deleteRequest DeleteRequest, l log.Logger) deleteRequestWithLogger { - logger := log.With(l, "user_id", deleteRequest.UserID, "request_id", deleteRequest.RequestID) - return deleteRequestWithLogger{deleteRequest, logger} -} - -// inProcessRequestsCollection stores DeleteRequests which are in process by each user. -// Currently we only allow processing of one delete request per user so it stores single DeleteRequest per user. -type inProcessRequestsCollection struct { - requests map[string]*DeleteRequest - usersWithFailedRequests map[string]struct{} - mtx sync.RWMutex -} - -func newInProcessRequestsCollection() *inProcessRequestsCollection { - return &inProcessRequestsCollection{ - requests: map[string]*DeleteRequest{}, - usersWithFailedRequests: map[string]struct{}{}, - } -} - -func (i *inProcessRequestsCollection) set(userID string, request *DeleteRequest) { - i.mtx.Lock() - defer i.mtx.Unlock() - - i.requests[userID] = request -} - -func (i *inProcessRequestsCollection) get(userID string) *DeleteRequest { - i.mtx.RLock() - defer i.mtx.RUnlock() - - return i.requests[userID] -} - -func (i *inProcessRequestsCollection) remove(userID string) { - i.mtx.Lock() - defer i.mtx.Unlock() - - delete(i.requests, userID) -} - -func (i *inProcessRequestsCollection) len() int { - i.mtx.RLock() - defer i.mtx.RUnlock() - - return len(i.requests) -} - -func (i *inProcessRequestsCollection) getOldest() *DeleteRequest { - i.mtx.RLock() - defer i.mtx.RUnlock() - - var oldestRequest *DeleteRequest - for _, request := range i.requests { - if oldestRequest == nil || request.CreatedAt.Before(oldestRequest.CreatedAt) { - oldestRequest = request - } - } - - return oldestRequest -} - -func (i *inProcessRequestsCollection) setFailedRequestForUser(userID string) { - i.mtx.Lock() - defer i.mtx.Unlock() - - i.usersWithFailedRequests[userID] = struct{}{} -} - -func (i *inProcessRequestsCollection) unsetFailedRequestForUser(userID string) { - i.mtx.Lock() - defer i.mtx.Unlock() - - delete(i.usersWithFailedRequests, userID) -} - -func (i *inProcessRequestsCollection) listUsersWithFailedRequest() []string { - i.mtx.RLock() - defer i.mtx.RUnlock() - - userIDs := make([]string, 0, len(i.usersWithFailedRequests)) - for userID := range i.usersWithFailedRequests { - userIDs = append(userIDs, userID) - } - - return userIDs -} diff --git a/pkg/chunk/purger/purger_test.go b/pkg/chunk/purger/purger_test.go deleted file mode 100644 index 58453e0e299..00000000000 --- a/pkg/chunk/purger/purger_test.go +++ /dev/null @@ -1,532 +0,0 @@ -package purger - -import ( - "context" - "fmt" - "sort" - "strings" - "testing" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/promql/parser" - "github.com/stretchr/testify/require" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/chunk/testutils" - "github.com/cortexproject/cortex/pkg/util/flagext" - util_log "github.com/cortexproject/cortex/pkg/util/log" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/pkg/util/test" -) - -const ( - userID = "userID" - modelTimeDay = model.Time(millisecondPerDay) - modelTimeHour = model.Time(time.Hour / time.Millisecond) -) - -func setupTestDeleteStore(t *testing.T) *DeleteStore { - var ( - deleteStoreConfig DeleteStoreConfig - tbmConfig chunk.TableManagerConfig - schemaCfg = chunk.DefaultSchemaConfig("", "v10", 0) - ) - flagext.DefaultValues(&deleteStoreConfig) - flagext.DefaultValues(&tbmConfig) - - mockStorage := chunk.NewMockStorage() - - extraTables := []chunk.ExtraTables{{TableClient: mockStorage, Tables: deleteStoreConfig.GetTables()}} - tableManager, err := chunk.NewTableManager(tbmConfig, schemaCfg, 12*time.Hour, mockStorage, nil, extraTables, nil) - require.NoError(t, err) - - require.NoError(t, tableManager.SyncTables(context.Background())) - - deleteStore, err := NewDeleteStore(deleteStoreConfig, mockStorage) - require.NoError(t, err) - - return deleteStore -} - -func setupStoresAndPurger(t *testing.T) (*DeleteStore, chunk.Store, chunk.ObjectClient, *Purger, *prometheus.Registry) { - deleteStore := setupTestDeleteStore(t) - - chunkStore, err := testutils.SetupTestChunkStore() - require.NoError(t, err) - - storageClient, err := testutils.SetupTestObjectStore() - require.NoError(t, err) - - purger, registry := setupPurger(t, deleteStore, chunkStore, storageClient) - - return deleteStore, chunkStore, storageClient, purger, registry -} - -func setupPurger(t *testing.T, deleteStore *DeleteStore, chunkStore chunk.Store, storageClient chunk.ObjectClient) (*Purger, *prometheus.Registry) { - registry := prometheus.NewRegistry() - - var cfg Config - flagext.DefaultValues(&cfg) - - purger, err := NewPurger(cfg, deleteStore, chunkStore, storageClient, registry) - require.NoError(t, err) - - return purger, registry -} - -func buildChunks(from, through model.Time, batchSize int) ([]chunk.Chunk, error) { - var chunks []chunk.Chunk - for ; from < through; from = from.Add(time.Hour) { - // creating batchSize number of chunks chunks per hour - _, testChunks, err := testutils.CreateChunks(0, batchSize, from, from.Add(time.Hour)) - if err != nil { - return nil, err - } - - chunks = append(chunks, testChunks...) - } - - return chunks, nil -} - -var purgePlanTestCases = []struct { - name string - chunkStoreDataInterval model.Interval - deleteRequestInterval model.Interval - expectedNumberOfPlans int - numChunksToDelete int - firstChunkPartialDeletionInterval *Interval - lastChunkPartialDeletionInterval *Interval - batchSize int -}{ - { - name: "deleting whole hour from a one hour data", - chunkStoreDataInterval: model.Interval{End: modelTimeHour}, - deleteRequestInterval: model.Interval{End: modelTimeHour}, - expectedNumberOfPlans: 1, - numChunksToDelete: 1, - }, - { - name: "deleting half a day from a days data", - chunkStoreDataInterval: model.Interval{End: modelTimeDay}, - deleteRequestInterval: model.Interval{End: model.Time(millisecondPerDay / 2)}, - expectedNumberOfPlans: 1, - numChunksToDelete: 12 + 1, // one chunk for each hour + end time touches chunk at boundary - lastChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(millisecondPerDay / 2), - EndTimestampMs: int64(millisecondPerDay / 2)}, - }, - { - name: "deleting a full day from 2 days data", - chunkStoreDataInterval: model.Interval{End: modelTimeDay * 2}, - deleteRequestInterval: model.Interval{End: modelTimeDay}, - expectedNumberOfPlans: 1, - numChunksToDelete: 24 + 1, // one chunk for each hour + end time touches chunk at boundary - lastChunkPartialDeletionInterval: &Interval{StartTimestampMs: millisecondPerDay, - EndTimestampMs: millisecondPerDay}, - }, - { - name: "deleting 2 days partially from 2 days data", - chunkStoreDataInterval: model.Interval{End: modelTimeDay * 2}, - deleteRequestInterval: model.Interval{Start: model.Time(millisecondPerDay / 2), - End: model.Time(millisecondPerDay + millisecondPerDay/2)}, - expectedNumberOfPlans: 2, - numChunksToDelete: 24 + 2, // one chunk for each hour + start and end time touches chunk at boundary - firstChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(millisecondPerDay / 2), - EndTimestampMs: int64(millisecondPerDay / 2)}, - lastChunkPartialDeletionInterval: &Interval{StartTimestampMs: millisecondPerDay + millisecondPerDay/2, - EndTimestampMs: millisecondPerDay + millisecondPerDay/2}, - }, - { - name: "deleting 2 days partially, not aligned with hour, from 2 days data", - chunkStoreDataInterval: model.Interval{End: modelTimeDay * 2}, - deleteRequestInterval: model.Interval{Start: model.Time(millisecondPerDay / 2).Add(time.Minute), - End: model.Time(millisecondPerDay + millisecondPerDay/2).Add(-time.Minute)}, - expectedNumberOfPlans: 2, - numChunksToDelete: 24, // one chunk for each hour, no chunks touched at boundary - firstChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(model.Time(millisecondPerDay / 2).Add(time.Minute)), - EndTimestampMs: int64(model.Time(millisecondPerDay / 2).Add(time.Hour))}, - lastChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(model.Time(millisecondPerDay + millisecondPerDay/2).Add(-time.Hour)), - EndTimestampMs: int64(model.Time(millisecondPerDay + millisecondPerDay/2).Add(-time.Minute))}, - }, - { - name: "deleting data outside of period of existing data", - chunkStoreDataInterval: model.Interval{End: modelTimeDay}, - deleteRequestInterval: model.Interval{Start: model.Time(millisecondPerDay * 2), End: model.Time(millisecondPerDay * 3)}, - expectedNumberOfPlans: 1, - numChunksToDelete: 0, - }, - { - name: "building multi-day chunk and deleting part of it from first day", - chunkStoreDataInterval: model.Interval{Start: modelTimeDay.Add(-30 * time.Minute), End: modelTimeDay.Add(30 * time.Minute)}, - deleteRequestInterval: model.Interval{Start: modelTimeDay.Add(-30 * time.Minute), End: modelTimeDay.Add(-15 * time.Minute)}, - expectedNumberOfPlans: 1, - numChunksToDelete: 1, - firstChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(modelTimeDay.Add(-30 * time.Minute)), - EndTimestampMs: int64(modelTimeDay.Add(-15 * time.Minute))}, - }, - { - name: "building multi-day chunk and deleting part of it for each day", - chunkStoreDataInterval: model.Interval{Start: modelTimeDay.Add(-30 * time.Minute), End: modelTimeDay.Add(30 * time.Minute)}, - deleteRequestInterval: model.Interval{Start: modelTimeDay.Add(-15 * time.Minute), End: modelTimeDay.Add(15 * time.Minute)}, - expectedNumberOfPlans: 2, - numChunksToDelete: 1, - firstChunkPartialDeletionInterval: &Interval{StartTimestampMs: int64(modelTimeDay.Add(-15 * time.Minute)), - EndTimestampMs: int64(modelTimeDay.Add(15 * time.Minute))}, - }, -} - -func TestPurger_BuildPlan(t *testing.T) { - for _, tc := range purgePlanTestCases { - for batchSize := 1; batchSize <= 5; batchSize++ { - t.Run(fmt.Sprintf("%s/batch-size=%d", tc.name, batchSize), func(t *testing.T) { - deleteStore, chunkStore, storageClient, purger, _ := setupStoresAndPurger(t) - defer func() { - purger.StopAsync() - chunkStore.Stop() - }() - - chunks, err := buildChunks(tc.chunkStoreDataInterval.Start, tc.chunkStoreDataInterval.End, batchSize) - require.NoError(t, err) - - require.NoError(t, chunkStore.Put(context.Background(), chunks)) - - err = deleteStore.AddDeleteRequest(context.Background(), userID, tc.deleteRequestInterval.Start, - tc.deleteRequestInterval.End, []string{"foo"}) - require.NoError(t, err) - - deleteRequests, err := deleteStore.GetAllDeleteRequestsForUser(context.Background(), userID) - require.NoError(t, err) - - deleteRequest := deleteRequests[0] - requestWithLogger := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - - err = purger.buildDeletePlan(requestWithLogger) - require.NoError(t, err) - planPath := fmt.Sprintf("%s:%s/", userID, deleteRequest.RequestID) - - plans, _, err := storageClient.List(context.Background(), planPath, "/") - require.NoError(t, err) - require.Equal(t, tc.expectedNumberOfPlans, len(plans)) - - numPlans := tc.expectedNumberOfPlans - var nilPurgePlanInterval *Interval - numChunks := 0 - - chunkIDs := map[string]struct{}{} - - for i := range plans { - deletePlan, err := purger.getDeletePlan(context.Background(), userID, deleteRequest.RequestID, i) - require.NoError(t, err) - for _, chunksGroup := range deletePlan.ChunksGroup { - numChunksInGroup := len(chunksGroup.Chunks) - chunks := chunksGroup.Chunks - numChunks += numChunksInGroup - - sort.Slice(chunks, func(i, j int) bool { - chunkI, err := chunk.ParseExternalKey(userID, chunks[i].ID) - require.NoError(t, err) - - chunkJ, err := chunk.ParseExternalKey(userID, chunks[j].ID) - require.NoError(t, err) - - return chunkI.From < chunkJ.From - }) - - for j, chunkDetails := range chunksGroup.Chunks { - chunkIDs[chunkDetails.ID] = struct{}{} - if i == 0 && j == 0 && tc.firstChunkPartialDeletionInterval != nil { - require.Equal(t, *tc.firstChunkPartialDeletionInterval, *chunkDetails.PartiallyDeletedInterval) - } else if i == numPlans-1 && j == numChunksInGroup-1 && tc.lastChunkPartialDeletionInterval != nil { - require.Equal(t, *tc.lastChunkPartialDeletionInterval, *chunkDetails.PartiallyDeletedInterval) - } else { - require.Equal(t, nilPurgePlanInterval, chunkDetails.PartiallyDeletedInterval) - } - } - } - } - - require.Equal(t, tc.numChunksToDelete*batchSize, len(chunkIDs)) - require.Equal(t, float64(tc.numChunksToDelete*batchSize), testutil.ToFloat64(purger.metrics.deleteRequestsChunksSelectedTotal)) - }) - } - } -} - -func TestPurger_ExecutePlan(t *testing.T) { - fooMetricNameMatcher, err := parser.ParseMetricSelector(`foo`) - if err != nil { - t.Fatal(err) - } - - for _, tc := range purgePlanTestCases { - for batchSize := 1; batchSize <= 5; batchSize++ { - t.Run(fmt.Sprintf("%s/batch-size=%d", tc.name, batchSize), func(t *testing.T) { - deleteStore, chunkStore, _, purger, _ := setupStoresAndPurger(t) - defer func() { - purger.StopAsync() - chunkStore.Stop() - }() - - chunks, err := buildChunks(tc.chunkStoreDataInterval.Start, tc.chunkStoreDataInterval.End, batchSize) - require.NoError(t, err) - - require.NoError(t, chunkStore.Put(context.Background(), chunks)) - - // calculate the expected number of chunks that should be there in store before deletion - chunkStoreDataIntervalTotal := tc.chunkStoreDataInterval.End - tc.chunkStoreDataInterval.Start - numChunksExpected := int(chunkStoreDataIntervalTotal / model.Time(time.Hour/time.Millisecond)) - - // see if store actually has expected number of chunks - chunks, err = chunkStore.Get(context.Background(), userID, tc.chunkStoreDataInterval.Start, tc.chunkStoreDataInterval.End, fooMetricNameMatcher...) - require.NoError(t, err) - require.Equal(t, numChunksExpected*batchSize, len(chunks)) - - // delete chunks - err = deleteStore.AddDeleteRequest(context.Background(), userID, tc.deleteRequestInterval.Start, - tc.deleteRequestInterval.End, []string{"foo"}) - require.NoError(t, err) - - // get the delete request - deleteRequests, err := deleteStore.GetAllDeleteRequestsForUser(context.Background(), userID) - require.NoError(t, err) - - deleteRequest := deleteRequests[0] - requestWithLogger := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - err = purger.buildDeletePlan(requestWithLogger) - require.NoError(t, err) - - // execute all the plans - for i := 0; i < tc.expectedNumberOfPlans; i++ { - err := purger.executePlan(userID, deleteRequest.RequestID, i, requestWithLogger.logger) - require.NoError(t, err) - } - - // calculate the expected number of chunks that should be there in store after deletion - numChunksExpectedAfterDeletion := 0 - for chunkStart := tc.chunkStoreDataInterval.Start; chunkStart < tc.chunkStoreDataInterval.End; chunkStart += modelTimeHour { - numChunksExpectedAfterDeletion += len(getNonDeletedIntervals(model.Interval{Start: chunkStart, End: chunkStart + modelTimeHour}, tc.deleteRequestInterval)) - } - - // see if store actually has expected number of chunks - chunks, err = chunkStore.Get(context.Background(), userID, tc.chunkStoreDataInterval.Start, tc.chunkStoreDataInterval.End, fooMetricNameMatcher...) - require.NoError(t, err) - require.Equal(t, numChunksExpectedAfterDeletion*batchSize, len(chunks)) - }) - } - } -} - -func TestPurger_Restarts(t *testing.T) { - fooMetricNameMatcher, err := parser.ParseMetricSelector(`foo`) - if err != nil { - t.Fatal(err) - } - - deleteStore, chunkStore, storageClient, purger, _ := setupStoresAndPurger(t) - defer func() { - chunkStore.Stop() - }() - - chunks, err := buildChunks(0, model.Time(0).Add(10*24*time.Hour), 1) - require.NoError(t, err) - - require.NoError(t, chunkStore.Put(context.Background(), chunks)) - - // delete chunks - err = deleteStore.AddDeleteRequest(context.Background(), userID, model.Time(0).Add(24*time.Hour), - model.Time(0).Add(8*24*time.Hour), []string{"foo"}) - require.NoError(t, err) - - // get the delete request - deleteRequests, err := deleteStore.GetAllDeleteRequestsForUser(context.Background(), userID) - require.NoError(t, err) - - deleteRequest := deleteRequests[0] - requestWithLogger := makeDeleteRequestWithLogger(deleteRequest, util_log.Logger) - err = purger.buildDeletePlan(requestWithLogger) - require.NoError(t, err) - - // stop the existing purger - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), purger)) - - // create a new purger to check whether it picks up in process delete requests - newPurger, _ := setupPurger(t, deleteStore, chunkStore, storageClient) - - // load in process delete requests by calling Run - require.NoError(t, services.StartAndAwaitRunning(context.Background(), newPurger)) - - defer newPurger.StopAsync() - - test.Poll(t, time.Minute, 0, func() interface{} { - return newPurger.inProcessRequests.len() - }) - - // check whether data got deleted from the store since delete request has been processed - chunks, err = chunkStore.Get(context.Background(), userID, 0, model.Time(0).Add(10*24*time.Hour), fooMetricNameMatcher...) - require.NoError(t, err) - - // we are deleting 7 days out of 10 so there should we 3 days data left in store which means 72 chunks - require.Equal(t, 72, len(chunks)) - - deleteRequests, err = deleteStore.GetAllDeleteRequestsForUser(context.Background(), userID) - require.NoError(t, err) - require.Equal(t, StatusProcessed, deleteRequests[0].Status) - - require.Equal(t, float64(1), testutil.ToFloat64(newPurger.metrics.deleteRequestsProcessedTotal)) - require.PanicsWithError(t, "collected 0 metrics instead of exactly 1", func() { - testutil.ToFloat64(newPurger.metrics.deleteRequestsProcessingFailures) - }) -} - -func TestPurger_Metrics(t *testing.T) { - deleteStore, chunkStore, storageClient, purger, registry := setupStoresAndPurger(t) - defer func() { - purger.StopAsync() - chunkStore.Stop() - }() - - // add delete requests without starting purger loops to load and process delete requests. - // add delete request whose createdAt is now - err := deleteStore.AddDeleteRequest(context.Background(), userID, model.Time(0).Add(24*time.Hour), - model.Time(0).Add(2*24*time.Hour), []string{"foo"}) - require.NoError(t, err) - - // add delete request whose createdAt is 2 days back - err = deleteStore.addDeleteRequest(context.Background(), userID, model.Now().Add(-2*24*time.Hour), model.Time(0).Add(24*time.Hour), - model.Time(0).Add(2*24*time.Hour), []string{"foo"}) - require.NoError(t, err) - - // add delete request whose createdAt is 3 days back - err = deleteStore.addDeleteRequest(context.Background(), userID, model.Now().Add(-3*24*time.Hour), model.Time(0).Add(24*time.Hour), - model.Time(0).Add(8*24*time.Hour), []string{"foo"}) - require.NoError(t, err) - - // load new delete requests for processing - require.NoError(t, purger.pullDeleteRequestsToPlanDeletes()) - - // there must be 2 pending delete requests, oldest being 2 days old since its cancellation time is over - require.InDelta(t, float64(2*86400), testutil.ToFloat64(purger.metrics.oldestPendingDeleteRequestAgeSeconds), 1) - require.Equal(t, float64(2), testutil.ToFloat64(purger.metrics.pendingDeleteRequestsCount)) - - // stop the existing purger - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), purger)) - - // create a new purger - purger, registry = setupPurger(t, deleteStore, chunkStore, storageClient) - - // load in process delete requests by starting the service - require.NoError(t, services.StartAndAwaitRunning(context.Background(), purger)) - - defer purger.StopAsync() - - // wait until purger_delete_requests_processed_total starts to show up. - test.Poll(t, 2*time.Second, 1, func() interface{} { - count, err := testutil.GatherAndCount(registry, "cortex_purger_delete_requests_processed_total") - require.NoError(t, err) - return count - }) - - // wait until both the pending delete requests are processed. - test.Poll(t, 2*time.Second, float64(2), func() interface{} { - return testutil.ToFloat64(purger.metrics.deleteRequestsProcessedTotal) - }) - - // wait until oldest pending request age becomes 0 - test.Poll(t, 2*time.Second, float64(0), func() interface{} { - return testutil.ToFloat64(purger.metrics.oldestPendingDeleteRequestAgeSeconds) - }) - - // wait until pending delete requests count becomes 0 - test.Poll(t, 2*time.Second, float64(0), func() interface{} { - return testutil.ToFloat64(purger.metrics.pendingDeleteRequestsCount) - }) -} - -func TestPurger_retryFailedRequests(t *testing.T) { - // setup chunks store - indexMockStorage := chunk.NewMockStorage() - chunksMockStorage := chunk.NewMockStorage() - - deleteStore := setupTestDeleteStore(t) - chunkStore, err := testutils.SetupTestChunkStoreWithClients(indexMockStorage, chunksMockStorage, indexMockStorage) - require.NoError(t, err) - - // create a purger instance - purgerMockStorage := chunk.NewMockStorage() - purger, _ := setupPurger(t, deleteStore, chunkStore, purgerMockStorage) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), purger)) - - defer func() { - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), purger)) - }() - - // add some chunks - chunks, err := buildChunks(0, model.Time(0).Add(3*24*time.Hour), 1) - require.NoError(t, err) - - require.NoError(t, chunkStore.Put(context.Background(), chunks)) - - // add a request to delete some chunks - err = deleteStore.addDeleteRequest(context.Background(), userID, model.Now().Add(-25*time.Hour), model.Time(0).Add(24*time.Hour), - model.Time(0).Add(2*24*time.Hour), []string{"foo"}) - require.NoError(t, err) - - // change purgerMockStorage to allow only reads. This would fail putting plans to the storage and hence fail build plans operation. - purgerMockStorage.SetMode(chunk.MockStorageModeReadOnly) - - // pull requests to process and ensure that it has failed. - err = purger.pullDeleteRequestsToPlanDeletes() - require.Error(t, err) - require.True(t, strings.Contains(err.Error(), "permission denied")) - - // there must be 1 delete request in process and the userID must be in failed requests list. - require.NotNil(t, purger.inProcessRequests.get(userID)) - require.Len(t, purger.inProcessRequests.listUsersWithFailedRequest(), 1) - - // now allow writes to purgerMockStorage to allow building plans to succeed. - purgerMockStorage.SetMode(chunk.MockStorageModeReadWrite) - - // but change mode of chunksMockStorage to read only which would deny permission to delete any chunks and in turn - // fail to execute delete plans. - chunksMockStorage.SetMode(chunk.MockStorageModeReadOnly) - - // retry processing of failed requests - purger.retryFailedRequests() - - // the delete request status should now change to StatusDeleting since the building of plan should have succeeded. - test.Poll(t, time.Second, StatusDeleting, func() interface{} { - return purger.inProcessRequests.get(userID).Status - }) - // the request should have failed again since we did not give permission to delete chunks. - test.Poll(t, time.Second, 1, func() interface{} { - return len(purger.inProcessRequests.listUsersWithFailedRequest()) - }) - - // now allow writes to chunksMockStorage so the requests do not fail anymore. - chunksMockStorage.SetMode(chunk.MockStorageModeReadWrite) - - // retry processing of failed requests. - purger.retryFailedRequests() - // there must be no in process requests anymore. - test.Poll(t, time.Second, true, func() interface{} { - return purger.inProcessRequests.get(userID) == nil - }) - // there must be no users having failed requests. - require.Len(t, purger.inProcessRequests.listUsersWithFailedRequest(), 0) -} - -func getNonDeletedIntervals(originalInterval, deletedInterval model.Interval) []model.Interval { - nonDeletedIntervals := []model.Interval{} - if deletedInterval.Start > originalInterval.Start { - nonDeletedIntervals = append(nonDeletedIntervals, model.Interval{Start: originalInterval.Start, End: deletedInterval.Start - 1}) - } - - if deletedInterval.End < originalInterval.End { - nonDeletedIntervals = append(nonDeletedIntervals, model.Interval{Start: deletedInterval.End + 1, End: originalInterval.End}) - } - - return nonDeletedIntervals -} diff --git a/pkg/chunk/purger/request_handler.go b/pkg/chunk/purger/request_handler.go deleted file mode 100644 index d9657b3ee79..00000000000 --- a/pkg/chunk/purger/request_handler.go +++ /dev/null @@ -1,183 +0,0 @@ -package purger - -import ( - "encoding/json" - "fmt" - "net/http" - "time" - - "github.com/go-kit/log/level" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/promql/parser" - - "github.com/cortexproject/cortex/pkg/tenant" - "github.com/cortexproject/cortex/pkg/util" - util_log "github.com/cortexproject/cortex/pkg/util/log" -) - -type deleteRequestHandlerMetrics struct { - deleteRequestsReceivedTotal *prometheus.CounterVec -} - -func newDeleteRequestHandlerMetrics(r prometheus.Registerer) *deleteRequestHandlerMetrics { - m := deleteRequestHandlerMetrics{} - - m.deleteRequestsReceivedTotal = promauto.With(r).NewCounterVec(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "purger_delete_requests_received_total", - Help: "Number of delete requests received per user", - }, []string{"user"}) - - return &m -} - -// DeleteRequestHandler provides handlers for delete requests -type DeleteRequestHandler struct { - deleteStore *DeleteStore - metrics *deleteRequestHandlerMetrics - deleteRequestCancelPeriod time.Duration -} - -// NewDeleteRequestHandler creates a DeleteRequestHandler -func NewDeleteRequestHandler(deleteStore *DeleteStore, deleteRequestCancelPeriod time.Duration, registerer prometheus.Registerer) *DeleteRequestHandler { - deleteMgr := DeleteRequestHandler{ - deleteStore: deleteStore, - deleteRequestCancelPeriod: deleteRequestCancelPeriod, - metrics: newDeleteRequestHandlerMetrics(registerer), - } - - return &deleteMgr -} - -// AddDeleteRequestHandler handles addition of new delete request -func (dm *DeleteRequestHandler) AddDeleteRequestHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - userID, err := tenant.TenantID(ctx) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - params := r.URL.Query() - match := params["match[]"] - if len(match) == 0 { - http.Error(w, "selectors not set", http.StatusBadRequest) - return - } - - for i := range match { - _, err := parser.ParseMetricSelector(match[i]) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - } - - startParam := params.Get("start") - startTime := int64(0) - if startParam != "" { - startTime, err = util.ParseTime(startParam) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - } - - endParam := params.Get("end") - endTime := int64(model.Now()) - - if endParam != "" { - endTime, err = util.ParseTime(endParam) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - if endTime > int64(model.Now()) { - http.Error(w, "deletes in future not allowed", http.StatusBadRequest) - return - } - } - - if startTime > endTime { - http.Error(w, "start time can't be greater than end time", http.StatusBadRequest) - return - } - - if err := dm.deleteStore.AddDeleteRequest(ctx, userID, model.Time(startTime), model.Time(endTime), match); err != nil { - level.Error(util_log.Logger).Log("msg", "error adding delete request to the store", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - dm.metrics.deleteRequestsReceivedTotal.WithLabelValues(userID).Inc() - w.WriteHeader(http.StatusNoContent) -} - -// GetAllDeleteRequestsHandler handles get all delete requests -func (dm *DeleteRequestHandler) GetAllDeleteRequestsHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - userID, err := tenant.TenantID(ctx) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - deleteRequests, err := dm.deleteStore.GetAllDeleteRequestsForUser(ctx, userID) - if err != nil { - level.Error(util_log.Logger).Log("msg", "error getting delete requests from the store", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - if err := json.NewEncoder(w).Encode(deleteRequests); err != nil { - level.Error(util_log.Logger).Log("msg", "error marshalling response", "err", err) - http.Error(w, fmt.Sprintf("Error marshalling response: %v", err), http.StatusInternalServerError) - } -} - -// CancelDeleteRequestHandler handles delete request cancellation -func (dm *DeleteRequestHandler) CancelDeleteRequestHandler(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - userID, err := tenant.TenantID(ctx) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - params := r.URL.Query() - requestID := params.Get("request_id") - - deleteRequest, err := dm.deleteStore.GetDeleteRequest(ctx, userID, requestID) - if err != nil { - level.Error(util_log.Logger).Log("msg", "error getting delete request from the store", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - if deleteRequest == nil { - http.Error(w, "could not find delete request with given id", http.StatusBadRequest) - return - } - - if deleteRequest.Status != StatusReceived { - http.Error(w, "deletion of request which is in process or already processed is not allowed", http.StatusBadRequest) - return - } - - if deleteRequest.CreatedAt.Add(dm.deleteRequestCancelPeriod).Before(model.Now()) { - http.Error(w, fmt.Sprintf("deletion of request past the deadline of %s since its creation is not allowed", dm.deleteRequestCancelPeriod.String()), http.StatusBadRequest) - return - } - - if err := dm.deleteStore.RemoveDeleteRequest(ctx, userID, requestID, deleteRequest.CreatedAt, deleteRequest.StartTime, deleteRequest.EndTime); err != nil { - level.Error(util_log.Logger).Log("msg", "error cancelling the delete request", "err", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusNoContent) -} diff --git a/pkg/chunk/purger/table_provisioning.go b/pkg/chunk/purger/table_provisioning.go deleted file mode 100644 index e8ce5d6364a..00000000000 --- a/pkg/chunk/purger/table_provisioning.go +++ /dev/null @@ -1,30 +0,0 @@ -package purger - -import ( - "flag" - - "github.com/cortexproject/cortex/pkg/chunk" -) - -// TableProvisioningConfig holds config for table throuput and autoscaling. Currently only used by DynamoDB. -type TableProvisioningConfig struct { - chunk.ActiveTableProvisionConfig `yaml:",inline"` - TableTags chunk.Tags `yaml:"tags"` -} - -// RegisterFlags adds the flags required to config this to the given FlagSet. -// Adding a separate RegisterFlags here instead of using it from embedded chunk.ActiveTableProvisionConfig to be able to manage defaults separately. -// Defaults for WriteScale and ReadScale are shared for now to avoid adding further complexity since autoscaling is disabled anyways by default. -func (cfg *TableProvisioningConfig) RegisterFlags(argPrefix string, f *flag.FlagSet) { - // default values ActiveTableProvisionConfig - cfg.ProvisionedWriteThroughput = 1 - cfg.ProvisionedReadThroughput = 300 - cfg.ProvisionedThroughputOnDemandMode = false - - cfg.ActiveTableProvisionConfig.RegisterFlags(argPrefix, f) - f.Var(&cfg.TableTags, argPrefix+".tags", "Tag (of the form key=value) to be added to the tables. Supported by DynamoDB") -} - -func (cfg DeleteStoreConfig) GetTables() []chunk.TableDesc { - return []chunk.TableDesc{cfg.ProvisionConfig.BuildTableDesc(cfg.RequestsTableName, cfg.ProvisionConfig.TableTags)} -} diff --git a/pkg/chunk/purger/tombstones.go b/pkg/chunk/purger/tombstones.go index 00eeeee1d69..31084dd529b 100644 --- a/pkg/chunk/purger/tombstones.go +++ b/pkg/chunk/purger/tombstones.go @@ -1,450 +1,74 @@ package purger import ( - "context" - "sort" - "strconv" - "sync" - "time" - - "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/promql/parser" - - util_log "github.com/cortexproject/cortex/pkg/util/log" ) -const tombstonesReloadDuration = 5 * time.Minute - -type tombstonesLoaderMetrics struct { - cacheGenLoadFailures prometheus.Counter - deleteRequestsLoadFailures prometheus.Counter -} - -func newtombstonesLoaderMetrics(r prometheus.Registerer) *tombstonesLoaderMetrics { - m := tombstonesLoaderMetrics{} - - m.cacheGenLoadFailures = promauto.With(r).NewCounter(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "tombstones_loader_cache_gen_load_failures_total", - Help: "Total number of failures while loading cache generation number using tombstones loader", - }) - m.deleteRequestsLoadFailures = promauto.With(r).NewCounter(prometheus.CounterOpts{ - Namespace: "cortex", - Name: "tombstones_loader_cache_delete_requests_load_failures_total", - Help: "Total number of failures while loading delete requests using tombstones loader", - }) +// TombstonesSet holds all the pending delete requests for a user +type TombstonesSet interface { + // GetDeletedIntervals returns non-overlapping, sorted deleted intervals. + GetDeletedIntervals(lbls labels.Labels, from, to model.Time) []model.Interval - return &m -} + // Len returns number of tombstones that are there + Len() int -// TombstonesSet holds all the pending delete requests for a user -type TombstonesSet struct { - tombstones []DeleteRequest - oldestTombstoneStart, newestTombstoneEnd model.Time // Used as optimization to find whether we want to iterate over tombstones or not + // HasTombstonesForInterval tells whether there are any tombstones which overlapping given interval + HasTombstonesForInterval(from, to model.Time) bool } -// Used for easier injection of mocks. -type DeleteStoreAPI interface { - getCacheGenerationNumbers(ctx context.Context, user string) (*cacheGenNumbers, error) - GetPendingDeleteRequestsForUser(ctx context.Context, id string) ([]DeleteRequest, error) +type noopTombstonesSet struct { } // TombstonesLoader loads delete requests and gen numbers from store and keeps checking for updates. // It keeps checking for changes in gen numbers, which also means changes in delete requests and reloads specific users delete requests. -type TombstonesLoader struct { - tombstones map[string]*TombstonesSet - tombstonesMtx sync.RWMutex - - cacheGenNumbers map[string]*cacheGenNumbers - cacheGenNumbersMtx sync.RWMutex - - deleteStore DeleteStoreAPI - metrics *tombstonesLoaderMetrics - quit chan struct{} -} - -// NewTombstonesLoader creates a TombstonesLoader -func NewTombstonesLoader(deleteStore DeleteStoreAPI, registerer prometheus.Registerer) *TombstonesLoader { - tl := TombstonesLoader{ - tombstones: map[string]*TombstonesSet{}, - cacheGenNumbers: map[string]*cacheGenNumbers{}, - deleteStore: deleteStore, - metrics: newtombstonesLoaderMetrics(registerer), - } - go tl.loop() - - return &tl -} - -// Stop stops TombstonesLoader -func (tl *TombstonesLoader) Stop() { - close(tl.quit) -} - -func (tl *TombstonesLoader) loop() { - if tl.deleteStore == nil { - return - } - - tombstonesReloadTimer := time.NewTicker(tombstonesReloadDuration) - for { - select { - case <-tombstonesReloadTimer.C: - err := tl.reloadTombstones() - if err != nil { - level.Error(util_log.Logger).Log("msg", "error reloading tombstones", "err", err) - } - case <-tl.quit: - return - } - } -} - -func (tl *TombstonesLoader) reloadTombstones() error { - updatedGenNumbers := make(map[string]*cacheGenNumbers) - tl.cacheGenNumbersMtx.RLock() - - // check for updates in loaded gen numbers - for userID, oldGenNumbers := range tl.cacheGenNumbers { - newGenNumbers, err := tl.deleteStore.getCacheGenerationNumbers(context.Background(), userID) - if err != nil { - tl.cacheGenNumbersMtx.RUnlock() - return err - } - - if *oldGenNumbers != *newGenNumbers { - updatedGenNumbers[userID] = newGenNumbers - } - } - - tl.cacheGenNumbersMtx.RUnlock() - - // in frontend we load only cache gen numbers so short circuit here if there are no loaded deleted requests - // first call to GetPendingTombstones would avoid doing this. - tl.tombstonesMtx.RLock() - if len(tl.tombstones) == 0 { - tl.tombstonesMtx.RUnlock() - return nil - } - tl.tombstonesMtx.RUnlock() - - // for all the updated gen numbers, reload delete requests - for userID, genNumbers := range updatedGenNumbers { - err := tl.loadPendingTombstones(userID) - if err != nil { - return err - } - - tl.cacheGenNumbersMtx.Lock() - tl.cacheGenNumbers[userID] = genNumbers - tl.cacheGenNumbersMtx.Unlock() - } - - return nil -} - -// GetPendingTombstones returns all pending tombstones -func (tl *TombstonesLoader) GetPendingTombstones(userID string) (*TombstonesSet, error) { - tl.tombstonesMtx.RLock() - - tombstoneSet, isOK := tl.tombstones[userID] - if isOK { - tl.tombstonesMtx.RUnlock() - return tombstoneSet, nil - } - - tl.tombstonesMtx.RUnlock() - err := tl.loadPendingTombstones(userID) - if err != nil { - return nil, err - } - - tl.tombstonesMtx.RLock() - defer tl.tombstonesMtx.RUnlock() - - return tl.tombstones[userID], nil -} - -// GetPendingTombstones returns all pending tombstones -func (tl *TombstonesLoader) GetPendingTombstonesForInterval(userID string, from, to model.Time) (*TombstonesSet, error) { - allTombstones, err := tl.GetPendingTombstones(userID) - if err != nil { - return nil, err - } - - if !allTombstones.HasTombstonesForInterval(from, to) { - return &TombstonesSet{}, nil - } - - filteredSet := TombstonesSet{oldestTombstoneStart: model.Now()} - - for _, tombstone := range allTombstones.tombstones { - if !intervalsOverlap(model.Interval{Start: from, End: to}, model.Interval{Start: tombstone.StartTime, End: tombstone.EndTime}) { - continue - } - - filteredSet.tombstones = append(filteredSet.tombstones, tombstone) - - if tombstone.StartTime < filteredSet.oldestTombstoneStart { - filteredSet.oldestTombstoneStart = tombstone.StartTime - } - - if tombstone.EndTime > filteredSet.newestTombstoneEnd { - filteredSet.newestTombstoneEnd = tombstone.EndTime - } - } - - return &filteredSet, nil -} - -func (tl *TombstonesLoader) loadPendingTombstones(userID string) error { - if tl.deleteStore == nil { - tl.tombstonesMtx.Lock() - defer tl.tombstonesMtx.Unlock() - - tl.tombstones[userID] = &TombstonesSet{oldestTombstoneStart: 0, newestTombstoneEnd: 0} - return nil - } - - pendingDeleteRequests, err := tl.deleteStore.GetPendingDeleteRequestsForUser(context.Background(), userID) - if err != nil { - tl.metrics.deleteRequestsLoadFailures.Inc() - return errors.Wrap(err, "error loading delete requests") - } - - tombstoneSet := TombstonesSet{tombstones: pendingDeleteRequests, oldestTombstoneStart: model.Now()} - for i := range tombstoneSet.tombstones { - tombstoneSet.tombstones[i].Matchers = make([][]*labels.Matcher, len(tombstoneSet.tombstones[i].Selectors)) - - for j, selector := range tombstoneSet.tombstones[i].Selectors { - tombstoneSet.tombstones[i].Matchers[j], err = parser.ParseMetricSelector(selector) +type TombstonesLoader interface { + // GetPendingTombstones returns all pending tombstones + GetPendingTombstones(userID string) (TombstonesSet, error) - if err != nil { - tl.metrics.deleteRequestsLoadFailures.Inc() - return errors.Wrapf(err, "error parsing metric selector") - } - } + // GetPendingTombstonesForInterval returns all pending tombstones between two times + GetPendingTombstonesForInterval(userID string, from, to model.Time) (TombstonesSet, error) - if tombstoneSet.tombstones[i].StartTime < tombstoneSet.oldestTombstoneStart { - tombstoneSet.oldestTombstoneStart = tombstoneSet.tombstones[i].StartTime - } - - if tombstoneSet.tombstones[i].EndTime > tombstoneSet.newestTombstoneEnd { - tombstoneSet.newestTombstoneEnd = tombstoneSet.tombstones[i].EndTime - } - } - - tl.tombstonesMtx.Lock() - defer tl.tombstonesMtx.Unlock() - tl.tombstones[userID] = &tombstoneSet - - return nil -} + // GetStoreCacheGenNumber returns store cache gen number for a user + GetStoreCacheGenNumber(tenantIDs []string) string -// GetStoreCacheGenNumber returns store cache gen number for a user -func (tl *TombstonesLoader) GetStoreCacheGenNumber(tenantIDs []string) string { - return tl.getCacheGenNumbersPerTenants(tenantIDs).store + // GetResultsCacheGenNumber returns results cache gen number for a user + GetResultsCacheGenNumber(tenantIDs []string) string } -// GetResultsCacheGenNumber returns results cache gen number for a user -func (tl *TombstonesLoader) GetResultsCacheGenNumber(tenantIDs []string) string { - return tl.getCacheGenNumbersPerTenants(tenantIDs).results +type noopTombstonesLoader struct { + ts noopTombstonesSet } -func (tl *TombstonesLoader) getCacheGenNumbersPerTenants(tenantIDs []string) *cacheGenNumbers { - var result cacheGenNumbers - - if len(tenantIDs) == 0 { - return &result - } - - // keep the maximum value that's currently in result - var maxResults, maxStore int - - for pos, tenantID := range tenantIDs { - numbers := tl.getCacheGenNumbers(tenantID) - - // handle first tenant in the list - if pos == 0 { - // short cut if there is only one tenant - if len(tenantIDs) == 1 { - return numbers - } - - // set first tenant string whatever happens next - result.results = numbers.results - result.store = numbers.store - } - - // set results number string if it's higher than the ones before - if numbers.results != "" { - results, err := strconv.Atoi(numbers.results) - if err != nil { - level.Error(util_log.Logger).Log("msg", "error parsing resultsCacheGenNumber", "user", tenantID, "err", err) - } else if maxResults < results { - maxResults = results - result.results = numbers.results - } - } - - // set store number string if it's higher than the ones before - if numbers.store != "" { - store, err := strconv.Atoi(numbers.store) - if err != nil { - level.Error(util_log.Logger).Log("msg", "error parsing storeCacheGenNumber", "user", tenantID, "err", err) - } else if maxStore < store { - maxStore = store - result.store = numbers.store - } - } - } - - return &result +// NewNoopTombstonesLoader creates a TombstonesLoader that does nothing +func NewNoopTombstonesLoader() TombstonesLoader { + return &noopTombstonesLoader{} } -func (tl *TombstonesLoader) getCacheGenNumbers(userID string) *cacheGenNumbers { - tl.cacheGenNumbersMtx.RLock() - if genNumbers, isOK := tl.cacheGenNumbers[userID]; isOK { - tl.cacheGenNumbersMtx.RUnlock() - return genNumbers - } - - tl.cacheGenNumbersMtx.RUnlock() - - if tl.deleteStore == nil { - tl.cacheGenNumbersMtx.Lock() - defer tl.cacheGenNumbersMtx.Unlock() - - tl.cacheGenNumbers[userID] = &cacheGenNumbers{} - return tl.cacheGenNumbers[userID] - } - - genNumbers, err := tl.deleteStore.getCacheGenerationNumbers(context.Background(), userID) - if err != nil { - level.Error(util_log.Logger).Log("msg", "error loading cache generation numbers", "err", err) - tl.metrics.cacheGenLoadFailures.Inc() - return &cacheGenNumbers{} - } - - tl.cacheGenNumbersMtx.Lock() - defer tl.cacheGenNumbersMtx.Unlock() - - tl.cacheGenNumbers[userID] = genNumbers - return genNumbers +func (tl *noopTombstonesLoader) GetPendingTombstones(userID string) (TombstonesSet, error) { + return &tl.ts, nil } -// GetDeletedIntervals returns non-overlapping, sorted deleted intervals. -func (ts TombstonesSet) GetDeletedIntervals(lbls labels.Labels, from, to model.Time) []model.Interval { - if len(ts.tombstones) == 0 || to < ts.oldestTombstoneStart || from > ts.newestTombstoneEnd { - return nil - } - - var deletedIntervals []model.Interval - requestedInterval := model.Interval{Start: from, End: to} - - for i := range ts.tombstones { - overlaps, overlappingInterval := getOverlappingInterval(requestedInterval, - model.Interval{Start: ts.tombstones[i].StartTime, End: ts.tombstones[i].EndTime}) - - if !overlaps { - continue - } - - matches := false - for _, matchers := range ts.tombstones[i].Matchers { - if labels.Selector(matchers).Matches(lbls) { - matches = true - break - } - } - - if !matches { - continue - } - - if overlappingInterval == requestedInterval { - // whole interval deleted - return []model.Interval{requestedInterval} - } - - deletedIntervals = append(deletedIntervals, overlappingInterval) - } - - if len(deletedIntervals) == 0 { - return nil - } - - return mergeIntervals(deletedIntervals) +func (tl *noopTombstonesLoader) GetPendingTombstonesForInterval(userID string, from, to model.Time) (TombstonesSet, error) { + return &tl.ts, nil } -// Len returns number of tombstones that are there -func (ts TombstonesSet) Len() int { - return len(ts.tombstones) +func (tl *noopTombstonesLoader) GetStoreCacheGenNumber(tenantIDs []string) string { + return "" } -// HasTombstonesForInterval tells whether there are any tombstones which overlapping given interval -func (ts TombstonesSet) HasTombstonesForInterval(from, to model.Time) bool { - if len(ts.tombstones) == 0 || to < ts.oldestTombstoneStart || from > ts.newestTombstoneEnd { - return false - } - - return true +func (tl *noopTombstonesLoader) GetResultsCacheGenNumber(tenantIDs []string) string { + return "" } -// sorts and merges overlapping intervals -func mergeIntervals(intervals []model.Interval) []model.Interval { - if len(intervals) <= 1 { - return intervals - } - - mergedIntervals := make([]model.Interval, 0, len(intervals)) - sort.Slice(intervals, func(i, j int) bool { - return intervals[i].Start < intervals[j].Start - }) - - ongoingTrFrom, ongoingTrTo := intervals[0].Start, intervals[0].End - for i := 1; i < len(intervals); i++ { - // if there is no overlap add it to mergedIntervals - if intervals[i].Start > ongoingTrTo { - mergedIntervals = append(mergedIntervals, model.Interval{Start: ongoingTrFrom, End: ongoingTrTo}) - ongoingTrFrom = intervals[i].Start - ongoingTrTo = intervals[i].End - continue - } - - // there is an overlap but check whether existing time range is bigger than the current one - if intervals[i].End > ongoingTrTo { - ongoingTrTo = intervals[i].End - } - } - - // add the last time range - mergedIntervals = append(mergedIntervals, model.Interval{Start: ongoingTrFrom, End: ongoingTrTo}) - - return mergedIntervals +func (ts noopTombstonesSet) GetDeletedIntervals(lbls labels.Labels, from, to model.Time) []model.Interval { + return nil } -func getOverlappingInterval(interval1, interval2 model.Interval) (bool, model.Interval) { - if interval2.Start > interval1.Start { - interval1.Start = interval2.Start - } - - if interval2.End < interval1.End { - interval1.End = interval2.End - } - - return interval1.Start < interval1.End, interval1 +func (ts noopTombstonesSet) Len() int { + return 0 } -func intervalsOverlap(interval1, interval2 model.Interval) bool { - if interval1.Start > interval2.End || interval2.Start > interval1.End { - return false - } - - return true +func (ts noopTombstonesSet) HasTombstonesForInterval(from, to model.Time) bool { + return false } diff --git a/pkg/chunk/purger/tombstones_test.go b/pkg/chunk/purger/tombstones_test.go deleted file mode 100644 index e04d6ef02fc..00000000000 --- a/pkg/chunk/purger/tombstones_test.go +++ /dev/null @@ -1,230 +0,0 @@ -package purger - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/promql/parser" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestTombstonesLoader(t *testing.T) { - deleteRequestSelectors := []string{"foo"} - metric, err := parser.ParseMetric(deleteRequestSelectors[0]) - require.NoError(t, err) - - for _, tc := range []struct { - name string - deleteRequestIntervals []model.Interval - queryForInterval model.Interval - expectedIntervals []model.Interval - }{ - { - name: "no delete requests", - queryForInterval: model.Interval{End: modelTimeDay}, - }, - { - name: "query out of range of delete requests", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - }, - queryForInterval: model.Interval{Start: modelTimeDay.Add(time.Hour), End: modelTimeDay * 2}, - }, - { - name: "no overlap but disjoint deleted intervals", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - {Start: modelTimeDay.Add(time.Hour), End: modelTimeDay.Add(2 * time.Hour)}, - }, - queryForInterval: model.Interval{End: modelTimeDay.Add(2 * time.Hour)}, - expectedIntervals: []model.Interval{ - {End: modelTimeDay}, - {Start: modelTimeDay.Add(time.Hour), End: modelTimeDay.Add(2 * time.Hour)}, - }, - }, - { - name: "no overlap but continuous deleted intervals", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - {Start: modelTimeDay, End: modelTimeDay.Add(2 * time.Hour)}, - }, - queryForInterval: model.Interval{End: modelTimeDay.Add(2 * time.Hour)}, - expectedIntervals: []model.Interval{ - {End: modelTimeDay.Add(2 * time.Hour)}, - }, - }, - { - name: "some overlap in deleted intervals", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - {Start: modelTimeDay.Add(-time.Hour), End: modelTimeDay.Add(2 * time.Hour)}, - }, - queryForInterval: model.Interval{End: modelTimeDay.Add(2 * time.Hour)}, - expectedIntervals: []model.Interval{ - {End: modelTimeDay.Add(2 * time.Hour)}, - }, - }, - { - name: "complete overlap in deleted intervals", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - {End: modelTimeDay}, - }, - queryForInterval: model.Interval{End: modelTimeDay.Add(2 * time.Hour)}, - expectedIntervals: []model.Interval{ - {End: modelTimeDay}, - }, - }, - { - name: "mix of overlaps in deleted intervals", - deleteRequestIntervals: []model.Interval{ - {End: modelTimeDay}, - {End: modelTimeDay}, - {Start: modelTimeDay.Add(time.Hour), End: modelTimeDay.Add(2 * time.Hour)}, - {Start: modelTimeDay.Add(2 * time.Hour), End: modelTimeDay.Add(24 * time.Hour)}, - {Start: modelTimeDay.Add(23 * time.Hour), End: modelTimeDay * 3}, - }, - queryForInterval: model.Interval{End: modelTimeDay * 10}, - expectedIntervals: []model.Interval{ - {End: modelTimeDay}, - {Start: modelTimeDay.Add(time.Hour), End: modelTimeDay * 3}, - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - deleteStore := setupTestDeleteStore(t) - tombstonesLoader := NewTombstonesLoader(deleteStore, nil) - - // add delete requests - for _, interval := range tc.deleteRequestIntervals { - err := deleteStore.AddDeleteRequest(context.Background(), userID, interval.Start, interval.End, deleteRequestSelectors) - require.NoError(t, err) - } - - // get all delete requests for user - tombstonesAnalyzer, err := tombstonesLoader.GetPendingTombstones(userID) - require.NoError(t, err) - - // verify whether number of delete requests is same as what we added - require.Equal(t, len(tc.deleteRequestIntervals), tombstonesAnalyzer.Len()) - - // if we are expecting to get deleted intervals then HasTombstonesForInterval should return true else false - expectedHasTombstonesForInterval := true - if len(tc.expectedIntervals) == 0 { - expectedHasTombstonesForInterval = false - } - - hasTombstonesForInterval := tombstonesAnalyzer.HasTombstonesForInterval(tc.queryForInterval.Start, tc.queryForInterval.End) - require.Equal(t, expectedHasTombstonesForInterval, hasTombstonesForInterval) - - // get deleted intervals - intervals := tombstonesAnalyzer.GetDeletedIntervals(metric, tc.queryForInterval.Start, tc.queryForInterval.End) - require.Equal(t, len(tc.expectedIntervals), len(intervals)) - - // verify whether we got expected intervals back - for i, interval := range intervals { - require.Equal(t, tc.expectedIntervals[i].Start, interval.Start) - require.Equal(t, tc.expectedIntervals[i].End, interval.End) - } - }) - } -} - -func TestTombstonesLoader_GetCacheGenNumber(t *testing.T) { - s := &store{ - numbers: map[string]*cacheGenNumbers{ - "tenant-a": { - results: "1000", - store: "2050", - }, - "tenant-b": { - results: "1050", - store: "2000", - }, - "tenant-c": { - results: "", - store: "", - }, - "tenant-d": { - results: "results-c", - store: "store-c", - }, - }, - } - tombstonesLoader := NewTombstonesLoader(s, nil) - - for _, tc := range []struct { - name string - expectedResultsCacheGenNumber string - expectedStoreCacheGenNumber string - tenantIDs []string - }{ - { - name: "single tenant with numeric values", - tenantIDs: []string{"tenant-a"}, - expectedResultsCacheGenNumber: "1000", - expectedStoreCacheGenNumber: "2050", - }, - { - name: "single tenant with non-numeric values", - tenantIDs: []string{"tenant-d"}, - expectedResultsCacheGenNumber: "results-c", - expectedStoreCacheGenNumber: "store-c", - }, - { - name: "multiple tenants with numeric values", - tenantIDs: []string{"tenant-a", "tenant-b"}, - expectedResultsCacheGenNumber: "1050", - expectedStoreCacheGenNumber: "2050", - }, - { - name: "multiple tenants with numeric and non-numeric values", - tenantIDs: []string{"tenant-d", "tenant-c", "tenant-b", "tenant-a"}, - expectedResultsCacheGenNumber: "1050", - expectedStoreCacheGenNumber: "2050", - }, - { - name: "no tenants", // not really an expected call, edge case check to avoid any panics - }, - } { - t.Run(tc.name, func(t *testing.T) { - assert.Equal(t, tc.expectedResultsCacheGenNumber, tombstonesLoader.GetResultsCacheGenNumber(tc.tenantIDs)) - assert.Equal(t, tc.expectedStoreCacheGenNumber, tombstonesLoader.GetStoreCacheGenNumber(tc.tenantIDs)) - }) - } -} - -func TestTombstonesReloadDoesntDeadlockOnFailure(t *testing.T) { - s := &store{} - tombstonesLoader := NewTombstonesLoader(s, nil) - tombstonesLoader.getCacheGenNumbers("test") - - s.err = errors.New("error") - require.NotNil(t, tombstonesLoader.reloadTombstones()) - - s.err = nil - require.NotNil(t, tombstonesLoader.getCacheGenNumbers("test2")) -} - -type store struct { - numbers map[string]*cacheGenNumbers - err error -} - -func (f *store) getCacheGenerationNumbers(ctx context.Context, user string) (*cacheGenNumbers, error) { - if f.numbers != nil { - number, ok := f.numbers[user] - if ok { - return number, nil - } - } - return &cacheGenNumbers{}, f.err -} - -func (f *store) GetPendingDeleteRequestsForUser(ctx context.Context, id string) ([]DeleteRequest, error) { - return nil, nil -} diff --git a/pkg/chunk/storage/factory.go b/pkg/chunk/storage/factory.go index 8076590d5d8..c78cb89d693 100644 --- a/pkg/chunk/storage/factory.go +++ b/pkg/chunk/storage/factory.go @@ -22,7 +22,6 @@ import ( "github.com/cortexproject/cortex/pkg/chunk/local" "github.com/cortexproject/cortex/pkg/chunk/objectclient" "github.com/cortexproject/cortex/pkg/chunk/openstack" - "github.com/cortexproject/cortex/pkg/chunk/purger" util_log "github.com/cortexproject/cortex/pkg/util/log" ) @@ -93,8 +92,6 @@ type Config struct { IndexQueriesCacheConfig cache.Config `yaml:"index_queries_cache_config"` - DeleteStoreConfig purger.DeleteStoreConfig `yaml:"delete_store"` - GrpcConfig grpc.Config `yaml:"grpc_store"` } @@ -107,19 +104,18 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.CassandraStorageConfig.RegisterFlags(f) cfg.BoltDBConfig.RegisterFlags(f) cfg.FSConfig.RegisterFlags(f) - cfg.DeleteStoreConfig.RegisterFlags(f) cfg.Swift.RegisterFlags(f) cfg.GrpcConfig.RegisterFlags(f) - f.StringVar(&cfg.Engine, "store.engine", "chunks", "The storage engine to use: chunks (deprecated) or blocks.") + f.StringVar(&cfg.Engine, "store.engine", "blocks", "The storage engine to use: blocks is the only supported option today.") cfg.IndexQueriesCacheConfig.RegisterFlagsWithPrefix("store.index-cache-read.", "Cache config for index entry reading. ", f) f.DurationVar(&cfg.IndexCacheValidity, "store.index-cache-validity", 5*time.Minute, "Cache validity for active index entries. Should be no higher than -ingester.max-chunk-idle.") } // Validate config and returns error on failure func (cfg *Config) Validate() error { - if cfg.Engine != StorageEngineChunks && cfg.Engine != StorageEngineBlocks { - return errors.New("unsupported storage engine") + if cfg.Engine != StorageEngineBlocks { + return errors.New("unsupported storage engine (only blocks is supported for ingest)") } if err := cfg.CassandraStorageConfig.Validate(); err != nil { return errors.Wrap(err, "invalid Cassandra Storage config") diff --git a/pkg/cortex/cortex.go b/pkg/cortex/cortex.go index 355bde61287..ac76795c307 100644 --- a/pkg/cortex/cortex.go +++ b/pkg/cortex/cortex.go @@ -112,7 +112,6 @@ type Config struct { BlocksStorage tsdb.BlocksStorageConfig `yaml:"blocks_storage"` Compactor compactor.Config `yaml:"compactor"` StoreGateway storegateway.Config `yaml:"store_gateway"` - PurgerConfig purger.Config `yaml:"purger"` TenantFederation tenantfederation.Config `yaml:"tenant_federation"` Ruler ruler.Config `yaml:"ruler"` @@ -161,7 +160,6 @@ func (c *Config) RegisterFlags(f *flag.FlagSet) { c.BlocksStorage.RegisterFlags(f) c.Compactor.RegisterFlags(f) c.StoreGateway.RegisterFlags(f) - c.PurgerConfig.RegisterFlags(f) c.TenantFederation.RegisterFlags(f) c.Ruler.RegisterFlags(f) @@ -319,12 +317,9 @@ type Cortex struct { Ingester *ingester.Ingester Flusher *flusher.Flusher Store chunk.Store - DeletesStore *purger.DeleteStore Frontend *frontendv1.Frontend - TableManager *chunk.TableManager RuntimeConfig *runtimeconfig.Manager - Purger *purger.Purger - TombstonesLoader *purger.TombstonesLoader + TombstonesLoader purger.TombstonesLoader QuerierQueryable prom_storage.SampleAndChunkQueryable ExemplarQueryable prom_storage.ExemplarQueryable QuerierEngine *promql.Engine diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index 2568029087d..fe93c95f82a 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -5,7 +5,6 @@ import ( "flag" "fmt" "net/http" - "os" "time" "github.com/go-kit/log/level" @@ -79,7 +78,6 @@ const ( Compactor string = "compactor" StoreGateway string = "store-gateway" MemberlistKV string = "memberlist-kv" - ChunksPurger string = "chunks-purger" TenantDeletion string = "tenant-deletion" Purger string = "purger" QueryScheduler string = "query-scheduler" @@ -426,7 +424,7 @@ func (t *Cortex) initIngesterService() (serv services.Service, err error) { t.Cfg.Ingester.InstanceLimitsFn = ingesterInstanceLimits(t.RuntimeConfig) t.tsdbIngesterConfig() - t.Ingester, err = ingester.New(t.Cfg.Ingester, t.Cfg.IngesterClient, t.Overrides, t.Store, prometheus.DefaultRegisterer, util_log.Logger) + t.Ingester, err = ingester.New(t.Cfg.Ingester, t.Overrides, prometheus.DefaultRegisterer, util_log.Logger) if err != nil { return } @@ -446,7 +444,6 @@ func (t *Cortex) initFlusher() (serv services.Service, err error) { t.Flusher, err = flusher.New( t.Cfg.Flusher, t.Cfg.Ingester, - t.Store, t.Overrides, prometheus.DefaultRegisterer, util_log.Logger, @@ -459,7 +456,10 @@ func (t *Cortex) initFlusher() (serv services.Service, err error) { } func (t *Cortex) initChunkStore() (serv services.Service, err error) { - if t.Cfg.Storage.Engine != storage.StorageEngineChunks && t.Cfg.Querier.SecondStoreEngine != storage.StorageEngineChunks { + if t.Cfg.Storage.Engine == storage.StorageEngineChunks { + return nil, errors.New("should not get here: ingesting into chunks storage is no longer supported") + } + if t.Cfg.Querier.SecondStoreEngine != storage.StorageEngineChunks { return nil, nil } err = t.Cfg.Schema.Load() @@ -479,28 +479,8 @@ func (t *Cortex) initChunkStore() (serv services.Service, err error) { } func (t *Cortex) initDeleteRequestsStore() (serv services.Service, err error) { - if t.Cfg.Storage.Engine != storage.StorageEngineChunks || !t.Cfg.PurgerConfig.Enable { - // until we need to explicitly enable delete series support we need to do create TombstonesLoader without DeleteStore which acts as noop - t.TombstonesLoader = purger.NewTombstonesLoader(nil, nil) - - return - } - - var indexClient chunk.IndexClient - reg := prometheus.WrapRegistererWith( - prometheus.Labels{"component": DeleteRequestsStore}, prometheus.DefaultRegisterer) - indexClient, err = storage.NewIndexClient(t.Cfg.Storage.DeleteStoreConfig.Store, t.Cfg.Storage, t.Cfg.Schema, reg) - if err != nil { - return - } - - t.DeletesStore, err = purger.NewDeleteStore(t.Cfg.Storage.DeleteStoreConfig, indexClient) - if err != nil { - return - } - - t.TombstonesLoader = purger.NewTombstonesLoader(t.DeletesStore, prometheus.DefaultRegisterer) - + // no-op while blocks store does not support series deletion + t.TombstonesLoader = purger.NewNoopTombstonesLoader() return } @@ -578,61 +558,6 @@ func (t *Cortex) initQueryFrontend() (serv services.Service, err error) { return nil, nil } -func (t *Cortex) initTableManager() (services.Service, error) { - if t.Cfg.Storage.Engine == storage.StorageEngineBlocks { - return nil, nil // table manager isn't used in v2 - } - - err := t.Cfg.Schema.Load() - if err != nil { - return nil, err - } - - // Assume the newest config is the one to use - lastConfig := &t.Cfg.Schema.Configs[len(t.Cfg.Schema.Configs)-1] - - if (t.Cfg.TableManager.ChunkTables.WriteScale.Enabled || - t.Cfg.TableManager.IndexTables.WriteScale.Enabled || - t.Cfg.TableManager.ChunkTables.InactiveWriteScale.Enabled || - t.Cfg.TableManager.IndexTables.InactiveWriteScale.Enabled || - t.Cfg.TableManager.ChunkTables.ReadScale.Enabled || - t.Cfg.TableManager.IndexTables.ReadScale.Enabled || - t.Cfg.TableManager.ChunkTables.InactiveReadScale.Enabled || - t.Cfg.TableManager.IndexTables.InactiveReadScale.Enabled) && - t.Cfg.Storage.AWSStorageConfig.Metrics.URL == "" { - level.Error(util_log.Logger).Log("msg", "WriteScale is enabled but no Metrics URL has been provided") - os.Exit(1) - } - - reg := prometheus.WrapRegistererWith( - prometheus.Labels{"component": "table-manager-store"}, prometheus.DefaultRegisterer) - - tableClient, err := storage.NewTableClient(lastConfig.IndexType, t.Cfg.Storage, reg) - if err != nil { - return nil, err - } - - bucketClient, err := storage.NewBucketClient(t.Cfg.Storage) - util_log.CheckFatal("initializing bucket client", err) - - var extraTables []chunk.ExtraTables - if t.Cfg.PurgerConfig.Enable { - reg := prometheus.WrapRegistererWith( - prometheus.Labels{"component": "table-manager-" + DeleteRequestsStore}, prometheus.DefaultRegisterer) - - deleteStoreTableClient, err := storage.NewTableClient(t.Cfg.Storage.DeleteStoreConfig.Store, t.Cfg.Storage, reg) - if err != nil { - return nil, err - } - - extraTables = append(extraTables, chunk.ExtraTables{TableClient: deleteStoreTableClient, Tables: t.Cfg.Storage.DeleteStoreConfig.GetTables()}) - } - - t.TableManager, err = chunk.NewTableManager(t.Cfg.TableManager, t.Cfg.Schema, t.Cfg.Ingester.MaxChunkAge, tableClient, - bucketClient, extraTables, prometheus.DefaultRegisterer) - return t.TableManager, err -} - func (t *Cortex) initRulerStorage() (serv services.Service, err error) { // if the ruler is not configured and we're in single binary then let's just log an error and continue. // unfortunately there is no way to generate a "default" config and compare default against actual @@ -790,26 +715,6 @@ func (t *Cortex) initMemberlistKV() (services.Service, error) { return t.MemberlistKV, nil } -func (t *Cortex) initChunksPurger() (services.Service, error) { - if t.Cfg.Storage.Engine != storage.StorageEngineChunks || !t.Cfg.PurgerConfig.Enable { - return nil, nil - } - - storageClient, err := storage.NewObjectClient(t.Cfg.PurgerConfig.ObjectStoreType, t.Cfg.Storage) - if err != nil { - return nil, err - } - - t.Purger, err = purger.NewPurger(t.Cfg.PurgerConfig, t.DeletesStore, t.Store, storageClient, prometheus.DefaultRegisterer) - if err != nil { - return nil, err - } - - t.API.RegisterChunksPurger(t.DeletesStore, t.Cfg.PurgerConfig.DeleteRequestCancelPeriod) - - return t.Purger, nil -} - func (t *Cortex) initTenantDeletionAPI() (services.Service, error) { if t.Cfg.Storage.Engine != storage.StorageEngineBlocks { return nil, nil @@ -859,14 +764,12 @@ func (t *Cortex) setupModuleManager() error { mm.RegisterModule(StoreQueryable, t.initStoreQueryables, modules.UserInvisibleModule) mm.RegisterModule(QueryFrontendTripperware, t.initQueryFrontendTripperware, modules.UserInvisibleModule) mm.RegisterModule(QueryFrontend, t.initQueryFrontend) - mm.RegisterModule(TableManager, t.initTableManager) mm.RegisterModule(RulerStorage, t.initRulerStorage, modules.UserInvisibleModule) mm.RegisterModule(Ruler, t.initRuler) mm.RegisterModule(Configs, t.initConfig) mm.RegisterModule(AlertManager, t.initAlertManager) mm.RegisterModule(Compactor, t.initCompactor) mm.RegisterModule(StoreGateway, t.initStoreGateway) - mm.RegisterModule(ChunksPurger, t.initChunksPurger, modules.UserInvisibleModule) mm.RegisterModule(TenantDeletion, t.initTenantDeletionAPI, modules.UserInvisibleModule) mm.RegisterModule(Purger, nil) mm.RegisterModule(QueryScheduler, t.initQueryScheduler) @@ -893,18 +796,16 @@ func (t *Cortex) setupModuleManager() error { QueryFrontendTripperware: {API, Overrides, DeleteRequestsStore}, QueryFrontend: {QueryFrontendTripperware}, QueryScheduler: {API, Overrides}, - TableManager: {API}, Ruler: {DistributorService, Store, StoreQueryable, RulerStorage}, RulerStorage: {Overrides}, Configs: {API}, AlertManager: {API, MemberlistKV, Overrides}, Compactor: {API, MemberlistKV, Overrides}, StoreGateway: {API, Overrides, MemberlistKV}, - ChunksPurger: {Store, DeleteRequestsStore, API}, TenantDeletion: {Store, API, Overrides}, - Purger: {ChunksPurger, TenantDeletion}, + Purger: {TenantDeletion}, TenantFederation: {Queryable}, - All: {QueryFrontend, Querier, Ingester, Distributor, TableManager, Purger, StoreGateway, Ruler}, + All: {QueryFrontend, Querier, Ingester, Distributor, Purger, StoreGateway, Ruler}, } for mod, targets := range deps { if err := mm.AddDependency(mod, targets...); err != nil { diff --git a/pkg/flusher/flusher.go b/pkg/flusher/flusher.go index ee0992a3a2f..39bd4e10f42 100644 --- a/pkg/flusher/flusher.go +++ b/pkg/flusher/flusher.go @@ -26,20 +26,18 @@ type Config struct { // RegisterFlags adds the flags required to config this to the given FlagSet func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - f.StringVar(&cfg.WALDir, "flusher.wal-dir", "wal", "Directory to read WAL from (chunks storage engine only).") - f.IntVar(&cfg.ConcurrentFlushes, "flusher.concurrent-flushes", 50, "Number of concurrent goroutines flushing to storage (chunks storage engine only).") - f.DurationVar(&cfg.FlushOpTimeout, "flusher.flush-op-timeout", 2*time.Minute, "Timeout for individual flush operations (chunks storage engine only).") + f.StringVar(&cfg.WALDir, "flusher.wal-dir", "wal", "Has no effect: directory to read WAL from (chunks storage engine only).") + f.IntVar(&cfg.ConcurrentFlushes, "flusher.concurrent-flushes", 50, "Has no effect: number of concurrent goroutines flushing to storage (chunks storage engine only).") + f.DurationVar(&cfg.FlushOpTimeout, "flusher.flush-op-timeout", 2*time.Minute, "Has no effect: timeout for individual flush operations (chunks storage engine only).") f.BoolVar(&cfg.ExitAfterFlush, "flusher.exit-after-flush", true, "Stop Cortex after flush has finished. If false, Cortex process will keep running, doing nothing.") } -// Flusher is designed to be used as a job to flush the data from the WAL on disk. -// Flusher works with both chunks-based and blocks-based ingesters. +// Flusher is designed to be used as a job to flush the data from the TSDB/WALs on disk. type Flusher struct { services.Service cfg Config ingesterConfig ingester.Config - chunkStore ingester.ChunkStore limits *validation.Overrides registerer prometheus.Registerer logger log.Logger @@ -54,21 +52,14 @@ const ( func New( cfg Config, ingesterConfig ingester.Config, - chunkStore ingester.ChunkStore, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger, ) (*Flusher, error) { - // These are ignored by blocks-ingester, but that's fine. - ingesterConfig.WALConfig.Dir = cfg.WALDir - ingesterConfig.ConcurrentFlushes = cfg.ConcurrentFlushes - ingesterConfig.FlushOpTimeout = cfg.FlushOpTimeout - f := &Flusher{ cfg: cfg, ingesterConfig: ingesterConfig, - chunkStore: chunkStore, limits: limits, registerer: registerer, logger: logger, @@ -78,7 +69,7 @@ func New( } func (f *Flusher) running(ctx context.Context) error { - ing, err := ingester.NewForFlusher(f.ingesterConfig, f.chunkStore, f.limits, f.registerer, f.logger) + ing, err := ingester.NewForFlusher(f.ingesterConfig, f.limits, f.registerer, f.logger) if err != nil { return errors.Wrap(err, "create ingester") } diff --git a/pkg/ingester/client/cortex_mock_test.go b/pkg/ingester/client/cortex_mock_test.go index d8ef07c78a4..54ade1e0860 100644 --- a/pkg/ingester/client/cortex_mock_test.go +++ b/pkg/ingester/client/cortex_mock_test.go @@ -61,8 +61,3 @@ func (m *IngesterServerMock) MetricsMetadata(ctx context.Context, r *MetricsMeta args := m.Called(ctx, r) return args.Get(0).(*MetricsMetadataResponse), args.Error(1) } - -func (m *IngesterServerMock) TransferChunks(s Ingester_TransferChunksServer) error { - args := m.Called(s) - return args.Error(0) -} diff --git a/pkg/ingester/client/cortex_util.go b/pkg/ingester/client/cortex_util.go index 2f917c03d70..b5dcecae354 100644 --- a/pkg/ingester/client/cortex_util.go +++ b/pkg/ingester/client/cortex_util.go @@ -12,14 +12,6 @@ func SendQueryStream(s Ingester_QueryStreamServer, m *QueryStreamResponse) error }) } -// SendTimeSeriesChunk wraps the stream's Send() checking if the context is done -// before calling Send(). -func SendTimeSeriesChunk(s Ingester_TransferChunksClient, m *TimeSeriesChunk) error { - return sendWithContextErrChecking(s.Context(), func() error { - return s.Send(m) - }) -} - func sendWithContextErrChecking(ctx context.Context, send func() error) error { // If the context has been canceled or its deadline exceeded, we should return it // instead of the cryptic error the Send() will return. diff --git a/pkg/ingester/client/ingester.pb.go b/pkg/ingester/client/ingester.pb.go index 8949becf7f1..7fd1997fe2a 100644 --- a/pkg/ingester/client/ingester.pb.go +++ b/pkg/ingester/client/ingester.pb.go @@ -1348,86 +1348,85 @@ func init() { func init() { proto.RegisterFile("ingester.proto", fileDescriptor_60f6df4f3586b478) } var fileDescriptor_60f6df4f3586b478 = []byte{ - // 1253 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0xcd, 0x6f, 0x13, 0x57, - 0x10, 0xdf, 0x17, 0x7f, 0x10, 0x8f, 0x1d, 0xe3, 0xbc, 0x00, 0x31, 0x4b, 0xd9, 0xd0, 0x95, 0x68, - 0xad, 0xb6, 0x38, 0x90, 0x7e, 0x08, 0xaa, 0x56, 0xc8, 0x81, 0x00, 0x29, 0x98, 0xc0, 0xc6, 0xb4, - 0x55, 0xa5, 0x6a, 0xb5, 0xb6, 0x5f, 0x9c, 0x2d, 0xfb, 0xc5, 0xbe, 0xb7, 0x15, 0xdc, 0x2a, 0xf5, - 0x0f, 0x68, 0xd5, 0x53, 0xaf, 0xbd, 0xf5, 0xdc, 0x4b, 0x6f, 0x3d, 0xf5, 0xc0, 0x91, 0x23, 0xea, - 0x01, 0x15, 0x73, 0xe9, 0x91, 0xfe, 0x07, 0xd5, 0xbe, 0x7d, 0xbb, 0xde, 0xdd, 0xd8, 0x40, 0x24, - 0xd2, 0x9b, 0x77, 0xe6, 0x37, 0xf3, 0x7e, 0x6f, 0x66, 0xde, 0xcc, 0x18, 0xea, 0xa6, 0x33, 0x22, - 0x94, 0x11, 0xbf, 0xed, 0xf9, 0x2e, 0x73, 0x71, 0x79, 0xe0, 0xfa, 0x8c, 0xdc, 0x97, 0xcf, 0x8c, - 0x4c, 0xb6, 0x1b, 0xf4, 0xdb, 0x03, 0xd7, 0x5e, 0x1d, 0xb9, 0x23, 0x77, 0x95, 0xab, 0xfb, 0xc1, - 0x0e, 0xff, 0xe2, 0x1f, 0xfc, 0x57, 0x64, 0x26, 0x5f, 0x48, 0xc1, 0x23, 0x0f, 0x9e, 0xef, 0x7e, - 0x43, 0x06, 0x4c, 0x7c, 0xad, 0x7a, 0x77, 0x47, 0xb1, 0xa2, 0x2f, 0x7e, 0x44, 0xa6, 0xea, 0xa7, - 0x50, 0xd5, 0x88, 0x31, 0xd4, 0xc8, 0xbd, 0x80, 0x50, 0x86, 0xdb, 0x70, 0xe8, 0x5e, 0x40, 0x7c, - 0x93, 0xd0, 0x26, 0x3a, 0x55, 0x68, 0x55, 0xd7, 0x8e, 0xb4, 0x05, 0xfc, 0x76, 0x40, 0xfc, 0x07, - 0x02, 0xa6, 0xc5, 0x20, 0xf5, 0x22, 0xd4, 0x22, 0x73, 0xea, 0xb9, 0x0e, 0x25, 0x78, 0x15, 0x0e, - 0xf9, 0x84, 0x06, 0x16, 0x8b, 0xed, 0x8f, 0xe6, 0xec, 0x23, 0x9c, 0x16, 0xa3, 0xd4, 0x9f, 0x11, - 0xd4, 0xd2, 0xae, 0xf1, 0x7b, 0x80, 0x29, 0x33, 0x7c, 0xa6, 0x33, 0xd3, 0x26, 0x94, 0x19, 0xb6, - 0xa7, 0xdb, 0xa1, 0x33, 0xd4, 0x2a, 0x68, 0x0d, 0xae, 0xe9, 0xc5, 0x8a, 0x2e, 0xc5, 0x2d, 0x68, - 0x10, 0x67, 0x98, 0xc5, 0xce, 0x71, 0x6c, 0x9d, 0x38, 0xc3, 0x34, 0xf2, 0x2c, 0xcc, 0xdb, 0x06, - 0x1b, 0xec, 0x12, 0x9f, 0x36, 0x0b, 0xd9, 0xab, 0xdd, 0x30, 0xfa, 0xc4, 0xea, 0x46, 0x4a, 0x2d, - 0x41, 0xa9, 0xbf, 0x20, 0x38, 0xb2, 0x71, 0x9f, 0xd8, 0x9e, 0x65, 0xf8, 0xff, 0x0b, 0xc5, 0x73, - 0x7b, 0x28, 0x1e, 0x9d, 0x46, 0x91, 0xa6, 0x38, 0x5e, 0x87, 0x85, 0x4c, 0x60, 0xf1, 0xc7, 0x00, - 0xfc, 0xa4, 0x69, 0x39, 0xf4, 0xfa, 0xed, 0xf0, 0xb8, 0x6d, 0xae, 0x5b, 0x2f, 0x3e, 0x7c, 0xb2, - 0x22, 0x69, 0x29, 0xb4, 0xfa, 0x13, 0x82, 0x25, 0xee, 0x6d, 0x9b, 0xf9, 0xc4, 0xb0, 0x13, 0x9f, - 0x17, 0xa1, 0x3a, 0xd8, 0x0d, 0x9c, 0xbb, 0x19, 0xa7, 0xcb, 0x31, 0xb5, 0x89, 0xcb, 0x4b, 0x21, - 0x48, 0xf8, 0x4d, 0x5b, 0xe4, 0x48, 0xcd, 0xed, 0x8b, 0xd4, 0x36, 0x1c, 0xcd, 0x25, 0xe1, 0x35, - 0xdc, 0xf4, 0x0f, 0x04, 0x98, 0x87, 0xf4, 0x73, 0xc3, 0x0a, 0x08, 0x8d, 0x13, 0x7b, 0x12, 0xc0, - 0x0a, 0xa5, 0xba, 0x63, 0xd8, 0x84, 0x27, 0xb4, 0xa2, 0x55, 0xb8, 0xe4, 0xa6, 0x61, 0x93, 0x19, - 0x79, 0x9f, 0xdb, 0x47, 0xde, 0x0b, 0x2f, 0xcd, 0x7b, 0xf1, 0x14, 0x7a, 0x95, 0xbc, 0x9f, 0x87, - 0xa5, 0x0c, 0x7f, 0x11, 0x93, 0x37, 0xa1, 0x16, 0x5d, 0xe0, 0x5b, 0x2e, 0xe7, 0x51, 0xa9, 0x68, - 0x55, 0x6b, 0x02, 0x55, 0xef, 0xc2, 0xe2, 0x8d, 0xf8, 0x46, 0xf4, 0x80, 0x2b, 0x5a, 0xfd, 0x50, - 0x84, 0x59, 0x1c, 0x26, 0x58, 0xae, 0x40, 0x75, 0x12, 0xe6, 0x98, 0x24, 0x24, 0x71, 0xa6, 0x2a, - 0x86, 0xc6, 0x1d, 0x4a, 0xfc, 0x6d, 0x66, 0xb0, 0x98, 0xa2, 0xfa, 0x3b, 0x82, 0xc5, 0x94, 0x50, - 0xb8, 0x3a, 0x1d, 0xb7, 0x50, 0xd3, 0x75, 0x74, 0xdf, 0x60, 0x51, 0xd6, 0x90, 0xb6, 0x90, 0x48, - 0x35, 0x83, 0x91, 0x30, 0xb1, 0x4e, 0x60, 0xeb, 0x49, 0x01, 0xa2, 0x56, 0x51, 0xab, 0x38, 0x81, - 0x1d, 0x15, 0x48, 0x78, 0x7d, 0xc3, 0x33, 0xf5, 0x9c, 0xa7, 0x02, 0xf7, 0xd4, 0x30, 0x3c, 0x73, - 0x33, 0xe3, 0xac, 0x0d, 0x4b, 0x7e, 0x60, 0x91, 0x3c, 0xbc, 0xc8, 0xe1, 0x8b, 0xa1, 0x2a, 0x83, - 0x57, 0xbf, 0x86, 0xa5, 0x90, 0xf8, 0xe6, 0xe5, 0x2c, 0xf5, 0x65, 0x38, 0x14, 0x50, 0xe2, 0xeb, - 0xe6, 0x50, 0x54, 0x5a, 0x39, 0xfc, 0xdc, 0x1c, 0xe2, 0x33, 0x50, 0x1c, 0x1a, 0xcc, 0xe0, 0x34, - 0xab, 0x6b, 0xc7, 0xe3, 0x52, 0xd8, 0x73, 0x79, 0x8d, 0xc3, 0xd4, 0xab, 0x80, 0x43, 0x15, 0xcd, - 0x7a, 0x3f, 0x07, 0x25, 0x1a, 0x0a, 0xc4, 0xc3, 0x38, 0x91, 0xf6, 0x92, 0x63, 0xa2, 0x45, 0x48, - 0xf5, 0x37, 0x04, 0x4a, 0x97, 0x30, 0xdf, 0x1c, 0xd0, 0x2b, 0xae, 0x9f, 0xad, 0xbc, 0x03, 0xee, - 0x7c, 0xe7, 0xa1, 0x16, 0x97, 0xb6, 0x4e, 0x09, 0x7b, 0x71, 0xf7, 0xab, 0xc6, 0xd0, 0x6d, 0xc2, - 0xd4, 0xeb, 0xb0, 0x32, 0x93, 0xb3, 0x08, 0x45, 0x0b, 0xca, 0x36, 0x87, 0x88, 0x58, 0x34, 0x26, - 0x4d, 0x22, 0x32, 0xd5, 0x84, 0x5e, 0x6d, 0xc2, 0x31, 0xe1, 0xac, 0x4b, 0x98, 0x11, 0x46, 0x37, - 0xae, 0xbe, 0x2d, 0x58, 0xde, 0xa3, 0x11, 0xee, 0x3f, 0x80, 0x79, 0x5b, 0xc8, 0xc4, 0x01, 0xcd, - 0xfc, 0x01, 0x89, 0x4d, 0x82, 0x54, 0xff, 0x45, 0x70, 0x38, 0xd7, 0x39, 0xc3, 0x78, 0xed, 0xf8, - 0xae, 0xad, 0xc7, 0x4b, 0xc1, 0xa4, 0x34, 0xea, 0xa1, 0x7c, 0x53, 0x88, 0x37, 0x87, 0xe9, 0xda, - 0x99, 0xcb, 0xd4, 0x8e, 0x03, 0x65, 0xfe, 0x8e, 0xe2, 0x01, 0xb2, 0x34, 0xa1, 0xc2, 0x83, 0x73, - 0xcb, 0x30, 0xfd, 0xf5, 0x4e, 0xd8, 0x0f, 0xff, 0x7a, 0xb2, 0xb2, 0xaf, 0xb5, 0x21, 0xb2, 0xef, - 0x0c, 0x0d, 0x8f, 0x11, 0x5f, 0x13, 0xa7, 0xe0, 0x77, 0xa1, 0x1c, 0x35, 0xfa, 0x66, 0x91, 0x9f, - 0xb7, 0x10, 0xa7, 0x2c, 0x3d, 0x0b, 0x04, 0x44, 0xfd, 0x01, 0x41, 0x29, 0xba, 0xe9, 0x41, 0xd5, - 0x91, 0x0c, 0xf3, 0xc4, 0x19, 0xb8, 0x43, 0xd3, 0x19, 0xf1, 0xe7, 0x5b, 0xd2, 0x92, 0x6f, 0x8c, - 0xc5, 0xb3, 0x0a, 0xdf, 0x69, 0x4d, 0xbc, 0x9d, 0x26, 0x1c, 0xeb, 0xf9, 0x86, 0x43, 0x77, 0x88, - 0xcf, 0x89, 0x25, 0x45, 0xa3, 0x76, 0x60, 0x21, 0x53, 0x4d, 0x99, 0xfd, 0x01, 0xbd, 0xd2, 0xfe, - 0xa0, 0x43, 0x2d, 0xad, 0xc1, 0xa7, 0xa1, 0xc8, 0x1e, 0x78, 0x51, 0x87, 0xaa, 0xaf, 0x2d, 0xc6, - 0xd6, 0x5c, 0xdd, 0x7b, 0xe0, 0x11, 0x8d, 0xab, 0x43, 0x9e, 0x7c, 0xfc, 0x44, 0x89, 0xe5, 0xbf, - 0xf1, 0x11, 0x28, 0xf1, 0x8e, 0xce, 0x2f, 0x55, 0xd1, 0xa2, 0x0f, 0xf5, 0x7b, 0x04, 0xf5, 0x49, - 0x0d, 0x5d, 0x31, 0x2d, 0xf2, 0x3a, 0x4a, 0x48, 0x86, 0xf9, 0x1d, 0xd3, 0x22, 0x9c, 0x43, 0x74, - 0x5c, 0xf2, 0x3d, 0x2d, 0x86, 0xef, 0x7c, 0x06, 0x95, 0xe4, 0x0a, 0xb8, 0x02, 0xa5, 0x8d, 0xdb, - 0x77, 0x3a, 0x37, 0x1a, 0x12, 0x5e, 0x80, 0xca, 0xcd, 0xad, 0x9e, 0x1e, 0x7d, 0x22, 0x7c, 0x18, - 0xaa, 0xda, 0xc6, 0xd5, 0x8d, 0x2f, 0xf5, 0x6e, 0xa7, 0x77, 0xe9, 0x5a, 0x63, 0x0e, 0x63, 0xa8, - 0x47, 0x82, 0x9b, 0x5b, 0x42, 0x56, 0x58, 0xfb, 0xb3, 0x0c, 0xf3, 0x31, 0x47, 0x7c, 0x01, 0x8a, - 0xb7, 0x02, 0xba, 0x8b, 0x8f, 0x4d, 0x6a, 0xf8, 0x0b, 0xdf, 0x64, 0x44, 0xbc, 0x49, 0x79, 0x79, - 0x8f, 0x5c, 0xe4, 0x4e, 0xc2, 0x1f, 0x41, 0x89, 0x2f, 0x0b, 0x78, 0xea, 0xfa, 0x2a, 0x4f, 0x5f, - 0x4a, 0x55, 0x09, 0x5f, 0x86, 0x6a, 0x6a, 0x01, 0x9a, 0x61, 0x7d, 0x22, 0x23, 0xcd, 0xee, 0x4a, - 0xaa, 0x74, 0x16, 0xe1, 0x2d, 0xa8, 0x73, 0x55, 0xbc, 0xb7, 0x50, 0xfc, 0x46, 0x6c, 0x32, 0x6d, - 0x9f, 0x94, 0x4f, 0xce, 0xd0, 0x26, 0xb4, 0xae, 0x41, 0x35, 0x35, 0xed, 0xb1, 0x9c, 0x29, 0xbc, - 0xcc, 0x0a, 0x33, 0x21, 0x37, 0x65, 0x3d, 0x50, 0x25, 0xbc, 0x01, 0x30, 0x19, 0xc8, 0xf8, 0x78, - 0x06, 0x9c, 0xde, 0x08, 0x64, 0x79, 0x9a, 0x2a, 0x71, 0xb3, 0x0e, 0x95, 0x64, 0x1c, 0xe1, 0xe6, - 0x94, 0x09, 0x15, 0x39, 0x99, 0x3d, 0xbb, 0x54, 0x09, 0x5f, 0x81, 0x5a, 0xc7, 0xb2, 0x5e, 0xc5, - 0x8d, 0x9c, 0xd6, 0xd0, 0xbc, 0x1f, 0x2b, 0x69, 0xcd, 0xf9, 0x09, 0x80, 0xdf, 0x4a, 0xde, 0xd8, - 0x0b, 0xc7, 0x9a, 0xfc, 0xf6, 0x4b, 0x71, 0xc9, 0x69, 0x3d, 0x38, 0x9c, 0x1b, 0x04, 0x58, 0xc9, - 0x59, 0xe7, 0x66, 0x87, 0xbc, 0x32, 0x53, 0x9f, 0x78, 0xed, 0x42, 0x3d, 0xdb, 0x87, 0xf0, 0xac, - 0xf5, 0x5a, 0x4e, 0x4e, 0x9b, 0xd1, 0xb8, 0xa4, 0x16, 0x5a, 0xff, 0xe4, 0xd1, 0x53, 0x45, 0x7a, - 0xfc, 0x54, 0x91, 0x9e, 0x3f, 0x55, 0xd0, 0x77, 0x63, 0x05, 0xfd, 0x3a, 0x56, 0xd0, 0xc3, 0xb1, - 0x82, 0x1e, 0x8d, 0x15, 0xf4, 0xf7, 0x58, 0x41, 0xff, 0x8c, 0x15, 0xe9, 0xf9, 0x58, 0x41, 0x3f, - 0x3e, 0x53, 0xa4, 0x47, 0xcf, 0x14, 0xe9, 0xf1, 0x33, 0x45, 0xfa, 0xaa, 0x3c, 0xb0, 0x4c, 0xe2, - 0xb0, 0x7e, 0x99, 0xff, 0x33, 0x7c, 0xff, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0e, 0x4f, 0x5c, - 0xe0, 0x9d, 0x0e, 0x00, 0x00, + // 1236 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0xcd, 0x6f, 0xd4, 0x46, + 0x14, 0xf7, 0x64, 0x3f, 0xc8, 0xbe, 0xdd, 0x2c, 0x9b, 0x09, 0x90, 0xc5, 0x14, 0x87, 0x5a, 0xa2, + 0x8d, 0xda, 0xb2, 0x81, 0xf4, 0x43, 0x50, 0xb5, 0x42, 0x1b, 0x08, 0x90, 0x42, 0x08, 0x38, 0x4b, + 0x5b, 0x55, 0xaa, 0x2c, 0xef, 0xee, 0x64, 0xe3, 0xe2, 0x2f, 0x3c, 0xe3, 0x0a, 0x6e, 0x95, 0xfa, + 0x07, 0xb4, 0xea, 0xa9, 0xd7, 0xde, 0x7a, 0xee, 0xa5, 0xb7, 0x9e, 0x39, 0xe6, 0x88, 0x7a, 0x40, + 0xcd, 0xe6, 0xd2, 0x23, 0x3d, 0xf4, 0x5e, 0x79, 0x3c, 0xf6, 0xda, 0xce, 0x2e, 0x24, 0x12, 0xe9, + 0xcd, 0xf3, 0xde, 0xef, 0xbd, 0xf9, 0xcd, 0x7b, 0x6f, 0xe6, 0x3d, 0x43, 0xdd, 0x74, 0x06, 0x84, + 0x32, 0xe2, 0xb7, 0x3c, 0xdf, 0x65, 0x2e, 0x2e, 0xf7, 0x5c, 0x9f, 0x91, 0xc7, 0xf2, 0x85, 0x81, + 0xc9, 0xb6, 0x83, 0x6e, 0xab, 0xe7, 0xda, 0x4b, 0x03, 0x77, 0xe0, 0x2e, 0x71, 0x75, 0x37, 0xd8, + 0xe2, 0x2b, 0xbe, 0xe0, 0x5f, 0x91, 0x99, 0x7c, 0x25, 0x05, 0x8f, 0x3c, 0x78, 0xbe, 0xfb, 0x0d, + 0xe9, 0x31, 0xb1, 0x5a, 0xf2, 0x1e, 0x0e, 0x62, 0x45, 0x57, 0x7c, 0x44, 0xa6, 0xea, 0xa7, 0x50, + 0xd5, 0x88, 0xd1, 0xd7, 0xc8, 0xa3, 0x80, 0x50, 0x86, 0x5b, 0x70, 0xec, 0x51, 0x40, 0x7c, 0x93, + 0xd0, 0x26, 0x3a, 0x57, 0x58, 0xac, 0x2e, 0x9f, 0x68, 0x09, 0xf8, 0xfd, 0x80, 0xf8, 0x4f, 0x04, + 0x4c, 0x8b, 0x41, 0xea, 0x55, 0xa8, 0x45, 0xe6, 0xd4, 0x73, 0x1d, 0x4a, 0xf0, 0x12, 0x1c, 0xf3, + 0x09, 0x0d, 0x2c, 0x16, 0xdb, 0x9f, 0xcc, 0xd9, 0x47, 0x38, 0x2d, 0x46, 0xa9, 0x3f, 0x23, 0xa8, + 0xa5, 0x5d, 0xe3, 0xf7, 0x00, 0x53, 0x66, 0xf8, 0x4c, 0x67, 0xa6, 0x4d, 0x28, 0x33, 0x6c, 0x4f, + 0xb7, 0x43, 0x67, 0x68, 0xb1, 0xa0, 0x35, 0xb8, 0xa6, 0x13, 0x2b, 0xd6, 0x29, 0x5e, 0x84, 0x06, + 0x71, 0xfa, 0x59, 0xec, 0x14, 0xc7, 0xd6, 0x89, 0xd3, 0x4f, 0x23, 0x2f, 0xc2, 0xb4, 0x6d, 0xb0, + 0xde, 0x36, 0xf1, 0x69, 0xb3, 0x90, 0x3d, 0xda, 0x1d, 0xa3, 0x4b, 0xac, 0xf5, 0x48, 0xa9, 0x25, + 0x28, 0xf5, 0x17, 0x04, 0x27, 0x56, 0x1f, 0x13, 0xdb, 0xb3, 0x0c, 0xff, 0x7f, 0xa1, 0x78, 0x69, + 0x1f, 0xc5, 0x93, 0xe3, 0x28, 0xd2, 0x14, 0xc7, 0xdb, 0x30, 0x93, 0x09, 0x2c, 0xfe, 0x18, 0x80, + 0xef, 0x34, 0x2e, 0x87, 0x5e, 0xb7, 0x15, 0x6e, 0xb7, 0xc9, 0x75, 0x2b, 0xc5, 0xa7, 0xcf, 0x17, + 0x24, 0x2d, 0x85, 0x56, 0x7f, 0x42, 0x30, 0xc7, 0xbd, 0x6d, 0x32, 0x9f, 0x18, 0x76, 0xe2, 0xf3, + 0x2a, 0x54, 0x7b, 0xdb, 0x81, 0xf3, 0x30, 0xe3, 0x74, 0x3e, 0xa6, 0x36, 0x72, 0x79, 0x2d, 0x04, + 0x09, 0xbf, 0x69, 0x8b, 0x1c, 0xa9, 0xa9, 0x43, 0x91, 0xda, 0x84, 0x93, 0xb9, 0x24, 0xbc, 0x86, + 0x93, 0xfe, 0x81, 0x00, 0xf3, 0x90, 0x7e, 0x6e, 0x58, 0x01, 0xa1, 0x71, 0x62, 0xcf, 0x02, 0x58, + 0xa1, 0x54, 0x77, 0x0c, 0x9b, 0xf0, 0x84, 0x56, 0xb4, 0x0a, 0x97, 0xdc, 0x35, 0x6c, 0x32, 0x21, + 0xef, 0x53, 0x87, 0xc8, 0x7b, 0xe1, 0x95, 0x79, 0x2f, 0x9e, 0x43, 0x07, 0xc9, 0xfb, 0x65, 0x98, + 0xcb, 0xf0, 0x17, 0x31, 0x79, 0x13, 0x6a, 0xd1, 0x01, 0xbe, 0xe5, 0x72, 0x1e, 0x95, 0x8a, 0x56, + 0xb5, 0x46, 0x50, 0xf5, 0x21, 0xcc, 0xde, 0x89, 0x4f, 0x44, 0x8f, 0xb8, 0xa2, 0xd5, 0x0f, 0x45, + 0x98, 0xc5, 0x66, 0x82, 0xe5, 0x02, 0x54, 0x47, 0x61, 0x8e, 0x49, 0x42, 0x12, 0x67, 0xaa, 0x62, + 0x68, 0x3c, 0xa0, 0xc4, 0xdf, 0x64, 0x06, 0x8b, 0x29, 0xaa, 0xbf, 0x23, 0x98, 0x4d, 0x09, 0x85, + 0xab, 0xf3, 0xf1, 0x13, 0x6a, 0xba, 0x8e, 0xee, 0x1b, 0x2c, 0xca, 0x1a, 0xd2, 0x66, 0x12, 0xa9, + 0x66, 0x30, 0x12, 0x26, 0xd6, 0x09, 0x6c, 0x3d, 0x29, 0x40, 0xb4, 0x58, 0xd4, 0x2a, 0x4e, 0x60, + 0x47, 0x05, 0x12, 0x1e, 0xdf, 0xf0, 0x4c, 0x3d, 0xe7, 0xa9, 0xc0, 0x3d, 0x35, 0x0c, 0xcf, 0x5c, + 0xcb, 0x38, 0x6b, 0xc1, 0x9c, 0x1f, 0x58, 0x24, 0x0f, 0x2f, 0x72, 0xf8, 0x6c, 0xa8, 0xca, 0xe0, + 0xd5, 0xaf, 0x61, 0x2e, 0x24, 0xbe, 0x76, 0x3d, 0x4b, 0x7d, 0x1e, 0x8e, 0x05, 0x94, 0xf8, 0xba, + 0xd9, 0x17, 0x95, 0x56, 0x0e, 0x97, 0x6b, 0x7d, 0x7c, 0x01, 0x8a, 0x7d, 0x83, 0x19, 0x9c, 0x66, + 0x75, 0xf9, 0x74, 0x5c, 0x0a, 0xfb, 0x0e, 0xaf, 0x71, 0x98, 0x7a, 0x13, 0x70, 0xa8, 0xa2, 0x59, + 0xef, 0x97, 0xa0, 0x44, 0x43, 0x81, 0xb8, 0x18, 0x67, 0xd2, 0x5e, 0x72, 0x4c, 0xb4, 0x08, 0xa9, + 0xfe, 0x86, 0x40, 0x59, 0x27, 0xcc, 0x37, 0x7b, 0xf4, 0x86, 0xeb, 0x67, 0x2b, 0xef, 0x88, 0x5f, + 0xbe, 0xcb, 0x50, 0x8b, 0x4b, 0x5b, 0xa7, 0x84, 0xbd, 0xfc, 0xf5, 0xab, 0xc6, 0xd0, 0x4d, 0xc2, + 0xd4, 0xdb, 0xb0, 0x30, 0x91, 0xb3, 0x08, 0xc5, 0x22, 0x94, 0x6d, 0x0e, 0x11, 0xb1, 0x68, 0x8c, + 0x1e, 0x89, 0xc8, 0x54, 0x13, 0x7a, 0xb5, 0x09, 0xa7, 0x84, 0xb3, 0x75, 0xc2, 0x8c, 0x30, 0xba, + 0x71, 0xf5, 0x6d, 0xc0, 0xfc, 0x3e, 0x8d, 0x70, 0xff, 0x01, 0x4c, 0xdb, 0x42, 0x26, 0x36, 0x68, + 0xe6, 0x37, 0x48, 0x6c, 0x12, 0xa4, 0xfa, 0x0f, 0x82, 0xe3, 0xb9, 0x97, 0x33, 0x8c, 0xd7, 0x96, + 0xef, 0xda, 0x7a, 0x3c, 0x14, 0x8c, 0x4a, 0xa3, 0x1e, 0xca, 0xd7, 0x84, 0x78, 0xad, 0x9f, 0xae, + 0x9d, 0xa9, 0x4c, 0xed, 0x38, 0x50, 0xe6, 0xf7, 0x28, 0x6e, 0x20, 0x73, 0x23, 0x2a, 0x3c, 0x38, + 0xf7, 0x0c, 0xd3, 0x5f, 0x69, 0x87, 0xef, 0xe1, 0x9f, 0xcf, 0x17, 0x0e, 0x35, 0x36, 0x44, 0xf6, + 0xed, 0xbe, 0xe1, 0x31, 0xe2, 0x6b, 0x62, 0x17, 0xfc, 0x2e, 0x94, 0xa3, 0x87, 0xbe, 0x59, 0xe4, + 0xfb, 0xcd, 0xc4, 0x29, 0x4b, 0xf7, 0x02, 0x01, 0x51, 0x7f, 0x40, 0x50, 0x8a, 0x4e, 0x7a, 0x54, + 0x75, 0x24, 0xc3, 0x34, 0x71, 0x7a, 0x6e, 0xdf, 0x74, 0x06, 0xfc, 0xfa, 0x96, 0xb4, 0x64, 0x8d, + 0xb1, 0xb8, 0x56, 0xe1, 0x3d, 0xad, 0x89, 0xbb, 0xd3, 0x84, 0x53, 0x1d, 0xdf, 0x70, 0xe8, 0x16, + 0xf1, 0x39, 0xb1, 0xa4, 0x68, 0xd4, 0x36, 0xcc, 0x64, 0xaa, 0x29, 0x33, 0x3f, 0xa0, 0x03, 0xcd, + 0x0f, 0x3a, 0xd4, 0xd2, 0x1a, 0x7c, 0x1e, 0x8a, 0xec, 0x89, 0x17, 0xbd, 0x50, 0xf5, 0xe5, 0xd9, + 0xd8, 0x9a, 0xab, 0x3b, 0x4f, 0x3c, 0xa2, 0x71, 0x75, 0xc8, 0x93, 0xb7, 0x9f, 0x28, 0xb1, 0xfc, + 0x1b, 0x9f, 0x80, 0x12, 0x7f, 0xd1, 0xf9, 0xa1, 0x2a, 0x5a, 0xb4, 0x50, 0xbf, 0x47, 0x50, 0x1f, + 0xd5, 0xd0, 0x0d, 0xd3, 0x22, 0xaf, 0xa3, 0x84, 0x64, 0x98, 0xde, 0x32, 0x2d, 0xc2, 0x39, 0x44, + 0xdb, 0x25, 0xeb, 0x71, 0x31, 0x7c, 0xe7, 0x33, 0xa8, 0x24, 0x47, 0xc0, 0x15, 0x28, 0xad, 0xde, + 0x7f, 0xd0, 0xbe, 0xd3, 0x90, 0xf0, 0x0c, 0x54, 0xee, 0x6e, 0x74, 0xf4, 0x68, 0x89, 0xf0, 0x71, + 0xa8, 0x6a, 0xab, 0x37, 0x57, 0xbf, 0xd4, 0xd7, 0xdb, 0x9d, 0x6b, 0xb7, 0x1a, 0x53, 0x18, 0x43, + 0x3d, 0x12, 0xdc, 0xdd, 0x10, 0xb2, 0xc2, 0xf2, 0xbf, 0x25, 0x98, 0x8e, 0x39, 0xe2, 0x2b, 0x50, + 0xbc, 0x17, 0xd0, 0x6d, 0x7c, 0x6a, 0x54, 0xc3, 0x5f, 0xf8, 0x26, 0x23, 0xe2, 0x4e, 0xca, 0xf3, + 0xfb, 0xe4, 0x22, 0x77, 0x12, 0xfe, 0x08, 0x4a, 0x7c, 0x58, 0xc0, 0x63, 0xc7, 0x57, 0x79, 0xfc, + 0x50, 0xaa, 0x4a, 0xf8, 0x3a, 0x54, 0x53, 0x03, 0xd0, 0x04, 0xeb, 0x33, 0x19, 0x69, 0x76, 0x56, + 0x52, 0xa5, 0x8b, 0x08, 0x6f, 0x40, 0x9d, 0xab, 0xe2, 0xb9, 0x85, 0xe2, 0x37, 0x62, 0x93, 0x71, + 0xf3, 0xa4, 0x7c, 0x76, 0x82, 0x36, 0xa1, 0x75, 0x0b, 0xaa, 0xa9, 0x6e, 0x8f, 0xe5, 0x4c, 0xe1, + 0x65, 0x46, 0x98, 0x11, 0xb9, 0x31, 0xe3, 0x81, 0x2a, 0xe1, 0x55, 0x80, 0x51, 0x43, 0xc6, 0xa7, + 0x33, 0xe0, 0xf4, 0x44, 0x20, 0xcb, 0xe3, 0x54, 0x89, 0x9b, 0x15, 0xa8, 0x24, 0xed, 0x08, 0x37, + 0xc7, 0x74, 0xa8, 0xc8, 0xc9, 0xe4, 0xde, 0xa5, 0x4a, 0xf8, 0x06, 0xd4, 0xda, 0x96, 0x75, 0x10, + 0x37, 0x72, 0x5a, 0x43, 0xf3, 0x7e, 0xac, 0xe4, 0x69, 0xce, 0x77, 0x00, 0xfc, 0x56, 0x72, 0xc7, + 0x5e, 0xda, 0xd6, 0xe4, 0xb7, 0x5f, 0x89, 0x4b, 0x76, 0xeb, 0xc0, 0xf1, 0x5c, 0x23, 0xc0, 0x4a, + 0xce, 0x3a, 0xd7, 0x3b, 0xe4, 0x85, 0x89, 0xfa, 0xd8, 0xeb, 0xca, 0x27, 0x3b, 0xbb, 0x8a, 0xf4, + 0x6c, 0x57, 0x91, 0x5e, 0xec, 0x2a, 0xe8, 0xbb, 0xa1, 0x82, 0x7e, 0x1d, 0x2a, 0xe8, 0xe9, 0x50, + 0x41, 0x3b, 0x43, 0x05, 0xfd, 0x35, 0x54, 0xd0, 0xdf, 0x43, 0x45, 0x7a, 0x31, 0x54, 0xd0, 0x8f, + 0x7b, 0x8a, 0xb4, 0xb3, 0xa7, 0x48, 0xcf, 0xf6, 0x14, 0xe9, 0xab, 0x72, 0xcf, 0x32, 0x89, 0xc3, + 0xba, 0x65, 0xfe, 0x2b, 0xf7, 0xfe, 0x7f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x92, 0x14, 0xdc, 0x04, + 0x4e, 0x0e, 0x00, 0x00, } func (x MatchType) String() string { @@ -2547,8 +2546,6 @@ type IngesterClient interface { AllUserStats(ctx context.Context, in *UserStatsRequest, opts ...grpc.CallOption) (*UsersStatsResponse, error) MetricsForLabelMatchers(ctx context.Context, in *MetricsForLabelMatchersRequest, opts ...grpc.CallOption) (*MetricsForLabelMatchersResponse, error) MetricsMetadata(ctx context.Context, in *MetricsMetadataRequest, opts ...grpc.CallOption) (*MetricsMetadataResponse, error) - // TransferChunks allows leaving ingester (client) to stream chunks directly to joining ingesters (server). - TransferChunks(ctx context.Context, opts ...grpc.CallOption) (Ingester_TransferChunksClient, error) } type ingesterClient struct { @@ -2672,40 +2669,6 @@ func (c *ingesterClient) MetricsMetadata(ctx context.Context, in *MetricsMetadat return out, nil } -func (c *ingesterClient) TransferChunks(ctx context.Context, opts ...grpc.CallOption) (Ingester_TransferChunksClient, error) { - stream, err := c.cc.NewStream(ctx, &_Ingester_serviceDesc.Streams[1], "/cortex.Ingester/TransferChunks", opts...) - if err != nil { - return nil, err - } - x := &ingesterTransferChunksClient{stream} - return x, nil -} - -type Ingester_TransferChunksClient interface { - Send(*TimeSeriesChunk) error - CloseAndRecv() (*TransferChunksResponse, error) - grpc.ClientStream -} - -type ingesterTransferChunksClient struct { - grpc.ClientStream -} - -func (x *ingesterTransferChunksClient) Send(m *TimeSeriesChunk) error { - return x.ClientStream.SendMsg(m) -} - -func (x *ingesterTransferChunksClient) CloseAndRecv() (*TransferChunksResponse, error) { - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - m := new(TransferChunksResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - // IngesterServer is the server API for Ingester service. type IngesterServer interface { Push(context.Context, *cortexpb.WriteRequest) (*cortexpb.WriteResponse, error) @@ -2718,8 +2681,6 @@ type IngesterServer interface { AllUserStats(context.Context, *UserStatsRequest) (*UsersStatsResponse, error) MetricsForLabelMatchers(context.Context, *MetricsForLabelMatchersRequest) (*MetricsForLabelMatchersResponse, error) MetricsMetadata(context.Context, *MetricsMetadataRequest) (*MetricsMetadataResponse, error) - // TransferChunks allows leaving ingester (client) to stream chunks directly to joining ingesters (server). - TransferChunks(Ingester_TransferChunksServer) error } // UnimplementedIngesterServer can be embedded to have forward compatible implementations. @@ -2756,9 +2717,6 @@ func (*UnimplementedIngesterServer) MetricsForLabelMatchers(ctx context.Context, func (*UnimplementedIngesterServer) MetricsMetadata(ctx context.Context, req *MetricsMetadataRequest) (*MetricsMetadataResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method MetricsMetadata not implemented") } -func (*UnimplementedIngesterServer) TransferChunks(srv Ingester_TransferChunksServer) error { - return status.Errorf(codes.Unimplemented, "method TransferChunks not implemented") -} func RegisterIngesterServer(s *grpc.Server, srv IngesterServer) { s.RegisterService(&_Ingester_serviceDesc, srv) @@ -2947,32 +2905,6 @@ func _Ingester_MetricsMetadata_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } -func _Ingester_TransferChunks_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(IngesterServer).TransferChunks(&ingesterTransferChunksServer{stream}) -} - -type Ingester_TransferChunksServer interface { - SendAndClose(*TransferChunksResponse) error - Recv() (*TimeSeriesChunk, error) - grpc.ServerStream -} - -type ingesterTransferChunksServer struct { - grpc.ServerStream -} - -func (x *ingesterTransferChunksServer) SendAndClose(m *TransferChunksResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *ingesterTransferChunksServer) Recv() (*TimeSeriesChunk, error) { - m := new(TimeSeriesChunk) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - var _Ingester_serviceDesc = grpc.ServiceDesc{ ServiceName: "cortex.Ingester", HandlerType: (*IngesterServer)(nil), @@ -3020,11 +2952,6 @@ var _Ingester_serviceDesc = grpc.ServiceDesc{ Handler: _Ingester_QueryStream_Handler, ServerStreams: true, }, - { - StreamName: "TransferChunks", - Handler: _Ingester_TransferChunks_Handler, - ClientStreams: true, - }, }, Metadata: "ingester.proto", } diff --git a/pkg/ingester/client/ingester.proto b/pkg/ingester/client/ingester.proto index 0314139fcec..942b8c04e39 100644 --- a/pkg/ingester/client/ingester.proto +++ b/pkg/ingester/client/ingester.proto @@ -23,9 +23,6 @@ service Ingester { rpc AllUserStats(UserStatsRequest) returns (UsersStatsResponse) {}; rpc MetricsForLabelMatchers(MetricsForLabelMatchersRequest) returns (MetricsForLabelMatchersResponse) {}; rpc MetricsMetadata(MetricsMetadataRequest) returns (MetricsMetadataResponse) {}; - - // TransferChunks allows leaving ingester (client) to stream chunks directly to joining ingesters (server). - rpc TransferChunks(stream TimeSeriesChunk) returns (TransferChunksResponse) {}; } message ReadRequest { diff --git a/pkg/ingester/errors.go b/pkg/ingester/errors.go index febdc1b4f03..b982f6ce09d 100644 --- a/pkg/ingester/errors.go +++ b/pkg/ingester/errors.go @@ -5,14 +5,12 @@ import ( "net/http" "github.com/prometheus/prometheus/model/labels" - "github.com/weaveworks/common/httpgrpc" ) type validationError struct { err error // underlying error errorType string code int - noReport bool // if true, error will be counted but not reported to caller labels labels.Labels } @@ -24,22 +22,6 @@ func makeLimitError(errorType string, err error) error { } } -func makeNoReportError(errorType string) error { - return &validationError{ - errorType: errorType, - noReport: true, - } -} - -func makeMetricValidationError(errorType string, labels labels.Labels, err error) error { - return &validationError{ - errorType: errorType, - err: err, - code: http.StatusBadRequest, - labels: labels, - } -} - func makeMetricLimitError(errorType string, labels labels.Labels, err error) error { return &validationError{ errorType: errorType, @@ -59,14 +41,6 @@ func (e *validationError) Error() string { return fmt.Sprintf("%s for series %s", e.err.Error(), e.labels.String()) } -// returns a HTTP gRPC error than is correctly forwarded over gRPC, with no reference to `e` retained. -func grpcForwardableError(userID string, code int, e error) error { - return httpgrpc.ErrorFromHTTPResponse(&httpgrpc.HTTPResponse{ - Code: int32(code), - Body: []byte(wrapWithUser(e, userID).Error()), - }) -} - // wrapWithUser prepends the user to the error. It does not retain a reference to err. func wrapWithUser(err error, userID string) error { return fmt.Errorf("user=%s: %s", userID, err) diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index b79e008091a..60f793f0790 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -1,430 +1,17 @@ package ingester import ( - "context" - "fmt" "net/http" - "time" - - "github.com/go-kit/log/level" - ot "github.com/opentracing/opentracing-go" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "golang.org/x/time/rate" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/log" -) - -const ( - // Backoff for retrying 'immediate' flushes. Only counts for queue - // position, not wallclock time. - flushBackoff = 1 * time.Second - // Lower bound on flushes per check period for rate-limiter - minFlushes = 100 ) // Flush triggers a flush of all the chunks and closes the flush queues. // Called from the Lifecycler as part of the ingester shutdown. func (i *Ingester) Flush() { - if i.cfg.BlocksStorageEnabled { - i.v2LifecyclerFlush() - return - } - - level.Info(i.logger).Log("msg", "starting to flush all the chunks") - i.sweepUsers(true) - level.Info(i.logger).Log("msg", "chunks queued for flushing") - - // Close the flush queues, to unblock waiting workers. - for _, flushQueue := range i.flushQueues { - flushQueue.Close() - } - - i.flushQueuesDone.Wait() - level.Info(i.logger).Log("msg", "flushing of chunks complete") + i.v2LifecyclerFlush() } // FlushHandler triggers a flush of all in memory chunks. Mainly used for // local testing. func (i *Ingester) FlushHandler(w http.ResponseWriter, r *http.Request) { - if i.cfg.BlocksStorageEnabled { - i.v2FlushHandler(w, r) - return - } - - level.Info(i.logger).Log("msg", "starting to flush all the chunks") - i.sweepUsers(true) - level.Info(i.logger).Log("msg", "chunks queued for flushing") - w.WriteHeader(http.StatusNoContent) -} - -type flushOp struct { - from model.Time - userID string - fp model.Fingerprint - immediate bool -} - -func (o *flushOp) Key() string { - return fmt.Sprintf("%s-%d-%v", o.userID, o.fp, o.immediate) -} - -func (o *flushOp) Priority() int64 { - return -int64(o.from) -} - -// sweepUsers periodically schedules series for flushing and garbage collects users with no series -func (i *Ingester) sweepUsers(immediate bool) { - if i.chunkStore == nil { - return - } - - oldest := model.Time(0) - - for id, state := range i.userStates.cp() { - for pair := range state.fpToSeries.iter() { - state.fpLocker.Lock(pair.fp) - i.sweepSeries(id, pair.fp, pair.series, immediate) - i.removeFlushedChunks(state, pair.fp, pair.series) - first := pair.series.firstUnflushedChunkTime() - state.fpLocker.Unlock(pair.fp) - - if first > 0 && (oldest == 0 || first < oldest) { - oldest = first - } - } - } - - i.metrics.oldestUnflushedChunkTimestamp.Set(float64(oldest.Unix())) - i.setFlushRate() -} - -// Compute a rate such to spread calls to the store over nearly all of the flush period, -// for example if we have 600 items in the queue and period 1 min we will send 10.5 per second. -// Note if the store can't keep up with this rate then it doesn't make any difference. -func (i *Ingester) setFlushRate() { - totalQueueLength := 0 - for _, q := range i.flushQueues { - totalQueueLength += q.Length() - } - const fudge = 1.05 // aim to finish a little bit before the end of the period - flushesPerSecond := float64(totalQueueLength) / i.cfg.FlushCheckPeriod.Seconds() * fudge - // Avoid going very slowly with tiny queues - if flushesPerSecond*i.cfg.FlushCheckPeriod.Seconds() < minFlushes { - flushesPerSecond = minFlushes / i.cfg.FlushCheckPeriod.Seconds() - } - level.Debug(i.logger).Log("msg", "computed flush rate", "rate", flushesPerSecond) - i.flushRateLimiter.SetLimit(rate.Limit(flushesPerSecond)) -} - -type flushReason int8 - -const ( - noFlush = iota - reasonImmediate - reasonMultipleChunksInSeries - reasonAged - reasonIdle - reasonStale - reasonSpreadFlush - // Following are flush outcomes - noUser - noSeries - noChunks - flushError - reasonDropped - maxFlushReason // Used for testing String() method. Should be last. -) - -func (f flushReason) String() string { - switch f { - case noFlush: - return "NoFlush" - case reasonImmediate: - return "Immediate" - case reasonMultipleChunksInSeries: - return "MultipleChunksInSeries" - case reasonAged: - return "Aged" - case reasonIdle: - return "Idle" - case reasonStale: - return "Stale" - case reasonSpreadFlush: - return "Spread" - case noUser: - return "NoUser" - case noSeries: - return "NoSeries" - case noChunks: - return "NoChunksToFlush" - case flushError: - return "FlushError" - case reasonDropped: - return "Dropped" - default: - panic("unrecognised flushReason") - } -} - -// sweepSeries schedules a series for flushing based on a set of criteria -// -// NB we don't close the head chunk here, as the series could wait in the queue -// for some time, and we want to encourage chunks to be as full as possible. -func (i *Ingester) sweepSeries(userID string, fp model.Fingerprint, series *memorySeries, immediate bool) { - if len(series.chunkDescs) <= 0 { - return - } - - firstTime := series.firstTime() - flush := i.shouldFlushSeries(series, fp, immediate) - if flush == noFlush { - return - } - - flushQueueIndex := int(uint64(fp) % uint64(i.cfg.ConcurrentFlushes)) - if i.flushQueues[flushQueueIndex].Enqueue(&flushOp{firstTime, userID, fp, immediate}) { - i.metrics.seriesEnqueuedForFlush.WithLabelValues(flush.String()).Inc() - util.Event().Log("msg", "add to flush queue", "userID", userID, "reason", flush, "firstTime", firstTime, "fp", fp, "series", series.metric, "nlabels", len(series.metric), "queue", flushQueueIndex) - } -} - -func (i *Ingester) shouldFlushSeries(series *memorySeries, fp model.Fingerprint, immediate bool) flushReason { - if len(series.chunkDescs) == 0 { - return noFlush - } - if immediate { - for _, cd := range series.chunkDescs { - if !cd.flushed { - return reasonImmediate - } - } - return noFlush // Everything is flushed. - } - - // Flush if we have more than one chunk, and haven't already flushed the first chunk - if len(series.chunkDescs) > 1 && !series.chunkDescs[0].flushed { - if series.chunkDescs[0].flushReason != noFlush { - return series.chunkDescs[0].flushReason - } - return reasonMultipleChunksInSeries - } - // Otherwise look in more detail at the first chunk - return i.shouldFlushChunk(series.chunkDescs[0], fp, series.isStale()) -} - -func (i *Ingester) shouldFlushChunk(c *desc, fp model.Fingerprint, lastValueIsStale bool) flushReason { - if c.flushed { // don't flush chunks we've already flushed - return noFlush - } - - // Adjust max age slightly to spread flushes out over time - var jitter time.Duration - if i.cfg.ChunkAgeJitter != 0 { - jitter = time.Duration(fp) % i.cfg.ChunkAgeJitter - } - // Chunks should be flushed if they span longer than MaxChunkAge - if c.LastTime.Sub(c.FirstTime) > (i.cfg.MaxChunkAge - jitter) { - return reasonAged - } - - // Chunk should be flushed if their last update is older then MaxChunkIdle. - if model.Now().Sub(c.LastUpdate) > i.cfg.MaxChunkIdle { - return reasonIdle - } - - // A chunk that has a stale marker can be flushed if possible. - if i.cfg.MaxStaleChunkIdle > 0 && - lastValueIsStale && - model.Now().Sub(c.LastUpdate) > i.cfg.MaxStaleChunkIdle { - return reasonStale - } - - return noFlush -} - -func (i *Ingester) flushLoop(j int) { - defer func() { - level.Debug(i.logger).Log("msg", "Ingester.flushLoop() exited") - i.flushQueuesDone.Done() - }() - - for { - o := i.flushQueues[j].Dequeue() - if o == nil { - return - } - op := o.(*flushOp) - - if !op.immediate { - _ = i.flushRateLimiter.Wait(context.Background()) - } - outcome, err := i.flushUserSeries(j, op.userID, op.fp, op.immediate) - i.metrics.seriesDequeuedOutcome.WithLabelValues(outcome.String()).Inc() - if err != nil { - level.Error(log.WithUserID(op.userID, i.logger)).Log("msg", "failed to flush user", "err", err) - } - - // If we're exiting & we failed to flush, put the failed operation - // back in the queue at a later point. - if op.immediate && err != nil { - op.from = op.from.Add(flushBackoff) - i.flushQueues[j].Enqueue(op) - } - } -} - -// Returns flush outcome (either original reason, if series was flushed, noFlush if it doesn't need flushing anymore, or one of the errors) -func (i *Ingester) flushUserSeries(flushQueueIndex int, userID string, fp model.Fingerprint, immediate bool) (flushReason, error) { - i.metrics.flushSeriesInProgress.Inc() - defer i.metrics.flushSeriesInProgress.Dec() - - if i.preFlushUserSeries != nil { - i.preFlushUserSeries() - } - - userState, ok := i.userStates.get(userID) - if !ok { - return noUser, nil - } - - series, ok := userState.fpToSeries.get(fp) - if !ok { - return noSeries, nil - } - - userState.fpLocker.Lock(fp) - reason := i.shouldFlushSeries(series, fp, immediate) - if reason == noFlush { - userState.fpLocker.Unlock(fp) - return noFlush, nil - } - - // shouldFlushSeries() has told us we have at least one chunk. - // Make a copy of chunks descriptors slice, to avoid possible issues related to removing (and niling) elements from chunkDesc. - // This can happen if first chunk is already flushed -- removeFlushedChunks may set such chunk to nil. - // Since elements in the slice are pointers, we can still safely update chunk descriptors after the copy. - chunks := append([]*desc(nil), series.chunkDescs...) - if immediate { - series.closeHead(reasonImmediate) - } else if chunkReason := i.shouldFlushChunk(series.head(), fp, series.isStale()); chunkReason != noFlush { - series.closeHead(chunkReason) - } else { - // The head chunk doesn't need flushing; step back by one. - chunks = chunks[:len(chunks)-1] - } - - if (reason == reasonIdle || reason == reasonStale) && series.headChunkClosed { - if minChunkLength := i.limits.MinChunkLength(userID); minChunkLength > 0 { - chunkLength := 0 - for _, c := range chunks { - chunkLength += c.C.Len() - } - if chunkLength < minChunkLength { - userState.removeSeries(fp, series.metric) - i.metrics.memoryChunks.Sub(float64(len(chunks))) - i.metrics.droppedChunks.Add(float64(len(chunks))) - util.Event().Log( - "msg", "dropped chunks", - "userID", userID, - "numChunks", len(chunks), - "chunkLength", chunkLength, - "fp", fp, - "series", series.metric, - "queue", flushQueueIndex, - ) - chunks = nil - reason = reasonDropped - } - } - } - - userState.fpLocker.Unlock(fp) - - if reason == reasonDropped { - return reason, nil - } - - // No need to flush these chunks again. - for len(chunks) > 0 && chunks[0].flushed { - chunks = chunks[1:] - } - - if len(chunks) == 0 { - return noChunks, nil - } - - // flush the chunks without locking the series, as we don't want to hold the series lock for the duration of the dynamo/s3 rpcs. - ctx, cancel := context.WithTimeout(context.Background(), i.cfg.FlushOpTimeout) - defer cancel() // releases resources if slowOperation completes before timeout elapses - - sp, ctx := ot.StartSpanFromContext(ctx, "flushUserSeries") - defer sp.Finish() - sp.SetTag("organization", userID) - - util.Event().Log("msg", "flush chunks", "userID", userID, "reason", reason, "numChunks", len(chunks), "firstTime", chunks[0].FirstTime, "fp", fp, "series", series.metric, "nlabels", len(series.metric), "queue", flushQueueIndex) - err := i.flushChunks(ctx, userID, fp, series.metric, chunks) - if err != nil { - return flushError, err - } - - userState.fpLocker.Lock(fp) - for i := 0; i < len(chunks); i++ { - // Mark the chunks as flushed, so we can remove them after the retention period. - // We can safely use chunks[i] here, because elements are pointers to chunk descriptors. - chunks[i].flushed = true - chunks[i].LastUpdate = model.Now() - } - userState.fpLocker.Unlock(fp) - return reason, err -} - -// must be called under fpLocker lock -func (i *Ingester) removeFlushedChunks(userState *userState, fp model.Fingerprint, series *memorySeries) { - now := model.Now() - for len(series.chunkDescs) > 0 { - if series.chunkDescs[0].flushed && now.Sub(series.chunkDescs[0].LastUpdate) > i.cfg.RetainPeriod { - series.chunkDescs[0] = nil // erase reference so the chunk can be garbage-collected - series.chunkDescs = series.chunkDescs[1:] - i.metrics.memoryChunks.Dec() - } else { - break - } - } - if len(series.chunkDescs) == 0 { - userState.removeSeries(fp, series.metric) - } -} - -func (i *Ingester) flushChunks(ctx context.Context, userID string, fp model.Fingerprint, metric labels.Labels, chunkDescs []*desc) error { - if i.preFlushChunks != nil { - i.preFlushChunks() - } - - wireChunks := make([]chunk.Chunk, 0, len(chunkDescs)) - for _, chunkDesc := range chunkDescs { - c := chunk.NewChunk(userID, fp, metric, chunkDesc.C, chunkDesc.FirstTime, chunkDesc.LastTime) - if err := c.Encode(); err != nil { - return err - } - wireChunks = append(wireChunks, c) - } - - if err := i.chunkStore.Put(ctx, wireChunks); err != nil { - return err - } - - // Record statistics only when actual put request did not return error. - for _, chunkDesc := range chunkDescs { - utilization, length, size := chunkDesc.C.Utilization(), chunkDesc.C.Len(), chunkDesc.C.Size() - util.Event().Log("msg", "chunk flushed", "userID", userID, "fp", fp, "series", metric, "nlabels", len(metric), "utilization", utilization, "length", length, "size", size, "firstTime", chunkDesc.FirstTime, "lastTime", chunkDesc.LastTime) - i.metrics.chunkUtilization.Observe(utilization) - i.metrics.chunkLength.Observe(float64(length)) - i.metrics.chunkSize.Observe(float64(size)) - i.metrics.chunkAge.Observe(model.Now().Sub(chunkDesc.FirstTime).Seconds()) - } - - return nil + i.v2FlushHandler(w, r) } diff --git a/pkg/ingester/flush_test.go b/pkg/ingester/flush_test.go deleted file mode 100644 index ffb00f09de0..00000000000 --- a/pkg/ingester/flush_test.go +++ /dev/null @@ -1,261 +0,0 @@ -package ingester - -import ( - "context" - "fmt" - "io/ioutil" - "os" - "testing" - "time" - - "github.com/go-kit/log" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/require" - "github.com/weaveworks/common/user" - "go.uber.org/atomic" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/ingester/client" - "github.com/cortexproject/cortex/pkg/ring" - "github.com/cortexproject/cortex/pkg/ring/kv" - "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/pkg/util/validation" -) - -var singleTestLabel = []labels.Labels{[]labels.Label{{Name: "__name__", Value: "test"}}} - -// This test case demonstrates problem with losing incoming samples while chunks are flushed with "immediate" mode. -func TestSweepImmediateDropsSamples(t *testing.T) { - cfg := emptyIngesterConfig() - cfg.FlushCheckPeriod = 1 * time.Minute - cfg.RetainPeriod = 10 * time.Millisecond - - st := &sleepyCountingStore{} - ing := createTestIngester(t, cfg, st) - - samples := newSampleGenerator(t, time.Now(), time.Millisecond) - - // Generates one sample. - pushSample(t, ing, <-samples) - - notify := make(chan struct{}) - ing.preFlushChunks = func() { - if ing.State() == services.Running { - pushSample(t, ing, <-samples) - notify <- struct{}{} - } - } - - // Simulate /flush. Sweeps everything, but also pushes another sample (in preFlushChunks) - ing.sweepUsers(true) - <-notify // Wait for flushing to happen. - - // Stopping ingester should sweep the remaining samples. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - - require.Equal(t, 2, st.samples) -} - -// There are several factors in this panic: -// Chunk is first flushed normally -// "/flush" is called (sweepUsers(true)), and that causes new flush of already flushed chunks -// During the flush to store (in flushChunks), chunk is actually removed from list of chunks (and its reference is niled) in removeFlushedChunks. -// After flushing to store, reference is nil, causing panic. -func TestFlushPanicIssue2743(t *testing.T) { - cfg := emptyIngesterConfig() - cfg.FlushCheckPeriod = 50 * time.Millisecond // We want to check for flush-able and removable chunks often. - cfg.RetainPeriod = 500 * time.Millisecond // Remove flushed chunks quickly. This triggers nil-ing. To get a panic, it should happen while Store is "writing" chunks. (We use "sleepy store" to enforce that) - cfg.MaxChunkAge = 1 * time.Hour // We don't use max chunk age for this test, as that is jittered. - cfg.MaxChunkIdle = 200 * time.Millisecond // Flush chunk 200ms after adding last sample. - - st := &sleepyCountingStore{d: 1 * time.Second} // Longer than retain period - - ing := createTestIngester(t, cfg, st) - samples := newSampleGenerator(t, time.Now(), 1*time.Second) - - notifyCh := make(chan bool, 10) - ing.preFlushChunks = func() { - select { - case notifyCh <- true: - default: - } - } - - // Generates one sample - pushSample(t, ing, <-samples) - - // Wait until regular flush kicks in (flushing due to chunk being idle) - <-notifyCh - - // Sweep again -- this causes the same chunks to be queued for flushing again. - // We must hit this *before* flushed chunk is removed from list of chunks. (RetainPeriod) - // While store is flushing (simulated by sleep in the store), previously flushed chunk is removed from memory. - ing.sweepUsers(true) - - // Wait a bit for flushing to end. In buggy version, we get panic while waiting. - time.Sleep(2 * time.Second) -} - -func pushSample(t *testing.T, ing *Ingester, sample cortexpb.Sample) { - _, err := ing.Push(user.InjectOrgID(context.Background(), userID), cortexpb.ToWriteRequest(singleTestLabel, []cortexpb.Sample{sample}, nil, cortexpb.API)) - require.NoError(t, err) -} - -func createTestIngester(t *testing.T, cfg Config, store ChunkStore) *Ingester { - l := validation.Limits{} - overrides, err := validation.NewOverrides(l, nil) - require.NoError(t, err) - - ing, err := New(cfg, client.Config{}, overrides, store, nil, log.NewNopLogger()) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) - t.Cleanup(func() { - _ = services.StopAndAwaitTerminated(context.Background(), ing) - }) - - return ing -} - -type sleepyCountingStore struct { - d time.Duration - samples int -} - -func (m *sleepyCountingStore) Put(_ context.Context, chunks []chunk.Chunk) error { - if m.d > 0 { - time.Sleep(m.d) - } - - for _, c := range chunks { - m.samples += c.Data.Len() - } - return nil -} - -func emptyIngesterConfig() Config { - return Config{ - WALConfig: WALConfig{}, - LifecyclerConfig: ring.LifecyclerConfig{ - RingConfig: ring.Config{ - KVStore: kv.Config{ - Store: "inmemory", - }, - ReplicationFactor: 1, - }, - InfNames: []string{"en0", "eth0", "lo0", "lo"}, - HeartbeatPeriod: 10 * time.Second, - }, - - ConcurrentFlushes: 1, // Single queue only. Doesn't really matter for this test (same series is always flushed by same worker), but must be positive. - RateUpdatePeriod: 1 * time.Hour, // Must be positive, doesn't matter for this test. - ActiveSeriesMetricsUpdatePeriod: 5 * time.Minute, // Must be positive. - } -} - -func newSampleGenerator(t *testing.T, initTime time.Time, step time.Duration) <-chan cortexpb.Sample { - ts := make(chan cortexpb.Sample) - - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - go func(ctx context.Context) { - c := initTime - for { - select { - case ts <- cortexpb.Sample{Value: 0, TimestampMs: util.TimeToMillis(c)}: - case <-ctx.Done(): - return - } - - c = c.Add(step) - } - }(ctx) - - return ts -} - -func TestFlushReasonString(t *testing.T) { - for fr := flushReason(0); fr < maxFlushReason; fr++ { - require.True(t, len(fr.String()) > 0) - } -} - -// Issue 3139 depends on a timing between immediate flush, and periodic flush, and the fact that "immediate" chunks get behind "idle" chunks. -// Periodic flush may still find "idle" chunks and put them onto queue, because "ingester for flusher" still runs all the loops. -// When flush of "immediate" chunk fails (eg. due to storage error), it is put back onto the queue, but behind Idle chunk. -// When handling Idle chunks, they are then compared against user limit (MinChunkLength), which panics -- because we were not setting limits. -func TestIssue3139(t *testing.T) { - dir, err := ioutil.TempDir("", "wal") - require.NoError(t, err) - t.Cleanup(func() { - _ = os.RemoveAll(dir) - }) - - cfg := emptyIngesterConfig() - cfg.WALConfig.FlushOnShutdown = false - cfg.WALConfig.Dir = dir - cfg.WALConfig.WALEnabled = true - - cfg.FlushCheckPeriod = 10 * time.Millisecond - cfg.MaxChunkAge = 1 * time.Hour // We don't want to hit "age" check, but idle-ness check. - cfg.MaxChunkIdle = 0 // Everything is idle immediately - - // Sleep long enough for period flush to happen. Also we want to return errors to the first attempts, so that - // series are flushed again. - st := &sleepyStoreWithErrors{d: 500 * time.Millisecond} - st.errorsToGenerate.Store(1) - - ing := createTestIngester(t, cfg, st) - - // Generates a sample. While it is flushed for the first time (which returns error), it will be put on the queue - // again. - pushSample(t, ing, cortexpb.Sample{Value: 100, TimestampMs: int64(model.Now())}) - - // stop ingester -- no flushing should happen yet - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - - // Make sure nothing was flushed yet... sample should be in WAL - require.Equal(t, int64(0), st.samples.Load()) - require.Equal(t, int64(1), st.errorsToGenerate.Load()) // no error was "consumed" - - // Start new ingester, for flushing only - ing, err = NewForFlusher(cfg, st, nil, nil, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) - t.Cleanup(func() { - // Just in case test fails earlier, stop ingester anyay. - _ = services.StopAndAwaitTerminated(context.Background(), ing) - }) - - ing.Flush() - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - - // Verify sample was flushed from WAL. - require.Equal(t, int64(1), st.samples.Load()) -} - -type sleepyStoreWithErrors struct { - d time.Duration - errorsToGenerate atomic.Int64 - samples atomic.Int64 -} - -func (m *sleepyStoreWithErrors) Put(_ context.Context, chunks []chunk.Chunk) error { - if m.d > 0 { - time.Sleep(m.d) - } - - if m.errorsToGenerate.Load() > 0 { - m.errorsToGenerate.Dec() - return fmt.Errorf("put error") - } - - for _, c := range chunks { - m.samples.Add(int64(c.Data.Len())) - } - return nil -} diff --git a/pkg/ingester/index/index.go b/pkg/ingester/index/index.go deleted file mode 100644 index 09d9d84eeab..00000000000 --- a/pkg/ingester/index/index.go +++ /dev/null @@ -1,324 +0,0 @@ -package index - -import ( - "sort" - "sync" - "unsafe" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/util" -) - -const indexShards = 32 - -// InvertedIndex implements a in-memory inverter index from label pairs to fingerprints. -// It is sharded to reduce lock contention on writes. -type InvertedIndex struct { - shards []indexShard -} - -// New returns a new InvertedIndex. -func New() *InvertedIndex { - shards := make([]indexShard, indexShards) - for i := 0; i < indexShards; i++ { - shards[i].idx = map[string]indexEntry{} - } - return &InvertedIndex{ - shards: shards, - } -} - -// Add a fingerprint under the specified labels. -// NOTE: memory for `labels` is unsafe; anything retained beyond the -// life of this function must be copied -func (ii *InvertedIndex) Add(labels []cortexpb.LabelAdapter, fp model.Fingerprint) labels.Labels { - shard := &ii.shards[util.HashFP(fp)%indexShards] - return shard.add(labels, fp) // add() returns 'interned' values so the original labels are not retained -} - -// Lookup all fingerprints for the provided matchers. -func (ii *InvertedIndex) Lookup(matchers []*labels.Matcher) []model.Fingerprint { - if len(matchers) == 0 { - return nil - } - - result := []model.Fingerprint{} - for i := range ii.shards { - fps := ii.shards[i].lookup(matchers) - result = append(result, fps...) - } - - return result -} - -// LabelNames returns all label names. -func (ii *InvertedIndex) LabelNames() []string { - results := make([][]string, 0, indexShards) - - for i := range ii.shards { - shardResult := ii.shards[i].labelNames() - results = append(results, shardResult) - } - - return mergeStringSlices(results) -} - -// LabelValues returns the values for the given label. -func (ii *InvertedIndex) LabelValues(name string) []string { - results := make([][]string, 0, indexShards) - - for i := range ii.shards { - shardResult := ii.shards[i].labelValues(name) - results = append(results, shardResult) - } - - return mergeStringSlices(results) -} - -// Delete a fingerprint with the given label pairs. -func (ii *InvertedIndex) Delete(labels labels.Labels, fp model.Fingerprint) { - shard := &ii.shards[util.HashFP(fp)%indexShards] - shard.delete(labels, fp) -} - -// NB slice entries are sorted in fp order. -type indexEntry struct { - name string - fps map[string]indexValueEntry -} - -type indexValueEntry struct { - value string - fps []model.Fingerprint -} - -type unlockIndex map[string]indexEntry - -// This is the prevalent value for Intel and AMD CPUs as-at 2018. -const cacheLineSize = 64 - -type indexShard struct { - mtx sync.RWMutex - idx unlockIndex - //nolint:structcheck,unused - pad [cacheLineSize - unsafe.Sizeof(sync.Mutex{}) - unsafe.Sizeof(unlockIndex{})]byte -} - -func copyString(s string) string { - return string([]byte(s)) -} - -// add metric to the index; return all the name/value pairs as a fresh -// sorted slice, referencing 'interned' strings from the index so that -// no references are retained to the memory of `metric`. -func (shard *indexShard) add(metric []cortexpb.LabelAdapter, fp model.Fingerprint) labels.Labels { - shard.mtx.Lock() - defer shard.mtx.Unlock() - - internedLabels := make(labels.Labels, len(metric)) - - for i, pair := range metric { - values, ok := shard.idx[pair.Name] - if !ok { - values = indexEntry{ - name: copyString(pair.Name), - fps: map[string]indexValueEntry{}, - } - shard.idx[values.name] = values - } - fingerprints, ok := values.fps[pair.Value] - if !ok { - fingerprints = indexValueEntry{ - value: copyString(pair.Value), - } - } - // Insert into the right position to keep fingerprints sorted - j := sort.Search(len(fingerprints.fps), func(i int) bool { - return fingerprints.fps[i] >= fp - }) - fingerprints.fps = append(fingerprints.fps, 0) - copy(fingerprints.fps[j+1:], fingerprints.fps[j:]) - fingerprints.fps[j] = fp - values.fps[fingerprints.value] = fingerprints - internedLabels[i] = labels.Label{Name: values.name, Value: fingerprints.value} - } - sort.Sort(internedLabels) - return internedLabels -} - -func (shard *indexShard) lookup(matchers []*labels.Matcher) []model.Fingerprint { - // index slice values must only be accessed under lock, so all - // code paths must take a copy before returning - shard.mtx.RLock() - defer shard.mtx.RUnlock() - - // per-shard intersection is initially nil, which is a special case - // meaning "everything" when passed to intersect() - // loop invariant: result is sorted - var result []model.Fingerprint - for _, matcher := range matchers { - values, ok := shard.idx[matcher.Name] - if !ok { - return nil - } - var toIntersect model.Fingerprints - if matcher.Type == labels.MatchEqual { - fps := values.fps[matcher.Value] - toIntersect = append(toIntersect, fps.fps...) // deliberate copy - } else if matcher.Type == labels.MatchRegexp && len(chunk.FindSetMatches(matcher.Value)) > 0 { - // The lookup is of the form `=~"a|b|c|d"` - set := chunk.FindSetMatches(matcher.Value) - for _, value := range set { - toIntersect = append(toIntersect, values.fps[value].fps...) - } - sort.Sort(toIntersect) - } else { - // accumulate the matching fingerprints (which are all distinct) - // then sort to maintain the invariant - for value, fps := range values.fps { - if matcher.Matches(value) { - toIntersect = append(toIntersect, fps.fps...) - } - } - sort.Sort(toIntersect) - } - result = intersect(result, toIntersect) - if len(result) == 0 { - return nil - } - } - - return result -} - -func (shard *indexShard) labelNames() []string { - shard.mtx.RLock() - defer shard.mtx.RUnlock() - - results := make([]string, 0, len(shard.idx)) - for name := range shard.idx { - results = append(results, name) - } - - sort.Strings(results) - return results -} - -func (shard *indexShard) labelValues(name string) []string { - shard.mtx.RLock() - defer shard.mtx.RUnlock() - - values, ok := shard.idx[name] - if !ok { - return nil - } - - results := make([]string, 0, len(values.fps)) - for val := range values.fps { - results = append(results, val) - } - - sort.Strings(results) - return results -} - -func (shard *indexShard) delete(labels labels.Labels, fp model.Fingerprint) { - shard.mtx.Lock() - defer shard.mtx.Unlock() - - for _, pair := range labels { - name, value := pair.Name, pair.Value - values, ok := shard.idx[name] - if !ok { - continue - } - fingerprints, ok := values.fps[value] - if !ok { - continue - } - - j := sort.Search(len(fingerprints.fps), func(i int) bool { - return fingerprints.fps[i] >= fp - }) - - // see if search didn't find fp which matches the condition which means we don't have to do anything. - if j >= len(fingerprints.fps) || fingerprints.fps[j] != fp { - continue - } - fingerprints.fps = fingerprints.fps[:j+copy(fingerprints.fps[j:], fingerprints.fps[j+1:])] - - if len(fingerprints.fps) == 0 { - delete(values.fps, value) - } else { - values.fps[value] = fingerprints - } - - if len(values.fps) == 0 { - delete(shard.idx, name) - } else { - shard.idx[name] = values - } - } -} - -// intersect two sorted lists of fingerprints. Assumes there are no duplicate -// fingerprints within the input lists. -func intersect(a, b []model.Fingerprint) []model.Fingerprint { - if a == nil { - return b - } - result := []model.Fingerprint{} - for i, j := 0, 0; i < len(a) && j < len(b); { - if a[i] == b[j] { - result = append(result, a[i]) - } - if a[i] < b[j] { - i++ - } else { - j++ - } - } - return result -} - -func mergeStringSlices(ss [][]string) []string { - switch len(ss) { - case 0: - return nil - case 1: - return ss[0] - case 2: - return mergeTwoStringSlices(ss[0], ss[1]) - default: - halfway := len(ss) / 2 - return mergeTwoStringSlices( - mergeStringSlices(ss[:halfway]), - mergeStringSlices(ss[halfway:]), - ) - } -} - -func mergeTwoStringSlices(a, b []string) []string { - result := make([]string, 0, len(a)+len(b)) - i, j := 0, 0 - for i < len(a) && j < len(b) { - if a[i] < b[j] { - result = append(result, a[i]) - i++ - } else if a[i] > b[j] { - result = append(result, b[j]) - j++ - } else { - result = append(result, a[i]) - i++ - j++ - } - } - result = append(result, a[i:]...) - result = append(result, b[j:]...) - return result -} diff --git a/pkg/ingester/index/index_test.go b/pkg/ingester/index/index_test.go deleted file mode 100644 index d9183d950a2..00000000000 --- a/pkg/ingester/index/index_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package index - -import ( - "fmt" - "strconv" - "strings" - "testing" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/promql/parser" - "github.com/stretchr/testify/assert" - - "github.com/cortexproject/cortex/pkg/cortexpb" -) - -func TestIndex(t *testing.T) { - index := New() - - for _, entry := range []struct { - m model.Metric - fp model.Fingerprint - }{ - {model.Metric{"foo": "bar", "flip": "flop"}, 3}, - {model.Metric{"foo": "bar", "flip": "flap"}, 2}, - {model.Metric{"foo": "baz", "flip": "flop"}, 1}, - {model.Metric{"foo": "baz", "flip": "flap"}, 0}, - } { - index.Add(cortexpb.FromMetricsToLabelAdapters(entry.m), entry.fp) - } - - for _, tc := range []struct { - matchers []*labels.Matcher - fps []model.Fingerprint - }{ - {nil, nil}, - {mustParseMatcher(`{fizz="buzz"}`), []model.Fingerprint{}}, - - {mustParseMatcher(`{foo="bar"}`), []model.Fingerprint{2, 3}}, - {mustParseMatcher(`{foo="baz"}`), []model.Fingerprint{0, 1}}, - {mustParseMatcher(`{flip="flop"}`), []model.Fingerprint{1, 3}}, - {mustParseMatcher(`{flip="flap"}`), []model.Fingerprint{0, 2}}, - - {mustParseMatcher(`{foo="bar", flip="flop"}`), []model.Fingerprint{3}}, - {mustParseMatcher(`{foo="bar", flip="flap"}`), []model.Fingerprint{2}}, - {mustParseMatcher(`{foo="baz", flip="flop"}`), []model.Fingerprint{1}}, - {mustParseMatcher(`{foo="baz", flip="flap"}`), []model.Fingerprint{0}}, - - {mustParseMatcher(`{fizz=~"b.*"}`), []model.Fingerprint{}}, - - {mustParseMatcher(`{foo=~"bar.*"}`), []model.Fingerprint{2, 3}}, - {mustParseMatcher(`{foo=~"ba.*"}`), []model.Fingerprint{0, 1, 2, 3}}, - {mustParseMatcher(`{flip=~"flop|flap"}`), []model.Fingerprint{0, 1, 2, 3}}, - {mustParseMatcher(`{flip=~"flaps"}`), []model.Fingerprint{}}, - - {mustParseMatcher(`{foo=~"bar|bax", flip="flop"}`), []model.Fingerprint{3}}, - {mustParseMatcher(`{foo=~"bar|baz", flip="flap"}`), []model.Fingerprint{0, 2}}, - {mustParseMatcher(`{foo=~"baz.+", flip="flop"}`), []model.Fingerprint{}}, - {mustParseMatcher(`{foo=~"baz", flip="flap"}`), []model.Fingerprint{0}}, - } { - assert.Equal(t, tc.fps, index.Lookup(tc.matchers)) - } - - assert.Equal(t, []string{"flip", "foo"}, index.LabelNames()) - assert.Equal(t, []string{"bar", "baz"}, index.LabelValues("foo")) - assert.Equal(t, []string{"flap", "flop"}, index.LabelValues("flip")) -} - -func BenchmarkSetRegexLookup(b *testing.B) { - // Prepare the benchmark. - seriesLabels := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"} - seriesPerLabel := 100000 - - idx := New() - for _, l := range seriesLabels { - for i := 0; i < seriesPerLabel; i++ { - lbls := labels.FromStrings("foo", l, "bar", strconv.Itoa(i)) - idx.Add(cortexpb.FromLabelsToLabelAdapters(lbls), model.Fingerprint(lbls.Hash())) - } - } - - selectionLabels := []string{} - for i := 0; i < 100; i++ { - selectionLabels = append(selectionLabels, strconv.Itoa(i)) - } - - tests := []struct { - name string - matcher string - }{ - { - name: "select all", - matcher: fmt.Sprintf(`{bar=~"%s"}`, strings.Join(selectionLabels, "|")), - }, - { - name: "select two", - matcher: fmt.Sprintf(`{bar=~"%s"}`, strings.Join(selectionLabels[:2], "|")), - }, - { - name: "select half", - matcher: fmt.Sprintf(`{bar=~"%s"}`, strings.Join(selectionLabels[:len(selectionLabels)/2], "|")), - }, - { - name: "select none", - matcher: `{bar=~"bleep|bloop"}`, - }, - { - name: "equality matcher", - matcher: `{bar="1"}`, - }, - { - name: "regex (non-set) matcher", - matcher: `{bar=~"1.*"}`, - }, - } - - b.ResetTimer() - - for _, tc := range tests { - b.Run(fmt.Sprintf("%s:%s", tc.name, tc.matcher), func(b *testing.B) { - matcher := mustParseMatcher(tc.matcher) - for n := 0; n < b.N; n++ { - idx.Lookup(matcher) - } - }) - } - -} - -func mustParseMatcher(s string) []*labels.Matcher { - ms, err := parser.ParseMetricSelector(s) - if err != nil { - panic(err) - } - return ms -} - -func TestIndex_Delete(t *testing.T) { - index := New() - - testData := []struct { - m model.Metric - fp model.Fingerprint - }{ - {model.Metric{"common": "label", "foo": "bar", "flip": "flop"}, 0}, - {model.Metric{"common": "label", "foo": "bar", "flip": "flap"}, 1}, - {model.Metric{"common": "label", "foo": "baz", "flip": "flop"}, 2}, - {model.Metric{"common": "label", "foo": "baz", "flip": "flap"}, 3}, - } - for _, entry := range testData { - index.Add(cortexpb.FromMetricsToLabelAdapters(entry.m), entry.fp) - } - - for _, tc := range []struct { - name string - labelsToDelete labels.Labels - fpToDelete model.Fingerprint - expectedFPs []model.Fingerprint - }{ - { - name: "existing labels and fp", - labelsToDelete: metricToLabels(testData[0].m), - fpToDelete: testData[0].fp, - expectedFPs: []model.Fingerprint{1, 2, 3}, - }, - { - name: "non-existing labels", - labelsToDelete: metricToLabels(model.Metric{"app": "fizz"}), - fpToDelete: testData[1].fp, - expectedFPs: []model.Fingerprint{1, 2, 3}, - }, - { - name: "non-existing fp", - labelsToDelete: metricToLabels(testData[1].m), - fpToDelete: 99, - expectedFPs: []model.Fingerprint{1, 2, 3}, - }, - } { - t.Run(tc.name, func(t *testing.T) { - index.Delete(tc.labelsToDelete, tc.fpToDelete) - assert.Equal(t, tc.expectedFPs, index.Lookup(mustParseMatcher(`{common="label"}`))) - }) - } - - assert.Equal(t, []string{"common", "flip", "foo"}, index.LabelNames()) - assert.Equal(t, []string{"label"}, index.LabelValues("common")) - assert.Equal(t, []string{"bar", "baz"}, index.LabelValues("foo")) - assert.Equal(t, []string{"flap", "flop"}, index.LabelValues("flip")) -} - -func metricToLabels(m model.Metric) labels.Labels { - ls := make(labels.Labels, 0, len(m)) - for k, v := range m { - ls = append(ls, labels.Label{ - Name: string(k), - Value: string(v), - }) - } - - return ls -} diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index cf94cfe4803..8688cf93170 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -5,7 +5,6 @@ import ( "flag" "fmt" "net/http" - "os" "strings" "sync" "time" @@ -15,26 +14,17 @@ import ( "github.com/gogo/status" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/tsdb/chunks" - tsdb_record "github.com/prometheus/prometheus/tsdb/record" - "github.com/weaveworks/common/httpgrpc" "go.uber.org/atomic" - "golang.org/x/time/rate" "google.golang.org/grpc/codes" - cortex_chunk "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/tenant" - "github.com/cortexproject/cortex/pkg/util" logutil "github.com/cortexproject/cortex/pkg/util/log" util_math "github.com/cortexproject/cortex/pkg/util/math" "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/pkg/util/spanlogger" "github.com/cortexproject/cortex/pkg/util/validation" ) @@ -51,15 +41,12 @@ const ( ) var ( - // This is initialised if the WAL is enabled and the records are fetched from this pool. - recordPool sync.Pool - errIngesterStopping = errors.New("ingester stopping") ) // Config for an Ingester. type Config struct { - WALConfig WALConfig `yaml:"walconfig" doc:"description=Configures the Write-Ahead Log (WAL) for the Cortex chunks storage. This config is ignored when running the Cortex blocks storage."` + WALConfig WALConfig `yaml:"walconfig" doc:"description=Configures the Write-Ahead Log (WAL) for the removed Cortex chunks storage. This config is now always ignored."` LifecyclerConfig ring.LifecyclerConfig `yaml:"lifecycler"` // Config for transferring chunks. Zero or negative = no retries. @@ -165,43 +152,23 @@ func (cfg *Config) getIgnoreSeriesLimitForMetricNamesMap() map[string]struct{} { type Ingester struct { *services.BasicService - cfg Config - clientConfig client.Config + cfg Config metrics *ingesterMetrics logger log.Logger - chunkStore ChunkStore lifecycler *ring.Lifecycler limits *validation.Overrides limiter *Limiter subservicesWatcher *services.FailureWatcher - userStatesMtx sync.RWMutex // protects userStates and stopped - userStates *userStates - stopped bool // protected by userStatesMtx + stoppedMtx sync.RWMutex // protects stopped + stopped bool // protected by stoppedMtx // For storing metadata ingested. usersMetadataMtx sync.RWMutex usersMetadata map[string]*userMetricsMetadata - // One queue per flush thread. Fingerprint is used to - // pick a queue. - flushQueues []*util.PriorityQueue - flushQueuesDone sync.WaitGroup - - // Spread out calls to the chunk store over the flush period - flushRateLimiter *rate.Limiter - - // This should never be nil. - wal WAL - // To be passed to the WAL. - registerer prometheus.Registerer - - // Hooks for injecting behaviour from tests. - preFlushUserSeries func() - preFlushChunks func() - // Prometheus block storage TSDBState TSDBState @@ -210,237 +177,28 @@ type Ingester struct { inflightPushRequests atomic.Int64 } -// ChunkStore is the interface we need to store chunks -type ChunkStore interface { - Put(ctx context.Context, chunks []cortex_chunk.Chunk) error -} - // New constructs a new Ingester. -func New(cfg Config, clientConfig client.Config, limits *validation.Overrides, chunkStore ChunkStore, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { +func New(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { + if !cfg.BlocksStorageEnabled { + // TODO FIXME error message + return nil, fmt.Errorf("chunks storage is no longer supported") + } + defaultInstanceLimits = &cfg.DefaultLimits if cfg.ingesterClientFactory == nil { cfg.ingesterClientFactory = client.MakeIngesterClient } - if cfg.BlocksStorageEnabled { - return NewV2(cfg, clientConfig, limits, registerer, logger) - } - - if cfg.WALConfig.WALEnabled { - // If WAL is enabled, we don't transfer out the data to any ingester. - // Either the next ingester which takes it's place should recover from WAL - // or the data has to be flushed during scaledown. - cfg.MaxTransferRetries = 0 - - // Transfers are disabled with WAL, hence no need to wait for transfers. - cfg.LifecyclerConfig.JoinAfter = 0 - - recordPool = sync.Pool{ - New: func() interface{} { - return &WALRecord{} - }, - } - } - - if cfg.WALConfig.WALEnabled || cfg.WALConfig.Recover { - if err := os.MkdirAll(cfg.WALConfig.Dir, os.ModePerm); err != nil { - return nil, err - } - } - - i := &Ingester{ - cfg: cfg, - clientConfig: clientConfig, - - limits: limits, - chunkStore: chunkStore, - flushQueues: make([]*util.PriorityQueue, cfg.ConcurrentFlushes), - flushRateLimiter: rate.NewLimiter(rate.Inf, 1), - usersMetadata: map[string]*userMetricsMetadata{}, - registerer: registerer, - logger: logger, - } - i.metrics = newIngesterMetrics(registerer, true, cfg.ActiveSeriesMetricsEnabled, i.getInstanceLimits, nil, &i.inflightPushRequests) - - var err error - // During WAL recovery, it will create new user states which requires the limiter. - // Hence initialise the limiter before creating the WAL. - // The '!cfg.WALConfig.WALEnabled' argument says don't flush on shutdown if the WAL is enabled. - i.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, i, "ingester", RingKey, !cfg.WALConfig.WALEnabled || cfg.WALConfig.FlushOnShutdown, logger, prometheus.WrapRegistererWithPrefix("cortex_", registerer)) - if err != nil { - return nil, err - } - - i.limiter = NewLimiter( - limits, - i.lifecycler, - cfg.DistributorShardingStrategy, - cfg.DistributorShardByAllLabels, - cfg.LifecyclerConfig.RingConfig.ReplicationFactor, - cfg.LifecyclerConfig.RingConfig.ZoneAwarenessEnabled) - - i.subservicesWatcher = services.NewFailureWatcher() - i.subservicesWatcher.WatchService(i.lifecycler) - - i.BasicService = services.NewBasicService(i.starting, i.loop, i.stopping) - return i, nil -} - -func (i *Ingester) starting(ctx context.Context) error { - if i.cfg.WALConfig.Recover { - level.Info(i.logger).Log("msg", "recovering from WAL") - start := time.Now() - if err := recoverFromWAL(i); err != nil { - level.Error(i.logger).Log("msg", "failed to recover from WAL", "time", time.Since(start).String()) - return errors.Wrap(err, "failed to recover from WAL") - } - elapsed := time.Since(start) - level.Info(i.logger).Log("msg", "recovery from WAL completed", "time", elapsed.String()) - i.metrics.walReplayDuration.Set(elapsed.Seconds()) - } - - // If the WAL recover happened, then the userStates would already be set. - if i.userStates == nil { - i.userStates = newUserStates(i.limiter, i.cfg, i.metrics, i.logger) - } - - var err error - i.wal, err = newWAL(i.cfg.WALConfig, i.userStates.cp, i.registerer, i.logger) - if err != nil { - return errors.Wrap(err, "starting WAL") - } - - // Now that user states have been created, we can start the lifecycler. - // Important: we want to keep lifecycler running until we ask it to stop, so we need to give it independent context - if err := i.lifecycler.StartAsync(context.Background()); err != nil { - return errors.Wrap(err, "failed to start lifecycler") - } - if err := i.lifecycler.AwaitRunning(ctx); err != nil { - return errors.Wrap(err, "failed to start lifecycler") - } - - i.startFlushLoops() - - return nil -} - -func (i *Ingester) startFlushLoops() { - i.flushQueuesDone.Add(i.cfg.ConcurrentFlushes) - for j := 0; j < i.cfg.ConcurrentFlushes; j++ { - i.flushQueues[j] = util.NewPriorityQueue(i.metrics.flushQueueLength) - go i.flushLoop(j) - } + return NewV2(cfg, limits, registerer, logger) } // NewForFlusher constructs a new Ingester to be used by flusher target. // Compared to the 'New' method: // * Always replays the WAL. // * Does not start the lifecycler. -func NewForFlusher(cfg Config, chunkStore ChunkStore, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { - if cfg.BlocksStorageEnabled { - return NewV2ForFlusher(cfg, limits, registerer, logger) - } - - i := &Ingester{ - cfg: cfg, - chunkStore: chunkStore, - flushQueues: make([]*util.PriorityQueue, cfg.ConcurrentFlushes), - flushRateLimiter: rate.NewLimiter(rate.Inf, 1), - wal: &noopWAL{}, - limits: limits, - logger: logger, - } - i.metrics = newIngesterMetrics(registerer, true, false, i.getInstanceLimits, nil, &i.inflightPushRequests) - - i.BasicService = services.NewBasicService(i.startingForFlusher, i.loopForFlusher, i.stopping) - return i, nil -} - -func (i *Ingester) startingForFlusher(ctx context.Context) error { - level.Info(i.logger).Log("msg", "recovering from WAL") - - // We recover from WAL always. - start := time.Now() - if err := recoverFromWAL(i); err != nil { - level.Error(i.logger).Log("msg", "failed to recover from WAL", "time", time.Since(start).String()) - return err - } - elapsed := time.Since(start) - - level.Info(i.logger).Log("msg", "recovery from WAL completed", "time", elapsed.String()) - i.metrics.walReplayDuration.Set(elapsed.Seconds()) - - i.startFlushLoops() - return nil -} - -func (i *Ingester) loopForFlusher(ctx context.Context) error { - for { - select { - case <-ctx.Done(): - return nil - - case err := <-i.subservicesWatcher.Chan(): - return errors.Wrap(err, "ingester subservice failed") - } - } -} - -func (i *Ingester) loop(ctx context.Context) error { - flushTicker := time.NewTicker(i.cfg.FlushCheckPeriod) - defer flushTicker.Stop() - - rateUpdateTicker := time.NewTicker(i.cfg.RateUpdatePeriod) - defer rateUpdateTicker.Stop() - - metadataPurgeTicker := time.NewTicker(metadataPurgePeriod) - defer metadataPurgeTicker.Stop() - - var activeSeriesTickerChan <-chan time.Time - if i.cfg.ActiveSeriesMetricsEnabled { - t := time.NewTicker(i.cfg.ActiveSeriesMetricsUpdatePeriod) - activeSeriesTickerChan = t.C - defer t.Stop() - } - - for { - select { - case <-metadataPurgeTicker.C: - i.purgeUserMetricsMetadata() - - case <-flushTicker.C: - i.sweepUsers(false) - - case <-rateUpdateTicker.C: - i.userStates.updateRates() - - case <-activeSeriesTickerChan: - i.userStates.purgeAndUpdateActiveSeries(time.Now().Add(-i.cfg.ActiveSeriesMetricsIdleTimeout)) - - case <-ctx.Done(): - return nil - - case err := <-i.subservicesWatcher.Chan(): - return errors.Wrap(err, "ingester subservice failed") - } - } -} - -// stopping is run when ingester is asked to stop -func (i *Ingester) stopping(_ error) error { - i.wal.Stop() - - // This will prevent us accepting any more samples - i.stopIncomingRequests() - - // Lifecycler can be nil if the ingester is for a flusher. - if i.lifecycler != nil { - // Next initiate our graceful exit from the ring. - return services.StopAndAwaitTerminated(context.Background(), i.lifecycler) - } - - return nil +func NewForFlusher(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { + return NewV2ForFlusher(cfg, limits, registerer, logger) } // ShutdownHandler triggers the following set of operations in order: @@ -463,13 +221,6 @@ func (i *Ingester) ShutdownHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } -// stopIncomingRequests is called during the shutdown process. -func (i *Ingester) stopIncomingRequests() { - i.userStatesMtx.Lock() - defer i.userStatesMtx.Unlock() - i.stopped = true -} - // check that ingester has finished starting, i.e. it is in Running or Stopping state. // Why Stopping? Because ingester still runs, even when it is transferring data out in Stopping state. // Ingester handles this state on its own (via `stopped` flag). @@ -508,159 +259,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte } } - if i.cfg.BlocksStorageEnabled { - return i.v2Push(ctx, req) - } - - // NOTE: because we use `unsafe` in deserialisation, we must not - // retain anything from `req` past the call to ReuseSlice - defer cortexpb.ReuseSlice(req.Timeseries) - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - // Given metadata is a best-effort approach, and we don't halt on errors - // process it before samples. Otherwise, we risk returning an error before ingestion. - i.pushMetadata(ctx, userID, req.GetMetadata()) - - var firstPartialErr *validationError - var record *WALRecord - if i.cfg.WALConfig.WALEnabled { - record = recordPool.Get().(*WALRecord) - record.UserID = userID - // Assuming there is not much churn in most cases, there is no use - // keeping the record.Labels slice hanging around. - record.Series = nil - if cap(record.Samples) < len(req.Timeseries) { - record.Samples = make([]tsdb_record.RefSample, 0, len(req.Timeseries)) - } else { - record.Samples = record.Samples[:0] - } - } - - for _, ts := range req.Timeseries { - seriesSamplesIngested := 0 - for _, s := range ts.Samples { - // append() copies the memory in `ts.Labels` except on the error path - err := i.append(ctx, userID, ts.Labels, model.Time(s.TimestampMs), model.SampleValue(s.Value), req.Source, record) - if err == nil { - seriesSamplesIngested++ - continue - } - - i.metrics.ingestedSamplesFail.Inc() - if ve, ok := err.(*validationError); ok { - if firstPartialErr == nil { - firstPartialErr = ve - } - continue - } - - // non-validation error: abandon this request - return nil, grpcForwardableError(userID, http.StatusInternalServerError, err) - } - - if i.cfg.ActiveSeriesMetricsEnabled && seriesSamplesIngested > 0 { - // updateActiveSeries will copy labels if necessary. - i.updateActiveSeries(userID, time.Now(), ts.Labels) - } - } - - if record != nil { - // Log the record only if there was no error in ingestion. - if err := i.wal.Log(record); err != nil { - return nil, err - } - recordPool.Put(record) - } - - if firstPartialErr != nil { - // grpcForwardableError turns the error into a string so it no longer references `req` - return &cortexpb.WriteResponse{}, grpcForwardableError(userID, firstPartialErr.code, firstPartialErr) - } - - return &cortexpb.WriteResponse{}, nil -} - -// NOTE: memory for `labels` is unsafe; anything retained beyond the -// life of this function must be copied -func (i *Ingester) append(ctx context.Context, userID string, labels labelPairs, timestamp model.Time, value model.SampleValue, source cortexpb.WriteRequest_SourceEnum, record *WALRecord) error { - labels.removeBlanks() - - var ( - state *userState - fp model.Fingerprint - ) - i.userStatesMtx.RLock() - defer func() { - i.userStatesMtx.RUnlock() - if state != nil { - state.fpLocker.Unlock(fp) - } - }() - if i.stopped { - return errIngesterStopping - } - - // getOrCreateSeries copies the memory for `labels`, except on the error path. - state, fp, series, err := i.userStates.getOrCreateSeries(ctx, userID, labels, record) - if err != nil { - if ve, ok := err.(*validationError); ok { - state.discardedSamples.WithLabelValues(ve.errorType).Inc() - } - - // Reset the state so that the defer will not try to unlock the fpLocker - // in case of error, because that lock has already been released on error. - state = nil - return err - } - - prevNumChunks := len(series.chunkDescs) - if i.cfg.SpreadFlushes && prevNumChunks > 0 { - // Map from the fingerprint hash to a point in the cycle of period MaxChunkAge - startOfCycle := timestamp.Add(-(timestamp.Sub(model.Time(0)) % i.cfg.MaxChunkAge)) - slot := startOfCycle.Add(time.Duration(uint64(fp) % uint64(i.cfg.MaxChunkAge))) - // If adding this sample means the head chunk will span that point in time, close so it will get flushed - if series.head().FirstTime < slot && timestamp >= slot { - series.closeHead(reasonSpreadFlush) - } - } - - if err := series.add(model.SamplePair{ - Value: value, - Timestamp: timestamp, - }); err != nil { - if ve, ok := err.(*validationError); ok { - state.discardedSamples.WithLabelValues(ve.errorType).Inc() - if ve.noReport { - return nil - } - } - return err - } - - if record != nil { - record.Samples = append(record.Samples, tsdb_record.RefSample{ - Ref: chunks.HeadSeriesRef(fp), - T: int64(timestamp), - V: float64(value), - }) - } - - i.metrics.memoryChunks.Add(float64(len(series.chunkDescs) - prevNumChunks)) - i.metrics.ingestedSamples.Inc() - switch source { - case cortexpb.RULE: - state.ingestedRuleSamples.Inc() - case cortexpb.API: - fallthrough - default: - state.ingestedAPISamples.Inc() - } - - return err + return i.v2Push(ctx, req) } // pushMetadata returns number of ingested metadata. @@ -696,12 +295,12 @@ func (i *Ingester) pushMetadata(ctx context.Context, userID string, metadata []* } func (i *Ingester) appendMetadata(userID string, m *cortexpb.MetricMetadata) error { - i.userStatesMtx.RLock() + i.stoppedMtx.RLock() if i.stopped { - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() return errIngesterStopping } - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() userMetadata := i.getOrCreateUserMetadata(userID) @@ -773,269 +372,42 @@ func (i *Ingester) purgeUserMetricsMetadata() { // Query implements service.IngesterServer func (i *Ingester) Query(ctx context.Context, req *client.QueryRequest) (*client.QueryResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2Query(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - from, through, matchers, err := client.FromQueryRequest(req) - if err != nil { - return nil, err - } - - i.metrics.queries.Inc() - - i.userStatesMtx.RLock() - state, ok, err := i.userStates.getViaContext(ctx) - i.userStatesMtx.RUnlock() - if err != nil { - return nil, err - } else if !ok { - return &client.QueryResponse{}, nil - } - - result := &client.QueryResponse{} - numSeries, numSamples := 0, 0 - maxSamplesPerQuery := i.limits.MaxSamplesPerQuery(userID) - err = state.forSeriesMatching(ctx, matchers, func(ctx context.Context, _ model.Fingerprint, series *memorySeries) error { - values, err := series.samplesForRange(from, through) - if err != nil { - return err - } - if len(values) == 0 { - return nil - } - numSeries++ - - numSamples += len(values) - if numSamples > maxSamplesPerQuery { - return httpgrpc.Errorf(http.StatusRequestEntityTooLarge, "exceeded maximum number of samples in a query (%d)", maxSamplesPerQuery) - } - - ts := cortexpb.TimeSeries{ - Labels: cortexpb.FromLabelsToLabelAdapters(series.metric), - Samples: make([]cortexpb.Sample, 0, len(values)), - } - for _, s := range values { - ts.Samples = append(ts.Samples, cortexpb.Sample{ - Value: float64(s.Value), - TimestampMs: int64(s.Timestamp), - }) - } - result.Timeseries = append(result.Timeseries, ts) - return nil - }, nil, 0) - i.metrics.queriedSeries.Observe(float64(numSeries)) - i.metrics.queriedSamples.Observe(float64(numSamples)) - return result, err + return i.v2Query(ctx, req) } // QueryStream implements service.IngesterServer func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_QueryStreamServer) error { - if i.cfg.BlocksStorageEnabled { - return i.v2QueryStream(req, stream) - } - - if err := i.checkRunningOrStopping(); err != nil { - return err - } - - spanLog, ctx := spanlogger.New(stream.Context(), "QueryStream") - defer spanLog.Finish() - - from, through, matchers, err := client.FromQueryRequest(req) - if err != nil { - return err - } - - i.metrics.queries.Inc() - - i.userStatesMtx.RLock() - state, ok, err := i.userStates.getViaContext(ctx) - i.userStatesMtx.RUnlock() - if err != nil { - return err - } else if !ok { - return nil - } - - numSeries, numChunks := 0, 0 - reuseWireChunks := [queryStreamBatchSize][]client.Chunk{} - batch := make([]client.TimeSeriesChunk, 0, queryStreamBatchSize) - // We'd really like to have series in label order, not FP order, so we - // can iteratively merge them with entries coming from the chunk store. But - // that would involve locking all the series & sorting, so until we have - // a better solution in the ingesters I'd rather take the hit in the queriers. - err = state.forSeriesMatching(stream.Context(), matchers, func(ctx context.Context, _ model.Fingerprint, series *memorySeries) error { - chunks := make([]*desc, 0, len(series.chunkDescs)) - for _, chunk := range series.chunkDescs { - if !(chunk.FirstTime.After(through) || chunk.LastTime.Before(from)) { - chunks = append(chunks, chunk.slice(from, through)) - } - } - - if len(chunks) == 0 { - return nil - } - - numSeries++ - reusePos := len(batch) - wireChunks, err := toWireChunks(chunks, reuseWireChunks[reusePos]) - if err != nil { - return err - } - reuseWireChunks[reusePos] = wireChunks - - numChunks += len(wireChunks) - batch = append(batch, client.TimeSeriesChunk{ - Labels: cortexpb.FromLabelsToLabelAdapters(series.metric), - Chunks: wireChunks, - }) - - return nil - }, func(ctx context.Context) error { - if len(batch) == 0 { - return nil - } - err = client.SendQueryStream(stream, &client.QueryStreamResponse{ - Chunkseries: batch, - }) - batch = batch[:0] - return err - }, queryStreamBatchSize) - if err != nil { - return err - } - - i.metrics.queriedSeries.Observe(float64(numSeries)) - i.metrics.queriedChunks.Observe(float64(numChunks)) - level.Debug(spanLog).Log("streams", numSeries) - level.Debug(spanLog).Log("chunks", numChunks) - return err + return i.v2QueryStream(req, stream) } // Query implements service.IngesterServer func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQueryRequest) (*client.ExemplarQueryResponse, error) { - if !i.cfg.BlocksStorageEnabled { - return nil, errors.New("not supported") - } - return i.v2QueryExemplars(ctx, req) } // LabelValues returns all label values that are associated with a given label name. func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesRequest) (*client.LabelValuesResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2LabelValues(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - state, ok, err := i.userStates.getViaContext(ctx) - if err != nil { - return nil, err - } else if !ok { - return &client.LabelValuesResponse{}, nil - } - - resp := &client.LabelValuesResponse{} - resp.LabelValues = append(resp.LabelValues, state.index.LabelValues(req.LabelName)...) - - return resp, nil + return i.v2LabelValues(ctx, req) } // LabelNames return all the label names. func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest) (*client.LabelNamesResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2LabelNames(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - state, ok, err := i.userStates.getViaContext(ctx) - if err != nil { - return nil, err - } else if !ok { - return &client.LabelNamesResponse{}, nil - } - - resp := &client.LabelNamesResponse{} - resp.LabelNames = append(resp.LabelNames, state.index.LabelNames()...) - - return resp, nil + return i.v2LabelNames(ctx, req) } // MetricsForLabelMatchers returns all the metrics which match a set of matchers. func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.MetricsForLabelMatchersRequest) (*client.MetricsForLabelMatchersResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2MetricsForLabelMatchers(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - state, ok, err := i.userStates.getViaContext(ctx) - if err != nil { - return nil, err - } else if !ok { - return &client.MetricsForLabelMatchersResponse{}, nil - } - - // TODO Right now we ignore start and end. - _, _, matchersSet, err := client.FromMetricsForLabelMatchersRequest(req) - if err != nil { - return nil, err - } - - lss := map[model.Fingerprint]labels.Labels{} - for _, matchers := range matchersSet { - if err := state.forSeriesMatching(ctx, matchers, func(ctx context.Context, fp model.Fingerprint, series *memorySeries) error { - if _, ok := lss[fp]; !ok { - lss[fp] = series.metric - } - return nil - }, nil, 0); err != nil { - return nil, err - } - } - - result := &client.MetricsForLabelMatchersResponse{ - Metric: make([]*cortexpb.Metric, 0, len(lss)), - } - for _, ls := range lss { - result.Metric = append(result.Metric, &cortexpb.Metric{Labels: cortexpb.FromLabelsToLabelAdapters(ls)}) - } - - return result, nil + return i.v2MetricsForLabelMatchers(ctx, req) } // MetricsMetadata returns all the metric metadata of a user. func (i *Ingester) MetricsMetadata(ctx context.Context, req *client.MetricsMetadataRequest) (*client.MetricsMetadataResponse, error) { - i.userStatesMtx.RLock() + i.stoppedMtx.RLock() if err := i.checkRunningOrStopping(); err != nil { - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() return nil, err } - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() userID, err := tenant.TenantID(ctx) if err != nil { @@ -1053,64 +425,12 @@ func (i *Ingester) MetricsMetadata(ctx context.Context, req *client.MetricsMetad // UserStats returns ingestion statistics for the current user. func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) (*client.UserStatsResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2UserStats(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - state, ok, err := i.userStates.getViaContext(ctx) - if err != nil { - return nil, err - } else if !ok { - return &client.UserStatsResponse{}, nil - } - - apiRate := state.ingestedAPISamples.Rate() - ruleRate := state.ingestedRuleSamples.Rate() - return &client.UserStatsResponse{ - IngestionRate: apiRate + ruleRate, - ApiIngestionRate: apiRate, - RuleIngestionRate: ruleRate, - NumSeries: uint64(state.fpToSeries.length()), - }, nil + return i.v2UserStats(ctx, req) } // AllUserStats returns ingestion statistics for all users known to this ingester. func (i *Ingester) AllUserStats(ctx context.Context, req *client.UserStatsRequest) (*client.UsersStatsResponse, error) { - if i.cfg.BlocksStorageEnabled { - return i.v2AllUserStats(ctx, req) - } - - if err := i.checkRunningOrStopping(); err != nil { - return nil, err - } - - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - users := i.userStates.cp() - - response := &client.UsersStatsResponse{ - Stats: make([]*client.UserIDStatsResponse, 0, len(users)), - } - for userID, state := range users { - apiRate := state.ingestedAPISamples.Rate() - ruleRate := state.ingestedRuleSamples.Rate() - response.Stats = append(response.Stats, &client.UserIDStatsResponse{ - UserId: userID, - Data: &client.UserStatsResponse{ - IngestionRate: apiRate + ruleRate, - ApiIngestionRate: apiRate, - RuleIngestionRate: ruleRate, - NumSeries: uint64(state.fpToSeries.length()), - }, - }) - } - return response, nil + return i.v2AllUserStats(ctx, req) } // CheckReady is the readiness handler used to indicate to k8s when the ingesters @@ -1121,11 +441,3 @@ func (i *Ingester) CheckReady(ctx context.Context) error { } return i.lifecycler.CheckReady(ctx) } - -// labels will be copied if needed. -func (i *Ingester) updateActiveSeries(userID string, now time.Time, labels []cortexpb.LabelAdapter) { - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() - - i.userStates.updateActiveSeriesForUser(userID, now, cortexpb.FromLabelAdaptersToLabels(labels)) -} diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index ef676614f3e..0259d88b505 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -4,148 +4,29 @@ import ( "context" "fmt" "io/ioutil" - "math" - "math/rand" "net/http" "os" "path/filepath" "sort" - "strconv" - "strings" - "sync" "testing" "time" - "github.com/go-kit/log" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/cortexproject/cortex/pkg/chunk" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/weaveworks/common/httpgrpc" "github.com/weaveworks/common/user" - "google.golang.org/grpc" - "github.com/cortexproject/cortex/pkg/chunk" - promchunk "github.com/cortexproject/cortex/pkg/chunk/encoding" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" - "github.com/cortexproject/cortex/pkg/util/chunkcompat" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/test" - "github.com/cortexproject/cortex/pkg/util/validation" ) -type testStore struct { - mtx sync.Mutex - // Chunks keyed by userID. - chunks map[string][]chunk.Chunk -} - -func newTestStore(t require.TestingT, cfg Config, clientConfig client.Config, limits validation.Limits, reg prometheus.Registerer) (*testStore, *Ingester) { - store := &testStore{ - chunks: map[string][]chunk.Chunk{}, - } - overrides, err := validation.NewOverrides(limits, nil) - require.NoError(t, err) - - ing, err := New(cfg, clientConfig, overrides, store, reg, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) - - return store, ing -} - -func newDefaultTestStore(t testing.TB) (*testStore, *Ingester) { - t.Helper() - - return newTestStore(t, - defaultIngesterTestConfig(t), - defaultClientTestConfig(), - defaultLimitsTestConfig(), nil) -} - -func (s *testStore) Put(ctx context.Context, chunks []chunk.Chunk) error { - if len(chunks) == 0 { - return nil - } - s.mtx.Lock() - defer s.mtx.Unlock() - - for _, chunk := range chunks { - for _, v := range chunk.Metric { - if v.Value == "" { - return fmt.Errorf("Chunk has blank label %q", v.Name) - } - } - } - userID := chunks[0].UserID - s.chunks[userID] = append(s.chunks[userID], chunks...) - return nil -} - -func (s *testStore) Stop() {} - -// check that the store is holding data equivalent to what we expect -func (s *testStore) checkData(t *testing.T, userIDs []string, testData map[string]model.Matrix) { - s.mtx.Lock() - defer s.mtx.Unlock() - for _, userID := range userIDs { - res, err := chunk.ChunksToMatrix(context.Background(), s.chunks[userID], model.Time(0), model.Time(math.MaxInt64)) - require.NoError(t, err) - sort.Sort(res) - assert.Equal(t, testData[userID], res, "userID %s", userID) - } -} - -func buildTestMatrix(numSeries int, samplesPerSeries int, offset int) model.Matrix { - m := make(model.Matrix, 0, numSeries) - for i := 0; i < numSeries; i++ { - ss := model.SampleStream{ - Metric: model.Metric{ - model.MetricNameLabel: model.LabelValue(fmt.Sprintf("testmetric_%d", i)), - model.JobLabel: model.LabelValue(fmt.Sprintf("testjob%d", i%2)), - }, - Values: make([]model.SamplePair, 0, samplesPerSeries), - } - for j := 0; j < samplesPerSeries; j++ { - ss.Values = append(ss.Values, model.SamplePair{ - Timestamp: model.Time(i + j + offset), - Value: model.SampleValue(i + j + offset), - }) - } - m = append(m, &ss) - } - sort.Sort(m) - return m -} - -func matrixToSamples(m model.Matrix) []cortexpb.Sample { - var samples []cortexpb.Sample - for _, ss := range m { - for _, sp := range ss.Values { - samples = append(samples, cortexpb.Sample{ - TimestampMs: int64(sp.Timestamp), - Value: float64(sp.Value), - }) - } - } - return samples -} - -// Return one copy of the labels per sample -func matrixToLables(m model.Matrix) []labels.Labels { - var labels []labels.Labels - for _, ss := range m { - for range ss.Values { - labels = append(labels, cortexpb.FromLabelAdaptersToLabels(cortexpb.FromMetricsToLabelAdapters(ss.Metric))) - } - } - return labels -} - func runTestQuery(ctx context.Context, t *testing.T, ing *Ingester, ty labels.MatchType, n, v string) (model.Matrix, *client.QueryRequest, error) { return runTestQueryTimes(ctx, t, ing, ty, n, v, model.Earliest, model.Latest) } @@ -168,351 +49,6 @@ func runTestQueryTimes(ctx context.Context, t *testing.T, ing *Ingester, ty labe return res, req, nil } -func pushTestMetadata(t *testing.T, ing *Ingester, numMetadata, metadataPerMetric int) ([]string, map[string][]*cortexpb.MetricMetadata) { - userIDs := []string{"1", "2", "3"} - - // Create test metadata. - // Map of userIDs, to map of metric => metadataSet - testData := map[string][]*cortexpb.MetricMetadata{} - for _, userID := range userIDs { - metadata := make([]*cortexpb.MetricMetadata, 0, metadataPerMetric) - for i := 0; i < numMetadata; i++ { - metricName := fmt.Sprintf("testmetric_%d", i) - for j := 0; j < metadataPerMetric; j++ { - m := &cortexpb.MetricMetadata{MetricFamilyName: metricName, Help: fmt.Sprintf("a help for %d", j), Unit: "", Type: cortexpb.COUNTER} - metadata = append(metadata, m) - } - } - testData[userID] = metadata - } - - // Append metadata. - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(nil, nil, testData[userID], cortexpb.API)) - require.NoError(t, err) - } - - return userIDs, testData -} - -func pushTestSamples(t testing.TB, ing *Ingester, numSeries, samplesPerSeries, offset int) ([]string, map[string]model.Matrix) { - userIDs := []string{"1", "2", "3"} - - // Create test samples. - testData := map[string]model.Matrix{} - for i, userID := range userIDs { - testData[userID] = buildTestMatrix(numSeries, samplesPerSeries, i+offset) - } - - // Append samples. - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(matrixToLables(testData[userID]), matrixToSamples(testData[userID]), nil, cortexpb.API)) - require.NoError(t, err) - } - - return userIDs, testData -} - -func retrieveTestSamples(t *testing.T, ing *Ingester, userIDs []string, testData map[string]model.Matrix) { - // Read samples back via ingester queries. - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - res, req, err := runTestQuery(ctx, t, ing, labels.MatchRegexp, model.JobLabel, ".+") - require.NoError(t, err) - assert.Equal(t, testData[userID], res) - - s := stream{ - ctx: ctx, - } - err = ing.QueryStream(req, &s) - require.NoError(t, err) - - res, err = chunkcompat.StreamsToMatrix(model.Earliest, model.Latest, s.responses) - require.NoError(t, err) - assert.Equal(t, testData[userID].String(), res.String()) - } -} - -func TestIngesterAppend(t *testing.T) { - store, ing := newDefaultTestStore(t) - userIDs, testData := pushTestSamples(t, ing, 10, 1000, 0) - retrieveTestSamples(t, ing, userIDs, testData) - - // Read samples back via chunk store. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - store.checkData(t, userIDs, testData) -} - -func TestIngesterMetadataAppend(t *testing.T) { - for _, tc := range []struct { - desc string - numMetadata int - metadataPerMetric int - expectedMetrics int - expectedMetadata int - err error - }{ - {"with no metadata", 0, 0, 0, 0, nil}, - {"with one metadata per metric", 10, 1, 10, 10, nil}, - {"with multiple metadata per metric", 10, 3, 10, 30, nil}, - } { - t.Run(tc.desc, func(t *testing.T) { - limits := defaultLimitsTestConfig() - limits.MaxLocalMetadataPerMetric = 50 - _, ing := newTestStore(t, defaultIngesterTestConfig(t), defaultClientTestConfig(), limits, nil) - userIDs, _ := pushTestMetadata(t, ing, tc.numMetadata, tc.metadataPerMetric) - - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - resp, err := ing.MetricsMetadata(ctx, nil) - - if tc.err != nil { - require.Equal(t, tc.err, err) - } else { - require.NoError(t, err) - require.NotNil(t, resp) - - metricTracker := map[string]bool{} - for _, m := range resp.Metadata { - _, ok := metricTracker[m.GetMetricFamilyName()] - if !ok { - metricTracker[m.GetMetricFamilyName()] = true - } - } - - require.Equal(t, tc.expectedMetrics, len(metricTracker)) - require.Equal(t, tc.expectedMetadata, len(resp.Metadata)) - } - } - }) - } -} - -func TestIngesterPurgeMetadata(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.MetadataRetainPeriod = 20 * time.Millisecond - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - userIDs, _ := pushTestMetadata(t, ing, 10, 3) - - time.Sleep(40 * time.Millisecond) - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - ing.purgeUserMetricsMetadata() - - resp, err := ing.MetricsMetadata(ctx, nil) - require.NoError(t, err) - assert.Equal(t, 0, len(resp.GetMetadata())) - } -} - -func TestIngesterMetadataMetrics(t *testing.T) { - reg := prometheus.NewPedanticRegistry() - cfg := defaultIngesterTestConfig(t) - cfg.MetadataRetainPeriod = 20 * time.Millisecond - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), reg) - _, _ = pushTestMetadata(t, ing, 10, 3) - - pushTestMetadata(t, ing, 10, 3) - pushTestMetadata(t, ing, 10, 3) // We push the _exact_ same metrics again to ensure idempotency. Metadata is kept as a set so there shouldn't be a change of metrics. - - metricNames := []string{ - "cortex_ingester_memory_metadata_created_total", - "cortex_ingester_memory_metadata_removed_total", - "cortex_ingester_memory_metadata", - } - - assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingester_memory_metadata The current number of metadata in memory. - # TYPE cortex_ingester_memory_metadata gauge - cortex_ingester_memory_metadata 90 - # HELP cortex_ingester_memory_metadata_created_total The total number of metadata that were created per user - # TYPE cortex_ingester_memory_metadata_created_total counter - cortex_ingester_memory_metadata_created_total{user="1"} 30 - cortex_ingester_memory_metadata_created_total{user="2"} 30 - cortex_ingester_memory_metadata_created_total{user="3"} 30 - `), metricNames...)) - - time.Sleep(40 * time.Millisecond) - ing.purgeUserMetricsMetadata() - assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingester_memory_metadata The current number of metadata in memory. - # TYPE cortex_ingester_memory_metadata gauge - cortex_ingester_memory_metadata 0 - # HELP cortex_ingester_memory_metadata_created_total The total number of metadata that were created per user - # TYPE cortex_ingester_memory_metadata_created_total counter - cortex_ingester_memory_metadata_created_total{user="1"} 30 - cortex_ingester_memory_metadata_created_total{user="2"} 30 - cortex_ingester_memory_metadata_created_total{user="3"} 30 - # HELP cortex_ingester_memory_metadata_removed_total The total number of metadata that were removed per user. - # TYPE cortex_ingester_memory_metadata_removed_total counter - cortex_ingester_memory_metadata_removed_total{user="1"} 30 - cortex_ingester_memory_metadata_removed_total{user="2"} 30 - cortex_ingester_memory_metadata_removed_total{user="3"} 30 - `), metricNames...)) - -} - -func TestIngesterSendsOnlySeriesWithData(t *testing.T) { - _, ing := newDefaultTestStore(t) - - userIDs, _ := pushTestSamples(t, ing, 10, 1000, 0) - - // Read samples back via ingester queries. - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - _, req, err := runTestQueryTimes(ctx, t, ing, labels.MatchRegexp, model.JobLabel, ".+", model.Latest.Add(-15*time.Second), model.Latest) - require.NoError(t, err) - - s := stream{ - ctx: ctx, - } - err = ing.QueryStream(req, &s) - require.NoError(t, err) - - // Nothing should be selected. - require.Equal(t, 0, len(s.responses)) - } - - // Read samples back via chunk store. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) -} - -func TestIngesterIdleFlush(t *testing.T) { - // Create test ingester with short flush cycle - cfg := defaultIngesterTestConfig(t) - cfg.FlushCheckPeriod = 20 * time.Millisecond - cfg.MaxChunkIdle = 100 * time.Millisecond - cfg.RetainPeriod = 500 * time.Millisecond - store, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - userIDs, testData := pushTestSamples(t, ing, 4, 100, 0) - - // wait beyond idle time so samples flush - time.Sleep(cfg.MaxChunkIdle * 3) - - store.checkData(t, userIDs, testData) - - // Check data is still retained by ingester - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - res, _, err := runTestQuery(ctx, t, ing, labels.MatchRegexp, model.JobLabel, ".+") - require.NoError(t, err) - assert.Equal(t, testData[userID], res) - } - - // now wait beyond retain time so chunks are removed from memory - time.Sleep(cfg.RetainPeriod) - - // Check data has gone from ingester - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - res, _, err := runTestQuery(ctx, t, ing, labels.MatchRegexp, model.JobLabel, ".+") - require.NoError(t, err) - assert.Equal(t, model.Matrix{}, res) - } - - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) -} - -func TestIngesterSpreadFlush(t *testing.T) { - // Create test ingester with short flush cycle - cfg := defaultIngesterTestConfig(t) - cfg.SpreadFlushes = true - cfg.FlushCheckPeriod = 20 * time.Millisecond - store, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - userIDs, testData := pushTestSamples(t, ing, 4, 100, 0) - - // add another sample with timestamp at the end of the cycle to trigger - // head closes and get an extra chunk so we will flush the first one - _, _ = pushTestSamples(t, ing, 4, 1, int(cfg.MaxChunkAge.Seconds()-1)*1000) - - // wait beyond flush time so first set of samples should be sent to store - // (you'd think a shorter wait, like period*2, would work, but Go timers are not reliable enough for that) - time.Sleep(cfg.FlushCheckPeriod * 10) - - // check the first set of samples has been sent to the store - store.checkData(t, userIDs, testData) - - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) -} - -type stream struct { - grpc.ServerStream - ctx context.Context - responses []*client.QueryStreamResponse -} - -func (s *stream) Context() context.Context { - return s.ctx -} - -func (s *stream) Send(response *client.QueryStreamResponse) error { - s.responses = append(s.responses, response) - return nil -} - -func TestIngesterAppendOutOfOrderAndDuplicate(t *testing.T) { - _, ing := newDefaultTestStore(t) - defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck - - m := labelPairs{ - {Name: model.MetricNameLabel, Value: "testmetric"}, - } - ctx := context.Background() - err := ing.append(ctx, userID, m, 1, 0, cortexpb.API, nil) - require.NoError(t, err) - - // Two times exactly the same sample (noop). - err = ing.append(ctx, userID, m, 1, 0, cortexpb.API, nil) - require.NoError(t, err) - - // Earlier sample than previous one. - err = ing.append(ctx, userID, m, 0, 0, cortexpb.API, nil) - require.Contains(t, err.Error(), "sample timestamp out of order") - errResp, ok := err.(*validationError) - require.True(t, ok) - require.Equal(t, errResp.code, 400) - - // Same timestamp as previous sample, but different value. - err = ing.append(ctx, userID, m, 1, 1, cortexpb.API, nil) - require.Contains(t, err.Error(), "sample with repeated timestamp but different value") - errResp, ok = err.(*validationError) - require.True(t, ok) - require.Equal(t, errResp.code, 400) -} - -// Test that blank labels are removed by the ingester -func TestIngesterAppendBlankLabel(t *testing.T) { - _, ing := newDefaultTestStore(t) - defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck - - lp := labelPairs{ - {Name: model.MetricNameLabel, Value: "testmetric"}, - {Name: "foo", Value: ""}, - {Name: "bar", Value: ""}, - } - ctx := user.InjectOrgID(context.Background(), userID) - err := ing.append(ctx, userID, lp, 1, 0, cortexpb.API, nil) - require.NoError(t, err) - - res, _, err := runTestQuery(ctx, t, ing, labels.MatchEqual, labels.MetricName, "testmetric") - require.NoError(t, err) - - expected := model.Matrix{ - { - Metric: model.Metric{labels.MetricName: "testmetric"}, - Values: []model.SamplePair{ - {Timestamp: 1, Value: 0}, - }, - }, - } - - assert.Equal(t, expected, res) -} - func TestIngesterUserLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() limits.MaxLocalSeriesPerUser = 1 @@ -529,17 +65,6 @@ func TestIngesterUserLimitExceeded(t *testing.T) { require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - chunksIngesterGenerator := func() *Ingester { - cfg := defaultIngesterTestConfig(t) - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.Recover = true - cfg.WALConfig.Dir = chunksDir - cfg.WALConfig.CheckpointDuration = 100 * time.Minute - - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), limits, nil) - return ing - } - blocksIngesterGenerator := func() *Ingester { ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, blocksDir, nil) require.NoError(t, err) @@ -552,8 +77,8 @@ func TestIngesterUserLimitExceeded(t *testing.T) { return ing } - tests := []string{"chunks", "blocks"} - for i, ingGenerator := range []func() *Ingester{chunksIngesterGenerator, blocksIngesterGenerator} { + tests := []string{"blocks"} + for i, ingGenerator := range []func() *Ingester{blocksIngesterGenerator} { t.Run(tests[i], func(t *testing.T) { ing := ingGenerator() @@ -636,6 +161,20 @@ func TestIngesterUserLimitExceeded(t *testing.T) { } +func benchmarkData(nSeries int) (allLabels []labels.Labels, allSamples []cortexpb.Sample) { + for j := 0; j < nSeries; j++ { + labels := chunk.BenchmarkLabels.Copy() + for i := range labels { + if labels[i].Name == "cpu" { + labels[i].Value = fmt.Sprintf("cpu%02d", j) + } + } + allLabels = append(allLabels, labels) + allSamples = append(allSamples, cortexpb.Sample{TimestampMs: 0, Value: float64(j)}) + } + return +} + func TestIngesterMetricLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() limits.MaxLocalSeriesPerMetric = 1 @@ -652,17 +191,6 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - chunksIngesterGenerator := func() *Ingester { - cfg := defaultIngesterTestConfig(t) - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.Recover = true - cfg.WALConfig.Dir = chunksDir - cfg.WALConfig.CheckpointDuration = 100 * time.Minute - - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), limits, nil) - return ing - } - blocksIngesterGenerator := func() *Ingester { ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, blocksDir, nil) require.NoError(t, err) @@ -676,7 +204,7 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { } tests := []string{"chunks", "blocks"} - for i, ingGenerator := range []func() *Ingester{chunksIngesterGenerator, blocksIngesterGenerator} { + for i, ingGenerator := range []func() *Ingester{blocksIngesterGenerator} { t.Run(tests[i], func(t *testing.T) { ing := ingGenerator() @@ -758,300 +286,6 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { } } -func TestIngesterValidation(t *testing.T) { - _, ing := newDefaultTestStore(t) - defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck - userID := "1" - ctx := user.InjectOrgID(context.Background(), userID) - m := labelPairs{{Name: labels.MetricName, Value: "testmetric"}} - - // As a setup, let's append samples. - err := ing.append(context.Background(), userID, m, 1, 0, cortexpb.API, nil) - require.NoError(t, err) - - for _, tc := range []struct { - desc string - lbls []labels.Labels - samples []cortexpb.Sample - err error - }{ - { - desc: "With multiple append failures, only return the first error.", - lbls: []labels.Labels{ - {{Name: labels.MetricName, Value: "testmetric"}}, - {{Name: labels.MetricName, Value: "testmetric"}}, - }, - samples: []cortexpb.Sample{ - {TimestampMs: 0, Value: 0}, // earlier timestamp, out of order. - {TimestampMs: 1, Value: 2}, // same timestamp different value. - }, - err: httpgrpc.Errorf(http.StatusBadRequest, `user=1: sample timestamp out of order; last timestamp: 0.001, incoming timestamp: 0 for series {__name__="testmetric"}`), - }, - } { - t.Run(tc.desc, func(t *testing.T) { - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(tc.lbls, tc.samples, nil, cortexpb.API)) - require.Equal(t, tc.err, err) - }) - } -} - -func BenchmarkIngesterSeriesCreationLocking(b *testing.B) { - for i := 1; i <= 32; i++ { - b.Run(strconv.Itoa(i), func(b *testing.B) { - for n := 0; n < b.N; n++ { - benchmarkIngesterSeriesCreationLocking(b, i) - } - }) - } -} - -func benchmarkIngesterSeriesCreationLocking(b *testing.B, parallelism int) { - _, ing := newDefaultTestStore(b) - defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck - - var ( - wg sync.WaitGroup - series = int(1e4) - ctx = context.Background() - ) - wg.Add(parallelism) - ctx = user.InjectOrgID(ctx, "1") - for i := 0; i < parallelism; i++ { - seriesPerGoroutine := series / parallelism - go func(from, through int) { - defer wg.Done() - - for j := from; j < through; j++ { - _, err := ing.Push(ctx, &cortexpb.WriteRequest{ - Timeseries: []cortexpb.PreallocTimeseries{ - { - TimeSeries: &cortexpb.TimeSeries{ - Labels: []cortexpb.LabelAdapter{ - {Name: model.MetricNameLabel, Value: fmt.Sprintf("metric_%d", j)}, - }, - Samples: []cortexpb.Sample{ - {TimestampMs: int64(j), Value: float64(j)}, - }, - }, - }, - }, - }) - require.NoError(b, err) - } - - }(i*seriesPerGoroutine, (i+1)*seriesPerGoroutine) - } - - wg.Wait() -} - -func BenchmarkIngesterPush(b *testing.B) { - limits := defaultLimitsTestConfig() - benchmarkIngesterPush(b, limits, false) -} - -func BenchmarkIngesterPushErrors(b *testing.B) { - limits := defaultLimitsTestConfig() - limits.MaxLocalSeriesPerMetric = 1 - benchmarkIngesterPush(b, limits, true) -} - -// Construct a set of realistic-looking samples, all with slightly different label sets -func benchmarkData(nSeries int) (allLabels []labels.Labels, allSamples []cortexpb.Sample) { - for j := 0; j < nSeries; j++ { - labels := chunk.BenchmarkLabels.Copy() - for i := range labels { - if labels[i].Name == "cpu" { - labels[i].Value = fmt.Sprintf("cpu%02d", j) - } - } - allLabels = append(allLabels, labels) - allSamples = append(allSamples, cortexpb.Sample{TimestampMs: 0, Value: float64(j)}) - } - return -} - -func benchmarkIngesterPush(b *testing.B, limits validation.Limits, errorsExpected bool) { - cfg := defaultIngesterTestConfig(b) - clientCfg := defaultClientTestConfig() - - const ( - series = 100 - samples = 100 - ) - - allLabels, allSamples := benchmarkData(series) - ctx := user.InjectOrgID(context.Background(), "1") - - encodings := []struct { - name string - e promchunk.Encoding - }{ - {"DoubleDelta", promchunk.DoubleDelta}, - {"Varbit", promchunk.Varbit}, - {"Bigchunk", promchunk.Bigchunk}, - } - - for _, enc := range encodings { - b.Run(fmt.Sprintf("encoding=%s", enc.name), func(b *testing.B) { - b.ResetTimer() - for iter := 0; iter < b.N; iter++ { - _, ing := newTestStore(b, cfg, clientCfg, limits, nil) - // Bump the timestamp on each of our test samples each time round the loop - for j := 0; j < samples; j++ { - for i := range allSamples { - allSamples[i].TimestampMs = int64(j + 1) - } - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(allLabels, allSamples, nil, cortexpb.API)) - if !errorsExpected { - require.NoError(b, err) - } - } - _ = services.StopAndAwaitTerminated(context.Background(), ing) - } - }) - } - -} - -func BenchmarkIngester_QueryStream(b *testing.B) { - cfg := defaultIngesterTestConfig(b) - clientCfg := defaultClientTestConfig() - limits := defaultLimitsTestConfig() - _, ing := newTestStore(b, cfg, clientCfg, limits, nil) - ctx := user.InjectOrgID(context.Background(), "1") - - const ( - series = 2000 - samples = 1000 - ) - - allLabels, allSamples := benchmarkData(series) - - // Bump the timestamp and set a random value on each of our test samples each time round the loop - for j := 0; j < samples; j++ { - for i := range allSamples { - allSamples[i].TimestampMs = int64(j + 1) - allSamples[i].Value = rand.Float64() - } - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(allLabels, allSamples, nil, cortexpb.API)) - require.NoError(b, err) - } - - req := &client.QueryRequest{ - StartTimestampMs: 0, - EndTimestampMs: samples + 1, - - Matchers: []*client.LabelMatcher{{ - Type: client.EQUAL, - Name: model.MetricNameLabel, - Value: "container_cpu_usage_seconds_total", - }}, - } - - mockStream := &mockQueryStreamServer{ctx: ctx} - - b.ResetTimer() - - for ix := 0; ix < b.N; ix++ { - err := ing.QueryStream(req, mockStream) - require.NoError(b, err) - } -} - -func TestIngesterActiveSeries(t *testing.T) { - metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} - metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) - metricNames := []string{ - "cortex_ingester_active_series", - } - userID := "test" - - tests := map[string]struct { - reqs []*cortexpb.WriteRequest - expectedMetrics string - disableActiveSeries bool - }{ - "should succeed on valid series and metadata": { - reqs: []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, - nil, - cortexpb.API), - }, - expectedMetrics: ` - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 1 - `, - }, - "successful push, active series disabled": { - disableActiveSeries: true, - reqs: []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, - nil, - cortexpb.API), - }, - expectedMetrics: ``, - }, - } - - for testName, testData := range tests { - t.Run(testName, func(t *testing.T) { - registry := prometheus.NewRegistry() - - // Create a mocked ingester - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries - - _, i := newTestStore(t, - cfg, - defaultClientTestConfig(), - defaultLimitsTestConfig(), registry) - - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - ctx := user.InjectOrgID(context.Background(), userID) - - // Wait until the ingester is ACTIVE - test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push timeseries - for _, req := range testData.reqs { - _, err := i.Push(ctx, req) - assert.NoError(t, err) - } - - // Update active series for metrics check. - if !testData.disableActiveSeries { - i.userStatesMtx.Lock() - i.userStates.purgeAndUpdateActiveSeries(time.Now().Add(-i.cfg.ActiveSeriesMetricsIdleTimeout)) - i.userStatesMtx.Unlock() - } - - // Check tracked Prometheus metrics - err := testutil.GatherAndCompare(registry, strings.NewReader(testData.expectedMetrics), metricNames...) - assert.NoError(t, err) - }) - } -} - func TestGetIgnoreSeriesLimitForMetricNamesMap(t *testing.T) { cfg := Config{} diff --git a/pkg/ingester/ingester_v2.go b/pkg/ingester/ingester_v2.go index c7627b72b4c..cc1038bb741 100644 --- a/pkg/ingester/ingester_v2.go +++ b/pkg/ingester/ingester_v2.go @@ -481,7 +481,7 @@ func newTSDBState(bucketClient objstore.Bucket, registerer prometheus.Registerer } // NewV2 returns a new Ingester that uses Cortex block storage instead of chunks storage. -func NewV2(cfg Config, clientConfig client.Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { +func NewV2(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { bucketClient, err := bucket.NewClient(context.Background(), cfg.BlocksStorageConfig.Bucket, "ingester", logger, registerer) if err != nil { return nil, errors.Wrap(err, "failed to create the bucket client") @@ -489,11 +489,8 @@ func NewV2(cfg Config, clientConfig client.Config, limits *validation.Overrides, i := &Ingester{ cfg: cfg, - clientConfig: clientConfig, limits: limits, - chunkStore: nil, usersMetadata: map[string]*userMetricsMetadata{}, - wal: &noopWAL{}, TSDBState: newTSDBState(bucketClient, registerer), logger: logger, ingestionRate: util_math.NewEWMARate(0.2, instanceIngestionRateTickInterval), @@ -553,7 +550,6 @@ func NewV2ForFlusher(cfg Config, limits *validation.Overrides, registerer promet i := &Ingester{ cfg: cfg, limits: limits, - wal: &noopWAL{}, TSDBState: newTSDBState(bucketClient, registerer), logger: logger, } @@ -681,12 +677,12 @@ func (i *Ingester) updateLoop(ctx context.Context) error { case <-ingestionRateTicker.C: i.ingestionRate.Tick() case <-rateUpdateTicker.C: - i.userStatesMtx.RLock() + i.stoppedMtx.RLock() for _, db := range i.TSDBState.dbs { db.ingestedAPISamples.Tick() db.ingestedRuleSamples.Tick() } - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() case <-activeSeriesTickerChan: i.v2UpdateActiveSeries() @@ -745,12 +741,12 @@ func (i *Ingester) v2Push(ctx context.Context, req *cortexpb.WriteRequest) (*cor } // Ensure the ingester shutdown procedure hasn't started - i.userStatesMtx.RLock() + i.stoppedMtx.RLock() if i.stopped { - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() return nil, errIngesterStopping } - i.userStatesMtx.RUnlock() + i.stoppedMtx.RUnlock() if err := db.acquireAppendLock(); err != nil { return &cortexpb.WriteResponse{}, httpgrpc.Errorf(http.StatusServiceUnavailable, wrapWithUser(err, userID).Error()) @@ -1266,8 +1262,8 @@ func (i *Ingester) v2AllUserStats(ctx context.Context, req *client.UserStatsRequ return nil, err } - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() users := i.TSDBState.dbs @@ -1518,8 +1514,8 @@ func (i *Ingester) v2QueryStreamChunks(ctx context.Context, db *userTSDB, from, } func (i *Ingester) getTSDB(userID string) *userTSDB { - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() db := i.TSDBState.dbs[userID] return db } @@ -1527,8 +1523,8 @@ func (i *Ingester) getTSDB(userID string) *userTSDB { // List all users for which we have a TSDB. We do it here in order // to keep the mutex locked for the shortest time possible. func (i *Ingester) getTSDBUsers() []string { - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() ids := make([]string, 0, len(i.TSDBState.dbs)) for userID := range i.TSDBState.dbs { @@ -1544,8 +1540,8 @@ func (i *Ingester) getOrCreateTSDB(userID string, force bool) (*userTSDB, error) return db, nil } - i.userStatesMtx.Lock() - defer i.userStatesMtx.Unlock() + i.stoppedMtx.Lock() + defer i.stoppedMtx.Unlock() // Check again for DB in the event it was created in-between locks var ok bool @@ -1688,7 +1684,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { } func (i *Ingester) closeAllTSDB() { - i.userStatesMtx.Lock() + i.stoppedMtx.Lock() wg := &sync.WaitGroup{} wg.Add(len(i.TSDBState.dbs)) @@ -1709,9 +1705,9 @@ func (i *Ingester) closeAllTSDB() { // set of open ones. This lock acquisition doesn't deadlock with the // outer one, because the outer one is released as soon as all go // routines are started. - i.userStatesMtx.Lock() + i.stoppedMtx.Lock() delete(i.TSDBState.dbs, userID) - i.userStatesMtx.Unlock() + i.stoppedMtx.Unlock() i.metrics.memUsers.Dec() i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) @@ -1719,7 +1715,7 @@ func (i *Ingester) closeAllTSDB() { } // Wait until all Close() completed - i.userStatesMtx.Unlock() + i.stoppedMtx.Unlock() wg.Wait() } @@ -1744,9 +1740,9 @@ func (i *Ingester) openExistingTSDB(ctx context.Context) error { } // Add the database to the map of user databases - i.userStatesMtx.Lock() + i.stoppedMtx.Lock() i.TSDBState.dbs[userID] = db - i.userStatesMtx.Unlock() + i.stoppedMtx.Unlock() i.metrics.memUsers.Inc() i.TSDBState.walReplayTime.Observe(time.Since(startTime).Seconds()) @@ -1829,8 +1825,8 @@ func (i *Ingester) getMemorySeriesMetric() float64 { return 0 } - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() count := uint64(0) for _, db := range i.TSDBState.dbs { @@ -1843,8 +1839,8 @@ func (i *Ingester) getMemorySeriesMetric() float64 { // getOldestUnshippedBlockMetric returns the unix timestamp of the oldest unshipped block or // 0 if all blocks have been shipped. func (i *Ingester) getOldestUnshippedBlockMetric() float64 { - i.userStatesMtx.RLock() - defer i.userStatesMtx.RUnlock() + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() oldest := uint64(0) for _, db := range i.TSDBState.dbs { @@ -2101,9 +2097,9 @@ func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckRes // If this happens now, the request will get reject as the push will not be able to acquire the lock as the tsdb will be // in closed state defer func() { - i.userStatesMtx.Lock() + i.stoppedMtx.Lock() delete(i.TSDBState.dbs, userID) - i.userStatesMtx.Unlock() + i.stoppedMtx.Unlock() }() i.metrics.memUsers.Dec() diff --git a/pkg/ingester/ingester_v2_test.go b/pkg/ingester/ingester_v2_test.go index d9c25c1002c..a82591f5209 100644 --- a/pkg/ingester/ingester_v2_test.go +++ b/pkg/ingester/ingester_v2_test.go @@ -2187,8 +2187,6 @@ func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, require.NoError(t, os.RemoveAll(bucketDir)) }) - clientCfg := defaultClientTestConfig() - overrides, err := validation.NewOverrides(limits, nil) if err != nil { return nil, err @@ -2199,7 +2197,7 @@ func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, ingesterCfg.BlocksStorageConfig.Bucket.Backend = "filesystem" ingesterCfg.BlocksStorageConfig.Bucket.Filesystem.Directory = bucketDir - ingester, err := NewV2(ingesterCfg, clientCfg, overrides, registerer, log.NewNopLogger()) + ingester, err := NewV2(ingesterCfg, overrides, registerer, log.NewNopLogger()) if err != nil { return nil, err } @@ -2323,7 +2321,6 @@ func TestIngester_v2OpenExistingTSDBOnStartup(t *testing.T) { testName := name testData := test t.Run(testName, func(t *testing.T) { - clientCfg := defaultClientTestConfig() limits := defaultLimitsTestConfig() overrides, err := validation.NewOverrides(limits, nil) @@ -2344,7 +2341,7 @@ func TestIngester_v2OpenExistingTSDBOnStartup(t *testing.T) { // setup the tsdbs dir testData.setup(t, tempDir) - ingester, err := NewV2(ingesterCfg, clientCfg, overrides, nil, log.NewNopLogger()) + ingester, err := NewV2(ingesterCfg, overrides, nil, log.NewNopLogger()) require.NoError(t, err) startErr := services.StartAndAwaitRunning(context.Background(), ingester) @@ -3211,8 +3208,8 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { // Wait until TSDB has been closed and removed. test.Poll(t, 10*time.Second, 0, func() interface{} { - i.userStatesMtx.Lock() - defer i.userStatesMtx.Unlock() + i.stoppedMtx.Lock() + defer i.stoppedMtx.Unlock() return len(i.TSDBState.dbs) }) @@ -3340,7 +3337,6 @@ func TestHeadCompactionOnStartup(t *testing.T) { require.NoError(t, db.Close()) } - clientCfg := defaultClientTestConfig() limits := defaultLimitsTestConfig() overrides, err := validation.NewOverrides(limits, nil) @@ -3353,7 +3349,7 @@ func TestHeadCompactionOnStartup(t *testing.T) { ingesterCfg.BlocksStorageConfig.Bucket.S3.Endpoint = "localhost" ingesterCfg.BlocksStorageConfig.TSDB.Retention = 2 * 24 * time.Hour // Make sure that no newly created blocks are deleted. - ingester, err := NewV2(ingesterCfg, clientCfg, overrides, nil, log.NewNopLogger()) + ingester, err := NewV2(ingesterCfg, overrides, nil, log.NewNopLogger()) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ingester)) diff --git a/pkg/ingester/label_pairs.go b/pkg/ingester/label_pairs.go deleted file mode 100644 index bd0e8af632b..00000000000 --- a/pkg/ingester/label_pairs.go +++ /dev/null @@ -1,90 +0,0 @@ -package ingester - -import ( - "sort" - "strings" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/util/extract" -) - -// A series is uniquely identified by its set of label name/value -// pairs, which may arrive in any order over the wire -type labelPairs []cortexpb.LabelAdapter - -func (a labelPairs) String() string { - var b strings.Builder - - metricName, err := extract.MetricNameFromLabelAdapters(a) - numLabels := len(a) - 1 - if err != nil { - numLabels = len(a) - } - b.WriteString(metricName) - b.WriteByte('{') - count := 0 - for _, pair := range a { - if pair.Name != model.MetricNameLabel { - b.WriteString(pair.Name) - b.WriteString("=\"") - b.WriteString(pair.Value) - b.WriteByte('"') - count++ - if count < numLabels { - b.WriteByte(',') - } - } - } - b.WriteByte('}') - return b.String() -} - -// Remove any label where the value is "" - Prometheus 2+ will remove these -// before sending, but other clients such as Prometheus 1.x might send us blanks. -func (a *labelPairs) removeBlanks() { - for i := 0; i < len(*a); { - if len((*a)[i].Value) == 0 { - // Delete by swap with the value at the end of the slice - (*a)[i] = (*a)[len(*a)-1] - (*a) = (*a)[:len(*a)-1] - continue // go round and check the data that is now at position i - } - i++ - } -} - -func valueForName(s labels.Labels, name string) (string, bool) { - pos := sort.Search(len(s), func(i int) bool { return s[i].Name >= name }) - if pos == len(s) || s[pos].Name != name { - return "", false - } - return s[pos].Value, true -} - -// Check if a and b contain the same name/value pairs -func (a labelPairs) equal(b labels.Labels) bool { - if len(a) != len(b) { - return false - } - // Check as many as we can where the two sets are in the same order - i := 0 - for ; i < len(a); i++ { - if b[i].Name != string(a[i].Name) { - break - } - if b[i].Value != string(a[i].Value) { - return false - } - } - // Now check remaining values using binary search - for ; i < len(a); i++ { - v, found := valueForName(b, a[i].Name) - if !found || v != a[i].Value { - return false - } - } - return true -} diff --git a/pkg/ingester/label_pairs_test.go b/pkg/ingester/label_pairs_test.go deleted file mode 100644 index bb2a8641a79..00000000000 --- a/pkg/ingester/label_pairs_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package ingester - -import ( - "testing" - - "github.com/prometheus/prometheus/model/labels" -) - -func TestLabelPairsEqual(t *testing.T) { - for _, test := range []struct { - name string - a labelPairs - b labels.Labels - equal bool - }{ - { - name: "both blank", - a: labelPairs{}, - b: labels.Labels{}, - equal: true, - }, - { - name: "labelPairs nonblank; labels blank", - a: labelPairs{ - {Name: "foo", Value: "a"}, - }, - b: labels.Labels{}, - equal: false, - }, - { - name: "labelPairs blank; labels nonblank", - a: labelPairs{}, - b: labels.Labels{ - {Name: "foo", Value: "a"}, - }, - equal: false, - }, - { - name: "same contents; labelPairs not sorted", - a: labelPairs{ - {Name: "foo", Value: "a"}, - {Name: "bar", Value: "b"}, - }, - b: labels.Labels{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - }, - equal: true, - }, - { - name: "same contents", - a: labelPairs{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - }, - b: labels.Labels{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - }, - equal: true, - }, - { - name: "same names, different value", - a: labelPairs{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "c"}, - }, - b: labels.Labels{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - }, - equal: false, - }, - { - name: "labels has one extra value", - a: labelPairs{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - }, - b: labels.Labels{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - {Name: "firble", Value: "c"}, - }, - equal: false, - }, - { - name: "labelPairs has one extra value", - a: labelPairs{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - {Name: "firble", Value: "c"}, - }, - b: labels.Labels{ - {Name: "bar", Value: "b"}, - {Name: "foo", Value: "a"}, - {Name: "firble", Value: "a"}, - }, - equal: false, - }, - } { - if test.a.equal(test.b) != test.equal { - t.Errorf("%s: expected equal=%t", test.name, test.equal) - } - } -} diff --git a/pkg/ingester/lifecycle_test.go b/pkg/ingester/lifecycle_test.go index 5298fa75c79..a688cb77070 100644 --- a/pkg/ingester/lifecycle_test.go +++ b/pkg/ingester/lifecycle_test.go @@ -3,24 +3,17 @@ package ingester import ( "context" "fmt" - "io" - "math" "net/http" "net/http/httptest" "testing" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/go-kit/log" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/weaveworks/common/user" - "google.golang.org/grpc" - "google.golang.org/grpc/health/grpc_health_v1" - "github.com/cortexproject/cortex/pkg/chunk" - "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv" @@ -71,12 +64,12 @@ func defaultLimitsTestConfig() validation.Limits { // TestIngesterRestart tests a restarting ingester doesn't keep adding more tokens. func TestIngesterRestart(t *testing.T) { config := defaultIngesterTestConfig(t) - clientConfig := defaultClientTestConfig() - limits := defaultLimitsTestConfig() config.LifecyclerConfig.UnregisterOnShutdown = false { - _, ingester := newTestStore(t, config, clientConfig, limits, nil) + ingester, err := prepareIngesterWithBlocksStorage(t, config, prometheus.NewRegistry()) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ingester)) time.Sleep(100 * time.Millisecond) // Doesn't actually unregister due to UnregisterFromRing: false. require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ingester)) @@ -87,7 +80,9 @@ func TestIngesterRestart(t *testing.T) { }) { - _, ingester := newTestStore(t, config, clientConfig, limits, nil) + ingester, err := prepareIngesterWithBlocksStorage(t, config, prometheus.NewRegistry()) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ingester)) time.Sleep(100 * time.Millisecond) // Doesn't actually unregister due to UnregisterFromRing: false. require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ingester)) @@ -103,11 +98,12 @@ func TestIngesterRestart(t *testing.T) { func TestIngester_ShutdownHandler(t *testing.T) { for _, unregister := range []bool{false, true} { t.Run(fmt.Sprintf("unregister=%t", unregister), func(t *testing.T) { + registry := prometheus.NewRegistry() config := defaultIngesterTestConfig(t) - clientConfig := defaultClientTestConfig() - limits := defaultLimitsTestConfig() config.LifecyclerConfig.UnregisterOnShutdown = unregister - _, ingester := newTestStore(t, config, clientConfig, limits, nil) + ingester, err := prepareIngesterWithBlocksStorage(t, config, registry) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ingester)) // Make sure the ingester has been added to the ring. test.Poll(t, 100*time.Millisecond, 1, func() interface{} { @@ -126,243 +122,6 @@ func TestIngester_ShutdownHandler(t *testing.T) { } } -func TestIngesterChunksTransfer(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) - require.NoError(t, err) - - // Start the first ingester, and get it into ACTIVE state. - cfg1 := defaultIngesterTestConfig(t) - cfg1.LifecyclerConfig.ID = "ingester1" - cfg1.LifecyclerConfig.Addr = "ingester1" - cfg1.LifecyclerConfig.JoinAfter = 0 * time.Second - cfg1.MaxTransferRetries = 10 - ing1, err := New(cfg1, defaultClientTestConfig(), limits, nil, nil, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing1)) - - test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return ing1.lifecycler.GetState() - }) - - // Now write a sample to this ingester - req, expectedResponse, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "foo"}}, 456, 123000) - ctx := user.InjectOrgID(context.Background(), userID) - _, err = ing1.Push(ctx, req) - require.NoError(t, err) - - // Start a second ingester, but let it go into PENDING - cfg2 := defaultIngesterTestConfig(t) - cfg2.LifecyclerConfig.RingConfig.KVStore.Mock = cfg1.LifecyclerConfig.RingConfig.KVStore.Mock - cfg2.LifecyclerConfig.ID = "ingester2" - cfg2.LifecyclerConfig.Addr = "ingester2" - cfg2.LifecyclerConfig.JoinAfter = 100 * time.Second - ing2, err := New(cfg2, defaultClientTestConfig(), limits, nil, nil, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing2)) - - // Let ing2 send chunks to ing1 - ing1.cfg.ingesterClientFactory = func(addr string, _ client.Config) (client.HealthAndIngesterClient, error) { - return ingesterClientAdapater{ - ingester: ing2, - }, nil - } - - // Now stop the first ingester, and wait for the second ingester to become ACTIVE. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing1)) - - test.Poll(t, 10*time.Second, ring.ACTIVE, func() interface{} { - return ing2.lifecycler.GetState() - }) - - // And check the second ingester has the sample - matcher, err := labels.NewMatcher(labels.MatchEqual, model.MetricNameLabel, "foo") - require.NoError(t, err) - - request, err := client.ToQueryRequest(model.TimeFromUnix(0), model.TimeFromUnix(200), []*labels.Matcher{matcher}) - require.NoError(t, err) - - response, err := ing2.Query(ctx, request) - require.NoError(t, err) - assert.Equal(t, expectedResponse, response) - - // Check we can send the same sample again to the new ingester and get the same result - req, _, _, _ = mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "foo"}}, 456, 123000) - _, err = ing2.Push(ctx, req) - require.NoError(t, err) - response, err = ing2.Query(ctx, request) - require.NoError(t, err) - assert.Equal(t, expectedResponse, response) -} - -func TestIngesterBadTransfer(t *testing.T) { - limits, err := validation.NewOverrides(defaultLimitsTestConfig(), nil) - require.NoError(t, err) - - // Start ingester in PENDING. - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.ID = "ingester1" - cfg.LifecyclerConfig.Addr = "ingester1" - cfg.LifecyclerConfig.JoinAfter = 100 * time.Second - ing, err := New(cfg, defaultClientTestConfig(), limits, nil, nil, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) - - test.Poll(t, 100*time.Millisecond, ring.PENDING, func() interface{} { - return ing.lifecycler.GetState() - }) - - // Now transfer 0 series to this ingester, ensure it errors. - client := ingesterClientAdapater{ingester: ing} - stream, err := client.TransferChunks(context.Background()) - require.NoError(t, err) - _, err = stream.CloseAndRecv() - require.Error(t, err) - - // Check the ingester is still waiting. - require.Equal(t, ring.PENDING, ing.lifecycler.GetState()) -} - -type ingesterTransferChunkStreamMock struct { - ctx context.Context - reqs chan *client.TimeSeriesChunk - resp chan *client.TransferChunksResponse - err chan error - - grpc.ServerStream - grpc.ClientStream -} - -func (s *ingesterTransferChunkStreamMock) Send(tsc *client.TimeSeriesChunk) error { - s.reqs <- tsc - return nil -} - -func (s *ingesterTransferChunkStreamMock) CloseAndRecv() (*client.TransferChunksResponse, error) { - close(s.reqs) - select { - case resp := <-s.resp: - return resp, nil - case err := <-s.err: - return nil, err - } -} - -func (s *ingesterTransferChunkStreamMock) SendAndClose(resp *client.TransferChunksResponse) error { - s.resp <- resp - return nil -} - -func (s *ingesterTransferChunkStreamMock) ErrorAndClose(err error) { - s.err <- err -} - -func (s *ingesterTransferChunkStreamMock) Recv() (*client.TimeSeriesChunk, error) { - req, ok := <-s.reqs - if !ok { - return nil, io.EOF - } - return req, nil -} - -func (s *ingesterTransferChunkStreamMock) Context() context.Context { - return s.ctx -} - -func (*ingesterTransferChunkStreamMock) SendMsg(m interface{}) error { - return nil -} - -func (*ingesterTransferChunkStreamMock) RecvMsg(m interface{}) error { - return nil -} - -type ingesterClientAdapater struct { - client.IngesterClient - grpc_health_v1.HealthClient - ingester client.IngesterServer -} - -func (i ingesterClientAdapater) TransferChunks(ctx context.Context, _ ...grpc.CallOption) (client.Ingester_TransferChunksClient, error) { - stream := &ingesterTransferChunkStreamMock{ - ctx: ctx, - reqs: make(chan *client.TimeSeriesChunk), - resp: make(chan *client.TransferChunksResponse), - err: make(chan error), - } - go func() { - err := i.ingester.TransferChunks(stream) - if err != nil { - stream.ErrorAndClose(err) - } - }() - return stream, nil -} - -func (i ingesterClientAdapater) Close() error { - return nil -} - -func (i ingesterClientAdapater) Check(ctx context.Context, in *grpc_health_v1.HealthCheckRequest, opts ...grpc.CallOption) (*grpc_health_v1.HealthCheckResponse, error) { - return nil, nil -} - -// TestIngesterFlush tries to test that the ingester flushes chunks before -// removing itself from the ring. -func TestIngesterFlush(t *testing.T) { - // Start the ingester, and get it into ACTIVE state. - store, ing := newDefaultTestStore(t) - - test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return ing.lifecycler.GetState() - }) - - // Now write a sample to this ingester - var ( - lbls = []labels.Labels{{{Name: labels.MetricName, Value: "foo"}}} - sampleData = []cortexpb.Sample{ - { - TimestampMs: 123000, - Value: 456, - }, - } - ) - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ing.Push(ctx, cortexpb.ToWriteRequest(lbls, sampleData, nil, cortexpb.API)) - require.NoError(t, err) - - // We add a 100ms sleep into the flush loop, such that we can reliably detect - // if the ingester is removing its token from Consul before flushing chunks. - ing.preFlushUserSeries = func() { - time.Sleep(100 * time.Millisecond) - } - - // Now stop the ingester. Don't call shutdown, as it waits for all goroutines - // to exit. We just want to check that by the time the token is removed from - // the ring, the data is in the chunk store. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing.lifecycler)) - test.Poll(t, 200*time.Millisecond, 0, func() interface{} { - r, err := ing.lifecycler.KVStore.Get(context.Background(), RingKey) - if err != nil { - return -1 - } - return len(r.(*ring.Desc).Ingesters) - }) - - // And check the store has the chunk - res, err := chunk.ChunksToMatrix(context.Background(), store.chunks[userID], model.Time(0), model.Time(math.MaxInt64)) - require.NoError(t, err) - assert.Equal(t, model.Matrix{ - &model.SampleStream{ - Metric: model.Metric{ - model.MetricNameLabel: "foo", - }, - Values: []model.SamplePair{ - {Timestamp: model.TimeFromUnix(123), Value: model.SampleValue(456)}, - }, - }, - }, res) -} - // numTokens determines the number of tokens owned by the specified // address func numTokens(c kv.Client, name, ringKey string) int { diff --git a/pkg/ingester/locker.go b/pkg/ingester/locker.go deleted file mode 100644 index 3c97f38ba1b..00000000000 --- a/pkg/ingester/locker.go +++ /dev/null @@ -1,58 +0,0 @@ -package ingester - -import ( - "sync" - "unsafe" - - "github.com/prometheus/common/model" - - "github.com/cortexproject/cortex/pkg/util" -) - -const ( - cacheLineSize = 64 -) - -// Avoid false sharing when using array of mutexes. -type paddedMutex struct { - sync.Mutex - //nolint:structcheck,unused - pad [cacheLineSize - unsafe.Sizeof(sync.Mutex{})]byte -} - -// fingerprintLocker allows locking individual fingerprints. To limit the number -// of mutexes needed for that, only a fixed number of mutexes are -// allocated. Fingerprints to be locked are assigned to those pre-allocated -// mutexes by their value. Collisions are not detected. If two fingerprints get -// assigned to the same mutex, only one of them can be locked at the same -// time. As long as the number of pre-allocated mutexes is much larger than the -// number of goroutines requiring a fingerprint lock concurrently, the loss in -// efficiency is small. However, a goroutine must never lock more than one -// fingerprint at the same time. (In that case a collision would try to acquire -// the same mutex twice). -type fingerprintLocker struct { - fpMtxs []paddedMutex - numFpMtxs uint32 -} - -// newFingerprintLocker returns a new fingerprintLocker ready for use. At least -// 1024 preallocated mutexes are used, even if preallocatedMutexes is lower. -func newFingerprintLocker(preallocatedMutexes int) *fingerprintLocker { - if preallocatedMutexes < 1024 { - preallocatedMutexes = 1024 - } - return &fingerprintLocker{ - make([]paddedMutex, preallocatedMutexes), - uint32(preallocatedMutexes), - } -} - -// Lock locks the given fingerprint. -func (l *fingerprintLocker) Lock(fp model.Fingerprint) { - l.fpMtxs[util.HashFP(fp)%l.numFpMtxs].Lock() -} - -// Unlock unlocks the given fingerprint. -func (l *fingerprintLocker) Unlock(fp model.Fingerprint) { - l.fpMtxs[util.HashFP(fp)%l.numFpMtxs].Unlock() -} diff --git a/pkg/ingester/locker_test.go b/pkg/ingester/locker_test.go deleted file mode 100644 index a1503973935..00000000000 --- a/pkg/ingester/locker_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package ingester - -import ( - "sync" - "testing" - - "github.com/prometheus/common/model" -) - -func BenchmarkFingerprintLockerParallel(b *testing.B) { - numGoroutines := 10 - numFingerprints := 10 - numLockOps := b.N - locker := newFingerprintLocker(100) - - wg := sync.WaitGroup{} - b.ResetTimer() - for i := 0; i < numGoroutines; i++ { - wg.Add(1) - go func(i int) { - for j := 0; j < numLockOps; j++ { - fp1 := model.Fingerprint(j % numFingerprints) - fp2 := model.Fingerprint(j%numFingerprints + numFingerprints) - locker.Lock(fp1) - locker.Lock(fp2) - locker.Unlock(fp2) - locker.Unlock(fp1) - } - wg.Done() - }(i) - } - wg.Wait() -} - -func BenchmarkFingerprintLockerSerial(b *testing.B) { - numFingerprints := 10 - locker := newFingerprintLocker(100) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - fp := model.Fingerprint(i % numFingerprints) - locker.Lock(fp) - locker.Unlock(fp) - } -} diff --git a/pkg/ingester/mapper.go b/pkg/ingester/mapper.go deleted file mode 100644 index 835f6253abf..00000000000 --- a/pkg/ingester/mapper.go +++ /dev/null @@ -1,155 +0,0 @@ -package ingester - -import ( - "fmt" - "sort" - "strings" - "sync" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/prometheus/common/model" - "go.uber.org/atomic" -) - -const maxMappedFP = 1 << 20 // About 1M fingerprints reserved for mapping. - -var separatorString = string([]byte{model.SeparatorByte}) - -// fpMappings maps original fingerprints to a map of string representations of -// metrics to the truly unique fingerprint. -type fpMappings map[model.Fingerprint]map[string]model.Fingerprint - -// fpMapper is used to map fingerprints in order to work around fingerprint -// collisions. -type fpMapper struct { - highestMappedFP atomic.Uint64 - - mtx sync.RWMutex // Protects mappings. - mappings fpMappings - - fpToSeries *seriesMap - - logger log.Logger -} - -// newFPMapper loads the collision map from the persistence and -// returns an fpMapper ready to use. -func newFPMapper(fpToSeries *seriesMap, logger log.Logger) *fpMapper { - return &fpMapper{ - fpToSeries: fpToSeries, - mappings: map[model.Fingerprint]map[string]model.Fingerprint{}, - logger: logger, - } -} - -// mapFP takes a raw fingerprint (as returned by Metrics.FastFingerprint) and -// returns a truly unique fingerprint. The caller must have locked the raw -// fingerprint. -// -// If an error is encountered, it is returned together with the unchanged raw -// fingerprint. -func (m *fpMapper) mapFP(fp model.Fingerprint, metric labelPairs) model.Fingerprint { - // First check if we are in the reserved FP space, in which case this is - // automatically a collision that has to be mapped. - if fp <= maxMappedFP { - return m.maybeAddMapping(fp, metric) - } - - // Then check the most likely case: This fp belongs to a series that is - // already in memory. - s, ok := m.fpToSeries.get(fp) - if ok { - // FP exists in memory, but is it for the same metric? - if metric.equal(s.metric) { - // Yup. We are done. - return fp - } - // Collision detected! - return m.maybeAddMapping(fp, metric) - } - // Metric is not in memory. Before doing the expensive archive lookup, - // check if we have a mapping for this metric in place already. - m.mtx.RLock() - mappedFPs, fpAlreadyMapped := m.mappings[fp] - m.mtx.RUnlock() - if fpAlreadyMapped { - // We indeed have mapped fp historically. - ms := metricToUniqueString(metric) - // fp is locked by the caller, so no further locking of - // 'collisions' required (it is specific to fp). - mappedFP, ok := mappedFPs[ms] - if ok { - // Historical mapping found, return the mapped FP. - return mappedFP - } - } - return fp -} - -// maybeAddMapping is only used internally. It takes a detected collision and -// adds it to the collisions map if not yet there. In any case, it returns the -// truly unique fingerprint for the colliding metric. -func (m *fpMapper) maybeAddMapping( - fp model.Fingerprint, - collidingMetric labelPairs, -) model.Fingerprint { - ms := metricToUniqueString(collidingMetric) - m.mtx.RLock() - mappedFPs, ok := m.mappings[fp] - m.mtx.RUnlock() - if ok { - // fp is locked by the caller, so no further locking required. - mappedFP, ok := mappedFPs[ms] - if ok { - return mappedFP // Existing mapping. - } - // A new mapping has to be created. - mappedFP = m.nextMappedFP() - mappedFPs[ms] = mappedFP - level.Debug(m.logger).Log( - "msg", "fingerprint collision detected, mapping to new fingerprint", - "old_fp", fp, - "new_fp", mappedFP, - "metric", collidingMetric, - ) - return mappedFP - } - // This is the first collision for fp. - mappedFP := m.nextMappedFP() - mappedFPs = map[string]model.Fingerprint{ms: mappedFP} - m.mtx.Lock() - m.mappings[fp] = mappedFPs - m.mtx.Unlock() - level.Debug(m.logger).Log( - "msg", "fingerprint collision detected, mapping to new fingerprint", - "old_fp", fp, - "new_fp", mappedFP, - "metric", collidingMetric, - ) - return mappedFP -} - -func (m *fpMapper) nextMappedFP() model.Fingerprint { - mappedFP := model.Fingerprint(m.highestMappedFP.Inc()) - if mappedFP > maxMappedFP { - panic(fmt.Errorf("more than %v fingerprints mapped in collision detection", maxMappedFP)) - } - return mappedFP -} - -// metricToUniqueString turns a metric into a string in a reproducible and -// unique way, i.e. the same metric will always create the same string, and -// different metrics will always create different strings. In a way, it is the -// "ideal" fingerprint function, only that it is more expensive than the -// FastFingerprint function, and its result is not suitable as a key for maps -// and indexes as it might become really large, causing a lot of hashing effort -// in maps and a lot of storage overhead in indexes. -func metricToUniqueString(m labelPairs) string { - parts := make([]string, 0, len(m)) - for _, pair := range m { - parts = append(parts, string(pair.Name)+separatorString+string(pair.Value)) - } - sort.Strings(parts) - return strings.Join(parts, separatorString) -} diff --git a/pkg/ingester/mapper_test.go b/pkg/ingester/mapper_test.go deleted file mode 100644 index d18c6cb3c31..00000000000 --- a/pkg/ingester/mapper_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package ingester - -import ( - "sort" - "testing" - - "github.com/go-kit/log" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" -) - -var ( - // cm11, cm12, cm13 are colliding with fp1. - // cm21, cm22 are colliding with fp2. - // cm31, cm32 are colliding with fp3, which is below maxMappedFP. - // Note that fingerprints are set and not actually calculated. - // The collision detection is independent from the actually used - // fingerprinting algorithm. - fp1 = model.Fingerprint(maxMappedFP + 1) - fp2 = model.Fingerprint(maxMappedFP + 2) - fp3 = model.Fingerprint(1) - cm11 = labelPairs{ - {Name: "foo", Value: "bar"}, - {Name: "dings", Value: "bumms"}, - } - cm12 = labelPairs{ - {Name: "bar", Value: "foo"}, - } - cm13 = labelPairs{ - {Name: "foo", Value: "bar"}, - } - cm21 = labelPairs{ - {Name: "foo", Value: "bumms"}, - {Name: "dings", Value: "bar"}, - } - cm22 = labelPairs{ - {Name: "dings", Value: "foo"}, - {Name: "bar", Value: "bumms"}, - } - cm31 = labelPairs{ - {Name: "bumms", Value: "dings"}, - } - cm32 = labelPairs{ - {Name: "bumms", Value: "dings"}, - {Name: "bar", Value: "foo"}, - } -) - -func (a labelPairs) copyValuesAndSort() labels.Labels { - c := make(labels.Labels, len(a)) - for i, pair := range a { - c[i].Name = pair.Name - c[i].Value = pair.Value - } - sort.Sort(c) - return c -} - -func TestFPMapper(t *testing.T) { - sm := newSeriesMap() - - mapper := newFPMapper(sm, log.NewNopLogger()) - - // Everything is empty, resolving a FP should do nothing. - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), fp1) - - // cm11 is in sm. Adding cm11 should do nothing. Mapping cm12 should resolve - // the collision. - sm.put(fp1, &memorySeries{metric: cm11.copyValuesAndSort()}) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1)) - - // The mapped cm12 is added to sm, too. That should not change the outcome. - sm.put(model.Fingerprint(1), &memorySeries{metric: cm12.copyValuesAndSort()}) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1)) - - // Now map cm13, should reproducibly result in the next mapped FP. - assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2)) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2)) - - // Add cm13 to sm. Should not change anything. - sm.put(model.Fingerprint(2), &memorySeries{metric: cm13.copyValuesAndSort()}) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1)) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2)) - - // Now add cm21 and cm22 in the same way, checking the mapped FPs. - assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2) - sm.put(fp2, &memorySeries{metric: cm21.copyValuesAndSort()}) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3)) - sm.put(model.Fingerprint(3), &memorySeries{metric: cm22.copyValuesAndSort()}) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3)) - - // Map cm31, resulting in a mapping straight away. - assertFingerprintEqual(t, mapper.mapFP(fp3, cm31), model.Fingerprint(4)) - sm.put(model.Fingerprint(4), &memorySeries{metric: cm31.copyValuesAndSort()}) - - // Map cm32, which is now mapped for two reasons... - assertFingerprintEqual(t, mapper.mapFP(fp3, cm32), model.Fingerprint(5)) - sm.put(model.Fingerprint(5), &memorySeries{metric: cm32.copyValuesAndSort()}) - - // Now check ALL the mappings, just to be sure. - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1)) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2)) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3)) - assertFingerprintEqual(t, mapper.mapFP(fp3, cm31), model.Fingerprint(4)) - assertFingerprintEqual(t, mapper.mapFP(fp3, cm32), model.Fingerprint(5)) - - // Remove all the fingerprints from sm, which should change nothing, as - // the existing mappings stay and should be detected. - sm.del(fp1) - sm.del(fp2) - sm.del(fp3) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm11), fp1) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm12), model.Fingerprint(1)) - assertFingerprintEqual(t, mapper.mapFP(fp1, cm13), model.Fingerprint(2)) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm21), fp2) - assertFingerprintEqual(t, mapper.mapFP(fp2, cm22), model.Fingerprint(3)) - assertFingerprintEqual(t, mapper.mapFP(fp3, cm31), model.Fingerprint(4)) - assertFingerprintEqual(t, mapper.mapFP(fp3, cm32), model.Fingerprint(5)) -} - -// assertFingerprintEqual asserts that two fingerprints are equal. -func assertFingerprintEqual(t *testing.T, gotFP, wantFP model.Fingerprint) { - if gotFP != wantFP { - t.Errorf("got fingerprint %v, want fingerprint %v", gotFP, wantFP) - } -} diff --git a/pkg/ingester/series.go b/pkg/ingester/series.go index a5dfcacde47..727cd1ce931 100644 --- a/pkg/ingester/series.go +++ b/pkg/ingester/series.go @@ -1,260 +1,7 @@ package ingester -import ( - "fmt" - "sort" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/value" - - "github.com/cortexproject/cortex/pkg/chunk/encoding" - "github.com/cortexproject/cortex/pkg/prom1/storage/metric" -) - const ( sampleOutOfOrder = "sample-out-of-order" newValueForTimestamp = "new-value-for-timestamp" sampleOutOfBounds = "sample-out-of-bounds" - duplicateSample = "duplicate-sample" - duplicateTimestamp = "duplicate-timestamp" ) - -type memorySeries struct { - metric labels.Labels - - // Sorted by start time, overlapping chunk ranges are forbidden. - chunkDescs []*desc - - // Whether the current head chunk has already been finished. If true, - // the current head chunk must not be modified anymore. - headChunkClosed bool - - // The timestamp & value of the last sample in this series. Needed to - // ensure timestamp monotonicity during ingestion. - lastSampleValueSet bool - lastTime model.Time - lastSampleValue model.SampleValue - - // Prometheus metrics. - createdChunks prometheus.Counter -} - -// newMemorySeries returns a pointer to a newly allocated memorySeries for the -// given metric. -func newMemorySeries(m labels.Labels, createdChunks prometheus.Counter) *memorySeries { - return &memorySeries{ - metric: m, - lastTime: model.Earliest, - createdChunks: createdChunks, - } -} - -// add adds a sample pair to the series, possibly creating a new chunk. -// The caller must have locked the fingerprint of the series. -func (s *memorySeries) add(v model.SamplePair) error { - // If sender has repeated the same timestamp, check more closely and perhaps return error. - if v.Timestamp == s.lastTime { - // If we don't know what the last sample value is, silently discard. - // This will mask some errors but better than complaining when we don't really know. - if !s.lastSampleValueSet { - return makeNoReportError(duplicateTimestamp) - } - // If both timestamp and sample value are the same as for the last append, - // ignore as they are a common occurrence when using client-side timestamps - // (e.g. Pushgateway or federation). - if v.Value.Equal(s.lastSampleValue) { - return makeNoReportError(duplicateSample) - } - return makeMetricValidationError(newValueForTimestamp, s.metric, - fmt.Errorf("sample with repeated timestamp but different value; last value: %v, incoming value: %v", s.lastSampleValue, v.Value)) - } - if v.Timestamp < s.lastTime { - return makeMetricValidationError(sampleOutOfOrder, s.metric, - fmt.Errorf("sample timestamp out of order; last timestamp: %v, incoming timestamp: %v", s.lastTime, v.Timestamp)) - } - - if len(s.chunkDescs) == 0 || s.headChunkClosed { - newHead := newDesc(encoding.New(), v.Timestamp, v.Timestamp) - s.chunkDescs = append(s.chunkDescs, newHead) - s.headChunkClosed = false - s.createdChunks.Inc() - } - - newChunk, err := s.head().add(v) - if err != nil { - return err - } - - // If we get a single chunk result, then just replace the head chunk with it - // (no need to update first/last time). Otherwise, we'll need to update first - // and last time. - if newChunk != nil { - first, last, err := firstAndLastTimes(newChunk) - if err != nil { - return err - } - s.chunkDescs = append(s.chunkDescs, newDesc(newChunk, first, last)) - s.createdChunks.Inc() - } - - s.lastTime = v.Timestamp - s.lastSampleValue = v.Value - s.lastSampleValueSet = true - - return nil -} - -func firstAndLastTimes(c encoding.Chunk) (model.Time, model.Time, error) { - var ( - first model.Time - last model.Time - firstSet bool - iter = c.NewIterator(nil) - ) - for iter.Scan() { - sample := iter.Value() - if !firstSet { - first = sample.Timestamp - firstSet = true - } - last = sample.Timestamp - } - return first, last, iter.Err() -} - -// closeHead marks the head chunk closed. The caller must have locked -// the fingerprint of the memorySeries. This method will panic if this -// series has no chunk descriptors. -func (s *memorySeries) closeHead(reason flushReason) { - s.chunkDescs[0].flushReason = reason - s.headChunkClosed = true -} - -// firstTime returns the earliest known time for the series. The caller must have -// locked the fingerprint of the memorySeries. This method will panic if this -// series has no chunk descriptors. -func (s *memorySeries) firstTime() model.Time { - return s.chunkDescs[0].FirstTime -} - -// Returns time of oldest chunk in the series, that isn't flushed. If there are -// no chunks, or all chunks are flushed, returns 0. -// The caller must have locked the fingerprint of the memorySeries. -func (s *memorySeries) firstUnflushedChunkTime() model.Time { - for _, c := range s.chunkDescs { - if !c.flushed { - return c.FirstTime - } - } - - return 0 -} - -// head returns a pointer to the head chunk descriptor. The caller must have -// locked the fingerprint of the memorySeries. This method will panic if this -// series has no chunk descriptors. -func (s *memorySeries) head() *desc { - return s.chunkDescs[len(s.chunkDescs)-1] -} - -func (s *memorySeries) samplesForRange(from, through model.Time) ([]model.SamplePair, error) { - // Find first chunk with start time after "from". - fromIdx := sort.Search(len(s.chunkDescs), func(i int) bool { - return s.chunkDescs[i].FirstTime.After(from) - }) - // Find first chunk with start time after "through". - throughIdx := sort.Search(len(s.chunkDescs), func(i int) bool { - return s.chunkDescs[i].FirstTime.After(through) - }) - if fromIdx == len(s.chunkDescs) { - // Even the last chunk starts before "from". Find out if the - // series ends before "from" and we don't need to do anything. - lt := s.chunkDescs[len(s.chunkDescs)-1].LastTime - if lt.Before(from) { - return nil, nil - } - } - if fromIdx > 0 { - fromIdx-- - } - if throughIdx == len(s.chunkDescs) { - throughIdx-- - } - var values []model.SamplePair - in := metric.Interval{ - OldestInclusive: from, - NewestInclusive: through, - } - var reuseIter encoding.Iterator - for idx := fromIdx; idx <= throughIdx; idx++ { - cd := s.chunkDescs[idx] - reuseIter = cd.C.NewIterator(reuseIter) - chValues, err := encoding.RangeValues(reuseIter, in) - if err != nil { - return nil, err - } - values = append(values, chValues...) - } - return values, nil -} - -func (s *memorySeries) setChunks(descs []*desc) error { - if len(s.chunkDescs) != 0 { - return fmt.Errorf("series already has chunks") - } - - s.chunkDescs = descs - if len(descs) > 0 { - s.lastTime = descs[len(descs)-1].LastTime - } - return nil -} - -func (s *memorySeries) isStale() bool { - return s.lastSampleValueSet && value.IsStaleNaN(float64(s.lastSampleValue)) -} - -type desc struct { - C encoding.Chunk // nil if chunk is evicted. - FirstTime model.Time // Timestamp of first sample. Populated at creation. Immutable. - LastTime model.Time // Timestamp of last sample. Populated at creation & on append. - LastUpdate model.Time // This server's local time on last change - flushReason flushReason // If chunk is closed, holds the reason why. - flushed bool // set to true when flush succeeds -} - -func newDesc(c encoding.Chunk, firstTime model.Time, lastTime model.Time) *desc { - return &desc{ - C: c, - FirstTime: firstTime, - LastTime: lastTime, - LastUpdate: model.Now(), - } -} - -// Add adds a sample pair to the underlying chunk. For safe concurrent access, -// The chunk must be pinned, and the caller must have locked the fingerprint of -// the series. -func (d *desc) add(s model.SamplePair) (encoding.Chunk, error) { - cs, err := d.C.Add(s) - if err != nil { - return nil, err - } - - if cs == nil { - d.LastTime = s.Timestamp // sample was added to this chunk - d.LastUpdate = model.Now() - } - - return cs, nil -} - -func (d *desc) slice(start, end model.Time) *desc { - return &desc{ - C: d.C.Slice(start, end), - FirstTime: start, - LastTime: end, - } -} diff --git a/pkg/ingester/series_map.go b/pkg/ingester/series_map.go deleted file mode 100644 index 4d4a9a5b669..00000000000 --- a/pkg/ingester/series_map.go +++ /dev/null @@ -1,110 +0,0 @@ -package ingester - -import ( - "sync" - "unsafe" - - "github.com/prometheus/common/model" - "go.uber.org/atomic" - - "github.com/cortexproject/cortex/pkg/util" -) - -const seriesMapShards = 128 - -// seriesMap maps fingerprints to memory series. All its methods are -// goroutine-safe. A seriesMap is effectively a goroutine-safe version of -// map[model.Fingerprint]*memorySeries. -type seriesMap struct { - size atomic.Int32 - shards []shard -} - -type shard struct { - mtx sync.Mutex - m map[model.Fingerprint]*memorySeries - - // Align this struct. - _ [cacheLineSize - unsafe.Sizeof(sync.Mutex{}) - unsafe.Sizeof(map[model.Fingerprint]*memorySeries{})]byte -} - -// fingerprintSeriesPair pairs a fingerprint with a memorySeries pointer. -type fingerprintSeriesPair struct { - fp model.Fingerprint - series *memorySeries -} - -// newSeriesMap returns a newly allocated empty seriesMap. To create a seriesMap -// based on a prefilled map, use an explicit initializer. -func newSeriesMap() *seriesMap { - shards := make([]shard, seriesMapShards) - for i := 0; i < seriesMapShards; i++ { - shards[i].m = map[model.Fingerprint]*memorySeries{} - } - return &seriesMap{ - shards: shards, - } -} - -// get returns a memorySeries for a fingerprint. Return values have the same -// semantics as the native Go map. -func (sm *seriesMap) get(fp model.Fingerprint) (*memorySeries, bool) { - shard := &sm.shards[util.HashFP(fp)%seriesMapShards] - shard.mtx.Lock() - ms, ok := shard.m[fp] - shard.mtx.Unlock() - return ms, ok -} - -// put adds a mapping to the seriesMap. -func (sm *seriesMap) put(fp model.Fingerprint, s *memorySeries) { - shard := &sm.shards[util.HashFP(fp)%seriesMapShards] - shard.mtx.Lock() - _, ok := shard.m[fp] - shard.m[fp] = s - shard.mtx.Unlock() - - if !ok { - sm.size.Inc() - } -} - -// del removes a mapping from the series Map. -func (sm *seriesMap) del(fp model.Fingerprint) { - shard := &sm.shards[util.HashFP(fp)%seriesMapShards] - shard.mtx.Lock() - _, ok := shard.m[fp] - delete(shard.m, fp) - shard.mtx.Unlock() - if ok { - sm.size.Dec() - } -} - -// iter returns a channel that produces all mappings in the seriesMap. The -// channel will be closed once all fingerprints have been received. Not -// consuming all fingerprints from the channel will leak a goroutine. The -// semantics of concurrent modification of seriesMap is the similar as the one -// for iterating over a map with a 'range' clause. However, if the next element -// in iteration order is removed after the current element has been received -// from the channel, it will still be produced by the channel. -func (sm *seriesMap) iter() <-chan fingerprintSeriesPair { - ch := make(chan fingerprintSeriesPair) - go func() { - for i := range sm.shards { - sm.shards[i].mtx.Lock() - for fp, ms := range sm.shards[i].m { - sm.shards[i].mtx.Unlock() - ch <- fingerprintSeriesPair{fp, ms} - sm.shards[i].mtx.Lock() - } - sm.shards[i].mtx.Unlock() - } - close(ch) - }() - return ch -} - -func (sm *seriesMap) length() int { - return int(sm.size.Load()) -} diff --git a/pkg/ingester/transfer.go b/pkg/ingester/transfer.go index ce31e9f42a6..e20f91d847a 100644 --- a/pkg/ingester/transfer.go +++ b/pkg/ingester/transfer.go @@ -1,390 +1,17 @@ package ingester import ( - "bytes" "context" - "fmt" - "io" - "os" - "time" "github.com/go-kit/log/level" - "github.com/pkg/errors" - "github.com/prometheus/common/model" - "github.com/weaveworks/common/user" - "github.com/cortexproject/cortex/pkg/chunk/encoding" - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" - "github.com/cortexproject/cortex/pkg/util/backoff" ) -var ( - errTransferNoPendingIngesters = errors.New("no pending ingesters") -) - -// returns source ingesterID, number of received series, added chunks and error -func (i *Ingester) fillUserStatesFromStream(userStates *userStates, stream client.Ingester_TransferChunksServer) (fromIngesterID string, seriesReceived int, retErr error) { - chunksAdded := 0.0 - - defer func() { - if retErr != nil { - // Ensure the in memory chunks are updated to reflect the number of dropped chunks from the transfer - i.metrics.memoryChunks.Sub(chunksAdded) - - // If an error occurs during the transfer and the user state is to be discarded, - // ensure the metrics it exports reflect this. - userStates.teardown() - } - }() - - for { - wireSeries, err := stream.Recv() - if err == io.EOF { - break - } - if err != nil { - retErr = errors.Wrap(err, "TransferChunks: Recv") - return - } - - // We can't send "extra" fields with a streaming call, so we repeat - // wireSeries.FromIngesterId and assume it is the same every time - // round this loop. - if fromIngesterID == "" { - fromIngesterID = wireSeries.FromIngesterId - level.Info(i.logger).Log("msg", "processing TransferChunks request", "from_ingester", fromIngesterID) - - // Before transfer, make sure 'from' ingester is in correct state to call ClaimTokensFor later - err := i.checkFromIngesterIsInLeavingState(stream.Context(), fromIngesterID) - if err != nil { - retErr = errors.Wrap(err, "TransferChunks: checkFromIngesterIsInLeavingState") - return - } - } - descs, err := fromWireChunks(wireSeries.Chunks) - if err != nil { - retErr = errors.Wrap(err, "TransferChunks: fromWireChunks") - return - } - - state, fp, series, err := userStates.getOrCreateSeries(stream.Context(), wireSeries.UserId, wireSeries.Labels, nil) - if err != nil { - retErr = errors.Wrapf(err, "TransferChunks: getOrCreateSeries: user %s series %s", wireSeries.UserId, wireSeries.Labels) - return - } - prevNumChunks := len(series.chunkDescs) - - err = series.setChunks(descs) - state.fpLocker.Unlock(fp) // acquired in getOrCreateSeries - if err != nil { - retErr = errors.Wrapf(err, "TransferChunks: setChunks: user %s series %s", wireSeries.UserId, wireSeries.Labels) - return - } - - seriesReceived++ - chunksDelta := float64(len(series.chunkDescs) - prevNumChunks) - chunksAdded += chunksDelta - i.metrics.memoryChunks.Add(chunksDelta) - i.metrics.receivedChunks.Add(float64(len(descs))) - } - - if seriesReceived == 0 { - level.Error(i.logger).Log("msg", "received TransferChunks request with no series", "from_ingester", fromIngesterID) - retErr = fmt.Errorf("TransferChunks: no series") - return - } - - if fromIngesterID == "" { - level.Error(i.logger).Log("msg", "received TransferChunks request with no ID from ingester") - retErr = fmt.Errorf("no ingester id") - return - } - - if err := i.lifecycler.ClaimTokensFor(stream.Context(), fromIngesterID); err != nil { - retErr = errors.Wrap(err, "TransferChunks: ClaimTokensFor") - return - } - - return -} - -// TransferChunks receives all the chunks from another ingester. -func (i *Ingester) TransferChunks(stream client.Ingester_TransferChunksServer) error { - fromIngesterID := "" - seriesReceived := 0 - - xfer := func() error { - userStates := newUserStates(i.limiter, i.cfg, i.metrics, i.logger) - - var err error - fromIngesterID, seriesReceived, err = i.fillUserStatesFromStream(userStates, stream) - - if err != nil { - return err - } - - i.userStatesMtx.Lock() - defer i.userStatesMtx.Unlock() - - i.userStates = userStates - - return nil - } - - if err := i.transfer(stream.Context(), xfer); err != nil { - return err - } - - // Close the stream last, as this is what tells the "from" ingester that - // it's OK to shut down. - if err := stream.SendAndClose(&client.TransferChunksResponse{}); err != nil { - level.Error(i.logger).Log("msg", "Error closing TransferChunks stream", "from_ingester", fromIngesterID, "err", err) - return err - } - level.Info(i.logger).Log("msg", "Successfully transferred chunks", "from_ingester", fromIngesterID, "series_received", seriesReceived) - - return nil -} - -// Ring gossiping: check if "from" ingester is in LEAVING state. It should be, but we may not see that yet -// when using gossip ring. If we cannot see ingester is the LEAVING state yet, we don't accept this -// transfer, as claiming tokens would possibly end up with this ingester owning no tokens, due to conflict -// resolution in ring merge function. Hopefully the leaving ingester will retry transfer again. -func (i *Ingester) checkFromIngesterIsInLeavingState(ctx context.Context, fromIngesterID string) error { - v, err := i.lifecycler.KVStore.Get(ctx, i.lifecycler.RingKey) - if err != nil { - return errors.Wrap(err, "get ring") - } - if v == nil { - return fmt.Errorf("ring not found when checking state of source ingester") - } - r, ok := v.(*ring.Desc) - if !ok || r == nil { - return fmt.Errorf("ring not found, got %T", v) - } - - if r.Ingesters == nil || r.Ingesters[fromIngesterID].State != ring.LEAVING { - return fmt.Errorf("source ingester is not in a LEAVING state, found state=%v", r.Ingesters[fromIngesterID].State) - } - - // all fine - return nil -} - -func (i *Ingester) transfer(ctx context.Context, xfer func() error) error { - // Enter JOINING state (only valid from PENDING) - if err := i.lifecycler.ChangeState(ctx, ring.JOINING); err != nil { - return err - } - - // The ingesters state effectively works as a giant mutex around this whole - // method, and as such we have to ensure we unlock the mutex. - defer func() { - state := i.lifecycler.GetState() - if i.lifecycler.GetState() == ring.ACTIVE { - return - } - - level.Error(i.logger).Log("msg", "TransferChunks failed, not in ACTIVE state.", "state", state) - - // Enter PENDING state (only valid from JOINING) - if i.lifecycler.GetState() == ring.JOINING { - if err := i.lifecycler.ChangeState(ctx, ring.PENDING); err != nil { - level.Error(i.logger).Log("msg", "error rolling back failed TransferChunks", "err", err) - os.Exit(1) - } - } - }() - - if err := xfer(); err != nil { - return err - } - - if err := i.lifecycler.ChangeState(ctx, ring.ACTIVE); err != nil { - return errors.Wrap(err, "Transfer: ChangeState") - } - - return nil -} - -// The passed wireChunks slice is for re-use. -func toWireChunks(descs []*desc, wireChunks []client.Chunk) ([]client.Chunk, error) { - if cap(wireChunks) < len(descs) { - wireChunks = make([]client.Chunk, len(descs)) - } else { - wireChunks = wireChunks[:len(descs)] - } - for i, d := range descs { - wireChunk := client.Chunk{ - StartTimestampMs: int64(d.FirstTime), - EndTimestampMs: int64(d.LastTime), - Encoding: int32(d.C.Encoding()), - } - - slice := wireChunks[i].Data[:0] // try to re-use the memory from last time - if cap(slice) < d.C.Size() { - slice = make([]byte, 0, d.C.Size()) - } - buf := bytes.NewBuffer(slice) - - if err := d.C.Marshal(buf); err != nil { - return nil, err - } - - wireChunk.Data = buf.Bytes() - wireChunks[i] = wireChunk - } - return wireChunks, nil -} - -func fromWireChunks(wireChunks []client.Chunk) ([]*desc, error) { - descs := make([]*desc, 0, len(wireChunks)) - for _, c := range wireChunks { - desc := &desc{ - FirstTime: model.Time(c.StartTimestampMs), - LastTime: model.Time(c.EndTimestampMs), - LastUpdate: model.Now(), - } - - var err error - desc.C, err = encoding.NewForEncoding(encoding.Encoding(byte(c.Encoding))) - if err != nil { - return nil, err - } - - if err := desc.C.UnmarshalFromBuf(c.Data); err != nil { - return nil, err - } - - descs = append(descs, desc) - } - return descs, nil -} - // TransferOut finds an ingester in PENDING state and transfers our chunks to it. // Called as part of the ingester shutdown process. func (i *Ingester) TransferOut(ctx context.Context) error { // The blocks storage doesn't support blocks transferring. - if i.cfg.BlocksStorageEnabled { - level.Info(i.logger).Log("msg", "transfer between a LEAVING ingester and a PENDING one is not supported for the blocks storage") - return ring.ErrTransferDisabled - } - - if i.cfg.MaxTransferRetries <= 0 { - return ring.ErrTransferDisabled - } - backoff := backoff.New(ctx, backoff.Config{ - MinBackoff: 100 * time.Millisecond, - MaxBackoff: 5 * time.Second, - MaxRetries: i.cfg.MaxTransferRetries, - }) - - // Keep track of the last error so that we can log it with the highest level - // once all retries have completed - var err error - - for backoff.Ongoing() { - err = i.transferOut(ctx) - if err == nil { - level.Info(i.logger).Log("msg", "transfer successfully completed") - return nil - } - - level.Warn(i.logger).Log("msg", "transfer attempt failed", "err", err, "attempt", backoff.NumRetries()+1, "max_retries", i.cfg.MaxTransferRetries) - - backoff.Wait() - } - - level.Error(i.logger).Log("msg", "all transfer attempts failed", "err", err) - return backoff.Err() -} - -func (i *Ingester) transferOut(ctx context.Context) error { - userStatesCopy := i.userStates.cp() - if len(userStatesCopy) == 0 { - level.Info(i.logger).Log("msg", "nothing to transfer") - return nil - } - - targetIngester, err := i.findTargetIngester(ctx) - if err != nil { - return fmt.Errorf("cannot find ingester to transfer chunks to: %w", err) - } - - level.Info(i.logger).Log("msg", "sending chunks", "to_ingester", targetIngester.Addr) - c, err := i.cfg.ingesterClientFactory(targetIngester.Addr, i.clientConfig) - if err != nil { - return err - } - defer c.Close() - - ctx = user.InjectOrgID(ctx, "-1") - stream, err := c.TransferChunks(ctx) - if err != nil { - return errors.Wrap(err, "TransferChunks") - } - - var chunks []client.Chunk - for userID, state := range userStatesCopy { - for pair := range state.fpToSeries.iter() { - state.fpLocker.Lock(pair.fp) - - if len(pair.series.chunkDescs) == 0 { // Nothing to send? - state.fpLocker.Unlock(pair.fp) - continue - } - - chunks, err = toWireChunks(pair.series.chunkDescs, chunks) - if err != nil { - state.fpLocker.Unlock(pair.fp) - return errors.Wrap(err, "toWireChunks") - } - - err = client.SendTimeSeriesChunk(stream, &client.TimeSeriesChunk{ - FromIngesterId: i.lifecycler.ID, - UserId: userID, - Labels: cortexpb.FromLabelsToLabelAdapters(pair.series.metric), - Chunks: chunks, - }) - state.fpLocker.Unlock(pair.fp) - if err != nil { - return errors.Wrap(err, "Send") - } - - i.metrics.sentChunks.Add(float64(len(chunks))) - } - } - - _, err = stream.CloseAndRecv() - if err != nil { - return errors.Wrap(err, "CloseAndRecv") - } - - // Close & empty all the flush queues, to unblock waiting workers. - for _, flushQueue := range i.flushQueues { - flushQueue.DiscardAndClose() - } - i.flushQueuesDone.Wait() - - level.Info(i.logger).Log("msg", "successfully sent chunks", "to_ingester", targetIngester.Addr) - return nil -} - -// findTargetIngester finds an ingester in PENDING state. -func (i *Ingester) findTargetIngester(ctx context.Context) (*ring.InstanceDesc, error) { - ringDesc, err := i.lifecycler.KVStore.Get(ctx, i.lifecycler.RingKey) - if err != nil { - return nil, err - } else if ringDesc == nil { - return nil, errTransferNoPendingIngesters - } - - ingesters := ringDesc.(*ring.Desc).FindIngestersByState(ring.PENDING) - if len(ingesters) <= 0 { - return nil, errTransferNoPendingIngesters - } - - return &ingesters[0], nil + level.Info(i.logger).Log("msg", "transfer between a LEAVING ingester and a PENDING one is not supported for the blocks storage") + return ring.ErrTransferDisabled } diff --git a/pkg/ingester/user_state.go b/pkg/ingester/user_state.go index 685dd54f38a..57973e7568e 100644 --- a/pkg/ingester/user_state.go +++ b/pkg/ingester/user_state.go @@ -1,358 +1,20 @@ package ingester import ( - "context" - "net/http" "sync" - "time" - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/tsdb/chunks" - tsdb_record "github.com/prometheus/prometheus/tsdb/record" "github.com/segmentio/fasthash/fnv1a" - "github.com/weaveworks/common/httpgrpc" - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/ingester/client" - "github.com/cortexproject/cortex/pkg/ingester/index" - "github.com/cortexproject/cortex/pkg/tenant" "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/extract" - util_math "github.com/cortexproject/cortex/pkg/util/math" - "github.com/cortexproject/cortex/pkg/util/spanlogger" - "github.com/cortexproject/cortex/pkg/util/validation" ) -// userStates holds the userState object for all users (tenants), -// each one containing all the in-memory series for a given user. -type userStates struct { - states sync.Map - limiter *Limiter - cfg Config - metrics *ingesterMetrics - logger log.Logger -} - -type userState struct { - limiter *Limiter - userID string - fpLocker *fingerprintLocker - fpToSeries *seriesMap - mapper *fpMapper - index *index.InvertedIndex - ingestedAPISamples *util_math.EwmaRate - ingestedRuleSamples *util_math.EwmaRate - activeSeries *ActiveSeries - logger log.Logger - - seriesInMetric *metricCounter - - // Series metrics. - memSeries prometheus.Gauge - memSeriesCreatedTotal prometheus.Counter - memSeriesRemovedTotal prometheus.Counter - discardedSamples *prometheus.CounterVec - createdChunks prometheus.Counter - activeSeriesGauge prometheus.Gauge -} - // DiscardedSamples metric labels const ( perUserSeriesLimit = "per_user_series_limit" perMetricSeriesLimit = "per_metric_series_limit" ) -func newUserStates(limiter *Limiter, cfg Config, metrics *ingesterMetrics, logger log.Logger) *userStates { - return &userStates{ - limiter: limiter, - cfg: cfg, - metrics: metrics, - logger: logger, - } -} - -func (us *userStates) cp() map[string]*userState { - states := map[string]*userState{} - us.states.Range(func(key, value interface{}) bool { - states[key.(string)] = value.(*userState) - return true - }) - return states -} - -//nolint:unused -func (us *userStates) gc() { - us.states.Range(func(key, value interface{}) bool { - state := value.(*userState) - if state.fpToSeries.length() == 0 { - us.states.Delete(key) - state.activeSeries.clear() - state.activeSeriesGauge.Set(0) - } - return true - }) -} - -func (us *userStates) updateRates() { - us.states.Range(func(key, value interface{}) bool { - state := value.(*userState) - state.ingestedAPISamples.Tick() - state.ingestedRuleSamples.Tick() - return true - }) -} - -// Labels will be copied if they are kept. -func (us *userStates) updateActiveSeriesForUser(userID string, now time.Time, lbls []labels.Label) { - if s, ok := us.get(userID); ok { - s.activeSeries.UpdateSeries(lbls, now, func(l labels.Labels) labels.Labels { return cortexpb.CopyLabels(l) }) - } -} - -func (us *userStates) purgeAndUpdateActiveSeries(purgeTime time.Time) { - us.states.Range(func(key, value interface{}) bool { - state := value.(*userState) - state.activeSeries.Purge(purgeTime) - state.activeSeriesGauge.Set(float64(state.activeSeries.Active())) - return true - }) -} - -func (us *userStates) get(userID string) (*userState, bool) { - state, ok := us.states.Load(userID) - if !ok { - return nil, ok - } - return state.(*userState), ok -} - -func (us *userStates) getOrCreate(userID string) *userState { - state, ok := us.get(userID) - if !ok { - - logger := log.With(us.logger, "user", userID) - // Speculatively create a userState object and try to store it - // in the map. Another goroutine may have got there before - // us, in which case this userState will be discarded - state = &userState{ - userID: userID, - limiter: us.limiter, - fpToSeries: newSeriesMap(), - fpLocker: newFingerprintLocker(16 * 1024), - index: index.New(), - ingestedAPISamples: util_math.NewEWMARate(0.2, us.cfg.RateUpdatePeriod), - ingestedRuleSamples: util_math.NewEWMARate(0.2, us.cfg.RateUpdatePeriod), - seriesInMetric: newMetricCounter(us.limiter, us.cfg.getIgnoreSeriesLimitForMetricNamesMap()), - logger: logger, - - memSeries: us.metrics.memSeries, - memSeriesCreatedTotal: us.metrics.memSeriesCreatedTotal.WithLabelValues(userID), - memSeriesRemovedTotal: us.metrics.memSeriesRemovedTotal.WithLabelValues(userID), - discardedSamples: validation.DiscardedSamples.MustCurryWith(prometheus.Labels{"user": userID}), - createdChunks: us.metrics.createdChunks, - - activeSeries: NewActiveSeries(), - activeSeriesGauge: us.metrics.activeSeriesPerUser.WithLabelValues(userID), - } - state.mapper = newFPMapper(state.fpToSeries, logger) - stored, ok := us.states.LoadOrStore(userID, state) - if !ok { - us.metrics.memUsers.Inc() - } - state = stored.(*userState) - } - - return state -} - -// teardown ensures metrics are accurately updated if a userStates struct is discarded -func (us *userStates) teardown() { - for _, u := range us.cp() { - u.memSeriesRemovedTotal.Add(float64(u.fpToSeries.length())) - u.memSeries.Sub(float64(u.fpToSeries.length())) - u.activeSeriesGauge.Set(0) - us.metrics.memUsers.Dec() - } -} - -func (us *userStates) getViaContext(ctx context.Context) (*userState, bool, error) { - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, false, err - } - state, ok := us.get(userID) - return state, ok, nil -} - -// NOTE: memory for `labels` is unsafe; anything retained beyond the -// life of this function must be copied -func (us *userStates) getOrCreateSeries(ctx context.Context, userID string, labels []cortexpb.LabelAdapter, record *WALRecord) (*userState, model.Fingerprint, *memorySeries, error) { - state := us.getOrCreate(userID) - // WARNING: `err` may have a reference to unsafe memory in `labels` - fp, series, err := state.getSeries(labels, record) - return state, fp, series, err -} - -// NOTE: memory for `metric` is unsafe; anything retained beyond the -// life of this function must be copied -func (u *userState) getSeries(metric labelPairs, record *WALRecord) (model.Fingerprint, *memorySeries, error) { - rawFP := client.FastFingerprint(metric) - u.fpLocker.Lock(rawFP) - fp := u.mapper.mapFP(rawFP, metric) - if fp != rawFP { - u.fpLocker.Unlock(rawFP) - u.fpLocker.Lock(fp) - } - - series, ok := u.fpToSeries.get(fp) - if ok { - return fp, series, nil - } - - series, err := u.createSeriesWithFingerprint(fp, metric, record, false) - if err != nil { - u.fpLocker.Unlock(fp) - return 0, nil, err - } - - return fp, series, nil -} - -func (u *userState) createSeriesWithFingerprint(fp model.Fingerprint, metric labelPairs, record *WALRecord, recovery bool) (*memorySeries, error) { - // There's theoretically a relatively harmless race here if multiple - // goroutines get the length of the series map at the same time, then - // all proceed to add a new series. This is likely not worth addressing, - // as this should happen rarely (all samples from one push are added - // serially), and the overshoot in allowed series would be minimal. - - if !recovery { - if err := u.limiter.AssertMaxSeriesPerUser(u.userID, u.fpToSeries.length()); err != nil { - return nil, makeLimitError(perUserSeriesLimit, u.limiter.FormatError(u.userID, err)) - } - } - - // MetricNameFromLabelAdapters returns a copy of the string in `metric` - metricName, err := extract.MetricNameFromLabelAdapters(metric) - if err != nil { - return nil, err - } - - if !recovery { - // Check if the per-metric limit has been exceeded - if err = u.seriesInMetric.canAddSeriesFor(u.userID, metricName); err != nil { - // WARNING: returns a reference to `metric` - return nil, makeMetricLimitError(perMetricSeriesLimit, cortexpb.FromLabelAdaptersToLabels(metric), u.limiter.FormatError(u.userID, err)) - } - } - - u.memSeriesCreatedTotal.Inc() - u.memSeries.Inc() - u.seriesInMetric.increaseSeriesForMetric(metricName) - - if record != nil { - lbls := make(labels.Labels, 0, len(metric)) - for _, m := range metric { - lbls = append(lbls, labels.Label(m)) - } - record.Series = append(record.Series, tsdb_record.RefSeries{ - Ref: chunks.HeadSeriesRef(fp), - Labels: lbls, - }) - } - - labels := u.index.Add(metric, fp) // Add() returns 'interned' values so the original labels are not retained - series := newMemorySeries(labels, u.createdChunks) - u.fpToSeries.put(fp, series) - - return series, nil -} - -func (u *userState) removeSeries(fp model.Fingerprint, metric labels.Labels) { - u.fpToSeries.del(fp) - u.index.Delete(metric, fp) - - metricName := metric.Get(model.MetricNameLabel) - if metricName == "" { - // Series without a metric name should never be able to make it into - // the ingester's memory storage. - panic("No metric name label") - } - - u.seriesInMetric.decreaseSeriesForMetric(metricName) - - u.memSeriesRemovedTotal.Inc() - u.memSeries.Dec() -} - -// forSeriesMatching passes all series matching the given matchers to the -// provided callback. Deals with locking and the quirks of zero-length matcher -// values. There are 2 callbacks: -// - The `add` callback is called for each series while the lock is held, and -// is intend to be used by the caller to build a batch. -// - The `send` callback is called at certain intervals specified by batchSize -// with no locks held, and is intended to be used by the caller to send the -// built batches. -func (u *userState) forSeriesMatching(ctx context.Context, allMatchers []*labels.Matcher, - add func(context.Context, model.Fingerprint, *memorySeries) error, - send func(context.Context) error, batchSize int, -) error { - log, ctx := spanlogger.New(ctx, "forSeriesMatching") - defer log.Finish() - - filters, matchers := util.SplitFiltersAndMatchers(allMatchers) - fps := u.index.Lookup(matchers) - if len(fps) > u.limiter.MaxSeriesPerQuery(u.userID) { - return httpgrpc.Errorf(http.StatusRequestEntityTooLarge, "exceeded maximum number of series in a query") - } - - level.Debug(log).Log("series", len(fps)) - - // We only hold one FP lock at once here, so no opportunity to deadlock. - sent := 0 -outer: - for _, fp := range fps { - if err := ctx.Err(); err != nil { - return err - } - - u.fpLocker.Lock(fp) - series, ok := u.fpToSeries.get(fp) - if !ok { - u.fpLocker.Unlock(fp) - continue - } - - for _, filter := range filters { - if !filter.Matches(series.metric.Get(filter.Name)) { - u.fpLocker.Unlock(fp) - continue outer - } - } - - err := add(ctx, fp, series) - u.fpLocker.Unlock(fp) - if err != nil { - return err - } - - sent++ - if batchSize > 0 && sent%batchSize == 0 && send != nil { - if err = send(ctx); err != nil { - return nil - } - } - } - - if batchSize > 0 && sent%batchSize > 0 && send != nil { - return send(ctx) - } - return nil -} - const numMetricCounterShards = 128 type metricCounterShard struct { diff --git a/pkg/ingester/user_state_test.go b/pkg/ingester/user_state_test.go index 1d5ec1238d0..ec6234e25a3 100644 --- a/pkg/ingester/user_state_test.go +++ b/pkg/ingester/user_state_test.go @@ -1,162 +1,15 @@ package ingester import ( - "context" - "fmt" - "math" - "strings" "testing" - "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/weaveworks/common/user" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/validation" ) -// Test forSeriesMatching correctly batches up series. -func TestForSeriesMatchingBatching(t *testing.T) { - matchAllNames, err := labels.NewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+") - require.NoError(t, err) - // We rely on pushTestSamples() creating jobs "testjob0" and "testjob1" in equal parts - matchNotJob0, err := labels.NewMatcher(labels.MatchNotEqual, model.JobLabel, "testjob0") - require.NoError(t, err) - matchNotJob1, err := labels.NewMatcher(labels.MatchNotEqual, model.JobLabel, "testjob1") - require.NoError(t, err) - - for _, tc := range []struct { - numSeries, batchSize int - matchers []*labels.Matcher - expected int - }{ - {100, 10, []*labels.Matcher{matchAllNames}, 100}, - {99, 10, []*labels.Matcher{matchAllNames}, 99}, - {98, 10, []*labels.Matcher{matchAllNames}, 98}, - {5, 10, []*labels.Matcher{matchAllNames}, 5}, - {10, 1, []*labels.Matcher{matchAllNames}, 10}, - {1, 1, []*labels.Matcher{matchAllNames}, 1}, - {10, 10, []*labels.Matcher{matchAllNames, matchNotJob0}, 5}, - {10, 10, []*labels.Matcher{matchAllNames, matchNotJob1}, 5}, - {100, 10, []*labels.Matcher{matchAllNames, matchNotJob0}, 50}, - {100, 10, []*labels.Matcher{matchAllNames, matchNotJob1}, 50}, - {99, 10, []*labels.Matcher{matchAllNames, matchNotJob0}, 49}, - {99, 10, []*labels.Matcher{matchAllNames, matchNotJob1}, 50}, - } { - t.Run(fmt.Sprintf("numSeries=%d,batchSize=%d,matchers=%s", tc.numSeries, tc.batchSize, tc.matchers), func(t *testing.T) { - _, ing := newDefaultTestStore(t) - userIDs, _ := pushTestSamples(t, ing, tc.numSeries, 100, 0) - - for _, userID := range userIDs { - ctx := user.InjectOrgID(context.Background(), userID) - instance, ok, err := ing.userStates.getViaContext(ctx) - require.NoError(t, err) - require.True(t, ok) - - total, batch, batches := 0, 0, 0 - err = instance.forSeriesMatching(ctx, tc.matchers, - func(_ context.Context, _ model.Fingerprint, s *memorySeries) error { - batch++ - return nil - }, - func(context.Context) error { - require.True(t, batch <= tc.batchSize) - total += batch - batch = 0 - batches++ - return nil - }, - tc.batchSize) - require.NoError(t, err) - require.Equal(t, tc.expected, total) - require.Equal(t, int(math.Ceil(float64(tc.expected)/float64(tc.batchSize))), batches) - } - }) - } -} - -// TestTeardown ensures metrics are updated correctly if the userState is discarded -func TestTeardown(t *testing.T) { - reg := prometheus.NewPedanticRegistry() - _, ing := newTestStore(t, - defaultIngesterTestConfig(t), - defaultClientTestConfig(), - defaultLimitsTestConfig(), - reg) - - pushTestSamples(t, ing, 100, 100, 0) - - // Assert exported metrics (3 blocks, 5 files per block, 2 files WAL). - metricNames := []string{ - "cortex_ingester_memory_series_created_total", - "cortex_ingester_memory_series_removed_total", - "cortex_ingester_memory_series", - "cortex_ingester_memory_users", - "cortex_ingester_active_series", - } - - ing.userStatesMtx.Lock() - ing.userStates.purgeAndUpdateActiveSeries(time.Now().Add(-5 * time.Minute)) - ing.userStatesMtx.Unlock() - - assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="1"} 0 - cortex_ingester_memory_series_removed_total{user="2"} 0 - cortex_ingester_memory_series_removed_total{user="3"} 0 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="1"} 100 - cortex_ingester_memory_series_created_total{user="2"} 100 - cortex_ingester_memory_series_created_total{user="3"} 100 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 300 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 3 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="1"} 100 - cortex_ingester_active_series{user="2"} 100 - cortex_ingester_active_series{user="3"} 100 - `), metricNames...)) - - ing.userStatesMtx.Lock() - defer ing.userStatesMtx.Unlock() - ing.userStates.teardown() - - assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="1"} 100 - cortex_ingester_memory_series_removed_total{user="2"} 100 - cortex_ingester_memory_series_removed_total{user="3"} 100 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="1"} 100 - cortex_ingester_memory_series_created_total{user="2"} 100 - cortex_ingester_memory_series_created_total{user="3"} 100 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 0 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 0 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="1"} 0 - cortex_ingester_active_series{user="2"} 0 - cortex_ingester_active_series{user="3"} 0 - `), metricNames...)) -} - func TestMetricCounter(t *testing.T) { const metric = "metric" diff --git a/pkg/ingester/wal.go b/pkg/ingester/wal.go index d714294be7e..704179c391b 100644 --- a/pkg/ingester/wal.go +++ b/pkg/ingester/wal.go @@ -2,32 +2,7 @@ package ingester import ( "flag" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "runtime" - "strconv" - "sync" "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/gogo/protobuf/proto" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/tsdb/encoding" - tsdb_errors "github.com/prometheus/prometheus/tsdb/errors" - "github.com/prometheus/prometheus/tsdb/fileutil" - tsdb_record "github.com/prometheus/prometheus/tsdb/record" - "github.com/prometheus/prometheus/tsdb/wal" - - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/ingester/client" ) // WALConfig is config for the Write Ahead Log. @@ -52,1083 +27,3 @@ func (cfg *WALConfig) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.FlushOnShutdown, "ingester.flush-on-shutdown-with-wal-enabled", false, "When WAL is enabled, should chunks be flushed to long-term storage on shutdown. Useful eg. for migration to blocks engine.") cfg.checkpointDuringShutdown = true } - -// WAL interface allows us to have a no-op WAL when the WAL is disabled. -type WAL interface { - // Log marshalls the records and writes it into the WAL. - Log(*WALRecord) error - // Stop stops all the WAL operations. - Stop() -} - -// RecordType represents the type of the WAL/Checkpoint record. -type RecordType byte - -const ( - // WALRecordSeries is the type for the WAL record on Prometheus TSDB record for series. - WALRecordSeries RecordType = 1 - // WALRecordSamples is the type for the WAL record based on Prometheus TSDB record for samples. - WALRecordSamples RecordType = 2 - - // CheckpointRecord is the type for the Checkpoint record based on protos. - CheckpointRecord RecordType = 3 -) - -type noopWAL struct{} - -func (noopWAL) Log(*WALRecord) error { return nil } -func (noopWAL) Stop() {} - -type walWrapper struct { - cfg WALConfig - quit chan struct{} - wait sync.WaitGroup - - wal *wal.WAL - getUserStates func() map[string]*userState - checkpointMtx sync.Mutex - bytesPool sync.Pool - - logger log.Logger - - // Metrics. - checkpointDeleteFail prometheus.Counter - checkpointDeleteTotal prometheus.Counter - checkpointCreationFail prometheus.Counter - checkpointCreationTotal prometheus.Counter - checkpointDuration prometheus.Summary - checkpointLoggedBytesTotal prometheus.Counter - walLoggedBytesTotal prometheus.Counter - walRecordsLogged prometheus.Counter -} - -// newWAL creates a WAL object. If the WAL is disabled, then the returned WAL is a no-op WAL. -func newWAL(cfg WALConfig, userStatesFunc func() map[string]*userState, registerer prometheus.Registerer, logger log.Logger) (WAL, error) { - if !cfg.WALEnabled { - return &noopWAL{}, nil - } - - var walRegistry prometheus.Registerer - if registerer != nil { - walRegistry = prometheus.WrapRegistererWith(prometheus.Labels{"kind": "wal"}, registerer) - } - tsdbWAL, err := wal.NewSize(logger, walRegistry, cfg.Dir, wal.DefaultSegmentSize/4, false) - if err != nil { - return nil, err - } - - w := &walWrapper{ - cfg: cfg, - quit: make(chan struct{}), - wal: tsdbWAL, - getUserStates: userStatesFunc, - bytesPool: sync.Pool{ - New: func() interface{} { - return make([]byte, 0, 512) - }, - }, - logger: logger, - } - - w.checkpointDeleteFail = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_checkpoint_deletions_failed_total", - Help: "Total number of checkpoint deletions that failed.", - }) - w.checkpointDeleteTotal = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_checkpoint_deletions_total", - Help: "Total number of checkpoint deletions attempted.", - }) - w.checkpointCreationFail = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_checkpoint_creations_failed_total", - Help: "Total number of checkpoint creations that failed.", - }) - w.checkpointCreationTotal = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_checkpoint_creations_total", - Help: "Total number of checkpoint creations attempted.", - }) - w.checkpointDuration = promauto.With(registerer).NewSummary(prometheus.SummaryOpts{ - Name: "cortex_ingester_checkpoint_duration_seconds", - Help: "Time taken to create a checkpoint.", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }) - w.walRecordsLogged = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_wal_records_logged_total", - Help: "Total number of WAL records logged.", - }) - w.checkpointLoggedBytesTotal = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_checkpoint_logged_bytes_total", - Help: "Total number of bytes written to disk for checkpointing.", - }) - w.walLoggedBytesTotal = promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_wal_logged_bytes_total", - Help: "Total number of bytes written to disk for WAL records.", - }) - - w.wait.Add(1) - go w.run() - return w, nil -} - -func (w *walWrapper) Stop() { - close(w.quit) - w.wait.Wait() - w.wal.Close() -} - -func (w *walWrapper) Log(record *WALRecord) error { - if record == nil || (len(record.Series) == 0 && len(record.Samples) == 0) { - return nil - } - select { - case <-w.quit: - return nil - default: - buf := w.bytesPool.Get().([]byte)[:0] - defer func() { - w.bytesPool.Put(buf) // nolint:staticcheck - }() - - if len(record.Series) > 0 { - buf = record.encodeSeries(buf) - if err := w.wal.Log(buf); err != nil { - return err - } - w.walRecordsLogged.Inc() - w.walLoggedBytesTotal.Add(float64(len(buf))) - buf = buf[:0] - } - if len(record.Samples) > 0 { - buf = record.encodeSamples(buf) - if err := w.wal.Log(buf); err != nil { - return err - } - w.walRecordsLogged.Inc() - w.walLoggedBytesTotal.Add(float64(len(buf))) - } - return nil - } -} - -func (w *walWrapper) run() { - defer w.wait.Done() - - if !w.cfg.CheckpointEnabled { - return - } - - ticker := time.NewTicker(w.cfg.CheckpointDuration) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - start := time.Now() - level.Info(w.logger).Log("msg", "starting checkpoint") - if err := w.performCheckpoint(false); err != nil { - level.Error(w.logger).Log("msg", "error checkpointing series", "err", err) - continue - } - elapsed := time.Since(start) - level.Info(w.logger).Log("msg", "checkpoint done", "time", elapsed.String()) - w.checkpointDuration.Observe(elapsed.Seconds()) - case <-w.quit: - if w.cfg.checkpointDuringShutdown { - level.Info(w.logger).Log("msg", "creating checkpoint before shutdown") - if err := w.performCheckpoint(true); err != nil { - level.Error(w.logger).Log("msg", "error checkpointing series during shutdown", "err", err) - } - } - return - } - } -} - -const checkpointPrefix = "checkpoint." - -func (w *walWrapper) performCheckpoint(immediate bool) (err error) { - if !w.cfg.CheckpointEnabled { - return nil - } - - // This method is called during shutdown which can interfere with ongoing checkpointing. - // Hence to avoid any race between file creation and WAL truncation, we hold this lock here. - w.checkpointMtx.Lock() - defer w.checkpointMtx.Unlock() - - w.checkpointCreationTotal.Inc() - defer func() { - if err != nil { - w.checkpointCreationFail.Inc() - } - }() - - if w.getUserStates == nil { - return errors.New("function to get user states not initialised") - } - - _, lastSegment, err := wal.Segments(w.wal.Dir()) - if err != nil { - return err - } - if lastSegment < 0 { - // There are no WAL segments. No need of checkpoint yet. - return nil - } - - _, lastCh, err := lastCheckpoint(w.wal.Dir()) - if err != nil { - return err - } - - if lastCh == lastSegment { - // As the checkpoint name is taken from last WAL segment, we need to ensure - // a new segment for every checkpoint so that the old checkpoint is not overwritten. - if err := w.wal.NextSegment(); err != nil { - return err - } - - _, lastSegment, err = wal.Segments(w.wal.Dir()) - if err != nil { - return err - } - } - - // Checkpoint is named after the last WAL segment present so that when replaying the WAL - // we can start from that particular WAL segment. - checkpointDir := filepath.Join(w.wal.Dir(), fmt.Sprintf(checkpointPrefix+"%06d", lastSegment)) - level.Info(w.logger).Log("msg", "attempting checkpoint for", "dir", checkpointDir) - checkpointDirTemp := checkpointDir + ".tmp" - - if err := os.MkdirAll(checkpointDirTemp, 0777); err != nil { - return errors.Wrap(err, "create checkpoint dir") - } - checkpoint, err := wal.New(nil, nil, checkpointDirTemp, false) - if err != nil { - return errors.Wrap(err, "open checkpoint") - } - defer func() { - checkpoint.Close() - os.RemoveAll(checkpointDirTemp) - }() - - // Count number of series - we'll use this to rate limit checkpoints. - numSeries := 0 - us := w.getUserStates() - for _, state := range us { - numSeries += state.fpToSeries.length() - } - if numSeries == 0 { - return nil - } - - perSeriesDuration := (95 * w.cfg.CheckpointDuration) / (100 * time.Duration(numSeries)) - - var wireChunkBuf []client.Chunk - var b []byte - bytePool := sync.Pool{ - New: func() interface{} { - return make([]byte, 0, 1024) - }, - } - records := [][]byte{} - totalSize := 0 - ticker := time.NewTicker(perSeriesDuration) - defer ticker.Stop() - start := time.Now() - for userID, state := range us { - for pair := range state.fpToSeries.iter() { - state.fpLocker.Lock(pair.fp) - wireChunkBuf, b, err = w.checkpointSeries(userID, pair.fp, pair.series, wireChunkBuf, bytePool.Get().([]byte)) - state.fpLocker.Unlock(pair.fp) - if err != nil { - return err - } - - records = append(records, b) - totalSize += len(b) - if totalSize >= 1*1024*1024 { // 1 MiB. - if err := checkpoint.Log(records...); err != nil { - return err - } - w.checkpointLoggedBytesTotal.Add(float64(totalSize)) - totalSize = 0 - for i := range records { - bytePool.Put(records[i]) // nolint:staticcheck - } - records = records[:0] - } - - if !immediate { - if time.Since(start) > 2*w.cfg.CheckpointDuration { - // This could indicate a surge in number of series and continuing with - // the old estimation of ticker can make checkpointing run indefinitely in worst case - // and disk running out of space. Re-adjust the ticker might not solve the problem - // as there can be another surge again. Hence let's checkpoint this one immediately. - immediate = true - continue - } - - select { - case <-ticker.C: - case <-w.quit: // When we're trying to shutdown, finish the checkpoint as fast as possible. - } - } - } - } - - if err := checkpoint.Log(records...); err != nil { - return err - } - - if err := checkpoint.Close(); err != nil { - return errors.Wrap(err, "close checkpoint") - } - if err := fileutil.Replace(checkpointDirTemp, checkpointDir); err != nil { - return errors.Wrap(err, "rename checkpoint directory") - } - - // We delete the WAL segments which are before the previous checkpoint and not before the - // current checkpoint created. This is because if the latest checkpoint is corrupted for any reason, we - // should be able to recover from the older checkpoint which would need the older WAL segments. - if err := w.wal.Truncate(lastCh); err != nil { - // It is fine to have old WAL segments hanging around if deletion failed. - // We can try again next time. - level.Error(w.logger).Log("msg", "error deleting old WAL segments", "err", err) - } - - if lastCh >= 0 { - if err := w.deleteCheckpoints(lastCh); err != nil { - // It is fine to have old checkpoints hanging around if deletion failed. - // We can try again next time. - level.Error(w.logger).Log("msg", "error deleting old checkpoint", "err", err) - } - } - - return nil -} - -// lastCheckpoint returns the directory name and index of the most recent checkpoint. -// If dir does not contain any checkpoints, -1 is returned as index. -func lastCheckpoint(dir string) (string, int, error) { - dirs, err := ioutil.ReadDir(dir) - if err != nil { - return "", -1, err - } - var ( - maxIdx = -1 - checkpointDir string - ) - // There may be multiple checkpoints left, so select the one with max index. - for i := 0; i < len(dirs); i++ { - di := dirs[i] - - idx, err := checkpointIndex(di.Name(), false) - if err != nil { - continue - } - if !di.IsDir() { - return "", -1, fmt.Errorf("checkpoint %s is not a directory", di.Name()) - } - if idx > maxIdx { - checkpointDir = di.Name() - maxIdx = idx - } - } - if maxIdx >= 0 { - return filepath.Join(dir, checkpointDir), maxIdx, nil - } - return "", -1, nil -} - -// deleteCheckpoints deletes all checkpoints in a directory which is < maxIndex. -func (w *walWrapper) deleteCheckpoints(maxIndex int) (err error) { - w.checkpointDeleteTotal.Inc() - defer func() { - if err != nil { - w.checkpointDeleteFail.Inc() - } - }() - - errs := tsdb_errors.NewMulti() - - files, err := ioutil.ReadDir(w.wal.Dir()) - if err != nil { - return err - } - for _, fi := range files { - index, err := checkpointIndex(fi.Name(), true) - if err != nil || index >= maxIndex { - continue - } - if err := os.RemoveAll(filepath.Join(w.wal.Dir(), fi.Name())); err != nil { - errs.Add(err) - } - } - return errs.Err() -} - -var checkpointRe = regexp.MustCompile("^" + regexp.QuoteMeta(checkpointPrefix) + "(\\d+)(\\.tmp)?$") - -// checkpointIndex returns the index of a given checkpoint file. It handles -// both regular and temporary checkpoints according to the includeTmp flag. If -// the file is not a checkpoint it returns an error. -func checkpointIndex(filename string, includeTmp bool) (int, error) { - result := checkpointRe.FindStringSubmatch(filename) - if len(result) < 2 { - return 0, errors.New("file is not a checkpoint") - } - // Filter out temporary checkpoints if desired. - if !includeTmp && len(result) == 3 && result[2] != "" { - return 0, errors.New("temporary checkpoint") - } - return strconv.Atoi(result[1]) -} - -// checkpointSeries write the chunks of the series to the checkpoint. -func (w *walWrapper) checkpointSeries(userID string, fp model.Fingerprint, series *memorySeries, wireChunks []client.Chunk, b []byte) ([]client.Chunk, []byte, error) { - var err error - wireChunks, err = toWireChunks(series.chunkDescs, wireChunks) - if err != nil { - return wireChunks, b, err - } - - b, err = encodeWithTypeHeader(&Series{ - UserId: userID, - Fingerprint: uint64(fp), - Labels: cortexpb.FromLabelsToLabelAdapters(series.metric), - Chunks: wireChunks, - }, CheckpointRecord, b) - - return wireChunks, b, err -} - -type walRecoveryParameters struct { - walDir string - ingester *Ingester - numWorkers int - stateCache []map[string]*userState - seriesCache []map[string]map[uint64]*memorySeries -} - -func recoverFromWAL(ingester *Ingester) error { - params := walRecoveryParameters{ - walDir: ingester.cfg.WALConfig.Dir, - numWorkers: runtime.GOMAXPROCS(0), - ingester: ingester, - } - - params.stateCache = make([]map[string]*userState, params.numWorkers) - params.seriesCache = make([]map[string]map[uint64]*memorySeries, params.numWorkers) - for i := 0; i < params.numWorkers; i++ { - params.stateCache[i] = make(map[string]*userState) - params.seriesCache[i] = make(map[string]map[uint64]*memorySeries) - } - - level.Info(ingester.logger).Log("msg", "recovering from checkpoint") - start := time.Now() - userStates, idx, err := processCheckpointWithRepair(params) - if err != nil { - return err - } - elapsed := time.Since(start) - level.Info(ingester.logger).Log("msg", "recovered from checkpoint", "time", elapsed.String()) - - if segExists, err := segmentsExist(params.walDir); err == nil && !segExists { - level.Info(ingester.logger).Log("msg", "no segments found, skipping recover from segments") - ingester.userStatesMtx.Lock() - ingester.userStates = userStates - ingester.userStatesMtx.Unlock() - return nil - } else if err != nil { - return err - } - - level.Info(ingester.logger).Log("msg", "recovering from WAL", "dir", params.walDir, "start_segment", idx) - start = time.Now() - if err := processWALWithRepair(idx, userStates, params); err != nil { - return err - } - elapsed = time.Since(start) - level.Info(ingester.logger).Log("msg", "recovered from WAL", "time", elapsed.String()) - - ingester.userStatesMtx.Lock() - ingester.userStates = userStates - ingester.userStatesMtx.Unlock() - return nil -} - -func processCheckpointWithRepair(params walRecoveryParameters) (*userStates, int, error) { - logger := params.ingester.logger - - // Use a local userStates, so we don't need to worry about locking. - userStates := newUserStates(params.ingester.limiter, params.ingester.cfg, params.ingester.metrics, params.ingester.logger) - - lastCheckpointDir, idx, err := lastCheckpoint(params.walDir) - if err != nil { - return nil, -1, err - } - if idx < 0 { - level.Info(logger).Log("msg", "no checkpoint found") - return userStates, -1, nil - } - - level.Info(logger).Log("msg", fmt.Sprintf("recovering from %s", lastCheckpointDir)) - - err = processCheckpoint(lastCheckpointDir, userStates, params) - if err == nil { - return userStates, idx, nil - } - - // We don't call repair on checkpoint as losing even a single record is like losing the entire data of a series. - // We try recovering from the older checkpoint instead. - params.ingester.metrics.walCorruptionsTotal.Inc() - level.Error(logger).Log("msg", "checkpoint recovery failed, deleting this checkpoint and trying to recover from old checkpoint", "err", err) - - // Deleting this checkpoint to try the previous checkpoint. - if err := os.RemoveAll(lastCheckpointDir); err != nil { - return nil, -1, errors.Wrapf(err, "unable to delete checkpoint directory %s", lastCheckpointDir) - } - - // If we have reached this point, it means the last checkpoint was deleted. - // Now the last checkpoint will be the one before the deleted checkpoint. - lastCheckpointDir, idx, err = lastCheckpoint(params.walDir) - if err != nil { - return nil, -1, err - } - - // Creating new userStates to discard the old chunks. - userStates = newUserStates(params.ingester.limiter, params.ingester.cfg, params.ingester.metrics, params.ingester.logger) - if idx < 0 { - // There was only 1 checkpoint. We don't error in this case - // as for the first checkpoint entire WAL will/should be present. - return userStates, -1, nil - } - - level.Info(logger).Log("msg", fmt.Sprintf("attempting recovery from %s", lastCheckpointDir)) - if err := processCheckpoint(lastCheckpointDir, userStates, params); err != nil { - // We won't attempt the repair again even if its the old checkpoint. - params.ingester.metrics.walCorruptionsTotal.Inc() - return nil, -1, err - } - - return userStates, idx, nil -} - -// segmentsExist is a stripped down version of -// https://github.com/prometheus/prometheus/blob/4c648eddf47d7e07fbc74d0b18244402200dca9e/tsdb/wal/wal.go#L739-L760. -func segmentsExist(dir string) (bool, error) { - files, err := ioutil.ReadDir(dir) - if err != nil { - return false, err - } - for _, f := range files { - if _, err := strconv.Atoi(f.Name()); err == nil { - // First filename which is a number. - // This is how Prometheus stores and this - // is how it checks too. - return true, nil - } - } - return false, nil -} - -// processCheckpoint loads the chunks of the series present in the last checkpoint. -func processCheckpoint(name string, userStates *userStates, params walRecoveryParameters) error { - - reader, closer, err := newWalReader(name, -1) - if err != nil { - return err - } - defer closer.Close() - - var ( - inputs = make([]chan *Series, params.numWorkers) - // errChan is to capture the errors from goroutine. - // The channel size is nWorkers+1 to not block any worker if all of them error out. - errChan = make(chan error, params.numWorkers) - wg = sync.WaitGroup{} - seriesPool = &sync.Pool{ - New: func() interface{} { - return &Series{} - }, - } - ) - - wg.Add(params.numWorkers) - for i := 0; i < params.numWorkers; i++ { - inputs[i] = make(chan *Series, 300) - go func(input <-chan *Series, stateCache map[string]*userState, seriesCache map[string]map[uint64]*memorySeries) { - processCheckpointRecord(userStates, seriesPool, stateCache, seriesCache, input, errChan, params.ingester.metrics.memoryChunks) - wg.Done() - }(inputs[i], params.stateCache[i], params.seriesCache[i]) - } - - var capturedErr error -Loop: - for reader.Next() { - s := seriesPool.Get().(*Series) - m, err := decodeCheckpointRecord(reader.Record(), s) - if err != nil { - // We don't return here in order to close/drain all the channels and - // make sure all goroutines exit. - capturedErr = err - break Loop - } - s = m.(*Series) - - // The yoloString from the unmarshal of LabelAdapter gets corrupted - // when travelling through the channel. Hence making a copy of that. - // This extra alloc during the read path is fine as it's only 1 time - // and saves extra allocs during write path by having LabelAdapter. - s.Labels = copyLabelAdapters(s.Labels) - - select { - case capturedErr = <-errChan: - // Exit early on an error. - // Only acts upon the first error received. - break Loop - default: - mod := s.Fingerprint % uint64(params.numWorkers) - inputs[mod] <- s - } - } - - for i := 0; i < params.numWorkers; i++ { - close(inputs[i]) - } - wg.Wait() - // If any worker errored out, some input channels might not be empty. - // Hence drain them. - for i := 0; i < params.numWorkers; i++ { - for range inputs[i] { - } - } - - if capturedErr != nil { - return capturedErr - } - select { - case capturedErr = <-errChan: - return capturedErr - default: - return reader.Err() - } -} - -func copyLabelAdapters(las []cortexpb.LabelAdapter) []cortexpb.LabelAdapter { - for i := range las { - n, v := make([]byte, len(las[i].Name)), make([]byte, len(las[i].Value)) - copy(n, las[i].Name) - copy(v, las[i].Value) - las[i].Name = string(n) - las[i].Value = string(v) - } - return las -} - -func processCheckpointRecord( - userStates *userStates, - seriesPool *sync.Pool, - stateCache map[string]*userState, - seriesCache map[string]map[uint64]*memorySeries, - seriesChan <-chan *Series, - errChan chan error, - memoryChunks prometheus.Counter, -) { - var la []cortexpb.LabelAdapter - for s := range seriesChan { - state, ok := stateCache[s.UserId] - if !ok { - state = userStates.getOrCreate(s.UserId) - stateCache[s.UserId] = state - seriesCache[s.UserId] = make(map[uint64]*memorySeries) - } - - la = la[:0] - for _, l := range s.Labels { - la = append(la, cortexpb.LabelAdapter{ - Name: string(l.Name), - Value: string(l.Value), - }) - } - series, err := state.createSeriesWithFingerprint(model.Fingerprint(s.Fingerprint), la, nil, true) - if err != nil { - errChan <- err - return - } - - descs, err := fromWireChunks(s.Chunks) - if err != nil { - errChan <- err - return - } - - if err := series.setChunks(descs); err != nil { - errChan <- err - return - } - memoryChunks.Add(float64(len(descs))) - - seriesCache[s.UserId][s.Fingerprint] = series - seriesPool.Put(s) - } -} - -type samplesWithUserID struct { - samples []tsdb_record.RefSample - userID string -} - -func processWALWithRepair(startSegment int, userStates *userStates, params walRecoveryParameters) error { - logger := params.ingester.logger - - corruptErr := processWAL(startSegment, userStates, params) - if corruptErr == nil { - return nil - } - - params.ingester.metrics.walCorruptionsTotal.Inc() - level.Error(logger).Log("msg", "error in replaying from WAL", "err", corruptErr) - - // Attempt repair. - level.Info(logger).Log("msg", "attempting repair of the WAL") - w, err := wal.New(logger, nil, params.walDir, true) - if err != nil { - return err - } - - err = w.Repair(corruptErr) - if err != nil { - level.Error(logger).Log("msg", "error in repairing WAL", "err", err) - } - - return tsdb_errors.NewMulti(err, w.Close()).Err() -} - -// processWAL processes the records in the WAL concurrently. -func processWAL(startSegment int, userStates *userStates, params walRecoveryParameters) error { - - reader, closer, err := newWalReader(params.walDir, startSegment) - if err != nil { - return err - } - defer closer.Close() - - var ( - wg sync.WaitGroup - inputs = make([]chan *samplesWithUserID, params.numWorkers) - outputs = make([]chan *samplesWithUserID, params.numWorkers) - // errChan is to capture the errors from goroutine. - // The channel size is nWorkers to not block any worker if all of them error out. - errChan = make(chan error, params.numWorkers) - shards = make([]*samplesWithUserID, params.numWorkers) - ) - - wg.Add(params.numWorkers) - for i := 0; i < params.numWorkers; i++ { - outputs[i] = make(chan *samplesWithUserID, 300) - inputs[i] = make(chan *samplesWithUserID, 300) - shards[i] = &samplesWithUserID{} - - go func(input <-chan *samplesWithUserID, output chan<- *samplesWithUserID, - stateCache map[string]*userState, seriesCache map[string]map[uint64]*memorySeries) { - processWALSamples(userStates, stateCache, seriesCache, input, output, errChan, params.ingester.logger) - wg.Done() - }(inputs[i], outputs[i], params.stateCache[i], params.seriesCache[i]) - } - - var ( - capturedErr error - walRecord = &WALRecord{} - lp labelPairs - ) -Loop: - for reader.Next() { - select { - case capturedErr = <-errChan: - // Exit early on an error. - // Only acts upon the first error received. - break Loop - default: - } - - if err := decodeWALRecord(reader.Record(), walRecord); err != nil { - // We don't return here in order to close/drain all the channels and - // make sure all goroutines exit. - capturedErr = err - break Loop - } - - if len(walRecord.Series) > 0 { - userID := walRecord.UserID - - state := userStates.getOrCreate(userID) - - for _, s := range walRecord.Series { - fp := model.Fingerprint(s.Ref) - _, ok := state.fpToSeries.get(fp) - if ok { - continue - } - - lp = lp[:0] - for _, l := range s.Labels { - lp = append(lp, cortexpb.LabelAdapter(l)) - } - if _, err := state.createSeriesWithFingerprint(fp, lp, nil, true); err != nil { - // We don't return here in order to close/drain all the channels and - // make sure all goroutines exit. - capturedErr = err - break Loop - } - } - } - - // We split up the samples into chunks of 5000 samples or less. - // With O(300 * #cores) in-flight sample batches, large scrapes could otherwise - // cause thousands of very large in flight buffers occupying large amounts - // of unused memory. - walRecordSamples := walRecord.Samples - for len(walRecordSamples) > 0 { - m := 5000 - userID := walRecord.UserID - if len(walRecordSamples) < m { - m = len(walRecordSamples) - } - - for i := 0; i < params.numWorkers; i++ { - if len(shards[i].samples) == 0 { - // It is possible that the previous iteration did not put - // anything in this shard. In that case no need to get a new buffer. - shards[i].userID = userID - continue - } - select { - case buf := <-outputs[i]: - buf.samples = buf.samples[:0] - buf.userID = userID - shards[i] = buf - default: - shards[i] = &samplesWithUserID{ - userID: userID, - } - } - } - - for _, sam := range walRecordSamples[:m] { - mod := uint64(sam.Ref) % uint64(params.numWorkers) - shards[mod].samples = append(shards[mod].samples, sam) - } - - for i := 0; i < params.numWorkers; i++ { - if len(shards[i].samples) > 0 { - inputs[i] <- shards[i] - } - } - - walRecordSamples = walRecordSamples[m:] - } - } - - for i := 0; i < params.numWorkers; i++ { - close(inputs[i]) - for range outputs[i] { - } - } - wg.Wait() - // If any worker errored out, some input channels might not be empty. - // Hence drain them. - for i := 0; i < params.numWorkers; i++ { - for range inputs[i] { - } - } - - if capturedErr != nil { - return capturedErr - } - select { - case capturedErr = <-errChan: - return capturedErr - default: - return reader.Err() - } -} - -func processWALSamples(userStates *userStates, stateCache map[string]*userState, seriesCache map[string]map[uint64]*memorySeries, - input <-chan *samplesWithUserID, output chan<- *samplesWithUserID, errChan chan error, logger log.Logger) { - defer close(output) - - sp := model.SamplePair{} - for samples := range input { - state, ok := stateCache[samples.userID] - if !ok { - state = userStates.getOrCreate(samples.userID) - stateCache[samples.userID] = state - seriesCache[samples.userID] = make(map[uint64]*memorySeries) - } - sc := seriesCache[samples.userID] - for i := range samples.samples { - series, ok := sc[uint64(samples.samples[i].Ref)] - if !ok { - series, ok = state.fpToSeries.get(model.Fingerprint(samples.samples[i].Ref)) - if !ok { - // This should ideally not happen. - // If the series was not created in recovering checkpoint or - // from the labels of any records previous to this, there - // is no way to get the labels for this fingerprint. - level.Warn(logger).Log("msg", "series not found for sample during wal recovery", "userid", samples.userID, "fingerprint", model.Fingerprint(samples.samples[i].Ref).String()) - continue - } - } - sp.Timestamp = model.Time(samples.samples[i].T) - sp.Value = model.SampleValue(samples.samples[i].V) - // There can be many out of order samples because of checkpoint and WAL overlap. - // Checking this beforehand avoids the allocation of lots of error messages. - if sp.Timestamp.After(series.lastTime) { - if err := series.add(sp); err != nil { - errChan <- err - return - } - } - } - output <- samples - } -} - -// If startSegment is <0, it means all the segments. -func newWalReader(name string, startSegment int) (*wal.Reader, io.Closer, error) { - var ( - segmentReader io.ReadCloser - err error - ) - if startSegment < 0 { - segmentReader, err = wal.NewSegmentsReader(name) - if err != nil { - return nil, nil, err - } - } else { - first, last, err := wal.Segments(name) - if err != nil { - return nil, nil, err - } - if startSegment > last { - return nil, nil, errors.New("start segment is beyond the last WAL segment") - } - if first > startSegment { - startSegment = first - } - segmentReader, err = wal.NewSegmentsRangeReader(wal.SegmentRange{ - Dir: name, - First: startSegment, - Last: -1, // Till the end. - }) - if err != nil { - return nil, nil, err - } - } - return wal.NewReader(segmentReader), segmentReader, nil -} - -func decodeCheckpointRecord(rec []byte, m proto.Message) (_ proto.Message, err error) { - switch RecordType(rec[0]) { - case CheckpointRecord: - if err := proto.Unmarshal(rec[1:], m); err != nil { - return m, err - } - default: - // The legacy proto record will have it's first byte >7. - // Hence it does not match any of the existing record types. - err := proto.Unmarshal(rec, m) - if err != nil { - return m, err - } - } - - return m, err -} - -func encodeWithTypeHeader(m proto.Message, typ RecordType, b []byte) ([]byte, error) { - buf, err := proto.Marshal(m) - if err != nil { - return b, err - } - - b = append(b[:0], byte(typ)) - b = append(b, buf...) - return b, nil -} - -// WALRecord is a struct combining the series and samples record. -type WALRecord struct { - UserID string - Series []tsdb_record.RefSeries - Samples []tsdb_record.RefSample -} - -func (record *WALRecord) encodeSeries(b []byte) []byte { - buf := encoding.Encbuf{B: b} - buf.PutByte(byte(WALRecordSeries)) - buf.PutUvarintStr(record.UserID) - - var enc tsdb_record.Encoder - // The 'encoded' already has the type header and userID here, hence re-using - // the remaining part of the slice (i.e. encoded[len(encoded):])) to encode the series. - encoded := buf.Get() - encoded = append(encoded, enc.Series(record.Series, encoded[len(encoded):])...) - - return encoded -} - -func (record *WALRecord) encodeSamples(b []byte) []byte { - buf := encoding.Encbuf{B: b} - buf.PutByte(byte(WALRecordSamples)) - buf.PutUvarintStr(record.UserID) - - var enc tsdb_record.Encoder - // The 'encoded' already has the type header and userID here, hence re-using - // the remaining part of the slice (i.e. encoded[len(encoded):]))to encode the samples. - encoded := buf.Get() - encoded = append(encoded, enc.Samples(record.Samples, encoded[len(encoded):])...) - - return encoded -} - -func decodeWALRecord(b []byte, walRec *WALRecord) (err error) { - var ( - userID string - dec tsdb_record.Decoder - rseries []tsdb_record.RefSeries - rsamples []tsdb_record.RefSample - - decbuf = encoding.Decbuf{B: b} - t = RecordType(decbuf.Byte()) - ) - - walRec.Series = walRec.Series[:0] - walRec.Samples = walRec.Samples[:0] - switch t { - case WALRecordSamples: - userID = decbuf.UvarintStr() - rsamples, err = dec.Samples(decbuf.B, walRec.Samples) - case WALRecordSeries: - userID = decbuf.UvarintStr() - rseries, err = dec.Series(decbuf.B, walRec.Series) - default: - return errors.New("unknown record type") - } - - // We reach here only if its a record with type header. - if decbuf.Err() != nil { - return decbuf.Err() - } - - if err != nil { - return err - } - - walRec.UserID = userID - walRec.Samples = rsamples - walRec.Series = rseries - - return nil -} diff --git a/pkg/ingester/wal.pb.go b/pkg/ingester/wal.pb.go deleted file mode 100644 index 65f04214700..00000000000 --- a/pkg/ingester/wal.pb.go +++ /dev/null @@ -1,607 +0,0 @@ -// Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: wal.proto - -package ingester - -import ( - fmt "fmt" - _ "github.com/cortexproject/cortex/pkg/cortexpb" - github_com_cortexproject_cortex_pkg_cortexpb "github.com/cortexproject/cortex/pkg/cortexpb" - client "github.com/cortexproject/cortex/pkg/ingester/client" - _ "github.com/gogo/protobuf/gogoproto" - proto "github.com/gogo/protobuf/proto" - io "io" - math "math" - math_bits "math/bits" - reflect "reflect" - strings "strings" -) - -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package - -type Series struct { - UserId string `protobuf:"bytes,1,opt,name=user_id,json=userId,proto3" json:"user_id,omitempty"` - Fingerprint uint64 `protobuf:"varint,2,opt,name=fingerprint,proto3" json:"fingerprint,omitempty"` - Labels []github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter `protobuf:"bytes,3,rep,name=labels,proto3,customtype=github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter" json:"labels"` - Chunks []client.Chunk `protobuf:"bytes,4,rep,name=chunks,proto3" json:"chunks"` -} - -func (m *Series) Reset() { *m = Series{} } -func (*Series) ProtoMessage() {} -func (*Series) Descriptor() ([]byte, []int) { - return fileDescriptor_ae6364fc8077884f, []int{0} -} -func (m *Series) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Series) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Series.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Series) XXX_Merge(src proto.Message) { - xxx_messageInfo_Series.Merge(m, src) -} -func (m *Series) XXX_Size() int { - return m.Size() -} -func (m *Series) XXX_DiscardUnknown() { - xxx_messageInfo_Series.DiscardUnknown(m) -} - -var xxx_messageInfo_Series proto.InternalMessageInfo - -func (m *Series) GetUserId() string { - if m != nil { - return m.UserId - } - return "" -} - -func (m *Series) GetFingerprint() uint64 { - if m != nil { - return m.Fingerprint - } - return 0 -} - -func (m *Series) GetChunks() []client.Chunk { - if m != nil { - return m.Chunks - } - return nil -} - -func init() { - proto.RegisterType((*Series)(nil), "ingester.Series") -} - -func init() { proto.RegisterFile("wal.proto", fileDescriptor_ae6364fc8077884f) } - -var fileDescriptor_ae6364fc8077884f = []byte{ - // 323 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x91, 0x31, 0x4e, 0xc3, 0x30, - 0x18, 0x85, 0x6d, 0x5a, 0x05, 0xea, 0x8a, 0x25, 0x0c, 0x44, 0x1d, 0xfe, 0x46, 0x4c, 0x95, 0x10, - 0x89, 0x04, 0x13, 0x0b, 0x52, 0xc3, 0x84, 0xc4, 0x80, 0xc2, 0xc6, 0x82, 0x92, 0xd4, 0x4d, 0x4d, - 0x43, 0x1c, 0x39, 0x8e, 0x60, 0xe4, 0x08, 0x1c, 0x83, 0xa3, 0x74, 0xec, 0x58, 0x31, 0x54, 0xad, - 0xbb, 0x30, 0xf6, 0x08, 0x28, 0xae, 0x5b, 0x75, 0x84, 0xed, 0x7f, 0x2f, 0xef, 0xcb, 0xfb, 0x6d, - 0x93, 0xd6, 0x5b, 0x94, 0x79, 0x85, 0xe0, 0x92, 0xdb, 0x47, 0x2c, 0x4f, 0x69, 0x29, 0xa9, 0xe8, - 0x5c, 0xa4, 0x4c, 0x8e, 0xaa, 0xd8, 0x4b, 0xf8, 0xab, 0x9f, 0xf2, 0x94, 0xfb, 0x3a, 0x10, 0x57, - 0x43, 0xad, 0xb4, 0xd0, 0xd3, 0x06, 0xec, 0x5c, 0xef, 0xc5, 0x13, 0x2e, 0x24, 0x7d, 0x2f, 0x04, - 0x7f, 0xa1, 0x89, 0x34, 0xca, 0x2f, 0xc6, 0xe9, 0xf6, 0x43, 0x6c, 0x06, 0x83, 0x06, 0x7f, 0x41, - 0xb7, 0x7b, 0xf9, 0x49, 0xc6, 0x68, 0x2e, 0x77, 0x7a, 0xf3, 0x8f, 0xb3, 0x05, 0x26, 0xd6, 0x23, - 0x15, 0x8c, 0x96, 0xf6, 0x29, 0x39, 0xac, 0x4a, 0x2a, 0x9e, 0xd9, 0xc0, 0xc1, 0x2e, 0xee, 0xb5, - 0x42, 0xab, 0x96, 0x77, 0x03, 0xdb, 0x25, 0xed, 0x61, 0x8d, 0x89, 0x42, 0xb0, 0x5c, 0x3a, 0x07, - 0x2e, 0xee, 0x35, 0xc3, 0x7d, 0xcb, 0xce, 0x89, 0x95, 0x45, 0x31, 0xcd, 0x4a, 0xa7, 0xe1, 0x36, - 0x7a, 0xed, 0xcb, 0x13, 0x6f, 0xbb, 0xb1, 0x77, 0x5f, 0xfb, 0x0f, 0x11, 0x13, 0x41, 0x7f, 0x32, - 0xef, 0xa2, 0xef, 0x79, 0xf7, 0x5f, 0x27, 0xde, 0xf0, 0xfd, 0x41, 0x54, 0x48, 0x2a, 0x42, 0xd3, - 0x62, 0x9f, 0x13, 0x2b, 0x19, 0x55, 0xf9, 0xb8, 0x74, 0x9a, 0xba, 0xef, 0xd8, 0xf4, 0x79, 0xb7, - 0xb5, 0x1b, 0x34, 0xeb, 0xa6, 0xd0, 0x44, 0x82, 0x9b, 0xe9, 0x12, 0xd0, 0x6c, 0x09, 0x68, 0xbd, - 0x04, 0xfc, 0xa1, 0x00, 0x7f, 0x29, 0xc0, 0x13, 0x05, 0x78, 0xaa, 0x00, 0x2f, 0x14, 0xe0, 0x1f, - 0x05, 0x68, 0xad, 0x00, 0x7f, 0xae, 0x00, 0x4d, 0x57, 0x80, 0x66, 0x2b, 0x40, 0x4f, 0xbb, 0x07, - 0x8d, 0x2d, 0x7d, 0x53, 0x57, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xd2, 0x67, 0x44, 0x9d, 0xee, - 0x01, 0x00, 0x00, -} - -func (this *Series) Equal(that interface{}) bool { - if that == nil { - return this == nil - } - - that1, ok := that.(*Series) - if !ok { - that2, ok := that.(Series) - if ok { - that1 = &that2 - } else { - return false - } - } - if that1 == nil { - return this == nil - } else if this == nil { - return false - } - if this.UserId != that1.UserId { - return false - } - if this.Fingerprint != that1.Fingerprint { - return false - } - if len(this.Labels) != len(that1.Labels) { - return false - } - for i := range this.Labels { - if !this.Labels[i].Equal(that1.Labels[i]) { - return false - } - } - if len(this.Chunks) != len(that1.Chunks) { - return false - } - for i := range this.Chunks { - if !this.Chunks[i].Equal(&that1.Chunks[i]) { - return false - } - } - return true -} -func (this *Series) GoString() string { - if this == nil { - return "nil" - } - s := make([]string, 0, 8) - s = append(s, "&ingester.Series{") - s = append(s, "UserId: "+fmt.Sprintf("%#v", this.UserId)+",\n") - s = append(s, "Fingerprint: "+fmt.Sprintf("%#v", this.Fingerprint)+",\n") - s = append(s, "Labels: "+fmt.Sprintf("%#v", this.Labels)+",\n") - if this.Chunks != nil { - vs := make([]*client.Chunk, len(this.Chunks)) - for i := range vs { - vs[i] = &this.Chunks[i] - } - s = append(s, "Chunks: "+fmt.Sprintf("%#v", vs)+",\n") - } - s = append(s, "}") - return strings.Join(s, "") -} -func valueToGoStringWal(v interface{}, typ string) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) -} -func (m *Series) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Series) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Series) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Chunks) > 0 { - for iNdEx := len(m.Chunks) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Chunks[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintWal(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x22 - } - } - if len(m.Labels) > 0 { - for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- { - { - size := m.Labels[iNdEx].Size() - i -= size - if _, err := m.Labels[iNdEx].MarshalTo(dAtA[i:]); err != nil { - return 0, err - } - i = encodeVarintWal(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a - } - } - if m.Fingerprint != 0 { - i = encodeVarintWal(dAtA, i, uint64(m.Fingerprint)) - i-- - dAtA[i] = 0x10 - } - if len(m.UserId) > 0 { - i -= len(m.UserId) - copy(dAtA[i:], m.UserId) - i = encodeVarintWal(dAtA, i, uint64(len(m.UserId))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - -func encodeVarintWal(dAtA []byte, offset int, v uint64) int { - offset -= sovWal(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ - } - dAtA[offset] = uint8(v) - return base -} -func (m *Series) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.UserId) - if l > 0 { - n += 1 + l + sovWal(uint64(l)) - } - if m.Fingerprint != 0 { - n += 1 + sovWal(uint64(m.Fingerprint)) - } - if len(m.Labels) > 0 { - for _, e := range m.Labels { - l = e.Size() - n += 1 + l + sovWal(uint64(l)) - } - } - if len(m.Chunks) > 0 { - for _, e := range m.Chunks { - l = e.Size() - n += 1 + l + sovWal(uint64(l)) - } - } - return n -} - -func sovWal(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozWal(x uint64) (n int) { - return sovWal(uint64((x << 1) ^ uint64((int64(x) >> 63)))) -} -func (this *Series) String() string { - if this == nil { - return "nil" - } - repeatedStringForChunks := "[]Chunk{" - for _, f := range this.Chunks { - repeatedStringForChunks += fmt.Sprintf("%v", f) + "," - } - repeatedStringForChunks += "}" - s := strings.Join([]string{`&Series{`, - `UserId:` + fmt.Sprintf("%v", this.UserId) + `,`, - `Fingerprint:` + fmt.Sprintf("%v", this.Fingerprint) + `,`, - `Labels:` + fmt.Sprintf("%v", this.Labels) + `,`, - `Chunks:` + repeatedStringForChunks + `,`, - `}`, - }, "") - return s -} -func valueToStringWal(v interface{}) string { - rv := reflect.ValueOf(v) - if rv.IsNil() { - return "nil" - } - pv := reflect.Indirect(rv).Interface() - return fmt.Sprintf("*%v", pv) -} -func (m *Series) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowWal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Series: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Series: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field UserId", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowWal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthWal - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthWal - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.UserId = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Fingerprint", wireType) - } - m.Fingerprint = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowWal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Fingerprint |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowWal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthWal - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthWal - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Labels = append(m.Labels, github_com_cortexproject_cortex_pkg_cortexpb.LabelAdapter{}) - if err := m.Labels[len(m.Labels)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Chunks", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowWal - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthWal - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthWal - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Chunks = append(m.Chunks, client.Chunk{}) - if err := m.Chunks[len(m.Chunks)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipWal(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthWal - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthWal - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func skipWal(dAtA []byte) (n int, err error) { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowWal - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - wireType := int(wire & 0x7) - switch wireType { - case 0: - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowWal - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - iNdEx++ - if dAtA[iNdEx-1] < 0x80 { - break - } - } - return iNdEx, nil - case 1: - iNdEx += 8 - return iNdEx, nil - case 2: - var length int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowWal - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - length |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if length < 0 { - return 0, ErrInvalidLengthWal - } - iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthWal - } - return iNdEx, nil - case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowWal - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipWal(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthWal - } - } - return iNdEx, nil - case 4: - return iNdEx, nil - case 5: - iNdEx += 4 - return iNdEx, nil - default: - return 0, fmt.Errorf("proto: illegal wireType %d", wireType) - } - } - panic("unreachable") -} - -var ( - ErrInvalidLengthWal = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowWal = fmt.Errorf("proto: integer overflow") -) diff --git a/pkg/ingester/wal.proto b/pkg/ingester/wal.proto deleted file mode 100644 index 1cd86f13c1d..00000000000 --- a/pkg/ingester/wal.proto +++ /dev/null @@ -1,16 +0,0 @@ -syntax = "proto3"; - -package ingester; - -option go_package = "ingester"; - -import "github.com/gogo/protobuf/gogoproto/gogo.proto"; -import "github.com/cortexproject/cortex/pkg/cortexpb/cortex.proto"; -import "github.com/cortexproject/cortex/pkg/ingester/client/ingester.proto"; - -message Series { - string user_id = 1; - uint64 fingerprint = 2; - repeated cortexpb.LabelPair labels = 3 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/cortexproject/cortex/pkg/cortexpb.LabelAdapter"]; - repeated cortex.Chunk chunks = 4 [(gogoproto.nullable) = false]; -} diff --git a/pkg/ingester/wal_test.go b/pkg/ingester/wal_test.go deleted file mode 100644 index ca1d8fbb600..00000000000 --- a/pkg/ingester/wal_test.go +++ /dev/null @@ -1,328 +0,0 @@ -package ingester - -import ( - "context" - "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" - "testing" - "time" - - prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/require" - "github.com/weaveworks/common/httpgrpc" - "github.com/weaveworks/common/user" - - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/util/services" -) - -func TestWAL(t *testing.T) { - dirname, err := ioutil.TempDir("", "cortex-wal") - require.NoError(t, err) - defer func() { - require.NoError(t, os.RemoveAll(dirname)) - }() - - cfg := defaultIngesterTestConfig(t) - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.CheckpointEnabled = true - cfg.WALConfig.Recover = true - cfg.WALConfig.Dir = dirname - cfg.WALConfig.CheckpointDuration = 100 * time.Minute - cfg.WALConfig.checkpointDuringShutdown = true - - numSeries := 100 - numSamplesPerSeriesPerPush := 10 - numRestarts := 5 - - // Build an ingester, add some samples, then shut it down. - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - userIDs, testData := pushTestSamples(t, ing, numSeries, numSamplesPerSeriesPerPush, 0) - // Checkpoint happens when stopping. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - - for r := 0; r < numRestarts; r++ { - if r == 2 { - // From 3rd restart onwards, we are disabling checkpointing during shutdown - // to test both checkpoint+WAL replay. - cfg.WALConfig.checkpointDuringShutdown = false - } - if r == numRestarts-1 { - cfg.WALConfig.WALEnabled = false - cfg.WALConfig.CheckpointEnabled = false - } - - // Start a new ingester and recover the WAL. - _, ing = newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - for i, userID := range userIDs { - testData[userID] = buildTestMatrix(numSeries, (r+1)*numSamplesPerSeriesPerPush, i) - } - // Check the samples are still there! - retrieveTestSamples(t, ing, userIDs, testData) - - if r != numRestarts-1 { - userIDs, testData = pushTestSamples(t, ing, numSeries, numSamplesPerSeriesPerPush, (r+1)*numSamplesPerSeriesPerPush) - } - - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - } - - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.CheckpointEnabled = true - - // Start a new ingester and recover the WAL. - _, ing = newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - userID := userIDs[0] - sampleStream := testData[userID][0] - lastSample := sampleStream.Values[len(sampleStream.Values)-1] - - // In-order and out of order sample in the same request. - metric := cortexpb.FromLabelAdaptersToLabels(cortexpb.FromMetricsToLabelAdapters(sampleStream.Metric)) - outOfOrderSample := cortexpb.Sample{TimestampMs: int64(lastSample.Timestamp - 10), Value: 99} - inOrderSample := cortexpb.Sample{TimestampMs: int64(lastSample.Timestamp + 10), Value: 999} - - ctx := user.InjectOrgID(context.Background(), userID) - _, err = ing.Push(ctx, cortexpb.ToWriteRequest( - []labels.Labels{metric, metric}, - []cortexpb.Sample{outOfOrderSample, inOrderSample}, nil, cortexpb.API)) - require.Equal(t, httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(makeMetricValidationError(sampleOutOfOrder, metric, - fmt.Errorf("sample timestamp out of order; last timestamp: %v, incoming timestamp: %v", lastSample.Timestamp, model.Time(outOfOrderSample.TimestampMs))), userID).Error()), err) - - // We should have logged the in-order sample. - testData[userID][0].Values = append(testData[userID][0].Values, model.SamplePair{ - Timestamp: model.Time(inOrderSample.TimestampMs), - Value: model.SampleValue(inOrderSample.Value), - }) - - // Check samples after restart from WAL. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - _, ing = newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - retrieveTestSamples(t, ing, userIDs, testData) -} - -func TestCheckpointRepair(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.CheckpointEnabled = true - cfg.WALConfig.Recover = true - cfg.WALConfig.CheckpointDuration = 100 * time.Hour // Basically no automatic checkpoint. - - numSeries := 100 - numSamplesPerSeriesPerPush := 10 - for _, numCheckpoints := range []int{0, 1, 2, 3} { - dirname, err := ioutil.TempDir("", "cortex-wal") - require.NoError(t, err) - defer func() { - require.NoError(t, os.RemoveAll(dirname)) - }() - cfg.WALConfig.Dir = dirname - - // Build an ingester, add some samples, then shut it down. - _, ing := newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - w, ok := ing.wal.(*walWrapper) - require.True(t, ok) - - var userIDs []string - // Push some samples for the 0 checkpoints case. - // We dont shutdown the ingester in that case, else it will create a checkpoint. - userIDs, _ = pushTestSamples(t, ing, numSeries, numSamplesPerSeriesPerPush, 0) - for i := 0; i < numCheckpoints; i++ { - // First checkpoint. - userIDs, _ = pushTestSamples(t, ing, numSeries, numSamplesPerSeriesPerPush, (i+1)*numSamplesPerSeriesPerPush) - if i == numCheckpoints-1 { - // Shutdown creates a checkpoint. This is only for the last checkpoint. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - } else { - require.NoError(t, w.performCheckpoint(true)) - } - } - - require.Equal(t, float64(numCheckpoints), prom_testutil.ToFloat64(w.checkpointCreationTotal)) - - // Verify checkpoint dirs. - files, err := ioutil.ReadDir(w.wal.Dir()) - require.NoError(t, err) - numDirs := 0 - for _, f := range files { - if f.IsDir() { - numDirs++ - } - } - if numCheckpoints <= 1 { - require.Equal(t, numCheckpoints, numDirs) - } else { - // At max there are last 2 checkpoints on the disk. - require.Equal(t, 2, numDirs) - } - - if numCheckpoints > 0 { - // Corrupt the last checkpoint. - lastChDir, _, err := lastCheckpoint(w.wal.Dir()) - require.NoError(t, err) - files, err = ioutil.ReadDir(lastChDir) - require.NoError(t, err) - - lastFile, err := os.OpenFile(filepath.Join(lastChDir, files[len(files)-1].Name()), os.O_WRONLY, os.ModeAppend) - require.NoError(t, err) - n, err := lastFile.WriteAt([]byte{1, 2, 3, 4}, 2) - require.NoError(t, err) - require.Equal(t, 4, n) - require.NoError(t, lastFile.Close()) - } - - // Open an ingester for the repair. - _, ing = newTestStore(t, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - w, ok = ing.wal.(*walWrapper) - require.True(t, ok) - // defer in case we hit an error though we explicitly close it later. - defer func() { - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - }() - - if numCheckpoints > 0 { - require.Equal(t, 1.0, prom_testutil.ToFloat64(ing.metrics.walCorruptionsTotal)) - } else { - require.Equal(t, 0.0, prom_testutil.ToFloat64(ing.metrics.walCorruptionsTotal)) - } - - // Verify checkpoint dirs after the corrupt checkpoint is deleted. - files, err = ioutil.ReadDir(w.wal.Dir()) - require.NoError(t, err) - numDirs = 0 - for _, f := range files { - if f.IsDir() { - numDirs++ - } - } - if numCheckpoints <= 1 { - // The only checkpoint is removed (or) there was no checkpoint at all. - require.Equal(t, 0, numDirs) - } else { - // There is at max last 2 checkpoints. Hence only 1 should be remaining. - require.Equal(t, 1, numDirs) - } - - testData := map[string]model.Matrix{} - // Verify we did not lose any data. - for i, userID := range userIDs { - // 'numCheckpoints*' because we ingested the data 'numCheckpoints' number of time. - testData[userID] = buildTestMatrix(numSeries, (numCheckpoints+1)*numSamplesPerSeriesPerPush, i) - } - retrieveTestSamples(t, ing, userIDs, testData) - - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), ing)) - } - -} - -func TestCheckpointIndex(t *testing.T) { - tcs := []struct { - filename string - includeTmp bool - index int - shouldError bool - }{ - { - filename: "checkpoint.123456", - includeTmp: false, - index: 123456, - shouldError: false, - }, - { - filename: "checkpoint.123456", - includeTmp: true, - index: 123456, - shouldError: false, - }, - { - filename: "checkpoint.123456.tmp", - includeTmp: true, - index: 123456, - shouldError: false, - }, - { - filename: "checkpoint.123456.tmp", - includeTmp: false, - shouldError: true, - }, - { - filename: "not-checkpoint.123456.tmp", - includeTmp: true, - shouldError: true, - }, - { - filename: "checkpoint.123456.tmp2", - shouldError: true, - }, - { - filename: "checkpoints123456", - shouldError: true, - }, - { - filename: "012345", - shouldError: true, - }, - } - for _, tc := range tcs { - index, err := checkpointIndex(tc.filename, tc.includeTmp) - if tc.shouldError { - require.Error(t, err, "filename: %s, includeTmp: %t", tc.filename, tc.includeTmp) - continue - } - - require.NoError(t, err, "filename: %s, includeTmp: %t", tc.filename, tc.includeTmp) - require.Equal(t, tc.index, index) - } -} - -func BenchmarkWALReplay(b *testing.B) { - dirname, err := ioutil.TempDir("", "cortex-wal") - require.NoError(b, err) - defer func() { - require.NoError(b, os.RemoveAll(dirname)) - }() - - cfg := defaultIngesterTestConfig(b) - cfg.WALConfig.WALEnabled = true - cfg.WALConfig.CheckpointEnabled = true - cfg.WALConfig.Recover = true - cfg.WALConfig.Dir = dirname - cfg.WALConfig.CheckpointDuration = 100 * time.Minute - cfg.WALConfig.checkpointDuringShutdown = false - - numSeries := 10 - numSamplesPerSeriesPerPush := 2 - numPushes := 100000 - - _, ing := newTestStore(b, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - - // Add samples for the checkpoint. - for r := 0; r < numPushes; r++ { - _, _ = pushTestSamples(b, ing, numSeries, numSamplesPerSeriesPerPush, r*numSamplesPerSeriesPerPush) - } - w, ok := ing.wal.(*walWrapper) - require.True(b, ok) - require.NoError(b, w.performCheckpoint(true)) - - // Add samples for the additional WAL not in checkpoint. - for r := 0; r < numPushes; r++ { - _, _ = pushTestSamples(b, ing, numSeries, numSamplesPerSeriesPerPush, (numPushes+r)*numSamplesPerSeriesPerPush) - } - - require.NoError(b, services.StopAndAwaitTerminated(context.Background(), ing)) - - var ing2 *Ingester - b.Run("wal replay", func(b *testing.B) { - // Replay will happen here. - _, ing2 = newTestStore(b, cfg, defaultClientTestConfig(), defaultLimitsTestConfig(), nil) - }) - require.NoError(b, services.StopAndAwaitTerminated(context.Background(), ing2)) -} diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 3d6952a72f0..2da95b10dc8 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -144,7 +144,7 @@ func NewChunkStoreQueryable(cfg Config, chunkStore chunkstore.ChunkStore) storag } // New builds a queryable and promql engine. -func New(cfg Config, limits *validation.Overrides, distributor Distributor, stores []QueryableWithFilter, tombstonesLoader *purger.TombstonesLoader, reg prometheus.Registerer, logger log.Logger) (storage.SampleAndChunkQueryable, storage.ExemplarQueryable, *promql.Engine) { +func New(cfg Config, limits *validation.Overrides, distributor Distributor, stores []QueryableWithFilter, tombstonesLoader purger.TombstonesLoader, reg prometheus.Registerer, logger log.Logger) (storage.SampleAndChunkQueryable, storage.ExemplarQueryable, *promql.Engine) { iteratorFunc := getChunksIteratorFunction(cfg) distributorQueryable := newDistributorQueryable(distributor, cfg.IngesterStreaming, iteratorFunc, cfg.QueryIngestersWithin) @@ -216,7 +216,7 @@ type QueryableWithFilter interface { } // NewQueryable creates a new Queryable for cortex. -func NewQueryable(distributor QueryableWithFilter, stores []QueryableWithFilter, chunkIterFn chunkIteratorFunc, cfg Config, limits *validation.Overrides, tombstonesLoader *purger.TombstonesLoader) storage.Queryable { +func NewQueryable(distributor QueryableWithFilter, stores []QueryableWithFilter, chunkIterFn chunkIteratorFunc, cfg Config, limits *validation.Overrides, tombstonesLoader purger.TombstonesLoader) storage.Queryable { return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { now := time.Now() @@ -284,7 +284,7 @@ type querier struct { ctx context.Context mint, maxt int64 - tombstonesLoader *purger.TombstonesLoader + tombstonesLoader purger.TombstonesLoader limits *validation.Overrides maxQueryIntoFuture time.Duration queryStoreForLabels bool diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index 232ce2e99e2..703c3c31d8b 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -161,7 +161,7 @@ func TestQuerier(t *testing.T) { require.NoError(t, err) queryables := []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore)), UseAlwaysQueryable(db)} - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) testRangeQuery(t, queryable, through, query) }) } @@ -283,7 +283,7 @@ func TestNoHistoricalQueryToIngester(t *testing.T) { overrides, err := validation.NewOverrides(DefaultLimitsConfig(), nil) require.NoError(t, err) - queryable, _, _ := New(cfg, overrides, distributor, []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))}, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))}, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) query, err := engine.NewRangeQuery(queryable, nil, "dummy", c.mint, c.maxt, 1*time.Minute) require.NoError(t, err) @@ -372,7 +372,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryIntoFuture(t *testing.T) { require.NoError(t, err) queryables := []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))} - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) query, err := engine.NewRangeQuery(queryable, nil, "dummy", c.queryStartTime, c.queryEndTime, time.Minute) require.NoError(t, err) @@ -448,7 +448,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLength(t *testing.T) { distributor := &emptyDistributor{} queryables := []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))} - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) // Create the PromQL engine to execute the query. engine := promql.NewEngine(promql.EngineOpts{ @@ -574,7 +574,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor.On("Query", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(model.Matrix{}, nil) distributor.On("QueryStream", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&client.QueryStreamResponse{}, nil) - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) require.NoError(t, err) query, err := engine.NewRangeQuery(queryable, nil, testData.query, testData.queryStartTime, testData.queryEndTime, time.Minute) @@ -602,7 +602,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor := &MockDistributor{} distributor.On("MetricsForLabelMatchers", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]metric.Metric{}, nil) - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) @@ -634,7 +634,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor := &MockDistributor{} distributor.On("LabelNames", mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) @@ -661,7 +661,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor := &MockDistributor{} distributor.On("MetricsForLabelMatchers", mock.Anything, mock.Anything, mock.Anything, matchers).Return([]metric.Metric{}, nil) - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) @@ -687,7 +687,7 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { distributor := &MockDistributor{} distributor.On("LabelValuesForLabelName", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) - queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, queryables, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) q, err := queryable.Querier(ctx, util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) require.NoError(t, err) @@ -905,7 +905,7 @@ func TestShortTermQueryToLTS(t *testing.T) { overrides, err := validation.NewOverrides(DefaultLimitsConfig(), nil) require.NoError(t, err) - queryable, _, _ := New(cfg, overrides, distributor, []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))}, purger.NewTombstonesLoader(nil, nil), nil, log.NewNopLogger()) + queryable, _, _ := New(cfg, overrides, distributor, []QueryableWithFilter{UseAlwaysQueryable(NewChunkStoreQueryable(cfg, chunkStore))}, purger.NewNoopTombstonesLoader(), nil, log.NewNopLogger()) query, err := engine.NewRangeQuery(queryable, nil, "dummy", c.mint, c.maxt, 1*time.Minute) require.NoError(t, err) diff --git a/pkg/querier/series/series_set.go b/pkg/querier/series/series_set.go index 76896c6e8a6..ee3a9a4ef08 100644 --- a/pkg/querier/series/series_set.go +++ b/pkg/querier/series/series_set.go @@ -197,11 +197,11 @@ func (b byLabels) Less(i, j int) bool { return labels.Compare(b[i].Labels(), b[j type DeletedSeriesSet struct { seriesSet storage.SeriesSet - tombstones *purger.TombstonesSet + tombstones purger.TombstonesSet queryInterval model.Interval } -func NewDeletedSeriesSet(seriesSet storage.SeriesSet, tombstones *purger.TombstonesSet, queryInterval model.Interval) storage.SeriesSet { +func NewDeletedSeriesSet(seriesSet storage.SeriesSet, tombstones purger.TombstonesSet, queryInterval model.Interval) storage.SeriesSet { return &DeletedSeriesSet{ seriesSet: seriesSet, tombstones: tombstones, diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index d356a742b2d..1f0d4f04010 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -825,18 +825,6 @@ func (i *Lifecycler) SetUnregisterOnShutdown(enabled bool) { func (i *Lifecycler) processShutdown(ctx context.Context) { flushRequired := i.flushOnShutdown.Load() - transferStart := time.Now() - if err := i.flushTransferer.TransferOut(ctx); err != nil { - if err == ErrTransferDisabled { - level.Info(i.logger).Log("msg", "transfers are disabled") - } else { - level.Error(i.logger).Log("msg", "failed to transfer chunks to another instance", "ring", i.RingName, "err", err) - i.lifecyclerMetrics.shutdownDuration.WithLabelValues("transfer", "fail").Observe(time.Since(transferStart).Seconds()) - } - } else { - flushRequired = false - i.lifecyclerMetrics.shutdownDuration.WithLabelValues("transfer", "success").Observe(time.Since(transferStart).Seconds()) - } if flushRequired { flushStart := time.Now() diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go index a05d24a2f24..9ef2f02a00f 100644 --- a/pkg/ruler/ruler_test.go +++ b/pkg/ruler/ruler_test.go @@ -131,7 +131,7 @@ func testQueryableFunc(querierTestConfig *querier.TestConfig, reg prometheus.Reg querierTestConfig.Cfg.ActiveQueryTrackerDir = "" overrides, _ := validation.NewOverrides(querier.DefaultLimitsConfig(), nil) - q, _, _ := querier.New(querierTestConfig.Cfg, overrides, querierTestConfig.Distributor, querierTestConfig.Stores, purger.NewTombstonesLoader(nil, nil), reg, logger) + q, _, _ := querier.New(querierTestConfig.Cfg, overrides, querierTestConfig.Distributor, querierTestConfig.Stores, purger.NewNoopTombstonesLoader(), reg, logger) return func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { return q.Querier(ctx, mint, maxt) } diff --git a/pkg/util/chunkcompat/compat.go b/pkg/util/chunkcompat/compat.go index 8021497e488..ae6d22aaa27 100644 --- a/pkg/util/chunkcompat/compat.go +++ b/pkg/util/chunkcompat/compat.go @@ -13,20 +13,6 @@ import ( "github.com/cortexproject/cortex/pkg/util" ) -// StreamsToMatrix converts a slice of QueryStreamResponse to a model.Matrix. -func StreamsToMatrix(from, through model.Time, responses []*client.QueryStreamResponse) (model.Matrix, error) { - result := model.Matrix{} - for _, response := range responses { - series, err := SeriesChunksToMatrix(from, through, response.Chunkseries) - if err != nil { - return nil, err - } - - result = append(result, series...) - } - return result, nil -} - // SeriesChunksToMatrix converts slice of []client.TimeSeriesChunk to a model.Matrix. func SeriesChunksToMatrix(from, through model.Time, serieses []client.TimeSeriesChunk) (model.Matrix, error) { if serieses == nil { diff --git a/tools/doc-generator/main.go b/tools/doc-generator/main.go index 946760de969..5aaf9f1e73a 100644 --- a/tools/doc-generator/main.go +++ b/tools/doc-generator/main.go @@ -15,7 +15,6 @@ import ( "github.com/cortexproject/cortex/pkg/alertmanager/alertstore" "github.com/cortexproject/cortex/pkg/chunk" "github.com/cortexproject/cortex/pkg/chunk/cache" - "github.com/cortexproject/cortex/pkg/chunk/purger" "github.com/cortexproject/cortex/pkg/chunk/storage" "github.com/cortexproject/cortex/pkg/compactor" "github.com/cortexproject/cortex/pkg/configs" @@ -194,11 +193,6 @@ var ( structType: reflect.TypeOf(storegateway.Config{}), desc: "The store_gateway_config configures the store-gateway service used by the blocks storage.", }, - { - name: "purger_config", - structType: reflect.TypeOf(purger.Config{}), - desc: "The purger_config configures the purger which takes care of delete requests.", - }, { name: "s3_sse_config", structType: reflect.TypeOf(s3.SSEConfig{}), diff --git a/vendor/modules.txt b/vendor/modules.txt index 3879be6b1c7..e876bf910b3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1011,3 +1011,4 @@ sigs.k8s.io/yaml # github.com/gocql/gocql => github.com/grafana/gocql v0.0.0-20200605141915-ba5dc39ece85 # github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab # github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b +# github.com/efficientgo/tools/core => github.com/efficientgo/tools/core v0.0.0-20210829154005-c7bad8450208