From 6ea7e00271bd742ee61c2562432e568a5d6704e2 Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Wed, 10 Aug 2022 21:05:46 -0700 Subject: [PATCH] remove ingester v2 Signed-off-by: Alan Protasio --- pkg/ingester/flush.go | 4 +- pkg/ingester/ingester.go | 2477 ++++++++++++++++++- pkg/ingester/ingester_test.go | 3871 ++++++++++++++++++++++++++++- pkg/ingester/ingester_v2.go | 2360 ------------------ pkg/ingester/ingester_v2_test.go | 3893 ------------------------------ 5 files changed, 6249 insertions(+), 6356 deletions(-) delete mode 100644 pkg/ingester/ingester_v2.go delete mode 100644 pkg/ingester/ingester_v2_test.go diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index 60f793f0790..abfc4d0f87e 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -7,11 +7,11 @@ import ( // Flush triggers a flush of all the chunks and closes the flush queues. // Called from the Lifecycler as part of the ingester shutdown. func (i *Ingester) Flush() { - i.v2LifecyclerFlush() + i.lifecyclerFlush() } // FlushHandler triggers a flush of all in memory chunks. Mainly used for // local testing. func (i *Ingester) FlushHandler(w http.ResponseWriter, r *http.Request) { - i.v2FlushHandler(w, r) + i.flushHandler(w, r) } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index b57414480dd..99ede3c93f9 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -4,7 +4,11 @@ import ( "context" "flag" "fmt" + "io" + "math" "net/http" + "os" + "path/filepath" "strings" "sync" "time" @@ -12,23 +16,56 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/gogo/status" + "github.com/oklog/ulid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/exemplar" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/thanos-io/thanos/pkg/block/metadata" + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/shipper" + "github.com/weaveworks/common/httpgrpc" "go.uber.org/atomic" + "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" + "github.com/cortexproject/cortex/pkg/chunk/encoding" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" - "github.com/cortexproject/cortex/pkg/storage/tsdb" + "github.com/cortexproject/cortex/pkg/storage/bucket" + cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" "github.com/cortexproject/cortex/pkg/tenant" + "github.com/cortexproject/cortex/pkg/util" + "github.com/cortexproject/cortex/pkg/util/concurrency" + "github.com/cortexproject/cortex/pkg/util/extract" logutil "github.com/cortexproject/cortex/pkg/util/log" util_math "github.com/cortexproject/cortex/pkg/util/math" "github.com/cortexproject/cortex/pkg/util/services" + "github.com/cortexproject/cortex/pkg/util/spanlogger" "github.com/cortexproject/cortex/pkg/util/validation" ) const ( + // RingKey is the key under which we store the ingesters ring in the KVStore. + RingKey = "ring" +) + +const ( + errTSDBCreateIncompatibleState = "cannot create a new TSDB while the ingester is not in active state (current state: %s)" + errTSDBIngest = "err: %v. timestamp=%s, series=%s" // Using error.Wrap puts the message before the error and if the series is too long, its truncated. + errTSDBIngestExemplar = "err: %v. timestamp=%s, series=%s, exemplar=%s" + + // Jitter applied to the idle timeout to prevent compaction in all ingesters concurrently. + compactionIdleTimeoutJitter = 0.25 + + instanceIngestionRateTickInterval = time.Second + // Number of timeseries to return in each batch of a QueryStream. queryStreamBatchSize = 128 metadataStreamBatchSize = 128 @@ -42,6 +79,7 @@ const ( ) var ( + errExemplarRef = errors.New("exemplars not ingested because series not already present") errIngesterStopping = errors.New("ingester stopping") ) @@ -59,8 +97,8 @@ type Config struct { ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout"` // Use blocks storage. - BlocksStorageConfig tsdb.BlocksStorageConfig `yaml:"-"` - StreamChunksWhenUsingBlocks bool `yaml:"-"` + BlocksStorageConfig cortex_tsdb.BlocksStorageConfig `yaml:"-"` + StreamChunksWhenUsingBlocks bool `yaml:"-"` // Runtime-override for type of streaming query to use (chunks or samples). StreamTypeFn func() QueryStreamType `yaml:"-"` @@ -149,23 +187,656 @@ type Ingester struct { inflightPushRequests atomic.Int64 } -// New constructs a new Ingester. +// Shipper interface is used to have an easy way to mock it in tests. +type Shipper interface { + Sync(ctx context.Context) (uploaded int, err error) +} + +type tsdbState int + +const ( + active tsdbState = iota // Pushes are allowed. + activeShipping // Pushes are allowed. Blocks shipping is in progress. + forceCompacting // TSDB is being force-compacted. + closing // Used while closing idle TSDB. + closed // Used to avoid setting closing back to active in closeAndDeleteIdleUsers method. +) + +// Describes result of TSDB-close check. String is used as metric label. +type tsdbCloseCheckResult string + +const ( + tsdbIdle tsdbCloseCheckResult = "idle" // Not reported via metrics. Metrics use tsdbIdleClosed on success. + tsdbShippingDisabled tsdbCloseCheckResult = "shipping_disabled" + tsdbNotIdle tsdbCloseCheckResult = "not_idle" + tsdbNotCompacted tsdbCloseCheckResult = "not_compacted" + tsdbNotShipped tsdbCloseCheckResult = "not_shipped" + tsdbCheckFailed tsdbCloseCheckResult = "check_failed" + tsdbCloseFailed tsdbCloseCheckResult = "close_failed" + tsdbNotActive tsdbCloseCheckResult = "not_active" + tsdbDataRemovalFailed tsdbCloseCheckResult = "data_removal_failed" + tsdbTenantMarkedForDeletion tsdbCloseCheckResult = "tenant_marked_for_deletion" + tsdbIdleClosed tsdbCloseCheckResult = "idle_closed" // Success. +) + +func (r tsdbCloseCheckResult) shouldClose() bool { + return r == tsdbIdle || r == tsdbTenantMarkedForDeletion +} + +// QueryStreamType defines type of function to use when doing query-stream operation. +type QueryStreamType int + +const ( + QueryStreamDefault QueryStreamType = iota // Use default configured value. + QueryStreamSamples // Stream individual samples. + QueryStreamChunks // Stream entire chunks. +) + +type userTSDB struct { + db *tsdb.DB + userID string + activeSeries *ActiveSeries + seriesInMetric *metricCounter + limiter *Limiter + + instanceSeriesCount *atomic.Int64 // Shared across all userTSDB instances created by ingester. + instanceLimitsFn func() *InstanceLimits + + stateMtx sync.RWMutex + state tsdbState + pushesInFlight sync.WaitGroup // Increased with stateMtx read lock held, only if state == active or activeShipping. + + // Used to detect idle TSDBs. + lastUpdate atomic.Int64 + + // Thanos shipper used to ship blocks to the storage. + shipper Shipper + + // When deletion marker is found for the tenant (checked before shipping), + // shipping stops and TSDB is closed before reaching idle timeout time (if enabled). + deletionMarkFound atomic.Bool + + // Unix timestamp of last deletion mark check. + lastDeletionMarkCheck atomic.Int64 + + // for statistics + ingestedAPISamples *util_math.EwmaRate + ingestedRuleSamples *util_math.EwmaRate + + // Cached shipped blocks. + shippedBlocksMtx sync.Mutex + shippedBlocks map[ulid.ULID]struct{} +} + +// Explicitly wrapping the tsdb.DB functions that we use. + +func (u *userTSDB) Appender(ctx context.Context) storage.Appender { + return u.db.Appender(ctx) +} + +func (u *userTSDB) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { + return u.db.Querier(ctx, mint, maxt) +} + +func (u *userTSDB) ChunkQuerier(ctx context.Context, mint, maxt int64) (storage.ChunkQuerier, error) { + return u.db.ChunkQuerier(ctx, mint, maxt) +} + +func (u *userTSDB) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) { + return u.db.ExemplarQuerier(ctx) +} + +func (u *userTSDB) Head() *tsdb.Head { + return u.db.Head() +} + +func (u *userTSDB) Blocks() []*tsdb.Block { + return u.db.Blocks() +} + +func (u *userTSDB) Close() error { + return u.db.Close() +} + +func (u *userTSDB) Compact() error { + return u.db.Compact() +} + +func (u *userTSDB) StartTime() (int64, error) { + return u.db.StartTime() +} + +func (u *userTSDB) casState(from, to tsdbState) bool { + u.stateMtx.Lock() + defer u.stateMtx.Unlock() + + if u.state != from { + return false + } + u.state = to + return true +} + +// compactHead compacts the Head block at specified block durations avoiding a single huge block. +func (u *userTSDB) compactHead(blockDuration int64) error { + if !u.casState(active, forceCompacting) { + return errors.New("TSDB head cannot be compacted because it is not in active state (possibly being closed or blocks shipping in progress)") + } + + defer u.casState(forceCompacting, active) + + // Ingestion of samples in parallel with forced compaction can lead to overlapping blocks, + // and possible invalidation of the references returned from Appender.GetRef(). + // So we wait for existing in-flight requests to finish. Future push requests would fail until compaction is over. + u.pushesInFlight.Wait() + + h := u.Head() + + minTime, maxTime := h.MinTime(), h.MaxTime() + + for (minTime/blockDuration)*blockDuration != (maxTime/blockDuration)*blockDuration { + // Data in Head spans across multiple block ranges, so we break it into blocks here. + // Block max time is exclusive, so we do a -1 here. + blockMaxTime := ((minTime/blockDuration)+1)*blockDuration - 1 + if err := u.db.CompactHead(tsdb.NewRangeHead(h, minTime, blockMaxTime)); err != nil { + return err + } + + // Get current min/max times after compaction. + minTime, maxTime = h.MinTime(), h.MaxTime() + } + + return u.db.CompactHead(tsdb.NewRangeHead(h, minTime, maxTime)) +} + +// PreCreation implements SeriesLifecycleCallback interface. +func (u *userTSDB) PreCreation(metric labels.Labels) error { + if u.limiter == nil { + return nil + } + + // Verify ingester's global limit + gl := u.instanceLimitsFn() + if gl != nil && gl.MaxInMemorySeries > 0 { + if series := u.instanceSeriesCount.Load(); series >= gl.MaxInMemorySeries { + return errMaxSeriesLimitReached + } + } + + // Total series limit. + if err := u.limiter.AssertMaxSeriesPerUser(u.userID, int(u.Head().NumSeries())); err != nil { + return err + } + + // Series per metric name limit. + metricName, err := extract.MetricNameFromLabels(metric) + if err != nil { + return err + } + if err := u.seriesInMetric.canAddSeriesFor(u.userID, metricName); err != nil { + return err + } + + return nil +} + +// PostCreation implements SeriesLifecycleCallback interface. +func (u *userTSDB) PostCreation(metric labels.Labels) { + u.instanceSeriesCount.Inc() + + metricName, err := extract.MetricNameFromLabels(metric) + if err != nil { + // This should never happen because it has already been checked in PreCreation(). + return + } + u.seriesInMetric.increaseSeriesForMetric(metricName) +} + +// PostDeletion implements SeriesLifecycleCallback interface. +func (u *userTSDB) PostDeletion(metrics ...labels.Labels) { + u.instanceSeriesCount.Sub(int64(len(metrics))) + + for _, metric := range metrics { + metricName, err := extract.MetricNameFromLabels(metric) + if err != nil { + // This should never happen because it has already been checked in PreCreation(). + continue + } + u.seriesInMetric.decreaseSeriesForMetric(metricName) + } +} + +// blocksToDelete filters the input blocks and returns the blocks which are safe to be deleted from the ingester. +func (u *userTSDB) blocksToDelete(blocks []*tsdb.Block) map[ulid.ULID]struct{} { + if u.db == nil { + return nil + } + deletable := tsdb.DefaultBlocksToDelete(u.db)(blocks) + if u.shipper == nil { + return deletable + } + + shippedBlocks := u.getCachedShippedBlocks() + + result := map[ulid.ULID]struct{}{} + for shippedID := range shippedBlocks { + if _, ok := deletable[shippedID]; ok { + result[shippedID] = struct{}{} + } + } + return result +} + +// updateCachedShipperBlocks reads the shipper meta file and updates the cached shipped blocks. +func (u *userTSDB) updateCachedShippedBlocks() error { + shipperMeta, err := shipper.ReadMetaFile(u.db.Dir()) + if os.IsNotExist(err) || os.IsNotExist(errors.Cause(err)) { + // If the meta file doesn't exist it means the shipper hasn't run yet. + shipperMeta = &shipper.Meta{} + } else if err != nil { + return err + } + + // Build a map. + shippedBlocks := make(map[ulid.ULID]struct{}, len(shipperMeta.Uploaded)) + for _, blockID := range shipperMeta.Uploaded { + shippedBlocks[blockID] = struct{}{} + } + + // Cache it. + u.shippedBlocksMtx.Lock() + u.shippedBlocks = shippedBlocks + u.shippedBlocksMtx.Unlock() + + return nil +} + +// getCachedShippedBlocks returns the cached shipped blocks. +func (u *userTSDB) getCachedShippedBlocks() map[ulid.ULID]struct{} { + u.shippedBlocksMtx.Lock() + defer u.shippedBlocksMtx.Unlock() + + // It's safe to directly return the map because it's never updated in-place. + return u.shippedBlocks +} + +// getOldestUnshippedBlockTime returns the unix timestamp with milliseconds precision of the oldest +// TSDB block not shipped to the storage yet, or 0 if all blocks have been shipped. +func (u *userTSDB) getOldestUnshippedBlockTime() uint64 { + shippedBlocks := u.getCachedShippedBlocks() + oldestTs := uint64(0) + + for _, b := range u.Blocks() { + if _, ok := shippedBlocks[b.Meta().ULID]; ok { + continue + } + + if oldestTs == 0 || b.Meta().ULID.Time() < oldestTs { + oldestTs = b.Meta().ULID.Time() + } + } + + return oldestTs +} + +func (u *userTSDB) isIdle(now time.Time, idle time.Duration) bool { + lu := u.lastUpdate.Load() + + return time.Unix(lu, 0).Add(idle).Before(now) +} + +func (u *userTSDB) setLastUpdate(t time.Time) { + u.lastUpdate.Store(t.Unix()) +} + +// Checks if TSDB can be closed. +func (u *userTSDB) shouldCloseTSDB(idleTimeout time.Duration) tsdbCloseCheckResult { + if u.deletionMarkFound.Load() { + return tsdbTenantMarkedForDeletion + } + + if !u.isIdle(time.Now(), idleTimeout) { + return tsdbNotIdle + } + + // If head is not compacted, we cannot close this yet. + if u.Head().NumSeries() > 0 { + return tsdbNotCompacted + } + + // Ensure that all blocks have been shipped. + if oldest := u.getOldestUnshippedBlockTime(); oldest > 0 { + return tsdbNotShipped + } + + return tsdbIdle +} + +// TSDBState holds data structures used by the TSDB storage engine +type TSDBState struct { + dbs map[string]*userTSDB // tsdb sharded by userID + bucket objstore.Bucket + + // Value used by shipper as external label. + shipperIngesterID string + + subservices *services.Manager + + tsdbMetrics *tsdbMetrics + + forceCompactTrigger chan requestWithUsersAndCallback + shipTrigger chan requestWithUsersAndCallback + + // Timeout chosen for idle compactions. + compactionIdleTimeout time.Duration + + // Number of series in memory, across all tenants. + seriesCount atomic.Int64 + + // Head compactions metrics. + compactionsTriggered prometheus.Counter + compactionsFailed prometheus.Counter + walReplayTime prometheus.Histogram + appenderAddDuration prometheus.Histogram + appenderCommitDuration prometheus.Histogram + idleTsdbChecks *prometheus.CounterVec +} + +type requestWithUsersAndCallback struct { + users *util.AllowedTenants // if nil, all tenants are allowed. + callback chan<- struct{} // when compaction/shipping is finished, this channel is closed +} + +func newTSDBState(bucketClient objstore.Bucket, registerer prometheus.Registerer) TSDBState { + idleTsdbChecks := promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_ingester_idle_tsdb_checks_total", + Help: "The total number of various results for idle TSDB checks.", + }, []string{"result"}) + + idleTsdbChecks.WithLabelValues(string(tsdbShippingDisabled)) + idleTsdbChecks.WithLabelValues(string(tsdbNotIdle)) + idleTsdbChecks.WithLabelValues(string(tsdbNotCompacted)) + idleTsdbChecks.WithLabelValues(string(tsdbNotShipped)) + idleTsdbChecks.WithLabelValues(string(tsdbCheckFailed)) + idleTsdbChecks.WithLabelValues(string(tsdbCloseFailed)) + idleTsdbChecks.WithLabelValues(string(tsdbNotActive)) + idleTsdbChecks.WithLabelValues(string(tsdbDataRemovalFailed)) + idleTsdbChecks.WithLabelValues(string(tsdbTenantMarkedForDeletion)) + idleTsdbChecks.WithLabelValues(string(tsdbIdleClosed)) + + return TSDBState{ + dbs: make(map[string]*userTSDB), + bucket: bucketClient, + tsdbMetrics: newTSDBMetrics(registerer), + forceCompactTrigger: make(chan requestWithUsersAndCallback), + shipTrigger: make(chan requestWithUsersAndCallback), + + compactionsTriggered: promauto.With(registerer).NewCounter(prometheus.CounterOpts{ + Name: "cortex_ingester_tsdb_compactions_triggered_total", + Help: "Total number of triggered compactions.", + }), + + compactionsFailed: promauto.With(registerer).NewCounter(prometheus.CounterOpts{ + Name: "cortex_ingester_tsdb_compactions_failed_total", + Help: "Total number of compactions that failed.", + }), + walReplayTime: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ + Name: "cortex_ingester_tsdb_wal_replay_duration_seconds", + Help: "The total time it takes to open and replay a TSDB WAL.", + Buckets: prometheus.DefBuckets, + }), + appenderAddDuration: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ + Name: "cortex_ingester_tsdb_appender_add_duration_seconds", + Help: "The total time it takes for a push request to add samples to the TSDB appender.", + Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}, + }), + appenderCommitDuration: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ + Name: "cortex_ingester_tsdb_appender_commit_duration_seconds", + Help: "The total time it takes for a push request to commit samples appended to TSDB.", + Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}, + }), + + idleTsdbChecks: idleTsdbChecks, + } +} + +// NewV2 returns a new Ingester that uses Cortex block storage instead of chunks storage. func New(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { defaultInstanceLimits = &cfg.DefaultLimits - if cfg.ingesterClientFactory == nil { cfg.ingesterClientFactory = client.MakeIngesterClient } - return NewV2(cfg, limits, registerer, logger) + bucketClient, err := bucket.NewClient(context.Background(), cfg.BlocksStorageConfig.Bucket, "ingester", logger, registerer) + if err != nil { + return nil, errors.Wrap(err, "failed to create the bucket client") + } + + i := &Ingester{ + cfg: cfg, + limits: limits, + usersMetadata: map[string]*userMetricsMetadata{}, + TSDBState: newTSDBState(bucketClient, registerer), + logger: logger, + ingestionRate: util_math.NewEWMARate(0.2, instanceIngestionRateTickInterval), + } + i.metrics = newIngesterMetrics(registerer, false, cfg.ActiveSeriesMetricsEnabled, i.getInstanceLimits, i.ingestionRate, &i.inflightPushRequests) + + // Replace specific metrics which we can't directly track but we need to read + // them from the underlying system (ie. TSDB). + if registerer != nil { + registerer.Unregister(i.metrics.memSeries) + + promauto.With(registerer).NewGaugeFunc(prometheus.GaugeOpts{ + Name: "cortex_ingester_memory_series", + Help: "The current number of series in memory.", + }, i.getMemorySeriesMetric) + + promauto.With(registerer).NewGaugeFunc(prometheus.GaugeOpts{ + Name: "cortex_ingester_oldest_unshipped_block_timestamp_seconds", + Help: "Unix timestamp of the oldest TSDB block not shipped to the storage yet. 0 if ingester has no blocks or all blocks have been shipped.", + }, i.getOldestUnshippedBlockMetric) + } + + i.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, i, "ingester", RingKey, cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown, logger, prometheus.WrapRegistererWithPrefix("cortex_", registerer)) + if err != nil { + return nil, err + } + i.subservicesWatcher = services.NewFailureWatcher() + i.subservicesWatcher.WatchService(i.lifecycler) + + // Init the limter and instantiate the user states which depend on it + i.limiter = NewLimiter( + limits, + i.lifecycler, + cfg.DistributorShardingStrategy, + cfg.DistributorShardByAllLabels, + cfg.LifecyclerConfig.RingConfig.ReplicationFactor, + cfg.LifecyclerConfig.RingConfig.ZoneAwarenessEnabled) + + i.TSDBState.shipperIngesterID = i.lifecycler.ID + + // Apply positive jitter only to ensure that the minimum timeout is adhered to. + i.TSDBState.compactionIdleTimeout = util.DurationWithPositiveJitter(i.cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout, compactionIdleTimeoutJitter) + level.Info(i.logger).Log("msg", "TSDB idle compaction timeout set", "timeout", i.TSDBState.compactionIdleTimeout) + + i.BasicService = services.NewBasicService(i.starting, i.updateLoop, i.stopping) + return i, nil } // NewForFlusher constructs a new Ingester to be used by flusher target. // Compared to the 'New' method: // * Always replays the WAL. // * Does not start the lifecycler. +// this is a special version of ingester used by Flusher. This ingester is not ingesting anything, its only purpose is to react +// on Flush method and flush all openened TSDBs when called. func NewForFlusher(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { - return NewV2ForFlusher(cfg, limits, registerer, logger) + bucketClient, err := bucket.NewClient(context.Background(), cfg.BlocksStorageConfig.Bucket, "ingester", logger, registerer) + if err != nil { + return nil, errors.Wrap(err, "failed to create the bucket client") + } + + i := &Ingester{ + cfg: cfg, + limits: limits, + TSDBState: newTSDBState(bucketClient, registerer), + logger: logger, + } + i.metrics = newIngesterMetrics(registerer, false, false, i.getInstanceLimits, nil, &i.inflightPushRequests) + + i.TSDBState.shipperIngesterID = "flusher" + + // This ingester will not start any subservices (lifecycler, compaction, shipping), + // and will only open TSDBs, wait for Flush to be called, and then close TSDBs again. + i.BasicService = services.NewIdleService(i.startingV2ForFlusher, i.stoppingV2ForFlusher) + return i, nil +} + +func (i *Ingester) startingV2ForFlusher(ctx context.Context) error { + if err := i.openExistingTSDB(ctx); err != nil { + // Try to rollback and close opened TSDBs before halting the ingester. + i.closeAllTSDB() + + return errors.Wrap(err, "opening existing TSDBs") + } + + // Don't start any sub-services (lifecycler, compaction, shipper) at all. + return nil +} + +func (i *Ingester) starting(ctx context.Context) error { + if err := i.openExistingTSDB(ctx); err != nil { + // Try to rollback and close opened TSDBs before halting the ingester. + i.closeAllTSDB() + + return errors.Wrap(err, "opening existing TSDBs") + } + + // Important: we want to keep lifecycler running until we ask it to stop, so we need to give it independent context + if err := i.lifecycler.StartAsync(context.Background()); err != nil { + return errors.Wrap(err, "failed to start lifecycler") + } + if err := i.lifecycler.AwaitRunning(ctx); err != nil { + return errors.Wrap(err, "failed to start lifecycler") + } + + // let's start the rest of subservices via manager + servs := []services.Service(nil) + + compactionService := services.NewBasicService(nil, i.compactionLoop, nil) + servs = append(servs, compactionService) + + if i.cfg.BlocksStorageConfig.TSDB.IsBlocksShippingEnabled() { + shippingService := services.NewBasicService(nil, i.shipBlocksLoop, nil) + servs = append(servs, shippingService) + } + + if i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout > 0 { + interval := i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval + if interval == 0 { + interval = cortex_tsdb.DefaultCloseIdleTSDBInterval + } + closeIdleService := services.NewTimerService(interval, nil, i.closeAndDeleteIdleUserTSDBs, nil) + servs = append(servs, closeIdleService) + } + + var err error + i.TSDBState.subservices, err = services.NewManager(servs...) + if err == nil { + err = services.StartManagerAndAwaitHealthy(ctx, i.TSDBState.subservices) + } + return errors.Wrap(err, "failed to start ingester components") +} + +func (i *Ingester) stoppingV2ForFlusher(_ error) error { + if !i.cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown { + i.closeAllTSDB() + } + return nil +} + +// runs when ingester is stopping +func (i *Ingester) stopping(_ error) error { + // This will prevent us accepting any more samples + i.stopIncomingRequests() + // It's important to wait until shipper is finished, + // because the blocks transfer should start only once it's guaranteed + // there's no shipping on-going. + if err := services.StopManagerAndAwaitStopped(context.Background(), i.TSDBState.subservices); err != nil { + level.Warn(i.logger).Log("msg", "failed to stop ingester subservices", "err", err) + } + + // Next initiate our graceful exit from the ring. + if err := services.StopAndAwaitTerminated(context.Background(), i.lifecycler); err != nil { + level.Warn(i.logger).Log("msg", "failed to stop ingester lifecycler", "err", err) + } + + if !i.cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown { + i.closeAllTSDB() + } + return nil +} + +func (i *Ingester) updateLoop(ctx context.Context) error { + if limits := i.getInstanceLimits(); limits != nil && *limits != (InstanceLimits{}) { + // This check will not cover enabling instance limits in runtime, but it will do for now. + logutil.WarnExperimentalUse("ingester instance limits") + } + + rateUpdateTicker := time.NewTicker(i.cfg.RateUpdatePeriod) + defer rateUpdateTicker.Stop() + + ingestionRateTicker := time.NewTicker(instanceIngestionRateTickInterval) + defer ingestionRateTicker.Stop() + + var activeSeriesTickerChan <-chan time.Time + if i.cfg.ActiveSeriesMetricsEnabled { + t := time.NewTicker(i.cfg.ActiveSeriesMetricsUpdatePeriod) + activeSeriesTickerChan = t.C + defer t.Stop() + } + + // Similarly to the above, this is a hardcoded value. + metadataPurgeTicker := time.NewTicker(metadataPurgePeriod) + defer metadataPurgeTicker.Stop() + + for { + select { + case <-metadataPurgeTicker.C: + i.purgeUserMetricsMetadata() + case <-ingestionRateTicker.C: + i.ingestionRate.Tick() + case <-rateUpdateTicker.C: + i.stoppedMtx.RLock() + for _, db := range i.TSDBState.dbs { + db.ingestedAPISamples.Tick() + db.ingestedRuleSamples.Tick() + } + i.stoppedMtx.RUnlock() + + case <-activeSeriesTickerChan: + i.updateActiveSeries() + + case <-ctx.Done(): + return nil + case err := <-i.subservicesWatcher.Chan(): + return errors.Wrap(err, "ingester subservice failed") + } + } +} + +func (i *Ingester) updateActiveSeries() { + purgeTime := time.Now().Add(-i.cfg.ActiveSeriesMetricsIdleTimeout) + + for _, userID := range i.getTSDBUsers() { + userDB := i.getTSDB(userID) + if userDB == nil { + continue + } + + userDB.activeSeries.Purge(purgeTime) + i.metrics.activeSeriesPerUser.WithLabelValues(userID).Set(float64(userDB.activeSeries.Active())) + } } // ShutdownHandler triggers the following set of operations in order: @@ -209,7 +880,13 @@ func (i *Ingester) checkRunning() error { return status.Error(codes.Unavailable, s.String()) } -// Push implements client.IngesterServer +// GetRef() is an extra method added to TSDB to let Cortex check before calling Add() +type extendedAppender interface { + storage.Appender + storage.GetRef +} + +// Push adds metrics to a block func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*cortexpb.WriteResponse, error) { if err := i.checkRunning(); err != nil { return nil, err @@ -226,53 +903,1570 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte } } - return i.v2Push(ctx, req) -} + var firstPartialErr error -// pushMetadata returns number of ingested metadata. -func (i *Ingester) pushMetadata(ctx context.Context, userID string, metadata []*cortexpb.MetricMetadata) int { - ingestedMetadata := 0 - failedMetadata := 0 + // NOTE: because we use `unsafe` in deserialisation, we must not + // retain anything from `req` past the call to ReuseSlice + defer cortexpb.ReuseSlice(req.Timeseries) - var firstMetadataErr error - for _, metadata := range metadata { - err := i.appendMetadata(userID, metadata) - if err == nil { - ingestedMetadata++ - continue - } + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, err + } - failedMetadata++ - if firstMetadataErr == nil { - firstMetadataErr = err + il := i.getInstanceLimits() + if il != nil && il.MaxIngestionRate > 0 { + if rate := i.ingestionRate.Rate(); rate >= il.MaxIngestionRate { + return nil, errMaxSamplesPushRateLimitReached } } - i.metrics.ingestedMetadata.Add(float64(ingestedMetadata)) - i.metrics.ingestedMetadataFail.Add(float64(failedMetadata)) - - // If we have any error with regard to metadata we just log and no-op. - // We consider metadata a best effort approach, errors here should not stop processing. - if firstMetadataErr != nil { - logger := logutil.WithContext(ctx, i.logger) - level.Warn(logger).Log("msg", "failed to ingest some metadata", "err", firstMetadataErr) + db, err := i.getOrCreateTSDB(userID, false) + if err != nil { + return nil, wrapWithUser(err, userID) } - return ingestedMetadata -} - -func (i *Ingester) appendMetadata(userID string, m *cortexpb.MetricMetadata) error { + // Ensure the ingester shutdown procedure hasn't started i.stoppedMtx.RLock() if i.stopped { i.stoppedMtx.RUnlock() - return errIngesterStopping + return nil, errIngesterStopping } i.stoppedMtx.RUnlock() - userMetadata := i.getOrCreateUserMetadata(userID) + if err := db.acquireAppendLock(); err != nil { + return &cortexpb.WriteResponse{}, httpgrpc.Errorf(http.StatusServiceUnavailable, wrapWithUser(err, userID).Error()) + } + defer db.releaseAppendLock() + + // Given metadata is a best-effort approach, and we don't halt on errors + // process it before samples. Otherwise, we risk returning an error before ingestion. + ingestedMetadata := i.pushMetadata(ctx, userID, req.GetMetadata()) + + // Keep track of some stats which are tracked only if the samples will be + // successfully committed + var ( + succeededSamplesCount = 0 + failedSamplesCount = 0 + succeededExemplarsCount = 0 + failedExemplarsCount = 0 + startAppend = time.Now() + sampleOutOfBoundsCount = 0 + sampleOutOfOrderCount = 0 + newValueForTimestampCount = 0 + perUserSeriesLimitCount = 0 + perMetricSeriesLimitCount = 0 + + updateFirstPartial = func(errFn func() error) { + if firstPartialErr == nil { + firstPartialErr = errFn() + } + } + ) + + // Walk the samples, appending them to the users database + app := db.Appender(ctx).(extendedAppender) + for _, ts := range req.Timeseries { + // The labels must be sorted (in our case, it's guaranteed a write request + // has sorted labels once hit the ingester). + + // Look up a reference for this series. + ref, copiedLabels := app.GetRef(cortexpb.FromLabelAdaptersToLabels(ts.Labels)) + + // To find out if any sample was added to this series, we keep old value. + oldSucceededSamplesCount := succeededSamplesCount + + for _, s := range ts.Samples { + var err error + + // If the cached reference exists, we try to use it. + if ref != 0 { + if _, err = app.Append(ref, copiedLabels, s.TimestampMs, s.Value); err == nil { + succeededSamplesCount++ + continue + } + + } else { + // Copy the label set because both TSDB and the active series tracker may retain it. + copiedLabels = cortexpb.FromLabelAdaptersToLabelsWithCopy(ts.Labels) + + // Retain the reference in case there are multiple samples for the series. + if ref, err = app.Append(0, copiedLabels, s.TimestampMs, s.Value); err == nil { + succeededSamplesCount++ + continue + } + } + + failedSamplesCount++ + + // Check if the error is a soft error we can proceed on. If so, we keep track + // of it, so that we can return it back to the distributor, which will return a + // 400 error to the client. The client (Prometheus) will not retry on 400, and + // we actually ingested all samples which haven't failed. + switch cause := errors.Cause(err); cause { + case storage.ErrOutOfBounds: + sampleOutOfBoundsCount++ + updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(s.TimestampMs), ts.Labels) }) + continue + + case storage.ErrOutOfOrderSample: + sampleOutOfOrderCount++ + updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(s.TimestampMs), ts.Labels) }) + continue + + case storage.ErrDuplicateSampleForTimestamp: + newValueForTimestampCount++ + updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(s.TimestampMs), ts.Labels) }) + continue + + case errMaxSeriesPerUserLimitExceeded: + perUserSeriesLimitCount++ + updateFirstPartial(func() error { return makeLimitError(perUserSeriesLimit, i.limiter.FormatError(userID, cause)) }) + continue + + case errMaxSeriesPerMetricLimitExceeded: + perMetricSeriesLimitCount++ + updateFirstPartial(func() error { + return makeMetricLimitError(perMetricSeriesLimit, copiedLabels, i.limiter.FormatError(userID, cause)) + }) + continue + } + + // The error looks an issue on our side, so we should rollback + if rollbackErr := app.Rollback(); rollbackErr != nil { + level.Warn(i.logger).Log("msg", "failed to rollback on error", "user", userID, "err", rollbackErr) + } + + return nil, wrapWithUser(err, userID) + } - return userMetadata.add(m.GetMetricFamilyName(), m) -} + if i.cfg.ActiveSeriesMetricsEnabled && succeededSamplesCount > oldSucceededSamplesCount { + db.activeSeries.UpdateSeries(cortexpb.FromLabelAdaptersToLabels(ts.Labels), startAppend, func(l labels.Labels) labels.Labels { + // we must already have copied the labels if succeededSamplesCount has been incremented. + return copiedLabels + }) + } + + if i.cfg.BlocksStorageConfig.TSDB.MaxExemplars > 0 { + // app.AppendExemplar currently doesn't create the series, it must + // already exist. If it does not then drop. + if ref == 0 && len(ts.Exemplars) > 0 { + updateFirstPartial(func() error { + return wrappedTSDBIngestExemplarErr(errExemplarRef, + model.Time(ts.Exemplars[0].TimestampMs), ts.Labels, ts.Exemplars[0].Labels) + }) + failedExemplarsCount += len(ts.Exemplars) + } else { // Note that else is explicit, rather than a continue in the above if, in case of additional logic post exemplar processing. + for _, ex := range ts.Exemplars { + e := exemplar.Exemplar{ + Value: ex.Value, + Ts: ex.TimestampMs, + HasTs: true, + Labels: cortexpb.FromLabelAdaptersToLabelsWithCopy(ex.Labels), + } + + if _, err = app.AppendExemplar(ref, nil, e); err == nil { + succeededExemplarsCount++ + continue + } + + // Error adding exemplar + updateFirstPartial(func() error { + return wrappedTSDBIngestExemplarErr(err, model.Time(ex.TimestampMs), ts.Labels, ex.Labels) + }) + failedExemplarsCount++ + } + } + } + } + + // At this point all samples have been added to the appender, so we can track the time it took. + i.TSDBState.appenderAddDuration.Observe(time.Since(startAppend).Seconds()) + + startCommit := time.Now() + if err := app.Commit(); err != nil { + return nil, wrapWithUser(err, userID) + } + i.TSDBState.appenderCommitDuration.Observe(time.Since(startCommit).Seconds()) + + // If only invalid samples are pushed, don't change "last update", as TSDB was not modified. + if succeededSamplesCount > 0 { + db.setLastUpdate(time.Now()) + } + + // Increment metrics only if the samples have been successfully committed. + // If the code didn't reach this point, it means that we returned an error + // which will be converted into an HTTP 5xx and the client should/will retry. + i.metrics.ingestedSamples.Add(float64(succeededSamplesCount)) + i.metrics.ingestedSamplesFail.Add(float64(failedSamplesCount)) + i.metrics.ingestedExemplars.Add(float64(succeededExemplarsCount)) + i.metrics.ingestedExemplarsFail.Add(float64(failedExemplarsCount)) + + if sampleOutOfBoundsCount > 0 { + validation.DiscardedSamples.WithLabelValues(sampleOutOfBounds, userID).Add(float64(sampleOutOfBoundsCount)) + } + if sampleOutOfOrderCount > 0 { + validation.DiscardedSamples.WithLabelValues(sampleOutOfOrder, userID).Add(float64(sampleOutOfOrderCount)) + } + if newValueForTimestampCount > 0 { + validation.DiscardedSamples.WithLabelValues(newValueForTimestamp, userID).Add(float64(newValueForTimestampCount)) + } + if perUserSeriesLimitCount > 0 { + validation.DiscardedSamples.WithLabelValues(perUserSeriesLimit, userID).Add(float64(perUserSeriesLimitCount)) + } + if perMetricSeriesLimitCount > 0 { + validation.DiscardedSamples.WithLabelValues(perMetricSeriesLimit, userID).Add(float64(perMetricSeriesLimitCount)) + } + + // Distributor counts both samples and metadata, so for consistency ingester does the same. + i.ingestionRate.Add(int64(succeededSamplesCount + ingestedMetadata)) + + switch req.Source { + case cortexpb.RULE: + db.ingestedRuleSamples.Add(int64(succeededSamplesCount)) + case cortexpb.API: + fallthrough + default: + db.ingestedAPISamples.Add(int64(succeededSamplesCount)) + } + + if firstPartialErr != nil { + code := http.StatusBadRequest + var ve *validationError + if errors.As(firstPartialErr, &ve) { + code = ve.code + } + return &cortexpb.WriteResponse{}, httpgrpc.Errorf(code, wrapWithUser(firstPartialErr, userID).Error()) + } + + return &cortexpb.WriteResponse{}, nil +} + +func (u *userTSDB) acquireAppendLock() error { + u.stateMtx.RLock() + defer u.stateMtx.RUnlock() + + switch u.state { + case active: + case activeShipping: + // Pushes are allowed. + case forceCompacting: + return errors.New("forced compaction in progress") + case closing: + return errors.New("TSDB is closing") + default: + return errors.New("TSDB is not active") + } + + u.pushesInFlight.Add(1) + return nil +} + +func (u *userTSDB) releaseAppendLock() { + u.pushesInFlight.Done() +} + +// Query implements service.IngesterServer +func (i *Ingester) Query(ctx context.Context, req *client.QueryRequest) (*client.QueryResponse, error) { + if err := i.checkRunning(); err != nil { + return nil, err + } + + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, err + } + + from, through, matchers, err := client.FromQueryRequest(req) + if err != nil { + return nil, err + } + + i.metrics.queries.Inc() + + db := i.getTSDB(userID) + if db == nil { + return &client.QueryResponse{}, nil + } + + q, err := db.Querier(ctx, int64(from), int64(through)) + if err != nil { + return nil, err + } + defer q.Close() + + // It's not required to return sorted series because series are sorted by the Cortex querier. + ss := q.Select(false, nil, matchers...) + if ss.Err() != nil { + return nil, ss.Err() + } + + numSamples := 0 + + result := &client.QueryResponse{} + for ss.Next() { + series := ss.At() + + ts := cortexpb.TimeSeries{ + Labels: cortexpb.FromLabelsToLabelAdapters(series.Labels()), + } + + it := series.Iterator() + for it.Next() { + t, v := it.At() + ts.Samples = append(ts.Samples, cortexpb.Sample{Value: v, TimestampMs: t}) + } + + numSamples += len(ts.Samples) + result.Timeseries = append(result.Timeseries, ts) + } + + i.metrics.queriedSeries.Observe(float64(len(result.Timeseries))) + i.metrics.queriedSamples.Observe(float64(numSamples)) + + return result, ss.Err() +} + +// Query implements service.IngesterServer +func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQueryRequest) (*client.ExemplarQueryResponse, error) { + if err := i.checkRunning(); err != nil { + return nil, err + } + + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, err + } + + from, through, matchers, err := client.FromExemplarQueryRequest(req) + if err != nil { + return nil, err + } + + i.metrics.queries.Inc() + + db := i.getTSDB(userID) + if db == nil { + return &client.ExemplarQueryResponse{}, nil + } + + q, err := db.ExemplarQuerier(ctx) + if err != nil { + return nil, err + } + + // It's not required to sort series from a single ingester because series are sorted by the Exemplar Storage before returning from Select. + res, err := q.Select(from, through, matchers...) + if err != nil { + return nil, err + } + + numExemplars := 0 + + result := &client.ExemplarQueryResponse{} + for _, es := range res { + ts := cortexpb.TimeSeries{ + Labels: cortexpb.FromLabelsToLabelAdapters(es.SeriesLabels), + Exemplars: cortexpb.FromExemplarsToExemplarProtos(es.Exemplars), + } + + numExemplars += len(ts.Exemplars) + result.Timeseries = append(result.Timeseries, ts) + } + + i.metrics.queriedExemplars.Observe(float64(numExemplars)) + + return result, nil +} + +// LabelValues returns all label values that are associated with a given label name. +func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesRequest) (*client.LabelValuesResponse, error) { + if err := i.checkRunning(); err != nil { + return nil, err + } + + labelName, startTimestampMs, endTimestampMs, matchers, err := client.FromLabelValuesRequest(req) + if err != nil { + return nil, err + } + + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, err + } + + db := i.getTSDB(userID) + if db == nil { + return &client.LabelValuesResponse{}, nil + } + + mint, maxt, err := metadataQueryRange(startTimestampMs, endTimestampMs, db) + if err != nil { + return nil, err + } + + q, err := db.Querier(ctx, mint, maxt) + if err != nil { + return nil, err + } + defer q.Close() + + vals, _, err := q.LabelValues(labelName, matchers...) + if err != nil { + return nil, err + } + + return &client.LabelValuesResponse{ + LabelValues: vals, + }, nil +} + +func (i *Ingester) LabelValuesStream(req *client.LabelValuesRequest, stream client.Ingester_LabelValuesStreamServer) error { + resp, err := i.LabelValues(stream.Context(), req) + + if err != nil { + return err + } + + for i := 0; i < len(resp.LabelValues); i += metadataStreamBatchSize { + j := i + metadataStreamBatchSize + if j > len(resp.LabelValues) { + j = len(resp.LabelValues) + } + resp := &client.LabelValuesStreamResponse{ + LabelValues: resp.LabelValues[i:j], + } + err := client.SendLabelValuesStream(stream, resp) + if err != nil { + return err + } + } + + return nil +} + +// LabelNames return all the label names. +func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest) (*client.LabelNamesResponse, error) { + if err := i.checkRunning(); err != nil { + return nil, err + } + + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, err + } + + db := i.getTSDB(userID) + if db == nil { + return &client.LabelNamesResponse{}, nil + } + + mint, maxt, err := metadataQueryRange(req.StartTimestampMs, req.EndTimestampMs, db) + if err != nil { + return nil, err + } + + q, err := db.Querier(ctx, mint, maxt) + if err != nil { + return nil, err + } + defer q.Close() + + names, _, err := q.LabelNames() + if err != nil { + return nil, err + } + + return &client.LabelNamesResponse{ + LabelNames: names, + }, nil +} + +// LabelNamesStream return all the label names. +func (i *Ingester) LabelNamesStream(req *client.LabelNamesRequest, stream client.Ingester_LabelNamesStreamServer) error { + resp, err := i.LabelNames(stream.Context(), req) + + if err != nil { + return err + } + + for i := 0; i < len(resp.LabelNames); i += metadataStreamBatchSize { + j := i + metadataStreamBatchSize + if j > len(resp.LabelNames) { + j = len(resp.LabelNames) + } + resp := &client.LabelNamesStreamResponse{ + LabelNames: resp.LabelNames[i:j], + } + err := client.SendLabelNamesStream(stream, resp) + if err != nil { + return err + } + } + + return nil +} + +// MetricsForLabelMatchers returns all the metrics which match a set of matchers. +func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.MetricsForLabelMatchersRequest) (*client.MetricsForLabelMatchersResponse, error) { + if err := i.checkRunning(); err != nil { + return nil, err + } + + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, err + } + + db := i.getTSDB(userID) + if db == nil { + return &client.MetricsForLabelMatchersResponse{}, nil + } + + // Parse the request + _, _, matchersSet, err := client.FromMetricsForLabelMatchersRequest(req) + if err != nil { + return nil, err + } + + mint, maxt, err := metadataQueryRange(req.StartTimestampMs, req.EndTimestampMs, db) + if err != nil { + return nil, err + } + + q, err := db.Querier(ctx, mint, maxt) + if err != nil { + return nil, err + } + defer q.Close() + + // Run a query for each matchers set and collect all the results. + var sets []storage.SeriesSet + + for _, matchers := range matchersSet { + // Interrupt if the context has been canceled. + if ctx.Err() != nil { + return nil, ctx.Err() + } + + hints := &storage.SelectHints{ + Start: mint, + End: maxt, + Func: "series", // There is no series function, this token is used for lookups that don't need samples. + } + + seriesSet := q.Select(true, hints, matchers...) + sets = append(sets, seriesSet) + } + + // Generate the response merging all series sets. + result := &client.MetricsForLabelMatchersResponse{ + Metric: make([]*cortexpb.Metric, 0), + } + + mergedSet := storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge) + for mergedSet.Next() { + // Interrupt if the context has been canceled. + if ctx.Err() != nil { + return nil, ctx.Err() + } + + result.Metric = append(result.Metric, &cortexpb.Metric{ + Labels: cortexpb.FromLabelsToLabelAdapters(mergedSet.At().Labels()), + }) + } + + return result, nil +} + +func (i *Ingester) MetricsForLabelMatchersStream(req *client.MetricsForLabelMatchersRequest, stream client.Ingester_MetricsForLabelMatchersStreamServer) error { + result, err := i.MetricsForLabelMatchers(stream.Context(), req) + if err != nil { + return err + } + + for i := 0; i < len(result.Metric); i += metadataStreamBatchSize { + j := i + metadataStreamBatchSize + if j > len(result.Metric) { + j = len(result.Metric) + } + resp := &client.MetricsForLabelMatchersStreamResponse{ + Metric: result.Metric[i:j], + } + err := client.SendMetricsForLabelMatchersStream(stream, resp) + if err != nil { + return err + } + } + + return nil +} + +// MetricsMetadata returns all the metric metadata of a user. +func (i *Ingester) MetricsMetadata(ctx context.Context, _ *client.MetricsMetadataRequest) (*client.MetricsMetadataResponse, error) { + i.stoppedMtx.RLock() + if err := i.checkRunningOrStopping(); err != nil { + i.stoppedMtx.RUnlock() + return nil, err + } + i.stoppedMtx.RUnlock() + + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, err + } + + userMetadata := i.getUserMetadata(userID) + + if userMetadata == nil { + return &client.MetricsMetadataResponse{}, nil + } + + return &client.MetricsMetadataResponse{Metadata: userMetadata.toClientMetadata()}, nil +} + +// CheckReady is the readiness handler used to indicate to k8s when the ingesters +// are ready for the addition or removal of another ingester. +func (i *Ingester) CheckReady(ctx context.Context) error { + if err := i.checkRunningOrStopping(); err != nil { + return fmt.Errorf("ingester not ready: %v", err) + } + return i.lifecycler.CheckReady(ctx) +} + +// UserStats returns ingestion statistics for the current user. +func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) (*client.UserStatsResponse, error) { + if err := i.checkRunning(); err != nil { + return nil, err + } + + userID, err := tenant.TenantID(ctx) + if err != nil { + return nil, err + } + + db := i.getTSDB(userID) + if db == nil { + return &client.UserStatsResponse{}, nil + } + + return createUserStats(db), nil +} + +// AllUserStats returns ingestion statistics for all users known to this ingester. +func (i *Ingester) AllUserStats(_ context.Context, _ *client.UserStatsRequest) (*client.UsersStatsResponse, error) { + if err := i.checkRunning(); err != nil { + return nil, err + } + + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() + + users := i.TSDBState.dbs + + response := &client.UsersStatsResponse{ + Stats: make([]*client.UserIDStatsResponse, 0, len(users)), + } + for userID, db := range users { + response.Stats = append(response.Stats, &client.UserIDStatsResponse{ + UserId: userID, + Data: createUserStats(db), + }) + } + return response, nil +} + +func createUserStats(db *userTSDB) *client.UserStatsResponse { + apiRate := db.ingestedAPISamples.Rate() + ruleRate := db.ingestedRuleSamples.Rate() + return &client.UserStatsResponse{ + IngestionRate: apiRate + ruleRate, + ApiIngestionRate: apiRate, + RuleIngestionRate: ruleRate, + NumSeries: db.Head().NumSeries(), + } +} + +const queryStreamBatchMessageSize = 1 * 1024 * 1024 + +// QueryStream implements service.IngesterServer +// Streams metrics from a TSDB. This implements the client.IngesterServer interface +func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_QueryStreamServer) error { + if err := i.checkRunning(); err != nil { + return err + } + + spanlog, ctx := spanlogger.New(stream.Context(), "QueryStream") + defer spanlog.Finish() + + userID, err := tenant.TenantID(ctx) + if err != nil { + return err + } + + from, through, matchers, err := client.FromQueryRequest(req) + if err != nil { + return err + } + + i.metrics.queries.Inc() + + db := i.getTSDB(userID) + if db == nil { + return nil + } + + numSamples := 0 + numSeries := 0 + + streamType := QueryStreamSamples + if i.cfg.StreamChunksWhenUsingBlocks { + streamType = QueryStreamChunks + } + + if i.cfg.StreamTypeFn != nil { + runtimeType := i.cfg.StreamTypeFn() + switch runtimeType { + case QueryStreamChunks: + streamType = QueryStreamChunks + case QueryStreamSamples: + streamType = QueryStreamSamples + default: + // no change from config value. + } + } + + if streamType == QueryStreamChunks { + level.Debug(spanlog).Log("msg", "using queryStreamChunks") + numSeries, numSamples, err = i.queryStreamChunks(ctx, db, int64(from), int64(through), matchers, stream) + } else { + level.Debug(spanlog).Log("msg", "using QueryStreamSamples") + numSeries, numSamples, err = i.queryStreamSamples(ctx, db, int64(from), int64(through), matchers, stream) + } + if err != nil { + return err + } + + i.metrics.queriedSeries.Observe(float64(numSeries)) + i.metrics.queriedSamples.Observe(float64(numSamples)) + level.Debug(spanlog).Log("series", numSeries, "samples", numSamples) + return nil +} + +func (i *Ingester) queryStreamSamples(ctx context.Context, db *userTSDB, from, through int64, matchers []*labels.Matcher, stream client.Ingester_QueryStreamServer) (numSeries, numSamples int, _ error) { + q, err := db.Querier(ctx, from, through) + if err != nil { + return 0, 0, err + } + defer q.Close() + + // It's not required to return sorted series because series are sorted by the Cortex querier. + ss := q.Select(false, nil, matchers...) + if ss.Err() != nil { + return 0, 0, ss.Err() + } + + timeseries := make([]cortexpb.TimeSeries, 0, queryStreamBatchSize) + batchSizeBytes := 0 + for ss.Next() { + series := ss.At() + + // convert labels to LabelAdapter + ts := cortexpb.TimeSeries{ + Labels: cortexpb.FromLabelsToLabelAdapters(series.Labels()), + } + + it := series.Iterator() + for it.Next() { + t, v := it.At() + ts.Samples = append(ts.Samples, cortexpb.Sample{Value: v, TimestampMs: t}) + } + numSamples += len(ts.Samples) + numSeries++ + tsSize := ts.Size() + + if (batchSizeBytes > 0 && batchSizeBytes+tsSize > queryStreamBatchMessageSize) || len(timeseries) >= queryStreamBatchSize { + // Adding this series to the batch would make it too big, + // flush the data and add it to new batch instead. + err = client.SendQueryStream(stream, &client.QueryStreamResponse{ + Timeseries: timeseries, + }) + if err != nil { + return 0, 0, err + } + + batchSizeBytes = 0 + timeseries = timeseries[:0] + } + + timeseries = append(timeseries, ts) + batchSizeBytes += tsSize + } + + // Ensure no error occurred while iterating the series set. + if err := ss.Err(); err != nil { + return 0, 0, err + } + + // Final flush any existing metrics + if batchSizeBytes != 0 { + err = client.SendQueryStream(stream, &client.QueryStreamResponse{ + Timeseries: timeseries, + }) + if err != nil { + return 0, 0, err + } + } + + return numSeries, numSamples, nil +} + +// queryStreamChunks streams metrics from a TSDB. This implements the client.IngesterServer interface +func (i *Ingester) queryStreamChunks(ctx context.Context, db *userTSDB, from, through int64, matchers []*labels.Matcher, stream client.Ingester_QueryStreamServer) (numSeries, numSamples int, _ error) { + q, err := db.ChunkQuerier(ctx, from, through) + if err != nil { + return 0, 0, err + } + defer q.Close() + + // It's not required to return sorted series because series are sorted by the Cortex querier. + ss := q.Select(false, nil, matchers...) + if ss.Err() != nil { + return 0, 0, ss.Err() + } + + chunkSeries := make([]client.TimeSeriesChunk, 0, queryStreamBatchSize) + batchSizeBytes := 0 + for ss.Next() { + series := ss.At() + + // convert labels to LabelAdapter + ts := client.TimeSeriesChunk{ + Labels: cortexpb.FromLabelsToLabelAdapters(series.Labels()), + } + + it := series.Iterator() + for it.Next() { + // Chunks are ordered by min time. + meta := it.At() + + // It is not guaranteed that chunk returned by iterator is populated. + // For now just return error. We could also try to figure out how to read the chunk. + if meta.Chunk == nil { + return 0, 0, errors.Errorf("unfilled chunk returned from TSDB chunk querier") + } + + ch := client.Chunk{ + StartTimestampMs: meta.MinTime, + EndTimestampMs: meta.MaxTime, + Data: meta.Chunk.Bytes(), + } + + switch meta.Chunk.Encoding() { + case chunkenc.EncXOR: + ch.Encoding = int32(encoding.PrometheusXorChunk) + default: + return 0, 0, errors.Errorf("unknown chunk encoding from TSDB chunk querier: %v", meta.Chunk.Encoding()) + } + + ts.Chunks = append(ts.Chunks, ch) + numSamples += meta.Chunk.NumSamples() + } + numSeries++ + tsSize := ts.Size() + + if (batchSizeBytes > 0 && batchSizeBytes+tsSize > queryStreamBatchMessageSize) || len(chunkSeries) >= queryStreamBatchSize { + // Adding this series to the batch would make it too big, + // flush the data and add it to new batch instead. + err = client.SendQueryStream(stream, &client.QueryStreamResponse{ + Chunkseries: chunkSeries, + }) + if err != nil { + return 0, 0, err + } + + batchSizeBytes = 0 + chunkSeries = chunkSeries[:0] + } + + chunkSeries = append(chunkSeries, ts) + batchSizeBytes += tsSize + } + + // Ensure no error occurred while iterating the series set. + if err := ss.Err(); err != nil { + return 0, 0, err + } + + // Final flush any existing metrics + if batchSizeBytes != 0 { + err = client.SendQueryStream(stream, &client.QueryStreamResponse{ + Chunkseries: chunkSeries, + }) + if err != nil { + return 0, 0, err + } + } + + return numSeries, numSamples, nil +} + +func (i *Ingester) getTSDB(userID string) *userTSDB { + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() + db := i.TSDBState.dbs[userID] + return db +} + +// List all users for which we have a TSDB. We do it here in order +// to keep the mutex locked for the shortest time possible. +func (i *Ingester) getTSDBUsers() []string { + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() + + ids := make([]string, 0, len(i.TSDBState.dbs)) + for userID := range i.TSDBState.dbs { + ids = append(ids, userID) + } + + return ids +} + +func (i *Ingester) getOrCreateTSDB(userID string, force bool) (*userTSDB, error) { + db := i.getTSDB(userID) + if db != nil { + return db, nil + } + + i.stoppedMtx.Lock() + defer i.stoppedMtx.Unlock() + + // Check again for DB in the event it was created in-between locks + var ok bool + db, ok = i.TSDBState.dbs[userID] + if ok { + return db, nil + } + + // We're ready to create the TSDB, however we must be sure that the ingester + // is in the ACTIVE state, otherwise it may conflict with the transfer in/out. + // The TSDB is created when the first series is pushed and this shouldn't happen + // to a non-ACTIVE ingester, however we want to protect from any bug, cause we + // may have data loss or TSDB WAL corruption if the TSDB is created before/during + // a transfer in occurs. + if ingesterState := i.lifecycler.GetState(); !force && ingesterState != ring.ACTIVE { + return nil, fmt.Errorf(errTSDBCreateIncompatibleState, ingesterState) + } + + gl := i.getInstanceLimits() + if gl != nil && gl.MaxInMemoryTenants > 0 { + if users := int64(len(i.TSDBState.dbs)); users >= gl.MaxInMemoryTenants { + return nil, errMaxUsersLimitReached + } + } + + // Create the database and a shipper for a user + db, err := i.createTSDB(userID) + if err != nil { + return nil, err + } + + // Add the db to list of user databases + i.TSDBState.dbs[userID] = db + i.metrics.memUsers.Inc() + + return db, nil +} + +// createTSDB creates a TSDB for a given userID, and returns the created db. +func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { + tsdbPromReg := prometheus.NewRegistry() + udir := i.cfg.BlocksStorageConfig.TSDB.BlocksDir(userID) + userLogger := logutil.WithUserID(userID, i.logger) + + blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() + + userDB := &userTSDB{ + userID: userID, + activeSeries: NewActiveSeries(), + seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), + ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), + ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), + + instanceLimitsFn: i.getInstanceLimits, + instanceSeriesCount: &i.TSDBState.seriesCount, + } + + enableExemplars := false + if i.cfg.BlocksStorageConfig.TSDB.MaxExemplars > 0 { + enableExemplars = true + } + // Create a new user database + db, err := tsdb.Open(udir, userLogger, tsdbPromReg, &tsdb.Options{ + RetentionDuration: i.cfg.BlocksStorageConfig.TSDB.Retention.Milliseconds(), + MinBlockDuration: blockRanges[0], + MaxBlockDuration: blockRanges[len(blockRanges)-1], + NoLockfile: true, + StripeSize: i.cfg.BlocksStorageConfig.TSDB.StripeSize, + HeadChunksWriteBufferSize: i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteBufferSize, + WALCompression: i.cfg.BlocksStorageConfig.TSDB.WALCompressionEnabled, + WALSegmentSize: i.cfg.BlocksStorageConfig.TSDB.WALSegmentSizeBytes, + SeriesLifecycleCallback: userDB, + BlocksToDelete: userDB.blocksToDelete, + EnableExemplarStorage: enableExemplars, + MaxExemplars: int64(i.cfg.BlocksStorageConfig.TSDB.MaxExemplars), + }, nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to open TSDB: %s", udir) + } + db.DisableCompactions() // we will compact on our own schedule + + // Run compaction before using this TSDB. If there is data in head that needs to be put into blocks, + // this will actually create the blocks. If there is no data (empty TSDB), this is a no-op, although + // local blocks compaction may still take place if configured. + level.Info(userLogger).Log("msg", "Running compaction after WAL replay") + err = db.Compact() + if err != nil { + return nil, errors.Wrapf(err, "failed to compact TSDB: %s", udir) + } + + userDB.db = db + // We set the limiter here because we don't want to limit + // series during WAL replay. + userDB.limiter = i.limiter + + if db.Head().NumSeries() > 0 { + // If there are series in the head, use max time from head. If this time is too old, + // TSDB will be eligible for flushing and closing sooner, unless more data is pushed to it quickly. + userDB.setLastUpdate(util.TimeFromMillis(db.Head().MaxTime())) + } else { + // If head is empty (eg. new TSDB), don't close it right after. + userDB.setLastUpdate(time.Now()) + } + + // Thanos shipper requires at least 1 external label to be set. For this reason, + // we set the tenant ID as external label and we'll filter it out when reading + // the series from the storage. + l := labels.Labels{ + { + Name: cortex_tsdb.TenantIDExternalLabel, + Value: userID, + }, { + Name: cortex_tsdb.IngesterIDExternalLabel, + Value: i.TSDBState.shipperIngesterID, + }, + } + + // Create a new shipper for this database + if i.cfg.BlocksStorageConfig.TSDB.IsBlocksShippingEnabled() { + userDB.shipper = shipper.New( + userLogger, + tsdbPromReg, + udir, + bucket.NewUserBucketClient(userID, i.TSDBState.bucket, i.limits), + func() labels.Labels { return l }, + metadata.ReceiveSource, + false, // No need to upload compacted blocks. Cortex compactor takes care of that. + true, // Allow out of order uploads. It's fine in Cortex's context. + metadata.NoneFunc, + ) + + // Initialise the shipper blocks cache. + if err := userDB.updateCachedShippedBlocks(); err != nil { + level.Error(userLogger).Log("msg", "failed to update cached shipped blocks after shipper initialisation", "err", err) + } + } + + i.TSDBState.tsdbMetrics.setRegistryForUser(userID, tsdbPromReg) + return userDB, nil +} + +func (i *Ingester) closeAllTSDB() { + i.stoppedMtx.Lock() + + wg := &sync.WaitGroup{} + wg.Add(len(i.TSDBState.dbs)) + + // Concurrently close all users TSDB + for userID, userDB := range i.TSDBState.dbs { + userID := userID + + go func(db *userTSDB) { + defer wg.Done() + + if err := db.Close(); err != nil { + level.Warn(i.logger).Log("msg", "unable to close TSDB", "err", err, "user", userID) + return + } + + // Now that the TSDB has been closed, we should remove it from the + // set of open ones. This lock acquisition doesn't deadlock with the + // outer one, because the outer one is released as soon as all go + // routines are started. + i.stoppedMtx.Lock() + delete(i.TSDBState.dbs, userID) + i.stoppedMtx.Unlock() + + i.metrics.memUsers.Dec() + i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) + }(userDB) + } + + // Wait until all Close() completed + i.stoppedMtx.Unlock() + wg.Wait() +} + +// openExistingTSDB walks the user tsdb dir, and opens a tsdb for each user. This may start a WAL replay, so we limit the number of +// concurrently opening TSDB. +func (i *Ingester) openExistingTSDB(ctx context.Context) error { + level.Info(i.logger).Log("msg", "opening existing TSDBs") + + queue := make(chan string) + group, groupCtx := errgroup.WithContext(ctx) + + // Create a pool of workers which will open existing TSDBs. + for n := 0; n < i.cfg.BlocksStorageConfig.TSDB.MaxTSDBOpeningConcurrencyOnStartup; n++ { + group.Go(func() error { + for userID := range queue { + startTime := time.Now() + + db, err := i.createTSDB(userID) + if err != nil { + level.Error(i.logger).Log("msg", "unable to open TSDB", "err", err, "user", userID) + return errors.Wrapf(err, "unable to open TSDB for user %s", userID) + } + + // Add the database to the map of user databases + i.stoppedMtx.Lock() + i.TSDBState.dbs[userID] = db + i.stoppedMtx.Unlock() + i.metrics.memUsers.Inc() + + i.TSDBState.walReplayTime.Observe(time.Since(startTime).Seconds()) + } + + return nil + }) + } + + // Spawn a goroutine to find all users with a TSDB on the filesystem. + group.Go(func() error { + // Close the queue once filesystem walking is done. + defer close(queue) + + walkErr := filepath.Walk(i.cfg.BlocksStorageConfig.TSDB.Dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + // If the root directory doesn't exist, we're OK (not needed to be created upfront). + if os.IsNotExist(err) && path == i.cfg.BlocksStorageConfig.TSDB.Dir { + return filepath.SkipDir + } + + level.Error(i.logger).Log("msg", "an error occurred while iterating the filesystem storing TSDBs", "path", path, "err", err) + return errors.Wrapf(err, "an error occurred while iterating the filesystem storing TSDBs at %s", path) + } + + // Skip root dir and all other files + if path == i.cfg.BlocksStorageConfig.TSDB.Dir || !info.IsDir() { + return nil + } + + // Top level directories are assumed to be user TSDBs + userID := info.Name() + f, err := os.Open(path) + if err != nil { + level.Error(i.logger).Log("msg", "unable to open TSDB dir", "err", err, "user", userID, "path", path) + return errors.Wrapf(err, "unable to open TSDB dir %s for user %s", path, userID) + } + defer f.Close() + + // If the dir is empty skip it + if _, err := f.Readdirnames(1); err != nil { + if err == io.EOF { + return filepath.SkipDir + } + + level.Error(i.logger).Log("msg", "unable to read TSDB dir", "err", err, "user", userID, "path", path) + return errors.Wrapf(err, "unable to read TSDB dir %s for user %s", path, userID) + } + + // Enqueue the user to be processed. + select { + case queue <- userID: + // Nothing to do. + case <-groupCtx.Done(): + // Interrupt in case a failure occurred in another goroutine. + return nil + } + + // Don't descend into subdirectories. + return filepath.SkipDir + }) + + return errors.Wrapf(walkErr, "unable to walk directory %s containing existing TSDBs", i.cfg.BlocksStorageConfig.TSDB.Dir) + }) + + // Wait for all workers to complete. + err := group.Wait() + if err != nil { + level.Error(i.logger).Log("msg", "error while opening existing TSDBs", "err", err) + return err + } + + level.Info(i.logger).Log("msg", "successfully opened existing TSDBs") + return nil +} + +// getMemorySeriesMetric returns the total number of in-memory series across all open TSDBs. +func (i *Ingester) getMemorySeriesMetric() float64 { + if err := i.checkRunning(); err != nil { + return 0 + } + + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() + + count := uint64(0) + for _, db := range i.TSDBState.dbs { + count += db.Head().NumSeries() + } + + return float64(count) +} + +// getOldestUnshippedBlockMetric returns the unix timestamp of the oldest unshipped block or +// 0 if all blocks have been shipped. +func (i *Ingester) getOldestUnshippedBlockMetric() float64 { + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() + + oldest := uint64(0) + for _, db := range i.TSDBState.dbs { + if ts := db.getOldestUnshippedBlockTime(); oldest == 0 || ts < oldest { + oldest = ts + } + } + + return float64(oldest / 1000) +} + +func (i *Ingester) shipBlocksLoop(ctx context.Context) error { + // We add a slight jitter to make sure that if the head compaction interval and ship interval are set to the same + // value they don't clash (if they both continuously run at the same exact time, the head compaction may not run + // because can't successfully change the state). + shipTicker := time.NewTicker(util.DurationWithJitter(i.cfg.BlocksStorageConfig.TSDB.ShipInterval, 0.01)) + defer shipTicker.Stop() + + for { + select { + case <-shipTicker.C: + i.shipBlocks(ctx, nil) + + case req := <-i.TSDBState.shipTrigger: + i.shipBlocks(ctx, req.users) + close(req.callback) // Notify back. + + case <-ctx.Done(): + return nil + } + } +} + +// shipBlocks runs shipping for all users. +func (i *Ingester) shipBlocks(ctx context.Context, allowed *util.AllowedTenants) { + // Do not ship blocks if the ingester is PENDING or JOINING. It's + // particularly important for the JOINING state because there could + // be a blocks transfer in progress (from another ingester) and if we + // run the shipper in such state we could end up with race conditions. + if i.lifecycler != nil { + if ingesterState := i.lifecycler.GetState(); ingesterState == ring.PENDING || ingesterState == ring.JOINING { + level.Info(i.logger).Log("msg", "TSDB blocks shipping has been skipped because of the current ingester state", "state", ingesterState) + return + } + } + + // Number of concurrent workers is limited in order to avoid to concurrently sync a lot + // of tenants in a large cluster. + _ = concurrency.ForEachUser(ctx, i.getTSDBUsers(), i.cfg.BlocksStorageConfig.TSDB.ShipConcurrency, func(ctx context.Context, userID string) error { + if !allowed.IsAllowed(userID) { + return nil + } + + // Get the user's DB. If the user doesn't exist, we skip it. + userDB := i.getTSDB(userID) + if userDB == nil || userDB.shipper == nil { + return nil + } + + if userDB.deletionMarkFound.Load() { + return nil + } + + if time.Since(time.Unix(userDB.lastDeletionMarkCheck.Load(), 0)) > cortex_tsdb.DeletionMarkCheckInterval { + // Even if check fails with error, we don't want to repeat it too often. + userDB.lastDeletionMarkCheck.Store(time.Now().Unix()) + + deletionMarkExists, err := cortex_tsdb.TenantDeletionMarkExists(ctx, i.TSDBState.bucket, userID) + if err != nil { + // If we cannot check for deletion mark, we continue anyway, even though in production shipper will likely fail too. + // This however simplifies unit tests, where tenant deletion check is enabled by default, but tests don't setup bucket. + level.Warn(i.logger).Log("msg", "failed to check for tenant deletion mark before shipping blocks", "user", userID, "err", err) + } else if deletionMarkExists { + userDB.deletionMarkFound.Store(true) + + level.Info(i.logger).Log("msg", "tenant deletion mark exists, not shipping blocks", "user", userID) + return nil + } + } + + // Run the shipper's Sync() to upload unshipped blocks. Make sure the TSDB state is active, in order to + // avoid any race condition with closing idle TSDBs. + if !userDB.casState(active, activeShipping) { + level.Info(i.logger).Log("msg", "shipper skipped because the TSDB is not active", "user", userID) + return nil + } + defer userDB.casState(activeShipping, active) + + uploaded, err := userDB.shipper.Sync(ctx) + if err != nil { + level.Warn(i.logger).Log("msg", "shipper failed to synchronize TSDB blocks with the storage", "user", userID, "uploaded", uploaded, "err", err) + } else { + level.Debug(i.logger).Log("msg", "shipper successfully synchronized TSDB blocks with storage", "user", userID, "uploaded", uploaded) + } + + // The shipper meta file could be updated even if the Sync() returned an error, + // so it's safer to update it each time at least a block has been uploaded. + // Moreover, the shipper meta file could be updated even if no blocks are uploaded + // (eg. blocks removed due to retention) but doesn't cause any harm not updating + // the cached list of blocks in such case, so we're not handling it. + if uploaded > 0 { + if err := userDB.updateCachedShippedBlocks(); err != nil { + level.Error(i.logger).Log("msg", "failed to update cached shipped blocks after shipper synchronisation", "user", userID, "err", err) + } + } + + return nil + }) +} + +func (i *Ingester) compactionLoop(ctx context.Context) error { + ticker := time.NewTicker(i.cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval) + defer ticker.Stop() + + for ctx.Err() == nil { + select { + case <-ticker.C: + i.compactBlocks(ctx, false, nil) + + case req := <-i.TSDBState.forceCompactTrigger: + i.compactBlocks(ctx, true, req.users) + close(req.callback) // Notify back. + + case <-ctx.Done(): + return nil + } + } + return nil +} + +// Compacts all compactable blocks. Force flag will force compaction even if head is not compactable yet. +func (i *Ingester) compactBlocks(ctx context.Context, force bool, allowed *util.AllowedTenants) { + // Don't compact TSDB blocks while JOINING as there may be ongoing blocks transfers. + // Compaction loop is not running in LEAVING state, so if we get here in LEAVING state, we're flushing blocks. + if i.lifecycler != nil { + if ingesterState := i.lifecycler.GetState(); ingesterState == ring.JOINING { + level.Info(i.logger).Log("msg", "TSDB blocks compaction has been skipped because of the current ingester state", "state", ingesterState) + return + } + } + + _ = concurrency.ForEachUser(ctx, i.getTSDBUsers(), i.cfg.BlocksStorageConfig.TSDB.HeadCompactionConcurrency, func(ctx context.Context, userID string) error { + if !allowed.IsAllowed(userID) { + return nil + } + + userDB := i.getTSDB(userID) + if userDB == nil { + return nil + } + + // Don't do anything, if there is nothing to compact. + h := userDB.Head() + if h.NumSeries() == 0 { + return nil + } + + var err error + + i.TSDBState.compactionsTriggered.Inc() + + reason := "" + switch { + case force: + reason = "forced" + err = userDB.compactHead(i.cfg.BlocksStorageConfig.TSDB.BlockRanges[0].Milliseconds()) + + case i.TSDBState.compactionIdleTimeout > 0 && userDB.isIdle(time.Now(), i.TSDBState.compactionIdleTimeout): + reason = "idle" + level.Info(i.logger).Log("msg", "TSDB is idle, forcing compaction", "user", userID) + err = userDB.compactHead(i.cfg.BlocksStorageConfig.TSDB.BlockRanges[0].Milliseconds()) + + default: + reason = "regular" + err = userDB.Compact() + } + + if err != nil { + i.TSDBState.compactionsFailed.Inc() + level.Warn(i.logger).Log("msg", "TSDB blocks compaction for user has failed", "user", userID, "err", err, "compactReason", reason) + } else { + level.Debug(i.logger).Log("msg", "TSDB blocks compaction completed successfully", "user", userID, "compactReason", reason) + } + + return nil + }) +} + +func (i *Ingester) closeAndDeleteIdleUserTSDBs(ctx context.Context) error { + for _, userID := range i.getTSDBUsers() { + if ctx.Err() != nil { + return nil + } + + result := i.closeAndDeleteUserTSDBIfIdle(userID) + + i.TSDBState.idleTsdbChecks.WithLabelValues(string(result)).Inc() + } + + return nil +} + +func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckResult { + userDB := i.getTSDB(userID) + if userDB == nil || userDB.shipper == nil { + // We will not delete local data when not using shipping to storage. + return tsdbShippingDisabled + } + + if result := userDB.shouldCloseTSDB(i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout); !result.shouldClose() { + return result + } + + // This disables pushes and force-compactions. Not allowed to close while shipping is in progress. + if !userDB.casState(active, closing) { + return tsdbNotActive + } + + // If TSDB is fully closed, we will set state to 'closed', which will prevent this defered closing -> active transition. + defer userDB.casState(closing, active) + + // Make sure we don't ignore any possible inflight pushes. + userDB.pushesInFlight.Wait() + + // Verify again, things may have changed during the checks and pushes. + tenantDeleted := false + if result := userDB.shouldCloseTSDB(i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout); !result.shouldClose() { + // This will also change TSDB state back to active (via defer above). + return result + } else if result == tsdbTenantMarkedForDeletion { + tenantDeleted = true + } + + // At this point there are no more pushes to TSDB, and no possible compaction. Normally TSDB is empty, + // but if we're closing TSDB because of tenant deletion mark, then it may still contain some series. + // We need to remove these series from series count. + i.TSDBState.seriesCount.Sub(int64(userDB.Head().NumSeries())) + + dir := userDB.db.Dir() + + if err := userDB.Close(); err != nil { + level.Error(i.logger).Log("msg", "failed to close idle TSDB", "user", userID, "err", err) + return tsdbCloseFailed + } + + level.Info(i.logger).Log("msg", "closed idle TSDB", "user", userID) + + // This will prevent going back to "active" state in deferred statement. + userDB.casState(closing, closed) + + // Only remove user from TSDBState when everything is cleaned up + // This will prevent concurrency problems when cortex are trying to open new TSDB - Ie: New request for a given tenant + // came in - while closing the tsdb for the same tenant. + // If this happens now, the request will get reject as the push will not be able to acquire the lock as the tsdb will be + // in closed state + defer func() { + i.stoppedMtx.Lock() + delete(i.TSDBState.dbs, userID) + i.stoppedMtx.Unlock() + }() + + i.metrics.memUsers.Dec() + i.TSDBState.tsdbMetrics.removeRegistryForUser(userID) + + i.deleteUserMetadata(userID) + i.metrics.deletePerUserMetrics(userID) + + validation.DeletePerUserValidationMetrics(userID, i.logger) + + // And delete local data. + if err := os.RemoveAll(dir); err != nil { + level.Error(i.logger).Log("msg", "failed to delete local TSDB", "user", userID, "err", err) + return tsdbDataRemovalFailed + } + + if tenantDeleted { + level.Info(i.logger).Log("msg", "deleted local TSDB, user marked for deletion", "user", userID, "dir", dir) + return tsdbTenantMarkedForDeletion + } + + level.Info(i.logger).Log("msg", "deleted local TSDB, due to being idle", "user", userID, "dir", dir) + return tsdbIdleClosed +} + +// pushMetadata returns number of ingested metadata. +func (i *Ingester) pushMetadata(ctx context.Context, userID string, metadata []*cortexpb.MetricMetadata) int { + ingestedMetadata := 0 + failedMetadata := 0 + + var firstMetadataErr error + for _, metadata := range metadata { + err := i.appendMetadata(userID, metadata) + if err == nil { + ingestedMetadata++ + continue + } + + failedMetadata++ + if firstMetadataErr == nil { + firstMetadataErr = err + } + } + + i.metrics.ingestedMetadata.Add(float64(ingestedMetadata)) + i.metrics.ingestedMetadataFail.Add(float64(failedMetadata)) + + // If we have any error with regard to metadata we just log and no-op. + // We consider metadata a best effort approach, errors here should not stop processing. + if firstMetadataErr != nil { + logger := logutil.WithContext(ctx, i.logger) + level.Warn(logger).Log("msg", "failed to ingest some metadata", "err", firstMetadataErr) + } + + return ingestedMetadata +} + +func (i *Ingester) appendMetadata(userID string, m *cortexpb.MetricMetadata) error { + i.stoppedMtx.RLock() + if i.stopped { + i.stoppedMtx.RUnlock() + return errIngesterStopping + } + i.stoppedMtx.RUnlock() + + userMetadata := i.getOrCreateUserMetadata(userID) + + return userMetadata.add(m.GetMetricFamilyName(), m) +} func (i *Ingester) getOrCreateUserMetadata(userID string) *userMetricsMetadata { userMetadata := i.getUserMetadata(userID) @@ -337,87 +2531,172 @@ func (i *Ingester) purgeUserMetricsMetadata() { } } -// Query implements service.IngesterServer -func (i *Ingester) Query(ctx context.Context, req *client.QueryRequest) (*client.QueryResponse, error) { - return i.v2Query(ctx, req) -} +// This method will flush all data. It is called as part of Lifecycler's shutdown (if flush on shutdown is configured), or from the flusher. +// +// When called as during Lifecycler shutdown, this happens as part of normal Ingester shutdown (see stopping method). +// Samples are not received at this stage. Compaction and Shipping loops have already been stopped as well. +// +// When used from flusher, ingester is constructed in a way that compaction, shipping and receiving of samples is never started. +func (i *Ingester) lifecyclerFlush() { + level.Info(i.logger).Log("msg", "starting to flush and ship TSDB blocks") -// QueryStream implements service.IngesterServer -func (i *Ingester) QueryStream(req *client.QueryRequest, stream client.Ingester_QueryStreamServer) error { - return i.v2QueryStream(req, stream) -} + ctx := context.Background() -// Query implements service.IngesterServer -func (i *Ingester) QueryExemplars(ctx context.Context, req *client.ExemplarQueryRequest) (*client.ExemplarQueryResponse, error) { - return i.v2QueryExemplars(ctx, req) -} + i.compactBlocks(ctx, true, nil) + if i.cfg.BlocksStorageConfig.TSDB.IsBlocksShippingEnabled() { + i.shipBlocks(ctx, nil) + } -// LabelValues returns all label values that are associated with a given label name. -func (i *Ingester) LabelValues(ctx context.Context, req *client.LabelValuesRequest) (*client.LabelValuesResponse, error) { - return i.v2LabelValues(ctx, req) + level.Info(i.logger).Log("msg", "finished flushing and shipping TSDB blocks") } -func (i *Ingester) LabelValuesStream(req *client.LabelValuesRequest, stream client.Ingester_LabelValuesStreamServer) error { - return i.v2LabelValuesStream(req, stream) -} +const ( + tenantParam = "tenant" + waitParam = "wait" +) -// LabelNames return all the label names. -func (i *Ingester) LabelNames(ctx context.Context, req *client.LabelNamesRequest) (*client.LabelNamesResponse, error) { - return i.v2LabelNames(ctx, req) -} +// Blocks version of Flush handler. It force-compacts blocks, and triggers shipping. +func (i *Ingester) flushHandler(w http.ResponseWriter, r *http.Request) { + err := r.ParseForm() + if err != nil { + level.Warn(i.logger).Log("msg", "failed to parse HTTP request in flush handler", "err", err) + w.WriteHeader(http.StatusBadRequest) + return + } -// LabelNames return all the label names. -func (i *Ingester) LabelNamesStream(req *client.LabelNamesRequest, stream client.Ingester_LabelNamesStreamServer) error { - return i.v2LabelNamesStream(req, stream) -} + tenants := r.Form[tenantParam] -// MetricsForLabelMatchers returns all the metrics which match a set of matchers. -func (i *Ingester) MetricsForLabelMatchers(ctx context.Context, req *client.MetricsForLabelMatchersRequest) (*client.MetricsForLabelMatchersResponse, error) { - return i.v2MetricsForLabelMatchers(ctx, req) -} + allowedUsers := util.NewAllowedTenants(tenants, nil) + run := func() { + ingCtx := i.BasicService.ServiceContext() + if ingCtx == nil || ingCtx.Err() != nil { + level.Info(i.logger).Log("msg", "flushing TSDB blocks: ingester not running, ignoring flush request") + return + } -func (i *Ingester) MetricsForLabelMatchersStream(req *client.MetricsForLabelMatchersRequest, stream client.Ingester_MetricsForLabelMatchersStreamServer) error { - return i.v2MetricsForLabelMatchersStream(req, stream) -} + compactionCallbackCh := make(chan struct{}) -// MetricsMetadata returns all the metric metadata of a user. -func (i *Ingester) MetricsMetadata(ctx context.Context, _ *client.MetricsMetadataRequest) (*client.MetricsMetadataResponse, error) { - i.stoppedMtx.RLock() - if err := i.checkRunningOrStopping(); err != nil { - i.stoppedMtx.RUnlock() - return nil, err + level.Info(i.logger).Log("msg", "flushing TSDB blocks: triggering compaction") + select { + case i.TSDBState.forceCompactTrigger <- requestWithUsersAndCallback{users: allowedUsers, callback: compactionCallbackCh}: + // Compacting now. + case <-ingCtx.Done(): + level.Warn(i.logger).Log("msg", "failed to compact TSDB blocks, ingester not running anymore") + return + } + + // Wait until notified about compaction being finished. + select { + case <-compactionCallbackCh: + level.Info(i.logger).Log("msg", "finished compacting TSDB blocks") + case <-ingCtx.Done(): + level.Warn(i.logger).Log("msg", "failed to compact TSDB blocks, ingester not running anymore") + return + } + + if i.cfg.BlocksStorageConfig.TSDB.IsBlocksShippingEnabled() { + shippingCallbackCh := make(chan struct{}) // must be new channel, as compactionCallbackCh is closed now. + + level.Info(i.logger).Log("msg", "flushing TSDB blocks: triggering shipping") + + select { + case i.TSDBState.shipTrigger <- requestWithUsersAndCallback{users: allowedUsers, callback: shippingCallbackCh}: + // shipping now + case <-ingCtx.Done(): + level.Warn(i.logger).Log("msg", "failed to ship TSDB blocks, ingester not running anymore") + return + } + + // Wait until shipping finished. + select { + case <-shippingCallbackCh: + level.Info(i.logger).Log("msg", "shipping of TSDB blocks finished") + case <-ingCtx.Done(): + level.Warn(i.logger).Log("msg", "failed to ship TSDB blocks, ingester not running anymore") + return + } + } + + level.Info(i.logger).Log("msg", "flushing TSDB blocks: finished") } - i.stoppedMtx.RUnlock() - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err + if len(r.Form[waitParam]) > 0 && r.Form[waitParam][0] == "true" { + // Run synchronously. This simplifies and speeds up tests. + run() + } else { + go run() } - userMetadata := i.getUserMetadata(userID) + w.WriteHeader(http.StatusNoContent) +} - if userMetadata == nil { - return &client.MetricsMetadataResponse{}, nil +// metadataQueryRange returns the best range to query for metadata queries based on the timerange in the ingester. +func metadataQueryRange(queryStart, queryEnd int64, db *userTSDB) (mint, maxt int64, err error) { + // Ingesters are run with limited retention and we don't support querying the store-gateway for labels yet. + // This means if someone loads a dashboard that is outside the range of the ingester, and we only return the + // data for the timerange requested (which will be empty), the dashboards will break. To fix this we should + // return the "head block" range until we can query the store-gateway. + + // Now the question would be what to do when the query is partially in the ingester range. I would err on the side + // of caution and query the entire db, as I can't think of a good way to query the head + the overlapping range. + mint, maxt = queryStart, queryEnd + + lowestTs, err := db.StartTime() + if err != nil { + return mint, maxt, err } - return &client.MetricsMetadataResponse{Metadata: userMetadata.toClientMetadata()}, nil + // Completely outside. + if queryEnd < lowestTs { + mint, maxt = db.Head().MinTime(), db.Head().MaxTime() + } else if queryStart < lowestTs { + // Partially inside. + mint, maxt = 0, math.MaxInt64 + } + + return } -// UserStats returns ingestion statistics for the current user. -func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) (*client.UserStatsResponse, error) { - return i.v2UserStats(ctx, req) +func wrappedTSDBIngestErr(ingestErr error, timestamp model.Time, labels []cortexpb.LabelAdapter) error { + if ingestErr == nil { + return nil + } + + return fmt.Errorf(errTSDBIngest, ingestErr, timestamp.Time().UTC().Format(time.RFC3339Nano), cortexpb.FromLabelAdaptersToLabels(labels).String()) } -// AllUserStats returns ingestion statistics for all users known to this ingester. -func (i *Ingester) AllUserStats(ctx context.Context, req *client.UserStatsRequest) (*client.UsersStatsResponse, error) { - return i.v2AllUserStats(ctx, req) +func wrappedTSDBIngestExemplarErr(ingestErr error, timestamp model.Time, seriesLabels, exemplarLabels []cortexpb.LabelAdapter) error { + if ingestErr == nil { + return nil + } + + return fmt.Errorf(errTSDBIngestExemplar, ingestErr, timestamp.Time().UTC().Format(time.RFC3339Nano), + cortexpb.FromLabelAdaptersToLabels(seriesLabels).String(), + cortexpb.FromLabelAdaptersToLabels(exemplarLabels).String(), + ) } -// CheckReady is the readiness handler used to indicate to k8s when the ingesters -// are ready for the addition or removal of another ingester. -func (i *Ingester) CheckReady(ctx context.Context) error { - if err := i.checkRunningOrStopping(); err != nil { - return fmt.Errorf("ingester not ready: %v", err) +func (i *Ingester) getInstanceLimits() *InstanceLimits { + // Don't apply any limits while starting. We especially don't want to apply series in memory limit while replaying WAL. + if i.State() == services.Starting { + return nil } - return i.lifecycler.CheckReady(ctx) + + if i.cfg.InstanceLimitsFn == nil { + return defaultInstanceLimits + } + + l := i.cfg.InstanceLimitsFn() + if l == nil { + return defaultInstanceLimits + } + + return l +} + +// stopIncomingRequests is called during the shutdown process. +func (i *Ingester) stopIncomingRequests() { + i.stoppedMtx.Lock() + defer i.stoppedMtx.Unlock() + i.stopped = true } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 629a65775eb..5b876092dbc 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -1,29 +1,57 @@ package ingester import ( + "bytes" "context" "fmt" + "io" + "io/ioutil" + "math" + "net" "net/http" + "net/http/httptest" + "net/url" "os" "path/filepath" "sort" + "strconv" + "strings" + "sync" "testing" "time" - "github.com/cortexproject/cortex/pkg/chunk" - + "github.com/go-kit/log" + "github.com/oklog/ulid" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "github.com/thanos-io/thanos/pkg/objstore" + "github.com/thanos-io/thanos/pkg/shipper" "github.com/weaveworks/common/httpgrpc" + "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/user" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc" + "github.com/cortexproject/cortex/pkg/chunk" + "github.com/cortexproject/cortex/pkg/chunk/encoding" "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" + cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" + "github.com/cortexproject/cortex/pkg/util" + util_math "github.com/cortexproject/cortex/pkg/util/math" "github.com/cortexproject/cortex/pkg/util/services" "github.com/cortexproject/cortex/pkg/util/test" + "github.com/cortexproject/cortex/pkg/util/validation" ) func runTestQuery(ctx context.Context, t *testing.T, ing *Ingester, ty labels.MatchType, n, v string) (model.Matrix, *client.QueryRequest, error) { @@ -288,3 +316,3842 @@ func TestGetIgnoreSeriesLimitForMetricNamesMap(t *testing.T) { cfg.IgnoreSeriesLimitForMetricNames = "foo, bar, ," require.Equal(t, map[string]struct{}{"foo": {}, "bar": {}}, cfg.getIgnoreSeriesLimitForMetricNamesMap()) } + +func TestIngester_Push(t *testing.T) { + metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} + metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) + metricNames := []string{ + "cortex_ingester_ingested_samples_total", + "cortex_ingester_ingested_samples_failures_total", + "cortex_ingester_memory_series", + "cortex_ingester_memory_users", + "cortex_ingester_memory_series_created_total", + "cortex_ingester_memory_series_removed_total", + "cortex_discarded_samples_total", + "cortex_ingester_active_series", + } + userID := "test" + + tests := map[string]struct { + reqs []*cortexpb.WriteRequest + expectedErr error + expectedIngested []cortexpb.TimeSeries + expectedMetadataIngested []*cortexpb.MetricMetadata + expectedExemplarsIngested []cortexpb.TimeSeries + expectedMetrics string + additionalMetrics []string + disableActiveSeries bool + maxExemplars int + }{ + "should succeed on valid series and metadata": { + reqs: []*cortexpb.WriteRequest{ + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, + []*cortexpb.MetricMetadata{ + {MetricFamilyName: "metric_name_1", Help: "a help for metric_name_1", Unit: "", Type: cortexpb.COUNTER}, + }, + cortexpb.API), + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, + []*cortexpb.MetricMetadata{ + {MetricFamilyName: "metric_name_2", Help: "a help for metric_name_2", Unit: "", Type: cortexpb.GAUGE}, + }, + cortexpb.API), + }, + expectedErr: nil, + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 9}, {Value: 2, TimestampMs: 10}}}, + }, + expectedMetadataIngested: []*cortexpb.MetricMetadata{ + {MetricFamilyName: "metric_name_2", Help: "a help for metric_name_2", Unit: "", Type: cortexpb.GAUGE}, + {MetricFamilyName: "metric_name_1", Help: "a help for metric_name_1", Unit: "", Type: cortexpb.COUNTER}, + }, + additionalMetrics: []string{ + // Metadata. + "cortex_ingester_memory_metadata", + "cortex_ingester_memory_metadata_created_total", + "cortex_ingester_ingested_metadata_total", + "cortex_ingester_ingested_metadata_failures_total", + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_metadata_failures_total The total number of metadata that errored on ingestion. + # TYPE cortex_ingester_ingested_metadata_failures_total counter + cortex_ingester_ingested_metadata_failures_total 0 + # HELP cortex_ingester_ingested_metadata_total The total number of metadata ingested. + # TYPE cortex_ingester_ingested_metadata_total counter + cortex_ingester_ingested_metadata_total 2 + # HELP cortex_ingester_memory_metadata The current number of metadata in memory. + # TYPE cortex_ingester_memory_metadata gauge + cortex_ingester_memory_metadata 2 + # HELP cortex_ingester_memory_metadata_created_total The total number of metadata that were created per user + # TYPE cortex_ingester_memory_metadata_created_total counter + cortex_ingester_memory_metadata_created_total{user="test"} 2 + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 2 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "should succeed on valid series with exemplars": { + maxExemplars: 2, + reqs: []*cortexpb.WriteRequest{ + // Ingesting an exemplar requires a sample to create the series first + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, + nil, + cortexpb.API), + { + Timeseries: []cortexpb.PreallocTimeseries{ + { + TimeSeries: &cortexpb.TimeSeries{ + Labels: []cortexpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool + Exemplars: []cortexpb.Exemplar{ + { + Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "123"}}, + TimestampMs: 1000, + Value: 1000, + }, + { + Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "456"}}, + TimestampMs: 1001, + Value: 1001, + }, + }, + }, + }, + }, + }, + }, + expectedErr: nil, + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 9}}}, + }, + expectedExemplarsIngested: []cortexpb.TimeSeries{ + { + Labels: metricLabelAdapters, + Exemplars: []cortexpb.Exemplar{ + { + Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "123"}}, + TimestampMs: 1000, + Value: 1000, + }, + { + Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "456"}}, + TimestampMs: 1001, + Value: 1001, + }, + }, + }, + }, + expectedMetadataIngested: nil, + additionalMetrics: []string{ + "cortex_ingester_tsdb_exemplar_exemplars_appended_total", + "cortex_ingester_tsdb_exemplar_exemplars_in_storage", + "cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage", + "cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds", + "cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total", + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + + # HELP cortex_ingester_tsdb_exemplar_exemplars_appended_total Total number of TSDB exemplars appended. + # TYPE cortex_ingester_tsdb_exemplar_exemplars_appended_total counter + cortex_ingester_tsdb_exemplar_exemplars_appended_total 2 + + # HELP cortex_ingester_tsdb_exemplar_exemplars_in_storage Number of TSDB exemplars currently in storage. + # TYPE cortex_ingester_tsdb_exemplar_exemplars_in_storage gauge + cortex_ingester_tsdb_exemplar_exemplars_in_storage 2 + + # HELP cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage Number of TSDB series with exemplars currently in storage. + # TYPE cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage gauge + cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{user="test"} 1 + + # HELP cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds The timestamp of the oldest exemplar stored in circular storage. Useful to check for what time range the current exemplar buffer limit allows. This usually means the last timestamp for all exemplars for a typical setup. This is not true though if one of the series timestamp is in future compared to rest series. + # TYPE cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds gauge + cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds{user="test"} 1 + + # HELP cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total Total number of out of order exemplar ingestion failed attempts. + # TYPE cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total counter + cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total 0 + `, + }, + "successful push, active series disabled": { + disableActiveSeries: true, + reqs: []*cortexpb.WriteRequest{ + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, + nil, + cortexpb.API), + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, + nil, + cortexpb.API), + }, + expectedErr: nil, + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 9}, {Value: 2, TimestampMs: 10}}}, + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 2 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + `, + }, + "should soft fail on sample out of order": { + reqs: []*cortexpb.WriteRequest{ + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, + nil, + cortexpb.API), + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, + nil, + cortexpb.API), + }, + expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestErr(storage.ErrOutOfOrderSample, model.Time(9), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 10}}}, + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 1 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{reason="sample-out-of-order",user="test"} 1 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "should soft fail on sample out of bound": { + reqs: []*cortexpb.WriteRequest{ + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}, + nil, + cortexpb.API), + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 1, TimestampMs: 1575043969 - (86400 * 1000)}}, + nil, + cortexpb.API), + }, + expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestErr(storage.ErrOutOfBounds, model.Time(1575043969-(86400*1000)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}}, + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 1 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{reason="sample-out-of-bounds",user="test"} 1 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "should soft fail on two different sample values at the same timestamp": { + reqs: []*cortexpb.WriteRequest{ + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}, + nil, + cortexpb.API), + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 1, TimestampMs: 1575043969}}, + nil, + cortexpb.API), + }, + expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestErr(storage.ErrDuplicateSampleForTimestamp, model.Time(1575043969), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), + expectedIngested: []cortexpb.TimeSeries{ + {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}}, + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 1 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 1 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 1 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_discarded_samples_total The total number of samples that were discarded. + # TYPE cortex_discarded_samples_total counter + cortex_discarded_samples_total{reason="new-value-for-timestamp",user="test"} 1 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 1 + `, + }, + "should soft fail on exemplar with unknown series": { + maxExemplars: 1, + reqs: []*cortexpb.WriteRequest{ + // Ingesting an exemplar requires a sample to create the series first + // This is not done here. + { + Timeseries: []cortexpb.PreallocTimeseries{ + { + TimeSeries: &cortexpb.TimeSeries{ + Labels: []cortexpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool + Exemplars: []cortexpb.Exemplar{ + { + Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "123"}}, + TimestampMs: 1000, + Value: 1000, + }, + }, + }, + }, + }, + }, + }, + expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestExemplarErr(errExemplarRef, model.Time(1000), cortexpb.FromLabelsToLabelAdapters(metricLabels), []cortexpb.LabelAdapter{{Name: "traceID", Value: "123"}}), userID).Error()), + expectedIngested: nil, + expectedMetadataIngested: nil, + additionalMetrics: []string{ + "cortex_ingester_tsdb_exemplar_exemplars_appended_total", + "cortex_ingester_tsdb_exemplar_exemplars_in_storage", + "cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage", + "cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds", + "cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total", + }, + expectedMetrics: ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 0 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 0 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test"} 0 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test"} 0 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test"} 0 + + # HELP cortex_ingester_tsdb_exemplar_exemplars_appended_total Total number of TSDB exemplars appended. + # TYPE cortex_ingester_tsdb_exemplar_exemplars_appended_total counter + cortex_ingester_tsdb_exemplar_exemplars_appended_total 0 + + # HELP cortex_ingester_tsdb_exemplar_exemplars_in_storage Number of TSDB exemplars currently in storage. + # TYPE cortex_ingester_tsdb_exemplar_exemplars_in_storage gauge + cortex_ingester_tsdb_exemplar_exemplars_in_storage 0 + + # HELP cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage Number of TSDB series with exemplars currently in storage. + # TYPE cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage gauge + cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{user="test"} 0 + + # HELP cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds The timestamp of the oldest exemplar stored in circular storage. Useful to check for what time range the current exemplar buffer limit allows. This usually means the last timestamp for all exemplars for a typical setup. This is not true though if one of the series timestamp is in future compared to rest series. + # TYPE cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds gauge + cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds{user="test"} 0 + + # HELP cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total Total number of out of order exemplar ingestion failed attempts. + # TYPE cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total counter + cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total 0 + `, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + registry := prometheus.NewRegistry() + + registry.MustRegister(validation.DiscardedSamples) + validation.DiscardedSamples.Reset() + + // Create a mocked ingester + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries + cfg.BlocksStorageConfig.TSDB.MaxExemplars = testData.maxExemplars + + i, err := prepareIngesterWithBlocksStorage(t, cfg, registry) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + ctx := user.InjectOrgID(context.Background(), userID) + + // Wait until the ingester is ACTIVE + test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push timeseries + for idx, req := range testData.reqs { + _, err := i.Push(ctx, req) + + // We expect no error on any request except the last one + // which may error (and in that case we assert on it) + if idx < len(testData.reqs)-1 { + assert.NoError(t, err) + } else { + assert.Equal(t, testData.expectedErr, err) + } + } + + // Read back samples to see what has been really ingested + res, err := i.Query(ctx, &client.QueryRequest{ + StartTimestampMs: math.MinInt64, + EndTimestampMs: math.MaxInt64, + Matchers: []*client.LabelMatcher{{Type: client.REGEX_MATCH, Name: labels.MetricName, Value: ".*"}}, + }) + + require.NoError(t, err) + require.NotNil(t, res) + assert.Equal(t, testData.expectedIngested, res.Timeseries) + + // Read back samples to see what has been really ingested + exemplarRes, err := i.QueryExemplars(ctx, &client.ExemplarQueryRequest{ + StartTimestampMs: math.MinInt64, + EndTimestampMs: math.MaxInt64, + Matchers: []*client.LabelMatchers{ + {Matchers: []*client.LabelMatcher{{Type: client.REGEX_MATCH, Name: labels.MetricName, Value: ".*"}}}, + }, + }) + + require.NoError(t, err) + require.NotNil(t, exemplarRes) + assert.Equal(t, testData.expectedExemplarsIngested, exemplarRes.Timeseries) + + // Read back metadata to see what has been really ingested. + mres, err := i.MetricsMetadata(ctx, &client.MetricsMetadataRequest{}) + + require.NoError(t, err) + require.NotNil(t, res) + + // Order is never guaranteed. + assert.ElementsMatch(t, testData.expectedMetadataIngested, mres.Metadata) + + // Update active series for metrics check. + if !testData.disableActiveSeries { + i.updateActiveSeries() + } + + // Append additional metrics to assert on. + mn := append(metricNames, testData.additionalMetrics...) + + // Check tracked Prometheus metrics + err = testutil.GatherAndCompare(registry, strings.NewReader(testData.expectedMetrics), mn...) + assert.NoError(t, err) + }) + } +} + +func TestIngester_Push_ShouldCorrectlyTrackMetricsInMultiTenantScenario(t *testing.T) { + metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} + metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) + metricNames := []string{ + "cortex_ingester_ingested_samples_total", + "cortex_ingester_ingested_samples_failures_total", + "cortex_ingester_memory_series", + "cortex_ingester_memory_users", + "cortex_ingester_memory_series_created_total", + "cortex_ingester_memory_series_removed_total", + "cortex_ingester_active_series", + } + + registry := prometheus.NewRegistry() + + // Create a mocked ingester + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + + i, err := prepareIngesterWithBlocksStorage(t, cfg, registry) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until the ingester is ACTIVE + test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push timeseries for each user + for _, userID := range []string{"test-1", "test-2"} { + reqs := []*cortexpb.WriteRequest{ + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, + nil, + cortexpb.API), + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, + nil, + cortexpb.API), + } + + for _, req := range reqs { + ctx := user.InjectOrgID(context.Background(), userID) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + } + + // Update active series for metrics check. + i.updateActiveSeries() + + // Check tracked Prometheus metrics + expectedMetrics := ` + # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. + # TYPE cortex_ingester_ingested_samples_total counter + cortex_ingester_ingested_samples_total 4 + # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. + # TYPE cortex_ingester_ingested_samples_failures_total counter + cortex_ingester_ingested_samples_failures_total 0 + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 2 + # HELP cortex_ingester_memory_series The current number of series in memory. + # TYPE cortex_ingester_memory_series gauge + cortex_ingester_memory_series 2 + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test-1"} 1 + cortex_ingester_memory_series_created_total{user="test-2"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test-1"} 0 + cortex_ingester_memory_series_removed_total{user="test-2"} 0 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test-1"} 1 + cortex_ingester_active_series{user="test-2"} 1 + ` + + assert.NoError(t, testutil.GatherAndCompare(registry, strings.NewReader(expectedMetrics), metricNames...)) +} + +func TestIngester_Push_DecreaseInactiveSeries(t *testing.T) { + metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} + metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) + metricNames := []string{ + "cortex_ingester_memory_series_created_total", + "cortex_ingester_memory_series_removed_total", + "cortex_ingester_active_series", + } + + registry := prometheus.NewRegistry() + + // Create a mocked ingester + cfg := defaultIngesterTestConfig(t) + cfg.ActiveSeriesMetricsIdleTimeout = 100 * time.Millisecond + cfg.LifecyclerConfig.JoinAfter = 0 + + i, err := prepareIngesterWithBlocksStorage(t, cfg, registry) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until the ingester is ACTIVE + test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push timeseries for each user + for _, userID := range []string{"test-1", "test-2"} { + reqs := []*cortexpb.WriteRequest{ + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, + nil, + cortexpb.API), + cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, + nil, + cortexpb.API), + } + + for _, req := range reqs { + ctx := user.InjectOrgID(context.Background(), userID) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + } + + // Wait a bit to make series inactive (set to 100ms above). + time.Sleep(200 * time.Millisecond) + + // Update active series for metrics check. This will remove inactive series. + i.updateActiveSeries() + + // Check tracked Prometheus metrics + expectedMetrics := ` + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="test-1"} 1 + cortex_ingester_memory_series_created_total{user="test-2"} 1 + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="test-1"} 0 + cortex_ingester_memory_series_removed_total{user="test-2"} 0 + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test-1"} 0 + cortex_ingester_active_series{user="test-2"} 0 + ` + + assert.NoError(t, testutil.GatherAndCompare(registry, strings.NewReader(expectedMetrics), metricNames...)) +} + +func BenchmarkIngesterPush(b *testing.B) { + limits := defaultLimitsTestConfig() + benchmarkIngesterPush(b, limits, false) +} + +func benchmarkIngesterPush(b *testing.B, limits validation.Limits, errorsExpected bool) { + registry := prometheus.NewRegistry() + ctx := user.InjectOrgID(context.Background(), userID) + + // Create a mocked ingester + cfg := defaultIngesterTestConfig(b) + cfg.LifecyclerConfig.JoinAfter = 0 + + ingester, err := prepareIngesterWithBlocksStorage(b, cfg, registry) + require.NoError(b, err) + require.NoError(b, services.StartAndAwaitRunning(context.Background(), ingester)) + defer services.StopAndAwaitTerminated(context.Background(), ingester) //nolint:errcheck + + // Wait until the ingester is ACTIVE + test.Poll(b, 100*time.Millisecond, ring.ACTIVE, func() interface{} { + return ingester.lifecycler.GetState() + }) + + // Push a single time series to set the TSDB min time. + metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} + metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) + startTime := util.TimeToMillis(time.Now()) + + currTimeReq := cortexpb.ToWriteRequest( + []labels.Labels{metricLabels}, + []cortexpb.Sample{{Value: 1, TimestampMs: startTime}}, + nil, + cortexpb.API) + _, err = ingester.Push(ctx, currTimeReq) + require.NoError(b, err) + + const ( + series = 10000 + samples = 10 + ) + + allLabels, allSamples := benchmarkData(series) + + b.ResetTimer() + for iter := 0; iter < b.N; iter++ { + // Bump the timestamp on each of our test samples each time round the loop + for j := 0; j < samples; j++ { + for i := range allSamples { + allSamples[i].TimestampMs = startTime + int64(iter*samples+j+1) + } + _, err := ingester.Push(ctx, cortexpb.ToWriteRequest(allLabels, allSamples, nil, cortexpb.API)) + if !errorsExpected { + require.NoError(b, err) + } + } + } +} + +func verifyErrorString(tb testing.TB, err error, expectedErr string) { + if err == nil || !strings.Contains(err.Error(), expectedErr) { + tb.Helper() + tb.Fatalf("unexpected error. expected: %s actual: %v", expectedErr, err) + } +} + +func Benchmark_Ingester_PushOnError(b *testing.B) { + var ( + ctx = user.InjectOrgID(context.Background(), userID) + sampleTimestamp = int64(100) + metricName = "test" + ) + + scenarios := map[string]struct { + numSeriesPerRequest int + numConcurrentClients int + }{ + "no concurrency": { + numSeriesPerRequest: 1000, + numConcurrentClients: 1, + }, + "low concurrency": { + numSeriesPerRequest: 1000, + numConcurrentClients: 100, + }, + "high concurrency": { + numSeriesPerRequest: 1000, + numConcurrentClients: 1000, + }, + } + + instanceLimits := map[string]*InstanceLimits{ + "no limits": nil, + "limits set": {MaxIngestionRate: 1000, MaxInMemoryTenants: 1, MaxInMemorySeries: 1000, MaxInflightPushRequests: 1000}, // these match max values from scenarios + } + + tests := map[string]struct { + // If this returns false, test is skipped. + prepareConfig func(limits *validation.Limits, instanceLimits *InstanceLimits) bool + beforeBenchmark func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) + runBenchmark func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) + }{ + "out of bound samples": { + prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { return true }, + beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { + // Push a single time series to set the TSDB min time. + currTimeReq := cortexpb.ToWriteRequest( + []labels.Labels{{{Name: labels.MetricName, Value: metricName}}}, + []cortexpb.Sample{{Value: 1, TimestampMs: util.TimeToMillis(time.Now())}}, + nil, + cortexpb.API) + _, err := ingester.Push(ctx, currTimeReq) + require.NoError(b, err) + }, + runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { + expectedErr := storage.ErrOutOfBounds.Error() + + // Push out of bound samples. + for n := 0; n < b.N; n++ { + _, err := ingester.Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) // nolint:errcheck + + verifyErrorString(b, err, expectedErr) + } + }, + }, + "out of order samples": { + prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { return true }, + beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { + // For each series, push a single sample with a timestamp greater than next pushes. + for i := 0; i < numSeriesPerRequest; i++ { + currTimeReq := cortexpb.ToWriteRequest( + []labels.Labels{{{Name: labels.MetricName, Value: metricName}, {Name: "cardinality", Value: strconv.Itoa(i)}}}, + []cortexpb.Sample{{Value: 1, TimestampMs: sampleTimestamp + 1}}, + nil, + cortexpb.API) + + _, err := ingester.Push(ctx, currTimeReq) + require.NoError(b, err) + } + }, + runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { + expectedErr := storage.ErrOutOfOrderSample.Error() + + // Push out of order samples. + for n := 0; n < b.N; n++ { + _, err := ingester.Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) // nolint:errcheck + + verifyErrorString(b, err, expectedErr) + } + }, + }, + "per-user series limit reached": { + prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { + limits.MaxLocalSeriesPerUser = 1 + return true + }, + beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { + // Push a series with a metric name different than the one used during the benchmark. + currTimeReq := cortexpb.ToWriteRequest( + []labels.Labels{labels.FromStrings(labels.MetricName, "another")}, + []cortexpb.Sample{{Value: 1, TimestampMs: sampleTimestamp + 1}}, + nil, + cortexpb.API) + _, err := ingester.Push(ctx, currTimeReq) + require.NoError(b, err) + }, + runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { + // Push series with a different name than the one already pushed. + for n := 0; n < b.N; n++ { + _, err := ingester.Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) // nolint:errcheck + verifyErrorString(b, err, "per-user series limit") + } + }, + }, + "per-metric series limit reached": { + prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { + limits.MaxLocalSeriesPerMetric = 1 + return true + }, + beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { + // Push a series with the same metric name but different labels than the one used during the benchmark. + currTimeReq := cortexpb.ToWriteRequest( + []labels.Labels{labels.FromStrings(labels.MetricName, metricName, "cardinality", "another")}, + []cortexpb.Sample{{Value: 1, TimestampMs: sampleTimestamp + 1}}, + nil, + cortexpb.API) + _, err := ingester.Push(ctx, currTimeReq) + require.NoError(b, err) + }, + runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { + // Push series with different labels than the one already pushed. + for n := 0; n < b.N; n++ { + _, err := ingester.Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) // nolint:errcheck + verifyErrorString(b, err, "per-metric series limit") + } + }, + }, + "very low ingestion rate limit": { + prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { + if instanceLimits == nil { + return false + } + instanceLimits.MaxIngestionRate = 0.00001 // very low + return true + }, + beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { + // Send a lot of samples + _, err := ingester.Push(ctx, generateSamplesForLabel(labels.FromStrings(labels.MetricName, "test"), 10000)) + require.NoError(b, err) + + ingester.ingestionRate.Tick() + }, + runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { + // Push series with different labels than the one already pushed. + for n := 0; n < b.N; n++ { + _, err := ingester.Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) + verifyErrorString(b, err, "push rate reached") + } + }, + }, + "max number of tenants reached": { + prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { + if instanceLimits == nil { + return false + } + instanceLimits.MaxInMemoryTenants = 1 + return true + }, + beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { + // Send some samples for one tenant (not the same that is used during the test) + ctx := user.InjectOrgID(context.Background(), "different_tenant") + _, err := ingester.Push(ctx, generateSamplesForLabel(labels.FromStrings(labels.MetricName, "test"), 10000)) + require.NoError(b, err) + }, + runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { + // Push series with different labels than the one already pushed. + for n := 0; n < b.N; n++ { + _, err := ingester.Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) + verifyErrorString(b, err, "max tenants limit reached") + } + }, + }, + "max number of series reached": { + prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { + if instanceLimits == nil { + return false + } + instanceLimits.MaxInMemorySeries = 1 + return true + }, + beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { + _, err := ingester.Push(ctx, generateSamplesForLabel(labels.FromStrings(labels.MetricName, "test"), 10000)) + require.NoError(b, err) + }, + runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { + for n := 0; n < b.N; n++ { + _, err := ingester.Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) + verifyErrorString(b, err, "max series limit reached") + } + }, + }, + "max inflight requests reached": { + prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { + if instanceLimits == nil { + return false + } + instanceLimits.MaxInflightPushRequests = 1 + return true + }, + beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { + ingester.inflightPushRequests.Inc() + }, + runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { + for n := 0; n < b.N; n++ { + _, err := ingester.Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) + verifyErrorString(b, err, "too many inflight push requests") + } + }, + }, + } + + for testName, testData := range tests { + for scenarioName, scenario := range scenarios { + for limitsName, limits := range instanceLimits { + b.Run(fmt.Sprintf("failure: %s, scenario: %s, limits: %s", testName, scenarioName, limitsName), func(b *testing.B) { + registry := prometheus.NewRegistry() + + instanceLimits := limits + if instanceLimits != nil { + // make a copy, to avoid changing value in the instanceLimits map. + newLimits := &InstanceLimits{} + *newLimits = *instanceLimits + instanceLimits = newLimits + } + + // Create a mocked ingester + cfg := defaultIngesterTestConfig(b) + cfg.LifecyclerConfig.JoinAfter = 0 + + limits := defaultLimitsTestConfig() + if !testData.prepareConfig(&limits, instanceLimits) { + b.SkipNow() + } + + cfg.InstanceLimitsFn = func() *InstanceLimits { + return instanceLimits + } + + ingester, err := prepareIngesterWithBlocksStorageAndLimits(b, cfg, limits, "", registry) + require.NoError(b, err) + require.NoError(b, services.StartAndAwaitRunning(context.Background(), ingester)) + defer services.StopAndAwaitTerminated(context.Background(), ingester) //nolint:errcheck + + // Wait until the ingester is ACTIVE + test.Poll(b, 100*time.Millisecond, ring.ACTIVE, func() interface{} { + return ingester.lifecycler.GetState() + }) + + testData.beforeBenchmark(b, ingester, scenario.numSeriesPerRequest) + + // Prepare the request. + metrics := make([]labels.Labels, 0, scenario.numSeriesPerRequest) + samples := make([]cortexpb.Sample, 0, scenario.numSeriesPerRequest) + for i := 0; i < scenario.numSeriesPerRequest; i++ { + metrics = append(metrics, labels.Labels{{Name: labels.MetricName, Value: metricName}, {Name: "cardinality", Value: strconv.Itoa(i)}}) + samples = append(samples, cortexpb.Sample{Value: float64(i), TimestampMs: sampleTimestamp}) + } + + // Run the benchmark. + wg := sync.WaitGroup{} + wg.Add(scenario.numConcurrentClients) + start := make(chan struct{}) + + b.ReportAllocs() + b.ResetTimer() + + for c := 0; c < scenario.numConcurrentClients; c++ { + go func() { + defer wg.Done() + <-start + + testData.runBenchmark(b, ingester, metrics, samples) + }() + } + + b.ResetTimer() + close(start) + wg.Wait() + }) + } + } + } +} + +func Test_Ingester_LabelNames(t *testing.T) { + series := []struct { + lbls labels.Labels + value float64 + timestamp int64 + }{ + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "200"}, {Name: "route", Value: "get_user"}}, 1, 100000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "500"}, {Name: "route", Value: "get_user"}}, 1, 110000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_2"}}, 2, 200000}, + } + + expected := []string{"__name__", "status", "route"} + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push series + ctx := user.InjectOrgID(context.Background(), "test") + + for _, series := range series { + req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + + // Get label names + res, err := i.LabelNames(ctx, &client.LabelNamesRequest{}) + require.NoError(t, err) + assert.ElementsMatch(t, expected, res.LabelNames) +} + +func Test_Ingester_LabelValues(t *testing.T) { + series := []struct { + lbls labels.Labels + value float64 + timestamp int64 + }{ + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "200"}, {Name: "route", Value: "get_user"}}, 1, 100000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "500"}, {Name: "route", Value: "get_user"}}, 1, 110000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_2"}}, 2, 200000}, + } + + expected := map[string][]string{ + "__name__": {"test_1", "test_2"}, + "status": {"200", "500"}, + "route": {"get_user"}, + "unknown": {}, + } + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push series + ctx := user.InjectOrgID(context.Background(), "test") + + for _, series := range series { + req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + + // Get label values + for labelName, expectedValues := range expected { + req := &client.LabelValuesRequest{LabelName: labelName} + res, err := i.LabelValues(ctx, req) + require.NoError(t, err) + assert.ElementsMatch(t, expectedValues, res.LabelValues) + } +} + +func Test_Ingester_Query(t *testing.T) { + series := []struct { + lbls labels.Labels + value float64 + timestamp int64 + }{ + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "200"}, {Name: "route", Value: "get_user"}}, 1, 100000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "500"}, {Name: "route", Value: "get_user"}}, 1, 110000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_2"}}, 2, 200000}, + } + + tests := map[string]struct { + from int64 + to int64 + matchers []*client.LabelMatcher + expected []cortexpb.TimeSeries + }{ + "should return an empty response if no metric matches": { + from: math.MinInt64, + to: math.MaxInt64, + matchers: []*client.LabelMatcher{ + {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "unknown"}, + }, + expected: []cortexpb.TimeSeries{}, + }, + "should filter series by == matcher": { + from: math.MinInt64, + to: math.MaxInt64, + matchers: []*client.LabelMatcher{ + {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, + }, + expected: []cortexpb.TimeSeries{ + {Labels: cortexpb.FromLabelsToLabelAdapters(series[0].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 100000}}}, + {Labels: cortexpb.FromLabelsToLabelAdapters(series[1].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 110000}}}, + }, + }, + "should filter series by != matcher": { + from: math.MinInt64, + to: math.MaxInt64, + matchers: []*client.LabelMatcher{ + {Type: client.NOT_EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, + }, + expected: []cortexpb.TimeSeries{ + {Labels: cortexpb.FromLabelsToLabelAdapters(series[2].lbls), Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 200000}}}, + }, + }, + "should filter series by =~ matcher": { + from: math.MinInt64, + to: math.MaxInt64, + matchers: []*client.LabelMatcher{ + {Type: client.REGEX_MATCH, Name: model.MetricNameLabel, Value: ".*_1"}, + }, + expected: []cortexpb.TimeSeries{ + {Labels: cortexpb.FromLabelsToLabelAdapters(series[0].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 100000}}}, + {Labels: cortexpb.FromLabelsToLabelAdapters(series[1].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 110000}}}, + }, + }, + "should filter series by !~ matcher": { + from: math.MinInt64, + to: math.MaxInt64, + matchers: []*client.LabelMatcher{ + {Type: client.REGEX_NO_MATCH, Name: model.MetricNameLabel, Value: ".*_1"}, + }, + expected: []cortexpb.TimeSeries{ + {Labels: cortexpb.FromLabelsToLabelAdapters(series[2].lbls), Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 200000}}}, + }, + }, + "should filter series by multiple matchers": { + from: math.MinInt64, + to: math.MaxInt64, + matchers: []*client.LabelMatcher{ + {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, + {Type: client.REGEX_MATCH, Name: "status", Value: "5.."}, + }, + expected: []cortexpb.TimeSeries{ + {Labels: cortexpb.FromLabelsToLabelAdapters(series[1].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 110000}}}, + }, + }, + "should filter series by matcher and time range": { + from: 100000, + to: 100000, + matchers: []*client.LabelMatcher{ + {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, + }, + expected: []cortexpb.TimeSeries{ + {Labels: cortexpb.FromLabelsToLabelAdapters(series[0].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 100000}}}, + }, + }, + } + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push series + ctx := user.InjectOrgID(context.Background(), "test") + + for _, series := range series { + req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + + // Run tests + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + req := &client.QueryRequest{ + StartTimestampMs: testData.from, + EndTimestampMs: testData.to, + Matchers: testData.matchers, + } + + res, err := i.Query(ctx, req) + require.NoError(t, err) + assert.ElementsMatch(t, testData.expected, res.Timeseries) + }) + } +} +func TestIngester_Query_ShouldNotCreateTSDBIfDoesNotExists(t *testing.T) { + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Mock request + userID := "test" + ctx := user.InjectOrgID(context.Background(), userID) + req := &client.QueryRequest{} + + res, err := i.Query(ctx, req) + require.NoError(t, err) + assert.Equal(t, &client.QueryResponse{}, res) + + // Check if the TSDB has been created + _, tsdbCreated := i.TSDBState.dbs[userID] + assert.False(t, tsdbCreated) +} + +func TestIngester_LabelValues_ShouldNotCreateTSDBIfDoesNotExists(t *testing.T) { + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Mock request + userID := "test" + ctx := user.InjectOrgID(context.Background(), userID) + req := &client.LabelValuesRequest{} + + res, err := i.LabelValues(ctx, req) + require.NoError(t, err) + assert.Equal(t, &client.LabelValuesResponse{}, res) + + // Check if the TSDB has been created + _, tsdbCreated := i.TSDBState.dbs[userID] + assert.False(t, tsdbCreated) +} + +func TestIngester_LabelNames_ShouldNotCreateTSDBIfDoesNotExists(t *testing.T) { + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Mock request + userID := "test" + ctx := user.InjectOrgID(context.Background(), userID) + req := &client.LabelNamesRequest{} + + res, err := i.LabelNames(ctx, req) + require.NoError(t, err) + assert.Equal(t, &client.LabelNamesResponse{}, res) + + // Check if the TSDB has been created + _, tsdbCreated := i.TSDBState.dbs[userID] + assert.False(t, tsdbCreated) +} + +func TestIngester_Push_ShouldNotCreateTSDBIfNotInActiveState(t *testing.T) { + // Configure the lifecycler to not immediately join the ring, to make sure + // the ingester will NOT be in the ACTIVE state when we'll push samples. + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 10 * time.Second + + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + require.Equal(t, ring.PENDING, i.lifecycler.GetState()) + + // Mock request + userID := "test" + ctx := user.InjectOrgID(context.Background(), userID) + req := &cortexpb.WriteRequest{} + + res, err := i.Push(ctx, req) + assert.Equal(t, wrapWithUser(fmt.Errorf(errTSDBCreateIncompatibleState, "PENDING"), userID).Error(), err.Error()) + assert.Nil(t, res) + + // Check if the TSDB has been created + _, tsdbCreated := i.TSDBState.dbs[userID] + assert.False(t, tsdbCreated) +} + +func TestIngester_getOrCreateTSDB_ShouldNotAllowToCreateTSDBIfIngesterStateIsNotActive(t *testing.T) { + tests := map[string]struct { + state ring.InstanceState + expectedErr error + }{ + "not allow to create TSDB if in PENDING state": { + state: ring.PENDING, + expectedErr: fmt.Errorf(errTSDBCreateIncompatibleState, ring.PENDING), + }, + "not allow to create TSDB if in JOINING state": { + state: ring.JOINING, + expectedErr: fmt.Errorf(errTSDBCreateIncompatibleState, ring.JOINING), + }, + "not allow to create TSDB if in LEAVING state": { + state: ring.LEAVING, + expectedErr: fmt.Errorf(errTSDBCreateIncompatibleState, ring.LEAVING), + }, + "allow to create TSDB if in ACTIVE state": { + state: ring.ACTIVE, + expectedErr: nil, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 60 * time.Second + + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Switch ingester state to the expected one in the test + if i.lifecycler.GetState() != testData.state { + var stateChain []ring.InstanceState + + if testData.state == ring.LEAVING { + stateChain = []ring.InstanceState{ring.ACTIVE, ring.LEAVING} + } else { + stateChain = []ring.InstanceState{testData.state} + } + + for _, s := range stateChain { + err = i.lifecycler.ChangeState(context.Background(), s) + require.NoError(t, err) + } + } + + db, err := i.getOrCreateTSDB("test", false) + assert.Equal(t, testData.expectedErr, err) + + if testData.expectedErr != nil { + assert.Nil(t, db) + } else { + assert.NotNil(t, db) + } + }) + } +} + +func Test_Ingester_MetricsForLabelMatchers(t *testing.T) { + fixtures := []struct { + lbls labels.Labels + value float64 + timestamp int64 + }{ + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "200"}}, 1, 100000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "500"}}, 1, 110000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_2"}}, 2, 200000}, + // The two following series have the same FastFingerprint=e002a3a451262627 + {labels.Labels{{Name: labels.MetricName, Value: "collision"}, {Name: "app", Value: "l"}, {Name: "uniq0", Value: "0"}, {Name: "uniq1", Value: "1"}}, 1, 300000}, + {labels.Labels{{Name: labels.MetricName, Value: "collision"}, {Name: "app", Value: "m"}, {Name: "uniq0", Value: "1"}, {Name: "uniq1", Value: "1"}}, 1, 300000}, + } + + tests := map[string]struct { + from int64 + to int64 + matchers []*client.LabelMatchers + expected []*cortexpb.Metric + }{ + "should return an empty response if no metric match": { + from: math.MinInt64, + to: math.MaxInt64, + matchers: []*client.LabelMatchers{{ + Matchers: []*client.LabelMatcher{ + {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "unknown"}, + }, + }}, + expected: []*cortexpb.Metric{}, + }, + "should filter metrics by single matcher": { + from: math.MinInt64, + to: math.MaxInt64, + matchers: []*client.LabelMatchers{{ + Matchers: []*client.LabelMatcher{ + {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, + }, + }}, + expected: []*cortexpb.Metric{ + {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[0].lbls)}, + {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[1].lbls)}, + }, + }, + "should filter metrics by multiple matchers": { + from: math.MinInt64, + to: math.MaxInt64, + matchers: []*client.LabelMatchers{ + { + Matchers: []*client.LabelMatcher{ + {Type: client.EQUAL, Name: "status", Value: "200"}, + }, + }, + { + Matchers: []*client.LabelMatcher{ + {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_2"}, + }, + }, + }, + expected: []*cortexpb.Metric{ + {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[0].lbls)}, + {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[2].lbls)}, + }, + }, + "should NOT filter metrics by time range to always return known metrics even when queried for older time ranges": { + from: 100, + to: 1000, + matchers: []*client.LabelMatchers{{ + Matchers: []*client.LabelMatcher{ + {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, + }, + }}, + expected: []*cortexpb.Metric{ + {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[0].lbls)}, + {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[1].lbls)}, + }, + }, + "should not return duplicated metrics on overlapping matchers": { + from: math.MinInt64, + to: math.MaxInt64, + matchers: []*client.LabelMatchers{ + { + Matchers: []*client.LabelMatcher{ + {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, + }, + }, + { + Matchers: []*client.LabelMatcher{ + {Type: client.REGEX_MATCH, Name: model.MetricNameLabel, Value: "test.*"}, + }, + }, + }, + expected: []*cortexpb.Metric{ + {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[0].lbls)}, + {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[1].lbls)}, + {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[2].lbls)}, + }, + }, + "should return all matching metrics even if their FastFingerprint collide": { + from: math.MinInt64, + to: math.MaxInt64, + matchers: []*client.LabelMatchers{{ + Matchers: []*client.LabelMatcher{ + {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "collision"}, + }, + }}, + expected: []*cortexpb.Metric{ + {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[3].lbls)}, + {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[4].lbls)}, + }, + }, + } + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push fixtures + ctx := user.InjectOrgID(context.Background(), "test") + + for _, series := range fixtures { + req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + + // Run tests + for testName, testData := range tests { + testData := testData + + t.Run(testName, func(t *testing.T) { + req := &client.MetricsForLabelMatchersRequest{ + StartTimestampMs: testData.from, + EndTimestampMs: testData.to, + MatchersSet: testData.matchers, + } + + res, err := i.MetricsForLabelMatchers(ctx, req) + require.NoError(t, err) + assert.ElementsMatch(t, testData.expected, res.Metric) + }) + } +} + +func Test_Ingester_MetricsForLabelMatchers_Deduplication(t *testing.T) { + const ( + userID = "test" + numSeries = 100000 + ) + + now := util.TimeToMillis(time.Now()) + i := createIngesterWithSeries(t, userID, numSeries, 1, now, 1) + ctx := user.InjectOrgID(context.Background(), "test") + + req := &client.MetricsForLabelMatchersRequest{ + StartTimestampMs: now, + EndTimestampMs: now, + // Overlapping matchers to make sure series are correctly deduplicated. + MatchersSet: []*client.LabelMatchers{ + {Matchers: []*client.LabelMatcher{ + {Type: client.REGEX_MATCH, Name: model.MetricNameLabel, Value: "test.*"}, + }}, + {Matchers: []*client.LabelMatcher{ + {Type: client.REGEX_MATCH, Name: model.MetricNameLabel, Value: "test.*0"}, + }}, + }, + } + + res, err := i.MetricsForLabelMatchers(ctx, req) + require.NoError(t, err) + require.Len(t, res.GetMetric(), numSeries) +} + +func Benchmark_Ingester_MetricsForLabelMatchers(b *testing.B) { + var ( + userID = "test" + numSeries = 10000 + numSamplesPerSeries = 60 * 6 // 6h on 1 sample per minute + startTimestamp = util.TimeToMillis(time.Now()) + step = int64(60000) // 1 sample per minute + ) + + i := createIngesterWithSeries(b, userID, numSeries, numSamplesPerSeries, startTimestamp, step) + ctx := user.InjectOrgID(context.Background(), "test") + + // Flush the ingester to ensure blocks have been compacted, so we'll test + // fetching labels from blocks. + i.Flush() + + b.ResetTimer() + b.ReportAllocs() + + for n := 0; n < b.N; n++ { + req := &client.MetricsForLabelMatchersRequest{ + StartTimestampMs: math.MinInt64, + EndTimestampMs: math.MaxInt64, + MatchersSet: []*client.LabelMatchers{{Matchers: []*client.LabelMatcher{ + {Type: client.REGEX_MATCH, Name: model.MetricNameLabel, Value: "test.*"}, + }}}, + } + + res, err := i.MetricsForLabelMatchers(ctx, req) + require.NoError(b, err) + require.Len(b, res.GetMetric(), numSeries) + } +} + +// createIngesterWithSeries creates an ingester and push numSeries with numSamplesPerSeries each. +func createIngesterWithSeries(t testing.TB, userID string, numSeries, numSamplesPerSeries int, startTimestamp, step int64) *Ingester { + const maxBatchSize = 1000 + + // Create ingester. + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + t.Cleanup(func() { + require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) + }) + + // Wait until it's ACTIVE. + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push fixtures. + ctx := user.InjectOrgID(context.Background(), userID) + + for ts := startTimestamp; ts < startTimestamp+(step*int64(numSamplesPerSeries)); ts += step { + for o := 0; o < numSeries; o += maxBatchSize { + batchSize := util_math.Min(maxBatchSize, numSeries-o) + + // Generate metrics and samples (1 for each series). + metrics := make([]labels.Labels, 0, batchSize) + samples := make([]cortexpb.Sample, 0, batchSize) + + for s := 0; s < batchSize; s++ { + metrics = append(metrics, labels.Labels{ + {Name: labels.MetricName, Value: fmt.Sprintf("test_%d", o+s)}, + }) + + samples = append(samples, cortexpb.Sample{ + TimestampMs: ts, + Value: 1, + }) + } + + // Send metrics to the ingester. + req := cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + } + + return i +} + +func TestIngester_QueryStream(t *testing.T) { + // Create ingester. + cfg := defaultIngesterTestConfig(t) + + // change stream type in runtime. + var streamType QueryStreamType + cfg.StreamTypeFn = func() QueryStreamType { + return streamType + } + + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE. + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push series. + ctx := user.InjectOrgID(context.Background(), userID) + lbls := labels.Labels{{Name: labels.MetricName, Value: "foo"}} + req, _, expectedResponseSamples, expectedResponseChunks := mockWriteRequest(t, lbls, 123000, 456) + _, err = i.Push(ctx, req) + require.NoError(t, err) + + // Create a GRPC server used to query back the data. + serv := grpc.NewServer(grpc.StreamInterceptor(middleware.StreamServerUserHeaderInterceptor)) + defer serv.GracefulStop() + client.RegisterIngesterServer(serv, i) + + listener, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + + go func() { + require.NoError(t, serv.Serve(listener)) + }() + + // Query back the series using GRPC streaming. + c, err := client.MakeIngesterClient(listener.Addr().String(), defaultClientTestConfig()) + require.NoError(t, err) + defer c.Close() + + queryRequest := &client.QueryRequest{ + StartTimestampMs: 0, + EndTimestampMs: 200000, + Matchers: []*client.LabelMatcher{{ + Type: client.EQUAL, + Name: model.MetricNameLabel, + Value: "foo", + }}, + } + + samplesTest := func(t *testing.T) { + s, err := c.QueryStream(ctx, queryRequest) + require.NoError(t, err) + + count := 0 + var lastResp *client.QueryStreamResponse + for { + resp, err := s.Recv() + if err == io.EOF { + break + } + require.NoError(t, err) + require.Zero(t, len(resp.Chunkseries)) // No chunks expected + count += len(resp.Timeseries) + lastResp = resp + } + require.Equal(t, 1, count) + require.Equal(t, expectedResponseSamples, lastResp) + } + + chunksTest := func(t *testing.T) { + s, err := c.QueryStream(ctx, queryRequest) + require.NoError(t, err) + + count := 0 + var lastResp *client.QueryStreamResponse + for { + resp, err := s.Recv() + if err == io.EOF { + break + } + require.NoError(t, err) + require.Zero(t, len(resp.Timeseries)) // No samples expected + count += len(resp.Chunkseries) + lastResp = resp + } + require.Equal(t, 1, count) + require.Equal(t, expectedResponseChunks, lastResp) + } + + streamType = QueryStreamDefault + t.Run("default", samplesTest) + + streamType = QueryStreamSamples + t.Run("samples", samplesTest) + + streamType = QueryStreamChunks + t.Run("chunks", chunksTest) +} + +func TestIngester_QueryStreamManySamples(t *testing.T) { + // Create ingester. + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE. + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push series. + ctx := user.InjectOrgID(context.Background(), userID) + + const samplesCount = 100000 + samples := make([]cortexpb.Sample, 0, samplesCount) + + for i := 0; i < samplesCount; i++ { + samples = append(samples, cortexpb.Sample{ + Value: float64(i), + TimestampMs: int64(i), + }) + } + + // 10k samples encode to around 140 KiB, + _, err = i.Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "1"}}, samples[0:10000])) + require.NoError(t, err) + + // 100k samples encode to around 1.4 MiB, + _, err = i.Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "2"}}, samples)) + require.NoError(t, err) + + // 50k samples encode to around 716 KiB, + _, err = i.Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "3"}}, samples[0:50000])) + require.NoError(t, err) + + // Create a GRPC server used to query back the data. + serv := grpc.NewServer(grpc.StreamInterceptor(middleware.StreamServerUserHeaderInterceptor)) + defer serv.GracefulStop() + client.RegisterIngesterServer(serv, i) + + listener, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + + go func() { + require.NoError(t, serv.Serve(listener)) + }() + + // Query back the series using GRPC streaming. + c, err := client.MakeIngesterClient(listener.Addr().String(), defaultClientTestConfig()) + require.NoError(t, err) + defer c.Close() + + s, err := c.QueryStream(ctx, &client.QueryRequest{ + StartTimestampMs: 0, + EndTimestampMs: samplesCount + 1, + + Matchers: []*client.LabelMatcher{{ + Type: client.EQUAL, + Name: model.MetricNameLabel, + Value: "foo", + }}, + }) + require.NoError(t, err) + + recvMsgs := 0 + series := 0 + totalSamples := 0 + + for { + resp, err := s.Recv() + if err == io.EOF { + break + } + require.NoError(t, err) + require.True(t, len(resp.Timeseries) > 0) // No empty messages. + + recvMsgs++ + series += len(resp.Timeseries) + + for _, ts := range resp.Timeseries { + totalSamples += len(ts.Samples) + } + } + + // As ingester doesn't guarantee sorting of series, we can get 2 (10k + 50k in first, 100k in second) + // or 3 messages (small series first, 100k second, small series last). + + require.True(t, 2 <= recvMsgs && recvMsgs <= 3) + require.Equal(t, 3, series) + require.Equal(t, 10000+50000+samplesCount, totalSamples) +} + +func TestIngester_QueryStreamManySamplesChunks(t *testing.T) { + // Create ingester. + cfg := defaultIngesterTestConfig(t) + cfg.StreamChunksWhenUsingBlocks = true + + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE. + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push series. + ctx := user.InjectOrgID(context.Background(), userID) + + const samplesCount = 1000000 + samples := make([]cortexpb.Sample, 0, samplesCount) + + for i := 0; i < samplesCount; i++ { + samples = append(samples, cortexpb.Sample{ + Value: float64(i), + TimestampMs: int64(i), + }) + } + + // 100k samples in chunks use about 154 KiB, + _, err = i.Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "1"}}, samples[0:100000])) + require.NoError(t, err) + + // 1M samples in chunks use about 1.51 MiB, + _, err = i.Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "2"}}, samples)) + require.NoError(t, err) + + // 500k samples in chunks need 775 KiB, + _, err = i.Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "3"}}, samples[0:500000])) + require.NoError(t, err) + + // Create a GRPC server used to query back the data. + serv := grpc.NewServer(grpc.StreamInterceptor(middleware.StreamServerUserHeaderInterceptor)) + defer serv.GracefulStop() + client.RegisterIngesterServer(serv, i) + + listener, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + + go func() { + require.NoError(t, serv.Serve(listener)) + }() + + // Query back the series using GRPC streaming. + c, err := client.MakeIngesterClient(listener.Addr().String(), defaultClientTestConfig()) + require.NoError(t, err) + defer c.Close() + + s, err := c.QueryStream(ctx, &client.QueryRequest{ + StartTimestampMs: 0, + EndTimestampMs: samplesCount + 1, + + Matchers: []*client.LabelMatcher{{ + Type: client.EQUAL, + Name: model.MetricNameLabel, + Value: "foo", + }}, + }) + require.NoError(t, err) + + recvMsgs := 0 + series := 0 + totalSamples := 0 + + for { + resp, err := s.Recv() + if err == io.EOF { + break + } + require.NoError(t, err) + require.True(t, len(resp.Chunkseries) > 0) // No empty messages. + + recvMsgs++ + series += len(resp.Chunkseries) + + for _, ts := range resp.Chunkseries { + for _, c := range ts.Chunks { + ch, err := encoding.NewForEncoding(encoding.Encoding(c.Encoding)) + require.NoError(t, err) + require.NoError(t, ch.UnmarshalFromBuf(c.Data)) + + totalSamples += ch.Len() + } + } + } + + // As ingester doesn't guarantee sorting of series, we can get 2 (100k + 500k in first, 1M in second) + // or 3 messages (100k or 500k first, 1M second, and 500k or 100k last). + + require.True(t, 2 <= recvMsgs && recvMsgs <= 3) + require.Equal(t, 3, series) + require.Equal(t, 100000+500000+samplesCount, totalSamples) +} + +func writeRequestSingleSeries(lbls labels.Labels, samples []cortexpb.Sample) *cortexpb.WriteRequest { + req := &cortexpb.WriteRequest{ + Source: cortexpb.API, + } + + ts := cortexpb.TimeSeries{} + ts.Labels = cortexpb.FromLabelsToLabelAdapters(lbls) + ts.Samples = samples + req.Timeseries = append(req.Timeseries, cortexpb.PreallocTimeseries{TimeSeries: &ts}) + + return req +} + +type mockQueryStreamServer struct { + grpc.ServerStream + ctx context.Context +} + +func (m *mockQueryStreamServer) Send(response *client.QueryStreamResponse) error { + return nil +} + +func (m *mockQueryStreamServer) Context() context.Context { + return m.ctx +} + +func BenchmarkIngester_QueryStream_Samples(b *testing.B) { + benchmarkQueryStream(b, false) +} + +func BenchmarkIngester_QueryStream_Chunks(b *testing.B) { + benchmarkQueryStream(b, true) +} + +func benchmarkQueryStream(b *testing.B, streamChunks bool) { + cfg := defaultIngesterTestConfig(b) + cfg.StreamChunksWhenUsingBlocks = streamChunks + + // Create ingester. + i, err := prepareIngesterWithBlocksStorage(b, cfg, nil) + require.NoError(b, err) + require.NoError(b, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE. + test.Poll(b, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push series. + ctx := user.InjectOrgID(context.Background(), userID) + + const samplesCount = 1000 + samples := make([]cortexpb.Sample, 0, samplesCount) + + for i := 0; i < samplesCount; i++ { + samples = append(samples, cortexpb.Sample{ + Value: float64(i), + TimestampMs: int64(i), + }) + } + + const seriesCount = 100 + for s := 0; s < seriesCount; s++ { + _, err = i.Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: strconv.Itoa(s)}}, samples)) + require.NoError(b, err) + } + + req := &client.QueryRequest{ + StartTimestampMs: 0, + EndTimestampMs: samplesCount + 1, + + Matchers: []*client.LabelMatcher{{ + Type: client.EQUAL, + Name: model.MetricNameLabel, + Value: "foo", + }}, + } + + mockStream := &mockQueryStreamServer{ctx: ctx} + + b.ResetTimer() + + for ix := 0; ix < b.N; ix++ { + err := i.QueryStream(req, mockStream) + require.NoError(b, err) + } +} + +func mockWriteRequest(t *testing.T, lbls labels.Labels, value float64, timestampMs int64) (*cortexpb.WriteRequest, *client.QueryResponse, *client.QueryStreamResponse, *client.QueryStreamResponse) { + samples := []cortexpb.Sample{ + { + TimestampMs: timestampMs, + Value: value, + }, + } + + req := cortexpb.ToWriteRequest([]labels.Labels{lbls}, samples, nil, cortexpb.API) + + // Generate the expected response + expectedQueryRes := &client.QueryResponse{ + Timeseries: []cortexpb.TimeSeries{ + { + Labels: cortexpb.FromLabelsToLabelAdapters(lbls), + Samples: samples, + }, + }, + } + + expectedQueryStreamResSamples := &client.QueryStreamResponse{ + Timeseries: []cortexpb.TimeSeries{ + { + Labels: cortexpb.FromLabelsToLabelAdapters(lbls), + Samples: samples, + }, + }, + } + + chunk := chunkenc.NewXORChunk() + app, err := chunk.Appender() + require.NoError(t, err) + app.Append(timestampMs, value) + chunk.Compact() + + expectedQueryStreamResChunks := &client.QueryStreamResponse{ + Chunkseries: []client.TimeSeriesChunk{ + { + Labels: cortexpb.FromLabelsToLabelAdapters(lbls), + Chunks: []client.Chunk{ + { + StartTimestampMs: timestampMs, + EndTimestampMs: timestampMs, + Encoding: int32(encoding.PrometheusXorChunk), + Data: chunk.Bytes(), + }, + }, + }, + }, + } + + return req, expectedQueryRes, expectedQueryStreamResSamples, expectedQueryStreamResChunks +} + +func prepareIngesterWithBlocksStorage(t testing.TB, ingesterCfg Config, registerer prometheus.Registerer) (*Ingester, error) { + return prepareIngesterWithBlocksStorageAndLimits(t, ingesterCfg, defaultLimitsTestConfig(), "", registerer) +} + +func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, limits validation.Limits, dataDir string, registerer prometheus.Registerer) (*Ingester, error) { + // Create a data dir if none has been provided. + if dataDir == "" { + dataDir = t.TempDir() + } + + bucketDir := t.TempDir() + + overrides, err := validation.NewOverrides(limits, nil) + if err != nil { + return nil, err + } + + ingesterCfg.BlocksStorageConfig.TSDB.Dir = dataDir + ingesterCfg.BlocksStorageConfig.Bucket.Backend = "filesystem" + ingesterCfg.BlocksStorageConfig.Bucket.Filesystem.Directory = bucketDir + + ingester, err := New(ingesterCfg, overrides, registerer, log.NewNopLogger()) + if err != nil { + return nil, err + } + + return ingester, nil +} + +func TestIngester_OpenExistingTSDBOnStartup(t *testing.T) { + t.Parallel() + + tests := map[string]struct { + concurrency int + setup func(*testing.T, string) + check func(*testing.T, *Ingester) + expectedErr string + }{ + "should not load TSDB if the user directory is empty": { + concurrency: 10, + setup: func(t *testing.T, dir string) { + require.NoError(t, os.Mkdir(filepath.Join(dir, "user0"), 0700)) + }, + check: func(t *testing.T, i *Ingester) { + require.Nil(t, i.getTSDB("user0")) + }, + }, + "should not load any TSDB if the root directory is empty": { + concurrency: 10, + setup: func(t *testing.T, dir string) {}, + check: func(t *testing.T, i *Ingester) { + require.Zero(t, len(i.TSDBState.dbs)) + }, + }, + "should not load any TSDB is the root directory is missing": { + concurrency: 10, + setup: func(t *testing.T, dir string) { + require.NoError(t, os.Remove(dir)) + }, + check: func(t *testing.T, i *Ingester) { + require.Zero(t, len(i.TSDBState.dbs)) + }, + }, + "should load TSDB for any non-empty user directory": { + concurrency: 10, + setup: func(t *testing.T, dir string) { + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "dummy"), 0700)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1", "dummy"), 0700)) + require.NoError(t, os.Mkdir(filepath.Join(dir, "user2"), 0700)) + }, + check: func(t *testing.T, i *Ingester) { + require.Equal(t, 2, len(i.TSDBState.dbs)) + require.NotNil(t, i.getTSDB("user0")) + require.NotNil(t, i.getTSDB("user1")) + require.Nil(t, i.getTSDB("user2")) + }, + }, + "should load all TSDBs on concurrency < number of TSDBs": { + concurrency: 2, + setup: func(t *testing.T, dir string) { + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "dummy"), 0700)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1", "dummy"), 0700)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user2", "dummy"), 0700)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user3", "dummy"), 0700)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user4", "dummy"), 0700)) + }, + check: func(t *testing.T, i *Ingester) { + require.Equal(t, 5, len(i.TSDBState.dbs)) + require.NotNil(t, i.getTSDB("user0")) + require.NotNil(t, i.getTSDB("user1")) + require.NotNil(t, i.getTSDB("user2")) + require.NotNil(t, i.getTSDB("user3")) + require.NotNil(t, i.getTSDB("user4")) + }, + }, + "should fail and rollback if an error occur while loading a TSDB on concurrency > number of TSDBs": { + concurrency: 10, + setup: func(t *testing.T, dir string) { + // Create a fake TSDB on disk with an empty chunks head segment file (it's invalid unless + // it's the last one and opening TSDB should fail). + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "wal", ""), 0700)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "chunks_head", ""), 0700)) + require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user0", "chunks_head", "00000001"), nil, 0700)) + require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user0", "chunks_head", "00000002"), nil, 0700)) + + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1", "dummy"), 0700)) + }, + check: func(t *testing.T, i *Ingester) { + require.Equal(t, 0, len(i.TSDBState.dbs)) + require.Nil(t, i.getTSDB("user0")) + require.Nil(t, i.getTSDB("user1")) + }, + expectedErr: "unable to open TSDB for user user0", + }, + "should fail and rollback if an error occur while loading a TSDB on concurrency < number of TSDBs": { + concurrency: 2, + setup: func(t *testing.T, dir string) { + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "dummy"), 0700)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1", "dummy"), 0700)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user3", "dummy"), 0700)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user4", "dummy"), 0700)) + + // Create a fake TSDB on disk with an empty chunks head segment file (it's invalid unless + // it's the last one and opening TSDB should fail). + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user2", "wal", ""), 0700)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "user2", "chunks_head", ""), 0700)) + require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user2", "chunks_head", "00000001"), nil, 0700)) + require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user2", "chunks_head", "00000002"), nil, 0700)) + }, + check: func(t *testing.T, i *Ingester) { + require.Equal(t, 0, len(i.TSDBState.dbs)) + require.Nil(t, i.getTSDB("user0")) + require.Nil(t, i.getTSDB("user1")) + require.Nil(t, i.getTSDB("user2")) + require.Nil(t, i.getTSDB("user3")) + require.Nil(t, i.getTSDB("user4")) + }, + expectedErr: "unable to open TSDB for user user2", + }, + } + + for name, test := range tests { + testName := name + testData := test + t.Run(testName, func(t *testing.T) { + limits := defaultLimitsTestConfig() + + overrides, err := validation.NewOverrides(limits, nil) + require.NoError(t, err) + + // Create a temporary directory for TSDB + tempDir := t.TempDir() + + ingesterCfg := defaultIngesterTestConfig(t) + ingesterCfg.BlocksStorageConfig.TSDB.Dir = tempDir + ingesterCfg.BlocksStorageConfig.TSDB.MaxTSDBOpeningConcurrencyOnStartup = testData.concurrency + ingesterCfg.BlocksStorageConfig.Bucket.Backend = "s3" + ingesterCfg.BlocksStorageConfig.Bucket.S3.Endpoint = "localhost" + + // setup the tsdbs dir + testData.setup(t, tempDir) + + ingester, err := New(ingesterCfg, overrides, nil, log.NewNopLogger()) + require.NoError(t, err) + + startErr := services.StartAndAwaitRunning(context.Background(), ingester) + if testData.expectedErr == "" { + require.NoError(t, startErr) + } else { + require.Error(t, startErr) + assert.Contains(t, startErr.Error(), testData.expectedErr) + } + + defer services.StopAndAwaitTerminated(context.Background(), ingester) //nolint:errcheck + testData.check(t, ingester) + }) + } +} + +func TestIngester_shipBlocks(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 2 + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Create the TSDB for 3 users and then replace the shipper with the mocked one + mocks := []*shipperMock{} + for _, userID := range []string{"user-1", "user-2", "user-3"} { + userDB, err := i.getOrCreateTSDB(userID, false) + require.NoError(t, err) + require.NotNil(t, userDB) + + m := &shipperMock{} + m.On("Sync", mock.Anything).Return(0, nil) + mocks = append(mocks, m) + + userDB.shipper = m + } + + // Ship blocks and assert on the mocked shipper + i.shipBlocks(context.Background(), nil) + + for _, m := range mocks { + m.AssertNumberOfCalls(t, "Sync", 1) + } +} + +func TestIngester_dontShipBlocksWhenTenantDeletionMarkerIsPresent(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 2 + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + + // Use in-memory bucket. + bucket := objstore.NewInMemBucket() + + i.TSDBState.bucket = bucket + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + pushSingleSampleWithMetadata(t, i) + require.Equal(t, int64(1), i.TSDBState.seriesCount.Load()) + i.compactBlocks(context.Background(), true, nil) + require.Equal(t, int64(0), i.TSDBState.seriesCount.Load()) + i.shipBlocks(context.Background(), nil) + + numObjects := len(bucket.Objects()) + require.NotZero(t, numObjects) + + require.NoError(t, cortex_tsdb.WriteTenantDeletionMark(context.Background(), bucket, userID, nil, cortex_tsdb.NewTenantDeletionMark(time.Now()))) + numObjects++ // For deletion marker + + db := i.getTSDB(userID) + require.NotNil(t, db) + db.lastDeletionMarkCheck.Store(0) + + // After writing tenant deletion mark, + pushSingleSampleWithMetadata(t, i) + require.Equal(t, int64(1), i.TSDBState.seriesCount.Load()) + i.compactBlocks(context.Background(), true, nil) + require.Equal(t, int64(0), i.TSDBState.seriesCount.Load()) + i.shipBlocks(context.Background(), nil) + + numObjectsAfterMarkingTenantForDeletion := len(bucket.Objects()) + require.Equal(t, numObjects, numObjectsAfterMarkingTenantForDeletion) + require.Equal(t, tsdbTenantMarkedForDeletion, i.closeAndDeleteUserTSDBIfIdle(userID)) +} + +func TestIngester_seriesCountIsCorrectAfterClosingTSDBForDeletedTenant(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 2 + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + + // Use in-memory bucket. + bucket := objstore.NewInMemBucket() + + // Write tenant deletion mark. + require.NoError(t, cortex_tsdb.WriteTenantDeletionMark(context.Background(), bucket, userID, nil, cortex_tsdb.NewTenantDeletionMark(time.Now()))) + + i.TSDBState.bucket = bucket + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + pushSingleSampleWithMetadata(t, i) + require.Equal(t, int64(1), i.TSDBState.seriesCount.Load()) + + // We call shipBlocks to check for deletion marker (it happens inside this method). + i.shipBlocks(context.Background(), nil) + + // Verify that tenant deletion mark was found. + db := i.getTSDB(userID) + require.NotNil(t, db) + require.True(t, db.deletionMarkFound.Load()) + + // If we try to close TSDB now, it should succeed, even though TSDB is not idle and empty. + require.Equal(t, uint64(1), db.Head().NumSeries()) + require.Equal(t, tsdbTenantMarkedForDeletion, i.closeAndDeleteUserTSDBIfIdle(userID)) + + // Closing should decrease series count. + require.Equal(t, int64(0), i.TSDBState.seriesCount.Load()) +} + +func TestIngester_sholdUpdateCacheShippedBlocks(t *testing.T) { + ctx := context.Background() + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 2 + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(ctx, i)) + defer services.StopAndAwaitTerminated(ctx, i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + mockUserShipper(t, i) + + // Mock the shipper meta (no blocks). + db := i.getTSDB(userID) + err = db.updateCachedShippedBlocks() + require.NoError(t, err) + + require.Equal(t, len(db.getCachedShippedBlocks()), 0) + shippedBlock, _ := ulid.Parse("01D78XZ44G0000000000000000") + + require.NoError(t, shipper.WriteMetaFile(log.NewNopLogger(), db.db.Dir(), &shipper.Meta{ + Version: shipper.MetaVersion1, + Uploaded: []ulid.ULID{shippedBlock}, + })) + + err = db.updateCachedShippedBlocks() + require.NoError(t, err) + + require.Equal(t, len(db.getCachedShippedBlocks()), 1) +} + +func TestIngester_closeAndDeleteUserTSDBIfIdle_shouldNotCloseTSDBIfShippingIsInProgress(t *testing.T) { + ctx := context.Background() + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 2 + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(ctx, i)) + defer services.StopAndAwaitTerminated(ctx, i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Mock the shipper to slow down Sync() execution. + s := mockUserShipper(t, i) + s.On("Sync", mock.Anything).Run(func(args mock.Arguments) { + time.Sleep(3 * time.Second) + }).Return(0, nil) + + // Mock the shipper meta (no blocks). + db := i.getTSDB(userID) + require.NoError(t, shipper.WriteMetaFile(log.NewNopLogger(), db.db.Dir(), &shipper.Meta{ + Version: shipper.MetaVersion1, + })) + + // Run blocks shipping in a separate go routine. + go i.shipBlocks(ctx, nil) + + // Wait until shipping starts. + test.Poll(t, 1*time.Second, activeShipping, func() interface{} { + db.stateMtx.RLock() + defer db.stateMtx.RUnlock() + return db.state + }) + assert.Equal(t, tsdbNotActive, i.closeAndDeleteUserTSDBIfIdle(userID)) +} + +func TestIngester_closingAndOpeningTsdbConcurrently(t *testing.T) { + ctx := context.Background() + cfg := defaultIngesterTestConfig(t) + cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast. + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(ctx, i)) + defer services.StopAndAwaitTerminated(ctx, i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + _, err = i.getOrCreateTSDB(userID, false) + require.NoError(t, err) + + iterations := 5000 + chanErr := make(chan error, 1) + quit := make(chan bool) + + go func() { + for { + select { + case <-quit: + return + default: + _, err = i.getOrCreateTSDB(userID, false) + if err != nil { + chanErr <- err + } + } + } + }() + + for k := 0; k < iterations; k++ { + i.closeAndDeleteUserTSDBIfIdle(userID) + } + + select { + case err := <-chanErr: + assert.Fail(t, err.Error()) + quit <- true + default: + quit <- true + } +} + +func TestIngester_idleCloseEmptyTSDB(t *testing.T) { + ctx := context.Background() + cfg := defaultIngesterTestConfig(t) + cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Minute + cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Minute + cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast. + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(ctx, i)) + defer services.StopAndAwaitTerminated(ctx, i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + db, err := i.getOrCreateTSDB(userID, true) + require.NoError(t, err) + require.NotNil(t, db) + + // Run compaction and shipping. + i.compactBlocks(context.Background(), true, nil) + i.shipBlocks(context.Background(), nil) + + // Make sure we can close completely empty TSDB without problems. + require.Equal(t, tsdbIdleClosed, i.closeAndDeleteUserTSDBIfIdle(userID)) + + // Verify that it was closed. + db = i.getTSDB(userID) + require.Nil(t, db) + + // And we can recreate it again, if needed. + db, err = i.getOrCreateTSDB(userID, true) + require.NoError(t, err) + require.NotNil(t, db) +} + +type shipperMock struct { + mock.Mock +} + +// Sync mocks Shipper.Sync() +func (m *shipperMock) Sync(ctx context.Context) (uploaded int, err error) { + args := m.Called(ctx) + return args.Int(0), args.Error(1) +} + +func TestIngester_invalidSamplesDontChangeLastUpdateTime(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + ctx := user.InjectOrgID(context.Background(), userID) + sampleTimestamp := int64(model.Now()) + + { + req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, sampleTimestamp) + _, err = i.Push(ctx, req) + require.NoError(t, err) + } + + db := i.getTSDB(userID) + lastUpdate := db.lastUpdate.Load() + + // Wait until 1 second passes. + test.Poll(t, 1*time.Second, time.Now().Unix()+1, func() interface{} { + return time.Now().Unix() + }) + + // Push another sample to the same metric and timestamp, with different value. We expect to get error. + { + req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 1, sampleTimestamp) + _, err = i.Push(ctx, req) + require.Error(t, err) + } + + // Make sure last update hasn't changed. + require.Equal(t, lastUpdate, db.lastUpdate.Load()) +} + +func TestIngester_flushing(t *testing.T) { + for name, tc := range map[string]struct { + setupIngester func(cfg *Config) + action func(t *testing.T, i *Ingester, reg *prometheus.Registry) + }{ + "ingesterShutdown": { + setupIngester: func(cfg *Config) { + cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown = true + cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown = true + }, + action: func(t *testing.T, i *Ingester, reg *prometheus.Registry) { + pushSingleSampleWithMetadata(t, i) + + // Nothing shipped yet. + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 0 + `), "cortex_ingester_shipper_uploads_total")) + + // Shutdown ingester. This triggers flushing of the block. + require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) + + verifyCompactedHead(t, i, true) + + // Verify that block has been shipped. + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 1 + `), "cortex_ingester_shipper_uploads_total")) + }, + }, + + "shutdownHandler": { + setupIngester: func(cfg *Config) { + cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown = false + cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown = true + }, + + action: func(t *testing.T, i *Ingester, reg *prometheus.Registry) { + pushSingleSampleWithMetadata(t, i) + + // Nothing shipped yet. + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 0 + `), "cortex_ingester_shipper_uploads_total")) + + i.ShutdownHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/shutdown", nil)) + + verifyCompactedHead(t, i, true) + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 1 + `), "cortex_ingester_shipper_uploads_total")) + }, + }, + + "flushHandler": { + setupIngester: func(cfg *Config) { + cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown = false + }, + + action: func(t *testing.T, i *Ingester, reg *prometheus.Registry) { + pushSingleSampleWithMetadata(t, i) + + // Nothing shipped yet. + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 0 + `), "cortex_ingester_shipper_uploads_total")) + + // Using wait=true makes this a synchronous call. + i.FlushHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/flush?wait=true", nil)) + + verifyCompactedHead(t, i, true) + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 1 + `), "cortex_ingester_shipper_uploads_total")) + }, + }, + + "flushHandlerWithListOfTenants": { + setupIngester: func(cfg *Config) { + cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown = false + }, + + action: func(t *testing.T, i *Ingester, reg *prometheus.Registry) { + pushSingleSampleWithMetadata(t, i) + + // Nothing shipped yet. + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 0 + `), "cortex_ingester_shipper_uploads_total")) + + users := url.Values{} + users.Add(tenantParam, "unknown-user") + users.Add(tenantParam, "another-unknown-user") + + // Using wait=true makes this a synchronous call. + i.FlushHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/flush?wait=true&"+users.Encode(), nil)) + + // Still nothing shipped or compacted. + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 0 + `), "cortex_ingester_shipper_uploads_total")) + verifyCompactedHead(t, i, false) + + users = url.Values{} + users.Add(tenantParam, "different-user") + users.Add(tenantParam, userID) // Our user + users.Add(tenantParam, "yet-another-user") + + i.FlushHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/flush?wait=true&"+users.Encode(), nil)) + + verifyCompactedHead(t, i, true) + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 1 + `), "cortex_ingester_shipper_uploads_total")) + }, + }, + + "flushMultipleBlocksWithDataSpanning3Days": { + setupIngester: func(cfg *Config) { + cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown = false + }, + + action: func(t *testing.T, i *Ingester, reg *prometheus.Registry) { + // Pushing 5 samples, spanning over 3 days. + // First block + pushSingleSampleAtTime(t, i, 23*time.Hour.Milliseconds()) + pushSingleSampleAtTime(t, i, 24*time.Hour.Milliseconds()-1) + + // Second block + pushSingleSampleAtTime(t, i, 24*time.Hour.Milliseconds()+1) + pushSingleSampleAtTime(t, i, 25*time.Hour.Milliseconds()) + + // Third block, far in the future. + pushSingleSampleAtTime(t, i, 50*time.Hour.Milliseconds()) + + // Nothing shipped yet. + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 0 + `), "cortex_ingester_shipper_uploads_total")) + + i.FlushHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/flush?wait=true", nil)) + + verifyCompactedHead(t, i, true) + + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 3 + `), "cortex_ingester_shipper_uploads_total")) + + userDB := i.getTSDB(userID) + require.NotNil(t, userDB) + + blocks := userDB.Blocks() + require.Equal(t, 3, len(blocks)) + require.Equal(t, 23*time.Hour.Milliseconds(), blocks[0].Meta().MinTime) + require.Equal(t, 24*time.Hour.Milliseconds(), blocks[0].Meta().MaxTime) // Block maxt is exclusive. + + require.Equal(t, 24*time.Hour.Milliseconds()+1, blocks[1].Meta().MinTime) + require.Equal(t, 26*time.Hour.Milliseconds(), blocks[1].Meta().MaxTime) + + require.Equal(t, 50*time.Hour.Milliseconds()+1, blocks[2].Meta().MaxTime) // Block maxt is exclusive. + }, + }, + } { + t.Run(name, func(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1 + cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Minute // Long enough to not be reached during the test. + + if tc.setupIngester != nil { + tc.setupIngester(&cfg) + } + + // Create ingester + reg := prometheus.NewPedanticRegistry() + i, err := prepareIngesterWithBlocksStorage(t, cfg, reg) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + t.Cleanup(func() { + _ = services.StopAndAwaitTerminated(context.Background(), i) + }) + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // mock user's shipper + tc.action(t, i, reg) + }) + } +} + +func TestIngester_ForFlush(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1 + cfg.BlocksStorageConfig.TSDB.ShipInterval = 10 * time.Minute // Long enough to not be reached during the test. + + // Create ingester + reg := prometheus.NewPedanticRegistry() + i, err := prepareIngesterWithBlocksStorage(t, cfg, reg) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + t.Cleanup(func() { + _ = services.StopAndAwaitTerminated(context.Background(), i) + }) + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push some data. + pushSingleSampleWithMetadata(t, i) + + // Stop ingester. + require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) + + // Nothing shipped yet. + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 0 + `), "cortex_ingester_shipper_uploads_total")) + + // Restart ingester in "For Flusher" mode. We reuse the same config (esp. same dir) + reg = prometheus.NewPedanticRegistry() + i, err = NewForFlusher(i.cfg, i.limits, reg, log.NewNopLogger()) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + + // Our single sample should be reloaded from WAL + verifyCompactedHead(t, i, false) + i.Flush() + + // Head should be empty after flushing. + verifyCompactedHead(t, i, true) + + // Verify that block has been shipped. + require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` + # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks + # TYPE cortex_ingester_shipper_uploads_total counter + cortex_ingester_shipper_uploads_total 1 + `), "cortex_ingester_shipper_uploads_total")) + + require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) +} + +func mockUserShipper(t *testing.T, i *Ingester) *shipperMock { + m := &shipperMock{} + userDB, err := i.getOrCreateTSDB(userID, false) + require.NoError(t, err) + require.NotNil(t, userDB) + + userDB.shipper = m + return m +} + +func Test_Ingester_UserStats(t *testing.T) { + series := []struct { + lbls labels.Labels + value float64 + timestamp int64 + }{ + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "200"}, {Name: "route", Value: "get_user"}}, 1, 100000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "500"}, {Name: "route", Value: "get_user"}}, 1, 110000}, + {labels.Labels{{Name: labels.MetricName, Value: "test_2"}}, 2, 200000}, + } + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push series + ctx := user.InjectOrgID(context.Background(), "test") + + for _, series := range series { + req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + + // force update statistics + for _, db := range i.TSDBState.dbs { + db.ingestedAPISamples.Tick() + db.ingestedRuleSamples.Tick() + } + + // Get label names + res, err := i.UserStats(ctx, &client.UserStatsRequest{}) + require.NoError(t, err) + assert.InDelta(t, 0.2, res.ApiIngestionRate, 0.0001) + assert.InDelta(t, float64(0), res.RuleIngestionRate, 0.0001) + assert.Equal(t, uint64(3), res.NumSeries) +} + +func Test_Ingester_AllUserStats(t *testing.T) { + series := []struct { + user string + lbls labels.Labels + value float64 + timestamp int64 + }{ + {"user-1", labels.Labels{{Name: labels.MetricName, Value: "test_1_1"}, {Name: "status", Value: "200"}, {Name: "route", Value: "get_user"}}, 1, 100000}, + {"user-1", labels.Labels{{Name: labels.MetricName, Value: "test_1_1"}, {Name: "status", Value: "500"}, {Name: "route", Value: "get_user"}}, 1, 110000}, + {"user-1", labels.Labels{{Name: labels.MetricName, Value: "test_1_2"}}, 2, 200000}, + {"user-2", labels.Labels{{Name: labels.MetricName, Value: "test_2_1"}}, 2, 200000}, + {"user-2", labels.Labels{{Name: labels.MetricName, Value: "test_2_2"}}, 2, 200000}, + } + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + for _, series := range series { + ctx := user.InjectOrgID(context.Background(), series.user) + req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + + // force update statistics + for _, db := range i.TSDBState.dbs { + db.ingestedAPISamples.Tick() + db.ingestedRuleSamples.Tick() + } + + // Get label names + res, err := i.AllUserStats(context.Background(), &client.UserStatsRequest{}) + require.NoError(t, err) + + expect := []*client.UserIDStatsResponse{ + { + UserId: "user-1", + Data: &client.UserStatsResponse{ + IngestionRate: 0.2, + NumSeries: 3, + ApiIngestionRate: 0.2, + RuleIngestionRate: 0, + }, + }, + { + UserId: "user-2", + Data: &client.UserStatsResponse{ + IngestionRate: 0.13333333333333333, + NumSeries: 2, + ApiIngestionRate: 0.13333333333333333, + RuleIngestionRate: 0, + }, + }, + } + assert.ElementsMatch(t, expect, res.Stats) +} + +func TestIngesterCompactIdleBlock(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1 + cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Hour // Long enough to not be reached during the test. + cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout = 1 * time.Second // Testing this. + + r := prometheus.NewRegistry() + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, cfg, r) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + t.Cleanup(func() { + _ = services.StopAndAwaitTerminated(context.Background(), i) + }) + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + pushSingleSampleWithMetadata(t, i) + + i.compactBlocks(context.Background(), false, nil) + verifyCompactedHead(t, i, false) + require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="1"} 1 + + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="1"} 0 + + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + `), memSeriesCreatedTotalName, memSeriesRemovedTotalName, "cortex_ingester_memory_users")) + + // wait one second (plus maximum jitter) -- TSDB is now idle. + time.Sleep(time.Duration(float64(cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout) * (1 + compactionIdleTimeoutJitter))) + + i.compactBlocks(context.Background(), false, nil) + verifyCompactedHead(t, i, true) + require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="1"} 1 + + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="1"} 1 + + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + `), memSeriesCreatedTotalName, memSeriesRemovedTotalName, "cortex_ingester_memory_users")) + + // Pushing another sample still works. + pushSingleSampleWithMetadata(t, i) + verifyCompactedHead(t, i, false) + + require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="1"} 2 + + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="1"} 1 + + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + `), memSeriesCreatedTotalName, memSeriesRemovedTotalName, "cortex_ingester_memory_users")) +} + +func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Second // Required to enable shipping. + cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1 + cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Second + cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout = 1 * time.Second + cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 1 * time.Second + cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval = 100 * time.Millisecond + + r := prometheus.NewRegistry() + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, cfg, r) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + t.Cleanup(func() { + require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) + }) + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + pushSingleSampleWithMetadata(t, i) + i.updateActiveSeries() + + require.Equal(t, int64(1), i.TSDBState.seriesCount.Load()) + + metricsToCheck := []string{memSeriesCreatedTotalName, memSeriesRemovedTotalName, "cortex_ingester_memory_users", "cortex_ingester_active_series", + "cortex_ingester_memory_metadata", "cortex_ingester_memory_metadata_created_total", "cortex_ingester_memory_metadata_removed_total"} + + require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="1"} 1 + + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="1"} 0 + + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="1"} 1 + + # HELP cortex_ingester_memory_metadata The current number of metadata in memory. + # TYPE cortex_ingester_memory_metadata gauge + cortex_ingester_memory_metadata 1 + + # HELP cortex_ingester_memory_metadata_created_total The total number of metadata that were created per user + # TYPE cortex_ingester_memory_metadata_created_total counter + cortex_ingester_memory_metadata_created_total{user="1"} 1 + `), metricsToCheck...)) + + // Wait until TSDB has been closed and removed. + test.Poll(t, 10*time.Second, 0, func() interface{} { + i.stoppedMtx.Lock() + defer i.stoppedMtx.Unlock() + return len(i.TSDBState.dbs) + }) + + require.Greater(t, testutil.ToFloat64(i.TSDBState.idleTsdbChecks.WithLabelValues(string(tsdbIdleClosed))), float64(0)) + i.updateActiveSeries() + require.Equal(t, int64(0), i.TSDBState.seriesCount.Load()) // Flushing removed all series from memory. + + // Verify that user has disappeared from metrics. + require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 0 + + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + + # HELP cortex_ingester_memory_metadata The current number of metadata in memory. + # TYPE cortex_ingester_memory_metadata gauge + cortex_ingester_memory_metadata 0 + `), metricsToCheck...)) + + // Pushing another sample will recreate TSDB. + pushSingleSampleWithMetadata(t, i) + i.updateActiveSeries() + + // User is back. + require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` + # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. + # TYPE cortex_ingester_memory_series_created_total counter + cortex_ingester_memory_series_created_total{user="1"} 1 + + # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. + # TYPE cortex_ingester_memory_series_removed_total counter + cortex_ingester_memory_series_removed_total{user="1"} 0 + + # HELP cortex_ingester_memory_users The current number of users in memory. + # TYPE cortex_ingester_memory_users gauge + cortex_ingester_memory_users 1 + + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="1"} 1 + + # HELP cortex_ingester_memory_metadata The current number of metadata in memory. + # TYPE cortex_ingester_memory_metadata gauge + cortex_ingester_memory_metadata 1 + + # HELP cortex_ingester_memory_metadata_created_total The total number of metadata that were created per user + # TYPE cortex_ingester_memory_metadata_created_total counter + cortex_ingester_memory_metadata_created_total{user="1"} 1 + `), metricsToCheck...)) +} + +func verifyCompactedHead(t *testing.T, i *Ingester, expected bool) { + db := i.getTSDB(userID) + require.NotNil(t, db) + + h := db.Head() + require.Equal(t, expected, h.NumSeries() == 0) +} + +func pushSingleSampleWithMetadata(t *testing.T, i *Ingester) { + ctx := user.InjectOrgID(context.Background(), userID) + req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, util.TimeToMillis(time.Now())) + req.Metadata = append(req.Metadata, &cortexpb.MetricMetadata{MetricFamilyName: "test", Help: "a help for metric", Unit: "", Type: cortexpb.COUNTER}) + _, err := i.Push(ctx, req) + require.NoError(t, err) +} + +func pushSingleSampleAtTime(t *testing.T, i *Ingester, ts int64) { + ctx := user.InjectOrgID(context.Background(), userID) + req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, ts) + _, err := i.Push(ctx, req) + require.NoError(t, err) +} + +func TestHeadCompactionOnStartup(t *testing.T) { + // Create a temporary directory for TSDB + tempDir := t.TempDir() + + // Build TSDB for user, with data covering 24 hours. + { + // Number of full chunks, 12 chunks for 24hrs. + numFullChunks := 12 + chunkRange := 2 * time.Hour.Milliseconds() + + userDir := filepath.Join(tempDir, userID) + require.NoError(t, os.Mkdir(userDir, 0700)) + + db, err := tsdb.Open(userDir, nil, nil, &tsdb.Options{ + RetentionDuration: int64(time.Hour * 25 / time.Millisecond), + NoLockfile: true, + MinBlockDuration: chunkRange, + MaxBlockDuration: chunkRange, + }, nil) + require.NoError(t, err) + + db.DisableCompactions() + head := db.Head() + + l := labels.Labels{{Name: "n", Value: "v"}} + for i := 0; i < numFullChunks; i++ { + // Not using db.Appender() as it checks for compaction. + app := head.Appender(context.Background()) + _, err := app.Append(0, l, int64(i)*chunkRange+1, 9.99) + require.NoError(t, err) + _, err = app.Append(0, l, int64(i+1)*chunkRange, 9.99) + require.NoError(t, err) + require.NoError(t, app.Commit()) + } + + dur := time.Duration(head.MaxTime()-head.MinTime()) * time.Millisecond + require.True(t, dur > 23*time.Hour) + require.Equal(t, 0, len(db.Blocks())) + require.NoError(t, db.Close()) + } + + limits := defaultLimitsTestConfig() + + overrides, err := validation.NewOverrides(limits, nil) + require.NoError(t, err) + + ingesterCfg := defaultIngesterTestConfig(t) + ingesterCfg.BlocksStorageConfig.TSDB.Dir = tempDir + ingesterCfg.BlocksStorageConfig.Bucket.Backend = "s3" + ingesterCfg.BlocksStorageConfig.Bucket.S3.Endpoint = "localhost" + ingesterCfg.BlocksStorageConfig.TSDB.Retention = 2 * 24 * time.Hour // Make sure that no newly created blocks are deleted. + + ingester, err := New(ingesterCfg, overrides, nil, log.NewNopLogger()) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ingester)) + + defer services.StopAndAwaitTerminated(context.Background(), ingester) //nolint:errcheck + + db := ingester.getTSDB(userID) + require.NotNil(t, db) + + h := db.Head() + + dur := time.Duration(h.MaxTime()-h.MinTime()) * time.Millisecond + require.True(t, dur <= 2*time.Hour) + require.Equal(t, 11, len(db.Blocks())) +} + +func TestIngester_CloseTSDBsOnShutdown(t *testing.T) { + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + t.Cleanup(func() { + _ = services.StopAndAwaitTerminated(context.Background(), i) + }) + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push some data. + pushSingleSampleWithMetadata(t, i) + + db := i.getTSDB(userID) + require.NotNil(t, db) + + // Stop ingester. + require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) + + // Verify that DB is no longer in memory, but was closed + db = i.getTSDB(userID) + require.Nil(t, db) +} + +func TestIngesterNotDeleteUnshippedBlocks(t *testing.T) { + chunkRange := 2 * time.Hour + chunkRangeMilliSec := chunkRange.Milliseconds() + cfg := defaultIngesterTestConfig(t) + cfg.BlocksStorageConfig.TSDB.BlockRanges = []time.Duration{chunkRange} + cfg.BlocksStorageConfig.TSDB.Retention = time.Millisecond // Which means delete all but first block. + cfg.LifecyclerConfig.JoinAfter = 0 + + // Create ingester + reg := prometheus.NewPedanticRegistry() + i, err := prepareIngesterWithBlocksStorage(t, cfg, reg) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + t.Cleanup(func() { + _ = services.StopAndAwaitTerminated(context.Background(), i) + }) + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_ingester_oldest_unshipped_block_timestamp_seconds Unix timestamp of the oldest TSDB block not shipped to the storage yet. 0 if ingester has no blocks or all blocks have been shipped. + # TYPE cortex_ingester_oldest_unshipped_block_timestamp_seconds gauge + cortex_ingester_oldest_unshipped_block_timestamp_seconds 0 + `), "cortex_ingester_oldest_unshipped_block_timestamp_seconds")) + + // Push some data to create 3 blocks. + ctx := user.InjectOrgID(context.Background(), userID) + for j := int64(0); j < 5; j++ { + req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, j*chunkRangeMilliSec) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + + db := i.getTSDB(userID) + require.NotNil(t, db) + require.Nil(t, db.Compact()) + + oldBlocks := db.Blocks() + require.Equal(t, 3, len(oldBlocks)) + + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(fmt.Sprintf(` + # HELP cortex_ingester_oldest_unshipped_block_timestamp_seconds Unix timestamp of the oldest TSDB block not shipped to the storage yet. 0 if ingester has no blocks or all blocks have been shipped. + # TYPE cortex_ingester_oldest_unshipped_block_timestamp_seconds gauge + cortex_ingester_oldest_unshipped_block_timestamp_seconds %d + `, oldBlocks[0].Meta().ULID.Time()/1000)), "cortex_ingester_oldest_unshipped_block_timestamp_seconds")) + + // Saying that we have shipped the second block, so only that should get deleted. + require.Nil(t, shipper.WriteMetaFile(nil, db.db.Dir(), &shipper.Meta{ + Version: shipper.MetaVersion1, + Uploaded: []ulid.ULID{oldBlocks[1].Meta().ULID}, + })) + require.NoError(t, db.updateCachedShippedBlocks()) + + // Add more samples that could trigger another compaction and hence reload of blocks. + for j := int64(5); j < 6; j++ { + req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, j*chunkRangeMilliSec) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + require.Nil(t, db.Compact()) + + // Only the second block should be gone along with a new block. + newBlocks := db.Blocks() + require.Equal(t, 3, len(newBlocks)) + require.Equal(t, oldBlocks[0].Meta().ULID, newBlocks[0].Meta().ULID) // First block remains same. + require.Equal(t, oldBlocks[2].Meta().ULID, newBlocks[1].Meta().ULID) // 3rd block becomes 2nd now. + require.NotEqual(t, oldBlocks[1].Meta().ULID, newBlocks[2].Meta().ULID) // The new block won't match previous 2nd block. + + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(fmt.Sprintf(` + # HELP cortex_ingester_oldest_unshipped_block_timestamp_seconds Unix timestamp of the oldest TSDB block not shipped to the storage yet. 0 if ingester has no blocks or all blocks have been shipped. + # TYPE cortex_ingester_oldest_unshipped_block_timestamp_seconds gauge + cortex_ingester_oldest_unshipped_block_timestamp_seconds %d + `, newBlocks[0].Meta().ULID.Time()/1000)), "cortex_ingester_oldest_unshipped_block_timestamp_seconds")) + + // Shipping 2 more blocks, hence all the blocks from first round. + require.Nil(t, shipper.WriteMetaFile(nil, db.db.Dir(), &shipper.Meta{ + Version: shipper.MetaVersion1, + Uploaded: []ulid.ULID{oldBlocks[1].Meta().ULID, newBlocks[0].Meta().ULID, newBlocks[1].Meta().ULID}, + })) + require.NoError(t, db.updateCachedShippedBlocks()) + + // Add more samples that could trigger another compaction and hence reload of blocks. + for j := int64(6); j < 7; j++ { + req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, j*chunkRangeMilliSec) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + require.Nil(t, db.Compact()) + + // All blocks from the old blocks should be gone now. + newBlocks2 := db.Blocks() + require.Equal(t, 2, len(newBlocks2)) + + require.Equal(t, newBlocks[2].Meta().ULID, newBlocks2[0].Meta().ULID) // Block created in last round. + for _, b := range oldBlocks { + // Second block is not one among old blocks. + require.NotEqual(t, b.Meta().ULID, newBlocks2[1].Meta().ULID) + } + + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(fmt.Sprintf(` + # HELP cortex_ingester_oldest_unshipped_block_timestamp_seconds Unix timestamp of the oldest TSDB block not shipped to the storage yet. 0 if ingester has no blocks or all blocks have been shipped. + # TYPE cortex_ingester_oldest_unshipped_block_timestamp_seconds gauge + cortex_ingester_oldest_unshipped_block_timestamp_seconds %d + `, newBlocks2[0].Meta().ULID.Time()/1000)), "cortex_ingester_oldest_unshipped_block_timestamp_seconds")) +} + +func TestIngesterPushErrorDuringForcedCompaction(t *testing.T) { + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + t.Cleanup(func() { + _ = services.StopAndAwaitTerminated(context.Background(), i) + }) + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push a sample, it should succeed. + pushSingleSampleWithMetadata(t, i) + + // We mock a flushing by setting the boolean. + db := i.getTSDB(userID) + require.NotNil(t, db) + require.True(t, db.casState(active, forceCompacting)) + + // Ingestion should fail with a 503. + req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, util.TimeToMillis(time.Now())) + ctx := user.InjectOrgID(context.Background(), userID) + _, err = i.Push(ctx, req) + require.Equal(t, httpgrpc.Errorf(http.StatusServiceUnavailable, wrapWithUser(errors.New("forced compaction in progress"), userID).Error()), err) + + // Ingestion is successful after a flush. + require.True(t, db.casState(forceCompacting, active)) + pushSingleSampleWithMetadata(t, i) +} + +func TestIngesterNoFlushWithInFlightRequest(t *testing.T) { + registry := prometheus.NewRegistry() + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), registry) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + t.Cleanup(func() { + _ = services.StopAndAwaitTerminated(context.Background(), i) + }) + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Push few samples. + for j := 0; j < 5; j++ { + pushSingleSampleWithMetadata(t, i) + } + + // Verifying that compaction won't happen when a request is in flight. + + // This mocks a request in flight. + db := i.getTSDB(userID) + require.NoError(t, db.acquireAppendLock()) + + // Flush handler only triggers compactions, but doesn't wait for them to finish. We cannot use ?wait=true here, + // because it would deadlock -- flush will wait for appendLock to be released. + i.FlushHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/flush", nil)) + + // Flushing should not have succeeded even after 5 seconds. + time.Sleep(5 * time.Second) + require.NoError(t, testutil.GatherAndCompare(registry, strings.NewReader(` + # HELP cortex_ingester_tsdb_compactions_total Total number of TSDB compactions that were executed. + # TYPE cortex_ingester_tsdb_compactions_total counter + cortex_ingester_tsdb_compactions_total 0 + `), "cortex_ingester_tsdb_compactions_total")) + + // No requests in flight after this. + db.releaseAppendLock() + + // Let's wait until all head series have been flushed. + test.Poll(t, 5*time.Second, uint64(0), func() interface{} { + db := i.getTSDB(userID) + if db == nil { + return false + } + return db.Head().NumSeries() + }) + + require.NoError(t, testutil.GatherAndCompare(registry, strings.NewReader(` + # HELP cortex_ingester_tsdb_compactions_total Total number of TSDB compactions that were executed. + # TYPE cortex_ingester_tsdb_compactions_total counter + cortex_ingester_tsdb_compactions_total 1 + `), "cortex_ingester_tsdb_compactions_total")) +} + +func TestIngester_PushInstanceLimits(t *testing.T) { + tests := map[string]struct { + limits InstanceLimits + reqs map[string][]*cortexpb.WriteRequest + expectedErr error + expectedErrType interface{} + }{ + "should succeed creating one user and series": { + limits: InstanceLimits{MaxInMemorySeries: 1, MaxInMemoryTenants: 1}, + reqs: map[string][]*cortexpb.WriteRequest{ + "test": { + cortexpb.ToWriteRequest( + []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}})}, + []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, + []*cortexpb.MetricMetadata{ + {MetricFamilyName: "metric_name_1", Help: "a help for metric_name_1", Unit: "", Type: cortexpb.COUNTER}, + }, + cortexpb.API), + }, + }, + expectedErr: nil, + }, + + "should fail creating two series": { + limits: InstanceLimits{MaxInMemorySeries: 1, MaxInMemoryTenants: 1}, + + reqs: map[string][]*cortexpb.WriteRequest{ + "test": { + cortexpb.ToWriteRequest( + []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, + []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, + nil, + cortexpb.API), + + cortexpb.ToWriteRequest( + []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test2"}})}, // another series + []cortexpb.Sample{{Value: 1, TimestampMs: 10}}, + nil, + cortexpb.API), + }, + }, + + expectedErr: wrapWithUser(errMaxSeriesLimitReached, "test"), + }, + + "should fail creating two users": { + limits: InstanceLimits{MaxInMemorySeries: 1, MaxInMemoryTenants: 1}, + + reqs: map[string][]*cortexpb.WriteRequest{ + "user1": { + cortexpb.ToWriteRequest( + []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, + []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, + nil, + cortexpb.API), + }, + + "user2": { + cortexpb.ToWriteRequest( + []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test2"}})}, // another series + []cortexpb.Sample{{Value: 1, TimestampMs: 10}}, + nil, + cortexpb.API), + }, + }, + expectedErr: wrapWithUser(errMaxUsersLimitReached, "user2"), + }, + + "should fail pushing samples in two requests due to rate limit": { + limits: InstanceLimits{MaxInMemorySeries: 1, MaxInMemoryTenants: 1, MaxIngestionRate: 0.001}, + + reqs: map[string][]*cortexpb.WriteRequest{ + "user1": { + cortexpb.ToWriteRequest( + []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, + []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, + nil, + cortexpb.API), + + cortexpb.ToWriteRequest( + []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, + []cortexpb.Sample{{Value: 1, TimestampMs: 10}}, + nil, + cortexpb.API), + }, + }, + expectedErr: errMaxSamplesPushRateLimitReached, + }, + } + + defaultInstanceLimits = nil + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + // Create a mocked ingester + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.InstanceLimitsFn = func() *InstanceLimits { + return &testData.limits + } + + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until the ingester is ACTIVE + test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + // Iterate through users in sorted order (by username). + uids := []string{} + totalPushes := 0 + for uid, requests := range testData.reqs { + uids = append(uids, uid) + totalPushes += len(requests) + } + sort.Strings(uids) + + pushIdx := 0 + for _, uid := range uids { + ctx := user.InjectOrgID(context.Background(), uid) + + for _, req := range testData.reqs[uid] { + pushIdx++ + _, err := i.Push(ctx, req) + + if pushIdx < totalPushes { + require.NoError(t, err) + } else { + // Last push may expect error. + if testData.expectedErr != nil { + assert.Equal(t, testData.expectedErr, err) + } else if testData.expectedErrType != nil { + assert.True(t, errors.As(err, testData.expectedErrType), "expected error type %T, got %v", testData.expectedErrType, err) + } else { + assert.NoError(t, err) + } + } + + // imitate time ticking between each push + i.ingestionRate.Tick() + + rate := testutil.ToFloat64(i.metrics.ingestionRate) + require.NotZero(t, rate) + } + } + }) + } +} + +func TestIngester_instanceLimitsMetrics(t *testing.T) { + reg := prometheus.NewRegistry() + + l := InstanceLimits{ + MaxIngestionRate: 10, + MaxInMemoryTenants: 20, + MaxInMemorySeries: 30, + } + + cfg := defaultIngesterTestConfig(t) + cfg.InstanceLimitsFn = func() *InstanceLimits { + return &l + } + cfg.LifecyclerConfig.JoinAfter = 0 + + _, err := prepareIngesterWithBlocksStorage(t, cfg, reg) + require.NoError(t, err) + + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_ingester_instance_limits Instance limits used by this ingester. + # TYPE cortex_ingester_instance_limits gauge + cortex_ingester_instance_limits{limit="max_inflight_push_requests"} 0 + cortex_ingester_instance_limits{limit="max_ingestion_rate"} 10 + cortex_ingester_instance_limits{limit="max_series"} 30 + cortex_ingester_instance_limits{limit="max_tenants"} 20 + `), "cortex_ingester_instance_limits")) + + l.MaxInMemoryTenants = 1000 + l.MaxInMemorySeries = 2000 + + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_ingester_instance_limits Instance limits used by this ingester. + # TYPE cortex_ingester_instance_limits gauge + cortex_ingester_instance_limits{limit="max_inflight_push_requests"} 0 + cortex_ingester_instance_limits{limit="max_ingestion_rate"} 10 + cortex_ingester_instance_limits{limit="max_series"} 2000 + cortex_ingester_instance_limits{limit="max_tenants"} 1000 + `), "cortex_ingester_instance_limits")) +} + +func TestIngester_inflightPushRequests(t *testing.T) { + limits := InstanceLimits{MaxInflightPushRequests: 1} + + // Create a mocked ingester + cfg := defaultIngesterTestConfig(t) + cfg.InstanceLimitsFn = func() *InstanceLimits { return &limits } + cfg.LifecyclerConfig.JoinAfter = 0 + + i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until the ingester is ACTIVE + test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + + ctx := user.InjectOrgID(context.Background(), "test") + + startCh := make(chan struct{}) + + g, ctx := errgroup.WithContext(ctx) + g.Go(func() error { + count := 100000 + target := time.Second + + // find right count to make sure that push takes given target duration. + for { + req := generateSamplesForLabel(labels.FromStrings(labels.MetricName, fmt.Sprintf("test-%d", count)), count) + + start := time.Now() + _, err := i.Push(ctx, req) + require.NoError(t, err) + + elapsed := time.Since(start) + t.Log(count, elapsed) + if elapsed > time.Second { + break + } + + count = int(float64(count) * float64(target/elapsed) * 1.5) // Adjust number of samples to hit our target push duration. + } + + // Now repeat push with number of samples calibrated to our target. + req := generateSamplesForLabel(labels.FromStrings(labels.MetricName, fmt.Sprintf("real-%d", count)), count) + + // Signal that we're going to do the real push now. + close(startCh) + + _, err := i.Push(ctx, req) + return err + }) + + g.Go(func() error { + select { + case <-ctx.Done(): + // failed to setup + case <-startCh: + // we can start the test. + } + + time.Sleep(10 * time.Millisecond) // Give first goroutine a chance to start pushing... + req := generateSamplesForLabel(labels.FromStrings(labels.MetricName, "testcase"), 1024) + + _, err := i.Push(ctx, req) + require.Equal(t, errTooManyInflightPushRequests, err) + return nil + }) + + require.NoError(t, g.Wait()) +} + +func generateSamplesForLabel(l labels.Labels, count int) *cortexpb.WriteRequest { + var lbls = make([]labels.Labels, 0, count) + var samples = make([]cortexpb.Sample, 0, count) + + for i := 0; i < count; i++ { + samples = append(samples, cortexpb.Sample{ + Value: float64(i), + TimestampMs: int64(i), + }) + lbls = append(lbls, l) + } + + return cortexpb.ToWriteRequest(lbls, samples, nil, cortexpb.API) +} diff --git a/pkg/ingester/ingester_v2.go b/pkg/ingester/ingester_v2.go deleted file mode 100644 index a2e08803a17..00000000000 --- a/pkg/ingester/ingester_v2.go +++ /dev/null @@ -1,2360 +0,0 @@ -package ingester - -import ( - "context" - "fmt" - "io" - "math" - "net/http" - "os" - "path/filepath" - "sync" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/oklog/ulid" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/exemplar" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/thanos-io/thanos/pkg/block/metadata" - "github.com/thanos-io/thanos/pkg/objstore" - "github.com/thanos-io/thanos/pkg/shipper" - "github.com/weaveworks/common/httpgrpc" - "go.uber.org/atomic" - "golang.org/x/sync/errgroup" - - "github.com/cortexproject/cortex/pkg/chunk/encoding" - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/ingester/client" - "github.com/cortexproject/cortex/pkg/ring" - "github.com/cortexproject/cortex/pkg/storage/bucket" - cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" - "github.com/cortexproject/cortex/pkg/tenant" - "github.com/cortexproject/cortex/pkg/util" - "github.com/cortexproject/cortex/pkg/util/concurrency" - "github.com/cortexproject/cortex/pkg/util/extract" - logutil "github.com/cortexproject/cortex/pkg/util/log" - util_math "github.com/cortexproject/cortex/pkg/util/math" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/pkg/util/spanlogger" - "github.com/cortexproject/cortex/pkg/util/validation" -) - -const ( - // RingKey is the key under which we store the ingesters ring in the KVStore. - RingKey = "ring" -) - -const ( - errTSDBCreateIncompatibleState = "cannot create a new TSDB while the ingester is not in active state (current state: %s)" - errTSDBIngest = "err: %v. timestamp=%s, series=%s" // Using error.Wrap puts the message before the error and if the series is too long, its truncated. - errTSDBIngestExemplar = "err: %v. timestamp=%s, series=%s, exemplar=%s" - - // Jitter applied to the idle timeout to prevent compaction in all ingesters concurrently. - compactionIdleTimeoutJitter = 0.25 - - instanceIngestionRateTickInterval = time.Second -) - -var ( - errExemplarRef = errors.New("exemplars not ingested because series not already present") -) - -// Shipper interface is used to have an easy way to mock it in tests. -type Shipper interface { - Sync(ctx context.Context) (uploaded int, err error) -} - -type tsdbState int - -const ( - active tsdbState = iota // Pushes are allowed. - activeShipping // Pushes are allowed. Blocks shipping is in progress. - forceCompacting // TSDB is being force-compacted. - closing // Used while closing idle TSDB. - closed // Used to avoid setting closing back to active in closeAndDeleteIdleUsers method. -) - -// Describes result of TSDB-close check. String is used as metric label. -type tsdbCloseCheckResult string - -const ( - tsdbIdle tsdbCloseCheckResult = "idle" // Not reported via metrics. Metrics use tsdbIdleClosed on success. - tsdbShippingDisabled tsdbCloseCheckResult = "shipping_disabled" - tsdbNotIdle tsdbCloseCheckResult = "not_idle" - tsdbNotCompacted tsdbCloseCheckResult = "not_compacted" - tsdbNotShipped tsdbCloseCheckResult = "not_shipped" - tsdbCheckFailed tsdbCloseCheckResult = "check_failed" - tsdbCloseFailed tsdbCloseCheckResult = "close_failed" - tsdbNotActive tsdbCloseCheckResult = "not_active" - tsdbDataRemovalFailed tsdbCloseCheckResult = "data_removal_failed" - tsdbTenantMarkedForDeletion tsdbCloseCheckResult = "tenant_marked_for_deletion" - tsdbIdleClosed tsdbCloseCheckResult = "idle_closed" // Success. -) - -func (r tsdbCloseCheckResult) shouldClose() bool { - return r == tsdbIdle || r == tsdbTenantMarkedForDeletion -} - -// QueryStreamType defines type of function to use when doing query-stream operation. -type QueryStreamType int - -const ( - QueryStreamDefault QueryStreamType = iota // Use default configured value. - QueryStreamSamples // Stream individual samples. - QueryStreamChunks // Stream entire chunks. -) - -type userTSDB struct { - db *tsdb.DB - userID string - activeSeries *ActiveSeries - seriesInMetric *metricCounter - limiter *Limiter - - instanceSeriesCount *atomic.Int64 // Shared across all userTSDB instances created by ingester. - instanceLimitsFn func() *InstanceLimits - - stateMtx sync.RWMutex - state tsdbState - pushesInFlight sync.WaitGroup // Increased with stateMtx read lock held, only if state == active or activeShipping. - - // Used to detect idle TSDBs. - lastUpdate atomic.Int64 - - // Thanos shipper used to ship blocks to the storage. - shipper Shipper - - // When deletion marker is found for the tenant (checked before shipping), - // shipping stops and TSDB is closed before reaching idle timeout time (if enabled). - deletionMarkFound atomic.Bool - - // Unix timestamp of last deletion mark check. - lastDeletionMarkCheck atomic.Int64 - - // for statistics - ingestedAPISamples *util_math.EwmaRate - ingestedRuleSamples *util_math.EwmaRate - - // Cached shipped blocks. - shippedBlocksMtx sync.Mutex - shippedBlocks map[ulid.ULID]struct{} -} - -// Explicitly wrapping the tsdb.DB functions that we use. - -func (u *userTSDB) Appender(ctx context.Context) storage.Appender { - return u.db.Appender(ctx) -} - -func (u *userTSDB) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { - return u.db.Querier(ctx, mint, maxt) -} - -func (u *userTSDB) ChunkQuerier(ctx context.Context, mint, maxt int64) (storage.ChunkQuerier, error) { - return u.db.ChunkQuerier(ctx, mint, maxt) -} - -func (u *userTSDB) ExemplarQuerier(ctx context.Context) (storage.ExemplarQuerier, error) { - return u.db.ExemplarQuerier(ctx) -} - -func (u *userTSDB) Head() *tsdb.Head { - return u.db.Head() -} - -func (u *userTSDB) Blocks() []*tsdb.Block { - return u.db.Blocks() -} - -func (u *userTSDB) Close() error { - return u.db.Close() -} - -func (u *userTSDB) Compact() error { - return u.db.Compact() -} - -func (u *userTSDB) StartTime() (int64, error) { - return u.db.StartTime() -} - -func (u *userTSDB) casState(from, to tsdbState) bool { - u.stateMtx.Lock() - defer u.stateMtx.Unlock() - - if u.state != from { - return false - } - u.state = to - return true -} - -// compactHead compacts the Head block at specified block durations avoiding a single huge block. -func (u *userTSDB) compactHead(blockDuration int64) error { - if !u.casState(active, forceCompacting) { - return errors.New("TSDB head cannot be compacted because it is not in active state (possibly being closed or blocks shipping in progress)") - } - - defer u.casState(forceCompacting, active) - - // Ingestion of samples in parallel with forced compaction can lead to overlapping blocks, - // and possible invalidation of the references returned from Appender.GetRef(). - // So we wait for existing in-flight requests to finish. Future push requests would fail until compaction is over. - u.pushesInFlight.Wait() - - h := u.Head() - - minTime, maxTime := h.MinTime(), h.MaxTime() - - for (minTime/blockDuration)*blockDuration != (maxTime/blockDuration)*blockDuration { - // Data in Head spans across multiple block ranges, so we break it into blocks here. - // Block max time is exclusive, so we do a -1 here. - blockMaxTime := ((minTime/blockDuration)+1)*blockDuration - 1 - if err := u.db.CompactHead(tsdb.NewRangeHead(h, minTime, blockMaxTime)); err != nil { - return err - } - - // Get current min/max times after compaction. - minTime, maxTime = h.MinTime(), h.MaxTime() - } - - return u.db.CompactHead(tsdb.NewRangeHead(h, minTime, maxTime)) -} - -// PreCreation implements SeriesLifecycleCallback interface. -func (u *userTSDB) PreCreation(metric labels.Labels) error { - if u.limiter == nil { - return nil - } - - // Verify ingester's global limit - gl := u.instanceLimitsFn() - if gl != nil && gl.MaxInMemorySeries > 0 { - if series := u.instanceSeriesCount.Load(); series >= gl.MaxInMemorySeries { - return errMaxSeriesLimitReached - } - } - - // Total series limit. - if err := u.limiter.AssertMaxSeriesPerUser(u.userID, int(u.Head().NumSeries())); err != nil { - return err - } - - // Series per metric name limit. - metricName, err := extract.MetricNameFromLabels(metric) - if err != nil { - return err - } - if err := u.seriesInMetric.canAddSeriesFor(u.userID, metricName); err != nil { - return err - } - - return nil -} - -// PostCreation implements SeriesLifecycleCallback interface. -func (u *userTSDB) PostCreation(metric labels.Labels) { - u.instanceSeriesCount.Inc() - - metricName, err := extract.MetricNameFromLabels(metric) - if err != nil { - // This should never happen because it has already been checked in PreCreation(). - return - } - u.seriesInMetric.increaseSeriesForMetric(metricName) -} - -// PostDeletion implements SeriesLifecycleCallback interface. -func (u *userTSDB) PostDeletion(metrics ...labels.Labels) { - u.instanceSeriesCount.Sub(int64(len(metrics))) - - for _, metric := range metrics { - metricName, err := extract.MetricNameFromLabels(metric) - if err != nil { - // This should never happen because it has already been checked in PreCreation(). - continue - } - u.seriesInMetric.decreaseSeriesForMetric(metricName) - } -} - -// blocksToDelete filters the input blocks and returns the blocks which are safe to be deleted from the ingester. -func (u *userTSDB) blocksToDelete(blocks []*tsdb.Block) map[ulid.ULID]struct{} { - if u.db == nil { - return nil - } - deletable := tsdb.DefaultBlocksToDelete(u.db)(blocks) - if u.shipper == nil { - return deletable - } - - shippedBlocks := u.getCachedShippedBlocks() - - result := map[ulid.ULID]struct{}{} - for shippedID := range shippedBlocks { - if _, ok := deletable[shippedID]; ok { - result[shippedID] = struct{}{} - } - } - return result -} - -// updateCachedShipperBlocks reads the shipper meta file and updates the cached shipped blocks. -func (u *userTSDB) updateCachedShippedBlocks() error { - shipperMeta, err := shipper.ReadMetaFile(u.db.Dir()) - if os.IsNotExist(err) || os.IsNotExist(errors.Cause(err)) { - // If the meta file doesn't exist it means the shipper hasn't run yet. - shipperMeta = &shipper.Meta{} - } else if err != nil { - return err - } - - // Build a map. - shippedBlocks := make(map[ulid.ULID]struct{}, len(shipperMeta.Uploaded)) - for _, blockID := range shipperMeta.Uploaded { - shippedBlocks[blockID] = struct{}{} - } - - // Cache it. - u.shippedBlocksMtx.Lock() - u.shippedBlocks = shippedBlocks - u.shippedBlocksMtx.Unlock() - - return nil -} - -// getCachedShippedBlocks returns the cached shipped blocks. -func (u *userTSDB) getCachedShippedBlocks() map[ulid.ULID]struct{} { - u.shippedBlocksMtx.Lock() - defer u.shippedBlocksMtx.Unlock() - - // It's safe to directly return the map because it's never updated in-place. - return u.shippedBlocks -} - -// getOldestUnshippedBlockTime returns the unix timestamp with milliseconds precision of the oldest -// TSDB block not shipped to the storage yet, or 0 if all blocks have been shipped. -func (u *userTSDB) getOldestUnshippedBlockTime() uint64 { - shippedBlocks := u.getCachedShippedBlocks() - oldestTs := uint64(0) - - for _, b := range u.Blocks() { - if _, ok := shippedBlocks[b.Meta().ULID]; ok { - continue - } - - if oldestTs == 0 || b.Meta().ULID.Time() < oldestTs { - oldestTs = b.Meta().ULID.Time() - } - } - - return oldestTs -} - -func (u *userTSDB) isIdle(now time.Time, idle time.Duration) bool { - lu := u.lastUpdate.Load() - - return time.Unix(lu, 0).Add(idle).Before(now) -} - -func (u *userTSDB) setLastUpdate(t time.Time) { - u.lastUpdate.Store(t.Unix()) -} - -// Checks if TSDB can be closed. -func (u *userTSDB) shouldCloseTSDB(idleTimeout time.Duration) tsdbCloseCheckResult { - if u.deletionMarkFound.Load() { - return tsdbTenantMarkedForDeletion - } - - if !u.isIdle(time.Now(), idleTimeout) { - return tsdbNotIdle - } - - // If head is not compacted, we cannot close this yet. - if u.Head().NumSeries() > 0 { - return tsdbNotCompacted - } - - // Ensure that all blocks have been shipped. - if oldest := u.getOldestUnshippedBlockTime(); oldest > 0 { - return tsdbNotShipped - } - - return tsdbIdle -} - -// TSDBState holds data structures used by the TSDB storage engine -type TSDBState struct { - dbs map[string]*userTSDB // tsdb sharded by userID - bucket objstore.Bucket - - // Value used by shipper as external label. - shipperIngesterID string - - subservices *services.Manager - - tsdbMetrics *tsdbMetrics - - forceCompactTrigger chan requestWithUsersAndCallback - shipTrigger chan requestWithUsersAndCallback - - // Timeout chosen for idle compactions. - compactionIdleTimeout time.Duration - - // Number of series in memory, across all tenants. - seriesCount atomic.Int64 - - // Head compactions metrics. - compactionsTriggered prometheus.Counter - compactionsFailed prometheus.Counter - walReplayTime prometheus.Histogram - appenderAddDuration prometheus.Histogram - appenderCommitDuration prometheus.Histogram - idleTsdbChecks *prometheus.CounterVec -} - -type requestWithUsersAndCallback struct { - users *util.AllowedTenants // if nil, all tenants are allowed. - callback chan<- struct{} // when compaction/shipping is finished, this channel is closed -} - -func newTSDBState(bucketClient objstore.Bucket, registerer prometheus.Registerer) TSDBState { - idleTsdbChecks := promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ - Name: "cortex_ingester_idle_tsdb_checks_total", - Help: "The total number of various results for idle TSDB checks.", - }, []string{"result"}) - - idleTsdbChecks.WithLabelValues(string(tsdbShippingDisabled)) - idleTsdbChecks.WithLabelValues(string(tsdbNotIdle)) - idleTsdbChecks.WithLabelValues(string(tsdbNotCompacted)) - idleTsdbChecks.WithLabelValues(string(tsdbNotShipped)) - idleTsdbChecks.WithLabelValues(string(tsdbCheckFailed)) - idleTsdbChecks.WithLabelValues(string(tsdbCloseFailed)) - idleTsdbChecks.WithLabelValues(string(tsdbNotActive)) - idleTsdbChecks.WithLabelValues(string(tsdbDataRemovalFailed)) - idleTsdbChecks.WithLabelValues(string(tsdbTenantMarkedForDeletion)) - idleTsdbChecks.WithLabelValues(string(tsdbIdleClosed)) - - return TSDBState{ - dbs: make(map[string]*userTSDB), - bucket: bucketClient, - tsdbMetrics: newTSDBMetrics(registerer), - forceCompactTrigger: make(chan requestWithUsersAndCallback), - shipTrigger: make(chan requestWithUsersAndCallback), - - compactionsTriggered: promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_tsdb_compactions_triggered_total", - Help: "Total number of triggered compactions.", - }), - - compactionsFailed: promauto.With(registerer).NewCounter(prometheus.CounterOpts{ - Name: "cortex_ingester_tsdb_compactions_failed_total", - Help: "Total number of compactions that failed.", - }), - walReplayTime: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ - Name: "cortex_ingester_tsdb_wal_replay_duration_seconds", - Help: "The total time it takes to open and replay a TSDB WAL.", - Buckets: prometheus.DefBuckets, - }), - appenderAddDuration: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ - Name: "cortex_ingester_tsdb_appender_add_duration_seconds", - Help: "The total time it takes for a push request to add samples to the TSDB appender.", - Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}, - }), - appenderCommitDuration: promauto.With(registerer).NewHistogram(prometheus.HistogramOpts{ - Name: "cortex_ingester_tsdb_appender_commit_duration_seconds", - Help: "The total time it takes for a push request to commit samples appended to TSDB.", - Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}, - }), - - idleTsdbChecks: idleTsdbChecks, - } -} - -// NewV2 returns a new Ingester that uses Cortex block storage instead of chunks storage. -func NewV2(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { - bucketClient, err := bucket.NewClient(context.Background(), cfg.BlocksStorageConfig.Bucket, "ingester", logger, registerer) - if err != nil { - return nil, errors.Wrap(err, "failed to create the bucket client") - } - - i := &Ingester{ - cfg: cfg, - limits: limits, - usersMetadata: map[string]*userMetricsMetadata{}, - TSDBState: newTSDBState(bucketClient, registerer), - logger: logger, - ingestionRate: util_math.NewEWMARate(0.2, instanceIngestionRateTickInterval), - } - i.metrics = newIngesterMetrics(registerer, false, cfg.ActiveSeriesMetricsEnabled, i.getInstanceLimits, i.ingestionRate, &i.inflightPushRequests) - - // Replace specific metrics which we can't directly track but we need to read - // them from the underlying system (ie. TSDB). - if registerer != nil { - registerer.Unregister(i.metrics.memSeries) - - promauto.With(registerer).NewGaugeFunc(prometheus.GaugeOpts{ - Name: "cortex_ingester_memory_series", - Help: "The current number of series in memory.", - }, i.getMemorySeriesMetric) - - promauto.With(registerer).NewGaugeFunc(prometheus.GaugeOpts{ - Name: "cortex_ingester_oldest_unshipped_block_timestamp_seconds", - Help: "Unix timestamp of the oldest TSDB block not shipped to the storage yet. 0 if ingester has no blocks or all blocks have been shipped.", - }, i.getOldestUnshippedBlockMetric) - } - - i.lifecycler, err = ring.NewLifecycler(cfg.LifecyclerConfig, i, "ingester", RingKey, cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown, logger, prometheus.WrapRegistererWithPrefix("cortex_", registerer)) - if err != nil { - return nil, err - } - i.subservicesWatcher = services.NewFailureWatcher() - i.subservicesWatcher.WatchService(i.lifecycler) - - // Init the limter and instantiate the user states which depend on it - i.limiter = NewLimiter( - limits, - i.lifecycler, - cfg.DistributorShardingStrategy, - cfg.DistributorShardByAllLabels, - cfg.LifecyclerConfig.RingConfig.ReplicationFactor, - cfg.LifecyclerConfig.RingConfig.ZoneAwarenessEnabled) - - i.TSDBState.shipperIngesterID = i.lifecycler.ID - - // Apply positive jitter only to ensure that the minimum timeout is adhered to. - i.TSDBState.compactionIdleTimeout = util.DurationWithPositiveJitter(i.cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout, compactionIdleTimeoutJitter) - level.Info(i.logger).Log("msg", "TSDB idle compaction timeout set", "timeout", i.TSDBState.compactionIdleTimeout) - - i.BasicService = services.NewBasicService(i.startingV2, i.updateLoop, i.stoppingV2) - return i, nil -} - -// NewV2ForFlusher is a special version of ingester used by Flusher. This ingester is not ingesting anything, its only purpose is to react -// on Flush method and flush all openened TSDBs when called. -func NewV2ForFlusher(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { - bucketClient, err := bucket.NewClient(context.Background(), cfg.BlocksStorageConfig.Bucket, "ingester", logger, registerer) - if err != nil { - return nil, errors.Wrap(err, "failed to create the bucket client") - } - - i := &Ingester{ - cfg: cfg, - limits: limits, - TSDBState: newTSDBState(bucketClient, registerer), - logger: logger, - } - i.metrics = newIngesterMetrics(registerer, false, false, i.getInstanceLimits, nil, &i.inflightPushRequests) - - i.TSDBState.shipperIngesterID = "flusher" - - // This ingester will not start any subservices (lifecycler, compaction, shipping), - // and will only open TSDBs, wait for Flush to be called, and then close TSDBs again. - i.BasicService = services.NewIdleService(i.startingV2ForFlusher, i.stoppingV2ForFlusher) - return i, nil -} - -func (i *Ingester) startingV2ForFlusher(ctx context.Context) error { - if err := i.openExistingTSDB(ctx); err != nil { - // Try to rollback and close opened TSDBs before halting the ingester. - i.closeAllTSDB() - - return errors.Wrap(err, "opening existing TSDBs") - } - - // Don't start any sub-services (lifecycler, compaction, shipper) at all. - return nil -} - -func (i *Ingester) startingV2(ctx context.Context) error { - if err := i.openExistingTSDB(ctx); err != nil { - // Try to rollback and close opened TSDBs before halting the ingester. - i.closeAllTSDB() - - return errors.Wrap(err, "opening existing TSDBs") - } - - // Important: we want to keep lifecycler running until we ask it to stop, so we need to give it independent context - if err := i.lifecycler.StartAsync(context.Background()); err != nil { - return errors.Wrap(err, "failed to start lifecycler") - } - if err := i.lifecycler.AwaitRunning(ctx); err != nil { - return errors.Wrap(err, "failed to start lifecycler") - } - - // let's start the rest of subservices via manager - servs := []services.Service(nil) - - compactionService := services.NewBasicService(nil, i.compactionLoop, nil) - servs = append(servs, compactionService) - - if i.cfg.BlocksStorageConfig.TSDB.IsBlocksShippingEnabled() { - shippingService := services.NewBasicService(nil, i.shipBlocksLoop, nil) - servs = append(servs, shippingService) - } - - if i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout > 0 { - interval := i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval - if interval == 0 { - interval = cortex_tsdb.DefaultCloseIdleTSDBInterval - } - closeIdleService := services.NewTimerService(interval, nil, i.closeAndDeleteIdleUserTSDBs, nil) - servs = append(servs, closeIdleService) - } - - var err error - i.TSDBState.subservices, err = services.NewManager(servs...) - if err == nil { - err = services.StartManagerAndAwaitHealthy(ctx, i.TSDBState.subservices) - } - return errors.Wrap(err, "failed to start ingester components") -} - -func (i *Ingester) stoppingV2ForFlusher(_ error) error { - if !i.cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown { - i.closeAllTSDB() - } - return nil -} - -// runs when V2 ingester is stopping -func (i *Ingester) stoppingV2(_ error) error { - // It's important to wait until shipper is finished, - // because the blocks transfer should start only once it's guaranteed - // there's no shipping on-going. - - if err := services.StopManagerAndAwaitStopped(context.Background(), i.TSDBState.subservices); err != nil { - level.Warn(i.logger).Log("msg", "failed to stop ingester subservices", "err", err) - } - - // Next initiate our graceful exit from the ring. - if err := services.StopAndAwaitTerminated(context.Background(), i.lifecycler); err != nil { - level.Warn(i.logger).Log("msg", "failed to stop ingester lifecycler", "err", err) - } - - if !i.cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown { - i.closeAllTSDB() - } - return nil -} - -func (i *Ingester) updateLoop(ctx context.Context) error { - if limits := i.getInstanceLimits(); limits != nil && *limits != (InstanceLimits{}) { - // This check will not cover enabling instance limits in runtime, but it will do for now. - logutil.WarnExperimentalUse("ingester instance limits") - } - - rateUpdateTicker := time.NewTicker(i.cfg.RateUpdatePeriod) - defer rateUpdateTicker.Stop() - - ingestionRateTicker := time.NewTicker(instanceIngestionRateTickInterval) - defer ingestionRateTicker.Stop() - - var activeSeriesTickerChan <-chan time.Time - if i.cfg.ActiveSeriesMetricsEnabled { - t := time.NewTicker(i.cfg.ActiveSeriesMetricsUpdatePeriod) - activeSeriesTickerChan = t.C - defer t.Stop() - } - - // Similarly to the above, this is a hardcoded value. - metadataPurgeTicker := time.NewTicker(metadataPurgePeriod) - defer metadataPurgeTicker.Stop() - - for { - select { - case <-metadataPurgeTicker.C: - i.purgeUserMetricsMetadata() - case <-ingestionRateTicker.C: - i.ingestionRate.Tick() - case <-rateUpdateTicker.C: - i.stoppedMtx.RLock() - for _, db := range i.TSDBState.dbs { - db.ingestedAPISamples.Tick() - db.ingestedRuleSamples.Tick() - } - i.stoppedMtx.RUnlock() - - case <-activeSeriesTickerChan: - i.v2UpdateActiveSeries() - - case <-ctx.Done(): - return nil - case err := <-i.subservicesWatcher.Chan(): - return errors.Wrap(err, "ingester subservice failed") - } - } -} - -func (i *Ingester) v2UpdateActiveSeries() { - purgeTime := time.Now().Add(-i.cfg.ActiveSeriesMetricsIdleTimeout) - - for _, userID := range i.getTSDBUsers() { - userDB := i.getTSDB(userID) - if userDB == nil { - continue - } - - userDB.activeSeries.Purge(purgeTime) - i.metrics.activeSeriesPerUser.WithLabelValues(userID).Set(float64(userDB.activeSeries.Active())) - } -} - -// GetRef() is an extra method added to TSDB to let Cortex check before calling Add() -type extendedAppender interface { - storage.Appender - storage.GetRef -} - -// v2Push adds metrics to a block -func (i *Ingester) v2Push(ctx context.Context, req *cortexpb.WriteRequest) (*cortexpb.WriteResponse, error) { - var firstPartialErr error - - // NOTE: because we use `unsafe` in deserialisation, we must not - // retain anything from `req` past the call to ReuseSlice - defer cortexpb.ReuseSlice(req.Timeseries) - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - il := i.getInstanceLimits() - if il != nil && il.MaxIngestionRate > 0 { - if rate := i.ingestionRate.Rate(); rate >= il.MaxIngestionRate { - return nil, errMaxSamplesPushRateLimitReached - } - } - - db, err := i.getOrCreateTSDB(userID, false) - if err != nil { - return nil, wrapWithUser(err, userID) - } - - // Ensure the ingester shutdown procedure hasn't started - i.stoppedMtx.RLock() - if i.stopped { - i.stoppedMtx.RUnlock() - return nil, errIngesterStopping - } - i.stoppedMtx.RUnlock() - - if err := db.acquireAppendLock(); err != nil { - return &cortexpb.WriteResponse{}, httpgrpc.Errorf(http.StatusServiceUnavailable, wrapWithUser(err, userID).Error()) - } - defer db.releaseAppendLock() - - // Given metadata is a best-effort approach, and we don't halt on errors - // process it before samples. Otherwise, we risk returning an error before ingestion. - ingestedMetadata := i.pushMetadata(ctx, userID, req.GetMetadata()) - - // Keep track of some stats which are tracked only if the samples will be - // successfully committed - var ( - succeededSamplesCount = 0 - failedSamplesCount = 0 - succeededExemplarsCount = 0 - failedExemplarsCount = 0 - startAppend = time.Now() - sampleOutOfBoundsCount = 0 - sampleOutOfOrderCount = 0 - newValueForTimestampCount = 0 - perUserSeriesLimitCount = 0 - perMetricSeriesLimitCount = 0 - - updateFirstPartial = func(errFn func() error) { - if firstPartialErr == nil { - firstPartialErr = errFn() - } - } - ) - - // Walk the samples, appending them to the users database - app := db.Appender(ctx).(extendedAppender) - for _, ts := range req.Timeseries { - // The labels must be sorted (in our case, it's guaranteed a write request - // has sorted labels once hit the ingester). - - // Look up a reference for this series. - ref, copiedLabels := app.GetRef(cortexpb.FromLabelAdaptersToLabels(ts.Labels)) - - // To find out if any sample was added to this series, we keep old value. - oldSucceededSamplesCount := succeededSamplesCount - - for _, s := range ts.Samples { - var err error - - // If the cached reference exists, we try to use it. - if ref != 0 { - if _, err = app.Append(ref, copiedLabels, s.TimestampMs, s.Value); err == nil { - succeededSamplesCount++ - continue - } - - } else { - // Copy the label set because both TSDB and the active series tracker may retain it. - copiedLabels = cortexpb.FromLabelAdaptersToLabelsWithCopy(ts.Labels) - - // Retain the reference in case there are multiple samples for the series. - if ref, err = app.Append(0, copiedLabels, s.TimestampMs, s.Value); err == nil { - succeededSamplesCount++ - continue - } - } - - failedSamplesCount++ - - // Check if the error is a soft error we can proceed on. If so, we keep track - // of it, so that we can return it back to the distributor, which will return a - // 400 error to the client. The client (Prometheus) will not retry on 400, and - // we actually ingested all samples which haven't failed. - switch cause := errors.Cause(err); cause { - case storage.ErrOutOfBounds: - sampleOutOfBoundsCount++ - updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(s.TimestampMs), ts.Labels) }) - continue - - case storage.ErrOutOfOrderSample: - sampleOutOfOrderCount++ - updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(s.TimestampMs), ts.Labels) }) - continue - - case storage.ErrDuplicateSampleForTimestamp: - newValueForTimestampCount++ - updateFirstPartial(func() error { return wrappedTSDBIngestErr(err, model.Time(s.TimestampMs), ts.Labels) }) - continue - - case errMaxSeriesPerUserLimitExceeded: - perUserSeriesLimitCount++ - updateFirstPartial(func() error { return makeLimitError(perUserSeriesLimit, i.limiter.FormatError(userID, cause)) }) - continue - - case errMaxSeriesPerMetricLimitExceeded: - perMetricSeriesLimitCount++ - updateFirstPartial(func() error { - return makeMetricLimitError(perMetricSeriesLimit, copiedLabels, i.limiter.FormatError(userID, cause)) - }) - continue - } - - // The error looks an issue on our side, so we should rollback - if rollbackErr := app.Rollback(); rollbackErr != nil { - level.Warn(i.logger).Log("msg", "failed to rollback on error", "user", userID, "err", rollbackErr) - } - - return nil, wrapWithUser(err, userID) - } - - if i.cfg.ActiveSeriesMetricsEnabled && succeededSamplesCount > oldSucceededSamplesCount { - db.activeSeries.UpdateSeries(cortexpb.FromLabelAdaptersToLabels(ts.Labels), startAppend, func(l labels.Labels) labels.Labels { - // we must already have copied the labels if succeededSamplesCount has been incremented. - return copiedLabels - }) - } - - if i.cfg.BlocksStorageConfig.TSDB.MaxExemplars > 0 { - // app.AppendExemplar currently doesn't create the series, it must - // already exist. If it does not then drop. - if ref == 0 && len(ts.Exemplars) > 0 { - updateFirstPartial(func() error { - return wrappedTSDBIngestExemplarErr(errExemplarRef, - model.Time(ts.Exemplars[0].TimestampMs), ts.Labels, ts.Exemplars[0].Labels) - }) - failedExemplarsCount += len(ts.Exemplars) - } else { // Note that else is explicit, rather than a continue in the above if, in case of additional logic post exemplar processing. - for _, ex := range ts.Exemplars { - e := exemplar.Exemplar{ - Value: ex.Value, - Ts: ex.TimestampMs, - HasTs: true, - Labels: cortexpb.FromLabelAdaptersToLabelsWithCopy(ex.Labels), - } - - if _, err = app.AppendExemplar(ref, nil, e); err == nil { - succeededExemplarsCount++ - continue - } - - // Error adding exemplar - updateFirstPartial(func() error { - return wrappedTSDBIngestExemplarErr(err, model.Time(ex.TimestampMs), ts.Labels, ex.Labels) - }) - failedExemplarsCount++ - } - } - } - } - - // At this point all samples have been added to the appender, so we can track the time it took. - i.TSDBState.appenderAddDuration.Observe(time.Since(startAppend).Seconds()) - - startCommit := time.Now() - if err := app.Commit(); err != nil { - return nil, wrapWithUser(err, userID) - } - i.TSDBState.appenderCommitDuration.Observe(time.Since(startCommit).Seconds()) - - // If only invalid samples are pushed, don't change "last update", as TSDB was not modified. - if succeededSamplesCount > 0 { - db.setLastUpdate(time.Now()) - } - - // Increment metrics only if the samples have been successfully committed. - // If the code didn't reach this point, it means that we returned an error - // which will be converted into an HTTP 5xx and the client should/will retry. - i.metrics.ingestedSamples.Add(float64(succeededSamplesCount)) - i.metrics.ingestedSamplesFail.Add(float64(failedSamplesCount)) - i.metrics.ingestedExemplars.Add(float64(succeededExemplarsCount)) - i.metrics.ingestedExemplarsFail.Add(float64(failedExemplarsCount)) - - if sampleOutOfBoundsCount > 0 { - validation.DiscardedSamples.WithLabelValues(sampleOutOfBounds, userID).Add(float64(sampleOutOfBoundsCount)) - } - if sampleOutOfOrderCount > 0 { - validation.DiscardedSamples.WithLabelValues(sampleOutOfOrder, userID).Add(float64(sampleOutOfOrderCount)) - } - if newValueForTimestampCount > 0 { - validation.DiscardedSamples.WithLabelValues(newValueForTimestamp, userID).Add(float64(newValueForTimestampCount)) - } - if perUserSeriesLimitCount > 0 { - validation.DiscardedSamples.WithLabelValues(perUserSeriesLimit, userID).Add(float64(perUserSeriesLimitCount)) - } - if perMetricSeriesLimitCount > 0 { - validation.DiscardedSamples.WithLabelValues(perMetricSeriesLimit, userID).Add(float64(perMetricSeriesLimitCount)) - } - - // Distributor counts both samples and metadata, so for consistency ingester does the same. - i.ingestionRate.Add(int64(succeededSamplesCount + ingestedMetadata)) - - switch req.Source { - case cortexpb.RULE: - db.ingestedRuleSamples.Add(int64(succeededSamplesCount)) - case cortexpb.API: - fallthrough - default: - db.ingestedAPISamples.Add(int64(succeededSamplesCount)) - } - - if firstPartialErr != nil { - code := http.StatusBadRequest - var ve *validationError - if errors.As(firstPartialErr, &ve) { - code = ve.code - } - return &cortexpb.WriteResponse{}, httpgrpc.Errorf(code, wrapWithUser(firstPartialErr, userID).Error()) - } - - return &cortexpb.WriteResponse{}, nil -} - -func (u *userTSDB) acquireAppendLock() error { - u.stateMtx.RLock() - defer u.stateMtx.RUnlock() - - switch u.state { - case active: - case activeShipping: - // Pushes are allowed. - case forceCompacting: - return errors.New("forced compaction in progress") - case closing: - return errors.New("TSDB is closing") - default: - return errors.New("TSDB is not active") - } - - u.pushesInFlight.Add(1) - return nil -} - -func (u *userTSDB) releaseAppendLock() { - u.pushesInFlight.Done() -} - -func (i *Ingester) v2Query(ctx context.Context, req *client.QueryRequest) (*client.QueryResponse, error) { - if err := i.checkRunning(); err != nil { - return nil, err - } - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - from, through, matchers, err := client.FromQueryRequest(req) - if err != nil { - return nil, err - } - - i.metrics.queries.Inc() - - db := i.getTSDB(userID) - if db == nil { - return &client.QueryResponse{}, nil - } - - q, err := db.Querier(ctx, int64(from), int64(through)) - if err != nil { - return nil, err - } - defer q.Close() - - // It's not required to return sorted series because series are sorted by the Cortex querier. - ss := q.Select(false, nil, matchers...) - if ss.Err() != nil { - return nil, ss.Err() - } - - numSamples := 0 - - result := &client.QueryResponse{} - for ss.Next() { - series := ss.At() - - ts := cortexpb.TimeSeries{ - Labels: cortexpb.FromLabelsToLabelAdapters(series.Labels()), - } - - it := series.Iterator() - for it.Next() { - t, v := it.At() - ts.Samples = append(ts.Samples, cortexpb.Sample{Value: v, TimestampMs: t}) - } - - numSamples += len(ts.Samples) - result.Timeseries = append(result.Timeseries, ts) - } - - i.metrics.queriedSeries.Observe(float64(len(result.Timeseries))) - i.metrics.queriedSamples.Observe(float64(numSamples)) - - return result, ss.Err() -} - -func (i *Ingester) v2QueryExemplars(ctx context.Context, req *client.ExemplarQueryRequest) (*client.ExemplarQueryResponse, error) { - if err := i.checkRunning(); err != nil { - return nil, err - } - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - from, through, matchers, err := client.FromExemplarQueryRequest(req) - if err != nil { - return nil, err - } - - i.metrics.queries.Inc() - - db := i.getTSDB(userID) - if db == nil { - return &client.ExemplarQueryResponse{}, nil - } - - q, err := db.ExemplarQuerier(ctx) - if err != nil { - return nil, err - } - - // It's not required to sort series from a single ingester because series are sorted by the Exemplar Storage before returning from Select. - res, err := q.Select(from, through, matchers...) - if err != nil { - return nil, err - } - - numExemplars := 0 - - result := &client.ExemplarQueryResponse{} - for _, es := range res { - ts := cortexpb.TimeSeries{ - Labels: cortexpb.FromLabelsToLabelAdapters(es.SeriesLabels), - Exemplars: cortexpb.FromExemplarsToExemplarProtos(es.Exemplars), - } - - numExemplars += len(ts.Exemplars) - result.Timeseries = append(result.Timeseries, ts) - } - - i.metrics.queriedExemplars.Observe(float64(numExemplars)) - - return result, nil -} - -func (i *Ingester) v2LabelValues(ctx context.Context, req *client.LabelValuesRequest) (*client.LabelValuesResponse, error) { - if err := i.checkRunning(); err != nil { - return nil, err - } - - labelName, startTimestampMs, endTimestampMs, matchers, err := client.FromLabelValuesRequest(req) - if err != nil { - return nil, err - } - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - db := i.getTSDB(userID) - if db == nil { - return &client.LabelValuesResponse{}, nil - } - - mint, maxt, err := metadataQueryRange(startTimestampMs, endTimestampMs, db) - if err != nil { - return nil, err - } - - q, err := db.Querier(ctx, mint, maxt) - if err != nil { - return nil, err - } - defer q.Close() - - vals, _, err := q.LabelValues(labelName, matchers...) - if err != nil { - return nil, err - } - - return &client.LabelValuesResponse{ - LabelValues: vals, - }, nil -} - -func (i *Ingester) v2LabelValuesStream(req *client.LabelValuesRequest, stream client.Ingester_LabelValuesStreamServer) error { - resp, err := i.v2LabelValues(stream.Context(), req) - - if err != nil { - return err - } - - for i := 0; i < len(resp.LabelValues); i += metadataStreamBatchSize { - j := i + metadataStreamBatchSize - if j > len(resp.LabelValues) { - j = len(resp.LabelValues) - } - resp := &client.LabelValuesStreamResponse{ - LabelValues: resp.LabelValues[i:j], - } - err := client.SendLabelValuesStream(stream, resp) - if err != nil { - return err - } - } - - return nil -} - -func (i *Ingester) v2LabelNames(ctx context.Context, req *client.LabelNamesRequest) (*client.LabelNamesResponse, error) { - if err := i.checkRunning(); err != nil { - return nil, err - } - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - db := i.getTSDB(userID) - if db == nil { - return &client.LabelNamesResponse{}, nil - } - - mint, maxt, err := metadataQueryRange(req.StartTimestampMs, req.EndTimestampMs, db) - if err != nil { - return nil, err - } - - q, err := db.Querier(ctx, mint, maxt) - if err != nil { - return nil, err - } - defer q.Close() - - names, _, err := q.LabelNames() - if err != nil { - return nil, err - } - - return &client.LabelNamesResponse{ - LabelNames: names, - }, nil -} - -func (i *Ingester) v2LabelNamesStream(req *client.LabelNamesRequest, stream client.Ingester_LabelNamesStreamServer) error { - resp, err := i.v2LabelNames(stream.Context(), req) - - if err != nil { - return err - } - - for i := 0; i < len(resp.LabelNames); i += metadataStreamBatchSize { - j := i + metadataStreamBatchSize - if j > len(resp.LabelNames) { - j = len(resp.LabelNames) - } - resp := &client.LabelNamesStreamResponse{ - LabelNames: resp.LabelNames[i:j], - } - err := client.SendLabelNamesStream(stream, resp) - if err != nil { - return err - } - } - - return nil -} - -func (i *Ingester) v2MetricsForLabelMatchers(ctx context.Context, req *client.MetricsForLabelMatchersRequest) (*client.MetricsForLabelMatchersResponse, error) { - if err := i.checkRunning(); err != nil { - return nil, err - } - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - db := i.getTSDB(userID) - if db == nil { - return &client.MetricsForLabelMatchersResponse{}, nil - } - - // Parse the request - _, _, matchersSet, err := client.FromMetricsForLabelMatchersRequest(req) - if err != nil { - return nil, err - } - - mint, maxt, err := metadataQueryRange(req.StartTimestampMs, req.EndTimestampMs, db) - if err != nil { - return nil, err - } - - q, err := db.Querier(ctx, mint, maxt) - if err != nil { - return nil, err - } - defer q.Close() - - // Run a query for each matchers set and collect all the results. - var sets []storage.SeriesSet - - for _, matchers := range matchersSet { - // Interrupt if the context has been canceled. - if ctx.Err() != nil { - return nil, ctx.Err() - } - - hints := &storage.SelectHints{ - Start: mint, - End: maxt, - Func: "series", // There is no series function, this token is used for lookups that don't need samples. - } - - seriesSet := q.Select(true, hints, matchers...) - sets = append(sets, seriesSet) - } - - // Generate the response merging all series sets. - result := &client.MetricsForLabelMatchersResponse{ - Metric: make([]*cortexpb.Metric, 0), - } - - mergedSet := storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge) - for mergedSet.Next() { - // Interrupt if the context has been canceled. - if ctx.Err() != nil { - return nil, ctx.Err() - } - - result.Metric = append(result.Metric, &cortexpb.Metric{ - Labels: cortexpb.FromLabelsToLabelAdapters(mergedSet.At().Labels()), - }) - } - - return result, nil -} - -func (i *Ingester) v2MetricsForLabelMatchersStream(req *client.MetricsForLabelMatchersRequest, stream client.Ingester_MetricsForLabelMatchersStreamServer) error { - result, err := i.v2MetricsForLabelMatchers(stream.Context(), req) - if err != nil { - return err - } - - for i := 0; i < len(result.Metric); i += metadataStreamBatchSize { - j := i + metadataStreamBatchSize - if j > len(result.Metric) { - j = len(result.Metric) - } - resp := &client.MetricsForLabelMatchersStreamResponse{ - Metric: result.Metric[i:j], - } - err := client.SendMetricsForLabelMatchersStream(stream, resp) - if err != nil { - return err - } - } - - return nil -} - -func (i *Ingester) v2UserStats(ctx context.Context, req *client.UserStatsRequest) (*client.UserStatsResponse, error) { - if err := i.checkRunning(); err != nil { - return nil, err - } - - userID, err := tenant.TenantID(ctx) - if err != nil { - return nil, err - } - - db := i.getTSDB(userID) - if db == nil { - return &client.UserStatsResponse{}, nil - } - - return createUserStats(db), nil -} - -func (i *Ingester) v2AllUserStats(ctx context.Context, req *client.UserStatsRequest) (*client.UsersStatsResponse, error) { - if err := i.checkRunning(); err != nil { - return nil, err - } - - i.stoppedMtx.RLock() - defer i.stoppedMtx.RUnlock() - - users := i.TSDBState.dbs - - response := &client.UsersStatsResponse{ - Stats: make([]*client.UserIDStatsResponse, 0, len(users)), - } - for userID, db := range users { - response.Stats = append(response.Stats, &client.UserIDStatsResponse{ - UserId: userID, - Data: createUserStats(db), - }) - } - return response, nil -} - -func createUserStats(db *userTSDB) *client.UserStatsResponse { - apiRate := db.ingestedAPISamples.Rate() - ruleRate := db.ingestedRuleSamples.Rate() - return &client.UserStatsResponse{ - IngestionRate: apiRate + ruleRate, - ApiIngestionRate: apiRate, - RuleIngestionRate: ruleRate, - NumSeries: db.Head().NumSeries(), - } -} - -const queryStreamBatchMessageSize = 1 * 1024 * 1024 - -// v2QueryStream streams metrics from a TSDB. This implements the client.IngesterServer interface -func (i *Ingester) v2QueryStream(req *client.QueryRequest, stream client.Ingester_QueryStreamServer) error { - if err := i.checkRunning(); err != nil { - return err - } - - spanlog, ctx := spanlogger.New(stream.Context(), "v2QueryStream") - defer spanlog.Finish() - - userID, err := tenant.TenantID(ctx) - if err != nil { - return err - } - - from, through, matchers, err := client.FromQueryRequest(req) - if err != nil { - return err - } - - i.metrics.queries.Inc() - - db := i.getTSDB(userID) - if db == nil { - return nil - } - - numSamples := 0 - numSeries := 0 - - streamType := QueryStreamSamples - if i.cfg.StreamChunksWhenUsingBlocks { - streamType = QueryStreamChunks - } - - if i.cfg.StreamTypeFn != nil { - runtimeType := i.cfg.StreamTypeFn() - switch runtimeType { - case QueryStreamChunks: - streamType = QueryStreamChunks - case QueryStreamSamples: - streamType = QueryStreamSamples - default: - // no change from config value. - } - } - - if streamType == QueryStreamChunks { - level.Debug(spanlog).Log("msg", "using v2QueryStreamChunks") - numSeries, numSamples, err = i.v2QueryStreamChunks(ctx, db, int64(from), int64(through), matchers, stream) - } else { - level.Debug(spanlog).Log("msg", "using v2QueryStreamSamples") - numSeries, numSamples, err = i.v2QueryStreamSamples(ctx, db, int64(from), int64(through), matchers, stream) - } - if err != nil { - return err - } - - i.metrics.queriedSeries.Observe(float64(numSeries)) - i.metrics.queriedSamples.Observe(float64(numSamples)) - level.Debug(spanlog).Log("series", numSeries, "samples", numSamples) - return nil -} - -func (i *Ingester) v2QueryStreamSamples(ctx context.Context, db *userTSDB, from, through int64, matchers []*labels.Matcher, stream client.Ingester_QueryStreamServer) (numSeries, numSamples int, _ error) { - q, err := db.Querier(ctx, from, through) - if err != nil { - return 0, 0, err - } - defer q.Close() - - // It's not required to return sorted series because series are sorted by the Cortex querier. - ss := q.Select(false, nil, matchers...) - if ss.Err() != nil { - return 0, 0, ss.Err() - } - - timeseries := make([]cortexpb.TimeSeries, 0, queryStreamBatchSize) - batchSizeBytes := 0 - for ss.Next() { - series := ss.At() - - // convert labels to LabelAdapter - ts := cortexpb.TimeSeries{ - Labels: cortexpb.FromLabelsToLabelAdapters(series.Labels()), - } - - it := series.Iterator() - for it.Next() { - t, v := it.At() - ts.Samples = append(ts.Samples, cortexpb.Sample{Value: v, TimestampMs: t}) - } - numSamples += len(ts.Samples) - numSeries++ - tsSize := ts.Size() - - if (batchSizeBytes > 0 && batchSizeBytes+tsSize > queryStreamBatchMessageSize) || len(timeseries) >= queryStreamBatchSize { - // Adding this series to the batch would make it too big, - // flush the data and add it to new batch instead. - err = client.SendQueryStream(stream, &client.QueryStreamResponse{ - Timeseries: timeseries, - }) - if err != nil { - return 0, 0, err - } - - batchSizeBytes = 0 - timeseries = timeseries[:0] - } - - timeseries = append(timeseries, ts) - batchSizeBytes += tsSize - } - - // Ensure no error occurred while iterating the series set. - if err := ss.Err(); err != nil { - return 0, 0, err - } - - // Final flush any existing metrics - if batchSizeBytes != 0 { - err = client.SendQueryStream(stream, &client.QueryStreamResponse{ - Timeseries: timeseries, - }) - if err != nil { - return 0, 0, err - } - } - - return numSeries, numSamples, nil -} - -// v2QueryStream streams metrics from a TSDB. This implements the client.IngesterServer interface -func (i *Ingester) v2QueryStreamChunks(ctx context.Context, db *userTSDB, from, through int64, matchers []*labels.Matcher, stream client.Ingester_QueryStreamServer) (numSeries, numSamples int, _ error) { - q, err := db.ChunkQuerier(ctx, from, through) - if err != nil { - return 0, 0, err - } - defer q.Close() - - // It's not required to return sorted series because series are sorted by the Cortex querier. - ss := q.Select(false, nil, matchers...) - if ss.Err() != nil { - return 0, 0, ss.Err() - } - - chunkSeries := make([]client.TimeSeriesChunk, 0, queryStreamBatchSize) - batchSizeBytes := 0 - for ss.Next() { - series := ss.At() - - // convert labels to LabelAdapter - ts := client.TimeSeriesChunk{ - Labels: cortexpb.FromLabelsToLabelAdapters(series.Labels()), - } - - it := series.Iterator() - for it.Next() { - // Chunks are ordered by min time. - meta := it.At() - - // It is not guaranteed that chunk returned by iterator is populated. - // For now just return error. We could also try to figure out how to read the chunk. - if meta.Chunk == nil { - return 0, 0, errors.Errorf("unfilled chunk returned from TSDB chunk querier") - } - - ch := client.Chunk{ - StartTimestampMs: meta.MinTime, - EndTimestampMs: meta.MaxTime, - Data: meta.Chunk.Bytes(), - } - - switch meta.Chunk.Encoding() { - case chunkenc.EncXOR: - ch.Encoding = int32(encoding.PrometheusXorChunk) - default: - return 0, 0, errors.Errorf("unknown chunk encoding from TSDB chunk querier: %v", meta.Chunk.Encoding()) - } - - ts.Chunks = append(ts.Chunks, ch) - numSamples += meta.Chunk.NumSamples() - } - numSeries++ - tsSize := ts.Size() - - if (batchSizeBytes > 0 && batchSizeBytes+tsSize > queryStreamBatchMessageSize) || len(chunkSeries) >= queryStreamBatchSize { - // Adding this series to the batch would make it too big, - // flush the data and add it to new batch instead. - err = client.SendQueryStream(stream, &client.QueryStreamResponse{ - Chunkseries: chunkSeries, - }) - if err != nil { - return 0, 0, err - } - - batchSizeBytes = 0 - chunkSeries = chunkSeries[:0] - } - - chunkSeries = append(chunkSeries, ts) - batchSizeBytes += tsSize - } - - // Ensure no error occurred while iterating the series set. - if err := ss.Err(); err != nil { - return 0, 0, err - } - - // Final flush any existing metrics - if batchSizeBytes != 0 { - err = client.SendQueryStream(stream, &client.QueryStreamResponse{ - Chunkseries: chunkSeries, - }) - if err != nil { - return 0, 0, err - } - } - - return numSeries, numSamples, nil -} - -func (i *Ingester) getTSDB(userID string) *userTSDB { - i.stoppedMtx.RLock() - defer i.stoppedMtx.RUnlock() - db := i.TSDBState.dbs[userID] - return db -} - -// List all users for which we have a TSDB. We do it here in order -// to keep the mutex locked for the shortest time possible. -func (i *Ingester) getTSDBUsers() []string { - i.stoppedMtx.RLock() - defer i.stoppedMtx.RUnlock() - - ids := make([]string, 0, len(i.TSDBState.dbs)) - for userID := range i.TSDBState.dbs { - ids = append(ids, userID) - } - - return ids -} - -func (i *Ingester) getOrCreateTSDB(userID string, force bool) (*userTSDB, error) { - db := i.getTSDB(userID) - if db != nil { - return db, nil - } - - i.stoppedMtx.Lock() - defer i.stoppedMtx.Unlock() - - // Check again for DB in the event it was created in-between locks - var ok bool - db, ok = i.TSDBState.dbs[userID] - if ok { - return db, nil - } - - // We're ready to create the TSDB, however we must be sure that the ingester - // is in the ACTIVE state, otherwise it may conflict with the transfer in/out. - // The TSDB is created when the first series is pushed and this shouldn't happen - // to a non-ACTIVE ingester, however we want to protect from any bug, cause we - // may have data loss or TSDB WAL corruption if the TSDB is created before/during - // a transfer in occurs. - if ingesterState := i.lifecycler.GetState(); !force && ingesterState != ring.ACTIVE { - return nil, fmt.Errorf(errTSDBCreateIncompatibleState, ingesterState) - } - - gl := i.getInstanceLimits() - if gl != nil && gl.MaxInMemoryTenants > 0 { - if users := int64(len(i.TSDBState.dbs)); users >= gl.MaxInMemoryTenants { - return nil, errMaxUsersLimitReached - } - } - - // Create the database and a shipper for a user - db, err := i.createTSDB(userID) - if err != nil { - return nil, err - } - - // Add the db to list of user databases - i.TSDBState.dbs[userID] = db - i.metrics.memUsers.Inc() - - return db, nil -} - -// createTSDB creates a TSDB for a given userID, and returns the created db. -func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { - tsdbPromReg := prometheus.NewRegistry() - udir := i.cfg.BlocksStorageConfig.TSDB.BlocksDir(userID) - userLogger := logutil.WithUserID(userID, i.logger) - - blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() - - userDB := &userTSDB{ - userID: userID, - activeSeries: NewActiveSeries(), - seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), - ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), - ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), - - instanceLimitsFn: i.getInstanceLimits, - instanceSeriesCount: &i.TSDBState.seriesCount, - } - - enableExemplars := false - if i.cfg.BlocksStorageConfig.TSDB.MaxExemplars > 0 { - enableExemplars = true - } - // Create a new user database - db, err := tsdb.Open(udir, userLogger, tsdbPromReg, &tsdb.Options{ - RetentionDuration: i.cfg.BlocksStorageConfig.TSDB.Retention.Milliseconds(), - MinBlockDuration: blockRanges[0], - MaxBlockDuration: blockRanges[len(blockRanges)-1], - NoLockfile: true, - StripeSize: i.cfg.BlocksStorageConfig.TSDB.StripeSize, - HeadChunksWriteBufferSize: i.cfg.BlocksStorageConfig.TSDB.HeadChunksWriteBufferSize, - WALCompression: i.cfg.BlocksStorageConfig.TSDB.WALCompressionEnabled, - WALSegmentSize: i.cfg.BlocksStorageConfig.TSDB.WALSegmentSizeBytes, - SeriesLifecycleCallback: userDB, - BlocksToDelete: userDB.blocksToDelete, - EnableExemplarStorage: enableExemplars, - MaxExemplars: int64(i.cfg.BlocksStorageConfig.TSDB.MaxExemplars), - }, nil) - if err != nil { - return nil, errors.Wrapf(err, "failed to open TSDB: %s", udir) - } - db.DisableCompactions() // we will compact on our own schedule - - // Run compaction before using this TSDB. If there is data in head that needs to be put into blocks, - // this will actually create the blocks. If there is no data (empty TSDB), this is a no-op, although - // local blocks compaction may still take place if configured. - level.Info(userLogger).Log("msg", "Running compaction after WAL replay") - err = db.Compact() - if err != nil { - return nil, errors.Wrapf(err, "failed to compact TSDB: %s", udir) - } - - userDB.db = db - // We set the limiter here because we don't want to limit - // series during WAL replay. - userDB.limiter = i.limiter - - if db.Head().NumSeries() > 0 { - // If there are series in the head, use max time from head. If this time is too old, - // TSDB will be eligible for flushing and closing sooner, unless more data is pushed to it quickly. - userDB.setLastUpdate(util.TimeFromMillis(db.Head().MaxTime())) - } else { - // If head is empty (eg. new TSDB), don't close it right after. - userDB.setLastUpdate(time.Now()) - } - - // Thanos shipper requires at least 1 external label to be set. For this reason, - // we set the tenant ID as external label and we'll filter it out when reading - // the series from the storage. - l := labels.Labels{ - { - Name: cortex_tsdb.TenantIDExternalLabel, - Value: userID, - }, { - Name: cortex_tsdb.IngesterIDExternalLabel, - Value: i.TSDBState.shipperIngesterID, - }, - } - - // Create a new shipper for this database - if i.cfg.BlocksStorageConfig.TSDB.IsBlocksShippingEnabled() { - userDB.shipper = shipper.New( - userLogger, - tsdbPromReg, - udir, - bucket.NewUserBucketClient(userID, i.TSDBState.bucket, i.limits), - func() labels.Labels { return l }, - metadata.ReceiveSource, - false, // No need to upload compacted blocks. Cortex compactor takes care of that. - true, // Allow out of order uploads. It's fine in Cortex's context. - metadata.NoneFunc, - ) - - // Initialise the shipper blocks cache. - if err := userDB.updateCachedShippedBlocks(); err != nil { - level.Error(userLogger).Log("msg", "failed to update cached shipped blocks after shipper initialisation", "err", err) - } - } - - i.TSDBState.tsdbMetrics.setRegistryForUser(userID, tsdbPromReg) - return userDB, nil -} - -func (i *Ingester) closeAllTSDB() { - i.stoppedMtx.Lock() - - wg := &sync.WaitGroup{} - wg.Add(len(i.TSDBState.dbs)) - - // Concurrently close all users TSDB - for userID, userDB := range i.TSDBState.dbs { - userID := userID - - go func(db *userTSDB) { - defer wg.Done() - - if err := db.Close(); err != nil { - level.Warn(i.logger).Log("msg", "unable to close TSDB", "err", err, "user", userID) - return - } - - // Now that the TSDB has been closed, we should remove it from the - // set of open ones. This lock acquisition doesn't deadlock with the - // outer one, because the outer one is released as soon as all go - // routines are started. - i.stoppedMtx.Lock() - delete(i.TSDBState.dbs, userID) - i.stoppedMtx.Unlock() - - i.metrics.memUsers.Dec() - i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) - }(userDB) - } - - // Wait until all Close() completed - i.stoppedMtx.Unlock() - wg.Wait() -} - -// openExistingTSDB walks the user tsdb dir, and opens a tsdb for each user. This may start a WAL replay, so we limit the number of -// concurrently opening TSDB. -func (i *Ingester) openExistingTSDB(ctx context.Context) error { - level.Info(i.logger).Log("msg", "opening existing TSDBs") - - queue := make(chan string) - group, groupCtx := errgroup.WithContext(ctx) - - // Create a pool of workers which will open existing TSDBs. - for n := 0; n < i.cfg.BlocksStorageConfig.TSDB.MaxTSDBOpeningConcurrencyOnStartup; n++ { - group.Go(func() error { - for userID := range queue { - startTime := time.Now() - - db, err := i.createTSDB(userID) - if err != nil { - level.Error(i.logger).Log("msg", "unable to open TSDB", "err", err, "user", userID) - return errors.Wrapf(err, "unable to open TSDB for user %s", userID) - } - - // Add the database to the map of user databases - i.stoppedMtx.Lock() - i.TSDBState.dbs[userID] = db - i.stoppedMtx.Unlock() - i.metrics.memUsers.Inc() - - i.TSDBState.walReplayTime.Observe(time.Since(startTime).Seconds()) - } - - return nil - }) - } - - // Spawn a goroutine to find all users with a TSDB on the filesystem. - group.Go(func() error { - // Close the queue once filesystem walking is done. - defer close(queue) - - walkErr := filepath.Walk(i.cfg.BlocksStorageConfig.TSDB.Dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - // If the root directory doesn't exist, we're OK (not needed to be created upfront). - if os.IsNotExist(err) && path == i.cfg.BlocksStorageConfig.TSDB.Dir { - return filepath.SkipDir - } - - level.Error(i.logger).Log("msg", "an error occurred while iterating the filesystem storing TSDBs", "path", path, "err", err) - return errors.Wrapf(err, "an error occurred while iterating the filesystem storing TSDBs at %s", path) - } - - // Skip root dir and all other files - if path == i.cfg.BlocksStorageConfig.TSDB.Dir || !info.IsDir() { - return nil - } - - // Top level directories are assumed to be user TSDBs - userID := info.Name() - f, err := os.Open(path) - if err != nil { - level.Error(i.logger).Log("msg", "unable to open TSDB dir", "err", err, "user", userID, "path", path) - return errors.Wrapf(err, "unable to open TSDB dir %s for user %s", path, userID) - } - defer f.Close() - - // If the dir is empty skip it - if _, err := f.Readdirnames(1); err != nil { - if err == io.EOF { - return filepath.SkipDir - } - - level.Error(i.logger).Log("msg", "unable to read TSDB dir", "err", err, "user", userID, "path", path) - return errors.Wrapf(err, "unable to read TSDB dir %s for user %s", path, userID) - } - - // Enqueue the user to be processed. - select { - case queue <- userID: - // Nothing to do. - case <-groupCtx.Done(): - // Interrupt in case a failure occurred in another goroutine. - return nil - } - - // Don't descend into subdirectories. - return filepath.SkipDir - }) - - return errors.Wrapf(walkErr, "unable to walk directory %s containing existing TSDBs", i.cfg.BlocksStorageConfig.TSDB.Dir) - }) - - // Wait for all workers to complete. - err := group.Wait() - if err != nil { - level.Error(i.logger).Log("msg", "error while opening existing TSDBs", "err", err) - return err - } - - level.Info(i.logger).Log("msg", "successfully opened existing TSDBs") - return nil -} - -// getMemorySeriesMetric returns the total number of in-memory series across all open TSDBs. -func (i *Ingester) getMemorySeriesMetric() float64 { - if err := i.checkRunning(); err != nil { - return 0 - } - - i.stoppedMtx.RLock() - defer i.stoppedMtx.RUnlock() - - count := uint64(0) - for _, db := range i.TSDBState.dbs { - count += db.Head().NumSeries() - } - - return float64(count) -} - -// getOldestUnshippedBlockMetric returns the unix timestamp of the oldest unshipped block or -// 0 if all blocks have been shipped. -func (i *Ingester) getOldestUnshippedBlockMetric() float64 { - i.stoppedMtx.RLock() - defer i.stoppedMtx.RUnlock() - - oldest := uint64(0) - for _, db := range i.TSDBState.dbs { - if ts := db.getOldestUnshippedBlockTime(); oldest == 0 || ts < oldest { - oldest = ts - } - } - - return float64(oldest / 1000) -} - -func (i *Ingester) shipBlocksLoop(ctx context.Context) error { - // We add a slight jitter to make sure that if the head compaction interval and ship interval are set to the same - // value they don't clash (if they both continuously run at the same exact time, the head compaction may not run - // because can't successfully change the state). - shipTicker := time.NewTicker(util.DurationWithJitter(i.cfg.BlocksStorageConfig.TSDB.ShipInterval, 0.01)) - defer shipTicker.Stop() - - for { - select { - case <-shipTicker.C: - i.shipBlocks(ctx, nil) - - case req := <-i.TSDBState.shipTrigger: - i.shipBlocks(ctx, req.users) - close(req.callback) // Notify back. - - case <-ctx.Done(): - return nil - } - } -} - -// shipBlocks runs shipping for all users. -func (i *Ingester) shipBlocks(ctx context.Context, allowed *util.AllowedTenants) { - // Do not ship blocks if the ingester is PENDING or JOINING. It's - // particularly important for the JOINING state because there could - // be a blocks transfer in progress (from another ingester) and if we - // run the shipper in such state we could end up with race conditions. - if i.lifecycler != nil { - if ingesterState := i.lifecycler.GetState(); ingesterState == ring.PENDING || ingesterState == ring.JOINING { - level.Info(i.logger).Log("msg", "TSDB blocks shipping has been skipped because of the current ingester state", "state", ingesterState) - return - } - } - - // Number of concurrent workers is limited in order to avoid to concurrently sync a lot - // of tenants in a large cluster. - _ = concurrency.ForEachUser(ctx, i.getTSDBUsers(), i.cfg.BlocksStorageConfig.TSDB.ShipConcurrency, func(ctx context.Context, userID string) error { - if !allowed.IsAllowed(userID) { - return nil - } - - // Get the user's DB. If the user doesn't exist, we skip it. - userDB := i.getTSDB(userID) - if userDB == nil || userDB.shipper == nil { - return nil - } - - if userDB.deletionMarkFound.Load() { - return nil - } - - if time.Since(time.Unix(userDB.lastDeletionMarkCheck.Load(), 0)) > cortex_tsdb.DeletionMarkCheckInterval { - // Even if check fails with error, we don't want to repeat it too often. - userDB.lastDeletionMarkCheck.Store(time.Now().Unix()) - - deletionMarkExists, err := cortex_tsdb.TenantDeletionMarkExists(ctx, i.TSDBState.bucket, userID) - if err != nil { - // If we cannot check for deletion mark, we continue anyway, even though in production shipper will likely fail too. - // This however simplifies unit tests, where tenant deletion check is enabled by default, but tests don't setup bucket. - level.Warn(i.logger).Log("msg", "failed to check for tenant deletion mark before shipping blocks", "user", userID, "err", err) - } else if deletionMarkExists { - userDB.deletionMarkFound.Store(true) - - level.Info(i.logger).Log("msg", "tenant deletion mark exists, not shipping blocks", "user", userID) - return nil - } - } - - // Run the shipper's Sync() to upload unshipped blocks. Make sure the TSDB state is active, in order to - // avoid any race condition with closing idle TSDBs. - if !userDB.casState(active, activeShipping) { - level.Info(i.logger).Log("msg", "shipper skipped because the TSDB is not active", "user", userID) - return nil - } - defer userDB.casState(activeShipping, active) - - uploaded, err := userDB.shipper.Sync(ctx) - if err != nil { - level.Warn(i.logger).Log("msg", "shipper failed to synchronize TSDB blocks with the storage", "user", userID, "uploaded", uploaded, "err", err) - } else { - level.Debug(i.logger).Log("msg", "shipper successfully synchronized TSDB blocks with storage", "user", userID, "uploaded", uploaded) - } - - // The shipper meta file could be updated even if the Sync() returned an error, - // so it's safer to update it each time at least a block has been uploaded. - // Moreover, the shipper meta file could be updated even if no blocks are uploaded - // (eg. blocks removed due to retention) but doesn't cause any harm not updating - // the cached list of blocks in such case, so we're not handling it. - if uploaded > 0 { - if err := userDB.updateCachedShippedBlocks(); err != nil { - level.Error(i.logger).Log("msg", "failed to update cached shipped blocks after shipper synchronisation", "user", userID, "err", err) - } - } - - return nil - }) -} - -func (i *Ingester) compactionLoop(ctx context.Context) error { - ticker := time.NewTicker(i.cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval) - defer ticker.Stop() - - for ctx.Err() == nil { - select { - case <-ticker.C: - i.compactBlocks(ctx, false, nil) - - case req := <-i.TSDBState.forceCompactTrigger: - i.compactBlocks(ctx, true, req.users) - close(req.callback) // Notify back. - - case <-ctx.Done(): - return nil - } - } - return nil -} - -// Compacts all compactable blocks. Force flag will force compaction even if head is not compactable yet. -func (i *Ingester) compactBlocks(ctx context.Context, force bool, allowed *util.AllowedTenants) { - // Don't compact TSDB blocks while JOINING as there may be ongoing blocks transfers. - // Compaction loop is not running in LEAVING state, so if we get here in LEAVING state, we're flushing blocks. - if i.lifecycler != nil { - if ingesterState := i.lifecycler.GetState(); ingesterState == ring.JOINING { - level.Info(i.logger).Log("msg", "TSDB blocks compaction has been skipped because of the current ingester state", "state", ingesterState) - return - } - } - - _ = concurrency.ForEachUser(ctx, i.getTSDBUsers(), i.cfg.BlocksStorageConfig.TSDB.HeadCompactionConcurrency, func(ctx context.Context, userID string) error { - if !allowed.IsAllowed(userID) { - return nil - } - - userDB := i.getTSDB(userID) - if userDB == nil { - return nil - } - - // Don't do anything, if there is nothing to compact. - h := userDB.Head() - if h.NumSeries() == 0 { - return nil - } - - var err error - - i.TSDBState.compactionsTriggered.Inc() - - reason := "" - switch { - case force: - reason = "forced" - err = userDB.compactHead(i.cfg.BlocksStorageConfig.TSDB.BlockRanges[0].Milliseconds()) - - case i.TSDBState.compactionIdleTimeout > 0 && userDB.isIdle(time.Now(), i.TSDBState.compactionIdleTimeout): - reason = "idle" - level.Info(i.logger).Log("msg", "TSDB is idle, forcing compaction", "user", userID) - err = userDB.compactHead(i.cfg.BlocksStorageConfig.TSDB.BlockRanges[0].Milliseconds()) - - default: - reason = "regular" - err = userDB.Compact() - } - - if err != nil { - i.TSDBState.compactionsFailed.Inc() - level.Warn(i.logger).Log("msg", "TSDB blocks compaction for user has failed", "user", userID, "err", err, "compactReason", reason) - } else { - level.Debug(i.logger).Log("msg", "TSDB blocks compaction completed successfully", "user", userID, "compactReason", reason) - } - - return nil - }) -} - -func (i *Ingester) closeAndDeleteIdleUserTSDBs(ctx context.Context) error { - for _, userID := range i.getTSDBUsers() { - if ctx.Err() != nil { - return nil - } - - result := i.closeAndDeleteUserTSDBIfIdle(userID) - - i.TSDBState.idleTsdbChecks.WithLabelValues(string(result)).Inc() - } - - return nil -} - -func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckResult { - userDB := i.getTSDB(userID) - if userDB == nil || userDB.shipper == nil { - // We will not delete local data when not using shipping to storage. - return tsdbShippingDisabled - } - - if result := userDB.shouldCloseTSDB(i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout); !result.shouldClose() { - return result - } - - // This disables pushes and force-compactions. Not allowed to close while shipping is in progress. - if !userDB.casState(active, closing) { - return tsdbNotActive - } - - // If TSDB is fully closed, we will set state to 'closed', which will prevent this defered closing -> active transition. - defer userDB.casState(closing, active) - - // Make sure we don't ignore any possible inflight pushes. - userDB.pushesInFlight.Wait() - - // Verify again, things may have changed during the checks and pushes. - tenantDeleted := false - if result := userDB.shouldCloseTSDB(i.cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout); !result.shouldClose() { - // This will also change TSDB state back to active (via defer above). - return result - } else if result == tsdbTenantMarkedForDeletion { - tenantDeleted = true - } - - // At this point there are no more pushes to TSDB, and no possible compaction. Normally TSDB is empty, - // but if we're closing TSDB because of tenant deletion mark, then it may still contain some series. - // We need to remove these series from series count. - i.TSDBState.seriesCount.Sub(int64(userDB.Head().NumSeries())) - - dir := userDB.db.Dir() - - if err := userDB.Close(); err != nil { - level.Error(i.logger).Log("msg", "failed to close idle TSDB", "user", userID, "err", err) - return tsdbCloseFailed - } - - level.Info(i.logger).Log("msg", "closed idle TSDB", "user", userID) - - // This will prevent going back to "active" state in deferred statement. - userDB.casState(closing, closed) - - // Only remove user from TSDBState when everything is cleaned up - // This will prevent concurrency problems when cortex are trying to open new TSDB - Ie: New request for a given tenant - // came in - while closing the tsdb for the same tenant. - // If this happens now, the request will get reject as the push will not be able to acquire the lock as the tsdb will be - // in closed state - defer func() { - i.stoppedMtx.Lock() - delete(i.TSDBState.dbs, userID) - i.stoppedMtx.Unlock() - }() - - i.metrics.memUsers.Dec() - i.TSDBState.tsdbMetrics.removeRegistryForUser(userID) - - i.deleteUserMetadata(userID) - i.metrics.deletePerUserMetrics(userID) - - validation.DeletePerUserValidationMetrics(userID, i.logger) - - // And delete local data. - if err := os.RemoveAll(dir); err != nil { - level.Error(i.logger).Log("msg", "failed to delete local TSDB", "user", userID, "err", err) - return tsdbDataRemovalFailed - } - - if tenantDeleted { - level.Info(i.logger).Log("msg", "deleted local TSDB, user marked for deletion", "user", userID, "dir", dir) - return tsdbTenantMarkedForDeletion - } - - level.Info(i.logger).Log("msg", "deleted local TSDB, due to being idle", "user", userID, "dir", dir) - return tsdbIdleClosed -} - -// This method will flush all data. It is called as part of Lifecycler's shutdown (if flush on shutdown is configured), or from the flusher. -// -// When called as during Lifecycler shutdown, this happens as part of normal Ingester shutdown (see stoppingV2 method). -// Samples are not received at this stage. Compaction and Shipping loops have already been stopped as well. -// -// When used from flusher, ingester is constructed in a way that compaction, shipping and receiving of samples is never started. -func (i *Ingester) v2LifecyclerFlush() { - level.Info(i.logger).Log("msg", "starting to flush and ship TSDB blocks") - - ctx := context.Background() - - i.compactBlocks(ctx, true, nil) - if i.cfg.BlocksStorageConfig.TSDB.IsBlocksShippingEnabled() { - i.shipBlocks(ctx, nil) - } - - level.Info(i.logger).Log("msg", "finished flushing and shipping TSDB blocks") -} - -const ( - tenantParam = "tenant" - waitParam = "wait" -) - -// Blocks version of Flush handler. It force-compacts blocks, and triggers shipping. -func (i *Ingester) v2FlushHandler(w http.ResponseWriter, r *http.Request) { - err := r.ParseForm() - if err != nil { - level.Warn(i.logger).Log("msg", "failed to parse HTTP request in flush handler", "err", err) - w.WriteHeader(http.StatusBadRequest) - return - } - - tenants := r.Form[tenantParam] - - allowedUsers := util.NewAllowedTenants(tenants, nil) - run := func() { - ingCtx := i.BasicService.ServiceContext() - if ingCtx == nil || ingCtx.Err() != nil { - level.Info(i.logger).Log("msg", "flushing TSDB blocks: ingester not running, ignoring flush request") - return - } - - compactionCallbackCh := make(chan struct{}) - - level.Info(i.logger).Log("msg", "flushing TSDB blocks: triggering compaction") - select { - case i.TSDBState.forceCompactTrigger <- requestWithUsersAndCallback{users: allowedUsers, callback: compactionCallbackCh}: - // Compacting now. - case <-ingCtx.Done(): - level.Warn(i.logger).Log("msg", "failed to compact TSDB blocks, ingester not running anymore") - return - } - - // Wait until notified about compaction being finished. - select { - case <-compactionCallbackCh: - level.Info(i.logger).Log("msg", "finished compacting TSDB blocks") - case <-ingCtx.Done(): - level.Warn(i.logger).Log("msg", "failed to compact TSDB blocks, ingester not running anymore") - return - } - - if i.cfg.BlocksStorageConfig.TSDB.IsBlocksShippingEnabled() { - shippingCallbackCh := make(chan struct{}) // must be new channel, as compactionCallbackCh is closed now. - - level.Info(i.logger).Log("msg", "flushing TSDB blocks: triggering shipping") - - select { - case i.TSDBState.shipTrigger <- requestWithUsersAndCallback{users: allowedUsers, callback: shippingCallbackCh}: - // shipping now - case <-ingCtx.Done(): - level.Warn(i.logger).Log("msg", "failed to ship TSDB blocks, ingester not running anymore") - return - } - - // Wait until shipping finished. - select { - case <-shippingCallbackCh: - level.Info(i.logger).Log("msg", "shipping of TSDB blocks finished") - case <-ingCtx.Done(): - level.Warn(i.logger).Log("msg", "failed to ship TSDB blocks, ingester not running anymore") - return - } - } - - level.Info(i.logger).Log("msg", "flushing TSDB blocks: finished") - } - - if len(r.Form[waitParam]) > 0 && r.Form[waitParam][0] == "true" { - // Run synchronously. This simplifies and speeds up tests. - run() - } else { - go run() - } - - w.WriteHeader(http.StatusNoContent) -} - -// metadataQueryRange returns the best range to query for metadata queries based on the timerange in the ingester. -func metadataQueryRange(queryStart, queryEnd int64, db *userTSDB) (mint, maxt int64, err error) { - // Ingesters are run with limited retention and we don't support querying the store-gateway for labels yet. - // This means if someone loads a dashboard that is outside the range of the ingester, and we only return the - // data for the timerange requested (which will be empty), the dashboards will break. To fix this we should - // return the "head block" range until we can query the store-gateway. - - // Now the question would be what to do when the query is partially in the ingester range. I would err on the side - // of caution and query the entire db, as I can't think of a good way to query the head + the overlapping range. - mint, maxt = queryStart, queryEnd - - lowestTs, err := db.StartTime() - if err != nil { - return mint, maxt, err - } - - // Completely outside. - if queryEnd < lowestTs { - mint, maxt = db.Head().MinTime(), db.Head().MaxTime() - } else if queryStart < lowestTs { - // Partially inside. - mint, maxt = 0, math.MaxInt64 - } - - return -} - -func wrappedTSDBIngestErr(ingestErr error, timestamp model.Time, labels []cortexpb.LabelAdapter) error { - if ingestErr == nil { - return nil - } - - return fmt.Errorf(errTSDBIngest, ingestErr, timestamp.Time().UTC().Format(time.RFC3339Nano), cortexpb.FromLabelAdaptersToLabels(labels).String()) -} - -func wrappedTSDBIngestExemplarErr(ingestErr error, timestamp model.Time, seriesLabels, exemplarLabels []cortexpb.LabelAdapter) error { - if ingestErr == nil { - return nil - } - - return fmt.Errorf(errTSDBIngestExemplar, ingestErr, timestamp.Time().UTC().Format(time.RFC3339Nano), - cortexpb.FromLabelAdaptersToLabels(seriesLabels).String(), - cortexpb.FromLabelAdaptersToLabels(exemplarLabels).String(), - ) -} - -func (i *Ingester) getInstanceLimits() *InstanceLimits { - // Don't apply any limits while starting. We especially don't want to apply series in memory limit while replaying WAL. - if i.State() == services.Starting { - return nil - } - - if i.cfg.InstanceLimitsFn == nil { - return defaultInstanceLimits - } - - l := i.cfg.InstanceLimitsFn() - if l == nil { - return defaultInstanceLimits - } - - return l -} diff --git a/pkg/ingester/ingester_v2_test.go b/pkg/ingester/ingester_v2_test.go deleted file mode 100644 index af03306d5d2..00000000000 --- a/pkg/ingester/ingester_v2_test.go +++ /dev/null @@ -1,3893 +0,0 @@ -package ingester - -import ( - "bytes" - "context" - "fmt" - "io" - "io/ioutil" - "math" - "net" - "net/http" - "net/http/httptest" - "net/url" - "os" - "path/filepath" - "sort" - "strconv" - "strings" - "sync" - "testing" - "time" - - "github.com/go-kit/log" - "github.com/oklog/ulid" - "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/testutil" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/tsdb" - "github.com/prometheus/prometheus/tsdb/chunkenc" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "github.com/thanos-io/thanos/pkg/objstore" - "github.com/thanos-io/thanos/pkg/shipper" - "github.com/weaveworks/common/httpgrpc" - "github.com/weaveworks/common/middleware" - "github.com/weaveworks/common/user" - "golang.org/x/sync/errgroup" - "google.golang.org/grpc" - - "github.com/cortexproject/cortex/pkg/chunk/encoding" - "github.com/cortexproject/cortex/pkg/cortexpb" - "github.com/cortexproject/cortex/pkg/ingester/client" - "github.com/cortexproject/cortex/pkg/ring" - cortex_tsdb "github.com/cortexproject/cortex/pkg/storage/tsdb" - "github.com/cortexproject/cortex/pkg/util" - util_math "github.com/cortexproject/cortex/pkg/util/math" - "github.com/cortexproject/cortex/pkg/util/services" - "github.com/cortexproject/cortex/pkg/util/test" - "github.com/cortexproject/cortex/pkg/util/validation" -) - -func TestIngester_v2Push(t *testing.T) { - metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} - metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) - metricNames := []string{ - "cortex_ingester_ingested_samples_total", - "cortex_ingester_ingested_samples_failures_total", - "cortex_ingester_memory_series", - "cortex_ingester_memory_users", - "cortex_ingester_memory_series_created_total", - "cortex_ingester_memory_series_removed_total", - "cortex_discarded_samples_total", - "cortex_ingester_active_series", - } - userID := "test" - - tests := map[string]struct { - reqs []*cortexpb.WriteRequest - expectedErr error - expectedIngested []cortexpb.TimeSeries - expectedMetadataIngested []*cortexpb.MetricMetadata - expectedExemplarsIngested []cortexpb.TimeSeries - expectedMetrics string - additionalMetrics []string - disableActiveSeries bool - maxExemplars int - }{ - "should succeed on valid series and metadata": { - reqs: []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - []*cortexpb.MetricMetadata{ - {MetricFamilyName: "metric_name_1", Help: "a help for metric_name_1", Unit: "", Type: cortexpb.COUNTER}, - }, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, - []*cortexpb.MetricMetadata{ - {MetricFamilyName: "metric_name_2", Help: "a help for metric_name_2", Unit: "", Type: cortexpb.GAUGE}, - }, - cortexpb.API), - }, - expectedErr: nil, - expectedIngested: []cortexpb.TimeSeries{ - {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 9}, {Value: 2, TimestampMs: 10}}}, - }, - expectedMetadataIngested: []*cortexpb.MetricMetadata{ - {MetricFamilyName: "metric_name_2", Help: "a help for metric_name_2", Unit: "", Type: cortexpb.GAUGE}, - {MetricFamilyName: "metric_name_1", Help: "a help for metric_name_1", Unit: "", Type: cortexpb.COUNTER}, - }, - additionalMetrics: []string{ - // Metadata. - "cortex_ingester_memory_metadata", - "cortex_ingester_memory_metadata_created_total", - "cortex_ingester_ingested_metadata_total", - "cortex_ingester_ingested_metadata_failures_total", - }, - expectedMetrics: ` - # HELP cortex_ingester_ingested_metadata_failures_total The total number of metadata that errored on ingestion. - # TYPE cortex_ingester_ingested_metadata_failures_total counter - cortex_ingester_ingested_metadata_failures_total 0 - # HELP cortex_ingester_ingested_metadata_total The total number of metadata ingested. - # TYPE cortex_ingester_ingested_metadata_total counter - cortex_ingester_ingested_metadata_total 2 - # HELP cortex_ingester_memory_metadata The current number of metadata in memory. - # TYPE cortex_ingester_memory_metadata gauge - cortex_ingester_memory_metadata 2 - # HELP cortex_ingester_memory_metadata_created_total The total number of metadata that were created per user - # TYPE cortex_ingester_memory_metadata_created_total counter - cortex_ingester_memory_metadata_created_total{user="test"} 2 - # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. - # TYPE cortex_ingester_ingested_samples_total counter - cortex_ingester_ingested_samples_total 2 - # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. - # TYPE cortex_ingester_ingested_samples_failures_total counter - cortex_ingester_ingested_samples_failures_total 0 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 1 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="test"} 1 - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="test"} 0 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 1 - `, - }, - "should succeed on valid series with exemplars": { - maxExemplars: 2, - reqs: []*cortexpb.WriteRequest{ - // Ingesting an exemplar requires a sample to create the series first - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - { - Timeseries: []cortexpb.PreallocTimeseries{ - { - TimeSeries: &cortexpb.TimeSeries{ - Labels: []cortexpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool - Exemplars: []cortexpb.Exemplar{ - { - Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "123"}}, - TimestampMs: 1000, - Value: 1000, - }, - { - Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "456"}}, - TimestampMs: 1001, - Value: 1001, - }, - }, - }, - }, - }, - }, - }, - expectedErr: nil, - expectedIngested: []cortexpb.TimeSeries{ - {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 9}}}, - }, - expectedExemplarsIngested: []cortexpb.TimeSeries{ - { - Labels: metricLabelAdapters, - Exemplars: []cortexpb.Exemplar{ - { - Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "123"}}, - TimestampMs: 1000, - Value: 1000, - }, - { - Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "456"}}, - TimestampMs: 1001, - Value: 1001, - }, - }, - }, - }, - expectedMetadataIngested: nil, - additionalMetrics: []string{ - "cortex_ingester_tsdb_exemplar_exemplars_appended_total", - "cortex_ingester_tsdb_exemplar_exemplars_in_storage", - "cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage", - "cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds", - "cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total", - }, - expectedMetrics: ` - # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. - # TYPE cortex_ingester_ingested_samples_total counter - cortex_ingester_ingested_samples_total 1 - # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. - # TYPE cortex_ingester_ingested_samples_failures_total counter - cortex_ingester_ingested_samples_failures_total 0 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 1 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="test"} 1 - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="test"} 0 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 1 - - # HELP cortex_ingester_tsdb_exemplar_exemplars_appended_total Total number of TSDB exemplars appended. - # TYPE cortex_ingester_tsdb_exemplar_exemplars_appended_total counter - cortex_ingester_tsdb_exemplar_exemplars_appended_total 2 - - # HELP cortex_ingester_tsdb_exemplar_exemplars_in_storage Number of TSDB exemplars currently in storage. - # TYPE cortex_ingester_tsdb_exemplar_exemplars_in_storage gauge - cortex_ingester_tsdb_exemplar_exemplars_in_storage 2 - - # HELP cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage Number of TSDB series with exemplars currently in storage. - # TYPE cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage gauge - cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{user="test"} 1 - - # HELP cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds The timestamp of the oldest exemplar stored in circular storage. Useful to check for what time range the current exemplar buffer limit allows. This usually means the last timestamp for all exemplars for a typical setup. This is not true though if one of the series timestamp is in future compared to rest series. - # TYPE cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds gauge - cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds{user="test"} 1 - - # HELP cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total Total number of out of order exemplar ingestion failed attempts. - # TYPE cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total counter - cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total 0 - `, - }, - "successful push, active series disabled": { - disableActiveSeries: true, - reqs: []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, - nil, - cortexpb.API), - }, - expectedErr: nil, - expectedIngested: []cortexpb.TimeSeries{ - {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 9}, {Value: 2, TimestampMs: 10}}}, - }, - expectedMetrics: ` - # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. - # TYPE cortex_ingester_ingested_samples_total counter - cortex_ingester_ingested_samples_total 2 - # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. - # TYPE cortex_ingester_ingested_samples_failures_total counter - cortex_ingester_ingested_samples_failures_total 0 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 1 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="test"} 1 - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="test"} 0 - `, - }, - "should soft fail on sample out of order": { - reqs: []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, - nil, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - }, - expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestErr(storage.ErrOutOfOrderSample, model.Time(9), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), - expectedIngested: []cortexpb.TimeSeries{ - {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 10}}}, - }, - expectedMetrics: ` - # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. - # TYPE cortex_ingester_ingested_samples_total counter - cortex_ingester_ingested_samples_total 1 - # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. - # TYPE cortex_ingester_ingested_samples_failures_total counter - cortex_ingester_ingested_samples_failures_total 1 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 1 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="test"} 1 - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="test"} 0 - # HELP cortex_discarded_samples_total The total number of samples that were discarded. - # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-out-of-order",user="test"} 1 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 1 - `, - }, - "should soft fail on sample out of bound": { - reqs: []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}, - nil, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 1575043969 - (86400 * 1000)}}, - nil, - cortexpb.API), - }, - expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestErr(storage.ErrOutOfBounds, model.Time(1575043969-(86400*1000)), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), - expectedIngested: []cortexpb.TimeSeries{ - {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}}, - }, - expectedMetrics: ` - # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. - # TYPE cortex_ingester_ingested_samples_total counter - cortex_ingester_ingested_samples_total 1 - # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. - # TYPE cortex_ingester_ingested_samples_failures_total counter - cortex_ingester_ingested_samples_failures_total 1 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 1 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="test"} 1 - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="test"} 0 - # HELP cortex_discarded_samples_total The total number of samples that were discarded. - # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="sample-out-of-bounds",user="test"} 1 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 1 - `, - }, - "should soft fail on two different sample values at the same timestamp": { - reqs: []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}, - nil, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 1575043969}}, - nil, - cortexpb.API), - }, - expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestErr(storage.ErrDuplicateSampleForTimestamp, model.Time(1575043969), cortexpb.FromLabelsToLabelAdapters(metricLabels)), userID).Error()), - expectedIngested: []cortexpb.TimeSeries{ - {Labels: metricLabelAdapters, Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 1575043969}}}, - }, - expectedMetrics: ` - # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. - # TYPE cortex_ingester_ingested_samples_total counter - cortex_ingester_ingested_samples_total 1 - # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. - # TYPE cortex_ingester_ingested_samples_failures_total counter - cortex_ingester_ingested_samples_failures_total 1 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 1 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="test"} 1 - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="test"} 0 - # HELP cortex_discarded_samples_total The total number of samples that were discarded. - # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="new-value-for-timestamp",user="test"} 1 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 1 - `, - }, - "should soft fail on exemplar with unknown series": { - maxExemplars: 1, - reqs: []*cortexpb.WriteRequest{ - // Ingesting an exemplar requires a sample to create the series first - // This is not done here. - { - Timeseries: []cortexpb.PreallocTimeseries{ - { - TimeSeries: &cortexpb.TimeSeries{ - Labels: []cortexpb.LabelAdapter{metricLabelAdapters[0]}, // Cannot reuse test slice var because it is cleared and returned to the pool - Exemplars: []cortexpb.Exemplar{ - { - Labels: []cortexpb.LabelAdapter{{Name: "traceID", Value: "123"}}, - TimestampMs: 1000, - Value: 1000, - }, - }, - }, - }, - }, - }, - }, - expectedErr: httpgrpc.Errorf(http.StatusBadRequest, wrapWithUser(wrappedTSDBIngestExemplarErr(errExemplarRef, model.Time(1000), cortexpb.FromLabelsToLabelAdapters(metricLabels), []cortexpb.LabelAdapter{{Name: "traceID", Value: "123"}}), userID).Error()), - expectedIngested: nil, - expectedMetadataIngested: nil, - additionalMetrics: []string{ - "cortex_ingester_tsdb_exemplar_exemplars_appended_total", - "cortex_ingester_tsdb_exemplar_exemplars_in_storage", - "cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage", - "cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds", - "cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total", - }, - expectedMetrics: ` - # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. - # TYPE cortex_ingester_ingested_samples_total counter - cortex_ingester_ingested_samples_total 0 - # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. - # TYPE cortex_ingester_ingested_samples_failures_total counter - cortex_ingester_ingested_samples_failures_total 0 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 0 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="test"} 0 - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="test"} 0 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 0 - - # HELP cortex_ingester_tsdb_exemplar_exemplars_appended_total Total number of TSDB exemplars appended. - # TYPE cortex_ingester_tsdb_exemplar_exemplars_appended_total counter - cortex_ingester_tsdb_exemplar_exemplars_appended_total 0 - - # HELP cortex_ingester_tsdb_exemplar_exemplars_in_storage Number of TSDB exemplars currently in storage. - # TYPE cortex_ingester_tsdb_exemplar_exemplars_in_storage gauge - cortex_ingester_tsdb_exemplar_exemplars_in_storage 0 - - # HELP cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage Number of TSDB series with exemplars currently in storage. - # TYPE cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage gauge - cortex_ingester_tsdb_exemplar_series_with_exemplars_in_storage{user="test"} 0 - - # HELP cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds The timestamp of the oldest exemplar stored in circular storage. Useful to check for what time range the current exemplar buffer limit allows. This usually means the last timestamp for all exemplars for a typical setup. This is not true though if one of the series timestamp is in future compared to rest series. - # TYPE cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds gauge - cortex_ingester_tsdb_exemplar_last_exemplars_timestamp_seconds{user="test"} 0 - - # HELP cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total Total number of out of order exemplar ingestion failed attempts. - # TYPE cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total counter - cortex_ingester_tsdb_exemplar_out_of_order_exemplars_total 0 - `, - }, - } - - for testName, testData := range tests { - t.Run(testName, func(t *testing.T) { - registry := prometheus.NewRegistry() - - registry.MustRegister(validation.DiscardedSamples) - validation.DiscardedSamples.Reset() - - // Create a mocked ingester - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries - cfg.BlocksStorageConfig.TSDB.MaxExemplars = testData.maxExemplars - - i, err := prepareIngesterWithBlocksStorage(t, cfg, registry) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - ctx := user.InjectOrgID(context.Background(), userID) - - // Wait until the ingester is ACTIVE - test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push timeseries - for idx, req := range testData.reqs { - _, err := i.v2Push(ctx, req) - - // We expect no error on any request except the last one - // which may error (and in that case we assert on it) - if idx < len(testData.reqs)-1 { - assert.NoError(t, err) - } else { - assert.Equal(t, testData.expectedErr, err) - } - } - - // Read back samples to see what has been really ingested - res, err := i.v2Query(ctx, &client.QueryRequest{ - StartTimestampMs: math.MinInt64, - EndTimestampMs: math.MaxInt64, - Matchers: []*client.LabelMatcher{{Type: client.REGEX_MATCH, Name: labels.MetricName, Value: ".*"}}, - }) - - require.NoError(t, err) - require.NotNil(t, res) - assert.Equal(t, testData.expectedIngested, res.Timeseries) - - // Read back samples to see what has been really ingested - exemplarRes, err := i.v2QueryExemplars(ctx, &client.ExemplarQueryRequest{ - StartTimestampMs: math.MinInt64, - EndTimestampMs: math.MaxInt64, - Matchers: []*client.LabelMatchers{ - {Matchers: []*client.LabelMatcher{{Type: client.REGEX_MATCH, Name: labels.MetricName, Value: ".*"}}}, - }, - }) - - require.NoError(t, err) - require.NotNil(t, exemplarRes) - assert.Equal(t, testData.expectedExemplarsIngested, exemplarRes.Timeseries) - - // Read back metadata to see what has been really ingested. - mres, err := i.MetricsMetadata(ctx, &client.MetricsMetadataRequest{}) - - require.NoError(t, err) - require.NotNil(t, res) - - // Order is never guaranteed. - assert.ElementsMatch(t, testData.expectedMetadataIngested, mres.Metadata) - - // Update active series for metrics check. - if !testData.disableActiveSeries { - i.v2UpdateActiveSeries() - } - - // Append additional metrics to assert on. - mn := append(metricNames, testData.additionalMetrics...) - - // Check tracked Prometheus metrics - err = testutil.GatherAndCompare(registry, strings.NewReader(testData.expectedMetrics), mn...) - assert.NoError(t, err) - }) - } -} - -func TestIngester_v2Push_ShouldCorrectlyTrackMetricsInMultiTenantScenario(t *testing.T) { - metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} - metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) - metricNames := []string{ - "cortex_ingester_ingested_samples_total", - "cortex_ingester_ingested_samples_failures_total", - "cortex_ingester_memory_series", - "cortex_ingester_memory_users", - "cortex_ingester_memory_series_created_total", - "cortex_ingester_memory_series_removed_total", - "cortex_ingester_active_series", - } - - registry := prometheus.NewRegistry() - - // Create a mocked ingester - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - - i, err := prepareIngesterWithBlocksStorage(t, cfg, registry) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until the ingester is ACTIVE - test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push timeseries for each user - for _, userID := range []string{"test-1", "test-2"} { - reqs := []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, - nil, - cortexpb.API), - } - - for _, req := range reqs { - ctx := user.InjectOrgID(context.Background(), userID) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - } - - // Update active series for metrics check. - i.v2UpdateActiveSeries() - - // Check tracked Prometheus metrics - expectedMetrics := ` - # HELP cortex_ingester_ingested_samples_total The total number of samples ingested. - # TYPE cortex_ingester_ingested_samples_total counter - cortex_ingester_ingested_samples_total 4 - # HELP cortex_ingester_ingested_samples_failures_total The total number of samples that errored on ingestion. - # TYPE cortex_ingester_ingested_samples_failures_total counter - cortex_ingester_ingested_samples_failures_total 0 - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 2 - # HELP cortex_ingester_memory_series The current number of series in memory. - # TYPE cortex_ingester_memory_series gauge - cortex_ingester_memory_series 2 - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="test-1"} 1 - cortex_ingester_memory_series_created_total{user="test-2"} 1 - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="test-1"} 0 - cortex_ingester_memory_series_removed_total{user="test-2"} 0 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test-1"} 1 - cortex_ingester_active_series{user="test-2"} 1 - ` - - assert.NoError(t, testutil.GatherAndCompare(registry, strings.NewReader(expectedMetrics), metricNames...)) -} - -func TestIngester_v2Push_DecreaseInactiveSeries(t *testing.T) { - metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} - metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) - metricNames := []string{ - "cortex_ingester_memory_series_created_total", - "cortex_ingester_memory_series_removed_total", - "cortex_ingester_active_series", - } - - registry := prometheus.NewRegistry() - - // Create a mocked ingester - cfg := defaultIngesterTestConfig(t) - cfg.ActiveSeriesMetricsIdleTimeout = 100 * time.Millisecond - cfg.LifecyclerConfig.JoinAfter = 0 - - i, err := prepareIngesterWithBlocksStorage(t, cfg, registry) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until the ingester is ACTIVE - test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push timeseries for each user - for _, userID := range []string{"test-1", "test-2"} { - reqs := []*cortexpb.WriteRequest{ - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 2, TimestampMs: 10}}, - nil, - cortexpb.API), - } - - for _, req := range reqs { - ctx := user.InjectOrgID(context.Background(), userID) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - } - - // Wait a bit to make series inactive (set to 100ms above). - time.Sleep(200 * time.Millisecond) - - // Update active series for metrics check. This will remove inactive series. - i.v2UpdateActiveSeries() - - // Check tracked Prometheus metrics - expectedMetrics := ` - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="test-1"} 1 - cortex_ingester_memory_series_created_total{user="test-2"} 1 - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="test-1"} 0 - cortex_ingester_memory_series_removed_total{user="test-2"} 0 - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test-1"} 0 - cortex_ingester_active_series{user="test-2"} 0 - ` - - assert.NoError(t, testutil.GatherAndCompare(registry, strings.NewReader(expectedMetrics), metricNames...)) -} - -func BenchmarkIngesterV2Push(b *testing.B) { - limits := defaultLimitsTestConfig() - benchmarkIngesterV2Push(b, limits, false) -} - -func benchmarkIngesterV2Push(b *testing.B, limits validation.Limits, errorsExpected bool) { - registry := prometheus.NewRegistry() - ctx := user.InjectOrgID(context.Background(), userID) - - // Create a mocked ingester - cfg := defaultIngesterTestConfig(b) - cfg.LifecyclerConfig.JoinAfter = 0 - - ingester, err := prepareIngesterWithBlocksStorage(b, cfg, registry) - require.NoError(b, err) - require.NoError(b, services.StartAndAwaitRunning(context.Background(), ingester)) - defer services.StopAndAwaitTerminated(context.Background(), ingester) //nolint:errcheck - - // Wait until the ingester is ACTIVE - test.Poll(b, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return ingester.lifecycler.GetState() - }) - - // Push a single time series to set the TSDB min time. - metricLabelAdapters := []cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} - metricLabels := cortexpb.FromLabelAdaptersToLabels(metricLabelAdapters) - startTime := util.TimeToMillis(time.Now()) - - currTimeReq := cortexpb.ToWriteRequest( - []labels.Labels{metricLabels}, - []cortexpb.Sample{{Value: 1, TimestampMs: startTime}}, - nil, - cortexpb.API) - _, err = ingester.v2Push(ctx, currTimeReq) - require.NoError(b, err) - - const ( - series = 10000 - samples = 10 - ) - - allLabels, allSamples := benchmarkData(series) - - b.ResetTimer() - for iter := 0; iter < b.N; iter++ { - // Bump the timestamp on each of our test samples each time round the loop - for j := 0; j < samples; j++ { - for i := range allSamples { - allSamples[i].TimestampMs = startTime + int64(iter*samples+j+1) - } - _, err := ingester.v2Push(ctx, cortexpb.ToWriteRequest(allLabels, allSamples, nil, cortexpb.API)) - if !errorsExpected { - require.NoError(b, err) - } - } - } -} - -func verifyErrorString(tb testing.TB, err error, expectedErr string) { - if err == nil || !strings.Contains(err.Error(), expectedErr) { - tb.Helper() - tb.Fatalf("unexpected error. expected: %s actual: %v", expectedErr, err) - } -} - -func Benchmark_Ingester_v2PushOnError(b *testing.B) { - var ( - ctx = user.InjectOrgID(context.Background(), userID) - sampleTimestamp = int64(100) - metricName = "test" - ) - - scenarios := map[string]struct { - numSeriesPerRequest int - numConcurrentClients int - }{ - "no concurrency": { - numSeriesPerRequest: 1000, - numConcurrentClients: 1, - }, - "low concurrency": { - numSeriesPerRequest: 1000, - numConcurrentClients: 100, - }, - "high concurrency": { - numSeriesPerRequest: 1000, - numConcurrentClients: 1000, - }, - } - - instanceLimits := map[string]*InstanceLimits{ - "no limits": nil, - "limits set": {MaxIngestionRate: 1000, MaxInMemoryTenants: 1, MaxInMemorySeries: 1000, MaxInflightPushRequests: 1000}, // these match max values from scenarios - } - - tests := map[string]struct { - // If this returns false, test is skipped. - prepareConfig func(limits *validation.Limits, instanceLimits *InstanceLimits) bool - beforeBenchmark func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) - runBenchmark func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) - }{ - "out of bound samples": { - prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { return true }, - beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { - // Push a single time series to set the TSDB min time. - currTimeReq := cortexpb.ToWriteRequest( - []labels.Labels{{{Name: labels.MetricName, Value: metricName}}}, - []cortexpb.Sample{{Value: 1, TimestampMs: util.TimeToMillis(time.Now())}}, - nil, - cortexpb.API) - _, err := ingester.v2Push(ctx, currTimeReq) - require.NoError(b, err) - }, - runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { - expectedErr := storage.ErrOutOfBounds.Error() - - // Push out of bound samples. - for n := 0; n < b.N; n++ { - _, err := ingester.v2Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) // nolint:errcheck - - verifyErrorString(b, err, expectedErr) - } - }, - }, - "out of order samples": { - prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { return true }, - beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { - // For each series, push a single sample with a timestamp greater than next pushes. - for i := 0; i < numSeriesPerRequest; i++ { - currTimeReq := cortexpb.ToWriteRequest( - []labels.Labels{{{Name: labels.MetricName, Value: metricName}, {Name: "cardinality", Value: strconv.Itoa(i)}}}, - []cortexpb.Sample{{Value: 1, TimestampMs: sampleTimestamp + 1}}, - nil, - cortexpb.API) - - _, err := ingester.v2Push(ctx, currTimeReq) - require.NoError(b, err) - } - }, - runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { - expectedErr := storage.ErrOutOfOrderSample.Error() - - // Push out of order samples. - for n := 0; n < b.N; n++ { - _, err := ingester.v2Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) // nolint:errcheck - - verifyErrorString(b, err, expectedErr) - } - }, - }, - "per-user series limit reached": { - prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { - limits.MaxLocalSeriesPerUser = 1 - return true - }, - beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { - // Push a series with a metric name different than the one used during the benchmark. - currTimeReq := cortexpb.ToWriteRequest( - []labels.Labels{labels.FromStrings(labels.MetricName, "another")}, - []cortexpb.Sample{{Value: 1, TimestampMs: sampleTimestamp + 1}}, - nil, - cortexpb.API) - _, err := ingester.v2Push(ctx, currTimeReq) - require.NoError(b, err) - }, - runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { - // Push series with a different name than the one already pushed. - for n := 0; n < b.N; n++ { - _, err := ingester.v2Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) // nolint:errcheck - verifyErrorString(b, err, "per-user series limit") - } - }, - }, - "per-metric series limit reached": { - prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { - limits.MaxLocalSeriesPerMetric = 1 - return true - }, - beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { - // Push a series with the same metric name but different labels than the one used during the benchmark. - currTimeReq := cortexpb.ToWriteRequest( - []labels.Labels{labels.FromStrings(labels.MetricName, metricName, "cardinality", "another")}, - []cortexpb.Sample{{Value: 1, TimestampMs: sampleTimestamp + 1}}, - nil, - cortexpb.API) - _, err := ingester.v2Push(ctx, currTimeReq) - require.NoError(b, err) - }, - runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { - // Push series with different labels than the one already pushed. - for n := 0; n < b.N; n++ { - _, err := ingester.v2Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) // nolint:errcheck - verifyErrorString(b, err, "per-metric series limit") - } - }, - }, - "very low ingestion rate limit": { - prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { - if instanceLimits == nil { - return false - } - instanceLimits.MaxIngestionRate = 0.00001 // very low - return true - }, - beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { - // Send a lot of samples - _, err := ingester.v2Push(ctx, generateSamplesForLabel(labels.FromStrings(labels.MetricName, "test"), 10000)) - require.NoError(b, err) - - ingester.ingestionRate.Tick() - }, - runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { - // Push series with different labels than the one already pushed. - for n := 0; n < b.N; n++ { - _, err := ingester.v2Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) - verifyErrorString(b, err, "push rate reached") - } - }, - }, - "max number of tenants reached": { - prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { - if instanceLimits == nil { - return false - } - instanceLimits.MaxInMemoryTenants = 1 - return true - }, - beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { - // Send some samples for one tenant (not the same that is used during the test) - ctx := user.InjectOrgID(context.Background(), "different_tenant") - _, err := ingester.v2Push(ctx, generateSamplesForLabel(labels.FromStrings(labels.MetricName, "test"), 10000)) - require.NoError(b, err) - }, - runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { - // Push series with different labels than the one already pushed. - for n := 0; n < b.N; n++ { - _, err := ingester.v2Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) - verifyErrorString(b, err, "max tenants limit reached") - } - }, - }, - "max number of series reached": { - prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { - if instanceLimits == nil { - return false - } - instanceLimits.MaxInMemorySeries = 1 - return true - }, - beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { - _, err := ingester.v2Push(ctx, generateSamplesForLabel(labels.FromStrings(labels.MetricName, "test"), 10000)) - require.NoError(b, err) - }, - runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { - for n := 0; n < b.N; n++ { - _, err := ingester.v2Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) - verifyErrorString(b, err, "max series limit reached") - } - }, - }, - "max inflight requests reached": { - prepareConfig: func(limits *validation.Limits, instanceLimits *InstanceLimits) bool { - if instanceLimits == nil { - return false - } - instanceLimits.MaxInflightPushRequests = 1 - return true - }, - beforeBenchmark: func(b *testing.B, ingester *Ingester, numSeriesPerRequest int) { - ingester.inflightPushRequests.Inc() - }, - runBenchmark: func(b *testing.B, ingester *Ingester, metrics []labels.Labels, samples []cortexpb.Sample) { - for n := 0; n < b.N; n++ { - _, err := ingester.Push(ctx, cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API)) - verifyErrorString(b, err, "too many inflight push requests") - } - }, - }, - } - - for testName, testData := range tests { - for scenarioName, scenario := range scenarios { - for limitsName, limits := range instanceLimits { - b.Run(fmt.Sprintf("failure: %s, scenario: %s, limits: %s", testName, scenarioName, limitsName), func(b *testing.B) { - registry := prometheus.NewRegistry() - - instanceLimits := limits - if instanceLimits != nil { - // make a copy, to avoid changing value in the instanceLimits map. - newLimits := &InstanceLimits{} - *newLimits = *instanceLimits - instanceLimits = newLimits - } - - // Create a mocked ingester - cfg := defaultIngesterTestConfig(b) - cfg.LifecyclerConfig.JoinAfter = 0 - - limits := defaultLimitsTestConfig() - if !testData.prepareConfig(&limits, instanceLimits) { - b.SkipNow() - } - - cfg.InstanceLimitsFn = func() *InstanceLimits { - return instanceLimits - } - - ingester, err := prepareIngesterWithBlocksStorageAndLimits(b, cfg, limits, "", registry) - require.NoError(b, err) - require.NoError(b, services.StartAndAwaitRunning(context.Background(), ingester)) - defer services.StopAndAwaitTerminated(context.Background(), ingester) //nolint:errcheck - - // Wait until the ingester is ACTIVE - test.Poll(b, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return ingester.lifecycler.GetState() - }) - - testData.beforeBenchmark(b, ingester, scenario.numSeriesPerRequest) - - // Prepare the request. - metrics := make([]labels.Labels, 0, scenario.numSeriesPerRequest) - samples := make([]cortexpb.Sample, 0, scenario.numSeriesPerRequest) - for i := 0; i < scenario.numSeriesPerRequest; i++ { - metrics = append(metrics, labels.Labels{{Name: labels.MetricName, Value: metricName}, {Name: "cardinality", Value: strconv.Itoa(i)}}) - samples = append(samples, cortexpb.Sample{Value: float64(i), TimestampMs: sampleTimestamp}) - } - - // Run the benchmark. - wg := sync.WaitGroup{} - wg.Add(scenario.numConcurrentClients) - start := make(chan struct{}) - - b.ReportAllocs() - b.ResetTimer() - - for c := 0; c < scenario.numConcurrentClients; c++ { - go func() { - defer wg.Done() - <-start - - testData.runBenchmark(b, ingester, metrics, samples) - }() - } - - b.ResetTimer() - close(start) - wg.Wait() - }) - } - } - } -} - -func Test_Ingester_v2LabelNames(t *testing.T) { - series := []struct { - lbls labels.Labels - value float64 - timestamp int64 - }{ - {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "200"}, {Name: "route", Value: "get_user"}}, 1, 100000}, - {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "500"}, {Name: "route", Value: "get_user"}}, 1, 110000}, - {labels.Labels{{Name: labels.MetricName, Value: "test_2"}}, 2, 200000}, - } - - expected := []string{"__name__", "status", "route"} - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push series - ctx := user.InjectOrgID(context.Background(), "test") - - for _, series := range series { - req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - - // Get label names - res, err := i.v2LabelNames(ctx, &client.LabelNamesRequest{}) - require.NoError(t, err) - assert.ElementsMatch(t, expected, res.LabelNames) -} - -func Test_Ingester_v2LabelValues(t *testing.T) { - series := []struct { - lbls labels.Labels - value float64 - timestamp int64 - }{ - {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "200"}, {Name: "route", Value: "get_user"}}, 1, 100000}, - {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "500"}, {Name: "route", Value: "get_user"}}, 1, 110000}, - {labels.Labels{{Name: labels.MetricName, Value: "test_2"}}, 2, 200000}, - } - - expected := map[string][]string{ - "__name__": {"test_1", "test_2"}, - "status": {"200", "500"}, - "route": {"get_user"}, - "unknown": {}, - } - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push series - ctx := user.InjectOrgID(context.Background(), "test") - - for _, series := range series { - req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - - // Get label values - for labelName, expectedValues := range expected { - req := &client.LabelValuesRequest{LabelName: labelName} - res, err := i.v2LabelValues(ctx, req) - require.NoError(t, err) - assert.ElementsMatch(t, expectedValues, res.LabelValues) - } -} - -func Test_Ingester_v2Query(t *testing.T) { - series := []struct { - lbls labels.Labels - value float64 - timestamp int64 - }{ - {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "200"}, {Name: "route", Value: "get_user"}}, 1, 100000}, - {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "500"}, {Name: "route", Value: "get_user"}}, 1, 110000}, - {labels.Labels{{Name: labels.MetricName, Value: "test_2"}}, 2, 200000}, - } - - tests := map[string]struct { - from int64 - to int64 - matchers []*client.LabelMatcher - expected []cortexpb.TimeSeries - }{ - "should return an empty response if no metric matches": { - from: math.MinInt64, - to: math.MaxInt64, - matchers: []*client.LabelMatcher{ - {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "unknown"}, - }, - expected: []cortexpb.TimeSeries{}, - }, - "should filter series by == matcher": { - from: math.MinInt64, - to: math.MaxInt64, - matchers: []*client.LabelMatcher{ - {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, - }, - expected: []cortexpb.TimeSeries{ - {Labels: cortexpb.FromLabelsToLabelAdapters(series[0].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 100000}}}, - {Labels: cortexpb.FromLabelsToLabelAdapters(series[1].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 110000}}}, - }, - }, - "should filter series by != matcher": { - from: math.MinInt64, - to: math.MaxInt64, - matchers: []*client.LabelMatcher{ - {Type: client.NOT_EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, - }, - expected: []cortexpb.TimeSeries{ - {Labels: cortexpb.FromLabelsToLabelAdapters(series[2].lbls), Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 200000}}}, - }, - }, - "should filter series by =~ matcher": { - from: math.MinInt64, - to: math.MaxInt64, - matchers: []*client.LabelMatcher{ - {Type: client.REGEX_MATCH, Name: model.MetricNameLabel, Value: ".*_1"}, - }, - expected: []cortexpb.TimeSeries{ - {Labels: cortexpb.FromLabelsToLabelAdapters(series[0].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 100000}}}, - {Labels: cortexpb.FromLabelsToLabelAdapters(series[1].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 110000}}}, - }, - }, - "should filter series by !~ matcher": { - from: math.MinInt64, - to: math.MaxInt64, - matchers: []*client.LabelMatcher{ - {Type: client.REGEX_NO_MATCH, Name: model.MetricNameLabel, Value: ".*_1"}, - }, - expected: []cortexpb.TimeSeries{ - {Labels: cortexpb.FromLabelsToLabelAdapters(series[2].lbls), Samples: []cortexpb.Sample{{Value: 2, TimestampMs: 200000}}}, - }, - }, - "should filter series by multiple matchers": { - from: math.MinInt64, - to: math.MaxInt64, - matchers: []*client.LabelMatcher{ - {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, - {Type: client.REGEX_MATCH, Name: "status", Value: "5.."}, - }, - expected: []cortexpb.TimeSeries{ - {Labels: cortexpb.FromLabelsToLabelAdapters(series[1].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 110000}}}, - }, - }, - "should filter series by matcher and time range": { - from: 100000, - to: 100000, - matchers: []*client.LabelMatcher{ - {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, - }, - expected: []cortexpb.TimeSeries{ - {Labels: cortexpb.FromLabelsToLabelAdapters(series[0].lbls), Samples: []cortexpb.Sample{{Value: 1, TimestampMs: 100000}}}, - }, - }, - } - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push series - ctx := user.InjectOrgID(context.Background(), "test") - - for _, series := range series { - req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - - // Run tests - for testName, testData := range tests { - t.Run(testName, func(t *testing.T) { - req := &client.QueryRequest{ - StartTimestampMs: testData.from, - EndTimestampMs: testData.to, - Matchers: testData.matchers, - } - - res, err := i.v2Query(ctx, req) - require.NoError(t, err) - assert.ElementsMatch(t, testData.expected, res.Timeseries) - }) - } -} -func TestIngester_v2Query_ShouldNotCreateTSDBIfDoesNotExists(t *testing.T) { - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Mock request - userID := "test" - ctx := user.InjectOrgID(context.Background(), userID) - req := &client.QueryRequest{} - - res, err := i.v2Query(ctx, req) - require.NoError(t, err) - assert.Equal(t, &client.QueryResponse{}, res) - - // Check if the TSDB has been created - _, tsdbCreated := i.TSDBState.dbs[userID] - assert.False(t, tsdbCreated) -} - -func TestIngester_v2LabelValues_ShouldNotCreateTSDBIfDoesNotExists(t *testing.T) { - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Mock request - userID := "test" - ctx := user.InjectOrgID(context.Background(), userID) - req := &client.LabelValuesRequest{} - - res, err := i.v2LabelValues(ctx, req) - require.NoError(t, err) - assert.Equal(t, &client.LabelValuesResponse{}, res) - - // Check if the TSDB has been created - _, tsdbCreated := i.TSDBState.dbs[userID] - assert.False(t, tsdbCreated) -} - -func TestIngester_v2LabelNames_ShouldNotCreateTSDBIfDoesNotExists(t *testing.T) { - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Mock request - userID := "test" - ctx := user.InjectOrgID(context.Background(), userID) - req := &client.LabelNamesRequest{} - - res, err := i.v2LabelNames(ctx, req) - require.NoError(t, err) - assert.Equal(t, &client.LabelNamesResponse{}, res) - - // Check if the TSDB has been created - _, tsdbCreated := i.TSDBState.dbs[userID] - assert.False(t, tsdbCreated) -} - -func TestIngester_v2Push_ShouldNotCreateTSDBIfNotInActiveState(t *testing.T) { - // Configure the lifecycler to not immediately join the ring, to make sure - // the ingester will NOT be in the ACTIVE state when we'll push samples. - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 10 * time.Second - - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - require.Equal(t, ring.PENDING, i.lifecycler.GetState()) - - // Mock request - userID := "test" - ctx := user.InjectOrgID(context.Background(), userID) - req := &cortexpb.WriteRequest{} - - res, err := i.v2Push(ctx, req) - assert.Equal(t, wrapWithUser(fmt.Errorf(errTSDBCreateIncompatibleState, "PENDING"), userID).Error(), err.Error()) - assert.Nil(t, res) - - // Check if the TSDB has been created - _, tsdbCreated := i.TSDBState.dbs[userID] - assert.False(t, tsdbCreated) -} - -func TestIngester_getOrCreateTSDB_ShouldNotAllowToCreateTSDBIfIngesterStateIsNotActive(t *testing.T) { - tests := map[string]struct { - state ring.InstanceState - expectedErr error - }{ - "not allow to create TSDB if in PENDING state": { - state: ring.PENDING, - expectedErr: fmt.Errorf(errTSDBCreateIncompatibleState, ring.PENDING), - }, - "not allow to create TSDB if in JOINING state": { - state: ring.JOINING, - expectedErr: fmt.Errorf(errTSDBCreateIncompatibleState, ring.JOINING), - }, - "not allow to create TSDB if in LEAVING state": { - state: ring.LEAVING, - expectedErr: fmt.Errorf(errTSDBCreateIncompatibleState, ring.LEAVING), - }, - "allow to create TSDB if in ACTIVE state": { - state: ring.ACTIVE, - expectedErr: nil, - }, - } - - for testName, testData := range tests { - t.Run(testName, func(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 60 * time.Second - - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Switch ingester state to the expected one in the test - if i.lifecycler.GetState() != testData.state { - var stateChain []ring.InstanceState - - if testData.state == ring.LEAVING { - stateChain = []ring.InstanceState{ring.ACTIVE, ring.LEAVING} - } else { - stateChain = []ring.InstanceState{testData.state} - } - - for _, s := range stateChain { - err = i.lifecycler.ChangeState(context.Background(), s) - require.NoError(t, err) - } - } - - db, err := i.getOrCreateTSDB("test", false) - assert.Equal(t, testData.expectedErr, err) - - if testData.expectedErr != nil { - assert.Nil(t, db) - } else { - assert.NotNil(t, db) - } - }) - } -} - -func Test_Ingester_v2MetricsForLabelMatchers(t *testing.T) { - fixtures := []struct { - lbls labels.Labels - value float64 - timestamp int64 - }{ - {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "200"}}, 1, 100000}, - {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "500"}}, 1, 110000}, - {labels.Labels{{Name: labels.MetricName, Value: "test_2"}}, 2, 200000}, - // The two following series have the same FastFingerprint=e002a3a451262627 - {labels.Labels{{Name: labels.MetricName, Value: "collision"}, {Name: "app", Value: "l"}, {Name: "uniq0", Value: "0"}, {Name: "uniq1", Value: "1"}}, 1, 300000}, - {labels.Labels{{Name: labels.MetricName, Value: "collision"}, {Name: "app", Value: "m"}, {Name: "uniq0", Value: "1"}, {Name: "uniq1", Value: "1"}}, 1, 300000}, - } - - tests := map[string]struct { - from int64 - to int64 - matchers []*client.LabelMatchers - expected []*cortexpb.Metric - }{ - "should return an empty response if no metric match": { - from: math.MinInt64, - to: math.MaxInt64, - matchers: []*client.LabelMatchers{{ - Matchers: []*client.LabelMatcher{ - {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "unknown"}, - }, - }}, - expected: []*cortexpb.Metric{}, - }, - "should filter metrics by single matcher": { - from: math.MinInt64, - to: math.MaxInt64, - matchers: []*client.LabelMatchers{{ - Matchers: []*client.LabelMatcher{ - {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, - }, - }}, - expected: []*cortexpb.Metric{ - {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[0].lbls)}, - {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[1].lbls)}, - }, - }, - "should filter metrics by multiple matchers": { - from: math.MinInt64, - to: math.MaxInt64, - matchers: []*client.LabelMatchers{ - { - Matchers: []*client.LabelMatcher{ - {Type: client.EQUAL, Name: "status", Value: "200"}, - }, - }, - { - Matchers: []*client.LabelMatcher{ - {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_2"}, - }, - }, - }, - expected: []*cortexpb.Metric{ - {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[0].lbls)}, - {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[2].lbls)}, - }, - }, - "should NOT filter metrics by time range to always return known metrics even when queried for older time ranges": { - from: 100, - to: 1000, - matchers: []*client.LabelMatchers{{ - Matchers: []*client.LabelMatcher{ - {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, - }, - }}, - expected: []*cortexpb.Metric{ - {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[0].lbls)}, - {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[1].lbls)}, - }, - }, - "should not return duplicated metrics on overlapping matchers": { - from: math.MinInt64, - to: math.MaxInt64, - matchers: []*client.LabelMatchers{ - { - Matchers: []*client.LabelMatcher{ - {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "test_1"}, - }, - }, - { - Matchers: []*client.LabelMatcher{ - {Type: client.REGEX_MATCH, Name: model.MetricNameLabel, Value: "test.*"}, - }, - }, - }, - expected: []*cortexpb.Metric{ - {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[0].lbls)}, - {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[1].lbls)}, - {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[2].lbls)}, - }, - }, - "should return all matching metrics even if their FastFingerprint collide": { - from: math.MinInt64, - to: math.MaxInt64, - matchers: []*client.LabelMatchers{{ - Matchers: []*client.LabelMatcher{ - {Type: client.EQUAL, Name: model.MetricNameLabel, Value: "collision"}, - }, - }}, - expected: []*cortexpb.Metric{ - {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[3].lbls)}, - {Labels: cortexpb.FromLabelsToLabelAdapters(fixtures[4].lbls)}, - }, - }, - } - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push fixtures - ctx := user.InjectOrgID(context.Background(), "test") - - for _, series := range fixtures { - req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - - // Run tests - for testName, testData := range tests { - testData := testData - - t.Run(testName, func(t *testing.T) { - req := &client.MetricsForLabelMatchersRequest{ - StartTimestampMs: testData.from, - EndTimestampMs: testData.to, - MatchersSet: testData.matchers, - } - - res, err := i.v2MetricsForLabelMatchers(ctx, req) - require.NoError(t, err) - assert.ElementsMatch(t, testData.expected, res.Metric) - }) - } -} - -func Test_Ingester_v2MetricsForLabelMatchers_Deduplication(t *testing.T) { - const ( - userID = "test" - numSeries = 100000 - ) - - now := util.TimeToMillis(time.Now()) - i := createIngesterWithSeries(t, userID, numSeries, 1, now, 1) - ctx := user.InjectOrgID(context.Background(), "test") - - req := &client.MetricsForLabelMatchersRequest{ - StartTimestampMs: now, - EndTimestampMs: now, - // Overlapping matchers to make sure series are correctly deduplicated. - MatchersSet: []*client.LabelMatchers{ - {Matchers: []*client.LabelMatcher{ - {Type: client.REGEX_MATCH, Name: model.MetricNameLabel, Value: "test.*"}, - }}, - {Matchers: []*client.LabelMatcher{ - {Type: client.REGEX_MATCH, Name: model.MetricNameLabel, Value: "test.*0"}, - }}, - }, - } - - res, err := i.v2MetricsForLabelMatchers(ctx, req) - require.NoError(t, err) - require.Len(t, res.GetMetric(), numSeries) -} - -func Benchmark_Ingester_v2MetricsForLabelMatchers(b *testing.B) { - var ( - userID = "test" - numSeries = 10000 - numSamplesPerSeries = 60 * 6 // 6h on 1 sample per minute - startTimestamp = util.TimeToMillis(time.Now()) - step = int64(60000) // 1 sample per minute - ) - - i := createIngesterWithSeries(b, userID, numSeries, numSamplesPerSeries, startTimestamp, step) - ctx := user.InjectOrgID(context.Background(), "test") - - // Flush the ingester to ensure blocks have been compacted, so we'll test - // fetching labels from blocks. - i.Flush() - - b.ResetTimer() - b.ReportAllocs() - - for n := 0; n < b.N; n++ { - req := &client.MetricsForLabelMatchersRequest{ - StartTimestampMs: math.MinInt64, - EndTimestampMs: math.MaxInt64, - MatchersSet: []*client.LabelMatchers{{Matchers: []*client.LabelMatcher{ - {Type: client.REGEX_MATCH, Name: model.MetricNameLabel, Value: "test.*"}, - }}}, - } - - res, err := i.v2MetricsForLabelMatchers(ctx, req) - require.NoError(b, err) - require.Len(b, res.GetMetric(), numSeries) - } -} - -// createIngesterWithSeries creates an ingester and push numSeries with numSamplesPerSeries each. -func createIngesterWithSeries(t testing.TB, userID string, numSeries, numSamplesPerSeries int, startTimestamp, step int64) *Ingester { - const maxBatchSize = 1000 - - // Create ingester. - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - t.Cleanup(func() { - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) - }) - - // Wait until it's ACTIVE. - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push fixtures. - ctx := user.InjectOrgID(context.Background(), userID) - - for ts := startTimestamp; ts < startTimestamp+(step*int64(numSamplesPerSeries)); ts += step { - for o := 0; o < numSeries; o += maxBatchSize { - batchSize := util_math.Min(maxBatchSize, numSeries-o) - - // Generate metrics and samples (1 for each series). - metrics := make([]labels.Labels, 0, batchSize) - samples := make([]cortexpb.Sample, 0, batchSize) - - for s := 0; s < batchSize; s++ { - metrics = append(metrics, labels.Labels{ - {Name: labels.MetricName, Value: fmt.Sprintf("test_%d", o+s)}, - }) - - samples = append(samples, cortexpb.Sample{ - TimestampMs: ts, - Value: 1, - }) - } - - // Send metrics to the ingester. - req := cortexpb.ToWriteRequest(metrics, samples, nil, cortexpb.API) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - } - - return i -} - -func TestIngester_v2QueryStream(t *testing.T) { - // Create ingester. - cfg := defaultIngesterTestConfig(t) - - // change stream type in runtime. - var streamType QueryStreamType - cfg.StreamTypeFn = func() QueryStreamType { - return streamType - } - - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE. - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push series. - ctx := user.InjectOrgID(context.Background(), userID) - lbls := labels.Labels{{Name: labels.MetricName, Value: "foo"}} - req, _, expectedResponseSamples, expectedResponseChunks := mockWriteRequest(t, lbls, 123000, 456) - _, err = i.v2Push(ctx, req) - require.NoError(t, err) - - // Create a GRPC server used to query back the data. - serv := grpc.NewServer(grpc.StreamInterceptor(middleware.StreamServerUserHeaderInterceptor)) - defer serv.GracefulStop() - client.RegisterIngesterServer(serv, i) - - listener, err := net.Listen("tcp", "localhost:0") - require.NoError(t, err) - - go func() { - require.NoError(t, serv.Serve(listener)) - }() - - // Query back the series using GRPC streaming. - c, err := client.MakeIngesterClient(listener.Addr().String(), defaultClientTestConfig()) - require.NoError(t, err) - defer c.Close() - - queryRequest := &client.QueryRequest{ - StartTimestampMs: 0, - EndTimestampMs: 200000, - Matchers: []*client.LabelMatcher{{ - Type: client.EQUAL, - Name: model.MetricNameLabel, - Value: "foo", - }}, - } - - samplesTest := func(t *testing.T) { - s, err := c.QueryStream(ctx, queryRequest) - require.NoError(t, err) - - count := 0 - var lastResp *client.QueryStreamResponse - for { - resp, err := s.Recv() - if err == io.EOF { - break - } - require.NoError(t, err) - require.Zero(t, len(resp.Chunkseries)) // No chunks expected - count += len(resp.Timeseries) - lastResp = resp - } - require.Equal(t, 1, count) - require.Equal(t, expectedResponseSamples, lastResp) - } - - chunksTest := func(t *testing.T) { - s, err := c.QueryStream(ctx, queryRequest) - require.NoError(t, err) - - count := 0 - var lastResp *client.QueryStreamResponse - for { - resp, err := s.Recv() - if err == io.EOF { - break - } - require.NoError(t, err) - require.Zero(t, len(resp.Timeseries)) // No samples expected - count += len(resp.Chunkseries) - lastResp = resp - } - require.Equal(t, 1, count) - require.Equal(t, expectedResponseChunks, lastResp) - } - - streamType = QueryStreamDefault - t.Run("default", samplesTest) - - streamType = QueryStreamSamples - t.Run("samples", samplesTest) - - streamType = QueryStreamChunks - t.Run("chunks", chunksTest) -} - -func TestIngester_v2QueryStreamManySamples(t *testing.T) { - // Create ingester. - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE. - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push series. - ctx := user.InjectOrgID(context.Background(), userID) - - const samplesCount = 100000 - samples := make([]cortexpb.Sample, 0, samplesCount) - - for i := 0; i < samplesCount; i++ { - samples = append(samples, cortexpb.Sample{ - Value: float64(i), - TimestampMs: int64(i), - }) - } - - // 10k samples encode to around 140 KiB, - _, err = i.v2Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "1"}}, samples[0:10000])) - require.NoError(t, err) - - // 100k samples encode to around 1.4 MiB, - _, err = i.v2Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "2"}}, samples)) - require.NoError(t, err) - - // 50k samples encode to around 716 KiB, - _, err = i.v2Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "3"}}, samples[0:50000])) - require.NoError(t, err) - - // Create a GRPC server used to query back the data. - serv := grpc.NewServer(grpc.StreamInterceptor(middleware.StreamServerUserHeaderInterceptor)) - defer serv.GracefulStop() - client.RegisterIngesterServer(serv, i) - - listener, err := net.Listen("tcp", "localhost:0") - require.NoError(t, err) - - go func() { - require.NoError(t, serv.Serve(listener)) - }() - - // Query back the series using GRPC streaming. - c, err := client.MakeIngesterClient(listener.Addr().String(), defaultClientTestConfig()) - require.NoError(t, err) - defer c.Close() - - s, err := c.QueryStream(ctx, &client.QueryRequest{ - StartTimestampMs: 0, - EndTimestampMs: samplesCount + 1, - - Matchers: []*client.LabelMatcher{{ - Type: client.EQUAL, - Name: model.MetricNameLabel, - Value: "foo", - }}, - }) - require.NoError(t, err) - - recvMsgs := 0 - series := 0 - totalSamples := 0 - - for { - resp, err := s.Recv() - if err == io.EOF { - break - } - require.NoError(t, err) - require.True(t, len(resp.Timeseries) > 0) // No empty messages. - - recvMsgs++ - series += len(resp.Timeseries) - - for _, ts := range resp.Timeseries { - totalSamples += len(ts.Samples) - } - } - - // As ingester doesn't guarantee sorting of series, we can get 2 (10k + 50k in first, 100k in second) - // or 3 messages (small series first, 100k second, small series last). - - require.True(t, 2 <= recvMsgs && recvMsgs <= 3) - require.Equal(t, 3, series) - require.Equal(t, 10000+50000+samplesCount, totalSamples) -} - -func TestIngester_v2QueryStreamManySamplesChunks(t *testing.T) { - // Create ingester. - cfg := defaultIngesterTestConfig(t) - cfg.StreamChunksWhenUsingBlocks = true - - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE. - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push series. - ctx := user.InjectOrgID(context.Background(), userID) - - const samplesCount = 1000000 - samples := make([]cortexpb.Sample, 0, samplesCount) - - for i := 0; i < samplesCount; i++ { - samples = append(samples, cortexpb.Sample{ - Value: float64(i), - TimestampMs: int64(i), - }) - } - - // 100k samples in chunks use about 154 KiB, - _, err = i.v2Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "1"}}, samples[0:100000])) - require.NoError(t, err) - - // 1M samples in chunks use about 1.51 MiB, - _, err = i.v2Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "2"}}, samples)) - require.NoError(t, err) - - // 500k samples in chunks need 775 KiB, - _, err = i.v2Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: "3"}}, samples[0:500000])) - require.NoError(t, err) - - // Create a GRPC server used to query back the data. - serv := grpc.NewServer(grpc.StreamInterceptor(middleware.StreamServerUserHeaderInterceptor)) - defer serv.GracefulStop() - client.RegisterIngesterServer(serv, i) - - listener, err := net.Listen("tcp", "localhost:0") - require.NoError(t, err) - - go func() { - require.NoError(t, serv.Serve(listener)) - }() - - // Query back the series using GRPC streaming. - c, err := client.MakeIngesterClient(listener.Addr().String(), defaultClientTestConfig()) - require.NoError(t, err) - defer c.Close() - - s, err := c.QueryStream(ctx, &client.QueryRequest{ - StartTimestampMs: 0, - EndTimestampMs: samplesCount + 1, - - Matchers: []*client.LabelMatcher{{ - Type: client.EQUAL, - Name: model.MetricNameLabel, - Value: "foo", - }}, - }) - require.NoError(t, err) - - recvMsgs := 0 - series := 0 - totalSamples := 0 - - for { - resp, err := s.Recv() - if err == io.EOF { - break - } - require.NoError(t, err) - require.True(t, len(resp.Chunkseries) > 0) // No empty messages. - - recvMsgs++ - series += len(resp.Chunkseries) - - for _, ts := range resp.Chunkseries { - for _, c := range ts.Chunks { - ch, err := encoding.NewForEncoding(encoding.Encoding(c.Encoding)) - require.NoError(t, err) - require.NoError(t, ch.UnmarshalFromBuf(c.Data)) - - totalSamples += ch.Len() - } - } - } - - // As ingester doesn't guarantee sorting of series, we can get 2 (100k + 500k in first, 1M in second) - // or 3 messages (100k or 500k first, 1M second, and 500k or 100k last). - - require.True(t, 2 <= recvMsgs && recvMsgs <= 3) - require.Equal(t, 3, series) - require.Equal(t, 100000+500000+samplesCount, totalSamples) -} - -func writeRequestSingleSeries(lbls labels.Labels, samples []cortexpb.Sample) *cortexpb.WriteRequest { - req := &cortexpb.WriteRequest{ - Source: cortexpb.API, - } - - ts := cortexpb.TimeSeries{} - ts.Labels = cortexpb.FromLabelsToLabelAdapters(lbls) - ts.Samples = samples - req.Timeseries = append(req.Timeseries, cortexpb.PreallocTimeseries{TimeSeries: &ts}) - - return req -} - -type mockQueryStreamServer struct { - grpc.ServerStream - ctx context.Context -} - -func (m *mockQueryStreamServer) Send(response *client.QueryStreamResponse) error { - return nil -} - -func (m *mockQueryStreamServer) Context() context.Context { - return m.ctx -} - -func BenchmarkIngester_v2QueryStream_Samples(b *testing.B) { - benchmarkV2QueryStream(b, false) -} - -func BenchmarkIngester_v2QueryStream_Chunks(b *testing.B) { - benchmarkV2QueryStream(b, true) -} - -func benchmarkV2QueryStream(b *testing.B, streamChunks bool) { - cfg := defaultIngesterTestConfig(b) - cfg.StreamChunksWhenUsingBlocks = streamChunks - - // Create ingester. - i, err := prepareIngesterWithBlocksStorage(b, cfg, nil) - require.NoError(b, err) - require.NoError(b, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE. - test.Poll(b, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push series. - ctx := user.InjectOrgID(context.Background(), userID) - - const samplesCount = 1000 - samples := make([]cortexpb.Sample, 0, samplesCount) - - for i := 0; i < samplesCount; i++ { - samples = append(samples, cortexpb.Sample{ - Value: float64(i), - TimestampMs: int64(i), - }) - } - - const seriesCount = 100 - for s := 0; s < seriesCount; s++ { - _, err = i.v2Push(ctx, writeRequestSingleSeries(labels.Labels{{Name: labels.MetricName, Value: "foo"}, {Name: "l", Value: strconv.Itoa(s)}}, samples)) - require.NoError(b, err) - } - - req := &client.QueryRequest{ - StartTimestampMs: 0, - EndTimestampMs: samplesCount + 1, - - Matchers: []*client.LabelMatcher{{ - Type: client.EQUAL, - Name: model.MetricNameLabel, - Value: "foo", - }}, - } - - mockStream := &mockQueryStreamServer{ctx: ctx} - - b.ResetTimer() - - for ix := 0; ix < b.N; ix++ { - err := i.v2QueryStream(req, mockStream) - require.NoError(b, err) - } -} - -func mockWriteRequest(t *testing.T, lbls labels.Labels, value float64, timestampMs int64) (*cortexpb.WriteRequest, *client.QueryResponse, *client.QueryStreamResponse, *client.QueryStreamResponse) { - samples := []cortexpb.Sample{ - { - TimestampMs: timestampMs, - Value: value, - }, - } - - req := cortexpb.ToWriteRequest([]labels.Labels{lbls}, samples, nil, cortexpb.API) - - // Generate the expected response - expectedQueryRes := &client.QueryResponse{ - Timeseries: []cortexpb.TimeSeries{ - { - Labels: cortexpb.FromLabelsToLabelAdapters(lbls), - Samples: samples, - }, - }, - } - - expectedQueryStreamResSamples := &client.QueryStreamResponse{ - Timeseries: []cortexpb.TimeSeries{ - { - Labels: cortexpb.FromLabelsToLabelAdapters(lbls), - Samples: samples, - }, - }, - } - - chunk := chunkenc.NewXORChunk() - app, err := chunk.Appender() - require.NoError(t, err) - app.Append(timestampMs, value) - chunk.Compact() - - expectedQueryStreamResChunks := &client.QueryStreamResponse{ - Chunkseries: []client.TimeSeriesChunk{ - { - Labels: cortexpb.FromLabelsToLabelAdapters(lbls), - Chunks: []client.Chunk{ - { - StartTimestampMs: timestampMs, - EndTimestampMs: timestampMs, - Encoding: int32(encoding.PrometheusXorChunk), - Data: chunk.Bytes(), - }, - }, - }, - }, - } - - return req, expectedQueryRes, expectedQueryStreamResSamples, expectedQueryStreamResChunks -} - -func prepareIngesterWithBlocksStorage(t testing.TB, ingesterCfg Config, registerer prometheus.Registerer) (*Ingester, error) { - return prepareIngesterWithBlocksStorageAndLimits(t, ingesterCfg, defaultLimitsTestConfig(), "", registerer) -} - -func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, limits validation.Limits, dataDir string, registerer prometheus.Registerer) (*Ingester, error) { - // Create a data dir if none has been provided. - if dataDir == "" { - dataDir = t.TempDir() - } - - bucketDir := t.TempDir() - - overrides, err := validation.NewOverrides(limits, nil) - if err != nil { - return nil, err - } - - ingesterCfg.BlocksStorageConfig.TSDB.Dir = dataDir - ingesterCfg.BlocksStorageConfig.Bucket.Backend = "filesystem" - ingesterCfg.BlocksStorageConfig.Bucket.Filesystem.Directory = bucketDir - - ingester, err := NewV2(ingesterCfg, overrides, registerer, log.NewNopLogger()) - if err != nil { - return nil, err - } - - return ingester, nil -} - -func TestIngester_v2OpenExistingTSDBOnStartup(t *testing.T) { - t.Parallel() - - tests := map[string]struct { - concurrency int - setup func(*testing.T, string) - check func(*testing.T, *Ingester) - expectedErr string - }{ - "should not load TSDB if the user directory is empty": { - concurrency: 10, - setup: func(t *testing.T, dir string) { - require.NoError(t, os.Mkdir(filepath.Join(dir, "user0"), 0700)) - }, - check: func(t *testing.T, i *Ingester) { - require.Nil(t, i.getTSDB("user0")) - }, - }, - "should not load any TSDB if the root directory is empty": { - concurrency: 10, - setup: func(t *testing.T, dir string) {}, - check: func(t *testing.T, i *Ingester) { - require.Zero(t, len(i.TSDBState.dbs)) - }, - }, - "should not load any TSDB is the root directory is missing": { - concurrency: 10, - setup: func(t *testing.T, dir string) { - require.NoError(t, os.Remove(dir)) - }, - check: func(t *testing.T, i *Ingester) { - require.Zero(t, len(i.TSDBState.dbs)) - }, - }, - "should load TSDB for any non-empty user directory": { - concurrency: 10, - setup: func(t *testing.T, dir string) { - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "dummy"), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1", "dummy"), 0700)) - require.NoError(t, os.Mkdir(filepath.Join(dir, "user2"), 0700)) - }, - check: func(t *testing.T, i *Ingester) { - require.Equal(t, 2, len(i.TSDBState.dbs)) - require.NotNil(t, i.getTSDB("user0")) - require.NotNil(t, i.getTSDB("user1")) - require.Nil(t, i.getTSDB("user2")) - }, - }, - "should load all TSDBs on concurrency < number of TSDBs": { - concurrency: 2, - setup: func(t *testing.T, dir string) { - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "dummy"), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1", "dummy"), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user2", "dummy"), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user3", "dummy"), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user4", "dummy"), 0700)) - }, - check: func(t *testing.T, i *Ingester) { - require.Equal(t, 5, len(i.TSDBState.dbs)) - require.NotNil(t, i.getTSDB("user0")) - require.NotNil(t, i.getTSDB("user1")) - require.NotNil(t, i.getTSDB("user2")) - require.NotNil(t, i.getTSDB("user3")) - require.NotNil(t, i.getTSDB("user4")) - }, - }, - "should fail and rollback if an error occur while loading a TSDB on concurrency > number of TSDBs": { - concurrency: 10, - setup: func(t *testing.T, dir string) { - // Create a fake TSDB on disk with an empty chunks head segment file (it's invalid unless - // it's the last one and opening TSDB should fail). - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "wal", ""), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "chunks_head", ""), 0700)) - require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user0", "chunks_head", "00000001"), nil, 0700)) - require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user0", "chunks_head", "00000002"), nil, 0700)) - - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1", "dummy"), 0700)) - }, - check: func(t *testing.T, i *Ingester) { - require.Equal(t, 0, len(i.TSDBState.dbs)) - require.Nil(t, i.getTSDB("user0")) - require.Nil(t, i.getTSDB("user1")) - }, - expectedErr: "unable to open TSDB for user user0", - }, - "should fail and rollback if an error occur while loading a TSDB on concurrency < number of TSDBs": { - concurrency: 2, - setup: func(t *testing.T, dir string) { - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user0", "dummy"), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user1", "dummy"), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user3", "dummy"), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user4", "dummy"), 0700)) - - // Create a fake TSDB on disk with an empty chunks head segment file (it's invalid unless - // it's the last one and opening TSDB should fail). - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user2", "wal", ""), 0700)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "user2", "chunks_head", ""), 0700)) - require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user2", "chunks_head", "00000001"), nil, 0700)) - require.NoError(t, ioutil.WriteFile(filepath.Join(dir, "user2", "chunks_head", "00000002"), nil, 0700)) - }, - check: func(t *testing.T, i *Ingester) { - require.Equal(t, 0, len(i.TSDBState.dbs)) - require.Nil(t, i.getTSDB("user0")) - require.Nil(t, i.getTSDB("user1")) - require.Nil(t, i.getTSDB("user2")) - require.Nil(t, i.getTSDB("user3")) - require.Nil(t, i.getTSDB("user4")) - }, - expectedErr: "unable to open TSDB for user user2", - }, - } - - for name, test := range tests { - testName := name - testData := test - t.Run(testName, func(t *testing.T) { - limits := defaultLimitsTestConfig() - - overrides, err := validation.NewOverrides(limits, nil) - require.NoError(t, err) - - // Create a temporary directory for TSDB - tempDir := t.TempDir() - - ingesterCfg := defaultIngesterTestConfig(t) - ingesterCfg.BlocksStorageConfig.TSDB.Dir = tempDir - ingesterCfg.BlocksStorageConfig.TSDB.MaxTSDBOpeningConcurrencyOnStartup = testData.concurrency - ingesterCfg.BlocksStorageConfig.Bucket.Backend = "s3" - ingesterCfg.BlocksStorageConfig.Bucket.S3.Endpoint = "localhost" - - // setup the tsdbs dir - testData.setup(t, tempDir) - - ingester, err := NewV2(ingesterCfg, overrides, nil, log.NewNopLogger()) - require.NoError(t, err) - - startErr := services.StartAndAwaitRunning(context.Background(), ingester) - if testData.expectedErr == "" { - require.NoError(t, startErr) - } else { - require.Error(t, startErr) - assert.Contains(t, startErr.Error(), testData.expectedErr) - } - - defer services.StopAndAwaitTerminated(context.Background(), ingester) //nolint:errcheck - testData.check(t, ingester) - }) - } -} - -func TestIngester_shipBlocks(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 2 - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Create the TSDB for 3 users and then replace the shipper with the mocked one - mocks := []*shipperMock{} - for _, userID := range []string{"user-1", "user-2", "user-3"} { - userDB, err := i.getOrCreateTSDB(userID, false) - require.NoError(t, err) - require.NotNil(t, userDB) - - m := &shipperMock{} - m.On("Sync", mock.Anything).Return(0, nil) - mocks = append(mocks, m) - - userDB.shipper = m - } - - // Ship blocks and assert on the mocked shipper - i.shipBlocks(context.Background(), nil) - - for _, m := range mocks { - m.AssertNumberOfCalls(t, "Sync", 1) - } -} - -func TestIngester_dontShipBlocksWhenTenantDeletionMarkerIsPresent(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 2 - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - - // Use in-memory bucket. - bucket := objstore.NewInMemBucket() - - i.TSDBState.bucket = bucket - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - pushSingleSampleWithMetadata(t, i) - require.Equal(t, int64(1), i.TSDBState.seriesCount.Load()) - i.compactBlocks(context.Background(), true, nil) - require.Equal(t, int64(0), i.TSDBState.seriesCount.Load()) - i.shipBlocks(context.Background(), nil) - - numObjects := len(bucket.Objects()) - require.NotZero(t, numObjects) - - require.NoError(t, cortex_tsdb.WriteTenantDeletionMark(context.Background(), bucket, userID, nil, cortex_tsdb.NewTenantDeletionMark(time.Now()))) - numObjects++ // For deletion marker - - db := i.getTSDB(userID) - require.NotNil(t, db) - db.lastDeletionMarkCheck.Store(0) - - // After writing tenant deletion mark, - pushSingleSampleWithMetadata(t, i) - require.Equal(t, int64(1), i.TSDBState.seriesCount.Load()) - i.compactBlocks(context.Background(), true, nil) - require.Equal(t, int64(0), i.TSDBState.seriesCount.Load()) - i.shipBlocks(context.Background(), nil) - - numObjectsAfterMarkingTenantForDeletion := len(bucket.Objects()) - require.Equal(t, numObjects, numObjectsAfterMarkingTenantForDeletion) - require.Equal(t, tsdbTenantMarkedForDeletion, i.closeAndDeleteUserTSDBIfIdle(userID)) -} - -func TestIngester_seriesCountIsCorrectAfterClosingTSDBForDeletedTenant(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 2 - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - - // Use in-memory bucket. - bucket := objstore.NewInMemBucket() - - // Write tenant deletion mark. - require.NoError(t, cortex_tsdb.WriteTenantDeletionMark(context.Background(), bucket, userID, nil, cortex_tsdb.NewTenantDeletionMark(time.Now()))) - - i.TSDBState.bucket = bucket - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - pushSingleSampleWithMetadata(t, i) - require.Equal(t, int64(1), i.TSDBState.seriesCount.Load()) - - // We call shipBlocks to check for deletion marker (it happens inside this method). - i.shipBlocks(context.Background(), nil) - - // Verify that tenant deletion mark was found. - db := i.getTSDB(userID) - require.NotNil(t, db) - require.True(t, db.deletionMarkFound.Load()) - - // If we try to close TSDB now, it should succeed, even though TSDB is not idle and empty. - require.Equal(t, uint64(1), db.Head().NumSeries()) - require.Equal(t, tsdbTenantMarkedForDeletion, i.closeAndDeleteUserTSDBIfIdle(userID)) - - // Closing should decrease series count. - require.Equal(t, int64(0), i.TSDBState.seriesCount.Load()) -} - -func TestIngester_sholdUpdateCacheShippedBlocks(t *testing.T) { - ctx := context.Background() - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 2 - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(ctx, i)) - defer services.StopAndAwaitTerminated(ctx, i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - mockUserShipper(t, i) - - // Mock the shipper meta (no blocks). - db := i.getTSDB(userID) - err = db.updateCachedShippedBlocks() - require.NoError(t, err) - - require.Equal(t, len(db.getCachedShippedBlocks()), 0) - shippedBlock, _ := ulid.Parse("01D78XZ44G0000000000000000") - - require.NoError(t, shipper.WriteMetaFile(log.NewNopLogger(), db.db.Dir(), &shipper.Meta{ - Version: shipper.MetaVersion1, - Uploaded: []ulid.ULID{shippedBlock}, - })) - - err = db.updateCachedShippedBlocks() - require.NoError(t, err) - - require.Equal(t, len(db.getCachedShippedBlocks()), 1) -} - -func TestIngester_closeAndDeleteUserTSDBIfIdle_shouldNotCloseTSDBIfShippingIsInProgress(t *testing.T) { - ctx := context.Background() - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 2 - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(ctx, i)) - defer services.StopAndAwaitTerminated(ctx, i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Mock the shipper to slow down Sync() execution. - s := mockUserShipper(t, i) - s.On("Sync", mock.Anything).Run(func(args mock.Arguments) { - time.Sleep(3 * time.Second) - }).Return(0, nil) - - // Mock the shipper meta (no blocks). - db := i.getTSDB(userID) - require.NoError(t, shipper.WriteMetaFile(log.NewNopLogger(), db.db.Dir(), &shipper.Meta{ - Version: shipper.MetaVersion1, - })) - - // Run blocks shipping in a separate go routine. - go i.shipBlocks(ctx, nil) - - // Wait until shipping starts. - test.Poll(t, 1*time.Second, activeShipping, func() interface{} { - db.stateMtx.RLock() - defer db.stateMtx.RUnlock() - return db.state - }) - assert.Equal(t, tsdbNotActive, i.closeAndDeleteUserTSDBIfIdle(userID)) -} - -func TestIngester_closingAndOpeningTsdbConcurrently(t *testing.T) { - ctx := context.Background() - cfg := defaultIngesterTestConfig(t) - cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast. - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(ctx, i)) - defer services.StopAndAwaitTerminated(ctx, i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - _, err = i.getOrCreateTSDB(userID, false) - require.NoError(t, err) - - iterations := 5000 - chanErr := make(chan error, 1) - quit := make(chan bool) - - go func() { - for { - select { - case <-quit: - return - default: - _, err = i.getOrCreateTSDB(userID, false) - if err != nil { - chanErr <- err - } - } - } - }() - - for k := 0; k < iterations; k++ { - i.closeAndDeleteUserTSDBIfIdle(userID) - } - - select { - case err := <-chanErr: - assert.Fail(t, err.Error()) - quit <- true - default: - quit <- true - } -} - -func TestIngester_idleCloseEmptyTSDB(t *testing.T) { - ctx := context.Background() - cfg := defaultIngesterTestConfig(t) - cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Minute - cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Minute - cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 0 // Will not run the loop, but will allow us to close any TSDB fast. - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(ctx, i)) - defer services.StopAndAwaitTerminated(ctx, i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - db, err := i.getOrCreateTSDB(userID, true) - require.NoError(t, err) - require.NotNil(t, db) - - // Run compaction and shipping. - i.compactBlocks(context.Background(), true, nil) - i.shipBlocks(context.Background(), nil) - - // Make sure we can close completely empty TSDB without problems. - require.Equal(t, tsdbIdleClosed, i.closeAndDeleteUserTSDBIfIdle(userID)) - - // Verify that it was closed. - db = i.getTSDB(userID) - require.Nil(t, db) - - // And we can recreate it again, if needed. - db, err = i.getOrCreateTSDB(userID, true) - require.NoError(t, err) - require.NotNil(t, db) -} - -type shipperMock struct { - mock.Mock -} - -// Sync mocks Shipper.Sync() -func (m *shipperMock) Sync(ctx context.Context) (uploaded int, err error) { - args := m.Called(ctx) - return args.Int(0), args.Error(1) -} - -func TestIngester_invalidSamplesDontChangeLastUpdateTime(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - ctx := user.InjectOrgID(context.Background(), userID) - sampleTimestamp := int64(model.Now()) - - { - req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, sampleTimestamp) - _, err = i.v2Push(ctx, req) - require.NoError(t, err) - } - - db := i.getTSDB(userID) - lastUpdate := db.lastUpdate.Load() - - // Wait until 1 second passes. - test.Poll(t, 1*time.Second, time.Now().Unix()+1, func() interface{} { - return time.Now().Unix() - }) - - // Push another sample to the same metric and timestamp, with different value. We expect to get error. - { - req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 1, sampleTimestamp) - _, err = i.v2Push(ctx, req) - require.Error(t, err) - } - - // Make sure last update hasn't changed. - require.Equal(t, lastUpdate, db.lastUpdate.Load()) -} - -func TestIngester_flushing(t *testing.T) { - for name, tc := range map[string]struct { - setupIngester func(cfg *Config) - action func(t *testing.T, i *Ingester, reg *prometheus.Registry) - }{ - "ingesterShutdown": { - setupIngester: func(cfg *Config) { - cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown = true - cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown = true - }, - action: func(t *testing.T, i *Ingester, reg *prometheus.Registry) { - pushSingleSampleWithMetadata(t, i) - - // Nothing shipped yet. - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 0 - `), "cortex_ingester_shipper_uploads_total")) - - // Shutdown ingester. This triggers flushing of the block. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) - - verifyCompactedHead(t, i, true) - - // Verify that block has been shipped. - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 1 - `), "cortex_ingester_shipper_uploads_total")) - }, - }, - - "shutdownHandler": { - setupIngester: func(cfg *Config) { - cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown = false - cfg.BlocksStorageConfig.TSDB.KeepUserTSDBOpenOnShutdown = true - }, - - action: func(t *testing.T, i *Ingester, reg *prometheus.Registry) { - pushSingleSampleWithMetadata(t, i) - - // Nothing shipped yet. - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 0 - `), "cortex_ingester_shipper_uploads_total")) - - i.ShutdownHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/shutdown", nil)) - - verifyCompactedHead(t, i, true) - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 1 - `), "cortex_ingester_shipper_uploads_total")) - }, - }, - - "flushHandler": { - setupIngester: func(cfg *Config) { - cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown = false - }, - - action: func(t *testing.T, i *Ingester, reg *prometheus.Registry) { - pushSingleSampleWithMetadata(t, i) - - // Nothing shipped yet. - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 0 - `), "cortex_ingester_shipper_uploads_total")) - - // Using wait=true makes this a synchronous call. - i.FlushHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/flush?wait=true", nil)) - - verifyCompactedHead(t, i, true) - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 1 - `), "cortex_ingester_shipper_uploads_total")) - }, - }, - - "flushHandlerWithListOfTenants": { - setupIngester: func(cfg *Config) { - cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown = false - }, - - action: func(t *testing.T, i *Ingester, reg *prometheus.Registry) { - pushSingleSampleWithMetadata(t, i) - - // Nothing shipped yet. - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 0 - `), "cortex_ingester_shipper_uploads_total")) - - users := url.Values{} - users.Add(tenantParam, "unknown-user") - users.Add(tenantParam, "another-unknown-user") - - // Using wait=true makes this a synchronous call. - i.FlushHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/flush?wait=true&"+users.Encode(), nil)) - - // Still nothing shipped or compacted. - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 0 - `), "cortex_ingester_shipper_uploads_total")) - verifyCompactedHead(t, i, false) - - users = url.Values{} - users.Add(tenantParam, "different-user") - users.Add(tenantParam, userID) // Our user - users.Add(tenantParam, "yet-another-user") - - i.FlushHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/flush?wait=true&"+users.Encode(), nil)) - - verifyCompactedHead(t, i, true) - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 1 - `), "cortex_ingester_shipper_uploads_total")) - }, - }, - - "flushMultipleBlocksWithDataSpanning3Days": { - setupIngester: func(cfg *Config) { - cfg.BlocksStorageConfig.TSDB.FlushBlocksOnShutdown = false - }, - - action: func(t *testing.T, i *Ingester, reg *prometheus.Registry) { - // Pushing 5 samples, spanning over 3 days. - // First block - pushSingleSampleAtTime(t, i, 23*time.Hour.Milliseconds()) - pushSingleSampleAtTime(t, i, 24*time.Hour.Milliseconds()-1) - - // Second block - pushSingleSampleAtTime(t, i, 24*time.Hour.Milliseconds()+1) - pushSingleSampleAtTime(t, i, 25*time.Hour.Milliseconds()) - - // Third block, far in the future. - pushSingleSampleAtTime(t, i, 50*time.Hour.Milliseconds()) - - // Nothing shipped yet. - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 0 - `), "cortex_ingester_shipper_uploads_total")) - - i.FlushHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/flush?wait=true", nil)) - - verifyCompactedHead(t, i, true) - - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 3 - `), "cortex_ingester_shipper_uploads_total")) - - userDB := i.getTSDB(userID) - require.NotNil(t, userDB) - - blocks := userDB.Blocks() - require.Equal(t, 3, len(blocks)) - require.Equal(t, 23*time.Hour.Milliseconds(), blocks[0].Meta().MinTime) - require.Equal(t, 24*time.Hour.Milliseconds(), blocks[0].Meta().MaxTime) // Block maxt is exclusive. - - require.Equal(t, 24*time.Hour.Milliseconds()+1, blocks[1].Meta().MinTime) - require.Equal(t, 26*time.Hour.Milliseconds(), blocks[1].Meta().MaxTime) - - require.Equal(t, 50*time.Hour.Milliseconds()+1, blocks[2].Meta().MaxTime) // Block maxt is exclusive. - }, - }, - } { - t.Run(name, func(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1 - cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Minute // Long enough to not be reached during the test. - - if tc.setupIngester != nil { - tc.setupIngester(&cfg) - } - - // Create ingester - reg := prometheus.NewPedanticRegistry() - i, err := prepareIngesterWithBlocksStorage(t, cfg, reg) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - t.Cleanup(func() { - _ = services.StopAndAwaitTerminated(context.Background(), i) - }) - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // mock user's shipper - tc.action(t, i, reg) - }) - } -} - -func TestIngester_ForFlush(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1 - cfg.BlocksStorageConfig.TSDB.ShipInterval = 10 * time.Minute // Long enough to not be reached during the test. - - // Create ingester - reg := prometheus.NewPedanticRegistry() - i, err := prepareIngesterWithBlocksStorage(t, cfg, reg) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - t.Cleanup(func() { - _ = services.StopAndAwaitTerminated(context.Background(), i) - }) - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push some data. - pushSingleSampleWithMetadata(t, i) - - // Stop ingester. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) - - // Nothing shipped yet. - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 0 - `), "cortex_ingester_shipper_uploads_total")) - - // Restart ingester in "For Flusher" mode. We reuse the same config (esp. same dir) - reg = prometheus.NewPedanticRegistry() - i, err = NewV2ForFlusher(i.cfg, i.limits, reg, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - - // Our single sample should be reloaded from WAL - verifyCompactedHead(t, i, false) - i.Flush() - - // Head should be empty after flushing. - verifyCompactedHead(t, i, true) - - // Verify that block has been shipped. - require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(` - # HELP cortex_ingester_shipper_uploads_total Total number of uploaded TSDB blocks - # TYPE cortex_ingester_shipper_uploads_total counter - cortex_ingester_shipper_uploads_total 1 - `), "cortex_ingester_shipper_uploads_total")) - - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) -} - -func mockUserShipper(t *testing.T, i *Ingester) *shipperMock { - m := &shipperMock{} - userDB, err := i.getOrCreateTSDB(userID, false) - require.NoError(t, err) - require.NotNil(t, userDB) - - userDB.shipper = m - return m -} - -func Test_Ingester_v2UserStats(t *testing.T) { - series := []struct { - lbls labels.Labels - value float64 - timestamp int64 - }{ - {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "200"}, {Name: "route", Value: "get_user"}}, 1, 100000}, - {labels.Labels{{Name: labels.MetricName, Value: "test_1"}, {Name: "status", Value: "500"}, {Name: "route", Value: "get_user"}}, 1, 110000}, - {labels.Labels{{Name: labels.MetricName, Value: "test_2"}}, 2, 200000}, - } - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push series - ctx := user.InjectOrgID(context.Background(), "test") - - for _, series := range series { - req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - - // force update statistics - for _, db := range i.TSDBState.dbs { - db.ingestedAPISamples.Tick() - db.ingestedRuleSamples.Tick() - } - - // Get label names - res, err := i.v2UserStats(ctx, &client.UserStatsRequest{}) - require.NoError(t, err) - assert.InDelta(t, 0.2, res.ApiIngestionRate, 0.0001) - assert.InDelta(t, float64(0), res.RuleIngestionRate, 0.0001) - assert.Equal(t, uint64(3), res.NumSeries) -} - -func Test_Ingester_v2AllUserStats(t *testing.T) { - series := []struct { - user string - lbls labels.Labels - value float64 - timestamp int64 - }{ - {"user-1", labels.Labels{{Name: labels.MetricName, Value: "test_1_1"}, {Name: "status", Value: "200"}, {Name: "route", Value: "get_user"}}, 1, 100000}, - {"user-1", labels.Labels{{Name: labels.MetricName, Value: "test_1_1"}, {Name: "status", Value: "500"}, {Name: "route", Value: "get_user"}}, 1, 110000}, - {"user-1", labels.Labels{{Name: labels.MetricName, Value: "test_1_2"}}, 2, 200000}, - {"user-2", labels.Labels{{Name: labels.MetricName, Value: "test_2_1"}}, 2, 200000}, - {"user-2", labels.Labels{{Name: labels.MetricName, Value: "test_2_2"}}, 2, 200000}, - } - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - for _, series := range series { - ctx := user.InjectOrgID(context.Background(), series.user) - req, _, _, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - - // force update statistics - for _, db := range i.TSDBState.dbs { - db.ingestedAPISamples.Tick() - db.ingestedRuleSamples.Tick() - } - - // Get label names - res, err := i.v2AllUserStats(context.Background(), &client.UserStatsRequest{}) - require.NoError(t, err) - - expect := []*client.UserIDStatsResponse{ - { - UserId: "user-1", - Data: &client.UserStatsResponse{ - IngestionRate: 0.2, - NumSeries: 3, - ApiIngestionRate: 0.2, - RuleIngestionRate: 0, - }, - }, - { - UserId: "user-2", - Data: &client.UserStatsResponse{ - IngestionRate: 0.13333333333333333, - NumSeries: 2, - ApiIngestionRate: 0.13333333333333333, - RuleIngestionRate: 0, - }, - }, - } - assert.ElementsMatch(t, expect, res.Stats) -} - -func TestIngesterCompactIdleBlock(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1 - cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Hour // Long enough to not be reached during the test. - cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout = 1 * time.Second // Testing this. - - r := prometheus.NewRegistry() - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, r) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - t.Cleanup(func() { - _ = services.StopAndAwaitTerminated(context.Background(), i) - }) - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - pushSingleSampleWithMetadata(t, i) - - i.compactBlocks(context.Background(), false, nil) - verifyCompactedHead(t, i, false) - require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="1"} 1 - - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="1"} 0 - - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - `), memSeriesCreatedTotalName, memSeriesRemovedTotalName, "cortex_ingester_memory_users")) - - // wait one second (plus maximum jitter) -- TSDB is now idle. - time.Sleep(time.Duration(float64(cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout) * (1 + compactionIdleTimeoutJitter))) - - i.compactBlocks(context.Background(), false, nil) - verifyCompactedHead(t, i, true) - require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="1"} 1 - - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="1"} 1 - - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - `), memSeriesCreatedTotalName, memSeriesRemovedTotalName, "cortex_ingester_memory_users")) - - // Pushing another sample still works. - pushSingleSampleWithMetadata(t, i) - verifyCompactedHead(t, i, false) - - require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="1"} 2 - - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="1"} 1 - - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - `), memSeriesCreatedTotalName, memSeriesRemovedTotalName, "cortex_ingester_memory_users")) -} - -func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.BlocksStorageConfig.TSDB.ShipInterval = 1 * time.Second // Required to enable shipping. - cfg.BlocksStorageConfig.TSDB.ShipConcurrency = 1 - cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval = 1 * time.Second - cfg.BlocksStorageConfig.TSDB.HeadCompactionIdleTimeout = 1 * time.Second - cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBTimeout = 1 * time.Second - cfg.BlocksStorageConfig.TSDB.CloseIdleTSDBInterval = 100 * time.Millisecond - - r := prometheus.NewRegistry() - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, r) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - t.Cleanup(func() { - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) - }) - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - pushSingleSampleWithMetadata(t, i) - i.v2UpdateActiveSeries() - - require.Equal(t, int64(1), i.TSDBState.seriesCount.Load()) - - metricsToCheck := []string{memSeriesCreatedTotalName, memSeriesRemovedTotalName, "cortex_ingester_memory_users", "cortex_ingester_active_series", - "cortex_ingester_memory_metadata", "cortex_ingester_memory_metadata_created_total", "cortex_ingester_memory_metadata_removed_total"} - - require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="1"} 1 - - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="1"} 0 - - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="1"} 1 - - # HELP cortex_ingester_memory_metadata The current number of metadata in memory. - # TYPE cortex_ingester_memory_metadata gauge - cortex_ingester_memory_metadata 1 - - # HELP cortex_ingester_memory_metadata_created_total The total number of metadata that were created per user - # TYPE cortex_ingester_memory_metadata_created_total counter - cortex_ingester_memory_metadata_created_total{user="1"} 1 - `), metricsToCheck...)) - - // Wait until TSDB has been closed and removed. - test.Poll(t, 10*time.Second, 0, func() interface{} { - i.stoppedMtx.Lock() - defer i.stoppedMtx.Unlock() - return len(i.TSDBState.dbs) - }) - - require.Greater(t, testutil.ToFloat64(i.TSDBState.idleTsdbChecks.WithLabelValues(string(tsdbIdleClosed))), float64(0)) - i.v2UpdateActiveSeries() - require.Equal(t, int64(0), i.TSDBState.seriesCount.Load()) // Flushing removed all series from memory. - - // Verify that user has disappeared from metrics. - require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 0 - - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - - # HELP cortex_ingester_memory_metadata The current number of metadata in memory. - # TYPE cortex_ingester_memory_metadata gauge - cortex_ingester_memory_metadata 0 - `), metricsToCheck...)) - - // Pushing another sample will recreate TSDB. - pushSingleSampleWithMetadata(t, i) - i.v2UpdateActiveSeries() - - // User is back. - require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` - # HELP cortex_ingester_memory_series_created_total The total number of series that were created per user. - # TYPE cortex_ingester_memory_series_created_total counter - cortex_ingester_memory_series_created_total{user="1"} 1 - - # HELP cortex_ingester_memory_series_removed_total The total number of series that were removed per user. - # TYPE cortex_ingester_memory_series_removed_total counter - cortex_ingester_memory_series_removed_total{user="1"} 0 - - # HELP cortex_ingester_memory_users The current number of users in memory. - # TYPE cortex_ingester_memory_users gauge - cortex_ingester_memory_users 1 - - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="1"} 1 - - # HELP cortex_ingester_memory_metadata The current number of metadata in memory. - # TYPE cortex_ingester_memory_metadata gauge - cortex_ingester_memory_metadata 1 - - # HELP cortex_ingester_memory_metadata_created_total The total number of metadata that were created per user - # TYPE cortex_ingester_memory_metadata_created_total counter - cortex_ingester_memory_metadata_created_total{user="1"} 1 - `), metricsToCheck...)) -} - -func verifyCompactedHead(t *testing.T, i *Ingester, expected bool) { - db := i.getTSDB(userID) - require.NotNil(t, db) - - h := db.Head() - require.Equal(t, expected, h.NumSeries() == 0) -} - -func pushSingleSampleWithMetadata(t *testing.T, i *Ingester) { - ctx := user.InjectOrgID(context.Background(), userID) - req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, util.TimeToMillis(time.Now())) - req.Metadata = append(req.Metadata, &cortexpb.MetricMetadata{MetricFamilyName: "test", Help: "a help for metric", Unit: "", Type: cortexpb.COUNTER}) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) -} - -func pushSingleSampleAtTime(t *testing.T, i *Ingester, ts int64) { - ctx := user.InjectOrgID(context.Background(), userID) - req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, ts) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) -} - -func TestHeadCompactionOnStartup(t *testing.T) { - // Create a temporary directory for TSDB - tempDir := t.TempDir() - - // Build TSDB for user, with data covering 24 hours. - { - // Number of full chunks, 12 chunks for 24hrs. - numFullChunks := 12 - chunkRange := 2 * time.Hour.Milliseconds() - - userDir := filepath.Join(tempDir, userID) - require.NoError(t, os.Mkdir(userDir, 0700)) - - db, err := tsdb.Open(userDir, nil, nil, &tsdb.Options{ - RetentionDuration: int64(time.Hour * 25 / time.Millisecond), - NoLockfile: true, - MinBlockDuration: chunkRange, - MaxBlockDuration: chunkRange, - }, nil) - require.NoError(t, err) - - db.DisableCompactions() - head := db.Head() - - l := labels.Labels{{Name: "n", Value: "v"}} - for i := 0; i < numFullChunks; i++ { - // Not using db.Appender() as it checks for compaction. - app := head.Appender(context.Background()) - _, err := app.Append(0, l, int64(i)*chunkRange+1, 9.99) - require.NoError(t, err) - _, err = app.Append(0, l, int64(i+1)*chunkRange, 9.99) - require.NoError(t, err) - require.NoError(t, app.Commit()) - } - - dur := time.Duration(head.MaxTime()-head.MinTime()) * time.Millisecond - require.True(t, dur > 23*time.Hour) - require.Equal(t, 0, len(db.Blocks())) - require.NoError(t, db.Close()) - } - - limits := defaultLimitsTestConfig() - - overrides, err := validation.NewOverrides(limits, nil) - require.NoError(t, err) - - ingesterCfg := defaultIngesterTestConfig(t) - ingesterCfg.BlocksStorageConfig.TSDB.Dir = tempDir - ingesterCfg.BlocksStorageConfig.Bucket.Backend = "s3" - ingesterCfg.BlocksStorageConfig.Bucket.S3.Endpoint = "localhost" - ingesterCfg.BlocksStorageConfig.TSDB.Retention = 2 * 24 * time.Hour // Make sure that no newly created blocks are deleted. - - ingester, err := NewV2(ingesterCfg, overrides, nil, log.NewNopLogger()) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), ingester)) - - defer services.StopAndAwaitTerminated(context.Background(), ingester) //nolint:errcheck - - db := ingester.getTSDB(userID) - require.NotNil(t, db) - - h := db.Head() - - dur := time.Duration(h.MaxTime()-h.MinTime()) * time.Millisecond - require.True(t, dur <= 2*time.Hour) - require.Equal(t, 11, len(db.Blocks())) -} - -func TestIngester_CloseTSDBsOnShutdown(t *testing.T) { - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - - // Create ingester - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - t.Cleanup(func() { - _ = services.StopAndAwaitTerminated(context.Background(), i) - }) - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push some data. - pushSingleSampleWithMetadata(t, i) - - db := i.getTSDB(userID) - require.NotNil(t, db) - - // Stop ingester. - require.NoError(t, services.StopAndAwaitTerminated(context.Background(), i)) - - // Verify that DB is no longer in memory, but was closed - db = i.getTSDB(userID) - require.Nil(t, db) -} - -func TestIngesterNotDeleteUnshippedBlocks(t *testing.T) { - chunkRange := 2 * time.Hour - chunkRangeMilliSec := chunkRange.Milliseconds() - cfg := defaultIngesterTestConfig(t) - cfg.BlocksStorageConfig.TSDB.BlockRanges = []time.Duration{chunkRange} - cfg.BlocksStorageConfig.TSDB.Retention = time.Millisecond // Which means delete all but first block. - cfg.LifecyclerConfig.JoinAfter = 0 - - // Create ingester - reg := prometheus.NewPedanticRegistry() - i, err := prepareIngesterWithBlocksStorage(t, cfg, reg) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - t.Cleanup(func() { - _ = services.StopAndAwaitTerminated(context.Background(), i) - }) - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingester_oldest_unshipped_block_timestamp_seconds Unix timestamp of the oldest TSDB block not shipped to the storage yet. 0 if ingester has no blocks or all blocks have been shipped. - # TYPE cortex_ingester_oldest_unshipped_block_timestamp_seconds gauge - cortex_ingester_oldest_unshipped_block_timestamp_seconds 0 - `), "cortex_ingester_oldest_unshipped_block_timestamp_seconds")) - - // Push some data to create 3 blocks. - ctx := user.InjectOrgID(context.Background(), userID) - for j := int64(0); j < 5; j++ { - req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, j*chunkRangeMilliSec) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - - db := i.getTSDB(userID) - require.NotNil(t, db) - require.Nil(t, db.Compact()) - - oldBlocks := db.Blocks() - require.Equal(t, 3, len(oldBlocks)) - - require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(fmt.Sprintf(` - # HELP cortex_ingester_oldest_unshipped_block_timestamp_seconds Unix timestamp of the oldest TSDB block not shipped to the storage yet. 0 if ingester has no blocks or all blocks have been shipped. - # TYPE cortex_ingester_oldest_unshipped_block_timestamp_seconds gauge - cortex_ingester_oldest_unshipped_block_timestamp_seconds %d - `, oldBlocks[0].Meta().ULID.Time()/1000)), "cortex_ingester_oldest_unshipped_block_timestamp_seconds")) - - // Saying that we have shipped the second block, so only that should get deleted. - require.Nil(t, shipper.WriteMetaFile(nil, db.db.Dir(), &shipper.Meta{ - Version: shipper.MetaVersion1, - Uploaded: []ulid.ULID{oldBlocks[1].Meta().ULID}, - })) - require.NoError(t, db.updateCachedShippedBlocks()) - - // Add more samples that could trigger another compaction and hence reload of blocks. - for j := int64(5); j < 6; j++ { - req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, j*chunkRangeMilliSec) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - require.Nil(t, db.Compact()) - - // Only the second block should be gone along with a new block. - newBlocks := db.Blocks() - require.Equal(t, 3, len(newBlocks)) - require.Equal(t, oldBlocks[0].Meta().ULID, newBlocks[0].Meta().ULID) // First block remains same. - require.Equal(t, oldBlocks[2].Meta().ULID, newBlocks[1].Meta().ULID) // 3rd block becomes 2nd now. - require.NotEqual(t, oldBlocks[1].Meta().ULID, newBlocks[2].Meta().ULID) // The new block won't match previous 2nd block. - - require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(fmt.Sprintf(` - # HELP cortex_ingester_oldest_unshipped_block_timestamp_seconds Unix timestamp of the oldest TSDB block not shipped to the storage yet. 0 if ingester has no blocks or all blocks have been shipped. - # TYPE cortex_ingester_oldest_unshipped_block_timestamp_seconds gauge - cortex_ingester_oldest_unshipped_block_timestamp_seconds %d - `, newBlocks[0].Meta().ULID.Time()/1000)), "cortex_ingester_oldest_unshipped_block_timestamp_seconds")) - - // Shipping 2 more blocks, hence all the blocks from first round. - require.Nil(t, shipper.WriteMetaFile(nil, db.db.Dir(), &shipper.Meta{ - Version: shipper.MetaVersion1, - Uploaded: []ulid.ULID{oldBlocks[1].Meta().ULID, newBlocks[0].Meta().ULID, newBlocks[1].Meta().ULID}, - })) - require.NoError(t, db.updateCachedShippedBlocks()) - - // Add more samples that could trigger another compaction and hence reload of blocks. - for j := int64(6); j < 7; j++ { - req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, j*chunkRangeMilliSec) - _, err := i.v2Push(ctx, req) - require.NoError(t, err) - } - require.Nil(t, db.Compact()) - - // All blocks from the old blocks should be gone now. - newBlocks2 := db.Blocks() - require.Equal(t, 2, len(newBlocks2)) - - require.Equal(t, newBlocks[2].Meta().ULID, newBlocks2[0].Meta().ULID) // Block created in last round. - for _, b := range oldBlocks { - // Second block is not one among old blocks. - require.NotEqual(t, b.Meta().ULID, newBlocks2[1].Meta().ULID) - } - - require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(fmt.Sprintf(` - # HELP cortex_ingester_oldest_unshipped_block_timestamp_seconds Unix timestamp of the oldest TSDB block not shipped to the storage yet. 0 if ingester has no blocks or all blocks have been shipped. - # TYPE cortex_ingester_oldest_unshipped_block_timestamp_seconds gauge - cortex_ingester_oldest_unshipped_block_timestamp_seconds %d - `, newBlocks2[0].Meta().ULID.Time()/1000)), "cortex_ingester_oldest_unshipped_block_timestamp_seconds")) -} - -func TestIngesterPushErrorDuringForcedCompaction(t *testing.T) { - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), nil) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - t.Cleanup(func() { - _ = services.StopAndAwaitTerminated(context.Background(), i) - }) - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push a sample, it should succeed. - pushSingleSampleWithMetadata(t, i) - - // We mock a flushing by setting the boolean. - db := i.getTSDB(userID) - require.NotNil(t, db) - require.True(t, db.casState(active, forceCompacting)) - - // Ingestion should fail with a 503. - req, _, _, _ := mockWriteRequest(t, labels.Labels{{Name: labels.MetricName, Value: "test"}}, 0, util.TimeToMillis(time.Now())) - ctx := user.InjectOrgID(context.Background(), userID) - _, err = i.v2Push(ctx, req) - require.Equal(t, httpgrpc.Errorf(http.StatusServiceUnavailable, wrapWithUser(errors.New("forced compaction in progress"), userID).Error()), err) - - // Ingestion is successful after a flush. - require.True(t, db.casState(forceCompacting, active)) - pushSingleSampleWithMetadata(t, i) -} - -func TestIngesterNoFlushWithInFlightRequest(t *testing.T) { - registry := prometheus.NewRegistry() - i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), registry) - require.NoError(t, err) - - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - t.Cleanup(func() { - _ = services.StopAndAwaitTerminated(context.Background(), i) - }) - - // Wait until it's ACTIVE - test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Push few samples. - for j := 0; j < 5; j++ { - pushSingleSampleWithMetadata(t, i) - } - - // Verifying that compaction won't happen when a request is in flight. - - // This mocks a request in flight. - db := i.getTSDB(userID) - require.NoError(t, db.acquireAppendLock()) - - // Flush handler only triggers compactions, but doesn't wait for them to finish. We cannot use ?wait=true here, - // because it would deadlock -- flush will wait for appendLock to be released. - i.FlushHandler(httptest.NewRecorder(), httptest.NewRequest("POST", "/flush", nil)) - - // Flushing should not have succeeded even after 5 seconds. - time.Sleep(5 * time.Second) - require.NoError(t, testutil.GatherAndCompare(registry, strings.NewReader(` - # HELP cortex_ingester_tsdb_compactions_total Total number of TSDB compactions that were executed. - # TYPE cortex_ingester_tsdb_compactions_total counter - cortex_ingester_tsdb_compactions_total 0 - `), "cortex_ingester_tsdb_compactions_total")) - - // No requests in flight after this. - db.releaseAppendLock() - - // Let's wait until all head series have been flushed. - test.Poll(t, 5*time.Second, uint64(0), func() interface{} { - db := i.getTSDB(userID) - if db == nil { - return false - } - return db.Head().NumSeries() - }) - - require.NoError(t, testutil.GatherAndCompare(registry, strings.NewReader(` - # HELP cortex_ingester_tsdb_compactions_total Total number of TSDB compactions that were executed. - # TYPE cortex_ingester_tsdb_compactions_total counter - cortex_ingester_tsdb_compactions_total 1 - `), "cortex_ingester_tsdb_compactions_total")) -} - -func TestIngester_v2PushInstanceLimits(t *testing.T) { - tests := map[string]struct { - limits InstanceLimits - reqs map[string][]*cortexpb.WriteRequest - expectedErr error - expectedErrType interface{} - }{ - "should succeed creating one user and series": { - limits: InstanceLimits{MaxInMemorySeries: 1, MaxInMemoryTenants: 1}, - reqs: map[string][]*cortexpb.WriteRequest{ - "test": { - cortexpb.ToWriteRequest( - []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}})}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - []*cortexpb.MetricMetadata{ - {MetricFamilyName: "metric_name_1", Help: "a help for metric_name_1", Unit: "", Type: cortexpb.COUNTER}, - }, - cortexpb.API), - }, - }, - expectedErr: nil, - }, - - "should fail creating two series": { - limits: InstanceLimits{MaxInMemorySeries: 1, MaxInMemoryTenants: 1}, - - reqs: map[string][]*cortexpb.WriteRequest{ - "test": { - cortexpb.ToWriteRequest( - []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - - cortexpb.ToWriteRequest( - []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test2"}})}, // another series - []cortexpb.Sample{{Value: 1, TimestampMs: 10}}, - nil, - cortexpb.API), - }, - }, - - expectedErr: wrapWithUser(errMaxSeriesLimitReached, "test"), - }, - - "should fail creating two users": { - limits: InstanceLimits{MaxInMemorySeries: 1, MaxInMemoryTenants: 1}, - - reqs: map[string][]*cortexpb.WriteRequest{ - "user1": { - cortexpb.ToWriteRequest( - []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - }, - - "user2": { - cortexpb.ToWriteRequest( - []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test2"}})}, // another series - []cortexpb.Sample{{Value: 1, TimestampMs: 10}}, - nil, - cortexpb.API), - }, - }, - expectedErr: wrapWithUser(errMaxUsersLimitReached, "user2"), - }, - - "should fail pushing samples in two requests due to rate limit": { - limits: InstanceLimits{MaxInMemorySeries: 1, MaxInMemoryTenants: 1, MaxIngestionRate: 0.001}, - - reqs: map[string][]*cortexpb.WriteRequest{ - "user1": { - cortexpb.ToWriteRequest( - []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, - []cortexpb.Sample{{Value: 1, TimestampMs: 9}}, - nil, - cortexpb.API), - - cortexpb.ToWriteRequest( - []labels.Labels{cortexpb.FromLabelAdaptersToLabels([]cortexpb.LabelAdapter{{Name: labels.MetricName, Value: "test1"}})}, - []cortexpb.Sample{{Value: 1, TimestampMs: 10}}, - nil, - cortexpb.API), - }, - }, - expectedErr: errMaxSamplesPushRateLimitReached, - }, - } - - defaultInstanceLimits = nil - - for testName, testData := range tests { - t.Run(testName, func(t *testing.T) { - // Create a mocked ingester - cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 - cfg.InstanceLimitsFn = func() *InstanceLimits { - return &testData.limits - } - - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until the ingester is ACTIVE - test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - // Iterate through users in sorted order (by username). - uids := []string{} - totalPushes := 0 - for uid, requests := range testData.reqs { - uids = append(uids, uid) - totalPushes += len(requests) - } - sort.Strings(uids) - - pushIdx := 0 - for _, uid := range uids { - ctx := user.InjectOrgID(context.Background(), uid) - - for _, req := range testData.reqs[uid] { - pushIdx++ - _, err := i.Push(ctx, req) - - if pushIdx < totalPushes { - require.NoError(t, err) - } else { - // Last push may expect error. - if testData.expectedErr != nil { - assert.Equal(t, testData.expectedErr, err) - } else if testData.expectedErrType != nil { - assert.True(t, errors.As(err, testData.expectedErrType), "expected error type %T, got %v", testData.expectedErrType, err) - } else { - assert.NoError(t, err) - } - } - - // imitate time ticking between each push - i.ingestionRate.Tick() - - rate := testutil.ToFloat64(i.metrics.ingestionRate) - require.NotZero(t, rate) - } - } - }) - } -} - -func TestIngester_instanceLimitsMetrics(t *testing.T) { - reg := prometheus.NewRegistry() - - l := InstanceLimits{ - MaxIngestionRate: 10, - MaxInMemoryTenants: 20, - MaxInMemorySeries: 30, - } - - cfg := defaultIngesterTestConfig(t) - cfg.InstanceLimitsFn = func() *InstanceLimits { - return &l - } - cfg.LifecyclerConfig.JoinAfter = 0 - - _, err := prepareIngesterWithBlocksStorage(t, cfg, reg) - require.NoError(t, err) - - require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingester_instance_limits Instance limits used by this ingester. - # TYPE cortex_ingester_instance_limits gauge - cortex_ingester_instance_limits{limit="max_inflight_push_requests"} 0 - cortex_ingester_instance_limits{limit="max_ingestion_rate"} 10 - cortex_ingester_instance_limits{limit="max_series"} 30 - cortex_ingester_instance_limits{limit="max_tenants"} 20 - `), "cortex_ingester_instance_limits")) - - l.MaxInMemoryTenants = 1000 - l.MaxInMemorySeries = 2000 - - require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingester_instance_limits Instance limits used by this ingester. - # TYPE cortex_ingester_instance_limits gauge - cortex_ingester_instance_limits{limit="max_inflight_push_requests"} 0 - cortex_ingester_instance_limits{limit="max_ingestion_rate"} 10 - cortex_ingester_instance_limits{limit="max_series"} 2000 - cortex_ingester_instance_limits{limit="max_tenants"} 1000 - `), "cortex_ingester_instance_limits")) -} - -func TestIngester_inflightPushRequests(t *testing.T) { - limits := InstanceLimits{MaxInflightPushRequests: 1} - - // Create a mocked ingester - cfg := defaultIngesterTestConfig(t) - cfg.InstanceLimitsFn = func() *InstanceLimits { return &limits } - cfg.LifecyclerConfig.JoinAfter = 0 - - i, err := prepareIngesterWithBlocksStorage(t, cfg, nil) - require.NoError(t, err) - require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) - defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck - - // Wait until the ingester is ACTIVE - test.Poll(t, 100*time.Millisecond, ring.ACTIVE, func() interface{} { - return i.lifecycler.GetState() - }) - - ctx := user.InjectOrgID(context.Background(), "test") - - startCh := make(chan struct{}) - - g, ctx := errgroup.WithContext(ctx) - g.Go(func() error { - count := 100000 - target := time.Second - - // find right count to make sure that push takes given target duration. - for { - req := generateSamplesForLabel(labels.FromStrings(labels.MetricName, fmt.Sprintf("test-%d", count)), count) - - start := time.Now() - _, err := i.Push(ctx, req) - require.NoError(t, err) - - elapsed := time.Since(start) - t.Log(count, elapsed) - if elapsed > time.Second { - break - } - - count = int(float64(count) * float64(target/elapsed) * 1.5) // Adjust number of samples to hit our target push duration. - } - - // Now repeat push with number of samples calibrated to our target. - req := generateSamplesForLabel(labels.FromStrings(labels.MetricName, fmt.Sprintf("real-%d", count)), count) - - // Signal that we're going to do the real push now. - close(startCh) - - _, err := i.Push(ctx, req) - return err - }) - - g.Go(func() error { - select { - case <-ctx.Done(): - // failed to setup - case <-startCh: - // we can start the test. - } - - time.Sleep(10 * time.Millisecond) // Give first goroutine a chance to start pushing... - req := generateSamplesForLabel(labels.FromStrings(labels.MetricName, "testcase"), 1024) - - _, err := i.Push(ctx, req) - require.Equal(t, errTooManyInflightPushRequests, err) - return nil - }) - - require.NoError(t, g.Wait()) -} - -func generateSamplesForLabel(l labels.Labels, count int) *cortexpb.WriteRequest { - var lbls = make([]labels.Labels, 0, count) - var samples = make([]cortexpb.Sample, 0, count) - - for i := 0; i < count; i++ { - samples = append(samples, cortexpb.Sample{ - Value: float64(i), - TimestampMs: int64(i), - }) - lbls = append(lbls, l) - } - - return cortexpb.ToWriteRequest(lbls, samples, nil, cortexpb.API) -}