Skip to content

Commit e8a101c

Browse files
harry671003harry671003
authored
Fix cleaner not discovering deleted users from global dir (#5691)
* Fix cleaner not discovering deleted users from global dir Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]> * Fix tests Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]> * Remove the prefix from entry Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]> * fix tests Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]> * Fix tests Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]> * Address comments Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]> * Update comments Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]> --------- Signed-off-by: 🌲 Harry 🌊 John 🏔 <[email protected]>
1 parent 61fe286 commit e8a101c

File tree

5 files changed

+61
-22
lines changed

5 files changed

+61
-22
lines changed

‎pkg/compactor/compactor_test.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ func TestCompactor_SkipCompactionWhenCmkError(t *testing.T) {
170170
// No user blocks stored in the bucket.
171171
bucketClient := &bucket.ClientMock{}
172172
bucketClient.MockIter("", []string{userID}, nil)
173+
bucketClient.MockIter("__markers__", []string{}, nil)
173174
bucketClient.MockIter(userID+"/", []string{}, nil)
174175
bucketClient.MockIter(userID+"/markers/", nil, nil)
175176
bucketClient.MockGet(userID+"/bucket-index-sync-status.json", string(content), nil)
@@ -198,6 +199,7 @@ func TestCompactor_ShouldDoNothingOnNoUserBlocks(t *testing.T) {
198199
// No user blocks stored in the bucket.
199200
bucketClient := &bucket.ClientMock{}
200201
bucketClient.MockIter("", []string{}, nil)
202+
bucketClient.MockIter("__markers__", []string{}, nil)
201203
cfg := prepareConfig()
202204
c, _, _, logs, registry := prepare(t, cfg, bucketClient, nil)
203205
require.NoError(t, services.StartAndAwaitRunning(context.Background(), c))
@@ -348,6 +350,7 @@ func TestCompactor_ShouldRetryCompactionOnFailureWhileDiscoveringUsersFromBucket
348350

349351
// Fail to iterate over the bucket while discovering users.
350352
bucketClient := &bucket.ClientMock{}
353+
bucketClient.MockIter("__markers__", nil, errors.New("failed to iterate the bucket"))
351354
bucketClient.MockIter("", nil, errors.New("failed to iterate the bucket"))
352355

353356
c, _, _, logs, registry := prepare(t, prepareConfig(), bucketClient, nil)
@@ -501,6 +504,7 @@ func TestCompactor_ShouldIncrementCompactionErrorIfFailedToCompactASingleTenant(
501504
userID := "test-user"
502505
bucketClient := &bucket.ClientMock{}
503506
bucketClient.MockIter("", []string{userID}, nil)
507+
bucketClient.MockIter("__markers__", []string{}, nil)
504508
bucketClient.MockIter(userID+"/", []string{userID + "/01DTVP434PA9VFXSW2JKB3392D/meta.json", userID + "/01FN6CDF3PNEWWRY5MPGJPE3EX/meta.json"}, nil)
505509
bucketClient.MockIter(userID+"/markers/", nil, nil)
506510
bucketClient.MockExists(cortex_tsdb.GetGlobalDeletionMarkPath(userID), false, nil)
@@ -553,6 +557,7 @@ func TestCompactor_ShouldIncrementCompactionErrorIfFailedToCompactASingleTenant(
553557
func TestCompactor_ShouldCompactAndRemoveUserFolder(t *testing.T) {
554558
bucketClient := &bucket.ClientMock{}
555559
bucketClient.MockIter("", []string{"user-1"}, nil)
560+
bucketClient.MockIter("__markers__", []string{}, nil)
556561
bucketClient.MockExists(cortex_tsdb.GetGlobalDeletionMarkPath("user-1"), false, nil)
557562
bucketClient.MockExists(cortex_tsdb.GetLocalDeletionMarkPath("user-1"), false, nil)
558563
bucketClient.MockIter("user-1/", []string{"user-1/01DTVP434PA9VFXSW2JKB3392D/meta.json", "user-1/01FN6CDF3PNEWWRY5MPGJPE3EX/meta.json"}, nil)
@@ -598,6 +603,7 @@ func TestCompactor_ShouldIterateOverUsersAndRunCompaction(t *testing.T) {
598603
// Mock the bucket to contain two users, each one with one block.
599604
bucketClient := &bucket.ClientMock{}
600605
bucketClient.MockIter("", []string{"user-1", "user-2"}, nil)
606+
bucketClient.MockIter("__markers__", []string{}, nil)
601607
bucketClient.MockExists(cortex_tsdb.GetGlobalDeletionMarkPath("user-1"), false, nil)
602608
bucketClient.MockExists(cortex_tsdb.GetLocalDeletionMarkPath("user-1"), false, nil)
603609
bucketClient.MockExists(cortex_tsdb.GetGlobalDeletionMarkPath("user-2"), false, nil)
@@ -741,6 +747,7 @@ func TestCompactor_ShouldNotCompactBlocksMarkedForDeletion(t *testing.T) {
741747
// Mock the bucket to contain two users, each one with one block.
742748
bucketClient := &bucket.ClientMock{}
743749
bucketClient.MockIter("", []string{"user-1"}, nil)
750+
bucketClient.MockIter("__markers__", []string{}, nil)
744751
bucketClient.MockIter("user-1/", []string{"user-1/01DTVP434PA9VFXSW2JKB3392D", "user-1/01DTW0ZCPDDNV4BV83Q2SV4QAZ"}, nil)
745752
bucketClient.MockExists(cortex_tsdb.GetGlobalDeletionMarkPath("user-1"), false, nil)
746753
bucketClient.MockExists(cortex_tsdb.GetLocalDeletionMarkPath("user-1"), false, nil)
@@ -866,6 +873,7 @@ func TestCompactor_ShouldNotCompactBlocksMarkedForSkipCompact(t *testing.T) {
866873
// Mock the bucket to contain two users, each one with one block.
867874
bucketClient := &bucket.ClientMock{}
868875
bucketClient.MockIter("", []string{"user-1", "user-2"}, nil)
876+
bucketClient.MockIter("__markers__", []string{}, nil)
869877
bucketClient.MockExists(cortex_tsdb.GetGlobalDeletionMarkPath("user-1"), false, nil)
870878
bucketClient.MockExists(cortex_tsdb.GetLocalDeletionMarkPath("user-1"), false, nil)
871879
bucketClient.MockExists(cortex_tsdb.GetGlobalDeletionMarkPath("user-2"), false, nil)
@@ -944,6 +952,7 @@ func TestCompactor_ShouldNotCompactBlocksForUsersMarkedForDeletion(t *testing.T)
944952
// Mock the bucket to contain two users, each one with one block.
945953
bucketClient := &bucket.ClientMock{}
946954
bucketClient.MockIter("", []string{"user-1"}, nil)
955+
bucketClient.MockIter("__markers__", []string{"__markers__/user-1/"}, nil)
947956
bucketClient.MockIter("user-1/", []string{"user-1/01DTVP434PA9VFXSW2JKB3392D"}, nil)
948957
bucketClient.MockGet(cortex_tsdb.GetGlobalDeletionMarkPath("user-1"), `{"deletion_time": 1}`, nil)
949958
bucketClient.MockUpload(cortex_tsdb.GetGlobalDeletionMarkPath("user-1"), nil)
@@ -1107,6 +1116,7 @@ func TestCompactor_ShouldCompactAllUsersOnShardingEnabledButOnlyOneInstanceRunni
11071116
// Mock the bucket to contain two users, each one with one block.
11081117
bucketClient := &bucket.ClientMock{}
11091118
bucketClient.MockIter("", []string{"user-1", "user-2"}, nil)
1119+
bucketClient.MockIter("__markers__", []string{}, nil)
11101120
bucketClient.MockExists(cortex_tsdb.GetGlobalDeletionMarkPath("user-1"), false, nil)
11111121
bucketClient.MockExists(cortex_tsdb.GetLocalDeletionMarkPath("user-1"), false, nil)
11121122
bucketClient.MockExists(cortex_tsdb.GetGlobalDeletionMarkPath("user-2"), false, nil)
@@ -1215,6 +1225,7 @@ func TestCompactor_ShouldCompactOnlyUsersOwnedByTheInstanceOnShardingEnabledAndM
12151225
// Mock the bucket to contain all users, each one with one block.
12161226
bucketClient := &bucket.ClientMock{}
12171227
bucketClient.MockIter("", userIDs, nil)
1228+
bucketClient.MockIter("__markers__", []string{}, nil)
12181229
for _, userID := range userIDs {
12191230
bucketClient.MockIter(userID+"/", []string{userID + "/01DTVP434PA9VFXSW2JKB3392D"}, nil)
12201231
bucketClient.MockIter(userID+"/markers/", nil, nil)
@@ -1321,6 +1332,7 @@ func TestCompactor_ShouldCompactOnlyShardsOwnedByTheInstanceOnShardingEnabledWit
13211332
// Mock the bucket to contain all users, each one with five blocks, 2 sets of overlapping blocks and 1 separate block.
13221333
bucketClient := &bucket.ClientMock{}
13231334
bucketClient.MockIter("", userIDs, nil)
1335+
bucketClient.MockIter("__markers__", []string{}, nil)
13241336

13251337
// Keys with a value greater than 1 will be groups that should be compacted
13261338
groupHashes := make(map[uint32]int)
@@ -1927,6 +1939,7 @@ func TestCompactor_ShouldFailCompactionOnTimeout(t *testing.T) {
19271939
// Mock the bucket
19281940
bucketClient := &bucket.ClientMock{}
19291941
bucketClient.MockIter("", []string{}, nil)
1942+
bucketClient.MockIter("__markers__", []string{}, nil)
19301943

19311944
ringStore, closer := consul.NewInMemoryClient(ring.GetCodec(), log.NewNopLogger(), nil)
19321945
t.Cleanup(func() { assert.NoError(t, closer.Close()) })

‎pkg/querier/blocks_finder_bucket_scan_test.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ func TestBucketScanBlocksFinder_InitialScanFailure(t *testing.T) {
9494

9595
// Mock the storage to simulate a failure when reading objects.
9696
bucket.MockIter("", []string{"user-1"}, nil)
97+
bucket.MockIter("__markers__", []string{}, nil)
9798
bucket.MockIter("user-1/", []string{"user-1/01DTVP434PA9VFXSW2JKB3392D/meta.json"}, nil)
9899
bucket.MockExists(cortex_tsdb.GetGlobalDeletionMarkPath("user-1"), false, nil)
99100
bucket.MockExists(cortex_tsdb.GetLocalDeletionMarkPath("user-1"), false, nil)
@@ -139,6 +140,7 @@ func TestBucketScanBlocksFinder_StopWhileRunningTheInitialScanOnManyTenants(t *t
139140
// Mock the bucket to introduce a 1s sleep while iterating each tenant in the bucket.
140141
bucket := &bucket.ClientMock{}
141142
bucket.MockIter("", tenantIDs, nil)
143+
bucket.MockIter("__markers__", []string{}, nil)
142144
for _, tenantID := range tenantIDs {
143145
bucket.MockIterWithCallback(tenantID+"/", []string{}, nil, func() {
144146
time.Sleep(time.Second)
@@ -177,6 +179,7 @@ func TestBucketScanBlocksFinder_StopWhileRunningTheInitialScanOnManyBlocks(t *te
177179
// Mock the bucket to introduce a 1s sleep while syncing each block in the bucket.
178180
bucket := &bucket.ClientMock{}
179181
bucket.MockIter("", []string{"user-1"}, nil)
182+
bucket.MockIter("__markers__", []string{}, nil)
180183
bucket.MockIter("user-1/", blockPaths, nil)
181184
bucket.On("Exists", mock.Anything, mock.Anything).Return(false, nil).Run(func(args mock.Arguments) {
182185
// We return the meta.json doesn't exist, but introduce a 1s delay for each call.

‎pkg/storage/tsdb/users_scanner.go

Lines changed: 23 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -38,38 +38,48 @@ func NewUsersScanner(bucketClient objstore.Bucket, isOwned func(userID string) (
3838
//
3939
// If sharding is enabled, returned lists contains only the users owned by this instance.
4040
func (s *UsersScanner) ScanUsers(ctx context.Context) (users, markedForDeletion []string, err error) {
41+
scannedUsers := make(map[string]struct{})
42+
43+
// Scan users in the bucket.
4144
err = s.bucketClient.Iter(ctx, "", func(entry string) error {
42-
users = append(users, strings.TrimSuffix(entry, "/"))
45+
userID := strings.TrimSuffix(entry, "/")
46+
scannedUsers[userID] = struct{}{}
4347
return nil
4448
})
4549
if err != nil {
4650
return nil, nil, err
4751
}
4852

49-
// Check users for being owned by instance, and split users into non-deleted and deleted.
50-
// We do these checks after listing all users, to improve cacheability of Iter (result is only cached at the end of Iter call).
51-
for ix := 0; ix < len(users); {
52-
userID := users[ix]
53+
// Scan users from the __markers__ directory.
54+
err = s.bucketClient.Iter(ctx, util.GlobalMarkersDir, func(entry string) error {
55+
// entry will be of the form __markers__/<user>/
56+
parts := strings.Split(entry, objstore.DirDelim)
57+
userID := parts[1]
58+
scannedUsers[userID] = struct{}{}
59+
return nil
60+
})
61+
if err != nil {
62+
return nil, nil, err
63+
}
5364

54-
// Check if it's owned by this instance.
55-
owned, err := s.isOwned(userID)
56-
if err != nil {
65+
for userID := range scannedUsers {
66+
// Filter out users not owned by this instance.
67+
if owned, err := s.isOwned(userID); err != nil {
5768
level.Warn(s.logger).Log("msg", "unable to check if user is owned by this shard", "user", userID, "err", err)
5869
} else if !owned {
59-
users = append(users[:ix], users[ix+1:]...)
6070
continue
6171
}
6272

63-
deletionMarkExists, err := TenantDeletionMarkExists(ctx, s.bucketClient, userID)
64-
if err != nil {
73+
// Filter users marked for deletion
74+
if deletionMarkExists, err := TenantDeletionMarkExists(ctx, s.bucketClient, userID); err != nil {
6575
level.Warn(s.logger).Log("msg", "unable to check if user is marked for deletion", "user", userID, "err", err)
6676
} else if deletionMarkExists {
67-
users = append(users[:ix], users[ix+1:]...)
6877
markedForDeletion = append(markedForDeletion, userID)
6978
continue
7079
}
7180

72-
ix++
81+
// The remaining are the active users owned by this instance.
82+
users = append(users, userID)
7383
}
7484

7585
return users, markedForDeletion, nil

‎pkg/storage/tsdb/users_scanner_test.go

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,35 +8,40 @@ import (
88
"github.com/go-kit/log"
99
"github.com/stretchr/testify/assert"
1010
"github.com/stretchr/testify/require"
11+
"golang.org/x/exp/slices"
1112

1213
"github.com/cortexproject/cortex/pkg/storage/bucket"
1314
)
1415

1516
func TestUsersScanner_ScanUsers_ShouldReturnedOwnedUsersOnly(t *testing.T) {
1617
bucketClient := &bucket.ClientMock{}
17-
bucketClient.MockIter("", []string{"user-1", "user-2", "user-3", "user-4"}, nil)
18+
bucketClient.MockIter("", []string{"user-1/", "user-2/", "user-3/", "user-4/"}, nil)
19+
bucketClient.MockIter("__markers__", []string{"__markers__/user-5/", "__markers__/user-6/", "__markers__/user-7/"}, nil)
1820
bucketClient.MockExists(GetGlobalDeletionMarkPath("user-1"), false, nil)
1921
bucketClient.MockExists(GetLocalDeletionMarkPath("user-1"), false, nil)
2022
bucketClient.MockExists(GetGlobalDeletionMarkPath("user-3"), true, nil)
21-
bucketClient.MockExists(GetLocalDeletionMarkPath("user-3"), true, nil)
23+
bucketClient.MockExists(GetLocalDeletionMarkPath("user-3"), false, nil)
24+
bucketClient.MockExists(GetGlobalDeletionMarkPath("user-7"), false, nil)
25+
bucketClient.MockExists(GetLocalDeletionMarkPath("user-7"), true, nil)
2226

2327
isOwned := func(userID string) (bool, error) {
24-
return userID == "user-1" || userID == "user-3", nil
28+
return userID == "user-1" || userID == "user-3" || userID == "user-7", nil
2529
}
2630

2731
s := NewUsersScanner(bucketClient, isOwned, log.NewNopLogger())
2832
actual, deleted, err := s.ScanUsers(context.Background())
2933
require.NoError(t, err)
3034
assert.Equal(t, []string{"user-1"}, actual)
31-
assert.Equal(t, []string{"user-3"}, deleted)
32-
35+
slices.Sort(deleted)
36+
assert.Equal(t, []string{"user-3", "user-7"}, deleted)
3337
}
3438

3539
func TestUsersScanner_ScanUsers_ShouldReturnUsersForWhichOwnerCheckOrTenantDeletionCheckFailed(t *testing.T) {
3640
expected := []string{"user-1", "user-2"}
3741

3842
bucketClient := &bucket.ClientMock{}
3943
bucketClient.MockIter("", expected, nil)
44+
bucketClient.MockIter("__markers__", []string{}, nil)
4045
bucketClient.MockExists(GetGlobalDeletionMarkPath("user-1"), false, nil)
4146
bucketClient.MockExists(GetLocalDeletionMarkPath("user-1"), false, nil)
4247

‎tools/thanosconvert/thanosconvert_test.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,17 +45,22 @@ func TestThanosBlockConverter(t *testing.T) {
4545
assertions func(*testing.T, *bucket.ClientMock, Results, error)
4646
}{
4747
{
48-
name: "empty bucket is a noop",
49-
bucketData: fakeBucket{},
48+
name: "empty bucket is a noop",
49+
bucketData: fakeBucket{
50+
"__markers__": map[string]metadata.Meta{},
51+
},
5052
assertions: func(t *testing.T, bkt *bucket.ClientMock, results Results, err error) {
5153
bkt.AssertNotCalled(t, "Get", mock.Anything, mock.Anything)
5254
bkt.AssertNotCalled(t, "Upload", mock.Anything, mock.Anything, mock.Anything)
5355
assert.Len(t, results, 0, "expected no users in results")
5456
},
5557
},
5658
{
57-
name: "user with no blocks is a noop",
58-
bucketData: fakeBucket{"user1": map[string]metadata.Meta{}},
59+
name: "user with no blocks is a noop",
60+
bucketData: fakeBucket{
61+
"user1": map[string]metadata.Meta{},
62+
"__markers__": map[string]metadata.Meta{},
63+
},
5964
assertions: func(t *testing.T, bkt *bucket.ClientMock, results Results, err error) {
6065
bkt.AssertNotCalled(t, "Get", mock.Anything, mock.Anything)
6166
bkt.AssertNotCalled(t, "Upload", mock.Anything, mock.Anything, mock.Anything)
@@ -80,6 +85,7 @@ func TestThanosBlockConverter(t *testing.T) {
8085
"user3": map[string]metadata.Meta{
8186
block1: cortexMeta("user3"),
8287
},
88+
"__markers__": map[string]metadata.Meta{},
8389
},
8490
assertions: func(t *testing.T, bkt *bucket.ClientMock, results Results, err error) {
8591
bkt.AssertNotCalled(t, "Upload", mock.Anything, mock.Anything, mock.Anything)
@@ -114,6 +120,7 @@ func TestThanosBlockConverter(t *testing.T) {
114120
"user3": map[string]metadata.Meta{
115121
block1: thanosMeta(),
116122
},
123+
"__markers__": map[string]metadata.Meta{},
117124
},
118125
assertions: func(t *testing.T, bkt *bucket.ClientMock, results Results, err error) {
119126
assert.Len(t, results, 3, "expected users in results")
@@ -149,6 +156,7 @@ func TestThanosBlockConverter(t *testing.T) {
149156
blockWithUploadFailure: thanosMeta(),
150157
blockWithMalformedMeta: thanosMeta(),
151158
},
159+
"__markers__": map[string]metadata.Meta{},
152160
},
153161
assertions: func(t *testing.T, bkt *bucket.ClientMock, results Results, err error) {
154162
assert.Len(t, results["user1"].FailedBlocks, 1)

0 commit comments

Comments
 (0)