Data inconsistent after adding unique index

[tangenta]

Bug Report

Please answer these questions before submitting your issue. Thanks!

1. Minimal reproduce step (Required)

Put this test to realtikvtest:

func TestAddIndexBackfillLostTempIndexValues(t *testing.T) {
	store := realtikvtest.CreateMockStoreAndSetup(t)
	tk := testkit.NewTestKit(t, store)
	tk.MustExec("drop database if exists addindexlit;")
	tk.MustExec("create database addindexlit;")
	tk.MustExec("use addindexlit;")
	tk.MustExec(`set global tidb_ddl_enable_fast_reorg=on;`)
	tk.MustExec("set global tidb_enable_dist_task = 0;")

	tk.MustExec("create table t(id int primary key, b int not null default 0);")
	tk.MustExec("insert into t values (1, 0);")

	tk1 := testkit.NewTestKit(t, store)
	tk1.MustExec("use addindexlit;")

	ddl.MockDMLExecutionBeforeScan = func() {
		_, err := tk1.Exec("insert into t values (2, 0);")
		assert.NoError(t, err)
		_, err = tk1.Exec("delete from t where id = 1;")
		assert.NoError(t, err)
		_, err = tk1.Exec("insert into t values (3, 0);")
		assert.NoError(t, err)
		_, err = tk1.Exec("delete from t where id = 2;")
		assert.NoError(t, err)
	}

	ingest.MockDMLExecutionStateBeforeImport = func() {
		_, err := tk1.Exec("insert into t(id) values (4);")
		assert.NoError(t, err)
		_, err = tk1.Exec("delete from t where id = 3;")
		assert.NoError(t, err)
	}
	var rows [][]any
	ddl.MockDMLExecutionStateBeforeMerge = func() {
		rows = tk1.MustQuery("select * from t use index();").Rows()
		_, err := tk1.Exec("insert into t values (3, 0);")
		assert.NoError(t, err)
	}

	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/skipReorgWorkForTempIndex", "return(false)"))
	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionBeforeScan", "return(true)"))
	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/ingest/mockDMLExecutionStateBeforeImport", "1*return"))
	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionStateBeforeMerge", "1*return"))
	t.Cleanup(func() {
		require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/skipReorgWorkForTempIndex"))
		require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionBeforeScan"))
		require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/ingest/mockDMLExecutionStateBeforeImport"))
		require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/pkg/ddl/mockDMLExecutionStateBeforeMerge"))
	})

	tk.MustGetErrMsg("alter table t add unique index idx(b);", "[kv:1062]Duplicate entry '0' for key 't.idx'")
	require.Len(t, rows, 1)
	require.Equal(t, rows[0][0].(string), "4")

	tk.MustExec("admin check table t;")
}

2. What did you expect to see? (Required)

Step	Operation	Expected	Actual
1	Index=write-only Index=write-reorg, scan rows	Rows = [h1 -> a] Index = [] Tmp index = []
2	Insert h2 -> a	Rows = [h1 -> a, h2 -> a] Index = [] Tmp index = [a -> h2]
3	Delete(h1 -> a)	Rows = [h2 -> a] Index = [] Tmp index = [a -> h2,h1d]
4	Insert(h3 -> a)	Rows = [h2 -> a, h3 -> a] Index = [] Tmp index = [a -> h2,h1d,h3]
5	Delete(h2 -> a)	Rows = [h3 -> a] Index = [] Tmp index = [a -> h2,h1d,h3,h2d]
6	Index=write-reorg, before import Insert(h4 -> a) Delete(h3 -> a)	Rows = [h4 -> a] Index = [] Tmp index = [a -> h2,h1d,h3,h2d,h4,h3d]
7	Backfill: do nothing	Rows = [h4 -> a] Index = [] Tmp index = [a -> h2,h1d,h3,h2d,h4,h3d]
8	Index=write-reorg, ready to merge Insert(h3 -> a)	Rows = [h4 -> a, h3 -> a] Index = [a -> h3] Tmp index = [a -> h2,h1d,h3,h2d,h4,h3d,h3]	Rows = [h4 -> a, h3 -> a] Index = [a -> h3] Tmp index = [a -> h3]
9	Merge	Rows = [h4 -> a, h3 -> a] Index = [a -> h3] [a -> h4] REPORT ERROR Tmp index = []	Rows = [h4 -> a, h3 -> a] Index = [a -> h3] Tmp index = []

3. What did you see instead (Required)

It means the temp index value is overwritten unexpectedly.

4. What is your TiDB version? (Required)

d916614

[tangenta]

Root cause: at step 8, if the origin index value is empty, the insert DML overwrites temp index value. The problematic code:

tidb/pkg/table/tables/index.go

Line 345 in d916614

val = tempVal.Encode(value)

We should reserve the original temp index value before inserting a new one (even though there is no conflict at that time).