kvfollowerreadsccl: deflake TestBoundedStalenessDataDriven

When using tenants, this test appears more prone to flakes. The flakes
seem to come from two sources:

1. A lease would move off of node 1 and thus we'd see a
   "local read" succeed. Here we try to make this less likely by turning
   off the lease, replicate, and split queues and adding an early
   assertion that all of the leaseholders live on n1.

2. The schema change the test performs would take > 10s, which then
   invalidates many later assumptions that 10s in the past should be
   strictly before the schema change. We've kicked the can here and
   bumped it to 20s.

Unfortunately, this still doesn't seem to be enough as we still
occasionally see this fail. One note is that this test is intended to be
skipped under stress, but in CI it is still run under stress because of
changes to the meaning of skip.UnderStress.

Informs #154710
Release note: None

Author: stevendanna

pkg/ccl/kvccl/kvfollowerreadsccl/boundedstaleness_test.go

@@ -311,6 +311,11 @@ func TestBoundedStalenessDataDriven(t *testing.T) {
 			skip.IgnoreLint(t, "test doesn't apply to external process multi-tenancy")
 		}
 
+		require.NoError(t, tc.WaitForFullReplication())
+		tc.ToggleLeaseQueues(false)
+		tc.ToggleSplitQueues(false)
+		tc.ToggleReplicateQueues(false)
+
 		savedTraceStmt := ""
 		datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {
 			// Early exit non-query execution related commands.
@@ -326,7 +331,7 @@ func TestBoundedStalenessDataDriven(t *testing.T) {
 				return ""
 			}
 
-			var showEvents bool
+			var showEvents *bool
 			var waitUntilFollowerReads bool
 			var waitUntilMatch bool
 			defer func() {
@@ -353,6 +358,9 @@ func TestBoundedStalenessDataDriven(t *testing.T) {
 					serverNum, err := strconv.ParseInt(arg.Vals[0], 10, 64)
 					require.NoError(t, err)
 					dbConn = tc.ServerConn(int(serverNum))
+				case "ignore-events":
+					f := false
+					showEvents = &f
 				default:
 					t.Fatalf("unknown arg: %s", arg.Key)
 				}
@@ -376,9 +384,12 @@ func TestBoundedStalenessDataDriven(t *testing.T) {
 					}
 					return ""
 				case "query":
-					// Always show events.
+					// Default to showing events
+					if showEvents == nil {
+						t := true
+						showEvents = &t
+					}
 					bse.setStmt(traceStmt)
-					showEvents = true
 					rows, err := dbConn.Query(d.Input)
 					if err != nil {
 						return err.Error()
@@ -394,7 +405,7 @@ func TestBoundedStalenessDataDriven(t *testing.T) {
 			testutils.SucceedsSoon(t, func() error {
 				ret = executeCmd()
 				// Append events to the output if desired.
-				if showEvents {
+				if showEvents != nil && *showEvents {
 					if !strings.HasSuffix(ret, "\n") {
 						ret += "\n"
 					}

pkg/ccl/kvccl/kvfollowerreadsccl/testdata/boundedstaleness/single_row

@@ -11,6 +11,12 @@ CREATE TABLE t(pk INT PRIMARY KEY) WITH (schema_locked = false);
 INSERT INTO t VALUES (1);
 ----
 
+# The tests assume that the leaseholder is node_idx=0 (n1), assert that here.
+query ignore-events
+SELECT DISTINCT(lease_holder) FROM [SHOW RANGES FROM TABLE t WITH DETAILS];
+----
+1
+
 # If we try to read a timestamp that is impossible to satisfy with a follower
 # read, we should always be looking at the leaseholder in the nearest_only=False
 # case. We always do bounded staleness reads from node_idx 2, as node_idx 0 in a
@@ -62,57 +68,57 @@ events (1 found):
 # Note we have to wait until a match here, in case a follower read reads an
 # older version of the data.
 query idx=2 wait-until-follower-read wait-until-match
-SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('10s') WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('20s') WHERE pk = 1
 ----
 1
 events (1 found):
  * event 1: colbatchscan trace on node_idx 2: local follower read
 
 query idx=2
-SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '10s') WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '20s') WHERE pk = 1
 ----
 1
 events (1 found):
  * event 1: colbatchscan trace on node_idx 2: local follower read
 
 query idx=2
-SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('10s', false) WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('20s', false) WHERE pk = 1
 ----
 1
 events (1 found):
  * event 1: colbatchscan trace on node_idx 2: local follower read
 
 query idx=2
-SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '10s', false) WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '20s', false) WHERE pk = 1
 ----
 1
 events (1 found):
  * event 1: colbatchscan trace on node_idx 2: local follower read
 
 query idx=2
-SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('10s', true) WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('20s', true) WHERE pk = 1
 ----
 1
 events (1 found):
  * event 1: colbatchscan trace on node_idx 2: local follower read
 
 query idx=2
-SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '10s', true) WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '20s', true) WHERE pk = 1
 ----
 1
 events (1 found):
  * event 1: colbatchscan trace on node_idx 2: local follower read
 
 exec idx=2
-PREPARE max_staleness_prep AS SELECT pk FROM t AS OF SYSTEM TIME with_max_staleness('10s') WHERE pk = 1;
+PREPARE max_staleness_prep AS SELECT pk FROM t AS OF SYSTEM TIME with_max_staleness('20s') WHERE pk = 1;
 ----
 
 exec idx=2
-PREPARE min_timestamp_prep AS SELECT pk FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '10s') WHERE pk = 1
+PREPARE min_timestamp_prep AS SELECT pk FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '20s') WHERE pk = 1
 ----
 
 override-matching-stmt-for-tracing
-SELECT pk FROM t AS OF SYSTEM TIME with_max_staleness('10s') WHERE pk = 1
+SELECT pk FROM t AS OF SYSTEM TIME with_max_staleness('20s') WHERE pk = 1
 ----
 
 query idx=2
@@ -123,7 +129,7 @@ events (1 found):
  * event 1: colbatchscan trace on node_idx 2: local follower read
 
 override-matching-stmt-for-tracing
-SELECT pk FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '10s') WHERE pk = 1
+SELECT pk FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '20s') WHERE pk = 1
 ----
 
 query idx=2
@@ -149,28 +155,28 @@ ALTER TABLE t ADD COLUMN new_col INT NOT NULL DEFAULT 2
 # Ensure we resort to the leaseholder as the schema change requires a recent read
 # in the nearest_only=False case.
 query idx=2
-SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('10s') WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('20s') WHERE pk = 1
 ----
 1 2
 events (1 found):
  * event 1: colbatchscan trace on node_idx 2: local read then remote leaseholder read
 
 query idx=2
-SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '10s') WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '20s') WHERE pk = 1
 ----
 1 2
 events (1 found):
  * event 1: colbatchscan trace on node_idx 2: local read then remote leaseholder read
 
 query idx=2
-SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('10s', false) WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('20s', false) WHERE pk = 1
 ----
 1 2
 events (1 found):
  * event 1: colbatchscan trace on node_idx 2: local read then remote leaseholder read
 
 query idx=2
-SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '10s', false) WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '20s', false) WHERE pk = 1
 ----
 1 2
 events (1 found):
@@ -182,7 +188,7 @@ events (1 found):
 # Note that we retry until follower read here as the first schema read of
 # historical schema descriptors result in non-follower reads.
 query idx=2 wait-until-follower-read
-SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('10s', true) WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('20s', true) WHERE pk = 1
 ----
 1
 events (17 found):
@@ -205,7 +211,7 @@ events (17 found):
  * event 17: colbatchscan trace on node_idx 2: local follower read
 
 query idx=2
-SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '10s', true) WHERE pk = 1
+SELECT * FROM t AS OF SYSTEM TIME with_min_timestamp(now() - '20s', true) WHERE pk = 1
 ----
 1
 events (17 found):
@@ -242,7 +248,7 @@ ALTER TABLE t2 ADD COLUMN new_col INT
 ----
 
 query idx=2
-SELECT * FROM t2 AS OF SYSTEM TIME with_min_timestamp(now() - '10s', true) WHERE pk = 2
+SELECT * FROM t2 AS OF SYSTEM TIME with_min_timestamp(now() - '20s', true) WHERE pk = 2
 ----
 pq: referenced descriptor ID 105: looking up ID 105: descriptor not found
 events (10 found):
@@ -258,7 +264,7 @@ events (10 found):
  * event 10: transaction retry on node_idx: 2
 
 query idx=2
-SELECT * FROM t2 AS OF SYSTEM TIME with_min_timestamp(now() - '10s', true) WHERE pk = 2
+SELECT * FROM t2 AS OF SYSTEM TIME with_min_timestamp(now() - '20s', true) WHERE pk = 2
 ----
 pq: referenced descriptor ID 105: looking up ID 105: descriptor not found
 events (10 found):