WIP make snapshot default mode

bdb/bdb_osqltrn.c

@@ -219,6 +219,13 @@ uint32_t bdb_latest_commit_gen = 0;
 pthread_mutex_t bdb_asof_current_lsn_mutex;
 pthread_cond_t bdb_asof_current_lsn_cond;
 
+void bdb_get_gbl_recoverable_lsn(void *lsn)
+{
+    Pthread_mutex_lock(&bdb_gbl_recoverable_lsn_mutex);
+    *(DB_LSN *)lsn = bdb_gbl_recoverable_lsn;
+    Pthread_mutex_unlock(&bdb_gbl_recoverable_lsn_mutex);
+}
+
 void bdb_set_gbl_recoverable_lsn(void *lsn, int32_t timestamp)
 {
     if (!gbl_new_snapisol_asof && !gbl_modsnap_asof)

bdb/bdb_osqltrn.h

@@ -83,7 +83,12 @@ struct page_logical_lsn_key;
 int bdb_osql_trn_asof_ok_to_delete_log(int filenum);
 
 /**
- * set global recoverabel lsn
+ * Get global recoverable lsn
+ */
+void bdb_get_gbl_recoverable_lsn(void *lsn);
+
+/**
+ * set global recoverable lsn
  */
 void bdb_set_gbl_recoverable_lsn(void *lsn, int32_t timestamp);
 

bdb/tran.c

@@ -2832,7 +2832,23 @@ static int get_modsnap_start_lsn(bdb_state_type *bdb_state, bdb_attr_type *bdb_a
             return rc;
         }
     } else {
-        (void)bdb_get_current_lsn(bdb_state, p_modsnap_start_lsn_file, p_modsnap_start_lsn_offset);
+        // The 'current' LSN is the last one that updated genids.
+        // This can be lower than the recoverable LSN. If it is, then
+        // use the recoverable LSN. Othwerwise, use the current LSN.
+
+        DB_LSN current_lsn;
+        (void)bdb_get_current_lsn(bdb_state, &(current_lsn.file), &(current_lsn.offset));
+
+        DB_LSN recoverable_lsn;
+        bdb_get_gbl_recoverable_lsn(&recoverable_lsn);
+
+        if (log_compare(&current_lsn, &recoverable_lsn) < 0) {
+            *p_modsnap_start_lsn_file = recoverable_lsn.file;
+            *p_modsnap_start_lsn_offset = recoverable_lsn.offset;
+        } else {
+            *p_modsnap_start_lsn_file = current_lsn.file;
+            *p_modsnap_start_lsn_offset = current_lsn.offset;
+        }
     }
 
     return 0;

tests/analyze_insert_burst.test/lrl.options

@@ -0,0 +1 @@
+sql_tranlevel_default blocksock

tests/basic.test/runit

@@ -259,9 +259,7 @@ if ! diff  dupins4.out dupins4.exp; then
 fi
 
 
-if echo $DBNAME | grep 'snapshot\|rowlocks' ; then
-  cdb2sql ${CDB2_OPTIONS} $dbnm default "insert into t2 values(1)"
-fi
+cdb2sql ${CDB2_OPTIONS} $dbnm default "insert into t2 values(1)"
 
 assertcnt t2 2
 do_verify t2

tests/cdb2api_chunk.test/lrl.options

@@ -0,0 +1 @@
+sql_tranlevel_default blocksock

tests/cdb2api_effects_on_chunk_error.test/lrl.options

@@ -0,0 +1 @@
+sql_tranlevel_default blocksock

tests/chunk_txn_effects.test/lrl.options

@@ -0,0 +1 @@
+sql_tranlevel_default blocksock

tests/comdb2_snapshot_lsn.test/runit

@@ -20,6 +20,7 @@ function run_test
 
     echo "Base-case, not snapshot, should return NULL"
     result=$($CDB2SQL_EXE $CDB2_OPTIONS $DBNAME --host $master - <<EOF
+set transaction blocksql
 select comdb2_snapshot_lsn()
 EOF
 )
@@ -31,6 +32,7 @@ EOF
     echo "use_current_lsn_for_non_snapshot"
     $CDB2SQL_EXE $CDB2_OPTIONS $DBNAME --host $master "put tunable 'use_current_lsn_for_non_snapshot' 1"
     result=$($CDB2SQL_EXE $CDB2_OPTIONS $DBNAME --host $master - <<EOF
+set transaction blocksql
 select comdb2_snapshot_lsn()
 EOF
 )

tests/consumer.test/t07.expected

@@ -32,7 +32,7 @@
 (old='{"a":"nan"}', new='{"a":"nan"}')
 (old='{"a":"-inf"}', new='{"a":"-inf"}')
 (old='{"a":"inf"}', new='{"a":"inf"}')
+(old='{"a":1}', new=NULL)
 (old='{"a":"nan"}', new=NULL)
 (old='{"a":"-inf"}', new=NULL)
 (old='{"a":"inf"}', new=NULL)
-(old='{"a":1}', new=NULL)

tests/dohsql_leaks.test/Makefile

@@ -4,5 +4,5 @@ else
   include $(TESTSROOTDIR)/testcase.mk
 endif
 ifeq ($(TEST_TIMEOUT),)
-  export TEST_TIMEOUT=20m
+  export TEST_TIMEOUT=25m
 endif

tests/dupewrite.test/t02.req.out

@@ -14,7 +14,5 @@
 [EXEC PROCEDURE test_dupe_insert_3()] failed with rc -3 [n = q:fetch()...]:7: statement must be read-only
 [EXEC PROCEDURE test_dupe_update_2()] failed with rc -3 [db:exec(sql):emit()...]:3: statement must be read-only
 [EXEC PROCEDURE test_dupe_update_3()] failed with rc -3 [n = q:fetch()...]:7: statement must be read-only
-[EXEC PROCEDURE test_dupe_update_4()] failed with rc -8 find old record failed unable to update record rc = 4
 [EXEC PROCEDURE test_dupe_delete_2()] failed with rc -3 [db:exec(sql):emit()...]:3: statement must be read-only
 [EXEC PROCEDURE test_dupe_delete_3()] failed with rc -3 [n = q:fetch()...]:7: statement must be read-only
-[EXEC PROCEDURE test_dupe_delete_4()] failed with rc -8 Table 'dupe4' find old record failed unable to delete genid =XXXXXX rc=4

tests/fingerprints.test/t02.req.out

@@ -1,7 +1,9 @@
 (version='1')
 (xyz=1)
+(xyz=0)
 (xyz=1)
 (xyz=3)
+(xyz=0)
 (xyz=2)
 (xyz=6)
 (xyz=4)

tests/fingerprints.test/t03.req.out

@@ -1,7 +1,7 @@
 (fingerprint='0a17a3dd4178baa9b98ef60fb54a1992', count=1, total_cost=0, total_rows=2, normalized_sql='SELECT?UNION SELECT?UNION SELECT?;')
 (fingerprint='162437a391bff4c53dd483b904a05184', count=1, total_cost=0, total_rows=2, normalized_sql='EXEC PROCEDURE test_fp_close();')
-(fingerprint='23476e0d735c4cf482b94c485e3d7127', count=1, total_cost=37, total_rows=7, normalized_sql='EXEC PROCEDURE test_fp();')
-(fingerprint='2d4fbab7f6a9c85c75bbb7fb6b43c863', count=1, total_cost=12, total_rows=1, normalized_sql='SELECT sum(x)AS z FROM fp1;')
+(fingerprint='23476e0d735c4cf482b94c485e3d7127', count=1, total_cost=40, total_rows=9, normalized_sql='EXEC PROCEDURE test_fp();')
+(fingerprint='2d4fbab7f6a9c85c75bbb7fb6b43c863', count=1, total_cost=13, total_rows=1, normalized_sql='SELECT sum(x)AS z FROM fp1;')
 (fingerprint='3dcb487a727f69d0d6753bf9046d642b', count=1, total_cost=0, total_rows=1, normalized_sql='EXEC PROCEDURE test_fp_emit();')
 (fingerprint='5bef07d72bc6a66e49bef817dd3bab9d', count=4, total_cost=0, total_rows=4, normalized_sql='INSERT INTO fp1(x)VALUES(?);')
 (fingerprint='5e7c0c2caf9a22cac49643a5a01d7760', count=1, total_cost=13, total_rows=3, normalized_sql='SELECT*FROM fp1 ORDER BY x;')
@@ -43,8 +43,8 @@ end};')
 (fingerprint='84496375cf558c93e22ce29e18ea7187', count=1, total_cost=0, total_rows=0, normalized_sql='CREATE TABLE fp1(x integer);')
 (fingerprint='b12bfa0963ad238396f59e2e1e96b218', count=1, total_cost=12, total_rows=1, normalized_sql='SELECT count(*)FROM fp1;')
 (fingerprint='b16c596087126535ffc0e991e3477d8a', count=1, total_cost=0, total_rows=1, normalized_sql='SELECT?AS xyz;')
-(fingerprint='c7f22188d3a63015bac874fd692fd350', count=1, total_cost=12, total_rows=2, normalized_sql='SELECT x AS w FROM fp1 ORDER BY x;')
-(fingerprint='cb377fe3503a8413402ff6fdbf8d9f11', count=1, total_cost=12, total_rows=2, normalized_sql='SELECT x+x AS y FROM fp1 ORDER BY x;')
+(fingerprint='c7f22188d3a63015bac874fd692fd350', count=1, total_cost=13, total_rows=3, normalized_sql='SELECT x AS w FROM fp1 ORDER BY x;')
+(fingerprint='cb377fe3503a8413402ff6fdbf8d9f11', count=1, total_cost=13, total_rows=3, normalized_sql='SELECT x+x AS y FROM fp1 ORDER BY x;')
 (fingerprint='d12630393a2ab9bf00df2f53a9355b10', count=1, total_cost=13, total_rows=1, normalized_sql='SELECT x,x+x,sum(x)FROM fp1 ORDER BY x;')
 (fingerprint='e2f0f5672b403145980c4535f8ad0253', count=1, total_cost=0, total_rows=1, normalized_sql='CREATE PROCEDURE test_fp_close VERSION?{
 local function main()

tests/logdelete.test/snapshot.testopts

@@ -1 +1,2 @@
 enable_snapshot_isolation
+sql_tranlevel_default snapshot

tests/multichunk.test/runit

@@ -4,6 +4,7 @@ bash -n "$0" | exit 1
 cdb2="cdb2sql --tabs -s ${CDB2_OPTIONS} $1 default "
 
 function multi_chunk {
+    echo "set transaction blocksql"
     echo "set transaction chunk $chunk"
     echo 'begin'
     from=1

tests/prepare.test/lrl.options

@@ -19,3 +19,5 @@ dtastripe 1
 utxnid_log on
 ufid_log on
 block_set_commit_genid_trace 1
+
+sql_tranlevel_default blocksock

tests/prepare.test/runit

@@ -666,7 +666,9 @@ function run_test
     prepare_and_abort
     prepare_and_commit
     recover_prepared_and_downgrade
-    prepare_fail_on_non_utxnid
+    if [[ $TESTCASE != "blkseq_serial_generated" ]] ; then
+        prepare_fail_on_non_utxnid
+    fi
 }
 
 [[ -z "$CLUSTER" ]] && failexit "This test requires a cluster"

tests/queuedb_locks.test/runit

@@ -12,6 +12,7 @@ cdb2sql --host $rep ${CDB2_OPTIONS} $dbname 'create table t(a int)'
 cdb2sql --host $rep ${CDB2_OPTIONS} $dbname 'create default lua consumer test on (table t for insert)'
 
 cdb2sql --host $rep ${CDB2_OPTIONS} $dbname - <<'EOF'
+set transaction blocksql
 set transaction chunk 100
 begin
 insert into t select value from generate_series(1, 1000000)

tests/recover_deadlock.test/lrl.options

@@ -0,0 +1 @@
+sql_tranlevel_default blocksock

tests/remsql.test/output.expected

@@ -19,7 +19,7 @@
 (a=3, b=3)
 (TEST='REMSQL DEDUP TEST 3')
 (a=0, b=0)
-(a=0, b=3)
+(a=3, b=3)
 (rows inserted=1)
 (hint='inserted 3 records from remote to source; source should have 3 records')
 (COUNT(*)=3)

tests/remsql.test/runit

@@ -102,6 +102,7 @@ EOF
 
 ###### inserting to source from remote ######
 cdb2sql -s --cdb2cfg ${CDB2_CONFIG} $srcdbname default - >>output.actual 2>&1 <<EOF
+set transaction blocksql
 DROP TABLE IF EXISTS chunky
 CREATE TABLE chunky (a INT)\$\$
 SET TRANSACTION CHUNK 1
@@ -114,6 +115,7 @@ EOF
 
 ###### inserting to remote from source ######
 cdb2sql -s --cdb2cfg ${CDB2_CONFIG} $srcdbname default - >>output.actual 2>&1 <<EOF
+set transaction blocksql
 SET TRANSACTION CHUNK 1
 BEGIN
 INSERT INTO LOCAL_${dbname}.chunky SELECT a FROM chunky
@@ -124,6 +126,7 @@ EOF
 
 ###### inserting to remote from remote ######
 cdb2sql -s --cdb2cfg ${CDB2_CONFIG} $srcdbname default - >>output.actual 2>&1 <<EOF
+set transaction blocksql
 SET TRANSACTION CHUNK 1
 BEGIN
 INSERT INTO LOCAL_${dbname}.chunky SELECT a FROM LOCAL_${dbname}.chunky
@@ -139,6 +142,7 @@ cdb2sql --cdb2cfg ${CDB2_CONFIG} $srcdbname default "SELECT * FROM LOCAL_${dbnam
 
 ###### Verify that we don't block writing results of a remote SELECT into another remote table, when chunking ######
 cdb2sql -s --cdb2cfg ${CDB2_CONFIG} $srcdbname default - >>output.actual 2>&1 <<EOF
+set transaction blocksql
 SET TRANSACTION CHUNK 1
 BEGIN
 INSERT INTO LOCAL_${dbname}.yet_another_table SELECT a FROM LOCAL_${dbname}.chunky

tests/runtestcase

@@ -390,7 +390,7 @@ verify_db_at_finish() {
 }
 
 exit_if_above_max_test_failures () {
-    MAX_TEST_FAILURES=${MAX_TEST_FAILURES:-10}    # stop running if more that this number of failures in test.log
+    MAX_TEST_FAILURES=${MAX_TEST_FAILURES:-300}    # stop running if more that this number of failures in test.log
     F=`grep failed ${TESTDIR}/test.log | wc -l`
     if [ "$F" -ge "$MAX_TEST_FAILURES" ] ; then 
         echo "$TESTCASE: Max number of allowed test failures ($MAX_TEST_FAILURES) reached" >> ${TESTDIR}/test.log

tests/sc_force.test/runit

@@ -11,6 +11,7 @@ master=`cdb2sql --tabs ${CDB2_OPTIONS} $dbnm --host $fixednode 'exec procedure s
 cat << EOF | cdb2sql ${CDB2_OPTIONS} -s --tabs $dbnm --host $fixednode - >actual 2>&1 
 drop table if exists t
 create table t {`cat t1.csc2`}\$\$
+set transaction blocksql
 set transaction chunk 10000
 BEGIN
 insert into t select * from generate_series(1,1000000)

tests/sc_multitable.test/run.log.alpha

@@ -15,6 +15,7 @@
   ]
  }
 ]
+[set transaction blocksql] rc 0
 [set transaction chunk 10000] rc 0
 [begin] rc 0
 [insert into testview1 select * from  generate_series(1,100000)] rc 0
@@ -94,6 +95,7 @@
 ]
 (rows inserted=10)
 (rows inserted=10)
+[set transaction blocksql] rc 0
 [set transaction chunk 1000] rc 0
 [begin] rc 0
 [insert into '$14_CC37A197' (a) select * from  generate_series(1,1000000)] rc 0

tests/sc_multitable.test/runit

@@ -37,6 +37,7 @@ if (( $? != 0 )) ; then
 fi
 
 cdb2sql ${CDB2_OPTIONS} $dbname default >> $OUT <<EOF
+set transaction blocksql
 set transaction chunk 10000
 begin
 insert into ${VIEW1} select * from  generate_series(1,100000)
@@ -97,6 +98,7 @@ if (( $? != 0 )) ; then
    exit 1
 fi
 cdb2sql ${CDB2_OPTIONS} $dbname default >> $OUT <<EOF
+set transaction blocksql
 set transaction chunk 1000
 begin
 insert into '\$14_CC37A197' (a) select * from  generate_series(1,1000000)

tests/sc_versmismatch.test/lrl.options

@@ -1,4 +1,4 @@
 setattr MIN_KEEP_LOGS 2
 setattr MIN_KEEP_LOGS_AGE 10
 setattr PRIVATE_BLKSEQ_MAXAGE 20
-
+sql_tranlevel_default blocksock

tests/sc_versmismatch.test/runit

@@ -192,6 +192,8 @@ function test2
 
     if [ $recom -ne 0 ] ; then
         echo "set transaction read committed" >&${COPROC[1]}
+    else
+        echo "set transaction blocksql" >&${COPROC[1]}
     fi
     echo "begin" >&${COPROC[1]}
     echo "update t1 set c=c+100000 where a = 300000" >&${COPROC[1]}

tests/seekscan.test/runit

@@ -11,6 +11,7 @@ create table t(a int, b int, c int); \$\$
 create index t_ix on t(a, b);
 truncate t;
 
+set transaction blocksql
 set transaction chunk 100;
 begin;
 insert into t select a.value as a, b.value as b, 0 from generate_series(1, 10000) as a cross join generate_series(1, 10) as b;

tests/selectv.test/t2_01.req.exp

@@ -5,7 +5,6 @@ done
 done
 (rows deleted=1)
 done
-[commit] failed with rc -103 constraints error, no genid
 done
 (rows inserted=1)
 done

tests/selectv_rcode.test/lrl.options

@@ -4,3 +4,4 @@ table jobinstance jobinstance.csc2
 table dbgopts dbgopts.csc2
 snapshot_serial_verify_retry off
 reorder_socksql_no_deadlock on
+min_keep_logs_age 20

tests/selectv_rcode.test/runit

@@ -80,6 +80,24 @@ function debug_flags
     fi
 }
 
+function get_max_db_time
+{
+    local maxtime=0
+    if [[ -z $CLUSTER ]]; then
+        maxtime=$($CDB2SQL_EXE -tabs $CDB2_OPTIONS $DBNAME default "select max(timestamp) from comdb2_transaction_logs")
+    else
+        for node in $CLUSTER; do
+            local time
+            time=$($CDB2SQL_EXE -tabs $CDB2_OPTIONS --host $node $DBNAME default "select max(timestamp) from comdb2_transaction_logs")
+            if (( time > maxtime )); then
+                maxtime=$time
+            fi
+        done
+    fi
+
+    echo $maxtime
+}
+
 function run_tests
 {
     [[ $debug == "1" ]] && set -x
@@ -89,28 +107,31 @@ function run_tests
     typeset dbug_flags=$(debug_flags)
     typeset isolation=""
 
+    local db_time_at_start
+    db_time_at_start=$(get_max_db_time)
+
     start=$(date +%s)
     isolation=socksql
     echo "$isolation test"
-    $selectv_rcode -d $DBNAME -c $CDB2_CONFIG $strict_mode $reads_like_writes $dbug_flags -i $isolation
+    $selectv_rcode -d $DBNAME -T $db_time_at_start -c $CDB2_CONFIG $strict_mode $reads_like_writes $dbug_flags -i $isolation
     export r=$?
     [[ $r != 0 ]] && failexit "$isolation selectv rcodes"
 
     isolation=reccom
     echo "$isolation test"
-    $selectv_rcode -d $DBNAME -c $CDB2_CONFIG $strict_mode $reads_like_writes $dbug_flags -i $isolation
+    $selectv_rcode -d $DBNAME -T $db_time_at_start -c $CDB2_CONFIG $strict_mode $reads_like_writes $dbug_flags -i $isolation
     export r=$?
     [[ $r != 0 ]] && failexit "$isolation selectv rcodes"
 
     isolation=snapshot
     echo "$isolation test"
-    $selectv_rcode -d $DBNAME -c $CDB2_CONFIG $strict_mode $reads_like_writes $dbug_flags -i $isolation
+    $selectv_rcode -d $DBNAME -T $db_time_at_start -c $CDB2_CONFIG $strict_mode $reads_like_writes $dbug_flags -i $isolation
     export r=$?
     [[ $r != 0 ]] && failexit "$isolation selectv rcodes"
 
     isolation=serial
     echo "$isolation test"
-    $selectv_rcode -d $DBNAME -c $CDB2_CONFIG $strict_mode $reads_like_writes $dbug_flags -i $isolation
+    $selectv_rcode -d $DBNAME -T $db_time_at_start -c $CDB2_CONFIG $strict_mode $reads_like_writes $dbug_flags -i $isolation
     export r=$?
     [[ $r != 0 ]] && failexit "$isolation selectv rcodes"
 }

tests/setup

@@ -154,6 +154,9 @@ setup_db() {
         echo "portmux_bind_path $pmux_socket" >> ${LRL}
     fi
 
+    echo "sql_tranlevel_default snapshot" >> ${LRL}
+    echo "enable_snapshot_isolation" >> ${LRL}
+
     # let test specific lrl options take precedence
     if [[ -f lrl ]]; then
         cat lrl >> ${LRL}

tests/socksql.test/lrl.options

@@ -1 +1,2 @@
 table t1 t1.csc2
+sql_tranlevel_default blocksock

tests/socksql.test/t1_04.req.exp

@@ -1 +1,6 @@
-[SELECT * FROM t1] failed with rc -3 Failed to set transaction mode
+(a=1, b=2, c=3, d=4)
+(a=1, b=1, c=11, d=12)
+(a=1, b=1, c=1, d=1)
+(a=1, b=12, c=1, d=14)
+(a=1, b=6, c=7, d=8)
+(a=1, b=8, c=1, d=10)

tests/socksql_master_swings.test/lrl.options

@@ -14,3 +14,4 @@ dump_blkseq on
 extended_sql_debug_trace on
 decoupled_logputs off
 nowatch
+sql_tranlevel_default blocksock

tests/sp.test/lrl.options

@@ -1,2 +1,2 @@
 dtastripe 1
-enable_snapshot_isolation
+sql_tranlevel_default blocksock