fix/TS-5651-trim-db-test

source/common/src/tglobal.c

@@ -857,7 +857,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
 
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_SERVER_LAZY,CFG_CATEGORY_LOCAL));
-  TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
+  TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsQueueMemoryAllowed, 1, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));
   TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_GLOBAL));

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -1845,6 +1845,8 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in
 
   pRsp->code = TSDB_CODE_SUCCESS;
 
+  taosMsleep(100000);
+
   void           *pAllocMsg = NULL;
   SSubmitReq2Msg *pMsg = (SSubmitReq2Msg *)pReq;
   if (0 == pMsg->version) {

tests/system-test/0-others/block_submit.py

@@ -0,0 +1,62 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+from util.sqlset import *
+import glob
+import os
+
+import threading
+
+class TDTestCase:
+    updatecfgDict = {'rpcQueueMemoryAllowed':100000}
+
+    def init(self, conn, logSql, replicaVar=1):
+
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+        self.setsql = TDSetSql()
+
+    def trim_database(self):
+
+        tdSql.execute('create database dbtest vgroups 12 replica 3 duration 60m keep 1d,365d,3650d;')
+        tdSql.execute('use dbtest')
+        tdSql.execute('create table stb (ts timestamp,c0 int) tags(t0 int)')
+        tdSql.execute('create table tb1 using stb tags(1)')
+
+        count = 0
+        while count < 1000:
+            newTdSql=tdCom.newTdSql()
+            threading.Thread(target=self.checkRunTimeError, args=(newTdSql, count)).start()
+            count += 1
+
+
+    def checkRunTimeError(self, newtdSql, i):
+        tdLog.info("Thread %d" % i)
+        newtdSql.execute(f'insert into dbtest.tb1 values(now-{i}d,10)')
+
+
+    def run(self):
+        self.trim_database()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

tests/system-test/0-others/multilevel_trimdb.py

@@ -0,0 +1,139 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+from util.sqlset import *
+import glob
+import os
+
+def scanFiles(pattern):
+    res = []
+    for f in glob.iglob(pattern):
+        res += [f]
+    return res
+
+def checkFiles(pattern, state):
+    res = scanFiles(pattern)
+    tdLog.info(res)
+    num = len(res)
+    if num:
+        if state:
+            tdLog.info("%s: %d files exist. expect: files exist" % (pattern, num))
+        else:
+            tdLog.exit("%s: %d files exist. expect: files not exist." % (pattern, num))
+    else:
+        if state:
+            tdLog.exit("%s: %d files exist. expect: files exist" % (pattern, num))
+        else:
+            tdLog.info("%s: %d files exist. expect: files not exist." % (pattern, num))
+
+class TDTestCase:
+    def init(self, conn, logSql, replicaVar=1):
+
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+        self.setsql = TDSetSql()
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        buildPath = ""
+        for root, dirs, files in os.walk(projPath):
+            if ("taosdump" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def trim_database(self):
+        tdLog.info("============== trim_database test ===============")
+        tdDnodes.stop(1)
+        cfg = {
+            '/mnt/data1 0 1' : 'dataDir'
+        }
+        tdSql.createDir('/mnt/data1')
+        tdDnodes.deploy(1,cfg)
+        tdDnodes.start(1)
+
+        # create dnode 'mys03d01:6030';
+        # create dnode 'mys04d01:6030';
+
+        tdSql.execute('create database dbtest vgroups 12 replica 3 duration 60m keep 1d,365d,3650d;')
+        tdSql.execute('use dbtest')
+        tdSql.execute('create table stb (ts timestamp,c0 int) tags(t0 int)')
+        tdSql.execute('create table tb1 using stb tags(1)')
+        for i in range(1,600, 30):
+            tdSql.execute(f'insert into tb1 values(now-{i}d,10)')
+        tdSql.execute('flush database dbtest')
+        time.sleep(3)
+        checkFiles('/mnt/data1/vnode/*/tsdb/v*',1)
+        tdDnodes.stop(1)
+
+        cfg={
+            '/mnt/data1 0 1' : 'dataDir',
+            '/mnt/data2 1 0 0' : 'dataDir',
+            '/mnt/data3 1 0 1' : 'dataDir',
+            '/mnt/data4 2 0' : 'dataDir',
+        }
+        tdSql.createDir('/mnt/data2')
+        tdSql.createDir('/mnt/data3')
+        tdSql.createDir('/mnt/data4')
+        tdDnodes.deploy(1,cfg)
+        tdDnodes.start(1) 
+
+        checkFiles('/mnt/data1/vnode/*/tsdb/v*',1)
+        checkFiles('/mnt/data2/vnode/*/tsdb/v*',0)
+        checkFiles('/mnt/data3/vnode/*/tsdb/v*',0)
+        checkFiles('/mnt/data4/vnode/*/tsdb/v*',0)
+        tdSql.execute('alter database dbtest keep 1d,365d,3650d')
+
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        binPath = buildPath+ "/build/bin/"
+
+        #-N:regular table  -d:database name   -t:table num  -n:rows num per table  -l:col num  -y:force
+        #regular old && new
+        os.system("%staosBenchmark -N -d dbtest -t 100 -n 5500000 -l 1023 -y" % binPath)
+
+        #taosBenchmark -N -d dbtest -t 100 -n 5500000 -l 1023 -y -h mys02d02 -P 6030 -Q
+
+        #tdSql.execute('trim database dbtest')
+        #time.sleep(3)
+        #checkFiles('/mnt/data1/vnode/*/tsdb/v*',1)
+        #checkFiles('/mnt/data2/vnode/*/tsdb/v*',1)
+        #checkFiles('/mnt/data3/vnode/*/tsdb/v*',0)
+        #checkFiles('/mnt/data4/vnode/*/tsdb/v*',1)
+
+    def run(self):
+        self.trim_database()
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())