rm TARANTOOL_AUTO_DISABLE_VSHARD_STORAGE

Author: Satbek

CHANGELOG.rst

@@ -19,9 +19,6 @@ Added
 - New role callback ``before_apply_config`` to run some code before applying
   configuration changes.
 
-- Env variable ``TARANTOOL_AUTO_DISABLE_VSHARD_STORAGE`` to automatically disable
-  VShard storage while applying config.
-
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Changed
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

cartridge/roles/vshard-storage.lua

@@ -25,7 +25,6 @@ vars:new('instance_uuid')
 vars:new('replicaset_uuid')
 vars:new('issues', {})
 vars:new('enable_alerting', false)
-vars:new('auto_disable', false)
 local _G_vshard_backup
 
 local function apply_config(conf, opts)
@@ -69,11 +68,9 @@ local function init()
 
     local opts, err = require('cartridge.argparse').get_opts({
         add_vshard_storage_alerts_to_issues = 'boolean',
-        auto_disable_vshard_storage = 'boolean',
     })
     if err == nil then
         vars.enable_alerting = opts.add_vshard_storage_alerts_to_issues
-        vars.auto_disable = opts.auto_disable_vshard_storage
     end
 end
 
@@ -82,9 +79,10 @@ end
 if rawget(_G, '__cartridge_vshard_first_apply_config') == nil then
     rawset(_G, '__cartridge_vshard_first_apply_config', true)
 end
+
 local function before_apply_config(_)
     local first_apply = rawget(_G, '__cartridge_vshard_first_apply_config')
-    if first_apply or vars.auto_disable then
+    if first_apply then
         vshard.storage.disable()
     end
     rawset(_G, '__cartridge_vshard_first_apply_config', false)

test/integration/vshard_storage_disabling_test.lua

@@ -1,21 +1,42 @@
 local fio = require('fio')
 local t = require('luatest')
-local g_auto = t.group('integration.vshard_storage_disabling.auto')
-local g_default = t.group('integration.vshard_storage_disabling.default')
+local g = t.group()
 
 local helpers = require('test.helper')
 
+
+local function initialize_functions(srv)
+    return srv:exec(function()
+        local confapplier = require('cartridge.confapplier')
+        local function get(...)
+            if confapplier.get_state() == 'OperationError' then
+                error('Invalid state')
+            end
+            return box.space.test:get(...)
+        end
+        local function put(...)
+            if confapplier.get_state() == 'OperationError' then
+                error('Invalid state')
+            end
+            return box.space.test:put(...)
+        end
+        rawset(_G, 'get', get)
+        rawset(_G, 'put', put)
+    end)
+end
+
 local function setup_replica_backoff_interval(srv)
     srv:exec(function()
         require('vshard.consts').REPLICA_BACKOFF_INTERVAL = 0.1
     end)
 end
-local function setup_cluster(g, auto_disable)
+
+g.before_all = function()
     t.skip_if(not helpers.tarantool_version_ge('1.10.1'))
     g.cluster = helpers.Cluster:new({
         datadir = fio.tempdir(),
         server_command = helpers.entrypoint('srv_basic'),
-        cookie = helpers.random_cookie(),
+        cookie = 'cookie',
         use_vshard = true,
         replicasets = {{
             alias = 'router',
@@ -33,16 +54,38 @@ local function setup_cluster(g, auto_disable)
         }},
         env = {
             TARANTOOL_BUCKET_COUNT = 300,
-            TARANTOOL_AUTO_DISABLE_VSHARD_STORAGE = tostring(auto_disable),
         }
     })
     g.cluster:start()
     g.router = g.cluster:server('router-1')
     g.storage_master = g.cluster:server('storage-1')
     g.storage_replica = g.cluster:server('storage-2')
 
+    local test_schema = {
+        engine = 'memtx',
+        is_local = false,
+        temporary = false,
+        format = {
+            {name = 'bucket_id', type = 'unsigned', is_nullable = false},
+            {name = 'record_id', type = 'unsigned', is_nullable = false},
+        },
+        indexes = {{
+            name = 'pk', type = 'TREE', unique = true,
+            parts = {{path = 'record_id', is_nullable = false, type = 'unsigned'}},
+        },  {
+            name = 'bucket_id', type = 'TREE', unique = false,
+            parts = {{path = 'bucket_id', is_nullable = false, type = 'unsigned'}},
+        }},
+        sharding_key = {'record_id'},
+    }
+    g.cluster.main_server:call('cartridge_set_schema',
+        {require('yaml').encode({spaces = {test = test_schema}})}
+    )
+
     g.cluster:wait_until_healthy()
 
+    initialize_functions(g.storage_master)
+    initialize_functions(g.storage_replica)
     -- Just to speedup tests
     setup_replica_backoff_interval(g.router)
 
@@ -56,6 +99,17 @@ local function setup_cluster(g, auto_disable)
     end)
 end
 
+g.after_all = function()
+    g.cluster:stop()
+    fio.rmtree(g.cluster.datadir)
+end
+
+g.after_each = function()
+    g.storage_master:exec(function()
+        box.space.test:truncate()
+    end)
+end
+
 local function inject_operation_error(srv)
     helpers.run_remotely(srv, function()
         local myrole = package.loaded['mymodule']
@@ -73,127 +127,161 @@ local function dismiss_operation_error(srv)
     end)
 end
 
+local function get_state(srv)
+    return srv:eval('return require("cartridge.confapplier").get_state()')
+end
+
+local function get(srv, i)
+    return srv:eval(
+        'return require("vshard").router.callro(...)',
+        {i, 'get', {i}}
+    )
+end
+
+local function put(srv, i)
+    return srv:eval('return require("vshard").router.callrw(...)',
+        {i, 'put', {{i, i, string.format('i%04d', i)}}}
+    )
+end
+
 local function apply_config(srv)
-    return srv:exec(function()
-        return require("cartridge").config_patch_clusterwide({uuid = require("uuid").str()})
-    end)
+    return srv:eval('return require("cartridge").config_patch_clusterwide({uuid = require("uuid").str()})')
 end
 
-for _, case in pairs({{g_auto, true}, {g_default, false}}) do
-    local g, auto_disable = case[1], case[2]
-
-    local function set_failover(mode)
-        local response = g.cluster.main_server:graphql({
-            query = [[
-                mutation($mode: String) {
-                    cluster {
-                        failover_params(
-                            mode: $mode
-                        ) {
-                            mode
-                        }
-                    }
-                }
-            ]],
-            variables = { mode = mode },
-            raise = false,
-        })
-        if response.errors then
-            error(response.errors[1].message, 2)
-        end
+function g.test_vshard_storage_disable()
+    -- Read/write data from/to healthy cluster
+    t.assert_equals(put(g.router, 1), {1, 1, 'i0001'})
+    t.assert_equals(get(g.router, 1), {1, 1, 'i0001'})
+
+    -- Break master instance
+    inject_operation_error(g.storage_master)
+    apply_config(g.router)
+    t.assert_equals(get_state(g.storage_master), 'OperationError')
+    t.assert_equals(get_state(g.storage_replica), 'RolesConfigured')
+
+    for _ = 1, 1000 do
+        t.assert_equals(get(g.router, 1), {1, 1, 'i0001'})
     end
 
-    g.before_all = function()
-        setup_cluster(g, auto_disable)
+    local data, err = put(g.router, 2)
+    t.assert_equals(data, box.NULL)
+    t.assert_equals(err.message, 'Storage is disabled: storage is disabled explicitly')
+
+    -- Fix master state
+    dismiss_operation_error(g.storage_master)
+    apply_config(g.router)
+    t.assert_not_equals(get_state(g.storage_master), 'OperationError')
+    t.assert_not_equals(get_state(g.storage_replica), 'OperationError')
+
+    t.assert_equals(put(g.router, 2), {2, 2, 'i0002'})
+    for _ = 1, 1000 do
+        t.assert_equals(get(g.router, 2), {2, 2, 'i0002'})
     end
 
-    g.after_all = function()
-        g.cluster:stop()
-        fio.rmtree(g.cluster.datadir)
+    -- Break replica instance
+    inject_operation_error(g.storage_replica)
+    apply_config(g.router)
+    t.assert_not_equals(get_state(g.storage_master), 'OperationError')
+    t.assert_equals(get_state(g.storage_replica), 'OperationError')
+
+    t.assert_equals(put(g.router, 3), {3, 3, 'i0003'})
+    for _ = 1, 1000 do
+        t.assert_equals(get(g.router, 3), {3, 3, 'i0003'})
     end
 
-    g.before_test('test_disabled_on_failover', function()
-        set_failover('eventual')
-    end)
+    -- Fix replica state
+    dismiss_operation_error(g.storage_replica)
+    apply_config(g.router)
+    t.assert_not_equals(get_state(g.storage_master), 'OperationError')
+    t.assert_not_equals(get_state(g.storage_replica), 'OperationError')
 
-    function g.test_disabled_on_failover()
-        -- Break replica instance
-        inject_operation_error(g.storage_replica)
+    t.assert_equals(put(g.router, 4), {4, 4, 'i0004'})
+    for _ = 1, 1000 do
+        t.assert_equals(get(g.router, 4), {4, 4, 'i0004'})
+    end
+end
 
-        g.storage_master:stop()
+local function set_failover(mode)
+    local response = g.cluster.main_server:graphql({
+        query = [[
+            mutation($mode: String) {
+                cluster {
+                    failover_params(
+                        mode: $mode
+                    ) {
+                        mode
+                    }
+                }
+            }
+        ]],
+        variables = { mode = mode },
+        raise = false,
+    })
+    if response.errors then
+        error(response.errors[1].message, 2)
+    end
+end
 
-        helpers.retrying({timeout = 30}, function()
-            g.storage_replica:exec(function()
-                assert(box.info.ro == false)
-            end)
-        end)
+function g.test_vshard_storage_disable_on_failover()
+    set_failover('eventual')
+
+    -- Read/write data from/to healthy cluster
+    t.assert_equals(put(g.router, 1), {1, 1, 'i0001'})
+    t.assert_equals(get(g.router, 1), {1, 1, 'i0001'})
+
+    -- Break replica instance
+    inject_operation_error(g.storage_replica)
+
+    g.storage_master:stop()
 
-        local res = g.storage_replica:exec(function()
-            local cartridge = require('cartridge')
-            return cartridge.service_get('myrole').was_vshard_enabled_on_apply()
+    helpers.retrying({timeout = 30}, function()
+        g.storage_replica:exec(function()
+            assert(box.info.ro == false)
         end)
-        if auto_disable then
-            for _, v in ipairs(res) do
-                t.assert_not(v)
-            end
-        else
-            local res = g.storage_replica:exec(function()
-                local vshard = rawget(_G, 'vshard')
-                return vshard.storage.internal.is_enabled
-            end)
-            t.assert_not(res)
-        end
-    end
+    end)
 
-    g.after_test('test_disabled_on_failover', function ()
-        g.storage_master:start()
-        g.cluster:wait_until_healthy()
-        dismiss_operation_error(g.storage_replica)
-        set_failover('disabled')
+    helpers.retrying({}, function()
+        local data, err = put(g.router, 2)
+        t.assert_equals(data, box.NULL)
+        t.assert_equals(err.message, 'Storage is disabled: storage is disabled explicitly')
     end)
 
-    function g.test_disabled_on_apply_config()
-        inject_operation_error(g.storage_replica)
-        apply_config(g.router)
+    g.storage_master:start()
+    g.cluster:wait_until_healthy()
+    initialize_functions(g.storage_master)
 
-        local res = g.storage_replica:exec(function()
-            local cartridge = require('cartridge')
-            return cartridge.service_get('myrole').was_vshard_enabled_on_apply()
-        end)
-        if auto_disable then
-            for _, v in ipairs(res) do
-                t.assert_not(v)
-            end
-        else
-            local res = g.storage_replica:exec(function()
-                local vshard = rawget(_G, 'vshard')
-                return vshard.storage.internal.is_enabled
-            end)
-            t.assert_not(res)
-        end
+    -- Fix replica state
+    dismiss_operation_error(g.storage_replica)
+
+    g.cluster:wait_until_healthy()
+    t.assert_not_equals(get_state(g.storage_master), 'OperationError')
+    t.assert_not_equals(get_state(g.storage_replica), 'OperationError')
+
+    t.assert_equals(put(g.router, 2), {2, 2, 'i0002'})
+    for _ = 1, 1000 do
+        t.assert_equals(get(g.router, 2), {2, 2, 'i0002'})
     end
 
-    g.after_test('test_disabled_on_apply_config', function ()
-        dismiss_operation_error(g.storage_replica)
-        apply_config(g.router)
-    end)
+    set_failover('disabled')
 end
 
-g_default.test_disabled_on_first_apply = function ()
-    local res = g_default.storage_master:exec(function()
+g.test_disabled_on_first_apply = function ()
+    local res = g.storage_master:exec(function()
         local cartridge = require('cartridge')
         return cartridge.service_get('myrole').was_vshard_enabled_on_apply()
     end)
 
+    -- last applied config
+    apply_config(g.router)
+
     t.assert_not(res[1])
-    for i = 2, #res do
-        t.assert(res[i])
-    end
+    t.assert(res[#res], 'enabled on last apply')
 
-    local res = g_default.storage_replica:exec(function()
+    res = g.storage_replica:exec(function()
         local cartridge = require('cartridge')
         return cartridge.service_get('myrole').was_vshard_enabled_on_apply()
     end)
 
     t.assert_not(res[1])
+    t.assert(res[#res], 'enabled on last apply')
 end