instance: total memory metric

CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- `tnt_memory` metric.
+
 ### Changed
 
 ### Fixed

CMakeLists.txt

@@ -1,7 +1,10 @@
-cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.10...3.31 FATAL_ERROR)
 
 project(metrics NONE)
 
+# Fix problem with old third_party dependencies
+set(CMAKE_POLICY_VERSION_MINIMUM 3.5)
+
 ## Install ####################################################################
 ###############################################################################
 

doc/monitoring/metrics_reference.rst

@@ -25,6 +25,26 @@ General instance information:
 
 ..  _metrics-reference-memory_general:
 
+Instance metrics
+----------------
+
+Instance activity stats.
+These metrics can be used to monitor instance CPU and RAM usage.
+
+.. container:: table
+
+    ..  list-table::
+        :widths: 25 75
+        :header-rows: 0
+        *   -   ``tnt_cpu_user_time``
+            -   Tarantool CPU user time.
+        *   -   ``tnt_cpu_system_time``
+            -   Tarantool CPU system time.
+        *   -   ``tnt_memory``
+            -   Total number of bytes used by Tarantool instance.
+
+.. _metrics-reference-instance:
+
 Memory general
 --------------
 

metrics/psutils/psutils_linux.lua

@@ -52,6 +52,8 @@ local function parse_process_stat(path)
         comm = stats[2]:gsub('[()]', ''), -- strip spaces
         utime = tonumber(stats[14]),
         stime = tonumber(stats[15]),
+        vsize = tonumber(stats[23]),
+        rss = tonumber(stats[24]),
     }
 end
 
@@ -66,8 +68,13 @@ local function get_process_cpu_time()
     return thread_time
 end
 
+local function get_process_stats()
+    return parse_process_stat('/proc/self/stat')
+end
+
 return {
     get_cpu_time = get_cpu_time,
     get_process_cpu_time = get_process_cpu_time,
+    get_process_stats = get_process_stats,
     get_cpu_count = get_nprocs_conf,
 }

metrics/tarantool/cpu.lua

@@ -68,12 +68,12 @@ local function ss_get_rusage()
 end
 
 local function update_info_metrics()
-    local cpu_time = ss_get_rusage()
-    if cpu_time then
+    local rusage = ss_get_rusage()
+    if rusage then
         collectors_list.cpu_user_time = utils.set_gauge('cpu_user_time', 'CPU user time usage',
-            cpu_time.ru_utime, nil, nil, {default = true})
+            rusage.ru_utime, nil, nil, {default = true})
         collectors_list.cpu_system_time = utils.set_gauge('cpu_system_time', 'CPU system time usage',
-            cpu_time.ru_stime, nil, nil, {default = true})
+            rusage.ru_stime, nil, nil, {default = true})
     end
 end
 

metrics/tarantool/memory.lua

@@ -1,7 +1,19 @@
+-- local ffi = require('ffi')
 local utils = require('metrics.utils')
+local psutils = require('metrics.psutils.psutils_linux')
 
 local collectors_list = {}
 
+local sys_mem_page_size
+if jit.os == 'Linux' then
+    local handler = io.popen("getconf PAGESIZE 2>&1")
+    local output = handler:read("*a")
+    handler:close()
+    if output then
+        sys_mem_page_size = tonumber(output)
+    end
+end
+
 local function update_memory_metrics()
     if not utils.box_is_configured() then
         return
@@ -15,6 +27,50 @@ local function update_memory_metrics()
                 nil, nil, {default = true})
         end
     end
+
+    local memory_stat = 0
+
+    if jit.os == 'Linux' then
+        local data = psutils.get_process_stats()
+        memory_stat = data.rss * sys_mem_page_size
+    else
+        --[[memory]]
+        -- Skip `memory_box.data` cause in fact this is `memory_box.index`
+        -- and `memory_box.cache` cause in fact this is `vinyl_stat.memory.tuple_cache`.
+        if box.info.memory ~= nil then
+            local memory_box = box.info.memory()
+            for _, value in pairs(memory_box) do
+                memory_stat = memory_stat + value
+            end
+        end
+
+        --[[memtx]]
+        local ok, memtx_stat_3 = pcall(box.stat.memtx)
+        if ok then
+            if memtx_stat_3.data ~= nil then
+                memory_stat = memory_stat + memtx_stat_3.data.total
+            end
+            if memtx_stat_3.index ~= nil then
+                memory_stat = memory_stat + memtx_stat_3.index.total
+            end
+        end
+
+        --[[vinyl]]
+        local vinyl_stat = box.stat.vinyl()
+        if vinyl_stat ~= nil then
+            memory_stat = memory_stat + vinyl_stat.memory.tuple_cache
+                + vinyl_stat.memory.level0 + vinyl_stat.memory.page_index
+                + vinyl_stat.memory.bloom_filter
+            if vinyl_stat.memory.tuple ~= nil then
+                memory_stat = memory_stat + vinyl_stat.memory.tuple
+            end
+        end
+    end
+
+    --[[metric]]
+    collectors_list.memory_stat_usage = utils.set_gauge('memory',
+        'Tarantool instance memory usage', memory_stat,
+        nil, nil, {default = true})
 end
 
 return {

test/psutils_linux_test.lua

@@ -37,10 +37,10 @@ g.test_get_process_cpu_time = function(cg)
     cg.server:exec(function()
         local psutils_linux = require('metrics.psutils.psutils_linux')
         local expected = {
-            {pid = 1, comm = 'tarantool', utime = 468, stime = 171},
-            {pid = 12, comm = 'coio', utime = 0, stime = 0},
-            {pid = 13, comm = 'iproto', utime = 118, stime = 534},
-            {pid = 14, comm = 'wal', utime = 0, stime = 0},
+            {pid = 1, comm = 'tarantool', utime = 468, stime = 171, vsize = 867078144, rss = 12731},
+            {pid = 12, comm = 'coio', utime = 0, stime = 0, vsize = 867078144, rss = 12805},
+            {pid = 13, comm = 'iproto', utime = 118, stime = 534, vsize = 867078144, rss = 12844},
+            {pid = 14, comm = 'wal', utime = 0, stime = 0, vsize = 867078144, rss = 12915},
         }
         t.assert_items_equals(psutils_linux.get_process_cpu_time(), expected)
     end)
@@ -52,3 +52,11 @@ g.test_get_cpu_count = function(cg)
         t.assert_gt(psutils_linux.get_cpu_count(), 0)
     end)
 end
+
+g.test_get_process_stats = function(cg)
+    cg.server:exec(function()
+        local psutils_linux = require('metrics.psutils.psutils_linux')
+        local expected = {pid = 280908, comm = 'tarantool', utime = 222, stime = 111, vsize = 8679424, rss = 448,}
+        t.assert_items_equals(psutils_linux.get_process_stats(), expected)
+    end)
+end

test/psutils_linux_test_payload/init.lua

@@ -1,10 +1,12 @@
 local payload_dir = debug.sourcedir()
 local stat_file_path = payload_dir .. '/proc_stat'
+local self_stat_file_path = payload_dir .. '/proc_self_stat'
 local task_dir_path = payload_dir .. '/proc_self_task'
 
 return {
     files = {
         ['/proc/stat'] = stat_file_path,
+        ['/proc/self/stat'] = self_stat_file_path,
         ['/proc/self/task/1/stat'] = task_dir_path .. '/1/stat',
         ['/proc/self/task/12/stat'] = task_dir_path .. '/12/stat',
         ['/proc/self/task/13/stat'] = task_dir_path .. '/13/stat',

test/psutils_linux_test_payload/proc_self_stat

@@ -0,0 +1 @@
+280908 (tarantool) R 12484 280908 12484 34819 280908 4194304 101 0 0 0 222 111 0 0 20 0 1 0 50085946 8679424 448 18446744073709551615 110672370106368 110672370123953 140731302033040 0 0 0 0 0 0 0 0 0 17 9 0 0 0 0 0 110672370137744 110672370139240 110673314615296 140731302035060 140731302035080 140731302035080 140731302039531 0

test/tarantool/cpu_metrics_test.lua

@@ -14,8 +14,6 @@ end)
 
 g.test_cpu = function(cg)
     cg.server:exec(function()
-        t.skip('Flaky test, see https://github.com/tarantool/metrics/issues/492')
-
         local metrics = require('metrics')
         local cpu = require('metrics.tarantool.cpu')
         local utils = require('test.utils') -- luacheck: ignore 431

test/tarantool/memory_metrics_test.lua

@@ -0,0 +1,40 @@
+#!/usr/bin/env tarantool
+
+local t = require('luatest')
+local g = t.group('instance_metric')
+local utils = require('test.utils')
+
+g.before_all(function(cg)
+    utils.create_server(cg)
+    cg.server:exec(function()
+        box.cfg{}
+        local space = box.schema.space.create(
+            'test_space',
+            {if_not_exists = true, engine = 'vinyl'})
+        space:create_index('pk', {if_not_exists = true})
+    end)
+end)
+
+g.after_all(utils.drop_server)
+
+g.after_each(function(cg)
+    cg.server:exec(function()
+        require('metrics').clear()
+    end)
+end)
+
+g.test_instance_metrics = function(cg)
+    cg.server:exec(function()
+        local metrics = require('metrics')
+        local memory = require('metrics.tarantool.memory')
+        local utils = require('test.utils') -- luacheck: ignore 431
+
+        metrics.enable_default_metrics()
+        memory.update()
+        local default_metrics = metrics.collect{invoke_callbacks = true}
+
+        local memory_metric = utils.find_metric('tnt_memory', default_metrics)
+        t.assert(memory_metric)
+        t.assert_gt(memory_metric[1].value, 0)
+    end)
+end