Add traffic to generate data for all dashboard panels

Add endpoints responding with 4xx/5xx codes and HTTP traffic on them with luatest.
Add space on every instance and run space operations of each type with luatest.

Author: DifferentialOrange

CHANGELOG.md

@@ -7,10 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 ### Added
 - Publish dashboard on Grafana official and community built dashboards
+- Generate HTTP and space operations traffic for example cluster with luatest
 
 ### Changed
 - Update metrics version to 0.3.0 in experimental cluster
 - Rework example Tarantool instance on "cartridge.roles.metrics"
+- Rework example Tarantool docker to start multiple instances with luatest
 - Documentation improvements and fixes
 
 ## [0.1.0] - 2020-06-30

example/project/.luacheckrc

@@ -1,3 +1,15 @@
-include_files = {'**/*.lua', '*.luacheckrc', '*.rockspec'}
-exclude_files = {'.rocks/', 'tmp/'}
-max_line_length = 120
+std = {
+    read_globals = {'require', 'debug', 'pcall', 'xpcall', 'tostring',
+        'tonumber', 'type', 'assert', 'ipairs', 'math', 'error', 'string',
+        'table', 'pairs', 'os', 'io', 'select', 'unpack', 'dofile', 'next',
+        'loadstring', 'setfenv',
+        'rawget', 'rawset', '_G',
+        'getmetatable', 'setmetatable',
+        'print', 'tonumber64', 'arg'
+
+    },
+    globals = {'box', 'vshard', 'package',
+        'applier'
+    }
+}
+redefined = false

example/project/app/roles/custom.lua

@@ -1,8 +1,12 @@
 local cartridge = require('cartridge')
+local config = require('cartridge.argparse')
 
 local function init(opts) -- luacheck: no unused args
-    -- if opts.is_master then
-    -- end
+    local local_cfg = config.get_opts({
+        user = 'string',
+        password = 'string'
+    })
+
     local metrics = cartridge.service_get('metrics')
     local http_middleware = metrics.http_middleware
 
@@ -13,11 +17,44 @@ local function init(opts) -- luacheck: no unused args
         { method = 'GET', path = '/hello' },
         http_middleware.v1(
             function()
-                return { body = 'Hello world!' }
+                return { status = 200, body = 'Hello world!' }
+            end,
+            http_collector
+        )
+    )
+    httpd:route(
+        { method = 'GET', path = '/hell0' },
+        http_middleware.v1(
+            function()
+                return { status = 400, body = 'Hell0 world!' }
             end,
             http_collector
         )
     )
+    httpd:route(
+        { method = 'POST', path = '/goodbye' },
+        http_middleware.v1(
+            function()
+                return { status = 500, body = 'Goodbye cruel world!' }
+            end,
+            http_collector
+        )
+    )
+
+    if opts.is_master then
+        local sp = box.schema.space.create('MY_SPACE', { if_not_exists = true })
+        sp:format({
+            { name = 'key', type = 'number', is_nullable = false },
+            { name = 'value', type = 'string', is_nullable = false },
+        })
+        sp:create_index('pk', { parts = { 'key' }, if_not_exists = true })
+
+        if local_cfg.user and local_cfg.password then
+            -- cluster-wide user privileges
+            box.schema.user.create(local_cfg.user, { password = local_cfg.password, if_not_exists = true })
+            box.schema.user.grant(local_cfg.user, 'read,write,execute', 'universe', nil, { if_not_exists = true })
+        end
+    end
 
     return true
 end

example/project/cluster/integration/bootstrap_test.lua

@@ -1,21 +1,31 @@
 local test = require('luatest')
 local group = test.group('cluster')
-local fiber = require('fiber')
 
 local helper = require('test.helper')
 local cluster_helpers = require('cartridge.test-helpers')
 
+local app = 'example-app'
+local log_level = 6
+
+local cluster_alias = {
+    tnt_router = 'tnt_router',
+    tnt_storage_1_master = 'tnt_storage_1_master',
+    tnt_storage_1_replica = 'tnt_storage_1_replica',
+    tnt_storage_2_master = 'tnt_storage_2_master',
+    tnt_storage_2_replica = 'tnt_storage_2_replica',
+}
+
 local replicasets = {
     {
         uuid = cluster_helpers.uuid('a'),
         roles = { 'vshard-router', 'app.roles.custom' },
         servers = {
             {
                 instance_uuid = cluster_helpers.uuid('a', 1),
-                alias = 'tnt-router',
+                alias = cluster_alias.tnt_router,
                 env = {
-                    ['TARANTOOL_APP_NAME'] = 'example-app',
-                    ['TARANTOOL_LOG_LEVEL'] = 6,
+                    ['TARANTOOL_APP_NAME'] = app,
+                    ['TARANTOOL_LOG_LEVEL'] = log_level,
                 }
             }
         },
@@ -26,18 +36,18 @@ local replicasets = {
         servers = {
             {
                 instance_uuid = cluster_helpers.uuid('b', 1),
-                alias = 'tnt-storage-1-master',
+                alias = cluster_alias.tnt_storage_1_master,
                 env = {
-                    ['TARANTOOL_APP_NAME'] = 'example-app',
-                    ['TARANTOOL_LOG_LEVEL'] = 6,
+                    ['TARANTOOL_APP_NAME'] = app,
+                    ['TARANTOOL_LOG_LEVEL'] = log_level,
                 }
             },
             {
                 instance_uuid = cluster_helpers.uuid('b', 2),
-                alias = 'tnt-storage-1-replica',
+                alias = cluster_alias.tnt_storage_1_replica,
                 env = {
-                    ['TARANTOOL_APP_NAME'] = 'example-app',
-                    ['TARANTOOL_LOG_LEVEL'] = 6,
+                    ['TARANTOOL_APP_NAME'] = app,
+                    ['TARANTOOL_LOG_LEVEL'] = log_level,
                 }
             },
         }
@@ -48,31 +58,24 @@ local replicasets = {
         servers = {
             {
                 instance_uuid = cluster_helpers.uuid('c', 1),
-                alias = 'tnt-storage-2-master',
+                alias = cluster_alias.tnt_storage_2_master,
                 env = {
-                    ['TARANTOOL_APP_NAME'] = 'example-app',
-                    ['TARANTOOL_LOG_LEVEL'] = 6,
+                    ['TARANTOOL_APP_NAME'] = app,
+                    ['TARANTOOL_LOG_LEVEL'] = log_level,
                 }
             },
             {
                 instance_uuid = cluster_helpers.uuid('c', 2),
-                alias = 'tnt-storage-2-replica',
+                alias = cluster_alias.tnt_storage_2_replica,
                 env = {
-                    ['TARANTOOL_APP_NAME'] = 'example-app',
-                    ['TARANTOOL_LOG_LEVEL'] = 6,
+                    ['TARANTOOL_APP_NAME'] = app,
+                    ['TARANTOOL_LOG_LEVEL'] = log_level,
                 }
             }
         },
     }
 }
 
-local instances = {}
-for _, replicaset in ipairs(replicasets) do
-    for _, server in ipairs(replicaset.servers) do
-        table.insert(instances, server.alias)
-    end
-end
-
 test.before_suite(function()
     group.cluster = cluster_helpers.Cluster:new({
         datadir = helper.datadir,
@@ -102,25 +105,128 @@ test.after_suite(function()
     group.cluster:stop()
 end)
 
-local function http_request(alias, count)
+local tnt_router = group.cluster:server(cluster_alias.tnt_router)
+local tnt_storage_1_master = group.cluster:server(cluster_alias.tnt_storage_1_master)
+local tnt_storage_1_replica = group.cluster:server(cluster_alias.tnt_storage_1_replica)
+local tnt_storage_2_master = group.cluster:server(cluster_alias.tnt_storage_2_master)
+local tnt_storage_2_replica = group.cluster:server(cluster_alias.tnt_storage_2_replica)
+
+-- Space operations constants
+local SELECT = 'select'
+local INSERT = 'insert'
+local UPDATE = 'update'
+local UPSERT = 'upsert'
+local REPLACE = 'replace'
+local DELETE = 'delete'
+
+-- HTTP methods constants
+local GET = 'get'
+local POST = 'post'
+
+local function http_request(server, method, endpoint, count)
     if count <= 0 then
         return
     end
 
     for _ = 1, count do
-        group.cluster:server(alias):http_request('get', '/hello')
+        server:http_request(method, endpoint, { raise = false })
+    end
+end
+
+local last_key = 1
+
+local charset = {} -- [0-9a-zA-Z]
+for c = 48, 57  do table.insert(charset, string.char(c)) end
+for c = 65, 90  do table.insert(charset, string.char(c)) end
+for c = 97, 122 do table.insert(charset, string.char(c)) end
+
+local function random_string(length)
+    if not length or length <= 0 then return '' end
+    math.randomseed(os.clock()^5)
+    return random_string(length - 1) .. charset[math.random(1, #charset)]
+end
+
+local function space_operations(server, operation, count)
+    if count <= 0 then
+        return
+    end
+
+    local space = server.net_box.space.MY_SPACE
+
+    if operation == SELECT then
+        for _ = 1, count do
+            space:select({}, { limit = 1 })
+        end
+
+    elseif operation == INSERT then
+        for _ = 1, count do
+            space:insert{ last_key, random_string(5) }
+            last_key = last_key + 1
+        end
+
+    elseif operation == UPDATE then
+        for _ = 1, count do
+            local key = space:select({}, { limit = 1 })[1][1]
+            space:update(key, {{ '=', 2, random_string(5) }})
+        end
+
+    elseif operation == UPSERT then
+        for _ = 1, count do
+            local tuple = space:select({}, { limit = 1 })[1]
+            space:upsert(tuple, {{ '=', 2, random_string(5) }})
+        end
+
+    elseif operation == REPLACE then
+        for _ = 1, count do
+            local key = space:select({}, { limit = 1 })[1][1]
+            space:replace{ key, random_string(5) }
+        end
+
+    elseif operation == DELETE then
+        for _ = 1, count do
+            local key = space:select({}, { limit = 1 })[1][1]
+            space:delete{ key }
+        end
     end
 end
 
 group.test_cluster = function()
     test.helpers.retrying({ timeout = math.huge },
         function()
-            fiber.sleep(10)
+            -- Generate some HTTP traffic
+            http_request(tnt_router, GET, '/hello', math.random(5, 10))
+            http_request(tnt_router, GET, '/hell0', math.random(1, 2))
+            http_request(tnt_router, POST, '/goodbye', math.random(0, 1))
+            http_request(tnt_storage_1_master, GET, '/hello', math.random(2, 5))
+            http_request(tnt_storage_1_master, GET, '/hell0', math.random(0, 1))
+            http_request(tnt_storage_1_master, POST, '/goodbye', math.random(0, 1))
+            http_request(tnt_storage_1_replica, GET, '/hello', math.random(1, 3))
+            http_request(tnt_storage_1_replica, GET, '/hell0', math.random(0, 1))
+            http_request(tnt_storage_1_replica, POST, '/goodbye', math.random(0, 1))
+            http_request(tnt_storage_2_master, GET, '/hello', math.random(2, 5))
+            http_request(tnt_storage_2_master, GET, '/hell0', math.random(0, 1))
+            http_request(tnt_storage_2_master, POST, '/goodbye', math.random(0, 1))
+            http_request(tnt_storage_2_replica, GET, '/hello', math.random(1, 3))
+            http_request(tnt_storage_2_replica, GET, '/hell0', math.random(0, 1))
+            http_request(tnt_storage_2_replica, POST, '/goodbye', math.random(0, 1))
 
-            -- Generate some traffic
-            for _, alias in ipairs(instances) do
-                http_request(alias, math.random(0, 5))
-            end
+            -- Generate some space traffic
+            space_operations(tnt_router, INSERT, math.random(1, 3))
+            space_operations(tnt_router, UPDATE, math.random(1, 3))
+            space_operations(tnt_storage_1_master, INSERT, math.random(5, 10))
+            space_operations(tnt_storage_1_master, SELECT, math.random(10, 20))
+            space_operations(tnt_storage_1_master, UPDATE, math.random(5, 10))
+            space_operations(tnt_storage_1_master, UPSERT, math.random(5, 10))
+            space_operations(tnt_storage_1_master, REPLACE, math.random(5, 10))
+            space_operations(tnt_storage_1_master, DELETE, math.random(1, 2))
+            space_operations(tnt_storage_1_replica, SELECT, math.random(3, 5))
+            space_operations(tnt_storage_2_master, INSERT, math.random(5, 10))
+            space_operations(tnt_storage_2_master, SELECT, math.random(10, 20))
+            space_operations(tnt_storage_2_master, UPDATE, math.random(5, 10))
+            space_operations(tnt_storage_2_master, UPSERT, math.random(5, 10))
+            space_operations(tnt_storage_2_master, REPLACE, math.random(5, 10))
+            space_operations(tnt_storage_2_master, DELETE, math.random(1, 2))
+            space_operations(tnt_storage_2_replica, SELECT, math.random(3, 5))
 
             -- Fail this function so cluster don't stop
             error('running cluster')