Merge pull request #9174 from romayalon/romy-nc-health-bucket-account-check

romayalon · web-flow · commit 5b2f75c18257 · 2025-08-06T11:12:54.000+03:00
NC | Health | Add buckets/accounts count and limit check
diff --git a/config.js b/config.js
@@ -1041,6 +1041,9 @@ config.NC_DISABLE_SCHEMA_CHECK = false;
 config.ENTROPY_DISK_SIZE_THRESHOLD = 100 * 1024 * 1024;
 config.ENTROPY_MIN_THRESHOLD = 512;
 
+config.NC_HEALTH_BUCKETS_COUNT_LIMIT_WARNING = 5000;
+config.NC_HEALTH_ACCOUNTS_COUNT_LIMIT_WARNING = 5000;
+
 ////////// NC LIFECYLE  //////////
 
 config.NC_LIFECYCLE_LOGS_DIR = '/var/log/noobaa/lifecycle';
diff --git a/docs/NooBaaNonContainerized/Health.md b/docs/NooBaaNonContainerized/Health.md
@@ -26,14 +26,16 @@ For more details about NooBaa RPM installation, see - [Getting Started](./Gettin
   - `NooBaa endpoints health`
     - Ensuring that the NooBaa endpoint is running.
   - `NooBaa accounts Health`
-    - Iterating accounts under the config directory.
+    - Counting and Iterating accounts under the config directory.
     - Confirming the existence of the account's configuration file and its validity as a JSON file.
     - Verifying that the account's `new_buckets_path` is defined and `allow_bucket_creation` is set to true.
     - Ensuring that the account has read and write access to its new_buckets_path.
+    - Verifying that the number of accounts didn't exceed the supported/tested limit.
   - `NooBaa buckets health` 
-    - Iterating buckets under the config directory.
+    - Counting and Iterating buckets under the config directory.
     - Confirming the existence of the bucket's configuration file and its validity as a JSON file.
     - Verifying that the underlying storage path of a bucket exists.
+    - Verifying that the number of buckets didn't exceed the supported/tested limit.
   - `Config directory health`
     - checks if config system and directory data exists
     - returns the config directory status 
@@ -128,6 +130,10 @@ The output of the Health CLI is a JSON object containing the following propertie
     - Enum: See - [Health Errors](#health-errors)
     - Description: Error message describing a health issue uncovered by the Health CLI. Will be displayed in Health response if status is NOTOK.
 
+- `warnings` 
+    - Type: Array [{warning_code: String, warning_message: String}]
+    - Description: Warnings is an array that consists warnings that raised during the health script.
+
 - `service_status`
     - Type: String
     - Enum: "active" | "inactive" | "missing service status info"
@@ -221,6 +227,7 @@ Output:
     "error_code": "NOOBAA_SERVICE_FAILED",
     "error_message": "NooBaa service is not started properly, Please verify the service with status command."
   },
+  "warnings": [],
   "checks": {
     "services": [
       {
@@ -244,6 +251,7 @@ Output:
       "error_type": "TEMPORARY"
     },
     "accounts_status": {
+      "count": 3,
       "invalid_accounts": [
         {
           "name": "account_invalid",
@@ -263,6 +271,7 @@ Output:
       "error_type": "PERSISTENT"
     },
     "buckets_status": {
+      "count": 3,
       "invalid_buckets": [
         {
           "name": "bucket1.json",
@@ -283,7 +292,8 @@ Output:
       ],
       "error_type": "PERSISTENT"
     },
-    "connectoins_status": {
+    "connectins_status": {
+      "count": 2,
       "invalid_connections": [
         {
           "name": "notif_invalid",
@@ -464,3 +474,23 @@ The following error codes will be associated with a specific Bucket or Account s
     - Start NooBaa service
     - Run `noobaa-cli upgrade`
     - Check the in_progress_upgrade the exact reason for the failure.
+
+## Health Warnings
+
+If the Health CLI encounters some issues, the following warnings will be pushed to a warnings array in the health output.
+
+#### 1. Buckets count limit warning
+  - Warning code: `BUCKETS_COUNT_LIMIT_WARNING`
+  - Warning message: Number of buckets exceeds the limit of ${config.NC_HEALTH_BUCKETS_COUNT_LIMIT_WARNING}
+  - Reasons:
+    - The number of buckets exceeds the supported or tested limit, which may lead to unexpected behavior.
+  - Resolutions:
+    - Delete buckets until the total count is below the supported or tested limit.
+
+#### 1. Accounts count limit warning
+  - Warning code: `ACCOUNTS_COUNT_LIMIT_WARNING`
+  - Warning message: Number of accounts exceeds the limit of ${config.NC_HEALTH_ACCOUNTS_COUNT_LIMIT_WARNING}
+  - Reasons:
+    - The number of accounts exceeds the supported or tested limit, which may lead to unexpected behavior.
+  - Resolutions:
+    - Delete accounts until the total count is below the supported or tested limit.
diff --git a/src/manage_nsfs/health.js b/src/manage_nsfs/health.js
@@ -73,6 +73,17 @@ const health_errors = {
     }
 };
 
+const health_warnings = {
+    BUCKETS_COUNT_LIMIT_WARNING: {
+        warning_code: 'BUCKETS_COUNT_LIMIT_WARNING',
+        warning_message: `Number of buckets exceeds the supported limit of ${config.NC_HEALTH_BUCKETS_COUNT_LIMIT_WARNING}.`,
+    },
+    ACCOUNTS_COUNT_LIMIT_WARNING: {
+        warning_code: 'ACCOUNTS_COUNT_LIMIT_WARNING',
+        warning_message: `Number of accounts exceeds the supported limit of ${config.NC_HEALTH_ACCOUNTS_COUNT_LIMIT_WARNING}.`,
+    }
+};
+
 const fork_response_code = {
     RUNNING: {
         response_code: 'RUNNING',
@@ -146,12 +157,13 @@ class NSFSHealth {
         if (this.all_connection_details) connection_details = await this.get_connection_status();
         if (this.notif_storage_threshold) notif_storage_threshold_details = this.get_notif_storage_threshold_status();
         if (this.lifecycle) latest_lifecycle_run_status = await this.get_lifecycle_health_status();
-
+        const warnings = this.get_warnings_array({ buckets_count: bucket_details?.count, accounts_count: account_details?.count });
         const health = {
             service_name: NOOBAA_SERVICE_NAME,
             status: service_health,
             memory: memory,
             error: error_code,
+            warnings,
             checks: {
                 services: [noobaa_service_state],
                 endpoint: {
@@ -160,11 +172,13 @@ class NSFSHealth {
                 },
                 config_directory_status,
                 accounts_status: {
+                    count: account_details?.count || undefined,
                     invalid_accounts: account_details === undefined ? undefined : account_details.invalid_storages,
                     valid_accounts: account_details === undefined ? undefined : account_details.valid_storages,
                     error_type: health_errors_tyes.PERSISTENT,
                 },
                 buckets_status: {
+                    count: bucket_details?.count || undefined,
                     invalid_buckets: bucket_details === undefined ? undefined : bucket_details.invalid_storages,
                     valid_buckets: bucket_details === undefined ? undefined : bucket_details.valid_storages,
                     error_type: health_errors_tyes.PERSISTENT,
@@ -220,6 +234,25 @@ class NSFSHealth {
         }
     }
 
+    /**
+     * get_warnings_array returns an array of warnings based on the health check parameters
+     * @param {{
+     * buckets_count?: Number,
+     * accounts_count?: Number
+     * }} params 
+     * @returns {Array[Object]}
+     */
+    get_warnings_array({ buckets_count = 0, accounts_count = 0 }) {
+        const warnings = [];
+        if (this.all_bucket_details && buckets_count > config.NC_HEALTH_BUCKETS_COUNT_LIMIT_WARNING) {
+            warnings.push(health_warnings.BUCKETS_COUNT_LIMIT_WARNING);
+        }
+        if (this.all_account_details && accounts_count > config.NC_HEALTH_ACCOUNTS_COUNT_LIMIT_WARNING) {
+            warnings.push(health_warnings.ACCOUNTS_COUNT_LIMIT_WARNING);
+        }
+        return warnings;
+    }
+
     async get_service_state(service_name) {
         let pid;
         const service_status = await get_service_status(service_name);
@@ -430,6 +463,7 @@ class NSFSHealth {
             }
         }
         return {
+            count: valid_storages.length + invalid_storages.length,
             invalid_storages: invalid_storages,
             valid_storages: valid_storages
         };
@@ -818,4 +852,5 @@ function _should_skip_health_access_check() {
 }
 
 exports.get_health_status = get_health_status;
+exports.health_warnings = health_warnings;
 exports.NSFSHealth = NSFSHealth;
diff --git a/src/test/integration_tests/nc/cli/test_nc_health.js b/src/test/integration_tests/nc/cli/test_nc_health.js
@@ -1,5 +1,5 @@
 /* Copyright (C) 2016 NooBaa */
-/*eslint max-lines-per-function: ['error', 700]*/
+/*eslint max-lines-per-function: ['error', 800]*/
 
 'use strict';
 
@@ -27,6 +27,7 @@ const DEFAULT_FS_CONFIG = get_process_fs_context();
 
 const bucket_storage_path = path.join(tmp_fs_path, 'bucket_storage_path');
 const os = require('os');
+const { health_warnings } = require('../../../../manage_nsfs/health');
 const hostname = os.hostname();
 
 const valid_system_json = {
@@ -134,6 +135,9 @@ mocha.describe('nsfs nc health', function() {
     mocha.describe('health check', function() {
         this.timeout(10000);// eslint-disable-line no-invalid-this
         const new_buckets_path = `${root_path}new_buckets_path_user1/`;
+        const orig_health_buckets_count_limit = config.NC_HEALTH_BUCKETS_COUNT_LIMIT_WARNING;
+        const orig_health_accounts_count_limit = config.NC_HEALTH_ACCOUNTS_COUNT_LIMIT_WARNING;
+
         const account1_options = {
             name: 'account1',
             uid: process.getuid(),
@@ -195,6 +199,8 @@ mocha.describe('nsfs nc health', function() {
         mocha.afterEach(async () => {
             await fs_utils.file_delete(config_fs.config_json_path);
             restore_health_if_needed(Health);
+            config.NC_HEALTH_BUCKETS_COUNT_LIMIT_WARNING = orig_health_buckets_count_limit;
+            config.NC_HEALTH_ACCOUNTS_COUNT_LIMIT_WARNING = orig_health_accounts_count_limit;
         });
 
         mocha.it('Health all condition is success', async function() {
@@ -640,6 +646,89 @@ mocha.describe('nsfs nc health', function() {
             await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_invalid.name });
         });
 
+        mocha.it('accounts count below limit', async function() {
+            Health.all_account_details = true;
+            Health.all_bucket_details = true;
+            test_utils.set_health_mock_functions(Health, {
+                get_service_state: get_service_state_mock_default_response,
+                get_endpoint_response: get_endpoint_response_mock_default_response,
+                get_system_config_file: get_system_config_mock_default_response
+            });
+            const health_status = await Health.nc_nsfs_health();
+            console.log('health_status', health_status);
+            assert.strictEqual(health_status.checks.buckets_status.count, 1);
+            assert.strictEqual(health_status.checks.accounts_status.count, 1);
+            assert.strictEqual(health_status.warnings.length, 0);
+        });
+
+        mocha.it('accounts count above limit - should emit warning', async function() {
+            Health.all_account_details = true;
+            Health.all_bucket_details = true;
+            config.NC_HEALTH_ACCOUNTS_COUNT_LIMIT_WARNING = 0;
+            test_utils.set_health_mock_functions(Health, {
+                get_service_state: get_service_state_mock_default_response,
+                get_endpoint_response: get_endpoint_response_mock_default_response,
+                get_system_config_file: get_system_config_mock_default_response
+            });
+            const health_status = await Health.nc_nsfs_health();
+            console.log('health_status', health_status);
+            assert.strictEqual(health_status.checks.buckets_status.count, 1);
+            assert.strictEqual(health_status.checks.accounts_status.count, 1);
+            assert.strictEqual(health_status.warnings.length, 1);
+            assert.strictEqual(health_status.warnings.includes(health_warnings.ACCOUNTS_COUNT_LIMIT_WARNING), true);
+        });
+
+        mocha.it('buckets count below limit', async function() {
+            Health.all_account_details = true;
+            Health.all_bucket_details = true;
+            test_utils.set_health_mock_functions(Health, {
+                get_service_state: get_service_state_mock_default_response,
+                get_endpoint_response: get_endpoint_response_mock_default_response,
+                get_system_config_file: get_system_config_mock_default_response
+            });
+            const health_status = await Health.nc_nsfs_health();
+            console.log('health_status', health_status);
+            assert.strictEqual(health_status.checks.buckets_status.count, 1);
+            assert.strictEqual(health_status.checks.accounts_status.count, 1);
+            assert.strictEqual(health_status.warnings.length, 0);
+        });
+
+        mocha.it('buckets count above limit - should emit warning', async function() {
+            config.NC_HEALTH_BUCKETS_COUNT_LIMIT_WARNING = 0;
+            Health.all_account_details = true;
+            Health.all_bucket_details = true;
+            test_utils.set_health_mock_functions(Health, {
+                get_service_state: get_service_state_mock_default_response,
+                get_endpoint_response: get_endpoint_response_mock_default_response,
+                get_system_config_file: get_system_config_mock_default_response
+            });
+            const health_status = await Health.nc_nsfs_health();
+            console.log('health_status', health_status);
+            assert.strictEqual(health_status.checks.buckets_status.count, 1);
+            assert.strictEqual(health_status.checks.accounts_status.count, 1);
+            assert.strictEqual(health_status.warnings.length, 1);
+            assert.strictEqual(health_status.warnings.includes(health_warnings.BUCKETS_COUNT_LIMIT_WARNING), true);
+        });
+
+        mocha.it('buckets and accounts count above limit - should emit warnings', async function() {
+            Health.all_account_details = true;
+            Health.all_bucket_details = true;
+            config.NC_HEALTH_BUCKETS_COUNT_LIMIT_WARNING = 0;
+            config.NC_HEALTH_ACCOUNTS_COUNT_LIMIT_WARNING = 0;
+            test_utils.set_health_mock_functions(Health, {
+                get_service_state: get_service_state_mock_default_response,
+                get_endpoint_response: get_endpoint_response_mock_default_response,
+                get_system_config_file: get_system_config_mock_default_response
+            });
+            const health_status = await Health.nc_nsfs_health();
+            console.log('health_status', health_status);
+            assert.strictEqual(health_status.checks.buckets_status.count, 1);
+            assert.strictEqual(health_status.checks.accounts_status.count, 1);
+            assert.strictEqual(health_status.warnings.length, 2);
+            assert.strictEqual(health_status.warnings.includes(health_warnings.ACCOUNTS_COUNT_LIMIT_WARNING), true);
+            assert.strictEqual(health_status.warnings.includes(health_warnings.BUCKETS_COUNT_LIMIT_WARNING), true);
+        });
+
         mocha.it('Health all condition - failed config directory upgrade status', async function() {
             valid_system_json.config_directory = {
                 config_dir_version: config_fs.config_dir_version,
