diff --git a/docs/NooBaaNonContainerized/Health.md b/docs/NooBaaNonContainerized/Health.md index 4f24da77bd..d3184256c2 100644 --- a/docs/NooBaaNonContainerized/Health.md +++ b/docs/NooBaaNonContainerized/Health.md @@ -351,3 +351,18 @@ The following error codes will be associated with a specific Bucket or Account s - Resolutions: - Check for FS user on the host running the Health CLI. +#### 6. Bucket with invalid account owner + - Error code: `INVALID_ACCOUNT_OWNER` + - Error message: Bucket account owner is invalid + - Reasons: + - The bucket owner account is invalid. + - Resolutions: + - Compare bucket account owner and account ids in account dir. + +#### 7. Bucket missing account owner + - Error code: `MISSING_ACCOUNT_OWNER` + - Error message: Bucket account owner not found + - Reasons: + - Bucket missing owner account. + - Resolutions: + - Check for owner_account property in bucket config file. \ No newline at end of file diff --git a/src/manage_nsfs/health.js b/src/manage_nsfs/health.js index efd3fb1af1..05b55bdb26 100644 --- a/src/manage_nsfs/health.js +++ b/src/manage_nsfs/health.js @@ -11,7 +11,7 @@ const native_fs_utils = require('../util/native_fs_utils'); const { read_stream_join } = require('../util/buffer_utils'); const { make_https_request } = require('../util/http_utils'); const { TYPES } = require('./manage_nsfs_constants'); -const { get_boolean_or_string_value, throw_cli_error, write_stdout_response } = require('./manage_nsfs_cli_utils'); +const { get_boolean_or_string_value, throw_cli_error, write_stdout_response, get_bucket_owner_account_by_id } = require('./manage_nsfs_cli_utils'); const { ManageCLIResponse } = require('./manage_nsfs_cli_responses'); const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError; @@ -52,6 +52,14 @@ const health_errors = { error_code: 'INVALID_DISTINGUISHED_NAME', error_message: 'Account distinguished name was not found', }, + INVALID_ACCOUNT_OWNER: { + error_code: 'INVALID_ACCOUNT_OWNER', + error_message: 'Bucket account owner is invalid', + }, + MISSING_ACCOUNT_OWNER: { + error_code: 'MISSING_ACCOUNT_OWNER', + error_message: 'Bucket account owner not found', + }, UNKNOWN_ERROR: { error_code: 'UNKNOWN_ERROR', error_message: 'An unknown error occurred', @@ -364,7 +372,7 @@ class NSFSHealth { const config_file_path = this.config_fs.get_account_path_by_name(config_file_name); res = await is_new_buckets_path_valid(config_file_path, config_data, storage_path); } else if (type === TYPES.BUCKET) { - res = await is_bucket_storage_path_exists(this.config_fs.fs_context, config_data, storage_path); + res = await is_bucket_storage_and_owner_exists(this.config_fs, config_data, storage_path); } if (all_details && res.valid_storage) { valid_storages.push(res.valid_storage); @@ -498,17 +506,18 @@ async function is_new_buckets_path_valid(config_file_path, config_data, new_buck } /** - * is_bucket_storage_path_exists checks if the underlying storage path of a bucket exists - * @param {nb.NativeFSContext} fs_context + * is_bucket_storage_and_owner_exists checks if the underlying storage path of a bucket exists + * @param {import('../sdk/config_fs').ConfigFS} config_fs * @param {object} config_data * @param {string} storage_path * @returns {Promise} */ -async function is_bucket_storage_path_exists(fs_context, config_data, storage_path) { +async function is_bucket_storage_and_owner_exists(config_fs, config_data, storage_path) { let res_obj; try { if (!_should_skip_health_access_check()) { - await nb_native().fs.stat(fs_context, storage_path); + const account_fs_context = await get_account_owner_context(config_fs, config_data.owner_account); + await nb_native().fs.stat(account_fs_context, storage_path); } res_obj = get_valid_object(config_data.name, undefined, storage_path); } catch (err) { @@ -519,13 +528,40 @@ async function is_bucket_storage_path_exists(fs_context, config_data, storage_pa } else if (err.code === 'EACCES' || (err.code === 'EPERM' && err.message === 'Operation not permitted')) { dbg.log1('Error: Storage path should be accessible to account: ', storage_path); err_code = health_errors.ACCESS_DENIED.error_code; + } else if (err.code === health_errors.INVALID_ACCOUNT_OWNER.error_code || + err.code === health_errors.MISSING_ACCOUNT_OWNER.error_code) { + dbg.log1('Error: Bucket account owner should be valid account id ', config_data.owner_account); + err_code = err.code; } res_obj = get_invalid_object(config_data.name, undefined, storage_path, err_code); } return res_obj; } - +/** + * get_account_owner_context return bucket account owner specific context + * @param {import('../sdk/config_fs').ConfigFS} config_fs + * @param {string} owner_account + * @returns {Promise} + */ +async function get_account_owner_context(config_fs, owner_account) { + if (!owner_account) { + const new_err = new Error(health_errors.MISSING_ACCOUNT_OWNER.error_message); + new_err.code = health_errors.MISSING_ACCOUNT_OWNER.error_code; + throw new_err; + } + try { + // when account owner is invalid method will throw error + const owner_account_data = await get_bucket_owner_account_by_id(config_fs, owner_account); + const account_fs_context = await native_fs_utils.get_fs_context(owner_account_data.nsfs_account_config, + owner_account_data.nsfs_account_config.fs_backend); + return account_fs_context; + } catch (err) { + const new_err = new Error(`Bucket account owner ${owner_account} is invalid`); + new_err.code = health_errors.INVALID_ACCOUNT_OWNER.error_code; + throw new_err; + } +} /** * get_valid_object returns an object which repersents a valid account/bucket and contains defined parameters * @param {string} name diff --git a/src/test/unit_tests/test_nc_nsfs_health.js b/src/test/unit_tests/test_nc_nsfs_health.js index a708e515c8..3c28e2aa51 100644 --- a/src/test/unit_tests/test_nc_nsfs_health.js +++ b/src/test/unit_tests/test_nc_nsfs_health.js @@ -30,14 +30,17 @@ mocha.describe('nsfs nc health', function() { const config_fs = new ConfigFS(config_root); const root_path = path.join(tmp_fs_path, 'root_path_nsfs_health/'); const config_root_invalid = path.join(tmp_fs_path, 'config_root_nsfs_health_invalid'); + const nc_conf_dir = config.NSFS_NC_CONF_DIR; let Health; mocha.before(async () => { + config.NSFS_NC_CONF_DIR = config_root; await fs_utils.create_fresh_path(root_path); await fs_utils.create_fresh_path(config_root_invalid); await nb_native().fs.mkdir(DEFAULT_FS_CONFIG, bucket_storage_path, 0o770); }); mocha.after(async () => { + config.NSFS_NC_CONF_DIR = nc_conf_dir; fs_utils.folder_delete(config_root); fs_utils.folder_delete(root_path); fs_utils.folder_delete(config_root_invalid); @@ -95,6 +98,7 @@ mocha.describe('nsfs nc health', function() { }); mocha.describe('health check', function() { + this.timeout(10000);// eslint-disable-line no-invalid-this const new_buckets_path = `${root_path}new_buckets_path_user1/`; const account1_options = { name: 'account1', @@ -107,7 +111,14 @@ mocha.describe('nsfs nc health', function() { path: new_buckets_path + '/bucket1', owner: account1_options.name }; + const account2_options = { + name: 'account2', + uid: process.getuid(), + gid: process.getgid(), + new_buckets_path: new_buckets_path + }; const account_inaccessible_options = { name: 'account_inaccessible', uid: 999, gid: 999, new_buckets_path: bucket_storage_path }; + const bucket_inaccessible_options = { name: 'bucket2', path: bucket_storage_path + '/bucket2', owner: account_inaccessible_options.name}; const account_inaccessible_dn_options = { name: 'account_inaccessible_dn', user: 'inaccessible_dn', new_buckets_path: bucket_storage_path }; const invalid_account_dn_options = { name: 'invalid_account_dn', user: 'invalid_account_dn', new_buckets_path: bucket_storage_path }; const fs_users = { @@ -118,6 +129,7 @@ mocha.describe('nsfs nc health', function() { } }; mocha.before(async () => { + this.timeout(5000);// eslint-disable-line no-invalid-this const https_port = 6443; Health = new NSFSHealth({ config_root, https_port, config_fs }); await fs_utils.create_fresh_path(new_buckets_path); @@ -126,6 +138,7 @@ mocha.describe('nsfs nc health', function() { await fs_utils.file_must_exist(new_buckets_path + '/bucket1'); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, {config_root, ...account1_options}); await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, {config_root, ...bucket1_options}); + await fs_utils.file_must_exist(path.join(config_root, 'master_keys.json')); const get_service_memory_usage = sinon.stub(Health, "get_service_memory_usage"); get_service_memory_usage.onFirstCall().returns(Promise.resolve(100)); for (const user of Object.values(fs_users)) { @@ -134,6 +147,7 @@ mocha.describe('nsfs nc health', function() { }); mocha.after(async () => { + this.timeout(5000);// eslint-disable-line no-invalid-this fs_utils.folder_delete(new_buckets_path); fs_utils.folder_delete(path.join(new_buckets_path, 'bucket1')); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, {config_root, name: bucket1_options.name}); @@ -209,10 +223,13 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('NSFS bucket with invalid storage path', async function() { + this.timeout(5000);// eslint-disable-line no-invalid-this Health.get_service_state.restore(); Health.get_endpoint_response.restore(); + const resp = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account2_options }); + const parsed_res = JSON.parse(resp).response.reply; // create it manually because we can not skip invalid storage path check on the CLI - const bucket_invalid = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid', path: new_buckets_path + '/bucket1/invalid', owner: account1_options.name }; + const bucket_invalid = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid', path: new_buckets_path + '/bucket1/invalid', owner_account: parsed_res._id }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid); const get_service_state = sinon.stub(Health, "get_service_state"); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) @@ -223,7 +240,81 @@ mocha.describe('nsfs nc health', function() { assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, bucket_invalid.name); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "STORAGE_NOT_EXIST"); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: bucket_invalid.name}); + await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account2_options.name}); + }); + + mocha.it('Bucket with inaccessible path - uid gid', async function() { + this.timeout(5000);// eslint-disable-line no-invalid-this + await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); + await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_options }); + await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, {config_root, ...bucket_inaccessible_options}); + await config_fs.delete_config_json_file(); + Health.get_service_state.restore(); + Health.get_endpoint_response.restore(); + Health.all_account_details = true; + Health.all_bucket_details = true; + const get_service_state = sinon.stub(Health, "get_service_state"); + get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) + .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); + const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + const health_status = await Health.nc_nsfs_health(); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, bucket_inaccessible_options.name); + assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible_options.name); + + await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: bucket_inaccessible_options.name}); + await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_inaccessible_options.name}); + }); + + mocha.it('Bucket with inaccessible owner', async function() { + this.timeout(5000);// eslint-disable-line no-invalid-this + //create bucket manually, cli wont allow bucket with invalid owner + const bucket_invalid_owner = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid_account', path: new_buckets_path + '/bucket_account', owner_account: 'invalid_account' }; + await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid_owner); + Health.get_service_state.restore(); + Health.get_endpoint_response.restore(); + Health.all_account_details = true; + Health.all_bucket_details = true; + const get_service_state = sinon.stub(Health, "get_service_state"); + get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) + .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); + const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + const health_status = await Health.nc_nsfs_health(); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "INVALID_ACCOUNT_OWNER"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, 'bucket_invalid_account'); + + await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: 'bucket_invalid_account'}); + }); + + mocha.it('Bucket with empty owner', async function() { + this.timeout(5000);// eslint-disable-line no-invalid-this + //create bucket manually, cli wont allow bucket with empty owner + const bucket_invalid_owner = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid_account', path: new_buckets_path + '/bucket_account' }; + await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid_owner); + Health.get_service_state.restore(); + Health.get_endpoint_response.restore(); + Health.all_account_details = true; + Health.all_bucket_details = true; + const get_service_state = sinon.stub(Health, "get_service_state"); + get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) + .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); + const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); + get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + const health_status = await Health.nc_nsfs_health(); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "MISSING_ACCOUNT_OWNER"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, 'bucket_invalid_account'); + + await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: 'bucket_invalid_account'}); }); mocha.it('NSFS invalid bucket schema json', async function() { @@ -527,4 +618,3 @@ mocha.describe('nsfs nc health', function() { }); }); }); -