Merge pull request #8751 from naveenpaul1/list_object_conflicting_files

NC | Multi Protocol Access | List object with conflicting ownership

Author: naveenpaul1

src/sdk/namespace_fs.js

@@ -721,7 +721,10 @@ class NamespaceFS {
                         if (pos < results.length) {
                             results.splice(pos, 0, r);
                         } else {
-                            results.push(r);
+                            const stat = await native_fs_utils.stat_ignore_eacces(this.bucket_path, r, fs_context);
+                            if (stat) {
+                                results.push(r);
+                            }
                         }
                         if (results.length > limit) {
                             results.length = limit;

src/test/unit_tests/test_nsfs_access.js

@@ -9,6 +9,11 @@ const fs_utils = require('../../util/fs_utils');
 const nb_native = require('../../util/nb_native');
 const test_utils = require('../system_tests/test_utils');
 const fs = require('fs');
+const { TMP_PATH } = require('../system_tests/test_utils');
+const NamespaceFS = require('../../sdk/namespace_fs');
+const buffer_utils = require('../../util/buffer_utils');
+const endpoint_stats_collector = require('../../sdk/endpoint_stats_collector');
+const SensitiveString = require('../../util/sensitive_string');
 
 const new_umask = process.env.NOOBAA_ENDPOINT_UMASK || 0o000;
 const old_umask = process.umask(new_umask);
@@ -175,5 +180,145 @@ mocha.describe('new tests check', async function() {
     });
 });
 
+mocha.describe('list object access check', function() {
+    this.timeout(10 * 60 * 1000); // eslint-disable-line no-invalid-this
 
+    const key_files_set_first = make_keys(10, i => `small_key_files_set_first${i}`);
+    const key_files_set_second = make_keys(10, i => `small_key_files_set_second${i}`);
+    const max_keys_files_set = make_keys(981, i => `max_keys_files_set${i}`);
+    const access_src_bkt = 'access_src';
+    const tmp_fs_path = path.join(TMP_PATH, 'test_namespace_access_fs');
+    const ns_tmp_bucket_path = path.join(tmp_fs_path, access_src_bkt);
+    const first_file_path = path.join(ns_tmp_bucket_path, 'small_key_files_set_first1');
+    const bucket1 = 'access_bucket1';
+    const ns_src = new NamespaceFS({
+        bucket_path: ns_tmp_bucket_path,
+        bucket_id: '5',
+        namespace_resource_id: undefined,
+        access_mode: undefined,
+        versioning: undefined,
+        force_md5_etag: false,
+        stats: endpoint_stats_collector.instance(),
+    });
+    const custom_dummy_object_sdk1 = make_custom_dummy_object_sdk(200, 200);
+    const custom_dummy_object_sdk2 = make_custom_dummy_object_sdk(300, 200);
+    const custom_dummy_object_sdk3 = make_custom_dummy_object_sdk(400, 400);
+    mocha.before(async function() {
+        await fs_utils.create_fresh_path(tmp_fs_path, 0o777);
+        await fs_utils.file_must_exist(tmp_fs_path);
+        await fs_utils.create_fresh_path(ns_tmp_bucket_path, 0o770);
+        await fs_utils.file_must_exist(ns_tmp_bucket_path);
+        await fs.promises.chmod(tmp_fs_path, 0o777);
+        await fs.promises.chmod(ns_tmp_bucket_path, 0o770);
+        await fs.promises.chown(ns_tmp_bucket_path, custom_dummy_object_sdk1.requesting_account.nsfs_account_config.uid,
+            custom_dummy_object_sdk1.requesting_account.nsfs_account_config.gid);
+    });
+    mocha.after(async function() {
+        fs_utils.folder_delete(ns_tmp_bucket_path);
+        fs_utils.folder_delete(tmp_fs_path);
+    });
+
+    mocha.it('list object with inaccessible item, smae UI and GID', async function() {
+        await upload_objects(key_files_set_first, custom_dummy_object_sdk1, bucket1, ns_src);
+        // change ownership for one file, and account can not access this file
+        await fs.promises.chown(first_file_path, 999, 999);
+        const r = await ns_src.list_objects({
+            bucket: bucket1,
+        }, custom_dummy_object_sdk1);
+        // skipping inaccessible file, list rest of the files
+        assert_list_items(r, [...key_files_set_first], 9);
+    });
+
+    mocha.it('list object with different account and same GID', async function() {
+        await upload_objects(key_files_set_second, custom_dummy_object_sdk2, bucket1, ns_src);
+        const r = await ns_src.list_objects({
+            bucket: bucket1,
+        }, custom_dummy_object_sdk2);
+        assert_list_items(r, [...key_files_set_first, ...key_files_set_second], 19);
+    });
+
+    mocha.it('list object with different account and different GID', async function() {
+        try {
+            await upload_objects(["Banana"], custom_dummy_object_sdk3, bucket1, ns_src);
+        } catch (err) {
+            assert.strictEqual(err instanceof Error, true);
+            assert.strictEqual(err.code, 'EACCES');
+        }
+        const r = await ns_src.list_objects({
+            bucket: bucket1,
+        }, custom_dummy_object_sdk2);
+        assert_list_items(r, [...key_files_set_first, ...key_files_set_second], 19);
+    });
+
+    mocha.it('max - list object with different account and same GID', async function() {
+        await upload_objects(max_keys_files_set, custom_dummy_object_sdk1, bucket1, ns_src);
+        const r = await ns_src.list_objects({
+            bucket: bucket1,
+        }, custom_dummy_object_sdk1);
+        // Total number of object would be 9+10+981 = 1000
+        assert_list_items(r, [...key_files_set_first, ...key_files_set_second, ...max_keys_files_set], 1000);
+    });
+});
+
+async function upload_objects(keys, custom_object_sdk, user_bucket, user_ns) {
+    return Promise.all(keys.map(async key => {
+        await user_ns.upload_object({
+            bucket: user_bucket,
+            key,
+            content_type: 'application/octet-stream',
+            source_stream: buffer_utils.buffer_to_read_stream(null),
+            size: 0
+        }, custom_object_sdk);
+    }));
+}
+
+function make_custom_dummy_object_sdk(uid, gid) {
+    return {
+        requesting_account: {
+            force_md5_etag: false,
+            nsfs_account_config: {
+                uid: uid,
+                gid: gid,
+            }
+        },
+        abort_controller: new AbortController(),
+        throw_if_aborted() {
+            if (this.abort_controller.signal.aborted) throw new Error('request aborted signal');
+        },
+
+        read_bucket_sdk_config_info(name) {
+            return {
+                bucket_owner: new SensitiveString('dummy-owner'),
+                owner_account: {
+                    id: 'dummy-id-123',
+                }
+            };
+        }
+    };
+}
+
+/**
+ * validate list objects counts and items
+ * @param {Object} r 
+ * @param {string[]} splice_array
+ * @param {number} object_items_count 
+ */
+function assert_list_items(r, splice_array, object_items_count) {
+    assert.equal(r.objects.length, object_items_count);
+    const index = splice_array.indexOf('small_key_files_set_first1');
+    splice_array.splice(index, 1);
+    assert.deepStrictEqual(r.objects.map(it => it.key), splice_array.sort());
+}
 
+/**
+ * @param {number} count 
+ * @param {(i:number)=>string} gen 
+ * @returns {string[]}
+ */
+function make_keys(count, gen) {
+    const arr = new Array(count);
+    for (let i = 0; i < count; ++i) arr[i] = gen(i);
+    arr.sort();
+    Object.freeze(arr);
+    return arr;
+}

src/util/native_fs_utils.js

@@ -696,6 +696,16 @@ function translate_error_codes(err, entity) {
     return err;
 }
 
+async function stat_ignore_eacces(bucket_path, result, fs_context) {
+    const entry_path = path.join(bucket_path, result.key);
+    try {
+        return await nb_native().fs.stat(fs_context, entry_path);
+    } catch (err) {
+        if (err.code !== 'EACCES') throw err;
+        dbg.log0('NamespaceFS: check_stats_for_object : couldnt access file entry_path', entry_path, ", skipping...");
+    }
+}
+
 exports.get_umasked_mode = get_umasked_mode;
 exports._make_path_dirs = _make_path_dirs;
 exports._create_path = _create_path;
@@ -737,3 +747,4 @@ exports.get_bucket_tmpdir_full_path = get_bucket_tmpdir_full_path;
 exports.get_bucket_tmpdir_name = get_bucket_tmpdir_name;
 exports.entity_enum = entity_enum;
 exports.translate_error_codes = translate_error_codes;
+exports.stat_ignore_eacces = stat_ignore_eacces;