Added x-amz-expiration, missing HTTP header in the response of object GET/PUT/HEAD

Signed-off-by: Aayush Chouhan <[email protected]>

Author: achouhan09

src/endpoint/s3/ops/s3_get_object.js

@@ -49,6 +49,7 @@ async function get_object(req, res) {
         }
     }
     http_utils.set_response_headers_from_request(req, res);
+    if (!version_id) await http_utils.set_expiration_header(req, res, object_md); // setting expiration header for bucket lifecycle
     const obj_size = object_md.size;
     const params = {
         object_md,

src/endpoint/s3/ops/s3_head_object.js

@@ -29,6 +29,7 @@ async function head_object(req, res) {
     s3_utils.set_response_object_md(res, object_md);
     s3_utils.set_encryption_response_headers(req, res, object_md.encryption);
     http_utils.set_response_headers_from_request(req, res);
+    if (!params.version_id) await http_utils.set_expiration_header(req, res, object_md); // setting expiration header for bucket lifecycle
 }
 
 module.exports = {

src/endpoint/s3/ops/s3_put_object.js

@@ -81,6 +81,14 @@ async function put_object(req, res) {
     }
     res.setHeader('ETag', `"${reply.etag}"`);
 
+    const object_info = {
+        key: req.params.key,
+        create_time: new Date().getTime(),
+        size: size,
+        tagging: tagging,
+    };
+    await http_utils.set_expiration_header(req, res, object_info); // setting expiration header for bucket lifecycle
+
     if (reply.seq) {
         res.seq = reply.seq;
         delete reply.seq;

src/manage_nsfs/nc_lifecycle.js

@@ -271,7 +271,7 @@ class NCLifecycle {
 
             if (candidates.delete_candidates?.length > 0) {
                 const expiration = lifecycle_rule.expiration ? this._get_expiration_time(lifecycle_rule.expiration) : 0;
-                const filter_func = this._build_lifecycle_filter({filter: lifecycle_rule.filter, expiration});
+                const filter_func = lifecycle_utils.build_lifecycle_filter({filter: lifecycle_rule.filter, expiration});
                 dbg.log0('process_rule: calling delete_multiple_objects, num of objects to be deleted', candidates.delete_candidates.length);
                 const delete_res = await this._call_op_and_update_status({
                     bucket_name,
@@ -478,7 +478,7 @@ class NCLifecycle {
         if (rule_state.is_finished) return [];
         const expiration = this._get_expiration_time(lifecycle_rule.expiration);
         if (expiration < 0) return [];
-        const filter_func = this._build_lifecycle_filter({filter: lifecycle_rule.filter, expiration});
+        const filter_func = lifecycle_utils.build_lifecycle_filter({filter: lifecycle_rule.filter, expiration});
 
         const filtered_objects = [];
         // TODO list_objects does not accept a filter and works in batch sizes of 1000. should handle batching
@@ -537,7 +537,7 @@ class NCLifecycle {
         const versions_list = params.versions_list;
         const candidates = [];
         const expiration = lifecycle_rule.expiration?.days ? this._get_expiration_time(lifecycle_rule.expiration) : 0;
-        const filter_func = this._build_lifecycle_filter({filter: lifecycle_rule.filter, expiration});
+        const filter_func = lifecycle_utils.build_lifecycle_filter({filter: lifecycle_rule.filter, expiration});
         for (let i = 0; i < versions_list.objects.length - 1; i++) {
             if (this.filter_expired_delete_marker(versions_list.objects[i], versions_list.objects[i + 1], filter_func)) {
                 candidates.push(versions_list.objects[i]);
@@ -640,7 +640,7 @@ class NCLifecycle {
         }
         const versions_list = params.versions_list;
 
-        const filter_func = this._build_lifecycle_filter({filter: lifecycle_rule.filter, expiration: 0});
+        const filter_func = lifecycle_utils.build_lifecycle_filter({filter: lifecycle_rule.filter, expiration: 0});
         const num_newer_versions = lifecycle_rule.noncurrent_version_expiration.newer_noncurrent_versions;
         const num_non_current_days = lifecycle_rule.noncurrent_version_expiration.noncurrent_days;
         const delete_candidates = [];
@@ -674,7 +674,7 @@ class NCLifecycle {
         const expiration = lifecycle_rule.abort_incomplete_multipart_upload.days_after_initiation;
         const res = [];
 
-        const filter_func = this._build_lifecycle_filter({filter, expiration});
+        const filter_func = lifecycle_utils.build_lifecycle_filter({filter, expiration});
         let dir_handle;
         //TODO this is almost identical to list_uploads except for error handling and support for pagination. should modify list-upload and use it in here instead
         try {
@@ -720,29 +720,6 @@ class NCLifecycle {
     /////////   FILTER HELPERS  ////////
     ////////////////////////////////////
 
-    /**
-     * @typedef {{
-     *     filter: Object
-     *     expiration: Number
-     * }} filter_params
-     *
-     * @param {filter_params} params
-     * @returns
-     */
-    _build_lifecycle_filter(params) {
-        /**
-         * @param {Object} object_info
-         */
-        return function(object_info) {
-            if (params.filter?.prefix && !object_info.key.startsWith(params.filter.prefix)) return false;
-            if (params.expiration && object_info.age < params.expiration) return false;
-            if (params.filter?.tags && !_file_contain_tags(object_info, params.filter.tags)) return false;
-            if (params.filter?.object_size_greater_than && object_info.size < params.filter.object_size_greater_than) return false;
-            if (params.filter?.object_size_less_than && object_info.size > params.filter.object_size_less_than) return false;
-            return true;
-        };
-    }
-
     /**
      * get the expiration time in days of an object
      * if rule is set with date, then rule is applied for all objects after that date
@@ -1468,38 +1445,6 @@ class NCLifecycle {
     }
 }
 
-//////////////////
-// TAGS HELPERS //
-//////////////////
-
-/**
- * checks if tag query_tag is in the list tag_set
- * @param {Object} query_tag
- * @param {Array<Object>} tag_set
- */
-function _list_contain_tag(query_tag, tag_set) {
-    for (const t of tag_set) {
-        if (t.key === query_tag.key && t.value === query_tag.value) return true;
-    }
-    return false;
-}
-
-/**
- * checks if object has all the tags in filter_tags
- * @param {Object} object_info
- * @param {Array<Object>} filter_tags
- * @returns
- */
-function _file_contain_tags(object_info, filter_tags) {
-    if (object_info.tags === undefined) return false;
-    for (const tag of filter_tags) {
-        if (!_list_contain_tag(tag, object_info.tags)) {
-            return false;
-        }
-    }
-    return true;
-}
-
 // EXPORTS
 exports.NCLifecycle = NCLifecycle;
 exports.ILM_POLICIES_TMP_DIR = ILM_POLICIES_TMP_DIR;

src/test/system_tests/test_utils.js

@@ -790,6 +790,80 @@ const run_or_skip_test = cond => {
     } else return it.skip;
 };
 
+/**
+ * update_file_mtime updates the mtime of the target path
+ * Warnings:
+ *  - This operation would change the mtime of the file to 5 days ago - which means that it changes the etag / obj_id of the object
+ *  - Please do not use on versioned objects (version_id will not be changed, but the mtime will be changed) - might cause issues.
+ * @param {String} target_path
+ * @returns {Promise<Void>}
+ */
+async function update_file_mtime(target_path) {
+    const update_file_mtime_cmp = os_utils.IS_MAC ? `touch -t $(date -v -5d +"%Y%m%d%H%M.%S") ${target_path}` : `touch -d "5 days ago" ${target_path}`;
+    await os_utils.exec(update_file_mtime_cmp, { return_stdout: true });
+}
+
+/////////////////////////////////
+//////  LIFECYCLE UTILS   ///////
+/////////////////////////////////
+
+/**
+ * generate_lifecycle_rule generate an S3 lifecycle rule with optional filters and expiration currently (can be extend to support more lifecycle rule params)
+ *
+ * @param {number} expiration_days
+ * @param {string} id
+ * @param {string} [prefix]
+ * @param {Array<Object>} [tags]
+ * @param {number} [size_gt]
+ * @param {number} [size_lt]
+ * @returns {Object}
+ */
+function generate_lifecycle_rule(expiration_days, id, prefix, tags, size_gt, size_lt) {
+    const filters = {};
+    if (prefix) filters.Prefix = prefix;
+    if (Array.isArray(tags) && tags.length) filters.Tags = tags;
+    if (size_gt !== undefined) filters.ObjectSizeGreaterThan = size_gt;
+    if (size_lt !== undefined) filters.ObjectSizeLessThan = size_lt;
+
+    const filter = Object.keys(filters).length > 1 ? { And: filters } : filters;
+
+    return {
+        ID: id,
+        Status: 'Enabled',
+        Filter: filter,
+        Expiration: { Days: expiration_days },
+    };
+}
+
+/**
+ * validate_expiration_header validates the `x-amz-expiration` header against the object creation time, expected rule ID and expiration days
+ *
+ * The header is expected to have the format:
+ *   expiry-date="YYYY-MM-DDTHH:MM:SS.SSSZ", rule-id="RULE_ID"
+ *
+ * @param {string} expiration_header - expiration header value
+ * @param {string|Date} start_time - start/create time (string or Date) of the object
+ * @param {string} expected_rule_id - expected rule ID to match in the header
+ * @param {number} delta_days - expected number of days between start_time and expiry-date
+ * @returns {boolean} true if the header is valid and matches the expected_rule_id and delta_days otherwise false
+ */
+function validate_expiration_header(expiration_header, start_time, expected_rule_id, delta_days) {
+    const match = expiration_header.match(/expiry-date="(.+)", rule-id="(.+)"/);
+    if (!match) return false;
+
+    const [, expiry_str, rule_id] = match;
+    const expiration = new Date(expiry_str);
+    const start = new Date(start_time);
+    start.setUTCHours(0, 0, 0, 0); // adjusting to midnight UTC otherwise the tests will fail - fix for ceph-s3 tests
+
+    const days_diff = Math.floor((expiration.getTime() - start.getTime()) / (24 * 60 * 60 * 1000));
+
+    return days_diff === delta_days && rule_id === expected_rule_id;
+}
+
+exports.update_file_mtime = update_file_mtime;
+exports.generate_lifecycle_rule = generate_lifecycle_rule;
+exports.validate_expiration_header = validate_expiration_header;
 exports.run_or_skip_test = run_or_skip_test;
 exports.blocks_exist_on_cloud = blocks_exist_on_cloud;
 exports.create_hosts_pool = create_hosts_pool;