From 22a06b76618e663337cee69c3f5386ae404d4f09 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 20 Feb 2024 03:17:17 +0100 Subject: [PATCH 01/14] Complete GCP namespace support Signed-off-by: Ben --- src/sdk/namespace_gcp.js | 152 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 144 insertions(+), 8 deletions(-) diff --git a/src/sdk/namespace_gcp.js b/src/sdk/namespace_gcp.js index 689b4e22f6..0eb7387535 100644 --- a/src/sdk/namespace_gcp.js +++ b/src/sdk/namespace_gcp.js @@ -7,9 +7,11 @@ const util = require('util'); const P = require('../util/promise'); const stream_utils = require('../util/stream_utils'); const dbg = require('../util/debug_module')(__filename); +const s3_utils = require('../endpoint/s3/s3_utils'); const S3Error = require('../endpoint/s3/s3_errors').S3Error; // we use this wrapper to set a custom user agent const GoogleCloudStorage = require('../util/google_storage_wrap'); +const aws_sdk = require('aws-sdk'); /** * @implements {nb.Namespace} @@ -47,6 +49,16 @@ class NamespaceGCP { private_key: this.private_key, } }); + this.gcs.createHmacKey(client_email).then((res) => { + this.hmac_key = res[0]; + this.hmac_secret = res[1]; + }); + this.s3_client = new aws_sdk.S3({ + endpoint: 'https://storage.googleapis.com', + accessKeyId: this.hmac_key, + secretAccessKey: this.hmac_secret + }); + this.bucket = target_bucket; this.access_mode = access_mode; this.stats = stats; @@ -297,27 +309,129 @@ class NamespaceGCP { async create_object_upload(params, object_sdk) { dbg.log0('NamespaceGCP.create_object_upload:', this.bucket, inspect(params)); - throw new S3Error(S3Error.NotImplemented); + const Tagging = params.tagging && params.tagging.map(tag => tag.key + '=' + tag.value).join('&'); + /** @type {AWS.S3.CreateMultipartUploadRequest} */ + const request = { + Bucket: this.bucket, + Key: params.key, + ContentType: params.content_type, + StorageClass: params.storage_class, + Metadata: params.xattr, + Tagging + }; + const res = await this.s3_client.createMultipartUpload(request).promise(); + + dbg.log0('NamespaceGCP.create_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); + return { obj_id: res.UploadId }; } async upload_multipart(params, object_sdk) { dbg.log0('NamespaceGCP.upload_multipart:', this.bucket, inspect(params)); - throw new S3Error(S3Error.NotImplemented); + let res; + if (params.copy_source) { + const { copy_source, copy_source_range } = s3_utils.format_copy_source(params.copy_source); + + /** @type {AWS.S3.UploadPartCopyRequest} */ + const request = { + Bucket: this.bucket, + Key: params.key, + UploadId: params.obj_id, + PartNumber: params.num, + CopySource: copy_source, + CopySourceRange: copy_source_range, + }; + + res = await this.s3_client.uploadPartCopy(request).promise(); + } else { + let count = 1; + const count_stream = stream_utils.get_tap_stream(data => { + this.stats?.update_namespace_write_stats({ + namespace_resource_id: this.namespace_resource_id, + size: data.length, + count + }); + // clear count for next updates + count = 0; + }); + /** @type {AWS.S3.UploadPartRequest} */ + const request = { + Bucket: this.bucket, + Key: params.key, + UploadId: params.obj_id, + PartNumber: params.num, + Body: params.source_stream.pipe(count_stream), + ContentMD5: params.md5_b64, + ContentLength: params.size, + }; + try { + res = await this.s3_client.uploadPart(request).promise(); + } catch (err) { + object_sdk.rpc_client.pool.update_issues_report({ + namespace_resource_id: this.namespace_resource_id, + error_code: String(err.code), + time: Date.now(), + }); + throw err; + } + } + dbg.log0('NamespaceGCP.upload_multipart:', this.bucket, inspect(params), 'res', inspect(res)); + const etag = s3_utils.parse_etag(res.ETag); + return { etag }; } async list_multiparts(params, object_sdk) { dbg.log0('NamespaceGCP.list_multiparts:', this.bucket, inspect(params)); - throw new S3Error(S3Error.NotImplemented); + + const res = await this.s3_client.listParts({ + Bucket: this.bucket, + Key: params.key, + UploadId: params.obj_id, + MaxParts: params.max, + PartNumberMarker: params.num_marker, + }).promise(); + + dbg.log0('NamespaceGCP.list_multiparts:', this.bucket, inspect(params), 'res', inspect(res)); + return { + is_truncated: res.IsTruncated, + next_num_marker: res.NextPartNumberMarker, + multiparts: _.map(res.Parts, p => ({ + num: p.PartNumber, + size: p.Size, + etag: s3_utils.parse_etag(p.ETag), + last_modified: p.LastModified, + })) + }; } async complete_object_upload(params, object_sdk) { dbg.log0('NamespaceGCP.complete_object_upload:', this.bucket, inspect(params)); - throw new S3Error(S3Error.NotImplemented); + + const res = await this.s3_client.completeMultipartUpload({ + Bucket: this.bucket, + Key: params.key, + UploadId: params.obj_id, + MultipartUpload: { + Parts: _.map(params.multiparts, p => ({ + PartNumber: p.num, + ETag: `"${p.etag}"`, + })) + } + }).promise(); + + dbg.log0('NamespaceGCP.complete_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); + const etag = s3_utils.parse_etag(res.ETag); + return { etag, version_id: res.VersionId }; } async abort_object_upload(params, object_sdk) { dbg.log0('NamespaceGCP.abort_object_upload:', this.bucket, inspect(params)); - throw new S3Error(S3Error.NotImplemented); + const res = await this.s3_client.abortMultipartUpload({ + Bucket: this.bucket, + Key: params.key, + UploadId: params.obj_id, + }).promise(); + + dbg.log0('NamespaceGCP.abort_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); } ////////// @@ -378,13 +492,35 @@ class NamespaceGCP { //////////////////// async get_object_tagging(params, object_sdk) { - throw new Error('TODO'); + dbg.log0('NamespaceGCP.get_object_tagging:', this.bucket, inspect(params)); + const obj_tags = (await this.read_object_md(params, object_sdk)).xattr + // Converting tag dictionary to array of key-value object pairs + const tags = Object.entries(obj_tags).map(([key, value]) => ({ key, value })); + return { + tagging: tags + }; } async delete_object_tagging(params, object_sdk) { - throw new Error('TODO'); + dbg.log0('NamespaceGCP.delete_object_tagging:', this.bucket, inspect(params)); + try { + // Set an empty metadata object to remove all tags + const res = await this.gcs.bucket(this.bucket).file(params.key).setMetadata({}); + dbg.log0('NamespaceGCP.delete_object_tagging:', this.bucket, inspect(params), 'res', inspect(res)); + } catch (err) { + dbg.error('NamespaceGCP.delete_object_tagging error:', err); + } } async put_object_tagging(params, object_sdk) { - throw new Error('TODO'); + dbg.log0('NamespaceGCP.put_object_tagging:', this.bucket, inspect(params)); + try { + // Convert the array of key-value object pairs to a dictionary + const tags_to_put = Object.fromEntries(params.tagging.map(tag => ([tag.key, tag.value]))); + const res = await this.gcs.bucket(this.bucket).file(params.key).setMetadata({ metadata: tags_to_put }); + dbg.log0('NamespaceGCP.put_object_tagging:', this.bucket, inspect(params), 'res', inspect(res)); + } catch (err) { + dbg.error('NamespaceGCP.put_object_tagging error:', err); + } + } /////////////////// From 014c33f2633c57bd9637451b26f4e81091b572af Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 20 Feb 2024 11:01:20 +0100 Subject: [PATCH 02/14] Fix linting Signed-off-by: Ben --- src/sdk/namespace_gcp.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sdk/namespace_gcp.js b/src/sdk/namespace_gcp.js index 0eb7387535..ed34972337 100644 --- a/src/sdk/namespace_gcp.js +++ b/src/sdk/namespace_gcp.js @@ -49,7 +49,7 @@ class NamespaceGCP { private_key: this.private_key, } }); - this.gcs.createHmacKey(client_email).then((res) => { + this.gcs.createHmacKey(client_email).then(res => { this.hmac_key = res[0]; this.hmac_secret = res[1]; }); @@ -493,7 +493,7 @@ class NamespaceGCP { async get_object_tagging(params, object_sdk) { dbg.log0('NamespaceGCP.get_object_tagging:', this.bucket, inspect(params)); - const obj_tags = (await this.read_object_md(params, object_sdk)).xattr + const obj_tags = (await this.read_object_md(params, object_sdk)).xattr; // Converting tag dictionary to array of key-value object pairs const tags = Object.entries(obj_tags).map(([key, value]) => ({ key, value })); return { From 1168325b5b6436c046d7eebad2090706f81ba72a Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 6 Mar 2024 14:59:47 +0100 Subject: [PATCH 03/14] Change AWS SDK require name to conform to the rest of hte code Signed-off-by: Ben --- src/sdk/namespace_gcp.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sdk/namespace_gcp.js b/src/sdk/namespace_gcp.js index ed34972337..808bd8e986 100644 --- a/src/sdk/namespace_gcp.js +++ b/src/sdk/namespace_gcp.js @@ -11,7 +11,7 @@ const s3_utils = require('../endpoint/s3/s3_utils'); const S3Error = require('../endpoint/s3/s3_errors').S3Error; // we use this wrapper to set a custom user agent const GoogleCloudStorage = require('../util/google_storage_wrap'); -const aws_sdk = require('aws-sdk'); +const AWS = require('aws-sdk'); /** * @implements {nb.Namespace} @@ -53,7 +53,7 @@ class NamespaceGCP { this.hmac_key = res[0]; this.hmac_secret = res[1]; }); - this.s3_client = new aws_sdk.S3({ + this.s3_client = new AWS.S3({ endpoint: 'https://storage.googleapis.com', accessKeyId: this.hmac_key, secretAccessKey: this.hmac_secret From 0aea11efaff8aebc40821b263cfc8ebc0a18b030 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 12 Mar 2024 10:25:22 +0100 Subject: [PATCH 04/14] Add missing ACL ops handling Signed-off-by: Ben --- src/sdk/namespace_gcp.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sdk/namespace_gcp.js b/src/sdk/namespace_gcp.js index 808bd8e986..8de49d9d83 100644 --- a/src/sdk/namespace_gcp.js +++ b/src/sdk/namespace_gcp.js @@ -446,12 +446,13 @@ class NamespaceGCP { */ async get_object_acl(params, object_sdk) { dbg.log0('NamespaceGCP.get_object_acl:', this.bucket, inspect(params)); - throw new S3Error(S3Error.NotImplemented); + await this.read_object_md(params, object_sdk); + return s3_utils.DEFAULT_OBJECT_ACL; } async put_object_acl(params, object_sdk) { dbg.log0('NamespaceGCP.put_object_acl:', this.bucket, inspect(params)); - throw new S3Error(S3Error.NotImplemented); + await this.read_object_md(params, object_sdk); } /////////////////// From eff4f596e433873f5e2ca717c348ce3d13d7ff86 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 15:06:59 +0100 Subject: [PATCH 05/14] Improve `version_id` response check Signed-off-by: Ben --- src/endpoint/s3/ops/s3_put_object_tagging.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/endpoint/s3/ops/s3_put_object_tagging.js b/src/endpoint/s3/ops/s3_put_object_tagging.js index 2282b6164c..37dda6ce47 100644 --- a/src/endpoint/s3/ops/s3_put_object_tagging.js +++ b/src/endpoint/s3/ops/s3_put_object_tagging.js @@ -15,7 +15,7 @@ async function put_object_tagging(req, res) { tagging: tag_set, version_id }); - if (reply.version_id) res.setHeader('x-amz-version-id', reply.version_id); + if (reply?.version_id) res.setHeader('x-amz-version-id', reply.version_id); } module.exports = { From e8d7aa936cceb11e03e5876345d25c38b3972d1b Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 20:01:26 +0100 Subject: [PATCH 06/14] WIP: Generate singular HMAC key and save it in the database Signed-off-by: Ben --- src/sdk/namespace_gcp.js | 20 +++++++------------ src/sdk/object_sdk.js | 2 ++ src/server/system_services/account_server.js | 20 +++++++++++++++++++ .../system_services/master_key_manager.js | 7 +++++++ 4 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/sdk/namespace_gcp.js b/src/sdk/namespace_gcp.js index 8de49d9d83..4d1212338d 100644 --- a/src/sdk/namespace_gcp.js +++ b/src/sdk/namespace_gcp.js @@ -27,6 +27,10 @@ class NamespaceGCP { * private_key: string, * access_mode: string, * stats: import('./endpoint_stats_collector').EndpointStatsCollector, + * hmac_key: { + * access_id: string, + * secret_key: string, + * } * }} params */ constructor({ @@ -37,26 +41,16 @@ class NamespaceGCP { private_key, access_mode, stats, + hmac_key, }) { this.namespace_resource_id = namespace_resource_id; this.project_id = project_id; this.client_email = client_email; this.private_key = private_key; - this.gcs = new GoogleCloudStorage({ - projectId: this.project_id, - credentials: { - client_email: this.client_email, - private_key: this.private_key, - } - }); - this.gcs.createHmacKey(client_email).then(res => { - this.hmac_key = res[0]; - this.hmac_secret = res[1]; - }); this.s3_client = new AWS.S3({ endpoint: 'https://storage.googleapis.com', - accessKeyId: this.hmac_key, - secretAccessKey: this.hmac_secret + accessKeyId: hmac_key.access_id, + secretAccessKey: hmac_key.secret_key }); this.bucket = target_bucket; diff --git a/src/sdk/object_sdk.js b/src/sdk/object_sdk.js index 787a2a0572..e695806ffe 100644 --- a/src/sdk/object_sdk.js +++ b/src/sdk/object_sdk.js @@ -443,6 +443,7 @@ class ObjectSDK { /** * @returns {nb.Namespace} */ + // resource is a namespace_resource _setup_single_namespace({ resource: r, path: p }, bucket_id, options) { if (r.endpoint_type === 'NOOBAA') { @@ -502,6 +503,7 @@ class ObjectSDK { private_key, access_mode: r.access_mode, stats: this.stats, + hmac_key: r.gcp_hmac_key, }); } if (r.fs_root_path || r.fs_root_path === '') { diff --git a/src/server/system_services/account_server.js b/src/server/system_services/account_server.js index 98a5d6be4d..ba89b310aa 100644 --- a/src/server/system_services/account_server.js +++ b/src/server/system_services/account_server.js @@ -784,6 +784,26 @@ async function add_external_connection(req) { }; } + // If the connection is for Google, generate an HMAC key for S3-compatible actions (e.g. multipart uploads) + if (info.endpoint_type === 'GOOGLE') { + dbg.log0('add_external_connection: creating HMAC key for Google connection') + const key_file = JSON.parse(req.rpc_params.secret.unwrap()); + const credentials = _.pick(key_file, 'client_email', 'private_key'); + const gs_client = new GoogleStorage({ credentials, projectId: key_file.project_id }); + try { + const [hmac_key, secret] = await gs_client.createHmacKey(credentials.client_email); + info.gcp_hmac_key = { + access_id: hmac_key.metadata.accessId, + secret_key: system_store.master_key_manager.encrypt_sensitive_string_with_master_key_id( + new SensitiveString(secret), req.account.master_key_id._id) + }; + } catch (err) { + // The most likely reason is that the storage account already has 10 existing HMAC keys, which is the limit + dbg.error('add_external_connection: failed to create HMAC key for Google connection', err); + throw new RpcError('INTERNAL_ERROR', 'Failed to create HMAC key for Google connection'); + } + } + info.cp_code = req.rpc_params.cp_code || undefined; info.auth_method = req.rpc_params.auth_method || config.DEFAULT_S3_AUTH_METHOD[info.endpoint_type] || undefined; info = _.omitBy(info, _.isUndefined); diff --git a/src/server/system_services/master_key_manager.js b/src/server/system_services/master_key_manager.js index 32bb9b8701..b32281653f 100644 --- a/src/server/system_services/master_key_manager.js +++ b/src/server/system_services/master_key_manager.js @@ -339,6 +339,13 @@ class MasterKeysManager { decipher: crypto.createDecipheriv(m_key.cipher_type, m_key.cipher_key, m_key.cipher_iv) }, undefined); } + if (keys.gcp_hmac_key?.secret_key) { + keys.gcp_hmac_key.secret_key = await this.secret_keys_cache.get_with_cache({ + encrypted_value: keys.gcp_hmac_key.secret_key, + decipher: crypto.createDecipheriv(m_key.cipher_type, m_key.cipher_key, m_key.cipher_iv) + }, undefined); + + } } } } From bd1c03bd5b71d3879b7f709fa33b77d90371127b Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 27 Mar 2024 19:50:50 +0100 Subject: [PATCH 07/14] Finish HMAC DB storage implementation Signed-off-by: Ben --- src/api/account_api.js | 3 +++ src/api/common_api.js | 11 +++++++++++ src/api/pool_api.js | 1 + src/sdk/namespace_gcp.js | 7 +++++++ src/sdk/object_sdk.js | 5 +++-- src/server/system_services/master_key_manager.js | 7 +++++++ src/server/system_services/pool_server.js | 12 ++++++++++++ src/server/system_services/schemas/account_schema.js | 1 + .../schemas/namespace_resource_schema.js | 1 + 9 files changed, 46 insertions(+), 2 deletions(-) diff --git a/src/api/account_api.js b/src/api/account_api.js index cbf152e5f6..e71fb77d05 100644 --- a/src/api/account_api.js +++ b/src/api/account_api.js @@ -464,6 +464,9 @@ module.exports = { }, endpoint_type: { $ref: 'common_api#/definitions/endpoint_type' + }, + gcp_hmac_key: { + $ref: 'common_api#/definitions/gcp_hmac_key' } } diff --git a/src/api/common_api.js b/src/api/common_api.js index 88ddbee4da..09ed7922e8 100644 --- a/src/api/common_api.js +++ b/src/api/common_api.js @@ -953,6 +953,17 @@ module.exports = { } }, + gcp_access_id: { wrapper: SensitiveString }, + gcp_secret_key: { wrapper: SensitiveString }, + + gcp_hmac_key: { + type: 'object', + properties: { + access_id: { $ref: '#/definitions/gcp_access_id' }, + secret_key: { $ref: '#/definitions/gcp_secret_key' }, + } + }, + ip_range: { type: 'object', required: ['start', 'end'], diff --git a/src/api/pool_api.js b/src/api/pool_api.js index c0cbf08f11..0581ed59ea 100644 --- a/src/api/pool_api.js +++ b/src/api/pool_api.js @@ -644,6 +644,7 @@ module.exports = { region: { type: 'string' }, + gcp_hmac_key: { $ref: 'common_api#/definitions/gcp_hmac_key' }, } }, diff --git a/src/sdk/namespace_gcp.js b/src/sdk/namespace_gcp.js index 4d1212338d..b1c7d03fe8 100644 --- a/src/sdk/namespace_gcp.js +++ b/src/sdk/namespace_gcp.js @@ -47,6 +47,13 @@ class NamespaceGCP { this.project_id = project_id; this.client_email = client_email; this.private_key = private_key; + this.gcs = new GoogleCloudStorage({ + projectId: this.project_id, + credentials: { + client_email: this.client_email, + private_key: this.private_key, + } + }); this.s3_client = new AWS.S3({ endpoint: 'https://storage.googleapis.com', accessKeyId: hmac_key.access_id, diff --git a/src/sdk/object_sdk.js b/src/sdk/object_sdk.js index e695806ffe..7782b5c74c 100644 --- a/src/sdk/object_sdk.js +++ b/src/sdk/object_sdk.js @@ -443,7 +443,7 @@ class ObjectSDK { /** * @returns {nb.Namespace} */ - // resource is a namespace_resource + // resource contains the values of namespace_resource_extended_info _setup_single_namespace({ resource: r, path: p }, bucket_id, options) { if (r.endpoint_type === 'NOOBAA') { @@ -503,7 +503,8 @@ class ObjectSDK { private_key, access_mode: r.access_mode, stats: this.stats, - hmac_key: r.gcp_hmac_key, + hmac_key: { access_id : r.gcp_hmac_key.access_id.unwrap(), + secret_key : r.gcp_hmac_key.secret_key.unwrap() } }); } if (r.fs_root_path || r.fs_root_path === '') { diff --git a/src/server/system_services/master_key_manager.js b/src/server/system_services/master_key_manager.js index b32281653f..0d9d5db2c9 100644 --- a/src/server/system_services/master_key_manager.js +++ b/src/server/system_services/master_key_manager.js @@ -378,6 +378,13 @@ class MasterKeysManager { master_key_id: ns_resource.account.master_key_id._id }, undefined); } + if (ns_resource.connection.gcp_hmac_key?.secret_key) { + ns_resource.connection.gcp_hmac_key.secret_key = await this.secret_keys_cache.get_with_cache({ + encrypted_value: ns_resource.connection.gcp_hmac_key.secret_key.unwrap(), + undefined, + master_key_id: ns_resource.account.master_key_id._id + }, undefined); + } } } } diff --git a/src/server/system_services/pool_server.js b/src/server/system_services/pool_server.js index 8795406613..76917f1bb6 100644 --- a/src/server/system_services/pool_server.js +++ b/src/server/system_services/pool_server.js @@ -300,6 +300,16 @@ async function create_namespace_resource(req) { }; } + let gcp_hmac_key; + if (connection?.gcp_hmac_key?.secret_key) { + gcp_hmac_key = { + access_id: connection.gcp_hmac_key.access_id, + secret_key: system_store.master_key_manager.encrypt_sensitive_string_with_master_key_id( + connection.gcp_hmac_key.secret_key, req.account.master_key_id._id + ) + }; + } + namespace_resource = new_namespace_resource_defaults(name, req.system._id, req.account._id, _.omitBy({ aws_sts_arn: connection.aws_sts_arn, endpoint: connection.endpoint, @@ -311,6 +321,7 @@ async function create_namespace_resource(req) { endpoint_type: connection.endpoint_type || 'AWS', region: connection.region, azure_log_access_keys, + gcp_hmac_key, }, _.isUndefined), undefined, req.rpc_params.access_mode); const cloud_buckets = await server_rpc.client.bucket.get_cloud_buckets({ @@ -1210,6 +1221,7 @@ function get_namespace_resource_extended_info(namespace_resource) { secret_key: namespace_resource.connection.secret_key, access_mode: namespace_resource.access_mode, aws_sts_arn: namespace_resource.connection.aws_sts_arn || undefined, + gcp_hmac_key: namespace_resource.connection.gcp_hmac_key, }; const nsfs_info = namespace_resource.nsfs_config && { fs_root_path: namespace_resource.nsfs_config.fs_root_path, diff --git a/src/server/system_services/schemas/account_schema.js b/src/server/system_services/schemas/account_schema.js index 3f3a7c94d3..c458c9ea80 100644 --- a/src/server/system_services/schemas/account_schema.js +++ b/src/server/system_services/schemas/account_schema.js @@ -70,6 +70,7 @@ module.exports = { access_key: { $ref: 'common_api#/definitions/access_key' }, secret_key: { $ref: 'common_api#/definitions/secret_key' }, azure_log_access_keys: { $ref: 'common_api#/definitions/azure_log_access_keys' }, + gcp_hmac_key: { $ref: 'common_api#/definitions/gcp_hmac_key' }, aws_sts_arn: { type: 'string' }, diff --git a/src/server/system_services/schemas/namespace_resource_schema.js b/src/server/system_services/schemas/namespace_resource_schema.js index bda78ee97a..2ba71e4054 100644 --- a/src/server/system_services/schemas/namespace_resource_schema.js +++ b/src/server/system_services/schemas/namespace_resource_schema.js @@ -53,6 +53,7 @@ module.exports = { access_key: { $ref: 'common_api#/definitions/access_key' }, secret_key: { $ref: 'common_api#/definitions/secret_key' }, azure_log_access_keys: { $ref: 'common_api#/definitions/azure_log_access_keys' }, + gcp_hmac_key: { $ref: 'common_api#/definitions/gcp_hmac_key' }, cp_code: { type: 'string' } From 8ff3283e47e94a71ad9afab7cde31fbbe0f92ca2 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 27 Mar 2024 19:53:16 +0100 Subject: [PATCH 08/14] Remove unneeded null check Signed-off-by: Ben --- src/server/system_services/pool_server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/system_services/pool_server.js b/src/server/system_services/pool_server.js index 76917f1bb6..83b03b8da6 100644 --- a/src/server/system_services/pool_server.js +++ b/src/server/system_services/pool_server.js @@ -301,7 +301,7 @@ async function create_namespace_resource(req) { } let gcp_hmac_key; - if (connection?.gcp_hmac_key?.secret_key) { + if (connection.gcp_hmac_key?.secret_key) { gcp_hmac_key = { access_id: connection.gcp_hmac_key.access_id, secret_key: system_store.master_key_manager.encrypt_sensitive_string_with_master_key_id( From 6cbd8c42aa1f219ced7c660f116701c8bb93e97d Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 27 Mar 2024 20:15:21 +0100 Subject: [PATCH 09/14] Style fixes Signed-off-by: Ben --- src/sdk/object_sdk.js | 4 ++-- src/server/system_services/account_server.js | 2 +- src/server/system_services/master_key_manager.js | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sdk/object_sdk.js b/src/sdk/object_sdk.js index 7782b5c74c..c69000a1c5 100644 --- a/src/sdk/object_sdk.js +++ b/src/sdk/object_sdk.js @@ -503,8 +503,8 @@ class ObjectSDK { private_key, access_mode: r.access_mode, stats: this.stats, - hmac_key: { access_id : r.gcp_hmac_key.access_id.unwrap(), - secret_key : r.gcp_hmac_key.secret_key.unwrap() } + hmac_key: { access_id: r.gcp_hmac_key.access_id.unwrap(), + secret_key: r.gcp_hmac_key.secret_key.unwrap() } }); } if (r.fs_root_path || r.fs_root_path === '') { diff --git a/src/server/system_services/account_server.js b/src/server/system_services/account_server.js index ba89b310aa..ce881929a3 100644 --- a/src/server/system_services/account_server.js +++ b/src/server/system_services/account_server.js @@ -786,7 +786,7 @@ async function add_external_connection(req) { // If the connection is for Google, generate an HMAC key for S3-compatible actions (e.g. multipart uploads) if (info.endpoint_type === 'GOOGLE') { - dbg.log0('add_external_connection: creating HMAC key for Google connection') + dbg.log0('add_external_connection: creating HMAC key for Google connection'); const key_file = JSON.parse(req.rpc_params.secret.unwrap()); const credentials = _.pick(key_file, 'client_email', 'private_key'); const gs_client = new GoogleStorage({ credentials, projectId: key_file.project_id }); diff --git a/src/server/system_services/master_key_manager.js b/src/server/system_services/master_key_manager.js index 0d9d5db2c9..60e7030c39 100644 --- a/src/server/system_services/master_key_manager.js +++ b/src/server/system_services/master_key_manager.js @@ -344,7 +344,7 @@ class MasterKeysManager { encrypted_value: keys.gcp_hmac_key.secret_key, decipher: crypto.createDecipheriv(m_key.cipher_type, m_key.cipher_key, m_key.cipher_iv) }, undefined); - + } } } From 39ea3e90a184ec1894997133756f04a134200d3b Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 28 Mar 2024 13:03:21 +0100 Subject: [PATCH 10/14] - Upgrade S3 SDK from v2 to v3 - Fix invalid call to stats collector Signed-off-by: Ben --- src/sdk/namespace_gcp.js | 73 +++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/src/sdk/namespace_gcp.js b/src/sdk/namespace_gcp.js index b1c7d03fe8..d3f9651a5c 100644 --- a/src/sdk/namespace_gcp.js +++ b/src/sdk/namespace_gcp.js @@ -11,7 +11,15 @@ const s3_utils = require('../endpoint/s3/s3_utils'); const S3Error = require('../endpoint/s3/s3_errors').S3Error; // we use this wrapper to set a custom user agent const GoogleCloudStorage = require('../util/google_storage_wrap'); -const AWS = require('aws-sdk'); +const { + AbortMultipartUploadCommand, + CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, + ListPartsCommand, + S3Client, + UploadPartCommand, + UploadPartCopyCommand, +} = require('@aws-sdk/client-s3'); /** * @implements {nb.Namespace} @@ -54,10 +62,13 @@ class NamespaceGCP { private_key: this.private_key, } }); - this.s3_client = new AWS.S3({ + this.s3_client = new S3Client({ endpoint: 'https://storage.googleapis.com', - accessKeyId: hmac_key.access_id, - secretAccessKey: hmac_key.secret_key + region: 'auto', //https://cloud.google.com/storage/docs/aws-simple-migration#storage-list-buckets-s3-python + credentials: { + accessKeyId: hmac_key.access_id, + secretAccessKey: hmac_key.secret_key, + }, }); this.bucket = target_bucket; @@ -204,7 +215,7 @@ class NamespaceGCP { read_stream.on('response', () => { let count = 1; const count_stream = stream_utils.get_tap_stream(data => { - this.stats_collector.update_namespace_write_stats({ + this.stats.update_namespace_write_stats({ namespace_resource_id: this.namespace_resource_id, bucket_name: params.bucket, size: data.length, @@ -311,8 +322,9 @@ class NamespaceGCP { async create_object_upload(params, object_sdk) { dbg.log0('NamespaceGCP.create_object_upload:', this.bucket, inspect(params)); const Tagging = params.tagging && params.tagging.map(tag => tag.key + '=' + tag.value).join('&'); - /** @type {AWS.S3.CreateMultipartUploadRequest} */ - const request = { + + /** @type {import('@aws-sdk/client-s3').CreateMultipartUploadRequest} */ + const mp_upload_input = { Bucket: this.bucket, Key: params.key, ContentType: params.content_type, @@ -320,7 +332,8 @@ class NamespaceGCP { Metadata: params.xattr, Tagging }; - const res = await this.s3_client.createMultipartUpload(request).promise(); + const mp_upload_cmd = new CreateMultipartUploadCommand(mp_upload_input); + const res = await this.s3_client.send(mp_upload_cmd); dbg.log0('NamespaceGCP.create_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); return { obj_id: res.UploadId }; @@ -328,11 +341,11 @@ class NamespaceGCP { async upload_multipart(params, object_sdk) { dbg.log0('NamespaceGCP.upload_multipart:', this.bucket, inspect(params)); + let etag; let res; if (params.copy_source) { const { copy_source, copy_source_range } = s3_utils.format_copy_source(params.copy_source); - - /** @type {AWS.S3.UploadPartCopyRequest} */ + /** @type {import('@aws-sdk/client-s3').UploadPartCopyRequest} */ const request = { Bucket: this.bucket, Key: params.key, @@ -342,7 +355,9 @@ class NamespaceGCP { CopySourceRange: copy_source_range, }; - res = await this.s3_client.uploadPartCopy(request).promise(); + const command = new UploadPartCopyCommand(request); + res = await this.s3_client.send(command); + etag = s3_utils.parse_etag(res.CopyPartResult.ETag); } else { let count = 1; const count_stream = stream_utils.get_tap_stream(data => { @@ -354,7 +369,7 @@ class NamespaceGCP { // clear count for next updates count = 0; }); - /** @type {AWS.S3.UploadPartRequest} */ + /** @type {import('@aws-sdk/client-s3').UploadPartRequest} */ const request = { Bucket: this.bucket, Key: params.key, @@ -365,7 +380,8 @@ class NamespaceGCP { ContentLength: params.size, }; try { - res = await this.s3_client.uploadPart(request).promise(); + const command = new UploadPartCommand(request); + res = await this.s3_client.send(command); } catch (err) { object_sdk.rpc_client.pool.update_issues_report({ namespace_resource_id: this.namespace_resource_id, @@ -374,22 +390,25 @@ class NamespaceGCP { }); throw err; } + etag = s3_utils.parse_etag(res.ETag); } dbg.log0('NamespaceGCP.upload_multipart:', this.bucket, inspect(params), 'res', inspect(res)); - const etag = s3_utils.parse_etag(res.ETag); return { etag }; } async list_multiparts(params, object_sdk) { dbg.log0('NamespaceGCP.list_multiparts:', this.bucket, inspect(params)); - const res = await this.s3_client.listParts({ + /** @type {import('@aws-sdk/client-s3').ListPartsRequest} */ + const request = { Bucket: this.bucket, Key: params.key, UploadId: params.obj_id, MaxParts: params.max, PartNumberMarker: params.num_marker, - }).promise(); + }; + const command = new ListPartsCommand(request); + const res = await this.s3_client.send(command); dbg.log0('NamespaceGCP.list_multiparts:', this.bucket, inspect(params), 'res', inspect(res)); return { @@ -407,7 +426,8 @@ class NamespaceGCP { async complete_object_upload(params, object_sdk) { dbg.log0('NamespaceGCP.complete_object_upload:', this.bucket, inspect(params)); - const res = await this.s3_client.completeMultipartUpload({ + /** @type {import('@aws-sdk/client-s3').CompleteMultipartUploadRequest} */ + const request = { Bucket: this.bucket, Key: params.key, UploadId: params.obj_id, @@ -417,7 +437,9 @@ class NamespaceGCP { ETag: `"${p.etag}"`, })) } - }).promise(); + }; + const command = new CompleteMultipartUploadCommand(request); + const res = await this.s3_client.send(command); dbg.log0('NamespaceGCP.complete_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); const etag = s3_utils.parse_etag(res.ETag); @@ -426,11 +448,14 @@ class NamespaceGCP { async abort_object_upload(params, object_sdk) { dbg.log0('NamespaceGCP.abort_object_upload:', this.bucket, inspect(params)); - const res = await this.s3_client.abortMultipartUpload({ + /** @type {import('@aws-sdk/client-s3').AbortMultipartUploadRequest} */ + const request = { Bucket: this.bucket, Key: params.key, UploadId: params.obj_id, - }).promise(); + }; + const command = new AbortMultipartUploadCommand(request); + const res = await this.s3_client.send(command); dbg.log0('NamespaceGCP.abort_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); } @@ -479,8 +504,8 @@ class NamespaceGCP { const res = await P.map_with_concurrency(10, params.objects, obj => this.gcs.bucket(this.bucket).file(obj.key).delete() - .then(() => ({})) - .catch(err => ({ err_code: err.code, err_message: err.errors[0].reason || 'InternalError' }))); + .then(() => ({})) + .catch(err => ({ err_code: err.code, err_message: err.errors[0].reason || 'InternalError' }))); dbg.log1('NamespaceGCP.delete_multiple_objects:', this.bucket, inspect(params), 'res', inspect(res)); @@ -508,9 +533,9 @@ class NamespaceGCP { // Set an empty metadata object to remove all tags const res = await this.gcs.bucket(this.bucket).file(params.key).setMetadata({}); dbg.log0('NamespaceGCP.delete_object_tagging:', this.bucket, inspect(params), 'res', inspect(res)); - } catch (err) { + } catch (err) { dbg.error('NamespaceGCP.delete_object_tagging error:', err); - } + } } async put_object_tagging(params, object_sdk) { dbg.log0('NamespaceGCP.put_object_tagging:', this.bucket, inspect(params)); From 4fdb4869217313d0cafc42ac4a45285e2b13ece1 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 28 Mar 2024 14:05:38 +0100 Subject: [PATCH 11/14] - Add error object handling for v3 - Explain why an S3 client is used Signed-off-by: Ben --- src/sdk/namespace_gcp.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/sdk/namespace_gcp.js b/src/sdk/namespace_gcp.js index d3f9651a5c..a3ce707786 100644 --- a/src/sdk/namespace_gcp.js +++ b/src/sdk/namespace_gcp.js @@ -21,6 +21,8 @@ const { UploadPartCopyCommand, } = require('@aws-sdk/client-s3'); +const { fix_error_object } = require('./noobaa_s3_client/noobaa_s3_client'); + /** * @implements {nb.Namespace} */ @@ -62,6 +64,11 @@ class NamespaceGCP { private_key: this.private_key, } }); + /* An S3 client is needed for multipart upload since GCP only supports multipart i[;pads] via the S3-compatible XML API + * https://cloud.google.com/storage/docs/multipart-uploads + * This is also the reason an HMAC key is generated as part of `add_external_connection` - since the standard GCP credentials + * cannot be used in conjunction with the S3 client. + */ this.s3_client = new S3Client({ endpoint: 'https://storage.googleapis.com', region: 'auto', //https://cloud.google.com/storage/docs/aws-simple-migration#storage-list-buckets-s3-python @@ -383,6 +390,7 @@ class NamespaceGCP { const command = new UploadPartCommand(request); res = await this.s3_client.send(command); } catch (err) { + fix_error_object(err); object_sdk.rpc_client.pool.update_issues_report({ namespace_resource_id: this.namespace_resource_id, error_code: String(err.code), From 3413e13b6770fb41def41da7eb69547dc2670aa5 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 28 Mar 2024 14:15:21 +0100 Subject: [PATCH 12/14] Shorten code Signed-off-by: Ben --- src/sdk/namespace_gcp.js | 121 ++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 66 deletions(-) diff --git a/src/sdk/namespace_gcp.js b/src/sdk/namespace_gcp.js index a3ce707786..76947216a9 100644 --- a/src/sdk/namespace_gcp.js +++ b/src/sdk/namespace_gcp.js @@ -330,17 +330,15 @@ class NamespaceGCP { dbg.log0('NamespaceGCP.create_object_upload:', this.bucket, inspect(params)); const Tagging = params.tagging && params.tagging.map(tag => tag.key + '=' + tag.value).join('&'); - /** @type {import('@aws-sdk/client-s3').CreateMultipartUploadRequest} */ - const mp_upload_input = { - Bucket: this.bucket, - Key: params.key, - ContentType: params.content_type, - StorageClass: params.storage_class, - Metadata: params.xattr, - Tagging - }; - const mp_upload_cmd = new CreateMultipartUploadCommand(mp_upload_input); - const res = await this.s3_client.send(mp_upload_cmd); + const res = await this.s3_client.send( + new CreateMultipartUploadCommand({ + Bucket: this.bucket, + Key: params.key, + ContentType: params.content_type, + StorageClass: params.storage_class, + Metadata: params.xattr, + Tagging + })); dbg.log0('NamespaceGCP.create_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); return { obj_id: res.UploadId }; @@ -352,18 +350,16 @@ class NamespaceGCP { let res; if (params.copy_source) { const { copy_source, copy_source_range } = s3_utils.format_copy_source(params.copy_source); - /** @type {import('@aws-sdk/client-s3').UploadPartCopyRequest} */ - const request = { - Bucket: this.bucket, - Key: params.key, - UploadId: params.obj_id, - PartNumber: params.num, - CopySource: copy_source, - CopySourceRange: copy_source_range, - }; - const command = new UploadPartCopyCommand(request); - res = await this.s3_client.send(command); + res = await this.s3_client.send( + new UploadPartCopyCommand({ + Bucket: this.bucket, + Key: params.key, + UploadId: params.obj_id, + PartNumber: params.num, + CopySource: copy_source, + CopySourceRange: copy_source_range, + })); etag = s3_utils.parse_etag(res.CopyPartResult.ETag); } else { let count = 1; @@ -376,19 +372,17 @@ class NamespaceGCP { // clear count for next updates count = 0; }); - /** @type {import('@aws-sdk/client-s3').UploadPartRequest} */ - const request = { - Bucket: this.bucket, - Key: params.key, - UploadId: params.obj_id, - PartNumber: params.num, - Body: params.source_stream.pipe(count_stream), - ContentMD5: params.md5_b64, - ContentLength: params.size, - }; try { - const command = new UploadPartCommand(request); - res = await this.s3_client.send(command); + res = await this.s3_client.send( + new UploadPartCommand({ + Bucket: this.bucket, + Key: params.key, + UploadId: params.obj_id, + PartNumber: params.num, + Body: params.source_stream.pipe(count_stream), + ContentMD5: params.md5_b64, + ContentLength: params.size, + })); } catch (err) { fix_error_object(err); object_sdk.rpc_client.pool.update_issues_report({ @@ -407,16 +401,14 @@ class NamespaceGCP { async list_multiparts(params, object_sdk) { dbg.log0('NamespaceGCP.list_multiparts:', this.bucket, inspect(params)); - /** @type {import('@aws-sdk/client-s3').ListPartsRequest} */ - const request = { - Bucket: this.bucket, - Key: params.key, - UploadId: params.obj_id, - MaxParts: params.max, - PartNumberMarker: params.num_marker, - }; - const command = new ListPartsCommand(request); - const res = await this.s3_client.send(command); + const res = await this.s3_client.send( + new ListPartsCommand({ + Bucket: this.bucket, + Key: params.key, + UploadId: params.obj_id, + MaxParts: params.max, + PartNumberMarker: params.num_marker, + })); dbg.log0('NamespaceGCP.list_multiparts:', this.bucket, inspect(params), 'res', inspect(res)); return { @@ -434,20 +426,18 @@ class NamespaceGCP { async complete_object_upload(params, object_sdk) { dbg.log0('NamespaceGCP.complete_object_upload:', this.bucket, inspect(params)); - /** @type {import('@aws-sdk/client-s3').CompleteMultipartUploadRequest} */ - const request = { - Bucket: this.bucket, - Key: params.key, - UploadId: params.obj_id, - MultipartUpload: { - Parts: _.map(params.multiparts, p => ({ - PartNumber: p.num, - ETag: `"${p.etag}"`, - })) - } - }; - const command = new CompleteMultipartUploadCommand(request); - const res = await this.s3_client.send(command); + const res = await this.s3_client.send( + new CompleteMultipartUploadCommand({ + Bucket: this.bucket, + Key: params.key, + UploadId: params.obj_id, + MultipartUpload: { + Parts: _.map(params.multiparts, p => ({ + PartNumber: p.num, + ETag: `"${p.etag}"`, + })) + } + })); dbg.log0('NamespaceGCP.complete_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); const etag = s3_utils.parse_etag(res.ETag); @@ -456,14 +446,13 @@ class NamespaceGCP { async abort_object_upload(params, object_sdk) { dbg.log0('NamespaceGCP.abort_object_upload:', this.bucket, inspect(params)); - /** @type {import('@aws-sdk/client-s3').AbortMultipartUploadRequest} */ - const request = { - Bucket: this.bucket, - Key: params.key, - UploadId: params.obj_id, - }; - const command = new AbortMultipartUploadCommand(request); - const res = await this.s3_client.send(command); + + const res = await this.s3_client.send( + new AbortMultipartUploadCommand({ + Bucket: this.bucket, + Key: params.key, + UploadId: params.obj_id, + })); dbg.log0('NamespaceGCP.abort_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); } From 0ab52e5137e1a3876938d6f8b6d72e66d9c57a75 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 13 May 2024 08:19:33 +0200 Subject: [PATCH 13/14] Fix typo Signed-off-by: Ben --- src/sdk/namespace_gcp.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sdk/namespace_gcp.js b/src/sdk/namespace_gcp.js index 76947216a9..a1cb69ab13 100644 --- a/src/sdk/namespace_gcp.js +++ b/src/sdk/namespace_gcp.js @@ -64,7 +64,7 @@ class NamespaceGCP { private_key: this.private_key, } }); - /* An S3 client is needed for multipart upload since GCP only supports multipart i[;pads] via the S3-compatible XML API + /* An S3 client is needed for multipart upload since GCP only supports multipart uploads via the S3-compatible XML API * https://cloud.google.com/storage/docs/multipart-uploads * This is also the reason an HMAC key is generated as part of `add_external_connection` - since the standard GCP credentials * cannot be used in conjunction with the S3 client. From 293eee35e335fa4dd0a6b20f0f715b3cfc81afcd Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 17 May 2024 12:19:00 +0200 Subject: [PATCH 14/14] - Rename const - Remove `else` branch by adding an early exit Signed-off-by: Ben --- src/sdk/namespace_gcp.js | 68 +++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/src/sdk/namespace_gcp.js b/src/sdk/namespace_gcp.js index a1cb69ab13..8eca4ea83c 100644 --- a/src/sdk/namespace_gcp.js +++ b/src/sdk/namespace_gcp.js @@ -328,7 +328,7 @@ class NamespaceGCP { async create_object_upload(params, object_sdk) { dbg.log0('NamespaceGCP.create_object_upload:', this.bucket, inspect(params)); - const Tagging = params.tagging && params.tagging.map(tag => tag.key + '=' + tag.value).join('&'); + const tagging = params.tagging && params.tagging.map(tag => tag.key + '=' + tag.value).join('&'); const res = await this.s3_client.send( new CreateMultipartUploadCommand({ @@ -337,7 +337,7 @@ class NamespaceGCP { ContentType: params.content_type, StorageClass: params.storage_class, Metadata: params.xattr, - Tagging + Tagging: tagging })); dbg.log0('NamespaceGCP.create_object_upload:', this.bucket, inspect(params), 'res', inspect(res)); @@ -361,39 +361,41 @@ class NamespaceGCP { CopySourceRange: copy_source_range, })); etag = s3_utils.parse_etag(res.CopyPartResult.ETag); - } else { - let count = 1; - const count_stream = stream_utils.get_tap_stream(data => { - this.stats?.update_namespace_write_stats({ - namespace_resource_id: this.namespace_resource_id, - size: data.length, - count - }); - // clear count for next updates - count = 0; + return { etag }; + } + + let count = 1; + const count_stream = stream_utils.get_tap_stream(data => { + this.stats?.update_namespace_write_stats({ + namespace_resource_id: this.namespace_resource_id, + size: data.length, + count }); - try { - res = await this.s3_client.send( - new UploadPartCommand({ - Bucket: this.bucket, - Key: params.key, - UploadId: params.obj_id, - PartNumber: params.num, - Body: params.source_stream.pipe(count_stream), - ContentMD5: params.md5_b64, - ContentLength: params.size, - })); - } catch (err) { - fix_error_object(err); - object_sdk.rpc_client.pool.update_issues_report({ - namespace_resource_id: this.namespace_resource_id, - error_code: String(err.code), - time: Date.now(), - }); - throw err; - } - etag = s3_utils.parse_etag(res.ETag); + // clear count for next updates + count = 0; + }); + try { + res = await this.s3_client.send( + new UploadPartCommand({ + Bucket: this.bucket, + Key: params.key, + UploadId: params.obj_id, + PartNumber: params.num, + Body: params.source_stream.pipe(count_stream), + ContentMD5: params.md5_b64, + ContentLength: params.size, + })); + } catch (err) { + fix_error_object(err); + object_sdk.rpc_client.pool.update_issues_report({ + namespace_resource_id: this.namespace_resource_id, + error_code: String(err.code), + time: Date.now(), + }); + throw err; } + etag = s3_utils.parse_etag(res.ETag); + dbg.log0('NamespaceGCP.upload_multipart:', this.bucket, inspect(params), 'res', inspect(res)); return { etag }; }