Skip to content

Add support for SSE-C encryption on S3 #88

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add support for SSE-C encryption on S3 #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions extension/httpfs/create_secret_functions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,10 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
secret->secret_map["use_ssl"] = Value::BOOLEAN(named_param.second.GetValue<bool>());
} else if (lower_name == "kms_key_id") {
secret->secret_map["kms_key_id"] = named_param.second.ToString();
} else if (lower_name == "sse_c_key") {
secret->secret_map["sse_c_key"] = named_param.second.ToString();
} else if (lower_name == "sse_c_key_md5") {
secret->secret_map["sse_c_key_md5"] = named_param.second.ToString();
} else if (lower_name == "url_compatibility_mode") {
if (named_param.second.type() != LogicalType::BOOLEAN) {
throw InvalidInputException("Invalid type past to secret option: '%s', found '%s', expected: 'BOOLEAN'",
Expand Down Expand Up @@ -120,6 +124,16 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
}
}

bool has_sse_c_key = secret->secret_map.find("sse_c_key") != secret->secret_map.end();
bool has_sse_c_key_md5 = secret->secret_map.find("sse_c_key_md5") != secret->secret_map.end();
if (has_sse_c_key != has_sse_c_key_md5) {
throw InvalidInputException("Both `sse_c_key` and `sse_c_key_md5` must be set together, or neither should be set");
}
bool has_kms_key_id = secret->secret_map.find("kms_key_id") != secret->secret_map.end();
if (has_kms_key_id && has_sse_c_key) {
throw InvalidInputException("Cannot set `kms_key_id` and `sse_c_key` at the same time");
}

return std::move(secret);
}

Expand Down Expand Up @@ -190,6 +204,8 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function,
function.named_parameters["url_style"] = LogicalType::VARCHAR;
function.named_parameters["use_ssl"] = LogicalType::BOOLEAN;
function.named_parameters["kms_key_id"] = LogicalType::VARCHAR;
function.named_parameters["sse_c_key"] = LogicalType::VARCHAR;
function.named_parameters["sse_c_key_md5"] = LogicalType::VARCHAR;
function.named_parameters["url_compatibility_mode"] = LogicalType::BOOLEAN;
function.named_parameters["requester_pays"] = LogicalType::BOOLEAN;

Expand Down
2 changes: 2 additions & 0 deletions extension/httpfs/httpfs_extension.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ static void LoadInternal(DatabaseInstance &instance) {
config.AddExtensionOption("s3_url_style", "S3 URL style", LogicalType::VARCHAR, Value("vhost"));
config.AddExtensionOption("s3_use_ssl", "S3 use SSL", LogicalType::BOOLEAN, Value(true));
config.AddExtensionOption("s3_kms_key_id", "S3 KMS Key ID", LogicalType::VARCHAR);
config.AddExtensionOption("s3_sse_c_key", "S3 SSE-C Key", LogicalType::VARCHAR);
config.AddExtensionOption("s3_sse_c_key_md5", "S3 SSE-C Key MD5", LogicalType::VARCHAR);
config.AddExtensionOption("s3_url_compatibility_mode", "Disable Globs and Query Parameters on S3 URLs",
LogicalType::BOOLEAN, Value(false));
config.AddExtensionOption("s3_requester_pays", "S3 use requester pays mode", LogicalType::BOOLEAN, Value(false));
Expand Down
5 changes: 4 additions & 1 deletion extension/httpfs/include/s3fs.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ struct S3AuthParams {
string session_token;
string endpoint;
string kms_key_id;
string sse_c_key;
string sse_c_key_md5;
string url_style;
bool use_ssl = true;
bool s3_url_compatibility_mode = false;
Expand All @@ -44,9 +46,10 @@ struct AWSEnvironmentCredentialsProvider {
static constexpr const char *DUCKDB_ENDPOINT_ENV_VAR = "DUCKDB_S3_ENDPOINT";
static constexpr const char *DUCKDB_USE_SSL_ENV_VAR = "DUCKDB_S3_USE_SSL";
static constexpr const char *DUCKDB_KMS_KEY_ID_ENV_VAR = "DUCKDB_S3_KMS_KEY_ID";
static constexpr const char *DUCKDB_SSE_C_KEY_ENV_VAR = "DUCKDB_S3_SSE_C_KEY";
static constexpr const char *DUCKDB_SSE_C_KEY_MD5_ENV_VAR = "DUCKDB_S3_SSE_C_KEY_MD5";
static constexpr const char *DUCKDB_REQUESTER_PAYS_ENV_VAR = "DUCKDB_S3_REQUESTER_PAYS";


explicit AWSEnvironmentCredentialsProvider(DBConfig &config) : config(config) {};

DBConfig &config;
Expand Down
26 changes: 24 additions & 2 deletions extension/httpfs/s3fs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/specifying-kms-encryption.html#sse-request-headers-kms
bool use_sse_kms = auth_params.kms_key_id.length() > 0 && (method == "POST" || method == "PUT") &&
query.find("uploadId") == std::string::npos;
bool use_sse_c = auth_params.sse_c_key.length() > 0 && auth_params.sse_c_key_md5.length() > 0;

res["x-amz-date"] = datetime_now;
res["x-amz-content-sha256"] = payload_hash;
Expand All @@ -61,13 +62,18 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
res["x-amz-server-side-encryption"] = "aws:kms";
res["x-amz-server-side-encryption-aws-kms-key-id"] = auth_params.kms_key_id;
}
if (use_sse_c) {
res["x-amz-server-side-encryption-customer-algorithm"] = "AES256";
res["x-amz-server-side-encryption-customer-key"] = auth_params.sse_c_key;
res["x-amz-server-side-encryption-customer-key-md5"] = auth_params.sse_c_key_md5;
}

bool use_requester_pays = auth_params.requester_pays;
if (use_requester_pays) {
res["x-amz-request-payer"] = "requester";
}

string signed_headers = "";
string signed_headers = "";
hash_bytes canonical_request_hash;
hash_str canonical_request_hash_str;
if (content_type.length() > 0) {
Expand All @@ -80,10 +86,13 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
if (use_sse_kms) {
signed_headers += ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id";
}
if (use_sse_c) {
signed_headers += ";x-amz-server-side-encryption-customer-algorithm;x-amz-server-side-encryption-customer-key;x-amz-server-side-encryption-customer-key-md5";
}
if (use_requester_pays) {
signed_headers += ";x-amz-request-payer";
}
auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query;
auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query;
if (content_type.length() > 0) {
canonical_request += "\ncontent-type:" + content_type;
}
Expand All @@ -95,6 +104,11 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
canonical_request += "\nx-amz-server-side-encryption:aws:kms";
canonical_request += "\nx-amz-server-side-encryption-aws-kms-key-id:" + auth_params.kms_key_id;
}
if (use_sse_c) {
canonical_request += "\nx-amz-server-side-encryption-customer-algorithm:AES256";
canonical_request += "\nx-amz-server-side-encryption-customer-key:" + auth_params.sse_c_key;
canonical_request += "\nx-amz-server-side-encryption-customer-key-md5:" + auth_params.sse_c_key_md5;
}
if (use_requester_pays) {
canonical_request += "\nx-amz-request-payer:requester";
}
Expand Down Expand Up @@ -154,6 +168,8 @@ void AWSEnvironmentCredentialsProvider::SetAll() {
this->SetExtensionOptionValue("s3_endpoint", DUCKDB_ENDPOINT_ENV_VAR);
this->SetExtensionOptionValue("s3_use_ssl", DUCKDB_USE_SSL_ENV_VAR);
this->SetExtensionOptionValue("s3_kms_key_id", DUCKDB_KMS_KEY_ID_ENV_VAR);
this->SetExtensionOptionValue("s3_sse_c_key", DUCKDB_SSE_C_KEY_ENV_VAR);
this->SetExtensionOptionValue("s3_sse_c_key_md5", DUCKDB_SSE_C_KEY_MD5_ENV_VAR);
this->SetExtensionOptionValue("s3_requester_pays", DUCKDB_REQUESTER_PAYS_ENV_VAR);
}

Expand All @@ -167,6 +183,8 @@ S3AuthParams AWSEnvironmentCredentialsProvider::CreateParams() {
params.session_token = SESSION_TOKEN_ENV_VAR;
params.endpoint = DUCKDB_ENDPOINT_ENV_VAR;
params.kms_key_id = DUCKDB_KMS_KEY_ID_ENV_VAR;
params.sse_c_key = DUCKDB_SSE_C_KEY_ENV_VAR;
params.sse_c_key_md5 = DUCKDB_SSE_C_KEY_MD5_ENV_VAR;
params.use_ssl = DUCKDB_USE_SSL_ENV_VAR;
params.requester_pays = DUCKDB_REQUESTER_PAYS_ENV_VAR;

Expand All @@ -192,6 +210,8 @@ S3AuthParams S3AuthParams::ReadFrom(optional_ptr<FileOpener> opener, FileOpenerI
secret_reader.TryGetSecretKeyOrSetting("region", "s3_region", result.region);
secret_reader.TryGetSecretKeyOrSetting("use_ssl", "s3_use_ssl", result.use_ssl);
secret_reader.TryGetSecretKeyOrSetting("kms_key_id", "s3_kms_key_id", result.kms_key_id);
secret_reader.TryGetSecretKeyOrSetting("sse_c_key", "s3_sse_c_key", result.sse_c_key);
secret_reader.TryGetSecretKeyOrSetting("sse_c_key_md5", "s3_sse_c_key_md5", result.sse_c_key_md5);
secret_reader.TryGetSecretKeyOrSetting("s3_url_compatibility_mode", "s3_url_compatibility_mode",
result.s3_url_compatibility_mode);
secret_reader.TryGetSecretKeyOrSetting("requester_pays", "s3_requester_pays",
Expand Down Expand Up @@ -233,6 +253,8 @@ unique_ptr<KeyValueSecret> CreateSecret(vector<string> &prefix_paths_p, string &
return_value->secret_map["url_style"] = params.url_style;
return_value->secret_map["use_ssl"] = params.use_ssl;
return_value->secret_map["kms_key_id"] = params.kms_key_id;
return_value->secret_map["sse_c_key"] = params.sse_c_key;
return_value->secret_map["sse_c_key_md5"] = params.sse_c_key_md5;
return_value->secret_map["s3_url_compatibility_mode"] = params.s3_url_compatibility_mode;
return_value->secret_map["requester_pays"] = params.requester_pays;

Expand Down