From 194e19ebf09e3330de1b1d310e0fe842729afa20 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 8 May 2023 15:48:24 -0500 Subject: [PATCH 01/11] Support long-term credentials --- src/AWSCredentials.jl | 19 ++++++++++++++++++- src/utilities/credentials.jl | 25 +++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 6f6c879508..4b25e63bd9 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -424,10 +424,14 @@ function dot_aws_config(profile=nothing) settings = _aws_profile_config(ini, p) isempty(settings) && return nothing + credential_process = get(settings, "credential_process", nothing) access_key = get(settings, "aws_access_key_id", nothing) sso_start_url = get(settings, "sso_start_url", nothing) - if !isnothing(access_key) + if !isnothing(credential_process) + cmd = Cmd(Base.shell_split(credential_process)) + return credential_process_credentials(cmd) + elseif !isnothing(access_key) access_key, secret_key, token = _aws_get_credential_details(p, ini) return AWSCredentials(access_key, secret_key, token) elseif !isnothing(sso_start_url) @@ -559,6 +563,19 @@ function credentials_from_webtoken() ) end +function credential_process_credentials(cmd::Base.AbstractCmd) + nt = open(cmd, "r") do io + _read_credential_process(io) + end + return AWSCredentials( + nt.access_key_id, + nt.secret_access_key, + nt.session_token; + expiry=@something(nt.expiration, typemax(DateTime)), + renew=() -> credential_process_credentials(cmd), + ) +end + """ aws_get_region(; profile=nothing, config=nothing, default="$DEFAULT_REGION") diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index be2d96e8e2..0d743dfd6f 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -212,3 +212,28 @@ function _aws_get_sso_credential_details(profile::AbstractString, ini::Inifile) return (access_key, secret_key, token, expiry) end + +function _read_credential_process(io::IO) + # `JSON.parse` chokes on `Base.Process` I/O streams. + json = JSON.parse(read(io, String)) + + version = json["Version"] + if version != 1 + error( + "Credential process returned unhandled version $version:\n" * + sprint(JSON.print, json, 2) + ) + end + + access_key_id = json["AccessKeyId"] + secret_access_key = json["SecretAccessKey"] + session_token = json["SessionToken"] + + expiration = if haskey(json, "Expiration") + parse(DateTime, json["Expiration"], dateformat"yyyy-mm-dd\THH:MM:SS\Z") + else + nothing + end + + return (; access_key_id, secret_access_key, session_token, expiration) +end From 0e1c643e25a2049450efb3431f807fcd0f134713 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 8 May 2023 15:51:07 -0500 Subject: [PATCH 02/11] Rename to external_process_credentials --- src/AWSCredentials.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 4b25e63bd9..8ecdb531c7 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -430,7 +430,7 @@ function dot_aws_config(profile=nothing) if !isnothing(credential_process) cmd = Cmd(Base.shell_split(credential_process)) - return credential_process_credentials(cmd) + return external_process_credentials(cmd) elseif !isnothing(access_key) access_key, secret_key, token = _aws_get_credential_details(p, ini) return AWSCredentials(access_key, secret_key, token) @@ -563,7 +563,7 @@ function credentials_from_webtoken() ) end -function credential_process_credentials(cmd::Base.AbstractCmd) +function external_process_credentials(cmd::Base.AbstractCmd) nt = open(cmd, "r") do io _read_credential_process(io) end @@ -572,7 +572,7 @@ function credential_process_credentials(cmd::Base.AbstractCmd) nt.secret_access_key, nt.session_token; expiry=@something(nt.expiration, typemax(DateTime)), - renew=() -> credential_process_credentials(cmd), + renew=() -> external_process_credentials(cmd), ) end From 7a81ce4925c9ea07886070392c7f5f7de4a8fc23 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 8 May 2023 15:51:27 -0500 Subject: [PATCH 03/11] Add docstrings --- src/AWSCredentials.jl | 7 +++++++ src/utilities/credentials.jl | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 8ecdb531c7..6e8532b3ba 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -563,6 +563,13 @@ function credentials_from_webtoken() ) end +""" + external_process_credentials(cmd::Base.AbstractCmd) -> AWSCredentials + +Sources AWS credentials from an external process as defined in the AWS CLI config file. +See https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sourcing-external.html +for details. +""" function external_process_credentials(cmd::Base.AbstractCmd) nt = open(cmd, "r") do io _read_credential_process(io) diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index 0d743dfd6f..764487cba3 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -213,6 +213,12 @@ function _aws_get_sso_credential_details(profile::AbstractString, ini::Inifile) return (access_key, secret_key, token, expiry) end +""" + _read_credential_process(io::IO) -> NamedTuple + +Parse the AWS CLI external process output out as defined in: +https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-sourcing-external.html +""" function _read_credential_process(io::IO) # `JSON.parse` chokes on `Base.Process` I/O streams. json = JSON.parse(read(io, String)) From 9e59451872bc6c3753ab19090cffc9ceb78a60ca Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 8 May 2023 15:51:48 -0500 Subject: [PATCH 04/11] Use compat for implicit keywords --- Project.toml | 2 +- src/AWS.jl | 2 +- src/utilities/credentials.jl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index efe0825efc..8c45f4a638 100644 --- a/Project.toml +++ b/Project.toml @@ -23,7 +23,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" XMLDict = "228000da-037f-5747-90a9-8195ccbf91a5" [compat] -Compat = "3.29, 4" +Compat = "3.32, 4" GitHub = "5" HTTP = "1" IniFile = "0.5" diff --git a/src/AWS.jl b/src/AWS.jl index 7055f09f09..a08a30a6c4 100644 --- a/src/AWS.jl +++ b/src/AWS.jl @@ -1,6 +1,6 @@ module AWS -using Compat: Compat, @something +using Compat: Compat, @compat, @something using Base64 using Dates using Downloads: Downloads, Downloader, Curl diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index 764487cba3..867f23fac6 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -241,5 +241,5 @@ function _read_credential_process(io::IO) nothing end - return (; access_key_id, secret_access_key, session_token, expiration) + return @compat (; access_key_id, secret_access_key, session_token, expiration) end From b261108a11e0daee798c90092458992e4181ad15 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 09:46:30 -0500 Subject: [PATCH 05/11] Add tests --- src/AWSCredentials.jl | 5 +- src/utilities/credentials.jl | 12 +++-- test/AWSCredentials.jl | 88 ++++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 6e8532b3ba..85a828d4f1 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -22,7 +22,8 @@ export AWSCredentials, localhost_is_ec2, localhost_maybe_ec2, localhost_is_lambda, - credentials_from_webtoken + credentials_from_webtoken, + external_process_credentials function localhost_maybe_ec2() return localhost_is_ec2() || isfile("/sys/devices/virtual/dmi/id/product_uuid") @@ -577,7 +578,7 @@ function external_process_credentials(cmd::Base.AbstractCmd) return AWSCredentials( nt.access_key_id, nt.secret_access_key, - nt.session_token; + @something(nt.session_token, ""); expiry=@something(nt.expiration, typemax(DateTime)), renew=() -> external_process_credentials(cmd), ) diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index 867f23fac6..58632dfa07 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -233,12 +233,16 @@ function _read_credential_process(io::IO) access_key_id = json["AccessKeyId"] secret_access_key = json["SecretAccessKey"] - session_token = json["SessionToken"] - expiration = if haskey(json, "Expiration") - parse(DateTime, json["Expiration"], dateformat"yyyy-mm-dd\THH:MM:SS\Z") + # The presence of the "Expiration" key determines if the provided credentials are + # long-term credentials or temporary credentials. Temporary credentials must include a + # session token (https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_temp_use-resources.html) + if haskey(json, "Expiration") || haskey(json, "SessionToken") + expiration = parse(DateTime, json["Expiration"], dateformat"yyyy-mm-dd\THH:MM:SS\Z") + session_token = json["SessionToken"] else - nothing + expiration = nothing + session_token = nothing end return @compat (; access_key_id, secret_access_key, session_token, expiration) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 24c5998ddc..7f38c266cb 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -511,6 +511,43 @@ end end end + @testset "~/.aws/config - Credential Process" begin + mktempdir() do dir + credential_process_file = joinpath(dir, "cred_process") + open(credential_process_file, "w") do io + println(io, "#!/bin/sh") + println(io, "cat < 1, + "AccessKeyId" => test_values["Test-AccessKeyId"], + "SecretAccessKey" => test_values["Test-SecretAccessKey"], + )) + println(io, "\nEOF") + end + chmod(credential_process_file, 0o700) + + config_file = joinpath(dir, "config") + open(config_file, "w") do io + write( + io, + """ + [profile $(test_values["Test-Config-Profile"])] + credential_process = $(abspath(credential_process_file)) + """, + ) + end + + withenv("AWS_CONFIG_FILE" => config_file) do + specified_result = dot_aws_config(test_values["Test-Config-Profile"]) + + @test specified_result.access_key_id == test_values["Test-AccessKeyId"] + @test specified_result.secret_key == test_values["Test-SecretAccessKey"] + @test isempty(specified_result.token) + @test specified_result.expiry == typemax(DateTime) + end + end + end + @testset "~/.aws/creds - Default Profile" begin mktemp() do creds_file, creds_io write( @@ -696,6 +733,57 @@ end end end + @testset "Credential Process" begin + gen_process(json) = Cmd(["echo", JSON.json(json)]) + + long_term_resp = Dict( + "Version" => 1, + "AccessKeyId" => "access-key", + "SecretAccessKey" => "secret-key", + ) + creds = external_process_credentials(gen_process(long_term_resp)) + @test creds.access_key_id == long_term_resp["AccessKeyId"] + @test creds.secret_key == long_term_resp["SecretAccessKey"] + @test isempty(creds.token) + @test creds.expiry == typemax(DateTime) + + expiration = floor(now(UTC), Second) + temporary_resp = Dict( + "Version" => 1, + "AccessKeyId" => "access-key", + "SecretAccessKey" => "secret-key", + "SessionToken" => "session-token", + "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), + ) + creds = external_process_credentials(gen_process(temporary_resp)) + @test creds.access_key_id == temporary_resp["AccessKeyId"] + @test creds.secret_key == temporary_resp["SecretAccessKey"] + @test creds.token == temporary_resp["SessionToken"] + @test creds.expiry == expiration + + unhandled_version_resp = Dict( + "Version" => 2, + ) + ex = ErrorException("Credential process returned unhandled version 2:\n{\n \"Version\": 2\n}\n") + @test_throws ex external_process_credentials(gen_process(unhandled_version_resp)) + + missing_token_resp = Dict( + "Version" => 1, + "AccessKeyId" => "access-key", + "SecretAccessKey" => "secret-key", + "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), + ) + @test_throws KeyError("SessionToken") external_process_credentials(gen_process(missing_token_resp)) + + missing_expiration_resp = Dict( + "Version" => 1, + "AccessKeyId" => "access-key", + "SecretAccessKey" => "secret-key", + "SessionToken" => "session-token", + ) + @test_throws KeyError("Expiration") external_process_credentials(gen_process(missing_expiration_resp)) + end + @testset "Credentials Not Found" begin patches = [ @patch HTTP.request(method::String, url; kwargs...) = nothing From 8b87c2d11d090c53d8f52a177b0bd5be59b3ba90 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 11:56:36 -0500 Subject: [PATCH 06/11] Add precedence test --- test/AWSCredentials.jl | 57 ++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 7f38c266cb..146273234a 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -513,6 +513,7 @@ end @testset "~/.aws/config - Credential Process" begin mktempdir() do dir + config_file = joinpath(dir, "config") credential_process_file = joinpath(dir, "cred_process") open(credential_process_file, "w") do io println(io, "#!/bin/sh") @@ -526,24 +527,48 @@ end end chmod(credential_process_file, 0o700) - config_file = joinpath(dir, "config") - open(config_file, "w") do io - write( - io, - """ - [profile $(test_values["Test-Config-Profile"])] - credential_process = $(abspath(credential_process_file)) - """, - ) - end - withenv("AWS_CONFIG_FILE" => config_file) do - specified_result = dot_aws_config(test_values["Test-Config-Profile"]) + @testset "support" begin + open(config_file, "w") do io + write( + io, + """ + [profile $(test_values["Test-Config-Profile"])] + credential_process = $(abspath(credential_process_file)) + """, + ) + end + + result = dot_aws_config(test_values["Test-Config-Profile"]) + + @test result.access_key_id == test_values["Test-AccessKeyId"] + @test result.secret_key == test_values["Test-SecretAccessKey"] + @test isempty(result.token) + @test result.expiry == typemax(DateTime) + end - @test specified_result.access_key_id == test_values["Test-AccessKeyId"] - @test specified_result.secret_key == test_values["Test-SecretAccessKey"] - @test isempty(specified_result.token) - @test specified_result.expiry == typemax(DateTime) + # The AWS CLI uses the config file `credential_process` setting over + # specifying the config file `aws_access_key_id`/`aws_secret_access_key`. + @testset "precedence" begin + open(config_file, "w") do io + write( + io, + """ + [profile $(test_values["Test-Config-Profile"])] + aws_access_key_id = invalid + aws_secret_access_key = invalid + credential_process = $(abspath(credential_process_file)) + """, + ) + end + + result = dot_aws_config(test_values["Test-Config-Profile"]) + + @test result.access_key_id == test_values["Test-AccessKeyId"] + @test result.secret_key == test_values["Test-SecretAccessKey"] + @test isempty(result.token) + @test result.expiry == typemax(DateTime) + end end end end From 93d6c4311b71231464976c2596b51359cb83f219 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 11:57:51 -0500 Subject: [PATCH 07/11] Set project version to 1.85.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 8c45f4a638..9d152ec62f 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "AWS" uuid = "fbe9abb3-538b-5e4e-ba9e-bc94f4f92ebc" license = "MIT" -version = "1.84.1" +version = "1.85.0" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" From 7aba415edb4baefa3544348c7de81e212b037c7b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 12:16:20 -0500 Subject: [PATCH 08/11] Formatting --- src/utilities/credentials.jl | 4 ++-- test/AWSCredentials.jl | 15 ++++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/utilities/credentials.jl b/src/utilities/credentials.jl index 58632dfa07..599489cf49 100644 --- a/src/utilities/credentials.jl +++ b/src/utilities/credentials.jl @@ -226,8 +226,8 @@ function _read_credential_process(io::IO) version = json["Version"] if version != 1 error( - "Credential process returned unhandled version $version:\n" * - sprint(JSON.print, json, 2) + "Credential process returned unhandled version $version:\n", + sprint(JSON.print, json, 2), ) end diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 146273234a..6e11d0e536 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -518,11 +518,12 @@ end open(credential_process_file, "w") do io println(io, "#!/bin/sh") println(io, "cat < 1, "AccessKeyId" => test_values["Test-AccessKeyId"], "SecretAccessKey" => test_values["Test-SecretAccessKey"], - )) + ) + JSON.print(io, json) println(io, "\nEOF") end chmod(credential_process_file, 0o700) @@ -765,6 +766,7 @@ end "Version" => 1, "AccessKeyId" => "access-key", "SecretAccessKey" => "secret-key", + # format trick: using this comment to force use of multiple lines ) creds = external_process_credentials(gen_process(long_term_resp)) @test creds.access_key_id == long_term_resp["AccessKeyId"] @@ -789,7 +791,8 @@ end unhandled_version_resp = Dict( "Version" => 2, ) - ex = ErrorException("Credential process returned unhandled version 2:\n{\n \"Version\": 2\n}\n") + json = JSON.print(unhandled_version_resp, 2) + ex = ErrorException("Credential process returned unhandled version 2:\n$json") @test_throws ex external_process_credentials(gen_process(unhandled_version_resp)) missing_token_resp = Dict( @@ -798,7 +801,8 @@ end "SecretAccessKey" => "secret-key", "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), ) - @test_throws KeyError("SessionToken") external_process_credentials(gen_process(missing_token_resp)) + ex = KeyError("SessionToken") + @test_throws ex external_process_credentials(gen_process(missing_token_resp)) missing_expiration_resp = Dict( "Version" => 1, @@ -806,7 +810,8 @@ end "SecretAccessKey" => "secret-key", "SessionToken" => "session-token", ) - @test_throws KeyError("Expiration") external_process_credentials(gen_process(missing_expiration_resp)) + ex = KeyError("Expiration") + @test_throws ex external_process_credentials(gen_process(missing_expiration_resp)) end @testset "Credentials Not Found" begin From e399c386e04c61f8133e21cb7731c72ed781e45f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 12:22:22 -0500 Subject: [PATCH 09/11] fixup! Formatting --- test/AWSCredentials.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index 6e11d0e536..aba2155fcf 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -788,9 +788,7 @@ end @test creds.token == temporary_resp["SessionToken"] @test creds.expiry == expiration - unhandled_version_resp = Dict( - "Version" => 2, - ) + unhandled_version_resp = Dict("Version" => 2) json = JSON.print(unhandled_version_resp, 2) ex = ErrorException("Credential process returned unhandled version 2:\n$json") @test_throws ex external_process_credentials(gen_process(unhandled_version_resp)) @@ -801,7 +799,7 @@ end "SecretAccessKey" => "secret-key", "Expiration" => Dates.format(expiration, dateformat"yyyy-mm-dd\THH:MM:SS\Z"), ) - ex = KeyError("SessionToken") + ex = KeyError("SessionToken") @test_throws ex external_process_credentials(gen_process(missing_token_resp)) missing_expiration_resp = Dict( From a0491c49b57fecc04cf40f3879c5845021a5742d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 12:42:39 -0500 Subject: [PATCH 10/11] fixup! Formatting --- test/AWSCredentials.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/AWSCredentials.jl b/test/AWSCredentials.jl index aba2155fcf..60c513b5d6 100644 --- a/test/AWSCredentials.jl +++ b/test/AWSCredentials.jl @@ -789,7 +789,7 @@ end @test creds.expiry == expiration unhandled_version_resp = Dict("Version" => 2) - json = JSON.print(unhandled_version_resp, 2) + json = sprint(JSON.print, unhandled_version_resp, 2) ex = ErrorException("Credential process returned unhandled version 2:\n$json") @test_throws ex external_process_credentials(gen_process(unhandled_version_resp)) From d4ec67815c285cdf5e97e3464eb1d7b5628b419e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 9 May 2023 15:13:30 -0500 Subject: [PATCH 11/11] Order list of exports --- src/AWSCredentials.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/AWSCredentials.jl b/src/AWSCredentials.jl index 85a828d4f1..d6ca0588dd 100644 --- a/src/AWSCredentials.jl +++ b/src/AWSCredentials.jl @@ -8,22 +8,22 @@ using ..AWSExceptions export AWSCredentials, aws_account_number, - aws_get_region, aws_get_profile_settings, + aws_get_region, aws_user_arn, check_credentials, + credentials_from_webtoken, dot_aws_config, + dot_aws_config_file, dot_aws_credentials, dot_aws_credentials_file, - dot_aws_config_file, ec2_instance_credentials, ecs_instance_credentials, env_var_credentials, + external_process_credentials, localhost_is_ec2, - localhost_maybe_ec2, localhost_is_lambda, - credentials_from_webtoken, - external_process_credentials + localhost_maybe_ec2 function localhost_maybe_ec2() return localhost_is_ec2() || isfile("/sys/devices/virtual/dmi/id/product_uuid")