diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..0c97a49 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: https://www.paypal.me/civitaspo diff --git a/CHANGELOG.md b/CHANGELOG.md index dafab84..a4ae6ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +0.4.0 (2019-10-20) +================== +* [Enhancement] Update dependencies: aws-sdk 1.11.587 -> 1.11.653, digdag 0.9.37 -> 0.9.39 +* [Fix] Fix README: athena.query> preview option is false by default +* [New feature] Support auth_method: web_identity_token + 0.3.2 (2019-08-06) ================== diff --git a/README.md b/README.md index 4abffec..6d64779 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ _export: repositories: - https://jitpack.io dependencies: - - pro.civitaspo:digdag-operator-athena:0.3.2 + - pro.civitaspo:digdag-operator-athena:0.4.0 athena: auth_method: profile @@ -26,10 +26,9 @@ _export: echo>: ${athena.last_query} +stap3: - athena.ctas>: - select_query: template.sql + athena.ctas>: template.sql table: hoge - output: s3://mybucket/prefix/ + location: s3://mybucket/prefix/ ``` See [examples](./example/example.dig) for more cases. @@ -51,7 +50,10 @@ Define the below options on properties (which is indicated by `-c`, `--config`). - **athena.allow_auth_method_instance**: Indicates whether users can use **auth_method** `"instance"` (boolean, default: `false`) - **athena.allow_auth_method_profile**: Indicates whether users can use **auth_method** `"profile"` (boolean, default: `false`) - **athena.allow_auth_method_properties**: Indicates whether users can use **auth_method** `"properties"` (boolean, default: `false`) +- **athena.allow_auth_method_web_identity_token**: Indicates whether users can use **auth_method** `"web_identity_token"` (boolean, default: `false`) - **athena.assume_role_timeout_duration**: Maximum duration which server administer allows when users assume **role_arn**. (`DurationParam`, default: `1h`) +- **athena.default_web_identity_token_file**: Path to a web identity token file. (string, optional) +- **athena.default_web_identity_role_arn**: AWS Role when using a web identity token. (string, optional) ### Secrets @@ -78,6 +80,9 @@ Define the below options on properties (which is indicated by `-c`, `--config`). - `"properties"`: uses aws.accessKeyId and aws.secretKey Java system properties. - `"anonymous"`: uses anonymous access. This auth method can access only public files. - `"session"`: uses temporary-generated access_key_id, secret_access_key and session_token. + - `"web_identity_token"`: uses web identity token. + - **web_identity_token_file**: path to a web identity token file. (string, default: given by **athena.default_web_identity_token_file**) + - **web_identity_role_arn**: aws role arn when using a web identity token. (string, default: given by **athena.default_web_identity_role_arn**) - **use_http_proxy**: Indicate whether using when accessing AWS via http proxy. (boolean, default: `false`) - **region**: The AWS region to use for Athena service. (string, optional) - **endpoint**: The Amazon Athena endpoint address to use. (string, optional) @@ -171,7 +176,7 @@ Nothing - **database**: The name of the database. (string, optional) - **workgroup**: The name of the workgroup in which the query is being started. (string, optional) - **timeout**: Specify timeout period. (`DurationParam`, default: `"10m"`) -- **preview**: Call `athena.preview>` operator after run `athena.query>`. (boolean, default: `true`) +- **preview**: Call `athena.preview>` operator after run `athena.query>`. (boolean, default: `false`) ### Output Parameters @@ -218,7 +223,6 @@ Nothing - **database**: The database name for query execution context. (string, optional) - **table**: The table name for the new table (string, default: `digdag_athena_ctas_${session_uuid.replaceAll("-", "")}_${random}`) - **workgroup**: The name of the workgroup in which the query is being started. (string, optional) -- **output**: [**Deprecated**] Use **location** option instead. - **location**: Output location for data created by CTAS (string, default: `"s3://aws-athena-query-results-${AWS_ACCOUNT_ID}-/Unsaved/${YEAR}/${MONTH}/${DAY}/${athena_query_id}/"`) - **format**: The data format for the CTAS query results, such as `"orc"`, `"parquet"`, `"avro"`, `"json"`, or `"textfile"`. (string, default: `"parquet"`) - **compression**: The compression type to use for `"orc"` or `"parquet"`. (string, default: `"snappy"`) diff --git a/build.gradle b/build.gradle index cc92794..d857fe9 100644 --- a/build.gradle +++ b/build.gradle @@ -5,10 +5,10 @@ plugins { } group = 'pro.civitaspo' -version = '0.3.2' +version = '0.4.0' -def digdagVersion = '0.9.37' -def awsSdkVersion = "1.11.587" +def digdagVersion = '0.9.39' +def awsSdkVersion = "1.11.653" def scalaSemanticVersion = "2.13.0" def depScalaVersion = "2.13" diff --git a/example/example.dig b/example/example.dig index 3b9d231..c4e63e7 100644 --- a/example/example.dig +++ b/example/example.dig @@ -4,7 +4,7 @@ _export: - file://${repos} # - https://jitpack.io dependencies: - - pro.civitaspo:digdag-operator-athena:0.3.2 + - pro.civitaspo:digdag-operator-athena:0.4.0 athena: auth_method: profile value: 5 diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/athena/AbstractAthenaOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/athena/AbstractAthenaOperator.scala index b59b357..59a7a19 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/athena/AbstractAthenaOperator.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/athena/AbstractAthenaOperator.scala @@ -39,12 +39,17 @@ abstract class AbstractAthenaOperator(operatorName: String, isAllowedAuthMethodInstance = systemConfig.get("athena.allow_auth_method_instance", classOf[Boolean], false), isAllowedAuthMethodProfile = systemConfig.get("athena.allow_auth_method_profile", classOf[Boolean], false), isAllowedAuthMethodProperties = systemConfig.get("athena.allow_auth_method_properties", classOf[Boolean], false), + isAllowedAuthMethodWebIdentityToken = systemConfig.get("athena.allow_auth_method_web_identity_token", classOf[Boolean], false), assumeRoleTimeoutDuration = systemConfig.get("athena.assume_role_timeout_duration", classOf[DurationParam], DurationParam.parse("1h")), accessKeyId = secrets.getSecretOptional("access_key_id"), secretAccessKey = secrets.getSecretOptional("secret_access_key"), sessionToken = secrets.getSecretOptional("session_token"), roleArn = secrets.getSecretOptional("role_arn"), roleSessionName = secrets.getSecretOptional("role_session_name").or(s"digdag-athena-$sessionUuid"), + defaultWebIdentityTokenFile = systemConfig.getOptional("athena.default_web_identity_token_file", classOf[String]), + webIdentityTokenFile = params.getOptional("web_identity_token_file", classOf[String]), + defaultWebIdentityRoleArn = systemConfig.getOptional("athena.default_web_identity_role_arn", classOf[String]), + webIdentityRoleArn = params.getOptional("web_identity_role_arn", classOf[String]), httpProxy = secrets.getSecrets("http_proxy"), authMethod = params.get("auth_method", classOf[String], "basic"), profileName = params.get("profile_name", classOf[String], "default"), diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/Aws.scala b/src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/Aws.scala index 2a07b86..04009b3 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/Aws.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/Aws.scala @@ -2,7 +2,7 @@ package pro.civitaspo.digdag.plugin.athena.aws import com.amazonaws.{ClientConfiguration, Protocol} -import com.amazonaws.auth.{AnonymousAWSCredentials, AWSCredentials, AWSCredentialsProvider, AWSStaticCredentialsProvider, BasicAWSCredentials, BasicSessionCredentials, EC2ContainerCredentialsProviderWrapper, EnvironmentVariableCredentialsProvider, SystemPropertiesCredentialsProvider} +import com.amazonaws.auth.{AnonymousAWSCredentials, AWSCredentials, AWSCredentialsProvider, AWSStaticCredentialsProvider, BasicAWSCredentials, BasicSessionCredentials, EC2ContainerCredentialsProviderWrapper, EnvironmentVariableCredentialsProvider, SystemPropertiesCredentialsProvider, WebIdentityTokenCredentialsProvider} import com.amazonaws.auth.profile.{ProfileCredentialsProvider, ProfilesConfigFile} import com.amazonaws.client.builder.AwsClientBuilder import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration @@ -78,14 +78,15 @@ case class Aws(conf: AwsConf) private def standardCredentialsProvider: AWSCredentialsProvider = { conf.authMethod match { - case "basic" => basicAuthMethodAWSCredentialsProvider - case "env" => envAuthMethodAWSCredentialsProvider - case "instance" => instanceAuthMethodAWSCredentialsProvider - case "profile" => profileAuthMethodAWSCredentialsProvider - case "properties" => propertiesAuthMethodAWSCredentialsProvider - case "anonymous" => anonymousAuthMethodAWSCredentialsProvider - case "session" => sessionAuthMethodAWSCredentialsProvider - case _ => + case "basic" => basicAuthMethodAWSCredentialsProvider + case "env" => envAuthMethodAWSCredentialsProvider + case "instance" => instanceAuthMethodAWSCredentialsProvider + case "profile" => profileAuthMethodAWSCredentialsProvider + case "properties" => propertiesAuthMethodAWSCredentialsProvider + case "anonymous" => anonymousAuthMethodAWSCredentialsProvider + case "session" => sessionAuthMethodAWSCredentialsProvider + case "web_identity_token" => webIdentityTokenAuthMethodAWSCredentialsProvider + case _ => throw new ConfigException( s"""auth_method: "${conf.authMethod}" is not supported. available `auth_method`s are "basic", "env", "instance", "profile", "properties", "anonymous", or "session".""" ) @@ -151,6 +152,18 @@ case class Aws(conf: AwsConf) new AWSStaticCredentialsProvider(credentials) } + private def webIdentityTokenAuthMethodAWSCredentialsProvider: AWSCredentialsProvider = + { + if (!conf.isAllowedAuthMethodWebIdentityToken) throw new ConfigException(s"""auth_method: "${conf.authMethod}" is not allowed.""") + if (!conf.webIdentityTokenFile.or(conf.defaultWebIdentityTokenFile).isPresent) throw new ConfigException(s"""`web_identity_token_file` or `athena.allow_auth_method_web_identity_token` (system) must be set when `auth_method` is "${conf.authMethod}".""") + if (!conf.webIdentityRoleArn.or(conf.defaultWebIdentityRoleArn).isPresent) throw new ConfigException(s"""`web_identity_role_arn` or `athena.allow_auth_method_web_identity_role_arn` (system) must be set when `auth_method` is "${conf.authMethod}".""") + WebIdentityTokenCredentialsProvider.builder() + .webIdentityTokenFile(conf.webIdentityTokenFile.or(conf.defaultWebIdentityTokenFile).get()) + .roleArn(conf.webIdentityRoleArn.or(conf.defaultWebIdentityRoleArn).get()) + .roleSessionName(conf.roleSessionName) + .build() + } + private def clientConfiguration: ClientConfiguration = { if (!conf.useHttpProxy) return new ClientConfiguration() diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/AwsConf.scala b/src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/AwsConf.scala index 61625e5..43fa732 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/AwsConf.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/athena/aws/AwsConf.scala @@ -10,6 +10,7 @@ case class AwsConf( isAllowedAuthMethodInstance: Boolean, isAllowedAuthMethodProfile: Boolean, isAllowedAuthMethodProperties: Boolean, + isAllowedAuthMethodWebIdentityToken: Boolean, assumeRoleTimeoutDuration: DurationParam, accessKeyId: Optional[String], secretAccessKey: Optional[String], @@ -20,6 +21,10 @@ case class AwsConf( authMethod: String, profileName: String, profileFile: Optional[String], + defaultWebIdentityTokenFile: Optional[String], + webIdentityTokenFile: Optional[String], + defaultWebIdentityRoleArn: Optional[String], + webIdentityRoleArn: Optional[String], useHttpProxy: Boolean, region: Optional[String], endpoint: Optional[String]