Skip to content

Commit 9af72f5

Browse files
landonxjameslandonxjames
authored
Add customization to ensure S3 Expires field is always a DateTime (#3730)
## Motivation and Context <!--- Why is this change required? What problem does it solve? --> <!--- If it fixes an open issue, please link to the issue here --> At some point in the near future (after this customization is applied to all existing SDKs) the S3 model will change the type of `Expires` members to `String` from the current `Timestamp`. This change would break backwards compatibility for us. ## Description <!--- Describe your changes in detail --> Add customization to ensure S3 `Expires` field is always a `Timestamp` and ass a new synthetic member `ExpiresString` that persists the un-parsed data from the `Expires` header. ## Testing <!--- Please describe in detail how you tested your changes --> <!--- Include details of your testing environment, and the tests you ran to --> <!--- see how your change affects other areas of the code, etc. --> Added tests to ensure that the model is pre-processed correctly. Added integration tests for S3. Considered making this more generic codegen tests, but since this customization will almost certainly only ever apply to S3 and I wanted to ensure that it was properly applied to the generated S3 SDK I opted for this route. ## Checklist <!--- If a checkbox below is not applicable, then please DELETE it rather than leaving it unchecked --> - [x] I have updated `CHANGELOG.next.toml` if I made changes to the AWS SDK, generated SDK code, or SDK runtime crates ---- _By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice._
1 parent 5c0baa7 commit 9af72f5

File tree

8 files changed

+423
-1
lines changed

8 files changed

+423
-1
lines changed

‎CHANGELOG.next.toml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,9 @@ Content-Type header validation now ignores parameter portion of media types.
2323
references = ["smithy-rs#3471","smithy-rs#3724"]
2424
meta = { "breaking" = false, "tada" = false, "bug" = true, target = "server" }
2525
authors = ["djedward"]
26+
27+
[[aws-sdk-rust]]
28+
message = "Add customizations for S3 Expires fields."
29+
references = ["smithy-rs#3730"]
30+
meta = { "breaking" = false, "tada" = false, "bug" = false }
31+
author = "landonxjames"

‎aws/rust-runtime/aws-inlineable/src/lib.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ pub mod endpoint_discovery;
5656
// the `presigning_interceptors` module can refer to it.
5757
mod serialization_settings;
5858

59+
/// Parse the Expires and ExpiresString fields correctly
60+
pub mod s3_expires_interceptor;
61+
5962
// just so docs work
6063
#[allow(dead_code)]
6164
/// allow docs to work

‎aws/rust-runtime/aws-inlineable/src/s3_expires_interceptor.rs

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
use aws_smithy_runtime_api::box_error::BoxError;
7+
use aws_smithy_runtime_api::client::interceptors::context::BeforeDeserializationInterceptorContextMut;
8+
use aws_smithy_runtime_api::client::interceptors::Intercept;
9+
use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents;
10+
use aws_smithy_types::config_bag::ConfigBag;
11+
use aws_smithy_types::date_time::{DateTime, Format};
12+
13+
/// An interceptor to implement custom parsing logic for S3's `Expires` header. This
14+
/// intercaptor copies the value of the `Expires` header to a (synthetically added)
15+
/// `ExpiresString` header. It also attempts to parse the header as an `HttpDate`, if
16+
/// that parsing fails the header is removed so the `Expires` field in the final output
17+
/// will be `None`.
18+
#[derive(Debug)]
19+
pub(crate) struct S3ExpiresInterceptor;
20+
const EXPIRES: &str = "Expires";
21+
const EXPIRES_STRING: &str = "ExpiresString";
22+
23+
impl Intercept for S3ExpiresInterceptor {
24+
fn name(&self) -> &'static str {
25+
"S3ExpiresInterceptor"
26+
}
27+
28+
fn modify_before_deserialization(
29+
&self,
30+
context: &mut BeforeDeserializationInterceptorContextMut<'_>,
31+
_: &RuntimeComponents,
32+
_: &mut ConfigBag,
33+
) -> Result<(), BoxError> {
34+
let headers = context.response_mut().headers_mut();
35+
36+
if headers.contains_key(EXPIRES) {
37+
let expires_header = headers.get(EXPIRES).unwrap().to_string();
38+
39+
// If the Expires header fails to parse to an HttpDate we remove the header so
40+
// it is parsed to None. We use HttpDate since that is the SEP defined default
41+
// if no other format is specified in the model.
42+
if DateTime::from_str(&expires_header, Format::HttpDate).is_err() {
43+
tracing::debug!(
44+
"Failed to parse the header `{EXPIRES}` = \"{expires_header}\" as an HttpDate. The raw string value can found in `{EXPIRES_STRING}`."
45+
);
46+
headers.remove(EXPIRES);
47+
}
48+
49+
// Regardless of parsing success we copy the value of the Expires header to the
50+
// ExpiresString header.
51+
headers.insert(EXPIRES_STRING, expires_header);
52+
}
53+
54+
Ok(())
55+
}
56+
}

‎aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/AwsCodegenDecorator.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import software.amazon.smithy.rustsdk.customize.lambda.LambdaDecorator
2121
import software.amazon.smithy.rustsdk.customize.onlyApplyTo
2222
import software.amazon.smithy.rustsdk.customize.route53.Route53Decorator
2323
import software.amazon.smithy.rustsdk.customize.s3.S3Decorator
24+
import software.amazon.smithy.rustsdk.customize.s3.S3ExpiresDecorator
2425
import software.amazon.smithy.rustsdk.customize.s3.S3ExpressDecorator
2526
import software.amazon.smithy.rustsdk.customize.s3.S3ExtendedRequestIdDecorator
2627
import software.amazon.smithy.rustsdk.customize.s3control.S3ControlDecorator
@@ -79,6 +80,7 @@ val DECORATORS: List<ClientCodegenDecorator> =
7980
S3ExpressDecorator(),
8081
S3ExtendedRequestIdDecorator(),
8182
IsTruncatedPaginatorDecorator(),
83+
S3ExpiresDecorator(),
8284
),
8385
S3ControlDecorator().onlyApplyTo("com.amazonaws.s3control#AWSS3ControlServiceV20180820"),
8486
STSDecorator().onlyApplyTo("com.amazonaws.sts#AWSSecurityTokenServiceV20110615"),

‎aws/sdk-codegen/src/main/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpiresDecorator.kt

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package software.amazon.smithy.rustsdk.customize.s3
7+
8+
import software.amazon.smithy.model.Model
9+
import software.amazon.smithy.model.shapes.MemberShape
10+
import software.amazon.smithy.model.shapes.OperationShape
11+
import software.amazon.smithy.model.shapes.ServiceShape
12+
import software.amazon.smithy.model.shapes.ShapeType
13+
import software.amazon.smithy.model.shapes.StringShape
14+
import software.amazon.smithy.model.shapes.StructureShape
15+
import software.amazon.smithy.model.traits.DeprecatedTrait
16+
import software.amazon.smithy.model.traits.DocumentationTrait
17+
import software.amazon.smithy.model.traits.HttpHeaderTrait
18+
import software.amazon.smithy.model.traits.OutputTrait
19+
import software.amazon.smithy.model.transform.ModelTransformer
20+
import software.amazon.smithy.rust.codegen.client.smithy.ClientCodegenContext
21+
import software.amazon.smithy.rust.codegen.client.smithy.ClientRustSettings
22+
import software.amazon.smithy.rust.codegen.client.smithy.customize.ClientCodegenDecorator
23+
import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationCustomization
24+
import software.amazon.smithy.rust.codegen.client.smithy.generators.OperationSection
25+
import software.amazon.smithy.rust.codegen.core.rustlang.Writable
26+
import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate
27+
import software.amazon.smithy.rust.codegen.core.rustlang.writable
28+
import software.amazon.smithy.rust.codegen.core.smithy.RuntimeType
29+
import software.amazon.smithy.rust.codegen.core.util.getTrait
30+
import software.amazon.smithy.rust.codegen.core.util.hasTrait
31+
import software.amazon.smithy.rust.codegen.core.util.outputShape
32+
import software.amazon.smithy.rustsdk.InlineAwsDependency
33+
import kotlin.streams.asSequence
34+
35+
/**
36+
* Enforces that Expires fields have the DateTime type (since in the future the model will change to model them as String),
37+
* and add an ExpiresString field to maintain the raw string value sent.
38+
*/
39+
class S3ExpiresDecorator : ClientCodegenDecorator {
40+
override val name: String = "S3ExpiresDecorator"
41+
override val order: Byte = 0
42+
private val expires = "Expires"
43+
private val expiresString = "ExpiresString"
44+
45+
override fun transformModel(
46+
service: ServiceShape,
47+
model: Model,
48+
settings: ClientRustSettings,
49+
): Model {
50+
val transformer = ModelTransformer.create()
51+
52+
// Ensure all `Expires` shapes are timestamps
53+
val expiresShapeTimestampMap =
54+
model.shapes()
55+
.asSequence()
56+
.mapNotNull { shape ->
57+
shape.members()
58+
.singleOrNull { member -> member.memberName.equals(expires, ignoreCase = true) }
59+
?.target
60+
}
61+
.associateWith { ShapeType.TIMESTAMP }
62+
var transformedModel = transformer.changeShapeType(model, expiresShapeTimestampMap)
63+
64+
// Add an `ExpiresString` string shape to the model
65+
val expiresStringShape = StringShape.builder().id("aws.sdk.rust.s3.synthetic#$expiresString").build()
66+
transformedModel = transformedModel.toBuilder().addShape(expiresStringShape).build()
67+
68+
// For output shapes only, deprecate `Expires` and add a synthetic member that targets `ExpiresString`
69+
transformedModel =
70+
transformer.mapShapes(transformedModel) { shape ->
71+
if (shape.hasTrait<OutputTrait>() && shape.memberNames.any { it.equals(expires, ignoreCase = true) }) {
72+
val builder = (shape as StructureShape).toBuilder()
73+
74+
// Deprecate `Expires`
75+
val expiresMember = shape.members().single { it.memberName.equals(expires, ignoreCase = true) }
76+
77+
builder.removeMember(expiresMember.memberName)
78+
val deprecatedTrait =
79+
DeprecatedTrait.builder()
80+
.message("Please use `expires_string` which contains the raw, unparsed value of this field.")
81+
.build()
82+
83+
builder.addMember(
84+
expiresMember.toBuilder()
85+
.addTrait(deprecatedTrait)
86+
.build(),
87+
)
88+
89+
// Add a synthetic member targeting `ExpiresString`
90+
val expiresStringMember = MemberShape.builder()
91+
expiresStringMember.target(expiresStringShape.id)
92+
expiresStringMember.id(expiresMember.id.toString() + "String") // i.e. com.amazonaws.s3.<MEMBER_NAME>$ExpiresString
93+
expiresStringMember.addTrait(HttpHeaderTrait(expiresString)) // Add HttpHeaderTrait to ensure the field is deserialized
94+
expiresMember.getTrait<DocumentationTrait>()?.let {
95+
expiresStringMember.addTrait(it) // Copy documentation from `Expires`
96+
}
97+
builder.addMember(expiresStringMember.build())
98+
builder.build()
99+
} else {
100+
shape
101+
}
102+
}
103+
104+
return transformedModel
105+
}
106+
107+
override fun operationCustomizations(
108+
codegenContext: ClientCodegenContext,
109+
operation: OperationShape,
110+
baseCustomizations: List<OperationCustomization>,
111+
): List<OperationCustomization> {
112+
val outputShape = operation.outputShape(codegenContext.model)
113+
114+
if (outputShape.memberNames.any { it.equals(expires, ignoreCase = true) }) {
115+
return baseCustomizations +
116+
ParseExpiresFieldsCustomization(
117+
codegenContext,
118+
)
119+
} else {
120+
return baseCustomizations
121+
}
122+
}
123+
}
124+
125+
class ParseExpiresFieldsCustomization(
126+
private val codegenContext: ClientCodegenContext,
127+
) : OperationCustomization() {
128+
override fun section(section: OperationSection): Writable =
129+
writable {
130+
when (section) {
131+
is OperationSection.AdditionalInterceptors -> {
132+
section.registerInterceptor(codegenContext.runtimeConfig, this) {
133+
val interceptor =
134+
RuntimeType.forInlineDependency(
135+
InlineAwsDependency.forRustFile("s3_expires_interceptor"),
136+
).resolve("S3ExpiresInterceptor")
137+
rustTemplate(
138+
"""
139+
#{S3ExpiresInterceptor}
140+
""",
141+
"S3ExpiresInterceptor" to interceptor,
142+
)
143+
}
144+
}
145+
146+
else -> {}
147+
}
148+
}
149+
}

‎aws/sdk-codegen/src/test/kotlin/software/amazon/smithy/rustsdk/customize/s3/S3ExpiresDecoratorTest.kt

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package software.amazon.smithy.rustsdk.customize.s3
7+
8+
import org.junit.jupiter.api.Assertions.assertNull
9+
import org.junit.jupiter.api.Assertions.assertTrue
10+
import org.junit.jupiter.api.Test
11+
import software.amazon.smithy.model.shapes.ServiceShape
12+
import software.amazon.smithy.model.shapes.ShapeId
13+
import software.amazon.smithy.model.traits.DeprecatedTrait
14+
import software.amazon.smithy.rust.codegen.client.testutil.testClientRustSettings
15+
import software.amazon.smithy.rust.codegen.core.testutil.asSmithyModel
16+
import software.amazon.smithy.rust.codegen.core.util.hasTrait
17+
import software.amazon.smithy.rust.codegen.core.util.targetOrSelf
18+
import kotlin.jvm.optionals.getOrNull
19+
20+
internal class S3ExpiresDecoratorTest {
21+
private val baseModel =
22+
"""
23+
namespace smithy.example
24+
use aws.protocols#restXml
25+
use aws.auth#sigv4
26+
use aws.api#service
27+
@restXml
28+
@sigv4(name: "s3")
29+
@service(
30+
sdkId: "S3"
31+
arnNamespace: "s3"
32+
)
33+
service S3 {
34+
version: "1.0.0",
35+
operations: [GetFoo, NewGetFoo]
36+
}
37+
operation GetFoo {
38+
input: GetFooInput
39+
output: GetFooOutput
40+
}
41+
42+
operation NewGetFoo {
43+
input: GetFooInput
44+
output: NewGetFooOutput
45+
}
46+
47+
structure GetFooInput {
48+
payload: String
49+
expires: String
50+
}
51+
52+
@output
53+
structure GetFooOutput {
54+
expires: Timestamp
55+
}
56+
57+
@output
58+
structure NewGetFooOutput {
59+
expires: String
60+
}
61+
""".asSmithyModel()
62+
63+
private val serviceShape = baseModel.expectShape(ShapeId.from("smithy.example#S3"), ServiceShape::class.java)
64+
private val settings = testClientRustSettings()
65+
private val model = S3ExpiresDecorator().transformModel(serviceShape, baseModel, settings)
66+
67+
@Test
68+
fun `Model is pre-processed correctly`() {
69+
val expiresShapes =
70+
listOf(
71+
model.expectShape(ShapeId.from("smithy.example#GetFooInput\$expires")),
72+
model.expectShape(ShapeId.from("smithy.example#GetFooOutput\$expires")),
73+
model.expectShape(ShapeId.from("smithy.example#NewGetFooOutput\$expires")),
74+
)
75+
76+
// Expires should always be Timestamp, even if not modeled as such since its
77+
// type will change in the future
78+
assertTrue(expiresShapes.all { it.targetOrSelf(model).isTimestampShape })
79+
80+
// All Expires output members should be marked with the deprecated trait
81+
assertTrue(
82+
expiresShapes
83+
.filter { it.id.toString().contains("Output") }
84+
.all { it.hasTrait<DeprecatedTrait>() },
85+
)
86+
87+
// No ExpiresString member should be added to the input shape
88+
assertNull(model.getShape(ShapeId.from("smithy.example#GetFooInput\$expiresString")).getOrNull())
89+
90+
val expiresStringOutputFields =
91+
listOf(
92+
model.expectShape(ShapeId.from("smithy.example#GetFooOutput\$expiresString")),
93+
model.expectShape(ShapeId.from("smithy.example#NewGetFooOutput\$expiresString")),
94+
)
95+
96+
// There should be a synthetic ExpiresString string member added to output shapes
97+
assertTrue(expiresStringOutputFields.all { it.targetOrSelf(model).isStringShape })
98+
99+
// The synthetic fields should not be deprecated
100+
assertTrue(expiresStringOutputFields.none { it.hasTrait<DeprecatedTrait>() })
101+
}
102+
}

0 commit comments

Comments
 (0)