Skip to content

Commit d2b8497

Browse files
YuriyHolinkoYuriyHolinko
authored
Retry on configurable exception (#6991)
1 parent cd3b0e7 commit d2b8497

File tree

9 files changed

+116
-41
lines changed

9 files changed

+116
-41
lines changed

‎buildSrc/src/main/kotlin/otel.japicmp-conventions.gradle.kts

+2-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ val latestReleasedVersion: String by lazy {
2727

2828
class AllowNewAbstractMethodOnAutovalueClasses : AbstractRecordingSeenMembers() {
2929
override fun maybeAddViolation(member: JApiCompatibility): Violation? {
30-
val allowableAutovalueChanges = setOf(JApiCompatibilityChangeType.METHOD_ABSTRACT_ADDED_TO_CLASS, JApiCompatibilityChangeType.METHOD_ADDED_TO_PUBLIC_CLASS)
30+
val allowableAutovalueChanges = setOf(JApiCompatibilityChangeType.METHOD_ABSTRACT_ADDED_TO_CLASS,
31+
JApiCompatibilityChangeType.METHOD_ADDED_TO_PUBLIC_CLASS, JApiCompatibilityChangeType.ANNOTATION_ADDED)
3132
if (member.compatibilityChanges.filter { !allowableAutovalueChanges.contains(it.type) }.isEmpty() &&
3233
member is JApiMethod && isAutoValueClass(member.getjApiClass()))
3334
{

‎docs/apidiffs/current_vs_latest/opentelemetry-sdk-common.txt

+7-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,8 @@
11
Comparing source compatibility of opentelemetry-sdk-common-1.47.0-SNAPSHOT.jar against opentelemetry-sdk-common-1.46.0.jar
2-
No changes.
2+
**** MODIFIED CLASS: PUBLIC ABSTRACT io.opentelemetry.sdk.common.export.RetryPolicy (not serializable)
3+
=== CLASS FILE FORMAT VERSION: 52.0 <- 52.0
4+
+++* NEW METHOD: PUBLIC(+) ABSTRACT(+) java.util.function.Predicate<java.io.IOException> getRetryExceptionPredicate()
5+
+++ NEW ANNOTATION: javax.annotation.Nullable
6+
**** MODIFIED CLASS: PUBLIC ABSTRACT STATIC io.opentelemetry.sdk.common.export.RetryPolicy$RetryPolicyBuilder (not serializable)
7+
=== CLASS FILE FORMAT VERSION: 52.0 <- 52.0
8+
+++* NEW METHOD: PUBLIC(+) ABSTRACT(+) io.opentelemetry.sdk.common.export.RetryPolicy$RetryPolicyBuilder setRetryExceptionPredicate(java.util.function.Predicate<java.io.IOException>)

‎exporters/otlp/testing-internal/src/main/java/io/opentelemetry/exporter/otlp/testing/internal/AbstractGrpcTelemetryExporterTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -1050,7 +1050,7 @@ void stringRepresentation() throws IOException, CertificateEncodingException {
10501050
+ ", "
10511051
+ "compressorEncoding=gzip, "
10521052
+ "headers=Headers\\{.*foo=OBFUSCATED.*\\}, "
1053-
+ "retryPolicy=RetryPolicy\\{maxAttempts=2, initialBackoff=PT0\\.05S, maxBackoff=PT3S, backoffMultiplier=1\\.3\\}"
1053+
+ "retryPolicy=RetryPolicy\\{maxAttempts=2, initialBackoff=PT0\\.05S, maxBackoff=PT3S, backoffMultiplier=1\\.3, retryExceptionPredicate=null\\}"
10541054
+ ".*" // Maybe additional grpcChannel field, signal specific fields
10551055
+ "\\}");
10561056
} finally {

‎exporters/otlp/testing-internal/src/main/java/io/opentelemetry/exporter/otlp/testing/internal/AbstractHttpTelemetryExporterTest.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -935,7 +935,7 @@ void stringRepresentation() throws IOException, CertificateEncodingException {
935935
+ ", "
936936
+ "exportAsJson=false, "
937937
+ "headers=Headers\\{.*foo=OBFUSCATED.*\\}, "
938-
+ "retryPolicy=RetryPolicy\\{maxAttempts=2, initialBackoff=PT0\\.05S, maxBackoff=PT3S, backoffMultiplier=1\\.3\\}"
938+
+ "retryPolicy=RetryPolicy\\{maxAttempts=2, initialBackoff=PT0\\.05S, maxBackoff=PT3S, backoffMultiplier=1\\.3, retryExceptionPredicate=null\\}"
939939
+ ".*" // Maybe additional signal specific fields
940940
+ "\\}");
941941
} finally {

‎exporters/sender/jdk/src/main/java/io/opentelemetry/exporter/sender/jdk/internal/JdkHttpSender.java

+8-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.time.Duration;
2727
import java.util.List;
2828
import java.util.Map;
29+
import java.util.Optional;
2930
import java.util.Set;
3031
import java.util.StringJoiner;
3132
import java.util.concurrent.CompletableFuture;
@@ -35,6 +36,7 @@
3536
import java.util.concurrent.ThreadLocalRandom;
3637
import java.util.concurrent.TimeUnit;
3738
import java.util.function.Consumer;
39+
import java.util.function.Predicate;
3840
import java.util.function.Supplier;
3941
import java.util.logging.Level;
4042
import java.util.logging.Logger;
@@ -68,6 +70,7 @@ public final class JdkHttpSender implements HttpSender {
6870
private final long timeoutNanos;
6971
private final Supplier<Map<String, List<String>>> headerSupplier;
7072
@Nullable private final RetryPolicy retryPolicy;
73+
private final Predicate<IOException> retryExceptionPredicate;
7174

7275
// Visible for testing
7376
JdkHttpSender(
@@ -91,6 +94,10 @@ public final class JdkHttpSender implements HttpSender {
9194
this.timeoutNanos = timeoutNanos;
9295
this.headerSupplier = headerSupplier;
9396
this.retryPolicy = retryPolicy;
97+
this.retryExceptionPredicate =
98+
Optional.ofNullable(retryPolicy)
99+
.map(RetryPolicy::getRetryExceptionPredicate)
100+
.orElse(JdkHttpSender::isRetryableException);
94101
}
95102

96103
JdkHttpSender(
@@ -235,7 +242,7 @@ HttpResponse<byte[]> sendInternal(Marshaler marshaler) throws IOException {
235242
}
236243
}
237244
if (exception != null) {
238-
boolean retryable = isRetryableException(exception);
245+
boolean retryable = retryExceptionPredicate.test(exception);
239246
if (logger.isLoggable(Level.FINER)) {
240247
logger.log(
241248
Level.FINER,

‎exporters/sender/okhttp/src/main/java/io/opentelemetry/exporter/sender/okhttp/internal/RetryInterceptor.java

+13-5
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import java.util.concurrent.ThreadLocalRandom;
1717
import java.util.concurrent.TimeUnit;
1818
import java.util.function.Function;
19+
import java.util.function.Predicate;
1920
import java.util.logging.Level;
2021
import java.util.logging.Logger;
2122
import okhttp3.Interceptor;
@@ -33,7 +34,7 @@ public final class RetryInterceptor implements Interceptor {
3334

3435
private final RetryPolicy retryPolicy;
3536
private final Function<Response, Boolean> isRetryable;
36-
private final Function<IOException, Boolean> isRetryableException;
37+
private final Predicate<IOException> retryExceptionPredicate;
3738
private final Sleeper sleeper;
3839
private final BoundedLongGenerator randomLong;
3940

@@ -42,7 +43,9 @@ public RetryInterceptor(RetryPolicy retryPolicy, Function<Response, Boolean> isR
4243
this(
4344
retryPolicy,
4445
isRetryable,
45-
RetryInterceptor::isRetryableException,
46+
retryPolicy.getRetryExceptionPredicate() == null
47+
? RetryInterceptor::isRetryableException
48+
: retryPolicy.getRetryExceptionPredicate(),
4649
TimeUnit.NANOSECONDS::sleep,
4750
bound -> ThreadLocalRandom.current().nextLong(bound));
4851
}
@@ -51,12 +54,12 @@ public RetryInterceptor(RetryPolicy retryPolicy, Function<Response, Boolean> isR
5154
RetryInterceptor(
5255
RetryPolicy retryPolicy,
5356
Function<Response, Boolean> isRetryable,
54-
Function<IOException, Boolean> isRetryableException,
57+
Predicate<IOException> retryExceptionPredicate,
5558
Sleeper sleeper,
5659
BoundedLongGenerator randomLong) {
5760
this.retryPolicy = retryPolicy;
5861
this.isRetryable = isRetryable;
59-
this.isRetryableException = isRetryableException;
62+
this.retryExceptionPredicate = retryExceptionPredicate;
6063
this.sleeper = sleeper;
6164
this.randomLong = randomLong;
6265
}
@@ -109,7 +112,7 @@ public Response intercept(Chain chain) throws IOException {
109112
}
110113
}
111114
if (exception != null) {
112-
boolean retryable = Boolean.TRUE.equals(isRetryableException.apply(exception));
115+
boolean retryable = retryExceptionPredicate.test(exception);
113116
if (logger.isLoggable(Level.FINER)) {
114117
logger.log(
115118
Level.FINER,
@@ -144,6 +147,11 @@ private static String responseStringRepresentation(Response response) {
144147
return joiner.toString();
145148
}
146149

150+
// Visible for testing
151+
boolean shouldRetryOnException(IOException e) {
152+
return retryExceptionPredicate.test(e);
153+
}
154+
147155
// Visible for testing
148156
static boolean isRetryableException(IOException e) {
149157
if (e instanceof SocketTimeoutException) {

‎exporters/sender/okhttp/src/test/java/io/opentelemetry/exporter/sender/okhttp/internal/RetryInterceptorTest.java

+69-31
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,14 @@
2424
import io.opentelemetry.sdk.common.export.RetryPolicy;
2525
import java.io.IOException;
2626
import java.net.ConnectException;
27+
import java.net.HttpRetryException;
2728
import java.net.ServerSocket;
2829
import java.net.SocketTimeoutException;
2930
import java.time.Duration;
3031
import java.util.concurrent.TimeUnit;
31-
import java.util.function.Function;
32+
import java.util.function.Predicate;
33+
import java.util.logging.Level;
34+
import java.util.logging.Logger;
3235
import okhttp3.OkHttpClient;
3336
import okhttp3.Request;
3437
import okhttp3.Response;
@@ -48,34 +51,38 @@ class RetryInterceptorTest {
4851

4952
@Mock private RetryInterceptor.Sleeper sleeper;
5053
@Mock private RetryInterceptor.BoundedLongGenerator random;
51-
private Function<IOException, Boolean> isRetryableException;
54+
private Predicate<IOException> retryExceptionPredicate;
5255

5356
private RetryInterceptor retrier;
5457
private OkHttpClient client;
5558

5659
@BeforeEach
5760
void setUp() {
58-
// Note: cannot replace this with lambda or method reference because we need to spy on it
59-
isRetryableException =
61+
Logger logger = java.util.logging.Logger.getLogger(RetryInterceptor.class.getName());
62+
logger.setLevel(Level.FINER);
63+
retryExceptionPredicate =
6064
spy(
61-
new Function<IOException, Boolean>() {
65+
new Predicate<IOException>() {
6266
@Override
63-
public Boolean apply(IOException exception) {
64-
return RetryInterceptor.isRetryableException(exception);
67+
public boolean test(IOException e) {
68+
return RetryInterceptor.isRetryableException(e)
69+
|| (e instanceof HttpRetryException
70+
&& e.getMessage().contains("timeout retry"));
6571
}
6672
});
73+
74+
RetryPolicy retryPolicy =
75+
RetryPolicy.builder()
76+
.setBackoffMultiplier(1.6)
77+
.setInitialBackoff(Duration.ofSeconds(1))
78+
.setMaxBackoff(Duration.ofSeconds(2))
79+
.setMaxAttempts(5)
80+
.setRetryExceptionPredicate(retryExceptionPredicate)
81+
.build();
82+
6783
retrier =
6884
new RetryInterceptor(
69-
RetryPolicy.builder()
70-
.setBackoffMultiplier(1.6)
71-
.setInitialBackoff(Duration.ofSeconds(1))
72-
.setMaxBackoff(Duration.ofSeconds(2))
73-
.setMaxAttempts(5)
74-
.build(),
75-
r -> !r.isSuccessful(),
76-
isRetryableException,
77-
sleeper,
78-
random);
85+
retryPolicy, r -> !r.isSuccessful(), retryExceptionPredicate, sleeper, random);
7986
client = new OkHttpClient.Builder().addInterceptor(retrier).build();
8087
}
8188

@@ -154,7 +161,7 @@ void connectTimeout() throws Exception {
154161
client.newCall(new Request.Builder().url("http://10.255.255.1").build()).execute())
155162
.isInstanceOf(SocketTimeoutException.class);
156163

157-
verify(isRetryableException, times(5)).apply(any());
164+
verify(retryExceptionPredicate, times(5)).test(any());
158165
// Should retry maxAttempts, and sleep maxAttempts - 1 times
159166
verify(sleeper, times(4)).sleep(anyLong());
160167
}
@@ -174,7 +181,7 @@ void connectException() throws Exception {
174181
.execute())
175182
.isInstanceOfAny(ConnectException.class, SocketTimeoutException.class);
176183

177-
verify(isRetryableException, times(5)).apply(any());
184+
verify(retryExceptionPredicate, times(5)).test(any());
178185
// Should retry maxAttempts, and sleep maxAttempts - 1 times
179186
verify(sleeper, times(4)).sleep(anyLong());
180187
}
@@ -190,16 +197,16 @@ private static int freePort() {
190197
@Test
191198
void nonRetryableException() throws InterruptedException {
192199
client = connectTimeoutClient();
193-
// Override isRetryableException so that no exception is retryable
194-
when(isRetryableException.apply(any())).thenReturn(false);
200+
// Override retryPredicate so that no exception is retryable
201+
when(retryExceptionPredicate.test(any())).thenReturn(false);
195202

196203
// Connecting to a non-routable IP address to trigger connection timeout
197204
assertThatThrownBy(
198205
() ->
199206
client.newCall(new Request.Builder().url("http://10.255.255.1").build()).execute())
200207
.isInstanceOf(SocketTimeoutException.class);
201208

202-
verify(isRetryableException, times(1)).apply(any());
209+
verify(retryExceptionPredicate, times(1)).test(any());
203210
verify(sleeper, never()).sleep(anyLong());
204211
}
205212

@@ -214,20 +221,51 @@ private OkHttpClient connectTimeoutClient() {
214221
void isRetryableException() {
215222
// Should retry on connection timeouts, where error message is "Connect timed out" or "connect
216223
// timed out"
217-
assertThat(
218-
RetryInterceptor.isRetryableException(new SocketTimeoutException("Connect timed out")))
224+
assertThat(retrier.shouldRetryOnException(new SocketTimeoutException("Connect timed out")))
219225
.isTrue();
220-
assertThat(
221-
RetryInterceptor.isRetryableException(new SocketTimeoutException("connect timed out")))
226+
assertThat(retrier.shouldRetryOnException(new SocketTimeoutException("connect timed out")))
222227
.isTrue();
223228
// Shouldn't retry on read timeouts, where error message is "Read timed out"
224-
assertThat(RetryInterceptor.isRetryableException(new SocketTimeoutException("Read timed out")))
229+
assertThat(retrier.shouldRetryOnException(new SocketTimeoutException("Read timed out")))
225230
.isFalse();
226-
// Shouldn't retry on write timeouts, where error message is "timeout", or other IOException
227-
assertThat(RetryInterceptor.isRetryableException(new SocketTimeoutException("timeout")))
231+
// Shouldn't retry on write timeouts or other IOException
232+
assertThat(retrier.shouldRetryOnException(new SocketTimeoutException("timeout"))).isFalse();
233+
assertThat(retrier.shouldRetryOnException(new SocketTimeoutException())).isTrue();
234+
assertThat(retrier.shouldRetryOnException(new IOException("error"))).isFalse();
235+
236+
// Testing configured predicate
237+
assertThat(retrier.shouldRetryOnException(new HttpRetryException("error", 400))).isFalse();
238+
assertThat(retrier.shouldRetryOnException(new HttpRetryException("timeout retry", 400)))
239+
.isTrue();
240+
}
241+
242+
@Test
243+
void isRetryableExceptionDefaultBehaviour() {
244+
RetryInterceptor retryInterceptor =
245+
new RetryInterceptor(RetryPolicy.getDefault(), OkHttpHttpSender::isRetryable);
246+
assertThat(
247+
retryInterceptor.shouldRetryOnException(
248+
new SocketTimeoutException("Connect timed out")))
249+
.isTrue();
250+
assertThat(retryInterceptor.shouldRetryOnException(new IOException("Connect timed out")))
251+
.isFalse();
252+
}
253+
254+
@Test
255+
void isRetryableExceptionCustomRetryPredicate() {
256+
RetryInterceptor retryInterceptor =
257+
new RetryInterceptor(
258+
RetryPolicy.builder()
259+
.setRetryExceptionPredicate((IOException e) -> e.getMessage().equals("retry"))
260+
.build(),
261+
OkHttpHttpSender::isRetryable);
262+
263+
assertThat(retryInterceptor.shouldRetryOnException(new IOException("some message"))).isFalse();
264+
assertThat(retryInterceptor.shouldRetryOnException(new IOException("retry"))).isTrue();
265+
assertThat(
266+
retryInterceptor.shouldRetryOnException(
267+
new SocketTimeoutException("Connect timed out")))
228268
.isFalse();
229-
assertThat(RetryInterceptor.isRetryableException(new SocketTimeoutException())).isTrue();
230-
assertThat(RetryInterceptor.isRetryableException(new IOException("error"))).isFalse();
231269
}
232270

233271
private Response sendRequest() throws IOException {

‎sdk/common/src/main/java/io/opentelemetry/sdk/common/export/RetryPolicy.java

+14
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,10 @@
88
import static io.opentelemetry.api.internal.Utils.checkArgument;
99

1010
import com.google.auto.value.AutoValue;
11+
import java.io.IOException;
1112
import java.time.Duration;
13+
import java.util.function.Predicate;
14+
import javax.annotation.Nullable;
1215

1316
/**
1417
* Configuration for exporter exponential retry policy.
@@ -66,6 +69,13 @@ public static RetryPolicyBuilder builder() {
6669
/** Returns the backoff multiplier. */
6770
public abstract double getBackoffMultiplier();
6871

72+
/**
73+
* Returns the predicate used to determine if thrown exception is retryableor {@code null} if no
74+
* predicate was set.
75+
*/
76+
@Nullable
77+
public abstract Predicate<IOException> getRetryExceptionPredicate();
78+
6979
/** Builder for {@link RetryPolicy}. */
7080
@AutoValue.Builder
7181
public abstract static class RetryPolicyBuilder {
@@ -96,6 +106,10 @@ public abstract static class RetryPolicyBuilder {
96106
*/
97107
public abstract RetryPolicyBuilder setBackoffMultiplier(double backoffMultiplier);
98108

109+
/** Set the predicate to determine if retry should happen based on exception. */
110+
public abstract RetryPolicyBuilder setRetryExceptionPredicate(
111+
Predicate<IOException> retryExceptionPredicate);
112+
99113
abstract RetryPolicy autoBuild();
100114

101115
/** Build and return a {@link RetryPolicy} with the values of this builder. */

‎sdk/common/src/test/java/io/opentelemetry/sdk/common/RetryPolicyTest.java

+1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ void build() {
4040
assertThat(retryPolicy.getInitialBackoff()).isEqualTo(Duration.ofMillis(2));
4141
assertThat(retryPolicy.getMaxBackoff()).isEqualTo(Duration.ofSeconds(1));
4242
assertThat(retryPolicy.getBackoffMultiplier()).isEqualTo(1.1);
43+
assertThat(retryPolicy.getRetryExceptionPredicate()).isEqualTo(null);
4344
}
4445

4546
@Test

0 commit comments

Comments
 (0)