From 1528079189d4658e5cc4224751709644237878ad Mon Sep 17 00:00:00 2001 From: box-sdk-build Date: Wed, 18 Nov 2026 02:59:34 -0800 Subject: [PATCH] fix: Prevent Java retries from resending consumed upload streams (box/box-codegen#1009) --- .codegen.json | 2 +- docs/sdkgen/configuration.md | 67 ++++ .../BoxNetworkClientUploadRetryTest.java | 296 ++++++++++++++++++ .../InputStreamRequestBodyTest.java | 102 ++++++ .../test/BoxNetworkClientRetryTest.java | 86 ++++- .../box/sdkgen/box/errors/RequestInfo.java | 14 + .../boxnetworkclient/BoxNetworkClient.java | 98 ++++-- .../DefaultNetworkClient.java | 72 +++-- .../InputStreamRequestBody.java | 86 +++++ 9 files changed, 764 insertions(+), 59 deletions(-) create mode 100644 src/intTest/java/com/box/sdkgen/networking/boxnetworkclient/BoxNetworkClientUploadRetryTest.java create mode 100644 src/intTest/java/com/box/sdkgen/networking/inputstreamrequestbody/InputStreamRequestBodyTest.java create mode 100644 src/main/java/com/box/sdkgen/networking/inputstreamrequestbody/InputStreamRequestBody.java diff --git a/.codegen.json b/.codegen.json index 06dd81ce5..a8581427c 100644 --- a/.codegen.json +++ b/.codegen.json @@ -1 +1 @@ -{ "engineHash": "6772894", "specHash": "7f8494a", "version": "5.20.0" } +{ "engineHash": "dabe8ed", "specHash": "7f8494a", "version": "5.20.0" } diff --git a/docs/sdkgen/configuration.md b/docs/sdkgen/configuration.md index a07fc503f..d4179d1bc 100644 --- a/docs/sdkgen/configuration.md +++ b/docs/sdkgen/configuration.md @@ -11,6 +11,9 @@ - [Example Delays (with default settings)](#example-delays-with-default-settings) - [Retry-After Header](#retry-after-header) - [Network Exception Handling](#network-exception-handling) + - [Retrying Uploads](#retrying-uploads) + - [Streams That Can Be Retried](#streams-that-can-be-retried) + - [Handling Errors for Streams That Cannot Be Retried](#handling-errors-for-streams-that-cannot-be-retried) - [Customizing Retry Parameters](#customizing-retry-parameters) - [Custom Retry Strategy](#custom-retry-strategy) - [Timeouts](#timeouts) @@ -129,6 +132,70 @@ This means: - Network exception retries are tracked independently from HTTP error retries, each with their own counter and backoff progression. - A request can fail up to `maxRetriesOnException` times due to network exceptions, but each exception retry also increments the overall attempt counter, so the total number of retries across both exception and HTTP error types is bounded by `maxAttempts`. +### Retrying Uploads + +To retry a request that uploads content, the SDK has to send the content again. The SDK does not copy the content into memory to do that. Instead, it rewinds the `InputStream` you passed to the position it had when the upload started. Whether the request can be retried therefore depends on the type of the stream. + +| Stream type | Retried automatically | How the content is sent again | +| ------------------------------------------------------------------------------------------------------- | --------------------- | -------------------------------------------------------------------------- | +| `FileInputStream` | Yes | The file position is moved back to where it was when the upload started. | +| `ByteArrayInputStream` | Yes | The stream is reset to where it was when the upload started. | +| Any other stream, for example from `Files.newInputStream`, `BufferedInputStream`, or a network response | No | The content is sent once. If that attempt fails, the SDK throws the error. | + +If a request with a stream that cannot be rewound fails with a retryable error, such as `429 Too Many Requests` or a `5xx` status, the SDK does not retry it. It throws the original `BoxAPIError` instead, which keeps the response status and headers, including `Retry-After`. For a `401` response, the SDK still refreshes the access token, so your next request uses a valid token. + +Uploads to an upload session, such as `uploadBigFile` and `uploadFilePart`, are always retried. Each part is read into memory before it is sent, so it can be sent again regardless of the stream type. + +The SDK closes the stream you pass after the request finishes, including all retries, whether it succeeds or fails. Closing it yourself as well, for example with try-with-resources, is safe. + +Interceptors run before every attempt, but a request body that reads a stream is built on the first attempt and reused on retries. Changes an interceptor makes to the upload stream on a retry are therefore not applied. + +#### Streams That Can Be Retried + +To let the SDK retry uploads automatically, pass a `FileInputStream` when uploading from a file: + +```java +try (FileInputStream fileStream = new FileInputStream(new File("report.pdf"))) { + client.getUploads().uploadFile( + new UploadFileRequestBody( + new UploadFileRequestBodyAttributesField( + "report.pdf", new UploadFileRequestBodyAttributesParentField("0")), + fileStream)); +} +``` + +If you open files with `Files.newInputStream(path)`, use `new FileInputStream(path.toFile())` instead. The stream returned by `Files.newInputStream` cannot be rewound. + +For content that is already in memory, wrap it in a `ByteArrayInputStream`. Do not copy large files into memory only to make them retryable; use a `FileInputStream` instead. + +#### Handling Errors for Streams That Cannot Be Retried + +If you upload from a stream that cannot be rewound, handle retryable errors in your code and send a new stream on each attempt: + +```java +BoxAPIError lastError = null; +for (int attempt = 0; attempt < 3; attempt++) { + try (InputStream stream = openContentStream()) { + return client.getUploads().uploadFile( + new UploadFileRequestBody( + new UploadFileRequestBodyAttributesField( + "report.pdf", new UploadFileRequestBodyAttributesParentField("0")), + stream)); + } catch (BoxAPIError error) { + int status = error.getResponseInfo().getStatusCode(); + if (status != 429 && status < 500) { + throw error; + } + lastError = error; + String retryAfter = error.getResponseInfo().getHeaders().get("Retry-After"); + Thread.sleep(retryAfter != null ? (long) (Double.parseDouble(retryAfter) * 1000) : 1000L); + } +} +throw lastError; +``` + +`openContentStream()` stands for your own code that opens the content from the beginning, for example a new request to the system the content comes from. + ### Customizing Retry Parameters You can customize all retry parameters by initializing `BoxRetryStrategy` with the desired values and passing it to `NetworkSession`: diff --git a/src/intTest/java/com/box/sdkgen/networking/boxnetworkclient/BoxNetworkClientUploadRetryTest.java b/src/intTest/java/com/box/sdkgen/networking/boxnetworkclient/BoxNetworkClientUploadRetryTest.java new file mode 100644 index 000000000..eda0cf46f --- /dev/null +++ b/src/intTest/java/com/box/sdkgen/networking/boxnetworkclient/BoxNetworkClientUploadRetryTest.java @@ -0,0 +1,296 @@ +package com.box.sdkgen.networking.boxnetworkclient; + +import static com.box.sdkgen.serialization.json.JsonManager.jsonToSerializedData; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.box.sdkgen.box.errors.BoxAPIError; +import com.box.sdkgen.networking.fetchoptions.FetchOptions; +import com.box.sdkgen.networking.fetchoptions.MultipartItem; +import com.box.sdkgen.networking.fetchresponse.FetchResponse; +import com.box.sdkgen.networking.interceptors.Interceptor; +import com.box.sdkgen.networking.network.NetworkSession; +import com.box.sdkgen.networking.retries.BoxRetryStrategy; +import com.box.sdkgen.networking.retries.RetryStrategy; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Queue; +import java.util.Random; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import okhttp3.ConnectionSpec; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +@Timeout(value = 30, threadMode = Timeout.ThreadMode.SEPARATE_THREAD) +public class BoxNetworkClientUploadRetryTest { + + private static final byte[] FILE_CONTENT = randomBytes(256 * 1024); + private static final int SUCCESS_STATUS = 201; + + private final List receivedBodies = Collections.synchronizedList(new ArrayList<>()); + private final List receivedCustomHeaders = + Collections.synchronizedList(new ArrayList<>()); + private final Queue failureStatuses = new ConcurrentLinkedQueue<>(); + private final BoxNetworkClient client = + new BoxNetworkClient( + BoxNetworkClient.getDefaultOkHttpClientBuilder() + .connectionSpecs(Collections.singletonList(ConnectionSpec.CLEARTEXT)) + .build()); + private final ExecutorService serverExecutor = Executors.newCachedThreadPool(); + private HttpServer server; + private String url; + + @BeforeEach + public void startServer() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", this::respond); + server.setExecutor(serverExecutor); + server.start(); + url = "http://127.0.0.1:" + server.getAddress().getPort() + "/files/content"; + } + + @AfterEach + public void stopServer() { + server.stop(0); + serverExecutor.shutdownNow(); + } + + @Test + public void testMultipartUploadWithByteArrayStreamIsRetriedWithFullContent() { + failureStatuses.add(429); + + FetchResponse response = client.fetch(multipartUpload(new ByteArrayInputStream(FILE_CONTENT))); + + assertEquals(SUCCESS_STATUS, response.getStatus()); + assertEquals(2, receivedBodies.size()); + assertContainsFileContent(receivedBodies.get(0)); + assertArrayEquals(receivedBodies.get(0), receivedBodies.get(1)); + } + + @Test + public void testMultipartUploadWithFileStreamIsRetriedWithFullContent() throws IOException { + failureStatuses.add(503); + + try (FileInputStream stream = new FileInputStream(createTempFile())) { + FetchResponse response = client.fetch(multipartUpload(stream)); + + assertEquals(SUCCESS_STATUS, response.getStatus()); + assertFalse(stream.getChannel().isOpen()); + } + assertEquals(2, receivedBodies.size()); + assertContainsFileContent(receivedBodies.get(0)); + assertArrayEquals(receivedBodies.get(0), receivedBodies.get(1)); + } + + @Test + public void testMultipartUploadWithInterceptorIsRetriedWithFullContent() { + failureStatuses.add(429); + Interceptor headerInterceptor = + new Interceptor() { + @Override + public FetchOptions beforeRequest(FetchOptions options) { + return new FetchOptions.Builder(options.getUrl(), options.getMethod()) + .headers(Collections.singletonMap("X-Custom-Header", "value")) + .contentType(options.getContentType()) + .multipartData(options.getMultipartData()) + .build(); + } + + @Override + public FetchResponse afterRequest(FetchResponse response) { + return response; + } + }; + NetworkSession networkSession = + networkSessionWithoutRetryDelay().withInterceptors(Arrays.asList(headerInterceptor)); + + FetchResponse response = + client.fetch(multipartUpload(new ByteArrayInputStream(FILE_CONTENT), networkSession)); + + assertEquals(SUCCESS_STATUS, response.getStatus()); + assertEquals(Arrays.asList("value", "value"), receivedCustomHeaders); + assertEquals(2, receivedBodies.size()); + assertContainsFileContent(receivedBodies.get(0)); + assertArrayEquals(receivedBodies.get(0), receivedBodies.get(1)); + } + + @Test + public void testMultipartUploadWithStreamThatCannotBeRewoundIsNotRetried() throws IOException { + failureStatuses.add(429); + InputStream stream = new BufferedInputStream(Files.newInputStream(createTempFile().toPath())); + + BoxAPIError error = + assertThrows(BoxAPIError.class, () -> client.fetch(multipartUpload(stream))); + + assertEquals(429, error.getResponseInfo().getStatusCode()); + assertEquals("0", error.getResponseInfo().getHeaders().get("Retry-After")); + assertEquals(1, receivedBodies.size()); + assertThrows(IOException.class, stream::available); + } + + @Test + public void testRetryStrategyIsCalledWhenUploadStreamCannotBeRewound() { + failureStatuses.add(429); + AtomicInteger retryStrategyCalls = new AtomicInteger(); + RetryStrategy retryStrategy = + new RetryStrategy() { + @Override + public boolean shouldRetry( + FetchOptions fetchOptions, FetchResponse fetchResponse, int attemptNumber) { + retryStrategyCalls.incrementAndGet(); + return true; + } + + @Override + public double retryAfter( + FetchOptions fetchOptions, FetchResponse fetchResponse, int attemptNumber) { + return 0; + } + }; + NetworkSession networkSession = + new NetworkSession.Builder().retryStrategy(retryStrategy).build(); + InputStream stream = new BufferedInputStream(new ByteArrayInputStream(FILE_CONTENT)); + + assertThrows(BoxAPIError.class, () -> client.fetch(multipartUpload(stream, networkSession))); + + assertEquals(1, retryStrategyCalls.get()); + assertEquals(1, receivedBodies.size()); + } + + @Test + public void testUploadPartWithStreamThatCannotBeRewoundIsRetriedWithFullContent() { + failureStatuses.add(429); + InputStream stream = new BufferedInputStream(new ByteArrayInputStream(FILE_CONTENT)); + FetchOptions options = + new FetchOptions.Builder(url, "PUT") + .contentType("application/octet-stream") + .fileStream(stream) + .networkSession(networkSessionWithoutRetryDelay()) + .build(); + + FetchResponse response = client.fetch(options); + + assertEquals(SUCCESS_STATUS, response.getStatus()); + assertEquals(2, receivedBodies.size()); + assertArrayEquals(FILE_CONTENT, receivedBodies.get(0)); + assertArrayEquals(FILE_CONTENT, receivedBodies.get(1)); + } + + @Test + public void testUploadErrorDoesNotIncludeFileContent() throws IOException { + failureStatuses.add(409); + + try (FileInputStream stream = new FileInputStream(createTempFile())) { + BoxAPIError error = + assertThrows(BoxAPIError.class, () -> client.fetch(multipartUpload(stream))); + + assertEquals(409, error.getResponseInfo().getStatusCode()); + assertTrue(error.toString().contains("body='null'")); + } + assertEquals(1, receivedBodies.size()); + } + + private FetchOptions multipartUpload(InputStream fileStream) { + return multipartUpload(fileStream, networkSessionWithoutRetryDelay()); + } + + private FetchOptions multipartUpload(InputStream fileStream, NetworkSession networkSession) { + List parts = + Arrays.asList( + new MultipartItem.Builder("attributes") + .data(jsonToSerializedData("{\"name\":\"file.bin\",\"parent\":{\"id\":\"0\"}}")) + .build(), + new MultipartItem.Builder("file").fileStream(fileStream).fileName("file.bin").build()); + return new FetchOptions.Builder(url, "POST") + .contentType("multipart/form-data") + .multipartData(parts) + .networkSession(networkSession) + .build(); + } + + private static NetworkSession networkSessionWithoutRetryDelay() { + return new NetworkSession.Builder() + .retryStrategy(new BoxRetryStrategy.Builder().retryBaseInterval(0).build()) + .build(); + } + + private void respond(HttpExchange exchange) throws IOException { + receivedBodies.add(readAll(exchange.getRequestBody())); + receivedCustomHeaders.add(exchange.getRequestHeaders().getFirst("X-Custom-Header")); + Integer failureStatus = failureStatuses.poll(); + byte[] responseBody = "{}".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + if (failureStatus != null && failureStatus == 429) { + exchange.getResponseHeaders().add("Retry-After", "0"); + } + exchange.sendResponseHeaders( + failureStatus != null ? failureStatus : SUCCESS_STATUS, responseBody.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(responseBody); + } + } + + private static void assertContainsFileContent(byte[] body) { + assertTrue( + indexOf(body, FILE_CONTENT) >= 0, "Request body does not contain the full file content"); + } + + private static int indexOf(byte[] array, byte[] target) { + outer: + for (int i = 0; i <= array.length - target.length; i++) { + for (int j = 0; j < target.length; j++) { + if (array[i + j] != target[j]) { + continue outer; + } + } + return i; + } + return -1; + } + + private static byte[] readAll(InputStream input) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + output.write(buffer, 0, read); + } + return output.toByteArray(); + } + + private static File createTempFile() throws IOException { + File file = Files.createTempFile("upload-retry", ".bin").toFile(); + file.deleteOnExit(); + Files.write(file.toPath(), FILE_CONTENT); + return file; + } + + private static byte[] randomBytes(int size) { + byte[] bytes = new byte[size]; + new Random(0).nextBytes(bytes); + return bytes; + } +} diff --git a/src/intTest/java/com/box/sdkgen/networking/inputstreamrequestbody/InputStreamRequestBodyTest.java b/src/intTest/java/com/box/sdkgen/networking/inputstreamrequestbody/InputStreamRequestBodyTest.java new file mode 100644 index 000000000..b031a6adc --- /dev/null +++ b/src/intTest/java/com/box/sdkgen/networking/inputstreamrequestbody/InputStreamRequestBodyTest.java @@ -0,0 +1,102 @@ +package com.box.sdkgen.networking.inputstreamrequestbody; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import okhttp3.MediaType; +import okio.Buffer; +import org.junit.jupiter.api.Test; + +public class InputStreamRequestBodyTest { + + private static final MediaType CONTENT_TYPE = MediaType.parse("application/octet-stream"); + private static final byte[] CONTENT = "0123456789abcdef".getBytes(StandardCharsets.UTF_8); + private static final int INITIAL_POSITION = 4; + + @Test + public void testByteArrayStreamIsSentAgainFromItsInitialPosition() throws IOException { + ByteArrayInputStream stream = new ByteArrayInputStream(CONTENT); + stream.skip(INITIAL_POSITION); + InputStreamRequestBody body = new InputStreamRequestBody(stream, CONTENT_TYPE); + byte[] expected = Arrays.copyOfRange(CONTENT, INITIAL_POSITION, CONTENT.length); + + assertArrayEquals(expected, write(body)); + assertTrue(body.canBeResent()); + assertArrayEquals(expected, write(body)); + } + + @Test + public void testFileStreamIsSentAgainFromItsInitialPosition() throws IOException { + File file = createTempFile(); + try (FileInputStream stream = new FileInputStream(file)) { + stream.skip(INITIAL_POSITION); + InputStreamRequestBody body = new InputStreamRequestBody(stream, CONTENT_TYPE); + byte[] expected = Arrays.copyOfRange(CONTENT, INITIAL_POSITION, CONTENT.length); + + assertArrayEquals(expected, write(body)); + assertTrue(body.canBeResent()); + assertArrayEquals(expected, write(body)); + } + } + + @Test + public void testStreamThatCannotBeRewoundIsSentOnlyOnce() throws IOException { + InputStream stream = new BufferedInputStream(new ByteArrayInputStream(CONTENT)); + InputStreamRequestBody body = new InputStreamRequestBody(stream, CONTENT_TYPE); + + assertTrue(body.canBeResent()); + assertArrayEquals(CONTENT, write(body)); + assertFalse(body.canBeResent()); + assertThrows(IOException.class, () -> write(body)); + } + + @Test + public void testStreamIsNotClosedAfterBeingSent() throws IOException { + CloseTrackingInputStream stream = + new CloseTrackingInputStream(new ByteArrayInputStream(CONTENT)); + InputStreamRequestBody body = new InputStreamRequestBody(stream, CONTENT_TYPE); + + write(body); + + assertFalse(stream.closed); + } + + private static byte[] write(InputStreamRequestBody body) throws IOException { + Buffer buffer = new Buffer(); + body.writeTo(buffer); + return buffer.readByteArray(); + } + + private static File createTempFile() throws IOException { + File file = Files.createTempFile("input-stream-request-body", ".bin").toFile(); + file.deleteOnExit(); + Files.write(file.toPath(), CONTENT); + return file; + } + + private static class CloseTrackingInputStream extends FilterInputStream { + private boolean closed; + + CloseTrackingInputStream(InputStream in) { + super(in); + } + + @Override + public void close() throws IOException { + closed = true; + super.close(); + } + } +} diff --git a/src/intTest/java/com/box/sdkgen/test/BoxNetworkClientRetryTest.java b/src/intTest/java/com/box/sdkgen/test/BoxNetworkClientRetryTest.java index 13dc52822..f714afe8a 100644 --- a/src/intTest/java/com/box/sdkgen/test/BoxNetworkClientRetryTest.java +++ b/src/intTest/java/com/box/sdkgen/test/BoxNetworkClientRetryTest.java @@ -124,7 +124,7 @@ public void testRetryStrategyReceivesRegeneratedRequestData() throws Exception { } @Test - public void testInterceptorRunsOnEachRetryWithRebuiltData() throws Exception { + public void testInterceptorRunsOnEachRetryBeforeDataIsRebuilt() throws Exception { try (MockWebServer server = new MockWebServer()) { server.enqueue(jsonResponse(500, "{\"error\":\"retry\"}")); server.enqueue(jsonResponse(200, "{\"ok\":true}")); @@ -158,7 +158,74 @@ public FetchResponse afterRequest(FetchResponse response) { .networkSession(networkSession) .build()); - assertEquals(Arrays.asList("D1", "D2"), interceptorData); + assertEquals(Arrays.asList("D1", "D1"), interceptorData); + assertEquals("attempt=D1", takeBody(server)); + assertEquals("attempt=D2", takeBody(server)); + assertEquals(1, dataFactory.calls); + } + } + + @Test + public void testDataFactoryRemovedByInterceptorIsNotUsed() throws Exception { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(jsonResponse(500, "{\"error\":\"retry\"}")); + server.enqueue(jsonResponse(200, "{\"ok\":true}")); + server.start(); + + NetworkSession networkSession = + new NetworkSession() + .withInterceptors( + Arrays.asList( + interceptorReplacingOptions( + options -> + baseOptions( + options.getUrl(), jsonToSerializedData("{\"attempt\":\"I\"}")) + .build()))) + .withRetryStrategy( + retryWhen((options, response, attempt) -> response.getStatus() == 500)); + + CountingDataFactory dataFactory = new CountingDataFactory(); + testNetworkClient() + .fetch( + baseOptions(server.url("/").toString(), jsonToSerializedData("{\"attempt\":\"D1\"}")) + .dataFactory(dataFactory) + .networkSession(networkSession) + .build()); + + assertEquals("attempt=I", takeBody(server)); + assertEquals("attempt=I", takeBody(server)); + assertEquals(0, dataFactory.calls); + } + } + + @Test + public void testDataFactoryAddedByInterceptorIsUsedOnRetry() throws Exception { + try (MockWebServer server = new MockWebServer()) { + server.enqueue(jsonResponse(500, "{\"error\":\"retry\"}")); + server.enqueue(jsonResponse(200, "{\"ok\":true}")); + server.start(); + + CountingDataFactory dataFactory = new CountingDataFactory(); + NetworkSession networkSession = + new NetworkSession() + .withInterceptors( + Arrays.asList( + interceptorReplacingOptions( + options -> + baseOptions(options.getUrl(), options.getData()) + .dataFactory(dataFactory) + .build()))) + .withRetryStrategy( + retryWhen((options, response, attempt) -> response.getStatus() == 500)); + + testNetworkClient() + .fetch( + baseOptions(server.url("/").toString(), jsonToSerializedData("{\"attempt\":\"D1\"}")) + .networkSession(networkSession) + .build()); + + assertEquals("attempt=D1", takeBody(server)); + assertEquals("attempt=D2", takeBody(server)); assertEquals(1, dataFactory.calls); } } @@ -353,6 +420,21 @@ private JwtPayload decodePayload(String assertion) throws Exception { return objectMapper.readValue(payload, JwtPayload.class); } + private Interceptor interceptorReplacingOptions( + java.util.function.UnaryOperator replace) { + return new Interceptor() { + @Override + public FetchOptions beforeRequest(FetchOptions options) { + return replace.apply(options); + } + + @Override + public FetchResponse afterRequest(FetchResponse response) { + return response; + } + }; + } + private interface RetryPredicate { boolean shouldRetry(FetchOptions options, FetchResponse response, int attemptNumber); } diff --git a/src/main/java/com/box/sdkgen/box/errors/RequestInfo.java b/src/main/java/com/box/sdkgen/box/errors/RequestInfo.java index dcdb9edec..963207cc7 100644 --- a/src/main/java/com/box/sdkgen/box/errors/RequestInfo.java +++ b/src/main/java/com/box/sdkgen/box/errors/RequestInfo.java @@ -1,8 +1,10 @@ package com.box.sdkgen.box.errors; import com.box.sdkgen.internal.logging.DataSanitizer; +import com.box.sdkgen.networking.inputstreamrequestbody.InputStreamRequestBody; import java.util.Map; import java.util.stream.Collectors; +import okhttp3.MultipartBody; import okhttp3.Request; import okhttp3.RequestBody; import okio.Buffer; @@ -120,6 +122,10 @@ public static RequestInfo fromRequest(Request request, String contentType) { } private static String getRequestBodyAsString(RequestBody requestBody) { + // Writing a stream body here would read the whole file content into memory. + if (containsStream(requestBody)) { + return null; + } try { if (requestBody != null) { Buffer buffer = new Buffer(); @@ -132,6 +138,14 @@ private static String getRequestBodyAsString(RequestBody requestBody) { return null; } + private static boolean containsStream(RequestBody requestBody) { + if (requestBody instanceof MultipartBody) { + return ((MultipartBody) requestBody) + .parts().stream().anyMatch(part -> containsStream(part.body())); + } + return requestBody instanceof InputStreamRequestBody; + } + String print(DataSanitizer dataSanitizer) { Map sanitizedHeaders = dataSanitizer == null ? headers : dataSanitizer.sanitizeHeaders(headers); diff --git a/src/main/java/com/box/sdkgen/networking/boxnetworkclient/BoxNetworkClient.java b/src/main/java/com/box/sdkgen/networking/boxnetworkclient/BoxNetworkClient.java index dd203170e..7a66f7ecd 100644 --- a/src/main/java/com/box/sdkgen/networking/boxnetworkclient/BoxNetworkClient.java +++ b/src/main/java/com/box/sdkgen/networking/boxnetworkclient/BoxNetworkClient.java @@ -16,12 +16,14 @@ import com.box.sdkgen.networking.fetchoptions.MultipartItem; import com.box.sdkgen.networking.fetchoptions.ResponseFormat; import com.box.sdkgen.networking.fetchresponse.FetchResponse; +import com.box.sdkgen.networking.inputstreamrequestbody.InputStreamRequestBody; import com.box.sdkgen.networking.network.NetworkSession; import com.box.sdkgen.networking.networkclient.NetworkClient; import com.box.sdkgen.networking.proxyconfig.ProxyConfig; import com.box.sdkgen.networking.timeoutconfig.TimeoutConfig; import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; +import java.io.InputStream; import java.net.InetSocketAddress; import java.net.Proxy; import java.net.URI; @@ -43,9 +45,6 @@ import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; -import okio.BufferedSink; -import okio.Okio; -import okio.Source; public class BoxNetworkClient implements NetworkClient { @@ -132,11 +131,20 @@ public BoxNetworkClient withTimeoutConfig(TimeoutConfig config) { } public FetchResponse fetch(FetchOptions options) { + try { + return fetchWithRetries(options); + } finally { + closeFileStreams(options); + } + } + + private FetchResponse fetchWithRetries(FetchOptions options) { NetworkSession networkSession = options.getNetworkSession() == null ? new NetworkSession() : options.getNetworkSession(); boolean authenticationNeeded = false; Request request; + RequestBody body = null; FetchResponse fetchResponse = new FetchResponse.Builder(0, new TreeMap<>()).build(); Exception exceptionThrown = null; @@ -146,21 +154,17 @@ public FetchResponse fetch(FetchOptions options) { boolean shouldRetry = false; while (true) { - FetchOptions attemptOptions = - (attemptNumber > 1 || numberOfRetriesOnException > 0 || options.getData() == null) - && options.getDataFactory() != null - ? withData(options, options.getDataFactory().create()) - : options; - FetchOptions fetchOptions = applyInterceptors(attemptOptions, networkSession); + boolean isFirstAttempt = attemptNumber == 1 && numberOfRetriesOnException == 0; + FetchOptions fetchOptions = applyInterceptors(options, networkSession); if (fetchOptions.getDataFactory() != null - && options.getDataFactory() == null - && (attemptNumber > 1 - || numberOfRetriesOnException > 0 - || fetchOptions.getData() == null)) { + && (!isFirstAttempt || fetchOptions.getData() == null)) { fetchOptions = withData(fetchOptions, fetchOptions.getDataFactory().create()); } - request = prepareRequest(fetchOptions, authenticationNeeded, networkSession); + if (isFirstAttempt || !containsFileStream(fetchOptions)) { + body = prepareRequestBody(fetchOptions); + } + request = prepareRequest(fetchOptions, body, authenticationNeeded, networkSession); Response response = null; String rawResponseBody = null; @@ -222,8 +226,9 @@ public FetchResponse fetch(FetchOptions options) { networkSession .getRetryStrategy() .shouldRetry(fetchOptions, fetchResponse, attemptForRetry); + boolean isRetryable = canBeResent(body); - if (shouldRetry) { + if (shouldRetry && isRetryable) { double retryDelay = networkSession .getRetryStrategy() @@ -301,11 +306,13 @@ private static FetchOptions withData(FetchOptions options, JsonNode data) { } private static Request prepareRequest( - FetchOptions options, boolean reauthenticate, NetworkSession networkSession) { + FetchOptions options, + RequestBody body, + boolean reauthenticate, + NetworkSession networkSession) { Request.Builder requestBuilder = new Request.Builder().url(options.getUrl()); Headers headers = prepareHeaders(options, reauthenticate, networkSession); HttpUrl url = prepareUrl(options); - RequestBody body = prepareRequestBody(options); requestBuilder.headers(headers); requestBuilder.url(url); @@ -438,23 +445,48 @@ private static int getRetryAfterTimeInSeconds(int attemptNumber, String retryAft return (int) (Math.pow(2, attemptNumber) * BASE_TIMEOUT * jitter); } - public static RequestBody createMultipartRequestBody(MultipartItem part) { - return new RequestBody() { - @Override - public MediaType contentType() { - if (part.contentType != null) { - return MediaType.parse(part.contentType); - } - return MediaType.parse("application/octet-stream"); - } + /** + * Returns whether the request body reads a file stream. Such a body is built once and reused on + * retries, because rebuilding it would read the stream from where the previous attempt stopped. + */ + private static boolean containsFileStream(FetchOptions options) { + return options.getFileStream() != null + || (options.getMultipartData() != null + && options.getMultipartData().stream().anyMatch(part -> part.getFileStream() != null)); + } - @Override - public void writeTo(BufferedSink sink) throws IOException { - try (Source source = Okio.source(part.getFileStream())) { - sink.writeAll(source); - } - } - }; + private static void closeFileStreams(FetchOptions options) { + if (options.getFileStream() != null) { + closeQuietly(options.getFileStream()); + } + if (options.getMultipartData() != null) { + options.getMultipartData().stream() + .map(MultipartItem::getFileStream) + .filter(Objects::nonNull) + .forEach(BoxNetworkClient::closeQuietly); + } + } + + /** A failure to close a stream must not replace the result or the error of the request. */ + private static void closeQuietly(InputStream stream) { + try { + stream.close(); + } catch (IOException ignored) { + } + } + + private static boolean canBeResent(RequestBody body) { + if (body instanceof MultipartBody) { + return ((MultipartBody) body).parts().stream().allMatch(part -> canBeResent(part.body())); + } + return !(body instanceof InputStreamRequestBody) + || ((InputStreamRequestBody) body).canBeResent(); + } + + public static RequestBody createMultipartRequestBody(MultipartItem part) { + MediaType contentType = + MediaType.parse(part.contentType != null ? part.contentType : "application/octet-stream"); + return new InputStreamRequestBody(part.getFileStream(), contentType); } public static OkHttpClient.Builder getDefaultOkHttpClientBuilder() { diff --git a/src/main/java/com/box/sdkgen/networking/defaultnetworkclient/DefaultNetworkClient.java b/src/main/java/com/box/sdkgen/networking/defaultnetworkclient/DefaultNetworkClient.java index f8bba1237..a8e4de908 100644 --- a/src/main/java/com/box/sdkgen/networking/defaultnetworkclient/DefaultNetworkClient.java +++ b/src/main/java/com/box/sdkgen/networking/defaultnetworkclient/DefaultNetworkClient.java @@ -11,9 +11,11 @@ import com.box.sdkgen.networking.fetchoptions.MultipartItem; import com.box.sdkgen.networking.fetchoptions.ResponseFormat; import com.box.sdkgen.networking.fetchresponse.FetchResponse; +import com.box.sdkgen.networking.inputstreamrequestbody.InputStreamRequestBody; import com.box.sdkgen.networking.network.NetworkSession; import com.box.sdkgen.networking.networkclient.NetworkClient; import java.io.IOException; +import java.io.InputStream; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -29,9 +31,6 @@ import okhttp3.Request; import okhttp3.RequestBody; import okhttp3.Response; -import okio.BufferedSink; -import okio.Okio; -import okio.Source; public class DefaultNetworkClient implements NetworkClient { @@ -46,6 +45,14 @@ public DefaultNetworkClient() { } public FetchResponse fetch(FetchOptions options) { + try { + return fetchWithRetries(options); + } finally { + closeFileStreams(options); + } + } + + private FetchResponse fetchWithRetries(FetchOptions options) { NetworkSession networkSession = options.getNetworkSession() == null ? new NetworkSession() : options.getNetworkSession(); @@ -58,6 +65,7 @@ public FetchResponse fetch(FetchOptions options) { boolean authenticationNeeded = false; Request request; + RequestBody body = prepareRequestBody(fetchOptions); FetchResponse fetchResponse = null; Exception exceptionThrown = null; @@ -67,7 +75,7 @@ public FetchResponse fetch(FetchOptions options) { boolean shouldRetry = false; while (true) { - request = prepareRequest(fetchOptions, authenticationNeeded, networkSession); + request = prepareRequest(fetchOptions, body, authenticationNeeded, networkSession); Response response = null; try { @@ -123,8 +131,9 @@ public FetchResponse fetch(FetchOptions options) { networkSession .getRetryStrategy() .shouldRetry(fetchOptions, fetchResponse, attemptForRetry); + boolean isRetryable = canBeResent(body); - if (shouldRetry) { + if (shouldRetry && isRetryable) { double retryDelay = networkSession .getRetryStrategy() @@ -167,11 +176,13 @@ public FetchResponse fetch(FetchOptions options) { } private static Request prepareRequest( - FetchOptions options, boolean reauthenticate, NetworkSession networkSession) { + FetchOptions options, + RequestBody body, + boolean reauthenticate, + NetworkSession networkSession) { Request.Builder requestBuilder = new Request.Builder().url(options.getUrl()); Headers headers = prepareHeaders(options, reauthenticate, networkSession); HttpUrl url = prepareUrl(options); - RequestBody body = prepareRequestBody(options); requestBuilder.headers(headers); requestBuilder.url(url); @@ -257,23 +268,38 @@ private Response executeOnClient(Request request) { } } - public static RequestBody createMultipartRequestBody(MultipartItem part) { - return new RequestBody() { - @Override - public MediaType contentType() { - if (part.contentType != null) { - return MediaType.parse(part.contentType); - } - return MediaType.parse("application/octet-stream"); - } + private static void closeFileStreams(FetchOptions options) { + if (options.getFileStream() != null) { + closeQuietly(options.getFileStream()); + } + if (options.getMultipartData() != null) { + options.getMultipartData().stream() + .map(MultipartItem::getFileStream) + .filter(Objects::nonNull) + .forEach(DefaultNetworkClient::closeQuietly); + } + } - @Override - public void writeTo(BufferedSink sink) throws IOException { - try (Source source = Okio.source(part.getFileStream())) { - sink.writeAll(source); - } - } - }; + /** A failure to close a stream must not replace the result or the error of the request. */ + private static void closeQuietly(InputStream stream) { + try { + stream.close(); + } catch (IOException ignored) { + } + } + + private static boolean canBeResent(RequestBody body) { + if (body instanceof MultipartBody) { + return ((MultipartBody) body).parts().stream().allMatch(part -> canBeResent(part.body())); + } + return !(body instanceof InputStreamRequestBody) + || ((InputStreamRequestBody) body).canBeResent(); + } + + public static RequestBody createMultipartRequestBody(MultipartItem part) { + MediaType contentType = + MediaType.parse(part.contentType != null ? part.contentType : "application/octet-stream"); + return new InputStreamRequestBody(part.getFileStream(), contentType); } public static OkHttpClient.Builder getDefaultOkHttpClientBuilder() { diff --git a/src/main/java/com/box/sdkgen/networking/inputstreamrequestbody/InputStreamRequestBody.java b/src/main/java/com/box/sdkgen/networking/inputstreamrequestbody/InputStreamRequestBody.java new file mode 100644 index 000000000..2105bc3a7 --- /dev/null +++ b/src/main/java/com/box/sdkgen/networking/inputstreamrequestbody/InputStreamRequestBody.java @@ -0,0 +1,86 @@ +package com.box.sdkgen.networking.inputstreamrequestbody; + +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import okhttp3.MediaType; +import okhttp3.RequestBody; +import okio.BufferedSink; +import okio.Okio; + +/** + * A request body that streams a caller-provided {@link InputStream} without buffering its content. + * + *

The body does not close the stream; the network client closes it once the request and all its + * retries finish. The body can be sent again on a retry only if the stream can be rewound without + * copying it: a {@link ByteArrayInputStream} via mark/reset, or a {@link FileInputStream} via its + * channel position. Any other stream can be sent only once. + */ +public class InputStreamRequestBody extends RequestBody { + + private static final long NO_FILE_POSITION = -1; + + private final InputStream stream; + private final MediaType contentType; + private final long initialFilePosition; + private boolean written; + + public InputStreamRequestBody(InputStream stream, MediaType contentType) { + this.stream = stream; + this.contentType = contentType; + if (stream instanceof ByteArrayInputStream) { + stream.mark(0); + } + this.initialFilePosition = readFilePosition(stream); + } + + @Override + public MediaType contentType() { + return contentType; + } + + @Override + public void writeTo(BufferedSink sink) throws IOException { + if (written) { + rewind(); + } + written = true; + sink.writeAll(Okio.source(stream)); + } + + /** + * Returns whether this body can be written again, either because it has not been written yet or + * because its stream can be rewound. + */ + public boolean canBeResent() { + return !written || isRewindable(); + } + + private boolean isRewindable() { + return stream instanceof ByteArrayInputStream || initialFilePosition != NO_FILE_POSITION; + } + + private void rewind() throws IOException { + if (stream instanceof ByteArrayInputStream) { + stream.reset(); + return; + } + if (initialFilePosition != NO_FILE_POSITION) { + ((FileInputStream) stream).getChannel().position(initialFilePosition); + return; + } + throw new IOException("Request body stream was already sent and cannot be rewound"); + } + + private static long readFilePosition(InputStream stream) { + if (!(stream instanceof FileInputStream)) { + return NO_FILE_POSITION; + } + try { + return ((FileInputStream) stream).getChannel().position(); + } catch (IOException e) { + return NO_FILE_POSITION; + } + } +}