diff --git a/client/src/main/java/org/asynchttpclient/util/HttpUtils.java b/client/src/main/java/org/asynchttpclient/util/HttpUtils.java index fd7e76db5..81c79d250 100644 --- a/client/src/main/java/org/asynchttpclient/util/HttpUtils.java +++ b/client/src/main/java/org/asynchttpclient/util/HttpUtils.java @@ -193,6 +193,9 @@ public static String cookiePairName(String pair) { return (eq >= 0 ? pair.substring(0, eq) : pair).trim(); } + // Decoded over HTTP/1.1 and HTTP/2 alike, unlike br and zstd: Http2ContentDecompressor decodes only these. + private static final AsciiString[] WILDCARD_CODINGS = {HttpHeaderValues.GZIP, HttpHeaderValues.DEFLATE}; + public static CharSequence filterOutBrotliFromAcceptEncoding(String acceptEncoding) { return filterOutAcceptEncoding(acceptEncoding, HttpHeaderValues.BR); } @@ -204,26 +207,24 @@ public static CharSequence filterOutZstdFromAcceptEncoding(String acceptEncoding /** * Drops every element offering {@code coding}, which may appear anywhere in the list, in any case and with a weight * (RFC 9110 section 12.5.3). Dropping the last element leaves an empty value, which accepts no content coding. - * An element with weight 0 refuses the coding and is kept. Next to a "*" element, which would still offer the coding - * at its own weight, the dropped elements are replaced by one such refusal. + * An element with weight 0 refuses the coding and is kept. A list with a "*" that accepts codings is rebuilt by + * {@link #replaceWildcard}. */ private static String filterOutAcceptEncoding(String acceptEncoding, AsciiString coding) { - if (!AsciiString.containsIgnoreCase(acceptEncoding, coding)) { + if (!AsciiString.containsIgnoreCase(acceptEncoding, coding) && acceptEncoding.indexOf('*') < 0) { return acceptEncoding; } String[] elements = acceptEncoding.split(","); - boolean wildcard = false; for (String element : elements) { - wildcard |= offers(element.trim(), "*"); + if (offers(element.trim(), "*")) { + return replaceWildcard(elements, coding); + } } StringBuilder kept = new StringBuilder(acceptEncoding.length()); boolean dropped = false; for (String element : elements) { String trimmed = element.trim(); if (offers(trimmed, coding)) { - if (wildcard && !dropped) { - appendElement(kept, coding + ";q=0"); - } dropped = true; } else if (!trimmed.isEmpty()) { appendElement(kept, trimmed); @@ -232,11 +233,77 @@ private static String filterOutAcceptEncoding(String acceptEncoding, AsciiString return dropped ? kept.toString() : acceptEncoding; } + /** + * A "*" would still offer {@code coding}, so it is dropped, with every element naming {@code coding} and every + * other token holding a "*". A coding the list does not name is then not acceptable, so the refusals are dropped + * too, but "identity;q=0" and "*;q=0", which decide whether an unencoded response is acceptable. When no content + * coding but br or zstd is left with a positive weight, the {@link #WILDCARD_CODINGS} the list does not name take + * the place of the "*", at its weight. + *

+ * Netty's {@code HttpContentCompressor} matches "*" and coding names by substring, picks a refused coding once a + * listed coding has a positive weight, and compares weights along br, zstd, snappy, gzip, deflate. Keeping the + * elements with a positive weight as they are, and adding gzip and deflate only when nothing else is offered, + * leaves it the choice it makes on the unfiltered list, unless that choice was {@code coding}, a refused coding or + * one offered only by the "*". + */ + private static String replaceWildcard(String[] elements, AsciiString coding) { + String parameters = ""; + boolean wildcardSeen = false; + boolean offersOther = false; + StringBuilder kept = new StringBuilder(); + for (String element : elements) { + String trimmed = element.trim(); + String token = token(trimmed); + if (offers(trimmed, "*")) { + if (!wildcardSeen) { + wildcardSeen = true; + int semicolon = trimmed.indexOf(';'); + parameters = semicolon < 0 ? "" : trimmed.substring(semicolon); + } + } else if ("*".equals(token) || AsciiString.contentEqualsIgnoreCase(token, HttpHeaderValues.IDENTITY)) { + appendElement(kept, trimmed); + } else if (!trimmed.isEmpty() && token.indexOf('*') < 0 && !refuses(trimmed) + && !AsciiString.contentEqualsIgnoreCase(token, coding)) { + appendElement(kept, trimmed); + offersOther |= !AsciiString.contentEqualsIgnoreCase(token, HttpHeaderValues.BR) + && !AsciiString.contentEqualsIgnoreCase(token, HttpHeaderValues.ZSTD); + } + } + if (!offersOther) { + for (AsciiString wildcardCoding : WILDCARD_CODINGS) { + if (!names(elements, wildcardCoding)) { + appendElement(kept, wildcardCoding + parameters); + } + } + } + return kept.toString(); + } + + private static String token(String element) { + int semicolon = element.indexOf(';'); + return semicolon < 0 ? element : element.substring(0, semicolon).trim(); + } + private static boolean offers(String element, CharSequence coding) { + return AsciiString.contentEqualsIgnoreCase(token(element), coding) && !refuses(element); + } + + private static boolean refuses(String element) { int semicolon = element.indexOf(';'); - String token = semicolon < 0 ? element : element.substring(0, semicolon).trim(); - return AsciiString.contentEqualsIgnoreCase(token, coding) - && (semicolon < 0 || !hasZeroWeight(element.substring(semicolon + 1))); + return semicolon >= 0 && hasZeroWeight(element.substring(semicolon + 1)); + } + + // A recipient should treat x-gzip as gzip (RFC 9110 section 8.4.1.3); AHC decodes x-deflate as deflate. + private static boolean names(String[] elements, AsciiString coding) { + AsciiString alias = HttpHeaderValues.GZIP.equals(coding) ? HttpHeaderValues.X_GZIP : HttpHeaderValues.X_DEFLATE; + for (String element : elements) { + String token = token(element.trim()); + if (AsciiString.contentEqualsIgnoreCase(token, coding) + || AsciiString.contentEqualsIgnoreCase(token, alias)) { + return true; + } + } + return false; } private static void appendElement(StringBuilder list, String element) { diff --git a/client/src/test/java/org/asynchttpclient/netty/request/NettyRequestFactoryAcceptEncodingTest.java b/client/src/test/java/org/asynchttpclient/netty/request/NettyRequestFactoryAcceptEncodingTest.java index 9ba0e5576..6f66f933d 100644 --- a/client/src/test/java/org/asynchttpclient/netty/request/NettyRequestFactoryAcceptEncodingTest.java +++ b/client/src/test/java/org/asynchttpclient/netty/request/NettyRequestFactoryAcceptEncodingTest.java @@ -15,19 +15,34 @@ */ package org.asynchttpclient.netty.request; +import io.netty.buffer.Unpooled; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.DefaultFullHttpResponse; import io.netty.handler.codec.http.DefaultHttpHeaders; +import io.netty.handler.codec.http.FullHttpRequest; +import io.netty.handler.codec.http.HttpContentCompressor; import io.netty.handler.codec.http.HttpHeaders; +import io.netty.handler.codec.http.HttpResponse; +import io.netty.util.ReferenceCountUtil; import org.asynchttpclient.AsyncHttpClientConfig; import org.asynchttpclient.DefaultAsyncHttpClientConfig; import org.asynchttpclient.RequestBuilder; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import static io.netty.handler.codec.http.HttpHeaderNames.ACCEPT_ENCODING; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_ENCODING; +import static io.netty.handler.codec.http.HttpMethod.GET; +import static io.netty.handler.codec.http.HttpResponseStatus.OK; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; +import static java.nio.charset.StandardCharsets.US_ASCII; import static org.asynchttpclient.netty.request.NettyRequestFactory.filterOutUndecodableAcceptEncodings; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Brotli and zstd are both on the test classpath, so the availability flags are passed in, to the helper or to the @@ -35,6 +50,9 @@ */ public class NettyRequestFactoryAcceptEncodingTest { + // Brotli and zstd availability; with both available the header is sent as the caller set it. + private static final boolean[][] A_LIBRARY_MISSING = {{false, false}, {false, true}, {true, false}}; + @Test public void dropsBothWhenNeitherIsAvailable() { assertEquals(List.of("gzip, deflate"), filter(false, false, "gzip, deflate, br, zstd")); @@ -70,9 +88,60 @@ public void newRequestDropsTheUnavailableCodingsWithAutomaticDecompression() { assertEquals(List.of("gzip, deflate, br, zstd"), newRequest(true, true, true, "gzip, deflate, br, zstd")); } + @Test + public void newRequestDropsAWildcardWithAutomaticDecompression() { + assertEquals(List.of("gzip, deflate"), newRequest(true, false, false, "br, *")); + assertEquals(List.of("gzip"), newRequest(true, false, false, "gzip, *")); + assertEquals(List.of("gzip"), newRequest(true, false, false, "gzip, br;q=0", "*")); + assertEquals(List.of("gzip, deflate"), newRequest(true, false, true, "*")); + assertEquals(List.of("zstd, gzip, deflate"), newRequest(true, false, true, "zstd, *")); + assertEquals(List.of("gzip;q=0.5, deflate;q=0.5"), newRequest(true, true, false, "*;q=0.5")); + assertEquals(List.of("br, *"), newRequest(true, true, true, "br, *")); + } + @Test public void newRequestLeavesTheHeaderToTheUserWithoutAutomaticDecompression() { assertEquals(List.of("gzip, deflate, br, zstd"), newRequest(false, false, false, "gzip, deflate, br, zstd")); + assertEquals(List.of("br, *"), newRequest(false, false, false, "br, *")); + } + + @Test + public void nettyServerAnswersWithADecodedCodingThatIsNotRefused() { + String[] acceptEncodings = {"*", "gzip, *", "br, *", "gzip, br, *", "gzip, br;q=0, *", "*, zstd;q=0.5", + "gzip;q=0.5, *;q=0.1", "gzip, deflate, br, zstd", "br;q=0, *", "zstd;q=0, *", + "x-br;q=0, ZSTD;q=0, *", "br;q=0, zstd;q=0, gzip;q=0, deflate;q=0, snappy;q=0, *, x*", + "compress, x*, *", "gzip, x-zstd;q=0, br;q=0, *", "snappy;q=0, deflate, *"}; + for (boolean[] available : A_LIBRARY_MISSING) { + boolean brotliAvailable = available[0]; + boolean zstdAvailable = available[1]; + List decoded = new ArrayList<>(List.of("gzip", "deflate")); + if (brotliAvailable) { + decoded.add("br"); + } + if (zstdAvailable) { + decoded.add("zstd"); + } + for (String acceptEncoding : acceptEncodings) { + String sent = String.join(", ", newRequest(true, brotliAvailable, zstdAvailable, acceptEncoding)); + String coding = nettyServerCoding(sent); + assertTrue(coding == null || (decoded.contains(coding) && !refuses(acceptEncoding, coding)), + acceptEncoding + " sent as " + sent + " got " + coding + " with br " + brotliAvailable + + ", zstd " + zstdAvailable); + } + } + } + + @Test + public void nettyServerStillPicksGzipForAWildcardNextToGzip() { + // Http2ContentDecompressor decodes only gzip and deflate, so the "*" must not turn into br or zstd. + for (boolean[] available : A_LIBRARY_MISSING) { + for (String acceptEncoding : new String[]{"gzip, *", "zstd;q=0.1, x-snappy;q=0.2, gzip, *"}) { + List sent = newRequest(true, available[0], available[1], acceptEncoding); + assertEquals("gzip", nettyServerCoding(String.join(", ", sent)), acceptEncoding + " sent as " + sent); + } + } + assertEquals("gzip", nettyServerCoding(String.join(", ", newRequest(true, false, false, "gzip, br, *")))); + assertEquals("gzip", nettyServerCoding(String.join(", ", newRequest(true, false, true, "gzip, br, *")))); } @Test @@ -99,6 +168,41 @@ private static List newRequest(boolean automaticDecompression, boolean b .getAll(ACCEPT_ENCODING); } + private static boolean refuses(String acceptEncoding, String coding) { + for (String element : acceptEncoding.split(",")) { + String[] parts = element.split(";"); + if (parts[0].trim().equalsIgnoreCase(coding)) { + for (int i = 1; i < parts.length; i++) { + String[] parameter = parts[i].split("=", 2); + if (parameter.length == 2 && parameter[0].trim().equalsIgnoreCase("q") + && Float.parseFloat(parameter[1].trim()) == 0) { + return true; + } + } + } + } + return false; + } + + // A Netty server with every coding Netty offers, br and zstd included as both are on the test classpath. + private static String nettyServerCoding(String acceptEncoding) { + EmbeddedChannel server = new EmbeddedChannel(new HttpContentCompressor()); + try { + FullHttpRequest request = new DefaultFullHttpRequest(HTTP_1_1, GET, "/"); + request.headers().set(ACCEPT_ENCODING, acceptEncoding); + server.writeInbound(request); + ReferenceCountUtil.release(server.readInbound()); + server.writeOutbound(new DefaultFullHttpResponse(HTTP_1_1, OK, + Unpooled.copiedBuffer("hello, hello, hello", US_ASCII))); + HttpResponse response = server.readOutbound(); + String coding = response.headers().get(CONTENT_ENCODING); + ReferenceCountUtil.release(response); + return coding; + } finally { + server.finishAndReleaseAll(); + } + } + private static List filter(boolean brotliAvailable, boolean zstdAvailable, String... acceptEncodings) { HttpHeaders headers = new DefaultHttpHeaders(); headers.add(ACCEPT_ENCODING, Arrays.asList(acceptEncodings)); diff --git a/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java b/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java index 25bcf1016..df77c4569 100644 --- a/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java +++ b/client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java @@ -260,18 +260,52 @@ public void filterOutZstdDropsTheCodingWherever() { } @Test - public void filterOutKeepsARefusalOfTheCoding() { - assertEquals("*, br;q=0", filterOutBrotli("*, br;q=0")); - assertEquals("*, BR ; Q = 0.000", filterOutBrotli("*, BR ; Q = 0.000")); - assertEquals("gzip, zstd;q=0., *", filterOutZstd("gzip, zstd;q=0., *")); + public void filterOutKeepsARefusalOfTheCodingWithoutAWildcard() { assertEquals("gzip, br;q=0", filterOutBrotli("gzip, br;q=0.001, br;q=0")); + assertEquals("ZSTD ; Q = 0.000", filterOutZstd("ZSTD ; Q = 0.000")); + assertEquals("*;q=0", filterOutBrotli("*;q=0, br")); + assertEquals("*;q=0, br;q=0", filterOutBrotli("*;q=0, br;q=0")); + assertEquals("br;q=0, x*", filterOutBrotli("br;q=0, x*")); + assertEquals("gzip;q=0, deflate", filterOutBrotli("gzip;q=0, deflate, br")); } @Test - public void filterOutRefusesTheCodingNextToAWildcard() { - assertEquals("*, zstd;q=0", filterOutZstd("*, zstd;q=0.5")); - assertEquals("gzip, br;q=0, *", filterOutBrotli("gzip, BR, br;q=0.2, *")); - assertEquals("*;q=0", filterOutBrotli("*;q=0, br")); + public void filterOutReplacesAWildcardOfferingNothingElseWithGzipAndDeflate() { + assertEquals("gzip, deflate", filterOutBrotli("*")); + assertEquals("gzip, deflate", filterOutZstd("*")); + assertEquals("gzip, deflate", filterOutBrotli("br, *")); + assertEquals("gzip, deflate", filterOutBrotli("*, br;q=0")); + assertEquals("gzip, deflate", filterOutBrotli("zstd;q=0, *")); + assertEquals("gzip, deflate", filterOutBrotli("x-br;q=0, ZSTD;q=0, *")); + assertEquals("gzip;q=0.5, deflate;q=0.5", filterOutZstd("*;q=0.5, zstd;q=0.5")); + assertEquals("zstd, gzip, deflate", filterOutBrotli("zstd, *")); + assertEquals("deflate", filterOutBrotli("gzip;q=0, *")); + assertEquals("deflate", filterOutBrotli("X-Gzip;q=0, *")); + assertEquals("gzip", filterOutBrotli("x-deflate;q=0, *")); + assertEquals("", filterOutBrotli("br;q=0, zstd;q=0, gzip;q=0, deflate;q=0, snappy;q=0, *, x*")); + } + + @Test + public void filterOutKeepsIdentityAndARefusedWildcardWithAReplacedWildcard() { + assertEquals("identity;q=0, gzip, deflate", filterOutBrotli("identity;q=0, *")); + assertEquals("identity;q=0.5, gzip, deflate", filterOutBrotli("identity;q=0.5, *")); + assertEquals("*;q=0, gzip, deflate", filterOutBrotli("*, *;q=0")); + } + + @Test + public void filterOutDropsAWildcardAndRefusalsNextToAnotherOfferedCoding() { + assertEquals("gzip", filterOutBrotli("gzip, *")); + assertEquals("gzip", filterOutBrotli("gzip, br;q=0, *")); + assertEquals("gzip", filterOutBrotli("gzip, BR, br;q=0.2, *")); + assertEquals("gzip", filterOutZstd("gzip, zstd;q=0., *")); + assertEquals("gzip", filterOutBrotli("*, gzip, *;q=0.5")); + assertEquals("gzip;q=0.5", filterOutZstd("gzip;q=0.5, * ; q=0.1")); + assertEquals("gzip, deflate, zstd", filterOutBrotli("gzip, deflate, br, zstd, *")); + assertEquals("snappy, X-Gzip;q=0.5, zstd", filterOutBrotli("snappy, X-Gzip;q=0.5, zstd, *")); + assertEquals("zstd;q=0.1, x-snappy;q=0.2, gzip", filterOutBrotli("zstd;q=0.1, x-snappy;q=0.2, gzip, *")); + assertEquals("compress", filterOutBrotli("compress, gzip;q=0, x*, *")); + assertEquals("gzip", filterOutZstd("gzip, x-zstd;q=0, br;q=0, *")); + assertEquals("deflate", filterOutBrotli("snappy;q=0, deflate, *")); } private static String filterOutBrotli(String acceptEncoding) {