Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 78 additions & 11 deletions client/src/main/java/org/asynchttpclient/util/HttpUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,9 @@ public static String cookiePairName(String pair) {
return (eq >= 0 ? pair.substring(0, eq) : pair).trim();
}

// Decoded over HTTP/1.1 and HTTP/2 alike, unlike br and zstd: Http2ContentDecompressor decodes only these.
private static final AsciiString[] WILDCARD_CODINGS = {HttpHeaderValues.GZIP, HttpHeaderValues.DEFLATE};

public static CharSequence filterOutBrotliFromAcceptEncoding(String acceptEncoding) {
return filterOutAcceptEncoding(acceptEncoding, HttpHeaderValues.BR);
}
Expand All @@ -204,26 +207,24 @@ public static CharSequence filterOutZstdFromAcceptEncoding(String acceptEncoding
/**
* Drops every element offering {@code coding}, which may appear anywhere in the list, in any case and with a weight
* (RFC 9110 section 12.5.3). Dropping the last element leaves an empty value, which accepts no content coding.
* An element with weight 0 refuses the coding and is kept. Next to a "*" element, which would still offer the coding
* at its own weight, the dropped elements are replaced by one such refusal.
* An element with weight 0 refuses the coding and is kept. A list with a "*" that accepts codings is rebuilt by
* {@link #replaceWildcard}.
*/
private static String filterOutAcceptEncoding(String acceptEncoding, AsciiString coding) {
if (!AsciiString.containsIgnoreCase(acceptEncoding, coding)) {
if (!AsciiString.containsIgnoreCase(acceptEncoding, coding) && acceptEncoding.indexOf('*') < 0) {
return acceptEncoding;
}
String[] elements = acceptEncoding.split(",");
boolean wildcard = false;
for (String element : elements) {
wildcard |= offers(element.trim(), "*");
if (offers(element.trim(), "*")) {
return replaceWildcard(elements, coding);
}
}
StringBuilder kept = new StringBuilder(acceptEncoding.length());
boolean dropped = false;
for (String element : elements) {
String trimmed = element.trim();
if (offers(trimmed, coding)) {
if (wildcard && !dropped) {
appendElement(kept, coding + ";q=0");
}
dropped = true;
} else if (!trimmed.isEmpty()) {
appendElement(kept, trimmed);
Expand All @@ -232,11 +233,77 @@ private static String filterOutAcceptEncoding(String acceptEncoding, AsciiString
return dropped ? kept.toString() : acceptEncoding;
}

/**
* A "*" would still offer {@code coding}, so it is dropped, with every element naming {@code coding} and every
* other token holding a "*". A coding the list does not name is then not acceptable, so the refusals are dropped
* too, but "identity;q=0" and "*;q=0", which decide whether an unencoded response is acceptable. When no content
* coding but br or zstd is left with a positive weight, the {@link #WILDCARD_CODINGS} the list does not name take
* the place of the "*", at its weight.
* <p>
* Netty's {@code HttpContentCompressor} matches "*" and coding names by substring, picks a refused coding once a
* listed coding has a positive weight, and compares weights along br, zstd, snappy, gzip, deflate. Keeping the
* elements with a positive weight as they are, and adding gzip and deflate only when nothing else is offered,
* leaves it the choice it makes on the unfiltered list, unless that choice was {@code coding}, a refused coding or
* one offered only by the "*".
*/
private static String replaceWildcard(String[] elements, AsciiString coding) {
String parameters = "";
boolean wildcardSeen = false;
boolean offersOther = false;
StringBuilder kept = new StringBuilder();
for (String element : elements) {
String trimmed = element.trim();
String token = token(trimmed);
if (offers(trimmed, "*")) {
if (!wildcardSeen) {
wildcardSeen = true;
int semicolon = trimmed.indexOf(';');
parameters = semicolon < 0 ? "" : trimmed.substring(semicolon);
}
} else if ("*".equals(token) || AsciiString.contentEqualsIgnoreCase(token, HttpHeaderValues.IDENTITY)) {
appendElement(kept, trimmed);
} else if (!trimmed.isEmpty() && token.indexOf('*') < 0 && !refuses(trimmed)
&& !AsciiString.contentEqualsIgnoreCase(token, coding)) {
appendElement(kept, trimmed);
offersOther |= !AsciiString.contentEqualsIgnoreCase(token, HttpHeaderValues.BR)
&& !AsciiString.contentEqualsIgnoreCase(token, HttpHeaderValues.ZSTD);
}
}
if (!offersOther) {
for (AsciiString wildcardCoding : WILDCARD_CODINGS) {
if (!names(elements, wildcardCoding)) {
appendElement(kept, wildcardCoding + parameters);
}
}
}
return kept.toString();
}

private static String token(String element) {
int semicolon = element.indexOf(';');
return semicolon < 0 ? element : element.substring(0, semicolon).trim();
}

private static boolean offers(String element, CharSequence coding) {
return AsciiString.contentEqualsIgnoreCase(token(element), coding) && !refuses(element);
}

private static boolean refuses(String element) {
int semicolon = element.indexOf(';');
String token = semicolon < 0 ? element : element.substring(0, semicolon).trim();
return AsciiString.contentEqualsIgnoreCase(token, coding)
&& (semicolon < 0 || !hasZeroWeight(element.substring(semicolon + 1)));
return semicolon >= 0 && hasZeroWeight(element.substring(semicolon + 1));
}

// A recipient should treat x-gzip as gzip (RFC 9110 section 8.4.1.3); AHC decodes x-deflate as deflate.
private static boolean names(String[] elements, AsciiString coding) {
AsciiString alias = HttpHeaderValues.GZIP.equals(coding) ? HttpHeaderValues.X_GZIP : HttpHeaderValues.X_DEFLATE;
for (String element : elements) {
String token = token(element.trim());
if (AsciiString.contentEqualsIgnoreCase(token, coding)
|| AsciiString.contentEqualsIgnoreCase(token, alias)) {
return true;
}
}
return false;
}

private static void appendElement(StringBuilder list, String element) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,26 +15,44 @@
*/
package org.asynchttpclient.netty.request;

import io.netty.buffer.Unpooled;
import io.netty.channel.embedded.EmbeddedChannel;
import io.netty.handler.codec.http.DefaultFullHttpRequest;
import io.netty.handler.codec.http.DefaultFullHttpResponse;
import io.netty.handler.codec.http.DefaultHttpHeaders;
import io.netty.handler.codec.http.FullHttpRequest;
import io.netty.handler.codec.http.HttpContentCompressor;
import io.netty.handler.codec.http.HttpHeaders;
import io.netty.handler.codec.http.HttpResponse;
import io.netty.util.ReferenceCountUtil;
import org.asynchttpclient.AsyncHttpClientConfig;
import org.asynchttpclient.DefaultAsyncHttpClientConfig;
import org.asynchttpclient.RequestBuilder;
import org.junit.jupiter.api.Test;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import static io.netty.handler.codec.http.HttpHeaderNames.ACCEPT_ENCODING;
import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_ENCODING;
import static io.netty.handler.codec.http.HttpMethod.GET;
import static io.netty.handler.codec.http.HttpResponseStatus.OK;
import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
import static java.nio.charset.StandardCharsets.US_ASCII;
import static org.asynchttpclient.netty.request.NettyRequestFactory.filterOutUndecodableAcceptEncodings;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;

/**
* Brotli and zstd are both on the test classpath, so the availability flags are passed in, to the helper or to the
* factory, rather than taken from Netty's {@code Brotli.isAvailable()} and {@code Zstd.isAvailable()}.
*/
public class NettyRequestFactoryAcceptEncodingTest {

// Brotli and zstd availability; with both available the header is sent as the caller set it.
private static final boolean[][] A_LIBRARY_MISSING = {{false, false}, {false, true}, {true, false}};

@Test
public void dropsBothWhenNeitherIsAvailable() {
assertEquals(List.of("gzip, deflate"), filter(false, false, "gzip, deflate, br, zstd"));
Expand Down Expand Up @@ -70,9 +88,60 @@ public void newRequestDropsTheUnavailableCodingsWithAutomaticDecompression() {
assertEquals(List.of("gzip, deflate, br, zstd"), newRequest(true, true, true, "gzip, deflate, br, zstd"));
}

@Test
public void newRequestDropsAWildcardWithAutomaticDecompression() {
assertEquals(List.of("gzip, deflate"), newRequest(true, false, false, "br, *"));
assertEquals(List.of("gzip"), newRequest(true, false, false, "gzip, *"));
assertEquals(List.of("gzip"), newRequest(true, false, false, "gzip, br;q=0", "*"));
assertEquals(List.of("gzip, deflate"), newRequest(true, false, true, "*"));
assertEquals(List.of("zstd, gzip, deflate"), newRequest(true, false, true, "zstd, *"));
assertEquals(List.of("gzip;q=0.5, deflate;q=0.5"), newRequest(true, true, false, "*;q=0.5"));
assertEquals(List.of("br, *"), newRequest(true, true, true, "br, *"));
}

@Test
public void newRequestLeavesTheHeaderToTheUserWithoutAutomaticDecompression() {
assertEquals(List.of("gzip, deflate, br, zstd"), newRequest(false, false, false, "gzip, deflate, br, zstd"));
assertEquals(List.of("br, *"), newRequest(false, false, false, "br, *"));
}

@Test
public void nettyServerAnswersWithADecodedCodingThatIsNotRefused() {
String[] acceptEncodings = {"*", "gzip, *", "br, *", "gzip, br, *", "gzip, br;q=0, *", "*, zstd;q=0.5",
"gzip;q=0.5, *;q=0.1", "gzip, deflate, br, zstd", "br;q=0, *", "zstd;q=0, *",
"x-br;q=0, ZSTD;q=0, *", "br;q=0, zstd;q=0, gzip;q=0, deflate;q=0, snappy;q=0, *, x*",
"compress, x*, *", "gzip, x-zstd;q=0, br;q=0, *", "snappy;q=0, deflate, *"};
for (boolean[] available : A_LIBRARY_MISSING) {
boolean brotliAvailable = available[0];
boolean zstdAvailable = available[1];
List<String> decoded = new ArrayList<>(List.of("gzip", "deflate"));
if (brotliAvailable) {
decoded.add("br");
}
if (zstdAvailable) {
decoded.add("zstd");
}
for (String acceptEncoding : acceptEncodings) {
String sent = String.join(", ", newRequest(true, brotliAvailable, zstdAvailable, acceptEncoding));
String coding = nettyServerCoding(sent);
assertTrue(coding == null || (decoded.contains(coding) && !refuses(acceptEncoding, coding)),
acceptEncoding + " sent as " + sent + " got " + coding + " with br " + brotliAvailable
+ ", zstd " + zstdAvailable);
}
}
}

@Test
public void nettyServerStillPicksGzipForAWildcardNextToGzip() {
// Http2ContentDecompressor decodes only gzip and deflate, so the "*" must not turn into br or zstd.
for (boolean[] available : A_LIBRARY_MISSING) {
for (String acceptEncoding : new String[]{"gzip, *", "zstd;q=0.1, x-snappy;q=0.2, gzip, *"}) {
List<String> sent = newRequest(true, available[0], available[1], acceptEncoding);
assertEquals("gzip", nettyServerCoding(String.join(", ", sent)), acceptEncoding + " sent as " + sent);
}
}
assertEquals("gzip", nettyServerCoding(String.join(", ", newRequest(true, false, false, "gzip, br, *"))));
assertEquals("gzip", nettyServerCoding(String.join(", ", newRequest(true, false, true, "gzip, br, *"))));
}

@Test
Expand All @@ -99,6 +168,41 @@ private static List<String> newRequest(boolean automaticDecompression, boolean b
.getAll(ACCEPT_ENCODING);
}

private static boolean refuses(String acceptEncoding, String coding) {
for (String element : acceptEncoding.split(",")) {
String[] parts = element.split(";");
if (parts[0].trim().equalsIgnoreCase(coding)) {
for (int i = 1; i < parts.length; i++) {
String[] parameter = parts[i].split("=", 2);
if (parameter.length == 2 && parameter[0].trim().equalsIgnoreCase("q")
&& Float.parseFloat(parameter[1].trim()) == 0) {
return true;
}
}
}
}
return false;
}

// A Netty server with every coding Netty offers, br and zstd included as both are on the test classpath.
private static String nettyServerCoding(String acceptEncoding) {
EmbeddedChannel server = new EmbeddedChannel(new HttpContentCompressor());
try {
FullHttpRequest request = new DefaultFullHttpRequest(HTTP_1_1, GET, "/");
request.headers().set(ACCEPT_ENCODING, acceptEncoding);
server.writeInbound(request);
ReferenceCountUtil.release(server.readInbound());
server.writeOutbound(new DefaultFullHttpResponse(HTTP_1_1, OK,
Unpooled.copiedBuffer("hello, hello, hello", US_ASCII)));
HttpResponse response = server.readOutbound();
String coding = response.headers().get(CONTENT_ENCODING);
ReferenceCountUtil.release(response);
return coding;
} finally {
server.finishAndReleaseAll();
}
}

private static List<String> filter(boolean brotliAvailable, boolean zstdAvailable, String... acceptEncodings) {
HttpHeaders headers = new DefaultHttpHeaders();
headers.add(ACCEPT_ENCODING, Arrays.asList(acceptEncodings));
Expand Down
50 changes: 42 additions & 8 deletions client/src/test/java/org/asynchttpclient/util/HttpUtilsTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -260,18 +260,52 @@ public void filterOutZstdDropsTheCodingWherever() {
}

@Test
public void filterOutKeepsARefusalOfTheCoding() {
assertEquals("*, br;q=0", filterOutBrotli("*, br;q=0"));
assertEquals("*, BR ; Q = 0.000", filterOutBrotli("*, BR ; Q = 0.000"));
assertEquals("gzip, zstd;q=0., *", filterOutZstd("gzip, zstd;q=0., *"));
public void filterOutKeepsARefusalOfTheCodingWithoutAWildcard() {
assertEquals("gzip, br;q=0", filterOutBrotli("gzip, br;q=0.001, br;q=0"));
assertEquals("ZSTD ; Q = 0.000", filterOutZstd("ZSTD ; Q = 0.000"));
assertEquals("*;q=0", filterOutBrotli("*;q=0, br"));
assertEquals("*;q=0, br;q=0", filterOutBrotli("*;q=0, br;q=0"));
assertEquals("br;q=0, x*", filterOutBrotli("br;q=0, x*"));
assertEquals("gzip;q=0, deflate", filterOutBrotli("gzip;q=0, deflate, br"));
}

@Test
public void filterOutRefusesTheCodingNextToAWildcard() {
assertEquals("*, zstd;q=0", filterOutZstd("*, zstd;q=0.5"));
assertEquals("gzip, br;q=0, *", filterOutBrotli("gzip, BR, br;q=0.2, *"));
assertEquals("*;q=0", filterOutBrotli("*;q=0, br"));
public void filterOutReplacesAWildcardOfferingNothingElseWithGzipAndDeflate() {
assertEquals("gzip, deflate", filterOutBrotli("*"));
assertEquals("gzip, deflate", filterOutZstd("*"));
assertEquals("gzip, deflate", filterOutBrotli("br, *"));
assertEquals("gzip, deflate", filterOutBrotli("*, br;q=0"));
assertEquals("gzip, deflate", filterOutBrotli("zstd;q=0, *"));
assertEquals("gzip, deflate", filterOutBrotli("x-br;q=0, ZSTD;q=0, *"));
assertEquals("gzip;q=0.5, deflate;q=0.5", filterOutZstd("*;q=0.5, zstd;q=0.5"));
assertEquals("zstd, gzip, deflate", filterOutBrotli("zstd, *"));
assertEquals("deflate", filterOutBrotli("gzip;q=0, *"));
assertEquals("deflate", filterOutBrotli("X-Gzip;q=0, *"));
assertEquals("gzip", filterOutBrotli("x-deflate;q=0, *"));
assertEquals("", filterOutBrotli("br;q=0, zstd;q=0, gzip;q=0, deflate;q=0, snappy;q=0, *, x*"));
}

@Test
public void filterOutKeepsIdentityAndARefusedWildcardWithAReplacedWildcard() {
assertEquals("identity;q=0, gzip, deflate", filterOutBrotli("identity;q=0, *"));
assertEquals("identity;q=0.5, gzip, deflate", filterOutBrotli("identity;q=0.5, *"));
assertEquals("*;q=0, gzip, deflate", filterOutBrotli("*, *;q=0"));
}

@Test
public void filterOutDropsAWildcardAndRefusalsNextToAnotherOfferedCoding() {
assertEquals("gzip", filterOutBrotli("gzip, *"));
assertEquals("gzip", filterOutBrotli("gzip, br;q=0, *"));
assertEquals("gzip", filterOutBrotli("gzip, BR, br;q=0.2, *"));
assertEquals("gzip", filterOutZstd("gzip, zstd;q=0., *"));
assertEquals("gzip", filterOutBrotli("*, gzip, *;q=0.5"));
assertEquals("gzip;q=0.5", filterOutZstd("gzip;q=0.5, * ; q=0.1"));
assertEquals("gzip, deflate, zstd", filterOutBrotli("gzip, deflate, br, zstd, *"));
assertEquals("snappy, X-Gzip;q=0.5, zstd", filterOutBrotli("snappy, X-Gzip;q=0.5, zstd, *"));
assertEquals("zstd;q=0.1, x-snappy;q=0.2, gzip", filterOutBrotli("zstd;q=0.1, x-snappy;q=0.2, gzip, *"));
assertEquals("compress", filterOutBrotli("compress, gzip;q=0, x*, *"));
assertEquals("gzip", filterOutZstd("gzip, x-zstd;q=0, br;q=0, *"));
assertEquals("deflate", filterOutBrotli("snappy;q=0, deflate, *"));
}

private static String filterOutBrotli(String acceptEncoding) {
Expand Down
Loading