Add zstd decompression (RFC8478)

* README.md: Mention zstandard
* configure.ac: Check for libzstd
* docs/wget2.md: Document new --compression type
* include/wget/wget.h: Add wget_content_encoding_zstd
* libwget/decompressor.c: Implement the zstd decompressor
* libwget/http_parse.c (wget_http_parse_content_encoding): Add zstd
* src/options.c (print_version): Add +/-zstd for --version,
  (parse_compression): Check for wget_content_encoding_zstd,
  (options): Add zstd to help text
* src/wget.c (http_create_request): Add zstd to Accept-Encoding:
* tests/test-compression.c: Add test for zstd

Author: rockdaboot

README.md

@@ -31,6 +31,7 @@ A non-exhaustive list of features
 
 - Support for HTTP/1.1 and HTTP/2.0 protocol
 - [brotli](https://github.com/google/brotli) decompression support (Accept-Encoding: br)
+- [zstandard](https://github.com/facebook/zstd) decompression support, RFC8478 (Accept-Encoding: zstd)
 - HPKP - HTTP Public Key Pinning (RFC7469) with persistent database
 - TCP Fast Open for plain text *and* for HTTPS
 - TLS Session Resumption including persistent session data cache

configure.ac

@@ -578,6 +578,21 @@ AS_IF([test "x$with_brotlidec" != xno], [
 ])
 AM_CONDITIONAL([WITH_BROTLIDEC], [test "x$with_brotlidec" = xyes])
 
+AC_ARG_WITH(zstd, AS_HELP_STRING([--without-zstd], [disable Zstandard compression support]), with_zstd=$withval, with_zstd=yes)
+AS_IF([test "x$with_zstd" != xno], [
+  PKG_CHECK_MODULES([ZSTD], libzstd, [
+    with_zstd=yes
+    LIBS="$ZSTD_LIBS $LIBS"
+    CFLAGS="$ZSTD_CFLAGS $CFLAGS"
+    AC_DEFINE([WITH_ZSTD], [1], [Use zstd])
+  ], [
+    AC_SEARCH_LIBS(ZSTD_decompressStream, zstd,
+      [with_zstd=yes; AC_DEFINE([WITH_ZSTD], [1], [Use libzstd])],
+      [with_zstd=no;  AC_MSG_WARN(*** libzstd was not found. You will not be able to use Zstandard decompression)])
+  ])
+])
+AM_CONDITIONAL([WITH_ZSTD], [test "x$with_zstd" = xyes])
+
 # Support for internationalized domain names.
 # IDN support in Wget2 is provided in multiple ways:
 #   1. libidn2 >= 0.14.0 (IDNA 2008)
@@ -766,6 +781,7 @@ AC_MSG_NOTICE([Summary of build options:
   BZIP2 compression:  $with_bzip2
   LZMA compression:   $with_lzma
   Brotli compression: $with_brotlidec
+  Zstd compression:   $with_zstd
   IDNA support:       $IDNA_INFO
   PSL support:        $with_libpsl
   HSTS support:       $with_libhsts

docs/wget2.md

@@ -354,7 +354,7 @@ Go to background immediately after startup. If no output file is specified via t
 
     `ResponseTime` ms between start of request and first response packet.
 
-    `Encoding` 0,1,2,3,4,5 mean server side compression was 'identity', 'gzip', 'deflate', 'lzma/xz', 'bzip2', 'brotli'
+    `Encoding` 0,1,2,3,4,5 mean server side compression was 'identity', 'gzip', 'deflate', 'lzma/xz', 'bzip2', 'brotli', 'zstd'
 
     `Verification` PGP verification status. 0,1,2,3 mean 'none',  'valid', 'invalid', 'bad', 'missing'.
 
@@ -1330,7 +1330,7 @@ Go to background immediately after startup. If no output file is specified via t
 
 ### `--compression=TYPE`
 
-  If this TYPE(`identity`, `gzip`, `deflate`, `xz`, `lzma`, `br`, `bzip2` or any combination of it) is given,
+  If this TYPE(`identity`, `gzip`, `deflate`, `xz`, `lzma`, `br`, `bzip2`, `zstd` or any combination of it) is given,
   Wget2 will set "Accept-Encoding" header accordingly. `--no-compression` means no "Accept-Encoding" header at all.
   To set "Accept-Encoding" to a custom value, use `--no-compression` in combination with
   `--header="Accept-Encoding: xxx"`.

include/wget/wget.h

@@ -850,7 +850,8 @@ typedef enum {
 	wget_content_encoding_lzma = 4,
 	wget_content_encoding_bzip2 = 5,
 	wget_content_encoding_brotli = 6,
-	wget_content_encoding_max = 7
+	wget_content_encoding_zstd = 7,
+	wget_content_encoding_max = 8
 } wget_content_encoding_type_t;
 
 WGETAPI G_GNUC_WGET_PURE wget_content_encoding_type_t

libwget/decompressor.c

@@ -54,6 +54,10 @@
 #include <brotli/decode.h>
 #endif
 
+#ifdef WITH_ZSTD
+#include <zstd.h>
+#endif
+
 #include <wget.h>
 #include "private.h"
 
@@ -77,6 +81,10 @@ struct _wget_decompressor_st {
 	BrotliDecoderState
 		*brotli_strm;
 #endif
+#ifdef WITH_ZSTD
+	ZSTD_DStream
+		*zstd_strm;
+#endif
 
 	wget_decompressor_sink_t
 		sink; // decompressed data goes here
@@ -279,6 +287,64 @@ static void brotli_exit(wget_decompressor_t *dc)
 }
 #endif // WITH_BROTLIDEC
 
+#ifdef WITH_ZSTD
+static int zstd_init(ZSTD_DStream **strm)
+{
+	if ((*strm = ZSTD_createDStream()) == NULL) {
+		error_printf(_("Failed to create Zstandard decompression\n"));
+		return -1;
+	}
+
+	size_t rc = ZSTD_initDStream(*strm);
+	if (ZSTD_isError(rc)) {
+		error_printf(_("Failed to init Zstandard decompression: %s\n"), ZSTD_getErrorName(rc));
+		ZSTD_freeDStream(*strm);
+		*strm = NULL;
+		return -1;
+	}
+
+	return 0;
+}
+
+static int zstd_decompress(wget_decompressor_t *dc, char *src, size_t srclen)
+{
+	ZSTD_DStream *strm;
+	uint8_t dst[10240];
+
+	if (!srclen) {
+		// special case to avoid decompress errors
+		if (dc->sink)
+			dc->sink(dc->context, "", 0);
+
+		return 0;
+	}
+
+	strm = dc->zstd_strm;
+
+	ZSTD_inBuffer input = { .src = src, .size = srclen, .pos = 0 };
+
+	while (input.pos < input.size) {
+		ZSTD_outBuffer output = { .dst = dst, .size = sizeof(dst), .pos = 0 };
+
+		size_t rc = ZSTD_decompressStream(strm, &output , &input);
+		if (ZSTD_isError(rc)) {
+			error_printf(_("Failed to init Zstandard decompression: %s\n"), ZSTD_getErrorName(rc));
+			return -1;
+		}
+
+		if (dc->sink)
+			dc->sink(dc->context, (char *)dst, output.pos);
+	}
+
+	return 0;
+}
+
+static void zstd_exit(wget_decompressor_t *dc)
+{
+	ZSTD_freeDStream(dc->zstd_strm);
+}
+#endif // WITH_ZSTD
+
 #ifdef WITH_BZIP2
 static int bzip2_init(bz_stream *strm)
 {
@@ -384,6 +450,13 @@ wget_decompressor_t *wget_decompress_open(
 			dc->decompress = brotli_decompress;
 			dc->exit = brotli_exit;
 		}
+#endif
+	} else if (encoding == wget_content_encoding_zstd) {
+#ifdef WITH_ZSTD
+		if ((rc = zstd_init(&dc->zstd_strm)) == 0) {
+			dc->decompress = zstd_decompress;
+			dc->exit = zstd_exit;
+		}
 #endif
 	}
 
@@ -444,6 +517,7 @@ static char _encoding_names[wget_content_encoding_max][9] = {
 	[wget_content_encoding_lzma] = "lzma",
 	[wget_content_encoding_bzip2] = "bzip2",
 	[wget_content_encoding_brotli] = "br",
+	[wget_content_encoding_zstd] = "zstd",
 };
 
 wget_content_encoding_type_t wget_content_encoding_by_name(const char *name)

libwget/http_parse.c

@@ -682,6 +682,8 @@ const char *wget_http_parse_content_encoding(const char *s, char *content_encodi
 		*content_encoding = wget_content_encoding_lzma;
 	else if (!wget_strcasecmp_ascii(s, "br"))
 		*content_encoding = wget_content_encoding_brotli;
+	else if (!wget_strcasecmp_ascii(s, "zstd"))
+		*content_encoding = wget_content_encoding_zstd;
 	else
 		*content_encoding = wget_content_encoding_identity;
 

src/options.c

@@ -209,6 +209,12 @@ static int print_version(G_GNUC_WGET_UNUSED option_t opt, G_GNUC_WGET_UNUSED con
 	" -brotlidec"
 #endif
 
+#if defined WITH_ZSTD
+	" +zstd"
+#else
+	" -zstd"
+#endif
+
 #if defined WITH_BZIP2
 	" +bzip2"
 #else
@@ -1017,6 +1023,10 @@ static int parse_compression(option_t opt, const char *val, const char invert)
 			if (type == wget_content_encoding_brotli)
 				not_built = 1;
 #endif
+#ifndef WITH_ZSTD
+			if (type == wget_content_encoding_zstd)
+				not_built = 1;
+#endif
 
 			if (not_built) {
 				wget_error_printf(_("Lib for type %s not built"), wget_content_encoding_to_name(type));
@@ -1242,7 +1252,7 @@ static const struct optionw options[] = {
 		"compression", &config.compression, parse_compression, -1, 0,
 		SECTION_HTTP,
 		{ "Customize Accept-Encoding with\n",
-		   "identity, gzip, deflate, xz, lzma, br, bzip2\n",
+		   "identity, gzip, deflate, xz, lzma, br, bzip2, zstd\n",
 		   "and any combination of it\n",
 		   "no-compression means no Accept-Encoding\n"
 		}

src/wget.c

@@ -3568,6 +3568,10 @@ static wget_http_request_t *http_create_request(wget_iri_t *iri, JOB *job)
 #ifdef WITH_BROTLIDEC
 		wget_buffer_strcat(&buf, buf.length ? ", br" : "br");
 #endif
+#ifdef WITH_ZSTD
+		wget_buffer_strcat(&buf, buf.length ? ", zstd" : "zstd");
+#endif
+
 		if (!buf.length)
 			wget_buffer_strcat(&buf, "identity");
 

tests/test-compression.c

@@ -30,6 +30,7 @@
 #define XZ "\xfd\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00\x21\x01\x16\x00\x00\x00\x74\x2f\xe5\xa3\x01\x00\x00\x78\x00\x00\x00\x00\x45\xae\xef\x83\xf8\xee\x16\x0a\x00\x01\x19\x01\xa5\x2c\x81\xcc\x1f\xb6\xf3\x7d\x01\x00\x00\x00\x00\x04\x59\x5a"	// xz
 #define LZMA "\x5d\x00\x00\x80\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x3c\x41\xfb\xff\xff\xff\xe0\x00\x00\x00"	// lzma
 #define BR "\x21\x00\x00\x04\x78\x03"	// br
+#define ZSTD "\x28\xb5\x2f\xfd\x24\x01\x09\x00\x00\x78\x23\x11\x04\x83"
 
 typedef struct  {
 	const char* body;
@@ -38,7 +39,7 @@ typedef struct  {
 	bool with_lib;
 } compression_test_t;
 
-#define KNOWN_TYPES 6
+#define KNOWN_TYPES countof(compressions)
 #define BUF_SIZE 40
 
 int main(void)
@@ -102,6 +103,16 @@ int main(void)
 		true,
 #else
 		false,
+#endif
+		},
+		{	.body = ZSTD,
+			.body_len = sizeof(ZSTD) - 1,
+			.type = "zstd",
+			.with_lib =
+#ifdef WITH_ZSTD
+		true,
+#else
+		false,
 #endif
 		},
 	};
@@ -179,6 +190,18 @@ int main(void)
 				"Accept-Encoding: br"
 			}
 		},
+		{	.name = "/zstd.html",
+			.code = "200 Dontcare",
+			.body = compressions[6].body,
+			.body_len =  compressions[6].body_len,
+			.headers = {
+				"Content-Type: text/html",
+				"Content-Encoding: zstd",
+			},
+			.expected_req_headers = {
+				"Accept-Encoding: zstd"
+			}
+		},
 		{	.name = "/identity.html",
 			.code = "200 Dontcare",
 			.body = uncompressed_body,
@@ -254,32 +277,32 @@ int main(void)
 	wget_test(
 		// WGET_TEST_KEEP_TMPFILES, 1,
 		WGET_TEST_OPTIONS, "--compression=none",
-		WGET_TEST_REQUEST_URL, urls[6].name + 1,
+		WGET_TEST_REQUEST_URL, urls[7].name + 1,
 		WGET_TEST_EXPECTED_ERROR_CODE, 0,
 		WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) {
-			{ urls[6].name + 1, uncompressed_body },
+			{ urls[7].name + 1, uncompressed_body },
 			{	NULL } },
 		0);
 
 	// test identity
 	wget_test(
 		// WGET_TEST_KEEP_TMPFILES, 1,
 		WGET_TEST_OPTIONS, "--compression=identity",
-		WGET_TEST_REQUEST_URL, urls[6].name + 1,
+		WGET_TEST_REQUEST_URL, urls[7].name + 1,
 		WGET_TEST_EXPECTED_ERROR_CODE, 0,
 		WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) {
-			{ urls[6].name + 1, uncompressed_body },
+			{ urls[7].name + 1, uncompressed_body },
 			{	NULL } },
 		0);
 
 	// test no "Accept-Encoding"
 	wget_test(
 		// WGET_TEST_KEEP_TMPFILES, 1,
 		WGET_TEST_OPTIONS, "--no-compression",
-		WGET_TEST_REQUEST_URL, urls[7].name + 1,
+		WGET_TEST_REQUEST_URL, urls[8].name + 1,
 		WGET_TEST_EXPECTED_ERROR_CODE, 0,
 		WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) {
-			{ urls[7].name + 1, uncompressed_body },
+			{ urls[8].name + 1, uncompressed_body },
 			{	NULL } },
 		0);
 
@@ -295,44 +318,44 @@ int main(void)
 	wget_test(
 		// WGET_TEST_KEEP_TMPFILES, 1,
 		WGET_TEST_OPTIONS, "--no-compression --header=\"Accept-Encoding: identity\"",
-		WGET_TEST_REQUEST_URL, urls[6].name + 1,
+		WGET_TEST_REQUEST_URL, urls[7].name + 1,
 		WGET_TEST_EXPECTED_ERROR_CODE, 0,
 		WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) {
-			{ urls[6].name + 1, uncompressed_body },
+			{ urls[7].name + 1, uncompressed_body },
 			{	NULL } },
 		0);
 
 	// test --compression overide
 	wget_test(
 		// WGET_TEST_KEEP_TMPFILES, 1,
 		WGET_TEST_OPTIONS, "--no-compression --compression=identity",
-		WGET_TEST_REQUEST_URL, urls[6].name + 1,
+		WGET_TEST_REQUEST_URL, urls[7].name + 1,
 		WGET_TEST_EXPECTED_ERROR_CODE, 0,
 		WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) {
-			{ urls[6].name + 1, uncompressed_body },
+			{ urls[7].name + 1, uncompressed_body },
 			{	NULL } },
 		0);
 
 	// test --no-compression overide
 	wget_test(
 		// WGET_TEST_KEEP_TMPFILES, 1,
 		WGET_TEST_OPTIONS, "--compression=identity --no-compression",
-		WGET_TEST_REQUEST_URL, urls[7].name + 1,
+		WGET_TEST_REQUEST_URL, urls[8].name + 1,
 		WGET_TEST_EXPECTED_ERROR_CODE, 0,
 		WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) {
-			{ urls[7].name + 1, uncompressed_body },
+			{ urls[8].name + 1, uncompressed_body },
 			{	NULL } },
 		0);
 
 	// test combination
 	wget_test(
         // WGET_TEST_KEEP_TMPFILES, 1,
         WGET_TEST_OPTIONS, "--compression=identity,br,lzma,gzip",
-        WGET_TEST_REQUEST_URL, urls[8].name + 1,
+        WGET_TEST_REQUEST_URL, urls[9].name + 1,
 #if defined WITH_BROTLIDEC && defined WITH_LZMA && defined WITH_ZLIB
         WGET_TEST_EXPECTED_ERROR_CODE, 0,
         WGET_TEST_EXPECTED_FILES, &(wget_test_file_t []) {
-            { urls[8].name + 1, uncompressed_body },
+            { urls[9].name + 1, uncompressed_body },
             {   NULL } },
 #else
         WGET_TEST_EXPECTED_ERROR_CODE, 2,
@@ -358,7 +381,7 @@ int main(void)
 	wget_test(
         // WGET_TEST_KEEP_TMPFILES, 1,
         WGET_TEST_OPTIONS, "--compression=identity,identity",
-        WGET_TEST_REQUEST_URL, urls[8].name + 1,
+        WGET_TEST_REQUEST_URL, urls[9].name + 1,
         WGET_TEST_EXPECTED_ERROR_CODE, 2,
 		0);