Add Brotli decompression to Http Client

Author: razielgn

go.mod

@@ -4,6 +4,7 @@ module miniflux.app
 
 require (
 	github.com/PuerkitoBio/goquery v1.8.0
+	github.com/andybalholm/brotli v1.0.4
 	github.com/coreos/go-oidc v2.2.1+incompatible
 	github.com/felixge/httpsnoop v1.0.1 // indirect
 	github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible

go.sum

@@ -42,6 +42,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=
+github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=

http/client/client.go

@@ -21,6 +21,8 @@ import (
 	"miniflux.app/errors"
 	"miniflux.app/logger"
 	"miniflux.app/timer"
+
+	"github.com/andybalholm/brotli"
 )
 
 const (
@@ -185,6 +187,25 @@ func (c *Client) PostJSON(data interface{}) (*Response, error) {
 	return c.executeRequest(request)
 }
 
+func isBodyBrotliEncoded(resp *http.Response) bool {
+	for _, encoding := range resp.Header["Content-Encoding"] {
+		if encoding == "br" {
+			return true
+		}
+	}
+
+	return false
+}
+
+func decodeBody(resp *http.Response) ([]byte, error) {
+	if isBodyBrotliEncoded(resp) {
+	        reader := brotli.NewReader(resp.Body)
+	        return io.ReadAll(reader)
+	}
+
+	return io.ReadAll(resp.Body)
+}
+
 func (c *Client) executeRequest(request *http.Request) (*Response, error) {
 	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[HttpClient] inputURL=%s", c.inputURL))
 
@@ -227,7 +248,7 @@ func (c *Client) executeRequest(request *http.Request) (*Response, error) {
 		return nil, fmt.Errorf("client: response too large (%d bytes)", resp.ContentLength)
 	}
 
-	buf, err := io.ReadAll(resp.Body)
+	buf, err := decodeBody(resp)
 	if err != nil {
 		return nil, fmt.Errorf("client: error while reading body %v", err)
 	}

http/client/client_test.go

@@ -6,6 +6,8 @@ package client // import "miniflux.app/http/client"
 
 import (
 	"testing"
+	"unicode/utf8"
+	"io"
 )
 
 func TestClientWithDelay(t *testing.T) {
@@ -43,6 +45,28 @@ func TestClientWithResponseTooLarge(t *testing.T) {
 	}
 }
 
+func TestClientWithBrotliResponse(t *testing.T) {
+	clt := New("http://httpbin.org/brotli")
+	response, err := clt.Get()
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if response.StatusCode != 200 {
+		t.Fatalf(`Unexpected response status code: %d`, response.StatusCode)
+	}
+
+	buf, err := io.ReadAll(response.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !utf8.Valid(buf) {
+		t.Fatal("Expected to decode brotli response")
+	}
+}
+
 func TestClientWithBasicAuth(t *testing.T) {
 	clt := New("http://httpbin.org/basic-auth/testuser/testpassword")
 	clt.WithCredentials("testuser", "testpassword")