fix: support bad utf8 chars in read

... not 500

Author: douyun-dtyq

modules/packages/composer/metadata.go

@@ -6,6 +6,7 @@ package composer
 import (
 	"archive/tar"
 	"archive/zip"
+	"bytes"
 	"compress/bzip2"
 	"compress/gzip"
 	"errors"
@@ -15,6 +16,7 @@ import (
 	"regexp"
 	"strings"
 
+	"gitea.dev/modules/charset"
 	"gitea.dev/modules/json"
 	"gitea.dev/modules/util"
 	"gitea.dev/modules/validation"
@@ -273,7 +275,8 @@ func ParsePackage(r ReadSeekAt, optVersion ...string) (*PackageInfo, error) {
 	if len(dataReadmeMd) == 0 {
 		cj.Readme = ""
 	} else {
-		cj.Readme = string(dataReadmeMd)
+		readmeContent := charset.ToUTF8WithFallback(dataReadmeMd, charset.ConvertOpts{ErrorReplacement: []byte{'?'}})
+		cj.Readme = string(bytes.ToValidUTF8(readmeContent, []byte{'?'}))
 	}
 
 	// FIXME: legacy format: strings.ToLower(fmt.Sprintf("%s.%s.zip", strings.ReplaceAll(cp.Name, "/", "-"), cp.Version)), doesn't read good

modules/packages/composer/metadata_test.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"strings"
 	"testing"
+	"unicode/utf8"
 
 	"gitea.dev/modules/json"
 
@@ -195,4 +196,16 @@ func TestParsePackage(t *testing.T) {
 		}, map[string]string{"composer.json": buildComposerContent(""), "README.md": readme})
 		assertValidPackage(t, data, "", "gitea-composer-package.tar.gz")
 	})
+
+	t.Run("InvalidUTF8Readme", func(t *testing.T) {
+		data := createArchive(map[string]string{"composer.json": buildComposerContent(""), "README.md": "readme\xffcontent"})
+
+		cp, err := ParsePackage(bytes.NewReader(data))
+		require.NoError(t, err)
+		require.NotNil(t, cp)
+
+		assert.True(t, utf8.ValidString(cp.Metadata.Readme))
+		_, err = json.Marshal(cp.Metadata)
+		assert.NoError(t, err)
+	})
 }