Fix commenting on non-utf8 encoded files (#11916) (#11950)

* Add comment on non-unicode line to force fail

Signed-off-by: Andrew Thornton <art27@cantab.net>

* Just quote/unquote patch

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: zeripath <art27@cantab.net>

Author: 6543

integrations/api_pull_review_test.go

@@ -86,14 +86,19 @@ func TestAPIPullReview(t *testing.T) {
 			Body:       "first old line",
 			OldLineNum: 1,
 			NewLineNum: 0,
+		}, {
+			Path:       "iso-8859-1.txt",
+			Body:       "this line contains a non-utf-8 character",
+			OldLineNum: 0,
+			NewLineNum: 1,
 		},
 		},
 	})
 	resp = session.MakeRequest(t, req, http.StatusOK)
 	DecodeJSON(t, resp, &review)
 	assert.EqualValues(t, 6, review.ID)
 	assert.EqualValues(t, "PENDING", review.State)
-	assert.EqualValues(t, 2, review.CodeCommentsCount)
+	assert.EqualValues(t, 3, review.CodeCommentsCount)
 
 	// test SubmitPullReview
 	req = NewRequestWithJSON(t, http.MethodPost, fmt.Sprintf("/api/v1/repos/%s/%s/pulls/%d/reviews/%d?token=%s", repo.OwnerName, repo.Name, pullIssue.Index, review.ID, token), &api.SubmitPullReviewOptions{
@@ -104,7 +109,7 @@ func TestAPIPullReview(t *testing.T) {
 	DecodeJSON(t, resp, &review)
 	assert.EqualValues(t, 6, review.ID)
 	assert.EqualValues(t, "APPROVED", review.State)
-	assert.EqualValues(t, 2, review.CodeCommentsCount)
+	assert.EqualValues(t, 3, review.CodeCommentsCount)
 
 	// test DeletePullReview
 	req = NewRequestWithJSON(t, http.MethodPost, fmt.Sprintf("/api/v1/repos/%s/%s/pulls/%d/reviews?token=%s", repo.OwnerName, repo.Name, pullIssue.Index, token), &api.CreatePullReviewOptions{

integrations/gitea-repositories-meta/user2/repo1.git/objects/5f/22f7d0d95d614d25a5b68592adb345a4b5c7fd

@@ -0,0 +1,2 @@
+x
e��N�0D�#���4 
+J�A
�5����,�x�zsV���5�D��ػ�7�,=��o.�E卢�q5J=����	r�=>4��O!�Š�����6ms�8��&\Ea�t�T��I�z��ԅ!	�dso@a��&�K5�B)�r4��Q�`Y�L�
��b	���o`�a�3�@(��e�ԭ5���H�\s�H�9�9R�3)��@�S��_"��4sE0�R��.�U|/�m�ۿ]U��z�

integrations/gitea-repositories-meta/user2/repo1.git/objects/90/dcd07da077d1e7cd6032b52d1f79ae2b5f19b2

@@ -1 +1 @@
-4a357436d925b5c974181ff12a994538ddc5a269
+5f22f7d0d95d614d25a5b68592adb345a4b5c7fd

integrations/gitea-repositories-meta/user2/repo1.git/objects/dd/59742c0f6672911f2b64cba5711ac00593ed32

@@ -9,7 +9,9 @@ package models
 import (
 	"fmt"
 	"regexp"
+	"strconv"
 	"strings"
+	"unicode/utf8"
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
@@ -139,7 +141,8 @@ type Comment struct {
 	RenderedContent string `xorm:"-"`
 
 	// Path represents the 4 lines of code cemented by this comment
-	Patch string `xorm:"TEXT"`
+	Patch       string `xorm:"-"`
+	PatchQuoted string `xorm:"TEXT patch"`
 
 	CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"`
 	UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"`
@@ -183,6 +186,33 @@ func (c *Comment) loadIssue(e Engine) (err error) {
 	return
 }
 
+// BeforeInsert will be invoked by XORM before inserting a record
+func (c *Comment) BeforeInsert() {
+	c.PatchQuoted = c.Patch
+	if !utf8.ValidString(c.Patch) {
+		c.PatchQuoted = strconv.Quote(c.Patch)
+	}
+}
+
+// BeforeUpdate will be invoked by XORM before updating a record
+func (c *Comment) BeforeUpdate() {
+	c.PatchQuoted = c.Patch
+	if !utf8.ValidString(c.Patch) {
+		c.PatchQuoted = strconv.Quote(c.Patch)
+	}
+}
+
+// AfterLoad is invoked from XORM after setting the values of all fields of this object.
+func (c *Comment) AfterLoad(session *xorm.Session) {
+	c.Patch = c.PatchQuoted
+	if len(c.PatchQuoted) > 0 && c.PatchQuoted[0] == '"' {
+		unquoted, err := strconv.Unquote(c.PatchQuoted)
+		if err == nil {
+			c.Patch = unquoted
+		}
+	}
+}
+
 func (c *Comment) loadPoster(e Engine) (err error) {
 	if c.PosterID <= 0 || c.Poster != nil {
 		return nil