Skip to content

Commit 5159055

Browse files
KN4CK3RKN4CK3R
authored
Read expected buffer size (#17409) (#17430)
Backport of #17409 * Read expected buffer size. * Changed name.
1 parent 06da10b commit 5159055

File tree

11 files changed

+51
-29
lines changed

11 files changed

+51
-29
lines changed

‎modules/charset/charset.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414

1515
"code.gitea.io/gitea/modules/log"
1616
"code.gitea.io/gitea/modules/setting"
17+
"code.gitea.io/gitea/modules/util"
1718

1819
"github.com/gogs/chardet"
1920
"golang.org/x/net/html/charset"
@@ -26,9 +27,9 @@ var UTF8BOM = []byte{'\xef', '\xbb', '\xbf'}
2627
// ToUTF8WithFallbackReader detects the encoding of content and coverts to UTF-8 reader if possible
2728
func ToUTF8WithFallbackReader(rd io.Reader) io.Reader {
2829
var buf = make([]byte, 2048)
29-
n, err := rd.Read(buf)
30+
n, err := util.ReadAtMost(rd, buf)
3031
if err != nil {
31-
return rd
32+
return io.MultiReader(bytes.NewReader(RemoveBOMIfPresent(buf[:n])), rd)
3233
}
3334

3435
charsetLabel, err := DetectEncoding(buf[:n])

‎modules/csv/csv.go

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,8 @@ func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader {
3030
// CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader.
3131
func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) {
3232
var data = make([]byte, 1e4)
33-
size, err := rd.Read(data)
33+
size, err := util.ReadAtMost(rd, data)
3434
if err != nil {
35-
if err == io.EOF {
36-
return CreateReader(bytes.NewReader([]byte{}), rune(',')), nil
37-
}
3835
return nil, err
3936
}
4037

‎modules/git/blob.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"io/ioutil"
1313

1414
"code.gitea.io/gitea/modules/typesniffer"
15+
"code.gitea.io/gitea/modules/util"
1516
)
1617

1718
// This file contains common functions between the gogit and !gogit variants for git Blobs
@@ -29,7 +30,7 @@ func (b *Blob) GetBlobContent() (string, error) {
2930
}
3031
defer dataRc.Close()
3132
buf := make([]byte, 1024)
32-
n, _ := dataRc.Read(buf)
33+
n, _ := util.ReadAtMost(dataRc, buf)
3334
buf = buf[:n]
3435
return string(buf), nil
3536
}

‎modules/repofiles/update.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
repo_module "code.gitea.io/gitea/modules/repository"
2020
"code.gitea.io/gitea/modules/setting"
2121
"code.gitea.io/gitea/modules/structs"
22+
"code.gitea.io/gitea/modules/util"
2223

2324
stdcharset "golang.org/x/net/html/charset"
2425
"golang.org/x/text/transform"
@@ -61,7 +62,7 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string
6162
}
6263
defer reader.Close()
6364
buf := make([]byte, 1024)
64-
n, err := reader.Read(buf)
65+
n, err := util.ReadAtMost(reader, buf)
6566
if err != nil {
6667
// return default
6768
return "UTF-8", false
@@ -84,7 +85,7 @@ func detectEncodingAndBOM(entry *git.TreeEntry, repo *models.Repository) (string
8485
}
8586
defer dataRc.Close()
8687
buf = make([]byte, 1024)
87-
n, err = dataRc.Read(buf)
88+
n, err = util.ReadAtMost(dataRc, buf)
8889
if err != nil {
8990
// return default
9091
return "UTF-8", false

‎modules/typesniffer/typesniffer.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import (
1010
"net/http"
1111
"regexp"
1212
"strings"
13+
14+
"code.gitea.io/gitea/modules/util"
1315
)
1416

1517
// Use at most this many bytes to determine Content Type.
@@ -86,8 +88,8 @@ func DetectContentType(data []byte) SniffedType {
8688
// DetectContentTypeFromReader guesses the content type contained in the reader.
8789
func DetectContentTypeFromReader(r io.Reader) (SniffedType, error) {
8890
buf := make([]byte, sniffLen)
89-
n, err := r.Read(buf)
90-
if err != nil && err != io.EOF {
91+
n, err := util.ReadAtMost(r, buf)
92+
if err != nil {
9193
return SniffedType{}, fmt.Errorf("DetectContentTypeFromReader io error: %w", err)
9294
}
9395
buf = buf[:n]

‎modules/util/io.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright 2021 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package util
6+
7+
import (
8+
"io"
9+
)
10+
11+
// ReadAtMost reads at most len(buf) bytes from r into buf.
12+
// It returns the number of bytes copied. n is only less then len(buf) if r provides fewer bytes.
13+
// If EOF occurs while reading, err will be nil.
14+
func ReadAtMost(r io.Reader, buf []byte) (n int, err error) {
15+
n, err = io.ReadFull(r, buf)
16+
if err == io.EOF || err == io.ErrUnexpectedEOF {
17+
err = nil
18+
}
19+
return
20+
}

‎routers/common/repo.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"code.gitea.io/gitea/modules/log"
1919
"code.gitea.io/gitea/modules/setting"
2020
"code.gitea.io/gitea/modules/typesniffer"
21+
"code.gitea.io/gitea/modules/util"
2122
)
2223

2324
// ServeBlob download a git.Blob
@@ -42,8 +43,8 @@ func ServeBlob(ctx *context.Context, blob *git.Blob) error {
4243
// ServeData download file from io.Reader
4344
func ServeData(ctx *context.Context, name string, size int64, reader io.Reader) error {
4445
buf := make([]byte, 1024)
45-
n, err := reader.Read(buf)
46-
if err != nil && err != io.EOF {
46+
n, err := util.ReadAtMost(reader, buf)
47+
if err != nil {
4748
return err
4849
}
4950
if n >= 0 {

‎routers/web/repo/attachment.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"code.gitea.io/gitea/modules/setting"
1616
"code.gitea.io/gitea/modules/storage"
1717
"code.gitea.io/gitea/modules/upload"
18+
"code.gitea.io/gitea/modules/util"
1819
"code.gitea.io/gitea/routers/common"
1920
)
2021

@@ -43,10 +44,8 @@ func uploadAttachment(ctx *context.Context, allowedTypes string) {
4344
defer file.Close()
4445

4546
buf := make([]byte, 1024)
46-
n, _ := file.Read(buf)
47-
if n > 0 {
48-
buf = buf[:n]
49-
}
47+
n, _ := util.ReadAtMost(file, buf)
48+
buf = buf[:n]
5049

5150
err = upload.Verify(buf, header.Filename, allowedTypes)
5251
if err != nil {

‎routers/web/repo/editor.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ func editFile(ctx *context.Context, isNewFile bool) {
114114
ctx.Data["FileName"] = blob.Name()
115115

116116
buf := make([]byte, 1024)
117-
n, _ := dataRc.Read(buf)
117+
n, _ := util.ReadAtMost(dataRc, buf)
118118
buf = buf[:n]
119119

120120
// Only some file types are editable online as text.
@@ -747,7 +747,7 @@ func UploadFileToServer(ctx *context.Context) {
747747
defer file.Close()
748748

749749
buf := make([]byte, 1024)
750-
n, _ := file.Read(buf)
750+
n, _ := util.ReadAtMost(file, buf)
751751
if n > 0 {
752752
buf = buf[:n]
753753
}

‎routers/web/repo/lfs.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"code.gitea.io/gitea/modules/setting"
2727
"code.gitea.io/gitea/modules/storage"
2828
"code.gitea.io/gitea/modules/typesniffer"
29+
"code.gitea.io/gitea/modules/util"
2930
)
3031

3132
const (
@@ -272,7 +273,7 @@ func LFSFileGet(ctx *context.Context) {
272273
}
273274
defer dataRc.Close()
274275
buf := make([]byte, 1024)
275-
n, err := dataRc.Read(buf)
276+
n, err := util.ReadAtMost(dataRc, buf)
276277
if err != nil {
277278
ctx.ServerError("Data", err)
278279
return
@@ -297,10 +298,10 @@ func LFSFileGet(ctx *context.Context) {
297298
break
298299
}
299300

300-
buf := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
301+
rd := charset.ToUTF8WithFallbackReader(io.MultiReader(bytes.NewReader(buf), dataRc))
301302

302303
// Building code view blocks with line number on server side.
303-
fileContent, _ := ioutil.ReadAll(buf)
304+
fileContent, _ := ioutil.ReadAll(rd)
304305

305306
var output bytes.Buffer
306307
lines := strings.Split(string(fileContent), "\n")

‎routers/web/repo/view.go

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"code.gitea.io/gitea/modules/setting"
3232
"code.gitea.io/gitea/modules/structs"
3333
"code.gitea.io/gitea/modules/typesniffer"
34+
"code.gitea.io/gitea/modules/util"
3435
)
3536

3637
const (
@@ -264,7 +265,7 @@ func renderDirectory(ctx *context.Context, treeLink string) {
264265
defer dataRc.Close()
265266

266267
buf := make([]byte, 1024)
267-
n, _ := dataRc.Read(buf)
268+
n, _ := util.ReadAtMost(dataRc, buf)
268269
buf = buf[:n]
269270

270271
st := typesniffer.DetectContentType(buf)
@@ -299,7 +300,7 @@ func renderDirectory(ctx *context.Context, treeLink string) {
299300
defer dataRc.Close()
300301

301302
buf = make([]byte, 1024)
302-
n, err = dataRc.Read(buf)
303+
n, err = util.ReadAtMost(dataRc, buf)
303304
if err != nil {
304305
ctx.ServerError("Data", err)
305306
return
@@ -413,7 +414,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
413414
ctx.Data["RawFileLink"] = rawLink + "/" + ctx.Repo.TreePath
414415

415416
buf := make([]byte, 1024)
416-
n, _ := dataRc.Read(buf)
417+
n, _ := util.ReadAtMost(dataRc, buf)
417418
buf = buf[:n]
418419

419420
st := typesniffer.DetectContentType(buf)
@@ -445,10 +446,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
445446
defer dataRc.Close()
446447

447448
buf = make([]byte, 1024)
448-
n, err = dataRc.Read(buf)
449-
// Error EOF don't mean there is an error, it just means we read to
450-
// the end
451-
if err != nil && err != io.EOF {
449+
n, err = util.ReadAtMost(dataRc, buf)
450+
if err != nil {
452451
ctx.ServerError("Data", err)
453452
return
454453
}

0 commit comments

Comments
 (0)