From 55241a32587a3317986b88b2e5532fad481170e3 Mon Sep 17 00:00:00 2001 From: marxangels <42709228+marxangels@users.noreply.github.com> Date: Sat, 29 Jan 2022 18:02:54 +0800 Subject: [PATCH 01/11] Serve LFS/attachment with http.ServeContent to Support Range-Request --- integrations/download_test.go | 16 ++- .../25/2b2353ba950ba3c0457c1acf214937e168db68 | Bin 0 -> 18 bytes .../80/8eedf6b8dd519aa89b59af2d815ed668580fc2 | Bin 0 -> 142 bytes .../ee/c3fee8b8e28307e5c8c9099fac5eb583f797b4 | Bin 0 -> 148 bytes .../user2/repo2.git/refs/heads/master | 2 +- modules/charset/charset.go | 12 ++ modules/lfs/content_store.go | 2 +- modules/setting/mime_type_map.go | 11 +- modules/typesniffer/typesniffer.go | 41 ++++-- routers/common/repo.go | 124 +++++++++++------- routers/web/repo/attachment.go | 2 +- routers/web/repo/download.go | 2 +- 12 files changed, 143 insertions(+), 69 deletions(-) create mode 100644 integrations/gitea-repositories-meta/user2/repo2.git/objects/25/2b2353ba950ba3c0457c1acf214937e168db68 create mode 100644 integrations/gitea-repositories-meta/user2/repo2.git/objects/80/8eedf6b8dd519aa89b59af2d815ed668580fc2 create mode 100644 integrations/gitea-repositories-meta/user2/repo2.git/objects/ee/c3fee8b8e28307e5c8c9099fac5eb583f797b4 diff --git a/integrations/download_test.go b/integrations/download_test.go index f46122d951080..8d34753e2102d 100644 --- a/integrations/download_test.go +++ b/integrations/download_test.go @@ -5,6 +5,7 @@ package integrations import ( + "mime" "net/http" "testing" @@ -70,24 +71,27 @@ func TestDownloadRawTextFileWithoutMimeTypeMapping(t *testing.T) { session := loginUser(t, "user2") - req := NewRequest(t, "GET", "/user2/repo2/raw/branch/master/test.xml") + req := NewRequest(t, "GET", "/user2/repo2/raw/branch/master/bin.foo") resp := session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, "text/plain; charset=utf-8", resp.HeaderMap.Get("Content-Type")) + assert.Equal(t, "application/octet-stream", resp.HeaderMap.Get("Content-Type")) } func TestDownloadRawTextFileWithMimeTypeMapping(t *testing.T) { defer prepareTestEnv(t)() - setting.MimeTypeMap.Map[".xml"] = "text/xml" + // Fixme: this doesn't take effect on the server-side? + setting.MimeTypeMap.Map[".foo"] = "audio/foo" setting.MimeTypeMap.Enabled = true + _ = mime.AddExtensionType(".foo", "audio/foo") session := loginUser(t, "user2") - req := NewRequest(t, "GET", "/user2/repo2/raw/branch/master/test.xml") + req := NewRequest(t, "GET", "/user2/repo2/raw/branch/master/bin.foo") resp := session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, "text/xml; charset=utf-8", resp.HeaderMap.Get("Content-Type")) + // assert.Equal(t, "audio/foo", resp.HeaderMap.Get("Content-Type")) + assert.Equal(t, "application/octet-stream", resp.HeaderMap.Get("Content-Type")) - delete(setting.MimeTypeMap.Map, ".xml") + delete(setting.MimeTypeMap.Map, ".foo") setting.MimeTypeMap.Enabled = false } diff --git a/integrations/gitea-repositories-meta/user2/repo2.git/objects/25/2b2353ba950ba3c0457c1acf214937e168db68 b/integrations/gitea-repositories-meta/user2/repo2.git/objects/25/2b2353ba950ba3c0457c1acf214937e168db68 new file mode 100644 index 0000000000000000000000000000000000000000..ca83a70756f88f3b9145a22c5ef863bbbb75bca7 GIT binary patch literal 18 Zcmb~ISNJaHDRKl-g(HeMB^P%wW$)Df{c+U1`8#VKwT=j1j!hc wKl9m#Zm6#f@9miK+FHM9dkJAhz@a`HSRnt7ru@UIM=6@xvDMzy7vp3)Hg`%wZU6uP literal 0 HcmV?d00001 diff --git a/integrations/gitea-repositories-meta/user2/repo2.git/objects/ee/c3fee8b8e28307e5c8c9099fac5eb583f797b4 b/integrations/gitea-repositories-meta/user2/repo2.git/objects/ee/c3fee8b8e28307e5c8c9099fac5eb583f797b4 new file mode 100644 index 0000000000000000000000000000000000000000..a0d2c16297a6b5f22e9cb8eb2f7ab779ee5af138 GIT binary patch literal 148 zcmV;F0Biqv0V^p=O;s>7F=8+@FfcPQQSivmP1VayVR+T_r@efUH~XP%EAKClN_3cu z?N&pTBxUC5rRC={sA?+*@0!ZJ_<(DT)Okfu^M@I?GoVUxGV@aPip$a&lBaI#lYh5c z&L_3ne}49&^ehb literal 0 HcmV?d00001 diff --git a/integrations/gitea-repositories-meta/user2/repo2.git/refs/heads/master b/integrations/gitea-repositories-meta/user2/repo2.git/refs/heads/master index 334d09ca02155..c0214d49eef42 100644 --- a/integrations/gitea-repositories-meta/user2/repo2.git/refs/heads/master +++ b/integrations/gitea-repositories-meta/user2/repo2.git/refs/heads/master @@ -1 +1 @@ -1032bbf17fbc0d9c95bb5418dabe8f8c99278700 +808eedf6b8dd519aa89b59af2d815ed668580fc2 diff --git a/modules/charset/charset.go b/modules/charset/charset.go index cf8aa0cb75e67..34b8518e50a65 100644 --- a/modules/charset/charset.go +++ b/modules/charset/charset.go @@ -129,6 +129,18 @@ func RemoveBOMIfPresent(content []byte) []byte { return content } +// DetectEncodingFromReader +// Read the head 1024 bytes from the reader and detect it's encoding +// Note: you may need reader.Seek(0, io.SeekStart) to reset the offset +func DetectEncodingFromReader(reader io.Reader) (string, error) { + buf := make([]byte, 1024) + n, err := util.ReadAtMost(reader, buf) + if err != nil { + return "", fmt.Errorf("DetectEncoding io error: %w", err) + } + return DetectEncoding(buf[:n]) +} + // DetectEncoding detect the encoding of content func DetectEncoding(content []byte) (string, error) { if utf8.Valid(content) { diff --git a/modules/lfs/content_store.go b/modules/lfs/content_store.go index 9fa2c7e3b2ce7..a8e1f4028465a 100644 --- a/modules/lfs/content_store.go +++ b/modules/lfs/content_store.go @@ -114,7 +114,7 @@ func (s *ContentStore) Verify(pointer Pointer) (bool, error) { } // ReadMetaObject will read a models.LFSMetaObject and return a reader -func ReadMetaObject(pointer Pointer) (io.ReadCloser, error) { +func ReadMetaObject(pointer Pointer) (io.ReadSeekCloser, error) { contentStore := NewContentStore() return contentStore.Get(pointer) } diff --git a/modules/setting/mime_type_map.go b/modules/setting/mime_type_map.go index 8e5b864e2413f..add2a3acc2400 100644 --- a/modules/setting/mime_type_map.go +++ b/modules/setting/mime_type_map.go @@ -4,7 +4,12 @@ package setting -import "strings" +import ( + "mime" + "strings" + + "code.gitea.io/gitea/modules/log" +) // MimeTypeMap defines custom mime type mapping settings var MimeTypeMap = struct { @@ -21,6 +26,10 @@ func newMimeTypeMap() { m := make(map[string]string, len(keys)) for _, key := range keys { m[strings.ToLower(key.Name())] = key.Value() + err := mime.AddExtensionType(key.Name(), key.Value()) + if err != nil { + log.Warn("mime.AddExtensionType(%s,%s): %v", key.Name(), key.Value(), err) + } } MimeTypeMap.Map = m if len(keys) > 0 { diff --git a/modules/typesniffer/typesniffer.go b/modules/typesniffer/typesniffer.go index e4bed6595a9c9..5cbcd935061ce 100644 --- a/modules/typesniffer/typesniffer.go +++ b/modules/typesniffer/typesniffer.go @@ -7,7 +7,9 @@ package typesniffer import ( "fmt" "io" + "mime" "net/http" + "path/filepath" "regexp" "strings" @@ -32,32 +34,32 @@ type SniffedType struct { // IsText etects if content format is plain text. func (ct SniffedType) IsText() bool { - return strings.Contains(ct.contentType, "text/") + return strings.HasPrefix(ct.contentType, "text/") } // IsImage detects if data is an image format func (ct SniffedType) IsImage() bool { - return strings.Contains(ct.contentType, "image/") + return strings.HasPrefix(ct.contentType, "image/") } // IsSvgImage detects if data is an SVG image format func (ct SniffedType) IsSvgImage() bool { - return strings.Contains(ct.contentType, SvgMimeType) + return strings.HasPrefix(ct.contentType, SvgMimeType) } // IsPDF detects if data is a PDF format func (ct SniffedType) IsPDF() bool { - return strings.Contains(ct.contentType, "application/pdf") + return strings.HasPrefix(ct.contentType, "application/pdf") } // IsVideo detects if data is an video format func (ct SniffedType) IsVideo() bool { - return strings.Contains(ct.contentType, "video/") + return strings.HasPrefix(ct.contentType, "video/") } // IsAudio detects if data is an video format func (ct SniffedType) IsAudio() bool { - return strings.Contains(ct.contentType, "audio/") + return strings.HasPrefix(ct.contentType, "audio/") } // IsRepresentableAsText returns true if file content can be represented as @@ -66,6 +68,11 @@ func (ct SniffedType) IsRepresentableAsText() bool { return ct.IsText() || ct.IsSvgImage() } +// Mime return the mime +func (ct SniffedType) Mime() string { + return strings.Split(ct.contentType, ";")[0] +} + // DetectContentType extends http.DetectContentType with more content types. Defaults to text/unknown if input is empty. func DetectContentType(data []byte) SniffedType { if len(data) == 0 { @@ -78,8 +85,8 @@ func DetectContentType(data []byte) SniffedType { data = data[:sniffLen] } - if (strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html")) && svgTagRegex.Match(data) || - strings.Contains(ct, "text/xml") && svgTagInXMLRegex.Match(data) { + if (strings.HasPrefix(ct, "text/plain") || strings.HasPrefix(ct, "text/html")) && svgTagRegex.Match(data) || + strings.HasPrefix(ct, "text/xml") && svgTagInXMLRegex.Match(data) { // SVG is unsupported. https://github.com/golang/go/issues/15888 ct = SvgMimeType } @@ -87,6 +94,24 @@ func DetectContentType(data []byte) SniffedType { return SniffedType{ct} } +// DetectContentTypeExtFirst +// detect content type by `name` first, if not found, detect by `reader` +// Note: you may need `reader.Seek(0, io.SeekStart)` to reset the offset +func DetectContentTypeExtFirst(name string, bytesOrReader interface{}) (SniffedType, error) { + ct := mime.TypeByExtension(filepath.Ext(name)) + if ct != "" && !strings.HasPrefix(ct, "text/") { + return SniffedType{ct}, nil + } + if r, ok := bytesOrReader.(io.Reader); ok { + st, err := DetectContentTypeFromReader(r) + if nil != err { + return SniffedType{}, err + } + return st, nil + } + return DetectContentType(bytesOrReader.([]byte)), nil +} + // DetectContentTypeFromReader guesses the content type contained in the reader. func DetectContentTypeFromReader(r io.Reader) (SniffedType, error) { buf := make([]byte, sniffLen) diff --git a/routers/common/repo.go b/routers/common/repo.go index b0e14b63f542c..64e45e4e73bb5 100644 --- a/routers/common/repo.go +++ b/routers/common/repo.go @@ -7,9 +7,10 @@ package common import ( "fmt" "io" - "path" - "path/filepath" + "net/http" + "strconv" "strings" + "time" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/context" @@ -21,7 +22,61 @@ import ( "code.gitea.io/gitea/modules/util" ) -// ServeBlob download a git.Blob +func setCommonHeaders(ctx *context.Context, name string, data interface{}) error { + // Google Chrome dislike commas in filenames, so let's change it to a space + name = strings.ReplaceAll(name, ",", " ") + + // Fixme: ctx.Repo.Repository.IsPrivate is nil + // if ctx.Repo.Repository.IsPrivate { + // ctx.Resp.Header().Set("Cache-Control", "private, max-age=300") + // } else { + // ctx.Resp.Header().Set("Cache-Control", "public, max-age=86400") + // } + ctx.Resp.Header().Set("Cache-Control", "public, max-age=86400") + + st, err := typesniffer.DetectContentTypeExtFirst(name, data) + if nil != err { + return err + } + + // reset the offset to the start of served file + if seeker, ok := data.(io.ReadSeeker); ok { + _, _ = seeker.Seek(0, io.SeekStart) + } + + if st.IsText() || ctx.FormBool("render") { + var cs string + var err error + if reader, ok := data.(io.ReadSeeker); ok { + cs, err = charset.DetectEncodingFromReader(reader) + _, _ = reader.Seek(0, io.SeekStart) + } else { + cs, err = charset.DetectEncoding(data.([]byte)) + } + if err != nil { + log.Error("Detect raw file %s charset failed: %v, using by default utf-8", name, err) + cs = "utf-8" + } + + // http.ServeContent has bug on detecting GBK charset + ctx.Resp.Header().Set("Content-Type", fmt.Sprintf("%s; charset=%s", st.Mime(), strings.ToLower(cs))) + } else if (st.IsImage() || st.IsPDF()) && (setting.UI.SVG.Enabled || !st.IsSvgImage()) { + ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition") + ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"`, name)) + if st.IsSvgImage() { + ctx.Resp.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox") + ctx.Resp.Header().Set("X-Content-Type-Options", "nosniff") + ctx.Resp.Header().Set("Content-Type", typesniffer.SvgMimeType) + } + } else { + ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition") + ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name)) + } + return nil +} + +// ServeBlob serve git.Blob which represents a normal(non-lfs) file stored in repositories +// todo: implement io.Seeker for git.Blob.blobReader to support Range-Request func ServeBlob(ctx *context.Context, blob *git.Blob) error { if httpcache.HandleGenericETagCache(ctx.Req, ctx.Resp, `"`+blob.ID.String()+`"`) { return nil @@ -37,13 +92,8 @@ func ServeBlob(ctx *context.Context, blob *git.Blob) error { } }() - return ServeData(ctx, ctx.Repo.TreePath, blob.Size(), dataRc) -} - -// ServeData download file from io.Reader -func ServeData(ctx *context.Context, name string, size int64, reader io.Reader) error { buf := make([]byte, 1024) - n, err := util.ReadAtMost(reader, buf) + n, err := util.ReadAtMost(dataRc, buf) if err != nil { return err } @@ -51,56 +101,30 @@ func ServeData(ctx *context.Context, name string, size int64, reader io.Reader) buf = buf[:n] } - ctx.Resp.Header().Set("Cache-Control", "public,max-age=86400") - + size := blob.Size() if size >= 0 { - ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", size)) + ctx.Resp.Header().Set("Content-Length", strconv.FormatInt(size, 10)) } else { - log.Error("ServeData called to serve data: %s with size < 0: %d", name, size) + log.Error("ServeData called to serve data: %s with size < 0: %d", ctx.Repo.TreePath, size) } - name = path.Base(name) - - // Google Chrome dislike commas in filenames, so let's change it to a space - name = strings.ReplaceAll(name, ",", " ") - st := typesniffer.DetectContentType(buf) - - mappedMimeType := "" - if setting.MimeTypeMap.Enabled { - fileExtension := strings.ToLower(filepath.Ext(name)) - mappedMimeType = setting.MimeTypeMap.Map[fileExtension] - } - if st.IsText() || ctx.FormBool("render") { - cs, err := charset.DetectEncoding(buf) - if err != nil { - log.Error("Detect raw file %s charset failed: %v, using by default utf-8", name, err) - cs = "utf-8" - } - if mappedMimeType == "" { - mappedMimeType = "text/plain" - } - ctx.Resp.Header().Set("Content-Type", mappedMimeType+"; charset="+strings.ToLower(cs)) - } else { - ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition") - if mappedMimeType != "" { - ctx.Resp.Header().Set("Content-Type", mappedMimeType) - } - if (st.IsImage() || st.IsPDF()) && (setting.UI.SVG.Enabled || !st.IsSvgImage()) { - ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"`, name)) - if st.IsSvgImage() { - ctx.Resp.Header().Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'; sandbox") - ctx.Resp.Header().Set("X-Content-Type-Options", "nosniff") - ctx.Resp.Header().Set("Content-Type", typesniffer.SvgMimeType) - } - } else { - ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name)) - } + if err := setCommonHeaders(ctx, ctx.Repo.TreePath, buf); err != nil { + return err } _, err = ctx.Resp.Write(buf) if err != nil { return err } - _, err = io.Copy(ctx.Resp, reader) + _, err = io.Copy(ctx.Resp, dataRc) return err } + +// ServeLargeFile Serve files stored with Git LFS and attachments uploaded on the Releases page +func ServeLargeFile(ctx *context.Context, name string, time time.Time, reader io.ReadSeeker) error { + if err := setCommonHeaders(ctx, name, reader); err != nil { + return err + } + http.ServeContent(ctx.Resp, ctx.Req, name, time, reader) + return nil +} diff --git a/routers/web/repo/attachment.go b/routers/web/repo/attachment.go index 4101d81ac5501..966ec32b00358 100644 --- a/routers/web/repo/attachment.go +++ b/routers/web/repo/attachment.go @@ -144,7 +144,7 @@ func GetAttachment(ctx *context.Context) { } defer fr.Close() - if err = common.ServeData(ctx, attach.Name, attach.Size, fr); err != nil { + if err = common.ServeLargeFile(ctx, attach.Name, attach.CreatedUnix.AsTime(), fr); err != nil { ctx.ServerError("ServeData", err) return } diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index 72d34cb937939..25670f8849e70 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -69,7 +69,7 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob) error { log.Error("ServeBlobOrLFS: Close: %v", err) } }() - return common.ServeData(ctx, ctx.Repo.TreePath, meta.Size, lfsDataRc) + return common.ServeLargeFile(ctx, ctx.Repo.TreePath, meta.CreatedUnix.AsTime(), lfsDataRc) } if err = dataRc.Close(); err != nil { log.Error("ServeBlobOrLFS: Close: %v", err) From 34fa2ced6ed824531316858c711e9ff9312e476a Mon Sep 17 00:00:00 2001 From: marxangels <42709228+marxangels@users.noreply.github.com> Date: Sun, 6 Feb 2022 11:35:58 +0800 Subject: [PATCH 02/11] ctx.Resp.Header().Set("Content-Type", st.Mime()) --- routers/common/repo.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/routers/common/repo.go b/routers/common/repo.go index 64e45e4e73bb5..a374760adcff7 100644 --- a/routers/common/repo.go +++ b/routers/common/repo.go @@ -28,9 +28,9 @@ func setCommonHeaders(ctx *context.Context, name string, data interface{}) error // Fixme: ctx.Repo.Repository.IsPrivate is nil // if ctx.Repo.Repository.IsPrivate { - // ctx.Resp.Header().Set("Cache-Control", "private, max-age=300") + // ctx.Resp.Header().Set("Cache-Control", "private, max-age=300") // } else { - // ctx.Resp.Header().Set("Cache-Control", "public, max-age=86400") + // ctx.Resp.Header().Set("Cache-Control", "public, max-age=86400") // } ctx.Resp.Header().Set("Cache-Control", "public, max-age=86400") @@ -61,6 +61,7 @@ func setCommonHeaders(ctx *context.Context, name string, data interface{}) error // http.ServeContent has bug on detecting GBK charset ctx.Resp.Header().Set("Content-Type", fmt.Sprintf("%s; charset=%s", st.Mime(), strings.ToLower(cs))) } else if (st.IsImage() || st.IsPDF()) && (setting.UI.SVG.Enabled || !st.IsSvgImage()) { + ctx.Resp.Header().Set("Content-Type", st.Mime()) ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition") ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"`, name)) if st.IsSvgImage() { @@ -69,6 +70,7 @@ func setCommonHeaders(ctx *context.Context, name string, data interface{}) error ctx.Resp.Header().Set("Content-Type", typesniffer.SvgMimeType) } } else { + ctx.Resp.Header().Set("Content-Type", st.Mime()) ctx.Resp.Header().Set("Access-Control-Expose-Headers", "Content-Disposition") ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, name)) } From f2ee607ca8cdc63a29fb2861143b0ecd722dea34 Mon Sep 17 00:00:00 2001 From: marxangels <42709228+marxangels@users.noreply.github.com> Date: Sun, 6 Feb 2022 12:20:48 +0800 Subject: [PATCH 03/11] Update download_test.go --- integrations/download_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integrations/download_test.go b/integrations/download_test.go index 8d34753e2102d..2fbab965dd0c8 100644 --- a/integrations/download_test.go +++ b/integrations/download_test.go @@ -79,7 +79,7 @@ func TestDownloadRawTextFileWithoutMimeTypeMapping(t *testing.T) { func TestDownloadRawTextFileWithMimeTypeMapping(t *testing.T) { defer prepareTestEnv(t)() - // Fixme: this doesn't take effect on the server-side? + setting.MimeTypeMap.Map[".foo"] = "audio/foo" setting.MimeTypeMap.Enabled = true _ = mime.AddExtensionType(".foo", "audio/foo") @@ -89,8 +89,7 @@ func TestDownloadRawTextFileWithMimeTypeMapping(t *testing.T) { req := NewRequest(t, "GET", "/user2/repo2/raw/branch/master/bin.foo") resp := session.MakeRequest(t, req, http.StatusOK) - // assert.Equal(t, "audio/foo", resp.HeaderMap.Get("Content-Type")) - assert.Equal(t, "application/octet-stream", resp.HeaderMap.Get("Content-Type")) + assert.Equal(t, "audio/foo", resp.HeaderMap.Get("Content-Type")) delete(setting.MimeTypeMap.Map, ".foo") setting.MimeTypeMap.Enabled = false From d0859442d46c970bf9f61f8050ae50136c6f57be Mon Sep 17 00:00:00 2001 From: marxangels <42709228+marxangels@users.noreply.github.com> Date: Sun, 6 Feb 2022 17:52:26 +0800 Subject: [PATCH 04/11] Update Cache-Control --- routers/common/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/common/repo.go b/routers/common/repo.go index a374760adcff7..c31269bbfde51 100644 --- a/routers/common/repo.go +++ b/routers/common/repo.go @@ -32,7 +32,7 @@ func setCommonHeaders(ctx *context.Context, name string, data interface{}) error // } else { // ctx.Resp.Header().Set("Cache-Control", "public, max-age=86400") // } - ctx.Resp.Header().Set("Cache-Control", "public, max-age=86400") + ctx.Resp.Header().Set("Cache-Control", "public, max-age=300") st, err := typesniffer.DetectContentTypeExtFirst(name, data) if nil != err { From 0f7f5837caf3c7a393a0c3b5e9f22d9e8e27f65b Mon Sep 17 00:00:00 2001 From: marxangels <42709228+marxangels@users.noreply.github.com> Date: Thu, 10 Feb 2022 19:36:46 +0800 Subject: [PATCH 05/11] Add a Comment --- modules/typesniffer/typesniffer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/typesniffer/typesniffer.go b/modules/typesniffer/typesniffer.go index 5cbcd935061ce..0bfc87171f4fc 100644 --- a/modules/typesniffer/typesniffer.go +++ b/modules/typesniffer/typesniffer.go @@ -99,6 +99,7 @@ func DetectContentType(data []byte) SniffedType { // Note: you may need `reader.Seek(0, io.SeekStart)` to reset the offset func DetectContentTypeExtFirst(name string, bytesOrReader interface{}) (SniffedType, error) { ct := mime.TypeByExtension(filepath.Ext(name)) + // FIXME: Not sure if `!strings.HasPrefix(ct, "text/")` is necessary to keep the old behavior. if ct != "" && !strings.HasPrefix(ct, "text/") { return SniffedType{ct}, nil } From 1953d843bc431e73fa5927ae3f95b2f922a89507 Mon Sep 17 00:00:00 2001 From: marxangels <42709228+marxangels@users.noreply.github.com> Date: Fri, 11 Feb 2022 08:37:00 +0800 Subject: [PATCH 06/11] add testes --- integrations/download_test.go | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/integrations/download_test.go b/integrations/download_test.go index 2fbab965dd0c8..d49f9ee45bcd5 100644 --- a/integrations/download_test.go +++ b/integrations/download_test.go @@ -71,13 +71,42 @@ func TestDownloadRawTextFileWithoutMimeTypeMapping(t *testing.T) { session := loginUser(t, "user2") + req := NewRequest(t, "GET", "/user2/repo2/raw/branch/master/test.xml") + resp := session.MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, "text/xml; charset=utf-8", resp.HeaderMap.Get("Content-Type")) +} + +func TestDownloadRawTextFileWithMimeTypeMapping(t *testing.T) { + defer prepareTestEnv(t)() + + setting.MimeTypeMap.Map[".xml"] = "text/xml" + setting.MimeTypeMap.Enabled = true + _ = mime.AddExtensionType(".xml", "text/xml") + + session := loginUser(t, "user2") + + req := NewRequest(t, "GET", "/user2/repo2/raw/branch/master/test.xml") + resp := session.MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, "text/xml; charset=utf-8", resp.HeaderMap.Get("Content-Type")) + + delete(setting.MimeTypeMap.Map, ".xml") + setting.MimeTypeMap.Enabled = false +} + +func TestDownloadRawBinaryFileWithoutMimeTypeMapping(t *testing.T) { + defer prepareTestEnv(t)() + + session := loginUser(t, "user2") + req := NewRequest(t, "GET", "/user2/repo2/raw/branch/master/bin.foo") resp := session.MakeRequest(t, req, http.StatusOK) assert.Equal(t, "application/octet-stream", resp.HeaderMap.Get("Content-Type")) } -func TestDownloadRawTextFileWithMimeTypeMapping(t *testing.T) { +func TestDownloadRawBinaryFileWithMimeTypeMapping(t *testing.T) { defer prepareTestEnv(t)() setting.MimeTypeMap.Map[".foo"] = "audio/foo" From 37813a69f140d59db3203d4d32553e5a18f15340 Mon Sep 17 00:00:00 2001 From: marxangels <42709228+marxangels@users.noreply.github.com> Date: Fri, 11 Feb 2022 20:42:58 +0800 Subject: [PATCH 07/11] Update typesniffer.go --- modules/typesniffer/typesniffer.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/typesniffer/typesniffer.go b/modules/typesniffer/typesniffer.go index 0bfc87171f4fc..b51683743ab47 100644 --- a/modules/typesniffer/typesniffer.go +++ b/modules/typesniffer/typesniffer.go @@ -99,8 +99,9 @@ func DetectContentType(data []byte) SniffedType { // Note: you may need `reader.Seek(0, io.SeekStart)` to reset the offset func DetectContentTypeExtFirst(name string, bytesOrReader interface{}) (SniffedType, error) { ct := mime.TypeByExtension(filepath.Ext(name)) - // FIXME: Not sure if `!strings.HasPrefix(ct, "text/")` is necessary to keep the old behavior. - if ct != "" && !strings.HasPrefix(ct, "text/") { + // FIXME: Not sure if it's necessary to keep the old behavior. + // if ct != "" && !strings.HasPrefix(ct, "text/") { + if ct != "" { return SniffedType{ct}, nil } if r, ok := bytesOrReader.(io.Reader); ok { From c6630dddb1425c5699a57f56a741de884e7694ec Mon Sep 17 00:00:00 2001 From: marxangels <42709228+marxangels@users.noreply.github.com> Date: Fri, 11 Feb 2022 22:22:02 +0800 Subject: [PATCH 08/11] Update repo.go --- routers/common/repo.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/routers/common/repo.go b/routers/common/repo.go index c31269bbfde51..8cf68a8137a0e 100644 --- a/routers/common/repo.go +++ b/routers/common/repo.go @@ -26,12 +26,6 @@ func setCommonHeaders(ctx *context.Context, name string, data interface{}) error // Google Chrome dislike commas in filenames, so let's change it to a space name = strings.ReplaceAll(name, ",", " ") - // Fixme: ctx.Repo.Repository.IsPrivate is nil - // if ctx.Repo.Repository.IsPrivate { - // ctx.Resp.Header().Set("Cache-Control", "private, max-age=300") - // } else { - // ctx.Resp.Header().Set("Cache-Control", "public, max-age=86400") - // } ctx.Resp.Header().Set("Cache-Control", "public, max-age=300") st, err := typesniffer.DetectContentTypeExtFirst(name, data) From 6743d56575c8d408a65c24678f61241dc2040128 Mon Sep 17 00:00:00 2001 From: marxangels <42709228+marxangels@users.noreply.github.com> Date: Sat, 12 Feb 2022 09:19:49 +0800 Subject: [PATCH 09/11] Update download_test.go --- integrations/download_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/download_test.go b/integrations/download_test.go index d49f9ee45bcd5..163740b201775 100644 --- a/integrations/download_test.go +++ b/integrations/download_test.go @@ -74,7 +74,7 @@ func TestDownloadRawTextFileWithoutMimeTypeMapping(t *testing.T) { req := NewRequest(t, "GET", "/user2/repo2/raw/branch/master/test.xml") resp := session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, "text/xml; charset=utf-8", resp.HeaderMap.Get("Content-Type")) + assert.Equal(t, "application/xml", resp.HeaderMap.Get("Content-Type")) } func TestDownloadRawTextFileWithMimeTypeMapping(t *testing.T) { From ea62aab129bd641260a4960ea58934e2eb1f098c Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 25 Jul 2022 05:01:02 +0200 Subject: [PATCH 10/11] ajust --- integrations/download_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/download_test.go b/integrations/download_test.go index 163740b201775..2d2514e0c4da0 100644 --- a/integrations/download_test.go +++ b/integrations/download_test.go @@ -74,7 +74,7 @@ func TestDownloadRawTextFileWithoutMimeTypeMapping(t *testing.T) { req := NewRequest(t, "GET", "/user2/repo2/raw/branch/master/test.xml") resp := session.MakeRequest(t, req, http.StatusOK) - assert.Equal(t, "application/xml", resp.HeaderMap.Get("Content-Type")) + assert.Equal(t, "text/plain; charset=utf-8", resp.HeaderMap.Get("Content-Type")) } func TestDownloadRawTextFileWithMimeTypeMapping(t *testing.T) { From 7c60fc8da7fb021fe5c62390431e542dfd37a074 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 25 Jul 2022 05:08:40 +0200 Subject: [PATCH 11/11] rm some unrelated diff --- integrations/download_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/integrations/download_test.go b/integrations/download_test.go index 2d2514e0c4da0..9594febc11e7d 100644 --- a/integrations/download_test.go +++ b/integrations/download_test.go @@ -79,10 +79,8 @@ func TestDownloadRawTextFileWithoutMimeTypeMapping(t *testing.T) { func TestDownloadRawTextFileWithMimeTypeMapping(t *testing.T) { defer prepareTestEnv(t)() - setting.MimeTypeMap.Map[".xml"] = "text/xml" setting.MimeTypeMap.Enabled = true - _ = mime.AddExtensionType(".xml", "text/xml") session := loginUser(t, "user2")