From 840d9385808ce319aab5f530c71767c930398c59 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 14 Sep 2022 17:14:07 +0100 Subject: [PATCH 1/6] Fix slight bug in katex There is a small bug in #20571 whereby `$a a$b b$` will not be correctly detected as a math inline block of `a a$b b`. This PR fixes this. Signed-off-by: Andrew Thornton --- modules/markup/markdown/math/inline_parser.go | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/modules/markup/markdown/math/inline_parser.go b/modules/markup/markdown/math/inline_parser.go index 0339674b6ca50..4a69daf3f31b3 100644 --- a/modules/markup/markdown/math/inline_parser.go +++ b/modules/markup/markdown/math/inline_parser.go @@ -59,20 +59,29 @@ func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser. } opener += len(parser.start) - ender := bytes.Index(line[opener:], parser.end) - if ender < 0 { - return nil - } - if len(line) > opener+ender+len(parser.end) && isAlphanumeric(line[opener+ender+len(parser.end)]) { - return nil + ender := opener + for pos := opener; pos < len(line); { + ender = bytes.Index(line[pos:], parser.end) + if ender < 0 { + return nil + } + + ender += pos + pos += len(parser.end) + if len(line) <= pos { + break + } + if !isAlphanumeric(line[pos]) { + break + } } block.Advance(opener) _, pos := block.Position() node := NewInline() - segment := pos.WithStop(pos.Start + ender) + segment := pos.WithStop(pos.Start + ender - opener) node.AppendChild(node, ast.NewRawTextSegment(segment)) - block.Advance(ender + len(parser.end)) + block.Advance(ender - opener + len(parser.end)) trimBlock(node, block) return node From 50d572be2517c985b8a7d866a9a83150139967dd Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 4 Oct 2022 21:31:46 +0100 Subject: [PATCH 2/6] re-enable testcases Signed-off-by: Andrew Thornton --- modules/markup/markdown/markdown_test.go | 50 +++++++++++++++++++ modules/markup/markdown/math/inline_parser.go | 30 +++++++---- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index fdbc291c947d2..49ed3d75d6c09 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -6,6 +6,7 @@ package markdown_test import ( "context" + "os" "strings" "testing" @@ -37,6 +38,7 @@ func TestMain(m *testing.M) { if err := git.InitSimple(context.Background()); err != nil { log.Fatal("git init failed, err: %v", err) } + os.Exit(m.Run()) } func TestRender_StandardLinks(t *testing.T) { @@ -426,3 +428,51 @@ func TestRenderEmojiInLinks_Issue12331(t *testing.T) { assert.NoError(t, err) assert.Equal(t, expected, res) } + +func TestMathBlock(t *testing.T) { + const nl = "\n" + testcases := []struct { + testcase string + expected string + }{ + { + "$a$", + `

a

` + nl, + }, + { + "$ a $", + `

a

` + nl, + }, + { + "$a$ $b$", + `

a b

` + nl, + }, + { + `\(a\) \(b\)`, + `

a b

` + nl, + }, + { + `$a a$b b$`, + `

a a$b b

` + nl, + }, + { + `a a$b b`, + `

a a$b b

` + nl, + }, + { + `a$b $a a$b b$`, + `

a$b a a$b b

` + nl, + }, + { + "$$a$$", + `
a
` + nl, + }, + } + + for _, test := range testcases { + res, err := RenderString(&markup.RenderContext{}, test.testcase) + assert.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) + assert.Equal(t, test.expected, res, "Unexpected result in testcase %q", test.testcase) + + } +} diff --git a/modules/markup/markdown/math/inline_parser.go b/modules/markup/markdown/math/inline_parser.go index 4a69daf3f31b3..31f3125390b6f 100644 --- a/modules/markup/markdown/math/inline_parser.go +++ b/modules/markup/markdown/math/inline_parser.go @@ -37,7 +37,7 @@ func NewInlineBracketParser() parser.InlineParser { return defaultInlineBracketParser } -// Trigger triggers this parser on $ +// Trigger returns nil func (parser *inlineParser) Trigger() []byte { return parser.start[0:1] } @@ -50,30 +50,42 @@ func isAlphanumeric(b byte) bool { // Parse parses the current line and returns a result of parsing. func (parser *inlineParser) Parse(parent ast.Node, block text.Reader, pc parser.Context) ast.Node { line, _ := block.PeekLine() - opener := bytes.Index(line, parser.start) - if opener < 0 { + + if !bytes.HasPrefix(line, parser.start) { + // We'll catch this one on the next time round return nil } - if opener != 0 && isAlphanumeric(line[opener-1]) { + + precedingCharacter := block.PrecendingCharacter() + if precedingCharacter < 256 && isAlphanumeric(byte(precedingCharacter)) { + // need to exclude things like `a$` from being considered a start return nil } - opener += len(parser.start) + // move the opener marker point at the start of the text + opener := len(parser.start) + + // Now look for an ending line ender := opener - for pos := opener; pos < len(line); { - ender = bytes.Index(line[pos:], parser.end) - if ender < 0 { + for { + pos := bytes.Index(line[ender:], parser.end) + if pos < 0 { return nil } ender += pos - pos += len(parser.end) + + // Now we want to check the character at the end of our parser section + // that is ender + len(parser.end) + pos = ender + len(parser.end) if len(line) <= pos { break } if !isAlphanumeric(line[pos]) { break } + // move the pointer onwards + ender += len(parser.end) } block.Advance(opener) From 4885ae3791927867ef29a6f705eaaf39c4aa4c79 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 4 Oct 2022 23:10:12 +0100 Subject: [PATCH 3/6] fix issue with renderconfig Signed-off-by: Andrew Thornton --- modules/markup/markdown/renderconfig.go | 96 +++++++++++--------- modules/markup/markdown/renderconfig_test.go | 22 +++-- 2 files changed, 64 insertions(+), 54 deletions(-) diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go index 003579115fc0c..9cba0c61a2a18 100644 --- a/modules/markup/markdown/renderconfig.go +++ b/modules/markup/markdown/renderconfig.go @@ -5,10 +5,9 @@ package markdown import ( + "fmt" "strings" - "code.gitea.io/gitea/modules/log" - "github.com/yuin/goldmark/ast" "gopkg.in/yaml.v3" ) @@ -33,17 +32,13 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { } rc.yamlNode = value - type basicRenderConfig struct { - Gitea *yaml.Node `yaml:"gitea"` - TOC bool `yaml:"include_toc"` - Lang string `yaml:"lang"` + type commonRenderConfig struct { + TOC bool `yaml:"include_toc"` + Lang string `yaml:"lang"` } - - var basic basicRenderConfig - - err := value.Decode(&basic) - if err != nil { - return err + var basic commonRenderConfig + if err := value.Decode(&basic); err != nil { + return fmt.Errorf("unable to decode into basicRenderConfig %w", err) } if basic.Lang != "" { @@ -51,54 +46,67 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { } rc.TOC = basic.TOC - if basic.Gitea == nil { - return nil + + type controlStringRenderConfig struct { + Gitea string `yaml:"gitea"` } - var control *string - if err := basic.Gitea.Decode(&control); err == nil && control != nil { - log.Info("control %v", control) - switch strings.TrimSpace(strings.ToLower(*control)) { - case "none": - rc.Meta = "none" - case "table": - rc.Meta = "table" - default: // "details" - rc.Meta = "details" + var stringBasic controlStringRenderConfig + + if err := value.Decode(&stringBasic); err == nil { + if stringBasic.Gitea != "" { + switch strings.TrimSpace(strings.ToLower(stringBasic.Gitea)) { + case "none": + rc.Meta = "none" + case "table": + rc.Meta = "table" + default: // "details" + rc.Meta = "details" + } } return nil } type giteaControl struct { - Meta string `yaml:"meta"` - Icon string `yaml:"details_icon"` - TOC *yaml.Node `yaml:"include_toc"` - Lang string `yaml:"lang"` + Meta *string `yaml:"meta"` + Icon *string `yaml:"details_icon"` + TOC *bool `yaml:"include_toc"` + Lang *string `yaml:"lang"` + } + + type complexGiteaConfig struct { + Gitea *giteaControl `yaml:"gitea"` + } + var complex complexGiteaConfig + if err := value.Decode(&complex); err != nil { + return fmt.Errorf("unable to decode into complexRenderConfig %w", err) } - var controlStruct *giteaControl - if err := basic.Gitea.Decode(controlStruct); err != nil || controlStruct == nil { - return err + if complex.Gitea == nil { + return nil } - switch strings.TrimSpace(strings.ToLower(controlStruct.Meta)) { - case "none": - rc.Meta = "none" - case "table": - rc.Meta = "table" - default: // "details" - rc.Meta = "details" + if complex.Gitea.Meta != nil { + switch strings.TrimSpace(strings.ToLower(*complex.Gitea.Meta)) { + case "none": + rc.Meta = "none" + case "table": + rc.Meta = "table" + default: // "details" + rc.Meta = "details" + } } - rc.Icon = strings.TrimSpace(strings.ToLower(controlStruct.Icon)) + if complex.Gitea.Icon != nil { + rc.Icon = strings.TrimSpace(strings.ToLower(*complex.Gitea.Icon)) + } - if controlStruct.Lang != "" { - rc.Lang = controlStruct.Lang + if complex.Gitea.Lang != nil && *complex.Gitea.Lang != "" { + rc.Lang = *complex.Gitea.Lang } - var toc bool - if err := controlStruct.TOC.Decode(&toc); err == nil { - rc.TOC = toc + if complex.Gitea.TOC != nil { + rc.TOC = *complex.Gitea.TOC } return nil diff --git a/modules/markup/markdown/renderconfig_test.go b/modules/markup/markdown/renderconfig_test.go index 1027035cda5a6..9d02770fafd34 100644 --- a/modules/markup/markdown/renderconfig_test.go +++ b/modules/markup/markdown/renderconfig_test.go @@ -5,8 +5,10 @@ package markdown import ( + "strings" "testing" + "code.gitea.io/gitea/modules/util" "gopkg.in/yaml.v3" ) @@ -80,20 +82,20 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { Icon: "table", TOC: true, Lang: "testlang", - }, ` - include_toc: true - lang: testlang -`, + }, util.Dedent(` + include_toc: true + lang: testlang + `), }, { "complexlang", &RenderConfig{ Meta: "table", Icon: "table", Lang: "testlang", - }, ` - gitea: - lang: testlang -`, + }, util.Dedent(` + gitea: + lang: testlang + `), }, { "complexlang2", &RenderConfig{ @@ -140,8 +142,8 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { Icon: "table", Lang: "", } - if err := yaml.Unmarshal([]byte(tt.args), got); err != nil { - t.Errorf("RenderConfig.UnmarshalYAML() error = %v", err) + if err := yaml.Unmarshal([]byte(strings.ReplaceAll(tt.args, "\t", " ")), got); err != nil { + t.Errorf("RenderConfig.UnmarshalYAML() error = %v\n%q", err, tt.args) return } From cb638d7e4b56173ca53c97badd7c2b60e75b6f52 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Wed, 5 Oct 2022 10:23:14 +0800 Subject: [PATCH 4/6] fix more tests --- modules/markup/html_test.go | 4 +++- modules/markup/markdown/meta.go | 4 +++- modules/markup/markdown/meta_test.go | 4 ++-- modules/markup/markdown/renderconfig_test.go | 11 +++++------ 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 370bc218225a5..e57187a67782b 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -7,6 +7,7 @@ package markup_test import ( "context" "io" + "os" "strings" "testing" @@ -32,6 +33,7 @@ func TestMain(m *testing.M) { if err := git.InitSimple(context.Background()); err != nil { log.Fatal("git init failed, err: %v", err) } + os.Exit(m.Run()) } func TestRender_Commits(t *testing.T) { @@ -336,7 +338,7 @@ func TestRender_emoji(t *testing.T) { `

Some text with 😄😄 2 emoji next to each other

`) test( "😎ðŸĪŠðŸ”ðŸĪ‘❓", - `

😎ðŸĪŠðŸ”ðŸĪ‘❓

`) + `

😎ðŸĪŠðŸ”ðŸĪ‘❓

`) // should match nothing test( diff --git a/modules/markup/markdown/meta.go b/modules/markup/markdown/meta.go index b08121e8680c2..45d79d537a737 100644 --- a/modules/markup/markdown/meta.go +++ b/modules/markup/markdown/meta.go @@ -88,7 +88,9 @@ func ExtractMetadataBytes(contents []byte, out interface{}) ([]byte, error) { line := contents[start:end] if isYAMLSeparator(line) { front = contents[frontMatterStart:start] - body = contents[end+1:] + if end+1 < len(contents) { + body = contents[end+1:] + } break } } diff --git a/modules/markup/markdown/meta_test.go b/modules/markup/markdown/meta_test.go index 9332b35b42782..720d0066f4f0b 100644 --- a/modules/markup/markdown/meta_test.go +++ b/modules/markup/markdown/meta_test.go @@ -61,7 +61,7 @@ func TestExtractMetadataBytes(t *testing.T) { var meta structs.IssueTemplate body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s\n%s", sepTest, frontTest, sepTest, bodyTest)), &meta) assert.NoError(t, err) - assert.Equal(t, bodyTest, body) + assert.Equal(t, bodyTest, string(body)) assert.Equal(t, metaTest, meta) assert.True(t, validateMetadata(meta)) }) @@ -82,7 +82,7 @@ func TestExtractMetadataBytes(t *testing.T) { var meta structs.IssueTemplate body, err := ExtractMetadataBytes([]byte(fmt.Sprintf("%s\n%s\n%s", sepTest, frontTest, sepTest)), &meta) assert.NoError(t, err) - assert.Equal(t, "", body) + assert.Equal(t, "", string(body)) assert.Equal(t, metaTest, meta) assert.True(t, validateMetadata(meta)) }) diff --git a/modules/markup/markdown/renderconfig_test.go b/modules/markup/markdown/renderconfig_test.go index 9d02770fafd34..672edbf46d321 100644 --- a/modules/markup/markdown/renderconfig_test.go +++ b/modules/markup/markdown/renderconfig_test.go @@ -8,7 +8,6 @@ import ( "strings" "testing" - "code.gitea.io/gitea/modules/util" "gopkg.in/yaml.v3" ) @@ -82,20 +81,20 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { Icon: "table", TOC: true, Lang: "testlang", - }, util.Dedent(` + }, ` include_toc: true lang: testlang - `), + `, }, { "complexlang", &RenderConfig{ Meta: "table", Icon: "table", Lang: "testlang", - }, util.Dedent(` + }, ` gitea: lang: testlang - `), + `, }, { "complexlang2", &RenderConfig{ @@ -142,7 +141,7 @@ func TestRenderConfig_UnmarshalYAML(t *testing.T) { Icon: "table", Lang: "", } - if err := yaml.Unmarshal([]byte(strings.ReplaceAll(tt.args, "\t", " ")), got); err != nil { + if err := yaml.Unmarshal([]byte(strings.ReplaceAll(tt.args, "\t", " ")), got); err != nil { t.Errorf("RenderConfig.UnmarshalYAML() error = %v\n%q", err, tt.args) return } From 967dec703698b511411ccd204530e0fdab59aff5 Mon Sep 17 00:00:00 2001 From: zeripath Date: Wed, 5 Oct 2022 05:09:40 +0100 Subject: [PATCH 5/6] Update modules/markup/markdown/math/inline_parser.go --- modules/markup/markdown/math/inline_parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/markup/markdown/math/inline_parser.go b/modules/markup/markdown/math/inline_parser.go index 31f3125390b6f..8dc88eb858f3a 100644 --- a/modules/markup/markdown/math/inline_parser.go +++ b/modules/markup/markdown/math/inline_parser.go @@ -37,7 +37,7 @@ func NewInlineBracketParser() parser.InlineParser { return defaultInlineBracketParser } -// Trigger returns nil +// Trigger triggers this parser on $ or \ func (parser *inlineParser) Trigger() []byte { return parser.start[0:1] } From a97d736114eb2c8a43bec857420d4bdfeb0163e6 Mon Sep 17 00:00:00 2001 From: zeripath Date: Wed, 5 Oct 2022 15:24:33 +0100 Subject: [PATCH 6/6] Update modules/markup/markdown/renderconfig.go Co-authored-by: delvh --- modules/markup/markdown/renderconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/markup/markdown/renderconfig.go b/modules/markup/markdown/renderconfig.go index 9cba0c61a2a18..1ba75dbb68ec3 100644 --- a/modules/markup/markdown/renderconfig.go +++ b/modules/markup/markdown/renderconfig.go @@ -38,7 +38,7 @@ func (rc *RenderConfig) UnmarshalYAML(value *yaml.Node) error { } var basic commonRenderConfig if err := value.Decode(&basic); err != nil { - return fmt.Errorf("unable to decode into basicRenderConfig %w", err) + return fmt.Errorf("unable to decode into commonRenderConfig %w", err) } if basic.Lang != "" {