Use bufio.Scanner to find line numbers in templates #5

Author: vorlif

.gitignore

@@ -31,3 +31,4 @@ _testmain.go
 *.exe
 *.test
 *.prof
+/xspreak

testdata/tmpl/five.tmpl

@@ -0,0 +1,9 @@
+<article>
+    <h2 class="font-display text-3xl tracking-tight text-slate-900 dark:text-white">Privacy</h2>
+    <div class="prose prose-slate max-w-none dark:prose-invert dark:text-slate-400 prose-headings:scroll-mt-28 prose-headings:font-display prose-headings:font-normal lg:prose-headings:scroll-mt-[8.5rem] prose-lead:text-slate-500 dark:prose-lead:text-slate-400 prose-a:font-semibold dark:prose-a:text-sky-400 prose-a:no-underline prose-a:shadow-[inset_0_-2px_0_0_var(--tw-prose-background,#fff),inset_0_calc(-1*(var(--tw-prose-underline-size,4px)+2px))_0_0_var(--tw-prose-underline,theme(colors.sky.300))] hover:prose-a:[--tw-prose-underline-size:6px] dark:[--tw-prose-background:theme(colors.slate.900)] dark:prose-a:shadow-[inset_0_calc(-1*var(--tw-prose-underline-size,2px))_0_0_var(--tw-prose-underline,theme(colors.sky.800))] dark:hover:prose-a:[--tw-prose-underline-size:6px] prose-pre:rounded-xl prose-pre:bg-slate-900 prose-pre:shadow-lg dark:prose-pre:bg-slate-800/60 dark:prose-pre:shadow-none dark:prose-pre:ring-1 dark:prose-pre:ring-slate-300/10 dark:prose-hr:border-slate-800">
+        <p>Emails you share with this site will be used to
+        provide you with the data about them you've requested. That data, including the entire content of the email, is made available without any authentication needed at a semi-private URL.</p>
+        <p>We don't currently plan on using any of that data for anything other than ensuring smooth running of the site.</p>
+        <p><a x-link href="contact">Contact us</a> if you have any questions.</p>
+    </div>
+</article>

testdata/tmpl/seven.parens

@@ -0,0 +1,4 @@
+
+
+{{ printf "%s" ( .X "foo" ) }}
+{{ .X "bar" }}

tmpl/parse.go

@@ -1,12 +1,12 @@
 package tmpl
 
 import (
+	"bufio"
 	"bytes"
 	"go/token"
 	"io"
 	"os"
 	"strings"
-	"text/scanner"
 	"text/template/parse"
 )
 
@@ -71,24 +71,14 @@ func extractLineInfos(filename string, src io.Reader) []token.Position {
 	infos := make([]token.Position, 0, 50)
 	infos = append(infos, token.Position{Filename: filename, Offset: 0, Line: 1, Column: 1})
 
-	var s scanner.Scanner
-	s.Filename = filename
-	s.Init(src)
-	s.Whitespace ^= 1<<'\t' | 1<<'\n'
-	s.Mode ^= scanner.SkipComments
-
-	for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
-		switch tok {
-		case '\n':
-			p := s.Pos()
-			tokPos := token.Position{
-				Filename: p.Filename,
-				Offset:   p.Offset,
-				Line:     p.Line,
-				Column:   p.Column,
-			}
-			infos = append(infos, tokPos)
-		}
+	scanner := bufio.NewScanner(src)
+	offset := 0
+	line := 1
+	for scanner.Scan() {
+		offset += len(scanner.Text())
+		infos = append(infos, token.Position{Filename: filename, Offset: offset, Line: line, Column: len(scanner.Text())})
+		offset++
+		line++
 	}
 
 	return infos

tmpl/parse_test.go

@@ -1,6 +1,7 @@
 package tmpl
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -37,3 +38,14 @@ multiline */}}
 	res.ExtractComments()
 	assert.Len(t, res.Comments, 3)
 }
+
+func TestParseHtml(t *testing.T) {
+	text, err := os.ReadFile("../testdata/tmpl/five.tmpl")
+	if err != nil {
+		panic(err)
+	}
+
+	res, err := ParseBytes("test", text)
+	assert.NoError(t, err)
+	require.NotNil(t, res)
+}

tmplextractors/command.go

@@ -43,13 +43,11 @@ func (c *commandExtractor) Run(_ context.Context, extractCtx *extractors.Context
 			}
 
 			for _, cmd := range pipe.Cmds {
-				iss := extractIssue(cmd, extractCtx, template)
-				if iss == nil {
-					continue
+				isses := extractIssues(cmd, extractCtx, template)
+				for _, iss := range isses {
+					iss.Flags = append(iss.Flags, "go-template")
+					issues = append(issues, iss)
 				}
-
-				iss.Flags = append(iss.Flags, "go-template")
-				issues = append(issues, *iss)
 			}
 
 			return
@@ -62,6 +60,29 @@ func (c *commandExtractor) Name() string {
 	return "tmpl_command"
 }
 
+func walkNode(n parse.Node, extractCtx *extractors.Context, template *tmpl.Template, results *[]result.Issue) {
+	switch v := n.(type) {
+	case *parse.CommandNode:
+		iss := extractIssue(v, extractCtx, template)
+		if iss != nil {
+			*results = append(*results, *iss)
+		}
+		for _, node := range v.Args {
+			walkNode(node, extractCtx, template, results)
+		}
+	case *parse.PipeNode:
+		for _, node := range v.Cmds {
+			walkNode(node, extractCtx, template, results)
+		}
+	}
+}
+
+func extractIssues(cmd *parse.CommandNode, extractCtx *extractors.Context, template *tmpl.Template) []result.Issue {
+	ret := []result.Issue{}
+	walkNode(cmd, extractCtx, template, &ret)
+	return ret
+}
+
 func extractIssue(cmd *parse.CommandNode, extractCtx *extractors.Context, template *tmpl.Template) *result.Issue {
 	if cmd == nil {
 		return nil

tmplextractors/command_test.go

@@ -146,3 +146,42 @@ func TestComplex(t *testing.T) {
 	got := collectIssueStrings(issues)
 	assert.ElementsMatch(t, want, got)
 }
+
+func TestParenthesised(t *testing.T) {
+	cfg := config.NewDefault()
+	cfg.SourceDir = testdataDir
+	cfg.ExtractErrors = false
+	cfg.Keywords = []*tmpl.Keyword{
+		{
+			Name:        ".X",
+			SingularPos: 0,
+			PluralPos:   -1,
+			ContextPos:  -1,
+			DomainPos:   -1,
+		},
+	}
+	cfg.TemplatePatterns = []string{
+		testdataTemplates + "/**/seven.parens",
+	}
+
+	require.NoError(t, cfg.Prepare())
+
+	ctx := context.Background()
+	contextLoader := extract.NewContextLoader(cfg)
+
+	extractCtx, err := contextLoader.Load(ctx)
+	require.NoError(t, err)
+
+	runner, err := extract.NewRunner(cfg, extractCtx.Packages)
+	require.NoError(t, err)
+
+	issues, err := runner.Run(ctx, extractCtx, []extractors.Extractor{NewCommandExtractor()})
+	require.NoError(t, err)
+
+	want := []string{
+		"foo",
+		"bar",
+	}
+	got := collectIssueStrings(issues)
+	assert.ElementsMatch(t, want, got)
+}