Skip to content

Commit 09c667e

Browse files
wolfogrelafriksdelvhlunny
authored
Fix sitemap (#22272) (#22320)
Backport #22272. Fix #22270. Related to #18407. The old code treated both sitemap and sitemap index as the format like: ```xml ... <url> <loc>http://localhost:3000/explore/users/sitemap-1.xml</loc> </url> ... ``` Actually, it's incorrect for sitemap index, it should be: ```xml ... <sitemap> <loc>http://localhost:3000/explore/users/sitemap-1.xml</loc> </sitemap> ... ``` See https://www.sitemaps.org/protocol.html Co-authored-by: Lauris BH <[email protected]> Co-authored-by: delvh <[email protected]> Co-authored-by: Lunny Xiao <[email protected]>
1 parent 791f290 commit 09c667e

File tree

2 files changed

+170
-66
lines changed

2 files changed

+170
-66
lines changed

‎modules/sitemap/sitemap.go

+29-15
Original file line numberDiff line numberDiff line change
@@ -12,48 +12,62 @@ import (
1212
"time"
1313
)
1414

15-
// sitemapFileLimit contains the maximum size of a sitemap file
16-
const sitemapFileLimit = 50 * 1024 * 1024
15+
const (
16+
sitemapFileLimit = 50 * 1024 * 1024 // the maximum size of a sitemap file
17+
urlsLimit = 50000
1718

18-
// Url represents a single sitemap entry
19+
schemaURL = "http://www.sitemaps.org/schemas/sitemap/0.9"
20+
urlsetName = "urlset"
21+
sitemapindexName = "sitemapindex"
22+
)
23+
24+
// URL represents a single sitemap entry
1925
type URL struct {
2026
URL string `xml:"loc"`
2127
LastMod *time.Time `xml:"lastmod,omitempty"`
2228
}
2329

24-
// SitemapUrl represents a sitemap
30+
// Sitemap represents a sitemap
2531
type Sitemap struct {
2632
XMLName xml.Name
2733
Namespace string `xml:"xmlns,attr"`
2834

29-
URLs []URL `xml:"url"`
35+
URLs []URL `xml:"url"`
36+
Sitemaps []URL `xml:"sitemap"`
3037
}
3138

3239
// NewSitemap creates a sitemap
3340
func NewSitemap() *Sitemap {
3441
return &Sitemap{
35-
XMLName: xml.Name{Local: "urlset"},
36-
Namespace: "http://www.sitemaps.org/schemas/sitemap/0.9",
42+
XMLName: xml.Name{Local: urlsetName},
43+
Namespace: schemaURL,
3744
}
3845
}
3946

40-
// NewSitemap creates a sitemap index.
47+
// NewSitemapIndex creates a sitemap index.
4148
func NewSitemapIndex() *Sitemap {
4249
return &Sitemap{
43-
XMLName: xml.Name{Local: "sitemapindex"},
44-
Namespace: "http://www.sitemaps.org/schemas/sitemap/0.9",
50+
XMLName: xml.Name{Local: sitemapindexName},
51+
Namespace: schemaURL,
4552
}
4653
}
4754

4855
// Add adds a URL to the sitemap
4956
func (s *Sitemap) Add(u URL) {
50-
s.URLs = append(s.URLs, u)
57+
if s.XMLName.Local == sitemapindexName {
58+
s.Sitemaps = append(s.Sitemaps, u)
59+
} else {
60+
s.URLs = append(s.URLs, u)
61+
}
5162
}
5263

53-
// Write writes the sitemap to a response
64+
// WriteTo writes the sitemap to a response
5465
func (s *Sitemap) WriteTo(w io.Writer) (int64, error) {
55-
if len(s.URLs) > 50000 {
56-
return 0, fmt.Errorf("The sitemap contains too many URLs: %d", len(s.URLs))
66+
if l := len(s.URLs); l > urlsLimit {
67+
return 0, fmt.Errorf("The sitemap contains %d URLs, but only %d are allowed", l, urlsLimit)
68+
}
69+
if l := len(s.Sitemaps); l > urlsLimit {
70+
return 0, fmt.Errorf("The sitemap contains %d sub-sitemaps, but only %d are allowed", l, urlsLimit)
5771
}
5872
buf := bytes.NewBufferString(xml.Header)
5973
if err := xml.NewEncoder(buf).Encode(s); err != nil {
@@ -63,7 +77,7 @@ func (s *Sitemap) WriteTo(w io.Writer) (int64, error) {
6377
return 0, err
6478
}
6579
if buf.Len() > sitemapFileLimit {
66-
return 0, fmt.Errorf("The sitemap is too big: %d", buf.Len())
80+
return 0, fmt.Errorf("The sitemap has %d bytes, but only %d are allowed", buf.Len(), sitemapFileLimit)
6781
}
6882
return buf.WriteTo(w)
6983
}

‎modules/sitemap/sitemap_test.go

+141-51
Original file line numberDiff line numberDiff line change
@@ -7,71 +7,161 @@ package sitemap
77
import (
88
"bytes"
99
"encoding/xml"
10-
"fmt"
1110
"strings"
1211
"testing"
1312
"time"
1413

1514
"github.com/stretchr/testify/assert"
1615
)
1716

18-
func TestOk(t *testing.T) {
19-
testReal := func(s *Sitemap, name string, urls []URL, expected string) {
20-
for _, url := range urls {
21-
s.Add(url)
22-
}
23-
buf := &bytes.Buffer{}
24-
_, err := s.WriteTo(buf)
25-
assert.NoError(t, nil, err)
26-
assert.Equal(t, xml.Header+"<"+name+" xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">"+expected+"</"+name+">\n", buf.String())
17+
func TestNewSitemap(t *testing.T) {
18+
ts := time.Unix(1651322008, 0).UTC()
19+
20+
tests := []struct {
21+
name string
22+
urls []URL
23+
want string
24+
wantErr string
25+
}{
26+
{
27+
name: "empty",
28+
urls: []URL{},
29+
want: xml.Header + `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
30+
"" +
31+
"</urlset>\n",
32+
},
33+
{
34+
name: "regular",
35+
urls: []URL{
36+
{URL: "https://gitea.io/test1", LastMod: &ts},
37+
},
38+
want: xml.Header + `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
39+
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>" +
40+
"</urlset>\n",
41+
},
42+
{
43+
name: "without lastmod",
44+
urls: []URL{
45+
{URL: "https://gitea.io/test1"},
46+
},
47+
want: xml.Header + `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
48+
"<url><loc>https://gitea.io/test1</loc></url>" +
49+
"</urlset>\n",
50+
},
51+
{
52+
name: "multiple",
53+
urls: []URL{
54+
{URL: "https://gitea.io/test1", LastMod: &ts},
55+
{URL: "https://gitea.io/test2", LastMod: nil},
56+
},
57+
want: xml.Header + `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
58+
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>" +
59+
"<url><loc>https://gitea.io/test2</loc></url>" +
60+
"</urlset>\n",
61+
},
62+
{
63+
name: "too many urls",
64+
urls: make([]URL, 50001),
65+
wantErr: "The sitemap contains 50001 URLs, but only 50000 are allowed",
66+
},
67+
{
68+
name: "too big file",
69+
urls: []URL{
70+
{URL: strings.Repeat("b", 50*1024*1024+1)},
71+
},
72+
wantErr: "The sitemap has 52428932 bytes, but only 52428800 are allowed",
73+
},
2774
}
28-
test := func(urls []URL, expected string) {
29-
testReal(NewSitemap(), "urlset", urls, expected)
30-
testReal(NewSitemapIndex(), "sitemapindex", urls, expected)
75+
for _, tt := range tests {
76+
t.Run(tt.name, func(t *testing.T) {
77+
s := NewSitemap()
78+
for _, url := range tt.urls {
79+
s.Add(url)
80+
}
81+
buf := &bytes.Buffer{}
82+
_, err := s.WriteTo(buf)
83+
if tt.wantErr != "" {
84+
assert.EqualError(t, err, tt.wantErr)
85+
} else {
86+
assert.NoError(t, err)
87+
assert.Equalf(t, tt.want, buf.String(), "NewSitemap()")
88+
}
89+
})
3190
}
91+
}
3292

93+
func TestNewSitemapIndex(t *testing.T) {
3394
ts := time.Unix(1651322008, 0).UTC()
3495

35-
test(
36-
[]URL{},
37-
"",
38-
)
39-
test(
40-
[]URL{
41-
{URL: "https://gitea.io/test1", LastMod: &ts},
96+
tests := []struct {
97+
name string
98+
urls []URL
99+
want string
100+
wantErr string
101+
}{
102+
{
103+
name: "empty",
104+
urls: []URL{},
105+
want: xml.Header + `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
106+
"" +
107+
"</sitemapindex>\n",
42108
},
43-
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>",
44-
)
45-
test(
46-
[]URL{
47-
{URL: "https://gitea.io/test2", LastMod: nil},
109+
{
110+
name: "regular",
111+
urls: []URL{
112+
{URL: "https://gitea.io/test1", LastMod: &ts},
113+
},
114+
want: xml.Header + `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
115+
"<sitemap><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></sitemap>" +
116+
"</sitemapindex>\n",
48117
},
49-
"<url><loc>https://gitea.io/test2</loc></url>",
50-
)
51-
test(
52-
[]URL{
53-
{URL: "https://gitea.io/test1", LastMod: &ts},
54-
{URL: "https://gitea.io/test2", LastMod: nil},
118+
{
119+
name: "without lastmod",
120+
urls: []URL{
121+
{URL: "https://gitea.io/test1"},
122+
},
123+
want: xml.Header + `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
124+
"<sitemap><loc>https://gitea.io/test1</loc></sitemap>" +
125+
"</sitemapindex>\n",
126+
},
127+
{
128+
name: "multiple",
129+
urls: []URL{
130+
{URL: "https://gitea.io/test1", LastMod: &ts},
131+
{URL: "https://gitea.io/test2", LastMod: nil},
132+
},
133+
want: xml.Header + `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` +
134+
"<sitemap><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></sitemap>" +
135+
"<sitemap><loc>https://gitea.io/test2</loc></sitemap>" +
136+
"</sitemapindex>\n",
137+
},
138+
{
139+
name: "too many sitemaps",
140+
urls: make([]URL, 50001),
141+
wantErr: "The sitemap contains 50001 sub-sitemaps, but only 50000 are allowed",
142+
},
143+
{
144+
name: "too big file",
145+
urls: []URL{
146+
{URL: strings.Repeat("b", 50*1024*1024+1)},
147+
},
148+
wantErr: "The sitemap has 52428952 bytes, but only 52428800 are allowed",
55149
},
56-
"<url><loc>https://gitea.io/test1</loc><lastmod>2022-04-30T12:33:28Z</lastmod></url>"+
57-
"<url><loc>https://gitea.io/test2</loc></url>",
58-
)
59-
}
60-
61-
func TestTooManyURLs(t *testing.T) {
62-
s := NewSitemap()
63-
for i := 0; i < 50001; i++ {
64-
s.Add(URL{URL: fmt.Sprintf("https://gitea.io/test%d", i)})
65150
}
66-
buf := &bytes.Buffer{}
67-
_, err := s.WriteTo(buf)
68-
assert.EqualError(t, err, "The sitemap contains too many URLs: 50001")
69-
}
70-
71-
func TestSitemapTooBig(t *testing.T) {
72-
s := NewSitemap()
73-
s.Add(URL{URL: strings.Repeat("b", sitemapFileLimit)})
74-
buf := &bytes.Buffer{}
75-
_, err := s.WriteTo(buf)
76-
assert.EqualError(t, err, "The sitemap is too big: 52428931")
151+
for _, tt := range tests {
152+
t.Run(tt.name, func(t *testing.T) {
153+
s := NewSitemapIndex()
154+
for _, url := range tt.urls {
155+
s.Add(url)
156+
}
157+
buf := &bytes.Buffer{}
158+
_, err := s.WriteTo(buf)
159+
if tt.wantErr != "" {
160+
assert.EqualError(t, err, tt.wantErr)
161+
} else {
162+
assert.NoError(t, err)
163+
assert.Equalf(t, tt.want, buf.String(), "NewSitemapIndex()")
164+
}
165+
})
166+
}
77167
}

0 commit comments

Comments
 (0)