Skip to content

Commit acb9d65

Browse files
daveshanleydaveshanley
committed
addresses #457 fully.
`—junit-fail-on-warn` will wrap errors AND warnings in `<failure>` otherwise just errors, everything else is marked as a pass.
1 parent 027e17b commit acb9d65

File tree

3 files changed

+223
-21
lines changed

3 files changed

+223
-21
lines changed

‎cmd/vacuum_report.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ vacuum report --globbed-files "api/**/*.json" -c`,
5555
noStyleFlag, _ := cmd.Flags().GetBool("no-style")
5656
baseFlag, _ := cmd.Flags().GetString("base")
5757
junitFlag, _ := cmd.Flags().GetBool("junit")
58+
junitFailOnWarn, _ := cmd.Flags().GetBool("junit-fail-on-warn")
5859
skipCheckFlag, _ := cmd.Flags().GetBool("skip-check")
5960
timeoutFlag, _ := cmd.Flags().GetInt("timeout")
6061
lookupTimeoutFlag, _ := cmd.Flags().GetInt("lookup-timeout")
@@ -351,7 +352,8 @@ vacuum report --globbed-files "api/**/*.json" -c`,
351352

352353
// if we want jUnit output, then build the report and be done with it.
353354
if junitFlag {
354-
junitXML := vacuum_report.BuildJUnitReport(resultSet, start, []string{specFile})
355+
junitConfig := vacuum_report.JUnitConfig{FailOnWarn: junitFailOnWarn}
356+
junitXML := vacuum_report.BuildJUnitReportWithConfig(resultSet, start, []string{specFile}, junitConfig)
355357
if stdOut {
356358
fmt.Print(string(junitXML))
357359
return nil
@@ -485,6 +487,7 @@ vacuum report --globbed-files "api/**/*.json" -c`,
485487
cmd.Flags().BoolP("stdin", "i", false, "Use stdin as input, instead of a file")
486488
cmd.Flags().BoolP("stdout", "o", false, "Use stdout as output, instead of a file")
487489
cmd.Flags().BoolP("junit", "j", false, "Generate report in JUnit format (cannot be compressed)")
490+
cmd.Flags().Bool("junit-fail-on-warn", false, "Treat warnings as failures in JUnit report (default: only errors are failures)")
488491
cmd.Flags().BoolP("compress", "c", false, "Compress results using gzip")
489492
cmd.Flags().BoolP("no-pretty", "n", false, "Render JSON with no formatting")
490493
cmd.Flags().BoolP("no-style", "q", false, "Disable styling and color output, just plain text (useful for CI/CD)")

‎vacuum-report/junit.go‎

Lines changed: 67 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,37 @@ type Failure struct {
5555
Contents string `xml:",innerxml"`
5656
}
5757

58+
// JUnitConfig controls how JUnit reports are generated
59+
type JUnitConfig struct {
60+
// FailOnWarn treats warnings as failures (default: false, only errors are failures)
61+
FailOnWarn bool
62+
}
63+
64+
// severityToUppercase converts severity to uppercase without allocation for known values
65+
func severityToUppercase(severity string) string {
66+
switch severity {
67+
case model.SeverityError:
68+
return "ERROR"
69+
case model.SeverityWarn:
70+
return "WARN"
71+
case model.SeverityInfo:
72+
return "INFO"
73+
case model.SeverityHint:
74+
return "HINT"
75+
default:
76+
return strings.ToUpper(severity)
77+
}
78+
}
79+
80+
// BuildJUnitReport generates a JUnit XML report from linting results.
81+
// By default, only errors create failure elements. Use config.FailOnWarn to include warnings.
82+
// Info and hint severities always create passing test cases.
5883
func BuildJUnitReport(resultSet *model.RuleResultSet, t time.Time, args []string) []byte {
84+
return BuildJUnitReportWithConfig(resultSet, t, args, JUnitConfig{FailOnWarn: true})
85+
}
86+
87+
// BuildJUnitReportWithConfig generates a JUnit XML report with configurable failure behavior.
88+
func BuildJUnitReportWithConfig(resultSet *model.RuleResultSet, t time.Time, args []string, config JUnitConfig) []byte {
5989

6090
since := time.Since(t)
6191
var suites []*TestSuite
@@ -74,7 +104,17 @@ func BuildJUnitReport(resultSet *model.RuleResultSet, t time.Time, args []string
74104

75105
gf, gtc := 0, 0 // global failure count, global test cases count.
76106

77-
// try a category print out.
107+
// isFailure determines if a severity level should be treated as a failure
108+
isFailure := func(severity string) bool {
109+
if severity == model.SeverityError {
110+
return true
111+
}
112+
if severity == model.SeverityWarn && config.FailOnWarn {
113+
return true
114+
}
115+
return false
116+
}
117+
78118
for _, val := range cats {
79119
categoryResults := resultSet.GetResultsByRuleCategory(val.Id)
80120

@@ -84,7 +124,9 @@ func BuildJUnitReport(resultSet *model.RuleResultSet, t time.Time, args []string
84124
for _, r := range categoryResults {
85125
var sb bytes.Buffer
86126
_ = parsedTemplate.Execute(&sb, r)
87-
if r.Rule.Severity == model.SeverityError || r.Rule.Severity == model.SeverityWarn {
127+
128+
treatAsFailure := isFailure(r.Rule.Severity)
129+
if treatAsFailure {
88130
f++
89131
gf++
90132
}
@@ -95,14 +137,9 @@ func BuildJUnitReport(resultSet *model.RuleResultSet, t time.Time, args []string
95137
}
96138

97139
tCase := &TestCase{
98-
Line: r.StartNode.Line,
140+
Line: line,
99141
Name: fmt.Sprintf("%s", val.Name),
100142
ClassName: r.Rule.Id,
101-
Failure: &Failure{
102-
Message: r.Message,
103-
Type: strings.ToUpper(r.Rule.Severity),
104-
Contents: sb.String(),
105-
},
106143
Properties: &Properties{
107144
Properties: []*Property{
108145
{
@@ -125,21 +162,32 @@ func BuildJUnitReport(resultSet *model.RuleResultSet, t time.Time, args []string
125162
},
126163
}
127164

165+
// only create failure element for errors (and warnings if configured)
166+
// info and hint severities become passing test cases
167+
if treatAsFailure {
168+
tCase.Failure = &Failure{
169+
Message: r.Message,
170+
Type: severityToUppercase(r.Rule.Severity),
171+
Contents: sb.String(),
172+
}
173+
}
174+
175+
// determine file path once and apply to all locations
176+
filePath := ""
128177
if r.Origin != nil && r.Origin.AbsoluteLocation != "" {
129-
tCase.File = r.Origin.AbsoluteLocation
178+
filePath = r.Origin.AbsoluteLocation
179+
} else if len(args) > 0 {
180+
filePath = args[0]
181+
}
182+
183+
if filePath != "" {
184+
tCase.File = filePath
130185
tCase.Properties.Properties = append(tCase.Properties.Properties, &Property{
131186
Name: "file",
132-
Value: r.Origin.AbsoluteLocation,
187+
Value: filePath,
133188
})
134-
tCase.Failure.File = r.Origin.AbsoluteLocation
135-
} else {
136-
if len(args) > 0 {
137-
tCase.File = args[0]
138-
tCase.Properties.Properties = append(tCase.Properties.Properties, &Property{
139-
Name: "file",
140-
Value: args[0],
141-
})
142-
tCase.Failure.File = args[0]
189+
if tCase.Failure != nil {
190+
tCase.Failure.File = filePath
143191
}
144192
}
145193

‎vacuum-report/junit_test.go‎

Lines changed: 152 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,13 @@
44
package vacuum_report
55

66
import (
7-
"github.com/stretchr/testify/assert"
7+
"strings"
88
"testing"
99
"time"
10+
11+
"github.com/daveshanley/vacuum/model"
12+
"github.com/stretchr/testify/assert"
13+
"go.yaml.in/yaml/v4"
1014
)
1115

1216
func TestBuildJUnitReport(t *testing.T) {
@@ -18,3 +22,150 @@ func TestBuildJUnitReport(t *testing.T) {
1822
data := BuildJUnitReport(j.ResultSet, f, []string{"test", "args"})
1923
assert.GreaterOrEqual(t, len(data), 407)
2024
}
25+
26+
func TestBuildJUnitReportWithConfig_DefaultOnlyErrorsAreFailures(t *testing.T) {
27+
// Create results with different severities
28+
results := []model.RuleFunctionResult{
29+
{
30+
Rule: &model.Rule{Id: "error-rule", Severity: model.SeverityError, RuleCategory: model.RuleCategories[model.CategoryInfo]},
31+
Message: "This is an error",
32+
StartNode: &yaml.Node{Line: 1, Column: 1},
33+
EndNode: &yaml.Node{Line: 1, Column: 10},
34+
},
35+
{
36+
Rule: &model.Rule{Id: "warn-rule", Severity: model.SeverityWarn, RuleCategory: model.RuleCategories[model.CategoryInfo]},
37+
Message: "This is a warning",
38+
StartNode: &yaml.Node{Line: 2, Column: 1},
39+
EndNode: &yaml.Node{Line: 2, Column: 10},
40+
},
41+
{
42+
Rule: &model.Rule{Id: "info-rule", Severity: model.SeverityInfo, RuleCategory: model.RuleCategories[model.CategoryInfo]},
43+
Message: "This is info",
44+
StartNode: &yaml.Node{Line: 3, Column: 1},
45+
EndNode: &yaml.Node{Line: 3, Column: 10},
46+
},
47+
{
48+
Rule: &model.Rule{Id: "hint-rule", Severity: model.SeverityHint, RuleCategory: model.RuleCategories[model.CategoryInfo]},
49+
Message: "This is a hint",
50+
StartNode: &yaml.Node{Line: 4, Column: 1},
51+
EndNode: &yaml.Node{Line: 4, Column: 10},
52+
},
53+
}
54+
55+
resultSet := model.NewRuleResultSet(results)
56+
config := JUnitConfig{FailOnWarn: false} // Default: only errors are failures
57+
58+
data := BuildJUnitReportWithConfig(resultSet, time.Now(), []string{"test.yaml"}, config)
59+
output := string(data)
60+
61+
// Should have 4 tests but only 1 failure (the error)
62+
assert.Contains(t, output, `tests="4"`)
63+
assert.Contains(t, output, `failures="1"`)
64+
65+
// Error should have failure element
66+
assert.Contains(t, output, `<failure message="This is an error" type="ERROR"`)
67+
68+
// Warning, info, hint should NOT have failure elements
69+
assert.NotContains(t, output, `<failure message="This is a warning"`)
70+
assert.NotContains(t, output, `<failure message="This is info"`)
71+
assert.NotContains(t, output, `<failure message="This is a hint"`)
72+
73+
// But all should still have testcase elements with properties
74+
assert.Contains(t, output, `classname="warn-rule"`)
75+
assert.Contains(t, output, `classname="info-rule"`)
76+
assert.Contains(t, output, `classname="hint-rule"`)
77+
}
78+
79+
func TestBuildJUnitReportWithConfig_FailOnWarn(t *testing.T) {
80+
// Create results with different severities
81+
results := []model.RuleFunctionResult{
82+
{
83+
Rule: &model.Rule{Id: "error-rule", Severity: model.SeverityError, RuleCategory: model.RuleCategories[model.CategoryInfo]},
84+
Message: "This is an error",
85+
StartNode: &yaml.Node{Line: 1, Column: 1},
86+
EndNode: &yaml.Node{Line: 1, Column: 10},
87+
},
88+
{
89+
Rule: &model.Rule{Id: "warn-rule", Severity: model.SeverityWarn, RuleCategory: model.RuleCategories[model.CategoryInfo]},
90+
Message: "This is a warning",
91+
StartNode: &yaml.Node{Line: 2, Column: 1},
92+
EndNode: &yaml.Node{Line: 2, Column: 10},
93+
},
94+
{
95+
Rule: &model.Rule{Id: "info-rule", Severity: model.SeverityInfo, RuleCategory: model.RuleCategories[model.CategoryInfo]},
96+
Message: "This is info",
97+
StartNode: &yaml.Node{Line: 3, Column: 1},
98+
EndNode: &yaml.Node{Line: 3, Column: 10},
99+
},
100+
}
101+
102+
resultSet := model.NewRuleResultSet(results)
103+
config := JUnitConfig{FailOnWarn: true} // Warnings are also failures
104+
105+
data := BuildJUnitReportWithConfig(resultSet, time.Now(), []string{"test.yaml"}, config)
106+
output := string(data)
107+
108+
// Should have 3 tests and 2 failures (error + warning)
109+
assert.Contains(t, output, `tests="3"`)
110+
assert.Contains(t, output, `failures="2"`)
111+
112+
// Error and warning should have failure elements
113+
assert.Contains(t, output, `<failure message="This is an error" type="ERROR"`)
114+
assert.Contains(t, output, `<failure message="This is a warning" type="WARN"`)
115+
116+
// Info should NOT have failure element
117+
assert.NotContains(t, output, `<failure message="This is info"`)
118+
}
119+
120+
func TestBuildJUnitReportWithConfig_InfoAndHintNeverFailures(t *testing.T) {
121+
// Even with FailOnWarn, info and hint should never be failures
122+
results := []model.RuleFunctionResult{
123+
{
124+
Rule: &model.Rule{Id: "info-rule", Severity: model.SeverityInfo, RuleCategory: model.RuleCategories[model.CategoryInfo]},
125+
Message: "This is info",
126+
StartNode: &yaml.Node{Line: 1, Column: 1},
127+
EndNode: &yaml.Node{Line: 1, Column: 10},
128+
},
129+
{
130+
Rule: &model.Rule{Id: "hint-rule", Severity: model.SeverityHint, RuleCategory: model.RuleCategories[model.CategoryInfo]},
131+
Message: "This is a hint",
132+
StartNode: &yaml.Node{Line: 2, Column: 1},
133+
EndNode: &yaml.Node{Line: 2, Column: 10},
134+
},
135+
}
136+
137+
resultSet := model.NewRuleResultSet(results)
138+
config := JUnitConfig{FailOnWarn: true} // Even with this on, info/hint should not be failures
139+
140+
data := BuildJUnitReportWithConfig(resultSet, time.Now(), []string{"test.yaml"}, config)
141+
output := string(data)
142+
143+
// Should have 2 tests and 0 failures
144+
assert.Contains(t, output, `tests="2"`)
145+
assert.Contains(t, output, `failures="0"`)
146+
147+
// Neither should have failure elements
148+
failureCount := strings.Count(output, "<failure")
149+
assert.Equal(t, 0, failureCount, "Info and hint should never create failure elements")
150+
}
151+
152+
func TestBuildJUnitReport_BackwardsCompatibility(t *testing.T) {
153+
// The original BuildJUnitReport function should maintain backwards compatibility
154+
// by treating warnings as failures (FailOnWarn: true)
155+
results := []model.RuleFunctionResult{
156+
{
157+
Rule: &model.Rule{Id: "warn-rule", Severity: model.SeverityWarn, RuleCategory: model.RuleCategories[model.CategoryInfo]},
158+
Message: "This is a warning",
159+
StartNode: &yaml.Node{Line: 1, Column: 1},
160+
EndNode: &yaml.Node{Line: 1, Column: 10},
161+
},
162+
}
163+
164+
resultSet := model.NewRuleResultSet(results)
165+
data := BuildJUnitReport(resultSet, time.Now(), []string{"test.yaml"})
166+
output := string(data)
167+
168+
// Original function should still treat warnings as failures for backwards compatibility
169+
assert.Contains(t, output, `failures="1"`)
170+
assert.Contains(t, output, `<failure message="This is a warning" type="WARN"`)
171+
}

0 commit comments

Comments
 (0)