From c0f6567ef2b135288e8a342d237990e50beb518e Mon Sep 17 00:00:00 2001 From: Valter Pires Date: Mon, 26 Oct 2020 14:41:18 +0400 Subject: [PATCH 1/7] Fix for atom/language-xml#91 and atom/language-xml#96 atom/language-xml#96 already provides all the details about the issues fixed here. https://github.com/atom/language-xml/pull/87#issuecomment-401249754 has the correct code but merge included some extra indent which causes the rule not to work properly. In relation with #91, a `begin` without `end` or `while` was added but this is not valid as `begin` should always have a corresponding `end` or `while`. `match` should be used instead of `begin` --- grammars/xml.cson | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/grammars/xml.cson b/grammars/xml.cson index be5fabd..46ae558 100644 --- a/grammars/xml.cson +++ b/grammars/xml.cson @@ -406,8 +406,8 @@ 'captures': '0': 'name': 'punctuation.definition.comment.xml' - 'end': '--%>' - 'name': 'comment.block.xml' + 'end': '--%>' + 'name': 'comment.block.xml' } { 'begin': '', scopes: ['text.xml', 'comment.block.xml'] + expect(tokens[3]).toEqual value: ' comment ', scopes: ['text.xml', 'comment.block.xml'] + expect(tokens[4]).toEqual value: '-->', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] it "tokenizes empty element meta.tag.no-content.xml", -> {tokens} = grammar.tokenizeLine('') From 6086fe0dfb9b6ca639e6b3aae87b4a733d61159f Mon Sep 17 00:00:00 2001 From: Valter Pires Date: Mon, 2 Nov 2020 18:29:47 +0400 Subject: [PATCH 3/7] Fix for atom/language-xml#91 and atom/language-xml#96 Updated grammar as per latest solution discussed in #91 Added test to document the new behavior when invalid comments are found --- grammars/xml.cson | 9 ++++----- spec/xml-spec.coffee | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/grammars/xml.cson b/grammars/xml.cson index 46ae558..ae8e79a 100644 --- a/grammars/xml.cson +++ b/grammars/xml.cson @@ -414,14 +414,13 @@ 'captures': '0': 'name': 'punctuation.definition.comment.xml' - 'end': '-->' + 'end': '(?' 'name': 'comment.block.xml' 'patterns': [ { - 'match': '--(?!>)' - 'captures': - '0': - 'name': 'invalid.illegal.bad-comments-or-CDATA.xml' + 'begin': '--' + 'end': '(?=-->)' + 'name': 'invalid.illegal.bad-comments-or-CDATA.xml' } ] } diff --git a/spec/xml-spec.coffee b/spec/xml-spec.coffee index 5028368..404fe95 100644 --- a/spec/xml-spec.coffee +++ b/spec/xml-spec.coffee @@ -30,15 +30,32 @@ describe "XML grammar", -> expect(tokens[0]).toEqual value: '') expect(tokens[0]).toEqual value: '', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] + it 'tokenizes after invalid comment only if comment was properly closed', -> + {tokens} = grammar.tokenizeLine('-->') + expect(tokens[0]).toEqual value: '', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] + expect(tokens[7]).toEqual value: '<', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] + expect(tokens[8]).toEqual value: 'n', scopes: ['text.xml', 'meta.tag.no-content.xml', 'entity.name.tag.xml', 'entity.name.tag.localname.xml'] + expect(tokens[9]).toEqual value: '>', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] + expect(tokens[10]).toEqual value: '', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] + it "tokenizes empty element meta.tag.no-content.xml", -> {tokens} = grammar.tokenizeLine('') expect(tokens[0]).toEqual value: '<', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] From f42aa7ef13fbfc45502281101bb36f2864a42d7d Mon Sep 17 00:00:00 2001 From: Valter Pires Date: Mon, 2 Nov 2020 18:34:26 +0400 Subject: [PATCH 4/7] Fix for atom/language-xml#91 and atom/language-xml#96 Minor test correction --- spec/xml-spec.coffee | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/xml-spec.coffee b/spec/xml-spec.coffee index 404fe95..2d9607b 100644 --- a/spec/xml-spec.coffee +++ b/spec/xml-spec.coffee @@ -45,9 +45,9 @@ describe "XML grammar", -> expect(tokens[0]).toEqual value: '', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] expect(tokens[7]).toEqual value: '<', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] expect(tokens[8]).toEqual value: 'n', scopes: ['text.xml', 'meta.tag.no-content.xml', 'entity.name.tag.xml', 'entity.name.tag.localname.xml'] From c293b84d4bf3dbe47c9575ce49452bab308106c4 Mon Sep 17 00:00:00 2001 From: Valter Pires Date: Mon, 2 Nov 2020 18:46:35 +0400 Subject: [PATCH 5/7] Fix for atom/language-xml#91 and atom/language-xml#96 Minor test correction --- spec/xml-spec.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/xml-spec.coffee b/spec/xml-spec.coffee index 2d9607b..994789b 100644 --- a/spec/xml-spec.coffee +++ b/spec/xml-spec.coffee @@ -47,7 +47,7 @@ describe "XML grammar", -> expect(tokens[2]).toEqual value: '--', scopes: ['text.xml', 'comment.block.xml', 'invalid.illegal.bad-comments-or-CDATA.xml'] expect(tokens[3]).toEqual value: ' comment -', scopes: ['text.xml', 'comment.block.xml', 'invalid.illegal.bad-comments-or-CDATA.xml'] expect(tokens[4]).toEqual value: '--', scopes: ['text.xml', 'comment.block.xml', 'invalid.illegal.bad-comments-or-CDATA.xml'] - expect(tokens[5]).toEqual value: '>', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] + expect(tokens[5]).toEqual value: '>', scopes: ['text.xml', 'comment.block.xml', 'invalid.illegal.bad-comments-or-CDATA.xml'] expect(tokens[6]).toEqual value: '-->', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] expect(tokens[7]).toEqual value: '<', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] expect(tokens[8]).toEqual value: 'n', scopes: ['text.xml', 'meta.tag.no-content.xml', 'entity.name.tag.xml', 'entity.name.tag.localname.xml'] From 15d1f0a68cccf7f91a86e0e58c37db2b77b9d240 Mon Sep 17 00:00:00 2001 From: Valter Pires Date: Thu, 5 Nov 2020 13:24:25 +0400 Subject: [PATCH 6/7] Fix for atom/language-xml#91 and atom/language-xml#96 Update invalid comment end regexp to reduce generated tokens Added test case for empty XML comment `` --- grammars/xml.cson | 6 +++--- spec/xml-spec.coffee | 23 +++++++++++++---------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/grammars/xml.cson b/grammars/xml.cson index ae8e79a..371ff94 100644 --- a/grammars/xml.cson +++ b/grammars/xml.cson @@ -414,12 +414,12 @@ 'captures': '0': 'name': 'punctuation.definition.comment.xml' - 'end': '(?' + 'end': '-->' 'name': 'comment.block.xml' 'patterns': [ { - 'begin': '--' - 'end': '(?=-->)' + 'begin': '--(?!>)' + 'end': '(?)' 'name': 'invalid.illegal.bad-comments-or-CDATA.xml' } ] diff --git a/spec/xml-spec.coffee b/spec/xml-spec.coffee index 994789b..2bc8252 100644 --- a/spec/xml-spec.coffee +++ b/spec/xml-spec.coffee @@ -25,6 +25,11 @@ describe "XML grammar", -> expect(lines[2][1]).toEqual value: '') + expect(tokens[0]).toEqual value: '', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] + it 'tokenizes comment endings with more than two dashes as invalid', -> {tokens} = grammar.tokenizeLine('') expect(tokens[0]).toEqual value: '', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] - expect(tokens[7]).toEqual value: '<', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] - expect(tokens[8]).toEqual value: 'n', scopes: ['text.xml', 'meta.tag.no-content.xml', 'entity.name.tag.xml', 'entity.name.tag.localname.xml'] - expect(tokens[9]).toEqual value: '>', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] - expect(tokens[10]).toEqual value: '', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] + expect(tokens[3]).toEqual value: ' comment --->', scopes: ['text.xml', 'comment.block.xml', 'invalid.illegal.bad-comments-or-CDATA.xml'] + expect(tokens[4]).toEqual value: '-->', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] + expect(tokens[5]).toEqual value: '<', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] + expect(tokens[6]).toEqual value: 'n', scopes: ['text.xml', 'meta.tag.no-content.xml', 'entity.name.tag.xml', 'entity.name.tag.localname.xml'] + expect(tokens[7]).toEqual value: '>', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] + expect(tokens[8]).toEqual value: '', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] it "tokenizes empty element meta.tag.no-content.xml", -> {tokens} = grammar.tokenizeLine('') From 60fc36feccfaa62ad8f7515bbd391ce865b4aa2a Mon Sep 17 00:00:00 2001 From: Valter Pires Date: Sun, 8 Nov 2020 14:59:13 +0400 Subject: [PATCH 7/7] Fix for atom/language-xml#91 and atom/language-xml#96 Updated invalid comment begin rule to reduce number of tokens generated --- grammars/xml.cson | 2 +- spec/xml-spec.coffee | 25 +++++++++++-------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/grammars/xml.cson b/grammars/xml.cson index 371ff94..ce30b98 100644 --- a/grammars/xml.cson +++ b/grammars/xml.cson @@ -418,7 +418,7 @@ 'name': 'comment.block.xml' 'patterns': [ { - 'begin': '--(?!>)' + 'begin': '(?=--(?!>))' 'end': '(?)' 'name': 'invalid.illegal.bad-comments-or-CDATA.xml' } diff --git a/spec/xml-spec.coffee b/spec/xml-spec.coffee index 2bc8252..7250c78 100644 --- a/spec/xml-spec.coffee +++ b/spec/xml-spec.coffee @@ -34,30 +34,27 @@ describe "XML grammar", -> {tokens} = grammar.tokenizeLine('') expect(tokens[0]).toEqual value: '', scopes: ['text.xml', 'comment.block.xml', 'invalid.illegal.bad-comments-or-CDATA.xml'] it 'tokenizes comments with two dashes not followed by ">" as invalid', -> {tokens} = grammar.tokenizeLine('') expect(tokens[0]).toEqual value: '', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] + expect(tokens[2]).toEqual value: '-- comment ', scopes: ['text.xml', 'comment.block.xml', 'invalid.illegal.bad-comments-or-CDATA.xml'] + expect(tokens[3]).toEqual value: '-->', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] it 'tokenizes after invalid comment only if comment was properly closed', -> {tokens} = grammar.tokenizeLine('-->') expect(tokens[0]).toEqual value: '', scopes: ['text.xml', 'comment.block.xml', 'invalid.illegal.bad-comments-or-CDATA.xml'] - expect(tokens[4]).toEqual value: '-->', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] - expect(tokens[5]).toEqual value: '<', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] - expect(tokens[6]).toEqual value: 'n', scopes: ['text.xml', 'meta.tag.no-content.xml', 'entity.name.tag.xml', 'entity.name.tag.localname.xml'] - expect(tokens[7]).toEqual value: '>', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] - expect(tokens[8]).toEqual value: '', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] + expect(tokens[2]).toEqual value: '-- comment --->', scopes: ['text.xml', 'comment.block.xml', 'invalid.illegal.bad-comments-or-CDATA.xml'] + expect(tokens[3]).toEqual value: '-->', scopes: ['text.xml', 'comment.block.xml', 'punctuation.definition.comment.xml'] + expect(tokens[4]).toEqual value: '<', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] + expect(tokens[5]).toEqual value: 'n', scopes: ['text.xml', 'meta.tag.no-content.xml', 'entity.name.tag.xml', 'entity.name.tag.localname.xml'] + expect(tokens[6]).toEqual value: '>', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] + expect(tokens[7]).toEqual value: '', scopes: ['text.xml', 'meta.tag.no-content.xml', 'punctuation.definition.tag.xml'] it "tokenizes empty element meta.tag.no-content.xml", -> {tokens} = grammar.tokenizeLine('')