diff --git a/include/swift/Parse/SyntaxParsingCache.h b/include/swift/Parse/SyntaxParsingCache.h index 466b3a5ea3552..b82b73a5640cb 100644 --- a/include/swift/Parse/SyntaxParsingCache.h +++ b/include/swift/Parse/SyntaxParsingCache.h @@ -18,7 +18,9 @@ #include "llvm/Support/raw_ostream.h" #include -namespace { +namespace swift { + +using namespace swift::syntax; /// A single edit to the original source file in which a continuous range of /// characters have been replaced by a new string @@ -38,16 +40,10 @@ struct SourceEdit { /// Check if the characters replaced by this edit fall into the given range /// or are directly adjacent to it bool intersectsOrTouchesRange(size_t RangeStart, size_t RangeEnd) { - return !(End <= RangeStart || Start >= RangeEnd); + return End >= RangeStart && Start <= RangeEnd; } }; -} // anonymous namespace - -namespace swift { - -using namespace swift::syntax; - struct SyntaxReuseRegion { AbsolutePosition Start; AbsolutePosition End; @@ -89,6 +85,13 @@ class SyntaxParsingCache { std::vector getReusedRegions(const SourceFileSyntax &SyntaxTree) const; + /// Translates a post-edit position to a pre-edit position by undoing the + /// specified edits. + /// Should not be invoked externally. Only public for testing purposes. + static size_t + translateToPreEditPosition(size_t PostEditPosition, + llvm::SmallVector Edits); + private: llvm::Optional lookUpFrom(const Syntax &Node, size_t NodeStart, size_t Position, SyntaxKind Kind); diff --git a/lib/Basic/SourceLoc.cpp b/lib/Basic/SourceLoc.cpp index a5791c3e9359f..4a5e5ad9af12d 100644 --- a/lib/Basic/SourceLoc.cpp +++ b/lib/Basic/SourceLoc.cpp @@ -331,7 +331,9 @@ llvm::Optional SourceManager::resolveFromLineCol(unsigned BufferId, return None; } Ptr = LineStart; - for (; Ptr < End; ++Ptr) { + + // The <= here is to allow for non-inclusive range end positions at EOF + for (; Ptr <= End; ++Ptr) { --Col; if (Col == 0) return Ptr - InputBuf->getBufferStart(); diff --git a/lib/Parse/SyntaxParsingCache.cpp b/lib/Parse/SyntaxParsingCache.cpp index c1709a39e7943..826d536c67cd8 100644 --- a/lib/Parse/SyntaxParsingCache.cpp +++ b/lib/Parse/SyntaxParsingCache.cpp @@ -79,17 +79,21 @@ llvm::Optional SyntaxParsingCache::lookUpFrom(const Syntax &Node, return llvm::None; } -llvm::Optional SyntaxParsingCache::lookUp(size_t NewPosition, - SyntaxKind Kind) { - // Undo the edits in reverse order - size_t OldPosition = NewPosition; - for (auto I = Edits.rbegin(), E = Edits.rend(); I != E; ++I) { +size_t SyntaxParsingCache::translateToPreEditPosition( + size_t PostEditPosition, llvm::SmallVector Edits) { + size_t Position = PostEditPosition; + for (auto I = Edits.begin(), E = Edits.end(); I != E; ++I) { auto Edit = *I; - if (Edit.End <= OldPosition) { - OldPosition = - OldPosition - Edit.ReplacementLength + Edit.originalLength(); + if (Edit.End + Edit.ReplacementLength - Edit.originalLength() <= Position) { + Position = Position - Edit.ReplacementLength + Edit.originalLength(); } } + return Position; +} + +llvm::Optional SyntaxParsingCache::lookUp(size_t NewPosition, + SyntaxKind Kind) { + size_t OldPosition = translateToPreEditPosition(NewPosition, Edits); auto Node = lookUpFrom(OldSyntaxTree, /*NodeStart=*/0, OldPosition, Kind); if (Node.hasValue()) { diff --git a/test/incrParse/Outputs/extend-identifier-at-eof.json b/test/incrParse/Outputs/extend-identifier-at-eof.json new file mode 100644 index 0000000000000..da989710f0a53 --- /dev/null +++ b/test/incrParse/Outputs/extend-identifier-at-eof.json @@ -0,0 +1,132 @@ +{ + "id": 39, + "kind": "SourceFile", + "layout": [ + { + "id": 38, + "kind": "CodeBlockItemList", + "layout": [ + { + "id": 13, + "omitted": true + }, + { + "id": 35, + "kind": "CodeBlockItem", + "layout": [ + { + "id": 34, + "kind": "SequenceExpr", + "layout": [ + { + "id": 33, + "kind": "ExprList", + "layout": [ + { + "id": 28, + "kind": "DiscardAssignmentExpr", + "layout": [ + { + "id": 27, + "tokenKind": { + "kind": "kw__" + }, + "leadingTrivia": [ + { + "kind": "Newline", + "value": 1 + }, + { + "kind": "LineComment", + "value": "\/\/ ATTENTION: This file is testing the EOF token. " + }, + { + "kind": "Newline", + "value": 1 + }, + { + "kind": "LineComment", + "value": "\/\/ DO NOT PUT ANYTHING AFTER THE CHANGE, NOT EVEN A NEWLINE" + }, + { + "kind": "Newline", + "value": 1 + } + ], + "trailingTrivia": [ + { + "kind": "Space", + "value": 1 + } + ], + "presence": "Present" + } + ], + "presence": "Present" + }, + { + "id": 30, + "kind": "AssignmentExpr", + "layout": [ + { + "id": 29, + "tokenKind": { + "kind": "equal" + }, + "leadingTrivia": [], + "trailingTrivia": [ + { + "kind": "Space", + "value": 1 + } + ], + "presence": "Present" + } + ], + "presence": "Present" + }, + { + "id": 32, + "kind": "IdentifierExpr", + "layout": [ + { + "id": 31, + "tokenKind": { + "kind": "identifier", + "text": "xx" + }, + "leadingTrivia": [], + "trailingTrivia": [], + "presence": "Present" + }, + null + ], + "presence": "Present" + } + ], + "presence": "Present" + } + ], + "presence": "Present" + }, + null, + null + ], + "presence": "Present" + } + ], + "presence": "Present" + }, + { + "id": 37, + "tokenKind": { + "kind": "eof", + "text": "" + }, + "leadingTrivia": [], + "trailingTrivia": [], + "presence": "Present" + } + ], + "presence": "Present" +} diff --git a/test/incrParse/extend-identifier-at-eof.swift b/test/incrParse/extend-identifier-at-eof.swift new file mode 100644 index 0000000000000..75c07d41320f9 --- /dev/null +++ b/test/incrParse/extend-identifier-at-eof.swift @@ -0,0 +1,6 @@ +// RUN: %incr-transfer-tree --expected-incremental-syntax-tree %S/Outputs/extend-identifier-at-eof.json %s + +func foo() {} +// ATTENTION: This file is testing the EOF token. +// DO NOT PUT ANYTHING AFTER THE CHANGE, NOT EVEN A NEWLINE +_ = x<<<|||x>>> \ No newline at end of file diff --git a/test/incrParse/multi-edit-mapping.swift b/test/incrParse/multi-edit-mapping.swift new file mode 100644 index 0000000000000..1ba407660796d --- /dev/null +++ b/test/incrParse/multi-edit-mapping.swift @@ -0,0 +1,4 @@ +// RUN: %empty-directory(%t) +// RUN: %validate-incrparse %s --test-case MULTI + +let one: Int;let two: Int; let three: Int; <>><>>let found: Int;let five: Int; diff --git a/test/incrParse/simple.swift b/test/incrParse/simple.swift index ea9c01a356118..12d8cbf88cb38 100644 --- a/test/incrParse/simple.swift +++ b/test/incrParse/simple.swift @@ -13,6 +13,7 @@ // RUN: %validate-incrparse %s --test-case LAST_CHARACTER_OF_STRUCT // RUN: %validate-incrparse %s --test-case ADD_ARRAY_CLOSE_BRACKET // RUN: %validate-incrparse %s --test-case ADD_IF_OPEN_BRACE +// RUN: %validate-incrparse %s --test-case EXTEND_IDENTIFIER func start() {} @@ -20,8 +21,8 @@ func start() {} func foo() { } -_ = <>> -_ = <>> +_ = <>> +_ = <>> _ = <>> <>> <>> @@ -52,3 +53,5 @@ var computedVar: [Int] { if true <>> _ = 5 } + +let y<>> = 42 diff --git a/tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp b/tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp index 1c08bab6687a5..58614e8c1bea2 100644 --- a/tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp +++ b/tools/SourceKit/tools/sourcekitd-test/sourcekitd-test.cpp @@ -1996,7 +1996,7 @@ static unsigned resolveFromLineCol(unsigned Line, unsigned Col, exit(1); } Ptr = LineStart; - for (; Ptr < End; ++Ptr) { + for (; Ptr <= End; ++Ptr) { --Col; if (Col == 0) return Ptr - InputBuf->getBufferStart(); diff --git a/tools/swift-syntax-test/swift-syntax-test.cpp b/tools/swift-syntax-test/swift-syntax-test.cpp index ff81788777043..7914b65acb4c5 100644 --- a/tools/swift-syntax-test/swift-syntax-test.cpp +++ b/tools/swift-syntax-test/swift-syntax-test.cpp @@ -246,11 +246,10 @@ struct ByteBasedSourceRange { bool empty() { return Start == End; } - SourceRange toSourceRange(SourceManager &SourceMgr, unsigned BufferID) { + CharSourceRange toCharSourceRange(SourceManager &SourceMgr, unsigned BufferID) { auto StartLoc = SourceMgr.getLocForOffset(BufferID, Start); - // SourceRange includes the last offset, we don't. So subtract 1 - auto EndLoc = SourceMgr.getLocForOffset(BufferID, End - 1); - return SourceRange(StartLoc, EndLoc); + auto EndLoc = SourceMgr.getLocForOffset(BufferID, End); + return CharSourceRange(SourceMgr, StartLoc, EndLoc); } }; @@ -539,12 +538,11 @@ bool verifyReusedRegions(ByteBasedSourceRangeSet ExpectedReparseRegions, bool NoUnexpectedParse = true; for (auto ReparseRegion : UnexpectedReparseRegions.Ranges) { - auto ReparseRange = ReparseRegion.toSourceRange(SourceMgr, BufferID); + auto ReparseRange = ReparseRegion.toCharSourceRange(SourceMgr, BufferID); // To improve the ergonomics when writing tests we do not want to complain // about reparsed whitespaces. - auto RangeStr = - CharSourceRange(SourceMgr, ReparseRange.Start, ReparseRange.End).str(); + auto RangeStr = ReparseRange.str(); llvm::Regex WhitespaceOnlyRegex("^[ \t\r\n]*$"); if (WhitespaceOnlyRegex.match(RangeStr)) { continue; diff --git a/unittests/Parse/CMakeLists.txt b/unittests/Parse/CMakeLists.txt index 1eb943df08a41..d94b310d92c43 100644 --- a/unittests/Parse/CMakeLists.txt +++ b/unittests/Parse/CMakeLists.txt @@ -2,6 +2,7 @@ add_swift_unittest(SwiftParseTests BuildConfigTests.cpp LexerTests.cpp LexerTriviaTests.cpp + SyntaxParsingCacheTests.cpp TokenizerTests.cpp ) diff --git a/unittests/Parse/SyntaxParsingCacheTests.cpp b/unittests/Parse/SyntaxParsingCacheTests.cpp new file mode 100644 index 0000000000000..628f48d968c9c --- /dev/null +++ b/unittests/Parse/SyntaxParsingCacheTests.cpp @@ -0,0 +1,119 @@ +#include "swift/Parse/SyntaxParsingCache.h" +#include "gtest/gtest.h" + +using namespace swift; +using namespace llvm; + +class TranslateToPreEditPositionTest : public ::testing::Test {}; + +TEST_F(TranslateToPreEditPositionTest, SingleEditBefore) { + // Old: ab_xy + // New: a1b_xy + // + // Edits: + // (1) 1-2: a -> a1 + // + // Lookup for _ at new position 4 + + llvm::SmallVector Edits = { + {1, 2, 2} + }; + + size_t PreEditPos = SyntaxParsingCache::translateToPreEditPosition(4, Edits); + EXPECT_EQ(PreEditPos, 3u); +} + +TEST_F(TranslateToPreEditPositionTest, SingleEditDirectlyBefore) { + // Old: ab_xy + // New: ablah_xy + // + // Edits: + // (1) 2-3: b -> blah + // + // Lookup for _ at new position 6 + + llvm::SmallVector Edits = { + {2, 3, 4} + }; + + size_t PreEditPos = SyntaxParsingCache::translateToPreEditPosition(6, Edits); + EXPECT_EQ(PreEditPos, 3u); +} + +TEST_F(TranslateToPreEditPositionTest, SingleMultiCharacterEdit) { + // Old: ab_xy + // New: abcdef_xy + // + // Edits: + // (1) 1-3: ab -> abcdef + // + // Lookup for _ at new position 7 + + llvm::SmallVector Edits = { + {1, 3, 6} + }; + + size_t PreEditPos = SyntaxParsingCache::translateToPreEditPosition(7, Edits); + EXPECT_EQ(PreEditPos, 3u); +} + +TEST_F(TranslateToPreEditPositionTest, EditAfterLookup) { + // Old: ab_xy + // New: ab_xyz + // + // Edits: + // (1) 4-6: xy -> xyz + // + // Lookup for _ at new position 3 + + llvm::SmallVector Edits = {{4, 6, 4}}; + + size_t PreEditPos = SyntaxParsingCache::translateToPreEditPosition(3, Edits); + EXPECT_EQ(PreEditPos, 3u); +} + +TEST_F(TranslateToPreEditPositionTest, SimpleMultiEdit) { + // Old: ab_xy + // New: a1b2_x3y4 + // + // Edits: + // (1) 1-2: a -> a1 + // (2) 2-3: b -> b2 + // (3) 4-5: x -> x3 + // (4) 5-6: y -> y4 + // + // Lookup for _ at new position 5 + + llvm::SmallVector Edits = { + {1, 2, 2}, + {2, 3, 2}, + {4, 5, 2}, + {5, 6, 2}, + }; + + size_t PreEditPos = SyntaxParsingCache::translateToPreEditPosition(5, Edits); + EXPECT_EQ(PreEditPos, 3u); +} + +TEST_F(TranslateToPreEditPositionTest, LongMultiEdit) { + // Old: ab_xy + // New: a11111b2_x3y4 + // + // Edits: + // (1) 1-2: a -> a11111 + // (2) 2-3: b -> b2 + // (3) 4-5: x -> x3 + // (4) 5-6: y -> y4 + // + // Lookup for _ at new position + + llvm::SmallVector Edits = { + {1, 2, 6}, + {2, 3, 2}, + {4, 5, 2}, + {5, 6, 2}, + }; + + size_t PreEditPos = SyntaxParsingCache::translateToPreEditPosition(9, Edits); + EXPECT_EQ(PreEditPos, 3u); +} \ No newline at end of file diff --git a/utils/incrparse/test_util.py b/utils/incrparse/test_util.py index 577d5c08495d1..56169530242d8 100755 --- a/utils/incrparse/test_util.py +++ b/utils/incrparse/test_util.py @@ -27,8 +27,6 @@ def run_command(cmd): def parseLine(line, line_no, test_case, incremental_edit_args, reparse_args, current_reparse_start): - pre_column_offset = 1 - post_column_offset = 1 pre_edit_line = "" post_edit_line = "" @@ -55,31 +53,29 @@ def parseLine(line, line_no, test_case, incremental_edit_args, reparse_args, suffix = subst_match.group(5) if match_test_case == test_case: - pre_edit_line += prefix + pre_edit - post_edit_line += prefix + post_edit - # Compute the -incremental-edit argument for swift-syntax-test - column = pre_column_offset + len(prefix) + column = len(pre_edit_line) + len(prefix) + 1 edit_arg = '%d:%d-%d:%d=%s' % \ (line_no, column, line_no, column + len(pre_edit), post_edit) incremental_edit_args.append('-incremental-edit') incremental_edit_args.append(edit_arg) + + pre_edit_line += prefix + pre_edit + post_edit_line += prefix + post_edit else: # For different test cases just take the pre-edit text pre_edit_line += prefix + pre_edit post_edit_line += prefix + pre_edit line = suffix - pre_column_offset += len(pre_edit_line) - post_column_offset += len(post_edit_line) elif reparse_match: prefix = reparse_match.group(1) is_closing = len(reparse_match.group(2)) > 0 match_test_case = reparse_match.group(3) suffix = reparse_match.group(4) if match_test_case == test_case: - column = post_column_offset + len(prefix) + column = len(post_edit_line) + len(prefix) + 1 if is_closing: if not current_reparse_start: raise TestFailedError('Closing unopened reparse tag '