Skip to content

[stdlib] Implement native grapheme breaking for String #37864

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
Nov 1, 2021
Merged

[stdlib] Implement native grapheme breaking for String #37864

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
f61754b
Implement GraphemeWalker that does native grapheme breaking
Azoy Jun 4, 2021
c4175d1
Bridged strings use native grapheme breaking for forward strides
Azoy Jun 9, 2021
2f7405a
Implement bidirectional native grapheme breaking for native and forei…
Azoy Jun 10, 2021
89b890c
Remove ICU's grapheme breaking support
Azoy Jun 10, 2021
af10a9b
Use UnicodeScalarView to implement GraphemeWalker
Azoy Jun 17, 2021
004ac2c
Incorporate Michael's feedback
Azoy Oct 14, 2021
ec31e3f
Try bringing back some old fast paths
Azoy Oct 21, 2021
85352c2
Parameterize nextBoundary and previousBoundary
Azoy Oct 25, 2021
16abfe5
Implement Michael's suggestions
Azoy Oct 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions stdlib/public/SwiftShims/UnicodeData.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ __swift_intptr_t _swift_stdlib_getMphIdx(__swift_uint32_t scalar,
const __swift_uint16_t * const *ranks,
const __swift_uint16_t * const sizes);

SWIFT_RUNTIME_STDLIB_INTERNAL
__swift_uint8_t _swift_stdlib_getGraphemeBreakProperty(__swift_uint32_t scalar);

#ifdef __cplusplus
} // extern "C"
#endif
Expand Down
85 changes: 0 additions & 85 deletions stdlib/public/SwiftShims/UnicodeShims.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,39 +29,6 @@
extern "C" {
#endif

SWIFT_RUNTIME_STDLIB_API
const __swift_uint8_t *_swift_stdlib_GraphemeClusterBreakPropertyTrie;

struct _swift_stdlib_GraphemeClusterBreakPropertyTrieMetadataTy {
unsigned BMPFirstLevelIndexBits;
unsigned BMPDataOffsetBits;
unsigned SuppFirstLevelIndexBits;
unsigned SuppSecondLevelIndexBits;
unsigned SuppDataOffsetBits;

unsigned BMPLookupBytesPerEntry;
unsigned BMPDataBytesPerEntry;
unsigned SuppLookup1BytesPerEntry;
unsigned SuppLookup2BytesPerEntry;
unsigned SuppDataBytesPerEntry;

unsigned TrieSize;

unsigned BMPLookupBytesOffset;
unsigned BMPDataBytesOffset;
unsigned SuppLookup1BytesOffset;
unsigned SuppLookup2BytesOffset;
unsigned SuppDataBytesOffset;
};

SWIFT_RUNTIME_STDLIB_API
const struct _swift_stdlib_GraphemeClusterBreakPropertyTrieMetadataTy
_swift_stdlib_GraphemeClusterBreakPropertyTrieMetadata;

SWIFT_RUNTIME_STDLIB_API
const __swift_uint16_t *
_swift_stdlib_ExtendedGraphemeClusterNoBoundaryRulesMatrix;

SWIFT_RUNTIME_STDLIB_API
__swift_int32_t _swift_stdlib_unicode_strToUpper(
__swift_uint16_t *Destination, __swift_int32_t DestinationCapacity,
Expand Down Expand Up @@ -362,17 +329,6 @@ typedef enum __swift_stdlib_UErrorCode {
__swift_stdlib_U_ERROR_LIMIT = __swift_stdlib_U_PLUGIN_ERROR_LIMIT
} __swift_stdlib_UErrorCode;

typedef enum __swift_stdlib_UBreakIteratorType {
__swift_stdlib_UBRK_CHARACTER = 0,
__swift_stdlib_UBRK_WORD = 1,
__swift_stdlib_UBRK_LINE = 2,
__swift_stdlib_UBRK_SENTENCE = 3,
#ifndef U_HIDE_DEPRECATED_API
__swift_stdlib_UBRK_TITLE = 4,
#endif
__swift_stdlib_UBRK_COUNT = 5
} __swift_stdlib_UBreakIteratorType;

typedef enum __swift_stdlib_UCharCategory {
__swift_stdlib_U_UNASSIGNED = 0,
__swift_stdlib_U_GENERAL_OTHER_TYPES = 0,
Expand Down Expand Up @@ -447,47 +403,6 @@ typedef __swift_uint16_t __swift_stdlib_UChar;
typedef __swift_uint8_t
__swift_stdlib_UVersionInfo[__SWIFT_STDLIB_U_MAX_VERSION_LENGTH];

SWIFT_RUNTIME_STDLIB_API
void __swift_stdlib_ubrk_close(__swift_stdlib_UBreakIterator *bi);

SWIFT_RUNTIME_STDLIB_API
__swift_stdlib_UBreakIterator *
__swift_stdlib_ubrk_open(__swift_stdlib_UBreakIteratorType type,
const char *_Nullable locale,
const __swift_stdlib_UChar *_Null_unspecified text,
__swift_int32_t textLength,
__swift_stdlib_UErrorCode *status);

SWIFT_RUNTIME_STDLIB_API
void __swift_stdlib_ubrk_setUText(__swift_stdlib_UBreakIterator *bi,
__swift_stdlib_UText *text,
__swift_stdlib_UErrorCode *status);

SWIFT_RUNTIME_STDLIB_API
void __swift_stdlib_ubrk_setText(__swift_stdlib_UBreakIterator *bi,
const __swift_stdlib_UChar *text,
__swift_int32_t textLength,
__swift_stdlib_UErrorCode *status);

SWIFT_RUNTIME_STDLIB_API
__swift_stdlib_UText *
__swift_stdlib_utext_openUTF8(__swift_stdlib_UText *_Nullable, const char *,
__swift_int64_t, __swift_stdlib_UErrorCode *);

SWIFT_RUNTIME_STDLIB_API
__swift_stdlib_UText *
__swift_stdlib_utext_openUChars(__swift_stdlib_UText *_Nullable,
const __swift_stdlib_UChar *, __swift_int64_t,
__swift_stdlib_UErrorCode *);

SWIFT_RUNTIME_STDLIB_API
__swift_int32_t __swift_stdlib_ubrk_preceding(__swift_stdlib_UBreakIterator *bi,
__swift_int32_t offset);

SWIFT_RUNTIME_STDLIB_API
__swift_int32_t __swift_stdlib_ubrk_following(__swift_stdlib_UBreakIterator *bi,
__swift_int32_t offset);

SWIFT_RUNTIME_STDLIB_API
__swift_stdlib_UBool
__swift_stdlib_u_hasBinaryProperty(__swift_stdlib_UChar32,
Expand Down
1 change: 1 addition & 0 deletions stdlib/public/core/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ set(SWIFTLIB_ESSENTIAL
UnavailableStringAPIs.swift
UnicodeData.swift
UnicodeEncoding.swift
UnicodeGraphemeBreakProperty.swift
UnicodeHelpers.swift
UnicodeParser.swift
UnicodeScalarProperties.swift
Expand Down
1 change: 1 addition & 0 deletions stdlib/public/core/GroupInfo.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
"Unicode.swift",
"UnicodeData.swift",
"UnicodeEncoding.swift",
"UnicodeGraphemeBreakProperty.swift",
"UnicodeHelpers.swift",
"UnicodeParser.swift",
"UnicodeScalar.swift",
Expand Down
Loading