Skip to content

Commit eefd826

Browse files
piotrkowalczukpiotrkowalczuk
committed
@dynamicMemberLookup dot notation used in favour of the subscript
1 parent c8ec50a commit eefd826

11 files changed

+96
-86
lines changed

‎Sources/Hub/Config.swift

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -574,6 +574,16 @@ public struct Config: Hashable, Sendable,
574574

575575
return nil // backward compatibility
576576
}
577+
}
578+
579+
public subscript(dynamicMember member: String) -> Config {
580+
get {
581+
if let dict = self.dictionary() {
582+
return dict[BinaryDistinctString(member)] ?? dict[self.uncamelCase(BinaryDistinctString(member))] ?? Config()
583+
}
584+
585+
return Config()
586+
}
577587
}
578588

579589
func uncamelCase(_ string: BinaryDistinctString) -> BinaryDistinctString {

‎Sources/Hub/Hub.swift

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ public class LanguageModelConfigurationFromHub {
105105
get async throws {
106106
if let hubConfig = try await configPromise!.value.tokenizerConfig {
107107
// Try to guess the class if it's not present and the modelType is
108-
if let _: String = hubConfig["tokenizerClass"].string() { return hubConfig }
108+
if let _: String = hubConfig.tokenizerClass?.string() { return hubConfig }
109109
guard let modelType = try await modelType else { return hubConfig }
110110

111111
// If the config exists but doesn't contain a tokenizerClass, use a fallback config if we have it
@@ -134,7 +134,7 @@ public class LanguageModelConfigurationFromHub {
134134

135135
public var modelType: String? {
136136
get async throws {
137-
try await modelConfig["modelType"].string()
137+
try await modelConfig.modelType.string()
138138
}
139139
}
140140

@@ -196,7 +196,7 @@ public class LanguageModelConfigurationFromHub {
196196
let chatTemplateURL = modelFolder.appending(path: "chat_template.json")
197197
if FileManager.default.fileExists(atPath: chatTemplateURL.path),
198198
let chatTemplateConfig = try? hubApi.configuration(fileURL: chatTemplateURL),
199-
let chatTemplate = chatTemplateConfig["chatTemplate"].string() {
199+
let chatTemplate = chatTemplateConfig.chatTemplate.string() {
200200
// Create or update tokenizer config with chat template
201201
if var configDict = tokenizerConfig?.dictionary() {
202202
configDict["chat_template"] = .init(chatTemplate)

‎Sources/Models/LanguageModel.swift

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -157,33 +157,33 @@ public extension LanguageModel {
157157

158158
var modelType: String? {
159159
get async throws {
160-
try await modelConfig["modelType"].string()
160+
try await modelConfig.modelType.string()
161161
}
162162
}
163163

164164
var textGenerationParameters: Config? {
165165
get async throws {
166-
try await modelConfig["taskSpecificParams"]["textGeneration"]
166+
try await modelConfig.taskSpecificParams.textGeneration
167167
}
168168
}
169169

170170
var defaultDoSample: Bool {
171171
get async throws {
172-
try await textGenerationParameters?["doSample"].boolean() ?? true
172+
try await textGenerationParameters?.doSample.boolean() ?? true
173173
}
174174
}
175175

176176
var bosTokenId: Int? {
177177
get async throws {
178178
let modelConfig = try await modelConfig
179-
return modelConfig["bosTokenId"].integer()
179+
return modelConfig.bosTokenId.integer()
180180
}
181181
}
182182

183183
var eosTokenId: Int? {
184184
get async throws {
185185
let modelConfig = try await modelConfig
186-
return modelConfig["eosTokenId"].integer()
186+
return modelConfig.eosTokenId.integer()
187187
}
188188
}
189189

‎Sources/Tokenizers/BPETokenizer.swift

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@ class BPETokenizer: PreTrainedTokenizerModel {
6464
}
6565

6666
required init(tokenizerConfig: Config, tokenizerData: Config, addedTokens: [String: Int]) throws {
67-
guard let merges = Self.mergesFromConfig(tokenizerData["model"]["merges"]) else { fatalError("BPETokenizer requires merges") }
68-
guard let vocab = tokenizerData["model"]["vocab"].dictionary() else {
67+
guard let merges = Self.mergesFromConfig(tokenizerData.model.merges) else { fatalError("BPETokenizer requires merges") }
68+
guard let vocab = tokenizerData.model.vocab.dictionary() else {
6969
throw TokenizerError.missingVocab
7070
}
7171
var bpeRanks: [BytePair: Int] = [:]
@@ -93,13 +93,13 @@ class BPETokenizer: PreTrainedTokenizerModel {
9393
self.unknownTokenId = nil
9494
}
9595

96-
eosToken = addedTokenAsString(tokenizerConfig["eosToken"])
96+
eosToken = addedTokenAsString(tokenizerConfig.eosToken)
9797
eosTokenId = eosToken == nil ? nil : tokensToIds[eosToken! as NSString]
9898

99-
bosToken = addedTokenAsString(tokenizerConfig["bosToken"])
99+
bosToken = addedTokenAsString(tokenizerConfig.bosToken)
100100
bosTokenId = bosToken == nil ? nil : tokensToIds[bosToken! as NSString]
101101

102-
fuseUnknownTokens = tokenizerConfig["fuseUnk"].boolean(or: false)
102+
fuseUnknownTokens = tokenizerConfig.fuseUnk.boolean(or: false)
103103
}
104104

105105
func convertTokenToId(_ token: String) -> Int? {

‎Sources/Tokenizers/BertTokenizer.swift

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -47,15 +47,15 @@ public class BertTokenizer {
4747
}
4848

4949
public required convenience init(tokenizerConfig: Config, tokenizerData: Config, addedTokens: [String: Int]) throws {
50-
guard let vocab = tokenizerData["model"]["vocab"].dictionary() else {
50+
guard let vocab = tokenizerData.model.vocab.dictionary() else {
5151
throw TokenizerError.missingVocab
5252
}
53-
let merges: [String]? = tokenizerData["model"]["merges"].get()
54-
let tokenizeChineseChars = tokenizerConfig["handleChineseChars"].boolean(or: true)
55-
let eosToken = tokenizerConfig["eosToken"].string()
56-
let bosToken = tokenizerConfig["bosToken"].string()
57-
let fuseUnknown = tokenizerConfig["fuseUnk"].boolean(or: false)
58-
let doLowerCase = tokenizerConfig["doLowerCase"].boolean(or: true)
53+
let merges: [String]? = tokenizerData.model.merges.get()
54+
let tokenizeChineseChars = tokenizerConfig.handleChineseChars.boolean(or: true)
55+
let eosToken = tokenizerConfig.eosToken.string()
56+
let bosToken = tokenizerConfig.bosToken.string()
57+
let fuseUnknown = tokenizerConfig.fuseUnk.boolean(or: false)
58+
let doLowerCase = tokenizerConfig.doLowerCase.boolean(or: true)
5959

6060
let vocabulary = vocab.reduce(into: [String: Int]()) { result, element in
6161
if let val = element.value.integer() {

‎Sources/Tokenizers/Decoder.swift

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ struct DecoderFactory {
3737
static func fromConfig(config: Config?, addedTokens: Set<String>? = nil) -> Decoder? {
3838
// TODO: not sure if we need to include `addedTokens` in all the decoder initializers (and the protocol)
3939
guard let config = config else { return nil }
40-
guard let typeName = config["type"].string() else { return nil }
40+
guard let typeName = config.type.string() else { return nil }
4141
let type = DecoderType(rawValue: typeName)
4242
switch type {
4343
case .Sequence : return DecoderSequence(config: config)
@@ -61,9 +61,9 @@ class WordPieceDecoder: Decoder {
6161
private let re = try! NSRegularExpression(pattern: "\\s(\\.|\\?|\\!|\\,|'\\s|n't|'m|'s|'ve|'re)", options: [])
6262

6363
required public init(config: Config) {
64-
guard let prefix = config["prefix"].string() else { fatalError("Missing `prefix` configuration for WordPieceDecoder.") }
64+
guard let prefix = config.prefix.string() else { fatalError("Missing `prefix` configuration for WordPieceDecoder.") }
6565
self.prefix = prefix
66-
self.cleanup = config["cleanup"].boolean(or: false)
66+
self.cleanup = config.cleanup.boolean(or: false)
6767
}
6868

6969
func decode(tokens: [String]) -> [String] {
@@ -86,7 +86,7 @@ class DecoderSequence: Decoder {
8686
let decoders: [Decoder]
8787

8888
required public init(config: Config) {
89-
guard let configs = config["decoders"].array() else { fatalError("No decoders in Sequence") }
89+
guard let configs = config.decoders.array() else { fatalError("No decoders in Sequence") }
9090
decoders = configs.compactMap { DecoderFactory.fromConfig(config: $0) }
9191
}
9292

@@ -199,9 +199,9 @@ class StripDecoder: Decoder {
199199
let stop: Int
200200

201201
required public init(config: Config) {
202-
guard let content = config["content"].string() else { fatalError("Incorrect StripDecoder configuration: can't parse `content`.") }
203-
guard let start = config["start"].integer() else { fatalError("Incorrect StripDecoder configuration: can't parse `start`.") }
204-
guard let stop = config["stop"].integer() else { fatalError("Incorrect StripDecoder configuration: can't parse `stop`.") }
202+
guard let content = config.content.string() else { fatalError("Incorrect StripDecoder configuration: can't parse `content`.") }
203+
guard let start = config.start.integer() else { fatalError("Incorrect StripDecoder configuration: can't parse `start`.") }
204+
guard let stop = config.stop.integer() else { fatalError("Incorrect StripDecoder configuration: can't parse `stop`.") }
205205
self.content = content
206206
self.start = start
207207
self.stop = stop
@@ -219,8 +219,8 @@ class MetaspaceDecoder: Decoder {
219219
let replacement: String
220220

221221
required public init(config: Config) {
222-
addPrefixSpace = config["addPrefixSpace"].boolean(or: false)
223-
replacement = config["replacement"].string(or: "_")
222+
addPrefixSpace = config.addPrefixSpace.boolean(or: false)
223+
replacement = config.replacement.string(or: "_")
224224
}
225225

226226
func decode(tokens: [String]) -> [String] {

‎Sources/Tokenizers/Normalizer.swift

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ enum NormalizerType: String {
4141
struct NormalizerFactory {
4242
static func fromConfig(config: Config?) -> Normalizer? {
4343
guard let config = config else { return nil }
44-
guard let typeName = config["type"].string() else { return nil }
44+
guard let typeName = config.type.string() else { return nil }
4545
let type = NormalizerType(rawValue: typeName)
4646
switch type {
4747
case .Sequence: return NormalizerSequence(config: config)
@@ -65,7 +65,7 @@ class NormalizerSequence: Normalizer {
6565
let normalizers: [Normalizer]
6666

6767
required public init(config: Config) {
68-
guard let configs = config["normalizers"].array() else {
68+
guard let configs = config.normalizers.array() else {
6969
fatalError("No normalizers in Sequence")
7070
}
7171
normalizers = configs.compactMap { NormalizerFactory.fromConfig(config: $0) }
@@ -82,7 +82,7 @@ class PrependNormalizer: Normalizer {
8282
let prepend: String
8383

8484
required public init(config: Config) {
85-
prepend = config["prepend"].string(or: "")
85+
prepend = config.prepend.string(or: "")
8686
}
8787

8888
public func normalize(text: String) -> String {
@@ -150,10 +150,10 @@ class BertNormalizer: Normalizer {
150150
let shouldLowercase: Bool
151151

152152
required init(config: Config) {
153-
self.shouldCleanText = config["cleanText"].boolean(or: true)
154-
self.shouldHandleChineseChars = config["handleChineseChars"].boolean(or: true)
155-
self.shouldLowercase = config["lowercase"].boolean(or: true)
156-
self.shouldStripAccents = config["stripAccents"].boolean(or: shouldLowercase)
153+
self.shouldCleanText = config.cleanText.boolean(or: true)
154+
self.shouldHandleChineseChars = config.handleChineseChars.boolean(or: true)
155+
self.shouldLowercase = config.lowercase.boolean(or: true)
156+
self.shouldStripAccents = config.stripAccents.boolean(or: shouldLowercase)
157157
}
158158

159159
func normalize(text: String) -> String {
@@ -281,8 +281,8 @@ class StripNormalizer: Normalizer {
281281
let rightStrip: Bool
282282

283283
required init(config: Config) {
284-
self.leftStrip = config["stripLeft"].boolean(or: true)
285-
self.rightStrip = config["stripRight"].boolean(or: true)
284+
self.leftStrip = config.stripLeft.boolean(or: true)
285+
self.rightStrip = config.stripRight.boolean(or: true)
286286
}
287287

288288
func normalize(text: String) -> String {
@@ -321,11 +321,11 @@ extension StringReplacePattern {
321321

322322
extension StringReplacePattern {
323323
static func from(config: Config) -> StringReplacePattern? {
324-
guard let replacement = config["content"].string() else { return nil }
325-
if let pattern = config["pattern"]["String"].string() {
324+
guard let replacement = config.content.string() else { return nil }
325+
if let pattern = config.pattern.String.string() {
326326
return StringReplacePattern.string(pattern: pattern, replacement: replacement)
327327
}
328-
if let pattern = config["pattern"]["Regex"].string() {
328+
if let pattern = config.pattern.Regex.string() {
329329
guard let regexp = try? NSRegularExpression(pattern: pattern, options: []) else {
330330
fatalError("Cannot build regexp from \(pattern)")
331331
}

‎Sources/Tokenizers/PostProcessor.swift

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ enum PostProcessorType: String {
3232
struct PostProcessorFactory {
3333
static func fromConfig(config: Config?) -> PostProcessor? {
3434
guard let config = config else { return nil }
35-
guard let typeName = config["type"].string() else { return nil }
35+
guard let typeName = config.type.string() else { return nil }
3636
let type = PostProcessorType(rawValue: typeName)
3737
switch type {
3838
case .TemplateProcessing: return TemplateProcessing(config: config)
@@ -50,8 +50,8 @@ class TemplateProcessing: PostProcessor {
5050
let pair: [Config]
5151

5252
required public init(config: Config) {
53-
guard let single = config["single"].array() else { fatalError("Missing `single` processor configuration") }
54-
guard let pair = config["pair"].array() else { fatalError("Missing `pair` processor configuration") }
53+
guard let single = config.single.array() else { fatalError("Missing `single` processor configuration") }
54+
guard let pair = config.pair.array() else { fatalError("Missing `pair` processor configuration") }
5555

5656
self.single = single
5757
self.pair = pair
@@ -62,13 +62,13 @@ class TemplateProcessing: PostProcessor {
6262

6363
var toReturn: [String] = []
6464
for item in config {
65-
if let id = item["SpecialToken"]["id"].string() {
65+
if let id = item.SpecialToken.id.string() {
6666
if addSpecialTokens {
6767
toReturn.append(id)
6868
}
69-
} else if item["Sequence"]["id"].string() == "A" {
69+
} else if item.Sequence.id.string() == "A" {
7070
toReturn += tokens
71-
} else if item["Sequence"]["id"].string() == "B" {
71+
} else if item.Sequence.id.string() == "B" {
7272
toReturn += tokensPair!
7373
}
7474
}
@@ -90,12 +90,12 @@ class RobertaProcessing: PostProcessor {
9090
private let addPrefixSpace: Bool
9191

9292
required public init(config: Config) {
93-
guard let sep = config["sep"].token() else { fatalError("Missing `sep` processor configuration") }
94-
guard let cls = config["cls"].token() else { fatalError("Missing `cls` processor configuration") }
93+
guard let sep = config.sep.token() else { fatalError("Missing `sep` processor configuration") }
94+
guard let cls = config.cls.token() else { fatalError("Missing `cls` processor configuration") }
9595
self.sep = sep
9696
self.cls = cls
97-
self.trimOffset = config["trimOffset"].boolean(or: true)
98-
self.addPrefixSpace = config["addPrefixSpace"].boolean(or: true)
97+
self.trimOffset = config.trimOffset.boolean(or: true)
98+
self.addPrefixSpace = config.addPrefixSpace.boolean(or: true)
9999
}
100100

101101
func postProcess(tokens: [String], tokensPair: [String]?, addSpecialTokens: Bool = true) -> [String] {
@@ -147,8 +147,8 @@ class BertProcessing: PostProcessor {
147147
private let cls: (UInt, String)
148148

149149
required public init(config: Config) {
150-
guard let sep = config["sep"].token() else { fatalError("Missing `sep` processor configuration") }
151-
guard let cls = config["cls"].token() else { fatalError("Missing `cls` processor configuration") }
150+
guard let sep = config.sep.token() else { fatalError("Missing `sep` processor configuration") }
151+
guard let cls = config.cls.token() else { fatalError("Missing `cls` processor configuration") }
152152
self.sep = sep
153153
self.cls = cls
154154
}
@@ -169,7 +169,7 @@ class SequenceProcessing: PostProcessor {
169169
private let processors: [PostProcessor]
170170

171171
required public init(config: Config) {
172-
guard let processorConfigs = config["processors"].array() else {
172+
guard let processorConfigs = config.processors.array() else {
173173
fatalError("Missing `processors` configuration")
174174
}
175175

0 commit comments

Comments
 (0)