Write spokenForms.json from Talon (cursorless-dev#1939)

- This PR is the Talon side of cursorless-dev#1940

## Checklist

- [-] I have added
[tests](https://www.cursorless.org/docs/contributing/test-case-recorder/)
(will do in follow-up)
- [-] I have updated the
[docs](https://github.com/cursorless-dev/cursorless/tree/main/docs) and
[cheatsheet](https://github.com/cursorless-dev/cursorless/tree/main/cursorless-talon/src/cheatsheet)
- [x] I have not broken the cheatsheet
- [x] have the json have its own version number

Author: pokey

src/csv_overrides.py

@@ -1,8 +1,10 @@
 import csv
+from collections import defaultdict
 from collections.abc import Container
+from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from typing import Optional
+from typing import Callable, Iterable, Optional, TypedDict
 
 from talon import Context, Module, actions, app, fs
 
@@ -25,50 +27,75 @@
     desc="The directory to use for cursorless settings csvs relative to talon user directory",
 )
 
-default_ctx = Context()
-default_ctx.matches = r"""
+# The global context we use for our lists
+ctx = Context()
+
+# A context that contains default vocabulary, for use in testing
+normalized_ctx = Context()
+normalized_ctx.matches = r"""
 tag: user.cursorless_default_vocabulary
 """
 
 
+# Maps from Talon list name to a map from spoken form to value
+ListToSpokenForms = dict[str, dict[str, str]]
+
+
+@dataclass
+class SpokenFormEntry:
+    list_name: str
+    id: str
+    spoken_forms: list[str]
+
+
 def init_csv_and_watch_changes(
     filename: str,
-    default_values: dict[str, dict[str, str]],
+    default_values: ListToSpokenForms,
+    handle_new_values: Optional[Callable[[list[SpokenFormEntry]], None]] = None,
+    *,
     extra_ignored_values: Optional[list[str]] = None,
     extra_allowed_values: Optional[list[str]] = None,
     allow_unknown_values: bool = False,
     default_list_name: Optional[str] = None,
     headers: list[str] = [SPOKEN_FORM_HEADER, CURSORLESS_IDENTIFIER_HEADER],
-    ctx: Context = Context(),
     no_update_file: bool = False,
     pluralize_lists: Optional[list[str]] = None,
 ):
     """
     Initialize a cursorless settings csv, creating it if necessary, and watch
     for changes to the csv.  Talon lists will be generated based on the keys of
     `default_values`.  For example, if there is a key `foo`, there will be a
-    list created called `user.cursorless_foo` that will contain entries from
-    the original dict at the key `foo`, updated according to customization in
-    the csv at
+    list created called `user.cursorless_foo` that will contain entries from the
+    original dict at the key `foo`, updated according to customization in the
+    csv at
 
-        actions.path.talon_user() / "cursorless-settings" / filename
+    ```
+    actions.path.talon_user() / "cursorless-settings" / filename
+    ```
 
     Note that the settings directory location can be customized using the
     `user.cursorless_settings_directory` setting.
 
     Args:
         filename (str): The name of the csv file to be placed in
-        `cursorles-settings` dir
-        default_values (dict[str, dict]): The default values for the lists to
-        be customized in the given csv
-        extra_ignored_values list[str]: Don't throw an exception if any of
-        these appear as values; just ignore them and don't add them to any list
-        allow_unknown_values bool: If unknown values appear, just put them in the list
-        default_list_name Optional[str]: If unknown values are allowed, put any
-        unknown values in this list
-        no_update_file Optional[bool]: Set this to `TRUE` to indicate that we should
-        not update the csv. This is used generally in case there was an issue coming up with the default set of values so we don't want to persist those to disk
-        pluralize_lists: Create plural version of given lists
+            `cursorles-settings` dir
+        default_values (ListToSpokenForms): The default values for the lists to
+            be customized in the given csv
+        handle_new_values (Optional[Callable[[list[SpokenFormEntry]], None]]): A
+            callback to be called when the lists are updated
+        extra_ignored_values (Optional[list[str]]): Don't throw an exception if
+            any of these appear as values; just ignore them and don't add them
+            to any list
+        allow_unknown_values (bool): If unknown values appear, just put them in
+            the list
+        default_list_name (Optional[str]): If unknown values are
+            allowed, put any unknown values in this list
+        headers (list[str]): The headers to use for the csv
+        no_update_file (bool): Set this to `True` to indicate that we should not
+            update the csv. This is used generally in case there was an issue
+            coming up with the default set of values so we don't want to persist
+            those to disk
+        pluralize_lists (list[str]): Create plural version of given lists
     """
     # Don't allow both `extra_allowed_values` and `allow_unknown_values`
     assert not (extra_allowed_values and allow_unknown_values)
@@ -112,7 +139,7 @@ def on_watch(path, flags):
                 allow_unknown_values=allow_unknown_values,
                 default_list_name=default_list_name,
                 pluralize_lists=pluralize_lists,
-                ctx=ctx,
+                handle_new_values=handle_new_values,
             )
 
     fs.watch(str(file_path.parent), on_watch)
@@ -135,7 +162,7 @@ def on_watch(path, flags):
             allow_unknown_values=allow_unknown_values,
             default_list_name=default_list_name,
             pluralize_lists=pluralize_lists,
-            ctx=ctx,
+            handle_new_values=handle_new_values,
         )
     else:
         if not no_update_file:
@@ -148,7 +175,7 @@ def on_watch(path, flags):
             allow_unknown_values=allow_unknown_values,
             default_list_name=default_list_name,
             pluralize_lists=pluralize_lists,
-            ctx=ctx,
+            handle_new_values=handle_new_values,
         )
 
     def unsubscribe():
@@ -184,68 +211,92 @@ def create_default_vocabulary_dicts(
             if active_key:
                 updated_dict[active_key] = value2
         default_values_updated[key] = updated_dict
-    assign_lists_to_context(default_ctx, default_values_updated, pluralize_lists)
+    assign_lists_to_context(normalized_ctx, default_values_updated, pluralize_lists)
 
 
 def update_dicts(
-    default_values: dict[str, dict],
-    current_values: dict,
+    default_values: ListToSpokenForms,
+    current_values: dict[str, str],
     extra_ignored_values: list[str],
     extra_allowed_values: list[str],
     allow_unknown_values: bool,
     default_list_name: Optional[str],
     pluralize_lists: list[str],
-    ctx: Context,
+    handle_new_values: Optional[Callable[[list[SpokenFormEntry]], None]],
 ):
     # Create map with all default values
-    results_map = {}
-    for list_name, dict in default_values.items():
-        for key, value in dict.items():
-            results_map[value] = {"key": key, "value": value, "list": list_name}
+    results_map: dict[str, ResultsListEntry] = {}
+    for list_name, obj in default_values.items():
+        for spoken, id in obj.items():
+            results_map[id] = {"spoken": spoken, "id": id, "list": list_name}
 
     # Update result with current values
-    for key, value in current_values.items():
+    for spoken, id in current_values.items():
         try:
-            results_map[value]["key"] = key
+            results_map[id]["spoken"] = spoken
         except KeyError:
-            if value in extra_ignored_values:
+            if id in extra_ignored_values:
                 pass
-            elif allow_unknown_values or value in extra_allowed_values:
-                results_map[value] = {
-                    "key": key,
-                    "value": value,
+            elif allow_unknown_values or id in extra_allowed_values:
+                assert default_list_name is not None
+                results_map[id] = {
+                    "spoken": spoken,
+                    "id": id,
                     "list": default_list_name,
                 }
             else:
                 raise
 
-    # Convert result map back to result list
-    results = {res["list"]: {} for res in results_map.values()}
-    for obj in results_map.values():
-        value = obj["value"]
-        key = obj["key"]
-        if not is_removed(key):
-            for k in key.split("|"):
-                if value == "pasteFromClipboard" and k.endswith(" to"):
+    spoken_form_entries = list(generate_spoken_forms(results_map.values()))
+
+    # Assign result to talon context list
+    lists: ListToSpokenForms = defaultdict(dict)
+    for entry in spoken_form_entries:
+        for spoken_form in entry.spoken_forms:
+            lists[entry.list_name][spoken_form] = entry.id
+    assign_lists_to_context(ctx, lists, pluralize_lists)
+
+    if handle_new_values is not None:
+        handle_new_values(spoken_form_entries)
+
+
+class ResultsListEntry(TypedDict):
+    spoken: str
+    id: str
+    list: str
+
+
+def generate_spoken_forms(results_list: Iterable[ResultsListEntry]):
+    for obj in results_list:
+        id = obj["id"]
+        spoken = obj["spoken"]
+
+        spoken_forms = []
+        if not is_removed(spoken):
+            for k in spoken.split("|"):
+                if id == "pasteFromClipboard" and k.endswith(" to"):
                     # FIXME: This is a hack to work around the fact that the
                     # spoken form of the `pasteFromClipboard` action used to be
                     # "paste to", but now the spoken form is just "paste" and
                     # the "to" is part of the positional target. Users who had
                     # cursorless before this change would have "paste to" as
                     # their spoken form and so would need to say "paste to to".
                     k = k[:-3]
-                results[obj["list"]][k.strip()] = value
+                spoken_forms.append(k.strip())
 
-    # Assign result to talon context list
-    assign_lists_to_context(ctx, results, pluralize_lists)
+        yield SpokenFormEntry(
+            list_name=obj["list"],
+            id=id,
+            spoken_forms=spoken_forms,
+        )
 
 
 def assign_lists_to_context(
     ctx: Context,
-    results: dict,
+    lists: ListToSpokenForms,
     pluralize_lists: list[str],
 ):
-    for list_name, dict in results.items():
+    for list_name, dict in lists.items():
         list_singular_name = get_cursorless_list_name(list_name)
         ctx.lists[list_singular_name] = dict
         if list_name in pluralize_lists:
@@ -410,7 +461,7 @@ def get_full_path(filename: str):
     return (settings_directory / filename).resolve()
 
 
-def get_super_values(values: dict[str, dict[str, str]]):
+def get_super_values(values: ListToSpokenForms):
     result: dict[str, str] = {}
     for value_dict in values.values():
         result.update(value_dict)

src/marks/decorated_mark.py

@@ -138,7 +138,7 @@ def setup_hat_styles_csv(hat_colors: dict[str, str], hat_shapes: dict[str, str])
             "hat_color": active_hat_colors,
             "hat_shape": active_hat_shapes,
         },
-        [*hat_colors.values(), *hat_shapes.values()],
+        extra_ignored_values=[*hat_colors.values(), *hat_shapes.values()],
         no_update_file=is_shape_error or is_color_error,
     )