Add token count post

Author: Kayce Basques

BUILD.bazel

@@ -43,6 +43,8 @@ sphinx_docs_library(
         "ai/agents/agents.png",
         "ai/agents/index.rst",
         "ai/index.rst",
+        "ai/tokens.py",
+        "ai/tokens.rst",
         "analytics/index.rst",
         "analytics/sheets.rst",
         "index.rst",

ai/index.rst

@@ -6,4 +6,5 @@ Artificial intelligence
    :hidden:
    :maxdepth: 1
 
-   agents/index
+   agents/index
+   tokens

ai/tokens.py

@@ -0,0 +1,75 @@
+from json import dump, load
+from os import environ, walk
+from pathlib import Path
+from subprocess import run
+from typing import Dict
+
+from google.genai import Client
+
+
+gemini = Client(api_key=environ["GEMINI_API_KEY"])
+
+
+def _is_ignored(root: Path, target: Path, ignored: list[Path]) -> bool:
+    """Check if Git is ignoring the path."""
+    # Ignore Git's directory itself.
+    if str(target).lower().endswith(".git"):
+        return True
+    # Check if this path matches something in ``.gitignore``.
+    command = ["git", "-C", str(root), "check-ignore", str(target)]
+    result = run(command, capture_output=True, text=True)
+    return str(target) in result.stdout
+
+
+def _is_in_ignored_dir(target: Path, ignored: list[Path]):
+    """Check if this path is in an ignored directory."""
+    for maybe_parent_dir in ignored:
+        if str(maybe_parent_dir) in str(target):
+            return True
+    return False
+
+
+def collect(root: Path) -> (list[Path], int):
+    """Collect all paths in the repository."""
+    paths: list[Path] = []
+    ignored: list[Path] = []
+    tokens = 0
+    for current_working_dir, _, files in walk(root):
+        cwd = Path(current_working_dir)
+        if _is_in_ignored_dir(cwd, ignored):
+            print(f"ignoring dir: {str(cwd)}")
+            continue
+        if _is_ignored(Path(root), cwd, ignored):
+            print(f"ignoring dir: {str(cwd)}")
+            ignored.append(cwd)
+            continue
+        for file in files:
+            path = cwd / Path(file)
+            if _is_ignored(Path(root), path, ignored):
+                print(f"ignoring file: {str(path)}")
+                ignored.append(path)
+                continue
+            paths.append(path)
+            with open(path, "r") as f:
+                try:
+                    contents = f.read()
+                except UnicodeDecodeError as e:
+                    continue
+                print(f"counting tokens: {str(path)}")
+                response = gemini.models.count_tokens(
+                    model="gemini-2.5-flash", contents=contents
+                )
+                tokens += response.total_tokens
+    return (paths, tokens)
+
+
+def main():
+    root = Path(".")
+    paths, tokens = collect(root)
+    print("*" * 80)
+    print(f"file count: {len(paths)}")
+    print(f"tokens: {tokens}")
+
+
+if __name__ == "__main__":
+    main()

ai/tokens.rst

@@ -0,0 +1,8 @@
+================================
+Git-aware token count for a repo
+================================
+
+"Git-aware" means that this script ignores files that match a ``.gitignore`` pattern.
+
+.. literalinclude:: ./tokens.py
+   :language: py