Repository navigation
feat(search): execute booru queries end-to-end (AST → SQL) #11
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| import { and, between, eq, gt, gte, lt, lte, not, or, sql, type SQL } from "drizzle-orm"; | ||
|
|
||
| import { assets, assetTags, tags, type Rating } from "@bunbooru/db"; | ||
| import { parse, type MetatagNode, type QueryNode } from "@bunbooru/search"; | ||
|
|
||
| /** | ||
| * The Search Engine (Core-owned per CLAUDE.md). Compiles a query string into a | ||
| * Drizzle `WHERE` over the assets table: `@bunbooru/search` parses text → AST, | ||
| * and this walks the AST → SQL. `db` never sees text; it only runs the condition. | ||
| * | ||
| * rating:safe width:>1000 1girl -monochrome | ||
| * → rating = 'safe' AND width > 1000 AND <tag 1girl> AND NOT <tag monochrome> | ||
| * | ||
| * Metatags map to asset columns and filter immediately. Tag terms compile to an | ||
| * `asset_tags` EXISTS subquery — valid SQL today, matching once tagging lands. | ||
| */ | ||
| export function compileAssetSearch(query: string): SQL | undefined { | ||
| return buildNode(parse(query)); | ||
| } | ||
|
|
||
| const RATINGS = new Set<Rating>(["safe", "questionable", "explicit"]); | ||
| const isRating = (value: string): value is Rating => RATINGS.has(value as Rating); | ||
|
|
||
| /** Metatag keys that map to a numeric asset column. */ | ||
| const NUMERIC_COLUMNS = { | ||
| width: assets.width, | ||
| height: assets.height, | ||
| size: assets.sizeBytes, | ||
| } as const; | ||
| type NumericColumn = (typeof NUMERIC_COLUMNS)[keyof typeof NUMERIC_COLUMNS]; | ||
|
|
||
| function buildNode(node: QueryNode): SQL | undefined { | ||
| switch (node.type) { | ||
| case "and": | ||
| // Drizzle's and/or drop undefined args, so unknown/empty terms just vanish. | ||
| return and(...node.children.map(buildNode)); | ||
| case "or": | ||
| return or(...node.children.map(buildNode)); | ||
| case "not": { | ||
| const child = buildNode(node.child); | ||
| return child ? not(child) : undefined; | ||
| } | ||
| case "tag": | ||
| return tagCondition(node.name); | ||
| case "metatag": | ||
| return metatagCondition(node); | ||
| } | ||
| } | ||
|
|
||
| /** Escape LIKE metacharacters, then map the tag glob `*` to SQL `%`. */ | ||
| function likePattern(name: string): string { | ||
| return name.replace(/[\\%_]/g, "\\$&").replace(/\*/g, "%"); | ||
| } | ||
|
|
||
| /** An asset has a tag matching `name` (supports `*` wildcards). */ | ||
| function tagCondition(name: string): SQL { | ||
| const match = name.includes("*") | ||
| ? sql`${tags.name} like ${likePattern(name)}` | ||
| : sql`${tags.name} = ${name}`; | ||
| return sql`exists (select 1 from ${assetTags} inner join ${tags} on ${assetTags.tagId} = ${tags.id} where ${assetTags.assetId} = ${assets.id} and ${match})`; | ||
| } | ||
|
|
||
| function metatagCondition(node: MetatagNode): SQL | undefined { | ||
| if (node.key === "rating") { | ||
| // Only equality is meaningful for an enum; an invalid value matches nothing. | ||
| return node.op === "eq" && isRating(node.value) ? eq(assets.rating, node.value) : undefined; | ||
| } | ||
|
|
||
| const column = NUMERIC_COLUMNS[node.key as keyof typeof NUMERIC_COLUMNS]; | ||
| if (!column) return undefined; // unknown metatag key → ignored | ||
|
|
||
| if (node.op === "range") return rangeCondition(column, node.value); | ||
|
|
||
| const n = Number(node.value); | ||
| if (!Number.isFinite(n)) return undefined; | ||
| switch (node.op) { | ||
| case "eq": | ||
| return eq(column, n); | ||
| case "gt": | ||
| return gt(column, n); | ||
| case "gte": | ||
| return gte(column, n); | ||
| case "lt": | ||
| return lt(column, n); | ||
| case "lte": | ||
| return lte(column, n); | ||
| default: | ||
| return undefined; | ||
| } | ||
| } | ||
|
|
||
| /** `lo..hi`, `lo..`, or `..hi` → bounded/half-open numeric range. */ | ||
| function rangeCondition(column: NumericColumn, value: string): SQL | undefined { | ||
| // Exactly one `..` separates the bounds; anything else (e.g. `1..2..3`) is | ||
| // malformed and ignored rather than silently truncated to the first two parts. | ||
| const parts = value.split(".."); | ||
| if (parts.length !== 2) return undefined; | ||
| const [loRaw, hiRaw] = parts; | ||
| const lo = loRaw ? Number(loRaw) : undefined; | ||
| const hi = hiRaw ? Number(hiRaw) : undefined; | ||
| if ((lo !== undefined && !Number.isFinite(lo)) || (hi !== undefined && !Number.isFinite(hi))) { | ||
| return undefined; | ||
| } | ||
| if (lo !== undefined && hi !== undefined) return between(column, lo, hi); | ||
| if (lo !== undefined) return gte(column, lo); | ||
| if (hi !== undefined) return lte(column, hi); | ||
| return undefined; | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| import { beforeAll, beforeEach, describe, expect, it } from "bun:test"; | ||
| import { sql } from "drizzle-orm"; | ||
|
|
||
| import { | ||
| createAssetRepository, | ||
| createDb, | ||
| type AssetRepository, | ||
| type DB, | ||
| type NewAsset, | ||
| } from "@bunbooru/db"; | ||
| import type { StorageProvider } from "@bunbooru/storage"; | ||
|
|
||
| import { createAssetService, type AssetService } from "../src/services/asset-service"; | ||
|
|
||
| /** | ||
| * Integration tests for the Search Engine (AST → SQL). They hit a real Postgres | ||
| * and TRUNCATE between cases, so they read a dedicated, opt-in `TEST_DATABASE_URL` | ||
| * — never the app's `DATABASE_URL` — so a misconfigured dev/prod env can't be | ||
| * wiped. Skipped when unset, keeping a bare `bun test` green without a database. | ||
| */ | ||
| const TEST_DATABASE_URL = Bun.env.TEST_DATABASE_URL?.trim(); | ||
|
|
||
| /** `list` never touches storage, so a throwing stub is fine here. */ | ||
| const noopStorage: StorageProvider = { | ||
| store: async () => undefined, | ||
| delete: async () => undefined, | ||
| exists: async () => false, | ||
| stream: async () => { | ||
| throw new Error("unused"); | ||
| }, | ||
| copy: async () => undefined, | ||
| move: async () => undefined, | ||
| getPublicUrl: async () => null, | ||
| }; | ||
|
|
||
| const hex = (algo: "sha256" | "md5", s: string) => | ||
| new Bun.CryptoHasher(algo).update(s).digest("hex"); | ||
|
|
||
| function asset(key: string, over: Partial<NewAsset>): NewAsset { | ||
| return { | ||
| storageKey: `k/${key}`, | ||
| mimeType: "image/png", | ||
| width: 100, | ||
| height: 100, | ||
| sizeBytes: 1, | ||
| sha256: hex("sha256", key), | ||
| md5: hex("md5", key), | ||
| ...over, | ||
| }; | ||
| } | ||
|
|
||
| describe.skipIf(!TEST_DATABASE_URL)("Search execution (integration)", () => { | ||
| let db: DB; | ||
| let repo: AssetRepository; | ||
| let service: AssetService; | ||
|
|
||
| beforeAll(() => { | ||
| db = createDb(TEST_DATABASE_URL as string); | ||
| repo = createAssetRepository(db); | ||
| service = createAssetService(repo, noopStorage); | ||
| }); | ||
|
|
||
| // Three assets: a(safe,1920×1080) b(explicit,500×800) c(safe,800×600). | ||
| beforeEach(async () => { | ||
| await db.execute(sql`TRUNCATE TABLE assets, asset_tags RESTART IDENTITY CASCADE`); | ||
| await repo.create(asset("a", { rating: "safe", width: 1920, height: 1080 })); | ||
| await repo.create(asset("b", { rating: "explicit", width: 500, height: 800 })); | ||
| await repo.create(asset("c", { rating: "safe", width: 800, height: 600 })); | ||
| }); | ||
|
|
||
| /** Sorted storage keys of a result page, for set comparison. */ | ||
| const keys = (p: { assets: Array<{ storageKey: string }> }) => | ||
| p.assets.map((a) => a.storageKey).sort(); | ||
|
|
||
| it("returns everything for an empty query", async () => { | ||
| const r = await service.list({}); | ||
| expect(r.total).toBe(3); | ||
| }); | ||
|
|
||
| it("filters by rating (enum equality)", async () => { | ||
| const r = await service.list({ query: "rating:safe" }); | ||
| expect(r.total).toBe(2); | ||
| expect(keys(r)).toEqual(["k/a", "k/c"]); | ||
| }); | ||
|
|
||
| it("filters by numeric comparison", async () => { | ||
| expect(keys(await service.list({ query: "width:>1000" }))).toEqual(["k/a"]); | ||
| expect(keys(await service.list({ query: "height:<=800" }))).toEqual(["k/b", "k/c"]); | ||
| }); | ||
|
|
||
| it("ANDs multiple metatags", async () => { | ||
| const r = await service.list({ query: "rating:safe width:>=800" }); | ||
| expect(keys(r)).toEqual(["k/a", "k/c"]); | ||
| }); | ||
|
|
||
| it("supports numeric ranges", async () => { | ||
| expect(keys(await service.list({ query: "width:600..900" }))).toEqual(["k/c"]); | ||
| expect(keys(await service.list({ query: "width:..600" }))).toEqual(["k/b"]); | ||
| }); | ||
|
|
||
| it("ignores a malformed multi-bound range (not silently truncated)", async () => { | ||
| // `1..2..3` is invalid syntax: the term is dropped, not compiled to 1..2. | ||
| expect((await service.list({ query: "width:1..2..3" })).total).toBe(3); | ||
| }); | ||
|
|
||
| it("negates a term", async () => { | ||
| expect(keys(await service.list({ query: "-rating:safe" }))).toEqual(["k/b"]); | ||
| }); | ||
|
|
||
| it("compiles tag terms but matches nothing until tags exist", async () => { | ||
| expect((await service.list({ query: "1girl" })).total).toBe(0); | ||
| }); | ||
|
|
||
| it("reflects the filter in pagination metadata", async () => { | ||
| const r = await service.list({ query: "rating:safe", perPage: 1 }); | ||
| expect(r.total).toBe(2); | ||
| expect(r.pageCount).toBe(2); | ||
| expect(r.assets).toHaveLength(1); | ||
| }); | ||
| }); |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.