Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,12 @@ POSTGRES_DB=bunbooru
POSTGRES_PORT=5432
DATABASE_URL=postgres://bunbooru:bunbooru@localhost:5432/bunbooru

# Integration tests (optional) — opt-in URL for the destructive integration
# suite, which TRUNCATEs between cases. Kept separate from DATABASE_URL so the
# tests can never wipe a real database; unset → those tests skip. Point it at a
# THROWAWAY database only.
# TEST_DATABASE_URL=postgres://bunbooru:bunbooru@localhost:5432/bunbooru_test

# Redis (optional accelerator) — only when running the `redis` compose profile
REDIS_PORT=6379
# REDIS_URL=redis://localhost:6379
4 changes: 4 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ jobs:

env:
DATABASE_URL: postgres://bunbooru:bunbooru@localhost:5432/bunbooru
# Destructive integration tests (they TRUNCATE) only run against this
# explicit, opt-in URL — never the app's DATABASE_URL. Safe here because
# the Postgres above is an ephemeral, per-job service container.
TEST_DATABASE_URL: postgres://bunbooru:bunbooru@localhost:5432/bunbooru

steps:
- uses: actions/checkout@v4
Expand Down
3 changes: 3 additions & 0 deletions apps/api/src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -120,11 +120,14 @@ export function createApp({ core, maxUploadBytes }: AppDependencies) {
const page = await core.assetService.list({
page: query.page,
perPage: query.per_page,
query: query.q,
});
return { ...page, assets: page.assets.map(serializeAsset) };
},
{
query: t.Object({
// Booru search query (tags + metatags); Core compiles it to SQL.
q: t.Optional(t.String({ maxLength: 1024 })),
// Integer-only (`multipleOf: 1`): reject fractional page sizes at
// the boundary so offset/limit math is never ambiguous. The upper
// bound keeps (page-1)*perPage within MAX_SAFE_INTEGER so a huge
Expand Down
15 changes: 15 additions & 0 deletions apps/api/test/server.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,21 @@ describe("GET /api/v1/assets", () => {
expect(received).toEqual({ page: 2, perPage: 5 });
});

it("forwards the q search query to the service", async () => {
let received: ListAssetsOptions | undefined;
const res = await buildApp(
stubCore({
list: async (options = {}) => {
received = options;
return emptyPage;
},
}),
).handle(new Request("http://localhost/api/v1/assets?q=rating%3Asafe+1girl"));

expect(res.status).toBe(200);
expect(received?.query).toBe("rating:safe 1girl");
});

it("rejects an out-of-range page with 422", async () => {
const res = await request("/api/v1/assets?page=0");
expect(res.status).toBe(422);
Expand Down
1 change: 1 addition & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"@bunbooru/db": "workspace:*",
"@bunbooru/events": "workspace:*",
"@bunbooru/search": "workspace:*",
"@bunbooru/storage": "workspace:*"
"@bunbooru/storage": "workspace:*",
"drizzle-orm": "^0.45.2"
}
}
108 changes: 108 additions & 0 deletions packages/core/src/search/asset-query.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
import { and, between, eq, gt, gte, lt, lte, not, or, sql, type SQL } from "drizzle-orm";

import { assets, assetTags, tags, type Rating } from "@bunbooru/db";
import { parse, type MetatagNode, type QueryNode } from "@bunbooru/search";

/**
* The Search Engine (Core-owned per CLAUDE.md). Compiles a query string into a
* Drizzle `WHERE` over the assets table: `@bunbooru/search` parses text → AST,
* and this walks the AST → SQL. `db` never sees text; it only runs the condition.
*
* rating:safe width:>1000 1girl -monochrome
* → rating = 'safe' AND width > 1000 AND <tag 1girl> AND NOT <tag monochrome>
*
* Metatags map to asset columns and filter immediately. Tag terms compile to an
* `asset_tags` EXISTS subquery — valid SQL today, matching once tagging lands.
*/
export function compileAssetSearch(query: string): SQL | undefined {
return buildNode(parse(query));
}

const RATINGS = new Set<Rating>(["safe", "questionable", "explicit"]);
const isRating = (value: string): value is Rating => RATINGS.has(value as Rating);

/** Metatag keys that map to a numeric asset column. */
const NUMERIC_COLUMNS = {
width: assets.width,
height: assets.height,
size: assets.sizeBytes,
} as const;
type NumericColumn = (typeof NUMERIC_COLUMNS)[keyof typeof NUMERIC_COLUMNS];

function buildNode(node: QueryNode): SQL | undefined {
switch (node.type) {
case "and":
// Drizzle's and/or drop undefined args, so unknown/empty terms just vanish.
return and(...node.children.map(buildNode));
case "or":
return or(...node.children.map(buildNode));
case "not": {
const child = buildNode(node.child);
return child ? not(child) : undefined;
}
case "tag":
return tagCondition(node.name);
case "metatag":
return metatagCondition(node);
}
}

/** Escape LIKE metacharacters, then map the tag glob `*` to SQL `%`. */
function likePattern(name: string): string {
return name.replace(/[\\%_]/g, "\\$&").replace(/\*/g, "%");
}

/** An asset has a tag matching `name` (supports `*` wildcards). */
function tagCondition(name: string): SQL {
const match = name.includes("*")
? sql`${tags.name} like ${likePattern(name)}`
: sql`${tags.name} = ${name}`;
return sql`exists (select 1 from ${assetTags} inner join ${tags} on ${assetTags.tagId} = ${tags.id} where ${assetTags.assetId} = ${assets.id} and ${match})`;
}

function metatagCondition(node: MetatagNode): SQL | undefined {
if (node.key === "rating") {
// Only equality is meaningful for an enum; an invalid value matches nothing.
return node.op === "eq" && isRating(node.value) ? eq(assets.rating, node.value) : undefined;
}

const column = NUMERIC_COLUMNS[node.key as keyof typeof NUMERIC_COLUMNS];
if (!column) return undefined; // unknown metatag key → ignored

if (node.op === "range") return rangeCondition(column, node.value);

const n = Number(node.value);
if (!Number.isFinite(n)) return undefined;
switch (node.op) {
case "eq":
return eq(column, n);
case "gt":
return gt(column, n);
case "gte":
return gte(column, n);
case "lt":
return lt(column, n);
case "lte":
return lte(column, n);
default:
return undefined;
}
}

/** `lo..hi`, `lo..`, or `..hi` → bounded/half-open numeric range. */
function rangeCondition(column: NumericColumn, value: string): SQL | undefined {
// Exactly one `..` separates the bounds; anything else (e.g. `1..2..3`) is
// malformed and ignored rather than silently truncated to the first two parts.
const parts = value.split("..");
if (parts.length !== 2) return undefined;
const [loRaw, hiRaw] = parts;
const lo = loRaw ? Number(loRaw) : undefined;
const hi = hiRaw ? Number(hiRaw) : undefined;
if ((lo !== undefined && !Number.isFinite(lo)) || (hi !== undefined && !Number.isFinite(hi))) {
return undefined;
}
if (lo !== undefined && hi !== undefined) return between(column, lo, hi);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
if (lo !== undefined) return gte(column, lo);
if (hi !== undefined) return lte(column, hi);
return undefined;
}
13 changes: 11 additions & 2 deletions packages/core/src/services/asset-service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type { Asset, AssetRepository, NewAsset } from "@bunbooru/db";
import type { StorageProvider } from "@bunbooru/storage";

import { UnsupportedMediaError } from "../errors";
import { compileAssetSearch } from "../search/asset-query";

/** Default and ceiling page sizes — the ceiling bounds the cost of one query. */
export const DEFAULT_PER_PAGE = 20;
Expand Down Expand Up @@ -43,6 +44,8 @@ export interface AssetListPage {
export interface ListAssetsOptions {
page?: number;
perPage?: number;
/** Booru query string (tags + metatags); compiled to a SQL filter. */
query?: string;
}

/** Raw upload: the bytes plus optional metadata the client may set. */
Expand Down Expand Up @@ -109,10 +112,16 @@ export function createAssetService(
const page = Math.min(toPositiveInt(options.page, 1), maxPage);
const offset = (page - 1) * perPage;

// Compile the query string to a SQL filter once; the count must use the
// same filter so pagination metadata matches the page. Trim first so
// surrounding whitespace is normalized away, not fed to the parser.
const query = options.query?.trim();
const where = query ? compileAssetSearch(query) : undefined;

// Count and page in parallel — they're independent reads.
const [list, total] = await Promise.all([
repository.findMany({ limit: perPage, offset }),
repository.count(),
repository.findMany({ limit: perPage, offset, where }),
repository.count(where),
]);

return {
Expand Down
120 changes: 120 additions & 0 deletions packages/core/test/asset-search.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
import { beforeAll, beforeEach, describe, expect, it } from "bun:test";
import { sql } from "drizzle-orm";

import {
createAssetRepository,
createDb,
type AssetRepository,
type DB,
type NewAsset,
} from "@bunbooru/db";
import type { StorageProvider } from "@bunbooru/storage";

import { createAssetService, type AssetService } from "../src/services/asset-service";

/**
* Integration tests for the Search Engine (AST → SQL). They hit a real Postgres
* and TRUNCATE between cases, so they read a dedicated, opt-in `TEST_DATABASE_URL`
* — never the app's `DATABASE_URL` — so a misconfigured dev/prod env can't be
* wiped. Skipped when unset, keeping a bare `bun test` green without a database.
*/
const TEST_DATABASE_URL = Bun.env.TEST_DATABASE_URL?.trim();

/** `list` never touches storage, so a throwing stub is fine here. */
const noopStorage: StorageProvider = {
store: async () => undefined,
delete: async () => undefined,
exists: async () => false,
stream: async () => {
throw new Error("unused");
},
copy: async () => undefined,
move: async () => undefined,
getPublicUrl: async () => null,
};

const hex = (algo: "sha256" | "md5", s: string) =>
new Bun.CryptoHasher(algo).update(s).digest("hex");

function asset(key: string, over: Partial<NewAsset>): NewAsset {
return {
storageKey: `k/${key}`,
mimeType: "image/png",
width: 100,
height: 100,
sizeBytes: 1,
sha256: hex("sha256", key),
md5: hex("md5", key),
...over,
};
}

describe.skipIf(!TEST_DATABASE_URL)("Search execution (integration)", () => {
let db: DB;
let repo: AssetRepository;
let service: AssetService;

beforeAll(() => {
db = createDb(TEST_DATABASE_URL as string);
repo = createAssetRepository(db);
service = createAssetService(repo, noopStorage);
});

// Three assets: a(safe,1920×1080) b(explicit,500×800) c(safe,800×600).
beforeEach(async () => {
await db.execute(sql`TRUNCATE TABLE assets, asset_tags RESTART IDENTITY CASCADE`);
await repo.create(asset("a", { rating: "safe", width: 1920, height: 1080 }));
await repo.create(asset("b", { rating: "explicit", width: 500, height: 800 }));
await repo.create(asset("c", { rating: "safe", width: 800, height: 600 }));
});

/** Sorted storage keys of a result page, for set comparison. */
const keys = (p: { assets: Array<{ storageKey: string }> }) =>
p.assets.map((a) => a.storageKey).sort();

it("returns everything for an empty query", async () => {
const r = await service.list({});
expect(r.total).toBe(3);
});

it("filters by rating (enum equality)", async () => {
const r = await service.list({ query: "rating:safe" });
expect(r.total).toBe(2);
expect(keys(r)).toEqual(["k/a", "k/c"]);
});

it("filters by numeric comparison", async () => {
expect(keys(await service.list({ query: "width:>1000" }))).toEqual(["k/a"]);
expect(keys(await service.list({ query: "height:<=800" }))).toEqual(["k/b", "k/c"]);
});

it("ANDs multiple metatags", async () => {
const r = await service.list({ query: "rating:safe width:>=800" });
expect(keys(r)).toEqual(["k/a", "k/c"]);
});

it("supports numeric ranges", async () => {
expect(keys(await service.list({ query: "width:600..900" }))).toEqual(["k/c"]);
expect(keys(await service.list({ query: "width:..600" }))).toEqual(["k/b"]);
});

it("ignores a malformed multi-bound range (not silently truncated)", async () => {
// `1..2..3` is invalid syntax: the term is dropped, not compiled to 1..2.
expect((await service.list({ query: "width:1..2..3" })).total).toBe(3);
});

it("negates a term", async () => {
expect(keys(await service.list({ query: "-rating:safe" }))).toEqual(["k/b"]);
});

it("compiles tag terms but matches nothing until tags exist", async () => {
expect((await service.list({ query: "1girl" })).total).toBe(0);
});

it("reflects the filter in pagination metadata", async () => {
const r = await service.list({ query: "rating:safe", perPage: 1 });
expect(r.total).toBe(2);
expect(r.pageCount).toBe(2);
expect(r.assets).toHaveLength(1);
});
});
Loading
Loading