Skip to content

Commit 21e192e

Browse files
committed
database: migrate "base"
1 parent e4cdd3b commit 21e192e

File tree

9 files changed

+831
-298
lines changed

9 files changed

+831
-298
lines changed

src/packages/database/DB_DEVELOPMENT.md

Lines changed: 315 additions & 239 deletions
Large diffs are not rendered by default.

src/packages/database/postgres-base.coffee

Lines changed: 12 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ required = defaults.required
5959

6060
metrics = require('@cocalc/backend/metrics')
6161

62+
# Group 1: Database Utilities - TypeScript implementations
63+
UtilTS = require('./postgres/core/util')
64+
6265
exports.PUBLIC_PROJECT_COLUMNS = ['project_id', 'last_edited', 'title', 'description', 'deleted', 'created', 'env']
6366
exports.PROJECT_COLUMNS = ['users'].concat(exports.PUBLIC_PROJECT_COLUMNS)
6467

@@ -115,7 +118,7 @@ class exports.PostgreSQL extends EventEmitter # emits a 'connect' event whene
115118
@connect() # start trying to connect
116119

117120
clear_cache: =>
118-
@_query_cache?.clear()
121+
UtilTS.clearCache(@)
119122

120123
close: =>
121124
if @_state == 'closed'
@@ -153,7 +156,7 @@ class exports.PostgreSQL extends EventEmitter # emits a 'connect' event whene
153156
clearInterval(@_test_query)
154157
delete @_test_query
155158

156-
engine: -> 'postgresql'
159+
engine: -> UtilTS.engine()
157160

158161
connect: (opts) =>
159162
opts = defaults opts,
@@ -396,24 +399,10 @@ class exports.PostgreSQL extends EventEmitter # emits a 'connect' event whene
396399
return db?.query.bind(db)
397400

398401
_dbg: (f) =>
399-
if @_debug
400-
return (m) => winston.debug("PostgreSQL.#{f}: #{misc.trunc_middle(JSON.stringify(m), 250)}")
401-
else
402-
return ->
402+
return UtilTS.dbg(@, f)
403403

404404
_init_metrics: =>
405-
# initialize metrics
406-
try
407-
@query_time_histogram = metrics.newHistogram('db', 'query_ms_histogram', 'db queries'
408-
buckets : [1, 5, 10, 20, 50, 100, 200, 500, 1000, 5000, 10000]
409-
labels: ['table']
410-
)
411-
@concurrent_counter = metrics.newCounter('db', 'concurrent_total',
412-
'Concurrent queries (started and finished)',
413-
['state']
414-
)
415-
catch err
416-
@_dbg("_init_metrics")("WARNING -- #{err}")
405+
UtilTS.initMetrics(@)
417406

418407
async_query: (opts) =>
419408
return await callback2(@_query.bind(@), opts)
@@ -864,32 +853,7 @@ class exports.PostgreSQL extends EventEmitter # emits a 'connect' event whene
864853
return true
865854

866855
_ensure_database_exists: (cb) =>
867-
dbg = @_dbg("_ensure_database_exists")
868-
dbg("ensure database '#{@_database}' exists")
869-
args = ['--user', @_user, '--host', @_host.split(',')[0], '--port', @_port, '--list', '--tuples-only']
870-
sslEnv = sslConfigToPsqlEnv(@_ssl)
871-
dbg("psql #{args.join(' ')}")
872-
misc_node.execute_code
873-
command : 'psql'
874-
args : args
875-
env : Object.assign sslEnv,
876-
PGPASSWORD : @_password
877-
cb : (err, output) =>
878-
if err
879-
cb(err)
880-
return
881-
databases = (x.split('|')[0].trim() for x in output.stdout.split('\n') when x)
882-
if @_database in databases
883-
dbg("database '#{@_database}' already exists")
884-
cb()
885-
return
886-
dbg("creating database '#{@_database}'")
887-
misc_node.execute_code
888-
command : 'createdb'
889-
args : ['--host', @_host, '--port', @_port, @_database]
890-
env :
891-
PGPASSWORD : @_password
892-
cb : cb
856+
return UtilTS.ensureDatabaseExists(@, cb)
893857

894858
_confirm_delete: (opts) =>
895859
opts = defaults opts,
@@ -1026,10 +990,10 @@ class exports.PostgreSQL extends EventEmitter # emits a 'connect' event whene
1026990

1027991
# Return the number of outstanding concurrent queries.
1028992
concurrent: =>
1029-
return @_concurrent_queries ? 0
993+
return UtilTS.concurrent(@)
1030994

1031995
is_heavily_loaded: =>
1032-
return @_concurrent_queries >= @_concurrent_heavily_loaded
996+
return UtilTS.isHeavilyLoaded(@)
1033997

1034998
# Compute the sha1 hash (in hex) of the input arguments, which are
1035999
# converted to strings (via json) if they are not strings, then concatenated.
@@ -1038,8 +1002,7 @@ class exports.PostgreSQL extends EventEmitter # emits a 'connect' event whene
10381002
# wouldn't be the end of the world. There is a similar client-only slower version
10391003
# of this function (in schema.coffee), so don't change it willy nilly.
10401004
sha1: (args...) ->
1041-
v = ((if typeof(x) == 'string' then x else JSON.stringify(x)) for x in args).join('')
1042-
return misc_node.sha1(v)
1005+
return UtilTS.sha1Hash(args...)
10431006

10441007
# Go through every table in the schema with a column called "expire", and
10451008
# delete every entry where expire is <= right now.
@@ -1084,7 +1047,7 @@ class exports.PostgreSQL extends EventEmitter # emits a 'connect' event whene
10841047

10851048
# sanitize strings before inserting them into a query string
10861049
sanitize: (s) =>
1087-
escapeString(s)
1050+
return UtilTS.sanitize(s)
10881051

10891052
###
10901053
Other misc functions

src/packages/database/postgres-server-queries.ts

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,7 @@
11
/*
2-
* decaffeinate suggestions:
3-
* DS002: Fix invalid constructor
4-
* DS101: Remove unnecessary use of Array.from
5-
* DS102: Remove unnecessary code created because of implicit returns
6-
* DS207: Consider shorter variations of null checks
7-
* Full docs: https://github.com/decaffeinate/decaffeinate/blob/main/docs/suggestions.md
2+
* This file is part of CoCalc: Copyright © 2025 Sagemath, Inc.
3+
* License: MS-RSL – see LICENSE.md for details
84
*/
9-
//########################################################################
10-
// This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
11-
// License: MS-RSL – see LICENSE.md for details
12-
//########################################################################
135

146
/*
157
PostgreSQL -- implementation of all the queries needed for the backend servers
Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
/*
2+
* This file is part of CoCalc: Copyright © 2025 Sagemath, Inc.
3+
* License: MS-RSL – see LICENSE.md for details
4+
*/
5+
6+
/*
7+
Group 1: Database Utilities - Core utility methods for PostgreSQL class
8+
9+
Tests for 9 utility methods:
10+
- _dbg(f) - Debug logger factory
11+
- _init_metrics() - Initialize Prometheus metrics
12+
- concurrent() - Get concurrent query count
13+
- is_heavily_loaded() - Check if heavily loaded
14+
- sha1(...args) - Generate SHA1 hash
15+
- sanitize(s) - Escape string for SQL
16+
- clear_cache() - Clear LRU cache
17+
- engine() - Return 'postgresql'
18+
- _ensure_database_exists(cb) - Create database if missing
19+
20+
TDD Workflow:
21+
- USE_TYPESCRIPT = false: Test against CoffeeScript implementation (db())
22+
- USE_TYPESCRIPT = true: Test against TypeScript implementation (direct import)
23+
*/
24+
25+
import { db } from "@cocalc/database";
26+
import { initEphemeralDatabase } from "@cocalc/database/pool";
27+
import { testCleanup } from "@cocalc/database/test-utils";
28+
29+
// These tests call CoffeeScript methods via db(), which now delegate to TypeScript implementations
30+
31+
describe("Database Utilities - Group 1", () => {
32+
let database: any; // Singleton database instance
33+
34+
beforeAll(async () => {
35+
await initEphemeralDatabase({});
36+
database = db(); // Get the singleton
37+
}, 15000);
38+
39+
afterAll(async () => {
40+
await testCleanup();
41+
});
42+
43+
describe("_dbg - Debug logger factory", () => {
44+
it("returns a debug function when debug is enabled", () => {
45+
const dbgFn = database._dbg("test_method");
46+
expect(typeof dbgFn).toBe("function");
47+
});
48+
49+
it("returns a no-op function when debug is disabled", () => {
50+
const database = db();
51+
// Save original debug state
52+
const originalDebug = database._debug;
53+
54+
// Disable debug
55+
database._debug = false;
56+
const dbgFn = database._dbg("test_method");
57+
expect(typeof dbgFn).toBe("function");
58+
59+
// Calling it should not throw
60+
expect(() => dbgFn("test message")).not.toThrow();
61+
62+
// Restore original state
63+
database._debug = originalDebug;
64+
});
65+
66+
it("logs messages with method name prefix", () => {
67+
const database = db();
68+
const originalDebug = database._debug;
69+
70+
database._debug = true;
71+
const dbgFn = database._dbg("test_method");
72+
73+
// Should not throw when called
74+
expect(() => dbgFn({ test: "data" })).not.toThrow();
75+
76+
database._debug = originalDebug;
77+
});
78+
});
79+
80+
describe("_init_metrics - Initialize Prometheus metrics", () => {
81+
it("initializes metrics without error", () => {
82+
const database = db();
83+
expect(() => database._init_metrics()).not.toThrow();
84+
});
85+
86+
it("creates query_time_histogram metric", () => {
87+
const database = db();
88+
database._init_metrics();
89+
expect(database.query_time_histogram).toBeDefined();
90+
});
91+
92+
it("creates concurrent_counter metric", () => {
93+
const database = db();
94+
database._init_metrics();
95+
expect(database.concurrent_counter).toBeDefined();
96+
});
97+
98+
it("handles metric initialization errors gracefully", () => {
99+
const database = db();
100+
// Should not throw even if metrics are already initialized
101+
expect(() => database._init_metrics()).not.toThrow();
102+
expect(() => database._init_metrics()).not.toThrow();
103+
});
104+
});
105+
106+
describe("concurrent - Get concurrent query count", () => {
107+
it("returns 0 when no queries are running", () => {
108+
const database = db();
109+
const count = database.concurrent();
110+
expect(count).toBe(0);
111+
});
112+
113+
it("returns a non-negative number", () => {
114+
const database = db();
115+
const count = database.concurrent();
116+
expect(typeof count).toBe("number");
117+
expect(count).toBeGreaterThanOrEqual(0);
118+
});
119+
});
120+
121+
describe("is_heavily_loaded - Check if heavily loaded", () => {
122+
it("returns false when concurrent queries are low", () => {
123+
const database = db();
124+
const loaded = database.is_heavily_loaded();
125+
expect(typeof loaded).toBe("boolean");
126+
// With no queries running, should not be heavily loaded
127+
expect(loaded).toBe(false);
128+
});
129+
130+
it("returns a boolean value", () => {
131+
const database = db();
132+
const loaded = database.is_heavily_loaded();
133+
expect(typeof loaded).toBe("boolean");
134+
});
135+
});
136+
137+
describe("sha1 - Generate SHA1 hash", () => {
138+
it("generates consistent hash for same input", () => {
139+
const database = db();
140+
const hash1 = database.sha1("test", "data");
141+
const hash2 = database.sha1("test", "data");
142+
expect(hash1).toBe(hash2);
143+
});
144+
145+
it("generates different hashes for different inputs", () => {
146+
const database = db();
147+
const hash1 = database.sha1("test", "data");
148+
const hash2 = database.sha1("different", "data");
149+
expect(hash1).not.toBe(hash2);
150+
});
151+
152+
it("handles object inputs by JSON stringifying", () => {
153+
const database = db();
154+
const hash = database.sha1({ foo: "bar" }, { baz: 123 });
155+
expect(typeof hash).toBe("string");
156+
expect(hash.length).toBe(40); // SHA1 produces 40 hex characters
157+
});
158+
159+
it("handles mixed string and object inputs", () => {
160+
const database = db();
161+
const hash = database.sha1("prefix", { data: "value" }, "suffix");
162+
expect(typeof hash).toBe("string");
163+
expect(hash.length).toBe(40);
164+
});
165+
166+
it("produces valid hex string", () => {
167+
const database = db();
168+
const hash = database.sha1("test");
169+
expect(hash).toMatch(/^[0-9a-f]{40}$/);
170+
});
171+
});
172+
173+
describe("sanitize - Escape string for SQL", () => {
174+
it("sanitizes simple strings", () => {
175+
const database = db();
176+
const result = database.sanitize("test string");
177+
expect(typeof result).toBe("string");
178+
});
179+
180+
it("escapes single quotes", () => {
181+
const database = db();
182+
const result = database.sanitize("test's string");
183+
// SQL escaping uses doubled single quotes
184+
expect(result).toBe("'test''s string'");
185+
});
186+
187+
it("handles empty string", () => {
188+
const database = db();
189+
const result = database.sanitize("");
190+
expect(typeof result).toBe("string");
191+
});
192+
193+
it("prevents SQL injection attempts", () => {
194+
const database = db();
195+
const malicious = "'; DROP TABLE users; --";
196+
const result = database.sanitize(malicious);
197+
// Should be safely escaped
198+
expect(result).not.toBe(malicious);
199+
});
200+
});
201+
202+
describe("clear_cache - Clear LRU cache", () => {
203+
it("clears cache without error", () => {
204+
const database = db();
205+
expect(() => database.clear_cache()).not.toThrow();
206+
});
207+
208+
it("can be called multiple times", () => {
209+
const database = db();
210+
expect(() => {
211+
database.clear_cache();
212+
database.clear_cache();
213+
database.clear_cache();
214+
}).not.toThrow();
215+
});
216+
});
217+
218+
describe("engine - Return database engine identifier", () => {
219+
it("returns 'postgresql'", () => {
220+
const database = db();
221+
expect(database.engine()).toBe("postgresql");
222+
});
223+
});
224+
225+
describe("_ensure_database_exists - Create database if missing", () => {
226+
it("completes without error for existing database", async () => {
227+
const database = db();
228+
229+
await new Promise<void>((resolve, reject) => {
230+
database._ensure_database_exists((err) => {
231+
if (err) {
232+
reject(err);
233+
} else {
234+
resolve();
235+
}
236+
});
237+
});
238+
});
239+
240+
it("uses callback pattern correctly", (done) => {
241+
const database = db();
242+
243+
database._ensure_database_exists((_err) => {
244+
// Should complete (may succeed or fail depending on environment)
245+
// The important thing is the callback is called
246+
done();
247+
});
248+
}, 10000); // Longer timeout for shell commands
249+
});
250+
});

0 commit comments

Comments
 (0)