Skip to content

Commit 28a2b6a

Browse files
authored
Merge pull request #37 from aniravi24/clickhouse-support
feat(typefusion): clickhouse support
2 parents db7b84d + f7fa6c7 commit 28a2b6a

File tree

17 files changed

+1112
-966
lines changed

17 files changed

+1112
-966
lines changed

.changeset/lovely-actors-cross.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"typefusion": patch
3+
---
4+
5+
Clickhouse support

.github/workflows/pr.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ jobs:
2020
# start preparing the database containers, if you run tests before these start up, you may get an error
2121
docker run -d -p 5432:5432 -e POSTGRES_DB=typefusion -e POSTGRES_PASSWORD=password postgres:alpine
2222
docker run -d -p 3306:3306 -e MYSQL_ROOT_PASSWORD=password -e MYSQL_DATABASE=typefusion mysql:latest
23+
docker run -d -p 8123:8123 clickhouse/clickhouse-server:latest
2324
2425
- name: Checkout repo
2526
uses: actions/checkout@v4
@@ -41,7 +42,7 @@ jobs:
4142
env:
4243
PG_DATABASE_URL: postgres://postgres:password@localhost:5432/typefusion?sslmode=disable
4344
MYSQL_DATABASE_URL: mysql://root:password@localhost:3306/typefusion
44-
45+
CLICKHOUSE_DATABASE_URL: http://localhost:8123/
4546
- name: Upload coverage reports to Codecov
4647
uses: codecov/codecov-action@v4
4748
env:

README.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,10 @@ To begin using Typefusion, follow these steps:
4848
bun add typefusion
4949
```
5050

51-
2. Configure your database connection using one of these methods (PostgreSQL and MySQL are supported):
51+
2. Configure your database connection using one of these methods (PostgreSQL, MySQL, and Clickhouse are supported):
5252

53-
- Set a full connection string in the `PG_DATABASE_URL` or `MYSQL_DATABASE_URL` environment variable.
54-
- Set individual environment variables: `PG_DATABASE`, `PG_HOST`, `PG_PORT`, `PG_PASSWORD`, and `PG_USER` (for postgres) or `MYSQL_DATABASE`, `MYSQL_HOST`, `MYSQL_PORT`, `MYSQL_PASSWORD`, and `MYSQL_USER` (for mysql).
53+
- Set a full connection string in the `PG_DATABASE_URL` or `MYSQL_DATABASE_URL` or `CLICKHOUSE_DATABASE_URL` (and optionally `CLICKHOUSE_USER`, `CLICKHOUSE_PASSWORD`, and `CLICKHOUSE_DATABASE` if not using the defaults) environment variable.
54+
- Set individual environment variables: `PG_DATABASE`, `PG_HOST`, `PG_PORT`, `PG_PASSWORD`, and `PG_USER` (for postgres) or `MYSQL_DATABASE`, `MYSQL_HOST`, `MYSQL_PORT`, `MYSQL_PASSWORD`, and `MYSQL_USER` (for mysql) or `CLICKHOUSE_DATABASE`, `CLICKHOUSE_HOST`, `CLICKHOUSE_PORT`, `CLICKHOUSE_PASSWORD`, and `CLICKHOUSE_USER` (for clickhouse).
5555

5656
3. Create a directory for your scripts (e.g., `workflows`).
5757

@@ -64,7 +64,7 @@ To begin using Typefusion, follow these steps:
6464
After following the above instructions, create a script file in the directory, for example, `main.ts`:
6565

6666
```ts
67-
// or mySqlType for mysql
67+
// or mySqlType for mysql or clickhouseType for clickhouse
6868
import { pgType, typefusionRef, TypefusionDbScript } from "typefusion";
6969

7070
const mainSchema = {

package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@
2626
"@commitlint/cli": "19.5.0",
2727
"@commitlint/config-conventional": "19.5.0",
2828
"@manypkg/cli": "0.22.0",
29-
"@typescript-eslint/eslint-plugin": "8.8.1",
30-
"@typescript-eslint/parser": "8.8.1",
29+
"@typescript-eslint/eslint-plugin": "8.11.0",
30+
"@typescript-eslint/parser": "8.11.0",
3131
"commitizen": "4.3.1",
3232
"cz-git": "1.10.1",
3333
"eslint": "9.13.0",
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import { clickhouseType, TypefusionDbScript } from "../../src/index.js";
2+
3+
const allClickhouseTypes = {
4+
text: clickhouseType.string().notNull(),
5+
int: clickhouseType.int32().notNull(),
6+
boolean: clickhouseType.boolean().notNull(),
7+
date: clickhouseType.date().notNull(),
8+
dateTime: clickhouseType.dateTime64(3).notNull(),
9+
bigint: clickhouseType.int64().notNull(),
10+
smallint: clickhouseType.int16().notNull(),
11+
float: clickhouseType.float32().notNull(),
12+
double: clickhouseType.float64().notNull(),
13+
decimal: clickhouseType.decimal(10, 2).notNull(),
14+
char: clickhouseType.fixedString(10).notNull(),
15+
varchar: clickhouseType.string().notNull(),
16+
time: clickhouseType.string().notNull(), // Clickhouse doesn't have a specific time type
17+
json: clickhouseType.json().notNull(),
18+
binary: clickhouseType.string().notNull(), // Using string as a close approximation
19+
};
20+
21+
export default {
22+
name: "typefusion_all_clickhouse_types",
23+
schema: allClickhouseTypes,
24+
resultDatabase: "clickhouse",
25+
run: async () => {
26+
console.log("TYPEFUSION ALL CLICKHOUSE TYPES IS RUN");
27+
return {
28+
data: [
29+
{
30+
text: "Sample text",
31+
int: 42,
32+
boolean: true,
33+
date: "2023-05-17",
34+
dateTime: "2023-05-17T12:34:56",
35+
bigint: BigInt("9007199254740991").toString(),
36+
smallint: 32767,
37+
float: 3.14,
38+
double: 3.141592653589793,
39+
decimal: 123.45,
40+
char: "Fixed ",
41+
varchar: "Variable length text",
42+
time: "12:34:56",
43+
// TODO this needs to be a json string
44+
json: { key: "value" },
45+
binary: Buffer.from([0x12]).toString("base64"), // Convert to base64 string
46+
},
47+
],
48+
};
49+
},
50+
} satisfies TypefusionDbScript<typeof allClickhouseTypes>;
Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import {
2+
typefusionRef,
3+
TypefusionDbScript,
4+
clickhouseType,
5+
} from "../../src/index.js";
6+
import main from "../main.js";
7+
8+
const smallSchema = {
9+
small: clickhouseType.string().notNull(),
10+
};
11+
12+
export default {
13+
name: "typefusion_clickhouse_result",
14+
resultDatabase: "clickhouse",
15+
schema: smallSchema,
16+
run: async () => {
17+
const result = await typefusionRef(main);
18+
console.log("TYPEFUSION CLICKHOUSE RESULT IS RUN", result);
19+
return {
20+
data: [
21+
{
22+
small: "smallString" as const,
23+
},
24+
],
25+
};
26+
},
27+
} satisfies TypefusionDbScript<typeof smallSchema>;

packages/typefusion/package.json

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,25 +42,25 @@
4242
"example-debug": "dotenv -- tsx src/cli.ts --log-level debug --ignore \"**/src/**\" ./test/examplejs"
4343
},
4444
"dependencies": {
45-
"@effect/cli": "0.46.1",
46-
"@effect/platform": "0.67.1",
47-
"@effect/platform-node": "0.62.1",
48-
"@effect/schema": "0.75.5",
49-
"@effect/sql": "0.15.1",
50-
"@effect/sql-mysql2": "0.15.2",
51-
"@effect/sql-pg": "0.15.2",
52-
"effect": "3.9.2",
45+
"@effect/cli": "0.48.5",
46+
"@effect/platform": "0.69.5",
47+
"@effect/platform-node": "0.64.6",
48+
"@effect/sql": "0.18.6",
49+
"@effect/sql-clickhouse": "0.2.6",
50+
"@effect/sql-mysql2": "0.18.6",
51+
"@effect/sql-pg": "0.18.6",
52+
"effect": "3.10.1",
5353
"postgres": "3.4.4",
5454
"skott": "0.35.3",
55-
"tslib": "2.7.0"
55+
"tslib": "2.8.0"
5656
},
5757
"devDependencies": {
58-
"@effect/vitest": "0.12.1",
58+
"@effect/vitest": "0.13.1",
5959
"@vitest/coverage-v8": "2.1.3",
6060
"dotenv-cli": "7.4.2",
6161
"tsx": "4.19.1",
6262
"type-fest": "4.26.1",
63-
"vite": "5.4.9",
63+
"vite": "5.4.10",
6464
"vite-tsconfig-paths": "5.0.1",
6565
"vitest": "2.1.3"
6666
}
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import { ClickhouseClient } from "@effect/sql-clickhouse";
2+
import { Config, Effect, Layer } from "effect";
3+
import { DatabaseHelper } from "../common/service.js";
4+
import {
5+
chDropTableIfExists,
6+
chCreateTableIfNotExists,
7+
chInsertIntoTable,
8+
chSelectAllFromTable,
9+
chColumnDDL,
10+
} from "./helpers.js";
11+
import { valueToClickhouseType, clickhouseIdColumn } from "./types.js";
12+
13+
/**
14+
* @internal
15+
*/
16+
export const ClickhouseDatabaseConfig = Config.all({
17+
databaseUrl: Config.orElse(Config.string("CLICKHOUSE_DATABASE_URL"), () =>
18+
Config.map(
19+
Config.all({
20+
CLICKHOUSE_HOST: Config.string("CLICKHOUSE_HOST"),
21+
CLICKHOUSE_PORT: Config.integer("CLICKHOUSE_PORT"),
22+
}),
23+
({ CLICKHOUSE_HOST, CLICKHOUSE_PORT }) =>
24+
`http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT}/`,
25+
),
26+
),
27+
username: Config.string("CLICKHOUSE_USER").pipe(
28+
Config.orElse(() => Config.succeed(undefined)),
29+
),
30+
password: Config.string("CLICKHOUSE_PASSWORD").pipe(
31+
Config.orElse(() => Config.succeed(undefined)),
32+
),
33+
database: Config.string("CLICKHOUSE_DATABASE").pipe(
34+
Config.orElse(() => Config.succeed(undefined)),
35+
),
36+
});
37+
38+
/**
39+
* @internal
40+
*/
41+
const ClickhouseLive = ClickhouseClient.layer({
42+
url: Config.map(ClickhouseDatabaseConfig, (c) => c.databaseUrl),
43+
username: Config.map(ClickhouseDatabaseConfig, (c) => c.username),
44+
password: Config.map(ClickhouseDatabaseConfig, (c) => c.password),
45+
database: Config.map(ClickhouseDatabaseConfig, (c) => c.database),
46+
clickhouse_settings: {
47+
allow_experimental_json_type: Config.succeed(true),
48+
},
49+
});
50+
51+
/**
52+
* @internal
53+
*/
54+
export class ClickhouseService extends Effect.Service<ClickhouseService>()(
55+
"@typefusion/clickhouse",
56+
{
57+
effect: ClickhouseClient.ClickhouseClient,
58+
dependencies: [ClickhouseLive],
59+
},
60+
) {}
61+
62+
/**
63+
* @internal
64+
*/
65+
export const ClickhouseDatabaseHelperLive = Layer.succeed(DatabaseHelper, {
66+
valueToDbType: valueToClickhouseType,
67+
idColumn: clickhouseIdColumn,
68+
dropTableIfExists: chDropTableIfExists,
69+
createTableIfNotExists: chCreateTableIfNotExists,
70+
insertIntoTable: chInsertIntoTable,
71+
selectAllFromTable: chSelectAllFromTable,
72+
columnDDL: chColumnDDL,
73+
});
74+
75+
/**
76+
* @internal
77+
*/
78+
export class ClickhouseDatabaseHelperService extends Effect.Service<ClickhouseDatabaseHelperService>()(
79+
"@typefusion/clickhouse/databasehelper",
80+
{
81+
effect: DatabaseHelper,
82+
dependencies: [ClickhouseDatabaseHelperLive],
83+
},
84+
) {}
85+
86+
/**
87+
* @internal
88+
*/
89+
export const ClickhouseFinalLive = Layer.mergeAll(
90+
ClickhouseService.Default,
91+
ClickhouseDatabaseHelperService.Default,
92+
);
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
import { ClickhouseClient } from "@effect/sql-clickhouse/ClickhouseClient";
2+
import { SqlClient } from "@effect/sql/SqlClient";
3+
import { Effect } from "effect";
4+
/**
5+
* @internal
6+
*/
7+
export const chDropTableIfExists = (
8+
sql: SqlClient | ClickhouseClient,
9+
tableName: string,
10+
) => sql`DROP TABLE IF EXISTS ${sql.unsafe(tableName)}`;
11+
12+
/**
13+
* @internal
14+
*/
15+
export const chCreateTableIfNotExists = (
16+
sql: SqlClient | ClickhouseClient,
17+
tableName: string,
18+
columnDefinitions: string,
19+
) => {
20+
const firstColumnName = columnDefinitions.split(" ")[0];
21+
return sql`CREATE TABLE IF NOT EXISTS ${sql.unsafe(tableName)} (${sql.unsafe(columnDefinitions)}) ORDER BY (${sql.unsafe(firstColumnName)})`;
22+
};
23+
24+
/**
25+
* @internal
26+
*/
27+
export const chInsertIntoTable = (
28+
sql: SqlClient | ClickhouseClient,
29+
tableName: string,
30+
data: unknown[],
31+
) =>
32+
(sql as ClickhouseClient)
33+
.insertQuery({ table: tableName, values: data })
34+
.pipe(Effect.asVoid);
35+
36+
/**
37+
* @internal
38+
*/
39+
export const chSelectAllFromTable = (
40+
sql: SqlClient | ClickhouseClient,
41+
tableName: string,
42+
) => sql`SELECT * FROM ${sql.unsafe(tableName)}`;
43+
44+
/**
45+
* @internal
46+
*/
47+
export const chColumnDDL = (columnName: string, columnType: string) =>
48+
`${columnName} ${columnType}`;

0 commit comments

Comments
 (0)