Skip to content

Commit a156eab

Browse files
rinOfTheStarsrinOfTheStars
authored
Refactor in accordance to cursorless issue 1488 specs (#1)
Should complete the first bullet point of cursorless-dev/cursorless#1488 as mentioned in the title. Things are a bit more hacky than I'd like, and in the future I might rewrite a good chunk of the code in order to only use `git clone`'d packages, but this is good enough for now. Obviously I can't test anything publishing related, so this is as far as I can take things. Note that the parser packages listed in `package.json` are only the ones present in [vscode-parse-tree](https://github.com/cursorless-dev/vscode-parse-tree) (plus gdscript since I have a pr open for that which this makes somewhat irrelevant if I understand things correctly?); please lmk if the parser packages that were unique to this project should be readded! Finally, please note that a few parser packages have had their versions upgraded. IIrc most of these were due to issues building older versions, as nearly all of the packages which are `git clone`'d in `build.ts` at runtime require special linking to build their `.wasm` file, which thankfully is present in fairly recent versions of said repos.
1 parent df1bed5 commit a156eab

File tree

6 files changed

+777
-1653
lines changed

6 files changed

+777
-1653
lines changed

‎.github/dependabot.yml

Lines changed: 0 additions & 6 deletions
This file was deleted.

‎.github/workflows/build-and-publish.yml

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,21 @@ jobs:
66
steps:
77
- uses: actions/checkout@v3
88

9-
- uses: pnpm/action-setup@v2
10-
- name: Install Node.js with pnpm
11-
uses: actions/setup-node@v4
9+
- uses: pnpm/action-setup@v4
10+
with:
11+
version: 10.0.0
12+
- uses: actions/setup-node@v4
1213
with:
1314
node-version: 20
1415
cache: "pnpm"
1516
- run: pnpm install
1617

17-
- uses: mymindstorm/setup-emsdk@v11
18+
- uses: mymindstorm/setup-emsdk@v14
1819
with:
19-
version: 2.0.24
20+
version: 3.1.74
2021

2122
- run: pnpm build
2223

2324
- uses: JS-DevTools/npm-publish@v1
2425
with:
25-
token: ${{ secrets.NPM_TOKEN }}
26+
token: ${{ secrets.NPM_TOKEN }}

‎.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
out/
2-
node_modules/
2+
node_modules/
3+
.DS_Store

‎build.ts

Lines changed: 73 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,51 @@ const outDir = path.join(__dirname, "out");
1616

1717
let hasErrors = false;
1818

19+
async function gitCloneOverload(name: string, repoUrl: string, useLatest: boolean, {commitHash}: {commitHash?: string}) {
20+
let packagePath;
21+
try {
22+
packagePath = findRoot(require.resolve(name));
23+
} catch (_) {
24+
packagePath = path.join(__dirname, "node_modules", name);
25+
}
26+
27+
let commitInfo = (commitHash === undefined) ? "latest" : commitHash;
28+
29+
try {
30+
console.log(`🗑️ Deleting cached node dep for ${ name }`);
31+
await exec(`rm -rf ${ packagePath }`)
32+
console.log(`⬇️ Cloning ${ name } from git (${commitInfo})`);
33+
await exec(`git clone ${ repoUrl } ${ packagePath }`)
34+
if (!useLatest) {
35+
if (commitHash !== undefined) {
36+
process.chdir(packagePath);
37+
await exec(`git reset --hard ${ commitHash }`);
38+
} else throw new Error("Latest commit is not being used, yet no commit hash was specified");
39+
}
40+
} catch (err) {
41+
console.error(`❗Failed to clone git repo for ${ name }:\n`, err);
42+
}
43+
}
44+
1945
async function buildParserWASM(
2046
name: string,
21-
{ subPath, generate }: { subPath?: string; generate?: boolean } = {}
47+
{ subPath, generate }:
48+
{ subPath?: string; generate?: boolean } = {}
2249
) {
50+
2351
const label = subPath ? path.join(name, subPath) : name;
52+
53+
let cliPackagePath;
54+
try {
55+
cliPackagePath = findRoot(require.resolve("tree-sitter-cli"));
56+
} catch(_) {
57+
cliPackagePath = path.join(__dirname, "node_modules", "tree-sitter-cli");
58+
}
59+
60+
let cliPath = path.join(cliPackagePath, "tree-sitter");
61+
let generateCommand = cliPath.concat(" generate");
62+
let buildCommand = cliPath.concat(" build --wasm");
63+
2464
try {
2565
console.log(`⏳ Building ${label}`);
2666
let packagePath;
@@ -31,9 +71,10 @@ async function buildParserWASM(
3171
}
3272
const cwd = subPath ? path.join(packagePath, subPath) : packagePath;
3373
if (generate) {
34-
await exec(`pnpm tree-sitter generate`, { cwd });
74+
await exec(generateCommand, { cwd });
3575
}
36-
await exec(`pnpm tree-sitter build-wasm ${cwd}`);
76+
await exec(buildCommand, { cwd });
77+
await exec(`mv *.wasm ${outDir}`, { cwd });
3778
console.log(`✅ Finished building ${label}`);
3879
} catch (e) {
3980
console.error(`🔥 Failed to build ${label}:\n`, e);
@@ -50,30 +91,48 @@ fs.mkdirSync(outDir);
5091
process.chdir(outDir);
5192

5293
const grammars = Object.keys(packageInfo.devDependencies)
53-
.filter((n) => n.startsWith("tree-sitter-") && n !== "tree-sitter-cli")
54-
.concat('@tree-sitter-grammars/tree-sitter-zig')
55-
.concat("@tlaplus/tree-sitter-tlaplus")
94+
.filter((n) => n.startsWith("tree-sitter-") && n !== "tree-sitter-cli" && n !== "tree-sitter")
5695
.filter((s) => !langArg || s.includes(langArg));
5796

5897
PromisePool.withConcurrency(os.cpus().length)
5998
.for(grammars)
60-
.process(async (name) => {
61-
if (name == "tree-sitter-rescript") {
62-
await buildParserWASM(name, { generate: true });
63-
} else if (name == "tree-sitter-ocaml") {
64-
await buildParserWASM(name, { subPath: "ocaml" });
99+
.process(async (name : string) => {
100+
if (name == "tree-sitter-agda") {
101+
await gitCloneOverload(name, "https://github.com/tree-sitter/tree-sitter-agda.git", false, {commitHash: "47802091de0cb8ac2533d67ac37e65692c5902c4"});
102+
await buildParserWASM(name)
103+
} else if (name == "tree-sitter-perl") {
104+
await gitCloneOverload(name, "https://github.com/tree-sitter-perl/tree-sitter-perl.git", false, {commitHash: "b467c41aed6e52c7f581c18a3b50821995a82c95"});
105+
await buildParserWASM(name, {generate: true});
65106
} else if (name == "tree-sitter-php") {
66107
await buildParserWASM(name, { subPath: "php" });
67108
} else if (name == "tree-sitter-typescript") {
68109
await buildParserWASM(name, { subPath: "typescript" });
69110
await buildParserWASM(name, { subPath: "tsx" });
70-
} else {
111+
} else if (name == "tree-sitter-latex") {
112+
await buildParserWASM(name, {generate: true});
113+
} else if (name == "tree-sitter-xml") {
114+
await buildParserWASM(name, {subPath: "xml"});
115+
await buildParserWASM(name, {subPath: "dtd"});
116+
} else if (name == "tree-sitter-query") {
117+
await gitCloneOverload(name, "https://github.com/tree-sitter-grammars/tree-sitter-query.git", false, {commitHash: "a6674e279b14958625d7a530cabe06119c7a1532"});
118+
await buildParserWASM(name);
119+
} else if (name == "tree-sitter-elixir") {
120+
await gitCloneOverload(name, "https://github.com/elixir-lang/tree-sitter-elixir.git", false, {commitHash: "02a6f7fd4be28dd94ee4dd2ca19cb777053ea74e"});
121+
await buildParserWASM(name);
122+
} else if (name == "tree-sitter-markdown") {
123+
await gitCloneOverload(name, "https://github.com/tree-sitter-grammars/tree-sitter-markdown", false, {commitHash: "192407ab5a24bfc24f13332979b5e7967518754a"});
124+
await buildParserWASM(name, {subPath: "tree-sitter-markdown"});
125+
await buildParserWASM(name, {subPath: "tree-sitter-markdown-inline"});
126+
}
127+
else {
71128
await buildParserWASM(name);
72129
}
73130
})
74131
.then(async () => {
75132
if (hasErrors) {
133+
//not sure if this failsafe is actually required, but it doesn't hurt to ensure that no wasms can be published from a failed build
134+
fs.rmSync(outDir, { recursive: true, force: true });
135+
fs.mkdirSync(outDir);
76136
process.exit(1);
77137
}
78-
await exec(`mv *.wasm ${outDir}`, { cwd: __dirname });
79-
});
138+
});

‎package.json

Lines changed: 51 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,73 @@
11
{
2-
"name": "tree-sitter-wasms",
3-
"version": "0.1.12",
2+
"name": "@cursorless/tree-sitter-wasms",
3+
"version": "0.2.0",
44
"description": "Prebuilt WASM binaries for tree-sitter's language parsers.",
5-
"repository": "https://github.com/Gregoor/tree-sitter-wasms",
6-
"author": "Gregor <[email protected]>, Menci <[email protected]>",
5+
"repository": "https://github.com/cursorless-dev/tree-sitter-wasms",
6+
"author": "Gregor <[email protected]>, Menci <[email protected]>, rinOfTheStars",
77
"license": "Unlicense",
88
"scripts": {
99
"build": "ts-node build.ts"
1010
},
1111
"devDependencies": {
1212
"@supercharge/promise-pool": "^3.1.1",
13-
"@tlaplus/tree-sitter-tlaplus": "^1.2.4",
14-
"@tree-sitter-grammars/tree-sitter-zig": "^1.0.0",
1513
"@types/find-root": "^1.1.4",
1614
"@types/node": "^20.11.17",
17-
"@willjouo/tree-sitter-r": "^0.0.1",
1815
"find-root": "^1.1.0",
19-
"tree-sitter-bash": "^0.20.5",
20-
"tree-sitter-c": "^0.20.7",
21-
"tree-sitter-c-sharp": "^0.20.0",
22-
"tree-sitter-cli": "^0.20.8",
23-
"tree-sitter-cpp": "^0.20.4",
24-
"tree-sitter-css": "^0.20.0",
25-
"tree-sitter-elisp": "^1.3.0",
26-
"tree-sitter-elixir": "^0.1.1",
27-
"tree-sitter-elm": "^4.5.0",
28-
"tree-sitter-embedded-template": "^0.20.0",
29-
"tree-sitter-go": "^0.20.0",
30-
"tree-sitter-html": "^0.20.0",
31-
"tree-sitter-java": "^0.20.2",
16+
"tree-sitter": "^0.22.4",
17+
"tree-sitter-agda": "github:tree-sitter/tree-sitter-agda#47802091de0cb8ac2533d67ac37e65692c5902c4",
18+
"tree-sitter-bash": "^0.23.3",
19+
"tree-sitter-c": "^0.23.4",
20+
"tree-sitter-c-sharp": "^0.23.1",
21+
"tree-sitter-cli": "^0.24.7",
22+
"tree-sitter-clojure": "github:sogaiu/tree-sitter-clojure#f4236d4da8aa92bc105d9c118746474c608e6af7",
23+
"tree-sitter-cpp": "^0.23.4",
24+
"tree-sitter-css": "^0.23.1",
25+
"tree-sitter-dart": "github:UserNobody14/tree-sitter-dart#e81af6ab94a728ed99c30083be72d88e6d56cf9e",
26+
"tree-sitter-elixir": "github:elixir-lang/tree-sitter-elixir#02a6f7fd4be28dd94ee4dd2ca19cb777053ea74e",
27+
"tree-sitter-elm": "github:elm-tooling/tree-sitter-elm#e34bdc5c512918628b05b48e633f711123204e45",
28+
"tree-sitter-gdscript": "github:PrestonKnopp/tree-sitter-gdscript#48b49330888a4669b48619b211cc8da573827725",
29+
"tree-sitter-gleam": "github:gleam-lang/tree-sitter-gleam#066704e4826699e754d351e3bbe12bf2e51de9d8",
30+
"tree-sitter-go": "^0.23.4",
31+
"tree-sitter-haskell": "^0.23.1",
32+
"tree-sitter-hcl": "github:tree-sitter-grammars/tree-sitter-hcl#9e3ec9848f28d26845ba300fd73c740459b83e9b",
33+
"tree-sitter-html": "^0.23.2",
34+
"tree-sitter-java": "^0.23.2",
3235
"tree-sitter-javascript": "^0.20.3",
33-
"tree-sitter-json": "^0.20.2",
34-
"tree-sitter-kotlin": "^0.3.1",
36+
"tree-sitter-json": "^0.24.8",
37+
"tree-sitter-julia": "^0.23.1",
38+
"tree-sitter-kotlin": "^0.3.8",
39+
"tree-sitter-latex": "github:latex-lsp/tree-sitter-latex#7b06f6ed394308e7407a1703d2724128c45fc9d7",
3540
"tree-sitter-lua": "^2.1.3",
36-
"tree-sitter-objc": "^2.1.0",
37-
"tree-sitter-ocaml": "^0.20.4",
38-
"tree-sitter-php": "^0.22.0",
39-
"tree-sitter-python": "^0.21.0",
40-
"tree-sitter-ql": "^1.0.0",
41-
"tree-sitter-rescript": "github:rescript-lang/tree-sitter-rescript#6376fa028f31aa4e26ca2c8f007e322cd2a5eb4a",
42-
"tree-sitter-ruby": "^0.20.1",
43-
"tree-sitter-rust": "^0.20.4",
44-
"tree-sitter-scala": "^0.19.0",
45-
"tree-sitter-solidity": "github:JoranHonig/tree-sitter-solidity#b239a95f94cfcc6e7b3e961bc73a28d55e214f02",
46-
"tree-sitter-swift": "^0.4.0",
47-
"tree-sitter-systemrdl": "^0.7.0",
48-
"tree-sitter-toml": "^0.5.1",
49-
"tree-sitter-typescript": "^0.20.5",
50-
"tree-sitter-vue": "https://github.com/tree-sitter-grammars/tree-sitter-vue#7e48557b903a9db9c38cea3b7839ef7e1f36c693",
51-
"tree-sitter-yaml": "^0.5.0",
41+
"tree-sitter-markdown": "github:tree-sitter-grammars/tree-sitter-markdown#192407ab5a24bfc24f13332979b5e7967518754a",
42+
"tree-sitter-nix": "github:nix-community/tree-sitter-nix#21897cc3dcd15325303e46b85295b743742af6ab",
43+
"tree-sitter-perl": "github:tree-sitter-perl/tree-sitter-perl#b467c41aed6e52c7f581c18a3b50821995a82c95",
44+
"tree-sitter-php": "^0.23.11",
45+
"tree-sitter-python": "^0.23.6",
46+
"tree-sitter-query": "github:tree-sitter-grammars/tree-sitter-query#a6674e279b14958625d7a530cabe06119c7a1532",
47+
"tree-sitter-ruby": "^0.23.1",
48+
"tree-sitter-rust": "^0.23.2",
49+
"tree-sitter-scala": "^0.23.4",
50+
"tree-sitter-scss": "github:tree-sitter-grammars/tree-sitter-scss#00a4cd98d5fc71aa5a55174e7ed5e34c9cc85a87",
51+
"tree-sitter-sparql": "github:IoannisNezis/tree-sitter-sparql#3c468620e7c2ffa69441de60161c4e4a919969f7",
52+
"tree-sitter-swift": "^0.6.0",
53+
"tree-sitter-talon": "github:wenkokke/tree-sitter-talon#7119d7f3fefd0b0c7e900120b51f85a23e6c3160",
54+
"tree-sitter-typescript": "^0.23.2",
55+
"tree-sitter-xml": "github:tree-sitter-grammars/tree-sitter-xml#ce150c5be7f617e18ffa2064213e5d9c4cb5d69e",
5256
"ts-node": "^10.9.2",
53-
"typescript": "5.3.3"
57+
"typescript": "^5.7.3"
58+
},
59+
"pnpm": {
60+
"onlyBuiltDependencies": [
61+
"tree-sitter-cli"
62+
]
5463
},
5564
"files": [
5665
"/out"
5766
],
58-
"packageManager": "pnpm@8.10.5",
67+
"packageManager": "pnpm@10.0.0",
5968
"main": "bindings/node",
6069
"dependencies": {
61-
"tree-sitter-wasms": "^0.1.11"
70+
"tree-sitter": "^0.22.4",
71+
"tree-sitter-cli": "^0.24.7"
6272
}
6373
}

0 commit comments

Comments
 (0)