Skip to content

Commit 1ff2075

Browse files
haoranpbbcbenchCopilot
authored
Enable AL-LSP for Claude Code (#652)
Co-authored-by: bcbench <bcbench@noreply> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 1758cb7 commit 1ff2075

9 files changed

Lines changed: 195 additions & 114 deletions

File tree

.github/workflows/claude-evaluation.yml

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,11 @@ on:
3333
required: false
3434
default: false
3535
type: boolean
36+
al-lsp:
37+
description: "Enable AL LSP server"
38+
required: false
39+
default: false
40+
type: boolean
3641
repeat:
3742
description: "Number of times to run sequentially (ignored for test runs)"
3843
required: false
@@ -100,9 +105,9 @@ jobs:
100105
node-version: 24
101106

102107
- name: Install AL Tool
103-
if: ${{ inputs.al-mcp }}
108+
if: ${{ inputs.al-mcp || inputs.al-lsp }}
104109
run: |
105-
dotnet tool install -g Microsoft.Dynamics.BusinessCentral.Development.Tools --version 17.0.33.55542
110+
dotnet tool install -g Microsoft.Dynamics.BusinessCentral.Development.Tools --version 18.0.36.64936-beta
106111
echo "$HOME\.dotnet\tools" >> $env:GITHUB_PATH
107112
108113
- name: Install Claude Code
@@ -120,7 +125,8 @@ jobs:
120125
--category "${{ inputs.category }}" `
121126
--repo-path "${{ steps.setup-env.outputs.repo_path }}" `
122127
--output-dir "${{ env.EVALUATION_RESULTS_DIR }}" `
123-
${{ inputs.al-mcp && '--al-mcp' || '' }}
128+
${{ inputs.al-mcp && '--al-mcp' || '' }} `
129+
${{ inputs.al-lsp && '--al-lsp' || '' }}
124130
125131
- name: Upload evaluation results
126132
uses: actions/upload-artifact@v6
@@ -155,4 +161,4 @@ jobs:
155161
workflow-file: claude-evaluation.yml
156162
repeat: ${{ inputs.repeat }}
157163
workflow-inputs: |
158-
{"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}"}
164+
{"model": "${{ inputs.model }}", "category": "${{ inputs.category }}", "test-run": "${{ inputs.test-run }}", "al-mcp": "${{ inputs.al-mcp }}", "al-lsp": "${{ inputs.al-lsp }}"}

EXPERIMENT.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ Trigger the evaluation workflow from the **Actions** tab:
7777

7878
- **Workflow:** `Evaluation with GitHub Copilot` or `Evaluation with Claude Code`
7979
- **`test-run`:** `true` (default — runs 4 entries, ~10 min)
80-
- **`model`**, **`category`**, **`al-mcp`**: as needed
80+
- **`model`**, **`category`**, **`al-mcp`**, **`al-lsp`**: as needed
8181

8282
This catches configuration mistakes cheaply. Do not skip it.
8383

src/bcbench/agent/claude/agent.py

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import yaml
77

88
from bcbench.agent.claude.metrics import parse_metrics
9-
from bcbench.agent.shared import build_mcp_config, build_prompt, parse_tool_usage_from_hooks
9+
from bcbench.agent.shared import build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks
1010
from bcbench.config import get_config
1111
from bcbench.dataset import BaseDatasetEntry
1212
from bcbench.exceptions import AgentError, AgentTimeoutError
@@ -19,7 +19,14 @@
1919

2020

2121
def run_claude_code(
22-
entry: BaseDatasetEntry, model: str, category: EvaluationCategory, repo_path: Path, output_dir: Path, al_mcp: bool = False, container_name: str = "bcbench"
22+
entry: BaseDatasetEntry,
23+
model: str,
24+
category: EvaluationCategory,
25+
repo_path: Path,
26+
output_dir: Path,
27+
al_mcp: bool = False,
28+
al_lsp: bool = False,
29+
container_name: str = "bcbench",
2330
) -> tuple[AgentMetrics | None, ExperimentConfiguration]:
2431
"""Run Claude Code on a single dataset entry.
2532
@@ -33,12 +40,14 @@ def run_claude_code(
3340

3441
prompt: str = build_prompt(entry, repo_path, claude_config, category, al_mcp=al_mcp)
3542
mcp_config_json, mcp_server_names = build_mcp_config(claude_config, entry, repo_path, al_mcp=al_mcp, container_name=container_name)
43+
lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentType.CLAUDE, al_lsp=al_lsp, container_name=container_name)
3644
instructions_enabled: bool = setup_instructions_from_config(claude_config, entry, repo_path, agent_type=AgentType.CLAUDE)
3745
skills_enabled: bool = setup_agent_skills(claude_config, entry, repo_path, agent_type=AgentType.CLAUDE)
3846
custom_agent: str | None = setup_custom_agent(claude_config, entry, repo_path, agent_type=AgentType.CLAUDE)
3947
tool_log_path: Path = setup_hooks(repo_path, AgentType.CLAUDE, output_dir)
4048
config = ExperimentConfiguration(
4149
mcp_servers=mcp_server_names,
50+
al_lsp_enabled=lsp_plugin_dir is not None,
4251
custom_instructions=instructions_enabled,
4352
skills_enabled=skills_enabled,
4453
custom_agent=custom_agent,
@@ -65,6 +74,8 @@ def run_claude_code(
6574
]
6675
if mcp_config_json:
6776
cmd_args.append(f"--mcp-config={mcp_config_json}")
77+
if lsp_plugin_dir is not None:
78+
cmd_args.append(f"--plugin-dir={lsp_plugin_dir}")
6879
if custom_agent:
6980
cmd_args.append(f"--agent={custom_agent}")
7081
cmd_args.extend(

src/bcbench/agent/copilot/agent.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import yaml
99

1010
from bcbench.agent.copilot.metrics import parse_metrics
11-
from bcbench.agent.shared import build_lsp_config, build_mcp_config, build_prompt, parse_tool_usage_from_hooks
11+
from bcbench.agent.shared import build_al_lsp_plugin, build_mcp_config, build_prompt, parse_tool_usage_from_hooks
1212
from bcbench.config import get_config
1313
from bcbench.dataset import BaseDatasetEntry
1414
from bcbench.exceptions import AgentError, AgentTimeoutError
@@ -42,14 +42,14 @@ def run_copilot_agent(
4242

4343
prompt: str = build_prompt(entry, repo_path, copilot_config, category, al_mcp=al_mcp)
4444
mcp_config_json, mcp_server_names = build_mcp_config(copilot_config, entry, repo_path, al_mcp=al_mcp, container_name=container_name)
45-
al_lsp_enabled: bool = build_lsp_config(entry, category, repo_path, al_lsp=al_lsp, container_name=container_name)
45+
lsp_plugin_dir: Path | None = build_al_lsp_plugin(entry, category, repo_path, AgentType.COPILOT, al_lsp=al_lsp, container_name=container_name)
4646
instructions_enabled: bool = setup_instructions_from_config(copilot_config, entry, repo_path, agent_type=AgentType.COPILOT)
4747
skills_enabled: bool = setup_agent_skills(copilot_config, entry, repo_path, agent_type=AgentType.COPILOT)
4848
custom_agent: str | None = setup_custom_agent(copilot_config, entry, repo_path, agent_type=AgentType.COPILOT)
4949
tool_log_path: Path = setup_hooks(repo_path, AgentType.COPILOT, output_dir)
5050
config = ExperimentConfiguration(
5151
mcp_servers=mcp_server_names,
52-
al_lsp_enabled=al_lsp_enabled,
52+
al_lsp_enabled=lsp_plugin_dir is not None,
5353
custom_instructions=instructions_enabled,
5454
skills_enabled=skills_enabled,
5555
custom_agent=custom_agent,
@@ -76,6 +76,8 @@ def run_copilot_agent(
7676
cmd_args.append("--no-custom-instructions")
7777
if mcp_config_json:
7878
cmd_args.append(f"--additional-mcp-config={mcp_config_json}")
79+
if lsp_plugin_dir is not None:
80+
cmd_args.append(f"--plugin-dir={lsp_plugin_dir}")
7981
if custom_agent:
8082
cmd_args.append(f"--agent={custom_agent}")
8183

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
"""Shared code for CLI-based agents (Claude, Copilot)."""
22

33
from bcbench.agent.shared.hooks_parser import parse_tool_usage_from_hooks
4-
from bcbench.agent.shared.lsp import build_lsp_config
4+
from bcbench.agent.shared.lsp import build_al_lsp_plugin
55
from bcbench.agent.shared.mcp import build_mcp_config
66
from bcbench.agent.shared.prompt import build_prompt
77

8-
__all__ = ["build_lsp_config", "build_mcp_config", "build_prompt", "parse_tool_usage_from_hooks"]
8+
__all__ = ["build_al_lsp_plugin", "build_mcp_config", "build_prompt", "parse_tool_usage_from_hooks"]

src/bcbench/agent/shared/lsp.py

Lines changed: 59 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import json
2+
import shutil
23
from pathlib import Path
34

45
from bcbench.agent.shared.altool_paths import (
@@ -10,11 +11,16 @@
1011
from bcbench.dataset import BaseDatasetEntry
1112
from bcbench.exceptions import AgentError
1213
from bcbench.logger import get_logger
13-
from bcbench.types import EvaluationCategory
14+
from bcbench.types import AgentType, EvaluationCategory
1415

1516
logger = get_logger(__name__)
1617

17-
_AL_LSP_RELATIVE_PATH = Path(".github") / "lsp.json"
18+
# Per-task plugin folder location. Both Copilot CLI and Claude Code accept
19+
# `--plugin-dir <path>` for ad-hoc plugin loading and both look for the
20+
# manifest under `.claude-plugin/plugin.json`, so a single neutral path works
21+
# for either agent. Lives under `.bcbench/` so it's visibly BC-Bench-owned
22+
# and won't collide with either agent's auto-discovery paths.
23+
_AL_LSP_PLUGIN_RELATIVE_PATH = Path(".bcbench") / "al-lsp-plugin"
1824

1925

2026
def _resolve_symbol_paths(entry: BaseDatasetEntry, category: EvaluationCategory, container_name: str) -> tuple[list[str], list[str]]:
@@ -48,48 +54,68 @@ def _build_lsp_args(project_paths: list[str], package_cache_paths: list[str], as
4854
return args
4955

5056

51-
def build_lsp_config(entry: BaseDatasetEntry, category: EvaluationCategory, repo_path: Path, al_lsp: bool, container_name: str = "") -> bool:
52-
"""Write Copilot's project-level LSP config to <repo_path>/.github/lsp.json.
57+
def _lsp_config_for(agent_type: AgentType, args: list[str]) -> dict:
58+
"""Build the agent-specific `.lsp.json` content.
5359
54-
When ``al_lsp=False``, removes any stale config left over from a previous run and returns False.
55-
When True, writes the `lspServers.altool` entry pointing at `altool launchlspserver` and returns True.
60+
Both agents launch the same `al launchlspserver` process — only the surrounding
61+
LSP-routing schema differs:
62+
63+
- Copilot CLI expects `{ "lspServers": { name: { ..., "fileExtensions": {".ext": "lang"} } } }`
64+
- Claude Code expects `{ name: { ..., "extensionToLanguage": {".ext": "lang"} } }` (no wrapper, different extension key)
65+
66+
`command: "al"` is unqualified by design: Copilot CLI silently rejects absolute paths in LSP
67+
`command` ("Server <name> is configured but not available"), so the published `altool` wrapper
68+
(`al`) must resolve via PATH on both sides.
69+
"""
70+
server = {"command": "al", "args": args}
71+
match agent_type:
72+
case AgentType.COPILOT:
73+
return {"lspServers": {"altool": {**server, "fileExtensions": {".al": "al"}}}}
74+
case AgentType.CLAUDE:
75+
return {"altool": {**server, "extensionToLanguage": {".al": "al"}}}
76+
77+
78+
def build_al_lsp_plugin(entry: BaseDatasetEntry, category: EvaluationCategory, repo_path: Path, agent_type: AgentType, al_lsp: bool, container_name: str = "") -> Path | None:
79+
"""Build a per-task plugin folder containing the AL LSP server, return its path or None.
80+
81+
Both Copilot CLI and Claude Code load this via ``--plugin-dir <path>`` for a single session
82+
— no marketplace registration, no global state, no cross-run plugin leakage. The plugin
83+
folder layout is identical between agents; only the LSP-routing schema in ``.lsp.json``
84+
differs (see :func:`_lsp_config_for`).
85+
86+
Layout written under ``<repo>/.bcbench/al-lsp-plugin/``::
87+
88+
.claude-plugin/plugin.json — minimal manifest (only ``name`` is required;
89+
both CLIs check this path)
90+
.lsp.json — LSP server config in the agent's schema
91+
92+
Returns the plugin folder path (to be passed as ``--plugin-dir``), or None when disabled.
5693
"""
57-
lsp_config_path = repo_path / _AL_LSP_RELATIVE_PATH
94+
plugin_dir = repo_path / _AL_LSP_PLUGIN_RELATIVE_PATH
5895

5996
if not al_lsp:
60-
if lsp_config_path.is_file():
61-
lsp_config_path.unlink()
62-
logger.info(f"Removed stale LSP config: {lsp_config_path}")
63-
return False
97+
if plugin_dir.exists():
98+
shutil.rmtree(plugin_dir)
99+
logger.info(f"Removed stale AL LSP plugin: {plugin_dir}")
100+
return None
64101

65102
project_paths = [str(repo_path / p) for p in entry.project_paths]
66103
set_runtime_version(project_paths)
67-
68104
package_cache_paths, assembly_probing_paths = _resolve_symbol_paths(entry, category, container_name)
105+
args = _build_lsp_args(project_paths, package_cache_paths, assembly_probing_paths)
69106

70-
args = _build_lsp_args(
71-
project_paths=project_paths,
72-
package_cache_paths=package_cache_paths,
73-
assembly_probing_paths=assembly_probing_paths,
74-
)
75-
76-
# Copilot CLI resolves `command` via PATH (absolute paths are silently rejected with
77-
# "Server <name> is configured but not available"). `al` is the published altool
78-
# wrapper installed via the .NET tool — it must be on PATH.
79-
lsp_config = {
80-
"lspServers": {
81-
"altool": {
82-
"command": "al",
83-
"args": args,
84-
"fileExtensions": {".al": "al"},
85-
}
86-
}
107+
plugin_manifest = {
108+
"name": "al-lsp",
109+
"version": "1.0.0",
110+
"description": "AL Language Server for Business Central agentic development",
87111
}
112+
lsp_config = _lsp_config_for(agent_type, args)
88113

89-
lsp_config_path.parent.mkdir(parents=True, exist_ok=True)
90-
lsp_config_path.write_text(json.dumps(lsp_config, indent=2), encoding="utf-8")
114+
(plugin_dir / ".claude-plugin").mkdir(parents=True, exist_ok=True)
115+
(plugin_dir / ".claude-plugin" / "plugin.json").write_text(json.dumps(plugin_manifest, indent=2), encoding="utf-8")
116+
(plugin_dir / ".lsp.json").write_text(json.dumps(lsp_config, indent=2), encoding="utf-8")
91117

92-
logger.info(f"Wrote AL LSP config: {lsp_config_path}")
118+
logger.info(f"Wrote AL LSP plugin for {agent_type.value}: {plugin_dir}")
93119
logger.debug(f"LSP configuration: {json.dumps(lsp_config, indent=2)}")
94120

95-
return True
121+
return plugin_dir

src/bcbench/commands/evaluate.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ def evaluate_claude_code(
106106
output_dir: OutputDir = _config.paths.evaluation_results_path,
107107
run_id: RunId = "claude_code_test_run",
108108
al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False,
109+
al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False,
109110
) -> None:
110111
"""
111112
Evaluate Claude Code on single dataset entry.
@@ -139,6 +140,7 @@ def evaluate_claude_code(
139140
model=ctx.model,
140141
output_dir=ctx.result_dir,
141142
al_mcp=al_mcp if ctx.container else False,
143+
al_lsp=al_lsp,
142144
container_name=ctx.get_container().name if ctx.container else "",
143145
),
144146
)

src/bcbench/commands/run.py

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ def run_claude(
6565
repo_path: RepoPath = _config.paths.testbed_path,
6666
output_dir: OutputDir = _config.paths.evaluation_results_path,
6767
al_mcp: Annotated[bool, typer.Option("--al-mcp", help="Enable AL MCP server")] = False,
68+
al_lsp: Annotated[bool, typer.Option("--al-lsp", help="Enable AL LSP server")] = False,
6869
) -> None:
6970
"""
7071
Run Claude Code on a single entry to generate a patch (without building/testing).
@@ -77,4 +78,13 @@ def run_claude(
7778
entry = category.entry_class.load(category.dataset_path, entry_id=entry_id)[0]
7879
category.pipeline.setup_workspace(entry, repo_path)
7980

80-
run_claude_code(entry=entry, repo_path=repo_path, model=model, category=category, output_dir=output_dir, al_mcp=al_mcp, container_name=container_name)
81+
run_claude_code(
82+
entry=entry,
83+
repo_path=repo_path,
84+
model=model,
85+
category=category,
86+
output_dir=output_dir,
87+
al_mcp=al_mcp if container_name else False,
88+
al_lsp=al_lsp,
89+
container_name=container_name or "",
90+
)

0 commit comments

Comments
 (0)