File based programs IDE support (#78488)

Co-authored-by: Cyrus Najmabadi <[email protected]>
Co-authored-by: Jason Malinowski <[email protected]>

Author: 3 people

.vscode/tasks.json

@@ -169,7 +169,7 @@
       "type": "process",
       "options": {
         "env": {
-          "DOTNET_ROSLYN_SERVER_PATH": "${workspaceRoot}/artifacts/bin/Microsoft.CodeAnalysis.LanguageServer/Debug/net8.0/Microsoft.CodeAnalysis.LanguageServer.dll"
+          "DOTNET_ROSLYN_SERVER_PATH": "${workspaceRoot}/artifacts/bin/Microsoft.CodeAnalysis.LanguageServer/Debug/net9.0/Microsoft.CodeAnalysis.LanguageServer.dll"
         }
       },
       "dependsOn": [ "build language server" ]

docs/features/file-based-programs-vscode.md

@@ -0,0 +1,59 @@
+# File-based programs VS Code support
+
+See also [dotnet-run-file.md](https://github.com/dotnet/sdk/blob/main/documentation/general/dotnet-run-file.md).
+
+## Feature overview
+
+A file-based program embeds a subset of MSBuild project capabilities into C# code, allowing single files to stand alone as ordinary projects.
+
+The following is a file-based program:
+
+```cs
+Console.WriteLine("Hello World!");
+```
+
+So is the following:
+
+```cs
+#!/usr/bin/env dotnet run
+#:sdk Microsoft.Net.Sdk
+#:package [email protected]
+#:property LangVersion=preview
+
+using Newtonsoft.Json;
+
+Main();
+
+void Main()
+{
+    if (args is not [_, var jsonPath, ..])
+    {
+        Console.Error.WriteLine("Usage: app <json-file>");
+        return;
+    }
+
+    var json = File.ReadAllText(jsonPath);
+    var data = JsonConvert.DeserializeObject<Data>(json);
+    // ...
+}
+
+record Data(string field1, int field2);
+```
+
+This basically works by having the `dotnet` command line interpret the `#:` directives in source files, produce a C# project XML document in memory, and pass it off to MSBuild. The in-memory project is sometimes called a "virtual project".
+
+## Miscellaneous files changes
+
+There is a long-standing backlog item to enhance the experience of working with miscellaneous files ("loose files" not associated with any project). We think that as part of the "file-based program" work, we can enable the following in such files without substantial issues:
+- Syntax diagnostics.
+- Intellisense for the "default" set of references. e.g. those references which are included in the project created by `dotnet new console` with the current SDK.
+
+### Heuristic
+The IDE considers a file to be a file-based program, if:
+- It has any `#:` directives which configure the file-based program project, or,
+- It has any top-level statements.
+Any of the above is met, and, the file is not included in an ordinary `.csproj` project (i.e. it is not part of any ordinary project's list of `Compile` items).
+
+### Opt-out
+
+We added an opt-out flag with option name `dotnet.projects.enableFileBasedPrograms`. If issues arise with the file-based program experience, then VS Code users should set the corresponding setting `"dotnet.projects.enableFileBasedPrograms": false` to revert back to the old miscellaneous files experience.

src/Features/Core/Portable/Workspace/MiscellaneousFileUtilities.cs

@@ -46,6 +46,12 @@ internal static ProjectInfo CreateMiscellaneousProjectInfoForDocument(
             compilationOptions = GetCompilationOptionsWithScriptReferenceResolvers(services, compilationOptions, filePath);
         }
 
+        if (parseOptions != null && fileExtension != languageInformation.ScriptExtension)
+        {
+            // Any non-script misc file should not complain about usage of '#:' ignored directives.
+            parseOptions = parseOptions.WithFeatures([.. parseOptions.Features, new("FileBasedProgram", "true")]);
+        }
+
         var projectId = ProjectId.CreateNewId(debugName: $"{workspace.GetType().Name} Files Project for {filePath}");
         var documentId = DocumentId.CreateNewId(projectId, debugName: filePath);
 

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer.UnitTests/TelemetryReporterTests.cs

@@ -14,13 +14,11 @@ public sealed class TelemetryReporterTests(ITestOutputHelper testOutputHelper)
 {
     private async Task<ITelemetryReporter> CreateReporterAsync()
     {
-        var exportProvider = await LanguageServerTestComposition.CreateExportProviderAsync(
+        var (exportProvider, _) = await LanguageServerTestComposition.CreateExportProviderAsync(
             LoggerFactory,
             includeDevKitComponents: true,
             MefCacheDirectory.Path,
-            [],
-            out var _,
-            out var _);
+            []);
 
         // VS Telemetry requires this environment variable to be set.
         Environment.SetEnvironmentVariable("CommonPropertyBagPath", Path.GetTempFileName());

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer.UnitTests/Utilities/AbstractLanguageServerHostTests.cs

@@ -46,8 +46,8 @@ protected sealed class TestLspServer : ILspClient, IAsyncDisposable
 
         internal static async Task<TestLspServer> CreateAsync(ClientCapabilities clientCapabilities, ILoggerFactory loggerFactory, string cacheDirectory, bool includeDevKitComponents = true, string[]? extensionPaths = null)
         {
-            var exportProvider = await LanguageServerTestComposition.CreateExportProviderAsync(
-                loggerFactory, includeDevKitComponents, cacheDirectory, extensionPaths, out var _, out var assemblyLoader);
+            var (exportProvider, assemblyLoader) = await LanguageServerTestComposition.CreateExportProviderAsync(
+                loggerFactory, includeDevKitComponents, cacheDirectory, extensionPaths);
             var testLspServer = new TestLspServer(exportProvider, loggerFactory, assemblyLoader);
             var initializeResponse = await testLspServer.ExecuteRequestAsync<InitializeParams, InitializeResult>(Methods.InitializeName, new InitializeParams { Capabilities = clientCapabilities }, CancellationToken.None);
             Assert.NotNull(initializeResponse?.Capabilities);

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer.UnitTests/Utilities/LanguageServerTestComposition.cs

@@ -10,16 +10,14 @@ namespace Microsoft.CodeAnalysis.LanguageServer.UnitTests;
 
 internal sealed class LanguageServerTestComposition
 {
-    public static Task<ExportProvider> CreateExportProviderAsync(
+    public static async Task<(ExportProvider exportProvider, IAssemblyLoader assemblyLoader)> CreateExportProviderAsync(
         ILoggerFactory loggerFactory,
         bool includeDevKitComponents,
         string cacheDirectory,
-        string[]? extensionPaths,
-        out ServerConfiguration serverConfiguration,
-        out IAssemblyLoader assemblyLoader)
+        string[]? extensionPaths)
     {
         var devKitDependencyPath = includeDevKitComponents ? TestPaths.GetDevKitExtensionPath() : null;
-        serverConfiguration = new ServerConfiguration(LaunchDebugger: false,
+        var serverConfiguration = new ServerConfiguration(LaunchDebugger: false,
             LogConfiguration: new LogConfiguration(LogLevel.Trace),
             StarredCompletionsPath: null,
             TelemetryLevel: null,
@@ -32,8 +30,10 @@ public static Task<ExportProvider> CreateExportProviderAsync(
             ServerPipeName: null,
             UseStdIo: false);
         var extensionManager = ExtensionAssemblyManager.Create(serverConfiguration, loggerFactory);
-        assemblyLoader = new CustomExportAssemblyLoader(extensionManager, loggerFactory);
+        var assemblyLoader = new CustomExportAssemblyLoader(extensionManager, loggerFactory);
 
-        return LanguageServerExportProviderBuilder.CreateExportProviderAsync(extensionManager, assemblyLoader, devKitDependencyPath, cacheDirectory, loggerFactory, CancellationToken.None);
+        var exportProvider = await LanguageServerExportProviderBuilder.CreateExportProviderAsync(extensionManager, assemblyLoader, devKitDependencyPath, cacheDirectory, loggerFactory, CancellationToken.None);
+        exportProvider.GetExportedValue<ServerConfigurationFactory>().InitializeConfiguration(serverConfiguration);
+        return (exportProvider, assemblyLoader);
     }
 }

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer.UnitTests/WorkspaceProjectFactoryServiceTests.cs

@@ -18,11 +18,10 @@ public sealed class WorkspaceProjectFactoryServiceTests(ITestOutputHelper testOu
     public async Task CreateProjectAndBatch()
     {
         var loggerFactory = new LoggerFactory();
-        using var exportProvider = await LanguageServerTestComposition.CreateExportProviderAsync(
-            loggerFactory, includeDevKitComponents: false, MefCacheDirectory.Path, [], out var serverConfiguration, out var _);
+        var (exportProvider, _) = await LanguageServerTestComposition.CreateExportProviderAsync(
+            loggerFactory, includeDevKitComponents: false, MefCacheDirectory.Path, []);
+        using var _ = exportProvider;
 
-        exportProvider.GetExportedValue<ServerConfigurationFactory>()
-            .InitializeConfiguration(serverConfiguration);
         await exportProvider.GetExportedValue<ServiceBrokerFactory>().CreateAsync();
 
         var workspaceFactory = exportProvider.GetExportedValue<LanguageServerWorkspaceFactory>();
@@ -48,7 +47,7 @@ public async Task CreateProjectAndBatch()
         await batch.ApplyAsync(CancellationToken.None);
 
         // Verify it actually did something; we won't exclusively test each method since those are tested at lower layers
-        var project = workspaceFactory.Workspace.CurrentSolution.Projects.Single();
+        var project = workspaceFactory.HostWorkspace.CurrentSolution.Projects.Single();
 
         var document = Assert.Single(project.Documents);
         Assert.Equal(sourceFilePath, document.FilePath);

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer/HostWorkspace/FileBasedProgramsProjectSystem.cs

@@ -0,0 +1,143 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Immutable;
+using System.Security;
+using Microsoft.CodeAnalysis.Features.Workspaces;
+using Microsoft.CodeAnalysis.Host;
+using Microsoft.CodeAnalysis.LanguageServer.HostWorkspace.ProjectTelemetry;
+using Microsoft.CodeAnalysis.MetadataAsSource;
+using Microsoft.CodeAnalysis.MSBuild;
+using Microsoft.CodeAnalysis.Options;
+using Microsoft.CodeAnalysis.ProjectSystem;
+using Microsoft.CodeAnalysis.Shared.Extensions;
+using Microsoft.CodeAnalysis.Shared.TestHooks;
+using Microsoft.CodeAnalysis.Text;
+using Microsoft.CodeAnalysis.Workspaces.ProjectSystem;
+using Microsoft.CommonLanguageServerProtocol.Framework;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.Composition;
+using Roslyn.LanguageServer.Protocol;
+using Roslyn.Utilities;
+using static Microsoft.CodeAnalysis.MSBuild.BuildHostProcessManager;
+
+namespace Microsoft.CodeAnalysis.LanguageServer.HostWorkspace;
+
+/// <summary>Handles loading both miscellaneous files and file-based program projects.</summary>
+internal sealed class FileBasedProgramsProjectSystem : LanguageServerProjectLoader, ILspMiscellaneousFilesWorkspaceProvider
+{
+    private readonly ILspServices _lspServices;
+    private readonly ILogger<FileBasedProgramsProjectSystem> _logger;
+    private readonly IMetadataAsSourceFileService _metadataAsSourceFileService;
+
+    public FileBasedProgramsProjectSystem(
+        ILspServices lspServices,
+        IMetadataAsSourceFileService metadataAsSourceFileService,
+        LanguageServerWorkspaceFactory workspaceFactory,
+        IFileChangeWatcher fileChangeWatcher,
+        IGlobalOptionService globalOptionService,
+        ILoggerFactory loggerFactory,
+        IAsynchronousOperationListenerProvider listenerProvider,
+        ProjectLoadTelemetryReporter projectLoadTelemetry,
+        ServerConfigurationFactory serverConfigurationFactory,
+        BinlogNamer binlogNamer)
+            : base(
+                workspaceFactory.FileBasedProgramsProjectFactory,
+                workspaceFactory.TargetFrameworkManager,
+                workspaceFactory.ProjectSystemHostInfo,
+                fileChangeWatcher,
+                globalOptionService,
+                loggerFactory,
+                listenerProvider,
+                projectLoadTelemetry,
+                serverConfigurationFactory,
+                binlogNamer)
+    {
+        _lspServices = lspServices;
+        _logger = loggerFactory.CreateLogger<FileBasedProgramsProjectSystem>();
+        _metadataAsSourceFileService = metadataAsSourceFileService;
+    }
+
+    public Workspace Workspace => ProjectFactory.Workspace;
+
+    private string GetDocumentFilePath(DocumentUri uri) => uri.ParsedUri is { } parsedUri ? ProtocolConversions.GetDocumentFilePathFromUri(parsedUri) : uri.UriString;
+
+    public async ValueTask<TextDocument?> AddMiscellaneousDocumentAsync(DocumentUri uri, SourceText documentText, string languageId, ILspLogger logger)
+    {
+        var documentFilePath = GetDocumentFilePath(uri);
+
+        // https://github.com/dotnet/roslyn/issues/78421: MetadataAsSource should be its own workspace
+        if (_metadataAsSourceFileService.TryAddDocumentToWorkspace(documentFilePath, documentText.Container, out var documentId))
+        {
+            var metadataWorkspace = _metadataAsSourceFileService.TryGetWorkspace();
+            Contract.ThrowIfNull(metadataWorkspace);
+            return metadataWorkspace.CurrentSolution.GetRequiredDocument(documentId);
+        }
+
+        var primordialDoc = AddPrimordialDocument(uri, documentText, languageId);
+        Contract.ThrowIfNull(primordialDoc.FilePath);
+
+        var doDesignTimeBuild = uri.ParsedUri?.IsFile is true
+            && primordialDoc.Project.Language == LanguageNames.CSharp
+            && GlobalOptionService.GetOption(LanguageServerProjectSystemOptionsStorage.EnableFileBasedPrograms);
+        await BeginLoadingProjectWithPrimordialAsync(primordialDoc.FilePath, primordialProjectId: primordialDoc.Project.Id, doDesignTimeBuild);
+
+        return primordialDoc;
+
+        TextDocument AddPrimordialDocument(DocumentUri uri, SourceText documentText, string languageId)
+        {
+            var languageInfoProvider = _lspServices.GetRequiredService<ILanguageInfoProvider>();
+            if (!languageInfoProvider.TryGetLanguageInformation(uri, languageId, out var languageInformation))
+            {
+                Contract.Fail($"Could not find language information for {uri} with absolute path {documentFilePath}");
+            }
+
+            var workspace = Workspace;
+            var sourceTextLoader = new SourceTextLoader(documentText, documentFilePath);
+            var projectInfo = MiscellaneousFileUtilities.CreateMiscellaneousProjectInfoForDocument(
+                workspace, documentFilePath, sourceTextLoader, languageInformation, documentText.ChecksumAlgorithm, workspace.Services.SolutionServices, []);
+
+            ProjectFactory.ApplyChangeToWorkspace(workspace => workspace.OnProjectAdded(projectInfo));
+
+            // https://github.com/dotnet/roslyn/pull/78267
+            // Work around an issue where opening a Razor file in the misc workspace causes a crash.
+            if (languageInformation.LanguageName == LanguageInfoProvider.RazorLanguageName)
+            {
+                var docId = projectInfo.AdditionalDocuments.Single().Id;
+                return workspace.CurrentSolution.GetRequiredAdditionalDocument(docId);
+            }
+
+            var id = projectInfo.Documents.Single().Id;
+            return workspace.CurrentSolution.GetRequiredDocument(id);
+        }
+    }
+
+    public async ValueTask TryRemoveMiscellaneousDocumentAsync(DocumentUri uri, bool removeFromMetadataWorkspace)
+    {
+        var documentPath = GetDocumentFilePath(uri);
+        if (removeFromMetadataWorkspace && _metadataAsSourceFileService.TryRemoveDocumentFromWorkspace(documentPath))
+        {
+            return;
+        }
+
+        await UnloadProjectAsync(documentPath);
+    }
+
+    protected override async Task<(RemoteProjectFile projectFile, bool hasAllInformation, BuildHostProcessKind preferred, BuildHostProcessKind actual)?> TryLoadProjectInMSBuildHostAsync(
+        BuildHostProcessManager buildHostProcessManager, string documentPath, CancellationToken cancellationToken)
+    {
+        const BuildHostProcessKind buildHostKind = BuildHostProcessKind.NetCore;
+        var buildHost = await buildHostProcessManager.GetBuildHostAsync(buildHostKind, cancellationToken);
+
+        var loader = ProjectFactory.CreateFileTextLoader(documentPath);
+        var textAndVersion = await loader.LoadTextAsync(new LoadTextOptions(SourceHashAlgorithms.Default), cancellationToken);
+        var (virtualProjectContent, isFileBasedProgram) = VirtualCSharpFileBasedProgramProject.MakeVirtualProjectContent(documentPath, textAndVersion.Text);
+
+        // When loading a virtual project, the path to the on-disk source file is not used. Instead the path is adjusted to end with .csproj.
+        // This is necessary in order to get msbuild to apply the standard c# props/targets to the project.
+        var virtualProjectPath = VirtualCSharpFileBasedProgramProject.GetVirtualProjectPath(documentPath);
+        var loadedFile = await buildHost.LoadProjectAsync(virtualProjectPath, virtualProjectContent, languageName: LanguageNames.CSharp, cancellationToken);
+        return (loadedFile, hasAllInformation: isFileBasedProgram, preferred: buildHostKind, actual: buildHostKind);
+    }
+}

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer/HostWorkspace/FileBasedProgramsWorkspaceProviderFactory.cs

@@ -0,0 +1,42 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Composition;
+using Microsoft.CodeAnalysis.Host;
+using Microsoft.CodeAnalysis.Host.Mef;
+using Microsoft.CodeAnalysis.LanguageServer.Handler;
+using Microsoft.CodeAnalysis.LanguageServer.HostWorkspace.ProjectTelemetry;
+using Microsoft.CodeAnalysis.MetadataAsSource;
+using Microsoft.CodeAnalysis.Options;
+using Microsoft.CodeAnalysis.ProjectSystem;
+using Microsoft.CodeAnalysis.Shared.TestHooks;
+using Microsoft.CommonLanguageServerProtocol.Framework;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.CodeAnalysis.LanguageServer.HostWorkspace;
+
+/// <summary>
+/// Service to create <see cref="LspMiscellaneousFilesWorkspaceProvider"/> instances.
+/// This is not exported as a <see cref="ILspServiceFactory"/> as it requires
+/// special base language server dependencies such as the <see cref="HostServices"/>
+/// </summary>
+[ExportCSharpVisualBasicStatelessLspService(typeof(ILspMiscellaneousFilesWorkspaceProviderFactory), WellKnownLspServerKinds.CSharpVisualBasicLspServer), Shared]
+[method: ImportingConstructor]
+[method: Obsolete(MefConstruction.ImportingConstructorMessage, error: true)]
+internal sealed class FileBasedProgramsWorkspaceProviderFactory(
+    IMetadataAsSourceFileService metadataAsSourceFileService,
+    LanguageServerWorkspaceFactory workspaceFactory,
+    IFileChangeWatcher fileChangeWatcher,
+    IGlobalOptionService globalOptionService,
+    ILoggerFactory loggerFactory,
+    IAsynchronousOperationListenerProvider listenerProvider,
+    ProjectLoadTelemetryReporter projectLoadTelemetry,
+    ServerConfigurationFactory serverConfigurationFactory,
+    BinlogNamer binlogNamer) : ILspMiscellaneousFilesWorkspaceProviderFactory
+{
+    public ILspMiscellaneousFilesWorkspaceProvider CreateLspMiscellaneousFilesWorkspaceProvider(ILspServices lspServices, HostServices hostServices)
+    {
+        return new FileBasedProgramsProjectSystem(lspServices, metadataAsSourceFileService, workspaceFactory, fileChangeWatcher, globalOptionService, loggerFactory, listenerProvider, projectLoadTelemetry, serverConfigurationFactory, binlogNamer);
+    }
+}