From c7d1e8be505586611782935dc4e2a39f41023cf1 Mon Sep 17 00:00:00 2001 From: Francisco-Gamino Date: Wed, 31 May 2023 20:15:20 -0700 Subject: [PATCH 01/10] Add support for the PowerShell new programming model --- src/DependencyManagement/DependencyManager.cs | 12 +-- src/RequestProcessor.cs | 50 ++++++++--- src/WorkerIndexing/BindingInformation.cs | 59 +++++++++++++ src/WorkerIndexing/FunctionInformation.cs | 40 +++++++++ src/WorkerIndexing/WorkerIndexingHelper.cs | 82 +++++++++++++++++++ src/resources/PowerShellWorkerStrings.resx | 6 ++ src/worker.config.json | 3 +- 7 files changed, 235 insertions(+), 17 deletions(-) create mode 100644 src/WorkerIndexing/BindingInformation.cs create mode 100644 src/WorkerIndexing/FunctionInformation.cs create mode 100644 src/WorkerIndexing/WorkerIndexingHelper.cs diff --git a/src/DependencyManagement/DependencyManager.cs b/src/DependencyManagement/DependencyManager.cs index 71caf2bd..fa4a8d9f 100644 --- a/src/DependencyManagement/DependencyManager.cs +++ b/src/DependencyManagement/DependencyManager.cs @@ -44,7 +44,7 @@ internal class DependencyManager : IDisposable #endregion public DependencyManager( - string requestMetadataDirectory = null, + string functionAppRootPath = null, IModuleProvider moduleProvider = null, IDependencyManagerStorage storage = null, IInstalledDependenciesLocator installedDependenciesLocator = null, @@ -54,7 +54,7 @@ public DependencyManager( IBackgroundDependencySnapshotContentLogger currentSnapshotContentLogger = null, ILogger logger = null) { - _storage = storage ?? new DependencyManagerStorage(GetFunctionAppRootPath(requestMetadataDirectory)); + _storage = storage ?? new DependencyManagerStorage(GetFunctionAppRootPath(functionAppRootPath)); _installedDependenciesLocator = installedDependenciesLocator ?? new InstalledDependenciesLocator(_storage, logger); var snapshotContentLogger = new PowerShellModuleSnapshotLogger(); _installer = installer ?? new DependencySnapshotInstaller( @@ -257,14 +257,14 @@ private bool AreAcceptableDependenciesAlreadyInstalled() return _storage.SnapshotExists(_currentSnapshotPath); } - private static string GetFunctionAppRootPath(string requestMetadataDirectory) + private static string GetFunctionAppRootPath(string functionAppRootPath) { - if (string.IsNullOrWhiteSpace(requestMetadataDirectory)) + if (string.IsNullOrWhiteSpace(functionAppRootPath)) { - throw new ArgumentException("Empty request metadata directory path", nameof(requestMetadataDirectory)); + throw new ArgumentException("Empty function app root path", nameof(functionAppRootPath)); } - return Path.GetFullPath(Path.Join(requestMetadataDirectory, "..")); + return functionAppRootPath; } #endregion diff --git a/src/RequestProcessor.cs b/src/RequestProcessor.cs index ad192aa0..f1977b6e 100644 --- a/src/RequestProcessor.cs +++ b/src/RequestProcessor.cs @@ -18,7 +18,10 @@ namespace Microsoft.Azure.Functions.PowerShellWorker { + using Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing; + using Microsoft.PowerShell; using System.Diagnostics; + using System.Text.Json; using LogLevel = Microsoft.Azure.WebJobs.Script.Grpc.Messages.RpcLog.Types.Level; using System.Runtime.InteropServices; @@ -69,6 +72,8 @@ internal RequestProcessor(MessagingStream msgStream, System.Management.Automatio // If an invocation is cancelled, host will receive an invocation response with status cancelled. _requestHandlers.Add(StreamingMessage.ContentOneofCase.InvocationCancel, ProcessInvocationCancelRequest); + _requestHandlers.Add(StreamingMessage.ContentOneofCase.FunctionsMetadataRequest, ProcessFunctionMetadataRequest); + _requestHandlers.Add(StreamingMessage.ContentOneofCase.FunctionEnvironmentReloadRequest, ProcessFunctionEnvironmentReloadRequest); } @@ -130,10 +135,26 @@ internal StreamingMessage ProcessWorkerInitRequest(StreamingMessage request) response.WorkerInitResponse.WorkerMetadata = GetWorkerMetadata(_pwshVersion); + // Previously, the dependency management would happen just prior to the dependency download in the + // first function load request. Now that we have the FunctionAppDirectory in the WorkerInitRequest, + // we can do the setup of these variables in the function load request. We need these variables initialized + // for the FunctionMetadataRequest, should it be sent. + _dependencyManager = new DependencyManager(request.WorkerInitRequest.FunctionAppDirectory, logger: rpcLogger); + + // The profile is invoke during this stage. + // TODO: Initialize the first PowerShell instance, but delay the invocation of the profile. + // The first PowerShell instance will be used to import the PowerShell SDK to generate the function metadata + // Issue: We do not know if Managed Dependencies is enabled until the first function load. + _powershellPool.Initialize(_firstPwshInstance); + rpcLogger.Log(isUserOnlyLog: false, LogLevel.Trace, string.Format(PowerShellWorkerStrings.WorkerInitCompleted, stopwatch.ElapsedMilliseconds)); } catch (Exception e) { + // This is a terminating failure: we will need to return a failure response to + // all subsequent 'FunctionLoadRequest'. Cache the exception so we can reuse it in future calls. + _initTerminatingError = e; + status.Status = StatusResult.Types.Status.Failure; status.Exception = e.ToRpcException(); return response; @@ -210,26 +231,20 @@ internal StreamingMessage ProcessFunctionLoadRequest(StreamingMessage request) { try { - _isFunctionAppInitialized = true; - var rpcLogger = new RpcLogger(_msgStream); rpcLogger.SetContext(request.RequestId, null); - _dependencyManager = new DependencyManager(request.FunctionLoadRequest.Metadata.Directory, logger: rpcLogger); - var managedDependenciesPath = _dependencyManager.Initialize(request, rpcLogger); - - SetupAppRootPathAndModulePath(functionLoadRequest, managedDependenciesPath); + _isFunctionAppInitialized = true; - _powershellPool.Initialize(_firstPwshInstance); + var managedDependenciesPath = _dependencyManager.Initialize(request, rpcLogger); + SetupAppRootPathAndModulePath(request.FunctionLoadRequest, managedDependenciesPath); // Start the download asynchronously if needed. _dependencyManager.StartDependencyInstallationIfNeeded(request, _firstPwshInstance, rpcLogger); - - rpcLogger.Log(isUserOnlyLog: false, LogLevel.Trace, string.Format(PowerShellWorkerStrings.FirstFunctionLoadCompleted, stopwatch.ElapsedMilliseconds)); } catch (Exception e) { - // Failure that happens during this step is terminating and we will need to return a failure response to + // This is a terminating failure: we will need to return a failure response to // all subsequent 'FunctionLoadRequest'. Cache the exception so we can reuse it in future calls. _initTerminatingError = e; @@ -362,6 +377,18 @@ internal StreamingMessage ProcessInvocationCancelRequest(StreamingMessage reques return null; } + private StreamingMessage ProcessFunctionMetadataRequest(StreamingMessage request) + { + StreamingMessage response = NewStreamingMessageTemplate( + request.RequestId, + StreamingMessage.ContentOneofCase.FunctionMetadataResponse, + out StatusResult status); + + response.FunctionMetadataResponse.FunctionMetadataResults.AddRange(WorkerIndexingHelper.IndexFunctions(request.FunctionsMetadataRequest.FunctionAppDirectory)); + + return response; + } + internal StreamingMessage ProcessFunctionEnvironmentReloadRequest(StreamingMessage request) { var stopwatch = new Stopwatch(); @@ -415,6 +442,9 @@ private StreamingMessage NewStreamingMessageTemplate(string requestId, Streaming case StreamingMessage.ContentOneofCase.FunctionEnvironmentReloadResponse: response.FunctionEnvironmentReloadResponse = new FunctionEnvironmentReloadResponse() { Result = status }; break; + case StreamingMessage.ContentOneofCase.FunctionMetadataResponse: + response.FunctionMetadataResponse = new FunctionMetadataResponse() { Result = status }; + break; default: throw new InvalidOperationException("Unreachable code."); } diff --git a/src/WorkerIndexing/BindingInformation.cs b/src/WorkerIndexing/BindingInformation.cs new file mode 100644 index 00000000..309adf3d --- /dev/null +++ b/src/WorkerIndexing/BindingInformation.cs @@ -0,0 +1,59 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using Microsoft.Azure.WebJobs.Script.Grpc.Messages; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; + +namespace Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing +{ + internal class BindingInformation + { + private const string BindingNameKey = "name"; + private const string BindingDirectionKey = "direction"; + private const string BindingTypeKey = "type"; + public enum Directions + { + Unknown = -1, + In = 0, + Out = 1, + Inout = 2 + } + + public Directions Direction { get; set; } = Directions.Unknown; + public string Type { get; set; } = ""; + public string Name { get; set; } = ""; + public Dictionary otherInformation { get; set; } = new Dictionary(); + + internal string ConvertToRpcRawBinding(out BindingInfo bindingInfo) + { + string rawBinding = string.Empty; + JObject rawBindingObject = new JObject(); + rawBindingObject.Add(BindingNameKey, Name); + BindingInfo outInfo = new BindingInfo(); + + + if (Direction == Directions.Unknown) + { + throw new Exception(string.Format(PowerShellWorkerStrings.InvalidBindingInfoDirection, Name)); + } + outInfo.Direction = (BindingInfo.Types.Direction)Direction; + rawBindingObject.Add(BindingDirectionKey, Enum.GetName(typeof(BindingInfo.Types.Direction), outInfo.Direction).ToLower()); + outInfo.Type = Type; + rawBindingObject.Add(BindingTypeKey, Type); + + foreach (KeyValuePair pair in otherInformation) + { + rawBindingObject.Add(pair.Key, JToken.FromObject(pair.Value)); + } + + rawBinding = JsonConvert.SerializeObject(rawBindingObject); + bindingInfo = outInfo; + return rawBinding; + } + } +} diff --git a/src/WorkerIndexing/FunctionInformation.cs b/src/WorkerIndexing/FunctionInformation.cs new file mode 100644 index 00000000..2408225b --- /dev/null +++ b/src/WorkerIndexing/FunctionInformation.cs @@ -0,0 +1,40 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using Microsoft.Azure.WebJobs.Script.Grpc.Messages; +using System.Collections.Generic; + +namespace Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing +{ + internal class FunctionInformation + { + private const string FunctionLanguagePowerShell = "powershell"; + + public string Directory { get; set; } = ""; + public string ScriptFile { get; set; } = ""; + public string Name { get; set; } = ""; + public string EntryPoint { get; set; } = ""; + public string FunctionId { get; set; } = ""; + public List Bindings { get; set; } = new List(); + + internal RpcFunctionMetadata ConvertToRpc() + { + RpcFunctionMetadata returnMetadata = new RpcFunctionMetadata(); + returnMetadata.FunctionId = FunctionId; + returnMetadata.Directory = Directory; + returnMetadata.EntryPoint = EntryPoint; + returnMetadata.Name = Name; + returnMetadata.ScriptFile = ScriptFile; + returnMetadata.Language = FunctionLanguagePowerShell; + foreach(BindingInformation binding in Bindings) + { + string rawBinding = binding.ConvertToRpcRawBinding(out BindingInfo bindingInfo); + returnMetadata.Bindings.Add(binding.Name, bindingInfo); + returnMetadata.RawBindings.Add(rawBinding); + } + return returnMetadata; + } + } +} diff --git a/src/WorkerIndexing/WorkerIndexingHelper.cs b/src/WorkerIndexing/WorkerIndexingHelper.cs new file mode 100644 index 00000000..945ed0c6 --- /dev/null +++ b/src/WorkerIndexing/WorkerIndexingHelper.cs @@ -0,0 +1,82 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +using Microsoft.Azure.WebJobs.Script.Grpc.Messages; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.IO; +using System.Management.Automation; +using System.Management.Automation.Runspaces; + +namespace Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing +{ + internal class WorkerIndexingHelper + { + // TODO: Follow up with the PowerShell on why we get a CommandNotFoundException when using the module qualified cmdlet name. + //const string GetFunctionsMetadataCmdletName = "AzureFunctions.PowerShell.SDK\\Get-FunctionsMetadata"; + const string GetFunctionsMetadataCmdletName = "Get-FunctionsMetadata"; + const string AzureFunctionsPowerShellSDKModuleName = "AzureFunctions.PowerShell.SDK"; + + internal static IEnumerable IndexFunctions(string baseDir) + { + List indexedFunctions = new List(); + + // This is not the correct way to deal with getting a runspace for the cmdlet. + + // Firstly, creating a runspace is expensive. If we are going to generate a runspace, it should be done on + // the function load request so that it can be created while the host is processing. + + // Secondly, this assumes that the AzureFunctions.PowerShell.SDK module is present on the machine/VM's + // PSModulePath. On an Azure instance, it will not be. What we need to do here is move the call + // to SetupAppRootPathAndModulePath in RequestProcessor to the init request, and then use the + // _firstPwshInstance to invoke the Get-FunctionsMetadata command. The only issue with this is that + // SetupAppRootPathAndModulePath needs the initial function init request in order to know if managed + // dependencies are enabled in this function app. + + // Proposed solutions: + // 1. Pass ManagedDependencyEnabled flag in the worker init request + // 2. Change the flow, so that _firstPwshInstance is initialized in worker init with the PSModulePath + // assuming that managed dependencies are enabled, and then revert the PSModulePath in the first function + // init request should the managed dependencies not be enabled. + // 3. Continue using a new runspace for invoking Get-FunctionsMetadata, but initialize it in worker init and + // point the PsModulePath to the module path bundled with the worker. + + InitialSessionState initial = InitialSessionState.CreateDefault(); + Runspace runspace = RunspaceFactory.CreateRunspace(initial); + runspace.Open(); + System.Management.Automation.PowerShell _powershell = System.Management.Automation.PowerShell.Create(); + _powershell.Runspace = runspace; + + var modulePath = Path.Join(AppDomain.CurrentDomain.BaseDirectory, "Modules", AzureFunctionsPowerShellSDKModuleName); + // Console.WriteLine("modulePath; {0}", modulePath); + // Console.WriteLine("Importing module..."); + _powershell.AddCommand("Import-Module").AddParameter("Name", modulePath) + .AddParameter("Force", true); + + // Console.WriteLine("Call Get-FunctionsMetadata"); + _powershell.AddCommand(GetFunctionsMetadataCmdletName).AddArgument(baseDir); + string outputString = string.Empty; + foreach (PSObject rawMetadata in _powershell.Invoke()) + { + if (outputString != string.Empty) + { + throw new Exception(PowerShellWorkerStrings.GetFunctionsMetadataMultipleResultsError); + } + outputString = rawMetadata.ToString(); + } + _powershell.Commands.Clear(); + + List functionInformations = JsonConvert.DeserializeObject>(outputString); + + foreach(FunctionInformation fi in functionInformations) + { + indexedFunctions.Add(fi.ConvertToRpc()); + } + + return indexedFunctions; + } + } +} diff --git a/src/resources/PowerShellWorkerStrings.resx b/src/resources/PowerShellWorkerStrings.resx index 0685d7b4..fee35278 100644 --- a/src/resources/PowerShellWorkerStrings.resx +++ b/src/resources/PowerShellWorkerStrings.resx @@ -379,4 +379,10 @@ Managed Dependencies is not supported in Linux Consumption on Legion. Please remove all module references from requirements.psd1 and include the function app dependencies with the function app content. For more information, please see https://aka.ms/functions-powershell-include-modules. + + Multiple results from metadata cmdlet. + + + Invalid binding direction. Binding name: {0} + \ No newline at end of file diff --git a/src/worker.config.json b/src/worker.config.json index 4dff9102..4dad98f5 100644 --- a/src/worker.config.json +++ b/src/worker.config.json @@ -6,6 +6,7 @@ "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/Microsoft.Azure.Functions.PowerShellWorker.dll", "supportedRuntimeVersions":["7", "7.2", "7.4"], "defaultRuntimeVersion": "7.2", - "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*" + "sanitizeRuntimeVersionRegex":"\\d+\\.?\\d*", + "workerIndexing": "true" } } From 87aabf26d3d4cac49ddae70e1f998753f9db1da3 Mon Sep 17 00:00:00 2001 From: Francisco-Gamino Date: Wed, 31 May 2023 20:15:34 -0700 Subject: [PATCH 02/10] Update tests --- .../DependencyManagementTests.cs | 51 +++++-------------- .../DependencyManagerTests.cs | 2 +- 2 files changed, 15 insertions(+), 38 deletions(-) diff --git a/test/Unit/DependencyManagement/DependencyManagementTests.cs b/test/Unit/DependencyManagement/DependencyManagementTests.cs index 49c70ee8..42f05a37 100644 --- a/test/Unit/DependencyManagement/DependencyManagementTests.cs +++ b/test/Unit/DependencyManagement/DependencyManagementTests.cs @@ -72,12 +72,10 @@ private string InitializeManagedDependenciesDirectory(string functionAppRootPath private void TestCaseCleanup() { // We run a test case clean up to reset DependencyManager.Dependencies and DependencyManager.DependenciesPath - var functionFolderPath = Path.Combine(_dependencyManagementDirectory, "DirectoryThatDoesNotExist"); - var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true); try { - using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory)) + using (var dependencyManager = new DependencyManager(_dependencyManagementDirectory)) { dependencyManager.Initialize(_testLogger); } @@ -95,14 +93,11 @@ public void TestManagedDependencyBasicRequirements() { // Test case setup. var requirementsDirectoryName = "BasicRequirements"; - var functionFolderPath = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName, "FunctionDirectory"); var functionAppRoot = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName); var managedDependenciesFolderPath = InitializeManagedDependenciesDirectory(functionAppRoot); - var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true); - // Create DependencyManager and process the requirements.psd1 file at the function app root. - using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, logger: _testLogger)) + using (var dependencyManager = new DependencyManager(functionAppRoot, logger: _testLogger)) { var currentDependenciesPath = dependencyManager.Initialize(_testLogger); @@ -126,13 +121,11 @@ public void TestManagedDependencyEmptyHashtableRequirement() { // Test case setup. var requirementsDirectoryName = "EmptyHashtableRequirement"; - var functionFolderPath = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName, "FunctionDirectory"); var functionAppRoot = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName); var managedDependenciesFolderPath = InitializeManagedDependenciesDirectory(functionAppRoot); - var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true); // Create DependencyManager and process the requirements.psd1 file at the function app root. - using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, logger: _testLogger)) + using (var dependencyManager = new DependencyManager(functionAppRoot, logger: _testLogger)) { var currentDependenciesPath = dependencyManager.Initialize(_testLogger); @@ -150,10 +143,9 @@ public void TestManagedDependencyNoHashtableRequirementShouldThrow() { // Test case setup. var requirementsDirectoryName = "NoHashtableRequirements"; - var functionFolderPath = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName, "FunctionDirectory"); - var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true); + var functionAppRoot = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName); - using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, logger: _testLogger)) + using (var dependencyManager = new DependencyManager(functionAppRoot, logger: _testLogger)) { // Trying to set the functionApp dependencies should throw since requirements.psd1 is not a hash table. var exception = Assert.Throws( @@ -169,10 +161,9 @@ public void TestManagedDependencyInvalidRequirementsFormatShouldThrow() { // Test case setup. var requirementsDirectoryName = "InvalidRequirementsFormat"; - var functionFolderPath = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName, "FunctionDirectory"); - var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true); + var functionAppRoot = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName); - using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, logger: _testLogger)) + using (var dependencyManager = new DependencyManager(functionAppRoot, logger: _testLogger)) { // Trying to set the functionApp dependencies should throw since the module version // in requirements.psd1 is not in a valid format. @@ -190,10 +181,9 @@ public void TestManagedDependencyNoRequirementsFileShouldThrow() { // Test case setup. var requirementsDirectoryName = "ModuleThatDoesNotExist"; - var functionFolderPath = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName, "FunctionDirectory"); - var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true); + var functionAppRoot = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName); - using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, logger: _testLogger)) + using (var dependencyManager = new DependencyManager(functionAppRoot, logger: _testLogger)) { // Trying to set the functionApp dependencies should throw since no // requirements.psd1 is found at the function app root. @@ -212,16 +202,14 @@ public void TestManagedDependencySuccessfulModuleDownload() { // Test case setup. var requirementsDirectoryName = "BasicRequirements"; - var functionFolderPath = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName, "FunctionDirectory"); var functionAppRoot = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName); var managedDependenciesFolderPath = InitializeManagedDependenciesDirectory(functionAppRoot); - var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true); // Configure MockModuleProvider to mimic a successful download. var mockModuleProvider = new MockModuleProvider { SuccessfulDownload = true }; // Create DependencyManager and process the requirements.psd1 file at the function app root. - using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, mockModuleProvider, logger: _testLogger)) + using (var dependencyManager = new DependencyManager(functionAppRoot, mockModuleProvider, logger: _testLogger)) { dependencyManager.Initialize(_testLogger); @@ -258,17 +246,14 @@ public void TestManagedDependencySuccessfulModuleDownloadAfterTwoTries() { // Test case setup var requirementsDirectoryName = "BasicRequirements"; - var functionFolderPath = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName, "FunctionDirectory"); var functionAppRoot = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName); var managedDependenciesFolderPath = InitializeManagedDependenciesDirectory(functionAppRoot); - var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true); - // Configure MockModuleProvider to not throw in the RunSaveModuleCommand call after 2 tries. var mockModuleProvider = new MockModuleProvider { ShouldNotThrowAfterCount = 2 }; // Create DependencyManager and process the requirements.psd1 file at the function app root. - using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, mockModuleProvider, logger: _testLogger)) + using (var dependencyManager = new DependencyManager(functionAppRoot, mockModuleProvider, logger: _testLogger)) { dependencyManager.Initialize(_testLogger); @@ -318,14 +303,11 @@ public void TestManagedDependencyRetryLogicMaxNumberOfTries() { // Test case setup var requirementsDirectoryName = "BasicRequirements"; - var functionFolderPath = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName, "FunctionDirectory"); var functionAppRoot = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName); var managedDependenciesFolderPath = InitializeManagedDependenciesDirectory(functionAppRoot); - var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true); - // Create DependencyManager and process the requirements.psd1 file at the function app root. - using (var dependencyManager = new DependencyManager(functionLoadRequest.Metadata.Directory, new MockModuleProvider(), logger: _testLogger)) + using (var dependencyManager = new DependencyManager(functionAppRoot, new MockModuleProvider(), logger: _testLogger)) { dependencyManager.Initialize(_testLogger); @@ -372,16 +354,13 @@ public void FunctionAppExecutionShouldStopIfNoPreviousDependenciesAreInstalled() { // Test case setup var requirementsDirectoryName = "BasicRequirements"; - var functionFolderPath = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName, "FunctionDirectory"); var functionAppRoot = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName); var managedDependenciesFolderPath = InitializeManagedDependenciesDirectory(functionAppRoot); - var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true); - // Create DependencyManager and configure it to mimic being unable to reach // the PSGallery to retrieve the latest module version using (var dependencyManager = new DependencyManager( - functionLoadRequest.Metadata.Directory, + functionAppRoot, new MockModuleProvider { GetLatestModuleVersionThrows = true }, logger: _testLogger)) { @@ -409,15 +388,13 @@ public void FunctionAppExecutionShouldContinueIfPreviousDependenciesExist() { // Test case setup var requirementsDirectoryName = "BasicRequirements"; - var functionFolderPath = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName, "FunctionDirectory"); var functionAppRoot = Path.Combine(_dependencyManagementDirectory, requirementsDirectoryName); var managedDependenciesFolderPath = InitializeManagedDependenciesDirectory(functionAppRoot); - var functionLoadRequest = GetFuncLoadRequest(functionFolderPath, true); // Create DependencyManager and configure it to mimic being unable to reach // the PSGallery to retrive the latest module version using (var dependencyManager = new DependencyManager( - functionLoadRequest.Metadata.Directory, + functionAppRoot, new MockModuleProvider { GetLatestModuleVersionThrows = true }, logger: _testLogger)) { diff --git a/test/Unit/DependencyManagement/DependencyManagerTests.cs b/test/Unit/DependencyManagement/DependencyManagerTests.cs index fbe8a13c..0280e3e8 100644 --- a/test/Unit/DependencyManagement/DependencyManagerTests.cs +++ b/test/Unit/DependencyManagement/DependencyManagerTests.cs @@ -372,7 +372,7 @@ private void VerifyMessageLogged(LogLevel expectedLogLevel, string expectedMessa private DependencyManager CreateDependencyManagerWithMocks() { return new DependencyManager( - requestMetadataDirectory: null, + functionAppRootPath: null, moduleProvider: null, storage: _mockStorage.Object, installedDependenciesLocator: _mockInstalledDependenciesLocator.Object, From 4faede88e73b380487a952395e07c52b80a9a470 Mon Sep 17 00:00:00 2001 From: Francisco-Gamino Date: Sun, 4 Jun 2023 21:09:18 -0700 Subject: [PATCH 03/10] Suppress NU5110 and NU5111 --- .../Microsoft.Azure.Functions.PowerShellWorker.Package.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/Microsoft.Azure.Functions.PowerShellWorker.Package.csproj b/package/Microsoft.Azure.Functions.PowerShellWorker.Package.csproj index 5d21fdd4..eb7b4bee 100644 --- a/package/Microsoft.Azure.Functions.PowerShellWorker.Package.csproj +++ b/package/Microsoft.Azure.Functions.PowerShellWorker.Package.csproj @@ -9,7 +9,7 @@ Licensed under the MIT license. See LICENSE file in the project root for full li true false true - NU5100;NU5123 + NU5100;NU5110;NU5111;NU5123 Microsoft.Azure.Functions.PowerShellWorker.nuspec configuration=$(Configuration);targetFramework=$(TargetFramework);version=$(Version) From 8656ad292446e41d1bcb82b125978cc30205ece5 Mon Sep 17 00:00:00 2001 From: Francisco-Gamino Date: Sun, 4 Jun 2023 21:09:48 -0700 Subject: [PATCH 04/10] Add AzureFunctions.PowerShell.SDK to requirements.psd1 --- src/requirements.psd1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/requirements.psd1 b/src/requirements.psd1 index b68bd0c6..15ec5c5a 100644 --- a/src/requirements.psd1 +++ b/src/requirements.psd1 @@ -16,4 +16,8 @@ Version = '1.4.7' Target = 'src/Modules' } + 'AzureFunctions.PowerShell.SDK' = @{ + Version = '0.0.3' + Target = 'src/Modules' + } } From 334eee899a7811d5c39b83e4d7e6b9721caec6f8 Mon Sep 17 00:00:00 2001 From: Francisco-Gamino Date: Sun, 4 Jun 2023 21:36:25 -0700 Subject: [PATCH 05/10] Update logic to show the error messages coming from Get-FunctionsMetadata --- src/RequestProcessor.cs | 6 +- src/WorkerIndexing/WorkerIndexingHelper.cs | 68 ++++++++++++++++++---- src/resources/PowerShellWorkerStrings.resx | 3 + 3 files changed, 65 insertions(+), 12 deletions(-) diff --git a/src/RequestProcessor.cs b/src/RequestProcessor.cs index f1977b6e..d6cdbbed 100644 --- a/src/RequestProcessor.cs +++ b/src/RequestProcessor.cs @@ -384,7 +384,11 @@ private StreamingMessage ProcessFunctionMetadataRequest(StreamingMessage request StreamingMessage.ContentOneofCase.FunctionMetadataResponse, out StatusResult status); - response.FunctionMetadataResponse.FunctionMetadataResults.AddRange(WorkerIndexingHelper.IndexFunctions(request.FunctionsMetadataRequest.FunctionAppDirectory)); + var rpcLogger = new RpcLogger(_msgStream); + rpcLogger.SetContext(request.RequestId, null); + + //response.FunctionMetadataResponse.FunctionMetadataResults.AddRange(WorkerIndexingHelper.IndexFunctions(request.FunctionsMetadataRequest.FunctionAppDirectory)); + response.FunctionMetadataResponse.FunctionMetadataResults.AddRange(WorkerIndexingHelper.IndexFunctions(request.FunctionsMetadataRequest.FunctionAppDirectory, rpcLogger)); return response; } diff --git a/src/WorkerIndexing/WorkerIndexingHelper.cs b/src/WorkerIndexing/WorkerIndexingHelper.cs index 945ed0c6..6d45389d 100644 --- a/src/WorkerIndexing/WorkerIndexingHelper.cs +++ b/src/WorkerIndexing/WorkerIndexingHelper.cs @@ -3,13 +3,18 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. // +using Microsoft.Azure.Functions.PowerShellWorker.Utility; using Microsoft.Azure.WebJobs.Script.Grpc.Messages; using Newtonsoft.Json; using System; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.IO; using System.Management.Automation; using System.Management.Automation.Runspaces; +using LogLevel = Microsoft.Azure.WebJobs.Script.Grpc.Messages.RpcLog.Types.Level; +using System.Text; +using Microsoft.Azure.Functions.PowerShellWorker.PowerShell; namespace Microsoft.Azure.Functions.PowerShellWorker.WorkerIndexing { @@ -19,8 +24,10 @@ internal class WorkerIndexingHelper //const string GetFunctionsMetadataCmdletName = "AzureFunctions.PowerShell.SDK\\Get-FunctionsMetadata"; const string GetFunctionsMetadataCmdletName = "Get-FunctionsMetadata"; const string AzureFunctionsPowerShellSDKModuleName = "AzureFunctions.PowerShell.SDK"; + private static readonly ErrorRecordFormatter _errorRecordFormatter = new ErrorRecordFormatter(); - internal static IEnumerable IndexFunctions(string baseDir) + //internal static IEnumerable IndexFunctions(string baseDir) + internal static IEnumerable IndexFunctions(string baseDir, ILogger logger) { List indexedFunctions = new List(); @@ -50,16 +57,56 @@ internal static IEnumerable IndexFunctions(string baseDir) System.Management.Automation.PowerShell _powershell = System.Management.Automation.PowerShell.Create(); _powershell.Runspace = runspace; - var modulePath = Path.Join(AppDomain.CurrentDomain.BaseDirectory, "Modules", AzureFunctionsPowerShellSDKModuleName); - // Console.WriteLine("modulePath; {0}", modulePath); - // Console.WriteLine("Importing module..."); - _powershell.AddCommand("Import-Module").AddParameter("Name", modulePath) - .AddParameter("Force", true); - - // Console.WriteLine("Call Get-FunctionsMetadata"); - _powershell.AddCommand(GetFunctionsMetadataCmdletName).AddArgument(baseDir); string outputString = string.Empty; - foreach (PSObject rawMetadata in _powershell.Invoke()) + Exception exception = null; + Collection results = null; + var cmdletExecutionHadErrors = false; + var exceptionThrown = false; + string errorMsg = null; + + try + { + _powershell.AddCommand(GetFunctionsMetadataCmdletName).AddArgument(baseDir); + results = _powershell.Invoke(); + cmdletExecutionHadErrors = _powershell.HadErrors; + } + catch (Exception ex) + { + exceptionThrown = true; + exception = ex; + throw; + } + finally + { + if (exceptionThrown) + { + errorMsg = string.Format(PowerShellWorkerStrings.ErrorsWhileExecutingGetFunctionsMetadata, exception.ToString()); + } + else if (cmdletExecutionHadErrors) + { + var errorCollection = _powershell.Streams.Error; + + var stringBuilder = new StringBuilder(); + foreach (var errorRecord in errorCollection) + { + var message = _errorRecordFormatter.Format(errorRecord); + stringBuilder.AppendLine(message); + } + + errorMsg = string.Format(PowerShellWorkerStrings.ErrorsWhileExecutingGetFunctionsMetadata, stringBuilder.ToString()); + } + + if (errorMsg != null) + { + logger.Log(isUserOnlyLog: true, LogLevel.Error, errorMsg, exception); + throw new Exception(errorMsg); + } + + _powershell.Commands.Clear(); + } + + // TODO: The GetFunctionsMetadataCmdlet should never return more than one result. Make sure that this is the case and remove this code. + foreach (PSObject rawMetadata in results) { if (outputString != string.Empty) { @@ -67,7 +114,6 @@ internal static IEnumerable IndexFunctions(string baseDir) } outputString = rawMetadata.ToString(); } - _powershell.Commands.Clear(); List functionInformations = JsonConvert.DeserializeObject>(outputString); diff --git a/src/resources/PowerShellWorkerStrings.resx b/src/resources/PowerShellWorkerStrings.resx index fee35278..df954b96 100644 --- a/src/resources/PowerShellWorkerStrings.resx +++ b/src/resources/PowerShellWorkerStrings.resx @@ -385,4 +385,7 @@ Invalid binding direction. Binding name: {0} + + Errors reported while executing Get-FunctionsMetadata cmldet. {0}. + \ No newline at end of file From e34b400110c0d2a4252f86ff33eca4a0805bf730 Mon Sep 17 00:00:00 2001 From: Francisco-Gamino Date: Thu, 8 Jun 2023 14:16:16 -0700 Subject: [PATCH 06/10] Fix typo --- src/resources/PowerShellWorkerStrings.resx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/PowerShellWorkerStrings.resx b/src/resources/PowerShellWorkerStrings.resx index df954b96..c58631bf 100644 --- a/src/resources/PowerShellWorkerStrings.resx +++ b/src/resources/PowerShellWorkerStrings.resx @@ -386,6 +386,6 @@ Invalid binding direction. Binding name: {0} - Errors reported while executing Get-FunctionsMetadata cmldet. {0}. + Errors reported while executing Get-FunctionsMetadata cmdlet. {0}. \ No newline at end of file From 94f303318800855b711acc049f835fb3f41ff861 Mon Sep 17 00:00:00 2001 From: Francisco-Gamino Date: Mon, 26 Jun 2023 13:42:58 -0700 Subject: [PATCH 07/10] Update logic to set the function app root for both programming models --- src/FunctionLoader.cs | 5 +-- src/RequestProcessor.cs | 48 ++++++++++++++-------- src/WorkerIndexing/WorkerIndexingHelper.cs | 10 +++-- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/src/FunctionLoader.cs b/src/FunctionLoader.cs index ae33dbf1..1611ccdd 100644 --- a/src/FunctionLoader.cs +++ b/src/FunctionLoader.cs @@ -72,10 +72,9 @@ internal static void ClearLoadedFunctions() /// Setup the well known paths about the FunctionApp. /// This method is called only once during the code start. /// - internal static void SetupWellKnownPaths(FunctionLoadRequest request, string managedDependenciesPath) + internal static void SetupWellKnownPaths(string functionAppRootPath, string managedDependenciesPath) { - // Resolve the FunctionApp root path - FunctionAppRootPath = Path.GetFullPath(Path.Join(request.Metadata.Directory, "..")); + FunctionAppRootPath = functionAppRootPath; // Resolve module paths var appLevelModulesPath = Path.Join(FunctionAppRootPath, "Modules"); diff --git a/src/RequestProcessor.cs b/src/RequestProcessor.cs index d6cdbbed..e79ff1ac 100644 --- a/src/RequestProcessor.cs +++ b/src/RequestProcessor.cs @@ -32,6 +32,7 @@ internal class RequestProcessor private readonly PowerShellManagerPool _powershellPool; private DependencyManager _dependencyManager; private string _pwshVersion; + private string _functionAppRootPath; // Holds the exception if an issue is encountered while processing the function app dependencies. private Exception _initTerminatingError; @@ -135,18 +136,6 @@ internal StreamingMessage ProcessWorkerInitRequest(StreamingMessage request) response.WorkerInitResponse.WorkerMetadata = GetWorkerMetadata(_pwshVersion); - // Previously, the dependency management would happen just prior to the dependency download in the - // first function load request. Now that we have the FunctionAppDirectory in the WorkerInitRequest, - // we can do the setup of these variables in the function load request. We need these variables initialized - // for the FunctionMetadataRequest, should it be sent. - _dependencyManager = new DependencyManager(request.WorkerInitRequest.FunctionAppDirectory, logger: rpcLogger); - - // The profile is invoke during this stage. - // TODO: Initialize the first PowerShell instance, but delay the invocation of the profile. - // The first PowerShell instance will be used to import the PowerShell SDK to generate the function metadata - // Issue: We do not know if Managed Dependencies is enabled until the first function load. - _powershellPool.Initialize(_firstPwshInstance); - rpcLogger.Log(isUserOnlyLog: false, LogLevel.Trace, string.Format(PowerShellWorkerStrings.WorkerInitCompleted, stopwatch.ElapsedMilliseconds)); } catch (Exception e) @@ -231,16 +220,39 @@ internal StreamingMessage ProcessFunctionLoadRequest(StreamingMessage request) { try { + _isFunctionAppInitialized = true; + var rpcLogger = new RpcLogger(_msgStream); rpcLogger.SetContext(request.RequestId, null); - _isFunctionAppInitialized = true; + // _functionAppRootPath is set in ProcessFunctionMetadataRequest for the v2 progamming model. + if (_functionAppRootPath == null) + { + // If _functionAppRootPath is null, this means that this is an app for the v1 programming model. + _functionAppRootPath = request.FunctionLoadRequest.Metadata.Directory; + } + if (string.IsNullOrWhiteSpace(_functionAppRootPath)) + { + throw new ArgumentException("Failed to resolve the function app root", nameof(_functionAppRootPath)); + } + + _dependencyManager = new DependencyManager(_functionAppRootPath, logger: rpcLogger); var managedDependenciesPath = _dependencyManager.Initialize(request, rpcLogger); - SetupAppRootPathAndModulePath(request.FunctionLoadRequest, managedDependenciesPath); + SetupAppRootPathAndModulePath(_functionAppRootPath, managedDependenciesPath); + + // The profile is invoke when the instance is initialized. + // TODO: Initialize the first PowerShell instance but delay the invocation of the profile until the first function load. + // The first PowerShell instance will be used to import the AzureFunctions.PowerShell.SDK to generate the function metadata + // Issue: We do not know if Managed Dependencies is enabled until the first function load. + _powershellPool.Initialize(_firstPwshInstance); + // Start the download asynchronously if needed. _dependencyManager.StartDependencyInstallationIfNeeded(request, _firstPwshInstance, rpcLogger); + + rpcLogger.Log(isUserOnlyLog: false, LogLevel.Trace, string.Format(PowerShellWorkerStrings.FirstFunctionLoadCompleted, stopwatch.ElapsedMilliseconds)); + } catch (Exception e) { @@ -387,8 +399,8 @@ private StreamingMessage ProcessFunctionMetadataRequest(StreamingMessage request var rpcLogger = new RpcLogger(_msgStream); rpcLogger.SetContext(request.RequestId, null); - //response.FunctionMetadataResponse.FunctionMetadataResults.AddRange(WorkerIndexingHelper.IndexFunctions(request.FunctionsMetadataRequest.FunctionAppDirectory)); - response.FunctionMetadataResponse.FunctionMetadataResults.AddRange(WorkerIndexingHelper.IndexFunctions(request.FunctionsMetadataRequest.FunctionAppDirectory, rpcLogger)); + _functionAppRootPath = request.FunctionsMetadataRequest.FunctionAppDirectory; + response.FunctionMetadataResponse.FunctionMetadataResults.AddRange(WorkerIndexingHelper.IndexFunctions(_functionAppRootPath, rpcLogger)); return response; } @@ -559,9 +571,9 @@ private static void BindOutputFromResult(InvocationResponse response, AzFunction } } - private void SetupAppRootPathAndModulePath(FunctionLoadRequest functionLoadRequest, string managedDependenciesPath) + private void SetupAppRootPathAndModulePath(string functionAppRootPath, string managedDependenciesPath) { - FunctionLoader.SetupWellKnownPaths(functionLoadRequest, managedDependenciesPath); + FunctionLoader.SetupWellKnownPaths(functionAppRootPath, managedDependenciesPath); if (FunctionLoader.FunctionAppRootPath == null) { diff --git a/src/WorkerIndexing/WorkerIndexingHelper.cs b/src/WorkerIndexing/WorkerIndexingHelper.cs index 6d45389d..87f2bc21 100644 --- a/src/WorkerIndexing/WorkerIndexingHelper.cs +++ b/src/WorkerIndexing/WorkerIndexingHelper.cs @@ -26,9 +26,13 @@ internal class WorkerIndexingHelper const string AzureFunctionsPowerShellSDKModuleName = "AzureFunctions.PowerShell.SDK"; private static readonly ErrorRecordFormatter _errorRecordFormatter = new ErrorRecordFormatter(); - //internal static IEnumerable IndexFunctions(string baseDir) - internal static IEnumerable IndexFunctions(string baseDir, ILogger logger) + internal static IEnumerable IndexFunctions(string functionAppRootPath, ILogger logger) { + if (string.IsNullOrWhiteSpace(functionAppRootPath)) + { + throw new ArgumentException("Empty function app root path", nameof(functionAppRootPath)); + } + List indexedFunctions = new List(); // This is not the correct way to deal with getting a runspace for the cmdlet. @@ -66,7 +70,7 @@ internal static IEnumerable IndexFunctions(string baseDir, try { - _powershell.AddCommand(GetFunctionsMetadataCmdletName).AddArgument(baseDir); + _powershell.AddCommand(GetFunctionsMetadataCmdletName).AddArgument(functionAppRootPath); results = _powershell.Invoke(); cmdletExecutionHadErrors = _powershell.HadErrors; } From 52cb4786ee6502ceff9bc67a3e56cde543864638 Mon Sep 17 00:00:00 2001 From: Francisco-Gamino Date: Mon, 26 Jun 2023 13:43:32 -0700 Subject: [PATCH 08/10] Update tests --- test/Unit/Modules/HelperModuleTests.cs | 2 +- test/Unit/PowerShell/PowerShellManagerTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Unit/Modules/HelperModuleTests.cs b/test/Unit/Modules/HelperModuleTests.cs index acf7c8c8..315ca200 100644 --- a/test/Unit/Modules/HelperModuleTests.cs +++ b/test/Unit/Modules/HelperModuleTests.cs @@ -49,7 +49,7 @@ static HelperModuleTests() }; var funcLoadReq = new FunctionLoadRequest { FunctionId = "FunctionId", Metadata = rpcFuncMetadata }; - FunctionLoader.SetupWellKnownPaths(funcLoadReq, managedDependenciesPath: null); + FunctionLoader.SetupWellKnownPaths(funcLoadReq.Metadata.Directory, managedDependenciesPath: null); s_pwsh = Utils.NewPwshInstance(); s_funcInfo = new AzFunctionInfo(rpcFuncMetadata); } diff --git a/test/Unit/PowerShell/PowerShellManagerTests.cs b/test/Unit/PowerShell/PowerShellManagerTests.cs index 36588611..726f5e4a 100644 --- a/test/Unit/PowerShell/PowerShellManagerTests.cs +++ b/test/Unit/PowerShell/PowerShellManagerTests.cs @@ -80,7 +80,7 @@ static PowerShellManagerTests() }; s_functionLoadRequest = new FunctionLoadRequest { FunctionId = "FunctionId", Metadata = rpcFunctionMetadata }; - FunctionLoader.SetupWellKnownPaths(s_functionLoadRequest, managedDependenciesPath: null); + FunctionLoader.SetupWellKnownPaths(s_functionLoadRequest.Metadata.Directory, managedDependenciesPath: null); } // Have a single place to get a PowerShellManager for testing. From 299cc4eb63d772b2817ca429477b75dcd87450b4 Mon Sep 17 00:00:00 2001 From: Francisco-Gamino Date: Mon, 26 Jun 2023 20:21:55 -0700 Subject: [PATCH 09/10] Fix functionAppRootPath for the v1 model --- src/RequestProcessor.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RequestProcessor.cs b/src/RequestProcessor.cs index e79ff1ac..9934c079 100644 --- a/src/RequestProcessor.cs +++ b/src/RequestProcessor.cs @@ -229,7 +229,7 @@ internal StreamingMessage ProcessFunctionLoadRequest(StreamingMessage request) if (_functionAppRootPath == null) { // If _functionAppRootPath is null, this means that this is an app for the v1 programming model. - _functionAppRootPath = request.FunctionLoadRequest.Metadata.Directory; + _functionAppRootPath = Path.GetFullPath(Path.Join(request.FunctionLoadRequest.Metadata.Directory, "..")); } if (string.IsNullOrWhiteSpace(_functionAppRootPath)) From 47db28cee10c5c28e2357b7bb6615cdff6c439b2 Mon Sep 17 00:00:00 2001 From: Francisco-Gamino Date: Mon, 26 Jun 2023 21:40:07 -0700 Subject: [PATCH 10/10] Add System.IO namespace --- src/RequestProcessor.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/RequestProcessor.cs b/src/RequestProcessor.cs index 9934c079..ed29a40a 100644 --- a/src/RequestProcessor.cs +++ b/src/RequestProcessor.cs @@ -4,6 +4,7 @@ // using System; +using System.IO; using System.Collections; using System.Collections.Generic; using System.Management.Automation.Remoting;