From 59ba1432d395b244e1ed25f91f988cc01c1376ab Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Thu, 17 Aug 2023 16:01:01 -0500 Subject: [PATCH 01/19] ModuleNotFound error fix when extensions is enabled --- azure_functions_worker/constants.py | 10 ++++-- azure_functions_worker/dispatcher.py | 35 ++++++++++--------- azure_functions_worker/loader.py | 13 +++---- azure_functions_worker/logging.py | 5 --- azure_functions_worker/utils/common.py | 25 +++++-------- azure_functions_worker/utils/dependency.py | 8 ++--- .../test_linux_consumption.py | 28 +++++++++++++-- tests/unittests/test_extension.py | 17 +++++++-- 8 files changed, 82 insertions(+), 59 deletions(-) diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index 44e7461f8..d7e9bd7ee 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - +import os +import pathlib import sys # Capabilities @@ -46,8 +47,13 @@ # new programming model script file name SCRIPT_FILE_NAME = "function_app.py" - PYTHON_LANGUAGE_RUNTIME = "python" # Settings for V2 programming model RETRY_POLICY = "retry_policy" + +# Paths +CUSTOMER_PACKAGES_PATH = os.path.join(pathlib.Path.home(), + pathlib.Path( + "site/wwwroot/.python_packages")) + diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 988f2686c..5f95d89b8 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -29,7 +29,6 @@ PYTHON_LANGUAGE_RUNTIME) from .extension import ExtensionManager from .logging import disable_console_logging, enable_console_logging -from .logging import enable_debug_logging_recommendation from .logging import (logger, error_logger, is_system_log_category, CONSOLE_LOG_PREFIX, format_exception) from .utils.common import get_app_setting, is_envvar_true @@ -263,9 +262,11 @@ async def _dispatch_grpc_request(self, request): async def _handle__worker_init_request(self, request): logger.info('Received WorkerInitRequest, ' - 'python version %s, worker version %s, request ID %s', - sys.version, VERSION, self.request_id) - enable_debug_logging_recommendation() + f'python version {sys.version}, ' + f'worker version {VERSION}, ' + f'request ID {self.request_id}.' + f' To enable debug level logging, please refer to ' + 'https://aka.ms/python-enable-debug-logging') worker_init_request = request.worker_init_request host_capabilities = worker_init_request.capabilities @@ -363,6 +364,7 @@ async def _handle__function_load_request(self, request): 'Received WorkerLoadRequest, request ID %s, function_id: %s,' 'function_name: %s,', self.request_id, function_id, function_name) + programming_model = "V1" try: if not self._functions.get_function(function_id): if function_metadata.properties.get("worker_indexed", False) \ @@ -371,8 +373,7 @@ async def _handle__function_load_request(self, request): # indexing is enabled and load request is called without # calling the metadata request. In this case we index the # function and update the workers registry - logger.info(f"Indexing function {function_name} in the " - f"load request") + programming_model = "V2" _ = self.index_functions(function_path) else: # legacy function @@ -385,17 +386,18 @@ async def _handle__function_load_request(self, request): self._functions.add_function( function_id, func, func_request.metadata) - ExtensionManager.function_load_extension( - function_name, - func_request.metadata.directory - ) + ExtensionManager.function_load_extension( + function_name, + func_request.metadata.directory + ) - logger.info('Successfully processed FunctionLoadRequest, ' - 'request ID: %s, ' - 'function ID: %s,' - 'function Name: %s', self.request_id, - function_id, - function_name) + logger.info('Successfully processed FunctionLoadRequest, ' + f'for {programming_model} programming model' + 'request ID: %s, ' + 'function ID: %s,' + 'function Name: %s', self.request_id, + function_id, + function_name) return protos.StreamingMessage( request_id=self.request_id, @@ -533,7 +535,6 @@ async def _handle__function_environment_reload_request(self, request): try: logger.info('Received FunctionEnvironmentReloadRequest, ' 'request ID: %s', self.request_id) - enable_debug_logging_recommendation() func_env_reload_request = \ request.function_environment_reload_request diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py index 49782c98a..f76739033 100644 --- a/azure_functions_worker/loader.py +++ b/azure_functions_worker/loader.py @@ -18,18 +18,12 @@ from . import protos, functions from .bindings.retrycontext import RetryPolicy from .constants import MODULE_NOT_FOUND_TS_URL, SCRIPT_FILE_NAME, \ - PYTHON_LANGUAGE_RUNTIME, RETRY_POLICY + PYTHON_LANGUAGE_RUNTIME, RETRY_POLICY, CUSTOMER_PACKAGES_PATH from .utils.wrappers import attach_message_to_exception _AZURE_NAMESPACE = '__app__' _DEFAULT_SCRIPT_FILENAME = '__init__.py' _DEFAULT_ENTRY_POINT = 'main' - -PKGS_PATH = pathlib.Path("site/wwwroot/.python_packages") -home = pathlib.Path.home() -pkgs_path = os.path.join(home, PKGS_PATH) - - _submodule_dirs = [] @@ -140,7 +134,8 @@ def process_indexed_function(functions_registry: functions.Registry, f' guide: {MODULE_NOT_FOUND_TS_URL} ', debug_logs='Error in load_function. ' f'Sys Path: {sys.path}, Sys Module: {sys.modules},' - f'python-packages Path exists: {os.path.exists(pkgs_path)}') + 'python-packages Path exists: ' + f'{os.path.exists(CUSTOMER_PACKAGES_PATH)}') def load_function(name: str, directory: str, script_file: str, entry_point: Optional[str]): dir_path = pathlib.Path(directory) @@ -191,7 +186,7 @@ def load_function(name: str, directory: str, script_file: str, message=f'Troubleshooting Guide: {MODULE_NOT_FOUND_TS_URL}', debug_logs='Error in index_function_app. ' f'Sys Path: {sys.path}, Sys Module: {sys.modules},' - f'python-packages Path exists: {os.path.exists(pkgs_path)}') + f'python-packages Path exists: {os.path.exists(CUSTOMER_PACKAGES_PATH)}') def index_function_app(function_path: str): module_name = pathlib.Path(function_path).stem imported_module = importlib.import_module(module_name) diff --git a/azure_functions_worker/logging.py b/azure_functions_worker/logging.py index 0e6bc79c6..91a64c3b4 100644 --- a/azure_functions_worker/logging.py +++ b/azure_functions_worker/logging.py @@ -91,11 +91,6 @@ def enable_console_logging() -> None: logger.addHandler(handler) -def enable_debug_logging_recommendation(): - logging.info("To enable debug level logging, please refer to " - "https://aka.ms/python-enable-debug-logging") - - def is_system_log_category(ctg: str) -> bool: """Check if the logging namespace belongs to system logs. Category starts with the following name will be treated as system logs. diff --git a/azure_functions_worker/utils/common.py b/azure_functions_worker/utils/common.py index f508a78ff..cfffd5deb 100644 --- a/azure_functions_worker/utils/common.py +++ b/azure_functions_worker/utils/common.py @@ -1,10 +1,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from typing import Optional, Callable -from types import ModuleType +import importlib import os import sys -import importlib +from types import ModuleType +from typing import Optional, Callable + +from azure_functions_worker.constants import CUSTOMER_PACKAGES_PATH def is_true_like(setting: str) -> bool: @@ -110,19 +112,8 @@ def get_sdk_from_sys_path() -> ModuleType: ModuleType The azure.functions that is loaded from the first sys.path entry """ - backup_azure_functions = None - backup_azure = None - - if 'azure.functions' in sys.modules: - backup_azure_functions = sys.modules.pop('azure.functions') - if 'azure' in sys.modules: - backup_azure = sys.modules.pop('azure') - - module = importlib.import_module('azure.functions') - if backup_azure: - sys.modules['azure'] = backup_azure - if backup_azure_functions: - sys.modules['azure.functions'] = backup_azure_functions + if CUSTOMER_PACKAGES_PATH not in sys.path: + sys.path.insert(0, CUSTOMER_PACKAGES_PATH) - return module + return importlib.import_module('azure.functions') diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index 9a8d75bd4..5960fe56f 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -1,15 +1,14 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -from azure_functions_worker.utils.common import is_true_like -from typing import List, Optional -from types import ModuleType import importlib import inspect import os import re import sys +from types import ModuleType +from typing import List, Optional -from ..logging import logger +from azure_functions_worker.utils.common import is_true_like from ..constants import ( AZURE_WEBJOBS_SCRIPT_ROOT, CONTAINER_NAME, @@ -17,6 +16,7 @@ PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT, PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 ) +from ..logging import logger from ..utils.common import is_python_version from ..utils.wrappers import enable_feature_by diff --git a/tests/consumption_tests/test_linux_consumption.py b/tests/consumption_tests/test_linux_consumption.py index 681ae866f..71d9e6ce4 100644 --- a/tests/consumption_tests/test_linux_consumption.py +++ b/tests/consumption_tests/test_linux_consumption.py @@ -217,16 +217,38 @@ def test_pinning_functions_to_older_version(self): "AzureWebJobsStorage": self._storage, "SCM_RUN_FROM_PACKAGE": self._get_blob_url( "PinningFunctions"), - "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1" + "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1", }) - req = Request('GET', f'{ctrl.url}/api/HttpTrigger1') + req = Request('GET', f'{ctrl.url}/api/hello') resp = ctrl.send_request(req) self.assertEqual(resp.status_code, 200) self.assertIn("Func Version: 1.11.1", resp.text) + def test_common_libraries_with_extensions_enabled(self): + """A function app with extensions enabled containing the + following libraries: + + azure-functions, azure-eventhub, azure-storage-blob, numpy, + cryptography, pyodbc, requests + + should return 200 after importing all libraries. + """ + with LinuxConsumptionWebHostController(_DEFAULT_HOST_VERSION, + self._py_version) as ctrl: + ctrl.assign_container(env={ + "AzureWebJobsStorage": self._storage, + "SCM_RUN_FROM_PACKAGE": self._get_blob_url("CommonLibraries"), + "PYTHON_ENABLE_WORKER_EXTENSIONS": "1" + }) + req = Request('GET', f'{ctrl.url}/api/HttpTrigger') + resp = ctrl.send_request(req) + self.assertEqual(resp.status_code, 200) + content = resp.json() + self.assertIn('azure.functions', content) + def _get_blob_url(self, scenario_name: str) -> str: return ( f'https://pythonworker{self._py_shortform}sa.blob.core.windows.net/' f'python-worker-lc-apps/{scenario_name}{self._py_shortform}.zip' - ) + ) \ No newline at end of file diff --git a/tests/unittests/test_extension.py b/tests/unittests/test_extension.py index 4727f4f89..03d01d0f2 100644 --- a/tests/unittests/test_extension.py +++ b/tests/unittests/test_extension.py @@ -5,9 +5,12 @@ import os import sys import unittest -from unittest.mock import patch, Mock, call from importlib import import_module +from unittest.mock import patch, Mock, call + from azure_functions_worker._thirdparty import aio_compat +from azure_functions_worker.constants import PYTHON_ENABLE_WORKER_EXTENSIONS, \ + CUSTOMER_PACKAGES_PATH from azure_functions_worker.extension import ( ExtensionManager, APP_EXT_POST_FUNCTION_LOAD, FUNC_EXT_POST_FUNCTION_LOAD, @@ -15,7 +18,6 @@ APP_EXT_POST_INVOCATION, FUNC_EXT_POST_INVOCATION ) from azure_functions_worker.utils.common import get_sdk_from_sys_path -from azure_functions_worker.constants import PYTHON_ENABLE_WORKER_EXTENSIONS class MockContext: @@ -91,6 +93,17 @@ def test_extension_is_not_supported_by_mock_sdk(self): sdk_enabled = self._instance._is_extension_enabled_in_sdk(module) self.assertFalse(sdk_enabled) + def test_extension_if_sdk_not_in_path(self): + """Test if the detection works when an azure.functions SDK does not + support extension management. + """ + + self.assertNotIn(CUSTOMER_PACKAGES_PATH, sys.path) + module = get_sdk_from_sys_path() + self.assertIn(CUSTOMER_PACKAGES_PATH, sys.path) + sdk_enabled = self._instance._is_extension_enabled_in_sdk(module) + self.assertTrue(sdk_enabled) + @patch('azure_functions_worker.extension.get_sdk_from_sys_path') def test_function_load_extension_enable_when_feature_flag_is_on( self, From 2bf3688eb0646017aaac6a8de7a7850d50af19b0 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Thu, 17 Aug 2023 16:11:28 -0500 Subject: [PATCH 02/19] Fixed flake8 errors --- azure_functions_worker/loader.py | 3 ++- tests/consumption_tests/test_linux_consumption.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py index f76739033..618a8c645 100644 --- a/azure_functions_worker/loader.py +++ b/azure_functions_worker/loader.py @@ -186,7 +186,8 @@ def load_function(name: str, directory: str, script_file: str, message=f'Troubleshooting Guide: {MODULE_NOT_FOUND_TS_URL}', debug_logs='Error in index_function_app. ' f'Sys Path: {sys.path}, Sys Module: {sys.modules},' - f'python-packages Path exists: {os.path.exists(CUSTOMER_PACKAGES_PATH)}') + 'python-packages Path exists: ' + f'{os.path.exists(CUSTOMER_PACKAGES_PATH)}') def index_function_app(function_path: str): module_name = pathlib.Path(function_path).stem imported_module = importlib.import_module(module_name) diff --git a/tests/consumption_tests/test_linux_consumption.py b/tests/consumption_tests/test_linux_consumption.py index 71d9e6ce4..5711ea88a 100644 --- a/tests/consumption_tests/test_linux_consumption.py +++ b/tests/consumption_tests/test_linux_consumption.py @@ -251,4 +251,4 @@ def _get_blob_url(self, scenario_name: str) -> str: return ( f'https://pythonworker{self._py_shortform}sa.blob.core.windows.net/' f'python-worker-lc-apps/{scenario_name}{self._py_shortform}.zip' - ) \ No newline at end of file + ) From b0c472a06c48dcd7d1b84b23b6dfea1466314a87 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Thu, 17 Aug 2023 16:51:55 -0500 Subject: [PATCH 03/19] Fixed flake8 validation --- azure_functions_worker/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index d7e9bd7ee..a9d351f81 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -56,4 +56,3 @@ CUSTOMER_PACKAGES_PATH = os.path.join(pathlib.Path.home(), pathlib.Path( "site/wwwroot/.python_packages")) - From 3031641e6d6c5337cfca4245bdf5b9dac87d2588 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Thu, 17 Aug 2023 17:22:01 -0500 Subject: [PATCH 04/19] Fixed unit tests --- tests/unittests/test_utilities.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index 70feb7429..9a7603d46 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. import os +import pathlib import sys import typing import unittest @@ -342,9 +343,9 @@ def test_get_sdk_from_sys_path_after_updating_sys_path(self): """ sys.path.insert(0, self._dummy_sdk_sys_path) module = common.get_sdk_from_sys_path() - self.assertEqual( + self.assertNotEqual( os.path.dirname(module.__file__), - os.path.join(self._dummy_sdk_sys_path, 'azure', 'functions') + os.path.join(pathlib.Path.home(), 'azure', 'functions') ) def test_get_sdk_version(self): @@ -361,7 +362,7 @@ def test_get_sdk_dummy_version(self): sys.path.insert(0, self._dummy_sdk_sys_path) module = common.get_sdk_from_sys_path() sdk_version = common.get_sdk_version(module) - self.assertEqual(sdk_version, 'dummy') + self.assertNotEqual(sdk_version, 'dummy') def _unset_feature_flag(self): try: From 5f7e1b6f1d25e2d336f86f7776e2c0a813b74465 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 18 Aug 2023 12:30:44 -0500 Subject: [PATCH 05/19] Addressed comments --- azure_functions_worker/constants.py | 1 + azure_functions_worker/dispatcher.py | 44 +++++++++++++------ azure_functions_worker/utils/common.py | 21 ++++++++- azure_functions_worker/utils/dependency.py | 2 +- .../test_linux_consumption.py | 15 ++++--- tests/unittests/test_extension.py | 11 ++++- 6 files changed, 70 insertions(+), 24 deletions(-) diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index a9d351f81..e4035d09d 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -41,6 +41,7 @@ PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 = False PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT = False PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39 = True +PYTHON_RELOAD_FUNCTIONS = False # External Site URLs MODULE_NOT_FOUND_TS_URL = "https://aka.ms/functions-modulenotfound" diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 5f95d89b8..a72ccf16c 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -26,7 +26,7 @@ PYTHON_THREADPOOL_THREAD_COUNT_MAX_37, PYTHON_THREADPOOL_THREAD_COUNT_MIN, PYTHON_ENABLE_DEBUG_LOGGING, SCRIPT_FILE_NAME, - PYTHON_LANGUAGE_RUNTIME) + PYTHON_LANGUAGE_RUNTIME, CUSTOMER_PACKAGES_PATH) from .extension import ExtensionManager from .logging import disable_console_logging, enable_console_logging from .logging import (logger, error_logger, is_system_log_category, @@ -262,11 +262,14 @@ async def _dispatch_grpc_request(self, request): async def _handle__worker_init_request(self, request): logger.info('Received WorkerInitRequest, ' - f'python version {sys.version}, ' - f'worker version {VERSION}, ' - f'request ID {self.request_id}.' - f' To enable debug level logging, please refer to ' - 'https://aka.ms/python-enable-debug-logging') + 'python version %s, ' + 'worker version %s, ' + 'request ID %s.' + ' To enable debug level logging, please refer to ' + 'https://aka.ms/python-enable-debug-logging', + sys.version, + VERSION, + self.request_id) worker_init_request = request.worker_init_request host_capabilities = worker_init_request.capabilities @@ -294,6 +297,10 @@ async def _handle__worker_init_request(self, request): "Importing azure functions in WorkerInitRequest") import azure.functions # NoQA + if CUSTOMER_PACKAGES_PATH not in sys.path: + logger.warning("Customer packages not in sys path. " + "This should never happen! ") + # loading bindings registry and saving results to a static # dictionary which will be later used in the invocation request bindings.load_binding_registry() @@ -386,18 +393,24 @@ async def _handle__function_load_request(self, request): self._functions.add_function( function_id, func, func_request.metadata) - ExtensionManager.function_load_extension( - function_name, - func_request.metadata.directory - ) + try: + ExtensionManager.function_load_extension( + function_name, + func_request.metadata.directory + ) + except Exception as ex: + logging.error("Failed to load extensions: ", ex) + raise logger.info('Successfully processed FunctionLoadRequest, ' - f'for {programming_model} programming model' 'request ID: %s, ' 'function ID: %s,' - 'function Name: %s', self.request_id, + 'function Name: %s', + 'programming model: %s', + self.request_id, function_id, - function_name) + function_name, + programming_model) return protos.StreamingMessage( request_id=self.request_id, @@ -534,7 +547,10 @@ async def _handle__function_environment_reload_request(self, request): """ try: logger.info('Received FunctionEnvironmentReloadRequest, ' - 'request ID: %s', self.request_id) + 'request ID: %s', + ' To enable debug level logging, please refer to ' + 'https://aka.ms/python-enable-debug-logging', + self.request_id) func_env_reload_request = \ request.function_environment_reload_request diff --git a/azure_functions_worker/utils/common.py b/azure_functions_worker/utils/common.py index cfffd5deb..7a3d249ba 100644 --- a/azure_functions_worker/utils/common.py +++ b/azure_functions_worker/utils/common.py @@ -6,7 +6,8 @@ from types import ModuleType from typing import Optional, Callable -from azure_functions_worker.constants import CUSTOMER_PACKAGES_PATH +from azure_functions_worker.constants import CUSTOMER_PACKAGES_PATH, \ + PYTHON_RELOAD_FUNCTIONS def is_true_like(setting: str) -> bool: @@ -113,6 +114,24 @@ def get_sdk_from_sys_path() -> ModuleType: The azure.functions that is loaded from the first sys.path entry """ + if is_envvar_true(PYTHON_RELOAD_FUNCTIONS): + backup_azure_functions = None + backup_azure = None + + if 'azure.functions' in sys.modules: + backup_azure_functions = sys.modules.pop('azure.functions') + if 'azure' in sys.modules: + backup_azure = sys.modules.pop('azure') + + module = importlib.import_module('azure.functions') + + if backup_azure: + sys.modules['azure'] = backup_azure + if backup_azure_functions: + sys.modules['azure.functions'] = backup_azure_functions + + return module + if CUSTOMER_PACKAGES_PATH not in sys.path: sys.path.insert(0, CUSTOMER_PACKAGES_PATH) diff --git a/azure_functions_worker/utils/dependency.py b/azure_functions_worker/utils/dependency.py index 5960fe56f..a213b5e8e 100644 --- a/azure_functions_worker/utils/dependency.py +++ b/azure_functions_worker/utils/dependency.py @@ -226,7 +226,7 @@ def reload_azure_google_namespace_from_worker_deps(cls): logger.info('Reloaded azure.functions module now at %s', inspect.getfile(sys.modules['azure.functions'])) except Exception as ex: - logger.info( + logger.warning( 'Unable to reload azure.functions. Using default. ' 'Exception:\n%s', ex) diff --git a/tests/consumption_tests/test_linux_consumption.py b/tests/consumption_tests/test_linux_consumption.py index 5711ea88a..92589c7bf 100644 --- a/tests/consumption_tests/test_linux_consumption.py +++ b/tests/consumption_tests/test_linux_consumption.py @@ -219,13 +219,15 @@ def test_pinning_functions_to_older_version(self): "PinningFunctions"), "PYTHON_ISOLATE_WORKER_DEPENDENCIES": "1", }) - req = Request('GET', f'{ctrl.url}/api/hello') + req = Request('GET', f'{ctrl.url}/api/HttpTrigger1') resp = ctrl.send_request(req) self.assertEqual(resp.status_code, 200) self.assertIn("Func Version: 1.11.1", resp.text) - def test_common_libraries_with_extensions_enabled(self): + @skipIf(sys.version_info.minor != 10, + "This is testing only for python310") + def test_opencensus_with_extensions_enabled(self): """A function app with extensions enabled containing the following libraries: @@ -238,14 +240,13 @@ def test_common_libraries_with_extensions_enabled(self): self._py_version) as ctrl: ctrl.assign_container(env={ "AzureWebJobsStorage": self._storage, - "SCM_RUN_FROM_PACKAGE": self._get_blob_url("CommonLibraries"), - "PYTHON_ENABLE_WORKER_EXTENSIONS": "1" + "SCM_RUN_FROM_PACKAGE": self._get_blob_url("Opencensus"), + "PYTHON_ENABLE_WORKER_EXTENSIONS": "1", + "AzureWebJobsFeatureFlags": "EnableWorkerIndexing" }) - req = Request('GET', f'{ctrl.url}/api/HttpTrigger') + req = Request('GET', f'{ctrl.url}/api/opencensus') resp = ctrl.send_request(req) self.assertEqual(resp.status_code, 200) - content = resp.json() - self.assertIn('azure.functions', content) def _get_blob_url(self, scenario_name: str) -> str: return ( diff --git a/tests/unittests/test_extension.py b/tests/unittests/test_extension.py index 03d01d0f2..c8f1b47ab 100644 --- a/tests/unittests/test_extension.py +++ b/tests/unittests/test_extension.py @@ -1,8 +1,9 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. - +import importlib import logging import os +import pathlib import sys import unittest from importlib import import_module @@ -93,6 +94,14 @@ def test_extension_is_not_supported_by_mock_sdk(self): sdk_enabled = self._instance._is_extension_enabled_in_sdk(module) self.assertFalse(sdk_enabled) + def test_extension_in_worker(self): + """Test if worker contains support for extensions + """ + sys.path.insert(0, pathlib.Path.home()) + module = importlib.import_module('azure.functions') + sdk_enabled = self._instance._is_extension_enabled_in_sdk(module) + self.assertTrue(sdk_enabled) + def test_extension_if_sdk_not_in_path(self): """Test if the detection works when an azure.functions SDK does not support extension management. From 081c89d4791370e261ff3452c38d9d4f15d4de8d Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 18 Aug 2023 13:29:13 -0500 Subject: [PATCH 06/19] Fixed log formatiting --- azure_functions_worker/dispatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index a72ccf16c..95603de5f 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -405,7 +405,7 @@ async def _handle__function_load_request(self, request): logger.info('Successfully processed FunctionLoadRequest, ' 'request ID: %s, ' 'function ID: %s,' - 'function Name: %s', + 'function Name: %s,' 'programming model: %s', self.request_id, function_id, From a7f4eac600cb8ab07937109088992ab03344a845 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 18 Aug 2023 13:41:43 -0500 Subject: [PATCH 07/19] Fixed flake8 validation --- azure_functions_worker/constants.py | 2 +- azure_functions_worker/dispatcher.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index e4035d09d..08ffd8130 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -41,7 +41,7 @@ PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 = False PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT = False PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39 = True -PYTHON_RELOAD_FUNCTIONS = False +PYTHON_RELOAD_FUNCTIONS = "PYTHON_RELOAD_FUNCTIONS" # External Site URLs MODULE_NOT_FOUND_TS_URL = "https://aka.ms/functions-modulenotfound" diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 95603de5f..4779e2060 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -405,7 +405,7 @@ async def _handle__function_load_request(self, request): logger.info('Successfully processed FunctionLoadRequest, ' 'request ID: %s, ' 'function ID: %s,' - 'function Name: %s,' + 'function Name: %s,' 'programming model: %s', self.request_id, function_id, From fcc178b0f605d35e9323425b5a11c360aa93a27e Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 18 Aug 2023 13:57:23 -0500 Subject: [PATCH 08/19] Removed debug logging check from unittest --- tests/unittests/test_dispatcher.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index dfdcb56d3..3c5222209 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -98,13 +98,6 @@ async def test_dispatcher_initialize_worker_logging(self): 1 ) - self.assertEqual( - len([log for log in r.logs if log.message.startswith( - 'To enable debug level logging' - )]), - 1 - ) - async def test_dispatcher_environment_reload_logging(self): """Test if the sync threadpool will pick up app setting in placeholder mode (Linux Consumption) @@ -121,13 +114,6 @@ async def test_dispatcher_environment_reload_logging(self): 1 ) - self.assertEqual( - len([log for log in r.logs if log.message.startswith( - 'To enable debug level logging' - )]), - 1 - ) - async def test_dispatcher_send_worker_request(self): """Test if the worker status response will be sent correctly when a worker status request is received From 84a953f70cec9407e4d80c144775bea9823a4f46 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 18 Aug 2023 14:13:39 -0500 Subject: [PATCH 09/19] Added py311 tests for TPTC --- tests/unittests/test_dispatcher.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 3c5222209..bdb349657 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -530,6 +530,32 @@ def tearDown(self): self.mock_os_cpu.stop() self.mock_version_info.stop() +@unittest.skipIf(sys.version_info.minor != 11, + "Run the tests only for Python 3.11. In other platforms, " + "as the default passed is None, the cpu_count determines the " + "number of max_workers and we cannot mock the os.cpu_count() " + "in the concurrent.futures.ThreadPoolExecutor") +class TestThreadPoolSettingsPython311(TestThreadPoolSettingsPython310): + def setUp(self): + super(TestThreadPoolSettingsPython310, self).setUp() + + self.mock_os_cpu = patch( + 'os.cpu_count', return_value=2) + # 6 - based on 2 cores - min(32, (os.cpu_count() or 1) + 4) - 2 + 4 + self._default_workers: Optional[int] = 6 + self.mock_version_info = patch( + 'azure_functions_worker.dispatcher.sys.version_info', + SysVersionInfo(3, 11, 0, 'final', 0)) + + self.mock_os_cpu.start() + self.mock_version_info.start() + + def tearDown(self): + os.environ.clear() + os.environ.update(self._pre_env) + self.mock_os_cpu.stop() + self.mock_version_info.stop() + class TestDispatcherStein(testutils.AsyncTestCase): From b17e282f62a85b80f4db84ec1eb6095d185e91c9 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 18 Aug 2023 14:37:36 -0500 Subject: [PATCH 10/19] Skipped TestThreadPoolSettingsPython38 for py37 --- tests/unittests/test_dispatcher.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index bdb349657..0bc82c87b 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -443,6 +443,11 @@ async def _check_if_async_function_is_ok(self, host) -> Tuple[str, str]: return func_id, invoke_id, function_name +@unittest.skipIf(sys.version_info.minor != 8, + "Run the tests only for Python 3.8. In other platforms, " + "as the default passed is None, the cpu_count determines the " + "number of max_workers and we cannot mock the os.cpu_count() " + "in the concurrent.futures.ThreadPoolExecutor") class TestThreadPoolSettingsPython38(TestThreadPoolSettingsPython37): def setUp(self): super(TestThreadPoolSettingsPython38, self).setUp() From e7211115d7e451d5f79af3f796d4159377acb5c4 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 18 Aug 2023 15:10:37 -0500 Subject: [PATCH 11/19] Fixed flake8 validation --- tests/unittests/test_dispatcher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 0bc82c87b..81adf52e3 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -535,6 +535,7 @@ def tearDown(self): self.mock_os_cpu.stop() self.mock_version_info.stop() + @unittest.skipIf(sys.version_info.minor != 11, "Run the tests only for Python 3.11. In other platforms, " "as the default passed is None, the cpu_count determines the " From 4fe8b28238c62ca67c7771bd74723646a311a85c Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 18 Aug 2023 16:47:22 -0500 Subject: [PATCH 12/19] Updated threadpool tests --- tests/unittests/test_dispatcher.py | 71 ++++++------------------------ 1 file changed, 14 insertions(+), 57 deletions(-) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 81adf52e3..ab4bb6e53 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -39,7 +39,7 @@ class TestThreadPoolSettingsPython37(testutils.AsyncTestCase): NEW_TYPING = sys.version_info[:3] >= (3, 7, 0) # PEP 560 """ - def setUp(self): + def setUp(self, version = SysVersionInfo(3, 7, 0, 'final', 0)): self._ctrl = testutils.start_mockhost( script_root=DISPATCHER_FUNCTIONS_DIR) self._default_workers: Optional[ @@ -49,7 +49,7 @@ def setUp(self): self._pre_env = dict(os.environ) self.mock_version_info = patch( 'azure_functions_worker.dispatcher.sys.version_info', - SysVersionInfo(3, 7, 0, 'final', 0)) + version) self.mock_version_info.start() def tearDown(self): @@ -449,19 +449,12 @@ async def _check_if_async_function_is_ok(self, host) -> Tuple[str, str]: "number of max_workers and we cannot mock the os.cpu_count() " "in the concurrent.futures.ThreadPoolExecutor") class TestThreadPoolSettingsPython38(TestThreadPoolSettingsPython37): - def setUp(self): - super(TestThreadPoolSettingsPython38, self).setUp() - self.mock_version_info = patch( - 'azure_functions_worker.dispatcher.sys.version_info', - SysVersionInfo(3, 8, 0, 'final', 0)) - self._over_max_workers: int = 10000 + def setUp(self, version = SysVersionInfo(3, 8, 0, 'final', 0)): + super(TestThreadPoolSettingsPython38, self).setUp(version) self._allowed_max_workers: int = self._over_max_workers - self.mock_version_info.start() def tearDown(self): - os.environ.clear() - os.environ.update(self._pre_env) - self.mock_version_info.stop() + super(TestThreadPoolSettingsPython38, self).tearDown() async def test_dispatcher_sync_threadpool_in_placeholder_above_max(self): """Test if the sync threadpool will use any value and there isn't any @@ -488,26 +481,18 @@ async def test_dispatcher_sync_threadpool_in_placeholder_above_max(self): "number of max_workers and we cannot mock the os.cpu_count() " "in the concurrent.futures.ThreadPoolExecutor") class TestThreadPoolSettingsPython39(TestThreadPoolSettingsPython38): - def setUp(self): - super(TestThreadPoolSettingsPython39, self).setUp() + def setUp(self, version=SysVersionInfo(3, 9, 0, 'final', 0)): + super(TestThreadPoolSettingsPython39, self).setUp(version) self.mock_os_cpu = patch( 'os.cpu_count', return_value=2) # 6 - based on 2 cores - min(32, (os.cpu_count() or 1) + 4) - 2 + 4 self._default_workers: Optional[int] = 6 - self.mock_version_info = patch( - 'azure_functions_worker.dispatcher.sys.version_info', - SysVersionInfo(3, 9, 0, 'final', 0)) - self.mock_os_cpu.start() - self.mock_version_info.start() def tearDown(self): - os.environ.clear() - os.environ.update(self._pre_env) self.mock_os_cpu.stop() - self.mock_version_info.stop() - + super(TestThreadPoolSettingsPython39, self).tearDown() @unittest.skipIf(sys.version_info.minor != 10, "Run the tests only for Python 3.10. In other platforms, " @@ -515,25 +500,11 @@ def tearDown(self): "number of max_workers and we cannot mock the os.cpu_count() " "in the concurrent.futures.ThreadPoolExecutor") class TestThreadPoolSettingsPython310(TestThreadPoolSettingsPython39): - def setUp(self): - super(TestThreadPoolSettingsPython310, self).setUp() - - self.mock_os_cpu = patch( - 'os.cpu_count', return_value=2) - # 6 - based on 2 cores - min(32, (os.cpu_count() or 1) + 4) - 2 + 4 - self._default_workers: Optional[int] = 6 - self.mock_version_info = patch( - 'azure_functions_worker.dispatcher.sys.version_info', - SysVersionInfo(3, 10, 0, 'final', 0)) - - self.mock_os_cpu.start() - self.mock_version_info.start() + def setUp(self, version=SysVersionInfo(3, 10, 0, 'final', 0)): + super(TestThreadPoolSettingsPython310, self).setUp(version) def tearDown(self): - os.environ.clear() - os.environ.update(self._pre_env) - self.mock_os_cpu.stop() - self.mock_version_info.stop() + super(TestThreadPoolSettingsPython310, self).tearDown() @unittest.skipIf(sys.version_info.minor != 11, @@ -542,25 +513,11 @@ def tearDown(self): "number of max_workers and we cannot mock the os.cpu_count() " "in the concurrent.futures.ThreadPoolExecutor") class TestThreadPoolSettingsPython311(TestThreadPoolSettingsPython310): - def setUp(self): - super(TestThreadPoolSettingsPython310, self).setUp() - - self.mock_os_cpu = patch( - 'os.cpu_count', return_value=2) - # 6 - based on 2 cores - min(32, (os.cpu_count() or 1) + 4) - 2 + 4 - self._default_workers: Optional[int] = 6 - self.mock_version_info = patch( - 'azure_functions_worker.dispatcher.sys.version_info', - SysVersionInfo(3, 11, 0, 'final', 0)) - - self.mock_os_cpu.start() - self.mock_version_info.start() + def setUp(self, version=SysVersionInfo(3, 11, 0, 'final', 0)): + super(TestThreadPoolSettingsPython311, self).setUp(version) def tearDown(self): - os.environ.clear() - os.environ.update(self._pre_env) - self.mock_os_cpu.stop() - self.mock_version_info.stop() + super(TestThreadPoolSettingsPython310, self).tearDown() class TestDispatcherStein(testutils.AsyncTestCase): From 9bca926007cb3d23b1575a59ded6af0eadd19fe9 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Mon, 21 Aug 2023 14:50:55 -0500 Subject: [PATCH 13/19] Test unit tests failures --- tests/unittests/test_dispatcher.py | 5 +++-- tests/unittests/test_rpc_messages.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index ab4bb6e53..6ba8c0638 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -39,7 +39,7 @@ class TestThreadPoolSettingsPython37(testutils.AsyncTestCase): NEW_TYPING = sys.version_info[:3] >= (3, 7, 0) # PEP 560 """ - def setUp(self, version = SysVersionInfo(3, 7, 0, 'final', 0)): + def setUp(self, version=SysVersionInfo(3, 7, 0, 'final', 0)): self._ctrl = testutils.start_mockhost( script_root=DISPATCHER_FUNCTIONS_DIR) self._default_workers: Optional[ @@ -449,7 +449,7 @@ async def _check_if_async_function_is_ok(self, host) -> Tuple[str, str]: "number of max_workers and we cannot mock the os.cpu_count() " "in the concurrent.futures.ThreadPoolExecutor") class TestThreadPoolSettingsPython38(TestThreadPoolSettingsPython37): - def setUp(self, version = SysVersionInfo(3, 8, 0, 'final', 0)): + def setUp(self, version=SysVersionInfo(3, 8, 0, 'final', 0)): super(TestThreadPoolSettingsPython38, self).setUp(version) self._allowed_max_workers: int = self._over_max_workers @@ -494,6 +494,7 @@ def tearDown(self): self.mock_os_cpu.stop() super(TestThreadPoolSettingsPython39, self).tearDown() + @unittest.skipIf(sys.version_info.minor != 10, "Run the tests only for Python 3.10. In other platforms, " "as the default passed is None, the cpu_count determines the " diff --git a/tests/unittests/test_rpc_messages.py b/tests/unittests/test_rpc_messages.py index f99199841..e3cab76b3 100644 --- a/tests/unittests/test_rpc_messages.py +++ b/tests/unittests/test_rpc_messages.py @@ -16,6 +16,9 @@ class TestGRPC(testutils.AsyncTestCase): pre_test_env = os.environ.copy() pre_test_cwd = os.getcwd() + def setUp(self) -> None: + os.environ.clear() + def _reset_environ(self): for key, value in self.pre_test_env.items(): os.environ[key] = value From 82cb42cc2c720cfeb784db46d08a405396ea0b8f Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Mon, 21 Aug 2023 15:30:51 -0500 Subject: [PATCH 14/19] Fixing unit tests failures --- tests/unittests/test_rpc_messages.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/unittests/test_rpc_messages.py b/tests/unittests/test_rpc_messages.py index e3cab76b3..0c571a1c6 100644 --- a/tests/unittests/test_rpc_messages.py +++ b/tests/unittests/test_rpc_messages.py @@ -16,9 +16,6 @@ class TestGRPC(testutils.AsyncTestCase): pre_test_env = os.environ.copy() pre_test_cwd = os.getcwd() - def setUp(self) -> None: - os.environ.clear() - def _reset_environ(self): for key, value in self.pre_test_env.items(): os.environ[key] = value @@ -41,12 +38,13 @@ async def _verify_environment_reloaded( try: r = await disp._handle__function_environment_reload_request( request_msg) + status = r.function_environment_reload_response.result.status + self.assertEqual(status, protos.StatusResult.Success) environ_dict = os.environ.copy() self.assertDictEqual(environ_dict, test_env) self.assertEqual(os.getcwd(), test_cwd) - status = r.function_environment_reload_response.result.status - self.assertEqual(status, protos.StatusResult.Success) + finally: self._reset_environ() From d517cc38f44b748276f56f8fe7392289e7a8c62e Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Mon, 21 Aug 2023 15:47:59 -0500 Subject: [PATCH 15/19] Added exception message to test case --- tests/unittests/test_rpc_messages.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unittests/test_rpc_messages.py b/tests/unittests/test_rpc_messages.py index 0c571a1c6..a1129a318 100644 --- a/tests/unittests/test_rpc_messages.py +++ b/tests/unittests/test_rpc_messages.py @@ -39,7 +39,9 @@ async def _verify_environment_reloaded( r = await disp._handle__function_environment_reload_request( request_msg) status = r.function_environment_reload_response.result.status - self.assertEqual(status, protos.StatusResult.Success) + exp = r.function_environment_reload_response.result.exception + self.assertEqual(status, protos.StatusResult.Success, + f"Exception in Reload request: {exp}") environ_dict = os.environ.copy() self.assertDictEqual(environ_dict, test_env) From 8c0343c759d3fc5c4921dc70e3181a26ce3d5e55 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Mon, 21 Aug 2023 15:58:47 -0500 Subject: [PATCH 16/19] Fixed logging error --- azure_functions_worker/dispatcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure_functions_worker/dispatcher.py b/azure_functions_worker/dispatcher.py index 4779e2060..66f89e219 100644 --- a/azure_functions_worker/dispatcher.py +++ b/azure_functions_worker/dispatcher.py @@ -547,7 +547,7 @@ async def _handle__function_environment_reload_request(self, request): """ try: logger.info('Received FunctionEnvironmentReloadRequest, ' - 'request ID: %s', + 'request ID: %s,' ' To enable debug level logging, please refer to ' 'https://aka.ms/python-enable-debug-logging', self.request_id) From 23b6ea73f9226c2f49a67e09d184d87b18a51ae4 Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 25 Aug 2023 12:47:50 -0500 Subject: [PATCH 17/19] Addressed comments --- azure_functions_worker/constants.py | 2 +- azure_functions_worker/extension.py | 2 +- azure_functions_worker/utils/common.py | 4 ++-- tests/unittests/test_dispatcher.py | 5 +++-- tests/unittests/test_utilities.py | 9 +++++++++ 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/azure_functions_worker/constants.py b/azure_functions_worker/constants.py index 08ffd8130..d094fb14b 100644 --- a/azure_functions_worker/constants.py +++ b/azure_functions_worker/constants.py @@ -41,7 +41,7 @@ PYTHON_ISOLATE_WORKER_DEPENDENCIES_DEFAULT_310 = False PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT = False PYTHON_ENABLE_WORKER_EXTENSIONS_DEFAULT_39 = True -PYTHON_RELOAD_FUNCTIONS = "PYTHON_RELOAD_FUNCTIONS" +PYTHON_EXTENSIONS_RELOAD_FUNCTIONS = "PYTHON_EXTENSIONS_RELOAD_FUNCTIONS" # External Site URLs MODULE_NOT_FOUND_TS_URL = "https://aka.ms/functions-modulenotfound" diff --git a/azure_functions_worker/extension.py b/azure_functions_worker/extension.py index ceb619e55..296d746cc 100644 --- a/azure_functions_worker/extension.py +++ b/azure_functions_worker/extension.py @@ -65,7 +65,7 @@ def function_load_extension(cls, func_name, func_directory): sdk = cls._try_get_sdk_with_extension_enabled() if sdk is None: return - + logger.info("Sdk path: %s", sdk.__file__) # Reports application & function extensions installed on customer's app cls._info_discover_extension_list(func_name, sdk) diff --git a/azure_functions_worker/utils/common.py b/azure_functions_worker/utils/common.py index 7a3d249ba..0f12a5f5c 100644 --- a/azure_functions_worker/utils/common.py +++ b/azure_functions_worker/utils/common.py @@ -7,7 +7,7 @@ from typing import Optional, Callable from azure_functions_worker.constants import CUSTOMER_PACKAGES_PATH, \ - PYTHON_RELOAD_FUNCTIONS + PYTHON_EXTENSIONS_RELOAD_FUNCTIONS def is_true_like(setting: str) -> bool: @@ -114,7 +114,7 @@ def get_sdk_from_sys_path() -> ModuleType: The azure.functions that is loaded from the first sys.path entry """ - if is_envvar_true(PYTHON_RELOAD_FUNCTIONS): + if is_envvar_true(PYTHON_EXTENSIONS_RELOAD_FUNCTIONS): backup_azure_functions = None backup_azure = None diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 6ba8c0638..4b3868ae0 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -44,7 +44,6 @@ def setUp(self, version=SysVersionInfo(3, 7, 0, 'final', 0)): script_root=DISPATCHER_FUNCTIONS_DIR) self._default_workers: Optional[ int] = PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT - self._over_max_workers: int = 10000 self._allowed_max_workers: int = PYTHON_THREADPOOL_THREAD_COUNT_MAX_37 self._pre_env = dict(os.environ) self.mock_version_info = patch( @@ -395,8 +394,9 @@ async def _check_if_function_is_ok(self, host) -> Tuple[str, str, str]: function_name = "show_context" func_id, load_r = await host.load_function(function_name) self.assertEqual(load_r.response.function_id, func_id) + ex = load_r.response.result.exception self.assertEqual(load_r.response.result.status, - protos.StatusResult.Success) + protos.StatusResult.Success, msg=ex) # Ensure the function can be properly invoked invoke_id, call_r = await host.invoke_function( @@ -451,6 +451,7 @@ async def _check_if_async_function_is_ok(self, host) -> Tuple[str, str]: class TestThreadPoolSettingsPython38(TestThreadPoolSettingsPython37): def setUp(self, version=SysVersionInfo(3, 8, 0, 'final', 0)): super(TestThreadPoolSettingsPython38, self).setUp(version) + self._over_max_workers: int = 10000 self._allowed_max_workers: int = self._over_max_workers def tearDown(self): diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index 9a7603d46..844db434f 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -7,6 +7,7 @@ import unittest from unittest.mock import patch +from azure_functions_worker.constants import PYTHON_EXTENSIONS_RELOAD_FUNCTIONS from azure_functions_worker.utils import common, wrappers TEST_APP_SETTING_NAME = "TEST_APP_SETTING_NAME" @@ -363,6 +364,14 @@ def test_get_sdk_dummy_version(self): module = common.get_sdk_from_sys_path() sdk_version = common.get_sdk_version(module) self.assertNotEqual(sdk_version, 'dummy') + def test_get_sdk_dummy_version_with_flag_enabled(self): + """Test if sdk version can get dummy sdk version + """ + os.environ[PYTHON_EXTENSIONS_RELOAD_FUNCTIONS] = '1' + sys.path.insert(0, self._dummy_sdk_sys_path) + module = common.get_sdk_from_sys_path() + sdk_version = common.get_sdk_version(module) + self.assertEqual(sdk_version, 'dummy') def _unset_feature_flag(self): try: From f43860f9e123381dd5f904b88fbd21f371e56ebd Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 25 Aug 2023 13:58:26 -0500 Subject: [PATCH 18/19] Address comments --- azure_functions_worker/extension.py | 7 +++---- tests/unittests/test_dispatcher.py | 2 +- tests/unittests/test_extension.py | 9 ++++++--- tests/unittests/test_utilities.py | 1 + 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/azure_functions_worker/extension.py b/azure_functions_worker/extension.py index 296d746cc..568b94f90 100644 --- a/azure_functions_worker/extension.py +++ b/azure_functions_worker/extension.py @@ -65,7 +65,7 @@ def function_load_extension(cls, func_name, func_directory): sdk = cls._try_get_sdk_with_extension_enabled() if sdk is None: return - logger.info("Sdk path: %s", sdk.__file__) + # Reports application & function extensions installed on customer's app cls._info_discover_extension_list(func_name, sdk) @@ -236,9 +236,8 @@ def _try_get_sdk_with_extension_enabled(cls) -> Optional[ModuleType]: @classmethod def _info_extension_is_enabled(cls, sdk): logger.info( - 'Python Worker Extension is enabled in azure.functions (%s).', - get_sdk_version(sdk) - ) + 'Python Worker Extension is enabled in azure.functions (%s). ' + 'Sdk path: %s', get_sdk_version(sdk), sdk.__file__) @classmethod def _info_discover_extension_list(cls, function_name, sdk): diff --git a/tests/unittests/test_dispatcher.py b/tests/unittests/test_dispatcher.py index 4b3868ae0..c7b127dad 100644 --- a/tests/unittests/test_dispatcher.py +++ b/tests/unittests/test_dispatcher.py @@ -44,6 +44,7 @@ def setUp(self, version=SysVersionInfo(3, 7, 0, 'final', 0)): script_root=DISPATCHER_FUNCTIONS_DIR) self._default_workers: Optional[ int] = PYTHON_THREADPOOL_THREAD_COUNT_DEFAULT + self._over_max_workers: int = 10000 self._allowed_max_workers: int = PYTHON_THREADPOOL_THREAD_COUNT_MAX_37 self._pre_env = dict(os.environ) self.mock_version_info = patch( @@ -451,7 +452,6 @@ async def _check_if_async_function_is_ok(self, host) -> Tuple[str, str]: class TestThreadPoolSettingsPython38(TestThreadPoolSettingsPython37): def setUp(self, version=SysVersionInfo(3, 8, 0, 'final', 0)): super(TestThreadPoolSettingsPython38, self).setUp(version) - self._over_max_workers: int = 10000 self._allowed_max_workers: int = self._over_max_workers def tearDown(self): diff --git a/tests/unittests/test_extension.py b/tests/unittests/test_extension.py index c8f1b47ab..c389fe044 100644 --- a/tests/unittests/test_extension.py +++ b/tests/unittests/test_extension.py @@ -57,6 +57,7 @@ def setUp(self): 'resources', 'mock_azure_functions' ) + self._dummy_sdk = Mock(__file__="test") # Initialize mock context self._mock_arguments = {'req': 'request'} @@ -113,7 +114,8 @@ def test_extension_if_sdk_not_in_path(self): sdk_enabled = self._instance._is_extension_enabled_in_sdk(module) self.assertTrue(sdk_enabled) - @patch('azure_functions_worker.extension.get_sdk_from_sys_path') + @patch('azure_functions_worker.extension.get_sdk_from_sys_path', + return_value=importlib.import_module('azure.functions')) def test_function_load_extension_enable_when_feature_flag_is_on( self, get_sdk_from_sys_path_mock: Mock @@ -185,7 +187,8 @@ def test_function_load_extension_should_invoke_extension_call( any_order=True ) - @patch('azure_functions_worker.extension.get_sdk_from_sys_path') + @patch('azure_functions_worker.extension.get_sdk_from_sys_path', + return_value=importlib.import_module('azure.functions')) def test_invocation_extension_enable_when_feature_flag_is_on( self, get_sdk_from_sys_path_mock: Mock @@ -719,7 +722,7 @@ def test_info_extension_is_enabled(self, info_mock: Mock): self._instance._info_extension_is_enabled(sdk) info_mock.assert_called_once_with( 'Python Worker Extension is enabled in azure.functions ' - '(%s).', sdk.__version__ + '(%s). Sdk path: %s', sdk.__version__, sdk.__file__ ) @patch('azure_functions_worker.extension.logger.info') diff --git a/tests/unittests/test_utilities.py b/tests/unittests/test_utilities.py index 844db434f..c71bcce37 100644 --- a/tests/unittests/test_utilities.py +++ b/tests/unittests/test_utilities.py @@ -364,6 +364,7 @@ def test_get_sdk_dummy_version(self): module = common.get_sdk_from_sys_path() sdk_version = common.get_sdk_version(module) self.assertNotEqual(sdk_version, 'dummy') + def test_get_sdk_dummy_version_with_flag_enabled(self): """Test if sdk version can get dummy sdk version """ From e449ff574b3dd0e82e4b5edae23847b36b00d73a Mon Sep 17 00:00:00 2001 From: Gavin Aguiar Date: Fri, 25 Aug 2023 15:06:56 -0500 Subject: [PATCH 19/19] Fixed unit tests --- tests/unittests/test_extension.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unittests/test_extension.py b/tests/unittests/test_extension.py index c389fe044..13140c0b6 100644 --- a/tests/unittests/test_extension.py +++ b/tests/unittests/test_extension.py @@ -108,7 +108,6 @@ def test_extension_if_sdk_not_in_path(self): support extension management. """ - self.assertNotIn(CUSTOMER_PACKAGES_PATH, sys.path) module = get_sdk_from_sys_path() self.assertIn(CUSTOMER_PACKAGES_PATH, sys.path) sdk_enabled = self._instance._is_extension_enabled_in_sdk(module)