diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..290806612 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,21 @@ +# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.234.0/containers/python-3/.devcontainer/base.Dockerfile + +# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster +ARG VARIANT="3.9" +FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT} + +# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10 +ARG NODE_VERSION="none" +RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi + +# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image. +# COPY requirements.txt /tmp/pip-tmp/ +# RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \ +# && rm -rf /tmp/pip-tmp + +# [Optional] Uncomment this section to install additional OS packages. +RUN sudo git clone --depth=1 https://github.com/P403n1x87/austin.git && cd austin \ + sudo autoreconf --install && sudo ./configure && sudo make && sudo make install + +# [Optional] Uncomment this line to install global node packages. +# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g " 2>&1 \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..91c3701d4 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,58 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.234.0/containers/python-3 +{ + "name": "Python 3", + "build": { + "dockerfile": "Dockerfile", + "context": "..", + "args": { + // Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6 + // Append -bullseye or -buster to pin to an OS version. + // Use -bullseye variants on local on arm64/Apple Silicon. + "VARIANT": "3.9", + // Options + "NODE_VERSION": "none" + } + }, + + // Set *default* container specific settings.json values on container create. + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", + "python.formatting.blackPath": "/usr/local/py-utils/bin/black", + "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", + "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", + "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", + "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", + "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", + "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", + "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", + "python.testing.pytestArgs": [ + "tests/unittests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "sudo python -m pip install -U pip && sudo python -m pip install -U -e .[dev] && sudo python setup.py webhost", + + // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. + "remoteUser": "vscode", + + "features": { + "dotnet": "latest" + }, + "runArgs": ["--cap-add", "SYS_PTRACE"] +} diff --git a/.gitignore b/.gitignore index e8d9736a3..cd5f27009 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,4 @@ prof/ tests/**/host.json tests/**/bin tests/**/extensions.csproj +.benchmarks diff --git a/azure_functions_worker/functions.py b/azure_functions_worker/functions.py index 085fb5de6..21022cd68 100644 --- a/azure_functions_worker/functions.py +++ b/azure_functions_worker/functions.py @@ -7,7 +7,9 @@ from azure.functions import DataType, Function -from . import bindings as bindings_utils +from .bindings import (has_implicit_output, + check_output_type_annotation, + check_input_type_annotation) from . import protos from ._thirdparty import typing_inspect from .protos import BindingInfo @@ -60,7 +62,7 @@ def get_explicit_and_implicit_return(binding_name: str, typing.Tuple[bool, bool]: if binding_name == '$return': explicit_return = True - elif bindings_utils.has_implicit_output( + elif has_implicit_output( binding.type): implicit_return = True bound_params[binding_name] = binding @@ -75,7 +77,7 @@ def get_return_binding(binding_name: str, if binding_name == "$return": return_binding_name = binding_type assert return_binding_name is not None - elif bindings_utils.has_implicit_output(binding_type): + elif has_implicit_output(binding_type): return_binding_name = binding_type return return_binding_name @@ -202,17 +204,17 @@ def validate_function_params(params: dict, bound_params: dict, 'is azure.functions.Out in Python') if param_has_anno and param_py_type in (str, bytes) and ( - not bindings_utils.has_implicit_output(binding.type)): + not has_implicit_output(binding.type)): param_bind_type = 'generic' else: param_bind_type = binding.type if param_has_anno: if is_param_out: - checks_out = bindings_utils.check_output_type_annotation( + checks_out = check_output_type_annotation( param_bind_type, param_py_type) else: - checks_out = bindings_utils.check_input_type_annotation( + checks_out = check_input_type_annotation( param_bind_type, param_py_type) if not checks_out: @@ -263,7 +265,7 @@ def get_function_return_type(annotations: dict, has_explicit_return: bool, if return_pytype is (str, bytes): binding_name = 'generic' - if not bindings_utils.check_output_type_annotation( + if not check_output_type_annotation( binding_name, return_pytype): raise FunctionLoadError( func_name, diff --git a/azure_functions_worker/loader.py b/azure_functions_worker/loader.py index e5120a2b9..a05aa9e48 100644 --- a/azure_functions_worker/loader.py +++ b/azure_functions_worker/loader.py @@ -8,13 +8,14 @@ import os.path import pathlib import sys -import uuid +from uuid import uuid4 from os import PathLike, fspath from typing import List, Optional, Dict from azure.functions import Function, FunctionApp -from . import protos, functions +from . import functions +from .protos import RpcFunctionMetadata, BindingInfo from .constants import MODULE_NOT_FOUND_TS_URL, SCRIPT_FILE_NAME, \ PYTHON_LANGUAGE_RUNTIME from .utils.wrappers import attach_message_to_exception @@ -48,28 +49,27 @@ def uninstall() -> None: def build_binding_protos(indexed_function: List[Function]) -> Dict: - binding_protos = {} - for binding in indexed_function.get_bindings(): - binding_protos[binding.name] = protos.BindingInfo( + return { + binding.name: BindingInfo( type=binding.type, data_type=binding.data_type, direction=binding.direction) - - return binding_protos + for binding in indexed_function.get_bindings() + } def process_indexed_function(functions_registry: functions.Registry, indexed_functions: List[Function]): fx_metadata_results = [] for indexed_function in indexed_functions: - function_id = str(uuid.uuid4()) + function_id = str(uuid4()) function_info = functions_registry.add_indexed_function( function_id, function=indexed_function) binding_protos = build_binding_protos(indexed_function) - function_metadata = protos.RpcFunctionMetadata( + function_metadata = RpcFunctionMetadata( name=function_info.name, function_id=function_id, managed_dependency_enabled=False, # only enabled for PowerShell diff --git a/setup.py b/setup.py index 16aedff83..b9aca1a66 100644 --- a/setup.py +++ b/setup.py @@ -127,6 +127,9 @@ "pytest-randomly", "pytest-instafail", "pytest-rerunfailures", + "pytest-benchmark", + "pytest-asyncio", + "pyinstrument", "ptvsd" ] } diff --git a/tests/.gitignore b/tests/.gitignore index 3e4ede76c..c84274a0b 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,3 +1,4 @@ *_functions/bin/ *_functions/host.json *_functions/ping/ +benchmarks/.profiles \ No newline at end of file diff --git a/tests/benchmarks/README.md b/tests/benchmarks/README.md new file mode 100644 index 000000000..da59d7f78 --- /dev/null +++ b/tests/benchmarks/README.md @@ -0,0 +1,26 @@ +This folder contains benchmarks written using `pytest` and profiled using `pyinstrument`. + +# Running the benchmarks + +Open this repository in VSCode and wait for the devcontainer to start up. + +Then you can run all the benchmarks with this command: + +`python -m pytest tests/benchmarks` + +Or use the keyword argument to run just one benchmark test: + +`python -m pytest tests/benchmarks -k test_process_indexed_function` + +When you run that test, a profile will also be saved in the `tests/benchmarks/.profiles` folder, in a file named after the profiled function. Open the file in a browser to see the profile. + +If you run a benchmark test multiple times (either on same code or different versions of the code), you probably want to save it. + +Either pass in `--benchmark-autosave` to save to an auto-generated filename or pass in `--benchmark-save=YOURNAME` to save with your specified name in the filename. All benchmark files will always start with a counter, beginning with 0001. + +Once saved, compare using the `pytest-benchmark` command and the counter numbers: + +`pytest-benchmark compare 0004 0005` + +You can sort the comparison using `--sort`, save it to a CSV using `--csv`, or save it to a histogram with `--histogram`. +More details available in the [pytest-benchmark reference](https://pytest-benchmark.readthedocs.io/en/latest/usage.html#comparison-cli). \ No newline at end of file diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py new file mode 100644 index 000000000..7e3e86e65 --- /dev/null +++ b/tests/benchmarks/conftest.py @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +import pytest_asyncio + + +@pytest_asyncio.fixture +async def aio_benchmark(benchmark, event_loop): + def _wrapper(func, *args, **kwargs): + if asyncio.iscoroutinefunction(func): + @benchmark + def _(): + return event_loop.run_until_complete(func(*args, **kwargs)) + else: + benchmark(func, *args, **kwargs) + + return _wrapper diff --git a/tests/benchmarks/dummy/__init__.py b/tests/benchmarks/dummy/__init__.py new file mode 100644 index 000000000..58da4b041 --- /dev/null +++ b/tests/benchmarks/dummy/__init__.py @@ -0,0 +1,26 @@ +import azure.functions as func + +def foo(): + pass + +app = func.FunctionApp() + +@app.route(route="func1") +def func1(req: func.HttpRequest) -> func.HttpResponse: + ... + + +@app.route(route="func1") +def func2(req: func.HttpRequest, arg1) -> func.HttpResponse: + ... + + +@app.route(route="func1") +def func3(req: func.HttpRequest, arg1, arg2) -> func.HttpResponse: + ... + + +@app.route(route="func1") +def func4(req: func.HttpRequest, arg1, arg2, arg3) -> func.HttpResponse: + ... + diff --git a/tests/benchmarks/test_dispatcher_benchmark.py b/tests/benchmarks/test_dispatcher_benchmark.py new file mode 100644 index 000000000..a38936840 --- /dev/null +++ b/tests/benchmarks/test_dispatcher_benchmark.py @@ -0,0 +1,103 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import asyncio +import pathlib +import typing +import sys + +from azure_functions_worker import protos, testutils, dispatcher + + +class _MockWebHostWithWorkerController: + + def __init__(self, scripts_dir: pathlib.PurePath, event_loop): + self._event_loop = event_loop + self._host: typing.Optional[testutils._MockWebHost] = None + self._scripts_dir: pathlib.PurePath = scripts_dir + self._worker: typing.Optional[dispatcher.Dispatcher] = None + + async def __aenter__(self) -> typing.Tuple[testutils._MockWebHost, dispatcher.Dispatcher]: + loop = self._event_loop + self._host = testutils._MockWebHost(loop, self._scripts_dir) + + await self._host.start() + + self._worker = await dispatcher.\ + Dispatcher.connect(testutils.LOCALHOST, self._host._port, + self._host.worker_id, self._host.request_id, + connect_timeout=5.0) + + self._worker_task = loop.create_task(self._worker.dispatch_forever()) + + done, pending = await asyncio. \ + wait([self._host._connected_fut, self._worker_task], + return_when=asyncio.FIRST_COMPLETED) + + # noinspection PyBroadException + try: + if self._worker_task in done: + self._worker_task.result() + + if self._host._connected_fut not in done: + raise RuntimeError('could not start a worker thread') + except Exception: + try: + await self._host.close() + self._worker.stop() + finally: + raise + + return self._host, self._worker + + async def __aexit__(self, *exc): + if not self._worker_task.done(): + self._worker_task.cancel() + try: + await self._worker_task + except asyncio.CancelledError: + pass + + self._worker_task = None + self._worker = None + + await self._host.close() + self._host = None + +def start_mockhost_with_worker(event_loop, script_root=testutils.FUNCS_PATH): + scripts_dir = testutils.TESTS_ROOT / script_root + if not (scripts_dir.exists() and scripts_dir.is_dir()): + raise RuntimeError( + f'invalid script_root argument: ' + f'{scripts_dir} directory does not exist') + + sys.path.append(str(scripts_dir)) + + return _MockWebHostWithWorkerController(scripts_dir, event_loop) + +def test_invoke_function_benchmark(aio_benchmark, event_loop): + async def invoke_function(): + wc = start_mockhost_with_worker(event_loop) + async with wc as (host, worker): + await host.load_function('return_http') + + func = host._available_functions['return_http'] + invocation_id = host.make_id() + input_data = [protos.ParameterBinding( + name='req', + data=protos.TypedData( + http=protos.RpcHttp( + method='GET')))] + message = protos.StreamingMessage( + invocation_request=protos.InvocationRequest( + invocation_id=invocation_id, + function_id=func.id, + input_data=input_data, + trigger_metadata={}, + ) + ) + for _ in range(1000): + event_loop.create_task(worker._handle__invocation_request(message)) + + aio_benchmark(invoke_function) + diff --git a/tests/benchmarks/test_loader_benchmark.py b/tests/benchmarks/test_loader_benchmark.py new file mode 100644 index 000000000..a5bf6d67d --- /dev/null +++ b/tests/benchmarks/test_loader_benchmark.py @@ -0,0 +1,87 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest +from pyinstrument import Profiler + +import azure_functions_worker.loader as loader +from azure_functions_worker.functions import Registry +from azure_functions_worker.testutils import TESTS_ROOT +from azure.functions import Function +from azure.functions.decorators.core import InputBinding + + +@pytest.fixture(autouse=True) +def auto_profile(request): + PROFILE_ROOT = (TESTS_ROOT / "benchmarks" / ".profiles") + # Turn profiling on + profiler = Profiler() + profiler.start() + + yield # Run test + + profiler.stop() + PROFILE_ROOT.mkdir(exist_ok=True) + results_file = PROFILE_ROOT / f"{request.node.name}.html" + with open(results_file, "w", encoding="utf-8") as f_html: + f_html.write(profiler.output_html()) + + +def dummy_func(): + ... + + +class FakeInputBinding(InputBinding): + + def __init__(self, + name): + super().__init__(name=name, data_type=None) + + @staticmethod + def get_binding_name() -> str: + return "test_binding" + + +@pytest.mark.parametrize("size", range(10)) +def test_build_binding_protos(benchmark, size): + f = Function(dummy_func, "foo.py") + for i in range(size): + f.add_binding(FakeInputBinding(f"test_binding{i}")) + benchmark(loader.build_binding_protos, f) + + +def test_process_indexed_function(benchmark): + def _test_func(test_binding0, test_binding1, test_binding2, test_binding3, test_binding4): + pass + + f = Function(_test_func, "foo.py") + for i in range(5): # Use 5 bindings + f.add_binding(FakeInputBinding(f"test_binding{i}")) + reg = Registry() + benchmark(loader.process_indexed_function, reg, [f, f, f, f, f]) + + +def test_load_function(benchmark): + loader.install() + benchmark( + loader.load_function, + "http_functions", + TESTS_ROOT / "benchmarks" / "dummy", + TESTS_ROOT / "benchmarks" / "dummy" / "__init__.py", + "foo" + ) + loader.uninstall() + + +def test_index_function_app(benchmark): + benchmark( + loader.index_function_app, + TESTS_ROOT / "benchmarks" / "dummy", + ) + + +def test_str_join(benchmark): + def j(_input): + return ", ".join([x for x in _input]) + + benchmark(j, ["a", "b", "c", "d"])