From b351b557972c8b96149f90774091adf6a2fb5770 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Mon, 29 Jun 2020 10:21:46 +0200 Subject: [PATCH 1/5] [CI] Fix build failed management Since arduino-cli v0.11.0 '-o' options has been removed. The script does not handle properly this case which raised a command usage error. Error: unknown shorthand flag: 'o' in -o Usage: arduino-cli compile [flags] So this fix allows to properly handle it. Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index e55f133796..ec2ce7f5d9 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -491,7 +491,7 @@ def check_status(status, build_conf, boardKo): # Check if failed due to a region overflowed logFile = os.path.join(build_conf[3], sketch_name + ".log") # error or fatal error - error_pattern = re.compile(r":\d+:\d+:\s.*error:\s") + error_pattern = re.compile(r":\d+:\d+:\s.*error:\s|^Error:") ld_pattern = re.compile("arm-none-eabi/bin/ld:") overflow_pattern = re.compile( r"(will not fit in |section .+ is not within )?region( .+ overflowed by [\d]+ bytes)?" From 7568025b124d5c6de3faf3dafdc6c353fa4aebc7 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Mon, 29 Jun 2020 14:03:35 +0200 Subject: [PATCH 2/5] [CI] Update binary management after cli change. With new arduino-cli v0.11.0, output management has changed. See https://github.com/arduino/arduino-cli/pull/687 Binaries are always copied. Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 39 ++------------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index ec2ce7f5d9..025447f70f 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -118,8 +118,6 @@ def create_output_log_tree(): # Folders for board in board_fqbn: createFolder(os.path.join(output_dir, board)) - if args.bin: - createFolder(os.path.join(output_dir, board, bin_dir)) createFolder(os.path.join(build_output_dir, board)) @@ -484,8 +482,6 @@ def check_status(status, build_conf, boardKo): if status[1] == 0: result = "\033[32msucceeded\033[0m" - if args.bin: - bin_copy(build_conf[0], sketch_name) nb_build_passed += 1 elif status[1] == 1: # Check if failed due to a region overflowed @@ -514,8 +510,6 @@ def check_status(status, build_conf, boardKo): else: # else consider it succeeded result = "\033[32msucceeded*\033[0m" - if args.bin: - empty_bin(build_conf[0], sketch_name) nb_build_passed += 1 else: result = "\033[31merror\033[0m" @@ -645,34 +639,6 @@ def log_final_result(): print(output_dir) -# Create an empty binary -def empty_bin(board_name, sketch_name): - empty_path = os.path.abspath(os.path.join(output_dir, board_name, bin_dir)) - createFolder(empty_path) - empty_file = os.path.join( - empty_path, sketch_name + "_COULD_NOT_FIT_IN_THIS_BOARD.bin" - ) - try: - f = open(empty_file, "w") - except IOError: - print("Cannot create empty binary: ", empty_file) - else: - f.close() - - -# Create a "bin" directory for each board and copy all binary files -# from the builder output directory into it -def bin_copy(board_name, sketch_name): - try: - shutil.copy( - os.path.join(build_output_dir, board_name, sketch_name + ".bin"), - os.path.abspath(os.path.join(output_dir, board_name, bin_dir)), - ) - except OSError as e: - print("Cannot copy the binary from the arduino-cli output: " + e.strerror) - raise - - # Set up specific options to customise arduino builder command def get_fqbn(b_name): if b_name in board_custom_fqbn and board_custom_fqbn[b_name]: @@ -696,8 +662,8 @@ def genBasicCommand(b_name): cmd.append(build_output_cache_dir) if args.verbose: cmd.append("--verbose") - cmd.append("-o") - cmd.append(os.path.join(build_output_dir, b_name, "sketch")) + cmd.append("--output-dir") + cmd.append(os.path.join(output_dir, b_name, bin_dir)) cmd.append("--fqbn") cmd.append(get_fqbn(b_name)) cmd.append("dummy_sketch") @@ -865,7 +831,6 @@ def build(build_conf): ) g1 = parser.add_mutually_exclusive_group() -g1.add_argument("--bin", help="save binaries", action="store_true") g1.add_argument("--ci", help="custom configuration for CI build", action="store_true") # Sketch options From c52765f7f45ccdbc38744bff3647c5348854c4f9 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Tue, 30 Jun 2020 17:01:27 +0200 Subject: [PATCH 3/5] [CI] Manage arduino-cli version Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index 025447f70f..25226df579 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -4,6 +4,7 @@ from datetime import timedelta import json import os +from packaging import version import re import shutil import subprocess @@ -52,6 +53,8 @@ arch = arch_default arduino_platform = arduino_platform_default arduino_cli = "" +arduino_cli_default_version = "0.10.0" +arduino_cli_version = arduino_cli_default_version # List sketch_list = [] @@ -117,6 +120,7 @@ def create_output_log_tree(): file.write(build_separator + "\n") # Folders for board in board_fqbn: + createFolder(os.path.join(output_dir, board, bin_dir)) createFolder(os.path.join(output_dir, board)) createFolder(os.path.join(build_output_dir, board)) @@ -150,6 +154,7 @@ def create_config(): def check_config(): global arduino_cli + global arduino_cli_version global arduino_cli_path global sketches_path_list global build_output_dir @@ -191,6 +196,23 @@ def check_config(): else: arduino_cli = "arduino-cli" + try: + output = subprocess.check_output( + [arduino_cli, "version"], stderr=subprocess.DEVNULL, + ) + res = re.match(r".*Version:\s+(\d+\.\d+\.\d+).*", output.decode("utf-8")) + + if res: + arduino_cli_version = res.group(1) + print("Arduino CLI version used: " + arduino_cli_version) + else: + raise subprocess.CalledProcessError(1, "re") + except subprocess.CalledProcessError: + print( + "Unable to define Arduino CLI version, use default: " + + arduino_cli_default_version + ) + try: output = subprocess.check_output( [arduino_cli, "core", "search", "stm32", "--additional-urls", stm32_url], @@ -662,8 +684,12 @@ def genBasicCommand(b_name): cmd.append(build_output_cache_dir) if args.verbose: cmd.append("--verbose") - cmd.append("--output-dir") - cmd.append(os.path.join(output_dir, b_name, bin_dir)) + if version.parse(arduino_cli_version) <= version.parse(arduino_cli_default_version): + cmd.append("--output") + cmd.append(os.path.join(output_dir, b_name, bin_dir, "dummy_sketch")) + else: + cmd.append("--output-dir") + cmd.append(os.path.join(output_dir, b_name, bin_dir)) cmd.append("--fqbn") cmd.append(get_fqbn(b_name)) cmd.append("dummy_sketch") @@ -693,6 +719,9 @@ def build_config(sketch, boardSkipped): for idx in reversed(range(len(build_conf_list))): build_conf_list[idx][4][-1] = sketch + build_conf_list[idx][4][-4] = build_conf_list[idx][4][-4].replace( + "dummy_sketch", os.path.basename(sketch) + ) if na_sketch_pattern: if build_conf_list[idx][0] in na_sketch_pattern: for pattern in na_sketch_pattern[build_conf_list[idx][0]]: From 6da8759b0517fbdd91d63e63e0b3904237b3719c Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Mon, 29 Jun 2020 17:04:43 +0200 Subject: [PATCH 4/5] [CI] Harden exception handling If the executed command failed return the output and the error code. Signed-off-by: Frederic Pillon --- CI/build/arduino-cli.py | 85 +++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/CI/build/arduino-cli.py b/CI/build/arduino-cli.py index 25226df579..fb4c8b0423 100644 --- a/CI/build/arduino-cli.py +++ b/CI/build/arduino-cli.py @@ -200,50 +200,62 @@ def check_config(): output = subprocess.check_output( [arduino_cli, "version"], stderr=subprocess.DEVNULL, ) + except subprocess.CalledProcessError as e: + print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)) + print(e.stdout) + quit(e.returncode) + else: res = re.match(r".*Version:\s+(\d+\.\d+\.\d+).*", output.decode("utf-8")) - if res: arduino_cli_version = res.group(1) print("Arduino CLI version used: " + arduino_cli_version) else: - raise subprocess.CalledProcessError(1, "re") - except subprocess.CalledProcessError: - print( - "Unable to define Arduino CLI version, use default: " - + arduino_cli_default_version - ) + print( + "Unable to define Arduino CLI version, use default: " + + arduino_cli_default_version + ) try: output = subprocess.check_output( [arduino_cli, "core", "search", "stm32", "--additional-urls", stm32_url], stderr=subprocess.DEVNULL, ) + except subprocess.CalledProcessError as e: + print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)) + print(e.stdout) + quit(e.returncode) + else: if arduino_platform not in output.decode("utf-8"): - raise subprocess.CalledProcessError(1, "re") + print(arduino_platform + " is not installed!") + quit(1) # Add core and library path to sketches_path_list try: output = subprocess.check_output( [arduino_cli, "config", "dump", "--format", "json"], stderr=subprocess.DEVNULL, ).decode("utf-8") + except subprocess.CalledProcessError as e: + print( + '"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode) + ) + print(e.stdout) + quit(e.returncode) + else: cli_config = json.loads(output) if cli_config is not None: if cli_config["directories"]["data"] is not None: sketches_path_list.append(cli_config["directories"]["data"]) else: - raise subprocess.CalledProcessError(3, "No data directory") + print("No data directory") + quit(1) if cli_config["directories"]["user"] is not None: sketches_path_list.append(cli_config["directories"]["user"]) else: - raise subprocess.CalledProcessError(2, "No user directory") + print("No user directory!") + quit(1) else: - raise subprocess.CalledProcessError(1, "No fqbn") - except subprocess.CalledProcessError: - print("No arduino-cli config!") - quit() - except subprocess.CalledProcessError: - print(arduino_platform + " is not installed!") - quit() + print("No arduino-cli config!") + quit(1) def load_core_config(): @@ -386,7 +398,7 @@ def manage_inos(): break else: print("Sketch {} path does not exist!".format(args.ino)) - quit() + quit(1) # Sketches listed in a file elif args.file: assert os.path.exists(args.file), "Sketches list file does not exist" @@ -417,7 +429,7 @@ def manage_inos(): sketch_list.append(sketch_default) if len(sketch_list) == 0: print("No sketch to build for " + arduino_platform + "!") - quit() + quit(1) # Find all .ino files and save directory @@ -449,32 +461,41 @@ def find_board(): try: output = subprocess.check_output( [arduino_cli, "board", "listall", "--format", "json"], - stderr=subprocess.DEVNULL, + stderr=subprocess.STDOUT, ).decode("utf-8") + except subprocess.CalledProcessError as e: + print('"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode)) + print(e.stdout) + quit(e.returncode) + else: boards_list = json.loads(output) if boards_list is not None: for board in boards_list["boards"]: if arduino_platform in board["FQBN"]: fqbn_list_tmp.append(board["FQBN"]) - if not len(fqbn_list_tmp): - raise subprocess.CalledProcessError(2, "No fqbn") - else: - raise subprocess.CalledProcessError(1, "No fqbn") - except subprocess.CalledProcessError: - print("No fqbn found for " + arduino_platform + "!") - quit() + if not len(fqbn_list_tmp): + print("No boards found for " + arduino_platform) + quit(1) # For STM32 core, pnum is requested for fqbn in fqbn_list_tmp: try: output = subprocess.check_output( [arduino_cli, "board", "details", "--format", "json", fqbn], - stderr=subprocess.DEVNULL, + stderr=subprocess.STDOUT, ).decode("utf-8") + except subprocess.CalledProcessError as e: + print( + '"' + " ".join(e.cmd) + '" failed with code: {}!'.format(e.returncode) + ) + print(e.stdout) + quit(e.returncode) + else: board_detail = json.loads(output) if board_detail is not None: if "config_options" not in board_detail: - raise subprocess.CalledProcessError(3, "No config_options") + print("No config_options found for " + fqbn) + quit(1) for option in board_detail["config_options"]: if option["option"] == "pnum": for value in option["values"]: @@ -486,14 +507,12 @@ def find_board(): ) break else: - raise subprocess.CalledProcessError(1, "No fqbn") - except subprocess.CalledProcessError as e: - print("No fqbn detail found for " + e.cmd + "!") + print('No detail found for:"' + fqbn + '"!') if board_found: board_fqbn = collections.OrderedDict(sorted(board_found.items())) else: print("No board found for " + arduino_platform + "!") - quit() + quit(1) # Check the status From 3e3cfb34e2fd9bd0418bff172f79b91851d74a88 Mon Sep 17 00:00:00 2001 From: Frederic Pillon Date: Fri, 3 Jul 2020 09:00:55 +0200 Subject: [PATCH 5/5] Revert "[CI] Force arduino-cli 0.10.0 usage" This reverts commit b12a55485250076e872fa286a925d6b3ae8d3016. --- .github/workflows/Continuous-Integration.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/Continuous-Integration.yml b/.github/workflows/Continuous-Integration.yml index a2c88c5039..148e9553bf 100644 --- a/.github/workflows/Continuous-Integration.yml +++ b/.github/workflows/Continuous-Integration.yml @@ -53,8 +53,6 @@ jobs: - name: Compilation id: Compile uses: stm32duino/actions/compile-examples@master - with: - cli-version: '0.10.0' # Use the output from the `Compile` step - name: Compilation Errors