From 51261053549f1559af14ccdd1bd8e7d050dc0fd7 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Wed, 4 Sep 2019 13:06:32 +0200 Subject: [PATCH 1/4] add whitebox testing to combo compile upload command integ test --- test/test_compile.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/test/test_compile.py b/test/test_compile.py index 4034b40718d..7ba486aa58c 100644 --- a/test/test_compile.py +++ b/test/test_compile.py @@ -57,7 +57,6 @@ def test_compile_with_simple_sketch(run_command, data_dir): @pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports") def test_compile_and_compile_combo(run_command, data_dir): - # Init the environment explicitly result = run_command("core update-index") assert result.ok @@ -68,7 +67,8 @@ def test_compile_and_compile_combo(run_command, data_dir): assert result.ok # Create a test sketch - sketch_path = os.path.join(data_dir, "CompileAndUploadIntegrationTest") + sketch_name = "CompileAndUploadIntegrationTest" + sketch_path = os.path.join(data_dir, sketch_name) result = run_command("sketch new CompileAndUploadIntegrationTest") assert result.ok assert "Sketch created in: {}".format(sketch_path) in result.stdout @@ -96,7 +96,7 @@ def test_compile_and_compile_combo(run_command, data_dir): # } # ] - detected_boards = [] + detected_boards = [] ports = json.loads(result.stdout) assert isinstance(ports, list) @@ -109,12 +109,21 @@ def test_compile_and_compile_combo(run_command, data_dir): assert len(detected_boards) >= 1, "There are no boards available for testing" # Build sketch for each detected board - for board in detected_boards: + for board in detected_boards: + log_file = "compile.log" result = run_command( - "compile -b {fqbn} --upload -p {address} {sketch_path}".format( + "compile -b {fqbn} --upload -p {address} {sketch_path} --log-format json --log-file {log_file} --log-level trace".format( fqbn=board.get('fqbn'), address=board.get('address'), - sketch_path=sketch_path) + sketch_path=sketch_path, + log_file=log_file + ) ) + log_json = open(log_file,'r') + log_json_lines = log_json.readlines() + assert is_value_in_any_json_log_message("copying sketch build output",log_json_lines) + assert is_value_in_any_json_log_message("Executing `arduino upload`",log_json_lines) assert result.ok - assert "Verify successful" in result.stdout + +def is_value_in_any_json_log_message(value,log_json_lines ): + return bool([index for index, line in enumerate(log_json_lines) if json.loads(line).get("msg") == value]) From 3a7aeabfc2d273761dc3b36e8343dc3724d5311f Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Wed, 4 Sep 2019 18:20:12 +0200 Subject: [PATCH 2/4] add log lines to compile and upload in case of success and related tests --- commands/compile/compile.go | 2 ++ commands/upload/upload.go | 3 +++ test/test_compile.py | 53 ++++++++++++++++++++++++++++--------- 3 files changed, 45 insertions(+), 13 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index e2a0a7e6f01..06b58a8550c 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -215,5 +215,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W return nil, fmt.Errorf("copying elf file: %s", err) } + logrus.Infof("Compile %s for %s successful", sketch.Name, fqbnIn) + return &rpc.CompileResp{}, nil } diff --git a/commands/upload/upload.go b/commands/upload/upload.go index 29f1895a236..840bc4e973d 100644 --- a/commands/upload/upload.go +++ b/commands/upload/upload.go @@ -248,6 +248,9 @@ func Upload(ctx context.Context, req *rpc.UploadReq, outStream io.Writer, errStr if err := cmd.Wait(); err != nil { return nil, fmt.Errorf("uploading error: %s", err) } + + logrus.Infof("Upload %s on %s successful", sketch.Name, fqbnIn) + return &rpc.UploadResp{}, nil } diff --git a/test/test_compile.py b/test/test_compile.py index 7ba486aa58c..2322e27ce3a 100644 --- a/test/test_compile.py +++ b/test/test_compile.py @@ -42,17 +42,31 @@ def test_compile_with_simple_sketch(run_command, data_dir): result = run_command("core install arduino:avr") assert result.ok - sketch_path = os.path.join(data_dir, "CompileIntegrationTest") + sketch_name = "CompileIntegrationTest" + sketch_path = os.path.join(data_dir, sketch_name) + fqbn = "arduino:avr:uno" # Create a test sketch - result = run_command("sketch new CompileIntegrationTest") + result = run_command("sketch new {}".format(sketch_name)) assert result.ok assert "Sketch created in: {}".format(sketch_path) in result.stdout # Build sketch for arduino:avr:uno - result = run_command("compile -b arduino:avr:uno {}".format(sketch_path)) + log_file_name = "compile.log" + log_file_path = os.path.join(data_dir, log_file_name) + result = run_command( + "compile -b {fqbn} {sketch_path} --log-format json --log-file {log_file} --log-level trace".format( + fqbn=fqbn, sketch_path=sketch_path, log_file=log_file_path)) assert result.ok - assert "Sketch uses" in result.stdout + + # let's test from the logs if the hex file produced by successful compile is moved to our sketch folder + log_json = open(log_file_path, 'r') + json_log_lines = log_json.readlines() + assert is_message_in_json_log_lines("Executing `arduino compile`", json_log_lines) + assert is_message_in_json_log_lines( + "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, + fqbn=fqbn), + json_log_lines) @pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports") @@ -110,20 +124,33 @@ def test_compile_and_compile_combo(run_command, data_dir): # Build sketch for each detected board for board in detected_boards: - log_file = "compile.log" + log_file_name = "{fqbn}-compile.log".format(fqbn=board.get('fqbn')) + log_file_path = os.path.join(data_dir, log_file_name) result = run_command( "compile -b {fqbn} --upload -p {address} {sketch_path} --log-format json --log-file {log_file} --log-level trace".format( fqbn=board.get('fqbn'), address=board.get('address'), sketch_path=sketch_path, - log_file=log_file + log_file=log_file_path ) ) - log_json = open(log_file,'r') - log_json_lines = log_json.readlines() - assert is_value_in_any_json_log_message("copying sketch build output",log_json_lines) - assert is_value_in_any_json_log_message("Executing `arduino upload`",log_json_lines) assert result.ok - -def is_value_in_any_json_log_message(value,log_json_lines ): - return bool([index for index, line in enumerate(log_json_lines) if json.loads(line).get("msg") == value]) + # check from the logs if the bin file were uploaded on the current board + log_json = open(log_file_path, 'r') + json_log_lines = log_json.readlines() + assert is_message_in_json_log_lines("Executing `arduino compile`", json_log_lines) + assert is_message_in_json_log_lines( + "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, + fqbn=board.get( + 'fqbn')), + json_log_lines) + assert is_message_in_json_log_lines("Executing `arduino upload`", json_log_lines) + assert is_message_in_json_log_lines( + "Upload {sketch} on {fqbn} successful".format(sketch=sketch_name, + fqbn=board.get( + 'fqbn')), + json_log_lines) + + +def is_message_in_json_log_lines(message, log_json_lines): + return len([index for index, entry in enumerate(log_json_lines) if json.loads(entry).get("msg") == message]) == 1 From 18a154fa04837543048761ff1b58896e2132d8bd Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Thu, 5 Sep 2019 11:25:04 +0200 Subject: [PATCH 3/4] check log traces for exact sequences --- commands/compile/compile.go | 8 ++--- commands/upload/upload.go | 4 +-- test/test_compile.py | 68 ++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 40 deletions(-) diff --git a/commands/compile/compile.go b/commands/compile/compile.go index 06b58a8550c..9b0a1f380f9 100644 --- a/commands/compile/compile.go +++ b/commands/compile/compile.go @@ -47,7 +47,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W return nil, errors.New("invalid instance") } - logrus.Info("Executing `arduino compile`") + logrus.Tracef("Compile %s for %s started", req.GetSketchPath(), req.GetFqbn()) if req.GetSketchPath() == "" { return nil, fmt.Errorf("missing sketchPath") } @@ -202,7 +202,7 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // Copy .hex file to sketch directory srcHex := paths.New(outputPath) dstHex := exportPath.Join(exportFile + ext) - logrus.WithField("from", srcHex).WithField("to", dstHex).Print("copying sketch build output") + logrus.WithField("from", srcHex).WithField("to", dstHex).Debug("copying sketch build output") if err = srcHex.CopyTo(dstHex); err != nil { return nil, fmt.Errorf("copying output file: %s", err) } @@ -210,12 +210,12 @@ func Compile(ctx context.Context, req *rpc.CompileReq, outStream, errStream io.W // Copy .elf file to sketch directory srcElf := paths.New(outputPath[:len(outputPath)-3] + "elf") dstElf := exportPath.Join(exportFile + ".elf") - logrus.WithField("from", srcElf).WithField("to", dstElf).Print("copying sketch build output") + logrus.WithField("from", srcElf).WithField("to", dstElf).Debug("copying sketch build output") if err = srcElf.CopyTo(dstElf); err != nil { return nil, fmt.Errorf("copying elf file: %s", err) } - logrus.Infof("Compile %s for %s successful", sketch.Name, fqbnIn) + logrus.Tracef("Compile %s for %s successful", sketch.Name, fqbnIn) return &rpc.CompileResp{}, nil } diff --git a/commands/upload/upload.go b/commands/upload/upload.go index 840bc4e973d..22c21d1a4af 100644 --- a/commands/upload/upload.go +++ b/commands/upload/upload.go @@ -40,7 +40,7 @@ import ( // Upload FIXMEDOC func Upload(ctx context.Context, req *rpc.UploadReq, outStream io.Writer, errStream io.Writer) (*rpc.UploadResp, error) { - logrus.Info("Executing `arduino upload`") + logrus.Tracef("Upload %s on %s started", req.GetSketchPath(), req.GetFqbn()) // TODO: make a generic function to extract sketch from request // and remove duplication in commands/compile.go @@ -249,7 +249,7 @@ func Upload(ctx context.Context, req *rpc.UploadReq, outStream io.Writer, errStr return nil, fmt.Errorf("uploading error: %s", err) } - logrus.Infof("Upload %s on %s successful", sketch.Name, fqbnIn) + logrus.Tracef("Upload %s on %s successful", sketch.Name, fqbnIn) return &rpc.UploadResp{}, nil } diff --git a/test/test_compile.py b/test/test_compile.py index 2322e27ce3a..a7c16f197ab 100644 --- a/test/test_compile.py +++ b/test/test_compile.py @@ -1,20 +1,20 @@ # This file is part of arduino-cli. - +# # Copyright 2019 ARDUINO SA (http://www.arduino.cc/) - +# # This software is released under the GNU General Public License version 3, # which covers the main part of arduino-cli. # The terms of this license can be found at: # https://www.gnu.org/licenses/gpl-3.0.en.html - +# # You can be released from the requirements of the above licenses by purchasing # a commercial license. Buying such a license is mandatory if you want to modify or # otherwise use the software for commercial activities involving the Arduino # software without disclosing the source code of your own applications. To purchase # a commercial license, send an email to license@arduino.cc. -import pytest import json import os +import pytest from .common import running_on_ci @@ -62,11 +62,11 @@ def test_compile_with_simple_sketch(run_command, data_dir): # let's test from the logs if the hex file produced by successful compile is moved to our sketch folder log_json = open(log_file_path, 'r') json_log_lines = log_json.readlines() - assert is_message_in_json_log_lines("Executing `arduino compile`", json_log_lines) - assert is_message_in_json_log_lines( - "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, - fqbn=fqbn), - json_log_lines) + expected_trace_sequence = [ + "Compile {sketch} for {fqbn} started".format(sketch=sketch_path, fqbn=fqbn), + "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, fqbn=fqbn) + ] + assert is_message_sequence_in_json_log_traces(expected_trace_sequence, json_log_lines) @pytest.mark.skipif(running_on_ci(), reason="VMs have no serial ports") @@ -77,7 +77,7 @@ def test_compile_and_compile_combo(run_command, data_dir): # Install required core(s) result = run_command("core install arduino:avr") - # result = run_command("core install arduino:samd") + result = run_command("core install arduino:samd") assert result.ok # Create a test sketch @@ -126,31 +126,31 @@ def test_compile_and_compile_combo(run_command, data_dir): for board in detected_boards: log_file_name = "{fqbn}-compile.log".format(fqbn=board.get('fqbn')) log_file_path = os.path.join(data_dir, log_file_name) - result = run_command( - "compile -b {fqbn} --upload -p {address} {sketch_path} --log-format json --log-file {log_file} --log-level trace".format( - fqbn=board.get('fqbn'), - address=board.get('address'), - sketch_path=sketch_path, - log_file=log_file_path - ) - ) + command_log_flags = "--log-format json --log-file {} --log-level trace".format(log_file_path) + result = run_command("compile -b {fqbn} --upload -p {address} {sketch_path} {log_flags}".format( + fqbn=board.get('fqbn'), + address=board.get('address'), + sketch_path=sketch_path, + log_flags=command_log_flags + )) assert result.ok # check from the logs if the bin file were uploaded on the current board log_json = open(log_file_path, 'r') json_log_lines = log_json.readlines() - assert is_message_in_json_log_lines("Executing `arduino compile`", json_log_lines) - assert is_message_in_json_log_lines( - "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, - fqbn=board.get( - 'fqbn')), - json_log_lines) - assert is_message_in_json_log_lines("Executing `arduino upload`", json_log_lines) - assert is_message_in_json_log_lines( - "Upload {sketch} on {fqbn} successful".format(sketch=sketch_name, - fqbn=board.get( - 'fqbn')), - json_log_lines) - - -def is_message_in_json_log_lines(message, log_json_lines): - return len([index for index, entry in enumerate(log_json_lines) if json.loads(entry).get("msg") == message]) == 1 + expected_trace_sequence = [ + "Compile {sketch} for {fqbn} started".format(sketch=sketch_path, fqbn=board.get('fqbn')), + "Compile {sketch} for {fqbn} successful".format(sketch=sketch_name, fqbn=board.get('fqbn')), + "Upload {sketch} on {fqbn} started".format(sketch=sketch_path, fqbn=board.get('fqbn')), + "Upload {sketch} on {fqbn} successful".format(sketch=sketch_name, fqbn=board.get('fqbn')) + ] + assert is_message_sequence_in_json_log_traces(expected_trace_sequence, json_log_lines) + + +def is_message_sequence_in_json_log_traces(message_sequence, log_json_lines): + trace_entries = [] + for entry in log_json_lines: + entry = json.loads(entry) + if entry.get("level") == "trace": + if entry.get("msg") in message_sequence: + trace_entries.append(entry.get("msg")) + return message_sequence == trace_entries From 004d599b4682a457f8cbd6b61094383ee1e82965 Mon Sep 17 00:00:00 2001 From: Roberto Sora Date: Thu, 5 Sep 2019 11:25:35 +0200 Subject: [PATCH 4/4] add virtualenv folder linting exclude and licence header cosmetics --- test/.flake8 | 3 ++- test/__init__.py | 14 ++++++++++++++ test/common.py | 6 +++--- test/conftest.py | 6 +++--- test/test_board.py | 6 +++--- test/test_lib.py | 6 +++--- test/test_main.py | 6 +++--- 7 files changed, 31 insertions(+), 16 deletions(-) diff --git a/test/.flake8 b/test/.flake8 index 79a16af7eeb..9a6c1b1bf2d 100644 --- a/test/.flake8 +++ b/test/.flake8 @@ -1,2 +1,3 @@ [flake8] -max-line-length = 120 \ No newline at end of file +exclude = venv +max-line-length = 120 diff --git a/test/__init__.py b/test/__init__.py index e69de29bb2d..4ff0074c804 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -0,0 +1,14 @@ +# This file is part of arduino-cli. +# +# Copyright 2019 ARDUINO SA (http://www.arduino.cc/) +# +# This software is released under the GNU General Public License version 3, +# which covers the main part of arduino-cli. +# The terms of this license can be found at: +# https://www.gnu.org/licenses/gpl-3.0.en.html +# +# You can be released from the requirements of the above licenses by purchasing +# a commercial license. Buying such a license is mandatory if you want to modify or +# otherwise use the software for commercial activities involving the Arduino +# software without disclosing the source code of your own applications. To purchase +# a commercial license, send an email to license@arduino.cc. diff --git a/test/common.py b/test/common.py index 81699e60239..67dd2cbb5c7 100644 --- a/test/common.py +++ b/test/common.py @@ -1,12 +1,12 @@ # This file is part of arduino-cli. - +# # Copyright 2019 ARDUINO SA (http://www.arduino.cc/) - +# # This software is released under the GNU General Public License version 3, # which covers the main part of arduino-cli. # The terms of this license can be found at: # https://www.gnu.org/licenses/gpl-3.0.en.html - +# # You can be released from the requirements of the above licenses by purchasing # a commercial license. Buying such a license is mandatory if you want to modify or # otherwise use the software for commercial activities involving the Arduino diff --git a/test/conftest.py b/test/conftest.py index b03bd20609a..b34bd5cc6e9 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,12 +1,12 @@ # This file is part of arduino-cli. - +# # Copyright 2019 ARDUINO SA (http://www.arduino.cc/) - +# # This software is released under the GNU General Public License version 3, # which covers the main part of arduino-cli. # The terms of this license can be found at: # https://www.gnu.org/licenses/gpl-3.0.en.html - +# # You can be released from the requirements of the above licenses by purchasing # a commercial license. Buying such a license is mandatory if you want to modify or # otherwise use the software for commercial activities involving the Arduino diff --git a/test/test_board.py b/test/test_board.py index 6249dc4956a..e1b9636cd5b 100644 --- a/test/test_board.py +++ b/test/test_board.py @@ -1,12 +1,12 @@ # This file is part of arduino-cli. - +# # Copyright 2019 ARDUINO SA (http://www.arduino.cc/) - +# # This software is released under the GNU General Public License version 3, # which covers the main part of arduino-cli. # The terms of this license can be found at: # https://www.gnu.org/licenses/gpl-3.0.en.html - +# # You can be released from the requirements of the above licenses by purchasing # a commercial license. Buying such a license is mandatory if you want to modify or # otherwise use the software for commercial activities involving the Arduino diff --git a/test/test_lib.py b/test/test_lib.py index 64446ece6c4..a899275a1ad 100644 --- a/test/test_lib.py +++ b/test/test_lib.py @@ -1,12 +1,12 @@ # This file is part of arduino-cli. - +# # Copyright 2019 ARDUINO SA (http://www.arduino.cc/) - +# # This software is released under the GNU General Public License version 3, # which covers the main part of arduino-cli. # The terms of this license can be found at: # https://www.gnu.org/licenses/gpl-3.0.en.html - +# # You can be released from the requirements of the above licenses by purchasing # a commercial license. Buying such a license is mandatory if you want to modify or # otherwise use the software for commercial activities involving the Arduino diff --git a/test/test_main.py b/test/test_main.py index bfdd903aa32..deab6497b0f 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -1,12 +1,12 @@ # This file is part of arduino-cli. - +# # Copyright 2019 ARDUINO SA (http://www.arduino.cc/) - +# # This software is released under the GNU General Public License version 3, # which covers the main part of arduino-cli. # The terms of this license can be found at: # https://www.gnu.org/licenses/gpl-3.0.en.html - +# # You can be released from the requirements of the above licenses by purchasing # a commercial license. Buying such a license is mandatory if you want to modify or # otherwise use the software for commercial activities involving the Arduino