Fix Python3 errors for device tests

tests/device/Makefile

@@ -11,9 +11,9 @@ UPLOAD_PORT ?= $(shell ls /dev/tty* | grep -m 1 -i USB)
 UPLOAD_BAUD ?= 460800
 UPLOAD_BOARD ?= nodemcu
 BS_DIR ?= libraries/BSTest
-DEBUG_LEVEL ?= DebugLevel=None____
+DEBUG_LEVEL ?= lvl=None____
 #FQBN ?= esp8266com:esp8266:generic:CpuFrequency=80,FlashFreq=40,FlashMode=dio,UploadSpeed=115200,FlashSize=4M1M,LwIPVariant=v2mss536,ResetMethod=none,Debug=Serial,$(DEBUG_LEVEL)
-FQBN ?= esp8266com:esp8266:generic:xtal=80,FlashFreq=40,FlashMode=dio,baud=115200,eesz=4M1M,ip=lm2f,ResetMethod=none,dbg=Serial,$(DEBUG_LEVEL)
+FQBN ?= esp8266com:esp8266:generic:xtal=160,FlashFreq=40,FlashMode=dio,baud=115200,eesz=4M1M,ip=lm2f,ResetMethod=none,dbg=Serial,$(DEBUG_LEVEL)
 BUILD_TOOL := $(ARDUINO_IDE_PATH)/arduino-builder
 TEST_CONFIG := test_env.cfg
 TEST_REPORT_XML := test_report.xml
@@ -104,7 +104,7 @@ ifneq ("$(NO_RUN)","1")
 endif
 
 $(TEST_REPORT_XML): $(HARDWARE_DIR) virtualenv
-	$(SILENT)$(BS_DIR)/virtualenv/bin/xunitmerge $(shell find $(BUILD_DIR) -name 'test_result.xml' | xargs echo) $(TEST_REPORT_XML)
+	$(SILENT)$(BS_DIR)/xunitmerge $(shell find $(BUILD_DIR) -name 'test_result.xml' | xargs echo) $(TEST_REPORT_XML)
 
 $(TEST_REPORT_HTML): $(TEST_REPORT_XML) | virtualenv
 	$(SILENT)$(BS_DIR)/virtualenv/bin/junit2html $< $@
@@ -124,7 +124,8 @@ virtualenv:
 
 clean:
 	rm -rf $(BUILD_DIR)
-	rm -rf $(HARDWARE_DIR)
+	rm -rf $(HARDWARE_DIR)A
+	rm -rf $(BS_DIR)/virtualenv
 	rm -f $(TEST_REPORT_HTML) $(TEST_REPORT_XML)
 
 distclean: clean

tests/device/libraries/BSTest/requirements.txt

@@ -3,6 +3,5 @@ junit-xml
 MarkupSafe
 pexpect
 pyserial
-xunitmerge
 junit2html
-poster
+poster3

tests/device/libraries/BSTest/runner.py

@@ -236,10 +236,10 @@ def request_env(self, key):
 def spawn_port(port_name, baudrate=115200):
     global ser
     ser = serial.serial_for_url(port_name, baudrate=baudrate)
-    return fdpexpect.fdspawn(ser, 'wb', timeout=0)
+    return fdpexpect.fdspawn(ser, 'wb', timeout=0, encoding='cp437')
 
 def spawn_exec(name):
-    return pexpect.spawn(name, timeout=0)
+    return pexpect.spawn(name, timeout=0, encoding='cp437')
 
 def run_tests(spawn, name, mocks, env_vars):
     tw = BSTestRunner(spawn, name, mocks, env_vars)

tests/device/libraries/BSTest/xmerge.py

@@ -0,0 +1,154 @@
+# Cloned from https://github.com/miki725/xunitmerge
+# to fix a Python3 error.
+#
+# xunitmerge is MIT licensed by Miroslav Shubernetskiy https://github.com/miki725
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+from contextlib import contextmanager
+from xml.etree import ElementTree as etree
+from xml.sax.saxutils import quoteattr
+
+import six
+
+
+CNAME_TAGS = ('system-out', 'skipped', 'error', 'failure')
+CNAME_PATTERN = '<![CDATA[{}]]>'
+TAG_PATTERN = '<{tag}{attrs}>{text}</{tag}>'
+
+
+@contextmanager
+def patch_etree_cname(etree):
+    """
+    Patch ElementTree's _serialize_xml function so that it will
+    write text as CDATA tag for tags tags defined in CNAME_TAGS.
+
+    >>> import re
+    >>> from xml.etree import ElementTree
+    >>> xml_string = '''
+    ... <testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0">
+    ...     <testcase classname="some.class.Foo" name="test_system_out" time="0.001">
+    ...         <system-out>Some output here</system-out>
+    ...     </testcase>
+    ...     <testcase classname="some.class.Foo" name="test_skipped" time="0.001">
+    ...         <skipped type="unittest.case.SkipTest" message="Skipped">Skipped</skipped>
+    ...     </testcase>
+    ...     <testcase classname="some.class.Foo" name="test_error" time="0.001">
+    ...         <error type="KeyError" message="Error here">Error here</error>
+    ...     </testcase>
+    ...     <testcase classname="some.class.Foo" name="test_failure" time="0.001">
+    ...         <failure type="AssertionError" message="Failure here">Failure here</failure>
+    ...     </testcase>
+    ... </testsuite>
+    ... '''
+    >>> tree = ElementTree.fromstring(xml_string)
+    >>> with patch_etree_cname(ElementTree):
+    ...    saved = str(ElementTree.tostring(tree))
+    >>> systemout = re.findall(r'(<system-out>.*?</system-out>)', saved)[0]
+    >>> print(systemout)
+    <system-out><![CDATA[Some output here]]></system-out>
+    >>> skipped = re.findall(r'(<skipped.*?</skipped>)', saved)[0]
+    >>> print(skipped)
+    <skipped message="Skipped" type="unittest.case.SkipTest"><![CDATA[Skipped]]></skipped>
+    >>> error = re.findall(r'(<error.*?</error>)', saved)[0]
+    >>> print(error)
+    <error message="Error here" type="KeyError"><![CDATA[Error here]]></error>
+    >>> failure = re.findall(r'(<failure.*?</failure>)', saved)[0]
+    >>> print(failure)
+    <failure message="Failure here" type="AssertionError"><![CDATA[Failure here]]></failure>
+    """
+    original_serialize = etree._serialize_xml
+
+    def _serialize_xml(write, elem, *args, **kwargs):
+        if elem.tag in CNAME_TAGS:
+            attrs = ' '.join(
+                ['{}={}'.format(k, quoteattr(v))
+                 for k, v in sorted(elem.attrib.items())]
+            )
+            attrs = ' ' + attrs if attrs else ''
+            text = CNAME_PATTERN.format(elem.text)
+            write(TAG_PATTERN.format(
+                tag=elem.tag,
+                attrs=attrs,
+                text=text
+            ))
+        else:
+            original_serialize(write, elem, *args, **kwargs)
+
+    etree._serialize_xml = etree._serialize['xml'] = _serialize_xml
+
+    yield
+
+    etree._serialize_xml = etree._serialize['xml'] = original_serialize
+
+
+def merge_trees(*trees):
+    """
+    Merge all given XUnit ElementTrees into a single ElementTree.
+    This combines all of the children test-cases and also merges
+    all of the metadata of how many tests were executed, etc.
+    """
+    first_tree = trees[0]
+    first_root = first_tree.getroot()
+
+    if len(trees) == 0:
+        return first_tree
+
+    for tree in trees[1:]:
+        root = tree.getroot()
+
+        # append children elements (testcases)
+        first_root.extend(root.getchildren())
+
+        # combine root attributes which stores the number
+        # of executed tests, skipped tests, etc
+        for key, value in first_root.attrib.items():
+            if not value.isdigit():
+                continue
+            combined = six.text_type(int(value) + int(root.attrib.get(key, '0')))
+            first_root.set(key, combined)
+
+    return first_tree
+
+
+def merge_xunit(files, output, callback=None):
+    """
+    Merge the given xunit xml files into a single output xml file.
+
+    If callback is not None, it will be called with the merged ElementTree
+    before the output file is written (useful for applying other fixes to
+    the merged file). This can either modify the element tree in place (and
+    return None) or return a completely new ElementTree to be written.
+    """
+    trees = []
+
+    for f in files:
+        trees.append(etree.parse(f))
+
+    merged = merge_trees(*trees)
+
+    if callback is not None:
+        result = callback(merged)
+        if result is not None:
+            merged = result
+
+    with patch_etree_cname(etree):
+        merged.write(output, encoding='utf-8', xml_declaration=True)

tests/device/libraries/BSTest/xunitmerge

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+# Cloned from https://github.com/miki725/xunitmerge
+# to fix a Python3 error.
+#
+# xunitmerge is MIT licensed by Miroslav Shubernetskiy https://github.com/miki725
+#
+# The MIT License (MIT)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import argparse
+from xmerge import merge_xunit
+
+
+parser = argparse.ArgumentParser(
+    description='Utility for merging multiple XUnit xml reports '
+                'into a single xml report.',
+)
+parser.add_argument(
+    'report',
+    nargs='+',
+    type=argparse.FileType('r'),
+    help='Path of XUnit xml report. Multiple can be provided.',
+)
+parser.add_argument(
+    'output',
+    help='Path where merged of XUnit will be saved.',
+)
+
+
+if __name__ == '__main__':
+    args = parser.parse_args()
+    merge_xunit(args.report, args.output)

tests/device/test_ClientContext/test_ClientContext.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 from mock_decorators import setup, teardown
 from flask import Flask, request
 from threading import Thread
@@ -21,7 +23,7 @@ def run():
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         for port in range(8266, 8285 + 1):
             try:
-                print >>sys.stderr, 'trying port', port
+                print ('trying port %d' %port, file=sys.stderr)
                 server_address = ("0.0.0.0", port)
                 sock.bind(server_address)
                 sock.listen(1)
@@ -31,17 +33,17 @@ def run():
                 print >>sys.stderr, 'busy'
         if not running:
             return
-        print >>sys.stderr, 'starting up on %s port %s' % server_address
-        print >>sys.stderr, 'waiting for connections'
+        print ('starting up on %s port %s' % server_address, file=sys.stderr)
+        print ( 'waiting for connections', file=sys.stderr)
         while running:
-            print >>sys.stderr, 'loop'
+            print ('loop', file=sys.stderr)
             readable, writable, errored = select.select([sock], [], [], 1.0)
             if readable:
                 connection, client_address = sock.accept()
                 try:
-                    print >>sys.stderr, 'client connected:', client_address
+                    print('client connected: %s' % str(client_address), file=sys.stderr)
                 finally:
-                    print >>sys.stderr, 'close'
+                    print ('close', file=sys.stderr)
                     connection.shutdown(socket.SHUT_RDWR)
                     connection.close()
 
@@ -54,7 +56,7 @@ def teardown_tcpsrv(e):
     global thread
     global running
 
-    print >>sys.stderr, 'closing'
+    print ('closing', file=sys.stderr)
     running = False
     thread.join()
     return 0

tests/device/test_http_client/test_http_client.py

@@ -1,7 +1,7 @@
 from mock_decorators import setup, teardown
 from flask import Flask, request, redirect
 from threading import Thread
-import urllib2
+import urllib
 import os
 import ssl
 import time
@@ -20,7 +20,7 @@ def shutdown():
         return 'Server shutting down...'
     @app.route("/", methods = ['GET', 'POST'])
     def root():
-        print('Got data: ' + request.data);
+        print('Got data: ' + request.data.decode());
         return 'hello!!!'
     @app.route("/data")
     def get_data():
@@ -48,7 +48,7 @@ def flaskThread():
 
 @teardown('HTTP GET & POST requests')
 def teardown_http_get(e):
-    response = urllib2.urlopen('http://localhost:8088/shutdown')
+    response = urllib.request.urlopen('http://localhost:8088/shutdown')
     html = response.read()
     time.sleep(1) # avoid address in use error on macOS
 
@@ -86,6 +86,6 @@ def teardown_http_get(e):
     ctx.check_hostname = False
     ctx.verify_mode = ssl.CERT_NONE
     p = os.path.dirname(os.path.abspath(__file__))
-    response = urllib2.urlopen('https://localhost:8088/shutdown', context=ctx)
+    response = urllib.request.urlopen('https://localhost:8088/shutdown', context=ctx)
     html = response.read()
 

tests/device/test_http_server/test_http_server.py

@@ -1,10 +1,9 @@
 from collections import OrderedDict
 from mock_decorators import setup, teardown
 from threading import Thread
-from poster.encode import MultipartParam
-from poster.encode import multipart_encode
-from poster.streaminghttp import register_openers
-import urllib2
+from poster3.encode import MultipartParam
+from poster3.encode import multipart_encode
+from poster3.streaminghttp import register_openers
 import urllib
 
 def http_test(res, url, get=None, post=None):
@@ -13,8 +12,8 @@ def http_test(res, url, get=None, post=None):
         if get:
             url += '?' + urllib.urlencode(get)
         if post:
-            post = urllib.urlencode(post)
-        request = urllib2.urlopen(url, post, 2)
+            post = urllib.parse.quote(post)
+        request = urllib.request.urlopen(url, post, 2)
         response = request.read()
     except:
         return 1
@@ -60,8 +59,8 @@ def testRun():
             register_openers()
             p = MultipartParam("file", "0123456789abcdef", "test.txt", "text/plain; charset=utf8")
             datagen, headers = multipart_encode( [("var4", "val with spaces"), p] )
-            request = urllib2.Request('http://etd.local/upload', datagen, headers)
-            response = urllib2.urlopen(request, None, 2).read()
+            request = urllib.request('http://etd.local/upload', datagen, headers)
+            response = urllib.request.urlopen(request, None, 2).read()
         except:
             return 1
         if response != 'test.txt:16\nvar4 = val with spaces':