From 4ddb07a5a2cb75b770126849c8b2d4d4a532c938 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 26 Oct 2019 09:17:22 -0700 Subject: [PATCH 1/6] Fix Python3 errors for device tests The Python3 migration didn't include fixes for local scripts in the device test tree. Fatal build and run Python errors fixed. The last update to xunitmerge is ~5 years ago, so it looks to be unsupported now. Use a local copy of the two components to allow patching to work with Python3. The serial test seems to send garbage chars (non-ASCII/etc.), so use a codepage 437 which supports all 255 chars. Fixes: #6660 --- tests/device/Makefile | 2 +- .../device/libraries/BSTest/requirements.txt | 3 +- tests/device/libraries/BSTest/runner.py | 4 +- tests/device/libraries/BSTest/xmerge.py | 130 ++++++++++++++++++ tests/device/libraries/BSTest/xunitmerge | 26 ++++ .../test_ClientContext/test_ClientContext.py | 16 ++- .../test_http_client/test_http_client.py | 8 +- .../test_http_server/test_http_server.py | 15 +- 8 files changed, 180 insertions(+), 24 deletions(-) create mode 100644 tests/device/libraries/BSTest/xmerge.py create mode 100755 tests/device/libraries/BSTest/xunitmerge diff --git a/tests/device/Makefile b/tests/device/Makefile index b34cb0dbbc..9b4349e000 100644 --- a/tests/device/Makefile +++ b/tests/device/Makefile @@ -104,7 +104,7 @@ ifneq ("$(NO_RUN)","1") endif $(TEST_REPORT_XML): $(HARDWARE_DIR) virtualenv - $(SILENT)$(BS_DIR)/virtualenv/bin/xunitmerge $(shell find $(BUILD_DIR) -name 'test_result.xml' | xargs echo) $(TEST_REPORT_XML) + $(SILENT)$(BS_DIR)/xunitmerge $(shell find $(BUILD_DIR) -name 'test_result.xml' | xargs echo) $(TEST_REPORT_XML) $(TEST_REPORT_HTML): $(TEST_REPORT_XML) | virtualenv $(SILENT)$(BS_DIR)/virtualenv/bin/junit2html $< $@ diff --git a/tests/device/libraries/BSTest/requirements.txt b/tests/device/libraries/BSTest/requirements.txt index d31484d2af..a65d9b1705 100644 --- a/tests/device/libraries/BSTest/requirements.txt +++ b/tests/device/libraries/BSTest/requirements.txt @@ -3,6 +3,5 @@ junit-xml MarkupSafe pexpect pyserial -xunitmerge junit2html -poster +poster3 diff --git a/tests/device/libraries/BSTest/runner.py b/tests/device/libraries/BSTest/runner.py index 425ac2a422..97849a5e32 100644 --- a/tests/device/libraries/BSTest/runner.py +++ b/tests/device/libraries/BSTest/runner.py @@ -236,10 +236,10 @@ def request_env(self, key): def spawn_port(port_name, baudrate=115200): global ser ser = serial.serial_for_url(port_name, baudrate=baudrate) - return fdpexpect.fdspawn(ser, 'wb', timeout=0) + return fdpexpect.fdspawn(ser, 'wb', timeout=0, encoding='cp437') def spawn_exec(name): - return pexpect.spawn(name, timeout=0) + return pexpect.spawn(name, timeout=0, encoding='cp437') def run_tests(spawn, name, mocks, env_vars): tw = BSTestRunner(spawn, name, mocks, env_vars) diff --git a/tests/device/libraries/BSTest/xmerge.py b/tests/device/libraries/BSTest/xmerge.py new file mode 100644 index 0000000000..ac0db6c54e --- /dev/null +++ b/tests/device/libraries/BSTest/xmerge.py @@ -0,0 +1,130 @@ +from __future__ import unicode_literals, print_function +from contextlib import contextmanager +from xml.etree import ElementTree as etree +from xml.sax.saxutils import quoteattr + +import six + + +CNAME_TAGS = ('system-out', 'skipped', 'error', 'failure') +CNAME_PATTERN = '' +TAG_PATTERN = '<{tag}{attrs}>{text}' + + +@contextmanager +def patch_etree_cname(etree): + """ + Patch ElementTree's _serialize_xml function so that it will + write text as CDATA tag for tags tags defined in CNAME_TAGS. + + >>> import re + >>> from xml.etree import ElementTree + >>> xml_string = ''' + ... + ... + ... Some output here + ... + ... + ... Skipped + ... + ... + ... Error here + ... + ... + ... Failure here + ... + ... + ... ''' + >>> tree = ElementTree.fromstring(xml_string) + >>> with patch_etree_cname(ElementTree): + ... saved = str(ElementTree.tostring(tree)) + >>> systemout = re.findall(r'(.*?)', saved)[0] + >>> print(systemout) + + >>> skipped = re.findall(r'()', saved)[0] + >>> print(skipped) + + >>> error = re.findall(r'()', saved)[0] + >>> print(error) + + >>> failure = re.findall(r'()', saved)[0] + >>> print(failure) + + """ + original_serialize = etree._serialize_xml + + def _serialize_xml(write, elem, *args, **kwargs): + if elem.tag in CNAME_TAGS: + attrs = ' '.join( + ['{}={}'.format(k, quoteattr(v)) + for k, v in sorted(elem.attrib.items())] + ) + attrs = ' ' + attrs if attrs else '' + text = CNAME_PATTERN.format(elem.text) + write(TAG_PATTERN.format( + tag=elem.tag, + attrs=attrs, + text=text + )) + else: + original_serialize(write, elem, *args, **kwargs) + + etree._serialize_xml = etree._serialize['xml'] = _serialize_xml + + yield + + etree._serialize_xml = etree._serialize['xml'] = original_serialize + + +def merge_trees(*trees): + """ + Merge all given XUnit ElementTrees into a single ElementTree. + This combines all of the children test-cases and also merges + all of the metadata of how many tests were executed, etc. + """ + first_tree = trees[0] + first_root = first_tree.getroot() + + if len(trees) == 0: + return first_tree + + for tree in trees[1:]: + root = tree.getroot() + + # append children elements (testcases) + first_root.extend(root.getchildren()) + + # combine root attributes which stores the number + # of executed tests, skipped tests, etc + for key, value in first_root.attrib.items(): + if not value.isdigit(): + continue + combined = six.text_type(int(value) + int(root.attrib.get(key, '0'))) + first_root.set(key, combined) + + return first_tree + + +def merge_xunit(files, output, callback=None): + """ + Merge the given xunit xml files into a single output xml file. + + If callback is not None, it will be called with the merged ElementTree + before the output file is written (useful for applying other fixes to + the merged file). This can either modify the element tree in place (and + return None) or return a completely new ElementTree to be written. + """ + trees = [] + + for f in files: + trees.append(etree.parse(f)) + + merged = merge_trees(*trees) + + if callback is not None: + result = callback(merged) + if result is not None: + merged = result + + with patch_etree_cname(etree): + merged.write(output, encoding='utf-8', xml_declaration=True) diff --git a/tests/device/libraries/BSTest/xunitmerge b/tests/device/libraries/BSTest/xunitmerge new file mode 100755 index 0000000000..8189035d6b --- /dev/null +++ b/tests/device/libraries/BSTest/xunitmerge @@ -0,0 +1,26 @@ +#!/home/earle/Arduino/hardware/esp8266com/esp8266/tests/device/libraries/BSTest/virtualenv/bin/python3 + +from __future__ import unicode_literals, print_function +import argparse +from xmerge import merge_xunit + + +parser = argparse.ArgumentParser( + description='Utility for merging multiple XUnit xml reports ' + 'into a single xml report.', +) +parser.add_argument( + 'report', + nargs='+', + type=argparse.FileType('r'), + help='Path of XUnit xml report. Multiple can be provided.', +) +parser.add_argument( + 'output', + help='Path where merged of XUnit will be saved.', +) + + +if __name__ == '__main__': + args = parser.parse_args() + merge_xunit(args.report, args.output) diff --git a/tests/device/test_ClientContext/test_ClientContext.py b/tests/device/test_ClientContext/test_ClientContext.py index ae29bcd2fe..6650e1cfdb 100644 --- a/tests/device/test_ClientContext/test_ClientContext.py +++ b/tests/device/test_ClientContext/test_ClientContext.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python3 + from mock_decorators import setup, teardown from flask import Flask, request from threading import Thread @@ -21,7 +23,7 @@ def run(): sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) for port in range(8266, 8285 + 1): try: - print >>sys.stderr, 'trying port', port + print ('trying port %d' %port, file=sys.stderr) server_address = ("0.0.0.0", port) sock.bind(server_address) sock.listen(1) @@ -31,17 +33,17 @@ def run(): print >>sys.stderr, 'busy' if not running: return - print >>sys.stderr, 'starting up on %s port %s' % server_address - print >>sys.stderr, 'waiting for connections' + print ('starting up on %s port %s' % server_address, file=sys.stderr) + print ( 'waiting for connections', file=sys.stderr) while running: - print >>sys.stderr, 'loop' + print ('loop', file=sys.stderr) readable, writable, errored = select.select([sock], [], [], 1.0) if readable: connection, client_address = sock.accept() try: - print >>sys.stderr, 'client connected:', client_address + print('client connected: %s' % str(client_address), file=sys.stderr) finally: - print >>sys.stderr, 'close' + print ('close', file=sys.stderr) connection.shutdown(socket.SHUT_RDWR) connection.close() @@ -54,7 +56,7 @@ def teardown_tcpsrv(e): global thread global running - print >>sys.stderr, 'closing' + print ('closing', file=sys.stderr) running = False thread.join() return 0 diff --git a/tests/device/test_http_client/test_http_client.py b/tests/device/test_http_client/test_http_client.py index d991ca985a..83bc4e8c17 100644 --- a/tests/device/test_http_client/test_http_client.py +++ b/tests/device/test_http_client/test_http_client.py @@ -1,7 +1,7 @@ from mock_decorators import setup, teardown from flask import Flask, request, redirect from threading import Thread -import urllib2 +import urllib import os import ssl import time @@ -20,7 +20,7 @@ def shutdown(): return 'Server shutting down...' @app.route("/", methods = ['GET', 'POST']) def root(): - print('Got data: ' + request.data); + print('Got data: ' + request.data.decode()); return 'hello!!!' @app.route("/data") def get_data(): @@ -48,7 +48,7 @@ def flaskThread(): @teardown('HTTP GET & POST requests') def teardown_http_get(e): - response = urllib2.urlopen('http://localhost:8088/shutdown') + response = urllib.request.urlopen('http://localhost:8088/shutdown') html = response.read() time.sleep(1) # avoid address in use error on macOS @@ -86,6 +86,6 @@ def teardown_http_get(e): ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE p = os.path.dirname(os.path.abspath(__file__)) - response = urllib2.urlopen('https://localhost:8088/shutdown', context=ctx) + response = urllib.request.urlopen('https://localhost:8088/shutdown', context=ctx) html = response.read() diff --git a/tests/device/test_http_server/test_http_server.py b/tests/device/test_http_server/test_http_server.py index 319a8a7101..e184e367e6 100644 --- a/tests/device/test_http_server/test_http_server.py +++ b/tests/device/test_http_server/test_http_server.py @@ -1,10 +1,9 @@ from collections import OrderedDict from mock_decorators import setup, teardown from threading import Thread -from poster.encode import MultipartParam -from poster.encode import multipart_encode -from poster.streaminghttp import register_openers -import urllib2 +from poster3.encode import MultipartParam +from poster3.encode import multipart_encode +from poster3.streaminghttp import register_openers import urllib def http_test(res, url, get=None, post=None): @@ -13,8 +12,8 @@ def http_test(res, url, get=None, post=None): if get: url += '?' + urllib.urlencode(get) if post: - post = urllib.urlencode(post) - request = urllib2.urlopen(url, post, 2) + post = urllib.parse.quote(post) + request = urllib.request.urlopen(url, post, 2) response = request.read() except: return 1 @@ -60,8 +59,8 @@ def testRun(): register_openers() p = MultipartParam("file", "0123456789abcdef", "test.txt", "text/plain; charset=utf8") datagen, headers = multipart_encode( [("var4", "val with spaces"), p] ) - request = urllib2.Request('http://etd.local/upload', datagen, headers) - response = urllib2.urlopen(request, None, 2).read() + request = urllib.request('http://etd.local/upload', datagen, headers) + response = urllib.request.urlopen(request, None, 2).read() except: return 1 if response != 'test.txt:16\nvar4 = val with spaces': From e0f7330efe9fee3a626cac4e29ed4c23bb3396b8 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 26 Oct 2019 09:58:38 -0700 Subject: [PATCH 2/6] Run tests at 160MHz (req'd for some SSL connections) --- tests/device/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/device/Makefile b/tests/device/Makefile index 9b4349e000..97d4ee7ea9 100644 --- a/tests/device/Makefile +++ b/tests/device/Makefile @@ -13,7 +13,7 @@ UPLOAD_BOARD ?= nodemcu BS_DIR ?= libraries/BSTest DEBUG_LEVEL ?= DebugLevel=None____ #FQBN ?= esp8266com:esp8266:generic:CpuFrequency=80,FlashFreq=40,FlashMode=dio,UploadSpeed=115200,FlashSize=4M1M,LwIPVariant=v2mss536,ResetMethod=none,Debug=Serial,$(DEBUG_LEVEL) -FQBN ?= esp8266com:esp8266:generic:xtal=80,FlashFreq=40,FlashMode=dio,baud=115200,eesz=4M1M,ip=lm2f,ResetMethod=none,dbg=Serial,$(DEBUG_LEVEL) +FQBN ?= esp8266com:esp8266:generic:xtal=160,FlashFreq=40,FlashMode=dio,baud=115200,eesz=4M1M,ip=lm2f,ResetMethod=none,dbg=Serial,$(DEBUG_LEVEL) BUILD_TOOL := $(ARDUINO_IDE_PATH)/arduino-builder TEST_CONFIG := test_env.cfg TEST_REPORT_XML := test_report.xml From b724be4d4efcf83a6bf3dd722dffc914c8c40a20 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 26 Oct 2019 10:37:23 -0700 Subject: [PATCH 3/6] Fix debuglevel options for builder --- tests/device/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/device/Makefile b/tests/device/Makefile index 97d4ee7ea9..a3f8cb63ca 100644 --- a/tests/device/Makefile +++ b/tests/device/Makefile @@ -11,7 +11,7 @@ UPLOAD_PORT ?= $(shell ls /dev/tty* | grep -m 1 -i USB) UPLOAD_BAUD ?= 460800 UPLOAD_BOARD ?= nodemcu BS_DIR ?= libraries/BSTest -DEBUG_LEVEL ?= DebugLevel=None____ +DEBUG_LEVEL ?= lvl=None____ #FQBN ?= esp8266com:esp8266:generic:CpuFrequency=80,FlashFreq=40,FlashMode=dio,UploadSpeed=115200,FlashSize=4M1M,LwIPVariant=v2mss536,ResetMethod=none,Debug=Serial,$(DEBUG_LEVEL) FQBN ?= esp8266com:esp8266:generic:xtal=160,FlashFreq=40,FlashMode=dio,baud=115200,eesz=4M1M,ip=lm2f,ResetMethod=none,dbg=Serial,$(DEBUG_LEVEL) BUILD_TOOL := $(ARDUINO_IDE_PATH)/arduino-builder From 4ad5ee9520da8c131010316fc867b5e888f6e0f0 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 26 Oct 2019 10:51:49 -0700 Subject: [PATCH 4/6] Fix Python3 interpreter path in xunitmerge --- tests/device/libraries/BSTest/xunitmerge | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/device/libraries/BSTest/xunitmerge b/tests/device/libraries/BSTest/xunitmerge index 8189035d6b..fced03bf97 100755 --- a/tests/device/libraries/BSTest/xunitmerge +++ b/tests/device/libraries/BSTest/xunitmerge @@ -1,4 +1,4 @@ -#!/home/earle/Arduino/hardware/esp8266com/esp8266/tests/device/libraries/BSTest/virtualenv/bin/python3 +#!/usr/bin/env python3 from __future__ import unicode_literals, print_function import argparse From 746620f06637c19396a1825af5123b0284774266 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 26 Oct 2019 11:05:36 -0700 Subject: [PATCH 5/6] Remove virtualenv on "make clean" --- tests/device/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/device/Makefile b/tests/device/Makefile index a3f8cb63ca..5062e4b2a2 100644 --- a/tests/device/Makefile +++ b/tests/device/Makefile @@ -124,7 +124,8 @@ virtualenv: clean: rm -rf $(BUILD_DIR) - rm -rf $(HARDWARE_DIR) + rm -rf $(HARDWARE_DIR)A + rm -rf $(BS_DIR)/virtualenv rm -f $(TEST_REPORT_HTML) $(TEST_REPORT_XML) distclean: clean From df9993ee16394a29400ed5e0166727b3746d3b37 Mon Sep 17 00:00:00 2001 From: "Earle F. Philhower, III" Date: Sat, 26 Oct 2019 11:18:41 -0700 Subject: [PATCH 6/6] Add appropriate attribution, license to xunitmerge Add like to the original sources with the author's license to the copied/fixed xunitmerge files. --- tests/device/libraries/BSTest/xmerge.py | 26 +++++++++++++++++++++++- tests/device/libraries/BSTest/xunitmerge | 26 +++++++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/tests/device/libraries/BSTest/xmerge.py b/tests/device/libraries/BSTest/xmerge.py index ac0db6c54e..c10ca7297e 100644 --- a/tests/device/libraries/BSTest/xmerge.py +++ b/tests/device/libraries/BSTest/xmerge.py @@ -1,4 +1,28 @@ -from __future__ import unicode_literals, print_function +# Cloned from https://github.com/miki725/xunitmerge +# to fix a Python3 error. +# +# xunitmerge is MIT licensed by Miroslav Shubernetskiy https://github.com/miki725 +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + from contextlib import contextmanager from xml.etree import ElementTree as etree from xml.sax.saxutils import quoteattr diff --git a/tests/device/libraries/BSTest/xunitmerge b/tests/device/libraries/BSTest/xunitmerge index fced03bf97..61a69f6ec0 100755 --- a/tests/device/libraries/BSTest/xunitmerge +++ b/tests/device/libraries/BSTest/xunitmerge @@ -1,6 +1,30 @@ #!/usr/bin/env python3 -from __future__ import unicode_literals, print_function +# Cloned from https://github.com/miki725/xunitmerge +# to fix a Python3 error. +# +# xunitmerge is MIT licensed by Miroslav Shubernetskiy https://github.com/miki725 +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + import argparse from xmerge import merge_xunit