Feature/python 3.9 (#30)

AlbertWeichselbraun · web-flow · commit 8bc85f65fc4a · 2020-12-07T20:38:01.000+01:00
* chg: improved publish script.

* add: test for different white-space handling strategies.

* add: support for Python 3.9

* add: support for Python 3.9

* add: testing of table handling.

* chg: updated package metadata.

* fix: issues reported by flake8

* chg: use Path for file handling.

* add: improved testing.

* chg: improved publish.sh based on codefactor feedback.
diff --git a/.gitignore b/.gitignore
@@ -14,3 +14,4 @@ dist/
 .pytest_cache/
 .coverage
 _build/
+.mypy_cache/
diff --git a/.travis.yml b/.travis.yml
@@ -3,6 +3,7 @@ python:
   - 3.5
   - 3.7
   - 3.8
+  - 3.9
 
 install:
   - python setup.py install
diff --git a/publish.sh b/publish.sh
@@ -8,11 +8,29 @@
 # - https://packaging.python.org/guides/distributing-packages-using-setuptools/#packaging-your-project
 # - https://packaging.python.org/guides/making-a-pypi-friendly-readme/
 
-# cleanup dist
-rm -rf ./dist
+VERSION=$(grep -Po "\b__version__ = '\K[^']+" src/inscriptis/__init__.py)
+IMAGE_NAME=inscriptis-web-service
 
-# build and verify packages
-python3 setup.py sdist bdist_wheel; twine check dist/*
+case "$1" in
+	python)
+		# cleanup dist
+		rm -rf ./dist
 
-# upload
-twine upload dist/*
+		# build and verify packages
+		python3 setup.py sdist bdist_wheel; twine check dist/*
+
+		# upload
+		twine upload dist/*
+		;;
+	docker)
+		echo "Publishing ${IMAGE_NAME} in version ${VERSION}"
+		docker login docker.pkg.github.com -u AlbertWeichselbraun --password-stdin < ../github-token.txt
+		docker build -t ${IMAGE_NAME}:${VERSION} .
+
+		# Step 2: Tag
+		docker tag ${IMAGE_NAME}:${VERSION} docker.pkg.github.com/weblyzard/inscriptis/${IMAGE_NAME}:${VERSION}
+
+		# Step 3: Publish
+		#docker push docker.pkg.github.com/weblyzard/inscriptis/${IMAGE_NAME}:${VERSION}
+		;;
+esac
diff --git a/scripts/inscript.py b/scripts/inscript.py
@@ -6,7 +6,7 @@
 
 import argparse
 import sys
-from os.path import isfile
+from pathlib import Path
 
 import requests
 
@@ -56,23 +56,24 @@ def get_parser():
 
     if args.version:
         print('Inscript HTML to text conversion '
-              '(based on the inscriptis library version {})'.format(
+              '(based on the inscriptis library version {0})'.format(
                   __version__))
         print('Copyright (C)', __copyright__)
         print('\nInscript comes with ABSOLUTELY NO WARRANTY.')
         print('This is free software and you are welcome to redistribute it '
-              'under the terms of the {}.'.format(__license__))
+              'under the terms of the {0}.'.format(__license__))
         sys.exit(0)
 
     if not args.input:
         html_content = sys.stdin.read()
-    elif isfile(args.input):
-        with open(args.input, encoding=args.encoding, errors='ignore') as f:
+    elif Path(args.input).is_file():
+        with Path(args.input).open(encoding=args.encoding,
+                                   errors='ignore') as f:
             html_content = f.read()
     elif args.input.startswith("http://") or args.input.startswith("https://"):
         html_content = requests.get(args.input).text
     else:
-        print("ERROR: Cannot open input file '{}'.\n".format(args.input))
+        print("ERROR: Cannot open input file '{0}'.\n".format(args.input))
         parser.print_help()
         sys.exit(-1)
 
@@ -85,7 +86,7 @@ def get_parser():
                           display_anchors=args.display_anchor_urls)
     text = get_text(html_content, config)
     if args.output:
-        with open(args.output, 'w', encoding=args.encoding) as open_file:
+        with Path(args.output).open('w', encoding=args.encoding) as open_file:
             open_file.write(text)
     else:
         print(text)
diff --git a/setup.py b/setup.py
@@ -2,17 +2,18 @@
 # -*- coding: utf-8 -*-
 
 import sys
+from pathlib import Path
 from setuptools import setup, find_packages
 from os import path
 
-here = path.abspath(path.dirname(__file__))
-sys.path.insert(0, path.join(here, 'src'))
+here = Path(path.dirname(__file__)).resolve()
+sys.path.insert(0, path.join(str(here), 'src'))
 
 from inscriptis import (__version__, __author__, __author_email__, __license__)
 
 
 # Get the long description from the README.md file
-with open(path.join(here, 'README.rst')) as f:  # , encoding='utf-8'
+with here.joinpath(Path('README.rst')).open() as f:  # , encoding='utf-8'
     long_description = f.read()
 
 setup(
@@ -25,16 +26,21 @@
     author_email=__author_email__,
     python_requires='>=3.5',
     classifiers=[
-           'Development Status :: 4 - Beta',
-           'License :: OSI Approved :: GNU General Public License v2 (GPLv2)',
-           'Topic :: Text Processing',
-           'Topic :: Text Processing :: Markup :: HTML',
-           'Programming Language :: Python :: 3.5',
-           'Programming Language :: Python :: 3.6',
-           'Programming Language :: Python :: 3.7',
-           'Programming Language :: Python :: 3.8',
+        'Development Status :: 5 - Production/Stable',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: GNU General Public License v2 (GPLv2)',
+        'Topic :: Text Processing',
+        'Topic :: Text Processing :: Markup :: HTML',
+        'Topic :: Utilities',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
     ],
-    url='http://github.com/weblyzard/inscriptis',
+    keywords='HTML,converter,text',
+    url='https://github.com/weblyzard/inscriptis',
     license=__license__,
     package_dir={'': 'src'},
 
diff --git a/src/inscriptis/__init__.py b/src/inscriptis/__init__.py
@@ -23,7 +23,7 @@
 __author_email__ = 'albert.weichselbraun@fhgr.ch, fabian.odoni@fhgr.ch'
 __copyright__ = '2016-2020 Albert Weichselbraun, Fabian Odoni'
 __license__ = 'GPL2'
-__version__ = '1.1'
+__version__ = '1.1.1'
 __status__ = 'Prototype'
 
 
diff --git a/src/inscriptis/html_engine.py b/src/inscriptis/html_engine.py
@@ -132,8 +132,8 @@ def _write_line(self, force=False):
           bool -- True, if a line has been writer, otherwise False.
         '''
         # only break the line if there is any relevant content
-        if not force and (not self.current_line[-1].content or
-                          self.current_line[-1].content.isspace()):
+        if not force and (not self.current_line[-1].content
+                          or self.current_line[-1].content.isspace()):
             self.current_line[-1].margin_before = \
                 max(self.current_line[-1].margin_before,
                     self.current_tag[-1].margin_before)
@@ -242,9 +242,9 @@ def _end_ul(self):
 
     def _start_img(self, attrs):
         image_text = attrs.get('alt', '') or attrs.get('title', '')
-        if image_text and not (self.config.deduplicate_captions and
-                               image_text == self.last_caption):
-            self.current_line[-1].content += '[{}]'.format(image_text)
+        if image_text and not (self.config.deduplicate_captions
+                               and image_text == self.last_caption):
+            self.current_line[-1].content += '[{0}]'.format(image_text)
             self.last_caption = image_text
 
     def _start_a(self, attrs):
@@ -259,7 +259,7 @@ def _start_a(self, attrs):
 
     def _end_a(self):
         if self.link_target:
-            self.current_line[-1].content += ']({})'.format(self.link_target)
+            self.current_line[-1].content += ']({0})'.format(self.link_target)
 
     def _start_ol(self, attrs):
         self.li_counter.append(1)
@@ -277,7 +277,7 @@ def _start_li(self, attrs):
             bullet = "* "
         if isinstance(bullet, int):
             self.li_counter[-1] += 1
-            self.current_line[-1].list_bullet = "{}. ".format(bullet)
+            self.current_line[-1].list_bullet = "{0}. ".format(bullet)
         else:
             self.current_line[-1].list_bullet = bullet
 
diff --git a/src/inscriptis/model/canvas.py b/src/inscriptis/model/canvas.py
@@ -4,7 +4,7 @@
 '''
 Elements used for rendering (parts) of the canvas.
 
- 1. the :class:`Line` determines how a single line is rendered.
+The :class:`Line` determines how a single line is rendered.
 '''
 
 
@@ -66,4 +66,4 @@ def get_text(self):
                         '\n' * self.margin_after))
 
     def __str__(self):
-        return "<Line: '{}'>".format(self.get_text())
+        return "<Line: '{0}'>".format(self.get_text())
diff --git a/src/inscriptis/model/css.py b/src/inscriptis/model/css.py
@@ -120,8 +120,8 @@ def get_style_attribute(style_attribute, html_element):
             key, value = (s.strip() for s in style_directive.split(':', 1))
 
             try:
-                apply_style = getattr(CssParse, "attr_" +
-                                      key.replace('-webkit-', '')
+                apply_style = getattr(CssParse, "attr_"
+                                      + key.replace('-webkit-', '')
                                       .replace("-", "_"))
                 apply_style(value, custome_html_element)
             except AttributeError:
@@ -144,7 +144,7 @@ def _get_em(length):
         unit = _m.group(2)
 
         if unit not in ('em', 'qem', 'rem'):
-            return int(round(value/8))
+            return int(round(value / 8))
         return int(round(value))
 
     # ------------------------------------------------------------------------
diff --git a/src/inscriptis/model/table.py b/src/inscriptis/model/table.py
@@ -116,14 +116,14 @@ def compute_column_width_and_height(self):
 
         # determine row height
         for row in self.rows:
-            max_row_height = max((len(cell.get_cell_lines())
-                                  for cell in row.columns)) \
-                                  if row.columns else 1
+            max_row_height = (max((len(cell.get_cell_lines())
+                                   for cell in row.columns))
+                              if row.columns else 1)
             for cell in row.columns:
                 cell.height = max_row_height
 
         # determine maximum number of columns
-        max_columns = max([len(row.columns) for row in self.rows])
+        max_columns = max((len(row.columns) for row in self.rows))
 
         for column_idx in range(max_columns):
             # determine max_column_width
diff --git a/tests/test_broken_table_handling.py b/tests/test_broken_table_handling.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+'''
+Tests the handling of tables that do not properly close all column tags.
+'''
+
+from inscriptis import get_text
+from inscriptis.css_profiles import CSS_PROFILES
+from inscriptis.model.config import ParserConfig
+
+config = ParserConfig(css=CSS_PROFILES['strict'])
+
+
+def test_forgotten_td_close_tag():
+    # one line (i.e., missing </td> before the next <td> and the next </tr>
+    html = (u'<body>hallo<table>'
+            '<tr><td>1<td>2</tr>'
+            u'</table>echo</body>')
+    assert get_text(html, config) == u'hallo\n1  2\necho'
+
+    # two lines (i.e. missing </td> before the <tr> and before the </table>
+    html = (u'<body>hallo<table>'
+            '<tr><td>1<td>2'
+            '<tr><td>3<td>4'
+            u'</table>echo</body>')
+    assert get_text(html, config) == u'hallo\n1  2\n3  4\necho'
diff --git a/tests/test_white_space_handling.py b/tests/test_white_space_handling.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+'''
+Tests different white-space handling.
+'''
+
+from inscriptis import get_text
+from inscriptis.css_profiles import CSS_PROFILES
+from inscriptis.model.config import ParserConfig
+
+config = ParserConfig(css=CSS_PROFILES['strict'])
+
+
+def test_white_space():
+    html = (u'<body><span style="white-space: normal"><i>1</i>2\n3</span>'
+            u'</body>')
+    assert get_text(html, config) == u'12 3'
+
+    html = (u'<body><span style="white-space: nowrap"><i>1</i>2\n3</span>'
+            u'</body>')
+    assert get_text(html, config) == u'12 3'
+
+    html = (u'<body><span style="white-space: pre"><i>1</i>2\n3</span>'
+            u'</body>')
+    assert get_text(html, config) == u'12\n3'
+
+    html = (u'<body><span style="white-space: pre-line"><i>1</i>2\n3</span>'
+            u'</body>')
+    assert get_text(html, config) == u'12\n3'
+
+    html = (u'<body><span style="white-space: pre-wrap"><i>1</i>2\n3</span>'
+            u'</body>')
+    assert get_text(html, config) == u'12\n3'
diff --git a/tox.ini b/tox.ini
@@ -1,9 +1,44 @@
-[tox]
-envlist = py27, py34
 
-[testenv]
-deps =
-    pytest
-    lxml
+# standard unit tests
+[testenv:pytest]
+deps = pytest
+commands = pytest ./tests
 
-commands = py.test
+# python packaging best practices
+[testenv:pyroma]
+deps = pyroma
+commands = pyroma .
+
+# coding style
+[testenv:pep8]
+deps = flake8
+       flake8-blind-except
+       flake8-bandit
+       flake8-bugbear
+       flake8-builtins
+       flake8-colors
+       flake8-comprehensions
+       flake8-mutable
+       flake8-pathlib
+       flake8-pytest
+       flake8-raise
+       flake8-string-format
+       flake8-tuple
+       flake8-logging-format
+commands = flake8
+
+[flake8]
+exclude = .tox 
+          docs
+          benchmarking
+          tests
+
+# S104 - do not cleanup XML data prior to processing
+# S410 - bind to all IPs is okay in the case of the Web service, since it is
+#        aimed for use with docker.
+# W503 - replaced with W504
+# E402 - required for importing inscriptis metadata in setup.py
+ignore = S104, S410, W503, E402
+show-source = true
+enable-extensions=G
+application-import-names = inscriptis
