Run without warnings on python 3.12 and 3.13 (#769)

* Drop the requirement on astor starting at python version 3.9.

* Fix deprecations by coping over a part of astor code and adjusting it not not trigger warnings

Author: tristanlatr

.github/workflows/unit.yaml

@@ -14,7 +14,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [pypy-3.7, 3.7, 3.8, 3.9, '3.10', 3.11, '3.12.0-rc.3']
+        python-version: [pypy-3.7, 3.7, 3.8, 3.9, '3.10', 3.11, '3.12', '3.13-dev']
         os: [ubuntu-20.04]
         include:
           - os: windows-latest

README.rst

@@ -78,8 +78,9 @@ in development
 
 This is the last major release to support Python 3.7.
 
-* Drop support for Python 3.6
-* Add support for Python 3.12
+* Drop support for Python 3.6.
+* Add support for Python 3.12 and Python 3.13.
+* Astor is no longer a requirement starting at Python 3.9.
 * `ExtRegistrar.register_post_processor()` now supports a `priority` argument that is an int.
   Highest priority callables will be called first during post-processing.
 * Fix too noisy ``--verbose`` mode (suppres some ambiguous annotations warnings).

pydoctor/astbuilder.py

@@ -13,12 +13,11 @@
     Type, TypeVar, Union, cast
 )
 
-import astor
 from pydoctor import epydoc2stan, model, node2stan, extensions, linker
 from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval
 from pydoctor.astutils import (is_none_literal, is_typing_annotation, is_using_annotations, is_using_typing_final, node2dottedname, node2fullname, 
                                is__name__equals__main__, unstring_annotation, iterassign, extract_docstring_linenum, infer_type, get_parents,
-                               get_docstring_node, NodeVisitor, Parentage, Str)
+                               get_docstring_node, unparse, NodeVisitor, Parentage, Str)
 
 
 def parseFile(path: Path) -> ast.Module:
@@ -230,8 +229,8 @@ def visit_ClassDef(self, node: ast.ClassDef) -> None:
                 name_node = base_node.value
 
             str_base = '.'.join(node2dottedname(name_node) or \
-                # Fallback on astor if the expression is unknown by node2dottedname().
-                [astor.to_source(base_node).strip()]) 
+                # Fallback on unparse() if the expression is unknown by node2dottedname().
+                [unparse(base_node).strip()]) 
 
             # Store the base as string and as ast.expr in rawbases list.
             rawbases += [(str_base, base_node)]

pydoctor/astutils.py

@@ -7,15 +7,28 @@
 import platform
 import sys
 from numbers import Number
-from typing import Any, Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union, cast
+from typing import Any, Callable, Collection, Iterator, Optional, List, Iterable, Sequence, TYPE_CHECKING, Tuple, Union, cast
 from inspect import BoundArguments, Signature
 import ast
 
+if sys.version_info >= (3, 9):
+    from ast import unparse as _unparse
+else:
+    from astor import to_source as _unparse
+
 from pydoctor import visitor
 
 if TYPE_CHECKING:
     from pydoctor import model
 
+def unparse(node:ast.AST) -> str:
+    """
+    This function convert a node tree back into python sourcecode.
+
+    Uses L{ast.unparse} or C{astor.to_source} for python versions before 3.9.
+    """
+    return _unparse(node)
+    
 # AST visitors
 
 def iter_values(node: ast.AST) -> Iterator[ast.AST]:
@@ -250,7 +263,7 @@ def visit_Subscript(self, node: ast.Subscript) -> ast.Subscript:
         else:
             # Other subscript; unstring the slice.
             slice = self.visit(node.slice)
-        return ast.copy_location(ast.Subscript(value, slice, node.ctx), node)
+        return ast.copy_location(ast.Subscript(value=value, slice=slice, ctx=node.ctx), node)
 
     # For Python >= 3.8:
 
@@ -488,13 +501,14 @@ def _annotation_for_value(value: object) -> Optional[ast.expr]:
             if ann_value is None:
                 ann_elem = None
             elif ann_elem is not None:
-                ann_elem = ast.Tuple(elts=[ann_elem, ann_value])
+                ann_elem = ast.Tuple(elts=[ann_elem, ann_value], ctx=ast.Load())
         if ann_elem is not None:
             if name == 'tuple':
-                ann_elem = ast.Tuple(elts=[ann_elem, ast.Constant(value=...)])
-            return ast.Subscript(value=ast.Name(id=name),
-                                 slice=ast.Index(value=ann_elem))
-    return ast.Name(id=name)
+                ann_elem = ast.Tuple(elts=[ann_elem, ast.Constant(value=...)], ctx=ast.Load())
+            return ast.Subscript(value=ast.Name(id=name, ctx=ast.Load()),
+                                 slice=ann_elem,
+                                 ctx=ast.Load())
+    return ast.Name(id=name, ctx=ast.Load())
 
 def _annotation_for_elements(sequence: Iterable[object]) -> Optional[ast.expr]:
     names = set()
@@ -507,7 +521,7 @@ def _annotation_for_elements(sequence: Iterable[object]) -> Optional[ast.expr]:
             return None
     if len(names) == 1:
         name = names.pop()
-        return ast.Name(id=name)
+        return ast.Name(id=name, ctx=ast.Load())
     else:
         # Empty sequence or no uniform type.
         return None
@@ -540,3 +554,118 @@ def _yield_parents(n:Optional[ast.AST]) -> Iterator[ast.AST]:
             yield from _yield_parents(p)
     yield from _yield_parents(getattr(node, 'parent', None))
 
+#Part of the astor library for Python AST manipulation.
+#License: 3-clause BSD
+#Copyright (c) 2015 Patrick Maupin
+_op_data = """
+    GeneratorExp                1
+
+          Assign                1
+       AnnAssign                1
+       AugAssign                0
+            Expr                0
+           Yield                1
+       YieldFrom                0
+              If                1
+             For                0
+        AsyncFor                0
+           While                0
+          Return                1
+
+           Slice                1
+       Subscript                0
+           Index                1
+        ExtSlice                1
+    comprehension_target        1
+           Tuple                0
+  FormattedValue                0
+
+           Comma                1
+       NamedExpr                1
+          Assert                0
+           Raise                0
+    call_one_arg                1
+
+          Lambda                1
+           IfExp                0
+
+   comprehension                1
+              Or   or           1
+             And   and          1
+             Not   not          1
+
+              Eq   ==           1
+              Gt   >            0
+             GtE   >=           0
+              In   in           0
+              Is   is           0
+           NotEq   !=           0
+              Lt   <            0
+             LtE   <=           0
+           NotIn   not in       0
+           IsNot   is not       0
+
+           BitOr   |            1
+          BitXor   ^            1
+          BitAnd   &            1
+          LShift   <<           1
+          RShift   >>           0
+             Add   +            1
+             Sub   -            0
+            Mult   *            1
+             Div   /            0
+             Mod   %            0
+        FloorDiv   //           0
+         MatMult   @            0
+          PowRHS                1
+          Invert   ~            1
+            UAdd   +            0
+            USub   -            0
+             Pow   **           1
+           Await                1
+             Num                1
+        Constant                1
+"""
+
+_op_data = [x.split() for x in _op_data.splitlines()] # type:ignore
+_op_data = [[x[0], ' '.join(x[1:-1]), int(x[-1])] for x in _op_data if x] # type:ignore
+for _index in range(1, len(_op_data)):
+    _op_data[_index][2] *= 2 # type:ignore
+    _op_data[_index][2] += _op_data[_index - 1][2] # type:ignore
+
+_deprecated: Collection[str] = ()
+if sys.version_info >= (3, 12):
+    _deprecated = ('Num', 'Str', 'Bytes', 'Ellipsis', 'NameConstant')
+_precedence_data = dict((getattr(ast, x, None), z) for x, y, z in _op_data if x not in _deprecated) # type:ignore
+_symbol_data = dict((getattr(ast, x, None), y) for x, y, z in _op_data if x not in _deprecated) # type:ignore
+
+class op_util:
+    """
+    This class provides data and functions for mapping
+    AST nodes to symbols and precedences.
+    """
+    @classmethod
+    def get_op_symbol(cls, obj:ast.operator|ast.boolop|ast.cmpop|ast.unaryop, 
+                      fmt:str='%s', 
+                      symbol_data:dict[type[ast.AST]|None, str]=_symbol_data, 
+                      type:Callable[[object], type[Any]]=type) -> str:
+        """Given an AST node object, returns a string containing the symbol.
+        """
+        return fmt % symbol_data[type(obj)]
+    @classmethod
+    def get_op_precedence(cls, obj:ast.operator|ast.boolop|ast.cmpop|ast.unaryop, 
+                          precedence_data:dict[type[ast.AST]|None, int]=_precedence_data, 
+                          type:Callable[[object], type[Any]]=type) -> int:
+        """Given an AST node object, returns the precedence.
+        """
+        return precedence_data[type(obj)]
+
+    if not TYPE_CHECKING:
+        class Precedence(object):
+            vars().update((cast(str, x), z) for x, _, z in _op_data)
+            highest = max(cast(int, z) for _, _, z in _op_data) + 2
+    else:
+        Precedence: Any
+
+del _op_data, _index, _precedence_data, _symbol_data, _deprecated
+# This was part of the astor library for Python AST manipulation.

pydoctor/epydoc/markup/_pyval_repr.py

@@ -43,15 +43,14 @@
 from typing import Any, AnyStr, Union, Callable, Dict, Iterable, Sequence, Optional, List, Tuple, cast
 
 import attr
-import astor.op_util
 from docutils import nodes
 from twisted.web.template import Tag
 
 from pydoctor.epydoc import sre_parse36, sre_constants36 as sre_constants
 from pydoctor.epydoc.markup import DocstringLinker
 from pydoctor.epydoc.markup.restructuredtext import ParsedRstDocstring
 from pydoctor.epydoc.docutils import set_node_attributes, wbr, obj_reference, new_document
-from pydoctor.astutils import node2dottedname, bind_args, Parentage, get_parents
+from pydoctor.astutils import node2dottedname, bind_args, Parentage, get_parents, unparse, op_util
 
 def decode_with_backslashreplace(s: bytes) -> str:
     r"""
@@ -76,6 +75,7 @@ class _MarkedColorizerState:
     charpos: int
     lineno: int
     linebreakok: bool
+    stacklength: int
 
 class _ColorizerState:
     """
@@ -87,18 +87,20 @@ class _ColorizerState:
     then fall back on a multi-line output if that fails.  
     """
     def __init__(self) -> None:
-        self.result: List[nodes.Node] = []
+        self.result: list[nodes.Node] = []
         self.charpos = 0
         self.lineno = 1
         self.linebreakok = True
-        self.warnings: List[str] = []
+        self.warnings: list[str] = []
+        self.stack: list[ast.AST] = []
 
     def mark(self) -> _MarkedColorizerState:
         return _MarkedColorizerState(
                     length=len(self.result), 
                     charpos=self.charpos,
                     lineno=self.lineno, 
-                    linebreakok=self.linebreakok)
+                    linebreakok=self.linebreakok,
+                    stacklength=len(self.stack))
 
     def restore(self, mark: _MarkedColorizerState) -> List[nodes.Node]:
         """
@@ -109,16 +111,17 @@ def restore(self, mark: _MarkedColorizerState) -> List[nodes.Node]:
                                         mark.linebreakok)
         trimmed = self.result[mark.length:]
         del self.result[mark.length:]
+        del self.stack[mark.stacklength:]
         return trimmed
 
 # TODO: add support for comparators when needed. 
 # _OperatorDelimitier is needed for:
-# - IfExp
-# - UnaryOp
-# - BinOp, needs special handling for power operator
-# - Compare
-# - BoolOp
-# - Lambda
+# - IfExp (TODO)
+# - UnaryOp (DONE)
+# - BinOp, needs special handling for power operator (DONE)
+# - Compare (TODO)
+# - BoolOp (DONE)
+# - Lambda (TODO)
 class _OperatorDelimiter:
     """
     A context manager that can add enclosing delimiters to nested operators when needed. 
@@ -145,14 +148,14 @@ def __init__(self, colorizer: 'PyvalColorizer', state: _ColorizerState,
 
         # avoid needless parenthesis, since we now collect parents for every nodes 
         if isinstance(parent_node, (ast.expr, ast.keyword, ast.comprehension)):
-            precedence = astor.op_util.get_op_precedence(node.op)
+            precedence = op_util.get_op_precedence(node.op)
             if isinstance(parent_node, (ast.UnaryOp, ast.BinOp, ast.BoolOp)):
-                parent_precedence = astor.op_util.get_op_precedence(parent_node.op)
+                parent_precedence = op_util.get_op_precedence(parent_node.op)
                 if isinstance(parent_node.op, ast.Pow) or isinstance(parent_node, ast.BoolOp):
                     parent_precedence+=1
             else:
                 parent_precedence = colorizer.explicit_precedence.get(
-                    node, astor.op_util.Precedence.highest)
+                    node, op_util.Precedence.highest)
 
             if precedence < parent_precedence:
                 self.discard = False
@@ -460,7 +463,7 @@ def _colorize_ast_dict(self, items: Iterable[Tuple[Optional[ast.AST], ast.AST]],
                 self._insert_comma(indent, state)
             state.result.append(self.WORD_BREAK_OPPORTUNITY)
             if key:
-                self._set_precedence(astor.op_util.Precedence.Comma, val)
+                self._set_precedence(op_util.Precedence.Comma, val)
                 self._colorize(key, state)
                 self._output(': ', self.COLON_TAG, state)
             else:
@@ -545,6 +548,7 @@ def _colorize_ast_constant(self, pyval: ast.AST, state: _ColorizerState) -> None
             self._output('...', self.ELLIPSIS_TAG, state)
 
     def _colorize_ast(self, pyval: ast.AST, state: _ColorizerState) -> None:
+        state.stack.append(pyval)
         # Set nodes parent in order to check theirs precedences and add delimiters when needed.
         try:
             next(get_parents(pyval))
@@ -588,6 +592,7 @@ def _colorize_ast(self, pyval: ast.AST, state: _ColorizerState) -> None:
             self._colorize_ast(pyval.value, state)
         else:
             self._colorize_ast_generic(pyval, state)
+        assert state.stack.pop() is pyval
 
     def _colorize_ast_unary_op(self, pyval: ast.UnaryOp, state: _ColorizerState) -> None:
         with _OperatorDelimiter(self, state, pyval):
@@ -761,7 +766,13 @@ def _colorize_ast_re(self, node:ast.Call, state: _ColorizerState) -> None:
 
     def _colorize_ast_generic(self, pyval: ast.AST, state: _ColorizerState) -> None:
         try:
-            source = astor.to_source(pyval).strip()
+            # Always wrap the expression inside parenthesis because we can't be sure 
+            # if there are required since we don;t have support for all operators 
+            # See TODO comment in _OperatorDelimiter.
+            source = unparse(pyval).strip()
+            if sys.version_info > (3,9) and isinstance(pyval, 
+                    (ast.IfExp, ast.Compare, ast.Lambda)) and len(state.stack)>1:
+                source = f'({source})'
         except Exception: #  No defined handler for node of type <type>
             state.result.append(self.UNKNOWN_REPR)
         else:

pydoctor/templatewriter/__init__.py

@@ -1,7 +1,7 @@
 """Render pydoctor data as HTML."""
 from __future__ import annotations
 
-from typing import Any, Iterable, Iterator, Optional, Union, TYPE_CHECKING
+from typing import Iterable, Iterator, Optional, Union, TYPE_CHECKING
 if TYPE_CHECKING:
     from typing_extensions import Protocol, runtime_checkable
 else:
@@ -11,15 +11,11 @@ def runtime_checkable(f):
 import abc
 from pathlib import Path, PurePath
 import warnings
-import sys
 from xml.dom import minidom
 
 # Newer APIs from importlib_resources should arrive to stdlib importlib.resources in Python 3.9.
 if TYPE_CHECKING:
-    if sys.version_info >= (3, 9):
-        from importlib.resources.abc import Traversable
-    else:
-        Traversable = Any
+    from importlib.resources.abc import Traversable
 else:
     Traversable = object