diff --git a/pydoctor/astbuilder.py b/pydoctor/astbuilder.py index 9aaf2275d..288ab8113 100644 --- a/pydoctor/astbuilder.py +++ b/pydoctor/astbuilder.py @@ -257,6 +257,7 @@ def depart_Module(self, node: ast.Module) -> None: self._tweak_constants_annotations(self.builder.current) self._infer_attr_annotations(self.builder.current) self.builder.pop(self.module) + epydoc2stan.transform_parsed_names(self.module) def visit_ClassDef(self, node: ast.ClassDef) -> None: # Ignore classes within functions. @@ -1153,6 +1154,8 @@ def _get_all_ast_annotations() -> Iterator[Tuple[str, Optional[ast.expr]]]: class ASTBuilder: """ Keeps tracks of the state of the AST build, creates documentable and adds objects to the system. + + One ASTBuilder instance is only suitable to build one Module. """ ModuleVistor = ModuleVistor @@ -1164,21 +1167,6 @@ def __init__(self, system: model.System): self._stack: List[model.Documentable] = [] - - def parseFile(self, path: Path, ctx: model.Module) -> Optional[ast.Module]: - try: - return self.system._ast_parser.parseFile(path) - except Exception as e: - ctx.report(f"cannot parse file, {e}") - return None - - def parseString(self, string:str, ctx: model.Module) -> Optional[ast.Module]: - try: - return self.system._ast_parser.parseString(string) - except Exception: - ctx.report("cannot parse string") - return None - def _push(self, cls: Type[DocumentableT], name: str, @@ -1270,14 +1258,6 @@ def addAttribute(self, def processModuleAST(self, mod_ast: ast.Module, mod: model.Module) -> None: - for name, node in findModuleLevelAssign(mod_ast): - try: - module_var_parser = MODULE_VARIABLES_META_PARSERS[name] - except KeyError: - continue - else: - module_var_parser(node, mod) - vis = self.ModuleVistor(self, mod) vis.extensions.add(*self.system._astbuilder_visitors) vis.extensions.attach_visitor(vis) @@ -1304,7 +1284,7 @@ def exception(self) -> Exception: def __init__(self) -> None: self.ast_cache: Dict[Path, ast.Module | SyntaxTreeParser._Error] = {} - def parseFile(self, path: Path) -> ast.Module: + def parseFileOnly(self, path: Path) -> ast.Module: try: r = self.ast_cache[path] except KeyError: @@ -1322,7 +1302,7 @@ def parseFile(self, path: Path) -> ast.Module: raise r.exception() return r - def parseString(self, string:str) -> ast.Module: + def parseStringOnly(self, string:str) -> ast.Module: mod = None try: mod = _parse(string) @@ -1330,6 +1310,20 @@ def parseString(self, string:str) -> ast.Module: raise SyntaxError("cannot parse string") from e return mod + def parseFile(self, path: Path, ctx: model.Module) -> model.ParsedAstModule | None: + try: + return model.ParsedAstModule(self.parseFileOnly(path)) + except Exception as e: + ctx.report(f"cannot parse file, {e}") + return None + + def parseString(self, string:str, ctx: model.Module) -> model.ParsedAstModule | None: + try: + return model.ParsedAstModule(self.parseStringOnly(string)) + except Exception: + ctx.report("cannot parse string") + return None + model.System.defaultBuilder = ASTBuilder model.System.syntaxTreeParser = SyntaxTreeParser diff --git a/pydoctor/epydoc/markup/__init__.py b/pydoctor/epydoc/markup/__init__.py index e1032ff0c..f026dc43f 100644 --- a/pydoctor/epydoc/markup/__init__.py +++ b/pydoctor/epydoc/markup/__init__.py @@ -315,7 +315,7 @@ def link_to(self, target: str, label: "Flattenable", *, is_annotation: bool = Fa @return: The link, or just the label if the target was not found. """ - def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag: + def link_xref(self, target: str, label: "Flattenable", lineno: int, rawtarget: str | None = None) -> Tag: """ Format a cross-reference link to a Python identifier. This will resolve the identifier to any reasonable target, @@ -326,8 +326,10 @@ def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag: @param label: The label to show for the link. @param lineno: The line number within the docstring at which the crossreference is located. + @param rawtarget: The name of the Python identifier that + should be linked to, as written in the docstring, for warning purposes. + If it's left None, the C{identifier} will be used instead. @return: The link, or just the label if the target was not found. - In either case, the returned top-level tag will be C{}. """ def switch_context(self, ob:Optional['Documentable']) -> ContextManager[None]: diff --git a/pydoctor/epydoc2stan.py b/pydoctor/epydoc2stan.py index 7a547b331..5daff100a 100644 --- a/pydoctor/epydoc2stan.py +++ b/pydoctor/epydoc2stan.py @@ -6,17 +6,20 @@ from collections import defaultdict import enum import inspect +import builtins +from itertools import chain from typing import ( TYPE_CHECKING, Any, Callable, ClassVar, DefaultDict, Dict, Generator, - Iterator, List, Mapping, Optional, Sequence, Tuple, Union, + Iterator, List, Mapping, Optional, Sequence, Tuple, TypeVar, Union, ) -import ast import re import attr +from docutils.transforms import Transform from docutils import nodes -from pydoctor import model, linker +from pydoctor import model, linker, node2stan +from pydoctor.node2stan import parse_reference from pydoctor.astutils import is_none_literal from pydoctor.epydoc.docutils import new_document, set_node_attributes, text_node, code from pydoctor.epydoc.markup import (Field as EpydocField, ParseError, get_parser_by_name, @@ -279,18 +282,19 @@ def __init__(self, obj: model.Documentable): self.sinces: List[Field] = [] self.unknowns: DefaultDict[str, List[FieldDesc]] = defaultdict(list) - def set_param_types_from_annotations( - self, annotations: Mapping[str, Optional[ast.expr]] - ) -> None: + def set_param_types_from_annotations(self) -> None: + if not isinstance(self.obj, model.Function): + return + annotations = self.obj.annotations _linker = self.obj.docstring_linker formatted_annotations = { - name: None if value is None - else ParamType(safe_to_stan(colorize_inline_pyval(value, is_annotation=True), _linker, + name: None if parsed_annotation is None + else ParamType(safe_to_stan(parsed_annotation, _linker, self.obj, fallback=colorized_pyval_fallback, section='annotation', report=False), # don't spam the log, invalid annotation are going to be reported when the signature gets colorized origin=FieldOrigin.FROM_AST) - for name, value in annotations.items() + for name, parsed_annotation in get_parsed_annotations(self.obj).items() } ret_type = formatted_annotations.pop('return', None) @@ -657,7 +661,12 @@ def ensure_parsed_docstring(obj: model.Documentable) -> Optional[model.Documenta Currently, it's not 100% clear at what point the L{Documentable.parsed_docstring} attribute is set. It can be set from the ast builder or later processing step. - This function ensures that the C{parsed_docstring} attribute of a documentable is set to it's final value. + This function ensures that the C{parsed_docstring} attribute of a + documentable is set to a relevant value at the given point of the processing. + Meaning if an obvious docstring is found it will be stored, if some some reason + (i.e import cycles) the docstring source has not been processed yet, this function is a no-op and + will return as-is the docstring is None. This function might have a different effect in a further + point of the processing. @returns: - If the C{obj.parsed_docstring} is set to a L{ParsedDocstring} instance: @@ -797,8 +806,7 @@ def format_docstring(obj: model.Documentable) -> Tag: ret(unwrap_docstring_stan(stan)) fh = FieldHandler(obj) - if isinstance(obj, model.Function): - fh.set_param_types_from_annotations(obj.annotations) + fh.set_param_types_from_annotations() if source is not None: assert obj.parsed_docstring is not None, "ensure_parsed_docstring() did not do it's job" for field in obj.parsed_docstring.fields: @@ -873,20 +881,90 @@ def type2stan(obj: model.Documentable) -> Optional[Tag]: return safe_to_stan(parsed_type, obj.docstring_linker, obj, fallback=colorized_pyval_fallback, section='annotation') +_T = TypeVar('_T') +def _memoize(o:object, attrname:str, getter:Callable[[], _T]) -> _T: + parsed = getattr(o, attrname, None) + if parsed is not None: + return parsed #type:ignore + parsed = getter() + setattr(o, attrname, parsed) + return parsed + def get_parsed_type(obj: model.Documentable) -> Optional[ParsedDocstring]: """ Get the type of this attribute as parsed docstring. """ - parsed_type = obj.parsed_type - if parsed_type is not None: - return parsed_type + def _get_parsed_type() -> Optional[ParsedDocstring]: + annotation = getattr(obj, 'annotation', None) + if annotation is not None: + v = colorize_inline_pyval(annotation, is_annotation=True) + reportWarnings(obj, v.warnings, section='colorize annotation') + return v + return None + return _memoize(obj, 'parsed_type', _get_parsed_type) - # Only Attribute instances have the 'annotation' attribute. - annotation: Optional[ast.expr] = getattr(obj, 'annotation', None) - if annotation is not None: - return colorize_inline_pyval(annotation, is_annotation=True) +def get_parsed_decorators(obj: model.Attribute | model.Function | model.FunctionOverload + ) -> Sequence[ParsedDocstring] | None: + """ + Get the decorators of this function as parsed docstring. + """ + def _get_parsed_decorators() -> Optional[Sequence[ParsedDocstring]]: + v = [colorize_inline_pyval(dec) for dec in obj.decorators] if \ + obj.decorators is not None else None + documentable_obj = obj if not isinstance(obj, model.FunctionOverload) else obj.primary + for c in v or (): + if c: + reportWarnings(documentable_obj, c.warnings, section='colorize decorators') + return v + return _memoize(obj, 'parsed_decorators', _get_parsed_decorators) + +def get_parsed_value(obj:model.Attribute) -> Optional[ParsedDocstring]: + """ + Get the value of this constant as parsed docstring. + """ + def _get_parsed_value() -> Optional[ParsedDocstring]: + v = colorize_pyval(obj.value, + linelen=obj.system.options.pyvalreprlinelen, + maxlines=obj.system.options.pyvalreprmaxlines) if obj.value is not None else None + # Report eventual warnings. + if v: + reportWarnings(obj, v.warnings, section='colorize constant') + return v + return _memoize(obj, 'parsed_value', _get_parsed_value) + +def get_parsed_annotations(obj:model.Function) -> Mapping[str, Optional[ParsedDocstring]]: + """ + Get the annotations of this function as dict from str to parsed docstring. + """ + def _get_parsed_annotations() -> Mapping[str, Optional[ParsedDocstring]]: + return {name:colorize_inline_pyval(ann, is_annotation=True) if ann else None for \ + (name, ann) in obj.annotations.items()} + # do not warn here + return _memoize(obj, 'parsed_annotations', _get_parsed_annotations) - return None +def get_parsed_bases(obj:model.Class) -> Sequence[ParsedDocstring]: + """ + Get the bases of this class as a seqeunce of parsed docstrings. + """ + def _get_parsed_bases() -> Sequence[ParsedDocstring]: + r = [] + for (str_base, base_node), base_obj in zip(obj.rawbases, obj.baseobjects): + # Make sure we bypass the linker’s resolver process for base object, + # because it has been resolved already (with two passes). + # Otherwise, since the class declaration wins over the imported names, + # a class with the same name as a base class confused pydoctor and it would link + # to it self: https://github.com/twisted/pydoctor/issues/662 + refmap = None + if base_obj is not None: + refmap = {str_base:base_obj.fullName()} + + # link to external class, using the colorizer here + # to link to classes with generics (subscripts and other AST expr). + p = colorize_inline_pyval(base_node, refmap=refmap, is_annotation=True) + r.append(p) + reportWarnings(obj, p.warnings, section='colorize bases') + return r + return _memoize(obj, 'parsed_bases', _get_parsed_bases) def format_toc(obj: model.Documentable) -> Optional[Tag]: # Load the parsed_docstring if it's not already done. @@ -985,23 +1063,19 @@ def colorized_pyval_fallback(_: List[ParseError], doc:ParsedDocstring, __:model. return tags.code(doc.to_text()) def _format_constant_value(obj: model.Attribute) -> Iterator["Flattenable"]: - + doc = get_parsed_value(obj) + if doc is None: + return + # yield the table title, "Value" row = tags.tr(class_="fieldStart") row(tags.td(class_="fieldName")("Value")) # yield the first row. yield row - doc = colorize_pyval(obj.value, - linelen=obj.system.options.pyvalreprlinelen, - maxlines=obj.system.options.pyvalreprmaxlines) - value_repr = safe_to_stan(doc, obj.docstring_linker, obj, fallback=colorized_pyval_fallback, section='rendering of constant') - # Report eventual warnings. It warns when a regex failed to parse. - reportWarnings(obj, doc.warnings, section='colorize constant') - # yield the value repr. row = tags.tr() row(tags.td(tags.pre(class_='constant-value')(value_repr))) @@ -1169,6 +1243,129 @@ def get_constructors_extra(cls:model.Class) -> ParsedDocstring | None: set_node_attributes(document, children=elements) return ParsedRstDocstring(document, ()) + +_builtin_names = set(dir(builtins)) + +class _ReferenceTransform(Transform): + + def __init__(self, document:nodes.document, + ctx:'model.Documentable'): + super().__init__(document) + self.ctx = ctx + self.module = ctx.module + + def _transform(self, node:nodes.title_reference) -> None: + ctx = self.ctx + module = self.module + ref = parse_reference(node) + target = ref.target + # we're setting two attributes here: 'refuri' and 'rawtarget'. + # 'refuri' might already be created by the colorizer or docstring parser, + # but 'rawtarget' is only created from within this transform, so we can + # use that information to ensure this process is only ever applied once + # per title_reference element. + attribs = node.attributes + if target == attribs.get('refuri', target) and 'rawtarget' not in attribs: + is_annotation = node.attributes.get('is_annotation') + # save the raw target name + attribs['rawtarget'] = target + name, *rest = target.split('.') + is_name_defined = ctx.isNameDefined(name) + # check if it's a non-shadowed builtins + if not is_name_defined and name in _builtin_names: + # transform bare builtin name into builtins. + attribs['refuri'] = '.'.join(('builtins', name, *rest)) + return + # no-op for unbound name + if not is_name_defined: + attribs['refuri'] = target + return + # kindda duplicate a little part of the annotation linker logic here, + # there are no simple way of doing it otherwise at the moment. + # Once all presented parsed elements are stored as Documentable attributes + # we might be able to simply use that and drop the use of the annotation linker, + # but for now this will do the trick: + lookup_context = ctx + if is_annotation and ctx is not module and module.isNameDefined(name, + only_locals=True) and ctx.isNameDefined(name, only_locals=True): + # If we're dealing with an annotation, give precedence to the module's + # lookup (wrt PEP 563) + lookup_context = module + + # save pre-resolved refuri + attribs['refuri'] = '.'.join(chain(lookup_context.expandName(name).split('.'), rest)) + + def apply(self) -> None: + for node in self.document.findall(nodes.title_reference): + self._transform(node) + + +def _apply_reference_transform(doc:ParsedDocstring, ctx:'model.Documentable') -> None: + """ + Runs L{_ReferenceTransform} on the underlying docutils document. + No-op if L{to_node} raises L{NotImplementedError}. + """ + try: + document = doc.to_node() + except NotImplementedError: + return + else: + _ReferenceTransform(document, ctx).apply() + +def transform_parsed_names(node: model.Module) -> None: + """ + Walk this module's content and apply in-place transformations to the + L{ParsedDocstring} instances that olds L{obj_reference} or L{nodes.title_reference} nodes. + + Fixing "Lookup of name in annotation fails on reparented object #295". + """ + from pydoctor import model + privacy = node.system.privacyClass + # resolve names early when possible + for ob in model.walk(node): + if privacy(ob) == model.PrivacyClass.HIDDEN: + # do not do anything with HIDDEN objects since they won't be renderred. + continue + + # resolve names in parsed_docstring, do not forget field bodies + if docsource:=ensure_parsed_docstring(ob): + assert ob.parsed_docstring is not None + _apply_reference_transform(ob.parsed_docstring, docsource) + for f in ob.parsed_docstring.fields: + _apply_reference_transform(f.body(), docsource) + + if isinstance(ob, model.Function): + if sig:=get_parsed_signature(ob): + _apply_reference_transform(sig, ob) + for _,ann in get_parsed_annotations(ob).items(): + if ann: + _apply_reference_transform(ann, ob) + for dec in get_parsed_decorators(ob) or (): + if dec: + _apply_reference_transform(dec, ob) + for overload in ob.overloads: + if sig:=get_parsed_signature(overload): + _apply_reference_transform(sig, ob) + for dec in get_parsed_decorators(overload) or (): + if dec: + _apply_reference_transform(dec, ob) + + elif isinstance(ob, model.Attribute): + # resolve attribute annotation with parsed_type attribute + parsed_type = get_parsed_type(ob) + if parsed_type: + _apply_reference_transform(parsed_type, ob) + if ob.kind in ob.system.show_attr_value: + parsed_value = get_parsed_value(ob) + if parsed_value: + _apply_reference_transform(parsed_value, ob) + for dec in get_parsed_decorators(ob) or (): + if dec: + _apply_reference_transform(dec, ob) + + elif isinstance(ob, model.Class): + for base in get_parsed_bases(ob): + _apply_reference_transform(base, ob) def get_namespace_docstring(ns: model.Package) -> str: """ diff --git a/pydoctor/linker.py b/pydoctor/linker.py index e0a633af5..4bb8dc9f0 100644 --- a/pydoctor/linker.py +++ b/pydoctor/linker.py @@ -137,9 +137,13 @@ def link_to(self, identifier: str, label: "Flattenable", *, is_annotation: bool else: fullID = self.obj.expandName(identifier) - target = self.obj.system.objForFullName(fullID) - if target is not None: - return taglink(target, self.page_url, label) + try: + target = self.obj.system.find_object(fullID) + except LookupError: + pass + else: + if target is not None: + return taglink(target, self.page_url, label) url = self.look_for_intersphinx(fullID) if url is not None: @@ -148,10 +152,10 @@ def link_to(self, identifier: str, label: "Flattenable", *, is_annotation: bool link = tags.transparent(label) return link - def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag: + def link_xref(self, target: str, label: "Flattenable", lineno: int, rawtarget: str | None = None) -> Tag: xref: "Flattenable" try: - resolved = self._resolve_identifier_xref(target, lineno) + resolved = self._resolve_identifier_xref(target, lineno, rawtarget) except LookupError: xref = tags.transparent(label) else: @@ -164,7 +168,8 @@ def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag: def _resolve_identifier_xref(self, identifier: str, - lineno: int + lineno: int, + rawtarget: str | None, ) -> Union[str, 'model.Documentable']: """ Resolve a crossreference link to a Python identifier. @@ -188,8 +193,18 @@ def _resolve_identifier_xref(self, if target is not None: return target - # Check if the fullID exists in an intersphinx inventory. fullID = self.obj.expandName(identifier) + + # Try fetching the name with it's outdated fullname + try: + target = self.obj.system.find_object(fullID) + except LookupError: + pass + else: + if target is not None: + return target + + # Check if the fullID exists in an intersphinx inventory. target_url = self.look_for_intersphinx(fullID) if not target_url: # FIXME: https://github.com/twisted/pydoctor/issues/125 @@ -233,8 +248,9 @@ def _resolve_identifier_xref(self, return target message = f'Cannot find link target for "{fullID}"' - if identifier != fullID: - message = f'{message}, resolved from "{identifier}"' + rawtarget = rawtarget or identifier + if rawtarget != fullID: + message = f'{message}, resolved from "{rawtarget}"' root_idx = fullID.find('.') if root_idx != -1 and fullID[:root_idx] not in self.obj.system.root_names: message += ' (you can link to external docs with --intersphinx)' @@ -252,7 +268,8 @@ class NotFoundLinker(DocstringLinker): def link_to(self, target: str, label: "Flattenable", *, is_annotation: bool = False) -> Tag: return tags.a(label) - def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag: + def link_xref(self, target: str, label: "Flattenable", + lineno: int, rawtarget: str | None = None) -> Tag: return tags.a(label) @contextlib.contextmanager diff --git a/pydoctor/model.py b/pydoctor/model.py index 081612df3..9bd37322c 100644 --- a/pydoctor/model.py +++ b/pydoctor/model.py @@ -116,6 +116,19 @@ class DocumentableKind(Enum): PROPERTY = 150 VARIABLE = 100 +def walk(node:'Documentable') -> Iterator['Documentable']: + """ + Recursively yield all descendant nodes in the tree starting at *node* + (including *node* itself), in no specified order. This is useful if you + only want to modify nodes in place and don't care about the context. + """ + from collections import deque + todo = deque([node]) + while todo: + node = todo.popleft() + todo.extend(node.contents.values()) + yield node + class Documentable: """An object that can be documented. @@ -305,7 +318,7 @@ def _handle_reparenting_post(self) -> None: def _localNameToFullName(self, name: str) -> str: raise NotImplementedError(self._localNameToFullName) - def isNameDefined(self, name:str) -> bool: + def isNameDefined(self, name:str, only_locals:bool=False) -> bool: """ Is the given name defined in the globals/locals of self-context? Only the first name of a dotted name is checked. @@ -412,7 +425,8 @@ def module(self) -> 'Module': def report(self, descr: str, section: str = 'parsing', lineno_offset: int = 0, thresh:int=-1) -> None: """ - Log an error or warning about this documentable object. + Log an error or warning about this documentable object. + A reported message will only be printed once. @param descr: The error/warning string @param section: What the warning is about. @@ -437,7 +451,8 @@ def report(self, descr: str, section: str = 'parsing', lineno_offset: int = 0, t self.system.msg( section, f'{self.description}:{linenumber}: {descr}', - thresh=thresh) + # some warnings can be reported more that once. + thresh=thresh, once=True) @property def docstring_linker(self) -> 'linker.DocstringLinker': @@ -456,13 +471,13 @@ def setup(self) -> None: super().setup() self._localNameToFullName_map: Dict[str, str] = {} - def isNameDefined(self, name: str) -> bool: + def isNameDefined(self, name: str, only_locals:bool=False) -> bool: name = name.split('.')[0] if name in self.contents: return True if name in self._localNameToFullName_map: return True - if not isinstance(self, Module): + if not isinstance(self, Module) and not only_locals: return self.module.isNameDefined(name) else: return False @@ -471,10 +486,23 @@ def localNames(self) -> Iterator[str]: return chain(self.contents.keys(), self._localNameToFullName_map.keys()) +@attr.s(auto_attribs=True) +class ParsedAstModule: + root: ast.Module + # will soon contain the source code lines as well + # this will enable to process tokens and eventually + # generate HTML for source code, see issue #??? + class Module(CanContainImportsDocumentable): kind = DocumentableKind.MODULE state = ProcessingState.UNPROCESSED + parsed_ast: ParsedAstModule | None = None + """ + When the AST of a module is succesfully parsed, it is encapsulated + in a L{ParsedAstModule} instance and stored here to be processed later. + """ + @property def privacyClass(self) -> PrivacyClass: if self.name == '__main__': @@ -793,6 +821,7 @@ def setup(self) -> None: self.subclasses: List[Class] = [] self._initialbases: List[str] = [] self._initialbaseobjects: List[Optional['Class']] = [] + self.parsed_bases:Optional[List[ParsedDocstring]] = None @overload def mro(self, include_external:'Literal[True]', include_self:bool=True) -> Sequence[Class | str]:... @@ -927,8 +956,8 @@ def docsources(self) -> Iterator[Documentable]: def _localNameToFullName(self, name: str) -> str: return self.parent._localNameToFullName(name) - def isNameDefined(self, name: str) -> bool: - return self.parent.isNameDefined(name) + def isNameDefined(self, name: str, only_locals:bool=False) -> bool: + return self.parent.isNameDefined(name, only_locals=only_locals) class Function(Inheritable): kind = DocumentableKind.FUNCTION @@ -946,6 +975,8 @@ def setup(self) -> None: self.kind = DocumentableKind.METHOD self.signature = None self.overloads = [] + self.parsed_decorators:Optional[Sequence[ParsedDocstring]] = None + self.parsed_annotations:Optional[Dict[str, Optional[ParsedDocstring]]] = None @attr.s(auto_attribs=True) class FunctionOverload: @@ -955,6 +986,7 @@ class FunctionOverload: primary: Function signature: Signature | None decorators: Sequence[ast.expr] + parsed_decorators:Optional[Sequence[ParsedDocstring]] = None parsed_signature: ParsedDocstring | None = None # set in get_parsed_signature() class Attribute(Inheritable): @@ -967,6 +999,8 @@ class Attribute(Inheritable): None value means the value is not initialized at the current point of the the process. """ + parsed_decorators:Optional[Sequence[ParsedDocstring]] = None + parsed_value:Optional[ParsedDocstring] = None # Work around the attributes of the same name within the System class. _ModuleT = Module @@ -1574,7 +1608,7 @@ def _is_pep420_namespace_package(self, path: Path) -> bool: def _is_oldschool_namespace_package(self, path: Path) -> bool: try: - tree = self._ast_parser.parseFile( + tree = self._ast_parser.parseFileOnly( path.joinpath('__init__.py')) except Exception: return False @@ -1647,18 +1681,17 @@ def processModule(self, mod: _ModuleT) -> None: assert head == mod.fullName() else: builder = self.defaultBuilder(self) - ast = None - if mod._py_string is not None: - ast = builder.parseString(mod._py_string, mod) - elif mod.kind is not DocumentableKind.NAMESPACE_PACKAGE: - # There is no AST for namespace packages. - assert mod.source_path is not None - ast = builder.parseFile(mod.source_path, mod) - if ast: + # if mod._py_string is not None: + # ast = builder.parseString(mod._py_string, mod) + # elif mod.kind is not DocumentableKind.NAMESPACE_PACKAGE: + # # There is no AST for namespace packages. + # assert mod.source_path is not None + # ast = builder.parseFile(mod.source_path, mod) + if mod.parsed_ast: self.processing_modules.append(mod.fullName()) if mod._py_string is None: self.msg("processModule", "processing %s"%(self.processing_modules), 1) - builder.processModuleAST(ast, mod) + builder.processModuleAST(mod.parsed_ast.root, mod) mod.state = ProcessingState.PROCESSED head = self.processing_modules.pop() assert head == mod.fullName() @@ -1668,8 +1701,39 @@ def processModule(self, mod: _ModuleT) -> None: self.module_count, f"modules processed, {self.violations} warnings") + def preProcess(self) -> None: + """ + Called before any module gets processed, at this point the only existing + objects are L{Modules }. + + Pre-processing is the place to compute informations needed at any point of + of the processing. Analysis of relations between documentables SHALL NOT be done here. + """ + # 1. parse ASTs of all modules + for mod in self.unprocessed_modules: + if mod._py_string is not None: + mod.parsed_ast = self._ast_parser.parseString(mod._py_string, mod) + elif mod.kind is not DocumentableKind.NAMESPACE_PACKAGE: + # There is no AST for namespace packages. + assert mod.source_path is not None + mod.parsed_ast = self._ast_parser.parseFile(mod.source_path, mod) + + # 2. (one-day we might need this) do some pre analysis + # 3. process meta-variables + from . import astbuilder + for mod in self.unprocessed_modules: + if not (parsed_ast:=mod.parsed_ast): + continue + for name, node in astbuilder.findModuleLevelAssign(parsed_ast.root): + try: + module_var_parser = astbuilder.MODULE_VARIABLES_META_PARSERS[name] + except KeyError: + continue + else: + module_var_parser(node, mod) def process(self) -> None: + self.preProcess() while self.unprocessed_modules: mod = next(iter(self.unprocessed_modules)) self.processModule(mod) diff --git a/pydoctor/node2stan.py b/pydoctor/node2stan.py index 6c56419ee..ffe543b8d 100644 --- a/pydoctor/node2stan.py +++ b/pydoctor/node2stan.py @@ -7,11 +7,13 @@ from itertools import chain import re import optparse -from typing import Any, Callable, ClassVar, Iterable, List, Optional, Union, TYPE_CHECKING +from typing import Any, Callable, ClassVar, Iterable, List, Optional, Sequence, Tuple, Union, TYPE_CHECKING from docutils.writers import html4css1 from docutils import nodes, frontend, __version_info__ as docutils_version_info from twisted.web.template import Tag +import attr + if TYPE_CHECKING: from twisted.web.template import Flattenable from pydoctor.epydoc.markup import DocstringLinker @@ -59,6 +61,30 @@ def gettext(node: Union[nodes.Node, List[nodes.Node]]) -> List[str]: filtered.extend(gettext(child)) return filtered +@attr.s(auto_attribs=True) +class Reference: + label: str | Sequence[nodes.Node] + target: str + +def parse_reference(node:nodes.title_reference) -> Reference: + """ + Split a reference into (label, target). + """ + label: Union[str, Sequence[nodes.Node]] + if 'refuri' in node.attributes: + # Epytext parsed or manually constructed nodes. + label, target = node.children, node.attributes['refuri'] + else: + # RST parsed. + m = _TARGET_RE.match(node.astext()) + if m: + label, target = m.groups() + else: + label = target = node.astext() + # Support linking to functions and methods with () at the end + if target.endswith('()'): + target = target[:len(target)-2] + return Reference(label, target) _TARGET_RE = re.compile(r'^(.*?)\s*<(?:URI:|URL:)?([^<>]+)>$') _VALID_IDENTIFIER_RE = re.compile('[^0-9a-zA-Z_]') @@ -110,13 +136,15 @@ def __init__(self, # Do not wrap links in tags if we're renderring a code-like parsed element. self._link_xref = self._linker.link_xref else: - self._link_xref = lambda target, label, lineno: Tag('code')(self._linker.link_xref(target, label, lineno)) + self._link_xref = lambda target, label, lineno, rawtarget = None: Tag('code')( + self._linker.link_xref(target, label, lineno, rawtarget)) # Handle interpreted text (crossreferences) def visit_title_reference(self, node: nodes.title_reference) -> None: lineno = get_lineno(node) - self._handle_reference(node, link_func=partial(self._link_xref, lineno=lineno)) + self._handle_reference(node, link_func=partial(self._link_xref, lineno=lineno, + rawtarget=node.attributes.get('rawtarget'))) # Handle internal references def visit_obj_reference(self, node: obj_reference) -> None: @@ -126,22 +154,14 @@ def visit_obj_reference(self, node: obj_reference) -> None: self._handle_reference(node, link_func=self._linker.link_to) def _handle_reference(self, node: nodes.title_reference, link_func: Callable[[str, "Flattenable"], "Flattenable"]) -> None: + ref = parse_reference(node) + node_label = ref.label + target = ref.target label: "Flattenable" - if 'refuri' in node.attributes: - # Epytext parsed or manually constructed nodes. - label, target = node2stan(node.children, self._linker), node.attributes['refuri'] + if not isinstance(node_label, str): + label = node2stan(node_label, self._linker) else: - # RST parsed. - m = _TARGET_RE.match(node.astext()) - if m: - label, target = m.groups() - else: - label = target = node.astext() - - # Support linking to functions and methods with () at the end - if target.endswith('()'): - target = target[:len(target)-2] - + label = node_label self.body.append(flatten(link_func(target, label))) raise nodes.SkipNode() diff --git a/pydoctor/sphinx.py b/pydoctor/sphinx.py index 240964dd6..6498a0797 100644 --- a/pydoctor/sphinx.py +++ b/pydoctor/sphinx.py @@ -149,6 +149,12 @@ def getLink(self, name: str) -> Optional[str]: """ Return link for `name` or None if no link is found. """ + # special casing the 'builtins' module because our name resolving + # replaces bare builtins names with builtins. in order not to confuse + # them with objects in the system when reparenting. + if name.startswith('builtins.'): + name = name[len('builtins.'):] + base_url, relative_link = self._links.get(name, (None, None)) if not relative_link: return None diff --git a/pydoctor/templatewriter/pages/__init__.py b/pydoctor/templatewriter/pages/__init__.py index fc6045b07..c427b552a 100644 --- a/pydoctor/templatewriter/pages/__init__.py +++ b/pydoctor/templatewriter/pages/__init__.py @@ -18,7 +18,6 @@ from pydoctor.templatewriter import util, TemplateLookup, TemplateElement from pydoctor.templatewriter.pages.table import ChildTable from pydoctor.templatewriter.pages.sidebar import SideBar -from pydoctor.epydoc.markup._pyval_repr import colorize_inline_pyval if TYPE_CHECKING: from typing_extensions import Final @@ -32,7 +31,8 @@ def _format_decorators(obj: Union[model.Function, model.Attribute, model.Functio # primary function for parts that requires an interface to Documentable methods or attributes documentable_obj = obj if not isinstance(obj, model.FunctionOverload) else obj.primary - for dec in obj.decorators or (): + for dec, doc in zip(obj.decorators or (), + epydoc2stan.get_parsed_decorators(obj) or ()): if isinstance(dec, ast.Call): fn = node2fullname(dec.func, documentable_obj) # We don't want to show the deprecated decorator; @@ -41,16 +41,9 @@ def _format_decorators(obj: Union[model.Function, model.Attribute, model.Functio if fn in ("twisted.python.deprecate.deprecated", "twisted.python.deprecate.deprecatedProperty"): break - - # Colorize decorators! - doc = colorize_inline_pyval(dec) stan = epydoc2stan.safe_to_stan(doc, documentable_obj.docstring_linker, documentable_obj, fallback=epydoc2stan.colorized_pyval_fallback, section='rendering of decorators') - - # Report eventual warnings. It warns when we can't colorize the expression for some reason. - epydoc2stan.reportWarnings(documentable_obj, doc.warnings, section='colorize decorator') - yield tags.span('@', stan.children, tags.br(), class_='decorator') def format_decorators(obj: Union[model.Function, model.Attribute, model.FunctionOverload]) -> Tag: @@ -83,30 +76,17 @@ def format_class_signature(cls: model.Class) -> "Flattenable": """ r: List["Flattenable"] = [] # the linker will only be used to resolve the generic arguments of the base classes, - # it won't actually resolve the base classes (see comment few lines below). - # this is why we're using the annotation linker. - _linker = cls.docstring_linker - if cls.rawbases: + # it won't actually resolve the base classes (see comment in epydoc2stan.get_parsed_bases). + # this is why we use is_annotation=True in get_parsed_bases(). + parsed_bases = epydoc2stan.get_parsed_bases(cls) + if parsed_bases: + _linker = cls.docstring_linker r.append('(') - for idx, ((str_base, base_node), base_obj) in enumerate(zip(cls.rawbases, cls.baseobjects)): + for idx, parsed_base in enumerate(parsed_bases): if idx != 0: r.append(', ') - - # Make sure we bypass the linker’s resolver process for base object, - # because it has been resolved already (with two passes). - # Otherwise, since the class declaration wins over the imported names, - # a class with the same name as a base class confused pydoctor and it would link - # to it self: https://github.com/twisted/pydoctor/issues/662 - - refmap = None - if base_obj is not None: - refmap = {str_base:base_obj.fullName()} - - # link to external class, using the colorizer here - # to link to classes with generics (subscripts and other AST expr). - # we use is_annotation=True because bases are unstringed, they can contain annotations. - stan = epydoc2stan.safe_to_stan(colorize_inline_pyval(base_node, refmap=refmap, is_annotation=True), _linker, cls, + stan = epydoc2stan.safe_to_stan(parsed_base, _linker, cls, fallback=epydoc2stan.colorized_pyval_fallback, section='rendering of class signature') r.extend(stan.children) diff --git a/pydoctor/test/test_astbuilder.py b/pydoctor/test/test_astbuilder.py index a9314e4f3..c65d8447b 100644 --- a/pydoctor/test/test_astbuilder.py +++ b/pydoctor/test/test_astbuilder.py @@ -3294,3 +3294,63 @@ def test_Final_constant_under_control_flow_block_is_still_constant(systemcls: Ty assert mod.contents['w'].kind == model.DocumentableKind.CONSTANT assert mod.contents['x'].kind == model.DocumentableKind.CONSTANT +def test_docformat_variable_ignored_corner_case(capsys: CapSys) -> None: + # test for https://github.com/twisted/pydoctor/pull/723/files#r2217424695 + + src_top = ''' + # test + # epytext is used by default but let's be explicit + __docformat__ = 'epytext' + + # this import might shortcut the processing of + # test.sub such that __docformat__ variable will be ignored + from test.sub.subsub import thing + ''' + + src_sub = ''' + # test.sub + __docformat__ = 'restructuredtext' + ''' + + src_sub_sub = ''' + # test.sub.subsub + # should be restructuredtext formatting + '`link `_' + thing = False + ''' + + builder = (s:=model.System()).systemBuilder(s) + builder.addModuleString(src_top, 'test', is_package=True) + builder.addModuleString(src_sub, 'sub', parent_name='test', is_package=True) + builder.addModuleString(src_sub_sub, 'subsub', parent_name='test.sub') + builder.buildModules() + + from .test_epydoc2stan import docstring2html + assert 'href' in docstring2html(s.allobjects['test.sub.subsub']) + +def test__all__variable_ignored_corner_case(capsys: CapSys) -> None: + raise NotImplementedError('unfinished!') + src_top = ''' + # test + ''' + + src_sub = ''' + # test.sub + ''' + + src_sub_sub = ''' + # test.sub.subsub + ''' + + builder = (s:=model.System()).systemBuilder(s) + builder.addModuleString(src_top, 'test', is_package=True) + builder.addModuleString(src_sub, 'sub', parent_name='test', is_package=True) + builder.addModuleString(src_sub_sub, 'subsub', parent_name='test.sub') + builder.buildModules() + +def test_preprocess_names_dont_draw_incorrect_conclusions() -> None: + # TODO: test the case of https://pydoctor.readthedocs.io/en/latest/api/pydoctor.model.System.html#postProcess + # docstring that have a link to L{extensions.PriorityProcessor} which can be interpreted as beeing + # in the extensions attribute of System but really is in the extensions package. + ... + raise NotImplementedError('unfinished!') \ No newline at end of file diff --git a/pydoctor/test/test_epydoc2stan.py b/pydoctor/test/test_epydoc2stan.py index e75c33ded..6fb7b548b 100644 --- a/pydoctor/test/test_epydoc2stan.py +++ b/pydoctor/test/test_epydoc2stan.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import List, Optional, Type, cast, TYPE_CHECKING import re @@ -52,9 +54,7 @@ class E: epydoc2stan.format_docstring(mod.contents['E']) -def docstring2html(obj: model.Documentable, docformat: Optional[str] = None) -> str: - if docformat: - obj.module.docformat = docformat +def docstring2html(obj: model.Documentable) -> str: stan = epydoc2stan.format_docstring(obj) assert stan.tagName == 'div', stan # We strip off break lines for the sake of simplicity. @@ -643,6 +643,7 @@ def f(args, kwargs, *a, **kwa) -> None: ''', modname='') mod_rst_with_asterixes = fromText(r''' + __docformat__ = 'restructuredtext' def f(args, kwargs, *a, **kwa) -> None: r""" Do something with var-positional and var-keyword arguments. @@ -655,6 +656,7 @@ def f(args, kwargs, *a, **kwa) -> None: ''', modname='') mod_rst_without_asterixes = fromText(''' + __docformat__ = 'restructuredtext' def f(args, kwargs, *a, **kwa) -> None: """ Do something with var-positional and var-keyword arguments. @@ -679,8 +681,8 @@ def f(args, kwargs, *a, **kwa) -> None: ''', modname='') epy_with_asterixes_fmt = docstring2html(mod_epy_with_asterixes.contents['f']) - rst_with_asterixes_fmt = docstring2html(mod_rst_with_asterixes.contents['f'], docformat='restructuredtext') - rst_without_asterixes_fmt = docstring2html(mod_rst_without_asterixes.contents['f'], docformat='restructuredtext') + rst_with_asterixes_fmt = docstring2html(mod_rst_with_asterixes.contents['f']) + rst_without_asterixes_fmt = docstring2html(mod_rst_without_asterixes.contents['f']) epy_without_asterixes_fmt = docstring2html(mod_epy_without_asterixes.contents['f']) assert epy_with_asterixes_fmt == rst_with_asterixes_fmt == rst_without_asterixes_fmt == epy_without_asterixes_fmt @@ -1029,21 +1031,22 @@ class Klass: mod.parsed_docstring.get_summary().to_stan(mod.docstring_linker) # type:ignore warnings = ['test:2: Cannot find link target for "thing.notfound" (you can link to external docs with --intersphinx)'] - if linkercls is linker._EpydocLinker: - warnings = warnings * 2 assert capsys.readouterr().out.strip().splitlines() == warnings - + + # reset warnings + mod.system.once_msgs = set() + # This is wrong: Klass.parsed_docstring.to_stan(mod.docstring_linker) # type:ignore Klass.parsed_docstring.get_summary().to_stan(mod.docstring_linker) # type:ignore # Because the warnings will be reported on line 2 warnings = ['test:2: Cannot find link target for "thing.notfound" (you can link to external docs with --intersphinx)'] - warnings = warnings * 2 assert capsys.readouterr().out.strip().splitlines() == warnings - # assert capsys.readouterr().out == '' + # reset warnings + mod.system.once_msgs = set() # Reset stan and summary, because they are supposed to be cached. Klass.parsed_docstring._stan = None # type:ignore @@ -1054,9 +1057,7 @@ class Klass: Klass.parsed_docstring.to_stan(mod.docstring_linker) # type:ignore Klass.parsed_docstring.get_summary().to_stan(mod.docstring_linker) # type:ignore - warnings = ['test:5: Cannot find link target for "thing.notfound" (you can link to external docs with --intersphinx)'] - warnings = warnings * 2 - + warnings = ['test:5: Cannot find link target for "thing.notfound" (you can link to external docs with --intersphinx)'] assert capsys.readouterr().out.strip().splitlines() == warnings def test_EpydocLinker_look_for_intersphinx_no_link() -> None: @@ -1123,7 +1124,7 @@ def test_EpydocLinker_resolve_identifier_xref_intersphinx_absolute_id() -> None: assert isinstance(sut, linker._EpydocLinker) url = sut.link_to('base.module.other', 'o').attributes['href'] - url_xref = sut._resolve_identifier_xref('base.module.other', 0) + url_xref = sut._resolve_identifier_xref('base.module.other', 0, 'base.module.other') assert "http://tm.tld/some.html" == url assert "http://tm.tld/some.html" == url_xref @@ -1150,7 +1151,7 @@ def test_EpydocLinker_resolve_identifier_xref_intersphinx_relative_id() -> None: # This is called for the L{ext_module} markup. url = sut.link_to('ext_module', 'ext').attributes['href'] - url_xref = sut._resolve_identifier_xref('ext_module', 0) + url_xref = sut._resolve_identifier_xref('ext_module', 0, 'ext_module') assert "http://tm.tld/some.html" == url assert "http://tm.tld/some.html" == url_xref @@ -1177,7 +1178,7 @@ def test_EpydocLinker_resolve_identifier_xref_intersphinx_link_not_found(capsys: assert sut.link_to('ext_module', 'ext').tagName == '' assert not capsys.readouterr().out with raises(LookupError): - sut._resolve_identifier_xref('ext_module', 0) + sut._resolve_identifier_xref('ext_module', 0, 'ext_module') captured = capsys.readouterr().out expected = ( @@ -1188,6 +1189,42 @@ def test_EpydocLinker_resolve_identifier_xref_intersphinx_link_not_found(capsys: assert expected == captured +def test_EpydocLinker_link_not_found_show_original(capsys: CapSys) -> None: + n = '' + m = '''\ + from n import Stuff + S = Stuff + ''' + src = '''\ + """ + L{S} + """ + class Cls: + """ + L{Stuff } + """ + from m import S + ''' + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(n, 'n') + builder.addModuleString(m, 'm') + builder.addModuleString(src, 'src') + builder.buildModules() + docstring2html(system.allobjects['src']) + captured = capsys.readouterr().out + expected = ( + 'src:2: Cannot find link target for "n.Stuff", resolved from "S"\n' + ) + assert expected == captured + + docstring2html(system.allobjects['src.Cls']) + captured = capsys.readouterr().out + expected = ( + 'src:6: Cannot find link target for "n.Stuff", resolved from "m.S"\n' + ) + assert expected == captured + class InMemoryInventory: """ A simple inventory implementation which has an in-memory API link mapping. @@ -1214,7 +1251,7 @@ class C: assert isinstance(_linker, linker._EpydocLinker) url = _linker.link_to('socket.socket', 's').attributes['href'] - url_xref = _linker._resolve_identifier_xref('socket.socket', 0) + url_xref = _linker._resolve_identifier_xref('socket.socket', 0, 'socket.socket') assert 'https://docs.python.org/3/library/socket.html#socket.socket' == url assert 'https://docs.python.org/3/library/socket.html#socket.socket' == url_xref @@ -1236,7 +1273,7 @@ class C: sut = target.docstring_linker assert isinstance(sut, linker._EpydocLinker) url = sut.link_to('internal_module.C','C').attributes['href'] - xref = sut._resolve_identifier_xref('internal_module.C', 0) + xref = sut._resolve_identifier_xref('internal_module.C', 0, 'internal_module.C') assert "internal_module.C.html" == url assert int_mod.contents['C'] is xref @@ -1291,8 +1328,6 @@ def test_EpydocLinker_warnings(capsys: CapSys) -> None: # The rationale about xref warnings is to warn when the target cannot be found. assert captured == ('module:3: Cannot find link target for "notfound"' - '\nmodule:3: Cannot find link target for "notfound"' - '\nmodule:5: Cannot find link target for "notfound"' '\nmodule:5: Cannot find link target for "notfound"\n') assert 'href="index.html#base"' in summary2html(mod) @@ -1303,6 +1338,61 @@ def test_EpydocLinker_warnings(capsys: CapSys) -> None: # No warnings are logged when generating the summary. assert captured == '' +def test_EpydocLinker_xref_look_for_name_multiple_candidates(capsys:CapSys) -> None: + """ + When the linker use look_for_name(), if 'identifier' refers to more than one object, it complains. + """ + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString('class C:...', modname='_one') + builder.addModuleString('class C:...', modname='_two') + builder.addModuleString('"L{C}"', modname='top') + builder.buildModules() + docstring2html(system.allobjects['top']) + assert capsys.readouterr().out == ( + 'top:1: ambiguous ref to C, could be _one.C, _two.C\n' + 'top:1: Cannot find link target for "C"\n') + +def test_EpydocLinker_xref_look_for_name_into_uncle_objects(capsys:CapSys) -> None: + """ + The linker walk up the object tree and see if 'identifier' refers to an + object in an "uncle" object. + """ + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString('', modname='pack', is_package=True) + builder.addModuleString('class C:...', modname='mod2', parent_name='pack') + builder.addModuleString('class I:\n var=1;"L{C}"', modname='mod1', parent_name='pack') + builder.buildModules() + assert 'href="pack.mod2.C.html"' in docstring2html(system.allobjects['pack.mod1.I.var']) + assert capsys.readouterr().out == '' + +def test_EpydocLinker_xref_look_for_name_into_all_modules(capsys:CapSys) -> None: + """ + The linker examine every module and package in the system and see if 'identifier' + names an object in each one. + """ + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString('class C:...', modname='_one') + builder.addModuleString('"L{C}"', modname='top') + builder.buildModules() + assert 'href="_one.C.html"' in docstring2html(system.allobjects['top']) + assert capsys.readouterr().out == '' + +def test_EpydocLinker_xref_walk_up_the_object_tree(capsys:CapSys) -> None: + """ + The linker walks up the object tree and see if 'identifier' refers + to an object by Python name resolution in each context. + """ + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString('class C:...', modname='pack', is_package=True) + builder.addModuleString('class I:\n var=1;"L{C}"', modname='mod1', parent_name='pack') + builder.buildModules() + assert 'href="pack.C.html"' in docstring2html(system.allobjects['pack.mod1.I.var']) + assert capsys.readouterr().out == '' + def test_xref_not_found_epytext(capsys: CapSys) -> None: """ When a link in an epytext docstring cannot be resolved, the reference @@ -1393,10 +1483,12 @@ def __init__(self) -> None: self.requests: List[str] = [] def link_to(self, target: str, label: "Flattenable", *, is_annotation: bool = False) -> Tag: + if target.startswith('builtins.'): + target = target[len('builtins.'):] self.requests.append(target) return tags.transparent(label) - def link_xref(self, target: str, label: "Flattenable", lineno: int) -> Tag: + def link_xref(self, target: str, label: "Flattenable", lineno: int, rawtarget: str | None = None) -> Tag: assert False @mark.parametrize('annotation', ( @@ -1993,6 +2085,7 @@ def f(self, x:typ) -> typ: assert isinstance(var, model.Attribute) assert 'href="index.html#typ"' in flatten(epydoc2stan.type2stan(var) or '') + assert not capsys.readouterr().out # Pydoctor is not a checker so no warning is beeing reported. def test_not_found_annotation_does_not_create_link() -> None: @@ -2065,8 +2158,7 @@ def test_invalid_epytext_renders_as_plaintext(capsys: CapSys) -> None: """ An invalid epytext docstring will be rederered as plaintext. """ - - mod = fromText(''' + src = ''' def func(): """ Title @@ -2078,7 +2170,8 @@ def func(): """ pass - ''', modname='invalid') + ''' + mod = fromText(src, modname='invalid') expected = """

Title @@ -2095,10 +2188,183 @@ def func(): 'invalid:8: bad docstring: Wrong underline character for heading.\n') assert actual == expected - assert docstring2html(mod.contents['func'], docformat='plaintext') == expected + mod = fromText(' __docformat__="plaintext"\n' + src, modname='invalid') + assert docstring2html(mod.contents['func']) == expected captured = capsys.readouterr().out assert captured == '' +def test_parsed_names_partially_resolved_early() -> None: + """ + Test for issue #295 + + Annotations are first locally resolved when we reach the end of the module, + then again when we actually resolve the name when generating the stan for the annotation. + """ + typing = '''\ + Callable = ClassVar = TypeVar = object() + ''' + + base = '''\ + import ast + class Vis(ast.NodeVisitor): + ... + ''' + src = '''\ + from typing import Callable + import typing as t + + from .base import Vis + + class Cls(Vis, t.Generic['_T']): + """ + L{Cls} + """ + clsvar:Callable[[], str] + clsvar2:t.ClassVar[Callable[[], str]] + + def __init__(self, a:'_T'): + self._a:'_T' = a + + C = Cls + _T = t.TypeVar('_T') + unknow: i|None|list + ann:Cls + ''' + + top = '''\ + # the order matters here + from .src import C, Cls, Vis + __all__ = ['Cls', 'C', 'Vis'] + ''' + + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(top, 'top', is_package=True) + builder.addModuleString(base, 'base', 'top') + builder.addModuleString(src, 'src', 'top') + builder.addModuleString(typing, 'typing') + builder.buildModules() + + Cls = system.allobjects['top.Cls'] + clsvar = Cls.contents['clsvar'] + clsvar2 = Cls.contents['clsvar2'] + a = Cls.contents['_a'] + assert clsvar.expandName('typing.Callable')=='typing.Callable' + assert 'refuri="typing.Callable"' in clsvar.parsed_type.to_node().pformat() #type: ignore + assert 'href="typing.html#Callable"' in flatten(clsvar.parsed_type.to_stan(clsvar.docstring_linker)) #type: ignore + assert 'href="typing.html#ClassVar"' in flatten(clsvar2.parsed_type.to_stan(clsvar2.docstring_linker)) #type: ignore + assert 'href="top.src.html#_T"' in flatten(a.parsed_type.to_stan(clsvar.docstring_linker)) #type: ignore + + # the reparenting/alias issue + ann = system.allobjects['top.src.ann'] + assert 'href="top.Cls.html"' in flatten(ann.parsed_type.to_stan(ann.docstring_linker)) #type: ignore + assert 'href="top.Cls.html"' in flatten(Cls.parsed_docstring.to_stan(Cls.docstring_linker)) #type: ignore + + unknow = system.allobjects['top.src.unknow'] + assert flatten_text(unknow.parsed_type.to_stan(unknow.docstring_linker)) == 'i | None | list' #type: ignore + + # test the __init__ signature + assert 'href="top.src.html#_T"' in flatten(format_signature(Cls.contents['__init__'])) #type: ignore + +def test_reparented_ambiguous_annotation_confusion() -> None: + """ + Like L{test_top_level_type_alias_wins_over_class_level} but with reparented class. + """ + src = ''' + typ = object() + class C: + typ = int|str + var: typ + ''' + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(src, modname='_m') + builder.addModuleString('from _m import C; __all__=["C"]', 'm') + builder.buildModules() + var = system.allobjects['m.C.var'] + assert 'href="_m.html#typ"' in flatten(var.parsed_type.to_stan(var.docstring_linker)) #type: ignore + +def test_reparented_builtins_confusion() -> None: + """ + - builtin links are resolved as such even when the new parent + declares a name shadowing a builtin. + """ + src = ''' + class C(int): + var: list + C = print('one') + @stuff(auto=object) + def __init__(self, v:bytes=bytes): + "L{str}" + ''' + top = ''' + list = object = int = print = str = bytes = True + + from src import C + __all__=["C"] + ''' + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(src, modname='src') + builder.addModuleString(top, modname='top') + builder.buildModules() + clsvar = system.allobjects['top.C.var'] + C = system.allobjects['top.C'] + Ci = system.allobjects['top.C.C'] + __init__ = system.allobjects['top.C.__init__'] + + assert 'refuri="builtins.list"' in clsvar.parsed_type.to_node().pformat() #type: ignore + assert 'refuri="builtins.print"' in Ci.parsed_value.to_node().pformat() #type: ignore + assert 'refuri="builtins.int"' in C.parsed_bases[0].to_node().pformat() #type: ignore + assert 'refuri="builtins.object"' in __init__.parsed_decorators[0].to_node().pformat() #type: ignore + assert 'refuri="builtins.bytes"' in __init__.parsed_signature.to_node().pformat() #type: ignore + assert 'refuri="builtins.bytes"' in __init__.parsed_signature.to_node().pformat() #type: ignore + assert 'refuri="builtins.bytes"' in __init__.parsed_annotations['v'].to_node().pformat() #type: ignore + assert 'refuri="builtins.str"' in __init__.parsed_docstring.to_node().pformat() #type: ignore + +def test_link_resolving_unbound_names() -> None: + """ + - unbdound names are not touched, and does not stop the process. + """ + src = ''' + class C: + var: unknown|list + ''' + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(src, modname='src') + builder.buildModules() + clsvar = system.allobjects['src.C.var'] + + assert 'refuri="builtins.list"' in clsvar.parsed_type.to_node().pformat() #type: ignore + assert 'refuri="unknown"' in clsvar.parsed_type.to_node().pformat() #type: ignore + # does not work for constant values at the moment + +def test_reference_transform_in_type_docstring() -> None: + """ + It will fail with ParsedTypeDocstring at the moment. + """ + src = ''' + __docformat__='google' + class C: + """ + Args: + a (list): the list + """ + ''' + system = model.System() + builder = system.systemBuilder(system) + builder.addModuleString(src, modname='src') + builder.addModuleString('from src import C;__all__=["C"];list=True', modname='top') + builder.buildModules() + clsvar = system.allobjects['top.C'] + assert 'refuri="builtins.list"' in clsvar.parsed_docstring.fields[1].body().to_node().pformat() #type: ignore + +# what to do with inherited documentation of reparented class attribute part of an +# import cycle? We can't set the value of parsed_docstring from the astbuilder because +# we havnen't resolved the mro yet. + + def test_regression_not_found_linenumbers(capsys: CapSys) -> None: """ Test for issue https://github.com/twisted/pydoctor/issues/745 @@ -2140,7 +2406,7 @@ def create_repository(self) -> repository.Repository: mod = fromText(code, ) docstring2html(mod.contents['Settings']) captured = capsys.readouterr().out - assert captured == ':15: Cannot find link target for "TypeError"\n' + assert captured == ':15: Cannot find link target for "builtins.TypeError", resolved from "TypeError" (you can link to external docs with --intersphinx)\n' def test_does_not_loose_type_linenumber(capsys: CapSys) -> None: # exmaple from numpy/distutils/ccompiler_opt.py @@ -2172,7 +2438,7 @@ def __init__(self): # the link not found warnings. getHTMLOf(mod.contents['C']) assert capsys.readouterr().out == (':16: Existing docstring at line 10 is overriden\n' - ':10: Cannot find link target for "bool"\n') + ':10: Cannot find link target for "builtins.bool", resolved from "bool" (you can link to external docs with --intersphinx)\n') def test_numpydoc_warns_about_unknown_types_in_explicit_references_at_line(capsys: CapSys) -> None: # we don't have a good knowledge of linenumber in numpy or google docstring @@ -2247,3 +2513,48 @@ def test_function_signature_html(signature: str, expected: str) -> None: # This little trick makes it possible to back reproduce the original signature from the genrated HTML. html = flatten(format_signature(docfunc)) assert html == expected + +def test_linker_reports_error_with_link_as_in_source(capsys: CapSys) -> None: + src1 = ''' + # test._impl + class notfoundthing: + from notfound import thing as t + ''' + + src2 = ''' + # test.lib + from ._impl import notfoundthing as a + thing = 123; 'L{a.t.bar}' # a symbol externally defined + foo = 456; 'L{str}' # a builtin + ''' + + builder = (s:=model.System()).systemBuilder(s) + builder.addModuleString('', 'test', is_package=True) + builder.addModuleString(src1, '_impl', parent_name='test') + builder.addModuleString(src2, 'lib', parent_name='test') + builder.buildModules() + + docstring2html(s.allobjects['test.lib.thing']) + docstring2html(s.allobjects['test.lib.foo']) + + assert capsys.readouterr().out == ( + 'test.lib:4: Cannot find link target for "notfound.thing.bar", resolved from "a.t.bar" (you can link to external docs with --intersphinx)\n' + 'test.lib:5: Cannot find link target for "builtins.str", resolved from "str" (you can link to external docs with --intersphinx)\n') + +def test_hidden_object_doesnt_gets_its_docstring_parsed(capsys: CapSys) -> None: + src = ''' + class C: + def __eq__(self, other): + """ + L{Invalid) epytext. + """ + ''' + + builder = (s:=model.System()).systemBuilder(s) + s.options.privacy.append((model.PrivacyClass.HIDDEN, '**.__eq__')) + builder.addModuleString(src, 'test') + builder.buildModules() + assert s.privacyClass(eq:=s.allobjects['test.C.__eq__']) == model.PrivacyClass.HIDDEN + assert not eq.parsed_docstring + assert not capsys.readouterr().out + diff --git a/pydoctor/test/test_sphinx.py b/pydoctor/test/test_sphinx.py index 4200f045e..2da8d4ebd 100644 --- a/pydoctor/test/test_sphinx.py +++ b/pydoctor/test/test_sphinx.py @@ -110,7 +110,8 @@ def test_generate_empty_functional() -> None: @contextmanager def openFileForWriting(path: str) -> Iterator[io.BytesIO]: yield output - inv_writer._openFileForWriting = openFileForWriting # type: ignore + + inv_writer._openFileForWriting = openFileForWriting # type:ignore inv_writer.generate(subjects=[], basepath='base-path') diff --git a/pydoctor/test/test_type_fields.py b/pydoctor/test/test_type_fields.py index d1c9c1718..d9ab9a52e 100644 --- a/pydoctor/test/test_type_fields.py +++ b/pydoctor/test/test_type_fields.py @@ -495,4 +495,4 @@ class MachAr: mod = fromText(src) [docstring2html(o) for o in mod.system.allobjects.values()] - assert capsys.readouterr().out.splitlines() == [':8: Cannot find link target for "int"'] \ No newline at end of file + assert capsys.readouterr().out.splitlines() == [':8: Cannot find link target for "builtins.int", resolved from "int" (you can link to external docs with --intersphinx)'] \ No newline at end of file