From aa0673f160cb3991e2f3a5c84b1756673b480d99 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Thu, 11 Apr 2019 16:55:33 +0100
Subject: [PATCH 01/50] Stubgen: Remove misplaced type comments before parsing

These happen with some frequency in third-party libraries.
---
 mypy/options.py             |  14 ++++
 mypy/parse.py               |   5 ++
 mypy/stubgen.py             |   4 +-
 mypy/stubutil.py            |  25 +++++++-
 mypy/test/teststubgen.py    | 125 +++++++++++++++++++++++++++++++++++-
 test-data/unit/stubgen.test |  10 +++
 6 files changed, 180 insertions(+), 3 deletions(-)

diff --git a/mypy/options.py b/mypy/options.py
index 5741c5a2e0b0..998f8f981661 100644
--- a/mypy/options.py
+++ b/mypy/options.py
@@ -3,8 +3,20 @@
 import pprint
 import sys
 
+<<<<<<< HEAD
 from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple
 from typing_extensions import Final
+||||||| merged common ancestors
+from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple
+MYPY = False
+if MYPY:
+    from typing_extensions import Final
+=======
+from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple, Callable, AnyStr
+MYPY = False
+if MYPY:
+    from typing_extensions import Final
+>>>>>>> Stubgen: Remove misplaced type comments before parsing
 
 from mypy import defaults
 from mypy.util import get_class_descriptors, replace_object_state
@@ -262,6 +274,8 @@ def __init__(self) -> None:
         self.cache_map = {}  # type: Dict[str, Tuple[str, str]]
         # Don't properly free objects on exit, just kill the current process.
         self.fast_exit = False
+        # Used to transform source code before parsing if not None
+        self.transform_source = None  # type: Optional[Callable[[AnyStr], AnyStr]]
         # Print full path to each file in the report.
         self.show_absolute_path = False  # type: bool
 
diff --git a/mypy/parse.py b/mypy/parse.py
index 149a0bbb6196..a8da1ba6db49 100644
--- a/mypy/parse.py
+++ b/mypy/parse.py
@@ -18,6 +18,11 @@ def parse(source: Union[str, bytes],
     The python_version (major, minor) option determines the Python syntax variant.
     """
     is_stub_file = fnam.endswith('.pyi')
+    if options.transform_source is not None:
+        if isinstance(source, str):  # Work around mypy issue
+            source = options.transform_source(source)
+        else:
+            source = options.transform_source(source)
     if options.python_version[0] >= 3 or is_stub_file:
         import mypy.fastparse
         return mypy.fastparse.parse(source,
diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index a901b4d153f9..7f971f2a6fa5 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -76,7 +76,7 @@
 from mypy.stubutil import (
     write_header, default_py2_interpreter, CantImport, generate_guarded,
     walk_packages, find_module_path_and_all_py2, find_module_path_and_all_py3,
-    report_missing, fail_missing
+    report_missing, fail_missing, remove_misplaced_type_comments
 )
 from mypy.stubdoc import parse_all_signatures, find_unique_signatures, Sig
 from mypy.options import Options as MypyOptions
@@ -977,6 +977,8 @@ def mypy_options(stubgen_options: Options) -> MypyOptions:
     options.ignore_errors = True
     options.semantic_analysis_only = True
     options.python_version = stubgen_options.pyversion
+    options.show_traceback = True
+    options.transform_source = remove_misplaced_type_comments
     return options
 
 
diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 0f2314bc8504..64834900d92a 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -7,10 +7,11 @@
 import pkgutil
 import importlib
 import subprocess
+import re
 from types import ModuleType
 from contextlib import contextmanager
 
-from typing import Optional, Tuple, List, IO, Iterator
+from typing import Optional, Tuple, List, IO, Iterator, AnyStr
 
 
 class CantImport(Exception):
@@ -165,3 +166,25 @@ def report_missing(mod: str, message: Optional[str] = '') -> None:
 
 def fail_missing(mod: str) -> None:
     raise SystemExit("Can't find module '{}' (consider using --search-path)".format(mod))
+
+
+def remove_misplaced_type_comments(source: AnyStr) -> AnyStr:
+    if isinstance(source, bytes):
+        # This gives us a 1-1 character code mapping, so it's roundtrippable.
+        text = source.decode('latin1')
+    else:
+        text = source
+
+    # Remove something that looks like a variable type comment but that's by itself
+    # on a line, as it will often generate a parse error (unless it's # type: ignore).
+    text = re.sub(r'^[ \t]*# +type: +[a-zA-Z_].*$', '', text, flags=re.MULTILINE)
+
+    # Remove something that looks like a function type annotation after docstring,
+    # which will result in a parse error.
+    text = re.sub(r'""" *\n[ \t\n]*# +type: +\(.*$', '"""\n', text, flags=re.MULTILINE)
+    text = re.sub(r"''' *\n[ \t\n]*# +type: +\(.*$", "'''\n", text, flags=re.MULTILINE)
+
+    if isinstance(source, bytes):
+        return text.encode('latin1')
+    else:
+        return text
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 438bfdeebe53..e45732beea9b 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -17,7 +17,7 @@
     generate_stubs, parse_options, Options, collect_build_targets,
     mypy_options
 )
-from mypy.stubutil import walk_packages
+from mypy.stubutil import walk_packages, remove_misplaced_type_comments
 from mypy.stubgenc import generate_c_type_stub, infer_method_sig, generate_c_function_stub
 from mypy.stubdoc import (
     parse_signature, parse_all_signatures, build_signature, find_unique_signatures,
@@ -270,6 +270,129 @@ def test_infer_prop_type_from_docstring(self) -> None:
                      'Tuple[int, int]')
         assert_equal(infer_prop_type_from_docstring('\nstr: A string.'), None)
 
+    def test_remove_misplaced_type_comments_1(self) -> None:
+        good = """
+        \u1234
+        def f(x):  # type: (int) -> int
+
+        def g(x):
+            # type: (int) -> int
+
+        def h():
+
+            # type: () int
+
+        x = 1  # type: int
+        """
+
+        assert_equal(remove_misplaced_type_comments(good), good)
+
+    def test_remove_misplaced_type_comments_2(self) -> None:
+        bad = """
+        def f(x):
+            # type: Callable[[int], int]
+            pass
+
+        x = 1
+        # type: int
+        """
+        bad_fixed = """
+        def f(x):
+
+            pass
+
+        x = 1
+
+        """
+        assert_equal(remove_misplaced_type_comments(bad), bad_fixed)
+
+    def test_remove_misplaced_type_comments_3(self) -> None:
+        bad = '''
+        def f(x):
+            """docstring"""
+            # type: (int) -> int
+            pass
+
+        def g(x):
+            """docstring
+            """
+            # type: (int) -> int
+            pass
+        '''
+        bad_fixed = '''
+        def f(x):
+            """docstring"""
+
+            pass
+
+        def g(x):
+            """docstring
+            """
+
+            pass
+        '''
+        assert_equal(remove_misplaced_type_comments(bad), bad_fixed)
+
+    def test_remove_misplaced_type_comments_4(self) -> None:
+        bad = """
+        def f(x):
+            '''docstring'''
+            # type: (int) -> int
+            pass
+
+        def g(x):
+            '''docstring
+            '''
+            # type: (int) -> int
+            pass
+        """
+        bad_fixed = """
+        def f(x):
+            '''docstring'''
+
+            pass
+
+        def g(x):
+            '''docstring
+            '''
+
+            pass
+        """
+        assert_equal(remove_misplaced_type_comments(bad), bad_fixed)
+
+    def test_remove_misplaced_type_comments_bytes(self) -> None:
+        original = b"""
+        \xbf
+        def f(x):  # type: (int) -> int
+
+        def g(x):
+            # type: (int) -> int
+            pass
+
+        def h():
+            # type: int
+            pass
+
+        x = 1  # type: int
+        """
+
+        dest = b"""
+        \xbf
+        def f(x):  # type: (int) -> int
+
+        def g(x):
+            # type: (int) -> int
+            pass
+
+        def h():
+
+            pass
+
+        x = 1  # type: int
+        """
+
+        assert_equal(remove_misplaced_type_comments(original), dest)
+
 
 class StubgenPythonSuite(DataSuite):
     required_out_section = True
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index b95e34f3f0d4..cf149bc3381d 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1511,3 +1511,13 @@ class B:
     y: str = ...
     @x.setter
     def x(self, value: Any) -> None: ...
+
+[case testMisplacedTypeComment]
+def f():
+    x = 0
+
+    # type: str
+    y = ''
+
+[out]
+def f() -> None: ...

From ae338766a3b76123882397baef7af5f188f7cf15 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Thu, 11 Apr 2019 17:05:16 +0100
Subject: [PATCH 02/50] stubgen: Don't fail if docstring cannot be tokenized

---
 mypy/stubdoc.py          | 8 ++++++--
 mypy/test/teststubgen.py | 7 +++++++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/mypy/stubdoc.py b/mypy/stubdoc.py
index bc10fe171ca5..48c678e6af31 100644
--- a/mypy/stubdoc.py
+++ b/mypy/stubdoc.py
@@ -199,8 +199,12 @@ def infer_sig_from_docstring(docstr: str, name: str) -> Optional[List[FunctionSi
     state = DocStringParser(name)
     # Return all found signatures, even if there is a parse error after some are found.
     with contextlib.suppress(tokenize.TokenError):
-        for token in tokenize.tokenize(io.BytesIO(docstr.encode('utf-8')).readline):
-            state.add_token(token)
+        try:
+            tokens = tokenize.tokenize(io.BytesIO(docstr.encode('utf-8')).readline)
+            for token in tokens:
+                state.add_token(token)
+        except IndentationError:
+            return None
     sigs = state.get_signatures()
 
     def is_unique_args(sig: FunctionSig) -> bool:
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index e45732beea9b..55cdbcaf7fa6 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -253,6 +253,13 @@ def test_infer_sig_from_docstring_duplicate_args(self) -> None:
                      [FunctionSig(name='func', args=[ArgSig(name='x'), ArgSig(name='y')],
                                   ret_type='int')])
 
+    def test_infer_sig_from_docstring_bad_indentation(self) -> None:
+        assert_equal(infer_sig_from_docstring("""
+            x
+              x
+             x
+            """, 'func'), None)
+
     def test_infer_arg_sig_from_docstring(self) -> None:
         assert_equal(infer_arg_sig_from_docstring("(*args, **kwargs)"),
                      [ArgSig(name='*args'), ArgSig(name='**kwargs')])

From 80108d3597cc477b5f16e9aff52e500a2e0ffbe4 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 10:45:39 +0100
Subject: [PATCH 03/50] stubgen: Handle None value for __file__

We just arbitrarily assume that this means that the module is not
a C module. This happens with paste.
---
 mypy/stubutil.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 64834900d92a..2f8c32035cc0 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -21,8 +21,11 @@ def __init__(self, module: str, message: str):
 
 
 def is_c_module(module: ModuleType) -> bool:
-    return ('__file__' not in module.__dict__ or
-            os.path.splitext(module.__dict__['__file__'])[-1] in ['.so', '.pyd'])
+    if '__file__' not in module.__dict__:
+        return True
+    if module.__dict__['__file__'] is None:
+        return False
+    return os.path.splitext(module.__dict__['__file__'])[-1] in ['.so', '.pyd']
 
 
 def write_header(file: IO[str], module_name: Optional[str] = None,

From 37a1ab930ae53e95b78cee4508b8b3faa4501709 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 10:47:57 +0100
Subject: [PATCH 04/50] stubgen: Fix None-related crash

---
 mypy/stubgen.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 7f971f2a6fa5..b64f8d024e5d 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -880,6 +880,12 @@ def get_qualified_name(o: Expression) -> str:
         return ERROR_MARKER
 
 
+def remove_blacklisted_modules(modules: List[StubSource]) -> List[StubSource]:
+    return [module for module in modules
+            if module.path is None or not any(substr in (module.path + '\n')
+                                              for substr in BLACKLIST)]
+
+
 def collect_build_targets(options: Options, mypy_opts: MypyOptions) -> Tuple[List[StubSource],
                                                                              List[StubSource]]:
     """Collect files for which we need to generate stubs.

From 020b11fc839637307a2f3468407a6c057f8b349c Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 10:54:45 +0100
Subject: [PATCH 05/50] stubgen: Attempt to fix namespace packages

---
 mypy/stubutil.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 2f8c32035cc0..2265cf4b66db 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -21,10 +21,10 @@ def __init__(self, module: str, message: str):
 
 
 def is_c_module(module: ModuleType) -> bool:
-    if '__file__' not in module.__dict__:
+    if module.__dict__.get('__file__') is None:
+        # Could be a namespace package. These must be handled through
+        # introspection, since there is no source file.
         return True
-    if module.__dict__['__file__'] is None:
-        return False
     return os.path.splitext(module.__dict__['__file__'])[-1] in ['.so', '.pyd']
 
 

From e1de86673011fb23df88c06ea091e7f554f9b47b Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 12:00:57 +0100
Subject: [PATCH 06/50] stubgen: Add --verbose and --quiet flags

---
 mypy/stubgen.py          | 69 +++++++++++++++++++++++++++++-----------
 mypy/stubutil.py         | 19 +++++++++--
 mypy/test/teststubgen.py | 23 ++++++++++++--
 3 files changed, 88 insertions(+), 23 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index b64f8d024e5d..8b046cbd027a 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -76,7 +76,7 @@
 from mypy.stubutil import (
     write_header, default_py2_interpreter, CantImport, generate_guarded,
     walk_packages, find_module_path_and_all_py2, find_module_path_and_all_py3,
-    report_missing, fail_missing, remove_misplaced_type_comments
+    report_missing, fail_missing, remove_misplaced_type_comments, common_dir_prefix
 )
 from mypy.stubdoc import parse_all_signatures, find_unique_signatures, Sig
 from mypy.options import Options as MypyOptions
@@ -96,10 +96,21 @@ class Options:
 
     This class is mutable to simplify testing.
     """
-    def __init__(self, pyversion: Tuple[int, int], no_import: bool, doc_dir: str,
-                 search_path: List[str], interpreter: str, parse_only: bool, ignore_errors: bool,
-                 include_private: bool, output_dir: str, modules: List[str], packages: List[str],
-                 files: List[str]) -> None:
+    def __init__(self,
+                 pyversion: Tuple[int, int],
+                 no_import: bool,
+                 doc_dir: str,
+                 search_path: List[str],
+                 interpreter: str,
+                 parse_only: bool,
+                 ignore_errors: bool,
+                 include_private: bool,
+                 output_dir: str,
+                 modules: List[str],
+                 packages: List[str],
+                 files: List[str],
+                 verbose: bool,
+                 quiet: bool) -> None:
         # See parse_options for descriptions of the flags.
         self.pyversion = pyversion
         self.no_import = no_import
@@ -114,6 +125,8 @@ def __init__(self, pyversion: Tuple[int, int], no_import: bool, doc_dir: str,
         self.modules = modules
         self.packages = packages
         self.files = files
+        self.verbose = verbose
+        self.quiet = quiet
 
 
 class StubSource(BuildSource):
@@ -904,7 +917,9 @@ def collect_build_targets(options: Options, mypy_opts: MypyOptions) -> Tuple[Lis
             py_modules, c_modules = find_module_paths_using_imports(options.modules,
                                                                     options.packages,
                                                                     options.interpreter,
-                                                                    options.pyversion)
+                                                                    options.pyversion,
+                                                                    options.verbose,
+                                                                    options.quiet)
     else:
         # Use mypy native source collection for files and directories.
         try:
@@ -917,10 +932,12 @@ def collect_build_targets(options: Options, mypy_opts: MypyOptions) -> Tuple[Lis
     return py_modules, c_modules
 
 
-def find_module_paths_using_imports(modules: List[str], packages: List[str],
+def find_module_paths_using_imports(modules: List[str],
+                                    packages: List[str],
                                     interpreter: str,
                                     pyversion: Tuple[int, int],
-                                    quiet: bool = True) -> Tuple[List[StubSource],
+                                    verbose: bool,
+                                    quiet: bool) -> Tuple[List[StubSource],
                                                                  List[StubSource]]:
     """Find path and runtime value of __all__ (if possible) for modules and packages.
 
@@ -936,9 +953,10 @@ def find_module_paths_using_imports(modules: List[str], packages: List[str],
             else:
                 result = find_module_path_and_all_py3(mod)
         except CantImport as e:
-            if not quiet:
+            if verbose:
                 traceback.print_exc()
-            report_missing(mod, e.message)
+            if not quiet:
+                report_missing(mod, e.message)
             continue
         if not result:
             c_modules.append(StubSource(mod))
@@ -1074,9 +1092,7 @@ def collect_docs_signatures(doc_dir: str) -> Tuple[Dict[str, str], Dict[str, str
     return sigs, class_sigs
 
 
-def generate_stubs(options: Options,
-                   # additional args for testing
-                   quiet: bool = False, add_header: bool = True) -> None:
+def generate_stubs(options: Options) -> None:
     """Main entry point for the program."""
     mypy_opts = mypy_options(options)
     py_modules, c_modules = collect_build_targets(options, mypy_opts)
@@ -1088,6 +1104,7 @@ def generate_stubs(options: Options,
 
     # Use parsed sources to generate stubs for Python modules.
     generate_asts_for_modules(py_modules, options.parse_only, mypy_opts)
+    files = []
     for mod in py_modules:
         assert mod.path is not None, "Not found module was not skipped"
         target = mod.module.replace('.', '/')
@@ -1096,7 +1113,8 @@ def generate_stubs(options: Options,
         else:
             target += '.pyi'
         target = os.path.join(options.output_dir, target)
-        with generate_guarded(mod.module, target, options.ignore_errors, quiet):
+        files.append(target)
+        with generate_guarded(mod.module, target, options.ignore_errors, options.verbose):
             generate_stub_from_ast(mod, target,
                                    options.parse_only, options.pyversion,
                                    options.include_private, add_header)
@@ -1105,9 +1123,16 @@ def generate_stubs(options: Options,
     for mod in c_modules:
         target = mod.module.replace('.', '/') + '.pyi'
         target = os.path.join(options.output_dir, target)
-        with generate_guarded(mod.module, target, options.ignore_errors, quiet):
-            generate_stub_for_c_module(mod.module, target, sigs=sigs, class_sigs=class_sigs,
-                                       add_header=add_header)
+        files.append(target)
+        with generate_guarded(mod.module, target, options.ignore_errors, options.verbose):
+            generate_stub_for_c_module(mod.module, target, sigs=sigs, class_sigs=class_sigs, add_header=add_header)
+    num_modules = len(py_modules) + len(c_modules)
+    if not options.quiet and num_modules > 0:
+        print('Processed %d modules' % num_modules)
+        if len(files) == 1:
+            print('Generated %s' % files[0])
+        else:
+            print('Generated files under %s' % common_dir_prefix(files) + os.sep)
 
 
 HEADER = """%(prog)s [-h] [--py2] [more options, see -h]
@@ -1140,6 +1165,10 @@ def parse_options(args: List[str]) -> Options:
     parser.add_argument('--include-private', action='store_true',
                         help="generate stubs for objects and members considered private "
                              "(single leading underscore and no trailing underscores)")
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help="show more verbose messages")
+    parser.add_argument('-q', '--quiet', action='store_true',
+                        help="show fewer messages")
     parser.add_argument('--doc-dir', metavar='PATH', default='',
                         help="use .rst documentation in PATH (this may result in "
                              "better stubs in some cases; consider setting this to "
@@ -1168,6 +1197,8 @@ def parse_options(args: List[str]) -> Options:
         ns.interpreter = sys.executable if pyversion[0] == 3 else default_py2_interpreter()
     if ns.modules + ns.packages and ns.files:
         parser.error("May only specify one of: modules/packages or files.")
+    if ns.quiet and ns.verbose:
+        parser.error('Cannot specify both quiet and verbose messages')
 
     # Create the output folder if it doesn't already exist.
     if not os.path.exists(ns.output_dir):
@@ -1184,7 +1215,9 @@ def parse_options(args: List[str]) -> Options:
                    output_dir=ns.output_dir,
                    modules=ns.modules,
                    packages=ns.packages,
-                   files=ns.files)
+                   files=ns.files,
+                   verbose=ns.verbose,
+                   quiet=ns.quiet)
 
 
 def main() -> None:
diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 2265cf4b66db..db169e8d8afa 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -143,11 +143,13 @@ def find_module_path_and_all_py3(module: str) -> Optional[Tuple[str, Optional[Li
 
 @contextmanager
 def generate_guarded(mod: str, target: str,
-                     ignore_errors: bool = True, quiet: bool = False) -> Iterator[None]:
+                     ignore_errors: bool = True, verbose: bool = False) -> Iterator[None]:
     """Ignore or report errors during stub generation.
 
     Optionally report success.
     """
+    if verbose:
+        print('Processing %s' % mod)
     try:
         yield
     except Exception as e:
@@ -157,7 +159,7 @@ def generate_guarded(mod: str, target: str,
             # --ignore-errors was passed
             print("Stub generation failed for", mod, file=sys.stderr)
     else:
-        if not quiet:
+        if verbose:
             print('Created %s' % target)
 
 
@@ -191,3 +193,16 @@ def remove_misplaced_type_comments(source: AnyStr) -> AnyStr:
         return text.encode('latin1')
     else:
         return text
+
+
+def common_dir_prefix(paths: List[str]) -> str:
+    if not paths:
+        return '.'
+    cur = os.path.dirname(paths[0])
+    for path in paths[1:]:
+        while True:
+            path = os.path.dirname(path)
+            if (cur + '/').startswith(path + '/'):
+                cur = path
+                break
+    return cur or '.'
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 55cdbcaf7fa6..865a09496b66 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -17,7 +17,7 @@
     generate_stubs, parse_options, Options, collect_build_targets,
     mypy_options
 )
-from mypy.stubutil import walk_packages, remove_misplaced_type_comments
+from mypy.stubutil import walk_packages, remove_misplaced_type_comments, common_dir_prefix
 from mypy.stubgenc import generate_c_type_stub, infer_method_sig, generate_c_function_stub
 from mypy.stubdoc import (
     parse_signature, parse_all_signatures, build_signature, find_unique_signatures,
@@ -400,6 +400,20 @@ def h():
 
         assert_equal(remove_misplaced_type_comments(original), dest)
 
+    def test_common_dir_prefix(self) -> None:
+        assert common_dir_prefix([]) == '.'
+        assert common_dir_prefix(['x.pyi']) == '.'
+        assert common_dir_prefix(['./x.pyi']) == '.'
+        assert common_dir_prefix(['foo/bar/x.pyi']) == 'foo/bar'
+        assert common_dir_prefix(['foo/bar/x.pyi',
+                                  'foo/bar/y.pyi']) == 'foo/bar'
+        assert common_dir_prefix(['foo/bar/x.pyi', 'foo/y.pyi']) == 'foo'
+        assert common_dir_prefix(['foo/x.pyi', 'foo/bar/y.pyi']) == 'foo'
+        assert common_dir_prefix(['foo/bar/zar/x.pyi', 'foo/y.pyi']) == 'foo'
+        assert common_dir_prefix(['foo/x.pyi', 'foo/bar/zar/y.pyi']) == 'foo'
+        assert common_dir_prefix(['foo/bar/zar/x.pyi', 'foo/bar/y.pyi']) == 'foo/bar'
+        assert common_dir_prefix(['foo/bar/x.pyi', 'foo/bar/zar/y.pyi']) == 'foo/bar'
+
 
 class StubgenPythonSuite(DataSuite):
     required_out_section = True
@@ -429,7 +443,7 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None:
                     options.no_import = True
                 if not testcase.name.endswith('_semanal'):
                     options.parse_only = True
-                generate_stubs(options, quiet=True, add_header=False)
+                generate_stubs(options, add_header=False)
                 a = []  # type: List[str]
                 self.add_file(os.path.join(out_dir, 'main.pyi'), a)
             except CompileError as e:
@@ -449,7 +463,10 @@ def parse_flags(self, program_text: str, extra: List[str]) -> Options:
             flag_list = flags.group(1).split()
         else:
             flag_list = []
-        return parse_options(flag_list + extra)
+        options = parse_options(flag_list + extra)
+        if '--verbose' not in flag_list:
+            options.quiet = True
+        return options
 
     def add_file(self, path: str, result: List[str]) -> None:
         with open(path, encoding='utf8') as file:

From 447307545a6fd88cdf9d3785eae3445f613c1f34 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 12:22:04 +0100
Subject: [PATCH 07/50] stubgen: Use __init__.pyi for C modules when needed

---
 mypy/stubgen.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 8b046cbd027a..e056e849bc5b 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -1121,7 +1121,11 @@ def generate_stubs(options: Options) -> None:
 
     # Separately analyse C modules using different logic.
     for mod in c_modules:
-        target = mod.module.replace('.', '/') + '.pyi'
+        if any(py_mod.module.startswith(mod.module + '.')
+               for py_mod in py_modules + c_modules):
+            target = mod.module.replace('.', '/') + '/__init__.pyi'
+        else:
+            target = mod.module.replace('.', '/') + '.pyi'
         target = os.path.join(options.output_dir, target)
         files.append(target)
         with generate_guarded(mod.module, target, options.ignore_errors, options.verbose):

From 3a0986d899d7aa0e8bb31a119530683c6313b1c9 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 12:39:49 +0100
Subject: [PATCH 08/50] stubgen: If we can't import a module, give more
 information

Show the module which we failed to import and hints about what
to do next.
---
 mypy/stubgen.py  |  5 +++--
 mypy/stubutil.py | 15 +++++++++++++--
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index e056e849bc5b..2afa9c01375f 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -953,10 +953,11 @@ def find_module_paths_using_imports(modules: List[str],
             else:
                 result = find_module_path_and_all_py3(mod)
         except CantImport as e:
+            tb = traceback.format_exc()
             if verbose:
-                traceback.print_exc()
+                sys.stdout.write(tb)
             if not quiet:
-                report_missing(mod, e.message)
+                report_missing(mod, e.message, tb)
             continue
         if not result:
             c_modules.append(StubSource(mod))
diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index db169e8d8afa..2e09d0bf822c 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -163,10 +163,21 @@ def generate_guarded(mod: str, target: str,
             print('Created %s' % target)
 
 
-def report_missing(mod: str, message: Optional[str] = '') -> None:
+PY2_MODULES = {'cStringIO', 'urlparse', 'collections.UserDict'}
+
+
+def report_missing(mod: str, message: Optional[str] = '', traceback: str = '') -> None:
     if message:
         message = ' with error: ' + message
-    print('Failed to import {}{}; skipping it'.format(mod, message))
+    print('{}: Failed to import, skipping{}'.format(mod, message))
+    m = re.search(r"ModuleNotFoundError: No module named '([^']*)'", traceback)
+    if m:
+        missing_module = m.group(1)
+        if missing_module in PY2_MODULES:
+            print('note: Could not import %r; try --py2 for Python 2 mode' % missing_module)
+        else:
+            print('note: Could not import %r; some dependency may be missing' % missing_module)
+    print()
 
 
 def fail_missing(mod: str) -> None:

From 049a2a49043a36b23c6d05c08abb778d5a3eab9f Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 13:05:37 +0100
Subject: [PATCH 09/50] stubgen: Work around crash

---
 mypy/stubgen.py             |  3 ++-
 test-data/unit/stubgen.test | 11 +++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 2afa9c01375f..606e834ee4a7 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -336,7 +336,8 @@ def import_lines(self) -> List[str]:
                 # We can already generate the import line
                 if name in self.reverse_alias:
                     name, alias = self.reverse_alias[name], name
-                    result.append("import {} as {}\n".format(self.direct_imports[name], alias))
+                    source = self.direct_imports.get(name, 'FIXME')
+                    result.append("import {} as {}\n".format(source, alias))
                 elif name in self.reexports:
                     assert '.' not in name  # Because reexports only has nonqualified names
                     result.append("import {} as {}\n".format(name, name))
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index cf149bc3381d..2091dd6bbd4c 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1521,3 +1521,14 @@ def f():
 
 [out]
 def f() -> None: ...
+
+[case testConditionalImportAll_semanal]
+__all__ = ['cookielib']
+
+if object():
+    from http import cookiejar as cookielib
+else:
+    import cookielib
+
+[out]
+import FIXME as cookielib

From bbf626879076fc8f804b362c39bf00c0d425b361 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 13:58:54 +0100
Subject: [PATCH 10/50] stubgen: Don't fail if a class has a cyclic MRO

---
 mypy/nodes.py               |  1 +
 mypy/semanal.py             |  9 +++++----
 test-data/unit/stubgen.test | 10 ++++++++++
 3 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/mypy/nodes.py b/mypy/nodes.py
index f294705ada01..89a9f03b61bb 100644
--- a/mypy/nodes.py
+++ b/mypy/nodes.py
@@ -2269,6 +2269,7 @@ class is generic then it will be a type constructor of higher kind.
     # Used to stash the names of the mro classes temporarily between
     # deserialization and fixup. See deserialize() for why.
     _mro_refs = None  # type: Optional[List[str]]
+    bad_mro = False  # Could not construct full MRO
 
     declared_metaclass = None  # type: Optional[mypy.types.Instance]
     metaclass_type = None  # type: Optional[mypy.types.Instance]
diff --git a/mypy/semanal.py b/mypy/semanal.py
index 18e45d684f64..5c7906a200b0 100644
--- a/mypy/semanal.py
+++ b/mypy/semanal.py
@@ -1493,6 +1493,7 @@ def configure_base_classes(self,
         if not self.verify_base_classes(defn):
             # Give it an MRO consisting of just the class itself and object.
             defn.info.mro = [defn.info, self.object_type().type]
+            defn.info.bad_mro = True
             return
         self.calculate_class_mro(defn, self.object_type)
 
@@ -1598,12 +1599,12 @@ def update_metaclass(self, defn: ClassDef) -> None:
 
     def verify_base_classes(self, defn: ClassDef) -> bool:
         info = defn.info
+        cycle = False
         for base in info.bases:
             baseinfo = base.type
             if self.is_base_class(info, baseinfo):
-                self.fail('Cycle in inheritance hierarchy', defn, blocker=True)
-                # Clear bases to forcefully get rid of the cycle.
-                info.bases = []
+                self.fail('Cycle in inheritance hierarchy', defn)
+                cycle = True
             if baseinfo.fullname() == 'builtins.bool':
                 self.fail("'%s' is not a valid base class" %
                           baseinfo.name(), defn, blocker=True)
@@ -1612,7 +1613,7 @@ def verify_base_classes(self, defn: ClassDef) -> bool:
         if dup:
             self.fail('Duplicate base class "%s"' % dup.name(), defn, blocker=True)
             return False
-        return True
+        return not cycle
 
     def is_base_class(self, t: TypeInfo, s: TypeInfo) -> bool:
         """Determine if t is a base class of s (but do not use mro)."""
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index 2091dd6bbd4c..b7484929b263 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1532,3 +1532,13 @@ else:
 
 [out]
 import FIXME as cookielib
+
+[case testCycleInheritanceHierarchy_semanal]
+class A: pass
+
+class int(int, A):
+    pass
+
+[out]
+class A: ...
+class int(int, A): ...

From a3ebf3c9ab99e3e287856e2468b3d2eb883febc6 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 14:08:57 +0100
Subject: [PATCH 11/50] stubgen: Log runtime imports in verbose mode

---
 mypy/stubgen.py  | 4 +++-
 mypy/stubutil.py | 6 ++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 606e834ee4a7..bed04ec93936 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -946,7 +946,9 @@ def find_module_paths_using_imports(modules: List[str],
     """
     py_modules = []  # type: List[StubSource]
     c_modules = []  # type: List[StubSource]
-    modules = modules + list(walk_packages(packages))
+    found = list(walk_packages(packages, verbose))
+    found = remove_test_modules(found)  # We don't want to run any tests
+    modules = modules + found
     for mod in modules:
         try:
             if pyversion[0] == 2:
diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 2e09d0bf822c..5ecf24f2ac81 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -55,7 +55,7 @@ def default_py2_interpreter() -> str:
                      "please use the --python-executable option")
 
 
-def walk_packages(packages: List[str]) -> Iterator[str]:
+def walk_packages(packages: List[str], verbose: bool = False) -> Iterator[str]:
     """Iterates through all packages and sub-packages in the given list.
 
     This uses runtime imports to find both Python and C modules. For Python packages
@@ -65,6 +65,8 @@ def walk_packages(packages: List[str]) -> Iterator[str]:
     all modules imported in the package that have matching names.
     """
     for package_name in packages:
+        if verbose:
+            print('Trying to import %r for runtime introspection' % package_name)
         try:
             package = importlib.import_module(package_name)
         except Exception:
@@ -84,7 +86,7 @@ def walk_packages(packages: List[str]) -> Iterator[str]:
                                if inspect.ismodule(val)
                                and val.__name__ == package.__name__ + "." + name]
                 # Recursively iterate through the subpackages
-                for submodule in walk_packages(subpackages):
+                for submodule in walk_packages(subpackages, verbose):
                     yield submodule
             # It's a module inside a package.  There's nothing else to walk/yield.
         else:

From a3a35fe2010464cdf68d1b3f453bcd6b83a94af6 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 14:11:30 +0100
Subject: [PATCH 12/50] stubgen: More verbose output

---
 mypy/stubgen.py  | 4 ++--
 mypy/stubutil.py | 5 ++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index bed04ec93936..6c75a42b4d06 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -954,8 +954,8 @@ def find_module_paths_using_imports(modules: List[str],
             if pyversion[0] == 2:
                 result = find_module_path_and_all_py2(mod, interpreter)
             else:
-                result = find_module_path_and_all_py3(mod)
-        except CantImport as e:
+                result = find_module_path_and_all_py3(mod, verbose)
+        except CantImport e:
             tb = traceback.format_exc()
             if verbose:
                 sys.stdout.write(tb)
diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 5ecf24f2ac81..0c923600845f 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -124,13 +124,16 @@ def find_module_path_and_all_py2(module: str,
     return module_path, module_all
 
 
-def find_module_path_and_all_py3(module: str) -> Optional[Tuple[str, Optional[List[str]]]]:
+def find_module_path_and_all_py3(module: str,
+                                 verbose: bool) -> Optional[Tuple[str, Optional[List[str]]]]:
     """Find module and determine __all__ for a Python 3 module.
 
     Return None if the module is a C module. Return (module_path, __all__) if
     it is a Python module. Raise CantImport if import failed.
     """
     # TODO: Support custom interpreters.
+    if verbose:
+        print('Trying to import %r for runtime introspection' % module)
     try:
         mod = importlib.import_module(module)
     except Exception as e:

From e2873016a75500f0034a1718288ebd71a12d2ebf Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 14:26:24 +0100
Subject: [PATCH 13/50] stubgen: Skip certain special cased packages

---
 mypy/stubutil.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 0c923600845f..a78ae62725d4 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -14,6 +14,12 @@
 from typing import Optional, Tuple, List, IO, Iterator, AnyStr
 
 
+# Modules that may fail when imported, or that may have side effects.
+NOT_IMPORTABLE_MODULES = {
+    'tensorflow.tools.pip_package.setup',
+}
+
+
 class CantImport(Exception):
     def __init__(self, module: str, message: str):
         self.module = module
@@ -65,6 +71,9 @@ def walk_packages(packages: List[str], verbose: bool = False) -> Iterator[str]:
     all modules imported in the package that have matching names.
     """
     for package_name in packages:
+        if package_name in NOT_IMPORTABLE_MODULES:
+            print('%s: Skipped (blacklisted)' % package_name)
+            continue
         if verbose:
             print('Trying to import %r for runtime introspection' % package_name)
         try:
@@ -131,6 +140,9 @@ def find_module_path_and_all_py3(module: str,
     Return None if the module is a C module. Return (module_path, __all__) if
     it is a Python module. Raise CantImport if import failed.
     """
+    if module in NOT_IMPORTABLE_MODULES:
+        raise CantImport(module)
+
     # TODO: Support custom interpreters.
     if verbose:
         print('Trying to import %r for runtime introspection' % module)

From 819a77d0adf9698af9d8f567631df2f8a230e971 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 14:39:27 +0100
Subject: [PATCH 14/50] stubgen: Filter out additional things that look like
 type comments

---
 mypy/stubutil.py         | 2 +-
 mypy/test/teststubgen.py | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index a78ae62725d4..75dc9406d016 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -210,7 +210,7 @@ def remove_misplaced_type_comments(source: AnyStr) -> AnyStr:
 
     # Remove something that looks like a variable type comment but that's by itself
     # on a line, as it will often generate a parse error (unless it's # type: ignore).
-    text = re.sub(r'^[ \t]*# +type: +[a-zA-Z_].*$', '', text, flags=re.MULTILINE)
+    text = re.sub(r'^[ \t]*# +type: +["\'a-zA-Z_].*$', '', text, flags=re.MULTILINE)
 
     # Remove something that looks like a function type annotation after docstring,
     # which will result in a parse error.
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 865a09496b66..f43e09c592aa 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -300,6 +300,8 @@ def f(x):
             # type: Callable[[int], int]
             pass
 
+        #  type:  "foo"
+        #  type:  'bar'
         x = 1
         # type: int
         """
@@ -308,6 +310,8 @@ def f(x):
 
             pass
 
+
+
         x = 1
 
         """

From 02d41848a77b09a979bf6d17a3b7c90309523c40 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 15:11:49 +0100
Subject: [PATCH 15/50] stubgen: Survive inconsistent MROs

---
 mypy/semanal.py             | 15 +++++++++------
 test-data/unit/stubgen.test | 19 +++++++++++++++----
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/mypy/semanal.py b/mypy/semanal.py
index 5c7906a200b0..06792c359703 100644
--- a/mypy/semanal.py
+++ b/mypy/semanal.py
@@ -1491,9 +1491,7 @@ def configure_base_classes(self,
 
         # Calculate the MRO.
         if not self.verify_base_classes(defn):
-            # Give it an MRO consisting of just the class itself and object.
-            defn.info.mro = [defn.info, self.object_type().type]
-            defn.info.bad_mro = True
+            self.set_dummy_mro(defn.info)
             return
         self.calculate_class_mro(defn, self.object_type)
 
@@ -1521,6 +1519,11 @@ def configure_tuple_base_class(self,
 
         return base.partial_fallback
 
+    def set_dummy_mro(self, info: TypeInfo) -> None:
+        # Give it an MRO consisting of just the class itself and object.
+        info.mro = [info, self.object_type().type]
+        info.bad_mro = True
+
     def calculate_class_mro(self, defn: ClassDef,
                             obj_type: Optional[Callable[[], Instance]] = None) -> None:
         """Calculate method resolution order for a class.
@@ -1532,9 +1535,9 @@ def calculate_class_mro(self, defn: ClassDef,
         try:
             calculate_mro(defn.info, obj_type)
         except MroError:
-            self.fail_blocker('Cannot determine consistent method resolution '
-                              'order (MRO) for "%s"' % defn.name, defn)
-            defn.info.mro = []
+            self.fail('Cannot determine consistent method resolution '
+                      'order (MRO) for "%s"' % defn.name, defn)
+            self.set_dummy_mro(defn.info)
         # Allow plugins to alter the MRO to handle the fact that `def mro()`
         # on metaclasses permits MRO rewriting.
         if defn.fullname:
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index b7484929b263..0639572cff9a 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1533,12 +1533,23 @@ else:
 [out]
 import FIXME as cookielib
 
-[case testCycleInheritanceHierarchy_semanal]
-class A: pass
+[case testCannotCalculateMRO_semanal]
+class X: pass
 
-class int(int, A):
+class int(int, X):  # Cycle
     pass
 
+class A: pass
+class B(A): pass
+class C(B): pass
+class D(A, B): pass # No consistent method resolution order
+class E(C, D): pass # Ditto
+
 [out]
+class X: ...
+class int(int, X): ...
 class A: ...
-class int(int, A): ...
+class B(A): ...
+class C(B): ...
+class D(A, B): ...
+class E(C, D): ...

From 67142d6ec6dc9881ecfcdbb313b54facc2e68446 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 12 Apr 2019 16:02:09 +0100
Subject: [PATCH 16/50] stubgen: Remove message when there are only C modules

There used to be a bogus "Nothing to do" message from
`mypy.build`.
---
 mypy/stubgen.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 6c75a42b4d06..8496c18968f5 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -1033,6 +1033,8 @@ def parse_source_file(mod: StubSource, mypy_options: MypyOptions) -> None:
 def generate_asts_for_modules(py_modules: List[StubSource],
                               parse_only: bool, mypy_options: MypyOptions) -> None:
     """Use mypy to parse (and optionally analyze) source files."""
+    if not py_modules:
+        return  # Nothing to do here, but there may be C modules
     if parse_only:
         for mod in py_modules:
             parse_source_file(mod, mypy_options)

From bc3ca0855f027cec762ea2b1f865afc1c7858604 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 4 Nov 2019 11:16:40 +0000
Subject: [PATCH 17/50] Fix rebase issue

---
 mypy/options.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/mypy/options.py b/mypy/options.py
index 998f8f981661..87e3524e1515 100644
--- a/mypy/options.py
+++ b/mypy/options.py
@@ -3,20 +3,10 @@
 import pprint
 import sys
 
-<<<<<<< HEAD
-from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple
-from typing_extensions import Final
-||||||| merged common ancestors
-from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple
-MYPY = False
-if MYPY:
-    from typing_extensions import Final
-=======
 from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple, Callable, AnyStr
 MYPY = False
 if MYPY:
     from typing_extensions import Final
->>>>>>> Stubgen: Remove misplaced type comments before parsing
 
 from mypy import defaults
 from mypy.util import get_class_descriptors, replace_object_state

From b1814d99b646c15832b1e61870bd4369fd58d3e4 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 4 Nov 2019 11:43:58 +0000
Subject: [PATCH 18/50] Fix after rebase

---
 mypy/stubgen.py  | 7 ++++---
 mypy/stubutil.py | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 8496c18968f5..b506bb71cc8a 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -955,7 +955,7 @@ def find_module_paths_using_imports(modules: List[str],
                 result = find_module_path_and_all_py2(mod, interpreter)
             else:
                 result = find_module_path_and_all_py3(mod, verbose)
-        except CantImport e:
+        except CantImport as e:
             tb = traceback.format_exc()
             if verbose:
                 sys.stdout.write(tb)
@@ -1098,7 +1098,8 @@ def collect_docs_signatures(doc_dir: str) -> Tuple[Dict[str, str], Dict[str, str
     return sigs, class_sigs
 
 
-def generate_stubs(options: Options) -> None:
+def generate_stubs(options: Options,
+                   add_header: bool) -> None:
     """Main entry point for the program."""
     mypy_opts = mypy_options(options)
     py_modules, c_modules = collect_build_targets(options, mypy_opts)
@@ -1238,7 +1239,7 @@ def main() -> None:
         sys.path.insert(0, '')
 
     options = parse_options(sys.argv[1:])
-    generate_stubs(options)
+    generate_stubs(options, add_header=False)
 
 
 if __name__ == '__main__':
diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 75dc9406d016..c23a88e0b4b8 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -141,7 +141,7 @@ def find_module_path_and_all_py3(module: str,
     it is a Python module. Raise CantImport if import failed.
     """
     if module in NOT_IMPORTABLE_MODULES:
-        raise CantImport(module)
+        raise CantImport(module, '')
 
     # TODO: Support custom interpreters.
     if verbose:

From 6f4a1fe90c67375030902867de9677a932e5be98 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 4 Nov 2019 11:52:54 +0000
Subject: [PATCH 19/50] Add missing stuff

---
 mypy/stubgen.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index b506bb71cc8a..506d96856bc3 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -91,6 +91,14 @@
 from mypy.traverser import has_return_statement
 
 
+# Avoid some file names that are unnecessary or likely to cause trouble (\n for end of path).
+BLACKLIST = [
+    '/six.py\n',  # Likely vendored six; too dynamic for us to handle
+    '/vendored/',  # Vendored packages
+    '/vendor/',  # Vendored packages
+]
+
+
 class Options:
     """Represents stubgen options.
 
@@ -970,6 +978,14 @@ def find_module_paths_using_imports(modules: List[str],
     return py_modules, c_modules
 
 
+def remove_test_modules(modules: List[str]) -> List[str]:
+    """Remove anything from modules that looks like a test."""
+    return [module for module in modules
+            if (not module.endswith(('.tests', '.test'))
+                and '.tests.' not in module
+                and '.test.' not in module)]
+
+
 def find_module_paths_using_search(modules: List[str], packages: List[str],
                                    search_path: List[str],
                                    pyversion: Tuple[int, int]) -> List[StubSource]:

From 25e3258b2453b703519e1747d1553ef3e0b016ad Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Wed, 10 Apr 2019 12:06:49 +0100
Subject: [PATCH 20/50] Fix signatures generated by stubgen for various dunder
 C methods

Add `self` arguments and special casing for many additional dunder
methods.
---
 mypy/stubgenc.py | 69 ++++++++++++++++++++++++++++++------------------
 1 file changed, 44 insertions(+), 25 deletions(-)

diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py
index cbe1575379dd..f792e4f3c080 100755
--- a/mypy/stubgenc.py
+++ b/mypy/stubgenc.py
@@ -339,33 +339,52 @@ def is_skipped_attribute(attr: str) -> bool:
 
 
 def infer_method_sig(name: str) -> List[ArgSig]:
+    args = None  # type: Optional[List[ArgSig]]
     if name.startswith('__') and name.endswith('__'):
         name = name[2:-2]
         if name in ('hash', 'iter', 'next', 'sizeof', 'copy', 'deepcopy', 'reduce', 'getinitargs',
-                    'int', 'float', 'trunc', 'complex', 'bool'):
-            return []
-        if name == 'getitem':
-            return [ArgSig(name='index')]
-        if name == 'setitem':
-            return [ArgSig(name='index'),
+                    'int', 'float', 'trunc', 'complex', 'bool', 'abs', 'bytes', 'dir', 'len',
+                    'reversed', 'round', 'index', 'enter'):
+            args = []
+        elif name == 'getitem':
+            args = [ArgSig(name='index')]
+        elif name == 'setitem':
+            args = [ArgSig(name='index'),
                     ArgSig(name='object')]
-        if name in ('delattr', 'getattr'):
-            return [ArgSig(name='name')]
-        if name == 'setattr':
-            return [ArgSig(name='name'),
+        elif name in ('delattr', 'getattr'):
+            args = [ArgSig(name='name')]
+        elif name == 'setattr':
+            args = [ArgSig(name='name'),
                     ArgSig(name='value')]
-        if name == 'getstate':
-            return []
-        if name == 'setstate':
-            return [ArgSig(name='state')]
-        if name in ('eq', 'ne', 'lt', 'le', 'gt', 'ge',
-                    'add', 'radd', 'sub', 'rsub', 'mul', 'rmul',
-                    'mod', 'rmod', 'floordiv', 'rfloordiv', 'truediv', 'rtruediv',
-                    'divmod', 'rdivmod', 'pow', 'rpow'):
-            return [ArgSig(name='other')]
-        if name in ('neg', 'pos'):
-            return []
-    return [
-        ArgSig(name='*args'),
-        ArgSig(name='**kwargs')
-    ]
+        elif name == 'getstate':
+            args = []
+        elif name == 'setstate':
+            args = [ArgSig(name='state')]
+        elif name in ('eq', 'ne', 'lt', 'le', 'gt', 'ge',
+                      'add', 'radd', 'sub', 'rsub', 'mul', 'rmul',
+                      'mod', 'rmod', 'floordiv', 'rfloordiv', 'truediv', 'rtruediv',
+                      'divmod', 'rdivmod', 'pow', 'rpow',
+                      'xor', 'rxor', 'or', 'ror', 'and', 'rand', 'lshift', 'rlshift',
+                      'rshift', 'rrshift',
+                      'contains', 'delitem',
+                      'iadd', 'iand', 'ifloordiv', 'ilshift', 'imod', 'imul', 'ior',
+                      'ipow', 'irshift', 'isub', 'itruediv', 'ixor'):
+            args = [ArgSig(name='other')]
+        elif name in ('neg', 'pos', 'invert'):
+            args = []
+        elif name == 'get':
+            args = [ArgSig(name='instance'),
+                    ArgSig(name='owner')]
+        elif name == 'set':
+            args = [ArgSig(name='instance'),
+                    ArgSig(name='value')]
+        elif name == 'reduce_ex':
+            args = [ArgSig(name='protocol')]
+        elif name == 'exit':
+            args = [ArgSig(name='type'),
+                    ArgSig(name='value'),
+                    ArgSig(name='traceback')]
+    if args is None:
+        args = [ArgSig(name='*args'),
+                ArgSig(name='**kwargs')]
+    return [ArgSig(name='self')] + args

From c9d2987a3adff8741a027a2d60aee7574ad8a8c6 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 4 Nov 2019 12:20:38 +0000
Subject: [PATCH 21/50] Add missing line

---
 mypy/stubgen.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 506d96856bc3..8597df92d9df 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -938,6 +938,8 @@ def collect_build_targets(options: Options, mypy_opts: MypyOptions) -> Tuple[Lis
         py_modules = [StubSource(m.module, m.path) for m in source_list]
         c_modules = []
 
+    py_modules = remove_blacklisted_modules(py_modules)
+
     return py_modules, c_modules
 
 

From 8777b5f3cf87cb72b35923d95f740b7c5326984b Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Tue, 9 Apr 2019 14:00:06 +0100
Subject: [PATCH 22/50] Ignore unreachable code in stubgen to avoid crashes

This may also prevent duplicate definitions from being generated.
However, some definitions that are needed on certain platforms
only or on Python 2 only, for example, may be omitted. This is
arguably less bad than crashing.
---
 mypy/stubgen.py             |  8 +++++++-
 test-data/unit/stubgen.test | 20 ++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 8597df92d9df..cdcc388b1377 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -69,7 +69,7 @@
     Expression, IntExpr, UnaryExpr, StrExpr, BytesExpr, NameExpr, FloatExpr, MemberExpr,
     TupleExpr, ListExpr, ComparisonExpr, CallExpr, IndexExpr, EllipsisExpr,
     ClassDef, MypyFile, Decorator, AssignmentStmt, TypeInfo,
-    IfStmt, ImportAll, ImportFrom, Import, FuncDef, FuncBase, TempNode,
+    IfStmt, ReturnStmt, ImportAll, ImportFrom, Import, FuncDef, FuncBase, TempNode, Block,
     ARG_POS, ARG_STAR, ARG_STAR2, ARG_NAMED, ARG_NAMED_OPT
 )
 from mypy.stubgenc import generate_stub_for_c_module
@@ -576,6 +576,12 @@ def get_base_types(self, cdef: ClassDef) -> List[str]:
                 base_types.append(base.accept(p))
         return base_types
 
+    def visit_block(self, o: Block) -> None:
+        # Unreachable statements may be partially uninitialized and that may
+        # cause trouble.
+        if not o.is_unreachable:
+            super().visit_block(o)
+
     def visit_assignment_stmt(self, o: AssignmentStmt) -> None:
         foundl = []
 
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index 0639572cff9a..f44eaf17f4ed 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1553,3 +1553,23 @@ class B(A): ...
 class C(B): ...
 class D(A, B): ...
 class E(C, D): ...
+
+[case testUnreachableCode_semanal]
+MYPY = False
+class A: pass
+if MYPY:
+    class C(A):
+        def f(self) -> None: pass
+else:
+    def f(i):
+        return i
+
+    class C(A):
+        def g(self) -> None: pass
+[out]
+MYPY: bool
+
+class A: ...
+
+class C(A):
+    def f(self) -> None: ...

From 5ccd902b27c822eb8dd2c8dcd722b53abd27b6ba Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 4 Nov 2019 12:26:21 +0000
Subject: [PATCH 23/50] Fix C dunder method inference tests

---
 mypy/test/teststubgen.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index f43e09c592aa..44267f3c4ad2 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -477,25 +477,28 @@ def add_file(self, path: str, result: List[str]) -> None:
             result.extend(file.read().splitlines())
 
 
+self_arg = ArgSig(name='self')
+
+
 class StubgencSuite(Suite):
     def test_infer_hash_sig(self) -> None:
-        assert_equal(infer_method_sig('__hash__'), [])
+        assert_equal(infer_method_sig('__hash__'), [self_arg])
 
     def test_infer_getitem_sig(self) -> None:
-        assert_equal(infer_method_sig('__getitem__'), [ArgSig(name='index')])
+        assert_equal(infer_method_sig('__getitem__'), [self_arg, ArgSig(name='index')])
 
     def test_infer_setitem_sig(self) -> None:
         assert_equal(infer_method_sig('__setitem__'),
-                     [ArgSig(name='index'), ArgSig(name='object')])
+                     [self_arg, ArgSig(name='index'), ArgSig(name='object')])
 
     def test_infer_binary_op_sig(self) -> None:
         for op in ('eq', 'ne', 'lt', 'le', 'gt', 'ge',
                    'add', 'radd', 'sub', 'rsub', 'mul', 'rmul'):
-            assert_equal(infer_method_sig('__%s__' % op), [ArgSig(name='other')])
+            assert_equal(infer_method_sig('__%s__' % op), [self_arg, ArgSig(name='other')])
 
     def test_infer_unary_op_sig(self) -> None:
         for op in ('neg', 'pos'):
-            assert_equal(infer_method_sig('__%s__' % op), [])
+            assert_equal(infer_method_sig('__%s__' % op), [self_arg])
 
     def test_generate_c_type_stub_no_crash_for_object(self) -> None:
         output = []  # type: List[str]

From 09f1b132bae69a23744f81a77e81560af647f2ec Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Wed, 10 Apr 2019 11:32:43 +0100
Subject: [PATCH 24/50] Preserve @abstractproperty in stubgen

Also refactor things a little to make it easier to make the change.
---
 mypy/stubgen.py             | 105 ++++++++++++++++++++----------------
 test-data/unit/stubgen.test |  33 ++++++++++++
 2 files changed, 93 insertions(+), 45 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index cdcc388b1377..4310e09c5cc2 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -480,51 +480,66 @@ def visit_decorator(self, o: Decorator) -> None:
         is_abstract = False
         for decorator in o.original_decorators:
             if isinstance(decorator, NameExpr):
-                if decorator.name in ('property',
-                                      'staticmethod',
-                                      'classmethod'):
-                    self.add_decorator('%s@%s\n' % (self._indent, decorator.name))
-                elif self.import_tracker.module_for.get(decorator.name) in ('asyncio',
-                                                                            'asyncio.coroutines',
-                                                                            'types'):
-                    self.add_coroutine_decorator(o.func, decorator.name, decorator.name)
-                elif (self.import_tracker.module_for.get(decorator.name) == 'abc' and
-                      (decorator.name == 'abstractmethod' or
-                       self.import_tracker.reverse_alias.get(decorator.name) == 'abstractmethod')):
-                    self.add_decorator('%s@%s\n' % (self._indent, decorator.name))
-                    self.import_tracker.require_name(decorator.name)
+                if self.process_name_expr_decorator(decorator, o):
                     is_abstract = True
             elif isinstance(decorator, MemberExpr):
-                if decorator.name == 'setter' and isinstance(decorator.expr, NameExpr):
-                    self.add_decorator('%s@%s.setter\n' % (self._indent, decorator.expr.name))
-                elif (isinstance(decorator.expr, NameExpr) and
-                      (decorator.expr.name == 'abc' or
-                       self.import_tracker.reverse_alias.get('abc')) and
-                      decorator.name == 'abstractmethod'):
-                    self.import_tracker.require_name(decorator.expr.name)
-                    self.add_decorator('%s@%s.%s\n' %
-                                       (self._indent, decorator.expr.name, decorator.name))
+                if self.process_member_expr_decorator(decorator, o):
                     is_abstract = True
-                elif decorator.name == 'coroutine':
-                    if (isinstance(decorator.expr, MemberExpr) and
-                        decorator.expr.name == 'coroutines' and
-                        isinstance(decorator.expr.expr, NameExpr) and
-                            (decorator.expr.expr.name == 'asyncio' or
-                             self.import_tracker.reverse_alias.get(decorator.expr.expr.name) ==
-                                'asyncio')):
-                        self.add_coroutine_decorator(o.func,
-                                                     '%s.coroutines.coroutine' %
-                                                     (decorator.expr.expr.name,),
-                                                     decorator.expr.expr.name)
-                    elif (isinstance(decorator.expr, NameExpr) and
-                          (decorator.expr.name in ('asyncio', 'types') or
-                           self.import_tracker.reverse_alias.get(decorator.expr.name) in
-                            ('asyncio', 'asyncio.coroutines', 'types'))):
-                        self.add_coroutine_decorator(o.func,
-                                                     decorator.expr.name + '.coroutine',
-                                                     decorator.expr.name)
         self.visit_func_def(o.func, is_abstract=is_abstract)
 
+    def process_name_expr_decorator(self, expr: NameExpr, context: Decorator) -> bool:
+        is_abstract = False
+        name = expr.name
+        if name in ('property', 'staticmethod', 'classmethod'):
+            self.add_decorator(name)
+        elif self.import_tracker.module_for.get(name) in ('asyncio',
+                                                          'asyncio.coroutines',
+                                                          'types'):
+            self.add_coroutine_decorator(context.func, name, name)
+        elif any(self.refers_to_fullname(name, target)
+                 for target in ('abc.abstractmethod', 'abc.abstractproperty')):
+            self.add_decorator(name)
+            self.import_tracker.require_name(name)
+            is_abstract = True
+        return is_abstract
+
+    def refers_to_fullname(self, name: str, fullname: str) -> bool:
+        module, short = fullname.rsplit('.', 1)
+        return (self.import_tracker.module_for.get(name) == module and
+                (name == short or
+                 self.import_tracker.reverse_alias.get(name) == short))
+
+    def process_member_expr_decorator(self, expr: MemberExpr, context: Decorator) -> bool:
+        is_abstract = False
+        if expr.name == 'setter' and isinstance(expr.expr, NameExpr):
+            self.add_decorator('%s.setter' % expr.expr.name)
+        elif (isinstance(expr.expr, NameExpr) and
+              (expr.expr.name == 'abc' or
+               self.import_tracker.reverse_alias.get('abc')) and
+              expr.name in ('abstractmethod', 'abstractproperty')):
+            self.import_tracker.require_name(expr.expr.name)
+            self.add_decorator('%s.%s' % (expr.expr.name, expr.name))
+            is_abstract = True
+        elif expr.name == 'coroutine':
+            if (isinstance(expr.expr, MemberExpr) and
+                expr.expr.name == 'coroutines' and
+                isinstance(expr.expr.expr, NameExpr) and
+                    (expr.expr.expr.name == 'asyncio' or
+                     self.import_tracker.reverse_alias.get(expr.expr.expr.name) ==
+                        'asyncio')):
+                self.add_coroutine_decorator(context.func,
+                                             '%s.coroutines.coroutine' %
+                                             (expr.expr.expr.name,),
+                                             expr.expr.expr.name)
+            elif (isinstance(expr.expr, NameExpr) and
+                  (expr.expr.name in ('asyncio', 'types') or
+                   self.import_tracker.reverse_alias.get(expr.expr.name) in
+                    ('asyncio', 'asyncio.coroutines', 'types'))):
+                self.add_coroutine_decorator(context.func,
+                                             expr.expr.name + '.coroutine',
+                                             expr.expr.name)
+        return is_abstract
+
     def visit_class_def(self, o: ClassDef) -> None:
         sep = None  # type: Optional[int]
         if not self._indent and self._state != EMPTY:
@@ -772,8 +787,10 @@ def add(self, string: str) -> None:
         """Add text to generated stub."""
         self._output.append(string)
 
-    def add_decorator(self, string: str) -> None:
-        self._decorators.append(string)
+    def add_decorator(self, name: str) -> None:
+        if not self._indent and self._state not in (EMPTY, FUNC):
+            self._decorators.append('\n')
+        self._decorators.append('%s@%s\n' % (self._indent, name))
 
     def clear_decorators(self) -> None:
         self._decorators.clear()
@@ -792,9 +809,7 @@ def add_import_line(self, line: str) -> None:
 
     def add_coroutine_decorator(self, func: FuncDef, name: str, require_name: str) -> None:
         func.is_awaitable_coroutine = True
-        if not self._indent and self._state not in (EMPTY, FUNC):
-            self.add_decorator('\n')
-        self.add_decorator('%s@%s\n' % (self._indent, name))
+        self.add_decorator(name)
         self.import_tracker.require_name(require_name)
 
     def output(self) -> str:
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index f44eaf17f4ed..184bf47b9815 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1573,3 +1573,36 @@ class A: ...
 
 class C(A):
     def f(self) -> None: ...
+
+[case testAbstractProperty1_semanal]
+import other
+import abc
+
+class A:
+    @abc.abstractproperty
+    def x(self): pass
+
+[out]
+import abc
+from typing import Any
+
+class A(metaclass=abc.ABCMeta):
+    @abc.abstractproperty
+    def x(self) -> Any: ...
+
+[case testAbstractProperty2_semanal]
+import other
+from abc import abstractproperty
+
+class A:
+    @abstractproperty
+    def x(self): pass
+
+[out]
+import abc
+from abc import abstractproperty
+from typing import Any
+
+class A(metaclass=abc.ABCMeta):
+    @abstractproperty
+    def x(self) -> Any: ...

From a9721edea955b1f9f786ea9ad8c146dd3f910e59 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Wed, 10 Apr 2019 13:44:20 +0100
Subject: [PATCH 25/50] Stubgen: Avoid name clashes between typing.Any and
 class Any etc.

Also support class named Optional.
---
 mypy/stubgen.py             | 54 ++++++++++++++++++++++++++++++++-----
 test-data/unit/stubgen.test | 26 ++++++++++++++++++
 2 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 4310e09c5cc2..c5b6648e4bd9 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -358,7 +358,33 @@ def import_lines(self) -> List[str]:
         return result
 
 
+def find_defined_names(file: MypyFile) -> Set[str]:
+    finder = DefinitionFinder()
+    file.accept(finder)
+    return finder.names
+
+
+class DefinitionFinder(mypy.traverser.TraverserVisitor):
+    """Find names of things defined at the top level of a module."""
+
+    # TODO: Assignment statements etc.
+
+    def __init__(self) -> None:
+        # Short names of things defined at the top level.
+        self.names = set()  # type: Set[str]
+
+    def visit_class_def(self, o: ClassDef) -> None:
+        # Don't recurse into classes, as we only keep track of top-level definitions.
+        self.names.add(o.name)
+
+    def visit_func_def(self, o: FuncDef) -> None:
+        # Don't recurse, as we only keep track of top-level definitions.
+        self.names.add(o.name())
+
+
 class StubGenerator(mypy.traverser.TraverserVisitor):
+    """Generate stub text from a mypy AST."""
+
     def __init__(self, _all_: Optional[List[str]], pyversion: Tuple[int, int],
                  include_private: bool = False, analyzed: bool = False) -> None:
         # Best known value of __all__.
@@ -380,13 +406,20 @@ def __init__(self, _all_: Optional[List[str]], pyversion: Tuple[int, int],
         self.analyzed = analyzed
         # Add imports that could be implicitly generated
         self.import_tracker.add_import_from("collections", [("namedtuple", None)])
-        typing_imports = "Any Optional TypeVar".split()
-        self.import_tracker.add_import_from("typing", [(t, None) for t in typing_imports])
         # Names in __all__ are required
         for name in _all_ or ():
             self.import_tracker.reexport(name)
+        self.defined_names = set()  # type: Set[str]
 
     def visit_mypy_file(self, o: MypyFile) -> None:
+        self.defined_names = find_defined_names(o)
+        typing_imports = ["Any", "Optional", "TypeVar"]
+        for t in typing_imports:
+            if t not in self.defined_names:
+                alias = None
+            else:
+                alias = '_' + t
+            self.import_tracker.add_import_from("typing", [(t, alias)])
         super().visit_mypy_file(o)
         undefined_names = [name for name in self._all_ or []
                            if name not in self._toplevel_names]
@@ -434,7 +467,7 @@ def visit_func_def(self, o: FuncDef, is_abstract: bool = False) -> None:
                     and not is_self_arg
                     and not is_cls_arg):
                 self.add_typing_import("Any")
-                annotation = ": Any"
+                annotation = ": {}".format(self.typing_name("Any"))
             elif annotated_type and not is_self_arg:
                 annotation = ": {}".format(self.print_annotation(annotated_type))
             else:
@@ -462,7 +495,7 @@ def visit_func_def(self, o: FuncDef, is_abstract: bool = False) -> None:
             retname = self.print_annotation(o.unanalyzed_type.ret_type)
         elif isinstance(o, FuncDef) and o.is_abstract:
             # Always assume abstract methods return Any unless explicitly annotated.
-            retname = 'Any'
+            retname = self.typing_name('Any')
             self.add_typing_import("Any")
         elif o.name() == '__init__' or not has_return_statement(o) and not is_abstract:
             retname = 'None'
@@ -795,11 +828,19 @@ def add_decorator(self, name: str) -> None:
     def clear_decorators(self) -> None:
         self._decorators.clear()
 
+    def typing_name(self, name: str) -> str:
+        if name in self.defined_names:
+            # Avoid name clash between name from typing and a name defined in stub.
+            return '_' + name
+        else:
+            return name
+
     def add_typing_import(self, name: str) -> None:
         """Add a name to be imported from typing, unless it's imported already.
 
         The import will be internal to the stub.
         """
+        name = self.typing_name(name)
         self.import_tracker.require_name(name)
 
     def add_import_line(self, line: str) -> None:
@@ -867,9 +908,10 @@ def get_str_type_of_node(self, rvalue: Expression,
                 isinstance(rvalue, NameExpr) and rvalue.name == 'None':
             self.add_typing_import('Optional')
             self.add_typing_import('Any')
-            return 'Optional[Any]'
+            return '{}[{}]'.format(self.typing_name('Optional'),
+                                   self.typing_name('Any'))
         self.add_typing_import('Any')
-        return 'Any'
+        return self.typing_name('Any')
 
     def print_annotation(self, t: Type) -> str:
         printer = AnnotationPrinter(self)
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index 184bf47b9815..1961309ff8e3 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1606,3 +1606,29 @@ from typing import Any
 class A(metaclass=abc.ABCMeta):
     @abstractproperty
     def x(self) -> Any: ...
+
+[case testClassWithNameAnyOrOptional]
+def f(x=object()):
+    return 1
+
+def g(x=None): pass
+
+x = g()
+
+class Any:
+    pass
+
+def Optional():
+    return 0
+
+[out]
+from typing import Any as _Any, Optional as _Optional
+
+def f(x: _Any = ...): ...
+def g(x: _Optional[_Any] = ...) -> None: ...
+
+x: _Any
+
+class Any: ...
+
+def Optional(): ...

From 6cefdc40663d1f8708a32c3b5031468ae1be5533 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Wed, 10 Apr 2019 15:28:59 +0100
Subject: [PATCH 26/50] Stubgen: special case certain names to be exported

---
 mypy/stubgen.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index c5b6648e4bd9..7cfaae089f55 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -98,6 +98,13 @@
     '/vendor/',  # Vendored packages
 ]
 
+# Special-cased names that are implicitly exported from the stub (from m import y as y).
+EXTRA_EXPORTED = {
+    'pyasn1_modules.rfc2437.univ',
+    'pyasn1_modules.rfc2459.char',
+    'pyasn1_modules.rfc2459.univ',
+}
+
 
 class Options:
     """Represents stubgen options.
@@ -412,6 +419,7 @@ def __init__(self, _all_: Optional[List[str]], pyversion: Tuple[int, int],
         self.defined_names = set()  # type: Set[str]
 
     def visit_mypy_file(self, o: MypyFile) -> None:
+        self.module = o.fullname()
         self.defined_names = find_defined_names(o)
         typing_imports = ["Any", "Optional", "TypeVar"]
         for t in typing_imports:
@@ -758,9 +766,14 @@ def visit_import_all(self, o: ImportAll) -> None:
 
     def visit_import_from(self, o: ImportFrom) -> None:
         exported_names = set()  # type: Set[str]
-        self.import_tracker.add_import_from('.' * o.relative + o.id, o.names)
-        self._vars[-1].extend(alias or name for name, alias in o.names)
-        for name, alias in o.names:
+        import_names = []
+        for name, as_name in o.names:
+            if as_name is None and (self.module + '.' + name) in EXTRA_EXPORTED:
+                as_name = name
+            import_names.append((name, as_name))
+        self.import_tracker.add_import_from('.' * o.relative + o.id, import_names)
+        self._vars[-1].extend(alias or name for name, alias in import_names)
+        for name, alias in import_names:
             self.record_name(alias or name)
 
         if self._all_:

From 95ab57c03ba63d83225ffccb64c340c17187aaa0 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 4 Nov 2019 12:50:33 +0000
Subject: [PATCH 27/50] Fix None errors

---
 mypy/stubgen.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 7cfaae089f55..2e10ebc143a6 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -768,7 +768,7 @@ def visit_import_from(self, o: ImportFrom) -> None:
         exported_names = set()  # type: Set[str]
         import_names = []
         for name, as_name in o.names:
-            if as_name is None and (self.module + '.' + name) in EXTRA_EXPORTED:
+            if as_name is None and self.module and (self.module + '.' + name) in EXTRA_EXPORTED:
                 as_name = name
             import_names.append((name, as_name))
         self.import_tracker.add_import_from('.' * o.relative + o.id, import_names)

From 156fe17e57d44546594c6293a8403fa85ed43854 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Wed, 10 Apr 2019 15:37:44 +0100
Subject: [PATCH 28/50] Add some docstrings and comments to stubgen tests

---
 mypy/test/teststubgen.py    | 29 +++++++++++++++++++++++++++--
 test-data/unit/stubgen.test |  2 ++
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 44267f3c4ad2..2c88e5c001b4 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -27,6 +27,8 @@
 
 
 class StubgenCmdLineSuite(Suite):
+    """Test cases for processing command-line options and finding files."""
+
     def test_files_found(self) -> None:
         current = os.getcwd()
         with tempfile.TemporaryDirectory() as tmp:
@@ -112,6 +114,8 @@ def test_walk_packages(self) -> None:
 
 
 class StubgenUtilSuite(Suite):
+    """Unit tests for stubgen utility functions."""
+
     def test_parse_signature(self) -> None:
         self.assert_parse_signature('func()', ('func', [], []))
 
@@ -420,6 +424,22 @@ def test_common_dir_prefix(self) -> None:
 
 
 class StubgenPythonSuite(DataSuite):
+    """Data-driven end-to-end test cases that generate stub files.
+
+    You can use these magic test case name suffixes:
+
+    *_semanal
+        Run semantic analysis (slow as this uses real stubs -- only use
+        when necessary)
+    *_import
+        Import module and perform runtime introspection (in the current
+        process!)
+
+    You can use this magic comment:
+
+        # flags: --some-stubgen-option ...
+    """
+
     required_out_section = True
     base_path = '.'
     files = ['stubgen.test']
@@ -429,8 +449,8 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             self.run_case_inner(testcase)
 
     def run_case_inner(self, testcase: DataDrivenTestCase) -> None:
-        extra = []
-        mods = []
+        extra = []  # Extra command-line args
+        mods = []  # Module names to process
         source = '\n'.join(testcase.input)
         for file, content in testcase.files + [('./main.py', source)]:
             mod = os.path.basename(file)[:-3]
@@ -481,6 +501,11 @@ def add_file(self, path: str, result: List[str]) -> None:
 
 
 class StubgencSuite(Suite):
+    """Unit tests for stub generation from C modules using introspection.
+
+    Note that these don't cover a lot!
+    """
+
     def test_infer_hash_sig(self) -> None:
         assert_equal(infer_method_sig('__hash__'), [self_arg])
 
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index 1961309ff8e3..5176f0c95543 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1,3 +1,5 @@
+-- Test cases for stubgen that generate stubs from Python code
+
 [case testEmptyFile]
 [out]
 

From 9b802e3e65c064244dab8f9d5bf20d490367cc04 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Wed, 10 Apr 2019 17:27:34 +0100
Subject: [PATCH 29/50] Stubgen: Generate exports for imported names that
 aren't referenced

---
 mypy/stubgen.py             | 60 +++++++++++++++++++++++++++++++++++--
 mypy/test/teststubgen.py    | 24 ++++++++++++---
 test-data/unit/stubgen.test | 56 ++++++++++++++++++++++++----------
 3 files changed, 119 insertions(+), 21 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 2e10ebc143a6..2128fa7a9aa0 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -62,6 +62,7 @@
 import mypy.parse
 import mypy.errors
 import mypy.traverser
+import mypy.mixedtraverser
 import mypy.util
 from mypy import defaults
 from mypy.modulefinder import FindModuleCache, SearchPaths, BuildSource, default_lib_path
@@ -81,8 +82,7 @@
 from mypy.stubdoc import parse_all_signatures, find_unique_signatures, Sig
 from mypy.options import Options as MypyOptions
 from mypy.types import (
-    Type, TypeStrVisitor, CallableType, AnyType,
-    UnboundType, NoneType, TupleType, TypeList,
+    Type, TypeStrVisitor, CallableType, UnboundType, NoneType, TupleType, TypeList, Instance, AnyType
 )
 from mypy.visitor import NodeVisitor
 from mypy.find_sources import create_source_list, InvalidSourceList
@@ -389,6 +389,51 @@ def visit_func_def(self, o: FuncDef) -> None:
         self.names.add(o.name())
 
 
+def find_referenced_names(file: MypyFile) -> Set[str]:
+    finder = ReferenceFinder()
+    file.accept(finder)
+    return finder.refs
+
+
+class ReferenceFinder(mypy.mixedtraverser.MixedTraverserVisitor):
+    """Find all name references (both local and global)."""
+
+    # TODO: Filter out local variable and class attribute references
+
+    def __init__(self) -> None:
+        # Short names of things defined at the top level.
+        self.refs = set()  # type: Set[str]
+
+    def visit_block(self, block: Block) -> None:
+        if not block.is_unreachable:
+            super().visit_block(block)
+
+    def visit_name_expr(self, e: NameExpr) -> None:
+        self.refs.add(e.name)
+
+    def visit_instance(self, t: Instance) -> None:
+        self.add_ref(t.type.fullname())
+        super().visit_instance(t)
+
+    def visit_unbound_type(self, t: UnboundType) -> None:
+        if t.name:
+            self.add_ref(t.name)
+
+    def visit_tuple_type(self, t: TupleType) -> None:
+        # Ignore fallback
+        for item in t.items:
+            item.accept(self)
+
+    def visit_callable_type(self, t: CallableType) -> None:
+        # Ignore fallback
+        for arg in t.arg_types:
+            arg.accept(self)
+        t.ret_type.accept(self)
+
+    def add_ref(self, fullname: str) -> None:
+        self.refs.add(fullname.split('.')[-1])
+
+
 class StubGenerator(mypy.traverser.TraverserVisitor):
     """Generate stub text from a mypy AST."""
 
@@ -421,6 +466,7 @@ def __init__(self, _all_: Optional[List[str]], pyversion: Tuple[int, int],
     def visit_mypy_file(self, o: MypyFile) -> None:
         self.module = o.fullname()
         self.defined_names = find_defined_names(o)
+        self.referenced_names = find_referenced_names(o)
         typing_imports = ["Any", "Optional", "TypeVar"]
         for t in typing_imports:
             if t not in self.defined_names:
@@ -768,7 +814,17 @@ def visit_import_from(self, o: ImportFrom) -> None:
         exported_names = set()  # type: Set[str]
         import_names = []
         for name, as_name in o.names:
+            exported = False
             if as_name is None and self.module and (self.module + '.' + name) in EXTRA_EXPORTED:
+                exported = True
+            if (as_name is None and name not in self.referenced_names and not self._all_
+                    and o.id not in ('abc', 'typing', 'asyncio')):
+                # An imported name that is never referenced in the module is assumed to be
+                # exported, unless there is an explicit __all__. Note that we need to special
+                # case 'abc' since some references are deleted during semantic analysis.
+                exported = True
+            if exported:
+                self.import_tracker.reexport(name)
                 as_name = name
             import_names.append((name, as_name))
         self.import_tracker.add_import_from('.' * o.relative + o.id, import_names)
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 2c88e5c001b4..7cda3608ce4f 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -435,9 +435,13 @@ class StubgenPythonSuite(DataSuite):
         Import module and perform runtime introspection (in the current
         process!)
 
-    You can use this magic comment:
+    You can use these magic comments:
 
-        # flags: --some-stubgen-option ...
+    # flags: --some-stubgen-option ...
+        Specify custom stubgen options
+
+    # modules: module1 module2 ...
+        Specify which modules to output (by default only 'main')
     """
 
     required_out_section = True
@@ -460,6 +464,7 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None:
                 f.write(content)
 
         options = self.parse_flags(source, extra)
+        modules = self.parse_modules(source)
         out_dir = 'out'
         try:
             try:
@@ -469,7 +474,9 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None:
                     options.parse_only = True
                 generate_stubs(options, add_header=False)
                 a = []  # type: List[str]
-                self.add_file(os.path.join(out_dir, 'main.pyi'), a)
+                for module in modules:
+                    fnam = os.path.join(out_dir, '{}.pyi'.format(module))
+                    self.add_file(fnam, a, header=len(modules) > 1)
             except CompileError as e:
                 a = e.messages
             assert_string_arrays_equal(testcase.output, a,
@@ -492,7 +499,16 @@ def parse_flags(self, program_text: str, extra: List[str]) -> Options:
             options.quiet = True
         return options
 
-    def add_file(self, path: str, result: List[str]) -> None:
+    def parse_modules(self, program_text: str) -> List[str]:
+        modules = re.search('# modules: (.*)$', program_text, flags=re.MULTILINE)
+        if modules:
+            return modules.group(1).split()
+        else:
+            return ['main']
+
+    def add_file(self, path: str, result: List[str], header: bool) -> None:
+        if header:
+            result.append('# {}'.format(os.path.basename(path)))
         with open(path, encoding='utf8') as file:
             result.extend(file.read().splitlines())
 
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index 5176f0c95543..9e50eb3a32b5 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -582,8 +582,9 @@ from collections import namedtuple
 X = namedtuple('X', ['a', 'b'])
 
 [case testNamedtupleAltSyntax]
-from collections import namedtuple, x
+from collections import namedtuple, xx
 X = namedtuple('X', 'a b')
+xx
 [out]
 from collections import namedtuple
 
@@ -615,10 +616,11 @@ _X = namedtuple('_X', ['a', 'b'])
 class Y(_X): ...
 
 [case testNamedtupleAltSyntaxFieldsTuples]
-from collections import namedtuple, x
+from collections import namedtuple, xx
 X = namedtuple('X', ())
 Y = namedtuple('Y', ('a',))
 Z = namedtuple('Z', ('a', 'b', 'c', 'd', 'e'))
+xx
 [out]
 from collections import namedtuple
 
@@ -703,15 +705,16 @@ class A:
 [case testExportViaRelativeImport]
 from .api import get
 [out]
-from .api import get
+from .api import get as get
 
 [case testExportViaRelativePackageImport]
 from .packages.urllib3.contrib import parse
 [out]
-from .packages.urllib3.contrib import parse
+from .packages.urllib3.contrib import parse as parse
 
 [case testNoExportViaRelativeImport]
 from . import get
+get()
 [out]
 
 [case testRelativeImportAndBase]
@@ -756,35 +759,35 @@ class A:
 
 [case testAnnotationImportsFrom]
 import foo
-from collection import defaultdict
+from collections import defaultdict
 x: defaultdict
 
 [out]
-from collection import defaultdict
+from collections import defaultdict
 
 x: defaultdict
 
 [case testAnnotationImports]
 import foo
-import collection
-x: collection.defaultdict
+import collections
+x: collections.defaultdict
 
 [out]
-import collection
+import collections
 
-x: collection.defaultdict
+x: collections.defaultdict
 
 
 [case testAnnotationImports]
 from typing import List
-import collection
-x: List[collection.defaultdict]
+import collections
+x: List[collections.defaultdict]
 
 [out]
-import collection
+import collections
 from typing import List
 
-x: List[collection.defaultdict]
+x: List[collections.defaultdict]
 
 
 [case testAnnotationFwRefs]
@@ -1429,7 +1432,7 @@ class A(metaclass=abc.ABCMeta):
     def meth(self): ...
 
 [case testABCMeta_semanal]
-from base import base
+from base import Base
 from abc import abstractmethod
 
 class C(Base):
@@ -1447,6 +1450,7 @@ class Base(metaclass=ABCMeta):
 [out]
 import abc
 from abc import abstractmethod
+from base import Base
 from typing import Any
 
 class C(Base, metaclass=abc.ABCMeta):
@@ -1634,3 +1638,25 @@ x: _Any
 class Any: ...
 
 def Optional(): ...
+
+[case testExportedNameImported]
+# modules: main a b
+from a import C
+
+class D(C): pass
+
+[file a.py]
+from b import C
+
+[file b.py]
+class C: pass
+
+[out]
+# main.pyi
+from a import C
+
+class D(C): ...
+# a.pyi
+from b import C as C
+# b.pyi
+class C: ...

From d7b29426af2a85587f9c06f8408d641cef0020f8 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Wed, 10 Apr 2019 17:32:46 +0100
Subject: [PATCH 30/50] Stubgen: allow special casing internal definitions to
 be exported

---
 mypy/stubgen.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 2128fa7a9aa0..acf71a71ba1a 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -103,6 +103,7 @@
     'pyasn1_modules.rfc2437.univ',
     'pyasn1_modules.rfc2459.char',
     'pyasn1_modules.rfc2459.univ',
+    'elasticsearch.client.utils._make_path',
 }
 
 
@@ -485,7 +486,7 @@ def visit_mypy_file(self, o: MypyFile) -> None:
                 self.add('#   %s\n' % name)
 
     def visit_func_def(self, o: FuncDef, is_abstract: bool = False) -> None:
-        if self.is_private_name(o.name()):
+        if self.is_private_name(o.name(), o.fullname()):
             return
         if self.is_not_in_all(o.name()):
             return
@@ -562,7 +563,7 @@ def visit_func_def(self, o: FuncDef, is_abstract: bool = False) -> None:
         self._state = FUNC
 
     def visit_decorator(self, o: Decorator) -> None:
-        if self.is_private_name(o.func.name()):
+        if self.is_private_name(o.func.name(), o.func.fullname()):
             return
         is_abstract = False
         for decorator in o.original_decorators:
@@ -939,9 +940,11 @@ def is_not_in_all(self, name: str) -> bool:
             return self.is_top_level() and name not in self._all_
         return False
 
-    def is_private_name(self, name: str) -> bool:
+    def is_private_name(self, name: str, fullname: Optional[str] = None) -> bool:
         if self._include_private:
             return False
+        if fullname in EXTRA_EXPORTED:
+            return False
         return name.startswith('_') and (not name.endswith('__')
                                          or name in ('__all__',
                                                      '__author__',

From b739314413fa80a92212fb602cb114bc67d02dca Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Wed, 10 Apr 2019 18:34:14 +0100
Subject: [PATCH 31/50] Stubgen: translate imports from vendored six to use
 real six

Since these are aliases, this is generally okay, and reduces
duplication.
---
 mypy/stubgen.py             | 30 ++++++++++++++++++++++++++----
 test-data/unit/stubgen.test | 17 +++++++++++++++++
 2 files changed, 43 insertions(+), 4 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index acf71a71ba1a..93d65fbe38a9 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -91,6 +91,13 @@
 from mypy.traverser import has_return_statement
 
 
+# Common ways of naming package containing vendored modules.
+VENDOR_PACKAGES = [
+    'packages',
+    'vendor',
+    'vendored',
+]
+
 # Avoid some file names that are unnecessary or likely to cause trouble (\n for end of path).
 BLACKLIST = [
     '/six.py\n',  # Likely vendored six; too dynamic for us to handle
@@ -814,12 +821,19 @@ def visit_import_all(self, o: ImportAll) -> None:
     def visit_import_from(self, o: ImportFrom) -> None:
         exported_names = set()  # type: Set[str]
         import_names = []
+        module, relative = self.translate_module_name(o.id, o.relative)
+        if module == '__future__':
+            return  # Not preserved
         for name, as_name in o.names:
+            if name == 'six':
+                # Vendored six -- translate into plain 'import six'.
+                self.visit_import(Import([('six', None)]))
+                continue
             exported = False
             if as_name is None and self.module and (self.module + '.' + name) in EXTRA_EXPORTED:
                 exported = True
             if (as_name is None and name not in self.referenced_names and not self._all_
-                    and o.id not in ('abc', 'typing', 'asyncio')):
+                    and module not in ('abc', 'typing', 'asyncio')):
                 # An imported name that is never referenced in the module is assumed to be
                 # exported, unless there is an explicit __all__. Note that we need to special
                 # case 'abc' since some references are deleted during semantic analysis.
@@ -828,7 +842,7 @@ def visit_import_from(self, o: ImportFrom) -> None:
                 self.import_tracker.reexport(name)
                 as_name = name
             import_names.append((name, as_name))
-        self.import_tracker.add_import_from('.' * o.relative + o.id, import_names)
+        self.import_tracker.add_import_from('.' * relative + module, import_names)
         self._vars[-1].extend(alias or name for name, alias in import_names)
         for name, alias in import_names:
             self.record_name(alias or name)
@@ -840,14 +854,22 @@ def visit_import_from(self, o: ImportFrom) -> None:
             exported_names.update(names)
         else:
             # Include import from targets that import from a submodule of a package.
-            if o.relative:
+            if relative:
                 sub_names = [name for name, alias in o.names
                              if alias is None]
                 exported_names.update(sub_names)
-                if o.id:
+                if module:
                     for name in sub_names:
                         self.import_tracker.require_name(name)
 
+    def translate_module_name(self, module: str, relative: int) -> Tuple[str, int]:
+        for pkg in VENDOR_PACKAGES:
+            for alt in 'six', 'six.moves':
+                if (module.endswith('.{}.{}'.format(pkg, alt))
+                        or (module == '{}.{}'.format(pkg, alt) and relative)):
+                    return alt, 0
+        return module, relative
+
     def visit_import(self, o: Import) -> None:
         for id, as_id in o.ids:
             self.import_tracker.add_import(id, as_id)
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index 9e50eb3a32b5..08634ec48be2 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1660,3 +1660,20 @@ class D(C): ...
 from b import C as C
 # b.pyi
 class C: ...
+
+[case testVendoredSix]
+from p1.vendored import six
+from p1.vendor.six import foobar
+from p1.packages.six.moves import http_client
+from .packages.six.moves import queue
+
+class C(http_client.HTTPMessage): pass
+class D(six.Iterator): pass
+
+[out]
+import six
+from six import foobar as foobar
+from six.moves import http_client, queue as queue
+
+class C(http_client.HTTPMessage): ...
+class D(six.Iterator): ...

From add4a64bcdf3947535dd3cdccc285733e5559e20 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Wed, 10 Apr 2019 18:48:22 +0100
Subject: [PATCH 32/50] Stubgen: Remove generated header

It doesn't seem useful and easily gets stale if the stub is
modified.
---
 mypy/stubgen.py          | 16 ++++++----------
 mypy/stubgenc.py         |  5 +----
 mypy/stubutil.py         | 10 ----------
 mypy/test/teststubgen.py |  2 +-
 4 files changed, 8 insertions(+), 25 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 93d65fbe38a9..4c4fe2024ecb 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -75,7 +75,7 @@
 )
 from mypy.stubgenc import generate_stub_for_c_module
 from mypy.stubutil import (
-    write_header, default_py2_interpreter, CantImport, generate_guarded,
+    default_py2_interpreter, CantImport, generate_guarded,
     walk_packages, find_module_path_and_all_py2, find_module_path_and_all_py3,
     report_missing, fail_missing, remove_misplaced_type_comments, common_dir_prefix
 )
@@ -1231,8 +1231,7 @@ def generate_stub_from_ast(mod: StubSource,
                            target: str,
                            parse_only: bool = False,
                            pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
-                           include_private: bool = False,
-                           add_header: bool = True) -> None:
+                           include_private: bool = False) -> None:
     """Use analysed (or just parsed) AST to generate type stub for single file.
 
     If directory for target doesn't exist it will created. Existing stub
@@ -1250,8 +1249,6 @@ def generate_stub_from_ast(mod: StubSource,
     if subdir and not os.path.isdir(subdir):
         os.makedirs(subdir)
     with open(target, 'w') as file:
-        if add_header:
-            write_header(file, mod.module, pyversion=pyversion)
         file.write(''.join(gen.output()))
 
 
@@ -1273,8 +1270,7 @@ def collect_docs_signatures(doc_dir: str) -> Tuple[Dict[str, str], Dict[str, str
     return sigs, class_sigs
 
 
-def generate_stubs(options: Options,
-                   add_header: bool) -> None:
+def generate_stubs(options: Options) -> None:
     """Main entry point for the program."""
     mypy_opts = mypy_options(options)
     py_modules, c_modules = collect_build_targets(options, mypy_opts)
@@ -1299,7 +1295,7 @@ def generate_stubs(options: Options,
         with generate_guarded(mod.module, target, options.ignore_errors, options.verbose):
             generate_stub_from_ast(mod, target,
                                    options.parse_only, options.pyversion,
-                                   options.include_private, add_header)
+                                   options.include_private)
 
     # Separately analyse C modules using different logic.
     for mod in c_modules:
@@ -1311,7 +1307,7 @@ def generate_stubs(options: Options,
         target = os.path.join(options.output_dir, target)
         files.append(target)
         with generate_guarded(mod.module, target, options.ignore_errors, options.verbose):
-            generate_stub_for_c_module(mod.module, target, sigs=sigs, class_sigs=class_sigs, add_header=add_header)
+            generate_stub_for_c_module(mod.module, target, sigs=sigs, class_sigs=class_sigs)
     num_modules = len(py_modules) + len(c_modules)
     if not options.quiet and num_modules > 0:
         print('Processed %d modules' % num_modules)
@@ -1414,7 +1410,7 @@ def main() -> None:
         sys.path.insert(0, '')
 
     options = parse_options(sys.argv[1:])
-    generate_stubs(options, add_header=False)
+    generate_stubs(options)
 
 
 if __name__ == '__main__':
diff --git a/mypy/stubgenc.py b/mypy/stubgenc.py
index f792e4f3c080..7de5f450aa6d 100755
--- a/mypy/stubgenc.py
+++ b/mypy/stubgenc.py
@@ -11,7 +11,7 @@
 from typing import List, Dict, Tuple, Optional, Mapping, Any, Set
 from types import ModuleType
 
-from mypy.stubutil import write_header, is_c_module
+from mypy.stubutil import is_c_module
 from mypy.stubdoc import (
     infer_sig_from_docstring, infer_prop_type_from_docstring, ArgSig,
     infer_arg_sig_from_docstring, FunctionSig
@@ -20,7 +20,6 @@
 
 def generate_stub_for_c_module(module_name: str,
                                target: str,
-                               add_header: bool = True,
                                sigs: Optional[Dict[str, str]] = None,
                                class_sigs: Optional[Dict[str, str]] = None) -> None:
     """Generate stub for C module.
@@ -76,8 +75,6 @@ def generate_stub_for_c_module(module_name: str,
         output.append(line)
     output = add_typing_import(output)
     with open(target, 'w') as file:
-        if add_header:
-            write_header(file, module_name)
         for line in output:
             file.write('%s\n' % line)
 
diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index c23a88e0b4b8..83fa6c4f9426 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -34,16 +34,6 @@ def is_c_module(module: ModuleType) -> bool:
     return os.path.splitext(module.__dict__['__file__'])[-1] in ['.so', '.pyd']
 
 
-def write_header(file: IO[str], module_name: Optional[str] = None,
-                 pyversion: Tuple[int, int] = (3, 5)) -> None:
-    """Write a header to file indicating this file is auto-generated by stubgen."""
-    if module_name:
-        file.write('# Stubs for %s (Python %s)\n' % (module_name, pyversion[0]))
-    file.write(
-        '#\n'
-        '# NOTE: This dynamically typed stub was automatically generated by stubgen.\n\n')
-
-
 def default_py2_interpreter() -> str:
     """Find a system Python 2 interpreter.
 
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 7cda3608ce4f..a0953fbc3129 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -472,7 +472,7 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None:
                     options.no_import = True
                 if not testcase.name.endswith('_semanal'):
                     options.parse_only = True
-                generate_stubs(options, add_header=False)
+                generate_stubs(options)
                 a = []  # type: List[str]
                 for module in modules:
                     fnam = os.path.join(out_dir, '{}.pyi'.format(module))

From 8da487df060107c185dea00bf65e6b5acb554146 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Fri, 10 May 2019 18:38:44 +0100
Subject: [PATCH 33/50] Add unit tests for skipping blacklisted and test
 modules

---
 mypy/stubgen.py          | 19 +++++++++++++------
 mypy/test/teststubgen.py | 30 +++++++++++++++++++++++++++++-
 2 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 4c4fe2024ecb..fc989419de11 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -1061,8 +1061,12 @@ def get_qualified_name(o: Expression) -> str:
 
 def remove_blacklisted_modules(modules: List[StubSource]) -> List[StubSource]:
     return [module for module in modules
-            if module.path is None or not any(substr in (module.path + '\n')
-                                              for substr in BLACKLIST)]
+            if module.path is None or not is_blacklisted_path(module.path)]
+
+
+def is_blacklisted_path(path: str) -> bool:
+    return any(substr in (path + '\n')
+               for substr in BLACKLIST)
 
 
 def collect_build_targets(options: Options, mypy_opts: MypyOptions) -> Tuple[List[StubSource],
@@ -1139,10 +1143,13 @@ def find_module_paths_using_imports(modules: List[str],
 
 def remove_test_modules(modules: List[str]) -> List[str]:
     """Remove anything from modules that looks like a test."""
-    return [module for module in modules
-            if (not module.endswith(('.tests', '.test'))
-                and '.tests.' not in module
-                and '.test.' not in module)]
+    return [mod for mod in modules if not is_test_module(mod)]
+
+
+def is_test_module(module: str) -> bool:
+    return (module.endswith(('.tests', '.test'))
+            or '.tests.' in module
+            or '.test.' in module)
 
 
 def find_module_paths_using_search(modules: List[str], packages: List[str],
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index a0953fbc3129..34483fdc1875 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -15,7 +15,7 @@
 from mypy.errors import CompileError
 from mypy.stubgen import (
     generate_stubs, parse_options, Options, collect_build_targets,
-    mypy_options
+    mypy_options, is_blacklisted_path, is_test_module
 )
 from mypy.stubutil import walk_packages, remove_misplaced_type_comments, common_dir_prefix
 from mypy.stubgenc import generate_c_type_stub, infer_method_sig, generate_c_function_stub
@@ -423,6 +423,34 @@ def test_common_dir_prefix(self) -> None:
         assert common_dir_prefix(['foo/bar/x.pyi', 'foo/bar/zar/y.pyi']) == 'foo/bar'
 
 
+class StubgenHelpersSuite(Suite):
+    def test_is_blacklisted_path(self) -> None:
+        assert not is_blacklisted_path('foo/bar.py')
+        assert not is_blacklisted_path('foo.py')
+        assert not is_blacklisted_path('foo/xvendor/bar.py')
+        assert not is_blacklisted_path('foo/vendorx/bar.py')
+        assert is_blacklisted_path('foo/vendor/bar.py')
+        assert is_blacklisted_path('foo/vendored/bar.py')
+        assert is_blacklisted_path('foo/vendored/bar/thing.py')
+        assert is_blacklisted_path('foo/six.py')
+
+    def test_is_test_module(self) -> None:
+        assert not is_test_module('foo')
+        assert not is_test_module('foo.bar')
+
+        # The following could be test modules, but we are very conservative and
+        # don't treat them as such since they could plausibly be real modules.
+        assert not is_test_module('foo.bartest')
+        assert not is_test_module('foo.bartests')
+        assert not is_test_module('foo.test_bar')
+        assert not is_test_module('foo.testbar')
+
+        assert is_test_module('foo.test')
+        assert is_test_module('foo.test.foo')
+        assert is_test_module('foo.tests')
+        assert is_test_module('foo.tests.foo')
+
+
 class StubgenPythonSuite(DataSuite):
     """Data-driven end-to-end test cases that generate stub files.
 

From 376513d5b7b59c99dc0314268df53d5d0431cb77 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 4 Nov 2019 14:25:06 +0000
Subject: [PATCH 34/50] Add docstring

---
 mypy/stubutil.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 83fa6c4f9426..a5650293ff0a 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -192,6 +192,11 @@ def fail_missing(mod: str) -> None:
 
 
 def remove_misplaced_type_comments(source: AnyStr) -> AnyStr:
+    """Remove comments from source that could be understood as misplaced type comments.
+
+    Normal comments may look like misplaced type comments, and since they cause blocking
+    parse errors, we want to avoid them.
+    """
     if isinstance(source, bytes):
         # This gives us a 1-1 character code mapping, so it's roundtrippable.
         text = source.decode('latin1')

From 41a1f22b70b99fe45cdea16f9ae64405937f0465 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 4 Nov 2019 14:25:14 +0000
Subject: [PATCH 35/50] Add test case for vendored package

---
 mypy/test/teststubgen.py    | 12 +++++++++---
 test-data/unit/stubgen.test | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 34483fdc1875..51b4e2be2a0a 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -485,7 +485,10 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None:
         mods = []  # Module names to process
         source = '\n'.join(testcase.input)
         for file, content in testcase.files + [('./main.py', source)]:
-            mod = os.path.basename(file)[:-3]
+            # Strip ./ prefix and .py suffix.
+            mod = file[2:-3].replace('/', '.')
+            if mod.endswith('.__init__'):
+                mod, _, _ = mod.rpartition('.')
             mods.append(mod)
             extra.extend(['-m', mod])
             with open(file, 'w') as f:
@@ -503,7 +506,7 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None:
                 generate_stubs(options)
                 a = []  # type: List[str]
                 for module in modules:
-                    fnam = os.path.join(out_dir, '{}.pyi'.format(module))
+                    fnam = os.path.join(out_dir, '{}.pyi'.format(module.replace('.', '/')))
                     self.add_file(fnam, a, header=len(modules) > 1)
             except CompileError as e:
                 a = e.messages
@@ -535,8 +538,11 @@ def parse_modules(self, program_text: str) -> List[str]:
             return ['main']
 
     def add_file(self, path: str, result: List[str], header: bool) -> None:
+        if not os.path.exists(path):
+            result.append('<%s was not generated>' % path)
+            return
         if header:
-            result.append('# {}'.format(os.path.basename(path)))
+            result.append('# {}'.format(path[4:]))
         with open(path, encoding='utf8') as file:
             result.extend(file.read().splitlines())
 
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index 08634ec48be2..8c87b02f9225 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1677,3 +1677,35 @@ from six.moves import http_client, queue as queue
 
 class C(http_client.HTTPMessage): ...
 class D(six.Iterator): ...
+
+[case testVendoredPackage]
+# modules: main p.vendored.requests p.sub.requests
+from p.vendored.requests import Request
+from p.sub.requests import Request2
+
+x = Request()
+y = Request2()
+
+[file p/__init__.py]
+
+[file p/vendored/__init__.py]
+
+[file p/vendored/requests.py]
+class Request:
+    pass
+
+[file p/sub/__init__.py]
+
+[file p/sub/requests.py]
+class Request2:
+    pass
+
+[out]
+# main.pyi
+from typing import Any
+
+x: Any
+y: Any
+<out/p/vendored/requests.pyi was not generated>
+# p/sub/requests.pyi
+class Request2: ...

From b6db629211fe34ed5a79c01d624d553ffd415170 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 11 Nov 2019 12:45:40 +0000
Subject: [PATCH 36/50] Fix lint

---
 mypy/stubgen.py  | 7 ++++---
 mypy/stubutil.py | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index fc989419de11..968f684599ed 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -70,7 +70,7 @@
     Expression, IntExpr, UnaryExpr, StrExpr, BytesExpr, NameExpr, FloatExpr, MemberExpr,
     TupleExpr, ListExpr, ComparisonExpr, CallExpr, IndexExpr, EllipsisExpr,
     ClassDef, MypyFile, Decorator, AssignmentStmt, TypeInfo,
-    IfStmt, ReturnStmt, ImportAll, ImportFrom, Import, FuncDef, FuncBase, TempNode, Block,
+    IfStmt, ImportAll, ImportFrom, Import, FuncDef, FuncBase, TempNode, Block,
     ARG_POS, ARG_STAR, ARG_STAR2, ARG_NAMED, ARG_NAMED_OPT
 )
 from mypy.stubgenc import generate_stub_for_c_module
@@ -82,7 +82,8 @@
 from mypy.stubdoc import parse_all_signatures, find_unique_signatures, Sig
 from mypy.options import Options as MypyOptions
 from mypy.types import (
-    Type, TypeStrVisitor, CallableType, UnboundType, NoneType, TupleType, TypeList, Instance, AnyType
+    Type, TypeStrVisitor, CallableType, UnboundType, NoneType, TupleType, TypeList, Instance,
+    AnyType
 )
 from mypy.visitor import NodeVisitor
 from mypy.find_sources import create_source_list, InvalidSourceList
@@ -1110,7 +1111,7 @@ def find_module_paths_using_imports(modules: List[str],
                                     pyversion: Tuple[int, int],
                                     verbose: bool,
                                     quiet: bool) -> Tuple[List[StubSource],
-                                                                 List[StubSource]]:
+                                                          List[StubSource]]:
     """Find path and runtime value of __all__ (if possible) for modules and packages.
 
     This function uses runtime Python imports to get the information.
diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index a5650293ff0a..a9e79bde4cff 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -11,7 +11,7 @@
 from types import ModuleType
 from contextlib import contextmanager
 
-from typing import Optional, Tuple, List, IO, Iterator, AnyStr
+from typing import Optional, Tuple, List, Iterator, AnyStr
 
 
 # Modules that may fail when imported, or that may have side effects.

From c6aca1b4760a4fea76023730634e7ed97502f5e7 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 11 Nov 2019 13:44:57 +0000
Subject: [PATCH 37/50] Update test case

---
 test-data/unit/check-classes.test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test-data/unit/check-classes.test b/test-data/unit/check-classes.test
index 78121bc70dad..8dffcfdd7d42 100644
--- a/test-data/unit/check-classes.test
+++ b/test-data/unit/check-classes.test
@@ -3715,7 +3715,7 @@ class A: pass
 class B(A): pass
 class C(B): pass
 class D(A, B): pass # E: Cannot determine consistent method resolution order (MRO) for "D"
-class E(C, D): pass # E: Cannot determine consistent method resolution order (MRO) for "E"
+class E(C, D): pass
 
 [case testInconsistentMroLocalRef]
 class A: pass

From 3309f745a8ce2dbcfaa24ad45027f3269d0d1dcd Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 4 Nov 2019 15:18:04 +0000
Subject: [PATCH 38/50] Always filter out tests

---
 mypy/stubgen.py             | 11 +++++-----
 mypy/test/teststubgen.py    | 11 +++++++++-
 test-data/unit/stubgen.test | 44 +++++++++++++++++++++++++++++++++++++
 3 files changed, 59 insertions(+), 7 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 968f684599ed..6eb7c25aa1af 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -1119,8 +1119,8 @@ def find_module_paths_using_imports(modules: List[str],
     py_modules = []  # type: List[StubSource]
     c_modules = []  # type: List[StubSource]
     found = list(walk_packages(packages, verbose))
-    found = remove_test_modules(found)  # We don't want to run any tests
     modules = modules + found
+    modules = [mod for mod in modules if not is_test_module(mod)]  # We don't want to run any tests
     for mod in modules:
         try:
             if pyversion[0] == 2:
@@ -1142,12 +1142,8 @@ def find_module_paths_using_imports(modules: List[str],
     return py_modules, c_modules
 
 
-def remove_test_modules(modules: List[str]) -> List[str]:
-    """Remove anything from modules that looks like a test."""
-    return [mod for mod in modules if not is_test_module(mod)]
-
-
 def is_test_module(module: str) -> bool:
+    """Does module look like a test module?"""
     return (module.endswith(('.tests', '.test'))
             or '.tests.' in module
             or '.test.' in module)
@@ -1177,6 +1173,9 @@ def find_module_paths_using_search(modules: List[str], packages: List[str],
             fail_missing(package)
         sources = [StubSource(m.module, m.path) for m in p_result]
         result.extend(sources)
+
+    result = [m for m in result if not is_test_module(m.module)]
+
     return result
 
 
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 51b4e2be2a0a..34ed5e90f256 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -506,7 +506,7 @@ def run_case_inner(self, testcase: DataDrivenTestCase) -> None:
                 generate_stubs(options)
                 a = []  # type: List[str]
                 for module in modules:
-                    fnam = os.path.join(out_dir, '{}.pyi'.format(module.replace('.', '/')))
+                    fnam = module_to_path(out_dir, module)
                     self.add_file(fnam, a, header=len(modules) > 1)
             except CompileError as e:
                 a = e.messages
@@ -764,3 +764,12 @@ def test_repr(self) -> None:
                      "ArgSig(name='func', type='str', default=False)")
         assert_equal(repr(ArgSig("func", 'str', default=True)),
                      "ArgSig(name='func', type='str', default=True)")
+
+
+def module_to_path(out_dir: str, module: str) -> str:
+    fnam = os.path.join(out_dir, '{}.pyi'.format(module.replace('.', '/')))
+    if not os.path.exists(fnam):
+        alt_fnam = fnam.replace('.pyi', '/__init__.pyi')
+        if os.path.exists(alt_fnam):
+            return alt_fnam
+    return fnam
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index 8c87b02f9225..8160c1c90149 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1709,3 +1709,47 @@ y: Any
 <out/p/vendored/requests.pyi was not generated>
 # p/sub/requests.pyi
 class Request2: ...
+
+[case testTestFiles]
+# modules: p p.x p.tests p.tests.test_foo
+
+[file p/__init__.py]
+def f(): pass
+
+[file p/x.py]
+def g(): pass
+
+[file p/tests/__init__.py]
+
+[file p/tests/test_foo.py]
+def test_thing(): pass
+
+[out]
+# p/__init__.pyi
+def f() -> None: ...
+# p/x.pyi
+def g() -> None: ...
+<out/p/tests.pyi was not generated>
+<out/p/tests/test_foo.pyi was not generated>
+
+[case testTestFiles_import]
+# modules: p p.x p.tests p.tests.test_foo
+
+[file p/__init__.py]
+def f(): pass
+
+[file p/x.py]
+def g(): pass
+
+[file p/tests/__init__.py]
+
+[file p/tests/test_foo.py]
+def test_thing(): pass
+
+[out]
+# p/__init__.pyi
+def f() -> None: ...
+# p/x.pyi
+def g() -> None: ...
+<out/p/tests.pyi was not generated>
+<out/p/tests/test_foo.pyi was not generated>

From 575cecd5b8cf3567e213d13089d2df160f44ce93 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 4 Nov 2019 18:17:15 +0000
Subject: [PATCH 39/50] Simplify redundant note

---
 mypy/stubutil.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index a9e79bde4cff..5407e040ef3f 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -181,10 +181,7 @@ def report_missing(mod: str, message: Optional[str] = '', traceback: str = '') -
     if m:
         missing_module = m.group(1)
         if missing_module in PY2_MODULES:
-            print('note: Could not import %r; try --py2 for Python 2 mode' % missing_module)
-        else:
-            print('note: Could not import %r; some dependency may be missing' % missing_module)
-    print()
+            print('note: Try --py2 for Python 2 mode' % missing_module)
 
 
 def fail_missing(mod: str) -> None:

From bde23a1022699ab256fe028323ec32ba1c671ad4 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 4 Nov 2019 18:23:11 +0000
Subject: [PATCH 40/50] Skip conftest modules (used for pytest tests)

---
 mypy/stubgen.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 6eb7c25aa1af..fa1bd00a0e7e 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -1144,7 +1144,7 @@ def find_module_paths_using_imports(modules: List[str],
 
 def is_test_module(module: str) -> bool:
     """Does module look like a test module?"""
-    return (module.endswith(('.tests', '.test'))
+    return (module.endswith(('.tests', '.test', '.conftest'))
             or '.tests.' in module
             or '.test.' in module)
 

From b4554e67df1d9e2a374e5ffd8214df33a7b57f34 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Tue, 5 Nov 2019 09:47:01 +0000
Subject: [PATCH 41/50] Fix crash

---
 mypy/stubutil.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 5407e040ef3f..6b5d4fa3b7e5 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -181,7 +181,7 @@ def report_missing(mod: str, message: Optional[str] = '', traceback: str = '') -
     if m:
         missing_module = m.group(1)
         if missing_module in PY2_MODULES:
-            print('note: Try --py2 for Python 2 mode' % missing_module)
+            print('note: Try --py2 for Python 2 mode')
 
 
 def fail_missing(mod: str) -> None:

From c84677be8e5112d8d0e39a3edd27a5cfb05b088b Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Tue, 5 Nov 2019 10:14:48 +0000
Subject: [PATCH 42/50] Filter out more kinds of bad function type comments

---
 mypy/stubutil.py         |  5 ++++-
 mypy/test/teststubgen.py | 23 +++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 6b5d4fa3b7e5..2adddefbe7f6 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -204,11 +204,14 @@ def remove_misplaced_type_comments(source: AnyStr) -> AnyStr:
     # on a line, as it will often generate a parse error (unless it's # type: ignore).
     text = re.sub(r'^[ \t]*# +type: +["\'a-zA-Z_].*$', '', text, flags=re.MULTILINE)
 
-    # Remove something that looks like a function type annotation after docstring,
+    # Remove something that looks like a function type comment after docstring,
     # which will result in a parse error.
     text = re.sub(r'""" *\n[ \t\n]*# +type: +\(.*$', '"""\n', text, flags=re.MULTILINE)
     text = re.sub(r"''' *\n[ \t\n]*# +type: +\(.*$", "'''\n", text, flags=re.MULTILINE)
 
+    # Remove something that looks like a badly formed function type comment.
+    text = re.sub(r'^[ \t]*# +type: +\([^()]+(\)[ \t]*)?$', '', text, flags=re.MULTILINE)
+
     if isinstance(source, bytes):
         return text.encode('latin1')
     else:
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 34ed5e90f256..b2ec42947951 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -375,6 +375,29 @@ def g(x):
         """
         assert_equal(remove_misplaced_type_comments(bad), bad_fixed)
 
+    def test_remove_misplaced_type_comments_5(self) -> None:
+        bad = """
+        def f(x):
+            # type: (int, List[Any],
+            #        float, bool) -> int
+            pass
+
+        def g(x):
+            # type: (int, List[Any])
+            pass
+        """
+        bad_fixed = """
+        def f(x):
+
+            #        float, bool) -> int
+            pass
+
+        def g(x):
+
+            pass
+        """
+        assert_equal(remove_misplaced_type_comments(bad), bad_fixed)
+
     def test_remove_misplaced_type_comments_bytes(self) -> None:
         original = b"""
         \xbf

From be7414ed12fbe59f3b969efd0f74fbdf922ecd98 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Tue, 5 Nov 2019 11:41:53 +0000
Subject: [PATCH 43/50] Recognize more modules as test modules

---
 mypy/stubgen.py          | 19 ++++++++++++++++---
 mypy/test/teststubgen.py | 10 +++++++++-
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index fa1bd00a0e7e..3cde8fb9274e 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -1144,9 +1144,22 @@ def find_module_paths_using_imports(modules: List[str],
 
 def is_test_module(module: str) -> bool:
     """Does module look like a test module?"""
-    return (module.endswith(('.tests', '.test', '.conftest'))
-            or '.tests.' in module
-            or '.test.' in module)
+    if module.endswith((
+            '.tests',
+            '.test',
+            '.testing',
+            '_tests',
+            '.conftest',
+            'test_util',
+            'test_utils',
+            'test_base',
+    )):
+        return True
+    if module.split('.')[-1].startswith('test_'):
+        return True
+    if '.tests.' in module or '.test.' in module or '.testing.' in module:
+        return True
+    return False
 
 
 def find_module_paths_using_search(modules: List[str], packages: List[str],
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index b2ec42947951..95b53141de0e 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -465,13 +465,21 @@ def test_is_test_module(self) -> None:
         # don't treat them as such since they could plausibly be real modules.
         assert not is_test_module('foo.bartest')
         assert not is_test_module('foo.bartests')
-        assert not is_test_module('foo.test_bar')
         assert not is_test_module('foo.testbar')
 
         assert is_test_module('foo.test')
         assert is_test_module('foo.test.foo')
         assert is_test_module('foo.tests')
         assert is_test_module('foo.tests.foo')
+        assert is_test_module('foo.testing.foo')
+
+        assert is_test_module('foo.test_bar')
+        assert is_test_module('foo.bar_tests')
+        assert is_test_module('foo.testing')
+        assert is_test_module('foo.conftest')
+        assert is_test_module('foo.bar_test_util')
+        assert is_test_module('foo.bar_test_utils')
+        assert is_test_module('foo.bar_test_base')
 
 
 class StubgenPythonSuite(DataSuite):

From 07e19550327fc0a1d13b51be4f3d55d31f9a9452 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 11 Nov 2019 14:09:28 +0000
Subject: [PATCH 44/50] Revert changes to imports

---
 mypy/options.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/mypy/options.py b/mypy/options.py
index 87e3524e1515..9171dcd45851 100644
--- a/mypy/options.py
+++ b/mypy/options.py
@@ -3,10 +3,8 @@
 import pprint
 import sys
 
+from typing_extensions import Final
 from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple, Callable, AnyStr
-MYPY = False
-if MYPY:
-    from typing_extensions import Final
 
 from mypy import defaults
 from mypy.util import get_class_descriptors, replace_object_state

From e6bac889f7160cd8f0dee89378a926238341e31f Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 11 Nov 2019 14:49:53 +0000
Subject: [PATCH 45/50] Try to fix Windows

---
 mypy/stubgen.py          | 8 +++++++-
 mypy/test/teststubgen.py | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 3cde8fb9274e..422652bde275 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -1066,10 +1066,16 @@ def remove_blacklisted_modules(modules: List[StubSource]) -> List[StubSource]:
 
 
 def is_blacklisted_path(path: str) -> bool:
-    return any(substr in (path + '\n')
+    return any(substr in (normalize_path_separators(path) + '\n')
                for substr in BLACKLIST)
 
 
+def normalize_path_separators(path: str) -> str:
+    if sys.platform == 'win32':
+        return path.replace('\\', '/')
+    return path
+
+
 def collect_build_targets(options: Options, mypy_opts: MypyOptions) -> Tuple[List[StubSource],
                                                                              List[StubSource]]:
     """Collect files for which we need to generate stubs.
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 95b53141de0e..846cd0a23e87 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -570,7 +570,7 @@ def parse_modules(self, program_text: str) -> List[str]:
 
     def add_file(self, path: str, result: List[str], header: bool) -> None:
         if not os.path.exists(path):
-            result.append('<%s was not generated>' % path)
+            result.append('<%s was not generated>' % path.replace('\\', '/'))
             return
         if header:
             result.append('# {}'.format(path[4:]))

From 548bb10178c12247b337633e8f66dbdbc5349aa0 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 11 Nov 2019 16:08:30 +0000
Subject: [PATCH 46/50] Attempt to fix compiled

---
 mypy/options.py  |  4 ++--
 mypy/stubutil.py | 10 ++++++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/mypy/options.py b/mypy/options.py
index 9171dcd45851..9ff167b0b351 100644
--- a/mypy/options.py
+++ b/mypy/options.py
@@ -4,7 +4,7 @@
 import sys
 
 from typing_extensions import Final
-from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple, Callable, AnyStr
+from typing import Dict, List, Mapping, Optional, Pattern, Set, Tuple, Callable, Any
 
 from mypy import defaults
 from mypy.util import get_class_descriptors, replace_object_state
@@ -263,7 +263,7 @@ def __init__(self) -> None:
         # Don't properly free objects on exit, just kill the current process.
         self.fast_exit = False
         # Used to transform source code before parsing if not None
-        self.transform_source = None  # type: Optional[Callable[[AnyStr], AnyStr]]
+        self.transform_source = None  # type: Optional[Callable[[Any], Any]]
         # Print full path to each file in the report.
         self.show_absolute_path = False  # type: bool
 
diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 2adddefbe7f6..c9eb86fe3d32 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -11,7 +11,7 @@
 from types import ModuleType
 from contextlib import contextmanager
 
-from typing import Optional, Tuple, List, Iterator, AnyStr
+from typing import Optional, Tuple, List, Iterator, AnyStr, Union, overload
 
 
 # Modules that may fail when imported, or that may have side effects.
@@ -188,7 +188,13 @@ def fail_missing(mod: str) -> None:
     raise SystemExit("Can't find module '{}' (consider using --search-path)".format(mod))
 
 
-def remove_misplaced_type_comments(source: AnyStr) -> AnyStr:
+@overload
+def remove_misplaced_type_comments(source: bytes) -> bytes: ...
+
+@overload
+def remove_misplaced_type_comments(source: str) -> str: ...
+
+def remove_misplaced_type_comments(source: Union[str, bytes]) -> Union[str, bytes]:
     """Remove comments from source that could be understood as misplaced type comments.
 
     Normal comments may look like misplaced type comments, and since they cause blocking

From f876154a401b693dcfcb857a05481134779bc4d1 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 11 Nov 2019 16:54:38 +0000
Subject: [PATCH 47/50] Fix lint

---
 mypy/stubutil.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index c9eb86fe3d32..4520fcee9e59 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -11,7 +11,7 @@
 from types import ModuleType
 from contextlib import contextmanager
 
-from typing import Optional, Tuple, List, Iterator, AnyStr, Union, overload
+from typing import Optional, Tuple, List, Iterator, Union, overload
 
 
 # Modules that may fail when imported, or that may have side effects.
@@ -191,9 +191,11 @@ def fail_missing(mod: str) -> None:
 @overload
 def remove_misplaced_type_comments(source: bytes) -> bytes: ...
 
+
 @overload
 def remove_misplaced_type_comments(source: str) -> str: ...
 
+
 def remove_misplaced_type_comments(source: Union[str, bytes]) -> Union[str, bytes]:
     """Remove comments from source that could be understood as misplaced type comments.
 

From 56d1a7336a3e5bf4b6915f7379387018e391aee2 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Mon, 11 Nov 2019 17:16:47 +0000
Subject: [PATCH 48/50] Try to fix Python 3.5

---
 mypy/stubutil.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/mypy/stubutil.py b/mypy/stubutil.py
index 4520fcee9e59..ad3c63d8f10a 100644
--- a/mypy/stubutil.py
+++ b/mypy/stubutil.py
@@ -11,7 +11,8 @@
 from types import ModuleType
 from contextlib import contextmanager
 
-from typing import Optional, Tuple, List, Iterator, Union, overload
+from typing import Optional, Tuple, List, Iterator, Union
+from typing_extensions import overload
 
 
 # Modules that may fail when imported, or that may have side effects.

From 58ae038c4f8e0f893e69d744814e887d878c4284 Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Thu, 14 Nov 2019 16:59:00 +0000
Subject: [PATCH 49/50] Respond to feedback

---
 mypy/options.py             |  1 +
 mypy/parse.py               |  5 +----
 mypy/stubgen.py             | 12 ++++++++++++
 mypy/test/teststubgen.py    |  2 ++
 test-data/unit/stubgen.test | 10 ++++++++++
 5 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/mypy/options.py b/mypy/options.py
index 9ff167b0b351..b5e3be6c68b4 100644
--- a/mypy/options.py
+++ b/mypy/options.py
@@ -263,6 +263,7 @@ def __init__(self) -> None:
         # Don't properly free objects on exit, just kill the current process.
         self.fast_exit = False
         # Used to transform source code before parsing if not None
+        # TODO: Make the type precise (AnyStr -> AnyStr)
         self.transform_source = None  # type: Optional[Callable[[Any], Any]]
         # Print full path to each file in the report.
         self.show_absolute_path = False  # type: bool
diff --git a/mypy/parse.py b/mypy/parse.py
index a8da1ba6db49..c39a2388028a 100644
--- a/mypy/parse.py
+++ b/mypy/parse.py
@@ -19,10 +19,7 @@ def parse(source: Union[str, bytes],
     """
     is_stub_file = fnam.endswith('.pyi')
     if options.transform_source is not None:
-        if isinstance(source, str):  # Work around mypy issue
-            source = options.transform_source(source)
-        else:
-            source = options.transform_source(source)
+        source = options.transform_source(source)
     if options.python_version[0] >= 3 or is_stub_file:
         import mypy.fastparse
         return mypy.fastparse.parse(source,
diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index 422652bde275..c15e1b8259a7 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -584,6 +584,12 @@ def visit_decorator(self, o: Decorator) -> None:
         self.visit_func_def(o.func, is_abstract=is_abstract)
 
     def process_name_expr_decorator(self, expr: NameExpr, context: Decorator) -> bool:
+        """Process a function decorator of form @foo.
+
+        Only preserve certain special decorators such as @abstractmethod.
+
+        Return True if the decorator makes a method abstract.
+        """
         is_abstract = False
         name = expr.name
         if name in ('property', 'staticmethod', 'classmethod'):
@@ -606,6 +612,12 @@ def refers_to_fullname(self, name: str, fullname: str) -> bool:
                  self.import_tracker.reverse_alias.get(name) == short))
 
     def process_member_expr_decorator(self, expr: MemberExpr, context: Decorator) -> bool:
+        """Process a function decorator of form @foo.bar.
+
+        Only preserve certain special decorators such as @abstractmethod.
+
+        Return True if the decorator makes a method abstract.
+        """
         is_abstract = False
         if expr.name == 'setter' and isinstance(expr.expr, NameExpr):
             self.add_decorator('%s.setter' % expr.expr.name)
diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py
index 846cd0a23e87..4d3eaef112c5 100644
--- a/mypy/test/teststubgen.py
+++ b/mypy/test/teststubgen.py
@@ -559,6 +559,8 @@ def parse_flags(self, program_text: str, extra: List[str]) -> Options:
         options = parse_options(flag_list + extra)
         if '--verbose' not in flag_list:
             options.quiet = True
+        else:
+            options.verbose = True
         return options
 
     def parse_modules(self, program_text: str) -> List[str]:
diff --git a/test-data/unit/stubgen.test b/test-data/unit/stubgen.test
index 8160c1c90149..18837e26dc79 100644
--- a/test-data/unit/stubgen.test
+++ b/test-data/unit/stubgen.test
@@ -1753,3 +1753,13 @@ def f() -> None: ...
 def g() -> None: ...
 <out/p/tests.pyi was not generated>
 <out/p/tests/test_foo.pyi was not generated>
+
+[case testVerboseFlag]
+# Just test that --verbose does not break anything in a basic test case.
+# flags: --verbose
+
+def f(x, y): pass
+[out]
+from typing import Any
+
+def f(x: Any, y: Any) -> None: ...

From 073832eb8be21049c82f142c003b718dd957c01e Mon Sep 17 00:00:00 2001
From: Jukka Lehtosalo <jukka.lehtosalo@iki.fi>
Date: Thu, 14 Nov 2019 17:03:37 +0000
Subject: [PATCH 50/50] Fix stuff

---
 mypy/stubgen.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mypy/stubgen.py b/mypy/stubgen.py
index ebef5fdfeae0..f7edbdf2d4c4 100755
--- a/mypy/stubgen.py
+++ b/mypy/stubgen.py
@@ -395,7 +395,7 @@ def visit_class_def(self, o: ClassDef) -> None:
 
     def visit_func_def(self, o: FuncDef) -> None:
         # Don't recurse, as we only keep track of top-level definitions.
-        self.names.add(o.name())
+        self.names.add(o.name)
 
 
 def find_referenced_names(file: MypyFile) -> Set[str]:
@@ -421,7 +421,7 @@ def visit_name_expr(self, e: NameExpr) -> None:
         self.refs.add(e.name)
 
     def visit_instance(self, t: Instance) -> None:
-        self.add_ref(t.type.fullname())
+        self.add_ref(t.type.fullname)
         super().visit_instance(t)
 
     def visit_unbound_type(self, t: UnboundType) -> None:
@@ -473,7 +473,7 @@ def __init__(self, _all_: Optional[List[str]], pyversion: Tuple[int, int],
         self.defined_names = set()  # type: Set[str]
 
     def visit_mypy_file(self, o: MypyFile) -> None:
-        self.module = o.fullname()
+        self.module = o.fullname
         self.defined_names = find_defined_names(o)
         self.referenced_names = find_referenced_names(o)
         typing_imports = ["Any", "Optional", "TypeVar"]