update the lazy_install mode

Author: ClericPy

README.md

@@ -373,6 +373,18 @@ Details:
 
 # Changelogs
 
+- 2021.04.22
+  - update the lazy_install mode (`-d`)
+    - simple use case:
+      - 1. `python -m zipapps -d six`
+      - 2. `python app.pyz -c "import six;print(six.__file__)"`
+    - it will not `rmtree` the `_zipapps_lazy_pip` folder for new build
+      - if you need to upgrade those requirements, use `-U` arg of pip
+      - lazy_install mode will install requirements with `pip install` while first running
+        - so you can just run like this to init its installation(`-m` not set): `python3 app.pyz -V`
+    - for now, `-d` is to server for the cross-platform and cross-python-version app
+      - pip target path is separated from different python version and system platform
+        - python version accurity can be reset with `-pva 5`, defaults to 2, means 3.8.3 and 3.8.4 are same `3.8`
 - 2021.04.11
   - add `--sys-paths` to include new paths of sys.path
     - support TEMP/HOME/SELF prefix, separated by commas

test_utils.py

@@ -439,6 +439,19 @@ def test_create_app_function():
     assert b'Installing collected packages' in stdout_output
     assert stdout_output.count(b'zipapps_cache') == 2, stdout_output.count(
         b'zipapps_cache')
+    app_path = create_app(lazy_install=True,
+                          pip_args=['bottle', '-r', '_requirements.txt'])
+    stdout_output, stderr_output = subprocess.Popen(
+        [
+            sys.executable,
+            str(app_path), '-c',
+            'import six,bottle;print(six.__file__, bottle.__file__)'
+        ],
+        stderr=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+    ).communicate()
+    # print(stdout_output, stderr_output)
+    assert b'already exists' in stderr_output
 
     # test sys_path
     _clean_paths()

zipapps/__init__.py

@@ -2,4 +2,4 @@
 from .main import create_app
 from .activate_zipapps import activate
 __all__ = ['create_app', 'activate']
-__version__ = '2021.04.11'
+__version__ = '2021.04.22'

zipapps/__main__.py

@@ -167,6 +167,15 @@ def main():
         dest='lazy_install',
         help='Install packages with pip while running, which means '
         'requirements will not be install into pyz file.')
+    parser.add_argument(
+        '-pva',
+        '--python-version-accuracy',
+        '--python-version-slice',
+        default='2',
+        dest='python_version_slice',
+        help='Only work for lazy-install mode, then `pip` target folders differ '
+        'according to sys.version_info[:_slice], defaults to 2, which means '
+        '3.8.3 equals to 3.8.4 for same version accuracy 3.8')
     parser.add_argument('--sys-paths',
                         '--sys-path',
                         '--py-path',
@@ -196,6 +205,7 @@ def main():
         env_paths=args.env_paths,
         lazy_install=args.lazy_install,
         sys_paths=args.sys_paths,
+        python_version_slice=int(args.python_version_slice),
     )
 
 

zipapps/ensure_zipapps_template.py

@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 import os
 import sys
 from pathlib import Path
@@ -19,15 +21,27 @@ def ensure_path(path):
     return _cache_folder_path
 
 
-def rm_dir(dir_path: Path):
+def rm_dir_and_file(path: Path):
     for _ in range(3):
         try:
-            if not dir_path.is_dir():
+            if path.is_dir():
+                rmtree(str(path.absolute()))
+            elif path.is_file():
+                path.unlink()
+            else:
                 break
-            # remove the exist folder
-            rmtree(dir_path)
         except FileNotFoundError:
             break
+    else:
+        return False
+    return True
+
+
+def clear_old_cache(_cache_folder_path: Path, LAZY_PIP_DIR_NAME=''):
+    for path in _cache_folder_path.glob('*'):
+        if path.name == LAZY_PIP_DIR_NAME:
+            continue
+        rm_dir_and_file(path)
 
 
 def prepare_path():
@@ -42,14 +56,25 @@ def prepare_path():
 
         _cache_folder_path = ensure_path(_cache_folder)
         _cache_folder_path = _cache_folder_path / zip_file_path.stem
+        _cache_folder_path.mkdir(parents=True, exist_ok=True)
         _cache_folder_path_str = str(_cache_folder_path.absolute())
         _zipapps_python_path_list.insert(0, _cache_folder_path_str)
         ts_file_name = '_zip_time_{ts}'
+        LAZY_PIP_DIR_NAME = r'''{LAZY_PIP_DIR_NAME}'''
+        if LAZY_PIP_DIR_NAME:
+            import platform
+            lazy_pip_dir = _cache_folder_path / LAZY_PIP_DIR_NAME
+            # pip target isolation with by python version and platform
+            platform_name = (platform.system() or '-')
+            version_name = '.'.join(
+                map(str, sys.version_info[:{python_version_slice}]))
+            _pip_target = lazy_pip_dir / version_name / platform_name
+            lazy_pip_dir_str = str(_pip_target.absolute())
+            _zipapps_python_path_list.insert(1, lazy_pip_dir_str)
         if not (_cache_folder_path / ts_file_name).is_file():
             # check timestamp difference by file name, need to refresh _cache_folder
             # rm the folder
-            rm_dir(_cache_folder_path)
-            _cache_folder_path.mkdir(parents=True)
+            clear_old_cache(_cache_folder_path, LAZY_PIP_DIR_NAME)
             _need_unzip_names = unzip.split(',')
             _need_unzip_names.append(ts_file_name)
             with ZipFile(zip_file_path, "r") as zf:
@@ -59,8 +84,7 @@ def prepare_path():
                     if unzip == '*' or member.filename in _need_unzip_names or file_dir_name in _need_unzip_names:
                         zf.extract(member, path=_cache_folder_path_str)
             # lazy pip install
-            lazy_pip_dir = _cache_folder_path / r'''{LAZY_PIP_DIR_NAME}'''
-            if lazy_pip_dir.is_dir():
+            if LAZY_PIP_DIR_NAME and lazy_pip_dir.is_dir():
                 try:
                     import pip
                 except ImportError:
@@ -69,7 +93,7 @@ def prepare_path():
                 import subprocess
                 shell_args = [
                     sys.executable, '-m', 'pip', 'install', '--target',
-                    _cache_folder_path_str
+                    lazy_pip_dir_str
                 ] + {pip_args_repr}
                 with subprocess.Popen(shell_args,
                                       cwd=_cache_folder_path_str) as proc:

zipapps/main.py

@@ -62,6 +62,7 @@ def prepare_entry(
     env_paths: str = '',
     pip_args: list = None,
     sys_paths: str = '',
+    python_version_slice: int = 2,
 ):
     unzip_names = set(unzip.split(',')) if unzip else set()
     warning_names: typing.Dict[str, dict] = {}
@@ -116,6 +117,7 @@ def prepare_entry(
         'LAZY_PIP_DIR_NAME': Config.LAZY_PIP_DIR_NAME,
         'pip_args_repr': repr(pip_args),
         'sys_paths': sys_paths,
+        'python_version_slice': python_version_slice,
     }
     code = get_data('zipapps', '_entry_point.py').decode('u8')
     (cache_path / '__main__.py').write_text(code.format(**kwargs))
@@ -217,6 +219,7 @@ def create_app(
     env_paths: str = '',
     lazy_install: bool = False,
     sys_paths: str = '',
+    python_version_slice: int = 2,
 ):
     tmp_dir: tempfile.TemporaryDirectory = None
     try:
@@ -237,7 +240,7 @@ def create_app(
                 )
             if lazy_install:
                 sys.stderr.write(
-                    f'WARNING: `unzip` arg has been changed `{unzip}` => `*` for `lazy_install` is True\n'
+                    'WARNING: `unzip` has been changed to "*" while `lazy_install` is True.\n'
                 )
                 unzip = '*'
                 # copy files to cache folder
@@ -252,7 +255,7 @@ def create_app(
                         pip_args[index] = _r_path.as_posix()
                 if not unzip_path:
                     sys.stderr.write(
-                        f'WARNING: unzip path has been set to `SELF/{Config.DEFAULT_UNZIP_CACHE_PATH}` by default for `lazy_install` is True.\n'
+                        f'WARNING: unzip path has been set to `SELF/{Config.DEFAULT_UNZIP_CACHE_PATH}` while `lazy_install` is True and `unzip_path` is null.\n'
                     )
                     unzip_path = f'SELF/{Config.DEFAULT_UNZIP_CACHE_PATH}'
             else:
@@ -273,6 +276,7 @@ def create_app(
             env_paths=env_paths,
             pip_args=pip_args,
             sys_paths=sys_paths,
+            python_version_slice=python_version_slice,
         )
         if compiled:
             if not unzip: