add --clear-zipapps-cache / -czc arg

Author: ClericPy

README.md

@@ -319,19 +319,23 @@ Details:
 - `-a xxx.py`
   - to add some files/folders into the zipped file.
 - `-u=AUTO`
+  - **Recommended**
   - auto unzip the .pyd / .so files
+    - or `-u=*` to unzip all the files
 - `-r requirements.txt`
   - install requirements with `pip install`
 - `-o my_app.pyz`
   - output the zipped file as given path
 - `-m app.__main__:main`
-  - set the entry point
+  - entry point
 - `-p /usr/bin/python3`
   - set the `shebang` line
 - `-d`
+  - **Recommended**
   - lazy install mode, requirements will be installed with `pip` while first running
-  - **Very useful**
   - zip file size will be very small, and the default unzip path is `SELF/zipapps_cache/`
+- `--clear-zipapps-cache`, `-czc`
+  - Clear the zipapps cache folder after running, but maybe failed for .pyd/.so files.
 
 Details: 
 
@@ -380,7 +384,7 @@ Details:
    1. Compile .py to .pyc for fast import, but zipapp does not work unless you unzip it(so NOT very useful).
    2. the `compiled` arg of `zipapps.create_app`
 10. ` --cache-path, --source-dir, -cp`
-       3. The cache path of zipapps to store site-packages and `includes` files. If not set, will create and clean-up in TEMP dir automately.
+       1. The cache path of zipapps to store site-packages and `includes` files. If not set, will create and clean-up in TEMP dir automately.
        2. the `cache_path` arg of `zipapps.create_app`
 11. `--shell, -s`
     1.  Only while `main` is not set, used for shell=True in `subprocess.run`.
@@ -407,7 +411,9 @@ Details:
     1. Layer mode for the `serverless` use case, `__main__.py / ensure_zipapps.py / activate_zipapps.py` files will not be set in this mode.
     2. `--layer-mode-prefix`
        1. Only work while `--layer-mode` is set, will move the files in the given prefix folder.
-19. all the other (or `unknown`) args will be used by `pip install`
+19. `--clear-zipapps-cache, -czc`
+    1.  Clear the zipapps cache folder after running, but maybe failed for .pyd/.so files.
+20. all the other (or `unknown`) args will be used by `pip install`
     1.  such as `-r requirements.txt`
     2.  such as `bottle aiohttp`
     3.  the `pip_args` arg of `zipapps.create_app`

changelog.md

@@ -1,6 +1,11 @@
 
 # Changelogs
 
+
+- 2021.11.01
+  - add `--clear-zipapps-cache` / `-czc` command line arg
+    - Clear the zipapps cache folder after running, but maybe failed for .pyd/.so files.
+  - <refactor> move the `const` to the top of `ensure_zipapps` template.
 - 2021.09.22
   - fix `--layer-mode-prefix` not work for `-a` files
 - 2021.09.21

test_utils.py

@@ -36,6 +36,18 @@ def _clean_paths():
 
 def test_create_app_function():
 
+    # test -czc
+    _clean_paths()
+    app_path = create_app(clear_zipapps_cache=False, unzip='*')
+    stdout_output, stderr_output = subprocess.Popen(
+        [sys.executable, str(app_path), '-V']).communicate()
+    assert Path('./zipapps_cache').is_dir()
+    _clean_paths()
+    app_path = create_app(clear_zipapps_cache=True, unzip='*')
+    stdout_output, stderr_output = subprocess.Popen(
+        [sys.executable, str(app_path), '-V']).communicate()
+    assert not Path('./zipapps_cache').is_dir()
+
     # test build_id
     _clean_paths()
     mock_requirement = Path('_requirements.txt')

zipapps/__main__.py

@@ -197,12 +197,24 @@ def main():
         '--layer-mode',
         action='store_true',
         dest='layer_mode',
-        help='Layer mode for the serverless use case, __main__.py / ensure_zipapps.py / activate_zipapps.py files will not be set in this mode.')
+        help=
+        'Layer mode for the serverless use case, __main__.py / ensure_zipapps.py / activate_zipapps.py files will not be set in this mode.'
+    )
     parser.add_argument(
         '--layer-mode-prefix',
         default='python',
         dest='layer_mode_prefix',
-        help='Only work while --layer-mode is set, will move the files in the given prefix folder.')
+        help=
+        'Only work while --layer-mode is set, will move the files in the given prefix folder.'
+    )
+    parser.add_argument(
+        '-czc',
+        '--clear-zipapps-cache',
+        action='store_true',
+        dest='clear_zipapps_cache',
+        help=
+        'Clear the zipapps cache folder after running, but maybe failed for .pyd/.so files.',
+    )
     if len(sys.argv) == 1:
         return parser.print_help()
     args, pip_args = parser.parse_known_args()
@@ -233,6 +245,7 @@ def main():
         ensure_pip=args.ensure_pip,
         layer_mode=args.layer_mode,
         layer_mode_prefix=args.layer_mode_prefix,
+        clear_zipapps_cache=args.clear_zipapps_cache,
     )
 
 

zipapps/ensure_zipapps_template.py

@@ -18,6 +18,7 @@
 pip_args_md5 = '{pip_args_md5}'
 py_version = '.'.join(map(str, sys.version_info[:{python_version_slice}]))
 _new_sys_paths = r'''{sys_paths}'''.strip()
+clear_zipapps_cache = {clear_zipapps_cache}
 
 
 def ensure_path(path):
@@ -37,13 +38,15 @@ def rm_dir_or_file(path: Path):
     for _ in range(3):
         try:
             if path.is_dir():
-                rmtree(str(path.absolute()))
+                rmtree(str(path.absolute()), ignore_errors=True)
             elif path.is_file():
                 path.unlink()
             else:
                 break
         except FileNotFoundError:
             break
+        except PermissionError:
+            break
     else:
         return False
     return True
@@ -62,8 +65,18 @@ def prepare_path():
     zip_file_path = Path(__file__).parent.absolute()
     _zipapps_python_path_list = [str(zip_file_path)]
     if unzip:
-        _cache_folder_path = ensure_path(_cache_folder)
-        _cache_folder_path = _cache_folder_path / zip_file_path.stem
+        _cache_folder_path_parent = ensure_path(_cache_folder)
+        _cache_folder_path = _cache_folder_path_parent / zip_file_path.stem
+        if clear_zipapps_cache:
+            import atexit
+
+            def _remove_cache_folder():
+                rm_dir_or_file(_cache_folder_path)
+                if not any(_cache_folder_path_parent.iterdir()):
+                    rm_dir_or_file(_cache_folder_path_parent)
+
+            atexit.register(_remove_cache_folder)
+
         _cache_folder_path.mkdir(parents=True, exist_ok=True)
         _cache_folder_path_str = str(_cache_folder_path.absolute())
         _zipapps_python_path_list.insert(0, _cache_folder_path_str)

zipapps/main.py

@@ -15,7 +15,7 @@
 from pkgutil import get_data
 from zipfile import ZIP_DEFLATED, ZIP_STORED, BadZipFile, ZipFile
 
-__version__ = '2021.09.22'
+__version__ = '2021.11.01'
 
 
 class ZipApp(object):
@@ -50,6 +50,7 @@ def __init__(
         ensure_pip: bool = False,
         layer_mode: bool = False,
         layer_mode_prefix: str = 'python',
+        clear_zipapps_cache: bool = False,
     ):
         """Zip your code.
 
@@ -95,6 +96,8 @@ def __init__(
         :type includes: bool, optional
         :param layer_mode_prefix: Only work while --layer-mode is set, will move the files in the given prefix folder.
         :type includes: str, optional
+        :param clear_zipapps_cache: Clear the zipapps cache folder after running, but maybe failed for .pyd/.so files..
+        :type includes: bool, optional
         """
         self.includes = includes
         self.cache_path = cache_path
@@ -118,6 +121,7 @@ def __init__(
         self.ensure_pip = ensure_pip
         self.layer_mode = layer_mode
         self.layer_mode_prefix = layer_mode_prefix
+        self.clear_zipapps_cache = clear_zipapps_cache
 
         self._tmp_dir: tempfile.TemporaryDirectory = None
         self._build_success = False
@@ -164,7 +168,7 @@ def prepare_ensure_pip(self):
 
     def build(self):
         self._log(
-            f'{"=" * 10} Start building `{self._output_path}` with zipapps version <{__version__}> {"=" * 10}'
+            f'[INFO]: {"=" * 10} Start building `{self._output_path}` with zipapps version <{__version__}> {"=" * 10}'
         )
         self.ensure_args()
         if self.build_exists():
@@ -237,6 +241,9 @@ def prepare_entry_point(self):
         auto_unzip_keys = ZipApp.AUTO_FIX_UNZIP_KEYS & unzip_names
         unzip_names -= auto_unzip_keys
         if warning_names:
+            if self.clear_zipapps_cache:
+                msg = f'[WARN]: clear_zipapps_cache is True but .pyd/.so files were found {warning_names}'
+                self._log(msg)
             if auto_unzip_keys:
                 unzip_names |= warning_names.keys()
             else:
@@ -279,6 +286,7 @@ def prepare_active_zipapps(self):
             'sys_paths': self.sys_paths,
             'python_version_slice': self.python_version_slice,
             'pip_args_md5': self.pip_args_md5,
+            'clear_zipapps_cache': repr(self.clear_zipapps_cache),
         }
         code = get_data('zipapps', '_entry_point.py').decode('u8')
         (self._cache_path / '__main__.py').write_text(code.format(**kwargs))
@@ -402,19 +410,19 @@ def create_app(cls, *args, **kwargs):
 
     @staticmethod
     def _log(text):
-        sys.stderr.write(f'{text}\n')
+        sys.stderr.write(f'{time.strftime("%Y-%m-%d %H:%M:%S")} | {text}\n')
 
     def __del__(self):
         if self._tmp_dir:
             self._tmp_dir.cleanup()
             self._log(
-                f'[INFO] Temp cache has been cleaned. ({self._tmp_dir!r})')
+                f'[INFO]: Temp cache has been cleaned. ({self._tmp_dir!r})')
         if self._build_success:
             self._log(
-                f'{"=" * 10} Successfully built `{self._output_path}` {"=" * 10}'
+                f'[INFO]: {"=" * 10} Successfully built `{self._output_path}` {"=" * 10}'
             )
         else:
-            self._log(f'{"=" * 10} Build failed {"=" * 10}')
+            self._log(f'[ERROR]: {"=" * 10} Build failed {"=" * 10}')
 
 
 def create_app(*args, **kwargs):