chore: deprecate python 3.9, reaching EOL soon

also update ruff rules to be consistent with template

Author: karlicoss

.github/workflows/main.yml

@@ -30,15 +30,13 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
+        python-version: ['3.10', '3.11', '3.12', '3.13']
         exclude: [
             # windows runners are pretty scarce, so let's only run lowest and highest python version
-            {platform: windows-latest, python-version: '3.10'},
             {platform: windows-latest, python-version: '3.11'},
             {platform: windows-latest, python-version: '3.12'},
 
             # same, macos is a bit too slow and ubuntu covers python quirks well
-            {platform: macos-latest  , python-version: '3.10'},
             {platform: macos-latest  , python-version: '3.11'},
             {platform: macos-latest  , python-version: '3.12'},
         ]
@@ -70,19 +68,18 @@ jobs:
 
     # explicit bash command is necessary for Windows CI runner, otherwise it thinks it's cmd...
     - run: bash .ci/run
+      env:
+        # only compute lxml coverage on ubuntu; it crashes on windows
+        CI_MYPY_COVERAGE: ${{ matrix.platform == 'ubuntu-latest' && '--cobertura-xml-report .coverage.mypy-all' || '' }}
 
     - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
-      uses: actions/upload-artifact@v4
+      uses: codecov/codecov-action@v5
       with:
-        include-hidden-files: true
-        name: .coverage.mypy-core_${{ matrix.platform }}_${{ matrix.python-version }}
-        path: .coverage.mypy-core/
-    - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
-      uses: actions/upload-artifact@v4
-      with:
-        include-hidden-files: true
-        name: .coverage.mypy-misc_${{ matrix.platform }}_${{ matrix.python-version }}
-        path: .coverage.mypy-misc/
+        fail_ci_if_error: true  # default false
+        token: ${{ secrets.CODECOV_TOKEN }}
+        flags: mypy-${{ matrix.python-version }}
+        files: .coverage.mypy-all/cobertura.xml
+
 
   pypi:
     # Do not run it for PRs/cron schedule etc.

conftest.py

@@ -4,7 +4,6 @@
 
 import os
 import pathlib
-from typing import Optional
 
 import _pytest.main
 import _pytest.pathlib
@@ -22,7 +21,7 @@
 resolve_pkg_path_orig = _pytest.pathlib.resolve_package_path
 
 
-def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]:
+def resolve_package_path(path: pathlib.Path) -> pathlib.Path | None:
     result = path  # search from the test file upwards
     for parent in result.parents:
         if str(parent) in namespace_pkg_dirs:

pyproject.toml

@@ -17,7 +17,7 @@ dependencies = [
     "promnesia[server]",
     ##
 ]
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 
 ## these need to be set if you're planning to upload to pypi
 description = "Enhancement of your browsing history"
@@ -82,8 +82,7 @@ testing = [
     "ruff",
     "mypy",
     "lxml",  # for mypy coverage
-
-    "ty>=0.0.1a14",
+    "ty>=0.0.1a20",
 
     "hypothesis",
 

ruff.toml

@@ -1,43 +1,9 @@
+line-length = 120  # impacts import sorting
+
 lint.extend-select = [
-    "F",    # flakes rules -- default, but extend just in case
-    "E",    # pycodestyle  -- default, but extend just in case
-    "W",    # various warnings
-
-    "B",    # 'bugbear' set -- various possible bugs
-    "C4",   # flake8-comprehensions -- unnecessary list/map/dict calls
-    "COM",  # trailing commas
-    "EXE",  # various checks wrt executable files
-    "I",    # sort imports
-    "ICN",  # various import conventions
-    "FBT",  # detect use of boolean arguments
-    "FURB", # various rules
-    "PERF", # various potential performance speedups
-    "PD",   # pandas rules
-    "PIE",  # 'misc' lints
-    "PLC",  # pylint convention rules
-    "PLR",  # pylint refactor rules
-    "PLW",  # pylint warnings
-    "PT",   # pytest stuff
-    "PYI",  # various type hinting rules
-    "RET",  # early returns
-    "RUF",  # various ruff-specific rules
-    "TID",  # various imports suggestions
-    "TRY",  # various exception handling rules
-    "UP",   # detect deprecated python stdlib stuff
-    "FA",   # suggest using from __future__ import annotations
-    "PTH",  # pathlib migration
-    "ARG",  # unused argument checks
-    "A",    # builtin shadowing
-    "G",    # logging stuff
-
-    # "ALL", # uncomment this to check for new rules!
+    "ALL",
 ]
 
-# Preserve types, even if a file imports `from __future__ import annotations`
-# we need this for cachew to work with HPI types on 3.9
-# can probably remove after 3.10?
-lint.pyupgrade.keep-runtime-typing = true
-
 lint.ignore = [
     "D",     # annoying nags about docstrings
     "N",     # pep naming
@@ -51,7 +17,6 @@ lint.ignore = [
 
 ### too opinionated style checks
     "E501",  # too long lines
-    "E702",  # Multiple statements on one line (semicolon)
     "E731",  # assigning lambda instead of using def
     "E741",  # Ambiguous variable name: `l`
     "E742",  # Ambiguous class name: `O
@@ -66,9 +31,6 @@ lint.ignore = [
 ## might be nice .. but later and I don't wanna make it strict
     "E402",  # Module level import not at top of file
 
-    "RUF100",  # unused noqa -- handle later
-    "RUF012",  # mutable class attrs should be annotated with ClassVar... ugh pretty annoying for user configs
-
 ### these are just nitpicky, we usually know better
     "PLR0911",  # too many return statements
     "PLR0912",  # too many branches
@@ -83,10 +45,8 @@ lint.ignore = [
 
     "B009",  # calling gettattr with constant attribute -- this is useful to convince mypy
     "B010",  # same as above, but setattr
-    "B011",  # complains about assert False
     "B017",  # pytest.raises(Exception)
     "B023",  # seems to result in false positives?
-    "B028",  # suggest using explicit stacklevel? TODO double check later, but not sure it's useful
 
     # complains about useless pass, but has sort of a false positive if the function has a docstring?
     # this is common for click entrypoints (e.g. in __main__), so disable
@@ -106,44 +66,43 @@ lint.ignore = [
     "PLW0603",  # global variable update.. we usually know why we are doing this
     "PLW2901",  # for loop variable overwritten, usually this is intentional
 
-    "PT011",  # pytest raises should is too broad
-    "PT012",  # pytest raises should contain a single statement
+    "PT011",  # pytest raises is too broad
 
     "COM812",  # trailing comma missing -- mostly just being annoying with long multiline strings
 
-    "PD901",   # generic variable name df
-
     "TRY003",  # suggests defining exception messages in exception class -- kinda annoying
-    "TRY004",  # prefer TypeError -- don't see the point
     "TRY201",  # raise without specifying exception name -- sometimes hurts readability
-    "TRY400",  # TODO double check this, might be useful
+    "TRY400",  # a bit dumb, and results in false positives (see https://github.com/astral-sh/ruff/issues/18070)
     "TRY401",  # redundant exception in logging.exception call? TODO double check, might result in excessive logging
 
-    "PGH",  # TODO force error code in mypy instead? although it also has blanket noqa rule
-
     "TID252",  # Prefer absolute imports over relative imports from parent modules
 
-    "UP038",  # suggests using | (union) in isisntance checks.. but it results in slower code
-
     ## too annoying
-    "T20",     # just complains about prints and pprints
+    "T20",     # just complains about prints and pprints (TODO maybe consider later?)
     "Q",       # flake quotes, too annoying
     "C90",     # some complexity checking
     "G004",    # logging statement uses f string
     "ERA001",  # commented out code
     "SLF001",  # private member accessed
     "BLE001",  # do not catch 'blind' Exception
     "INP001",  # complains about implicit namespace packages
-    "SIM",     # some if statements crap
+    "SIM102",  # if statements collapsing, often hurts readability
+    "SIM103",  # multiple conditions collapsing, often hurts readability
+    "SIM105",  # suggests using contextlib.suppress instad of try/except -- this wouldn't be mypy friendly
+    "SIM108",  # suggests using ternary operation instead of if -- hurts readability
+    "SIM110",  # suggests using any(...) instead of for look/return -- hurts readability
+    "SIM117",  # suggests using single with statement instead of nested -- doesn't work in tests
     "RSE102",  # complains about missing parens in exceptions
     ##
 
     "PLC0415", # "imports should be at the top level" -- not realistic
     "ARG001",  # ugh, kinda annoying when using pytest fixtures
     "RUF001", "RUF002", "RUF003",  # spams about non-latin characters that we do use for testing
     "A005",  #  we're using promnesia.logging module
+    "B028",  # warnings stacklevel -- will fix later
 ]
 
+
 extend-exclude = [
     "tests/testdata/**",
 ]

scripts/browser_history.py

@@ -94,7 +94,14 @@ def test_backup_history(tmp_path):
 
 def guess_db_date(db: Path) -> str:
     maxvisit = (
-        check_output(['sqlite3', '-csv', db, 'SELECT max(datetime(((visits.visit_time/1000000)-11644473600), "unixepoch")) FROM visits;'])
+        check_output(
+            [
+                'sqlite3',
+                '-csv',
+                db,
+                'SELECT max(datetime(((visits.visit_time/1000000)-11644473600), "unixepoch")) FROM visits;',
+            ]
+        )
         .decode('utf8')
         .strip()
         .strip('"')
@@ -115,7 +122,10 @@ def main():
     p = argparse.ArgumentParser()
     p.add_argument('--browser', type=Browser, required=True)
     p.add_argument(
-        '--profile', type=str, default='*', help='Use to pick the correct profile to back up. If unspecified, will assume a single profile'
+        '--profile',
+        type=str,
+        default='*',
+        help='Use to pick the correct profile to back up. If unspecified, will assume a single profile',
     )
     p.add_argument('--to', type=Path, required=True)
     args = p.parse_args()

src/promnesia/__main__.py

@@ -8,11 +8,10 @@
 import shlex
 import shutil
 import sys
-from collections.abc import Iterable, Iterator, Sequence
+from collections.abc import Callable, Iterable, Iterator, Sequence
 from pathlib import Path
 from subprocess import Popen, check_call, run
 from tempfile import TemporaryDirectory, gettempdir
-from typing import Callable
 
 from . import config, server
 from .common import (
@@ -83,7 +82,9 @@ def iter_all_visits(sources_subset: Iterable[str | int] = ()) -> Iterator[Res[Db
         logger.warning("unknown --sources: %s", ", ".join(repr(i) for i in sources_subset))
 
 
-def _do_index(*, dry: bool = False, sources_subset: Iterable[str | int] = (), overwrite_db: bool = False) -> Iterable[Exception]:
+def _do_index(
+    *, dry: bool = False, sources_subset: Iterable[str | int] = (), overwrite_db: bool = False
+) -> Iterable[Exception]:
     # also keep & return errors for further display
     errors: list[Exception] = []
 
@@ -185,7 +186,9 @@ def do_demo(
 
         dbp = config.get().db
         if port is None:
-            logger.warning(f"Port isn't specified, not serving!\nYou can inspect the database in the meantime, e.g. 'sqlitebrowser {dbp}'")
+            logger.warning(
+                f"Port isn't specified, not serving!\nYou can inspect the database in the meantime, e.g. 'sqlitebrowser {dbp}'"
+            )
         else:
             from .server import ServerConfig
 
@@ -329,7 +332,9 @@ def add_index_args(parser: argparse.ArgumentParser, default_config_path: PathIsh
             if not given, all :func:`demo_sources()` are run
         """
         parser.add_argument('--config', type=Path, default=default_config_path, help='Config path')
-        parser.add_argument('--dry', action='store_true', help="Dry run, won't touch the database, only print the results out")
+        parser.add_argument(
+            '--dry', action='store_true', help="Dry run, won't touch the database, only print the results out"
+        )
         parser.add_argument(
             '--sources',
             required=False,
@@ -365,7 +370,9 @@ def add_index_args(parser: argparse.ArgumentParser, default_config_path: PathIsh
 
     ap.add_argument('--name', type=str, default='demo', help='Set custom source name')
     add_port_arg(ap)
-    ap.add_argument('--no-serve', action='store_const', const=None, dest='port', help='Pass to only index without running server')
+    ap.add_argument(
+        '--no-serve', action='store_const', const=None, dest='port', help='Pass to only index without running server'
+    )
     ap.add_argument(
         '--as',
         choices=sorted(demo_sources().keys()),
@@ -375,7 +382,9 @@ def add_index_args(parser: argparse.ArgumentParser, default_config_path: PathIsh
     add_index_args(ap)
     ap.add_argument('params', nargs='*', help='Optional extra params for the indexer')
 
-    isp = subp.add_parser('install-server', help='Install server as a systemd service (for autostart)', formatter_class=F)
+    isp = subp.add_parser(
+        'install-server', help='Install server as a systemd service (for autostart)', formatter_class=F
+    )
     install_server.setup_parser(isp)
 
     cp = subp.add_parser('config', help='Config management')
@@ -441,9 +450,7 @@ def add_index_args(parser: argparse.ArgumentParser, default_config_path: PathIsh
             )
         elif mode == 'install-server':  # todo rename to 'autostart' or something?
             install_server.install(args)
-        elif mode == 'config':
-            args.func(args)
-        elif mode == 'doctor':
+        elif mode == 'config' or mode == 'doctor':
             args.func(args)
         else:
             raise AssertionError(f'unexpected mode {mode}')

src/promnesia/cannon.py

@@ -19,7 +19,7 @@
 
 # TODO eh?? they fixed mobile.twitter.com?
 from itertools import chain
-from typing import Any, NamedTuple, Union
+from typing import Any, NamedTuple
 from urllib.parse import SplitResult, parse_qsl, urlencode, urlsplit, urlunsplit
 
 # this has some benchmark, but quite a few librarires seem unmaintained, sadly
@@ -266,7 +266,7 @@ def _prenormalise(url: str) -> str:
     return url
 
 
-Left = Union[str, Sequence[str]]
+Left = str | Sequence[str]
 Right = tuple[str, str, str]