From 64848b900f4f65293d7a81a5e1229382c5b66f55 Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Wed, 31 Jul 2024 17:31:35 +0200 Subject: [PATCH 01/32] adding this flag due to https://github.com/astral-sh/ruff/issues/5434 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 23dd1b35..43fe23a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: rev: v3.16.0 hooks: - id: pyupgrade - args: [--py39-plus] + args: [--py39-plus, --keep-runtime-typing] - repo: https://github.com/asottile/add-trailing-comma rev: v2.2.3 hooks: From eb5ee3f6f65b8f2c6860e21281aa5fd0264b8d7b Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Wed, 31 Jul 2024 19:45:40 +0200 Subject: [PATCH 02/32] first pydantic model for config --- .pre-commit-config.yaml | 2 +- poetry.lock | 146 +++++++++++++++-- pyproject.toml | 1 + sed/config/config_model.py | 224 +++++++++++++++++++++++++++ sed/config/default.yaml | 43 ++--- sed/config/flash_example_config.yaml | 131 +++++++--------- sed/core/config.py | 9 +- 7 files changed, 445 insertions(+), 111 deletions(-) create mode 100644 sed/config/config_model.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 43fe23a0..23dd1b35 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: rev: v3.16.0 hooks: - id: pyupgrade - args: [--py39-plus, --keep-runtime-typing] + args: [--py39-plus] - repo: https://github.com/asottile/add-trailing-comma rev: v2.2.3 hooks: diff --git a/poetry.lock b/poetry.lock index f1b02cae..971e9be1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -40,6 +40,17 @@ files = [ {file = "alabaster-0.7.16.tar.gz", hash = "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65"}, ] +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + [[package]] name = "anyio" version = "3.7.1" @@ -2906,12 +2917,12 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, - {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, - {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\" and python_version < \"3.11\""}, - {version = ">=1.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version >= \"3.8\" and python_version < \"3.10\" or python_version > \"3.9\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_system != \"Darwin\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, {version = ">=1.21.0", markers = "python_version == \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\""}, {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\" and python_version < \"3.11\""}, + {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\" and python_version < \"3.11\""}, + {version = ">=1.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version >= \"3.8\" and python_version < \"3.10\" or python_version > \"3.9\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_system != \"Darwin\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, + {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] [[package]] @@ -2999,7 +3010,6 @@ optional = false python-versions = ">=3.9" files = [ {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"}, - {file = "pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238"}, {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"}, {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"}, {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"}, @@ -3013,14 +3023,12 @@ files = [ {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"}, {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"}, {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"}, - {file = "pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce"}, {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"}, {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"}, {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"}, {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"}, {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"}, {file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"}, - {file = "pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd"}, {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"}, {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"}, {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"}, @@ -3031,9 +3039,9 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -3455,6 +3463,126 @@ files = [ {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] +[[package]] +name = "pydantic" +version = "2.8.2" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic-2.8.2-py3-none-any.whl", hash = "sha256:73ee9fddd406dc318b885c7a2eab8a6472b68b8fb5ba8150949fc3db939f23c8"}, + {file = "pydantic-2.8.2.tar.gz", hash = "sha256:6f62c13d067b0755ad1c21a34bdd06c0c12625a22b0fc09c6b149816604f7c2a"}, +] + +[package.dependencies] +annotated-types = ">=0.4.0" +pydantic-core = "2.20.1" +typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""} + +[package.extras] +email = ["email-validator (>=2.0.0)"] + +[[package]] +name = "pydantic-core" +version = "2.20.1" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_core-2.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3acae97ffd19bf091c72df4d726d552c473f3576409b2a7ca36b2f535ffff4a3"}, + {file = "pydantic_core-2.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:41f4c96227a67a013e7de5ff8f20fb496ce573893b7f4f2707d065907bffdbd6"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f239eb799a2081495ea659d8d4a43a8f42cd1fe9ff2e7e436295c38a10c286a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53e431da3fc53360db73eedf6f7124d1076e1b4ee4276b36fb25514544ceb4a3"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1f62b2413c3a0e846c3b838b2ecd6c7a19ec6793b2a522745b0869e37ab5bc1"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d41e6daee2813ecceea8eda38062d69e280b39df793f5a942fa515b8ed67953"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d482efec8b7dc6bfaedc0f166b2ce349df0011f5d2f1f25537ced4cfc34fd98"}, + {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e93e1a4b4b33daed65d781a57a522ff153dcf748dee70b40c7258c5861e1768a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e7c4ea22b6739b162c9ecaaa41d718dfad48a244909fe7ef4b54c0b530effc5a"}, + {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4f2790949cf385d985a31984907fecb3896999329103df4e4983a4a41e13e840"}, + {file = "pydantic_core-2.20.1-cp310-none-win32.whl", hash = "sha256:5e999ba8dd90e93d57410c5e67ebb67ffcaadcea0ad973240fdfd3a135506250"}, + {file = "pydantic_core-2.20.1-cp310-none-win_amd64.whl", hash = "sha256:512ecfbefef6dac7bc5eaaf46177b2de58cdf7acac8793fe033b24ece0b9566c"}, + {file = "pydantic_core-2.20.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d2a8fa9d6d6f891f3deec72f5cc668e6f66b188ab14bb1ab52422fe8e644f312"}, + {file = "pydantic_core-2.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:175873691124f3d0da55aeea1d90660a6ea7a3cfea137c38afa0a5ffabe37b88"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37eee5b638f0e0dcd18d21f59b679686bbd18917b87db0193ae36f9c23c355fc"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25e9185e2d06c16ee438ed39bf62935ec436474a6ac4f9358524220f1b236e43"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:150906b40ff188a3260cbee25380e7494ee85048584998c1e66df0c7a11c17a6"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ad4aeb3e9a97286573c03df758fc7627aecdd02f1da04516a86dc159bf70121"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3f3ed29cd9f978c604708511a1f9c2fdcb6c38b9aae36a51905b8811ee5cbf1"}, + {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0dae11d8f5ded51699c74d9548dcc5938e0804cc8298ec0aa0da95c21fff57b"}, + {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:faa6b09ee09433b87992fb5a2859efd1c264ddc37280d2dd5db502126d0e7f27"}, + {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9dc1b507c12eb0481d071f3c1808f0529ad41dc415d0ca11f7ebfc666e66a18b"}, + {file = "pydantic_core-2.20.1-cp311-none-win32.whl", hash = "sha256:fa2fddcb7107e0d1808086ca306dcade7df60a13a6c347a7acf1ec139aa6789a"}, + {file = "pydantic_core-2.20.1-cp311-none-win_amd64.whl", hash = "sha256:40a783fb7ee353c50bd3853e626f15677ea527ae556429453685ae32280c19c2"}, + {file = "pydantic_core-2.20.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:595ba5be69b35777474fa07f80fc260ea71255656191adb22a8c53aba4479231"}, + {file = "pydantic_core-2.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a4f55095ad087474999ee28d3398bae183a66be4823f753cd7d67dd0153427c9"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9aa05d09ecf4c75157197f27cdc9cfaeb7c5f15021c6373932bf3e124af029f"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e97fdf088d4b31ff4ba35db26d9cc472ac7ef4a2ff2badeabf8d727b3377fc52"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc633a9fe1eb87e250b5c57d389cf28998e4292336926b0b6cdaee353f89a237"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d573faf8eb7e6b1cbbcb4f5b247c60ca8be39fe2c674495df0eb4318303137fe"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26dc97754b57d2fd00ac2b24dfa341abffc380b823211994c4efac7f13b9e90e"}, + {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:33499e85e739a4b60c9dac710c20a08dc73cb3240c9a0e22325e671b27b70d24"}, + {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bebb4d6715c814597f85297c332297c6ce81e29436125ca59d1159b07f423eb1"}, + {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:516d9227919612425c8ef1c9b869bbbee249bc91912c8aaffb66116c0b447ebd"}, + {file = "pydantic_core-2.20.1-cp312-none-win32.whl", hash = "sha256:469f29f9093c9d834432034d33f5fe45699e664f12a13bf38c04967ce233d688"}, + {file = "pydantic_core-2.20.1-cp312-none-win_amd64.whl", hash = "sha256:035ede2e16da7281041f0e626459bcae33ed998cca6a0a007a5ebb73414ac72d"}, + {file = "pydantic_core-2.20.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:0827505a5c87e8aa285dc31e9ec7f4a17c81a813d45f70b1d9164e03a813a686"}, + {file = "pydantic_core-2.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19c0fa39fa154e7e0b7f82f88ef85faa2a4c23cc65aae2f5aea625e3c13c735a"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa223cd1e36b642092c326d694d8bf59b71ddddc94cdb752bbbb1c5c91d833b"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c336a6d235522a62fef872c6295a42ecb0c4e1d0f1a3e500fe949415761b8a19"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7eb6a0587eded33aeefea9f916899d42b1799b7b14b8f8ff2753c0ac1741edac"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70c8daf4faca8da5a6d655f9af86faf6ec2e1768f4b8b9d0226c02f3d6209703"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9fa4c9bf273ca41f940bceb86922a7667cd5bf90e95dbb157cbb8441008482c"}, + {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:11b71d67b4725e7e2a9f6e9c0ac1239bbc0c48cce3dc59f98635efc57d6dac83"}, + {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:270755f15174fb983890c49881e93f8f1b80f0b5e3a3cc1394a255706cabd203"}, + {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c81131869240e3e568916ef4c307f8b99583efaa60a8112ef27a366eefba8ef0"}, + {file = "pydantic_core-2.20.1-cp313-none-win32.whl", hash = "sha256:b91ced227c41aa29c672814f50dbb05ec93536abf8f43cd14ec9521ea09afe4e"}, + {file = "pydantic_core-2.20.1-cp313-none-win_amd64.whl", hash = "sha256:65db0f2eefcaad1a3950f498aabb4875c8890438bc80b19362cf633b87a8ab20"}, + {file = "pydantic_core-2.20.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4745f4ac52cc6686390c40eaa01d48b18997cb130833154801a442323cc78f91"}, + {file = "pydantic_core-2.20.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8ad4c766d3f33ba8fd692f9aa297c9058970530a32c728a2c4bfd2616d3358b"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41e81317dd6a0127cabce83c0c9c3fbecceae981c8391e6f1dec88a77c8a569a"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04024d270cf63f586ad41fff13fde4311c4fc13ea74676962c876d9577bcc78f"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eaad4ff2de1c3823fddf82f41121bdf453d922e9a238642b1dedb33c4e4f98ad"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26ab812fa0c845df815e506be30337e2df27e88399b985d0bb4e3ecfe72df31c"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c5ebac750d9d5f2706654c638c041635c385596caf68f81342011ddfa1e5598"}, + {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2aafc5a503855ea5885559eae883978c9b6d8c8993d67766ee73d82e841300dd"}, + {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4868f6bd7c9d98904b748a2653031fc9c2f85b6237009d475b1008bfaeb0a5aa"}, + {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aa2f457b4af386254372dfa78a2eda2563680d982422641a85f271c859df1987"}, + {file = "pydantic_core-2.20.1-cp38-none-win32.whl", hash = "sha256:225b67a1f6d602de0ce7f6c1c3ae89a4aa25d3de9be857999e9124f15dab486a"}, + {file = "pydantic_core-2.20.1-cp38-none-win_amd64.whl", hash = "sha256:6b507132dcfc0dea440cce23ee2182c0ce7aba7054576efc65634f080dbe9434"}, + {file = "pydantic_core-2.20.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b03f7941783b4c4a26051846dea594628b38f6940a2fdc0df00b221aed39314c"}, + {file = "pydantic_core-2.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1eedfeb6089ed3fad42e81a67755846ad4dcc14d73698c120a82e4ccf0f1f9f6"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:635fee4e041ab9c479e31edda27fcf966ea9614fff1317e280d99eb3e5ab6fe2"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:77bf3ac639c1ff567ae3b47f8d4cc3dc20f9966a2a6dd2311dcc055d3d04fb8a"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ed1b0132f24beeec5a78b67d9388656d03e6a7c837394f99257e2d55b461611"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6514f963b023aeee506678a1cf821fe31159b925c4b76fe2afa94cc70b3222b"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10d4204d8ca33146e761c79f83cc861df20e7ae9f6487ca290a97702daf56006"}, + {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d036c7187b9422ae5b262badb87a20a49eb6c5238b2004e96d4da1231badef1"}, + {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ebfef07dbe1d93efb94b4700f2d278494e9162565a54f124c404a5656d7ff09"}, + {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6b9d9bb600328a1ce523ab4f454859e9d439150abb0906c5a1983c146580ebab"}, + {file = "pydantic_core-2.20.1-cp39-none-win32.whl", hash = "sha256:784c1214cb6dd1e3b15dd8b91b9a53852aed16671cc3fbe4786f4f1db07089e2"}, + {file = "pydantic_core-2.20.1-cp39-none-win_amd64.whl", hash = "sha256:d2fe69c5434391727efa54b47a1e7986bb0186e72a41b203df8f5b0a19a4f669"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a45f84b09ac9c3d35dfcf6a27fd0634d30d183205230a0ebe8373a0e8cfa0906"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d02a72df14dfdbaf228424573a07af10637bd490f0901cee872c4f434a735b94"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2b27e6af28f07e2f195552b37d7d66b150adbaa39a6d327766ffd695799780f"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084659fac3c83fd674596612aeff6041a18402f1e1bc19ca39e417d554468482"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:242b8feb3c493ab78be289c034a1f659e8826e2233786e36f2893a950a719bb6"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:38cf1c40a921d05c5edc61a785c0ddb4bed67827069f535d794ce6bcded919fc"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e0bbdd76ce9aa5d4209d65f2b27fc6e5ef1312ae6c5333c26db3f5ade53a1e99"}, + {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:254ec27fdb5b1ee60684f91683be95e5133c994cc54e86a0b0963afa25c8f8a6"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:407653af5617f0757261ae249d3fba09504d7a71ab36ac057c938572d1bc9331"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c693e916709c2465b02ca0ad7b387c4f8423d1db7b4649c551f27a529181c5ad"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b5ff4911aea936a47d9376fd3ab17e970cc543d1b68921886e7f64bd28308d1"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177f55a886d74f1808763976ac4efd29b7ed15c69f4d838bbd74d9d09cf6fa86"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:964faa8a861d2664f0c7ab0c181af0bea66098b1919439815ca8803ef136fc4e"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4dd484681c15e6b9a977c785a345d3e378d72678fd5f1f3c0509608da24f2ac0"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f6d6cff3538391e8486a431569b77921adfcdef14eb18fbf19b7c0a5294d4e6a"}, + {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a6d511cc297ff0883bc3708b465ff82d7560193169a8b93260f74ecb0a5e08a7"}, + {file = "pydantic_core-2.20.1.tar.gz", hash = "sha256:26ca695eeee5f9f1aeeb211ffc12f10bcb6f71e2989988fda61dabd65db878d4"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + [[package]] name = "pyerfa" version = "2.0.1.4" diff --git a/pyproject.toml b/pyproject.toml index 75e4892b..8081328b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ jupyter = {version = ">=1.0.0", optional = true} ipykernel = {version = ">=6.9.1", optional = true} jupyterlab = {version = "^3.4.0", optional = true} jupyterlab-h5web = {version = "^8.0.0", extras = ["full"]} +pydantic = "^2.8.2" [tool.poetry.extras] notebook = ["jupyter", "ipykernel", "jupyterlab", "jupyterlab-h5web"] diff --git a/sed/config/config_model.py b/sed/config/config_model.py new file mode 100644 index 00000000..b500e5b3 --- /dev/null +++ b/sed/config/config_model.py @@ -0,0 +1,224 @@ +from collections.abc import Sequence +from typing import Literal +from typing import Optional +from typing import Union + +from pydantic import BaseModel +from pydantic import DirectoryPath +from pydantic import field_validator +from pydantic import FilePath +from pydantic import HttpUrl +from pydantic import NewPath +from pydantic import SecretStr +from typing_extensions import NotRequired +from typing_extensions import TypedDict + +from sed.loader.loader_interface import get_names_of_all_loaders + +## Best to not use futures annotations with pydantic models +## https://github.com/astral-sh/ruff/issues/5434 + + +class Paths(BaseModel): + raw: DirectoryPath + processed: Union[DirectoryPath, NewPath] + + +class CoreModel(BaseModel): + loader: str = "generic" + paths: Optional[Paths] = None + num_cores: int = 4 + year: Optional[int] = None + beamtime_id: Optional[int] = None + instrument: Optional[str] = None + + @field_validator("loader") + @classmethod + def validate_loader(cls, v: str) -> str: + """Checks if the loader is one of the valid ones""" + names = get_names_of_all_loaders() + if v not in names: + raise ValueError(f"Invalid loader {v}. Available loaders are: {names}") + return v + + +class ColumnsModel(BaseModel): + x: str = "X" + y: str = "Y" + tof: str = "t" + tof_ns: str = "t_ns" + corrected_x: str = "Xm" + corrected_y: str = "Ym" + corrected_tof: str = "tm" + kx: str = "kx" + ky: str = "ky" + energy: str = "energy" + delay: str = "delay" + adc: str = "ADC" + bias: str = "sampleBias" + timestamp: str = "timeStamp" + + +class subChannelModel(TypedDict): + slice: int + dtype: NotRequired[str] + + +# Either channels is changed to not be dict or we use TypedDict +# However TypedDict can't accept default values +class ChannelModel(TypedDict): + format: Literal["per_train", "per_electron", "per_pulse", "per_file"] + dataset_key: str + index_key: NotRequired[str] + slice: NotRequired[int] + dtype: NotRequired[str] + sub_channels: NotRequired[dict[str, subChannelModel]] + + +class Dataframe(BaseModel): + columns: ColumnsModel = ColumnsModel() + units: Optional[dict[str, str]] = None + # Since channels are not fixed, we use a TypedDict to represent them. + channels: Optional[dict[str, ChannelModel]] = None + + tof_binwidth: float = 4.125e-12 + tof_binning: int = 1 + adc_binning: int = 1 + jitter_cols: Sequence[str] = ["@x", "@y", "@tof"] + jitter_amps: Union[float, Sequence[float]] = 0.5 + timed_dataframe_unit_time: float = 0.001 + # flash specific settings + forward_fill_iterations: Optional[int] = None + ubid_offset: Optional[int] = None + split_sector_id_from_dld_time: Optional[bool] = None + sector_id_reserved_bits: Optional[int] = None + sector_delays: Optional[Sequence[int]] = None + + +class BinningModel(BaseModel): + hist_mode: str = "numba" + mode: str = "fast" + pbar: bool = True + threads_per_worker: int = 4 + threadpool_API: str = "blas" + + +class HistogramModel(BaseModel): + bins: Sequence[int] = [80, 80, 80] + axes: Sequence[str] = ["@x", "@y", "@tof"] + ranges: Sequence[Sequence[int]] = [[0, 1800], [0, 1800], [0, 150000]] + + +class StaticModel(BaseModel): + """Static configuration settings that shouldn't be changed by users.""" + + # flash specific settings + stream_name_prefixes: Optional[dict] = None + stream_name_postfixes: Optional[dict] = None + beamtime_dir: Optional[dict] = None + + +class EnergyCalibrationModel(BaseModel): + d: float + t0: float + E0: float + energy_scale: str + + +class EnergyCorrectionModel(BaseModel): + correction_type: str = "Lorentzian" + amplitude: float + center: Sequence[float] + gamma: float + sigma: float + diameter: float + + +class EnergyModel(BaseModel): + bins: int = 1000 + ranges: Sequence[int] = [100000, 150000] + normalize: bool = True + normalize_span: int = 7 + normalize_order: int = 1 + fastdtw_radius: int = 2 + peak_window: int = 7 + calibration_method: str = "lmfit" + energy_scale: str = "kinetic" + tof_fermi: int = 132250 + tof_width: Sequence[int] = [-600, 1000] + x_width: Sequence[int] = [-20, 20] + y_width: Sequence[int] = [-20, 20] + color_clip: int = 300 + calibration: Optional[EnergyCalibrationModel] = None + correction: Optional[EnergyCorrectionModel] = None + + +class MomentumCalibrationModel(BaseModel): + kx_scale: float + ky_scale: float + x_center: float + y_center: float + rstart: float + cstart: float + rstep: float + cstep: float + + +class MomentumCorrectionModel(BaseModel): + feature_points: Sequence[Sequence[float]] + rotation_symmetry: int + include_center: bool + use_center: bool + + +class MomentumModel(BaseModel): + axes: Sequence[str] = ["@x", "@y", "@tof"] + bins: Sequence[int] = [512, 512, 300] + ranges: Sequence[Sequence[int]] = [[-256, 1792], [-256, 1792], [132000, 138000]] + detector_ranges: Sequence[Sequence[int]] = [[0, 2048], [0, 2048]] + center_pixel: Sequence[int] = [256, 256] + sigma: int = 5 + fwhm: int = 8 + sigma_radius: int = 1 + calibration: Optional[MomentumCalibrationModel] = None + correction: Optional[MomentumCorrectionModel] = None + + +class DelayModel(BaseModel): + adc_range: Sequence[int] = [1900, 25600] + time0: int = 0 + flip_time_axis: bool = False + p1_key: Optional[str] = None + p2_key: Optional[str] = None + p3_key: Optional[str] = None + + +class MetadataModel(BaseModel): + archiver_url: Optional[HttpUrl] = None + token: Optional[SecretStr] = None + epics_pvs: Optional[Sequence[str]] = None + fa_in_channel: Optional[str] = None + fa_hor_channel: Optional[str] = None + ca_in_channel: Optional[str] = None + aperture_config: Optional[dict] = None + lens_mode_config: Optional[dict] = None + + +class NexusModel(BaseModel): + reader: str # prob good to have validation here + # Currently only NXmpes definition is supported + definition: Literal["NXmpes"] + input_files: Sequence[FilePath] + + +class ConfigModel(BaseModel): + core: CoreModel + dataframe: Dataframe + energy: EnergyModel + momentum: MomentumModel + delay: DelayModel + binning: BinningModel + histogram: HistogramModel + metadata: Optional[MetadataModel] = None + nexus: Optional[NexusModel] = None + static: Optional[StaticModel] = None diff --git a/sed/config/default.yaml b/sed/config/default.yaml index b047d8a8..d6c15956 100644 --- a/sed/config/default.yaml +++ b/sed/config/default.yaml @@ -3,32 +3,23 @@ core: loader: generic dataframe: - # dataframe column containing x coordinates - x_column: "X" - # dataframe column containing y coordinates - y_column: "Y" - # dataframe column containing time-of-flight data - tof_column: "t" - # dataframe column containing time-of-flight data in nanoseconds - tof_ns_column: "t_ns" - # dataframe column containing analog-to-digital data - adc_column: "ADC" - # dataframe column containing bias voltage data - bias_column: "sampleBias" - # dataframe column containing corrected x coordinates - corrected_x_column: "Xm" - # dataframe column containing corrected y coordinates - corrected_y_column: "Ym" - # dataframe column containing corrected time-of-flight data - corrected_tof_column: "tm" - # dataframe column containing kx coordinates - kx_column: "kx" - # dataframe column containing ky coordinates - ky_column: "ky" - # dataframe column containing energy data - energy_column: "energy" - # dataframe column containing delay data - delay_column: "delay" + # Column settings + columns: + x: X # dataframe column containing x coordinates + y: Y # dataframe column containing y coordinates + tof: t # dataframe column containing time-of-flight data + tof_ns: t_ns # dataframe column containing time-of-flight data in nanoseconds + corrected_x: Xm # dataframe column containing corrected x coordinates + corrected_y: Ym # dataframe column containing corrected y coordinates + corrected_tof: tm # dataframe column containing corrected time-of-flight data + kx: kx # dataframe column containing kx coordinates + ky: ky # dataframe column containing ky coordinates + energy: energy # dataframe column containing energy data + delay: delay # dataframe column containing delay data + adc: ADC # dataframe column containing analog-to-digital data + bias: sampleBias # dataframe column containing bias voltage data + timestamp: timeStamp # dataframe column containing timestamp data + # time length of a base time-of-flight bin in s tof_binwidth: 4.125e-12 # Binning factor of the tof_column-data compared to tof_binwidth diff --git a/sed/config/flash_example_config.yaml b/sed/config/flash_example_config.yaml index ccc7dcac..03867513 100644 --- a/sed/config/flash_example_config.yaml +++ b/sed/config/flash_example_config.yaml @@ -1,9 +1,17 @@ # This file contains the default configuration for the flash loader. +# The paths to the raw and parquet data directories. If these are not +# provided, the loader will try to find the data based on year beamtimeID etc +paths: + # location of the raw data. + raw: "" + # location of the intermediate parquet files. + processed: "" + core: # defines the loader - loader: flash - # Since this will run on maxwell most probably, we have a lot of cores at our disposal + loader: 'flash' + # # Since this will run on maxwell most probably, we have a lot of cores at our disposal num_cores: 100 # the beamline where experiment took place beamline: pg2 @@ -14,14 +22,6 @@ core: # the instrument used instrument: hextof # hextof, wespe, etc - # The paths to the raw and parquet data directories. If these are not - # provided, the loader will try to find the data based on year beamtimeID etc - paths: - # location of the raw data. - raw: "" - # location of the intermediate parquet files. - processed: "" - binning: # Histogram computation mode to use. hist_mode: "numba" @@ -35,57 +35,39 @@ binning: threadpool_API: "blas" dataframe: - # The name of the DAQ system to use. Necessary to resolve the filenames/paths. - daq: fl1user3 - # The offset correction to the pulseId - ubid_offset: 5 - - # the number of iterations to fill the pulseId forward. - forward_fill_iterations: 2 - # if true, removes the 3 bits reserved for dldSectorID from the dldTimeSteps column - split_sector_id_from_dld_time: True - # bits reserved for dldSectorID in the dldTimeSteps column - sector_id_reserved_bits: 3 - # dataframe column containing x coordinates - x_column: dldPosX - # dataframe column containing corrected x coordinates - corrected_x_column: "X" - # dataframe column containing kx coordinates - kx_column: "kx" - # dataframe column containing y coordinates - y_column: dldPosY - # dataframe column containing corrected y coordinates - corrected_y_column: "Y" - # dataframe column containing kx coordinates - ky_column: "ky" - # dataframe column containing time-of-flight data - tof_column: dldTimeSteps - # dataframe column containing time-of-flight data in ns - tof_ns_column: dldTime - # dataframe column containing corrected time-of-flight data - corrected_tof_column: "tm" - # the time stamp column - time_stamp_alias: timeStamp - # auxiliary channel alias - aux_alias: dldAux - # aux subchannels alias - aux_subchannels_alias: dldAuxChannels - # time length of a base time-of-flight bin in seconds - tof_binwidth: 2.0576131995767355E-11 - # binning parameter for time-of-flight data. - tof_binning: 8 - # dataframe column containing sector ID. obtained from dldTimeSteps column - sector_id_column: dldSectorID - sector_delays: [0., 0., 0., 0., 0., 0., 0., 0.] - # the delay stage column - delay_column: delayStage - # the corrected pump-probe time axis - corrected_delay_column: pumpProbeTime - # the columns to be used for jitter correction - jitter_cols: ["dldPosX", "dldPosY", "dldTimeSteps"] - + daq: fl1user3 # DAQ system name to resolve filenames/paths + ubid_offset: 5 # Offset correction to the pulseId + forward_fill_iterations: 2 # Number of iterations to fill the pulseId forward + split_sector_id_from_dld_time: True # Remove reserved bits for dldSectorID from dldTimeSteps column + sector_id_reserved_bits: 3 # Bits reserved for dldSectorID in the dldTimeSteps column + sector_delays: [0., 0., 0., 0., 0., 0., 0., 0.] # Sector delays + + # Time and binning settings + tof_binwidth: 2.0576131995767355E-11 # Base time-of-flight bin width in seconds + tof_binning: 8 # Binning parameter for time-of-flight data + + # Columns used for jitter correction + jitter_columns: [dldPosX, dldPosY, dldTimeSteps] + + # Column settings + columns: + x: dldPosX + corrected_x: X + kx: kx + y: dldPosY + corrected_y: Y + ky: ky + tof: dldTimeSteps + tof_ns: dldTime + corrected_tof: tm + timestamp: timeStamp + auxiliary: dldAux + sector_id: dldSectorID + delay: delayStage + corrected_delay: pumpProbeTime + + # These are the units of the columns units: - # These are the units of the columns dldPosX: 'step' dldPosY: 'step' dldTimeSteps: 'step' @@ -102,7 +84,7 @@ dataframe: kx: '1/A' ky: '1/A' - # The channels to load. + # The channels to load from the raw data. The channels have the following structure: # channels have the following structure: # : # format: per_pulse/per_electron/per_train @@ -164,7 +146,7 @@ dataframe: index_key: "/uncategorised/FLASH.EXP/HEXTOF.DAQ/DLD1/index" dataset_key: "/uncategorised/FLASH.EXP/HEXTOF.DAQ/DLD1/value" slice: 4 - subChannels: + sub_channels: sampleBias: slice: 0 dtype: float32 @@ -215,8 +197,20 @@ dataframe: index_key: "/zraw/FLASH.SYNC/LASER.LOCK.EXP/F1.PG.OSC/FMC0.MD22.1.ENCODER_POSITION.RD/dGroup/index" dataset_key: "/zraw/FLASH.SYNC/LASER.LOCK.EXP/F1.PG.OSC/FMC0.MD22.1.ENCODER_POSITION.RD/dGroup/value" +# metadata collection from scicat +# metadata: +# scicat_url: +# scicat_token: + +# The nexus collection routine shall be finalized soon for both instruments +# nexus: +# reader: "flash" +# definition: "NXmpes" +# input_files: ["NXmpes_config_HEXTOF_light.json"] + +# (Not to be changed by user) +static: # The prefixes of the stream names for different DAQ systems for parsing filenames - # (Not to be changed by user) stream_name_prefixes: pbd: "GMD_DATA_gmd_data" pbd2: "FL2PhotDiag_pbd2_gmd_data" @@ -230,14 +224,3 @@ dataframe: # (Not to be changed by user) beamtime_dir: pg2: "/asap3/flash/gpfs/pg2/" - -# metadata collection from scicat -# metadata: -# scicat_url: -# scicat_token: - -# The nexus collection routine shall be finalized soon for both instruments -# nexus: -# reader: "flash" -# definition: "NXmpes" -# input_files: ["NXmpes_config_HEXTOF_light.json"] diff --git a/sed/core/config.py b/sed/core/config.py index 22e60fe5..8628352c 100644 --- a/sed/core/config.py +++ b/sed/core/config.py @@ -13,6 +13,7 @@ from platformdirs import user_config_path from sed.core.logging import setup_logging +from sed.config.config_model import ConfigModel package_dir = os.path.dirname(find_spec("sed").origin) @@ -29,6 +30,7 @@ def parse_config( system_config: dict | str = None, default_config: (dict | str) = f"{package_dir}/config/default.yaml", verbose: bool = True, + model: bool = False, ) -> dict: """Load the config dictionary from a file, or pass the provided config dictionary. The content of the loaded config dictionary is then completed from a set of pre-configured @@ -55,6 +57,7 @@ def parse_config( or file path. The loaded dictionary is completed with the default values. Defaults to *package_dir*/config/default.yaml". verbose (bool, optional): Option to report loaded config files. Defaults to True. + model (bool, optional): Option to return the config model instead of the dictionary. Raises: TypeError: Raised if the provided file is neither *json* nor *yaml*. FileNotFoundError: Raised if the provided file is not found. @@ -141,7 +144,11 @@ def parse_config( base_dictionary=default_dict, ) - return config_dict + # Run the config through the ConfigModel to ensure it is valid + config_model = ConfigModel(**config_dict) + if model: + return config_model + return config_model.model_dump() def load_config(config_path: str) -> dict: From 600ef5564c8baed71888fbdc2181048753fd85db Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Sat, 14 Sep 2024 21:19:36 +0200 Subject: [PATCH 03/32] without typeddict --- sed/config/config_model.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index b500e5b3..e70c4342 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -5,13 +5,12 @@ from pydantic import BaseModel from pydantic import DirectoryPath +from pydantic import Field from pydantic import field_validator from pydantic import FilePath from pydantic import HttpUrl from pydantic import NewPath from pydantic import SecretStr -from typing_extensions import NotRequired -from typing_extensions import TypedDict from sed.loader.loader_interface import get_names_of_all_loaders @@ -59,27 +58,25 @@ class ColumnsModel(BaseModel): timestamp: str = "timeStamp" -class subChannelModel(TypedDict): - slice: int - dtype: NotRequired[str] - - -# Either channels is changed to not be dict or we use TypedDict -# However TypedDict can't accept default values -class ChannelModel(TypedDict): +class ChannelModel(BaseModel): format: Literal["per_train", "per_electron", "per_pulse", "per_file"] dataset_key: str - index_key: NotRequired[str] - slice: NotRequired[int] - dtype: NotRequired[str] - sub_channels: NotRequired[dict[str, subChannelModel]] + index_key: Optional[str] = None + slice: Optional[int] = None + dtype: Optional[str] = None + + class subChannelModel(BaseModel): + slice: int + dtype: Optional[str] = None + + sub_channels: Optional[dict[str, subChannelModel]] = None class Dataframe(BaseModel): columns: ColumnsModel = ColumnsModel() units: Optional[dict[str, str]] = None # Since channels are not fixed, we use a TypedDict to represent them. - channels: Optional[dict[str, ChannelModel]] = None + channels: dict[str, ChannelModel] = Field(default_factory=dict) tof_binwidth: float = 4.125e-12 tof_binning: int = 1 From 24998df3a1b3d81f133d12949ebdaac33cf61040 Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Sun, 15 Sep 2024 01:38:41 +0200 Subject: [PATCH 04/32] remove defaults --- sed/config/config_model.py | 110 ++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index e70c4342..f7626908 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -1,3 +1,4 @@ +"""Pydantic model to validate the config for SED package.""" from collections.abc import Sequence from typing import Literal from typing import Optional @@ -42,20 +43,20 @@ def validate_loader(cls, v: str) -> str: class ColumnsModel(BaseModel): - x: str = "X" - y: str = "Y" - tof: str = "t" - tof_ns: str = "t_ns" - corrected_x: str = "Xm" - corrected_y: str = "Ym" - corrected_tof: str = "tm" - kx: str = "kx" - ky: str = "ky" - energy: str = "energy" - delay: str = "delay" - adc: str = "ADC" - bias: str = "sampleBias" - timestamp: str = "timeStamp" + x: str + y: str + tof: str + tof_ns: str + corrected_x: str + corrected_y: str + corrected_tof: str + kx: str + ky: str + energy: str + delay: str + adc: str + bias: str + timestamp: str class ChannelModel(BaseModel): @@ -75,15 +76,14 @@ class subChannelModel(BaseModel): class Dataframe(BaseModel): columns: ColumnsModel = ColumnsModel() units: Optional[dict[str, str]] = None - # Since channels are not fixed, we use a TypedDict to represent them. channels: dict[str, ChannelModel] = Field(default_factory=dict) - - tof_binwidth: float = 4.125e-12 - tof_binning: int = 1 - adc_binning: int = 1 - jitter_cols: Sequence[str] = ["@x", "@y", "@tof"] - jitter_amps: Union[float, Sequence[float]] = 0.5 - timed_dataframe_unit_time: float = 0.001 + # other settings + tof_binwidth: float + tof_binning: int + adc_binning: int + jitter_cols: Sequence[str] + jitter_amps: Union[float, Sequence[float]] + timed_dataframe_unit_time: float # flash specific settings forward_fill_iterations: Optional[int] = None ubid_offset: Optional[int] = None @@ -93,17 +93,17 @@ class Dataframe(BaseModel): class BinningModel(BaseModel): - hist_mode: str = "numba" - mode: str = "fast" - pbar: bool = True - threads_per_worker: int = 4 - threadpool_API: str = "blas" + hist_mode: str + mode: str + pbar: bool + threads_per_worker: int + threadpool_API: str class HistogramModel(BaseModel): - bins: Sequence[int] = [80, 80, 80] - axes: Sequence[str] = ["@x", "@y", "@tof"] - ranges: Sequence[Sequence[int]] = [[0, 1800], [0, 1800], [0, 150000]] + bins: Sequence[int] + axes: Sequence[str] + ranges: Sequence[Sequence[int]] class StaticModel(BaseModel): @@ -123,7 +123,7 @@ class EnergyCalibrationModel(BaseModel): class EnergyCorrectionModel(BaseModel): - correction_type: str = "Lorentzian" + correction_type: str amplitude: float center: Sequence[float] gamma: float @@ -132,20 +132,20 @@ class EnergyCorrectionModel(BaseModel): class EnergyModel(BaseModel): - bins: int = 1000 - ranges: Sequence[int] = [100000, 150000] - normalize: bool = True - normalize_span: int = 7 - normalize_order: int = 1 - fastdtw_radius: int = 2 - peak_window: int = 7 - calibration_method: str = "lmfit" - energy_scale: str = "kinetic" - tof_fermi: int = 132250 - tof_width: Sequence[int] = [-600, 1000] - x_width: Sequence[int] = [-20, 20] - y_width: Sequence[int] = [-20, 20] - color_clip: int = 300 + bins: int + ranges: Sequence[int] + normalize: bool + normalize_span: int + normalize_order: int + fastdtw_radius: int + peak_window: int + calibration_method: str + energy_scale: str + tof_fermi: int + tof_width: Sequence[int] + x_width: Sequence[int] + y_width: Sequence[int] + color_clip: int calibration: Optional[EnergyCalibrationModel] = None correction: Optional[EnergyCorrectionModel] = None @@ -169,21 +169,21 @@ class MomentumCorrectionModel(BaseModel): class MomentumModel(BaseModel): - axes: Sequence[str] = ["@x", "@y", "@tof"] - bins: Sequence[int] = [512, 512, 300] - ranges: Sequence[Sequence[int]] = [[-256, 1792], [-256, 1792], [132000, 138000]] - detector_ranges: Sequence[Sequence[int]] = [[0, 2048], [0, 2048]] - center_pixel: Sequence[int] = [256, 256] - sigma: int = 5 - fwhm: int = 8 - sigma_radius: int = 1 + axes: Sequence[str] + bins: Sequence[int] + ranges: Sequence[Sequence[int]] + detector_ranges: Sequence[Sequence[int]] + center_pixel: Sequence[int] + sigma: int + fwhm: int + sigma_radius: int calibration: Optional[MomentumCalibrationModel] = None correction: Optional[MomentumCorrectionModel] = None class DelayModel(BaseModel): - adc_range: Sequence[int] = [1900, 25600] - time0: int = 0 + adc_range: Sequence[int] + time0: int flip_time_axis: bool = False p1_key: Optional[str] = None p2_key: Optional[str] = None From bea6079fe1940e93387d10cf360b357d49cd99da Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Sun, 15 Sep 2024 01:49:25 +0200 Subject: [PATCH 05/32] update lock file with pydantic --- poetry.lock | 203 +++++++++++++++++++++++++------------------------ pyproject.toml | 2 +- 2 files changed, 103 insertions(+), 102 deletions(-) diff --git a/poetry.lock b/poetry.lock index 971e9be1..67de07be 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2917,12 +2917,12 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.21.0", markers = "python_version == \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\""}, - {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\" and python_version < \"3.11\""}, + {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, {version = ">=1.21.2", markers = "platform_system != \"Darwin\" and python_version >= \"3.10\" and python_version < \"3.11\""}, {version = ">=1.19.3", markers = "platform_system == \"Linux\" and platform_machine == \"aarch64\" and python_version >= \"3.8\" and python_version < \"3.10\" or python_version > \"3.9\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_system != \"Darwin\" and python_version < \"3.10\" or python_version >= \"3.9\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, - {version = ">=1.23.5", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, - {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.21.0", markers = "python_version == \"3.9\" and platform_system == \"Darwin\" and platform_machine == \"arm64\""}, + {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\" and python_version < \"3.11\""}, ] [[package]] @@ -3039,9 +3039,9 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.22.4", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -3465,119 +3465,120 @@ files = [ [[package]] name = "pydantic" -version = "2.8.2" +version = "2.9.1" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.8.2-py3-none-any.whl", hash = "sha256:73ee9fddd406dc318b885c7a2eab8a6472b68b8fb5ba8150949fc3db939f23c8"}, - {file = "pydantic-2.8.2.tar.gz", hash = "sha256:6f62c13d067b0755ad1c21a34bdd06c0c12625a22b0fc09c6b149816604f7c2a"}, + {file = "pydantic-2.9.1-py3-none-any.whl", hash = "sha256:7aff4db5fdf3cf573d4b3c30926a510a10e19a0774d38fc4967f78beb6deb612"}, + {file = "pydantic-2.9.1.tar.gz", hash = "sha256:1363c7d975c7036df0db2b4a61f2e062fbc0aa5ab5f2772e0ffc7191a4f4bce2"}, ] [package.dependencies] -annotated-types = ">=0.4.0" -pydantic-core = "2.20.1" +annotated-types = ">=0.6.0" +pydantic-core = "2.23.3" typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""} [package.extras] email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata"] [[package]] name = "pydantic-core" -version = "2.20.1" +version = "2.23.3" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.20.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3acae97ffd19bf091c72df4d726d552c473f3576409b2a7ca36b2f535ffff4a3"}, - {file = "pydantic_core-2.20.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:41f4c96227a67a013e7de5ff8f20fb496ce573893b7f4f2707d065907bffdbd6"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f239eb799a2081495ea659d8d4a43a8f42cd1fe9ff2e7e436295c38a10c286a"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:53e431da3fc53360db73eedf6f7124d1076e1b4ee4276b36fb25514544ceb4a3"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f1f62b2413c3a0e846c3b838b2ecd6c7a19ec6793b2a522745b0869e37ab5bc1"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d41e6daee2813ecceea8eda38062d69e280b39df793f5a942fa515b8ed67953"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d482efec8b7dc6bfaedc0f166b2ce349df0011f5d2f1f25537ced4cfc34fd98"}, - {file = "pydantic_core-2.20.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e93e1a4b4b33daed65d781a57a522ff153dcf748dee70b40c7258c5861e1768a"}, - {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e7c4ea22b6739b162c9ecaaa41d718dfad48a244909fe7ef4b54c0b530effc5a"}, - {file = "pydantic_core-2.20.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4f2790949cf385d985a31984907fecb3896999329103df4e4983a4a41e13e840"}, - {file = "pydantic_core-2.20.1-cp310-none-win32.whl", hash = "sha256:5e999ba8dd90e93d57410c5e67ebb67ffcaadcea0ad973240fdfd3a135506250"}, - {file = "pydantic_core-2.20.1-cp310-none-win_amd64.whl", hash = "sha256:512ecfbefef6dac7bc5eaaf46177b2de58cdf7acac8793fe033b24ece0b9566c"}, - {file = "pydantic_core-2.20.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d2a8fa9d6d6f891f3deec72f5cc668e6f66b188ab14bb1ab52422fe8e644f312"}, - {file = "pydantic_core-2.20.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:175873691124f3d0da55aeea1d90660a6ea7a3cfea137c38afa0a5ffabe37b88"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37eee5b638f0e0dcd18d21f59b679686bbd18917b87db0193ae36f9c23c355fc"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25e9185e2d06c16ee438ed39bf62935ec436474a6ac4f9358524220f1b236e43"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:150906b40ff188a3260cbee25380e7494ee85048584998c1e66df0c7a11c17a6"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ad4aeb3e9a97286573c03df758fc7627aecdd02f1da04516a86dc159bf70121"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3f3ed29cd9f978c604708511a1f9c2fdcb6c38b9aae36a51905b8811ee5cbf1"}, - {file = "pydantic_core-2.20.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0dae11d8f5ded51699c74d9548dcc5938e0804cc8298ec0aa0da95c21fff57b"}, - {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:faa6b09ee09433b87992fb5a2859efd1c264ddc37280d2dd5db502126d0e7f27"}, - {file = "pydantic_core-2.20.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9dc1b507c12eb0481d071f3c1808f0529ad41dc415d0ca11f7ebfc666e66a18b"}, - {file = "pydantic_core-2.20.1-cp311-none-win32.whl", hash = "sha256:fa2fddcb7107e0d1808086ca306dcade7df60a13a6c347a7acf1ec139aa6789a"}, - {file = "pydantic_core-2.20.1-cp311-none-win_amd64.whl", hash = "sha256:40a783fb7ee353c50bd3853e626f15677ea527ae556429453685ae32280c19c2"}, - {file = "pydantic_core-2.20.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:595ba5be69b35777474fa07f80fc260ea71255656191adb22a8c53aba4479231"}, - {file = "pydantic_core-2.20.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a4f55095ad087474999ee28d3398bae183a66be4823f753cd7d67dd0153427c9"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9aa05d09ecf4c75157197f27cdc9cfaeb7c5f15021c6373932bf3e124af029f"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e97fdf088d4b31ff4ba35db26d9cc472ac7ef4a2ff2badeabf8d727b3377fc52"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc633a9fe1eb87e250b5c57d389cf28998e4292336926b0b6cdaee353f89a237"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d573faf8eb7e6b1cbbcb4f5b247c60ca8be39fe2c674495df0eb4318303137fe"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26dc97754b57d2fd00ac2b24dfa341abffc380b823211994c4efac7f13b9e90e"}, - {file = "pydantic_core-2.20.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:33499e85e739a4b60c9dac710c20a08dc73cb3240c9a0e22325e671b27b70d24"}, - {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bebb4d6715c814597f85297c332297c6ce81e29436125ca59d1159b07f423eb1"}, - {file = "pydantic_core-2.20.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:516d9227919612425c8ef1c9b869bbbee249bc91912c8aaffb66116c0b447ebd"}, - {file = "pydantic_core-2.20.1-cp312-none-win32.whl", hash = "sha256:469f29f9093c9d834432034d33f5fe45699e664f12a13bf38c04967ce233d688"}, - {file = "pydantic_core-2.20.1-cp312-none-win_amd64.whl", hash = "sha256:035ede2e16da7281041f0e626459bcae33ed998cca6a0a007a5ebb73414ac72d"}, - {file = "pydantic_core-2.20.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:0827505a5c87e8aa285dc31e9ec7f4a17c81a813d45f70b1d9164e03a813a686"}, - {file = "pydantic_core-2.20.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19c0fa39fa154e7e0b7f82f88ef85faa2a4c23cc65aae2f5aea625e3c13c735a"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa223cd1e36b642092c326d694d8bf59b71ddddc94cdb752bbbb1c5c91d833b"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c336a6d235522a62fef872c6295a42ecb0c4e1d0f1a3e500fe949415761b8a19"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7eb6a0587eded33aeefea9f916899d42b1799b7b14b8f8ff2753c0ac1741edac"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70c8daf4faca8da5a6d655f9af86faf6ec2e1768f4b8b9d0226c02f3d6209703"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9fa4c9bf273ca41f940bceb86922a7667cd5bf90e95dbb157cbb8441008482c"}, - {file = "pydantic_core-2.20.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:11b71d67b4725e7e2a9f6e9c0ac1239bbc0c48cce3dc59f98635efc57d6dac83"}, - {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:270755f15174fb983890c49881e93f8f1b80f0b5e3a3cc1394a255706cabd203"}, - {file = "pydantic_core-2.20.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c81131869240e3e568916ef4c307f8b99583efaa60a8112ef27a366eefba8ef0"}, - {file = "pydantic_core-2.20.1-cp313-none-win32.whl", hash = "sha256:b91ced227c41aa29c672814f50dbb05ec93536abf8f43cd14ec9521ea09afe4e"}, - {file = "pydantic_core-2.20.1-cp313-none-win_amd64.whl", hash = "sha256:65db0f2eefcaad1a3950f498aabb4875c8890438bc80b19362cf633b87a8ab20"}, - {file = "pydantic_core-2.20.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:4745f4ac52cc6686390c40eaa01d48b18997cb130833154801a442323cc78f91"}, - {file = "pydantic_core-2.20.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a8ad4c766d3f33ba8fd692f9aa297c9058970530a32c728a2c4bfd2616d3358b"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41e81317dd6a0127cabce83c0c9c3fbecceae981c8391e6f1dec88a77c8a569a"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:04024d270cf63f586ad41fff13fde4311c4fc13ea74676962c876d9577bcc78f"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eaad4ff2de1c3823fddf82f41121bdf453d922e9a238642b1dedb33c4e4f98ad"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26ab812fa0c845df815e506be30337e2df27e88399b985d0bb4e3ecfe72df31c"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c5ebac750d9d5f2706654c638c041635c385596caf68f81342011ddfa1e5598"}, - {file = "pydantic_core-2.20.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2aafc5a503855ea5885559eae883978c9b6d8c8993d67766ee73d82e841300dd"}, - {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4868f6bd7c9d98904b748a2653031fc9c2f85b6237009d475b1008bfaeb0a5aa"}, - {file = "pydantic_core-2.20.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aa2f457b4af386254372dfa78a2eda2563680d982422641a85f271c859df1987"}, - {file = "pydantic_core-2.20.1-cp38-none-win32.whl", hash = "sha256:225b67a1f6d602de0ce7f6c1c3ae89a4aa25d3de9be857999e9124f15dab486a"}, - {file = "pydantic_core-2.20.1-cp38-none-win_amd64.whl", hash = "sha256:6b507132dcfc0dea440cce23ee2182c0ce7aba7054576efc65634f080dbe9434"}, - {file = "pydantic_core-2.20.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b03f7941783b4c4a26051846dea594628b38f6940a2fdc0df00b221aed39314c"}, - {file = "pydantic_core-2.20.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1eedfeb6089ed3fad42e81a67755846ad4dcc14d73698c120a82e4ccf0f1f9f6"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:635fee4e041ab9c479e31edda27fcf966ea9614fff1317e280d99eb3e5ab6fe2"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:77bf3ac639c1ff567ae3b47f8d4cc3dc20f9966a2a6dd2311dcc055d3d04fb8a"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ed1b0132f24beeec5a78b67d9388656d03e6a7c837394f99257e2d55b461611"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6514f963b023aeee506678a1cf821fe31159b925c4b76fe2afa94cc70b3222b"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10d4204d8ca33146e761c79f83cc861df20e7ae9f6487ca290a97702daf56006"}, - {file = "pydantic_core-2.20.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2d036c7187b9422ae5b262badb87a20a49eb6c5238b2004e96d4da1231badef1"}, - {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9ebfef07dbe1d93efb94b4700f2d278494e9162565a54f124c404a5656d7ff09"}, - {file = "pydantic_core-2.20.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:6b9d9bb600328a1ce523ab4f454859e9d439150abb0906c5a1983c146580ebab"}, - {file = "pydantic_core-2.20.1-cp39-none-win32.whl", hash = "sha256:784c1214cb6dd1e3b15dd8b91b9a53852aed16671cc3fbe4786f4f1db07089e2"}, - {file = "pydantic_core-2.20.1-cp39-none-win_amd64.whl", hash = "sha256:d2fe69c5434391727efa54b47a1e7986bb0186e72a41b203df8f5b0a19a4f669"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a45f84b09ac9c3d35dfcf6a27fd0634d30d183205230a0ebe8373a0e8cfa0906"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d02a72df14dfdbaf228424573a07af10637bd490f0901cee872c4f434a735b94"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2b27e6af28f07e2f195552b37d7d66b150adbaa39a6d327766ffd695799780f"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:084659fac3c83fd674596612aeff6041a18402f1e1bc19ca39e417d554468482"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:242b8feb3c493ab78be289c034a1f659e8826e2233786e36f2893a950a719bb6"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:38cf1c40a921d05c5edc61a785c0ddb4bed67827069f535d794ce6bcded919fc"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e0bbdd76ce9aa5d4209d65f2b27fc6e5ef1312ae6c5333c26db3f5ade53a1e99"}, - {file = "pydantic_core-2.20.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:254ec27fdb5b1ee60684f91683be95e5133c994cc54e86a0b0963afa25c8f8a6"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:407653af5617f0757261ae249d3fba09504d7a71ab36ac057c938572d1bc9331"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:c693e916709c2465b02ca0ad7b387c4f8423d1db7b4649c551f27a529181c5ad"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b5ff4911aea936a47d9376fd3ab17e970cc543d1b68921886e7f64bd28308d1"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177f55a886d74f1808763976ac4efd29b7ed15c69f4d838bbd74d9d09cf6fa86"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:964faa8a861d2664f0c7ab0c181af0bea66098b1919439815ca8803ef136fc4e"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4dd484681c15e6b9a977c785a345d3e378d72678fd5f1f3c0509608da24f2ac0"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f6d6cff3538391e8486a431569b77921adfcdef14eb18fbf19b7c0a5294d4e6a"}, - {file = "pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a6d511cc297ff0883bc3708b465ff82d7560193169a8b93260f74ecb0a5e08a7"}, - {file = "pydantic_core-2.20.1.tar.gz", hash = "sha256:26ca695eeee5f9f1aeeb211ffc12f10bcb6f71e2989988fda61dabd65db878d4"}, + {file = "pydantic_core-2.23.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:7f10a5d1b9281392f1bf507d16ac720e78285dfd635b05737c3911637601bae6"}, + {file = "pydantic_core-2.23.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c09a7885dd33ee8c65266e5aa7fb7e2f23d49d8043f089989726391dd7350c5"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6470b5a1ec4d1c2e9afe928c6cb37eb33381cab99292a708b8cb9aa89e62429b"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9172d2088e27d9a185ea0a6c8cebe227a9139fd90295221d7d495944d2367700"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86fc6c762ca7ac8fbbdff80d61b2c59fb6b7d144aa46e2d54d9e1b7b0e780e01"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0cb80fd5c2df4898693aa841425ea1727b1b6d2167448253077d2a49003e0ed"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03667cec5daf43ac4995cefa8aaf58f99de036204a37b889c24a80927b629cec"}, + {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:047531242f8e9c2db733599f1c612925de095e93c9cc0e599e96cf536aaf56ba"}, + {file = "pydantic_core-2.23.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5499798317fff7f25dbef9347f4451b91ac2a4330c6669821c8202fd354c7bee"}, + {file = "pydantic_core-2.23.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bbb5e45eab7624440516ee3722a3044b83fff4c0372efe183fd6ba678ff681fe"}, + {file = "pydantic_core-2.23.3-cp310-none-win32.whl", hash = "sha256:8b5b3ed73abb147704a6e9f556d8c5cb078f8c095be4588e669d315e0d11893b"}, + {file = "pydantic_core-2.23.3-cp310-none-win_amd64.whl", hash = "sha256:2b603cde285322758a0279995b5796d64b63060bfbe214b50a3ca23b5cee3e83"}, + {file = "pydantic_core-2.23.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c889fd87e1f1bbeb877c2ee56b63bb297de4636661cc9bbfcf4b34e5e925bc27"}, + {file = "pydantic_core-2.23.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea85bda3189fb27503af4c45273735bcde3dd31c1ab17d11f37b04877859ef45"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7f7f72f721223f33d3dc98a791666ebc6a91fa023ce63733709f4894a7dc611"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b2b55b0448e9da68f56b696f313949cda1039e8ec7b5d294285335b53104b61"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c24574c7e92e2c56379706b9a3f07c1e0c7f2f87a41b6ee86653100c4ce343e5"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2b05e6ccbee333a8f4b8f4d7c244fdb7a979e90977ad9c51ea31261e2085ce0"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c409ce1c219c091e47cb03feb3c4ed8c2b8e004efc940da0166aaee8f9d6c8"}, + {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d965e8b325f443ed3196db890d85dfebbb09f7384486a77461347f4adb1fa7f8"}, + {file = "pydantic_core-2.23.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f56af3a420fb1ffaf43ece3ea09c2d27c444e7c40dcb7c6e7cf57aae764f2b48"}, + {file = "pydantic_core-2.23.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5b01a078dd4f9a52494370af21aa52964e0a96d4862ac64ff7cea06e0f12d2c5"}, + {file = "pydantic_core-2.23.3-cp311-none-win32.whl", hash = "sha256:560e32f0df04ac69b3dd818f71339983f6d1f70eb99d4d1f8e9705fb6c34a5c1"}, + {file = "pydantic_core-2.23.3-cp311-none-win_amd64.whl", hash = "sha256:c744fa100fdea0d000d8bcddee95213d2de2e95b9c12be083370b2072333a0fa"}, + {file = "pydantic_core-2.23.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e0ec50663feedf64d21bad0809f5857bac1ce91deded203efc4a84b31b2e4305"}, + {file = "pydantic_core-2.23.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:db6e6afcb95edbe6b357786684b71008499836e91f2a4a1e55b840955b341dbb"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98ccd69edcf49f0875d86942f4418a4e83eb3047f20eb897bffa62a5d419c8fa"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a678c1ac5c5ec5685af0133262103defb427114e62eafeda12f1357a12140162"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01491d8b4d8db9f3391d93b0df60701e644ff0894352947f31fff3e52bd5c801"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fcf31facf2796a2d3b7fe338fe8640aa0166e4e55b4cb108dbfd1058049bf4cb"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7200fd561fb3be06827340da066df4311d0b6b8eb0c2116a110be5245dceb326"}, + {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dc1636770a809dee2bd44dd74b89cc80eb41172bcad8af75dd0bc182c2666d4c"}, + {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:67a5def279309f2e23014b608c4150b0c2d323bd7bccd27ff07b001c12c2415c"}, + {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:748bdf985014c6dd3e1e4cc3db90f1c3ecc7246ff5a3cd4ddab20c768b2f1dab"}, + {file = "pydantic_core-2.23.3-cp312-none-win32.whl", hash = "sha256:255ec6dcb899c115f1e2a64bc9ebc24cc0e3ab097775755244f77360d1f3c06c"}, + {file = "pydantic_core-2.23.3-cp312-none-win_amd64.whl", hash = "sha256:40b8441be16c1e940abebed83cd006ddb9e3737a279e339dbd6d31578b802f7b"}, + {file = "pydantic_core-2.23.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6daaf5b1ba1369a22c8b050b643250e3e5efc6a78366d323294aee54953a4d5f"}, + {file = "pydantic_core-2.23.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d015e63b985a78a3d4ccffd3bdf22b7c20b3bbd4b8227809b3e8e75bc37f9cb2"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3fc572d9b5b5cfe13f8e8a6e26271d5d13f80173724b738557a8c7f3a8a3791"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f6bd91345b5163ee7448bee201ed7dd601ca24f43f439109b0212e296eb5b423"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc379c73fd66606628b866f661e8785088afe2adaba78e6bbe80796baf708a63"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbdce4b47592f9e296e19ac31667daed8753c8367ebb34b9a9bd89dacaa299c9"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc3cf31edf405a161a0adad83246568647c54404739b614b1ff43dad2b02e6d5"}, + {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e22b477bf90db71c156f89a55bfe4d25177b81fce4aa09294d9e805eec13855"}, + {file = "pydantic_core-2.23.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:0a0137ddf462575d9bce863c4c95bac3493ba8e22f8c28ca94634b4a1d3e2bb4"}, + {file = "pydantic_core-2.23.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:203171e48946c3164fe7691fc349c79241ff8f28306abd4cad5f4f75ed80bc8d"}, + {file = "pydantic_core-2.23.3-cp313-none-win32.whl", hash = "sha256:76bdab0de4acb3f119c2a4bff740e0c7dc2e6de7692774620f7452ce11ca76c8"}, + {file = "pydantic_core-2.23.3-cp313-none-win_amd64.whl", hash = "sha256:37ba321ac2a46100c578a92e9a6aa33afe9ec99ffa084424291d84e456f490c1"}, + {file = "pydantic_core-2.23.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d063c6b9fed7d992bcbebfc9133f4c24b7a7f215d6b102f3e082b1117cddb72c"}, + {file = "pydantic_core-2.23.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6cb968da9a0746a0cf521b2b5ef25fc5a0bee9b9a1a8214e0a1cfaea5be7e8a4"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbefe079a520c5984e30e1f1f29325054b59534729c25b874a16a5048028d16"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbaaf2ef20d282659093913da9d402108203f7cb5955020bd8d1ae5a2325d1c4"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb539d7e5dc4aac345846f290cf504d2fd3c1be26ac4e8b5e4c2b688069ff4cf"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e6f33503c5495059148cc486867e1d24ca35df5fc064686e631e314d959ad5b"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04b07490bc2f6f2717b10c3969e1b830f5720b632f8ae2f3b8b1542394c47a8e"}, + {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03795b9e8a5d7fda05f3873efc3f59105e2dcff14231680296b87b80bb327295"}, + {file = "pydantic_core-2.23.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c483dab0f14b8d3f0df0c6c18d70b21b086f74c87ab03c59250dbf6d3c89baba"}, + {file = "pydantic_core-2.23.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8b2682038e255e94baf2c473dca914a7460069171ff5cdd4080be18ab8a7fd6e"}, + {file = "pydantic_core-2.23.3-cp38-none-win32.whl", hash = "sha256:f4a57db8966b3a1d1a350012839c6a0099f0898c56512dfade8a1fe5fb278710"}, + {file = "pydantic_core-2.23.3-cp38-none-win_amd64.whl", hash = "sha256:13dd45ba2561603681a2676ca56006d6dee94493f03d5cadc055d2055615c3ea"}, + {file = "pydantic_core-2.23.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:82da2f4703894134a9f000e24965df73cc103e31e8c31906cc1ee89fde72cbd8"}, + {file = "pydantic_core-2.23.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dd9be0a42de08f4b58a3cc73a123f124f65c24698b95a54c1543065baca8cf0e"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89b731f25c80830c76fdb13705c68fef6a2b6dc494402987c7ea9584fe189f5d"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c6de1ec30c4bb94f3a69c9f5f2182baeda5b809f806676675e9ef6b8dc936f28"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb68b41c3fa64587412b104294b9cbb027509dc2f6958446c502638d481525ef"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c3980f2843de5184656aab58698011b42763ccba11c4a8c35936c8dd6c7068c"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94f85614f2cba13f62c3c6481716e4adeae48e1eaa7e8bac379b9d177d93947a"}, + {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:510b7fb0a86dc8f10a8bb43bd2f97beb63cffad1203071dc434dac26453955cd"}, + {file = "pydantic_core-2.23.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1eba2f7ce3e30ee2170410e2171867ea73dbd692433b81a93758ab2de6c64835"}, + {file = "pydantic_core-2.23.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b259fd8409ab84b4041b7b3f24dcc41e4696f180b775961ca8142b5b21d0e70"}, + {file = "pydantic_core-2.23.3-cp39-none-win32.whl", hash = "sha256:40d9bd259538dba2f40963286009bf7caf18b5112b19d2b55b09c14dde6db6a7"}, + {file = "pydantic_core-2.23.3-cp39-none-win_amd64.whl", hash = "sha256:5a8cd3074a98ee70173a8633ad3c10e00dcb991ecec57263aacb4095c5efb958"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f399e8657c67313476a121a6944311fab377085ca7f490648c9af97fc732732d"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:6b5547d098c76e1694ba85f05b595720d7c60d342f24d5aad32c3049131fa5c4"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dda0290a6f608504882d9f7650975b4651ff91c85673341789a476b1159f211"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65b6e5da855e9c55a0c67f4db8a492bf13d8d3316a59999cfbaf98cc6e401961"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:09e926397f392059ce0afdcac920df29d9c833256354d0c55f1584b0b70cf07e"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:87cfa0ed6b8c5bd6ae8b66de941cece179281239d482f363814d2b986b79cedc"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e61328920154b6a44d98cabcb709f10e8b74276bc709c9a513a8c37a18786cc4"}, + {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce3317d155628301d649fe5e16a99528d5680af4ec7aa70b90b8dacd2d725c9b"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e89513f014c6be0d17b00a9a7c81b1c426f4eb9224b15433f3d98c1a071f8433"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4f62c1c953d7ee375df5eb2e44ad50ce2f5aff931723b398b8bc6f0ac159791a"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2718443bc671c7ac331de4eef9b673063b10af32a0bb385019ad61dcf2cc8f6c"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d90e08b2727c5d01af1b5ef4121d2f0c99fbee692c762f4d9d0409c9da6541"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2b676583fc459c64146debea14ba3af54e540b61762dfc0613dc4e98c3f66eeb"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:50e4661f3337977740fdbfbae084ae5693e505ca2b3130a6d4eb0f2281dc43b8"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:68f4cf373f0de6abfe599a38307f4417c1c867ca381c03df27c873a9069cda25"}, + {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:59d52cf01854cb26c46958552a21acb10dd78a52aa34c86f284e66b209db8cab"}, + {file = "pydantic_core-2.23.3.tar.gz", hash = "sha256:3cb0f65d8b4121c1b015c60104a685feb929a29d7cf204387c7f2688c7974690"}, ] [package.dependencies] diff --git a/pyproject.toml b/pyproject.toml index 8081328b..134c21d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,11 +38,11 @@ tqdm = ">=4.62.3" xarray = ">=0.20.2" joblib = ">=1.2.0" pyarrow = ">=14.0.1, <17.0" +pydantic = ">=2.8.2" jupyter = {version = ">=1.0.0", optional = true} ipykernel = {version = ">=6.9.1", optional = true} jupyterlab = {version = "^3.4.0", optional = true} jupyterlab-h5web = {version = "^8.0.0", extras = ["full"]} -pydantic = "^2.8.2" [tool.poetry.extras] notebook = ["jupyter", "ipykernel", "jupyterlab", "jupyterlab-h5web"] From c7aac14fbc430ae13e7c6ec4091c87ad39432913 Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Sun, 15 Sep 2024 20:32:37 +0200 Subject: [PATCH 06/32] nest the models --- sed/config/config_model.py | 101 +++++++++++++++++++++++-------------- sed/core/config.py | 4 +- 2 files changed, 64 insertions(+), 41 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index f7626908..658a317e 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -1,5 +1,6 @@ """Pydantic model to validate the config for SED package.""" from collections.abc import Sequence +from datetime import datetime from typing import Literal from typing import Optional from typing import Union @@ -50,6 +51,7 @@ class ColumnsModel(BaseModel): corrected_x: str corrected_y: str corrected_tof: str + corrected_delay: str kx: str ky: str energy: str @@ -66,15 +68,15 @@ class ChannelModel(BaseModel): slice: Optional[int] = None dtype: Optional[str] = None - class subChannelModel(BaseModel): + class subChannel(BaseModel): slice: int dtype: Optional[str] = None - sub_channels: Optional[dict[str, subChannelModel]] = None + sub_channels: Optional[dict[str, subChannel]] = None -class Dataframe(BaseModel): - columns: ColumnsModel = ColumnsModel() +class DataframeModel(BaseModel): + columns: ColumnsModel units: Optional[dict[str, str]] = None channels: dict[str, ChannelModel] = Field(default_factory=dict) # other settings @@ -115,22 +117,6 @@ class StaticModel(BaseModel): beamtime_dir: Optional[dict] = None -class EnergyCalibrationModel(BaseModel): - d: float - t0: float - E0: float - energy_scale: str - - -class EnergyCorrectionModel(BaseModel): - correction_type: str - amplitude: float - center: Sequence[float] - gamma: float - sigma: float - diameter: float - - class EnergyModel(BaseModel): bins: int ranges: Sequence[int] @@ -146,26 +132,24 @@ class EnergyModel(BaseModel): x_width: Sequence[int] y_width: Sequence[int] color_clip: int - calibration: Optional[EnergyCalibrationModel] = None - correction: Optional[EnergyCorrectionModel] = None + class EnergyCalibrationModel(BaseModel): + d: float + t0: float + E0: float + energy_scale: str -class MomentumCalibrationModel(BaseModel): - kx_scale: float - ky_scale: float - x_center: float - y_center: float - rstart: float - cstart: float - rstep: float - cstep: float + calibration: Optional[EnergyCalibrationModel] = None + class EnergyCorrectionModel(BaseModel): + correction_type: str + amplitude: float + center: Sequence[float] + gamma: float + sigma: float + diameter: float -class MomentumCorrectionModel(BaseModel): - feature_points: Sequence[Sequence[float]] - rotation_symmetry: int - include_center: bool - use_center: bool + correction: Optional[EnergyCorrectionModel] = None class MomentumModel(BaseModel): @@ -177,17 +161,56 @@ class MomentumModel(BaseModel): sigma: int fwhm: int sigma_radius: int + + class MomentumCalibrationModel(BaseModel): + kx_scale: float + ky_scale: float + x_center: float + y_center: float + rstart: float + cstart: float + rstep: float + cstep: float + calibration: Optional[MomentumCalibrationModel] = None + + class MomentumCorrectionModel(BaseModel): + feature_points: Sequence[Sequence[float]] + rotation_symmetry: int + include_center: bool + use_center: bool + correction: Optional[MomentumCorrectionModel] = None class DelayModel(BaseModel): adc_range: Sequence[int] - time0: int - flip_time_axis: bool = False + flip_time_axis: bool + # Group keys in the datafile p1_key: Optional[str] = None p2_key: Optional[str] = None p3_key: Optional[str] = None + t0_key: Optional[str] = None + + class Calibration(BaseModel): + creation_date: datetime + adc_range: Sequence[int] + delay_range: Sequence[float] + time0: float + delay_range_mm: Sequence[float] + datafile: FilePath # .h5 extension in filepath + + calibration: Optional[Calibration] = None + + class Offsets(BaseModel): + creation_date: datetime + constant: float + flip_delay_axis: bool + weight: float + preserve_mean: bool + reduction: str + + offsets: Optional[Offsets] = None class MetadataModel(BaseModel): @@ -210,7 +233,7 @@ class NexusModel(BaseModel): class ConfigModel(BaseModel): core: CoreModel - dataframe: Dataframe + dataframe: DataframeModel energy: EnergyModel momentum: MomentumModel delay: DelayModel diff --git a/sed/core/config.py b/sed/core/config.py index 8628352c..bd41fc24 100644 --- a/sed/core/config.py +++ b/sed/core/config.py @@ -12,8 +12,8 @@ import yaml from platformdirs import user_config_path -from sed.core.logging import setup_logging from sed.config.config_model import ConfigModel +from sed.core.logging import setup_logging package_dir = os.path.dirname(find_spec("sed").origin) @@ -31,7 +31,7 @@ def parse_config( default_config: (dict | str) = f"{package_dir}/config/default.yaml", verbose: bool = True, model: bool = False, -) -> dict: +) -> dict | ConfigModel: """Load the config dictionary from a file, or pass the provided config dictionary. The content of the loaded config dictionary is then completed from a set of pre-configured config files in hierarchical order, by adding missing items. These additional config files From c179d9ae0098fc92ead01f9585d19c65932a7b2e Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Sun, 15 Sep 2024 22:41:45 +0200 Subject: [PATCH 07/32] add copytool in configmodel and other attrs --- sed/config/config_model.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 658a317e..71840198 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -26,12 +26,18 @@ class Paths(BaseModel): class CoreModel(BaseModel): - loader: str = "generic" + loader: str + verbose: Optional[bool] = None paths: Optional[Paths] = None - num_cores: int = 4 + num_cores: Optional[int] = None year: Optional[int] = None beamtime_id: Optional[int] = None instrument: Optional[str] = None + # TODO: move copy tool to a separate model + use_copy_tool: Optional[bool] = None + copy_tool_source: Optional[DirectoryPath] = None + copy_tool_dest: Optional[DirectoryPath] = None + copy_tool_kwds: Optional[dict] = None @field_validator("loader") @classmethod @@ -48,10 +54,6 @@ class ColumnsModel(BaseModel): y: str tof: str tof_ns: str - corrected_x: str - corrected_y: str - corrected_tof: str - corrected_delay: str kx: str ky: str energy: str @@ -59,6 +61,12 @@ class ColumnsModel(BaseModel): adc: str bias: str timestamp: str + corrected_x: str + corrected_y: str + corrected_tof: str + corrected_delay: Optional[str] = None + sector_id: Optional[str] = None + auxiliary: Optional[str] = None class ChannelModel(BaseModel): From d7cab5f01f004b277bfc8a830a5dfb56f9d0a774 Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Sun, 15 Sep 2024 22:42:16 +0200 Subject: [PATCH 08/32] update config files to conform to model --- sed/config/default.yaml | 6 ++--- sed/config/mpes_example_config.yaml | 41 +++++++++++------------------ 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/sed/config/default.yaml b/sed/config/default.yaml index d6c15956..3d591926 100644 --- a/sed/config/default.yaml +++ b/sed/config/default.yaml @@ -27,7 +27,7 @@ dataframe: # binning factor used for the adc coordinate adc_binning: 1 # list of columns to apply jitter to. - jitter_cols: ["@x_column", "@y_column", "@tof_column"] + jitter_cols: ["@x", "@y", "@tof"] # Jitter amplitude or list of jitter amplitudes. Should equal half the digital step size of each jitter_column jitter_amps: 0.5 # Time stepping in seconds of the successive events in the timed dataframe @@ -68,7 +68,7 @@ energy: momentum: # binning axes to use for momentum correction/calibration. # Axes names starting with "@" refer to keys in the "dataframe" section - axes: ["@x_column", "@y_column", "@tof_column"] + axes: ["@x", "@y", "@tof"] # Bin numbers used for the respective axes bins: [512, 512, 300] # bin ranges to use (in unbinned detector coordinates) @@ -110,6 +110,6 @@ histogram: bins: [80, 80, 80] # default axes to use for histogram visualization. # Axes names starting with "@" refer to keys in the "dataframe" section - axes: ["@x_column", "@y_column", "@tof_column"] + axes: ["@x", "@y", "@tof"] # default ranges to use for histogram visualization (in unbinned detector coordinates) ranges: [[0, 1800], [0, 1800], [0, 150000]] diff --git a/sed/config/mpes_example_config.yaml b/sed/config/mpes_example_config.yaml index 4848945f..f191c97e 100644 --- a/sed/config/mpes_example_config.yaml +++ b/sed/config/mpes_example_config.yaml @@ -25,30 +25,19 @@ dataframe: timed_dataframe_unit_time: 0.001 # list of columns to apply jitter to jitter_cols: ["X", "Y", "t", "ADC"] - # dataframe column containing x coordinates - x_column: "X" - # dataframe column containing y coordinates - y_column: "Y" - # dataframe column containing time-of-flight data - tof_column: "t" - # dataframe column containing analog-to-digital data - adc_column: "ADC" - # dataframe column containing bias voltage data - bias_column: "sampleBias" - # dataframe column containing corrected x coordinates - corrected_x_column: "Xm" - # dataframe column containing corrected y coordinates - corrected_y_column: "Ym" - # dataframe column containing corrected time-of-flight data - corrected_tof_column: "tm" - # dataframe column containing kx coordinates - kx_column: "kx" - # dataframe column containing ky coordinates - ky_column: "ky" - # dataframe column containing energy data - energy_column: "energy" - # dataframe column containing delay data - delay_column: "delay" + columns: + x: X # dataframe column containing x coordinates + y: Y # dataframe column containing y coordinates + tof: t # dataframe column containing time-of-flight data + adc: ADC # dataframe column containing analog-to-digital data + bias: sampleBias # dataframe column containing bias voltage data + corrected_x: Xm # dataframe column containing corrected x coordinates + corrected_y: Ym # dataframe column containing corrected y coordinates + corrected_tof: tm # dataframe column containing corrected time-of-flight data + kx: kx # dataframe column containing kx coordinates + ky: ky # dataframe column containing ky coordinates + energy: energy # dataframe column containing energy data + delay: delay # dataframe column containing delay data # time length of a base time-of-flight bin in ns tof_binwidth: 4.125e-12 # Binning factor of the tof_column-data compared to tof_binwidth @@ -155,7 +144,7 @@ energy: momentum: # binning axes to use for momentum correction/calibration. # Axes names starting with "@" refer to keys in the "dataframe" section - axes: ["@x_column", "@y_column", "@tof_column"] + axes: ["@x", "@y", "@tof"] # Bin numbers used for the respective axes bins: [512, 512, 300] # bin ranges to use (in unbinned detector coordinates) @@ -226,7 +215,7 @@ histogram: bins: [80, 80, 80, 80] # default axes to use for histogram visualization. # Axes names starting with "@" refer to keys in the "dataframe" section - axes: ["@x_column", "@y_column", "@tof_column", "@adc_column"] + axes: ["@x", "@y", "@tof", "@adc"] # default ranges to use for histogram visualization (in unbinned detector coordinates) ranges: [[0, 1800], [0, 1800], [256000, 276000], [0, 32000]] From 75f1200765f81c0963a4136469828f28a35a0ae9 Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Mon, 16 Sep 2024 03:50:43 +0200 Subject: [PATCH 09/32] use configmodel in processor class --- sed/core/processor.py | 46 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/sed/core/processor.py b/sed/core/processor.py index e931d0a9..e4d5f550 100644 --- a/sed/core/processor.py +++ b/sed/core/processor.py @@ -823,8 +823,8 @@ def apply_momentum_correction( - **inv_dfield** (np.ndarray, optional): Inverse deformation field. """ - x_column = self._config["dataframe"]["x_column"] - y_column = self._config["dataframe"]["y_column"] + x_column = self._config["dataframe"]["columns"]["x"] + y_column = self._config["dataframe"]["columns"]["y"] if self._dataframe is not None: logger.info("Adding corrected X/Y columns to dataframe:") @@ -967,8 +967,8 @@ def apply_momentum_calibration( Defaults to False. **kwds: Keyword args passed to ``MomentumCalibrator.append_k_axis``. """ - x_column = self._config["dataframe"]["x_column"] - y_column = self._config["dataframe"]["y_column"] + x_column = self._config["dataframe"]["columns"]["x"] + y_column = self._config["dataframe"]["columns"]["y"] if self._dataframe is not None: logger.info("Adding kx/ky columns to dataframe:") @@ -1108,7 +1108,7 @@ def apply_energy_correction( **kwds: Keyword args passed to ``EnergyCalibrator.apply_energy_correction()``. """ - tof_column = self._config["dataframe"]["tof_column"] + tof_column = self._config["dataframe"]["columns"]["tof"] if self._dataframe is not None: logger.info("Applying energy correction to dataframe...") @@ -1187,16 +1187,16 @@ def load_bias_series( if binned_data is not None: if isinstance(binned_data, xr.DataArray): if ( - self._config["dataframe"]["tof_column"] not in binned_data.dims - or self._config["dataframe"]["bias_column"] not in binned_data.dims + self._config["dataframe"]["columns"]["tof"] not in binned_data.dims + or self._config["dataframe"]["columns"]["bias"] not in binned_data.dims ): raise ValueError( "If binned_data is provided as an xarray, it needs to contain dimensions " f"'{self._config['dataframe']['tof_column']}' and " f"'{self._config['dataframe']['bias_column']}'!.", ) - tof = binned_data.coords[self._config["dataframe"]["tof_column"]].values - biases = binned_data.coords[self._config["dataframe"]["bias_column"]].values + tof = binned_data.coords[self._config["dataframe"]["columns"]["tof"]].values + biases = binned_data.coords[self._config["dataframe"]["columns"]["bias"]].values traces = binned_data.values[:, :] else: try: @@ -1470,7 +1470,7 @@ def append_energy_axis( **kwds: Keyword args passed to ``EnergyCalibrator.append_energy_axis()``. """ - tof_column = self._config["dataframe"]["tof_column"] + tof_column = self._config["dataframe"]["columns"]["tof"] if self._dataframe is not None: logger.info("Adding energy column to dataframe:") @@ -1536,7 +1536,7 @@ def add_energy_offset( Raises: ValueError: If the energy column is not in the dataframe. """ - energy_column = self._config["dataframe"]["energy_column"] + energy_column = self._config["dataframe"]["columns"]["energy"] if energy_column not in self._dataframe.columns: raise ValueError( f"Energy column {energy_column} not found in dataframe! " @@ -1624,7 +1624,7 @@ def append_tof_ns_axis( **kwds: additional arguments are passed to ``EnergyCalibrator.append_tof_ns_axis()``. """ - tof_column = self._config["dataframe"]["tof_column"] + tof_column = self._config["dataframe"]["columns"]["tof"] if self._dataframe is not None: logger.info("Adding time-of-flight column in nanoseconds to dataframe.") @@ -1671,7 +1671,7 @@ def align_dld_sectors( Defaults to False. **kwds: additional arguments are passed to ``EnergyCalibrator.align_dld_sectors()``. """ - tof_column = self._config["dataframe"]["tof_column"] + tof_column = self._config["dataframe"]["columns"]["tof"] if self._dataframe is not None: logger.info("Aligning 8s sectors of dataframe") @@ -1725,7 +1725,7 @@ def calibrate_delay_axis( Defaults to False. **kwds: Keyword args passed to ``DelayCalibrator.append_delay_axis``. """ - adc_column = self._config["dataframe"]["adc_column"] + adc_column = self._config["dataframe"]["columns"]["adc"] if adc_column not in self._dataframe.columns: raise ValueError(f"ADC column {adc_column} not found in dataframe, cannot calibrate!") @@ -1841,7 +1841,7 @@ def add_delay_offset( Raises: ValueError: If the delay column is not in the dataframe. """ - delay_column = self._config["dataframe"]["delay_column"] + delay_column = self._config["dataframe"]["columns"]["delay"] if delay_column not in self._dataframe.columns: raise ValueError(f"Delay column {delay_column} not found in dataframe! ") @@ -1964,7 +1964,7 @@ def add_jitter( cols = self._config["dataframe"]["jitter_cols"] for loc, col in enumerate(cols): if col.startswith("@"): - cols[loc] = self._config["dataframe"].get(col.strip("@")) + cols[loc] = self._config["dataframe"]["columns"].get(col.strip("@")) if amps is None: amps = self._config["dataframe"]["jitter_amps"] @@ -2024,7 +2024,7 @@ def add_time_stamped_data( """ time_stamp_column = kwds.pop( "time_stamp_column", - self._config["dataframe"].get("time_stamp_alias", ""), + self._config["dataframe"]["columns"].get("timestamp", ""), ) if time_stamps is None and data is None: @@ -2099,7 +2099,7 @@ def pre_binning( axes = self._config["momentum"]["axes"] for loc, axis in enumerate(axes): if axis.startswith("@"): - axes[loc] = self._config["dataframe"].get(axis.strip("@")) + axes[loc] = self._config["dataframe"]["columns"].get(axis.strip("@")) if bins is None: bins = self._config["momentum"]["bins"] @@ -2333,14 +2333,14 @@ def get_normalization_histogram( self._dataframe.partitions[df_partitions], axis, self._binned.coords[axis].values, - self._config["dataframe"]["time_stamp_alias"], + self._config["dataframe"]["columns"]["timestamp"], ) else: self._normalization_histogram = normalization_histogram_from_timestamps( self._dataframe, axis, self._binned.coords[axis].values, - self._config["dataframe"]["time_stamp_alias"], + self._config["dataframe"]["columns"]["timestamp"], ) else: if df_partitions is not None: @@ -2406,13 +2406,13 @@ def view_event_histogram( axes = list(axes) for loc, axis in enumerate(axes): if axis.startswith("@"): - axes[loc] = self._config["dataframe"].get(axis.strip("@")) + axes[loc] = self._config["dataframe"]["columns"].get(axis.strip("@")) if ranges is None: ranges = list(self._config["histogram"]["ranges"]) for loc, axis in enumerate(axes): - if axis == self._config["dataframe"]["tof_column"]: + if axis == self._config["dataframe"]["columns"]["tof"]: ranges[loc] = np.asarray(ranges[loc]) / self._config["dataframe"]["tof_binning"] - elif axis == self._config["dataframe"]["adc_column"]: + elif axis == self._config["dataframe"]["columns"]["adc"]: ranges[loc] = np.asarray(ranges[loc]) / self._config["dataframe"]["adc_binning"] input_types = map(type, [axes, bins, ranges]) From 8ef58073cbb3b72b3afe8f7ce60084e30a874968 Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Mon, 16 Sep 2024 15:18:40 +0200 Subject: [PATCH 10/32] update modules to new config --- sed/calibrator/delay.py | 12 +++++------ sed/calibrator/energy.py | 32 ++++++++++++++-------------- sed/calibrator/momentum.py | 32 ++++++++++++++-------------- sed/config/config_model.py | 7 +++++- sed/config/flash_example_config.yaml | 16 ++++++-------- sed/config/mpes_example_config.yaml | 4 ++-- sed/core/config.py | 12 +++++------ sed/dataset/dataset.py | 1 + tests/data/loader/flash/config.yaml | 16 +++++++++++++- tests/test_config.py | 13 ++++++++--- 10 files changed, 85 insertions(+), 60 deletions(-) diff --git a/sed/calibrator/delay.py b/sed/calibrator/delay.py index 000fe9cf..a15edd15 100644 --- a/sed/calibrator/delay.py +++ b/sed/calibrator/delay.py @@ -51,10 +51,10 @@ def __init__( self._verbose = verbose set_verbosity(logger, self._verbose) - self.adc_column: str = self._config["dataframe"].get("adc_column", None) - self.delay_column: str = self._config["dataframe"]["delay_column"] - self.corrected_delay_column = self._config["dataframe"].get( - "corrected_delay_column", + self.adc_column: str = config["dataframe"]["columns"]["adc"] + self.delay_column: str = config["dataframe"]["columns"]["delay"] + self.corrected_delay_column = self._config["dataframe"]["columns"].get( + "corrected_delay", self.delay_column, ) self.calibration: dict[str, Any] = self._config["delay"].get("calibration", {}) @@ -102,9 +102,9 @@ def append_delay_axis( df (pd.DataFrame | dask.dataframe.DataFrame): The dataframe where to apply the delay calibration to. adc_column (str, optional): Source column for delay calibration. - Defaults to config["dataframe"]["adc_column"]. + Defaults to config["dataframe"]["columns"]["adc"]. delay_column (str, optional): Destination column for delay calibration. - Defaults to config["dataframe"]["delay_column"]. + Defaults to config["dataframe"]["columns"]["delay"]. calibration (dict, optional): Calibration dictionary with parameters for delay calibration. adc_range (tuple | list | np.ndarray, optional): The range of used diff --git a/sed/calibrator/energy.py b/sed/calibrator/energy.py index e3e7d1ca..83429ed2 100644 --- a/sed/calibrator/energy.py +++ b/sed/calibrator/energy.py @@ -107,12 +107,12 @@ def __init__( self.peaks: np.ndarray = np.asarray([]) self.calibration: dict[str, Any] = self._config["energy"].get("calibration", {}) - self.tof_column = self._config["dataframe"]["tof_column"] - self.tof_ns_column = self._config["dataframe"].get("tof_ns_column", None) - self.corrected_tof_column = self._config["dataframe"]["corrected_tof_column"] - self.energy_column = self._config["dataframe"]["energy_column"] - self.x_column = self._config["dataframe"]["x_column"] - self.y_column = self._config["dataframe"]["y_column"] + self.tof_column = self._config["dataframe"]["columns"]["tof"] + self.tof_ns_column = self._config["dataframe"]["columns"].get("tof_ns", None) + self.corrected_tof_column = self._config["dataframe"]["columns"]["corrected_tof"] + self.energy_column = self._config["dataframe"]["columns"]["energy"] + self.x_column = self._config["dataframe"]["columns"]["x"] + self.y_column = self._config["dataframe"]["columns"]["y"] self.binwidth: float = self._config["dataframe"]["tof_binwidth"] self.binning: int = self._config["dataframe"]["tof_binning"] self.x_width = self._config["energy"]["x_width"] @@ -121,7 +121,7 @@ def __init__( self.tof_fermi = self._config["energy"]["tof_fermi"] / self.binning self.color_clip = self._config["energy"]["color_clip"] self.sector_delays = self._config["dataframe"].get("sector_delays", None) - self.sector_id_column = self._config["dataframe"].get("sector_id_column", None) + self.sector_id_column = self._config["dataframe"]["columns"].get("sector_id", None) self.offsets: dict[str, Any] = self._config["energy"].get("offsets", {}) self.correction: dict[str, Any] = self._config["energy"].get("correction", {}) @@ -217,7 +217,7 @@ def bin_data( Args: data_files (list[str]): list of file names to bin axes (list[str], optional): bin axes. Defaults to - config["dataframe"]["tof_column"]. + config["dataframe"]["columns"]["tof"]. bins (list[int], optional): number of bins. Defaults to config["energy"]["bins"]. ranges (Sequence[tuple[float, float]], optional): bin ranges. @@ -802,9 +802,9 @@ def append_energy_axis( df (pd.DataFrame | dask.dataframe.DataFrame): Dataframe to apply the energy axis calibration to. tof_column (str, optional): Label of the source column. - Defaults to config["dataframe"]["tof_column"]. + Defaults to config["dataframe"]["columns"]["tof"]. energy_column (str, optional): Label of the destination column. - Defaults to config["dataframe"]["energy_column"]. + Defaults to config["dataframe"]["columns"]["energy"]. calibration (dict, optional): Calibration dictionary. If provided, overrides calibration from class or config. Defaults to self.calibration or config["energy"]["calibration"]. @@ -948,9 +948,9 @@ def append_tof_ns_axis( Args: df (pd.DataFrame | dask.dataframe.DataFrame): Dataframe to convert. tof_column (str, optional): Name of the column containing the - time-of-flight steps. Defaults to config["dataframe"]["tof_column"]. + time-of-flight steps. Defaults to config["dataframe"]["columns"]["tof"]. tof_ns_column (str, optional): Name of the column to store the - time-of-flight in nanoseconds. Defaults to config["dataframe"]["tof_ns_column"]. + time-of-flight in nanoseconds. Defaults to config["dataframe"]["columns"]["tof_ns"]. binwidth (float, optional): Time-of-flight binwidth in ns. Defaults to config["energy"]["tof_binwidth"]. binning (int, optional): Time-of-flight binning factor. @@ -1381,9 +1381,9 @@ def apply_energy_correction( df (pd.DataFrame | dask.dataframe.DataFrame): The dataframe where to apply the energy correction to. tof_column (str, optional): Name of the source column to convert. - Defaults to config["dataframe"]["tof_column"]. + Defaults to config["dataframe"]["columns"]["tof"]. new_tof_column (str, optional): Name of the destination column to convert. - Defaults to config["dataframe"]["corrected_tof_column"]. + Defaults to config["dataframe"]["columns"]["corrected_tof"]. correction_type (str, optional): Type of correction to apply to the TOF axis. Valid values are: @@ -1494,9 +1494,9 @@ def align_dld_sectors( Args: df (dask.dataframe.DataFrame): Dataframe to use. tof_column (str, optional): Name of the column containing the time-of-flight values. - Defaults to config["dataframe"]["tof_column"]. + Defaults to config["dataframe"]["columns"]["tof"]. sector_id_column (str, optional): Name of the column containing the sector id values. - Defaults to config["dataframe"]["sector_id_column"]. + Defaults to config["dataframe"]["columns"]["sector_id"]. sector_delays (np.ndarray, optional): Array containing the sector delays. Defaults to config["dataframe"]["sector_delays"]. diff --git a/sed/calibrator/momentum.py b/sed/calibrator/momentum.py index fed52473..f1852d8a 100644 --- a/sed/calibrator/momentum.py +++ b/sed/calibrator/momentum.py @@ -123,12 +123,12 @@ def __init__( self.adjust_params: dict[str, Any] = {} self.calibration: dict[str, Any] = self._config["momentum"].get("calibration", {}) - self.x_column = self._config["dataframe"]["x_column"] - self.y_column = self._config["dataframe"]["y_column"] - self.corrected_x_column = self._config["dataframe"]["corrected_x_column"] - self.corrected_y_column = self._config["dataframe"]["corrected_y_column"] - self.kx_column = self._config["dataframe"]["kx_column"] - self.ky_column = self._config["dataframe"]["ky_column"] + self.x_column = self._config["dataframe"]["columns"]["x"] + self.y_column = self._config["dataframe"]["columns"]["y"] + self.corrected_x_column = self._config["dataframe"]["columns"]["corrected_x"] + self.corrected_y_column = self._config["dataframe"]["columns"]["corrected_y"] + self.kx_column = self._config["dataframe"]["columns"]["kx"] + self.ky_column = self._config["dataframe"]["columns"]["ky"] self._state: int = 0 @@ -1749,15 +1749,15 @@ def apply_corrections( df (pd.DataFrame | dask.dataframe.DataFrame): Dataframe to apply the distortion correction to. x_column (str, optional): Label of the 'X' column before momentum - distortion correction. Defaults to config["momentum"]["x_column"]. + distortion correction. Defaults to config["dataframe"]["columns"]["x"]. y_column (str, optional): Label of the 'Y' column before momentum - distortion correction. Defaults to config["momentum"]["y_column"]. + distortion correction. Defaults to config["dataframe"]["columns"]["y"]. new_x_column (str, optional): Label of the 'X' column after momentum distortion correction. - Defaults to config["momentum"]["corrected_x_column"]. + Defaults to config["dataframe"]["columns"]["corrected_x"]. new_y_column (str, optional): Label of the 'Y' column after momentum distortion correction. - Defaults to config["momentum"]["corrected_y_column"]. + Defaults to config["dataframe"]["columns"]["corrected_y"]. Returns: tuple[pd.DataFrame | dask.dataframe.DataFrame, dict]: Dataframe with @@ -1898,15 +1898,15 @@ def append_k_axis( df (pd.DataFrame | dask.dataframe.DataFrame): Dataframe to apply the distortion correction to. x_column (str, optional): Label of the source 'X' column. - Defaults to config["momentum"]["corrected_x_column"] or - config["momentum"]["x_column"] (whichever is present). + Defaults to config["dataframe"]["columns"]["corrected_x"] or + config["dataframe"]["columns"]["x"] (whichever is present). y_column (str, optional): Label of the source 'Y' column. - Defaults to config["momentum"]["corrected_y_column"] or - config["momentum"]["y_column"] (whichever is present). + Defaults to config["dataframe"]["columns"]["corrected_y"] or + config["dataframe"]["columns"]["y"] (whichever is present). new_x_column (str, optional): Label of the destination 'X' column after - momentum calibration. Defaults to config["momentum"]["kx_column"]. + momentum calibration. Defaults to config["dataframe"]["columns"]["kx"]. new_y_column (str, optional): Label of the destination 'Y' column after - momentum calibration. Defaults to config["momentum"]["ky_column"]. + momentum calibration. Defaults to config["dataframe"]["columns"]["ky"]. calibration (dict, optional): Dictionary containing calibration parameters. Defaults to 'self.calibration' or config["momentum"]["calibration"]. suppress_output (bool, optional): Option to suppress output of diagnostic information. diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 71840198..ae2ef64b 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -6,6 +6,7 @@ from typing import Union from pydantic import BaseModel +from pydantic import ConfigDict from pydantic import DirectoryPath from pydantic import Field from pydantic import field_validator @@ -101,6 +102,8 @@ class DataframeModel(BaseModel): sector_id_reserved_bits: Optional[int] = None sector_delays: Optional[Sequence[int]] = None + # write validator for model so that x_column gets converted to columns: x + class BinningModel(BaseModel): hist_mode: str @@ -113,7 +116,7 @@ class BinningModel(BaseModel): class HistogramModel(BaseModel): bins: Sequence[int] axes: Sequence[str] - ranges: Sequence[Sequence[int]] + ranges: Sequence[tuple[float, float]] class StaticModel(BaseModel): @@ -250,3 +253,5 @@ class ConfigModel(BaseModel): metadata: Optional[MetadataModel] = None nexus: Optional[NexusModel] = None static: Optional[StaticModel] = None + + model_config = ConfigDict(extra="forbid") diff --git a/sed/config/flash_example_config.yaml b/sed/config/flash_example_config.yaml index 03867513..185fac78 100644 --- a/sed/config/flash_example_config.yaml +++ b/sed/config/flash_example_config.yaml @@ -1,13 +1,4 @@ # This file contains the default configuration for the flash loader. - -# The paths to the raw and parquet data directories. If these are not -# provided, the loader will try to find the data based on year beamtimeID etc -paths: - # location of the raw data. - raw: "" - # location of the intermediate parquet files. - processed: "" - core: # defines the loader loader: 'flash' @@ -21,6 +12,13 @@ core: year: 2023 # the instrument used instrument: hextof # hextof, wespe, etc + # The paths to the raw and parquet data directories. If these are not + # provided, the loader will try to find the data based on year beamtimeID etc + # paths: + # # location of the raw data. + # raw: "" + # # location of the intermediate parquet files. + # processed: "" binning: # Histogram computation mode to use. diff --git a/sed/config/mpes_example_config.yaml b/sed/config/mpes_example_config.yaml index f191c97e..3d1d3e39 100644 --- a/sed/config/mpes_example_config.yaml +++ b/sed/config/mpes_example_config.yaml @@ -6,9 +6,9 @@ core: # Option to use the copy tool to mirror data to a local storage location before processing. use_copy_tool: False # path to the root of the source data directory - copy_tool_source: "/path/to/data/" + copy_tool_source: null # "/path/to/data/" # path to the root or the local data storage - copy_tool_dest: "/path/to/localDataStore/" + copy_tool_dest: null # "/path/to/localDataStore/" # optional keywords for the copy tool: copy_tool_kwds: # group id to set for copied files and folders diff --git a/sed/core/config.py b/sed/core/config.py index bd41fc24..df44cfd1 100644 --- a/sed/core/config.py +++ b/sed/core/config.py @@ -30,8 +30,8 @@ def parse_config( system_config: dict | str = None, default_config: (dict | str) = f"{package_dir}/config/default.yaml", verbose: bool = True, - model: bool = False, -) -> dict | ConfigModel: + verify_config: bool = True, +) -> dict: """Load the config dictionary from a file, or pass the provided config dictionary. The content of the loaded config dictionary is then completed from a set of pre-configured config files in hierarchical order, by adding missing items. These additional config files @@ -57,13 +57,13 @@ def parse_config( or file path. The loaded dictionary is completed with the default values. Defaults to *package_dir*/config/default.yaml". verbose (bool, optional): Option to report loaded config files. Defaults to True. - model (bool, optional): Option to return the config model instead of the dictionary. + verify_config (bool, optional): Option to verify config file. Defaults to True. Raises: TypeError: Raised if the provided file is neither *json* nor *yaml*. FileNotFoundError: Raised if the provided file is not found. Returns: - dict: Loaded and possibly completed config dictionary. + ConfigModel: Loaded and possibly completed pydantic config model. """ if config is None: config = {} @@ -144,10 +144,10 @@ def parse_config( base_dictionary=default_dict, ) + if not verify_config: + return config_dict # Run the config through the ConfigModel to ensure it is valid config_model = ConfigModel(**config_dict) - if model: - return config_model return config_model.model_dump() diff --git a/sed/dataset/dataset.py b/sed/dataset/dataset.py index 15bf8b4f..fca7fc83 100644 --- a/sed/dataset/dataset.py +++ b/sed/dataset/dataset.py @@ -55,6 +55,7 @@ def load_datasets_dict() -> dict: system_config={}, default_config=DatasetsManager.json_path["module"], verbose=False, + verify_config=False, ) @staticmethod diff --git a/tests/data/loader/flash/config.yaml b/tests/data/loader/flash/config.yaml index 19e01a2a..d4209636 100644 --- a/tests/data/loader/flash/config.yaml +++ b/tests/data/loader/flash/config.yaml @@ -61,7 +61,21 @@ dataframe: sector_delays: [0., 0., 0., 0., 0., 0., 0., 0.] jitter_cols: ["dldPosX", "dldPosY", "dldTimeSteps"] - + columns: + x: dldPosX + corrected_x: X + kx: kx + y: dldPosY + corrected_y: Y + ky: ky + tof: dldTimeSteps + tof_ns: dldTime + corrected_tof: tm + timestamp: timeStamp + auxiliary: dldAux + sector_id: dldSectorID + delay: delayStage + corrected_delay: pumpProbeTime units: dldPosX: 'step' dldPosY: 'step' diff --git a/tests/test_config.py b/tests/test_config.py index df875b45..0ab54cbb 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -36,7 +36,7 @@ def test_default_config() -> None: """Test the config loader for the default config.""" - config = parse_config() + config = parse_config(verify_config=False) assert isinstance(config, dict) for key in default_config_keys: assert key in config.keys() @@ -49,7 +49,7 @@ def test_default_config() -> None: def test_load_dict() -> None: """Test the config loader for a dict.""" config_dict = {"test_entry": True} - config = parse_config(config_dict) + config = parse_config(config_dict, verify_config=False) assert isinstance(config, dict) for key in default_config_keys: assert key in config.keys() @@ -69,7 +69,14 @@ def test_load_does_not_modify() -> None: default_dict = {"a": 1, "b": {"c": 13}, "c": {"e": 11}} default_copy = copy.deepcopy(default_dict) - parse_config(config_dict, folder_dict, user_dict, system_dict, default_dict) + parse_config( + config_dict, + folder_dict, + user_dict, + system_dict, + default_dict, + verify_config=False, + ) assert config_dict == config_copy assert folder_dict == folder_copy assert user_dict == user_copy From 02aea0aed5a39e8fc842e3d06208342835821cac Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Mon, 16 Sep 2024 16:03:51 +0200 Subject: [PATCH 11/32] fix some config problems --- sed/config/config_model.py | 21 +++++++++++++-------- sed/loader/mpes/loader.py | 2 +- tests/calibrator/test_delay.py | 7 +++---- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index ae2ef64b..2356d4dd 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -8,7 +8,6 @@ from pydantic import BaseModel from pydantic import ConfigDict from pydantic import DirectoryPath -from pydantic import Field from pydantic import field_validator from pydantic import FilePath from pydantic import HttpUrl @@ -87,7 +86,7 @@ class subChannel(BaseModel): class DataframeModel(BaseModel): columns: ColumnsModel units: Optional[dict[str, str]] = None - channels: dict[str, ChannelModel] = Field(default_factory=dict) + channels: Optional[dict[str, ChannelModel]] = None # other settings tof_binwidth: float tof_binning: int @@ -214,12 +213,18 @@ class Calibration(BaseModel): calibration: Optional[Calibration] = None class Offsets(BaseModel): - creation_date: datetime - constant: float - flip_delay_axis: bool - weight: float - preserve_mean: bool - reduction: str + creation_date: Optional[datetime] = None + constant: Optional[float] = None + flip_delay_axis: Optional[bool] = False + + ## This seems rather complicated way to define offsets, + # inconsistent in how args vs config are for add_offsets + class OffsetColumn(BaseModel): + weight: float + preserve_mean: bool + reduction: Optional[str] = None + + columns: Optional[dict[str, OffsetColumn]] = None offsets: Optional[Offsets] = None diff --git a/sed/loader/mpes/loader.py b/sed/loader/mpes/loader.py index ebc9a08c..eaf296f0 100644 --- a/sed/loader/mpes/loader.py +++ b/sed/loader/mpes/loader.py @@ -72,7 +72,7 @@ def hdf5_to_dataframe( electron_channels = [] column_names = [] - + print("Print values: ", channels) for name, channel in channels.items(): if channel["format"] == "per_electron": if channel["dataset_key"] in test_proc: diff --git a/tests/calibrator/test_delay.py b/tests/calibrator/test_delay.py index d7677296..a81542e2 100644 --- a/tests/calibrator/test_delay.py +++ b/tests/calibrator/test_delay.py @@ -136,9 +136,8 @@ def test_delay_parameters_from_delay_range_mm() -> None: "offsets": { "constant": 1, "flip_delay_axis": True, - "bam": { - "weight": 0.001, - "preserve_mean": False, + "columns": { + "bam": {"weight": 0.001, "preserve_mean": False}, }, }, }, @@ -201,7 +200,7 @@ def test_add_offset_from_dict(df=test_dataframe) -> None: """test that the timing offset is corrected for correctly from config""" cfg_ = cfg.copy() offsets = cfg["delay"]["offsets"] # type:ignore - offsets["bam"].pop("weight") + offsets["columns"]["bam"].pop("weight") offsets["flip_delay_axis"] = False cfg_.pop("delay") config = parse_config( From d84f6e49a09b463bffc5d1a5102a18d7c33b8cba Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 7 Oct 2024 14:15:19 +0200 Subject: [PATCH 12/32] update lockfile --- poetry.lock | 592 +++++++++++++++++++++++++++------------------------- 1 file changed, 312 insertions(+), 280 deletions(-) diff --git a/poetry.lock b/poetry.lock index 67de07be..9ec11c08 100644 --- a/poetry.lock +++ b/poetry.lock @@ -196,13 +196,13 @@ test = ["pytest (>=7.0.0)", "pytest-xdist (>=2.1.0)"] [[package]] name = "asteval" -version = "1.0.4" +version = "1.0.5" description = "Safe, minimalistic evaluator of python expression using ast module" optional = false python-versions = ">=3.8" files = [ - {file = "asteval-1.0.4-py3-none-any.whl", hash = "sha256:7a88bfd0dd1eabdf20bb4995904df742cecf876f7f9e700f22231abf4e34d50c"}, - {file = "asteval-1.0.4.tar.gz", hash = "sha256:15e63bd01fce65aded51357f7e1debc6f46100d777c372af11c27c07cb740074"}, + {file = "asteval-1.0.5-py3-none-any.whl", hash = "sha256:082b95312578affc8a6d982f7d92b7ac5de05634985c87e7eedd3188d31149fa"}, + {file = "asteval-1.0.5.tar.gz", hash = "sha256:bac3c8dd6d2b789e959cfec9bb296fb8338eec066feae618c462132701fbc665"}, ] [package.extras] @@ -265,13 +265,13 @@ test-all = ["astropy[test]", "coverage[toml]", "ipython (>=4.2)", "objgraph", "s [[package]] name = "astropy-iers-data" -version = "0.2024.9.16.0.32.21" +version = "0.2024.10.7.0.32.46" description = "IERS Earth Rotation and Leap Second tables for the astropy core package" optional = false python-versions = ">=3.8" files = [ - {file = "astropy_iers_data-0.2024.9.16.0.32.21-py3-none-any.whl", hash = "sha256:adf111e1b596470c4437fa44cf767e56f6d4bc2e93068871fd0b30c73476d430"}, - {file = "astropy_iers_data-0.2024.9.16.0.32.21.tar.gz", hash = "sha256:2ff6fe868a623e616953a432698b05dd6adac9683d21ac780bfbb94e78f7c344"}, + {file = "astropy_iers_data-0.2024.10.7.0.32.46-py3-none-any.whl", hash = "sha256:4b5ce3e3c8ac17632a28a35f4b847d8a8db084509f5f0db3238140dbb5149eae"}, + {file = "astropy_iers_data-0.2024.10.7.0.32.46.tar.gz", hash = "sha256:349ff09294f03112d9391a5ad660135bf2fcef7de411a52c20bcf8598fb029d1"}, ] [package.extras] @@ -891,33 +891,33 @@ test = ["pandas[test]", "pre-commit", "pytest", "pytest-cov", "pytest-rerunfailu [[package]] name = "debugpy" -version = "1.8.5" +version = "1.8.6" description = "An implementation of the Debug Adapter Protocol for Python" optional = false python-versions = ">=3.8" files = [ - {file = "debugpy-1.8.5-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:7e4d594367d6407a120b76bdaa03886e9eb652c05ba7f87e37418426ad2079f7"}, - {file = "debugpy-1.8.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4413b7a3ede757dc33a273a17d685ea2b0c09dbd312cc03f5534a0fd4d40750a"}, - {file = "debugpy-1.8.5-cp310-cp310-win32.whl", hash = "sha256:dd3811bd63632bb25eda6bd73bea8e0521794cda02be41fa3160eb26fc29e7ed"}, - {file = "debugpy-1.8.5-cp310-cp310-win_amd64.whl", hash = "sha256:b78c1250441ce893cb5035dd6f5fc12db968cc07f91cc06996b2087f7cefdd8e"}, - {file = "debugpy-1.8.5-cp311-cp311-macosx_12_0_universal2.whl", hash = "sha256:606bccba19f7188b6ea9579c8a4f5a5364ecd0bf5a0659c8a5d0e10dcee3032a"}, - {file = "debugpy-1.8.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db9fb642938a7a609a6c865c32ecd0d795d56c1aaa7a7a5722d77855d5e77f2b"}, - {file = "debugpy-1.8.5-cp311-cp311-win32.whl", hash = "sha256:4fbb3b39ae1aa3e5ad578f37a48a7a303dad9a3d018d369bc9ec629c1cfa7408"}, - {file = "debugpy-1.8.5-cp311-cp311-win_amd64.whl", hash = "sha256:345d6a0206e81eb68b1493ce2fbffd57c3088e2ce4b46592077a943d2b968ca3"}, - {file = "debugpy-1.8.5-cp312-cp312-macosx_12_0_universal2.whl", hash = "sha256:5b5c770977c8ec6c40c60d6f58cacc7f7fe5a45960363d6974ddb9b62dbee156"}, - {file = "debugpy-1.8.5-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0a65b00b7cdd2ee0c2cf4c7335fef31e15f1b7056c7fdbce9e90193e1a8c8cb"}, - {file = "debugpy-1.8.5-cp312-cp312-win32.whl", hash = "sha256:c9f7c15ea1da18d2fcc2709e9f3d6de98b69a5b0fff1807fb80bc55f906691f7"}, - {file = "debugpy-1.8.5-cp312-cp312-win_amd64.whl", hash = "sha256:28ced650c974aaf179231668a293ecd5c63c0a671ae6d56b8795ecc5d2f48d3c"}, - {file = "debugpy-1.8.5-cp38-cp38-macosx_12_0_x86_64.whl", hash = "sha256:3df6692351172a42af7558daa5019651f898fc67450bf091335aa8a18fbf6f3a"}, - {file = "debugpy-1.8.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1cd04a73eb2769eb0bfe43f5bfde1215c5923d6924b9b90f94d15f207a402226"}, - {file = "debugpy-1.8.5-cp38-cp38-win32.whl", hash = "sha256:8f913ee8e9fcf9d38a751f56e6de12a297ae7832749d35de26d960f14280750a"}, - {file = "debugpy-1.8.5-cp38-cp38-win_amd64.whl", hash = "sha256:a697beca97dad3780b89a7fb525d5e79f33821a8bc0c06faf1f1289e549743cf"}, - {file = "debugpy-1.8.5-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:0a1029a2869d01cb777216af8c53cda0476875ef02a2b6ff8b2f2c9a4b04176c"}, - {file = "debugpy-1.8.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84c276489e141ed0b93b0af648eef891546143d6a48f610945416453a8ad406"}, - {file = "debugpy-1.8.5-cp39-cp39-win32.whl", hash = "sha256:ad84b7cde7fd96cf6eea34ff6c4a1b7887e0fe2ea46e099e53234856f9d99a34"}, - {file = "debugpy-1.8.5-cp39-cp39-win_amd64.whl", hash = "sha256:7b0fe36ed9d26cb6836b0a51453653f8f2e347ba7348f2bbfe76bfeb670bfb1c"}, - {file = "debugpy-1.8.5-py2.py3-none-any.whl", hash = "sha256:55919dce65b471eff25901acf82d328bbd5b833526b6c1364bd5133754777a44"}, - {file = "debugpy-1.8.5.zip", hash = "sha256:b2112cfeb34b4507399d298fe7023a16656fc553ed5246536060ca7bd0e668d0"}, + {file = "debugpy-1.8.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:30f467c5345d9dfdcc0afdb10e018e47f092e383447500f125b4e013236bf14b"}, + {file = "debugpy-1.8.6-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d73d8c52614432f4215d0fe79a7e595d0dd162b5c15233762565be2f014803b"}, + {file = "debugpy-1.8.6-cp310-cp310-win32.whl", hash = "sha256:e3e182cd98eac20ee23a00653503315085b29ab44ed66269482349d307b08df9"}, + {file = "debugpy-1.8.6-cp310-cp310-win_amd64.whl", hash = "sha256:e3a82da039cfe717b6fb1886cbbe5c4a3f15d7df4765af857f4307585121c2dd"}, + {file = "debugpy-1.8.6-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:67479a94cf5fd2c2d88f9615e087fcb4fec169ec780464a3f2ba4a9a2bb79955"}, + {file = "debugpy-1.8.6-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fb8653f6cbf1dd0a305ac1aa66ec246002145074ea57933978346ea5afdf70b"}, + {file = "debugpy-1.8.6-cp311-cp311-win32.whl", hash = "sha256:cdaf0b9691879da2d13fa39b61c01887c34558d1ff6e5c30e2eb698f5384cd43"}, + {file = "debugpy-1.8.6-cp311-cp311-win_amd64.whl", hash = "sha256:43996632bee7435583952155c06881074b9a742a86cee74e701d87ca532fe833"}, + {file = "debugpy-1.8.6-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:db891b141fc6ee4b5fc6d1cc8035ec329cabc64bdd2ae672b4550c87d4ecb128"}, + {file = "debugpy-1.8.6-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:567419081ff67da766c898ccf21e79f1adad0e321381b0dfc7a9c8f7a9347972"}, + {file = "debugpy-1.8.6-cp312-cp312-win32.whl", hash = "sha256:c9834dfd701a1f6bf0f7f0b8b1573970ae99ebbeee68314116e0ccc5c78eea3c"}, + {file = "debugpy-1.8.6-cp312-cp312-win_amd64.whl", hash = "sha256:e4ce0570aa4aca87137890d23b86faeadf184924ad892d20c54237bcaab75d8f"}, + {file = "debugpy-1.8.6-cp38-cp38-macosx_14_0_x86_64.whl", hash = "sha256:df5dc9eb4ca050273b8e374a4cd967c43be1327eeb42bfe2f58b3cdfe7c68dcb"}, + {file = "debugpy-1.8.6-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a85707c6a84b0c5b3db92a2df685b5230dd8fb8c108298ba4f11dba157a615a"}, + {file = "debugpy-1.8.6-cp38-cp38-win32.whl", hash = "sha256:538c6cdcdcdad310bbefd96d7850be1cd46e703079cc9e67d42a9ca776cdc8a8"}, + {file = "debugpy-1.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:22140bc02c66cda6053b6eb56dfe01bbe22a4447846581ba1dd6df2c9f97982d"}, + {file = "debugpy-1.8.6-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:c1cef65cffbc96e7b392d9178dbfd524ab0750da6c0023c027ddcac968fd1caa"}, + {file = "debugpy-1.8.6-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1e60bd06bb3cc5c0e957df748d1fab501e01416c43a7bdc756d2a992ea1b881"}, + {file = "debugpy-1.8.6-cp39-cp39-win32.whl", hash = "sha256:f7158252803d0752ed5398d291dee4c553bb12d14547c0e1843ab74ee9c31123"}, + {file = "debugpy-1.8.6-cp39-cp39-win_amd64.whl", hash = "sha256:3358aa619a073b620cd0d51d8a6176590af24abcc3fe2e479929a154bf591b51"}, + {file = "debugpy-1.8.6-py2.py3-none-any.whl", hash = "sha256:b48892df4d810eff21d3ef37274f4c60d32cdcafc462ad5647239036b0f0649f"}, + {file = "debugpy-1.8.6.zip", hash = "sha256:c931a9371a86784cee25dec8d65bc2dc7a21f3f1552e3833d9ef8f919d22280a"}, ] [[package]] @@ -944,13 +944,13 @@ files = [ [[package]] name = "dill" -version = "0.3.8" +version = "0.3.9" description = "serialize all of Python" optional = false python-versions = ">=3.8" files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, + {file = "dill-0.3.9-py3-none-any.whl", hash = "sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a"}, + {file = "dill-0.3.9.tar.gz", hash = "sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c"}, ] [package.extras] @@ -970,13 +970,13 @@ files = [ [[package]] name = "docutils" -version = "0.20.1" +version = "0.21.2" description = "Docutils -- Python Documentation Utilities" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" files = [ - {file = "docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6"}, - {file = "docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b"}, + {file = "docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2"}, + {file = "docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f"}, ] [[package]] @@ -1067,53 +1067,59 @@ typing = ["typing-extensions (>=4.12.2)"] [[package]] name = "fonttools" -version = "4.53.1" +version = "4.54.1" description = "Tools to manipulate font files" optional = false python-versions = ">=3.8" files = [ - {file = "fonttools-4.53.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0679a30b59d74b6242909945429dbddb08496935b82f91ea9bf6ad240ec23397"}, - {file = "fonttools-4.53.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e8bf06b94694251861ba7fdeea15c8ec0967f84c3d4143ae9daf42bbc7717fe3"}, - {file = "fonttools-4.53.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b96cd370a61f4d083c9c0053bf634279b094308d52fdc2dd9a22d8372fdd590d"}, - {file = "fonttools-4.53.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1c7c5aa18dd3b17995898b4a9b5929d69ef6ae2af5b96d585ff4005033d82f0"}, - {file = "fonttools-4.53.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:e013aae589c1c12505da64a7d8d023e584987e51e62006e1bb30d72f26522c41"}, - {file = "fonttools-4.53.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9efd176f874cb6402e607e4cc9b4a9cd584d82fc34a4b0c811970b32ba62501f"}, - {file = "fonttools-4.53.1-cp310-cp310-win32.whl", hash = "sha256:c8696544c964500aa9439efb6761947393b70b17ef4e82d73277413f291260a4"}, - {file = "fonttools-4.53.1-cp310-cp310-win_amd64.whl", hash = "sha256:8959a59de5af6d2bec27489e98ef25a397cfa1774b375d5787509c06659b3671"}, - {file = "fonttools-4.53.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:da33440b1413bad53a8674393c5d29ce64d8c1a15ef8a77c642ffd900d07bfe1"}, - {file = "fonttools-4.53.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5ff7e5e9bad94e3a70c5cd2fa27f20b9bb9385e10cddab567b85ce5d306ea923"}, - {file = "fonttools-4.53.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6e7170d675d12eac12ad1a981d90f118c06cf680b42a2d74c6c931e54b50719"}, - {file = "fonttools-4.53.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bee32ea8765e859670c4447b0817514ca79054463b6b79784b08a8df3a4d78e3"}, - {file = "fonttools-4.53.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6e08f572625a1ee682115223eabebc4c6a2035a6917eac6f60350aba297ccadb"}, - {file = "fonttools-4.53.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b21952c092ffd827504de7e66b62aba26fdb5f9d1e435c52477e6486e9d128b2"}, - {file = "fonttools-4.53.1-cp311-cp311-win32.whl", hash = "sha256:9dfdae43b7996af46ff9da520998a32b105c7f098aeea06b2226b30e74fbba88"}, - {file = "fonttools-4.53.1-cp311-cp311-win_amd64.whl", hash = "sha256:d4d0096cb1ac7a77b3b41cd78c9b6bc4a400550e21dc7a92f2b5ab53ed74eb02"}, - {file = "fonttools-4.53.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:d92d3c2a1b39631a6131c2fa25b5406855f97969b068e7e08413325bc0afba58"}, - {file = "fonttools-4.53.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3b3c8ebafbee8d9002bd8f1195d09ed2bd9ff134ddec37ee8f6a6375e6a4f0e8"}, - {file = "fonttools-4.53.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32f029c095ad66c425b0ee85553d0dc326d45d7059dbc227330fc29b43e8ba60"}, - {file = "fonttools-4.53.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10f5e6c3510b79ea27bb1ebfcc67048cde9ec67afa87c7dd7efa5c700491ac7f"}, - {file = "fonttools-4.53.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f677ce218976496a587ab17140da141557beb91d2a5c1a14212c994093f2eae2"}, - {file = "fonttools-4.53.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9e6ceba2a01b448e36754983d376064730690401da1dd104ddb543519470a15f"}, - {file = "fonttools-4.53.1-cp312-cp312-win32.whl", hash = "sha256:791b31ebbc05197d7aa096bbc7bd76d591f05905d2fd908bf103af4488e60670"}, - {file = "fonttools-4.53.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ed170b5e17da0264b9f6fae86073be3db15fa1bd74061c8331022bca6d09bab"}, - {file = "fonttools-4.53.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c818c058404eb2bba05e728d38049438afd649e3c409796723dfc17cd3f08749"}, - {file = "fonttools-4.53.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:651390c3b26b0c7d1f4407cad281ee7a5a85a31a110cbac5269de72a51551ba2"}, - {file = "fonttools-4.53.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e54f1bba2f655924c1138bbc7fa91abd61f45c68bd65ab5ed985942712864bbb"}, - {file = "fonttools-4.53.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9cd19cf4fe0595ebdd1d4915882b9440c3a6d30b008f3cc7587c1da7b95be5f"}, - {file = "fonttools-4.53.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:2af40ae9cdcb204fc1d8f26b190aa16534fcd4f0df756268df674a270eab575d"}, - {file = "fonttools-4.53.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:35250099b0cfb32d799fb5d6c651220a642fe2e3c7d2560490e6f1d3f9ae9169"}, - {file = "fonttools-4.53.1-cp38-cp38-win32.whl", hash = "sha256:f08df60fbd8d289152079a65da4e66a447efc1d5d5a4d3f299cdd39e3b2e4a7d"}, - {file = "fonttools-4.53.1-cp38-cp38-win_amd64.whl", hash = "sha256:7b6b35e52ddc8fb0db562133894e6ef5b4e54e1283dff606fda3eed938c36fc8"}, - {file = "fonttools-4.53.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75a157d8d26c06e64ace9df037ee93a4938a4606a38cb7ffaf6635e60e253b7a"}, - {file = "fonttools-4.53.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4824c198f714ab5559c5be10fd1adf876712aa7989882a4ec887bf1ef3e00e31"}, - {file = "fonttools-4.53.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:becc5d7cb89c7b7afa8321b6bb3dbee0eec2b57855c90b3e9bf5fb816671fa7c"}, - {file = "fonttools-4.53.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84ec3fb43befb54be490147b4a922b5314e16372a643004f182babee9f9c3407"}, - {file = "fonttools-4.53.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:73379d3ffdeecb376640cd8ed03e9d2d0e568c9d1a4e9b16504a834ebadc2dfb"}, - {file = "fonttools-4.53.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:02569e9a810f9d11f4ae82c391ebc6fb5730d95a0657d24d754ed7763fb2d122"}, - {file = "fonttools-4.53.1-cp39-cp39-win32.whl", hash = "sha256:aae7bd54187e8bf7fd69f8ab87b2885253d3575163ad4d669a262fe97f0136cb"}, - {file = "fonttools-4.53.1-cp39-cp39-win_amd64.whl", hash = "sha256:e5b708073ea3d684235648786f5f6153a48dc8762cdfe5563c57e80787c29fbb"}, - {file = "fonttools-4.53.1-py3-none-any.whl", hash = "sha256:f1f8758a2ad110bd6432203a344269f445a2907dc24ef6bccfd0ac4e14e0d71d"}, - {file = "fonttools-4.53.1.tar.gz", hash = "sha256:e128778a8e9bc11159ce5447f76766cefbd876f44bd79aff030287254e4752c4"}, + {file = "fonttools-4.54.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7ed7ee041ff7b34cc62f07545e55e1468808691dddfd315d51dd82a6b37ddef2"}, + {file = "fonttools-4.54.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:41bb0b250c8132b2fcac148e2e9198e62ff06f3cc472065dff839327945c5882"}, + {file = "fonttools-4.54.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7965af9b67dd546e52afcf2e38641b5be956d68c425bef2158e95af11d229f10"}, + {file = "fonttools-4.54.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:278913a168f90d53378c20c23b80f4e599dca62fbffae4cc620c8eed476b723e"}, + {file = "fonttools-4.54.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:0e88e3018ac809b9662615072dcd6b84dca4c2d991c6d66e1970a112503bba7e"}, + {file = "fonttools-4.54.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:4aa4817f0031206e637d1e685251ac61be64d1adef111060df84fdcbc6ab6c44"}, + {file = "fonttools-4.54.1-cp310-cp310-win32.whl", hash = "sha256:7e3b7d44e18c085fd8c16dcc6f1ad6c61b71ff463636fcb13df7b1b818bd0c02"}, + {file = "fonttools-4.54.1-cp310-cp310-win_amd64.whl", hash = "sha256:dd9cc95b8d6e27d01e1e1f1fae8559ef3c02c76317da650a19047f249acd519d"}, + {file = "fonttools-4.54.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5419771b64248484299fa77689d4f3aeed643ea6630b2ea750eeab219588ba20"}, + {file = "fonttools-4.54.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:301540e89cf4ce89d462eb23a89464fef50915255ece765d10eee8b2bf9d75b2"}, + {file = "fonttools-4.54.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76ae5091547e74e7efecc3cbf8e75200bc92daaeb88e5433c5e3e95ea8ce5aa7"}, + {file = "fonttools-4.54.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82834962b3d7c5ca98cb56001c33cf20eb110ecf442725dc5fdf36d16ed1ab07"}, + {file = "fonttools-4.54.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d26732ae002cc3d2ecab04897bb02ae3f11f06dd7575d1df46acd2f7c012a8d8"}, + {file = "fonttools-4.54.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:58974b4987b2a71ee08ade1e7f47f410c367cdfc5a94fabd599c88165f56213a"}, + {file = "fonttools-4.54.1-cp311-cp311-win32.whl", hash = "sha256:ab774fa225238986218a463f3fe151e04d8c25d7de09df7f0f5fce27b1243dbc"}, + {file = "fonttools-4.54.1-cp311-cp311-win_amd64.whl", hash = "sha256:07e005dc454eee1cc60105d6a29593459a06321c21897f769a281ff2d08939f6"}, + {file = "fonttools-4.54.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:54471032f7cb5fca694b5f1a0aaeba4af6e10ae989df408e0216f7fd6cdc405d"}, + {file = "fonttools-4.54.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8fa92cb248e573daab8d032919623cc309c005086d743afb014c836636166f08"}, + {file = "fonttools-4.54.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a911591200114969befa7f2cb74ac148bce5a91df5645443371aba6d222e263"}, + {file = "fonttools-4.54.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93d458c8a6a354dc8b48fc78d66d2a8a90b941f7fec30e94c7ad9982b1fa6bab"}, + {file = "fonttools-4.54.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5eb2474a7c5be8a5331146758debb2669bf5635c021aee00fd7c353558fc659d"}, + {file = "fonttools-4.54.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9c563351ddc230725c4bdf7d9e1e92cbe6ae8553942bd1fb2b2ff0884e8b714"}, + {file = "fonttools-4.54.1-cp312-cp312-win32.whl", hash = "sha256:fdb062893fd6d47b527d39346e0c5578b7957dcea6d6a3b6794569370013d9ac"}, + {file = "fonttools-4.54.1-cp312-cp312-win_amd64.whl", hash = "sha256:e4564cf40cebcb53f3dc825e85910bf54835e8a8b6880d59e5159f0f325e637e"}, + {file = "fonttools-4.54.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6e37561751b017cf5c40fce0d90fd9e8274716de327ec4ffb0df957160be3bff"}, + {file = "fonttools-4.54.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:357cacb988a18aace66e5e55fe1247f2ee706e01debc4b1a20d77400354cddeb"}, + {file = "fonttools-4.54.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8e953cc0bddc2beaf3a3c3b5dd9ab7554677da72dfaf46951e193c9653e515a"}, + {file = "fonttools-4.54.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:58d29b9a294573d8319f16f2f79e42428ba9b6480442fa1836e4eb89c4d9d61c"}, + {file = "fonttools-4.54.1-cp313-cp313-win32.whl", hash = "sha256:9ef1b167e22709b46bf8168368b7b5d3efeaaa746c6d39661c1b4405b6352e58"}, + {file = "fonttools-4.54.1-cp313-cp313-win_amd64.whl", hash = "sha256:262705b1663f18c04250bd1242b0515d3bbae177bee7752be67c979b7d47f43d"}, + {file = "fonttools-4.54.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ed2f80ca07025551636c555dec2b755dd005e2ea8fbeb99fc5cdff319b70b23b"}, + {file = "fonttools-4.54.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9dc080e5a1c3b2656caff2ac2633d009b3a9ff7b5e93d0452f40cd76d3da3b3c"}, + {file = "fonttools-4.54.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d152d1be65652fc65e695e5619e0aa0982295a95a9b29b52b85775243c06556"}, + {file = "fonttools-4.54.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8583e563df41fdecef31b793b4dd3af8a9caa03397be648945ad32717a92885b"}, + {file = "fonttools-4.54.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:0d1d353ef198c422515a3e974a1e8d5b304cd54a4c2eebcae708e37cd9eeffb1"}, + {file = "fonttools-4.54.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:fda582236fee135d4daeca056c8c88ec5f6f6d88a004a79b84a02547c8f57386"}, + {file = "fonttools-4.54.1-cp38-cp38-win32.whl", hash = "sha256:e7d82b9e56716ed32574ee106cabca80992e6bbdcf25a88d97d21f73a0aae664"}, + {file = "fonttools-4.54.1-cp38-cp38-win_amd64.whl", hash = "sha256:ada215fd079e23e060157aab12eba0d66704316547f334eee9ff26f8c0d7b8ab"}, + {file = "fonttools-4.54.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:f5b8a096e649768c2f4233f947cf9737f8dbf8728b90e2771e2497c6e3d21d13"}, + {file = "fonttools-4.54.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4e10d2e0a12e18f4e2dd031e1bf7c3d7017be5c8dbe524d07706179f355c5dac"}, + {file = "fonttools-4.54.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31c32d7d4b0958600eac75eaf524b7b7cb68d3a8c196635252b7a2c30d80e986"}, + {file = "fonttools-4.54.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c39287f5c8f4a0c5a55daf9eaf9ccd223ea59eed3f6d467133cc727d7b943a55"}, + {file = "fonttools-4.54.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a7a310c6e0471602fe3bf8efaf193d396ea561486aeaa7adc1f132e02d30c4b9"}, + {file = "fonttools-4.54.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d3b659d1029946f4ff9b6183984578041b520ce0f8fb7078bb37ec7445806b33"}, + {file = "fonttools-4.54.1-cp39-cp39-win32.whl", hash = "sha256:e96bc94c8cda58f577277d4a71f51c8e2129b8b36fd05adece6320dd3d57de8a"}, + {file = "fonttools-4.54.1-cp39-cp39-win_amd64.whl", hash = "sha256:e8a4b261c1ef91e7188a30571be6ad98d1c6d9fa2427244c545e2fa0a2494dd7"}, + {file = "fonttools-4.54.1-py3-none-any.whl", hash = "sha256:37cddd62d83dc4f72f7c3f3c2bcf2697e89a30efb152079896544a93907733bd"}, + {file = "fonttools-4.54.1.tar.gz", hash = "sha256:957f669d4922f92c171ba01bef7f29410668db09f6c02111e22b2bce446f3285"}, ] [package.extras] @@ -1205,36 +1211,41 @@ tornado = ["tornado"] [[package]] name = "h5py" -version = "3.11.0" +version = "3.12.1" description = "Read and write HDF5 files from Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "h5py-3.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1625fd24ad6cfc9c1ccd44a66dac2396e7ee74940776792772819fc69f3a3731"}, - {file = "h5py-3.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c072655ad1d5fe9ef462445d3e77a8166cbfa5e599045f8aa3c19b75315f10e5"}, - {file = "h5py-3.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77b19a40788e3e362b54af4dcf9e6fde59ca016db2c61360aa30b47c7b7cef00"}, - {file = "h5py-3.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:ef4e2f338fc763f50a8113890f455e1a70acd42a4d083370ceb80c463d803972"}, - {file = "h5py-3.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bbd732a08187a9e2a6ecf9e8af713f1d68256ee0f7c8b652a32795670fb481ba"}, - {file = "h5py-3.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75bd7b3d93fbeee40860fd70cdc88df4464e06b70a5ad9ce1446f5f32eb84007"}, - {file = "h5py-3.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c416f8eb0daae39dabe71415cb531f95dce2d81e1f61a74537a50c63b28ab3"}, - {file = "h5py-3.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:083e0329ae534a264940d6513f47f5ada617da536d8dccbafc3026aefc33c90e"}, - {file = "h5py-3.11.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a76cae64080210389a571c7d13c94a1a6cf8cb75153044fd1f822a962c97aeab"}, - {file = "h5py-3.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3736fe21da2b7d8a13fe8fe415f1272d2a1ccdeff4849c1421d2fb30fd533bc"}, - {file = "h5py-3.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa6ae84a14103e8dc19266ef4c3e5d7c00b68f21d07f2966f0ca7bdb6c2761fb"}, - {file = "h5py-3.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:21dbdc5343f53b2e25404673c4f00a3335aef25521bd5fa8c707ec3833934892"}, - {file = "h5py-3.11.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:754c0c2e373d13d6309f408325343b642eb0f40f1a6ad21779cfa9502209e150"}, - {file = "h5py-3.11.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:731839240c59ba219d4cb3bc5880d438248533366f102402cfa0621b71796b62"}, - {file = "h5py-3.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ec9df3dd2018904c4cc06331951e274f3f3fd091e6d6cc350aaa90fa9b42a76"}, - {file = "h5py-3.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:55106b04e2c83dfb73dc8732e9abad69d83a436b5b82b773481d95d17b9685e1"}, - {file = "h5py-3.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f4e025e852754ca833401777c25888acb96889ee2c27e7e629a19aee288833f0"}, - {file = "h5py-3.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c4b760082626120031d7902cd983d8c1f424cdba2809f1067511ef283629d4b"}, - {file = "h5py-3.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67462d0669f8f5459529de179f7771bd697389fcb3faab54d63bf788599a48ea"}, - {file = "h5py-3.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:d9c944d364688f827dc889cf83f1fca311caf4fa50b19f009d1f2b525edd33a3"}, - {file = "h5py-3.11.0.tar.gz", hash = "sha256:7b7e8f78072a2edec87c9836f25f34203fd492a4475709a18b417a33cfb21fa9"}, + {file = "h5py-3.12.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f0f1a382cbf494679c07b4371f90c70391dedb027d517ac94fa2c05299dacda"}, + {file = "h5py-3.12.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cb65f619dfbdd15e662423e8d257780f9a66677eae5b4b3fc9dca70b5fd2d2a3"}, + {file = "h5py-3.12.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b15d8dbd912c97541312c0e07438864d27dbca857c5ad634de68110c6beb1c2"}, + {file = "h5py-3.12.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59685fe40d8c1fbbee088c88cd4da415a2f8bee5c270337dc5a1c4aa634e3307"}, + {file = "h5py-3.12.1-cp310-cp310-win_amd64.whl", hash = "sha256:577d618d6b6dea3da07d13cc903ef9634cde5596b13e832476dd861aaf651f3e"}, + {file = "h5py-3.12.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ccd9006d92232727d23f784795191bfd02294a4f2ba68708825cb1da39511a93"}, + {file = "h5py-3.12.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ad8a76557880aed5234cfe7279805f4ab5ce16b17954606cca90d578d3e713ef"}, + {file = "h5py-3.12.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1473348139b885393125126258ae2d70753ef7e9cec8e7848434f385ae72069e"}, + {file = "h5py-3.12.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:018a4597f35092ae3fb28ee851fdc756d2b88c96336b8480e124ce1ac6fb9166"}, + {file = "h5py-3.12.1-cp311-cp311-win_amd64.whl", hash = "sha256:3fdf95092d60e8130ba6ae0ef7a9bd4ade8edbe3569c13ebbaf39baefffc5ba4"}, + {file = "h5py-3.12.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:06a903a4e4e9e3ebbc8b548959c3c2552ca2d70dac14fcfa650d9261c66939ed"}, + {file = "h5py-3.12.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7b3b8f3b48717e46c6a790e3128d39c61ab595ae0a7237f06dfad6a3b51d5351"}, + {file = "h5py-3.12.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:050a4f2c9126054515169c49cb900949814987f0c7ae74c341b0c9f9b5056834"}, + {file = "h5py-3.12.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5c4b41d1019322a5afc5082864dfd6359f8935ecd37c11ac0029be78c5d112c9"}, + {file = "h5py-3.12.1-cp312-cp312-win_amd64.whl", hash = "sha256:e4d51919110a030913201422fb07987db4338eba5ec8c5a15d6fab8e03d443fc"}, + {file = "h5py-3.12.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:513171e90ed92236fc2ca363ce7a2fc6f2827375efcbb0cc7fbdd7fe11fecafc"}, + {file = "h5py-3.12.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:59400f88343b79655a242068a9c900001a34b63e3afb040bd7cdf717e440f653"}, + {file = "h5py-3.12.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d3e465aee0ec353949f0f46bf6c6f9790a2006af896cee7c178a8c3e5090aa32"}, + {file = "h5py-3.12.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba51c0c5e029bb5420a343586ff79d56e7455d496d18a30309616fdbeed1068f"}, + {file = "h5py-3.12.1-cp313-cp313-win_amd64.whl", hash = "sha256:52ab036c6c97055b85b2a242cb540ff9590bacfda0c03dd0cf0661b311f522f8"}, + {file = "h5py-3.12.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d2b8dd64f127d8b324f5d2cd1c0fd6f68af69084e9e47d27efeb9e28e685af3e"}, + {file = "h5py-3.12.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4532c7e97fbef3d029735db8b6f5bf01222d9ece41e309b20d63cfaae2fb5c4d"}, + {file = "h5py-3.12.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6fdf6d7936fa824acfa27305fe2d9f39968e539d831c5bae0e0d83ed521ad1ac"}, + {file = "h5py-3.12.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84342bffd1f82d4f036433e7039e241a243531a1d3acd7341b35ae58cdab05bf"}, + {file = "h5py-3.12.1-cp39-cp39-win_amd64.whl", hash = "sha256:62be1fc0ef195891949b2c627ec06bc8e837ff62d5b911b6e42e38e0f20a897d"}, + {file = "h5py-3.12.1.tar.gz", hash = "sha256:326d70b53d31baa61f00b8aa5f95c2fcb9621a3ee8365d770c551a13dbbcbfdf"}, ] [package.dependencies] -numpy = ">=1.17.3" +numpy = ">=1.19.3" [[package]] name = "hdf5plugin" @@ -3004,37 +3015,53 @@ files = [ [[package]] name = "pandas" -version = "2.2.2" +version = "2.2.3" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" files = [ - {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"}, - {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"}, - {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"}, - {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"}, - {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8e5a0b00e1e56a842f922e7fae8ae4077aee4af0acb5ae3622bd4b4c30aedf99"}, - {file = "pandas-2.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:ddf818e4e6c7c6f4f7c8a12709696d193976b591cc7dc50588d3d1a6b5dc8772"}, - {file = "pandas-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288"}, - {file = "pandas-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151"}, - {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b"}, - {file = "pandas-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee"}, - {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db"}, - {file = "pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1"}, - {file = "pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24"}, - {file = "pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef"}, - {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad"}, - {file = "pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad"}, - {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76"}, - {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"}, - {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"}, - {file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"}, - {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"}, - {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"}, - {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"}, - {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92fd6b027924a7e178ac202cfbe25e53368db90d56872d20ffae94b96c7acc57"}, - {file = "pandas-2.2.2-cp39-cp39-win_amd64.whl", hash = "sha256:640cef9aa381b60e296db324337a554aeeb883ead99dc8f6c18e81a93942f5f4"}, - {file = "pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54"}, + {file = "pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5"}, + {file = "pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed"}, + {file = "pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42"}, + {file = "pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f"}, + {file = "pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039"}, + {file = "pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698"}, + {file = "pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3"}, + {file = "pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32"}, + {file = "pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9"}, + {file = "pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3"}, + {file = "pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8"}, + {file = "pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a"}, + {file = "pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015"}, + {file = "pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0"}, + {file = "pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659"}, + {file = "pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb"}, + {file = "pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468"}, + {file = "pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2"}, + {file = "pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d"}, + {file = "pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39"}, + {file = "pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c"}, + {file = "pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea"}, + {file = "pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761"}, + {file = "pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e"}, + {file = "pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667"}, ] [package.dependencies] @@ -3306,13 +3333,13 @@ files = [ [[package]] name = "pre-commit" -version = "3.8.0" +version = "4.0.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" files = [ - {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, - {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, + {file = "pre_commit-4.0.0-py2.py3-none-any.whl", hash = "sha256:0ca2341cf94ac1865350970951e54b1a50521e57b7b500403307aed4315a1234"}, + {file = "pre_commit-4.0.0.tar.gz", hash = "sha256:5d9807162cc5537940f94f266cbe2d716a75cfad0d78a317a92cac16287cfed6"}, ] [package.dependencies] @@ -3324,13 +3351,13 @@ virtualenv = ">=20.10.0" [[package]] name = "prometheus-client" -version = "0.20.0" +version = "0.21.0" description = "Python client for the Prometheus monitoring system." optional = false python-versions = ">=3.8" files = [ - {file = "prometheus_client-0.20.0-py3-none-any.whl", hash = "sha256:cde524a85bce83ca359cc837f28b8c0db5cac7aa653a588fd7e84ba061c329e7"}, - {file = "prometheus_client-0.20.0.tar.gz", hash = "sha256:287629d00b147a32dcb2be0b9df905da599b2d82f80377083ec8463309a4bb89"}, + {file = "prometheus_client-0.21.0-py3-none-any.whl", hash = "sha256:4fa6b4dd0ac16d58bb587c04b1caae65b8c5043e85f778f42f5f632f6af2e166"}, + {file = "prometheus_client-0.21.0.tar.gz", hash = "sha256:96c83c606b71ff2b0a433c98889d275f51ffec6c5e267de37c7a2b5c9aa9233e"}, ] [package.extras] @@ -3338,13 +3365,13 @@ twisted = ["twisted"] [[package]] name = "prompt-toolkit" -version = "3.0.47" +version = "3.0.48" description = "Library for building powerful interactive command lines in Python" optional = false python-versions = ">=3.7.0" files = [ - {file = "prompt_toolkit-3.0.47-py3-none-any.whl", hash = "sha256:0d7bfa67001d5e39d02c224b663abc33687405033a8c422d0d675a5a13361d10"}, - {file = "prompt_toolkit-3.0.47.tar.gz", hash = "sha256:1e1b29cb58080b1e69f207c893a1a7bf16d127a5c30c9d17a25a5d77792e5360"}, + {file = "prompt_toolkit-3.0.48-py3-none-any.whl", hash = "sha256:f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e"}, + {file = "prompt_toolkit-3.0.48.tar.gz", hash = "sha256:d6623ab0477a80df74e646bdbc93621143f5caf104206aa29294d53de1a03d90"}, ] [package.dependencies] @@ -3465,18 +3492,18 @@ files = [ [[package]] name = "pydantic" -version = "2.9.1" +version = "2.9.2" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.9.1-py3-none-any.whl", hash = "sha256:7aff4db5fdf3cf573d4b3c30926a510a10e19a0774d38fc4967f78beb6deb612"}, - {file = "pydantic-2.9.1.tar.gz", hash = "sha256:1363c7d975c7036df0db2b4a61f2e062fbc0aa5ab5f2772e0ffc7191a4f4bce2"}, + {file = "pydantic-2.9.2-py3-none-any.whl", hash = "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"}, + {file = "pydantic-2.9.2.tar.gz", hash = "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f"}, ] [package.dependencies] annotated-types = ">=0.6.0" -pydantic-core = "2.23.3" +pydantic-core = "2.23.4" typing-extensions = {version = ">=4.6.1", markers = "python_version < \"3.13\""} [package.extras] @@ -3485,100 +3512,100 @@ timezone = ["tzdata"] [[package]] name = "pydantic-core" -version = "2.23.3" +version = "2.23.4" description = "Core functionality for Pydantic validation and serialization" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic_core-2.23.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:7f10a5d1b9281392f1bf507d16ac720e78285dfd635b05737c3911637601bae6"}, - {file = "pydantic_core-2.23.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c09a7885dd33ee8c65266e5aa7fb7e2f23d49d8043f089989726391dd7350c5"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6470b5a1ec4d1c2e9afe928c6cb37eb33381cab99292a708b8cb9aa89e62429b"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9172d2088e27d9a185ea0a6c8cebe227a9139fd90295221d7d495944d2367700"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86fc6c762ca7ac8fbbdff80d61b2c59fb6b7d144aa46e2d54d9e1b7b0e780e01"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f0cb80fd5c2df4898693aa841425ea1727b1b6d2167448253077d2a49003e0ed"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03667cec5daf43ac4995cefa8aaf58f99de036204a37b889c24a80927b629cec"}, - {file = "pydantic_core-2.23.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:047531242f8e9c2db733599f1c612925de095e93c9cc0e599e96cf536aaf56ba"}, - {file = "pydantic_core-2.23.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5499798317fff7f25dbef9347f4451b91ac2a4330c6669821c8202fd354c7bee"}, - {file = "pydantic_core-2.23.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bbb5e45eab7624440516ee3722a3044b83fff4c0372efe183fd6ba678ff681fe"}, - {file = "pydantic_core-2.23.3-cp310-none-win32.whl", hash = "sha256:8b5b3ed73abb147704a6e9f556d8c5cb078f8c095be4588e669d315e0d11893b"}, - {file = "pydantic_core-2.23.3-cp310-none-win_amd64.whl", hash = "sha256:2b603cde285322758a0279995b5796d64b63060bfbe214b50a3ca23b5cee3e83"}, - {file = "pydantic_core-2.23.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c889fd87e1f1bbeb877c2ee56b63bb297de4636661cc9bbfcf4b34e5e925bc27"}, - {file = "pydantic_core-2.23.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea85bda3189fb27503af4c45273735bcde3dd31c1ab17d11f37b04877859ef45"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7f7f72f721223f33d3dc98a791666ebc6a91fa023ce63733709f4894a7dc611"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b2b55b0448e9da68f56b696f313949cda1039e8ec7b5d294285335b53104b61"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c24574c7e92e2c56379706b9a3f07c1e0c7f2f87a41b6ee86653100c4ce343e5"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2b05e6ccbee333a8f4b8f4d7c244fdb7a979e90977ad9c51ea31261e2085ce0"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c409ce1c219c091e47cb03feb3c4ed8c2b8e004efc940da0166aaee8f9d6c8"}, - {file = "pydantic_core-2.23.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d965e8b325f443ed3196db890d85dfebbb09f7384486a77461347f4adb1fa7f8"}, - {file = "pydantic_core-2.23.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f56af3a420fb1ffaf43ece3ea09c2d27c444e7c40dcb7c6e7cf57aae764f2b48"}, - {file = "pydantic_core-2.23.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5b01a078dd4f9a52494370af21aa52964e0a96d4862ac64ff7cea06e0f12d2c5"}, - {file = "pydantic_core-2.23.3-cp311-none-win32.whl", hash = "sha256:560e32f0df04ac69b3dd818f71339983f6d1f70eb99d4d1f8e9705fb6c34a5c1"}, - {file = "pydantic_core-2.23.3-cp311-none-win_amd64.whl", hash = "sha256:c744fa100fdea0d000d8bcddee95213d2de2e95b9c12be083370b2072333a0fa"}, - {file = "pydantic_core-2.23.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:e0ec50663feedf64d21bad0809f5857bac1ce91deded203efc4a84b31b2e4305"}, - {file = "pydantic_core-2.23.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:db6e6afcb95edbe6b357786684b71008499836e91f2a4a1e55b840955b341dbb"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98ccd69edcf49f0875d86942f4418a4e83eb3047f20eb897bffa62a5d419c8fa"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a678c1ac5c5ec5685af0133262103defb427114e62eafeda12f1357a12140162"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01491d8b4d8db9f3391d93b0df60701e644ff0894352947f31fff3e52bd5c801"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fcf31facf2796a2d3b7fe338fe8640aa0166e4e55b4cb108dbfd1058049bf4cb"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7200fd561fb3be06827340da066df4311d0b6b8eb0c2116a110be5245dceb326"}, - {file = "pydantic_core-2.23.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dc1636770a809dee2bd44dd74b89cc80eb41172bcad8af75dd0bc182c2666d4c"}, - {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:67a5def279309f2e23014b608c4150b0c2d323bd7bccd27ff07b001c12c2415c"}, - {file = "pydantic_core-2.23.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:748bdf985014c6dd3e1e4cc3db90f1c3ecc7246ff5a3cd4ddab20c768b2f1dab"}, - {file = "pydantic_core-2.23.3-cp312-none-win32.whl", hash = "sha256:255ec6dcb899c115f1e2a64bc9ebc24cc0e3ab097775755244f77360d1f3c06c"}, - {file = "pydantic_core-2.23.3-cp312-none-win_amd64.whl", hash = "sha256:40b8441be16c1e940abebed83cd006ddb9e3737a279e339dbd6d31578b802f7b"}, - {file = "pydantic_core-2.23.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:6daaf5b1ba1369a22c8b050b643250e3e5efc6a78366d323294aee54953a4d5f"}, - {file = "pydantic_core-2.23.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d015e63b985a78a3d4ccffd3bdf22b7c20b3bbd4b8227809b3e8e75bc37f9cb2"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3fc572d9b5b5cfe13f8e8a6e26271d5d13f80173724b738557a8c7f3a8a3791"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f6bd91345b5163ee7448bee201ed7dd601ca24f43f439109b0212e296eb5b423"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc379c73fd66606628b866f661e8785088afe2adaba78e6bbe80796baf708a63"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbdce4b47592f9e296e19ac31667daed8753c8367ebb34b9a9bd89dacaa299c9"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc3cf31edf405a161a0adad83246568647c54404739b614b1ff43dad2b02e6d5"}, - {file = "pydantic_core-2.23.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e22b477bf90db71c156f89a55bfe4d25177b81fce4aa09294d9e805eec13855"}, - {file = "pydantic_core-2.23.3-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:0a0137ddf462575d9bce863c4c95bac3493ba8e22f8c28ca94634b4a1d3e2bb4"}, - {file = "pydantic_core-2.23.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:203171e48946c3164fe7691fc349c79241ff8f28306abd4cad5f4f75ed80bc8d"}, - {file = "pydantic_core-2.23.3-cp313-none-win32.whl", hash = "sha256:76bdab0de4acb3f119c2a4bff740e0c7dc2e6de7692774620f7452ce11ca76c8"}, - {file = "pydantic_core-2.23.3-cp313-none-win_amd64.whl", hash = "sha256:37ba321ac2a46100c578a92e9a6aa33afe9ec99ffa084424291d84e456f490c1"}, - {file = "pydantic_core-2.23.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d063c6b9fed7d992bcbebfc9133f4c24b7a7f215d6b102f3e082b1117cddb72c"}, - {file = "pydantic_core-2.23.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6cb968da9a0746a0cf521b2b5ef25fc5a0bee9b9a1a8214e0a1cfaea5be7e8a4"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbefe079a520c5984e30e1f1f29325054b59534729c25b874a16a5048028d16"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbaaf2ef20d282659093913da9d402108203f7cb5955020bd8d1ae5a2325d1c4"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fb539d7e5dc4aac345846f290cf504d2fd3c1be26ac4e8b5e4c2b688069ff4cf"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7e6f33503c5495059148cc486867e1d24ca35df5fc064686e631e314d959ad5b"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04b07490bc2f6f2717b10c3969e1b830f5720b632f8ae2f3b8b1542394c47a8e"}, - {file = "pydantic_core-2.23.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03795b9e8a5d7fda05f3873efc3f59105e2dcff14231680296b87b80bb327295"}, - {file = "pydantic_core-2.23.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c483dab0f14b8d3f0df0c6c18d70b21b086f74c87ab03c59250dbf6d3c89baba"}, - {file = "pydantic_core-2.23.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8b2682038e255e94baf2c473dca914a7460069171ff5cdd4080be18ab8a7fd6e"}, - {file = "pydantic_core-2.23.3-cp38-none-win32.whl", hash = "sha256:f4a57db8966b3a1d1a350012839c6a0099f0898c56512dfade8a1fe5fb278710"}, - {file = "pydantic_core-2.23.3-cp38-none-win_amd64.whl", hash = "sha256:13dd45ba2561603681a2676ca56006d6dee94493f03d5cadc055d2055615c3ea"}, - {file = "pydantic_core-2.23.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:82da2f4703894134a9f000e24965df73cc103e31e8c31906cc1ee89fde72cbd8"}, - {file = "pydantic_core-2.23.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dd9be0a42de08f4b58a3cc73a123f124f65c24698b95a54c1543065baca8cf0e"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89b731f25c80830c76fdb13705c68fef6a2b6dc494402987c7ea9584fe189f5d"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c6de1ec30c4bb94f3a69c9f5f2182baeda5b809f806676675e9ef6b8dc936f28"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb68b41c3fa64587412b104294b9cbb027509dc2f6958446c502638d481525ef"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c3980f2843de5184656aab58698011b42763ccba11c4a8c35936c8dd6c7068c"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94f85614f2cba13f62c3c6481716e4adeae48e1eaa7e8bac379b9d177d93947a"}, - {file = "pydantic_core-2.23.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:510b7fb0a86dc8f10a8bb43bd2f97beb63cffad1203071dc434dac26453955cd"}, - {file = "pydantic_core-2.23.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1eba2f7ce3e30ee2170410e2171867ea73dbd692433b81a93758ab2de6c64835"}, - {file = "pydantic_core-2.23.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b259fd8409ab84b4041b7b3f24dcc41e4696f180b775961ca8142b5b21d0e70"}, - {file = "pydantic_core-2.23.3-cp39-none-win32.whl", hash = "sha256:40d9bd259538dba2f40963286009bf7caf18b5112b19d2b55b09c14dde6db6a7"}, - {file = "pydantic_core-2.23.3-cp39-none-win_amd64.whl", hash = "sha256:5a8cd3074a98ee70173a8633ad3c10e00dcb991ecec57263aacb4095c5efb958"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f399e8657c67313476a121a6944311fab377085ca7f490648c9af97fc732732d"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:6b5547d098c76e1694ba85f05b595720d7c60d342f24d5aad32c3049131fa5c4"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dda0290a6f608504882d9f7650975b4651ff91c85673341789a476b1159f211"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65b6e5da855e9c55a0c67f4db8a492bf13d8d3316a59999cfbaf98cc6e401961"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:09e926397f392059ce0afdcac920df29d9c833256354d0c55f1584b0b70cf07e"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:87cfa0ed6b8c5bd6ae8b66de941cece179281239d482f363814d2b986b79cedc"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:e61328920154b6a44d98cabcb709f10e8b74276bc709c9a513a8c37a18786cc4"}, - {file = "pydantic_core-2.23.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce3317d155628301d649fe5e16a99528d5680af4ec7aa70b90b8dacd2d725c9b"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e89513f014c6be0d17b00a9a7c81b1c426f4eb9224b15433f3d98c1a071f8433"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:4f62c1c953d7ee375df5eb2e44ad50ce2f5aff931723b398b8bc6f0ac159791a"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2718443bc671c7ac331de4eef9b673063b10af32a0bb385019ad61dcf2cc8f6c"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d90e08b2727c5d01af1b5ef4121d2f0c99fbee692c762f4d9d0409c9da6541"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2b676583fc459c64146debea14ba3af54e540b61762dfc0613dc4e98c3f66eeb"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:50e4661f3337977740fdbfbae084ae5693e505ca2b3130a6d4eb0f2281dc43b8"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:68f4cf373f0de6abfe599a38307f4417c1c867ca381c03df27c873a9069cda25"}, - {file = "pydantic_core-2.23.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:59d52cf01854cb26c46958552a21acb10dd78a52aa34c86f284e66b209db8cab"}, - {file = "pydantic_core-2.23.3.tar.gz", hash = "sha256:3cb0f65d8b4121c1b015c60104a685feb929a29d7cf204387c7f2688c7974690"}, + {file = "pydantic_core-2.23.4-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b"}, + {file = "pydantic_core-2.23.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2"}, + {file = "pydantic_core-2.23.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f"}, + {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3"}, + {file = "pydantic_core-2.23.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071"}, + {file = "pydantic_core-2.23.4-cp310-none-win32.whl", hash = "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119"}, + {file = "pydantic_core-2.23.4-cp310-none-win_amd64.whl", hash = "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8"}, + {file = "pydantic_core-2.23.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e"}, + {file = "pydantic_core-2.23.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0"}, + {file = "pydantic_core-2.23.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64"}, + {file = "pydantic_core-2.23.4-cp311-none-win32.whl", hash = "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f"}, + {file = "pydantic_core-2.23.4-cp311-none-win_amd64.whl", hash = "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231"}, + {file = "pydantic_core-2.23.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36"}, + {file = "pydantic_core-2.23.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e"}, + {file = "pydantic_core-2.23.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24"}, + {file = "pydantic_core-2.23.4-cp312-none-win32.whl", hash = "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84"}, + {file = "pydantic_core-2.23.4-cp312-none-win_amd64.whl", hash = "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9"}, + {file = "pydantic_core-2.23.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc"}, + {file = "pydantic_core-2.23.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b"}, + {file = "pydantic_core-2.23.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327"}, + {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6"}, + {file = "pydantic_core-2.23.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f"}, + {file = "pydantic_core-2.23.4-cp313-none-win32.whl", hash = "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769"}, + {file = "pydantic_core-2.23.4-cp313-none-win_amd64.whl", hash = "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5"}, + {file = "pydantic_core-2.23.4-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555"}, + {file = "pydantic_core-2.23.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad"}, + {file = "pydantic_core-2.23.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12"}, + {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2"}, + {file = "pydantic_core-2.23.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb"}, + {file = "pydantic_core-2.23.4-cp38-none-win32.whl", hash = "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6"}, + {file = "pydantic_core-2.23.4-cp38-none-win_amd64.whl", hash = "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556"}, + {file = "pydantic_core-2.23.4-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a"}, + {file = "pydantic_core-2.23.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c"}, + {file = "pydantic_core-2.23.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55"}, + {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040"}, + {file = "pydantic_core-2.23.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605"}, + {file = "pydantic_core-2.23.4-cp39-none-win32.whl", hash = "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6"}, + {file = "pydantic_core-2.23.4-cp39-none-win_amd64.whl", hash = "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433"}, + {file = "pydantic_core-2.23.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8"}, + {file = "pydantic_core-2.23.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e"}, + {file = "pydantic_core-2.23.4.tar.gz", hash = "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863"}, ] [package.dependencies] @@ -3638,13 +3665,13 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pynxtools" -version = "0.7.1" +version = "0.7.4" description = "Extend NeXus for experiments and characterization in Materials Science and Materials Engineering and serve as a NOMAD parser implementation for NeXus." optional = false python-versions = ">=3.8" files = [ - {file = "pynxtools-0.7.1-py3-none-any.whl", hash = "sha256:7f1a3eeef599b31a5571980cb2105074dc96c58dccb729703e1f4c4c46decbfd"}, - {file = "pynxtools-0.7.1.tar.gz", hash = "sha256:076b65cbe17ca822cb11d69841693a16c716fcb8c1452eba9bd27eec035242a1"}, + {file = "pynxtools-0.7.4-py3-none-any.whl", hash = "sha256:16a6a52ea8feb5847696bc6e474e31099840ae2fbcd1297bc5c37e756b7e6a62"}, + {file = "pynxtools-0.7.4.tar.gz", hash = "sha256:3e75a60cd92f0fea6eb9d456c135d62cf0f30cbdc54d73e71023d1db42920094"}, ] [package.dependencies] @@ -3820,25 +3847,29 @@ files = [ [[package]] name = "pywin32" -version = "306" +version = "307" description = "Python for Window Extensions" optional = false python-versions = "*" files = [ - {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, - {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, - {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, - {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, - {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, - {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, - {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, - {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, - {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, - {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, - {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, - {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, - {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, - {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, + {file = "pywin32-307-cp310-cp310-win32.whl", hash = "sha256:f8f25d893c1e1ce2d685ef6d0a481e87c6f510d0f3f117932781f412e0eba31b"}, + {file = "pywin32-307-cp310-cp310-win_amd64.whl", hash = "sha256:36e650c5e5e6b29b5d317385b02d20803ddbac5d1031e1f88d20d76676dd103d"}, + {file = "pywin32-307-cp310-cp310-win_arm64.whl", hash = "sha256:0c12d61e0274e0c62acee79e3e503c312426ddd0e8d4899c626cddc1cafe0ff4"}, + {file = "pywin32-307-cp311-cp311-win32.whl", hash = "sha256:fec5d27cc893178fab299de911b8e4d12c5954e1baf83e8a664311e56a272b75"}, + {file = "pywin32-307-cp311-cp311-win_amd64.whl", hash = "sha256:987a86971753ed7fdd52a7fb5747aba955b2c7fbbc3d8b76ec850358c1cc28c3"}, + {file = "pywin32-307-cp311-cp311-win_arm64.whl", hash = "sha256:fd436897c186a2e693cd0437386ed79f989f4d13d6f353f8787ecbb0ae719398"}, + {file = "pywin32-307-cp312-cp312-win32.whl", hash = "sha256:07649ec6b01712f36debf39fc94f3d696a46579e852f60157a729ac039df0815"}, + {file = "pywin32-307-cp312-cp312-win_amd64.whl", hash = "sha256:00d047992bb5dcf79f8b9b7c81f72e0130f9fe4b22df613f755ab1cc021d8347"}, + {file = "pywin32-307-cp312-cp312-win_arm64.whl", hash = "sha256:b53658acbfc6a8241d72cc09e9d1d666be4e6c99376bc59e26cdb6223c4554d2"}, + {file = "pywin32-307-cp313-cp313-win32.whl", hash = "sha256:ea4d56e48dc1ab2aa0a5e3c0741ad6e926529510516db7a3b6981a1ae74405e5"}, + {file = "pywin32-307-cp313-cp313-win_amd64.whl", hash = "sha256:576d09813eaf4c8168d0bfd66fb7cb3b15a61041cf41598c2db4a4583bf832d2"}, + {file = "pywin32-307-cp313-cp313-win_arm64.whl", hash = "sha256:b30c9bdbffda6a260beb2919f918daced23d32c79109412c2085cbc513338a0a"}, + {file = "pywin32-307-cp37-cp37m-win32.whl", hash = "sha256:5101472f5180c647d4525a0ed289ec723a26231550dbfd369ec19d5faf60e511"}, + {file = "pywin32-307-cp37-cp37m-win_amd64.whl", hash = "sha256:05de55a7c110478dc4b202230e98af5e0720855360d2b31a44bb4e296d795fba"}, + {file = "pywin32-307-cp38-cp38-win32.whl", hash = "sha256:13d059fb7f10792542082f5731d5d3d9645320fc38814759313e5ee97c3fac01"}, + {file = "pywin32-307-cp38-cp38-win_amd64.whl", hash = "sha256:7e0b2f93769d450a98ac7a31a087e07b126b6d571e8b4386a5762eb85325270b"}, + {file = "pywin32-307-cp39-cp39-win32.whl", hash = "sha256:55ee87f2f8c294e72ad9d4261ca423022310a6e79fb314a8ca76ab3f493854c6"}, + {file = "pywin32-307-cp39-cp39-win_amd64.whl", hash = "sha256:e9d5202922e74985b037c9ef46778335c102b74b95cec70f629453dbe7235d87"}, ] [[package]] @@ -4135,18 +4166,19 @@ files = [ [[package]] name = "rich" -version = "13.8.1" +version = "13.9.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" files = [ - {file = "rich-13.8.1-py3-none-any.whl", hash = "sha256:1760a3c0848469b97b558fc61c85233e3dafb69c7a071b4d60c38099d3cd4c06"}, - {file = "rich-13.8.1.tar.gz", hash = "sha256:8260cda28e3db6bf04d2d1ef4dbc03ba80a824c88b0e7668a0f23126a424844a"}, + {file = "rich-13.9.2-py3-none-any.whl", hash = "sha256:8c82a3d3f8dcfe9e734771313e606b39d8247bb6b826e196f4914b333b743cf1"}, + {file = "rich-13.9.2.tar.gz", hash = "sha256:51a2c62057461aaf7152b4d611168f93a9fc73068f8ded2790f29fe2b5366d0c"}, ] [package.dependencies] markdown-it-py = ">=2.2.0" pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.11\""} [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] @@ -4496,22 +4528,22 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.4.4)", "defusedxml (>=0.7.1)", [[package]] name = "sphinx-rtd-theme" -version = "2.0.0" +version = "3.0.0" description = "Read the Docs theme for Sphinx" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "sphinx_rtd_theme-2.0.0-py2.py3-none-any.whl", hash = "sha256:ec93d0856dc280cf3aee9a4c9807c60e027c7f7b461b77aeffed682e68f0e586"}, - {file = "sphinx_rtd_theme-2.0.0.tar.gz", hash = "sha256:bd5d7b80622406762073a04ef8fadc5f9151261563d47027de09910ce03afe6b"}, + {file = "sphinx_rtd_theme-3.0.0-py2.py3-none-any.whl", hash = "sha256:1ffe1539957775bfa0a7331370de7dc145b6eac705de23365dc55c5d94bb08e7"}, + {file = "sphinx_rtd_theme-3.0.0.tar.gz", hash = "sha256:905d67de03217fd3d76fbbdd992034ac8e77044ef8063a544dda1af74d409e08"}, ] [package.dependencies] -docutils = "<0.21" -sphinx = ">=5,<8" +docutils = ">0.18,<0.22" +sphinx = ">=6,<9" sphinxcontrib-jquery = ">=4,<5" [package.extras] -dev = ["bump2version", "sphinxcontrib-httpdomain", "transifex-client", "wheel"] +dev = ["bump2version", "transifex-client", "twine", "wheel"] [[package]] name = "sphinxcontrib-applehelp" @@ -4737,13 +4769,13 @@ test = ["pytest", "ruff"] [[package]] name = "tomli" -version = "2.0.1" +version = "2.0.2" description = "A lil' TOML parser" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, + {file = "tomli-2.0.2-py3-none-any.whl", hash = "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38"}, + {file = "tomli-2.0.2.tar.gz", hash = "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"}, ] [[package]] @@ -4759,13 +4791,13 @@ files = [ [[package]] name = "toolz" -version = "0.12.1" +version = "1.0.0" description = "List processing tools and functional utilities" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "toolz-0.12.1-py3-none-any.whl", hash = "sha256:d22731364c07d72eea0a0ad45bafb2c2937ab6fd38a3507bf55eae8744aa7d85"}, - {file = "toolz-0.12.1.tar.gz", hash = "sha256:ecca342664893f177a13dac0e6b41cbd8ac25a358e5f215316d43e2100224f4d"}, + {file = "toolz-1.0.0-py3-none-any.whl", hash = "sha256:292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236"}, + {file = "toolz-1.0.0.tar.gz", hash = "sha256:2c86e3d9a04798ac556793bced838816296a2f085017664e4995cb40a1047a02"}, ] [[package]] @@ -4825,13 +4857,13 @@ test = ["argcomplete (>=3.0.3)", "mypy (>=1.7.0)", "pre-commit", "pytest (>=7.0, [[package]] name = "types-python-dateutil" -version = "2.9.0.20240906" +version = "2.9.0.20241003" description = "Typing stubs for python-dateutil" optional = true python-versions = ">=3.8" files = [ - {file = "types-python-dateutil-2.9.0.20240906.tar.gz", hash = "sha256:9706c3b68284c25adffc47319ecc7947e5bb86b3773f843c73906fd598bc176e"}, - {file = "types_python_dateutil-2.9.0.20240906-py3-none-any.whl", hash = "sha256:27c8cc2d058ccb14946eebcaaa503088f4f6dbc4fb6093d3d456a49aef2753f6"}, + {file = "types-python-dateutil-2.9.0.20241003.tar.gz", hash = "sha256:58cb85449b2a56d6684e41aeefb4c4280631246a0da1a719bdbe6f3fb0317446"}, + {file = "types_python_dateutil-2.9.0.20241003-py3-none-any.whl", hash = "sha256:250e1d8e80e7bbc3a6c99b907762711d1a1cdd00e978ad39cb5940f6f0a87f3d"}, ] [[package]] @@ -4872,13 +4904,13 @@ files = [ [[package]] name = "tzdata" -version = "2024.1" +version = "2024.2" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" files = [ - {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, - {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, + {file = "tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"}, + {file = "tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc"}, ] [[package]] @@ -4931,13 +4963,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "virtualenv" -version = "20.26.5" +version = "20.26.6" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.26.5-py3-none-any.whl", hash = "sha256:4f3ac17b81fba3ce3bd6f4ead2749a72da5929c01774948e243db9ba41df4ff6"}, - {file = "virtualenv-20.26.5.tar.gz", hash = "sha256:ce489cac131aa58f4b25e321d6d186171f78e6cb13fafbf32a840cee67733ff4"}, + {file = "virtualenv-20.26.6-py3-none-any.whl", hash = "sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2"}, + {file = "virtualenv-20.26.6.tar.gz", hash = "sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48"}, ] [package.dependencies] @@ -5179,4 +5211,4 @@ notebook = ["ipykernel", "jupyter", "jupyterlab", "jupyterlab-h5web"] [metadata] lock-version = "2.0" python-versions = ">=3.9, <3.12.3, !=3.11.9" -content-hash = "8c1003f86a5981c9514947b1ba8ddb44933aacd6ec695a57ea2d94f610dc8858" +content-hash = "611c9371c0929b1f155919c2962e356799564a027196c2b06c4f0da75d6b2314" From 1146b4f94d4c5a8545807ca5f6115bbb961883d8 Mon Sep 17 00:00:00 2001 From: rettigl Date: Sat, 12 Oct 2024 21:59:05 +0200 Subject: [PATCH 13/32] fix tests for calibrators --- .cspell/custom-dictionary.txt | 1 + sed/calibrator/delay.py | 34 +++++++++--------- sed/calibrator/energy.py | 64 +++++++++++++++++---------------- sed/config/config_model.py | 49 +++++++++++++++++-------- sed/core/config.py | 2 +- tests/calibrator/test_delay.py | 6 ++-- tests/calibrator/test_energy.py | 58 ++++++++++++++++-------------- tests/test_processor.py | 2 +- 8 files changed, 124 insertions(+), 92 deletions(-) diff --git a/.cspell/custom-dictionary.txt b/.cspell/custom-dictionary.txt index 406f9f0f..f4cc3e5c 100644 --- a/.cspell/custom-dictionary.txt +++ b/.cspell/custom-dictionary.txt @@ -290,6 +290,7 @@ ptargs pullrequest pval pyarrow +pydantic pyenv pygments pynxtools diff --git a/sed/calibrator/delay.py b/sed/calibrator/delay.py index a15edd15..13c4436f 100644 --- a/sed/calibrator/delay.py +++ b/sed/calibrator/delay.py @@ -288,6 +288,7 @@ def add_offsets( offsets["creation_date"] = datetime.now().timestamp() # column-based offsets if columns is not None: + offsets["columns"] = {} if weights is None: weights = 1 if isinstance(weights, (int, float, np.integer, np.floating)): @@ -314,7 +315,7 @@ def add_offsets( # store in offsets dictionary for col, weight, pmean, red in zip(columns, weights, preserve_mean, reductions): - offsets[col] = { + offsets["columns"][col] = { "weight": weight, "preserve_mean": pmean, "reduction": red, @@ -359,21 +360,22 @@ def add_offsets( f"Invalid value for flip_delay_axis in config: {flip_delay_axis}.", ) log_str += f"\n Flip delay axis: {flip_delay_axis}" - else: - columns.append(k) - try: - weight = v["weight"] - except KeyError: - weight = 1 - weights.append(weight) - pm = v.get("preserve_mean", False) - preserve_mean.append(pm) - red = v.get("reduction", None) - reductions.append(red) - log_str += ( - f"\n Column[{k}]: Weight={weight}, Preserve Mean: {pm}, " - f"Reductions: {red}." - ) + elif k == "columns": + for k2, v2 in offsets["columns"].items(): + columns.append(k2) + try: + weight = v2["weight"] + except KeyError: + weight = 1 + weights.append(weight) + pm = v2.get("preserve_mean", False) + preserve_mean.append(pm) + red = v2.get("reduction", None) + reductions.append(red) + log_str += ( + f"\n Column[{k}]: Weight={weight}, Preserve Mean: {pm}, " + f"Reductions: {red}." + ) if not suppress_output: logger.info(log_str) diff --git a/sed/calibrator/energy.py b/sed/calibrator/energy.py index 83429ed2..72cfa801 100644 --- a/sed/calibrator/energy.py +++ b/sed/calibrator/energy.py @@ -915,17 +915,17 @@ def append_energy_axis( df[energy_column] = df[energy_column] + scale_sign * bias_voltage if not suppress_output: logger.debug(f"Shifted energy column by constant bias value: {bias_voltage}.") - elif self._config["dataframe"]["bias_column"] in df.columns: + elif self._config["dataframe"]["columns"]["bias"] in df.columns: df = dfops.offset_by_other_columns( df=df, target_column=energy_column, - offset_columns=self._config["dataframe"]["bias_column"], + offset_columns=self._config["dataframe"]["columns"]["bias"], weights=scale_sign, ) if not suppress_output: logger.debug( "Shifted energy column by bias column: " - f"{self._config['dataframe']['bias_column']}.", + f"{self._config['dataframe']['columns']['bias']}.", ) else: logger.warning( @@ -1595,6 +1595,7 @@ def add_offsets( offsets["creation_date"] = datetime.now().timestamp() # column-based offsets if columns is not None: + offsets["columns"] = {} if isinstance(columns, str): columns = [columns] @@ -1623,7 +1624,7 @@ def add_offsets( # store in offsets dictionary for col, weight, pmean, red in zip(columns, weights, preserve_mean, reductions): - offsets[col] = { + offsets["columns"][col] = { "weight": weight, "preserve_mean": pmean, "reduction": red, @@ -1652,35 +1653,38 @@ def add_offsets( for k, v in offsets.items(): if k == "creation_date": continue - if k == "constant": + elif k == "constant": # flip sign if binding energy scale constant = v * scale_sign log_str += f"\n Constant: {constant}" - else: - columns.append(k) - try: - weight = v["weight"] - except KeyError: - weight = 1 - if not isinstance(weight, (int, float, np.integer, np.floating)): - raise TypeError(f"Invalid type for weight of column {k}: {type(weight)}") - # flip sign if binding energy scale - weight = weight * scale_sign - weights.append(weight) - pm = v.get("preserve_mean", False) - if str(pm).lower() in ["false", "0", "no"]: - pm = False - elif str(pm).lower() in ["true", "1", "yes"]: - pm = True - preserve_mean.append(pm) - red = v.get("reduction", None) - if str(red).lower() in ["none", "null"]: - red = None - reductions.append(red) - log_str += ( - f"\n Column[{k}]: Weight={weight}, Preserve Mean: {pm}, " - f"Reductions: {red}." - ) + elif k == "columns": + for k2, v2 in offsets["columns"].items(): + columns.append(k2) + try: + weight = v2["weight"] + except KeyError: + weight = 1 + if not isinstance(weight, (int, float, np.integer, np.floating)): + raise TypeError( + f"Invalid type for weight of column {k}: {type(weight)}", + ) + # flip sign if binding energy scale + weight = weight * scale_sign + weights.append(weight) + pm = v2.get("preserve_mean", False) + if str(pm).lower() in ["false", "0", "no"]: + pm = False + elif str(pm).lower() in ["true", "1", "yes"]: + pm = True + preserve_mean.append(pm) + red = v2.get("reduction", None) + if str(red).lower() in ["none", "null"]: + red = None + reductions.append(red) + log_str += ( + f"\n Column[{k}]: Weight={weight}, Preserve Mean: {pm}, " + f"Reductions: {red}." + ) if not suppress_output: logger.info(log_str) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 2356d4dd..16095adf 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -1,6 +1,5 @@ """Pydantic model to validate the config for SED package.""" from collections.abc import Sequence -from datetime import datetime from typing import Literal from typing import Optional from typing import Union @@ -99,7 +98,7 @@ class DataframeModel(BaseModel): ubid_offset: Optional[int] = None split_sector_id_from_dld_time: Optional[bool] = None sector_id_reserved_bits: Optional[int] = None - sector_delays: Optional[Sequence[int]] = None + sector_delays: Optional[Sequence[float]] = None # write validator for model so that x_column gets converted to columns: x @@ -142,22 +141,42 @@ class EnergyModel(BaseModel): x_width: Sequence[int] y_width: Sequence[int] color_clip: int + bias_key: Optional[str] = None class EnergyCalibrationModel(BaseModel): - d: float - t0: float - E0: float + creation_date: Optional[float] = None + d: Optional[float] = None + t0: Optional[float] = None + E0: Optional[float] = None energy_scale: str calibration: Optional[EnergyCalibrationModel] = None + class EnergyOffsets(BaseModel): + creation_date: Optional[float] = None + constant: Optional[float] = None + + ## This seems rather complicated way to define offsets, + # inconsistent in how args vs config are for add_offsets + class OffsetColumn(BaseModel): + weight: float + preserve_mean: bool + reduction: Optional[str] = None + + columns: Optional[dict[str, OffsetColumn]] = None + + offsets: Optional[EnergyOffsets] = None + class EnergyCorrectionModel(BaseModel): + creation_date: Optional[float] = None correction_type: str amplitude: float center: Sequence[float] - gamma: float - sigma: float - diameter: float + gamma: Optional[float] = None + sigma: Optional[float] = None + diameter: Optional[float] = None + sigma2: Optional[float] = None + amplitude2: Optional[float] = None correction: Optional[EnergyCorrectionModel] = None @@ -173,6 +192,7 @@ class MomentumModel(BaseModel): sigma_radius: int class MomentumCalibrationModel(BaseModel): + creation_date: Optional[float] = None kx_scale: float ky_scale: float x_center: float @@ -185,6 +205,7 @@ class MomentumCalibrationModel(BaseModel): calibration: Optional[MomentumCalibrationModel] = None class MomentumCorrectionModel(BaseModel): + creation_date: Optional[float] = None feature_points: Sequence[Sequence[float]] rotation_symmetry: int include_center: bool @@ -202,18 +223,18 @@ class DelayModel(BaseModel): p3_key: Optional[str] = None t0_key: Optional[str] = None - class Calibration(BaseModel): - creation_date: datetime + class DelayCalibration(BaseModel): + creation_date: Optional[float] = None adc_range: Sequence[int] delay_range: Sequence[float] time0: float delay_range_mm: Sequence[float] datafile: FilePath # .h5 extension in filepath - calibration: Optional[Calibration] = None + calibration: Optional[DelayCalibration] = None - class Offsets(BaseModel): - creation_date: Optional[datetime] = None + class DelayOffsets(BaseModel): + creation_date: Optional[float] = None constant: Optional[float] = None flip_delay_axis: Optional[bool] = False @@ -226,7 +247,7 @@ class OffsetColumn(BaseModel): columns: Optional[dict[str, OffsetColumn]] = None - offsets: Optional[Offsets] = None + offsets: Optional[DelayOffsets] = None class MetadataModel(BaseModel): diff --git a/sed/core/config.py b/sed/core/config.py index df44cfd1..ff11bbd7 100644 --- a/sed/core/config.py +++ b/sed/core/config.py @@ -148,7 +148,7 @@ def parse_config( return config_dict # Run the config through the ConfigModel to ensure it is valid config_model = ConfigModel(**config_dict) - return config_model.model_dump() + return config_model.model_dump(exclude_unset=True, exclude_none=True, exclude_defaults=True) def load_config(config_path: str) -> dict: diff --git a/tests/calibrator/test_delay.py b/tests/calibrator/test_delay.py index a81542e2..5ec9fc2c 100644 --- a/tests/calibrator/test_delay.py +++ b/tests/calibrator/test_delay.py @@ -167,7 +167,7 @@ def test_add_offset_from_config(df=test_dataframe) -> None: dc = DelayCalibrator(config=config) df, _ = dc.add_offsets(df.copy()) assert "delay" in df.columns - assert "bam" in dc.offsets.keys() + assert "bam" in dc.offsets["columns"].keys() np.testing.assert_allclose(expected, df["delay"]) @@ -189,7 +189,7 @@ def test_add_offset_from_args(df=test_dataframe) -> None: columns="bam", ) assert "delay" in df.columns - assert "bam" in dc.offsets.keys() + assert "bam" in dc.offsets["columns"].keys() expected = -np.array( delay_stage_vals + bam_vals * 1 + 1, ) @@ -215,5 +215,5 @@ def test_add_offset_from_dict(df=test_dataframe) -> None: dc = DelayCalibrator(config=config) df, _ = dc.add_offsets(df.copy(), offsets=offsets) assert "delay" in df.columns - assert "bam" in dc.offsets.keys() + assert "bam" in dc.offsets["columns"].keys() np.testing.assert_allclose(expected, df["delay"]) diff --git a/tests/calibrator/test_energy.py b/tests/calibrator/test_energy.py index 32dd1a4b..9ec24229 100644 --- a/tests/calibrator/test_energy.py +++ b/tests/calibrator/test_energy.py @@ -210,7 +210,7 @@ def test_calibrate_append(energy_scale: str, calibration_method: str) -> None: method=calibration_method, ) df, metadata = ec.append_energy_axis(df) - assert config["dataframe"]["energy_column"] in df.columns + assert config["dataframe"]["columns"]["energy"] in df.columns axis = calibdict["axis"] diff = np.diff(axis) if energy_scale == "kinetic": @@ -256,7 +256,7 @@ def test_append_energy_axis_from_dict_kwds(calib_type: str, calib_dict: dict) -> df, _, _ = loader.read_dataframe(folders=df_folder, collect_metadata=False) ec = EnergyCalibrator(config=config, loader=loader) df, metadata = ec.append_energy_axis(df, calibration=calib_dict) - assert config["dataframe"]["energy_column"] in df.columns + assert config["dataframe"]["columns"]["energy"] in df.columns for key in calib_dict: np.testing.assert_equal(metadata["calibration"][key], calib_dict[key]) @@ -266,7 +266,7 @@ def test_append_energy_axis_from_dict_kwds(calib_type: str, calib_dict: dict) -> df, _, _ = loader.read_dataframe(folders=df_folder, collect_metadata=False) ec = EnergyCalibrator(config=config, loader=loader) df, metadata = ec.append_energy_axis(df, **calib_dict) - assert config["dataframe"]["energy_column"] in df.columns + assert config["dataframe"]["columns"]["energy"] in df.columns for key in calib_dict: np.testing.assert_equal(metadata["calibration"][key], calib_dict[key]) @@ -307,14 +307,14 @@ def test_append_tof_ns_axis() -> None: df, _, _ = loader.read_dataframe(folders=df_folder, collect_metadata=False) ec = EnergyCalibrator(config=config, loader=loader) df, _ = ec.append_tof_ns_axis(df, binwidth=2e-9, binning=2) - assert config["dataframe"]["tof_ns_column"] in df.columns + assert config["dataframe"]["columns"]["tof_ns"] in df.columns np.testing.assert_allclose(df[ec.tof_column], df[ec.tof_ns_column] / 4) # from config df, _, _ = loader.read_dataframe(folders=df_folder, collect_metadata=False) ec = EnergyCalibrator(config=config, loader=loader) df, _ = ec.append_tof_ns_axis(df) - assert config["dataframe"]["tof_ns_column"] in df.columns + assert config["dataframe"]["columns"]["tof_ns"] in df.columns np.testing.assert_allclose(df[ec.tof_column], df[ec.tof_ns_column] / 2) # illegal keywords: @@ -390,7 +390,7 @@ def test_energy_correction(correction_type: str, correction_kwd: dict) -> None: **correction_kwd, ) df, metadata = ec.apply_energy_correction(sample_df) - t = df[config["dataframe"]["corrected_tof_column"]] + t = df[config["dataframe"]["columns"]["corrected_tof"]] assert t[0] == t[2] assert t[0] < t[1] assert t[3] == t[5] @@ -426,7 +426,7 @@ def test_energy_correction(correction_type: str, correction_kwd: dict) -> None: **correction, ) df, metadata = ec.apply_energy_correction(sample_df) - t = df[config["dataframe"]["corrected_tof_column"]] + t = df[config["dataframe"]["columns"]["corrected_tof"]] assert t[0] == t[2] assert t[0] < t[1] assert t[3] == t[5] @@ -514,7 +514,7 @@ def test_energy_correction_from_dict_kwds(correction_type: str, correction_kwd: sample_df, correction=correction_dict, ) - t = df[config["dataframe"]["corrected_tof_column"]] + t = df[config["dataframe"]["columns"]["corrected_tof"]] assert t[0] == t[2] assert t[0] < t[1] assert t[3] == t[5] @@ -534,7 +534,7 @@ def test_energy_correction_from_dict_kwds(correction_type: str, correction_kwd: loader=get_loader("mpes", config=config), ) df, metadata = ec.apply_energy_correction(sample_df, **correction_dict) - t = df[config["dataframe"]["corrected_tof_column"]] + t = df[config["dataframe"]["columns"]["corrected_tof"]] assert t[0] == t[2] assert t[0] < t[1] assert t[3] == t[5] @@ -585,7 +585,7 @@ def test_apply_energy_correction_raises(correction_type: str) -> None: sample_df, correction=correction_dict, ) - assert config["dataframe"]["corrected_tof_column"] in df.columns + assert config["dataframe"]["columns"]["corrected_tof"] in df.columns @pytest.mark.parametrize( @@ -603,12 +603,14 @@ def test_add_offsets_functionality(energy_scale: str) -> None: }, "offsets": { "constant": 1, - "off1": { - "weight": 1, - "preserve_mean": True, + "columns": { + "off1": { + "weight": 1, + "preserve_mean": True, + }, + "off2": {"weight": -1, "preserve_mean": False}, + "off3": {"weight": 1, "preserve_mean": False, "reduction": "mean"}, }, - "off2": {"weight": -1, "preserve_mean": False}, - "off3": {"weight": 1, "preserve_mean": False, "reduction": "mean"}, }, }, }, @@ -684,9 +686,11 @@ def test_add_offset_raises() -> None: }, "offsets": { "constant": 1, - "off1": {"weight": -1, "preserve_mean": True}, - "off2": {"weight": -1, "preserve_mean": False}, - "off3": {"weight": 1, "preserve_mean": False, "reduction": "mean"}, + "columns": { + "off1": {"weight": -1, "preserve_mean": True}, + "off2": {"weight": -1, "preserve_mean": False}, + "off3": {"weight": 1, "preserve_mean": False, "reduction": "mean"}, + }, }, }, } @@ -719,17 +723,15 @@ def test_add_offset_raises() -> None: # invalid sign with pytest.raises(TypeError): - cfg = deepcopy(cfg_dict) - cfg["energy"]["offsets"]["off1"]["weight"] = "wrong_type" - config = parse_config(config=cfg, folder_config={}, user_config={}, system_config={}) + config = parse_config(config=cfg_dict, folder_config={}, user_config={}, system_config={}) + config["energy"]["offsets"]["columns"]["off1"]["weight"] = "wrong_type" ec = EnergyCalibrator(config=config, loader=get_loader("flash", config=config)) _ = ec.add_offsets(t_df) # invalid constant with pytest.raises(TypeError): - cfg = deepcopy(cfg_dict) - cfg["energy"]["offsets"]["constant"] = "wrong_type" - config = parse_config(config=cfg, folder_config={}, user_config={}, system_config={}) + config = parse_config(config=cfg_dict, folder_config={}, user_config={}, system_config={}) + config["energy"]["offsets"]["constant"] = "wrong_type" ec = EnergyCalibrator(config=config, loader=get_loader("flash", config=config)) _ = ec.add_offsets(t_df) @@ -738,8 +740,10 @@ def test_align_dld_sectors() -> None: """test functionality and error handling of align_dld_sectors""" cfg_dict: dict[str, Any] = { "dataframe": { - "tof_column": "dldTimeSteps", - "sector_id_column": "dldSectorId", + "columns": { + "tof": "dldTimeSteps", + "sector_id": "dldSectorId", + }, "sector_delays": [-0.35, -0.25, -0.15, -0.05, 0.05, 0.15, 0.25, 0.35], }, } @@ -767,7 +771,7 @@ def test_align_dld_sectors() -> None: t_df = dask.dataframe.from_pandas(df.copy(), npartitions=2) res, meta = ec.align_dld_sectors( t_df, - tof_column=cfg_dict["dataframe"]["tof_column"], + tof_column=cfg_dict["dataframe"]["columns"]["tof"], sector_delays=cfg_dict["dataframe"]["sector_delays"], sector_id_column="dldSectorId", ) diff --git a/tests/test_processor.py b/tests/test_processor.py index 21e4df86..7070dead 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -719,7 +719,7 @@ def test_append_tof_ns_axis() -> None: verbose=True, ) processor.append_tof_ns_axis() - assert processor.config["dataframe"]["tof_ns_column"] in processor.dataframe + assert processor.config["dataframe"]["columns"]["tof_ns"] in processor.dataframe def test_delay_calibration_workflow() -> None: From a1a9b27a4e5698f8cce39db9cfc1c592b9c60647 Mon Sep 17 00:00:00 2001 From: rettigl Date: Sat, 12 Oct 2024 23:08:34 +0200 Subject: [PATCH 14/32] make model fail on extra parameters --- sed/config/config_model.py | 48 ++++++++++++++++++++++++++++++++++++-- sed/config/default.yaml | 2 -- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 16095adf..d2cbcd96 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -20,11 +20,15 @@ class Paths(BaseModel): + model_config = ConfigDict(extra="forbid") + raw: DirectoryPath processed: Union[DirectoryPath, NewPath] class CoreModel(BaseModel): + model_config = ConfigDict(extra="forbid") + loader: str verbose: Optional[bool] = None paths: Optional[Paths] = None @@ -32,6 +36,7 @@ class CoreModel(BaseModel): year: Optional[int] = None beamtime_id: Optional[int] = None instrument: Optional[str] = None + beamline: Optional[str] = None # TODO: move copy tool to a separate model use_copy_tool: Optional[bool] = None copy_tool_source: Optional[DirectoryPath] = None @@ -49,6 +54,8 @@ def validate_loader(cls, v: str) -> str: class ColumnsModel(BaseModel): + model_config = ConfigDict(extra="forbid") + x: str y: str tof: str @@ -69,6 +76,8 @@ class ColumnsModel(BaseModel): class ChannelModel(BaseModel): + model_config = ConfigDict(extra="forbid") + format: Literal["per_train", "per_electron", "per_pulse", "per_file"] dataset_key: str index_key: Optional[str] = None @@ -76,6 +85,8 @@ class ChannelModel(BaseModel): dtype: Optional[str] = None class subChannel(BaseModel): + model_config = ConfigDict(extra="forbid") + slice: int dtype: Optional[str] = None @@ -83,6 +94,8 @@ class subChannel(BaseModel): class DataframeModel(BaseModel): + model_config = ConfigDict(extra="forbid") + columns: ColumnsModel units: Optional[dict[str, str]] = None channels: Optional[dict[str, ChannelModel]] = None @@ -99,11 +112,14 @@ class DataframeModel(BaseModel): split_sector_id_from_dld_time: Optional[bool] = None sector_id_reserved_bits: Optional[int] = None sector_delays: Optional[Sequence[float]] = None + daq: Optional[str] = None # write validator for model so that x_column gets converted to columns: x class BinningModel(BaseModel): + model_config = ConfigDict(extra="forbid") + hist_mode: str mode: str pbar: bool @@ -112,6 +128,8 @@ class BinningModel(BaseModel): class HistogramModel(BaseModel): + model_config = ConfigDict(extra="forbid") + bins: Sequence[int] axes: Sequence[str] ranges: Sequence[tuple[float, float]] @@ -121,12 +139,16 @@ class StaticModel(BaseModel): """Static configuration settings that shouldn't be changed by users.""" # flash specific settings + model_config = ConfigDict(extra="forbid") + stream_name_prefixes: Optional[dict] = None stream_name_postfixes: Optional[dict] = None beamtime_dir: Optional[dict] = None class EnergyModel(BaseModel): + model_config = ConfigDict(extra="forbid") + bins: int ranges: Sequence[int] normalize: bool @@ -144,6 +166,8 @@ class EnergyModel(BaseModel): bias_key: Optional[str] = None class EnergyCalibrationModel(BaseModel): + model_config = ConfigDict(extra="forbid") + creation_date: Optional[float] = None d: Optional[float] = None t0: Optional[float] = None @@ -153,6 +177,8 @@ class EnergyCalibrationModel(BaseModel): calibration: Optional[EnergyCalibrationModel] = None class EnergyOffsets(BaseModel): + model_config = ConfigDict(extra="forbid") + creation_date: Optional[float] = None constant: Optional[float] = None @@ -168,6 +194,8 @@ class OffsetColumn(BaseModel): offsets: Optional[EnergyOffsets] = None class EnergyCorrectionModel(BaseModel): + model_config = ConfigDict(extra="forbid") + creation_date: Optional[float] = None correction_type: str amplitude: float @@ -182,6 +210,8 @@ class EnergyCorrectionModel(BaseModel): class MomentumModel(BaseModel): + model_config = ConfigDict(extra="forbid") + axes: Sequence[str] bins: Sequence[int] ranges: Sequence[Sequence[int]] @@ -192,6 +222,8 @@ class MomentumModel(BaseModel): sigma_radius: int class MomentumCalibrationModel(BaseModel): + model_config = ConfigDict(extra="forbid") + creation_date: Optional[float] = None kx_scale: float ky_scale: float @@ -205,6 +237,8 @@ class MomentumCalibrationModel(BaseModel): calibration: Optional[MomentumCalibrationModel] = None class MomentumCorrectionModel(BaseModel): + model_config = ConfigDict(extra="forbid") + creation_date: Optional[float] = None feature_points: Sequence[Sequence[float]] rotation_symmetry: int @@ -215,6 +249,8 @@ class MomentumCorrectionModel(BaseModel): class DelayModel(BaseModel): + model_config = ConfigDict(extra="forbid") + adc_range: Sequence[int] flip_time_axis: bool # Group keys in the datafile @@ -224,6 +260,8 @@ class DelayModel(BaseModel): t0_key: Optional[str] = None class DelayCalibration(BaseModel): + model_config = ConfigDict(extra="forbid") + creation_date: Optional[float] = None adc_range: Sequence[int] delay_range: Sequence[float] @@ -234,6 +272,8 @@ class DelayCalibration(BaseModel): calibration: Optional[DelayCalibration] = None class DelayOffsets(BaseModel): + model_config = ConfigDict(extra="forbid") + creation_date: Optional[float] = None constant: Optional[float] = None flip_delay_axis: Optional[bool] = False @@ -251,6 +291,8 @@ class OffsetColumn(BaseModel): class MetadataModel(BaseModel): + model_config = ConfigDict(extra="forbid") + archiver_url: Optional[HttpUrl] = None token: Optional[SecretStr] = None epics_pvs: Optional[Sequence[str]] = None @@ -262,6 +304,8 @@ class MetadataModel(BaseModel): class NexusModel(BaseModel): + model_config = ConfigDict(extra="forbid") + reader: str # prob good to have validation here # Currently only NXmpes definition is supported definition: Literal["NXmpes"] @@ -269,6 +313,8 @@ class NexusModel(BaseModel): class ConfigModel(BaseModel): + model_config = ConfigDict(extra="forbid") + core: CoreModel dataframe: DataframeModel energy: EnergyModel @@ -279,5 +325,3 @@ class ConfigModel(BaseModel): metadata: Optional[MetadataModel] = None nexus: Optional[NexusModel] = None static: Optional[StaticModel] = None - - model_config = ConfigDict(extra="forbid") diff --git a/sed/config/default.yaml b/sed/config/default.yaml index 3d591926..28b9be3b 100644 --- a/sed/config/default.yaml +++ b/sed/config/default.yaml @@ -88,8 +88,6 @@ delay: # value ranges of the analog-to-digital converter axes used for encoding the delay stage position # (in unbinned coordinates) adc_range: [1900, 25600] - # pump probe time overlap in ps - time0: 0 # if to flip the time axis flip_time_axis: False From e3577bb8784b3637ba6f936819ddf086dbcdb644 Mon Sep 17 00:00:00 2001 From: rettigl Date: Sat, 12 Oct 2024 23:10:46 +0200 Subject: [PATCH 15/32] fix flash loader --- sed/config/flash_example_config.yaml | 2 +- sed/loader/flash/dataframe.py | 2 +- sed/loader/flash/loader.py | 6 +-- sed/loader/flash/utils.py | 2 +- sed/loader/utils.py | 8 +-- tests/data/loader/flash/config.yaml | 51 +++----------------- tests/loader/flash/test_dataframe_creator.py | 2 +- tests/loader/flash/test_flash_loader.py | 2 +- 8 files changed, 18 insertions(+), 57 deletions(-) diff --git a/sed/config/flash_example_config.yaml b/sed/config/flash_example_config.yaml index 185fac78..0bc95dd2 100644 --- a/sed/config/flash_example_config.yaml +++ b/sed/config/flash_example_config.yaml @@ -45,7 +45,7 @@ dataframe: tof_binning: 8 # Binning parameter for time-of-flight data # Columns used for jitter correction - jitter_columns: [dldPosX, dldPosY, dldTimeSteps] + jitter_cols: [dldPosX, dldPosY, dldTimeSteps] # Column settings columns: diff --git a/sed/loader/flash/dataframe.py b/sed/loader/flash/dataframe.py index 887cb9dd..6501c82a 100644 --- a/sed/loader/flash/dataframe.py +++ b/sed/loader/flash/dataframe.py @@ -248,7 +248,7 @@ def df_train(self) -> pd.DataFrame: aux_alias = self._config.get("aux_alias", "dldAux") if channel == aux_alias: try: - sub_channels = self._config["channels"][aux_alias]["subChannels"] + sub_channels = self._config["channels"][aux_alias]["sub_channels"] except KeyError: raise KeyError( f"Provide 'subChannels' for auxiliary channel '{aux_alias}'.", diff --git a/sed/loader/flash/loader.py b/sed/loader/flash/loader.py index 9b3524bc..f451783f 100644 --- a/sed/loader/flash/loader.py +++ b/sed/loader/flash/loader.py @@ -110,7 +110,7 @@ def _initialize_dirs(self) -> None: ) from exc beamtime_dir = Path( - self._config["dataframe"]["beamtime_dir"][self._config["core"]["beamline"]], + self._config["static"]["beamtime_dir"][self._config["core"]["beamline"]], ) beamtime_dir = beamtime_dir.joinpath(f"{year}/data/{beamtime_id}/") @@ -175,7 +175,7 @@ def get_files_from_run_id( # type: ignore[override] FileNotFoundError: If no files are found for the given run in the directory. """ # Define the stream name prefixes based on the data acquisition identifier - stream_name_prefixes = self._config["dataframe"]["stream_name_prefixes"] + stream_name_prefixes = self._config["static"]["stream_name_prefixes"] if folders is None: folders = self._config["core"]["base_folder"] @@ -183,7 +183,7 @@ def get_files_from_run_id( # type: ignore[override] if isinstance(folders, str): folders = [folders] - daq = self._config["dataframe"].get("daq") + daq = self._config["dataframe"]["daq"] # Generate the file patterns to search for in the directory file_pattern = f"{stream_name_prefixes[daq]}_run{run_id}_*." + extension diff --git a/sed/loader/flash/utils.py b/sed/loader/flash/utils.py index 6eb2ac30..85bca9a4 100644 --- a/sed/loader/flash/utils.py +++ b/sed/loader/flash/utils.py @@ -78,7 +78,7 @@ def get_channels( if format_ == FORMATS[2] and aux_alias in available_channels: if extend_aux: channels.extend( - channel_dict[aux_alias]["subChannels"].keys(), + channel_dict[aux_alias]["sub_channels"].keys(), ) else: channels.extend([aux_alias]) diff --git a/sed/loader/utils.py b/sed/loader/utils.py index 6bcce9f8..4f18cf0f 100644 --- a/sed/loader/utils.py +++ b/sed/loader/utils.py @@ -160,9 +160,9 @@ def split_dld_time_from_sector_id( Args: df (pd.DataFrame | dask.dataframe.DataFrame): Dataframe to use. tof_column (str, optional): Name of the column containing the - time-of-flight steps. Defaults to config["dataframe"]["tof_column"]. + time-of-flight steps. Defaults to config["dataframe"]["columns"]["tof"]. sector_id_column (str, optional): Name of the column containing the - sectorID. Defaults to config["dataframe"]["sector_id_column"]. + sectorID. Defaults to config["dataframe"]["columns"]["sector_id"]. sector_id_reserved_bits (int, optional): Number of bits reserved for the config (dict, optional): Dataframe configuration dictionary. Defaults to None. @@ -172,11 +172,11 @@ def split_dld_time_from_sector_id( if tof_column is None: if config is None: raise ValueError("Either tof_column or config must be given.") - tof_column = config["tof_column"] + tof_column = config["columns"]["tof"] if sector_id_column is None: if config is None: raise ValueError("Either sector_id_column or config must be given.") - sector_id_column = config["sector_id_column"] + sector_id_column = config["columns"]["sector_id"] if sector_id_reserved_bits is None: if config is None: raise ValueError("Either sector_id_reserved_bits or config must be given.") diff --git a/tests/data/loader/flash/config.yaml b/tests/data/loader/flash/config.yaml index d4209636..f7296ea5 100644 --- a/tests/data/loader/flash/config.yaml +++ b/tests/data/loader/flash/config.yaml @@ -29,34 +29,10 @@ dataframe: # if true, removes the 3 bits reserved for dldSectorID from the dldTimeSteps column split_sector_id_from_dld_time: True sector_id_reserved_bits: 3 - # dataframe column containing x coordinates - x_column: dldPosX - # dataframe column containing corrected x coordinates - corrected_x_column: "X" - # dataframe column containing kx coordinates - kx_column: "kx" - # dataframe column containing y coordinates - - y_column: dldPosY - # dataframe column containing corrected y coordinates - corrected_y_column: "Y" - # dataframe column containing kx coordinates - ky_column: "ky" - # dataframe column containing time-of-flight data - - tof_column: dldTimeSteps - # dataframe column containing time-of-flight data in ns - tof_ns_column: dldTime - # dataframe column containing corrected time-of-flight data - corrected_tof_column: "tm" - # the time stamp column - time_stamp_alias: timeStamp # time length of a base time-of-flight bin in seconds tof_binwidth: 2.0576131995767355E-11 # binning parameter for time-of-flight data. 2**tof_binning bins per base bin tof_binning: 3 # power of 2, 4 means 8 bins per step - # dataframe column containing sector ID. obtained from dldTimeSteps column - sector_id_column: dldSectorID sector_delays: [0., 0., 0., 0., 0., 0., 0., 0.] @@ -125,32 +101,27 @@ dataframe: # The auxiliary channel has a special structure where the group further contains # a multidimensional structure so further aliases are defined below dldAux: - format: per_train + format: per_pulse index_key: "/uncategorised/FLASH.EXP/HEXTOF.DAQ/DLD1/index" dataset_key: "/uncategorised/FLASH.EXP/HEXTOF.DAQ/DLD1/value" slice: 4 - subChannels: + sub_channels: sampleBias: slice: 0 - dtype: float64 + dtype: float32 tofVoltage: slice: 1 dtype: float64 extractorVoltage: slice: 2 - dtype: float64 extractorCurrent: slice: 3 - dtype: float64 cryoTemperature: slice: 4 - dtype: float64 sampleTemperature: slice: 5 - dtype: float64 dldTimeBinSize: slice: 15 - dtype: float64 timeStamp: format: per_train @@ -173,8 +144,10 @@ dataframe: dataset_key: "/FL1/Photon Diagnostic/GMD/Pulse resolved energy/energy tunnel/value" slice: 0 + +# (Not to be changed by user) +static: # The prefixes of the stream names for different DAQ systems for parsing filenames - # (Not to be changed by user) stream_name_prefixes: pbd: "GMD_DATA_gmd_data" pbd2: "FL2PhotDiag_pbd2_gmd_data" @@ -188,15 +161,3 @@ dataframe: # (Not to be changed by user) beamtime_dir: pg2: "/asap3/flash/gpfs/pg2/" - -# metadata collection from scicat -# metadata: -# scicat_url: -# scicat_username: -# scicat_password: - -# The nexus collection routine shall be finalized soon for both instruments -# nexus: -# reader: "flash" -# definition: "NXmpes" -# input_files: ["NXmpes_config_HEXTOF_light.json"] diff --git a/tests/loader/flash/test_dataframe_creator.py b/tests/loader/flash/test_dataframe_creator.py index 64e7712c..fe1c8f79 100644 --- a/tests/loader/flash/test_dataframe_creator.py +++ b/tests/loader/flash/test_dataframe_creator.py @@ -234,7 +234,7 @@ def test_create_dataframe_per_train(config_dataframe: dict, h5_paths: list[Path] # The subchannels are stored in the second dimension # Only index amount of values are stored in the first dimension, the rest are NaNs # hence the slicing - subchannels = config_dataframe["channels"]["dldAux"]["subChannels"] + subchannels = config_dataframe["channels"]["dldAux"]["sub_channels"] for subchannel, values in subchannels.items(): assert np.all(df.df_train[subchannel].dropna().values == data[: key.size, values["slice"]]) diff --git a/tests/loader/flash/test_flash_loader.py b/tests/loader/flash/test_flash_loader.py index a34a9977..3d9a5170 100644 --- a/tests/loader/flash/test_flash_loader.py +++ b/tests/loader/flash/test_flash_loader.py @@ -33,7 +33,7 @@ def test_initialize_dirs( config_["core"]["year"] = "2000" # Find base path of beamline from config. Here, we use pg2 - base_path = config_["dataframe"]["beamtime_dir"]["pg2"] + base_path = config_["static"]["beamtime_dir"]["pg2"] expected_path = ( Path(base_path) / config_["core"]["year"] / "data" / config_["core"]["beamtime_id"] ) From 8054bfa276bc1c7328fdc5eb0cef199f31032cd0 Mon Sep 17 00:00:00 2001 From: rettigl Date: Sat, 12 Oct 2024 23:16:58 +0200 Subject: [PATCH 16/32] fix calibrator tests again --- tests/calibrator/test_delay.py | 2 +- tests/calibrator/test_energy.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/calibrator/test_delay.py b/tests/calibrator/test_delay.py index 5ec9fc2c..b5bbe49c 100644 --- a/tests/calibrator/test_delay.py +++ b/tests/calibrator/test_delay.py @@ -131,7 +131,7 @@ def test_delay_parameters_from_delay_range_mm() -> None: delay_stage_vals = np.linspace(0, 99, 100) cfg = { "core": {"loader": "flash"}, - "dataframe": {"delay_column": "delay"}, + "dataframe": {"columns": {"delay": "delay"}}, "delay": { "offsets": { "constant": 1, diff --git a/tests/calibrator/test_energy.py b/tests/calibrator/test_energy.py index 9ec24229..21bac331 100644 --- a/tests/calibrator/test_energy.py +++ b/tests/calibrator/test_energy.py @@ -294,8 +294,10 @@ def test_append_tof_ns_axis() -> None: """ cfg = { "dataframe": { - "tof_column": "t", - "tof_ns_column": "t_ns", + "columns": { + "tof": "t", + "tof_ns": "t_ns", + }, "tof_binning": 2, "tof_binwidth": 1e-9, }, From 35dcd115dbec2b681fcede93428d1610e0a46d9c Mon Sep 17 00:00:00 2001 From: rettigl Date: Sat, 12 Oct 2024 23:53:07 +0200 Subject: [PATCH 17/32] fix processor tests --- sed/config/config_model.py | 23 ++++++++++++++++++++--- sed/config/mpes_example_config.yaml | 3 +-- sed/core/processor.py | 4 ++-- tests/test_processor.py | 7 ++++--- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index d2cbcd96..4d9a006a 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -9,7 +9,6 @@ from pydantic import DirectoryPath from pydantic import field_validator from pydantic import FilePath -from pydantic import HttpUrl from pydantic import NewPath from pydantic import SecretStr @@ -106,6 +105,8 @@ class DataframeModel(BaseModel): jitter_cols: Sequence[str] jitter_amps: Union[float, Sequence[float]] timed_dataframe_unit_time: float + first_event_time_stamp_key: Optional[str] = None + ms_markers_key: Optional[str] = None # flash specific settings forward_fill_iterations: Optional[int] = None ubid_offset: Optional[int] = None @@ -172,6 +173,8 @@ class EnergyCalibrationModel(BaseModel): d: Optional[float] = None t0: Optional[float] = None E0: Optional[float] = None + coeffs: Optional[Sequence[float]] = None + offset: Optional[float] = None energy_scale: str calibration: Optional[EnergyCalibrationModel] = None @@ -244,9 +247,23 @@ class MomentumCorrectionModel(BaseModel): rotation_symmetry: int include_center: bool use_center: bool + ascale: Optional[Sequence[float]] = None + center_point: Optional[Sequence[float]] = None + outer_points: Optional[Sequence[Sequence[float]]] = None correction: Optional[MomentumCorrectionModel] = None + class MomentumTransformationModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + creation_date: Optional[float] = None + scale: Optional[float] = None + angle: Optional[float] = None + xtrans: Optional[float] = None + ytrans: Optional[float] = None + + transformations: Optional[MomentumTransformationModel] = None + class DelayModel(BaseModel): model_config = ConfigDict(extra="forbid") @@ -293,7 +310,7 @@ class OffsetColumn(BaseModel): class MetadataModel(BaseModel): model_config = ConfigDict(extra="forbid") - archiver_url: Optional[HttpUrl] = None + archiver_url: Optional[str] = None token: Optional[SecretStr] = None epics_pvs: Optional[Sequence[str]] = None fa_in_channel: Optional[str] = None @@ -309,7 +326,7 @@ class NexusModel(BaseModel): reader: str # prob good to have validation here # Currently only NXmpes definition is supported definition: Literal["NXmpes"] - input_files: Sequence[FilePath] + input_files: Sequence[str] class ConfigModel(BaseModel): diff --git a/sed/config/mpes_example_config.yaml b/sed/config/mpes_example_config.yaml index 3d1d3e39..1d3753d9 100644 --- a/sed/config/mpes_example_config.yaml +++ b/sed/config/mpes_example_config.yaml @@ -15,8 +15,6 @@ core: gid: 1001 dataframe: - # dataframe column name for the time stamp column - time_stamp_alias: "timeStamps" # hdf5 group name containing eventIDs occurring at every millisecond (used to calculate timestamps) ms_markers_key: "msMarkers" # hdf5 attribute containing the timestamp of the first event in a file @@ -38,6 +36,7 @@ dataframe: ky: ky # dataframe column containing ky coordinates energy: energy # dataframe column containing energy data delay: delay # dataframe column containing delay data + timestamp: timeStamps # dataframe column containing timestamp data # time length of a base time-of-flight bin in ns tof_binwidth: 4.125e-12 # Binning factor of the tof_column-data compared to tof_binwidth diff --git a/sed/core/processor.py b/sed/core/processor.py index e4d5f550..f1294ce9 100644 --- a/sed/core/processor.py +++ b/sed/core/processor.py @@ -1192,8 +1192,8 @@ def load_bias_series( ): raise ValueError( "If binned_data is provided as an xarray, it needs to contain dimensions " - f"'{self._config['dataframe']['tof_column']}' and " - f"'{self._config['dataframe']['bias_column']}'!.", + f"'{self._config['dataframe']['columns']['tof']}' and " + f"'{self._config['dataframe']['columns']['bias']}'!.", ) tof = binned_data.coords[self._config["dataframe"]["columns"]["tof"]].values biases = binned_data.coords[self._config["dataframe"]["columns"]["bias"]].values diff --git a/tests/test_processor.py b/tests/test_processor.py index 7070dead..9471f721 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -660,8 +660,9 @@ def test_align_dld_sectors() -> None: user_config={}, system_config={}, ) - config["core"]["paths"]["processed"] = ( - config["core"]["paths"]["processed"] + "_align_dld_sectors" + config["core"]["paths"]["processed"] = Path( + config["core"]["paths"]["processed"], + "_align_dld_sectors", ) processor = SedProcessor( folder=df_folder + "../flash/", @@ -998,7 +999,7 @@ def test_compute_with_normalization() -> None: def test_get_normalization_histogram() -> None: """Test the generation function for the normalization histogram""" - config = {"core": {"loader": "mpes"}, "dataframe": {"time_stamp_alias": "timeStamps"}} + config = {"core": {"loader": "mpes"}, "dataframe": {"columns": {"timestamp": "timeStamps"}}} processor = SedProcessor( folder=df_folder, config=config, From 738cd85f7380d9184e698c6c146b53f18fd765cb Mon Sep 17 00:00:00 2001 From: rettigl Date: Sun, 13 Oct 2024 00:02:46 +0200 Subject: [PATCH 18/32] fix sxp loader --- sed/config/config_model.py | 1 + sed/config/sxp_example_config.yaml | 28 +++++++++++++++++---------- sed/loader/sxp/loader.py | 6 +++--- tests/data/loader/sxp/config.yaml | 30 +++++++++++++++++------------ tests/loader/sxp/test_sxp_loader.py | 4 ++-- 5 files changed, 42 insertions(+), 27 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 4d9a006a..2b716ee0 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -82,6 +82,7 @@ class ChannelModel(BaseModel): index_key: Optional[str] = None slice: Optional[int] = None dtype: Optional[str] = None + max_hits: Optional[int] = None class subChannel(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/sed/config/sxp_example_config.yaml b/sed/config/sxp_example_config.yaml index c0757fa5..1a7baa90 100644 --- a/sed/config/sxp_example_config.yaml +++ b/sed/config/sxp_example_config.yaml @@ -28,20 +28,27 @@ dataframe: daq: DA03 forward_fill_iterations: 2 num_trains: 10 - x_column: dldPosX - corrected_x_column: "X" - kx_column: "kx" - y_column: dldPosY - corrected_y_column: "Y" - ky_column: "ky" - tof_column: dldTimeSteps - tof_ns_column: dldTime - corrected_tof_column: "tm" - bias_column: "sampleBias" tof_binwidth: 6.875E-12 # in seconds tof_binning: 1 jitter_cols: ["dldPosX", "dldPosY", "dldTimeSteps"] + # Column settings + columns: + x: dldPosX + corrected_x: X + kx: kx + y: dldPosY + corrected_y: Y + ky: ky + tof: dldTimeSteps + tof_ns: dldTime + corrected_tof: tm + timestamp: timeStamp + auxiliary: dldAux + sector_id: dldSectorID + delay: delayStage + corrected_delay: pumpProbeTime + units: dldPosX: 'step' dldPosY: 'step' @@ -96,6 +103,7 @@ dataframe: dataset_key: "/CONTROL/SCS_ILH_LAS/MDL/OPTICALDELAY_PP800/actualPosition/value" index_key: "/INDEX/trainId" +static: stream_name_prefixes: DA03: "RAW-R" stream_name_postfixes: diff --git a/sed/loader/sxp/loader.py b/sed/loader/sxp/loader.py index a77e8ed6..c9d73a4c 100644 --- a/sed/loader/sxp/loader.py +++ b/sed/loader/sxp/loader.py @@ -116,7 +116,7 @@ def _initialize_dirs(self): ) from exc beamtime_dir = Path( - self._config["dataframe"]["beamtime_dir"][self._config["core"]["beamline"]], + self._config["static"]["beamtime_dir"][self._config["core"]["beamline"]], ) beamtime_dir = beamtime_dir.joinpath(f"{year}/{beamtime_id}/") @@ -158,8 +158,8 @@ def get_files_from_run_id( FileNotFoundError: If no files are found for the given run in the directory. """ # Define the stream name prefixes based on the data acquisition identifier - stream_name_prefixes = self._config["dataframe"]["stream_name_prefixes"] - stream_name_postfixes = self._config["dataframe"].get("stream_name_postfixes", {}) + stream_name_prefixes = self._config["static"]["stream_name_prefixes"] + stream_name_postfixes = self._config["static"].get("stream_name_postfixes", {}) if isinstance(run_id, (int, np.integer)): run_id = str(run_id).zfill(4) diff --git a/tests/data/loader/sxp/config.yaml b/tests/data/loader/sxp/config.yaml index 095178ff..c5680847 100644 --- a/tests/data/loader/sxp/config.yaml +++ b/tests/data/loader/sxp/config.yaml @@ -4,28 +4,33 @@ core: paths: raw: "tests/data/loader/sxp/" processed: "tests/data/loader/sxp/parquet" - -binning: num_cores: 10 dataframe: ubid_offset: 0 daq: DA03 forward_fill_iterations: 2 - x_column: dldPosX - corrected_x_column: "X" - kx_column: "kx" - y_column: dldPosY - corrected_y_column: "Y" - ky_column: "ky" - tof_column: dldTimeSteps - tof_ns_column: dldTime - corrected_tof_column: "tm" - bias_column: "sampleBias" tof_binwidth: 2.0576131995767355E-11 # in seconds tof_binning: 3 jitter_cols: ["dldPosX", "dldPosY", "dldTimeSteps"] + # Column settings + columns: + x: dldPosX + corrected_x: X + kx: kx + y: dldPosY + corrected_y: Y + ky: ky + tof: dldTimeSteps + tof_ns: dldTime + corrected_tof: tm + timestamp: timeStamp + auxiliary: dldAux + sector_id: dldSectorID + delay: delayStage + corrected_delay: pumpProbeTime + units: dldPosX: 'step' dldPosY: 'step' @@ -78,6 +83,7 @@ dataframe: dataset_key: "/CONTROL/SCS_ILH_LAS/MDL/OPTICALDELAY_PP800/actualPosition/value" index_key: "/INDEX/trainId" +static: stream_name_prefixes: DA03: "RAW-R" stream_name_postfixes: diff --git a/tests/loader/sxp/test_sxp_loader.py b/tests/loader/sxp/test_sxp_loader.py index 09588152..85a31eac 100644 --- a/tests/loader/sxp/test_sxp_loader.py +++ b/tests/loader/sxp/test_sxp_loader.py @@ -87,7 +87,7 @@ def test_initialize_dirs(config_file: dict, fs) -> None: config["core"]["year"] = "2000" # Find base path of beamline from config. - base_path = config["dataframe"]["beamtime_dir"]["sxp"] + base_path = config["static"]["beamtime_dir"]["sxp"] expected_path = Path(base_path) / config["core"]["year"] / config["core"]["beamtime_id"] # Create expected paths expected_raw_path = expected_path / "raw" @@ -150,7 +150,7 @@ def test_data_keys_not_in_h5(config_file: dict, key_type: str): sl = SXPLoader(config=config) with pytest.raises(ValueError) as e: - sl.create_dataframe_per_file(config["core"]["paths"]["raw"] + H5_PATH) + sl.create_dataframe_per_file(Path(config["core"]["paths"]["raw"], H5_PATH)) assert str(e.value.args[0]) == f"The {key_type} for channel dldPosX does not exist." From bc6f45799eabed540f6e4f93a7122a5475f2638a Mon Sep 17 00:00:00 2001 From: rettigl Date: Sun, 13 Oct 2024 00:28:21 +0200 Subject: [PATCH 19/32] update notebooks --- sed/config/config_model.py | 4 +++- sed/config/sxp_example_config.yaml | 4 ++-- ...ion_pipeline_for_example_time-resolved_ARPES_data.ipynb | 5 +---- tutorial/3_metadata_collection_and_export_to_NeXus.ipynb | 7 ++----- tutorial/4_hextof_workflow.ipynb | 4 +++- tutorial/5_sxp_workflow.ipynb | 6 +++--- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 2b716ee0..1ef3c4e4 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -33,7 +33,7 @@ class CoreModel(BaseModel): paths: Optional[Paths] = None num_cores: Optional[int] = None year: Optional[int] = None - beamtime_id: Optional[int] = None + beamtime_id: Optional[Union[int, str]] = None instrument: Optional[str] = None beamline: Optional[str] = None # TODO: move copy tool to a separate model @@ -83,6 +83,7 @@ class ChannelModel(BaseModel): slice: Optional[int] = None dtype: Optional[str] = None max_hits: Optional[int] = None + scale: Optional[float] = None class subChannel(BaseModel): model_config = ConfigDict(extra="forbid") @@ -115,6 +116,7 @@ class DataframeModel(BaseModel): sector_id_reserved_bits: Optional[int] = None sector_delays: Optional[Sequence[float]] = None daq: Optional[str] = None + num_trains: Optional[int] = None # write validator for model so that x_column gets converted to columns: x diff --git a/sed/config/sxp_example_config.yaml b/sed/config/sxp_example_config.yaml index 1a7baa90..b9e4d298 100644 --- a/sed/config/sxp_example_config.yaml +++ b/sed/config/sxp_example_config.yaml @@ -7,9 +7,9 @@ core: beamline: sxp instrument: sxp paths: - data_raw_dir: "/path/to/data" + raw: "/path/to/data" # change this to a local directory where you want to store the parquet files - data_parquet_dir: "/path/to/parquet" + processed: "/path/to/parquet" binning: # Histogram computation mode to use. diff --git a/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb b/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb index 82798d55..28865853 100644 --- a/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb +++ b/tutorial/2_conversion_pipeline_for_example_time-resolved_ARPES_data.ipynb @@ -59,7 +59,7 @@ "outputs": [], "source": [ "# create sed processor using the config file:\n", - "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", verbose=True)" + "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", system_config={}, verbose=True)" ] }, { @@ -649,9 +649,6 @@ } ], "metadata": { - "interpreter": { - "hash": "728003ee06929e5fa5ff815d1b96bf487266025e4b7440930c6bf4536d02d243" - }, "kernelspec": { "display_name": "python3", "language": "python", diff --git a/tutorial/3_metadata_collection_and_export_to_NeXus.ipynb b/tutorial/3_metadata_collection_and_export_to_NeXus.ipynb index 98c1258c..7ac5d39f 100644 --- a/tutorial/3_metadata_collection_and_export_to_NeXus.ipynb +++ b/tutorial/3_metadata_collection_and_export_to_NeXus.ipynb @@ -143,7 +143,7 @@ "outputs": [], "source": [ "# create sed processor using the config file, and collect the meta data from the files:\n", - "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", metadata=metadata, collect_metadata=True)" + "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", system_config={}, metadata=metadata, collect_metadata=True)" ] }, { @@ -290,9 +290,6 @@ } ], "metadata": { - "interpreter": { - "hash": "728003ee06929e5fa5ff815d1b96bf487266025e4b7440930c6bf4536d02d243" - }, "kernelspec": { "display_name": "python3", "language": "python", @@ -308,7 +305,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.9.19" } }, "nbformat": 4, diff --git a/tutorial/4_hextof_workflow.ipynb b/tutorial/4_hextof_workflow.ipynb index c9b36c04..e6484602 100644 --- a/tutorial/4_hextof_workflow.ipynb +++ b/tutorial/4_hextof_workflow.ipynb @@ -31,6 +31,8 @@ }, "outputs": [], "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", "from typing import List\n", "from pathlib import Path\n", "import os\n", @@ -964,7 +966,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.9", + "display_name": "python3", "language": "python", "name": "python3" }, diff --git a/tutorial/5_sxp_workflow.ipynb b/tutorial/5_sxp_workflow.ipynb index bb7c6a7b..3241545f 100644 --- a/tutorial/5_sxp_workflow.ipynb +++ b/tutorial/5_sxp_workflow.ipynb @@ -53,8 +53,8 @@ "config = {\n", " \"core\": {\n", " \"paths\": {\n", - " \"data_raw_dir\": \"/gpfs/exfel/exp/SXP/202302/p004316/raw/\",\n", - " \"data_parquet_dir\": os.path.expanduser(\"~\") + \"/sxp_parquet/\",\n", + " \"raw\": \"/gpfs/exfel/exp/SXP/202302/p004316/raw/\",\n", + " \"processed\": os.path.expanduser(\"~\") + \"/sxp_parquet/\",\n", " }\n", " }\n", "}\n", @@ -394,7 +394,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.12" + "version": "3.9.19" } }, "nbformat": 4, From bc40fea71674381735f0e265bccedd1ba89f259b Mon Sep 17 00:00:00 2001 From: rettigl Date: Sun, 13 Oct 2024 01:03:52 +0200 Subject: [PATCH 20/32] fix remaining tests --- sed/config/config_model.py | 2 +- sed/loader/mpes/loader.py | 2 +- tests/data/loader/generic/config.yaml | 3 +- tests/data/loader/mpes/config.yaml | 44 +++++++++++---------------- tests/loader/test_loaders.py | 23 +++++++++----- tests/test_diagnostics.py | 9 ++++-- 6 files changed, 44 insertions(+), 39 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 1ef3c4e4..61aed957 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -22,7 +22,7 @@ class Paths(BaseModel): model_config = ConfigDict(extra="forbid") raw: DirectoryPath - processed: Union[DirectoryPath, NewPath] + processed: Optional[Union[DirectoryPath, NewPath]] = None class CoreModel(BaseModel): diff --git a/sed/loader/mpes/loader.py b/sed/loader/mpes/loader.py index eaf296f0..4f5ed94b 100644 --- a/sed/loader/mpes/loader.py +++ b/sed/loader/mpes/loader.py @@ -749,7 +749,7 @@ def get_files_from_run_id( ) if folders is None: - folders = self._config["core"]["paths"]["data_raw_dir"] + folders = str(self._config["core"]["paths"]["raw"]) if isinstance(folders, str): folders = [folders] diff --git a/tests/data/loader/generic/config.yaml b/tests/data/loader/generic/config.yaml index e901b4d3..e1c6163f 100644 --- a/tests/data/loader/generic/config.yaml +++ b/tests/data/loader/generic/config.yaml @@ -1 +1,2 @@ -test: +core: + loader: generic diff --git a/tests/data/loader/mpes/config.yaml b/tests/data/loader/mpes/config.yaml index 1a411be2..62e46ebc 100644 --- a/tests/data/loader/mpes/config.yaml +++ b/tests/data/loader/mpes/config.yaml @@ -1,10 +1,8 @@ core: paths: - data_raw_dir: "tests/data/loader/mpes/" + raw: "tests/data/loader/mpes/" dataframe: - # dataframe column name for the time stamp column - time_stamp_alias: "timeStamps" # hdf5 group name containing eventIDs occuring at every millisecond (used to calculate timestamps) ms_markers_key: "msMarkers" # hdf5 attribute containing the timestamp of the first event in a file @@ -13,30 +11,6 @@ dataframe: timed_dataframe_unit_time: 0.001 # list of columns to apply jitter to jitter_cols: ["X", "Y", "t", "ADC"] - # dataframe column containing x coordinates - x_column: "X" - # dataframe column containing y coordinates - y_column: "Y" - # dataframe column containing time-of-flight data - tof_column: "t" - # dataframe column containing analog-to-digital data - adc_column: "ADC" - # dataframe column containing bias voltage data - bias_column: "sampleBias" - # dataframe column containing corrected x coordinates - corrected_x_column: "Xm" - # dataframe column containing corrected y coordinates - corrected_y_column: "Ym" - # dataframe column containing corrected time-of-flight data - corrected_tof_column: "tm" - # dataframe column containing kx coordinates - kx_column: "kx" - # dataframe column containing ky coordinates - ky_column: "ky" - # dataframe column containing energy data - energy_column: "energy" - # dataframe column containing delay data - delay_column: "delay" # time length of a base time-of-flight bin in ns tof_binwidth: 4.125e-12 # Binning factor of the tof_column-data compared to tof_binwidth (2^(tof_binning-1)) @@ -44,6 +18,22 @@ dataframe: # binning factor used for the adc coordinate (2^(adc_binning-1)) adc_binning: 3 # Default units for dataframe entries + + columns: + x: X # dataframe column containing x coordinates + y: Y # dataframe column containing y coordinates + tof: t # dataframe column containing time-of-flight data + adc: ADC # dataframe column containing analog-to-digital data + bias: sampleBias # dataframe column containing bias voltage data + corrected_x: Xm # dataframe column containing corrected x coordinates + corrected_y: Ym # dataframe column containing corrected y coordinates + corrected_tof: tm # dataframe column containing corrected time-of-flight data + kx: kx # dataframe column containing kx coordinates + ky: ky # dataframe column containing ky coordinates + energy: energy # dataframe column containing energy data + delay: delay # dataframe column containing delay data + timestamp: timeStamps # dataframe column containing timestamp data + units: X: 'step' Y: 'step' diff --git a/tests/loader/test_loaders.py b/tests/loader/test_loaders.py index 734e7b44..f3e0bf84 100644 --- a/tests/loader/test_loaders.py +++ b/tests/loader/test_loaders.py @@ -70,6 +70,9 @@ def get_all_loaders() -> list[ParameterSet]: loader_name, "config.yaml", ), + folder_config={}, + user_config={}, + system_config={}, ), ) for loader_name in get_names_of_all_loaders() @@ -95,8 +98,9 @@ def test_has_correct_read_dataframe_func(loader: BaseLoader, read_type: str) -> # Fix for race condition during parallel testing if loader.__name__ in {"flash", "sxp"}: config = deepcopy(loader._config) # pylint: disable=protected-access - config["core"]["paths"]["processed"] = ( - config["core"]["paths"]["processed"] + f"_{read_type}" + config["core"]["paths"]["processed"] = Path( + config["core"]["paths"]["processed"], + f"_{read_type}", ) loader = get_loader(loader_name=loader.__name__, config=config) @@ -183,8 +187,9 @@ def test_timed_dataframe(loader: BaseLoader) -> None: # Fix for race condition during parallel testing if loader.__name__ in {"flash", "sxp"}: config = deepcopy(loader._config) # pylint: disable=protected-access - config["core"]["paths"]["processed"] = ( - config["core"]["paths"]["processed"] + "_timed_dataframe" + config["core"]["paths"]["processed"] = Path( + config["core"]["paths"]["processed"], + "_timed_dataframe", ) loader = get_loader(loader_name=loader.__name__, config=config) @@ -226,7 +231,10 @@ def test_get_count_rate(loader: BaseLoader) -> None: # Fix for race condition during parallel testing if loader.__name__ in {"flash", "sxp"}: config = deepcopy(loader._config) # pylint: disable=protected-access - config["core"]["paths"]["processed"] = config["core"]["paths"]["processed"] + "_count_rate" + config["core"]["paths"]["processed"] = Path( + config["core"]["paths"]["processed"], + "_count_rate", + ) loader = get_loader(loader_name=loader.__name__, config=config) if loader.__name__ != "BaseLoader": @@ -273,8 +281,9 @@ def test_get_elapsed_time(loader: BaseLoader) -> None: # Fix for race condition during parallel testing if loader.__name__ in {"flash", "sxp"}: config = deepcopy(loader._config) # pylint: disable=protected-access - config["core"]["paths"]["processed"] = ( - config["core"]["paths"]["processed"] + "_elapsed_time" + config["core"]["paths"]["processed"] = Path( + config["core"]["paths"]["processed"], + "_elapsed_time", ) loader = get_loader(loader_name=loader.__name__, config=config) diff --git a/tests/test_diagnostics.py b/tests/test_diagnostics.py index 2614e69d..cb5bcce6 100644 --- a/tests/test_diagnostics.py +++ b/tests/test_diagnostics.py @@ -18,7 +18,12 @@ df_folder = package_dir + "/../tests/data/loader/mpes/" folder = package_dir + "/../tests/data/calibrator/" files = glob.glob(df_folder + "*.h5") -config = parse_config(package_dir + "/../tests/data/config/config.yaml") +config = parse_config( + package_dir + "/../tests/data/loader/mpes/config.yaml", + folder_config={}, + user_config={}, + system_config={}, +) loader = get_loader("mpes", config=config) @@ -39,7 +44,7 @@ def test_plot_histogram(ncols: int, backend: str) -> None: bins = config["histogram"]["bins"] for loc, axis in enumerate(axes): if axis.startswith("@"): - axes[loc] = config["dataframe"].get(axis.strip("@")) + axes[loc] = config["dataframe"]["columns"].get(axis.strip("@")) values = {axis: dataframe[axis].compute() for axis in axes} grid_histogram(values, ncols, axes, bins, ranges, backend) From b6db85f6eefd68fdbe0b61884171b4fd283fe695 Mon Sep 17 00:00:00 2001 From: rettigl Date: Sun, 13 Oct 2024 21:11:19 +0200 Subject: [PATCH 21/32] add config model for copy tool --- .cspell/custom-dictionary.txt | 1 + sed/config/config_model.py | 39 +++++++++++++++++++++++------ sed/config/mpes_example_config.yaml | 16 ++++++------ sed/core/processor.py | 15 ++++------- sed/loader/mpes/loader.py | 1 - tests/test_processor.py | 18 ++----------- 6 files changed, 47 insertions(+), 43 deletions(-) diff --git a/.cspell/custom-dictionary.txt b/.cspell/custom-dictionary.txt index f4cc3e5c..5fe97cab 100644 --- a/.cspell/custom-dictionary.txt +++ b/.cspell/custom-dictionary.txt @@ -126,6 +126,7 @@ ftype fwhm genindex getgid +getgrgid getmtime gpfs griddata diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 61aed957..b471e732 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -1,4 +1,5 @@ """Pydantic model to validate the config for SED package.""" +import grp from collections.abc import Sequence from typing import Literal from typing import Optional @@ -18,29 +19,45 @@ ## https://github.com/astral-sh/ruff/issues/5434 -class Paths(BaseModel): +class PathsModel(BaseModel): model_config = ConfigDict(extra="forbid") raw: DirectoryPath processed: Optional[Union[DirectoryPath, NewPath]] = None +class CopyToolModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + source: DirectoryPath + dest: DirectoryPath + safety_margin: Optional[float] = None + gid: Optional[int] = None + scheduler: Optional[str] = None + + @field_validator("gid") + @classmethod + def validate_gid(cls, v: int) -> int: + """Checks if the gid is valid on the system""" + try: + grp.getgrgid(v) + except KeyError: + raise ValueError(f"Invalid value {v} for gid. Group not found.") + return v + + class CoreModel(BaseModel): model_config = ConfigDict(extra="forbid") loader: str verbose: Optional[bool] = None - paths: Optional[Paths] = None + paths: Optional[PathsModel] = None num_cores: Optional[int] = None year: Optional[int] = None beamtime_id: Optional[Union[int, str]] = None instrument: Optional[str] = None beamline: Optional[str] = None - # TODO: move copy tool to a separate model - use_copy_tool: Optional[bool] = None - copy_tool_source: Optional[DirectoryPath] = None - copy_tool_dest: Optional[DirectoryPath] = None - copy_tool_kwds: Optional[dict] = None + copy_tool: Optional[CopyToolModel] = None @field_validator("loader") @classmethod @@ -51,6 +68,14 @@ def validate_loader(cls, v: str) -> str: raise ValueError(f"Invalid loader {v}. Available loaders are: {names}") return v + @field_validator("num_cores") + @classmethod + def validate_num_cores(cls, v: int) -> int: + """Checks if the num_cores field is a positive integer""" + if v < 1: + raise ValueError(f"Invalid value {v} for num_cores. Needs to be > 0.") + return v + class ColumnsModel(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/sed/config/mpes_example_config.yaml b/sed/config/mpes_example_config.yaml index 1d3753d9..c45ca865 100644 --- a/sed/config/mpes_example_config.yaml +++ b/sed/config/mpes_example_config.yaml @@ -4,15 +4,13 @@ core: # Number of parallel threads to use for parallelized jobs (e.g. binning, data conversion, copy, ...) num_cores: 20 # Option to use the copy tool to mirror data to a local storage location before processing. - use_copy_tool: False - # path to the root of the source data directory - copy_tool_source: null # "/path/to/data/" - # path to the root or the local data storage - copy_tool_dest: null # "/path/to/localDataStore/" - # optional keywords for the copy tool: - copy_tool_kwds: - # group id to set for copied files and folders - gid: 1001 + # copy_tool: + # # path to the root of the source data directory + # source: "/path/to/data/" + # # path to the root or the local data storage + # dest: "/path/to/localDataStore/" + # # group id to set for copied files and folders + # gid: 1000 dataframe: # hdf5 group name containing eventIDs occurring at every millisecond (used to calculate timestamps) diff --git a/sed/core/processor.py b/sed/core/processor.py index f1294ce9..03b7308b 100644 --- a/sed/core/processor.py +++ b/sed/core/processor.py @@ -161,22 +161,17 @@ def __init__( verbose=self._verbose, ) - self.use_copy_tool = self._config.get("core", {}).get( - "use_copy_tool", - False, - ) + self.use_copy_tool = "copy_tool" in self._config["core"] if self.use_copy_tool: try: self.ct = CopyTool( - source=self._config["core"]["copy_tool_source"], - dest=self._config["core"]["copy_tool_dest"], num_cores=self._config["core"]["num_cores"], - **self._config["core"].get("copy_tool_kwds", {}), + **self._config["core"]["copy_tool"], ) logger.debug( - f"Initialized copy tool: Copy file from " - f"'{self._config['core']['copy_tool_source']}' " - f"to '{self._config['core']['copy_tool_dest']}'.", + f"Initialized copy tool: Copy files from " + f"'{self._config['core']['copy_tool']['source']}' " + f"to '{self._config['core']['copy_tool']['dest']}'.", ) except KeyError: self.use_copy_tool = False diff --git a/sed/loader/mpes/loader.py b/sed/loader/mpes/loader.py index 4f5ed94b..16ad573d 100644 --- a/sed/loader/mpes/loader.py +++ b/sed/loader/mpes/loader.py @@ -72,7 +72,6 @@ def hdf5_to_dataframe( electron_channels = [] column_names = [] - print("Print values: ", channels) for name, channel in channels.items(): if channel["format"] == "per_electron": if channel["dataset_key"] in test_proc: diff --git a/tests/test_processor.py b/tests/test_processor.py index 9471f721..a98fa882 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -219,7 +219,7 @@ def test_attributes_setters() -> None: def test_copy_tool() -> None: """Test the copy tool functionality in the processor""" - config = {"core": {"loader": "mpes", "use_copy_tool": True}} + config: dict[str, dict[str, Any]] = {"core": {"loader": "mpes"}} processor = SedProcessor( config=config, folder_config={}, @@ -231,10 +231,7 @@ def test_copy_tool() -> None: config = { "core": { "loader": "mpes", - "use_copy_tool": True, - "copy_tool_source": source_folder, - "copy_tool_dest": dest_folder, - "copy_tool_kwds": {"gid": os.getgid()}, + "copy_tool": {"source": source_folder, "dest": dest_folder, "gid": os.getgid()}, }, } processor = SedProcessor( @@ -248,17 +245,6 @@ def test_copy_tool() -> None: processor.load(files=files) assert processor.files[0].find(dest_folder) > -1 - # test illegal keywords: - config["core"]["copy_tool_kwds"] = {"gid": os.getgid(), "illegal_keyword": True} - with pytest.raises(TypeError): - processor = SedProcessor( - config=config, - folder_config={}, - user_config={}, - system_config={}, - verbose=True, - ) - feature4 = np.array([[203.2, 341.96], [299.16, 345.32], [304.38, 149.88], [199.52, 152.48]]) feature5 = np.array( From bdc5bac5d778cff3c79aa300ded69c94a4bed55d Mon Sep 17 00:00:00 2001 From: rettigl Date: Sun, 13 Oct 2024 23:42:53 +0200 Subject: [PATCH 22/32] Add further type refinements to config model --- sed/calibrator/delay.py | 14 +++---- sed/calibrator/energy.py | 22 ++++------- sed/calibrator/momentum.py | 26 +++++-------- sed/config/config_model.py | 75 +++++++++++++++++++------------------- sed/core/processor.py | 64 ++++++++++++++++++++------------ sed/loader/mpes/loader.py | 4 +- 6 files changed, 102 insertions(+), 103 deletions(-) diff --git a/sed/calibrator/delay.py b/sed/calibrator/delay.py index 13c4436f..8035d456 100644 --- a/sed/calibrator/delay.py +++ b/sed/calibrator/delay.py @@ -146,7 +146,7 @@ def append_delay_axis( or datafile is not None ): calibration = {} - calibration["creation_date"] = datetime.now().timestamp() + calibration["creation_date"] = datetime.now() if adc_range is not None: calibration["adc_range"] = adc_range if delay_range is not None: @@ -158,9 +158,7 @@ def append_delay_axis( else: # report usage of loaded parameters if "creation_date" in calibration and not suppress_output: - datestring = datetime.fromtimestamp(calibration["creation_date"]).strftime( - "%m/%d/%Y, %H:%M:%S", - ) + datestring = calibration["creation_date"].strftime("%m/%d/%Y, %H:%M:%S") logger.info(f"Using delay calibration parameters generated on {datestring}") if adc_column is None: @@ -212,7 +210,7 @@ def append_delay_axis( ) if not suppress_output: logger.info(f"Converted delay_range (ps) = {calibration['delay_range']}") - calibration["creation_date"] = datetime.now().timestamp() + calibration["creation_date"] = datetime.now() if "delay_range" in calibration.keys(): df[delay_column] = calibration["delay_range"][0] + ( @@ -285,7 +283,7 @@ def add_offsets( # pylint:disable=duplicate-code # use passed parameters, overwrite config offsets = {} - offsets["creation_date"] = datetime.now().timestamp() + offsets["creation_date"] = datetime.now() # column-based offsets if columns is not None: offsets["columns"] = {} @@ -331,9 +329,7 @@ def add_offsets( offsets["flip_delay_axis"] = flip_delay_axis elif "creation_date" in offsets and not suppress_output: - datestring = datetime.fromtimestamp(offsets["creation_date"]).strftime( - "%m/%d/%Y, %H:%M:%S", - ) + datestring = offsets["creation_date"].strftime("%m/%d/%Y, %H:%M:%S") logger.info(f"Using delay offset parameters generated on {datestring}") if len(offsets) > 0: diff --git a/sed/calibrator/energy.py b/sed/calibrator/energy.py index 72cfa801..b004d25e 100644 --- a/sed/calibrator/energy.py +++ b/sed/calibrator/energy.py @@ -612,7 +612,7 @@ def calibrate( else: raise NotImplementedError() - self.calibration["creation_date"] = datetime.now().timestamp() + self.calibration["creation_date"] = datetime.now() return self.calibration def view( @@ -843,12 +843,10 @@ def append_energy_axis( if len(kwds) > 0: for key, value in kwds.items(): calibration[key] = value - calibration["creation_date"] = datetime.now().timestamp() + calibration["creation_date"] = datetime.now() elif "creation_date" in calibration and not suppress_output: - datestring = datetime.fromtimestamp(calibration["creation_date"]).strftime( - "%m/%d/%Y, %H:%M:%S", - ) + datestring = calibration["creation_date"].strftime("%m/%d/%Y, %H:%M:%S") logger.info(f"Using energy calibration parameters generated on {datestring}") # try to determine calibration type if not provided @@ -1202,7 +1200,7 @@ def common_apply_func(apply: bool): # noqa: ARG001 self.correction["amplitude"] = correction["amplitude"] self.correction["center"] = correction["center"] self.correction["correction_type"] = correction["correction_type"] - self.correction["creation_date"] = datetime.now().timestamp() + self.correction["creation_date"] = datetime.now() amplitude_slider.close() x_center_slider.close() y_center_slider.close() @@ -1440,12 +1438,10 @@ def apply_energy_correction( for key, value in kwds.items(): correction[key] = value - correction["creation_date"] = datetime.now().timestamp() + correction["creation_date"] = datetime.now() elif "creation_date" in correction and not suppress_output: - datestring = datetime.fromtimestamp(correction["creation_date"]).strftime( - "%m/%d/%Y, %H:%M:%S", - ) + datestring = correction["creation_date"].strftime("%m/%d/%Y, %H:%M:%S") logger.info(f"Using energy correction parameters generated on {datestring}") missing_keys = {"correction_type", "center", "amplitude"} - set(correction.keys()) @@ -1592,7 +1588,7 @@ def add_offsets( # pylint:disable=duplicate-code # use passed parameters, overwrite config offsets = {} - offsets["creation_date"] = datetime.now().timestamp() + offsets["creation_date"] = datetime.now() # column-based offsets if columns is not None: offsets["columns"] = {} @@ -1637,9 +1633,7 @@ def add_offsets( raise TypeError(f"Invalid type for constant: {type(constant)}") elif "creation_date" in offsets and not suppress_output: - datestring = datetime.fromtimestamp(offsets["creation_date"]).strftime( - "%m/%d/%Y, %H:%M:%S", - ) + datestring = offsets["creation_date"].strftime("%m/%d/%Y, %H:%M:%S") logger.info(f"Using energy offset parameters generated on {datestring}") if len(offsets) > 0: diff --git a/sed/calibrator/momentum.py b/sed/calibrator/momentum.py index f1852d8a..e2a40e38 100644 --- a/sed/calibrator/momentum.py +++ b/sed/calibrator/momentum.py @@ -678,7 +678,7 @@ def spline_warp_estimate( if self.pouter_ord is None: if self.pouter is not None: self.pouter_ord = po.pointset_order(self.pouter) - self.correction["creation_date"] = datetime.now().timestamp() + self.correction["creation_date"] = datetime.now() else: try: features = np.asarray( @@ -693,11 +693,7 @@ def spline_warp_estimate( ascale = np.asarray(ascale) if "creation_date" in self.correction: - datestring = datetime.fromtimestamp( - self.correction["creation_date"], - ).strftime( - "%m/%d/%Y, %H:%M:%S", - ) + datestring = self.correction["creation_date"].strftime("%m/%d/%Y, %H:%M:%S") logger.info( "No landmarks defined, using momentum correction parameters " f"generated on {datestring}", @@ -715,7 +711,7 @@ def spline_warp_estimate( self.add_features(features=features, rotsym=rotsym) else: - self.correction["creation_date"] = datetime.now().timestamp() + self.correction["creation_date"] = datetime.now() if ascale is not None: if isinstance(ascale, (int, float, np.floating, np.integer)): @@ -1135,9 +1131,7 @@ def pose_adjustment( ) elif "creation_date" in transformations: - datestring = datetime.fromtimestamp(transformations["creation_date"]).strftime( - "%m/%d/%Y, %H:%M:%S", - ) + datestring = transformations["creation_date"].strftime("%m/%d/%Y, %H:%M:%S") logger.info(f"Using transformation parameters generated on {datestring}") def update(scale: float, xtrans: float, ytrans: float, angle: float): @@ -1257,7 +1251,7 @@ def apply_func(apply: bool): # noqa: ARG001 fig.canvas.draw_idle() if transformations != self.transformations: - transformations["creation_date"] = datetime.now().timestamp() + transformations["creation_date"] = datetime.now() self.transformations = transformations if self._verbose: @@ -1714,7 +1708,7 @@ def calibrate( # Assemble into return dictionary self.calibration = {} - self.calibration["creation_date"] = datetime.now().timestamp() + self.calibration["creation_date"] = datetime.now() self.calibration["kx_axis"] = k_row self.calibration["ky_axis"] = k_col self.calibration["grid"] = (k_rowgrid, k_colgrid) @@ -1825,7 +1819,7 @@ def gather_correction_metadata(self) -> dict: pass if len(self.adjust_params) > 0: metadata["registration"] = self.adjust_params - metadata["registration"]["creation_date"] = datetime.now().timestamp() + metadata["registration"]["creation_date"] = datetime.now() metadata["registration"]["applied"] = True metadata["registration"]["depends_on"] = ( "/entry/process/registration/transformations/rot_z" @@ -1952,15 +1946,13 @@ def append_k_axis( ]: if key in kwds: calibration[key] = kwds.pop(key) - calibration["creation_date"] = datetime.now().timestamp() + calibration["creation_date"] = datetime.now() if len(kwds) > 0: raise TypeError(f"append_k_axis() got unexpected keyword arguments {kwds.keys()}.") if "creation_date" in calibration and not suppress_output: - datestring = datetime.fromtimestamp(calibration["creation_date"]).strftime( - "%m/%d/%Y, %H:%M:%S", - ) + datestring = calibration["creation_date"].strftime("%m/%d/%Y, %H:%M:%S") logger.info(f"Using momentum calibration parameters generated on {datestring}") try: diff --git a/sed/config/config_model.py b/sed/config/config_model.py index b471e732..5fde6513 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -1,6 +1,7 @@ """Pydantic model to validate the config for SED package.""" import grp from collections.abc import Sequence +from datetime import datetime from typing import Literal from typing import Optional from typing import Union @@ -132,6 +133,7 @@ class DataframeModel(BaseModel): jitter_cols: Sequence[str] jitter_amps: Union[float, Sequence[float]] timed_dataframe_unit_time: float + # mpes specific settings first_event_time_stamp_key: Optional[str] = None ms_markers_key: Optional[str] = None # flash specific settings @@ -149,11 +151,11 @@ class DataframeModel(BaseModel): class BinningModel(BaseModel): model_config = ConfigDict(extra="forbid") - hist_mode: str - mode: str + hist_mode: Literal["numpy", "numba"] + mode: Literal["fast", "lean", "legacy"] pbar: bool threads_per_worker: int - threadpool_API: str + threadpool_API: Literal["blas", "openmp"] class HistogramModel(BaseModel): @@ -185,36 +187,34 @@ class EnergyModel(BaseModel): normalize_order: int fastdtw_radius: int peak_window: int - calibration_method: str - energy_scale: str + calibration_method: Literal["lmfit", "lstsq", "lsq"] + energy_scale: Literal["binding", "kinetic"] tof_fermi: int - tof_width: Sequence[int] - x_width: Sequence[int] - y_width: Sequence[int] + tof_width: tuple[int, int] + x_width: tuple[int, int] + y_width: tuple[int, int] color_clip: int bias_key: Optional[str] = None class EnergyCalibrationModel(BaseModel): model_config = ConfigDict(extra="forbid") - creation_date: Optional[float] = None + creation_date: Optional[datetime] = None d: Optional[float] = None t0: Optional[float] = None E0: Optional[float] = None coeffs: Optional[Sequence[float]] = None offset: Optional[float] = None - energy_scale: str + energy_scale: Literal["binding", "kinetic"] calibration: Optional[EnergyCalibrationModel] = None class EnergyOffsets(BaseModel): model_config = ConfigDict(extra="forbid") - creation_date: Optional[float] = None + creation_date: Optional[datetime] = None constant: Optional[float] = None - ## This seems rather complicated way to define offsets, - # inconsistent in how args vs config are for add_offsets class OffsetColumn(BaseModel): weight: float preserve_mean: bool @@ -227,10 +227,10 @@ class OffsetColumn(BaseModel): class EnergyCorrectionModel(BaseModel): model_config = ConfigDict(extra="forbid") - creation_date: Optional[float] = None - correction_type: str + creation_date: Optional[datetime] = None + correction_type: Literal["Gaussian", "Lorentzian", "spherical", "Lorentzian_asymmetric"] amplitude: float - center: Sequence[float] + center: tuple[float, float] gamma: Optional[float] = None sigma: Optional[float] = None diameter: Optional[float] = None @@ -245,9 +245,9 @@ class MomentumModel(BaseModel): axes: Sequence[str] bins: Sequence[int] - ranges: Sequence[Sequence[int]] - detector_ranges: Sequence[Sequence[int]] - center_pixel: Sequence[int] + ranges: Sequence[tuple[int, int]] + detector_ranges: Sequence[tuple[int, int]] + center_pixel: tuple[int, int] sigma: int fwhm: int sigma_radius: int @@ -255,7 +255,7 @@ class MomentumModel(BaseModel): class MomentumCalibrationModel(BaseModel): model_config = ConfigDict(extra="forbid") - creation_date: Optional[float] = None + creation_date: Optional[datetime] = None kx_scale: float ky_scale: float x_center: float @@ -270,21 +270,21 @@ class MomentumCalibrationModel(BaseModel): class MomentumCorrectionModel(BaseModel): model_config = ConfigDict(extra="forbid") - creation_date: Optional[float] = None - feature_points: Sequence[Sequence[float]] + creation_date: Optional[datetime] = None + feature_points: Sequence[tuple[float, float]] rotation_symmetry: int include_center: bool use_center: bool ascale: Optional[Sequence[float]] = None - center_point: Optional[Sequence[float]] = None - outer_points: Optional[Sequence[Sequence[float]]] = None + center_point: Optional[tuple[float, float]] = None + outer_points: Optional[Sequence[tuple[float, float]]] = None correction: Optional[MomentumCorrectionModel] = None class MomentumTransformationModel(BaseModel): model_config = ConfigDict(extra="forbid") - creation_date: Optional[float] = None + creation_date: Optional[datetime] = None scale: Optional[float] = None angle: Optional[float] = None xtrans: Optional[float] = None @@ -296,7 +296,7 @@ class MomentumTransformationModel(BaseModel): class DelayModel(BaseModel): model_config = ConfigDict(extra="forbid") - adc_range: Sequence[int] + adc_range: tuple[int, int] flip_time_axis: bool # Group keys in the datafile p1_key: Optional[str] = None @@ -307,24 +307,22 @@ class DelayModel(BaseModel): class DelayCalibration(BaseModel): model_config = ConfigDict(extra="forbid") - creation_date: Optional[float] = None - adc_range: Sequence[int] - delay_range: Sequence[float] - time0: float - delay_range_mm: Sequence[float] - datafile: FilePath # .h5 extension in filepath + creation_date: Optional[datetime] = None + adc_range: Optional[tuple[int, int]] = None + delay_range: Optional[tuple[float, float]] = None + time0: Optional[float] = None + delay_range_mm: Optional[tuple[float, float]] = None + datafile: Optional[FilePath] # .h5 extension in filepath calibration: Optional[DelayCalibration] = None class DelayOffsets(BaseModel): model_config = ConfigDict(extra="forbid") - creation_date: Optional[float] = None + creation_date: Optional[datetime] = None constant: Optional[float] = None flip_delay_axis: Optional[bool] = False - ## This seems rather complicated way to define offsets, - # inconsistent in how args vs config are for add_offsets class OffsetColumn(BaseModel): weight: float preserve_mean: bool @@ -344,14 +342,15 @@ class MetadataModel(BaseModel): fa_in_channel: Optional[str] = None fa_hor_channel: Optional[str] = None ca_in_channel: Optional[str] = None - aperture_config: Optional[dict] = None - lens_mode_config: Optional[dict] = None + aperture_config: Optional[dict[datetime, dict]] = None + lens_mode_config: Optional[dict[str, dict]] = None class NexusModel(BaseModel): model_config = ConfigDict(extra="forbid") - reader: str # prob good to have validation here + # Currently only mpes reader is supported + reader: Literal["mpes"] # Currently only NXmpes definition is supported definition: Literal["NXmpes"] input_files: Sequence[str] diff --git a/sed/core/processor.py b/sed/core/processor.py index 03b7308b..d5f4c418 100644 --- a/sed/core/processor.py +++ b/sed/core/processor.py @@ -5,6 +5,7 @@ import pathlib from collections.abc import Sequence +from copy import deepcopy from datetime import datetime from typing import Any from typing import cast @@ -698,11 +699,13 @@ def save_splinewarp( correction[key] = [] for point in value: correction[key].append([float(i) for i in point]) + elif key == "creation_date": + correction[key] = value.isoformat() else: correction[key] = float(value) if "creation_date" not in correction: - correction["creation_date"] = datetime.now().timestamp() + correction["creation_date"] = datetime.now().isoformat() config = { "momentum": { @@ -785,10 +788,13 @@ def save_transformations( raise ValueError("No momentum transformation parameters to save!") transformations = {} for key, value in self.mc.transformations.items(): - transformations[key] = float(value) + if key == "creation_date": + transformations[key] = value.isoformat() + else: + transformations[key] = float(value) if "creation_date" not in transformations: - transformations["creation_date"] = datetime.now().timestamp() + transformations["creation_date"] = datetime.now().isoformat() config = { "momentum": { @@ -933,11 +939,13 @@ def save_momentum_calibration( for key, value in self.mc.calibration.items(): if key in ["kx_axis", "ky_axis", "grid", "extent"]: continue - - calibration[key] = float(value) + elif key == "creation_date": + calibration[key] = value.isoformat() + else: + calibration[key] = float(value) if "creation_date" not in calibration: - calibration["creation_date"] = datetime.now().timestamp() + calibration["creation_date"] = datetime.now().isoformat() config = {"momentum": {"calibration": calibration}} save_config(config, filename, overwrite) @@ -1069,16 +1077,18 @@ def save_energy_correction( if len(self.ec.correction) == 0: raise ValueError("No energy correction parameters to save!") correction = {} - for key, val in self.ec.correction.items(): + for key, value in self.ec.correction.items(): if key == "correction_type": - correction[key] = val + correction[key] = value elif key == "center": - correction[key] = [float(i) for i in val] + correction[key] = [float(i) for i in value] + elif key == "creation_date": + correction[key] = value.isoformat() else: - correction[key] = float(val) + correction[key] = float(value) if "creation_date" not in correction: - correction["creation_date"] = datetime.now().timestamp() + correction["creation_date"] = datetime.now().isoformat() config = {"energy": {"correction": correction}} save_config(config, filename, overwrite) @@ -1430,11 +1440,13 @@ def save_energy_calibration( calibration[key] = value elif key == "coeffs": calibration[key] = [float(i) for i in value] + elif key == "creation_date": + calibration[key] = value.isoformat() else: calibration[key] = float(value) if "creation_date" not in calibration: - calibration["creation_date"] = datetime.now().timestamp() + calibration["creation_date"] = datetime.now().isoformat() config = {"energy": {"calibration": calibration}} save_config(config, filename, overwrite) @@ -1595,10 +1607,14 @@ def save_energy_offset( if len(self.ec.offsets) == 0: raise ValueError("No energy offset parameters to save!") - if "creation_date" not in self.ec.offsets.keys(): - self.ec.offsets["creation_date"] = datetime.now().timestamp() + offsets = deepcopy(self.ec.offsets) + + if "creation_date" not in offsets.keys(): + offsets["creation_date"] = datetime.now() - config = {"energy": {"offsets": self.ec.offsets}} + offsets["creation_date"] = offsets["creation_date"].isoformat() + + config = {"energy": {"offsets": offsets}} save_config(config, filename, overwrite) logger.info(f'Saved energy offset parameters to "{filename}".') @@ -1791,11 +1807,13 @@ def save_delay_calibration( calibration[key] = value elif key in ["adc_range", "delay_range", "delay_range_mm"]: calibration[key] = [float(i) for i in value] + elif key == "creation_date": + calibration[key] = value.isoformat() else: calibration[key] = float(value) if "creation_date" not in calibration: - calibration["creation_date"] = datetime.now().timestamp() + calibration["creation_date"] = datetime.now().isoformat() config = { "delay": { @@ -1898,14 +1916,14 @@ def save_delay_offsets( if len(self.dc.offsets) == 0: raise ValueError("No delay offset parameters to save!") - if "creation_date" not in self.ec.offsets.keys(): - self.ec.offsets["creation_date"] = datetime.now().timestamp() + offsets = deepcopy(self.dc.offsets) - config = { - "delay": { - "offsets": self.dc.offsets, - }, - } + if "creation_date" not in offsets.keys(): + offsets["creation_date"] = datetime.now() + + offsets["creation_date"] = offsets["creation_date"].isoformat() + + config = {"delay": {"offsets": offsets}} save_config(config, filename, overwrite) logger.info(f'Saved delay offset parameters to "{filename}".') diff --git a/sed/loader/mpes/loader.py b/sed/loader/mpes/loader.py index 16ad573d..14a41ee2 100644 --- a/sed/loader/mpes/loader.py +++ b/sed/loader/mpes/loader.py @@ -853,7 +853,7 @@ def gather_metadata( # Get metadata from Epics archive if not present already epics_channels = self._config["metadata"]["epics_pvs"] - start = datetime.datetime.utcfromtimestamp(ts_from).isoformat() + start = datetime.datetime.utcfromtimestamp(ts_from) channels_missing = set(epics_channels) - set( metadata["file"].keys(), @@ -891,7 +891,7 @@ def gather_metadata( # Determine the correct aperture_config stamps = sorted( - list(self._config["metadata"]["aperture_config"]) + [start], + list(self._config["metadata"]["aperture_config"].keys()) + [start], ) current_index = stamps.index(start) timestamp = stamps[current_index - 1] # pick last configuration before file date From 022dc69406043047c6251453c88dfb64ca67a21d Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Mon, 14 Oct 2024 22:10:14 +0200 Subject: [PATCH 23/32] add tests for config model --- tests/test_config.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_config.py b/tests/test_config.py index 0ab54cbb..5ebf259b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -9,6 +9,7 @@ from pathlib import Path import pytest +from pydantic import ValidationError from sed.core.config import complete_dictionary from sed.core.config import load_config @@ -16,6 +17,10 @@ from sed.core.config import save_config package_dir = os.path.dirname(find_spec("sed").origin) + +test_config_dir = Path(package_dir).joinpath("../tests/data/loader/") +config_paths = test_config_dir.glob("*/*.yaml") + default_config_keys = [ "binning", "histogram", @@ -140,3 +145,28 @@ def test_save_dict() -> None: save_config(config_dict, filename, overwrite=True) config = load_config(filename) assert "test_entry" not in config.keys() + + +@pytest.mark.parametrize("config_path", config_paths) +def test_config_model_valid(config_path) -> None: + """Test the config model for a valid config.""" + config = parse_config(config_path, verify_config=True) + assert config is not None + + +def test_invalid_config_extra_field(): + """Test that an invalid config with an extra field fails validation.""" + default_config = parse_config(verify_config=False) + invalid_config = default_config.copy() + invalid_config["extra_field"] = "extra_value" + with pytest.raises(ValidationError): + parse_config(invalid_config, verify_config=True) + + +def test_invalid_config_missing_field(): + """Test that an invalid config with a missing required field fails validation.""" + default_config = parse_config(verify_config=False) + invalid_config = default_config.copy() + del invalid_config["core"]["loader"] + with pytest.raises(ValidationError): + parse_config(default_config=invalid_config, verify_config=True) From 52a11dcec657db9bbdb773ea970a030340ccec0d Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 14 Oct 2024 23:51:33 +0200 Subject: [PATCH 24/32] fix remaining tutorials --- tutorial/6_binning_with_time-stamped_data.ipynb | 4 ++-- tutorial/7_correcting_orthorhombic_symmetry.ipynb | 7 ++----- tutorial/8_jittering_tutorial.ipynb | 7 ++----- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/tutorial/6_binning_with_time-stamped_data.ipynb b/tutorial/6_binning_with_time-stamped_data.ipynb index 93080372..7978568f 100644 --- a/tutorial/6_binning_with_time-stamped_data.ipynb +++ b/tutorial/6_binning_with_time-stamped_data.ipynb @@ -68,7 +68,7 @@ "outputs": [], "source": [ "# create sed processor using the config file with time-stamps:\n", - "sp = sed.SedProcessor(folder=scandir, user_config=\"../sed/config/mpes_example_config.yaml\", time_stamps=True, verbose=True)" + "sp = sed.SedProcessor(folder=scandir, user_config=\"../sed/config/mpes_example_config.yaml\", system_config={}, time_stamps=True, verbose=True)" ] }, { @@ -162,7 +162,7 @@ "sp.load_bias_series(data_files=files, normalize=True, biases=voltages, ranges=[(64000, 76000)])\n", "rg = (65500, 66000)\n", "sp.find_bias_peaks(ranges=rg, ref_id=5, infer_others=True, apply=True)\n", - "sp.calibrate_energy_axis(ref_energy=-0.5, ref_id=4, energy_scale=\"kinetic\", method=\"lmfit\")" + "sp.calibrate_energy_axis(ref_energy=-0.5, energy_scale=\"kinetic\", method=\"lmfit\")" ] }, { diff --git a/tutorial/7_correcting_orthorhombic_symmetry.ipynb b/tutorial/7_correcting_orthorhombic_symmetry.ipynb index a385b3d1..faba30b6 100644 --- a/tutorial/7_correcting_orthorhombic_symmetry.ipynb +++ b/tutorial/7_correcting_orthorhombic_symmetry.ipynb @@ -60,7 +60,7 @@ "outputs": [], "source": [ "# create sed processor using the config file with time-stamps:\n", - "sp = sed.SedProcessor(folder=scandir, user_config=\"../sed/config/mpes_example_config.yaml\", time_stamps=True, verbose=True)\n", + "sp = sed.SedProcessor(folder=scandir, user_config=\"../sed/config/mpes_example_config.yaml\", system_config={}, time_stamps=True, verbose=True)\n", "sp.add_jitter()" ] }, @@ -216,11 +216,8 @@ } ], "metadata": { - "interpreter": { - "hash": "728003ee06929e5fa5ff815d1b96bf487266025e4b7440930c6bf4536d02d243" - }, "kernelspec": { - "display_name": "python3", + "display_name": ".pyenv", "language": "python", "name": "python3" }, diff --git a/tutorial/8_jittering_tutorial.ipynb b/tutorial/8_jittering_tutorial.ipynb index ef11af7a..d98a19b2 100644 --- a/tutorial/8_jittering_tutorial.ipynb +++ b/tutorial/8_jittering_tutorial.ipynb @@ -58,7 +58,7 @@ "outputs": [], "source": [ "# create sed processor using the config file:\n", - "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\")" + "sp = sed.SedProcessor(folder=scandir, config=\"../sed/config/mpes_example_config.yaml\", system_config={})" ] }, { @@ -358,11 +358,8 @@ } ], "metadata": { - "interpreter": { - "hash": "728003ee06929e5fa5ff815d1b96bf487266025e4b7440930c6bf4536d02d243" - }, "kernelspec": { - "display_name": "python3", + "display_name": ".pyenv", "language": "python", "name": "python3" }, From 9edbea2aa686d6e846118b57e5c82ef4230aaebc Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 15 Oct 2024 10:08:15 +0200 Subject: [PATCH 25/32] fix config model tests --- tests/test_config.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index 5ebf259b..df93520f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -150,23 +150,28 @@ def test_save_dict() -> None: @pytest.mark.parametrize("config_path", config_paths) def test_config_model_valid(config_path) -> None: """Test the config model for a valid config.""" - config = parse_config(config_path, verify_config=True) + config = parse_config(config_path, user_config={}, system_config={}, verify_config=True) assert config is not None def test_invalid_config_extra_field(): """Test that an invalid config with an extra field fails validation.""" - default_config = parse_config(verify_config=False) + default_config = parse_config(user_config={}, system_config={}, verify_config=True) invalid_config = default_config.copy() invalid_config["extra_field"] = "extra_value" with pytest.raises(ValidationError): - parse_config(invalid_config, verify_config=True) + parse_config(invalid_config, user_config={}, system_config={}, verify_config=True) def test_invalid_config_missing_field(): """Test that an invalid config with a missing required field fails validation.""" - default_config = parse_config(verify_config=False) + default_config = parse_config(user_config={}, system_config={}, verify_config=True) invalid_config = default_config.copy() del invalid_config["core"]["loader"] with pytest.raises(ValidationError): - parse_config(default_config=invalid_config, verify_config=True) + parse_config( + user_config={}, + system_config={}, + default_config=invalid_config, + verify_config=True, + ) From ad8705d360004d07af3af8c07cb8c767226b8b41 Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 15 Oct 2024 12:26:15 +0200 Subject: [PATCH 26/32] add review suggestions --- sed/config/config_model.py | 16 +++++----------- sed/config/flash_example_config.yaml | 2 +- sed/core/config.py | 2 +- sed/loader/mpes/loader.py | 2 +- tests/data/loader/flash/config.yaml | 2 +- tests/test_processor.py | 3 +++ 6 files changed, 12 insertions(+), 15 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 5fde6513..43e69e3d 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -11,7 +11,9 @@ from pydantic import DirectoryPath from pydantic import field_validator from pydantic import FilePath +from pydantic import HttpUrl from pydantic import NewPath +from pydantic import PositiveInt from pydantic import SecretStr from sed.loader.loader_interface import get_names_of_all_loaders @@ -53,7 +55,7 @@ class CoreModel(BaseModel): loader: str verbose: Optional[bool] = None paths: Optional[PathsModel] = None - num_cores: Optional[int] = None + num_cores: Optional[PositiveInt] = None year: Optional[int] = None beamtime_id: Optional[Union[int, str]] = None instrument: Optional[str] = None @@ -69,14 +71,6 @@ def validate_loader(cls, v: str) -> str: raise ValueError(f"Invalid loader {v}. Available loaders are: {names}") return v - @field_validator("num_cores") - @classmethod - def validate_num_cores(cls, v: int) -> int: - """Checks if the num_cores field is a positive integer""" - if v < 1: - raise ValueError(f"Invalid value {v} for num_cores. Needs to be > 0.") - return v - class ColumnsModel(BaseModel): model_config = ConfigDict(extra="forbid") @@ -336,7 +330,7 @@ class OffsetColumn(BaseModel): class MetadataModel(BaseModel): model_config = ConfigDict(extra="forbid") - archiver_url: Optional[str] = None + archiver_url: Optional[HttpUrl] = None token: Optional[SecretStr] = None epics_pvs: Optional[Sequence[str]] = None fa_in_channel: Optional[str] = None @@ -353,7 +347,7 @@ class NexusModel(BaseModel): reader: Literal["mpes"] # Currently only NXmpes definition is supported definition: Literal["NXmpes"] - input_files: Sequence[str] + input_files: Sequence[FilePath] class ConfigModel(BaseModel): diff --git a/sed/config/flash_example_config.yaml b/sed/config/flash_example_config.yaml index 0bc95dd2..bdd6a827 100644 --- a/sed/config/flash_example_config.yaml +++ b/sed/config/flash_example_config.yaml @@ -140,7 +140,7 @@ dataframe: # The auxiliary channel has a special structure where the group further contains # a multidimensional structure so further aliases are defined below dldAux: - format: per_pulse + format: per_train index_key: "/uncategorised/FLASH.EXP/HEXTOF.DAQ/DLD1/index" dataset_key: "/uncategorised/FLASH.EXP/HEXTOF.DAQ/DLD1/value" slice: 4 diff --git a/sed/core/config.py b/sed/core/config.py index ff11bbd7..723ef598 100644 --- a/sed/core/config.py +++ b/sed/core/config.py @@ -148,7 +148,7 @@ def parse_config( return config_dict # Run the config through the ConfigModel to ensure it is valid config_model = ConfigModel(**config_dict) - return config_model.model_dump(exclude_unset=True, exclude_none=True, exclude_defaults=True) + return config_model.model_dump(exclude_unset=True, exclude_none=True) def load_config(config_path: str) -> dict: diff --git a/sed/loader/mpes/loader.py b/sed/loader/mpes/loader.py index 14a41ee2..a7cb82d0 100644 --- a/sed/loader/mpes/loader.py +++ b/sed/loader/mpes/loader.py @@ -861,7 +861,7 @@ def gather_metadata( for channel in channels_missing: try: _, vals = get_archiver_data( - archiver_url=self._config["metadata"].get("archiver_url"), + archiver_url=str(self._config["metadata"].get("archiver_url")), archiver_channel=channel, ts_from=ts_from, ts_to=ts_to, diff --git a/tests/data/loader/flash/config.yaml b/tests/data/loader/flash/config.yaml index f7296ea5..db3aabbb 100644 --- a/tests/data/loader/flash/config.yaml +++ b/tests/data/loader/flash/config.yaml @@ -101,7 +101,7 @@ dataframe: # The auxiliary channel has a special structure where the group further contains # a multidimensional structure so further aliases are defined below dldAux: - format: per_pulse + format: per_train index_key: "/uncategorised/FLASH.EXP/HEXTOF.DAQ/DLD1/index" dataset_key: "/uncategorised/FLASH.EXP/HEXTOF.DAQ/DLD1/value" slice: 4 diff --git a/tests/test_processor.py b/tests/test_processor.py index a98fa882..6cc1d06c 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -829,6 +829,7 @@ def test_add_time_stamped_data() -> None: system_config={}, time_stamps=True, verbose=True, + verify_config=False, ) df_ts = processor.dataframe.timeStamps.compute().values data = np.linspace(0, 1, 20) @@ -1055,12 +1056,14 @@ def test_save(caplog) -> None: folder_config={}, user_config=package_dir + "/../sed/config/mpes_example_config.yaml", system_config={}, + verify_config=False, ) config["metadata"]["lens_mode_config"]["6kV_kmodem4.0_30VTOF_453ns_focus.sav"][ "MCPfront" ] = 21.0 config["metadata"]["lens_mode_config"]["6kV_kmodem4.0_30VTOF_453ns_focus.sav"]["Z1"] = 2450 config["metadata"]["lens_mode_config"]["6kV_kmodem4.0_30VTOF_453ns_focus.sav"]["F"] = 69.23 + config["nexus"]["input_files"] = [df_folder + "../../../../sed/config/NXmpes_config.json"] processor = SedProcessor( folder=df_folder, config=config, From 7aa7231a58aed5fb7774708d1cbbddc04749fef6 Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 15 Oct 2024 15:49:58 +0200 Subject: [PATCH 27/32] fix reporting of energy/delay offsets --- sed/calibrator/delay.py | 15 ++++++--------- sed/calibrator/energy.py | 19 ++++++------------- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/sed/calibrator/delay.py b/sed/calibrator/delay.py index 8035d456..9e0be940 100644 --- a/sed/calibrator/delay.py +++ b/sed/calibrator/delay.py @@ -357,19 +357,16 @@ def add_offsets( ) log_str += f"\n Flip delay axis: {flip_delay_axis}" elif k == "columns": - for k2, v2 in offsets["columns"].items(): - columns.append(k2) - try: - weight = v2["weight"] - except KeyError: - weight = 1 + for column_name, column_dict in offsets["columns"].items(): + columns.append(column_name) + weight = column_dict.get("weight", 1) weights.append(weight) - pm = v2.get("preserve_mean", False) + pm = column_dict.get("preserve_mean", False) preserve_mean.append(pm) - red = v2.get("reduction", None) + red = column_dict.get("reduction", None) reductions.append(red) log_str += ( - f"\n Column[{k}]: Weight={weight}, Preserve Mean: {pm}, " + f"\n Column[{column_name}]: Weight={weight}, Preserve Mean: {pm}, " f"Reductions: {red}." ) diff --git a/sed/calibrator/energy.py b/sed/calibrator/energy.py index b004d25e..831baa94 100644 --- a/sed/calibrator/energy.py +++ b/sed/calibrator/energy.py @@ -1652,12 +1652,9 @@ def add_offsets( constant = v * scale_sign log_str += f"\n Constant: {constant}" elif k == "columns": - for k2, v2 in offsets["columns"].items(): - columns.append(k2) - try: - weight = v2["weight"] - except KeyError: - weight = 1 + for column_name, column_dict in offsets["columns"].items(): + columns.append(column_name) + weight = column_dict.get("weight", 1) if not isinstance(weight, (int, float, np.integer, np.floating)): raise TypeError( f"Invalid type for weight of column {k}: {type(weight)}", @@ -1665,18 +1662,14 @@ def add_offsets( # flip sign if binding energy scale weight = weight * scale_sign weights.append(weight) - pm = v2.get("preserve_mean", False) - if str(pm).lower() in ["false", "0", "no"]: - pm = False - elif str(pm).lower() in ["true", "1", "yes"]: - pm = True + pm = column_dict.get("preserve_mean", False) preserve_mean.append(pm) - red = v2.get("reduction", None) + red = column_dict.get("reduction", None) if str(red).lower() in ["none", "null"]: red = None reductions.append(red) log_str += ( - f"\n Column[{k}]: Weight={weight}, Preserve Mean: {pm}, " + f"\n Column[{column_name}]: Weight={weight}, Preserve Mean: {pm}, " f"Reductions: {red}." ) From ec20e03661a07e172f76cc7fadece13c18c3409d Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 15 Oct 2024 16:30:59 +0200 Subject: [PATCH 28/32] fix handling of nexus input files and tests --- sed/core/processor.py | 2 +- tests/test_processor.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/sed/core/processor.py b/sed/core/processor.py index d5f4c418..0ea90517 100644 --- a/sed/core/processor.py +++ b/sed/core/processor.py @@ -2528,7 +2528,7 @@ def save( ) input_files = kwds.pop( "input_files", - self._config["nexus"]["input_files"], + [str(path) for path in self._config["nexus"]["input_files"]], ) except KeyError as exc: raise ValueError( diff --git a/tests/test_processor.py b/tests/test_processor.py index 6cc1d06c..709fc048 100644 --- a/tests/test_processor.py +++ b/tests/test_processor.py @@ -1063,7 +1063,7 @@ def test_save(caplog) -> None: ] = 21.0 config["metadata"]["lens_mode_config"]["6kV_kmodem4.0_30VTOF_453ns_focus.sav"]["Z1"] = 2450 config["metadata"]["lens_mode_config"]["6kV_kmodem4.0_30VTOF_453ns_focus.sav"]["F"] = 69.23 - config["nexus"]["input_files"] = [df_folder + "../../../../sed/config/NXmpes_config.json"] + config["nexus"]["input_files"] = [package_dir + "/../sed/config/NXmpes_config.json"] processor = SedProcessor( folder=df_folder, config=config, @@ -1095,7 +1095,6 @@ def test_save(caplog) -> None: # and error if any validation problems occur. processor.save( "output.nxs", - input_files=df_folder + "../../../../sed/config/NXmpes_config.json", fail=True, ) assert os.path.isfile("output.nxs") @@ -1104,7 +1103,6 @@ def test_save(caplog) -> None: with pytest.raises(ValidationFailed): processor.save( "result.nxs", - input_files=df_folder + "../../../../sed/config/NXmpes_config.json", fail=True, ) # Check that the issues are raised as warnings per default: @@ -1113,7 +1111,7 @@ def test_save(caplog) -> None: yaml.dump({"Instrument": {"undocumented_field": "undocumented entry"}}, f) with open("temp_config.json", "w") as f: with open( - df_folder + "../../../../sed/config/NXmpes_config.json", + package_dir + "/../sed/config/NXmpes_config.json", encoding="utf-8", ) as stream: config_dict = json.load(stream) From 4e535cf0fec203fda6a6b792e6f7d3bbfe1ad68b Mon Sep 17 00:00:00 2001 From: rettigl Date: Tue, 15 Oct 2024 23:09:00 +0200 Subject: [PATCH 29/32] fix error reporting --- sed/calibrator/delay.py | 4 ++++ sed/calibrator/energy.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sed/calibrator/delay.py b/sed/calibrator/delay.py index 9e0be940..14758b41 100644 --- a/sed/calibrator/delay.py +++ b/sed/calibrator/delay.py @@ -360,6 +360,10 @@ def add_offsets( for column_name, column_dict in offsets["columns"].items(): columns.append(column_name) weight = column_dict.get("weight", 1) + if not isinstance(weight, (int, float, np.integer, np.floating)): + raise TypeError( + f"Invalid type for weight of column {column_name}: {type(weight)}", + ) weights.append(weight) pm = column_dict.get("preserve_mean", False) preserve_mean.append(pm) diff --git a/sed/calibrator/energy.py b/sed/calibrator/energy.py index 831baa94..65493f66 100644 --- a/sed/calibrator/energy.py +++ b/sed/calibrator/energy.py @@ -1657,7 +1657,7 @@ def add_offsets( weight = column_dict.get("weight", 1) if not isinstance(weight, (int, float, np.integer, np.floating)): raise TypeError( - f"Invalid type for weight of column {k}: {type(weight)}", + f"Invalid type for weight of column {column_name}: {type(weight)}", ) # flip sign if binding energy scale weight = weight * scale_sign From 7a7441c41f101dc9be30ce6702344f5d8e51c5ef Mon Sep 17 00:00:00 2001 From: rettigl Date: Thu, 17 Oct 2024 14:48:16 +0200 Subject: [PATCH 30/32] fix sxp notebook --- tutorial/5_sxp_workflow.ipynb | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/tutorial/5_sxp_workflow.ipynb b/tutorial/5_sxp_workflow.ipynb index 3241545f..7e3d2dbc 100644 --- a/tutorial/5_sxp_workflow.ipynb +++ b/tutorial/5_sxp_workflow.ipynb @@ -297,7 +297,6 @@ "outputs": [], "source": [ "sp.calibrate_energy_axis(\n", - " ref_id=5,\n", " ref_energy=-33,\n", " method=\"lmfit\",\n", " energy_scale='kinetic',\n", @@ -316,15 +315,6 @@ "sp.save_energy_calibration()" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sp.append_energy_axis()" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -341,7 +331,7 @@ "sp.load(runs=np.arange(58, 62))\n", "sp.add_jitter()\n", "sp.filter_column(\"pulseId\", max_value=756)\n", - "sp.append_energy_axis()" + "sp.append_energy_axis(bias_voltage=957)" ] }, { From fb04ce6ddf8a828d4f01cdfa5c2950ab56621d92 Mon Sep 17 00:00:00 2001 From: rettigl Date: Mon, 21 Oct 2024 22:48:06 +0200 Subject: [PATCH 31/32] changes from review --- sed/config/config_model.py | 22 ++++----- sed/config/flash_example_config.yaml | 6 +-- sed/core/config.py | 2 +- sed/core/processor.py | 8 ++-- sed/loader/sxp/loader.py | 4 +- tests/data/loader/flash/config.yaml | 2 - tests/test_config.py | 68 ++++++++++++++++++++++++++-- 7 files changed, 83 insertions(+), 29 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 43e69e3d..5a840f3b 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -137,9 +137,8 @@ class DataframeModel(BaseModel): sector_id_reserved_bits: Optional[int] = None sector_delays: Optional[Sequence[float]] = None daq: Optional[str] = None - num_trains: Optional[int] = None - - # write validator for model so that x_column gets converted to columns: x + # SXP specific settings + num_trains: Optional[PositiveInt] = None class BinningModel(BaseModel): @@ -148,14 +147,14 @@ class BinningModel(BaseModel): hist_mode: Literal["numpy", "numba"] mode: Literal["fast", "lean", "legacy"] pbar: bool - threads_per_worker: int + threads_per_worker: PositiveInt threadpool_API: Literal["blas", "openmp"] class HistogramModel(BaseModel): model_config = ConfigDict(extra="forbid") - bins: Sequence[int] + bins: Sequence[PositiveInt] axes: Sequence[str] ranges: Sequence[tuple[float, float]] @@ -174,8 +173,8 @@ class StaticModel(BaseModel): class EnergyModel(BaseModel): model_config = ConfigDict(extra="forbid") - bins: int - ranges: Sequence[int] + bins: PositiveInt + ranges: tuple[int, int] normalize: bool normalize_span: int normalize_order: int @@ -238,7 +237,7 @@ class MomentumModel(BaseModel): model_config = ConfigDict(extra="forbid") axes: Sequence[str] - bins: Sequence[int] + bins: Sequence[PositiveInt] ranges: Sequence[tuple[int, int]] detector_ranges: Sequence[tuple[int, int]] center_pixel: tuple[int, int] @@ -266,7 +265,7 @@ class MomentumCorrectionModel(BaseModel): creation_date: Optional[datetime] = None feature_points: Sequence[tuple[float, float]] - rotation_symmetry: int + rotation_symmetry: PositiveInt include_center: bool use_center: bool ascale: Optional[Sequence[float]] = None @@ -275,7 +274,7 @@ class MomentumCorrectionModel(BaseModel): correction: Optional[MomentumCorrectionModel] = None - class MomentumTransformationModel(BaseModel): + class MomentumTransformationsModel(BaseModel): model_config = ConfigDict(extra="forbid") creation_date: Optional[datetime] = None @@ -284,7 +283,7 @@ class MomentumTransformationModel(BaseModel): xtrans: Optional[float] = None ytrans: Optional[float] = None - transformations: Optional[MomentumTransformationModel] = None + transformations: Optional[MomentumTransformationsModel] = None class DelayModel(BaseModel): @@ -295,7 +294,6 @@ class DelayModel(BaseModel): # Group keys in the datafile p1_key: Optional[str] = None p2_key: Optional[str] = None - p3_key: Optional[str] = None t0_key: Optional[str] = None class DelayCalibration(BaseModel): diff --git a/sed/config/flash_example_config.yaml b/sed/config/flash_example_config.yaml index bdd6a827..0f12402b 100644 --- a/sed/config/flash_example_config.yaml +++ b/sed/config/flash_example_config.yaml @@ -1,8 +1,8 @@ # This file contains the default configuration for the flash loader. core: # defines the loader - loader: 'flash' - # # Since this will run on maxwell most probably, we have a lot of cores at our disposal + loader: flash + # Since this will run on maxwell most probably, we have a lot of cores at our disposal num_cores: 100 # the beamline where experiment took place beamline: pg2 @@ -202,7 +202,7 @@ dataframe: # The nexus collection routine shall be finalized soon for both instruments # nexus: -# reader: "flash" +# reader: "mpes" # definition: "NXmpes" # input_files: ["NXmpes_config_HEXTOF_light.json"] diff --git a/sed/core/config.py b/sed/core/config.py index 723ef598..ca29285c 100644 --- a/sed/core/config.py +++ b/sed/core/config.py @@ -63,7 +63,7 @@ def parse_config( FileNotFoundError: Raised if the provided file is not found. Returns: - ConfigModel: Loaded and possibly completed pydantic config model. + dict: Loaded and completed config dict, possibly verified by pydantic config model. """ if config is None: config = {} diff --git a/sed/core/processor.py b/sed/core/processor.py index 0ea90517..1a7781f7 100644 --- a/sed/core/processor.py +++ b/sed/core/processor.py @@ -1167,11 +1167,11 @@ def load_bias_series( Args: binned_data (xr.DataArray | tuple[np.ndarray, np.ndarray, np.ndarray], optional): Binned data If provided as DataArray, Needs to contain dimensions - config["dataframe"]["tof_column"] and config["dataframe"]["bias_column"]. If - provided as tuple, needs to contain elements tof, biases, traces. + config["dataframe"]["columns"]["tof"] and config["dataframe"]["columns"]["bias"]. + If provided as tuple, needs to contain elements tof, biases, traces. data_files (list[str], optional): list of file paths to bin axes (list[str], optional): bin axes. - Defaults to config["dataframe"]["tof_column"]. + Defaults to config["dataframe"]["columns"]["tof"]. bins (list, optional): number of bins. Defaults to config["energy"]["bins"]. ranges (Sequence[tuple[float, float]], optional): bin ranges. @@ -1629,7 +1629,7 @@ def append_tof_ns_axis( Args: tof_ns_column (str, optional): Name of the generated column containing the time-of-flight in nanosecond. - Defaults to config["dataframe"]["tof_ns_column"]. + Defaults to config["dataframe"]["columns"]["tof_ns"]. preview (bool, optional): Option to preview the first elements of the data frame. Defaults to False. **kwds: additional arguments are passed to ``EnergyCalibrator.append_tof_ns_axis()``. diff --git a/sed/loader/sxp/loader.py b/sed/loader/sxp/loader.py index c9d73a4c..83d7c97c 100644 --- a/sed/loader/sxp/loader.py +++ b/sed/loader/sxp/loader.py @@ -686,7 +686,7 @@ def create_dataframe_per_file( with h5py.File(file_path, "r") as h5_file: self.reset_multi_index() # Reset MultiIndexes for next file df = self.concatenate_channels(h5_file) - df = df.dropna(subset=self._config["dataframe"].get("tof_column", "dldTimeSteps")) + df = df.dropna(subset=self._config["dataframe"]["columns"].get("tof", "dldTimeSteps")) # correct the 3 bit shift which encodes the detector ID in the 8s time if self._config["dataframe"].get("split_sector_id_from_dld_time", False): df, _ = split_dld_time_from_sector_id(df, config=self._config) @@ -763,7 +763,7 @@ def buffer_file_handler( parquet_schemas = [pq.read_schema(file) for file in existing_parquet_filenames] config_schema = set(self.get_channels(formats="all", index=True)) if self._config["dataframe"].get("split_sector_id_from_dld_time", False): - config_schema.add(self._config["dataframe"].get("sector_id_column", False)) + config_schema.add(self._config["dataframe"]["columns"].get("sector_id", False)) for i, schema in enumerate(parquet_schemas): schema_set = set(schema.names) diff --git a/tests/data/loader/flash/config.yaml b/tests/data/loader/flash/config.yaml index db3aabbb..8c57924b 100644 --- a/tests/data/loader/flash/config.yaml +++ b/tests/data/loader/flash/config.yaml @@ -108,10 +108,8 @@ dataframe: sub_channels: sampleBias: slice: 0 - dtype: float32 tofVoltage: slice: 1 - dtype: float64 extractorVoltage: slice: 2 extractorCurrent: diff --git a/tests/test_config.py b/tests/test_config.py index df93520f..bdbdd9c8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -41,7 +41,7 @@ def test_default_config() -> None: """Test the config loader for the default config.""" - config = parse_config(verify_config=False) + config = parse_config(config={}, folder_config={}, user_config={}, system_config={}) assert isinstance(config, dict) for key in default_config_keys: assert key in config.keys() @@ -150,28 +150,86 @@ def test_save_dict() -> None: @pytest.mark.parametrize("config_path", config_paths) def test_config_model_valid(config_path) -> None: """Test the config model for a valid config.""" - config = parse_config(config_path, user_config={}, system_config={}, verify_config=True) + config = parse_config( + config_path, + folder_config={}, + user_config={}, + system_config={}, + verify_config=True, + ) assert config is not None def test_invalid_config_extra_field(): """Test that an invalid config with an extra field fails validation.""" - default_config = parse_config(user_config={}, system_config={}, verify_config=True) + default_config = parse_config( + folder_config={}, + user_config={}, + system_config={}, + verify_config=True, + ) invalid_config = default_config.copy() invalid_config["extra_field"] = "extra_value" with pytest.raises(ValidationError): - parse_config(invalid_config, user_config={}, system_config={}, verify_config=True) + parse_config( + invalid_config, + folder_config={}, + user_config={}, + system_config={}, + verify_config=True, + ) def test_invalid_config_missing_field(): """Test that an invalid config with a missing required field fails validation.""" - default_config = parse_config(user_config={}, system_config={}, verify_config=True) + default_config = parse_config( + folder_config={}, + user_config={}, + system_config={}, + verify_config=True, + ) invalid_config = default_config.copy() del invalid_config["core"]["loader"] with pytest.raises(ValidationError): parse_config( + folder_config={}, + user_config={}, + system_config={}, + default_config=invalid_config, + verify_config=True, + ) + + +def test_invalid_config_wrong_values(): + """Test that the validators for certain fields fails validation if not fulfilled.""" + default_config = parse_config( + folder_config={}, + user_config={}, + system_config={}, + verify_config=True, + ) + invalid_config = default_config.copy() + invalid_config["core"]["loader"] = "nonexistent" + with pytest.raises(ValidationError) as e: + parse_config( + folder_config={}, + user_config={}, + system_config={}, + default_config=invalid_config, + verify_config=True, + ) + assert "Invalid loader nonexistent. Available loaders are:" in str(e.value) + invalid_config = default_config.copy() + invalid_config["core"]["copy_tool"] = {} + invalid_config["core"]["copy_tool"]["source"] = "./" + invalid_config["core"]["copy_tool"]["dest"] = "./" + invalid_config["core"]["copy_tool"]["gid"] = 9999 + with pytest.raises(ValidationError) as e: + parse_config( + folder_config={}, user_config={}, system_config={}, default_config=invalid_config, verify_config=True, ) + assert "Invalid value 9999 for gid. Group not found." in str(e.value) From fea015a34ade75b5b47ad92ab1bf94d6b5c33e7d Mon Sep 17 00:00:00 2001 From: Zain Sohail Date: Tue, 22 Oct 2024 10:13:44 +0200 Subject: [PATCH 32/32] Move static (#511) * move static config to core * config model linting * test fixes --- sed/config/config_model.py | 15 +++-------- sed/config/flash_example_config.yaml | 32 +++++++++++------------- sed/config/sxp_example_config.yaml | 15 +++++------ sed/loader/flash/loader.py | 4 +-- sed/loader/sxp/loader.py | 6 ++--- tests/data/loader/flash/config.yaml | 33 +++++++++++-------------- tests/data/loader/sxp/config.yaml | 16 ++++++------ tests/loader/flash/test_flash_loader.py | 2 +- tests/loader/sxp/test_sxp_loader.py | 2 +- 9 files changed, 52 insertions(+), 73 deletions(-) diff --git a/sed/config/config_model.py b/sed/config/config_model.py index 5a840f3b..9bdcdec1 100644 --- a/sed/config/config_model.py +++ b/sed/config/config_model.py @@ -61,6 +61,9 @@ class CoreModel(BaseModel): instrument: Optional[str] = None beamline: Optional[str] = None copy_tool: Optional[CopyToolModel] = None + stream_name_prefixes: Optional[dict] = None + stream_name_postfixes: Optional[dict] = None + beamtime_dir: Optional[dict] = None @field_validator("loader") @classmethod @@ -159,17 +162,6 @@ class HistogramModel(BaseModel): ranges: Sequence[tuple[float, float]] -class StaticModel(BaseModel): - """Static configuration settings that shouldn't be changed by users.""" - - # flash specific settings - model_config = ConfigDict(extra="forbid") - - stream_name_prefixes: Optional[dict] = None - stream_name_postfixes: Optional[dict] = None - beamtime_dir: Optional[dict] = None - - class EnergyModel(BaseModel): model_config = ConfigDict(extra="forbid") @@ -360,4 +352,3 @@ class ConfigModel(BaseModel): histogram: HistogramModel metadata: Optional[MetadataModel] = None nexus: Optional[NexusModel] = None - static: Optional[StaticModel] = None diff --git a/sed/config/flash_example_config.yaml b/sed/config/flash_example_config.yaml index 0f12402b..823a53fe 100644 --- a/sed/config/flash_example_config.yaml +++ b/sed/config/flash_example_config.yaml @@ -19,6 +19,19 @@ core: # raw: "" # # location of the intermediate parquet files. # processed: "" + # The prefixes of the stream names for different DAQ systems for parsing filenames + stream_name_prefixes: + pbd: "GMD_DATA_gmd_data" + pbd2: "FL2PhotDiag_pbd2_gmd_data" + fl1user1: "FLASH1_USER1_stream_2" + fl1user2: "FLASH1_USER2_stream_2" + fl1user3: "FLASH1_USER3_stream_2" + fl2user1: "FLASH2_USER1_stream_2" + fl2user2: "FLASH2_USER2_stream_2" + # The beamtime directories for different DAQ systems. + # (Not to be changed by user) + beamtime_dir: + pg2: "/asap3/flash/gpfs/pg2/" binning: # Histogram computation mode to use. @@ -204,21 +217,4 @@ dataframe: # nexus: # reader: "mpes" # definition: "NXmpes" -# input_files: ["NXmpes_config_HEXTOF_light.json"] - -# (Not to be changed by user) -static: - # The prefixes of the stream names for different DAQ systems for parsing filenames - stream_name_prefixes: - pbd: "GMD_DATA_gmd_data" - pbd2: "FL2PhotDiag_pbd2_gmd_data" - fl1user1: "FLASH1_USER1_stream_2" - fl1user2: "FLASH1_USER2_stream_2" - fl1user3: "FLASH1_USER3_stream_2" - fl2user1: "FLASH2_USER1_stream_2" - fl2user2: "FLASH2_USER2_stream_2" - - # The beamtime directories for different DAQ systems. - # (Not to be changed by user) - beamtime_dir: - pg2: "/asap3/flash/gpfs/pg2/" +# input_files: ["NXmpes_config_HEXTOF_light.json"] \ No newline at end of file diff --git a/sed/config/sxp_example_config.yaml b/sed/config/sxp_example_config.yaml index b9e4d298..9b48c28c 100644 --- a/sed/config/sxp_example_config.yaml +++ b/sed/config/sxp_example_config.yaml @@ -6,6 +6,12 @@ core: year: 202302 beamline: sxp instrument: sxp + stream_name_prefixes: + DA03: "RAW-R" + stream_name_postfixes: + DA03: "-DA03-" + beamtime_dir: + sxp: "/gpfs/exfel/exp/SXP/" paths: raw: "/path/to/data" # change this to a local directory where you want to store the parquet files @@ -102,12 +108,3 @@ dataframe: format: per_train dataset_key: "/CONTROL/SCS_ILH_LAS/MDL/OPTICALDELAY_PP800/actualPosition/value" index_key: "/INDEX/trainId" - -static: - stream_name_prefixes: - DA03: "RAW-R" - stream_name_postfixes: - DA03: "-DA03-" - - beamtime_dir: - sxp: "/gpfs/exfel/exp/SXP/" diff --git a/sed/loader/flash/loader.py b/sed/loader/flash/loader.py index f451783f..6c84fc34 100644 --- a/sed/loader/flash/loader.py +++ b/sed/loader/flash/loader.py @@ -110,7 +110,7 @@ def _initialize_dirs(self) -> None: ) from exc beamtime_dir = Path( - self._config["static"]["beamtime_dir"][self._config["core"]["beamline"]], + self._config["core"]["beamtime_dir"][self._config["core"]["beamline"]], ) beamtime_dir = beamtime_dir.joinpath(f"{year}/data/{beamtime_id}/") @@ -175,7 +175,7 @@ def get_files_from_run_id( # type: ignore[override] FileNotFoundError: If no files are found for the given run in the directory. """ # Define the stream name prefixes based on the data acquisition identifier - stream_name_prefixes = self._config["static"]["stream_name_prefixes"] + stream_name_prefixes = self._config["core"]["stream_name_prefixes"] if folders is None: folders = self._config["core"]["base_folder"] diff --git a/sed/loader/sxp/loader.py b/sed/loader/sxp/loader.py index 83d7c97c..9ff0ac9a 100644 --- a/sed/loader/sxp/loader.py +++ b/sed/loader/sxp/loader.py @@ -116,7 +116,7 @@ def _initialize_dirs(self): ) from exc beamtime_dir = Path( - self._config["static"]["beamtime_dir"][self._config["core"]["beamline"]], + self._config["core"]["beamtime_dir"][self._config["core"]["beamline"]], ) beamtime_dir = beamtime_dir.joinpath(f"{year}/{beamtime_id}/") @@ -158,8 +158,8 @@ def get_files_from_run_id( FileNotFoundError: If no files are found for the given run in the directory. """ # Define the stream name prefixes based on the data acquisition identifier - stream_name_prefixes = self._config["static"]["stream_name_prefixes"] - stream_name_postfixes = self._config["static"].get("stream_name_postfixes", {}) + stream_name_prefixes = self._config["core"]["stream_name_prefixes"] + stream_name_postfixes = self._config["core"].get("stream_name_postfixes", {}) if isinstance(run_id, (int, np.integer)): run_id = str(run_id).zfill(4) diff --git a/tests/data/loader/flash/config.yaml b/tests/data/loader/flash/config.yaml index 8c57924b..fbbcba25 100644 --- a/tests/data/loader/flash/config.yaml +++ b/tests/data/loader/flash/config.yaml @@ -17,6 +17,21 @@ core: # beamtime_id: xxxxxxxx # year: 20xx + # The prefixes of the stream names for different DAQ systems for parsing filenames + stream_name_prefixes: + pbd: "GMD_DATA_gmd_data" + pbd2: "FL2PhotDiag_pbd2_gmd_data" + fl1user1: "FLASH1_USER1_stream_2" + fl1user2: "FLASH1_USER2_stream_2" + fl1user3: "FLASH1_USER3_stream_2" + fl2user1: "FLASH2_USER1_stream_2" + fl2user2: "FLASH2_USER2_stream_2" + + # The beamtime directories for different DAQ systems. + # (Not to be changed by user) + beamtime_dir: + pg2: "/asap3/flash/gpfs/pg2/" + dataframe: # The name of the DAQ system to use. Necessary to resolve the filenames/paths. @@ -141,21 +156,3 @@ dataframe: index_key: "/FL1/Photon Diagnostic/GMD/Pulse resolved energy/energy tunnel/index" dataset_key: "/FL1/Photon Diagnostic/GMD/Pulse resolved energy/energy tunnel/value" slice: 0 - - -# (Not to be changed by user) -static: - # The prefixes of the stream names for different DAQ systems for parsing filenames - stream_name_prefixes: - pbd: "GMD_DATA_gmd_data" - pbd2: "FL2PhotDiag_pbd2_gmd_data" - fl1user1: "FLASH1_USER1_stream_2" - fl1user2: "FLASH1_USER2_stream_2" - fl1user3: "FLASH1_USER3_stream_2" - fl2user1: "FLASH2_USER1_stream_2" - fl2user2: "FLASH2_USER2_stream_2" - - # The beamtime directories for different DAQ systems. - # (Not to be changed by user) - beamtime_dir: - pg2: "/asap3/flash/gpfs/pg2/" diff --git a/tests/data/loader/sxp/config.yaml b/tests/data/loader/sxp/config.yaml index c5680847..2f88bc50 100644 --- a/tests/data/loader/sxp/config.yaml +++ b/tests/data/loader/sxp/config.yaml @@ -5,6 +5,13 @@ core: raw: "tests/data/loader/sxp/" processed: "tests/data/loader/sxp/parquet" num_cores: 10 + stream_name_prefixes: + DA03: "RAW-R" + stream_name_postfixes: + DA03: "-DA03-" + + beamtime_dir: + sxp: "/GPFS/exfel/exp/SXP/" dataframe: ubid_offset: 0 @@ -82,12 +89,3 @@ dataframe: format: per_train dataset_key: "/CONTROL/SCS_ILH_LAS/MDL/OPTICALDELAY_PP800/actualPosition/value" index_key: "/INDEX/trainId" - -static: - stream_name_prefixes: - DA03: "RAW-R" - stream_name_postfixes: - DA03: "-DA03-" - - beamtime_dir: - sxp: "/GPFS/exfel/exp/SXP/" diff --git a/tests/loader/flash/test_flash_loader.py b/tests/loader/flash/test_flash_loader.py index 3d9a5170..de0bdf35 100644 --- a/tests/loader/flash/test_flash_loader.py +++ b/tests/loader/flash/test_flash_loader.py @@ -33,7 +33,7 @@ def test_initialize_dirs( config_["core"]["year"] = "2000" # Find base path of beamline from config. Here, we use pg2 - base_path = config_["static"]["beamtime_dir"]["pg2"] + base_path = config_["core"]["beamtime_dir"]["pg2"] expected_path = ( Path(base_path) / config_["core"]["year"] / "data" / config_["core"]["beamtime_id"] ) diff --git a/tests/loader/sxp/test_sxp_loader.py b/tests/loader/sxp/test_sxp_loader.py index 85a31eac..1ee06d41 100644 --- a/tests/loader/sxp/test_sxp_loader.py +++ b/tests/loader/sxp/test_sxp_loader.py @@ -87,7 +87,7 @@ def test_initialize_dirs(config_file: dict, fs) -> None: config["core"]["year"] = "2000" # Find base path of beamline from config. - base_path = config["static"]["beamtime_dir"]["sxp"] + base_path = config["core"]["beamtime_dir"]["sxp"] expected_path = Path(base_path) / config["core"]["year"] / config["core"]["beamtime_id"] # Create expected paths expected_raw_path = expected_path / "raw"