#3 - visit date extraction (#9)

* 3 - initial work

* 3 - clean up a few things

* 3 - add pytests

* 3 - address feedback on the pull request

* 3 - double to single quotes for consistency

Author: rmcveigh

.github/workflows/run_pytests.yaml

@@ -0,0 +1,33 @@
+# This workflow will install Python dependencies and run pytests
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Run Pytests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -U pip setuptools wheel
+        if [ -f requirements/pytest_requirements.txt ]; then pip install -r requirements/pytest_requirements.txt; fi
+
+    - name: Run tests
+      run: |
+        pytest | tee output.txt
+        cat output.txt >> $GITHUB_STEP_SUMMARY

Dockerfile

@@ -3,7 +3,7 @@ COPY requirements.txt .
 ENV PYTHONPATH=\/workspace
 ARG USER_ID
 RUN apt-get update -y && \
-    apt-get -y install poppler-utils tesseract-ocr yq &&\
+    apt-get -y install poppler-utils tesseract-ocr yq build-essential libopenblas-dev libomp-dev &&\
     git clone https://github.com/tesseract-ocr/tessdata.git /usr/share/tessdata &&\
     useradd -l -u ${USER_ID} -g sudo jenkins && \
     mkdir -m 0755 /home/jenkins && chown jenkins /home/jenkins

README.md

@@ -16,11 +16,13 @@ Goals:
 The scripts are easily run via the Dockerfile included in this repo.
 
 1. Copy the example.env file to .env and fill in the values.  
-    a. The value for `SQL_CONNECTION_STRING` should be the connection string for the database container. (i.e. `mysql://user:password@vmrt-emr-process-log-mysql:3306/vmrt_emr_transcription`)
-2. The easiest way to spin up the docker images is to run the `run.sh` script within the repository root directory. You can also build the containers using the docker compose file. `docker build -t <container name> .`
-3. Set up your DB by running `python /workspace/scripts/database_setup.py install` within the container.
-4. Get ready for the transcription process by running `python scripts/create_transcription_process.py /data`
-5. Use the `transcribe_pdfs.py` script to transcribe the files needed.
+    a. The value for `SQL_CONNECTION_STRING` should be the connection string for the database container. (i.e. `mysql://user:password@vmrt-emr-process-log-mysql:3306/vmrt_emr_transcription`)
+2. The easiest way to spin up the docker images is to run the `run.sh` script within the repository root directory.
+3. Set up your DB by running `python /workspace/scripts/database_setup.py install` within the container.
+4. Get ready for the transcription process by running `python scripts/create_transcription_process.py /data`
+5. Use the `transcribe_pdfs.py` script to transcribe the files needed.
     - `python /workspace/scripts/transcribe_pdfs.py /workspace/output`
-6. Use the `pii_scrubber.py` script to remove PII from the text.
-    - `python /workspace/scripts/scrubbers/pii_scrubber.py`
+6. Use the `pii_scrubber.py` script to remove PII from the text.
+    - `python /workspace/scripts/scrubbers/pii_scrubber.py /workspace/output`
+7. Use the scripts in the `scripts/metadata_miners` directory to find data in the text.
+    - `python /workspace/scripts/metadata_miners/visit_date_miner.py /workspace/output --visit_date_tsv=/path/to/vet_visits.tsv --dog_profile_tsv=/path/to/dog_profile.tsv`

example.env

@@ -1,2 +1,2 @@
-SQL_CONNECTION_STRING=''
+SQL_CONNECTION_STRING='mysql+pymysql://{db_user}:{db_password}@{db_host}:3306/{db_name}'
 SQL_PASSWORD=''

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+testpaths = tests/pytests
+pythonpath = .
+addopts = --doctest-modules

requirements.ini

@@ -0,0 +1,8 @@
+numpy==2.0.2
+pdf2image==1.17.0
+presidio_analyzer==2.2.357
+presidio_anonymizer==2.2.357
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+SQLAlchemy==2.0.37
+tesserocr==2.7.1

requirements.txt

@@ -1,103 +1,172 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile requirements.ini
+#
 annotated-types==0.6.0
+    # via pydantic
 azure-core==1.32.0
-beautifulsoup4==4.12.3
+    # via presidio-anonymizer
 blis==1.1.0
+    # via thinc
 catalogue==2.0.10
+    # via
+    #   spacy
+    #   srsly
+    #   thinc
 certifi==2024.12.14
-cffi==1.16.0
+    # via requests
 charset-normalizer==3.4.0
+    # via requests
 click==8.1.8
+    # via typer
 cloudpathlib==0.20.0
-colorclass==2.2.2
-compressed-rtf==1.0.6
+    # via weasel
 confection==0.1.5
-cryptography==43.0.0
+    # via
+    #   thinc
+    #   weasel
 cymem==2.0.10
-dnspython==2.6.1
-easygui==0.98.3
-ebcdic==1.1.1
-email_validator==2.1.1
-en_core_web_lg @ https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl#sha256=293e9547a655b25499198ab15a525b05b9407a75f10255e405e8c3854329ab63
-en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl#sha256=1932429db727d4bff3deed6b34cfc05df17794f4a52eeb26cf8928f7c1a0fb85
-extract-msg==0.48.7
+    # via
+    #   preshed
+    #   spacy
+    #   thinc
 filelock==3.16.1
-flake8==7.1.1
-fsspec==2024.12.0
+    # via tldextract
 greenlet==3.1.1
-huggingface-hub==0.27.0
+    # via sqlalchemy
 idna==3.7
-isort==5.13.2
-Jinja2==3.1.5
+    # via
+    #   requests
+    #   tldextract
+jinja2==3.1.5
+    # via spacy
 langcodes==3.5.0
-language_data==1.3.0
-lark==1.1.9
+    # via spacy
+language-data==1.3.0
+    # via langcodes
 marisa-trie==1.2.1
+    # via language-data
 markdown-it-py==3.0.0
-MarkupSafe==3.0.2
-mccabe==0.7.0
+    # via rich
+markupsafe==3.0.2
+    # via jinja2
 mdurl==0.1.2
-mpmath==1.3.0
-msoffcrypto-tool==5.4.1
+    # via markdown-it-py
 murmurhash==1.0.11
-mysqlclient==2.2.6
-networkx==3.4.2
+    # via
+    #   preshed
+    #   spacy
+    #   thinc
 numpy==2.0.2
-olefile==0.47
-oletools==0.60.2
+    # via
+    #   -r requirements.ini
+    #   blis
+    #   spacy
+    #   thinc
 packaging==24.0
-pandas==2.2.2
-pcodedmp==1.2.6
+    # via
+    #   spacy
+    #   thinc
+    #   weasel
 pdf2image==1.17.0
+    # via -r requirements.ini
 phonenumbers==8.13.52
+    # via presidio-analyzer
 pillow==10.3.0
+    # via pdf2image
 preshed==3.0.9
-presidio_analyzer==2.2.355
-presidio_anonymizer==2.2.355
-pycodestyle==2.12.1
-pycparser==2.22
+    # via
+    #   spacy
+    #   thinc
+presidio-analyzer==2.2.357
+    # via -r requirements.ini
+presidio-anonymizer==2.2.357
+    # via -r requirements.ini
 pycryptodome==3.21.0
+    # via presidio-anonymizer
 pydantic==2.7.1
-pydantic_core==2.18.2
-pyflakes==3.2.0
-Pygments==2.18.0
-PyMySQL==1.1.1
-pyparsing==3.1.2
-pytesseract==0.3.10
+    # via
+    #   confection
+    #   spacy
+    #   thinc
+    #   weasel
+pydantic-core==2.18.2
+    # via pydantic
+pygments==2.18.0
+    # via rich
 python-dateutil==2.9.0.post0
+    # via -r requirements.ini
 python-dotenv==1.0.1
-pytz==2024.1
-PyYAML==6.0.2
-red-black-tree-mod==1.20
+    # via -r requirements.ini
+pyyaml==6.0.2
+    # via presidio-analyzer
 regex==2024.11.6
+    # via presidio-analyzer
 requests==2.32.3
+    # via
+    #   azure-core
+    #   requests-file
+    #   spacy
+    #   tldextract
+    #   weasel
 requests-file==2.1.0
+    # via tldextract
 rich==13.9.4
-RTFDE==0.1.2
-safetensors==0.4.5
+    # via typer
 shellingham==1.5.4
+    # via typer
 six==1.16.0
+    # via
+    #   azure-core
+    #   python-dateutil
 smart-open==7.1.0
-soupsieve==2.5
+    # via weasel
 spacy==3.8.2
-spacy-huggingface-pipelines==0.0.4
+    # via presidio-analyzer
 spacy-legacy==3.0.12
+    # via spacy
 spacy-loggers==1.0.5
-SQLAlchemy==2.0.36
+    # via spacy
+sqlalchemy==2.0.37
+    # via -r requirements.ini
 srsly==2.5.0
-sympy==1.13.1
-tesseract==0.1.3
-tesserocr==2.7.0
+    # via
+    #   confection
+    #   spacy
+    #   thinc
+    #   weasel
+tesserocr==2.7.1
+    # via -r requirements.ini
 thinc==8.3.3
+    # via spacy
 tldextract==5.1.3
-tokenizers==0.21.0
-torch==2.5.1
+    # via presidio-analyzer
 tqdm==4.67.1
-transformers==4.47.1
+    # via spacy
 typer==0.15.1
-typing_extensions==4.11.0
-tzdata==2024.1
-tzlocal==5.2
+    # via
+    #   spacy
+    #   weasel
+typing-extensions==4.11.0
+    # via
+    #   azure-core
+    #   pydantic
+    #   pydantic-core
+    #   sqlalchemy
+    #   typer
 urllib3==2.3.0
+    # via requests
 wasabi==1.1.3
+    # via
+    #   spacy
+    #   thinc
+    #   weasel
 weasel==0.4.1
+    # via spacy
 wrapt==1.17.0
+    # via smart-open
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

requirements/pytest_requirements.txt

@@ -0,0 +1,5 @@
+pytest==8.3.4
+pytest-mock==3.14.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+SQLAlchemy==2.0.37

scripts/database_setup.py

@@ -65,7 +65,7 @@ def parse_args() -> argparse.Namespace:
     """
     parser = argparse.ArgumentParser(
         prog='Performs database utility related functions.',)
-    parser.add_argument('operation', help='The database operation to perform, install or drop.')
+    parser.add_argument('operation', help='The database operation to perform, install or drop.', choices=['install', 'drop'])
     parser.add_argument('--debug-sql', action='store_true', help='Enable SQL debugging')
     return parser.parse_args()