Merge pull request #924 from mapswipe/dev

Backend Workers Refactoring

Author: Hagellach37

.github/workflows/actions.yml

@@ -63,6 +63,7 @@ jobs:
       env:
         FIREBASE_API_KEY: ${{ secrets.FIREBASE_API_KEY }}
         FIREBASE_DB: ${{ secrets.FIREBASE_DB }}
+        FIREBASE_STORAGE_BUCKET: ${{ secrets.FIREBASE_STORAGE_BUCKET }}
         POSTGRES_PASSWORD: postgres
         POSTGRES_USER: postgres
         POSTGRES_DB: postgres

Makefile

@@ -24,7 +24,9 @@ update_firebase_functions_and_db_rules:
 	docker compose run --rm firebase_deploy
 
 deploy_latest_workers_version:
-	git pull; docker compose build postgres django django-schedule-task manager_dashboard community_dashboard nginx api mapswipe_workers_creation mapswipe_workers_stats mapswipe_workers_firebase_to_postgres firebase_deploy; docker compose up -d postgres django django-schedule-task manager_dashboard community_dashboard nginx api mapswipe_workers_creation mapswipe_workers_stats mapswipe_workers_firebase_to_postgres firebase_deploy
-
+	git pull; docker compose build postgres django django-schedule-task manager_dashboard community_dashboard nginx api mapswipe_workers_creation mapswipe_workers_stats mapswipe_workers_firebase_to_postgres firebase_deploy; docker compose up -d postgres django django-schedule-task manager_dashboard community_dashboard nginx api mapswipe_workers_creation mapswipe_workers_stats mapswipe_workers_firebase_to_postgres firebase_deploy; docker compose restart nginx
 update_tms_api_key:
 	@echo "first update the .env file with the new api key and then run this script with tms_name=your_tms_provider_name to update API key"; docker compose run --rm mapswipe_workers_creation python python_scripts/change_tms_api_key_for_projects.py $(tms_name)
+
+list_all_project_managers:
+	docker compose run --rm mapswipe_workers_creation python python_scripts/list_all_project_managers.py

README.md

@@ -21,6 +21,10 @@ Please refer to the documentation for more information: https://mapswipe-workers
 - MapSwipe OSM-Wiki: https://wiki.openstreetmap.org/wiki/MapSwipe
 
 
+## Development Setup 
+
+Please see here: https://mapswipe-workers.readthedocs.io/en/master/dev_setup.html
+
 ## Contributing Guidelines
 
 ### Feature Branch

docker-compose.development.yaml

@@ -13,14 +13,14 @@ services:
     manager_dashboard:
         container_name: manager_dashboard
         build:
-            context: manager_dashboard/
+            context: manager-dashboard/
         restart: unless-stopped
         ports:
             - '5000:80'
         expose:
             - "80"
         volumes:
-            - ./manager_dashboard/manager_dashboard:/usr/share/nginx/html/manager_dashboard
+            - ./manager-dashboard:/usr/share/nginx/html/manager_dashboard
         networks:
             - manager_dashboard
 
@@ -39,6 +39,6 @@ services:
           - ./postgres-data:/var/lib/postgresql/mapswipe
         restart: unless-stopped
         ports:
-          - "5431:5432"
+          - "${POSTGRES_PORT}:5432"
         networks:
-            - postgres
+            - postgres

docker-compose.yaml

@@ -58,6 +58,7 @@ x-mapswipe-workers: &base_mapswipe_workers
         FIREBASE_DB: '${FIREBASE_DB}'
         FIREBASE_API_KEY: '${FIREBASE_API_KEY}'
         FIREBASE_TOKEN: '${FIREBASE_TOKEN}'
+        FIREBASE_STORAGE_BUCKET: '${FIREBASE_STORAGE_BUCKET}'
         GOOGLE_APPLICATION_CREDENTIALS: 'serviceAccountKey.json'
         POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
         POSTGRES_USER: '${POSTGRES_USER}'

docs/source/configuration.md

@@ -3,7 +3,7 @@
 Most of the configuration is stored in environment variables.
 At the root of the GitHub repository (in the same directory as `docker-compose.yml`) an example file (`example.env`) with all possible configuration variables exists. To get started copy this file to `.env` (no name is required) and fill in missing variables. The Docker Compose file will access those variables when needed.
 
-> Note: If you want those variables to be accessible as Environment Variables in your current shell (Eg. Inside a Python virtual environment for development.) your need to parse the file and export the variables: `export $(cat .env | xargs)`
+> Note: If you want those variables to be accessible as Environment Variables in your current shell (E.g. Inside a Python virtual environment for development.) your need to parse the file and export the variables: `export $(cat .env | xargs)`
 
 In following chapters configuration values and keys are discussed for each part of the MapSwipe Back-end.
 
@@ -17,6 +17,7 @@ Required environment variables are:
 - FIREBASE_API_KEY
 - FIREBASE_DB
 - FIREBASE_TOKEN
+- FIREBASE_STORAGE_BUCKET
 - GOOGLE_APPLICATION_CREDENTIALS
 
 ### Postgres DB

docs/source/dev_setup.md

@@ -46,17 +46,6 @@ The path to the Service Account Key is defined in the `GOOGLE_APPLICATION_CREDEN
 You could also set up your own Firebase instance. However, this is not recommended. 
 If you still want to do it, get your Service Account Key from Firebase from [Google Cloud Service Accounts](https://console.cloud.google.com/iam-admin/serviceaccounts).
 
-#### Directories
-
-MapSwipe Workers needs access to a data directory for logs and data for the API:
-
-To create this directories run:
-```
-mkdir --parents ~/.local/share/mapswipe_workers
-```
-
-> Note: XDG Base Directory Specification is respected
-
 
 ### Database
 
@@ -74,21 +63,33 @@ Or set up Postgres using the `initdb.sql` file in the `postgres/` folder.
 
 ### Mapswipe-Workers Python Package
 
-1. Export environment variables to current shell.
-2. Create a Python virtual environment with `system-site-packages` option enabled to get access to GDAL/OGR Python packages
-3. Activate the virtual environment.
-5. Install MapSwipe Workers using pip.
-6. Run it.
+#### Installation
+
+The Python virtual environment needs to be created with `system-site-packages` option enabled to get access to GDAL/OGR Python packages.
+Also the MapSwipe Workers Python Package needs access to a data directory for logs and data for the API.
 
 ```bash
-export $(cat .env | xargs)
 cd mapswipe_workers
 python -m venv --system-site-packages venv
 source venv/bin/activate
-pip install --editable mapswipe_workers/
+pip install --editable .
+pre-commit install
+mkdir --parents ~/.local/share/mapswipe_workers
+```
+
+#### Usage
+
+```bash
+export $(cat .env | xargs)  # See section on Configuration above
 mapswipe_workers --help
 ```
 
+#### Tests
+
+```bash
+python -m unittest
+```
+
 > Yeah! If you reached this point, you are ready to get into coding. Below you find some more information on Logging, Firebase Functions and Database Backup. However, you don't need this to get started for now.
 
 # Further Information

example.env

@@ -6,6 +6,7 @@ FIREBASE_API_KEY=
 FIREBASE_TOKEN=
 GOOGLE_APPLICATION_CREDENTIALS="$HOME/.config/mapswipe_workers/serviceAccountKey.json"
 FIREBASE_AUTH_SITE=
+FIREBASE_STORAGE_BUCKET=
 
 # postgres configuration
 POSTGRES_USER=mapswipe_workers

firebase/Dockerfile

@@ -3,4 +3,4 @@ RUN npm install firebase-functions@latest firebase-admin@latest --save
 RUN npm install -g [email protected]
 COPY . /firebase
 RUN cd firebase/functions && npm install
-WORKDIR /firebase/
+WORKDIR /firebase/

firebase/README.md

@@ -7,7 +7,7 @@ Create a docker image from the latest data.
 * `docker-compose build firebase_image`
 
 Alternatively you could do
-* `sudo docker build -t . firebase_image`
+* `sudo docker build -t firebase_image .`
 
 Then run the container interactively and open a bash shell.
 * `sudo docker run -it firebase_image /bin/bash`