switch scraper process to be on a different docker container

Author: skyeslattery

app.py

@@ -2,13 +2,9 @@
 import argparse
 from flask import Flask
 from flask_graphql import GraphQLView
-from flask_apscheduler import APScheduler
 from graphene import Schema
 from src.schema import Query, Mutation
-from src.scrapers.games_scraper import fetch_game_schedule
-from src.scrapers.youtube_stats import fetch_videos
 from src.utils.team_loader import TeamLoader
-import time
 import os
 import signal
 import sys
@@ -18,10 +14,6 @@
 
 app = Flask(__name__)
 
-# Set up the scheduler
-scheduler = APScheduler()
-scheduler.init_app(app)
-
 # Configure logging
 logging.basicConfig(
     format="%(asctime)s %(levelname)-8s %(message)s",
@@ -40,53 +32,11 @@ def create_context():
     "/graphql", view_func=GraphQLView.as_view("graphql", schema=schema, graphiql=True, get_context=create_context)
 )
 
-# Setup command line arguments
-def parse_args():
-    parser = argparse.ArgumentParser(description="Skip scraping tasks, for dev purposes.")
-    parser.add_argument(
-        "--no-scrape",
-        action="store_true",
-        help="Skips scraping tasks if set, useful for frontend development.",
-    )
-    return parser.parse_args()
-
-
 def signal_handler(sig, frame):
-    """Handle Ctrl+C by shutting down scheduler and exiting"""
-    scheduler.shutdown()
     sys.exit(0)
 
 signal.signal(signal.SIGINT, signal_handler)
 signal.signal(signal.SIGTERM, signal_handler)
 
 if __name__ == "__main__":
-    args = parse_args()
-    
-    if not args.no_scrape:
-        # scrapes games every 5 minutes. 
-        # need testing to see if this can be lower safely
-        @scheduler.task("interval", id="scrape_schedules", seconds=300)
-        def scrape_schedules():
-            start_time = time.time()
-            logging.info("Starting scraping games")
-            fetch_game_schedule()
-            elapsed_time = time.time() - start_time
-            logging.info(f"Completed scraping games in {elapsed_time:.2f} seconds")
-
-        @scheduler.task("interval", id="scrape_videos", seconds=43200)
-        def scrape_videos():
-            logging.info("Scraping YouTube videos")
-            fetch_videos()
-        
-        scheduler.start()
-        
-        scrape_schedules()
-        scrape_videos()
-
-
-    try:
-        debug = os.getenv("STAGE") != "prod"
-        app.run(debug=True, host="0.0.0.0", port=8000)
-    except KeyboardInterrupt:
-        scheduler.shutdown()
-        sys.exit(0)
+    app.run(debug=True, host="0.0.0.0", port=8000)

docker-compose.yml

@@ -8,3 +8,10 @@ services:
       - "8000:8000"
     volumes:
       - ./ca-certificate.crt:/etc/ssl/ca-certificate.crt:ro # Mount MongoDB cert inside the container, ro for read only
+
+  scraper:
+    image: cornellappdev/score-dev:${IMAGE_TAG}
+    env_file: .env
+    command: python scraper.py
+    volumes:
+      - ./ca-certificate.crt:/etc/ssl/ca-certificate.crt:ro

scraper.py

@@ -0,0 +1,47 @@
+import logging
+import time
+from flask import Flask
+from flask_apscheduler import APScheduler
+import signal
+import sys
+from src.scrapers.games_scraper import fetch_game_schedule
+from src.scrapers.youtube_stats import fetch_videos
+
+app = Flask(__name__)
+
+# Set up the scheduler
+scheduler = APScheduler()
+scheduler.init_app(app)
+
+# Configure logging
+logging.basicConfig(
+    format="%(asctime)s %(levelname)-8s %(message)s",
+    level=logging.INFO,
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+@scheduler.task("interval", id="scrape_schedules", seconds=300)
+def scrape_schedules():
+    start_time = time.time()
+    logging.info("Starting scraping games")
+    fetch_game_schedule()
+    elapsed_time = time.time() - start_time
+    logging.info(f"Completed scraping games in {elapsed_time:.2f} seconds")
+
+@scheduler.task("interval", id="scrape_videos", seconds=43200)
+def scrape_videos():
+    logging.info("Scraping YouTube videos")
+    fetch_videos()
+
+def signal_handler(sig, frame):
+    scheduler.shutdown()
+    sys.exit(0)
+
+signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
+
+if __name__ == "__main__":
+    scheduler.start()
+    scrape_schedules()
+    scrape_videos()
+    app.run(host="0.0.0.0", port=8001)

src/utils/helpers.py

@@ -1,3 +1,4 @@
+import logging
 import requests
 from PIL import Image
 from io import BytesIO
@@ -16,35 +17,41 @@ def get_dominant_color(image_url, white_threshold=200, black_threshold=50):
     Returns:
         color: The hex code of the dominant color.
     """
-    response = requests.get(image_url)
-    image = Image.open(BytesIO(response.content)).convert("RGBA")
-
-    image = image.resize((50, 50))
-    image = image.quantize(colors=5).convert("RGBA")
-    pixels = image.getdata()
-
-    filtered_pixels = [
-        pixel
-        for pixel in pixels
-        if not (
-            pixel[0] > white_threshold
-            and pixel[1] > white_threshold
-            and pixel[2] > white_threshold
+    default_color = "#000000" 
+
+    try:
+        response = requests.get(image_url)
+        image = Image.open(BytesIO(response.content)).convert("RGBA")
+
+        image = image.resize((50, 50))
+        image = image.quantize(colors=5).convert("RGBA")
+        pixels = image.getdata()
+
+        filtered_pixels = [
+            pixel
+            for pixel in pixels
+            if not (
+                pixel[0] > white_threshold
+                and pixel[1] > white_threshold
+                and pixel[2] > white_threshold
+            )
+            and not (
+                pixel[0] < black_threshold
+                and pixel[1] < black_threshold
+                and pixel[2] < black_threshold
+            )
+        ]
+
+        if filtered_pixels:
+            pixel_count = Counter(filtered_pixels)
+            dominant_color = pixel_count.most_common(1)[0][0]
+        else:
+            dominant_color = (0, 0, 0)
+
+        hex_color = "#{:02x}{:02x}{:02x}".format(
+            dominant_color[0], dominant_color[1], dominant_color[2]
         )
-        and not (
-            pixel[0] < black_threshold
-            and pixel[1] < black_threshold
-            and pixel[2] < black_threshold
-        )
-    ]
-
-    if filtered_pixels:
-        pixel_count = Counter(filtered_pixels)
-        dominant_color = pixel_count.most_common(1)[0][0]
-    else:
-        dominant_color = (0, 0, 0)
-
-    hex_color = "#{:02x}{:02x}{:02x}".format(
-        dominant_color[0], dominant_color[1], dominant_color[2]
-    )
-    return hex_color
+        return hex_color
+    except Exception as e:
+        logging.error(f"Error in get_dominant_color for {image_url}: {e}")
+        return default_color