Adding db indices

Author: Aayush-Agnihotri

scraper.py

@@ -38,8 +38,12 @@ def signal_handler(sig, frame):
 signal.signal(signal.SIGTERM, signal_handler)
 
 if __name__ == "__main__":
-    scheduler.add_job(scrape_schedules, "interval", seconds=300, id="scrape_schedules")
-    scheduler.add_job(scrape_videos, "interval", seconds=43200, id="scrape_videos")
+    scheduler.add_job(
+        scrape_schedules, "interval", seconds=60 * 60, id="scrape_schedules"
+    )
+    scheduler.add_job(
+        scrape_videos, "interval", seconds=60 * 60 * 24, id="scrape_videos"
+    )
     scheduler.start()
     scrape_schedules()
     scrape_videos()

src/database.py

@@ -16,19 +16,10 @@
 
 # Initialize MongoDB client
 if use_tls:
-    # client_options = {
-    #     "maxPoolSize": 100,
-    #     "minPoolSize": 10,
-    #     "waitQueueTimeoutMS": 2000,
-    #     "connectTimeoutMS": 30000,
-    #     "socketTimeoutMS": 45000,
-    #     "serverSelectionTimeoutMS": 30000
-    # }
     client = MongoClient(
         os.getenv("MONGO_URI"),
         tls=True,
         tlsCAFile=file_name,
-        # **client_options
     )
 else:
     client = MongoClient(os.getenv("MONGO_URI"))
@@ -57,3 +48,28 @@ def keep_connection_alive():
 
 # Access the database
 db = client[os.getenv("MONGO_DB", "score_db")]
+
+
+def setup_database_indexes():
+    """Set up MongoDB indexes for optimal query performance"""
+    try:
+        game_collection = db["game"]
+
+        # Create single field indexes for commonly queried fields
+        game_collection.create_index([("sport", 1)], background=True)
+        game_collection.create_index([("date", 1)], background=True)
+        game_collection.create_index([("gender", 1)], background=True)
+
+        # Create compound indexes for common query combinations
+        game_collection.create_index([("sport", 1), ("gender", 1)], background=True)
+
+        # Index for sorting operations
+        game_collection.create_index([("date", -1)], background=True)
+
+        print("✅ MongoDB indexes created successfully")
+    except Exception as e:
+        print(f"❌ Failed to create MongoDB indexes: {e}")
+        raise e
+
+
+setup_database_indexes()

src/models/game.py

@@ -35,7 +35,7 @@ def __init__(
         box_score=None,
         score_breakdown=None,
         team=None,
-        utc_date=None
+        utc_date=None,
     ):
         self.id = id if id else str(ObjectId())
         self.city = city
@@ -70,7 +70,7 @@ def to_dict(self):
             "box_score": self.box_score,
             "score_breakdown": self.score_breakdown,
             "team": self.team,
-            "utc_date": self.utc_date
+            "utc_date": self.utc_date,
         }
 
     @staticmethod
@@ -92,5 +92,5 @@ def from_dict(data) -> None:
             box_score=data.get("box_score"),
             score_breakdown=data.get("score_breakdown"),
             team=data.get("team"),
-            utc_date=data.get("utc_date")
-        )
+            utc_date=data.get("utc_date"),
+        )