performance enhancements with cacheing, split out session metrics to a separate page

Author: jeffoberlander

.streamlit/pages.toml

@@ -61,4 +61,10 @@ icon = ":small_orange_diamond:"
 [[pages]]
 path = "app_pages/engagement_over_time.py"
 name = "Engagement Over Time"
-icon = ":small_orange_diamond:"
+icon = ":small_orange_diamond:"
+
+[[pages]]
+path = "app_pages/unity_cr_engagement_metrics.py"
+name = "Unity and CR Session Metrics"
+icon = ":small_orange_diamond:"
+

app_pages/Unity_vs_CR.py

@@ -6,8 +6,7 @@
 import ui_widgets as ui
 import users
 import metrics
-from pyinstrument import Profiler
-from pyinstrument.renderers.console import ConsoleRenderer
+from settings import get_logger
 
 settings.initialize()
 settings.init_user_list()
@@ -17,6 +16,7 @@
 
 ui.colorize_multiselect_options()
 
+logger = settings.get_logger()
 
 # --- Filter Row ---
 col_date, col_lang, col_country = st.columns((1, 1, 1), gap="large")
@@ -53,72 +53,43 @@
 with col_cr_head:
     st.markdown("<strong><div style='text-align: center;'>Curious Reader</div></strong>", unsafe_allow_html=True)
 
-# --- Metrics + Funnels ---
-p = Profiler(async_mode="disabled")
-with p:
-    if len(daterange) == 2 and countries_list:
 
-        # --- Get user cohorts ---
-        user_cohort_list_unity = metrics.get_user_cohort_list(
-            daterange=daterange,
-            languages=language,
+if len(daterange) == 2 and countries_list:
+
+    # --- Get user cohorts ---
+    user_cohort_list_unity = metrics.get_user_cohort_list(
+        daterange=daterange,
+        languages=language,
+        countries_list=countries_list,
+        app="Unity"
+    )
+    user_cohort_list_cr = metrics.get_user_cohort_list(
+        daterange=daterange,
+        languages=language,
+        countries_list=countries_list,
+        app="CR"
+    )
+
+    col1, col2 = st.columns((1, 1), gap="large")
+
+    # --- Funnel Charts ---
+    with col1:
+        uic.create_funnels(
             countries_list=countries_list,
-            app="Unity"
-        )
-        user_cohort_list_cr = metrics.get_user_cohort_list(
             daterange=daterange,
-            languages=language,
-            countries_list=countries_list,
-            app="CR"
+            app="Unity",
+            key_prefix="u-1",
+            languages=languages,
+            user_list=user_cohort_list_unity
         )
 
-        # --- Calculate metrics ---
-        metrics_unity = {
-            "Avg # Sessions / User": metrics.calculate_average_metric_per_user(user_cohort_list_unity, app="Unity", column_name="engagement_event_count"),
-            "Avg Total Play Time / User": metrics.calculate_average_metric_per_user(user_cohort_list_unity, app="Unity", column_name="total_time_minutes"),
-            "Avg Session Length / User": metrics.calculate_average_metric_per_user(user_cohort_list_unity, app="Unity", column_name="avg_session_length_minutes"),
-            "Active Span / User": metrics.calculate_average_metric_per_user(user_cohort_list_unity, app="Unity", column_name="active_span"),
-        }
-
-        metrics_cr = {
-            "Avg # Sessions / User": metrics.calculate_average_metric_per_user(user_cohort_list_cr, app="CR", column_name="engagement_event_count"),
-            "Avg Total Play Time / User": metrics.calculate_average_metric_per_user(user_cohort_list_cr, app="CR", column_name="total_time_minutes"),
-            "Avg Session Length / User": metrics.calculate_average_metric_per_user(user_cohort_list_cr, app="CR", column_name="avg_session_length_minutes"),
-            "Active Span / User": metrics.calculate_average_metric_per_user(user_cohort_list_cr, app="CR", column_name="active_span"),
-        }
-
-        # --- Metric Display Columns ---
-        col_unity_metrics, col_cr_metrics = st.columns((1, 1), gap="large")
-
-        with col_unity_metrics:
-            for label, value in metrics_unity.items():
-                unit = " days" if "Span" in label else " min" if "Time" in label or "Length" in label else ""
-                st.metric(label=label, value=f"{value:.2f}{unit}")
-
-        with col_cr_metrics:
-            for label, value in metrics_cr.items():
-                unit = " days" if "Span" in label else " min" if "Time" in label or "Length" in label else ""
-                st.metric(label=label, value=f"{value:.2f}{unit}")
-
-        # --- Funnel Charts ---
-        with col_unity_metrics:
-            uic.create_funnels(
-                countries_list=countries_list,
-                daterange=daterange,
-                app="Unity",
-                key_prefix="u-1",
-                languages=languages,
-                user_list=user_cohort_list_unity
-            )
-
-        with col_cr_metrics:
-            uic.create_funnels(
-                countries_list=countries_list,
-                daterange=daterange,
-                app="CR",
-                funnel_size="compact",
-                key_prefix="u-2",
-                languages=languages,
-                user_list=user_cohort_list_cr)
-            
-print(p.output(ConsoleRenderer(show_all=False,timeline=True,color=True,unicode=True,short_mode=False)))
+    with col2:
+        uic.create_funnels(
+            countries_list=countries_list,
+            daterange=daterange,
+            app="CR",
+            funnel_size="compact",
+            key_prefix="u-2",
+            languages=languages,
+            user_list=user_cohort_list_cr)
+        

app_pages/unity_cr_engagement_metrics.py

@@ -0,0 +1,101 @@
+import streamlit as st
+import settings
+from rich import print as rprint
+from millify import prettify
+import ui_components as uic
+import ui_widgets as ui
+import users
+import metrics
+from settings import get_logger
+
+settings.initialize()
+settings.init_user_list()
+
+ui.display_definitions_table("Data Notes",ui.data_notes)
+countries_list = users.get_country_list()
+
+ui.colorize_multiselect_options()
+
+logger = settings.get_logger()
+
+# --- Filter Row ---
+col_date, col_lang, col_country = st.columns((1, 1, 1), gap="large")
+
+with col_date:
+    st.caption("Select a Date")
+    selected_date, option = ui.calendar_selector(placement="middle")
+    daterange = ui.convert_date_to_range(selected_date, option)
+
+with col_lang:
+    languages = users.get_language_list()
+    language = ui.single_selector(
+        languages, placement="middle", title="Select a language", key="acq-1"
+    )
+
+with col_country:
+    countries_list = ui.multi_select_all(
+        countries_list,
+        title="Country Selection",
+        key="acq-2",
+        placement="middle"
+    )
+
+# --- Date Display ---
+if len(daterange) == 2:
+    start = daterange[0].strftime("%b %d, %Y")
+    end = daterange[1].strftime("%b %d, %Y")
+    st.write(f"{start} to {end}")
+
+# --- Header Row ---
+col_unity_head, col_cr_head = st.columns((1, 1), gap="large")
+with col_unity_head:
+    st.markdown("<strong><div style='text-align: center;'>Unity</div></strong>", unsafe_allow_html=True)
+with col_cr_head:
+    st.markdown("<strong><div style='text-align: center;'>Curious Reader</div></strong>", unsafe_allow_html=True)
+
+
+if len(daterange) == 2 and countries_list:
+
+    # --- Get user cohorts ---
+    user_cohort_list_unity = metrics.get_user_cohort_list(
+        daterange=daterange,
+        languages=language,
+        countries_list=countries_list,
+        app="Unity"
+    )
+    user_cohort_list_cr = metrics.get_user_cohort_list(
+        daterange=daterange,
+        languages=language,
+        countries_list=countries_list,
+        app="CR"
+    )
+
+    # --- Calculate metrics ---
+    metrics_unity = {
+        "Avg # Sessions / User": metrics.calculate_average_metric_per_user(user_cohort_list_unity, app="Unity", column_name="engagement_event_count"),
+        "Avg Total Play Time / User": metrics.calculate_average_metric_per_user(user_cohort_list_unity, app="Unity", column_name="total_time_minutes"),
+        "Avg Session Length / User": metrics.calculate_average_metric_per_user(user_cohort_list_unity, app="Unity", column_name="avg_session_length_minutes"),
+        "Active Span / User": metrics.calculate_average_metric_per_user(user_cohort_list_unity, app="Unity", column_name="active_span"),
+    }
+
+    metrics_cr = {
+        "Avg # Sessions / User": metrics.calculate_average_metric_per_user(user_cohort_list_cr, app="CR", column_name="engagement_event_count"),
+        "Avg Total Play Time / User": metrics.calculate_average_metric_per_user(user_cohort_list_cr, app="CR", column_name="total_time_minutes"),
+        "Avg Session Length / User": metrics.calculate_average_metric_per_user(user_cohort_list_cr, app="CR", column_name="avg_session_length_minutes"),
+        "Active Span / User": metrics.calculate_average_metric_per_user(user_cohort_list_cr, app="CR", column_name="active_span"),
+    }
+
+    # --- Metric Display Columns ---
+    col_unity_metrics, col_cr_metrics = st.columns((1, 1), gap="large")
+
+    with col_unity_metrics:
+        for label, value in metrics_unity.items():
+            unit = " days" if "Span" in label else " min" if "Time" in label or "Length" in label else ""
+            st.metric(label=label, value=f"{value:.2f}{unit}")
+
+    with col_cr_metrics:
+        for label, value in metrics_cr.items():
+            unit = " days" if "Span" in label else " min" if "Time" in label or "Length" in label else ""
+            st.metric(label=label, value=f"{value:.2f}{unit}")
+
+            

metrics.py

@@ -180,7 +180,7 @@ def weeks_since(daterange):
 
 
 # Returns a DataFrame list of counts by language or counts by country
-#@st.cache_data(ttl="1d", show_spinner=False)
+@st.cache_data(ttl="1d", show_spinner=False)
 def get_counts(
     type="app_language",
     daterange=default_daterange,
@@ -266,7 +266,7 @@ def get_counts(
 
 #Added new parameter user_list.  If passed, only return the funnel based on that set of users
 
-#@st.cache_data(ttl="1d", show_spinner=False)
+@st.cache_data(ttl="1d", show_spinner=False)
 def build_funnel_dataframe(
     index_col="language",
     daterange=default_daterange,
@@ -469,7 +469,7 @@ def add_level_percents(df):
 
 
 # Get the campaign data and filter by date, language, and country selections
-#@st.cache_data(ttl="1d", show_spinner=False)
+@st.cache_data(ttl="1d", show_spinner=False)
 def filter_campaigns(df_campaigns_all,daterange,selected_languages,countries_list):
 
     # Drop the campaigns that don't meet the naming convention
@@ -514,7 +514,6 @@ def get_month_ranges(start_date, end_date):
     return month_ranges
 
 #Returns a dataframe of the totals of a stat for each month
-#@st.cache_data(ttl="1d", show_spinner=False)
 def get_totals_per_month(daterange, stat, countries_list, language):
     # First, get all campaign data
     df_campaigns_all = st.session_state["df_campaigns_all"]
@@ -560,7 +559,7 @@ def get_totals_per_month(daterange, stat, countries_list, language):
     # Display the DataFrame
     return df_totals
 
-#@st.cache_data(ttl="1d", show_spinner=False)
+@st.cache_data(ttl="1d", show_spinner=False)
 def get_date_cohort_dataframe(
     daterange=default_daterange,
     languages=["All"],
@@ -618,7 +617,7 @@ def get_user_cohort_list(
         return user_cohort_df
 
 
-#@st.cache_data(ttl="1d", show_spinner=False)
+
 def calculate_average_metric_per_user(user_cohort_list, app, column_name):
     df_cr_app_launch = st.session_state["df_cr_app_launch"]
     df_unity_users = st.session_state["df_unity_users"]
@@ -633,7 +632,9 @@ def calculate_average_metric_per_user(user_cohort_list, app, column_name):
         df_filtered = df_unity_users[df_unity_users["user_pseudo_id"].isin(user_cohort_list)]
 
     # Sum the selected column and calculate the average
-    total = df_filtered[column_name].sum()
+    import numpy as np
+
+    total = np.sum(df_filtered[column_name].values)
 
     average = total / len(user_cohort_list)
 

settings.py

@@ -12,6 +12,22 @@
 
 default_daterange = [dt.datetime(2021, 1, 1).date(), dt.date.today()]
 
+import logging
+
+def get_logger(name="dashboard_logger"):
+    logger = logging.getLogger(name)
+    if not logger.handlers:
+        logger.setLevel(logging.DEBUG)
+        
+        # Console handler or file handler
+        handler = logging.StreamHandler()  # or logging.FileHandler("logs/app.log")
+        
+        formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+        handler.setFormatter(formatter)
+        
+        logger.addHandler(handler)
+        logger.propagate = False  # Prevent double logging
+    return logger
 
 def get_gcp_credentials():
     # first get credentials to secret manager

ui_components.py

@@ -359,7 +359,7 @@ def lrc_scatter_chart(option,display_category,df_campaigns,daterange):
         st.write("No data for selected period")
 
 
-#@st.cache_data(ttl="1d", show_spinner=False)
+@st.cache_data(ttl="1d", show_spinner=False)
 def spend_by_country_map(df_campaigns,source):
 
     if source == 'Both':
@@ -402,7 +402,6 @@ def spend_by_country_map(df_campaigns,source):
     st.plotly_chart(country_fig)
 
 
-@st.cache_data(ttl="1d", show_spinner=False)
 def campaign_funnel_chart():
     df_campaigns = st.session_state.df_campaigns
     impressions = df_campaigns["impressions"].sum()
@@ -603,8 +602,6 @@ def funnel_change_line_chart(df, graph_type='sum'):
     st.plotly_chart(fig, use_container_width=True)
 
 
-
-@st.cache_data(ttl="1d", show_spinner=False)
 def top_campaigns_by_downloads_barchart(n):
     df_campaigns = st.session_state.df_campaigns
     df = df_campaigns.filter(["campaign_name", "mobile_app_install"], axis=1)
@@ -942,7 +939,7 @@ def top_and_bottom_languages_per_level(selection, min_LR):
 
 
 #Added user_list which is a list of cr_user_id to filter with
-#@st.cache_data(ttl="1d", show_spinner=False)
+
 def create_funnels(countries_list=["All"],
                    daterange=default_daterange,
                    languages=["All"],

users.py

@@ -5,6 +5,7 @@
 from pyinstrument import Profiler
 from pyinstrument.renderers.console import ConsoleRenderer
 import asyncio
+import settings
 
 
 # How far back to obtain user data.  Currently the queries pull back to 01/01/2021
@@ -103,8 +104,8 @@ async def run_query(query):
         max_level_indices_unity = df_unity_users.groupby('user_pseudo_id')['max_user_level'].idxmax()
         df_unity_users = df_unity_users.loc[max_level_indices_unity].reset_index()
 
-
-    print(p.output(ConsoleRenderer(show_all=False,timeline=True,color=True,unicode=True,short_mode=True)))
+    logger = settings.get_logger()
+    logger.debug(p.output(ConsoleRenderer(show_all=False,timeline=True,color=True,unicode=True,short_mode=False)))
 
     return  df_cr_users, df_unity_users,  df_cr_app_launch