Podigee total numbers (#101)

* Improve debug logging for Podigee refresh token usage, truncating token display for security
* Add endpoint for podigee total numbers of podcasts and episode
Fixes #100
* Refactor date handling for podcast metadata and analytics; improve date conversion utility
* Fix environment variable name for Podigee access token in .env.sample

Author: woolfg

connector_manager/manager/podigee_connector.py

@@ -83,7 +83,7 @@ def handle_podigee_refresh(db_connection, account_id, source_name, source_access
 
     # Get refresh token from source_access_keys
     refresh_token = source_access_keys.get("PODIGEE_REFRESH_TOKEN")
-    logger.debug(f"Using refresh token: {refresh_token}")    
+    logger.debug(f"Using refresh token: {refresh_token[:10]}... for {pod_name}")
     if not client_id or not client_secret or not refresh_token:
         logger.error(f"Missing required OAuth credentials for Podigee: {pod_name}")
         return None

podigee/.env.sample

@@ -5,4 +5,4 @@ PODIGEE_USERNAME="username"
 PODIGEE_PASSWORD="password"
 PODCAST_ID=12345
 
-OPENPODCAST_ACCESS_TOKEN=your_access_token_here
+PODIGEE_ACCESS_TOKEN=your_access_token_here

podigee/job/__main__.py

@@ -6,6 +6,7 @@
 
 from queue import Queue
 from datetime import datetime, timedelta
+import calendar
 
 from job.fetch_params import FetchParams
 from job.worker import worker
@@ -43,8 +44,10 @@
 
 # Start- and end-date for the data we want to fetch
 # Load from environment variable if set, otherwise set to defaults
+# Podigee default is last 30 days
+TODAY_DATE = dt.datetime.now()
 START_DATE = load_env(
-    "START_DATE", (dt.datetime.now() - dt.timedelta(days=30)
+    "START_DATE", (dt.datetime.now() - dt.timedelta(days=31)
                    ).strftime("%Y-%m-%d")
 )
 END_DATE = load_env(
@@ -130,6 +133,9 @@
 
 # Extract and validate podcast title
 podcast_title = podcast.get("title")
+# published at format is "2022-01-25T22:19:42Z"
+podcast_published_at = datetime.fromisoformat(podcast.get("published_at").replace("Z", "+00:00"))
+
 if not podcast_title:
     logger.error(f"Podcast with ID {PODCAST_ID} has no title")
     exit(1)
@@ -149,6 +155,36 @@
     exit(1)
 
 
+def get_date_string(date_obj):
+    """
+    Convert date object to string if needed, or return string as-is.
+    """
+    if isinstance(date_obj, str):
+        return date_obj
+    elif isinstance(date_obj, datetime):
+        return date_obj.strftime("%Y-%m-%d")
+    elif hasattr(date_obj, 'strftime'):  # handles date objects too
+        return date_obj.strftime("%Y-%m-%d")
+    else:
+        return str(date_obj)
+
+
+def extract_date_str_from_iso(iso_string):
+    """
+    Extract date string (YYYY-MM-DD) from ISO datetime string.
+    Since Podigee always sends UTC timestamps with 'Z', this preserves the UTC date.
+    """
+    if not iso_string:
+        return ""
+    try:
+        # Python 3.11+ handles 'Z' suffix directly
+        dt = datetime.fromisoformat(iso_string)
+        return dt.strftime("%Y-%m-%d")
+    except (ValueError, AttributeError):
+        # Fallback to split method if parsing fails
+        return iso_string.split("T")[0] if "T" in iso_string else iso_string
+
+
 def get_request_lambda(f, *args, **kwargs):
     """
     Capture arguments in the closure so we can use them later in the call
@@ -165,19 +201,89 @@ def get_podcast_metadata():
         "name": podcast_title
     }
 
+def get_end_date_on_granularity(granularity, start_date):
+    """
+    Get end date based on granularity and start date.
+    Returns a string in YYYY-MM-DD format.
+    """
+    if granularity == "day":
+        return get_date_string(start_date)
+    elif granularity == "month":
+        # Convert to datetime object if needed
+        if isinstance(start_date, str):
+            date_obj = datetime.strptime(start_date, "%Y-%m-%d")
+        else:
+            date_obj = start_date
+        
+        # Get last day of the month
+        last_day = calendar.monthrange(date_obj.year, date_obj.month)[1]
+        end_date = date_obj.replace(day=last_day)
+        return get_date_string(end_date)
+    return get_date_string(start_date)
+
+def transform_podigee_podcast_overview(overview_data):
+    """
+    Transform Podigee podcast overview data to OpenPodcast format.
+    Format is {"published_episodes_count":0,
+    "audio_published_minutes":0.0,
+    "unique_listeners_number":305,
+    "unique_subscribers_number":283,
+    "mean_audio_published_minutes":0,
+    "mean_episode_download":8.897435897435898,
+    "total_downloads":694.0,
+    "meta":{"from":"2025-08-01T00:00:00.000Z","to":"2025-08-31T23:59:59.999Z"}
+    """
+
+    if not overview_data or "meta" not in overview_data:
+        logger.error(f"Invalid overview data structure: {overview_data}")
+        return {"metrics": []}
+
+    metrics = []
+
+    if "unique_listeners_number" in overview_data:
+        metrics.append({
+            "start": extract_date_str_from_iso(overview_data["meta"]["from"]),
+            "end": extract_date_str_from_iso(overview_data["meta"]["to"]),
+            "dimension": "listeners",
+            "subdimension": "unique",
+            "value": overview_data["unique_listeners_number"]
+        })
+    if "unique_subscribers_number" in overview_data:
+        metrics.append({
+            "start": extract_date_str_from_iso(overview_data["meta"]["from"]),
+            "end": extract_date_str_from_iso(overview_data["meta"]["to"]),
+            "dimension": "subscribers",
+            "subdimension": "unique",
+            "value": overview_data["unique_subscribers_number"]
+        })
+    if "total_downloads" in overview_data:
+        metrics.append({
+            "start": extract_date_str_from_iso(overview_data["meta"]["from"]),
+            "end": extract_date_str_from_iso(overview_data["meta"]["to"]),
+            "dimension": "downloads",
+            "subdimension": "downloads",
+            "value": overview_data["total_downloads"]
+        })
+
+    if not metrics:
+        logger.warning(f"No valid metrics found in overview data: {overview_data}")
+
+    return {"metrics": metrics}
+
 
-def transform_podigee_analytics_to_metrics(analytics_data):
+def transform_podigee_analytics_to_metrics(analytics_data, store_downloads_only=False):
     """
     Transform Podigee analytics data to OpenPodcast metrics format.
     Expected format: {"metrics": [{"start": "date", "end": "date", "dimension": "string", "subdimension": "string", "value": number}]}
     """
     if not analytics_data or "objects" not in analytics_data:
         return {"metrics": []}
-    
+
+    aggregation_granularity = analytics_data.get("meta", {}).get("aggregation_granularity", "day")
     metrics = []
 
     for day_data in analytics_data["objects"]:
-        date = day_data.get("downloaded_on", "").split("T")[0]  # Extract YYYY-MM-DD
+        date = extract_date_str_from_iso(day_data.get("downloaded_on", ""))
         if not date:
             continue
 
@@ -186,44 +292,45 @@ def transform_podigee_analytics_to_metrics(analytics_data):
             for download_type, value in day_data["downloads"].items():
                 metrics.append({
                     "start": date,
-                    "end": date,
+                    "end": get_end_date_on_granularity(aggregation_granularity, date),
                     "dimension": "downloads",
                     "subdimension": download_type,
                     "value": value
                 })
-        
-        # Process platforms
-        if "platforms" in day_data:
-            for platform, value in day_data["platforms"].items():
-                metrics.append({
-                    "start": date,
-                    "end": date,
-                    "dimension": "platforms",
-                    "subdimension": platform,
-                    "value": value
-                })
-        
-        # Process clients
-        if "clients" in day_data:
-            for client, value in day_data["clients"].items():
-                metrics.append({
-                    "start": date,
-                    "end": date,
-                    "dimension": "clients",
-                    "subdimension": client,
-                    "value": value
-                })
-        
-        # Process sources
-        if "sources" in day_data:
-            for source, value in day_data["sources"].items():
-                metrics.append({
-                    "start": date,
-                    "end": date,
-                    "dimension": "sources",
-                    "subdimension": source,
-                    "value": value
-                })
+
+        if not store_downloads_only:
+            # Process platforms
+            if "platforms" in day_data:
+                for platform, value in day_data["platforms"].items():
+                    metrics.append({
+                        "start": date,
+                        "end": get_end_date_on_granularity(aggregation_granularity, date),
+                        "dimension": "platforms",
+                        "subdimension": platform,
+                        "value": value
+                    })
+
+            # Process clients
+            if "clients" in day_data:
+                for client, value in day_data["clients"].items():
+                    metrics.append({
+                        "start": date,
+                        "end": get_end_date_on_granularity(aggregation_granularity, date),
+                        "dimension": "clients",
+                        "subdimension": client,
+                        "value": value
+                    })
+            
+            # Process sources
+            if "sources" in day_data:
+                for source, value in day_data["sources"].items():
+                    metrics.append({
+                        "start": date,
+                        "end": get_end_date_on_granularity(aggregation_granularity, date),
+                        "dimension": "sources",
+                        "subdimension": source,
+                        "value": value
+                    })
 
     return {"metrics": metrics}
 
@@ -236,11 +343,33 @@ def transform_podigee_analytics_to_metrics(analytics_data):
         start_date=date_range.start,
         end_date=date_range.end,
     ),
-    # Podcast metrics - analytics data for the podcast
+    # Podcast metrics like apps and platforms and downloads per day of last 30 days
+    FetchParams(
+        openpodcast_endpoint="metrics",
+        podigee_call=lambda: transform_podigee_analytics_to_metrics(
+            podigee.podcast_analytics(PODCAST_ID, start=date_range.start, end=date_range.end),
+            # we fetch this just every week on Monday and the first day of the month
+            # daily downloads are stored every day
+            not (TODAY_DATE.weekday() == 0 or TODAY_DATE.day == 1)
+            ),
+            start_date=date_range.start,
+            end_date=date_range.end,
+        ),
+    # Fetch total downloads since beginning which is returned in months
     FetchParams(
-        openpodcast_endpoint="metrics", 
+        openpodcast_endpoint="metrics",
         podigee_call=lambda: transform_podigee_analytics_to_metrics(
-            podigee.podcast_analytics(PODCAST_ID, start=date_range.start, end=date_range.end)
+            podigee.podcast_analytics(PODCAST_ID, start=podcast_published_at, end=date_range.end),
+            store_downloads_only=True
+        ),
+        start_date=podcast_published_at,
+        end_date=date_range.end,
+    ),
+    # Fetch overview metrics for the podcast, endpoint "overview"
+    FetchParams(
+        openpodcast_endpoint="metrics",
+        podigee_call=lambda: transform_podigee_podcast_overview(
+            podigee.podcast_overview(PODCAST_ID, start=date_range.start, end=date_range.end),
         ),
         start_date=date_range.start,
         end_date=date_range.end,
@@ -251,6 +380,9 @@ def transform_podigee_analytics_to_metrics(analytics_data):
 
 for episode in episodes:
     print(episode)
+    episode_published_at_str = extract_date_str_from_iso(episode.get("published_at", ""))
+    # Convert to datetime object for API calls
+    episode_published_at = datetime.strptime(episode_published_at_str, "%Y-%m-%d") if episode_published_at_str else date_range.start
     endpoints += [
         # Episode metadata - basic episode information
         FetchParams(
@@ -269,14 +401,30 @@ def transform_podigee_analytics_to_metrics(analytics_data):
             openpodcast_endpoint="metrics",
             podigee_call=get_request_lambda(
                 lambda ep_id: transform_podigee_analytics_to_metrics(
-                    podigee.episode_analytics(ep_id, granularity=None, start=date_range.start, end=date_range.end)
-                ), 
+                    podigee.episode_analytics(ep_id, granularity=None, start=date_range.start, end=date_range.end),
+                    # for now we just store the downloads and do not store platforms etc. per episode
+                    store_downloads_only=True
+                ),
                 str(episode["id"])
             ),
             start_date=date_range.start,
             end_date=date_range.end,
             meta={"episode": str(episode["id"])},
         ),
+         # We store the downloads since publication. The Podigee API returns one data point per month.
+        FetchParams(
+            openpodcast_endpoint="metrics",
+            podigee_call=get_request_lambda(
+                lambda ep_id: transform_podigee_analytics_to_metrics(
+                    podigee.episode_analytics(ep_id, granularity="monthly", start=episode_published_at, end=date_range.end),
+                    store_downloads_only=True
+                ),
+                str(episode["id"])
+            ),
+            start_date=episode_published_at,
+            end_date=date_range.end,
+            meta={"episode": str(episode["id"])},
+        )
     ]
 
 # Create a queue to hold the FetchParams objects