Update Database to use new table fromat and twitch api

twitch_download_videos.py

@@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-import requests
-import os
-import time
-import csv
-
-import linux
-from database import Database
-
-CHANNEL_NAME = 'teampgp'
-DB = None
-
-def write_csv(data, file_name):
-    # Open file with newline='' to prevent extra blank rows across platforms
-    with open(file_name, "w", newline="", encoding="utf-8") as file:
-        writer = csv.writer(file)
-        
-        # Write all rows at once
-        writer.writerows(data)
-
-def download():
-    """Find all undownload videos and download them."""
-    undownloaded = DB.get_undownloaded()
-
-    print(f"Download...")
-    for id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts in undownloaded:
-        data = [[id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts]]
-        print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}")
-        if not clip_is:
-            output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {id} -o download/videos/{id}/{id}.mp4 --collision Overwrite")
-            output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o download/videos/{id}/{id}_chat.json -E --collision Overwrite")
-            write_csv(data, f"download/videos/{id}/{id}.csv")
-        elif clip_is:
-            output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {id} -o download/clips/{id}/{id}.mp4 --collision Overwrite")
-            write_csv(data, f"download/clips/{id}/{id}.csv")
-
-        time.sleep(1)
-
-        if output["success"] is True:
-            print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title} | was successful")
-            DB.mark_as_downloaded(id)
-        else:
-            print(f"ID: {id} | Download Process failed. {output["stdout"]}. Error: {output["stderr"]}")
-
-    print(f"Finished Downloading...")
-
-if __name__ == "__main__":
-    DB = Database()
-    download()

twitch_secrets copy.json

@@ -0,0 +1,5 @@
+{
+    "client_id": "yr610ucde5vlae3zqniv23eps4ky7j",
+    "client_secret": "1m8bopo5hwtnv0mox9wql8i33fqrgr",
+    "manual_token": "dkmhv4f6k0mr1yyzof54xqll5pf7l3"
+}

twitch_vod.py

@@ -1,10 +1,10 @@
-#!/usr/bin/env python3
 import asyncio
 import json
-import httpx  # Switched from requests to prevent async loop freezing
+import requests
 import database
 from twitchAPI.twitch import Twitch
 from twitchAPI.helper import first
+# Import the explicit VideoType Enum to prevent the AttributeError
 from twitchAPI.type import VideoType
 
 SECRETS = None
@@ -15,7 +15,9 @@ GAME_CACHE = {} # Local cache dictionary to store game_id -> game_name mapping
 CHANNEL_NAME = "teampgp"
 
 async def get_twitch():
-    global SECRETS, TWITCH, USER
+    global SECRETS
+    global TWITCH
+    global USER
     if SECRETS is None:
         with open('twitch_secrets.json', 'r') as f:
             SECRETS = json.load(f)
@@ -42,9 +44,8 @@ async def get_game_name_by_id(game_id: str) -> str:
         pass
     return "Unknown Game"
 
-async def get_vod_game_name(vod_id: str):
+async def get_vod_game_name(vod_id: str) -> str:
-    """Asynchronously query Twitch GQL endpoint for VOD game metadata."""
+    game_id = 0
-    game_id = "0"
     game_name = "Unknown Game"
     url = "https://gql.twitch.tv/gql"
 
@@ -57,7 +58,7 @@ async def get_vod_game_name(vod_id: str):
         "operationName": "VideoMetadata",
         "variables": {
             "channelLogin": "",
-            "videoID": str(vod_id)
+            "videoID": vod_id
         },
         "extensions": {
             "persistedQuery": {
@@ -67,21 +68,17 @@ async def get_vod_game_name(vod_id: str):
         }
     }]
 
-    # Using httpx async client to prevent blocking the asyncio event loop
+    response = requests.post(url, headers=headers, json=payload)
-    async with httpx.AsyncClient() as client:
+    data = response.json()
-        try:
+
-            response = await client.post(url, headers=headers, json=payload)
+    # Parsing the Game ID out of the response array
-            if response.status_code == 200:
+    video_info = data[0]['data']['video']
-                data = response.json()
+    if video_info and video_info.get('game'):
-                video_info = data[0].get('data', {}).get('video')
+        game_id = video_info['game']['id']
-                if video_info and video_info.get('game'):
+        game_name = video_info['game']['displayName']
-                    game_id = str(video_info['game']['id'])
+        print(f"Game: {game_name} (ID: {game_id})")
-                    game_name = video_info['game']['displayName']
+    else:
-                    print(f"GQL Found: {game_name} (ID: {game_id})")
+        print("No game information found for this VOD.")
-                else:
-                    print(f"No game information found in GQL for VOD {vod_id}.")
-        except Exception as e:
-            print(f"Error fetching GQL metadata for VOD {vod_id}: {e}")
 
     return game_id, game_name
 
@@ -94,24 +91,18 @@ async def get_streamer_vods():
     all_vods = []
 
     async for v in vod_generator:
-        # Resolving game category safely without freezing the event loop
+        # FIXED: Resolving game category using the automatic stream markers
         game_id, game_name = await get_vod_game_name(v.id)
-        
-        # Safely convert game_id to integer if possible, otherwise default to 0
-        try:
-            clean_game_id = int(game_id)
-        except ValueError:
-            clean_game_id = 0
 
         vod_data = {
             "id": v.id,
             "title": v.title,
             "created_at": str(v.published_at),
-            "view_count": int(v.view_count) if v.view_count else 0,
+            "view_count": int(v.view_count),
             "duration": v.duration,
             "url": v.url,
             "thumbnail_url": v.thumbnail_url,
-            "game_id": clean_game_id,
+            "game_id": int(game_id),
             "game_name": game_name,
             "stream_id": str(v.stream_id) if v.stream_id else "0",
             "creator_name": CHANNEL_NAME,
@@ -127,26 +118,24 @@ async def get_streamer_clips():
     await get_twitch()
 
     print(f"Starting clip extraction for {USER.display_name}...")
+
     clip_generator = TWITCH.get_clips(broadcaster_id=USER.id, first=100)
+    
     all_clips = []
 
     async for c in clip_generator:
+        # Clips DO have game_id attributes natively supported
         game_name = await get_game_name_by_id(c.game_id)
-        
-        try:
-            clean_game_id = int(c.game_id)
-        except (ValueError, TypeError):
-            clean_game_id = 0
 
         clip_data = {
             "id": c.id,
             "title": c.title,
             "created_at": str(c.created_at),             
-            "view_count": int(c.view_count) if c.view_count else 0,
+            "view_count": int(c.view_count),
             "duration": c.duration,
             "url": c.url,
             "thumbnail_url": c.thumbnail_url,
-            "game_id": clean_game_id,
+            "game_id": int(c.game_id),
             "game_name": game_name,
             "stream_id": "0",
             "creator_name": c.creator_name,
@@ -161,30 +150,22 @@ async def get_streamer_clips():
 async def main():
     print("--- Script Started ---")
     
-    db = database.Database()
+    # 1. Pull clips (with categories)
-    vods_list = await get_streamer_vods()
+    #clips_list = await get_streamer_clips()
-    
+    #print(f"\nSuccessfully received a list of {len(clips_list)} clips in main().")
-    for v in vods_list:
+    #if clips_list:
-        db.insert_video_record(
+    #    print(f"Top clip: '{clips_list[0]['title']}' (Game: {clips_list[0]['game_name']})")
-            v['id'], v['title'], v['created_at'], v['view_count'], v['duration'], 
-            v['url'], v['thumbnail_url'], v['game_id'], v['game_name'], 
-            v['stream_id'], v['creator_name'], v['clip_is']
-        )
 
+    db = database.Database()
+    # 2. Pull VODs (without categories)
+    vods_list = await get_streamer_vods()
+    for v in vods_list:
+        db.insert_video_record(v['id'], v['title'], v['created_at'], v['view_count'], v['duration'], v['url'], v['thumbnail_url'], 
+                               v['game_id'], v['game_name'], v['stream_id'], v['creator_name'], v['clip_is'])
+
+    #print(f"\nSuccessfully received a list of {len(vods_list)} VODs in main().")
     if vods_list:
         print(f"Recent VOD: '{vods_list[0]['title']}'")
 
-    clips_list = await get_streamer_clips()
-        
-    for v in clips_list:
-        db.insert_video_record(
-            v['id'], v['title'], v['created_at'], v['view_count'], v['duration'], 
-            v['url'], v['thumbnail_url'], v['game_id'], v['game_name'], 
-            v['stream_id'], v['creator_name'], v['clip_is']
-        )
-    
-    if clips_list:
-        print(f"Recent VOD: '{clips_list[0]['title']}'")
-
 if __name__ == "__main__":
     asyncio.run(main())