rearange files and updateing files

package/__init__.py

@@ -0,0 +1,35 @@
+# package/__init__.py
+import os
+
+# 1. Package Metadata
+__version__ = "1.0.0"
+__author__ = "Richard Allen"
+
+# 2. Initialization Logic
+print("Initializing package...")
+
+# Ensure a required directory exists when the package loads
+CACHE_DIR = os.path.join(os.path.expanduser("~"), ".package_cache")
+os.makedirs(CACHE_DIR, exist_ok=True)
+
+# 3. Expose Database Tools (Added)
+# Replace 'DatabaseClient' and 'connect_to_db' with your actual class/function names
+from .build_videos import main as build_videos
+from .config import CHANNEL_NAME, DB
+from .database import Database
+from .transcribe_video import transcribe_to_srt
+from .utils import run_command
+
+# 4. Expose Subpackages
+from . import twitch
+
+# (Optional) Defines what is exposed during 'from package import *'
+__all__ = [
+    "build_videos",
+    "CHANNEL_NAME",
+    "DB",
+    "Database",
+    "transcribe_to_srt", 
+    "utils", 
+    "twitch",
+]

package/build_videos.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+import youtube_short
+import package.transcribe_video as transcribe_video
+import package.twitch.chat_vod as chat_vod
+from package.config import DB as DB
+
+def build_chat_video():
+    chats = DB.get_unuploaded_chats()
+    
+    for chat in chats:
+        (
+            id,
+            title,
+            created_at,
+            view_count,
+            duration,
+            url,
+            thumbnail_url,
+            game_id,
+            game_name,
+            stream_id,
+            creator_name,
+            clip_is,
+            downloaded,
+            uploaded_yt,
+            uploaded_yt_chats,
+            uploaded_yt_shorts,
+        ) = chat
+        
+        print("====================================================")
+        print(f"🚀 Chat Video: {title}")
+        print("====================================================")
+
+        chat_vod.combine_twitch_vod_and_chat(f"download/videos/{id}/{id}.mp4", "side-by-side")
+
+    print("====================================================")
+    print(f"✅ Processing: Chat Video Done.")
+    print("====================================================")
+
+
+def build_transcribe():
+    unuploadeds = DB.get_unuploaded()
+
+    for unuploaded in unuploadeds:
+        (
+            id,
+            title,
+            created_at,
+            view_count,
+            duration,
+            url,
+            thumbnail_url,
+            game_id,
+            game_name,
+            stream_id,
+            creator_name,
+            clip_is,
+            downloaded,
+            uploaded_yt,
+            uploaded_yt_chats,
+            uploaded_yt_shorts,
+        ) = unuploaded
+
+        if clip_is:
+            target_dir = f"download/clips"
+        else:
+            target_dir = f"download/videos"
+
+        print("====================================================")
+        print(f"🚀 Transcribe: {title}")
+        print("====================================================")
+        
+        transcribe_video.transcribe_to_srt(f"{target_dir}/{id}/{id}.mp4")
+
+    print("====================================================")
+    print(f"✅ Processing: Transcribing Done.")
+    print("====================================================")
+
+
+def build_shorts():
+    shorts = DB.get_unuploaded_shorts()
+
+    top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
+
+    for short in shorts:
+        (
+            id,
+            title,
+            created_at,
+            view_count,
+            duration,
+            url,
+            thumbnail_url,
+            game_id,
+            game_name,
+            stream_id,
+            creator_name,
+            clip_is,
+            downloaded,
+            uploaded_yt,
+            uploaded_yt_chats,
+            uploaded_yt_shorts,
+        ) = short
+
+        target_dir = f"download/clips/{id}/{id}.mp4"
+
+        print("====================================================")
+        print(f"🚀 Processing: Clip to Youtube Short {title}")
+        print("====================================================")
+
+        youtube_short.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {creator_name}.")
+
+    print("====================================================")
+    print(f"✅ Processing: Clip to Youtube Short Done.")
+    print("====================================================")
+
+def main():
+    build_shorts()
+    build_transcribe()
+    build_chat_video()
+
+if __name__ == "__main__":
+    main()

package/config.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+import package.database as database
+
+CHANNEL_NAME = "teampgp"
+
+DB = database.Database()

package/database.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+import sqlite3
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+class Database:
+    def __init__(self, db_path: str = "database.db"):
+        self.columns = "id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts"
+
+        file_exists = Path(db_path).is_file()
+
+        self.conn = sqlite3.connect(db_path)
+        self.cursor = self.conn.cursor()
+
+        if not file_exists:
+            self.create_database()
+
+    def __exit__(self):
+        # Destructors are unpredictable in Python; explicitly close when done instead
+        try:
+            self.close_database()
+        except:
+            pass
+
+    def create_database(self):
+        """Creates a table structured explicitly for Twitch videos properties."""
+        self.cursor.execute("""
+        CREATE TABLE IF NOT EXISTS twitch_videos (
+            id TEXT PRIMARY KEY,
+            title TEXT NOT NULL,
+            created_at TEXT NOT NULL,
+            view_count INTEGER NOT NULL,
+            duration TEXT NOT NULL,
+            url TEXT NOT NULL,
+            thumbnail_url TEXT NOT NULL,
+            game_id TEXT NOT NULL,
+            game_name TEXT NOT NULL,
+            stream_id TEXT NOT NULL,
+            creator_name TEXT NOT NULL,
+            clip_is BOOLEAN NOT NULL DEFAULT 0,
+            downloaded BOOLEAN NOT NULL DEFAULT 0,
+            uploaded_yt BOOLEAN NOT NULL DEFAULT 0,
+            uploaded_yt_chats BOOLEAN NOT NULL DEFAULT 0,
+            uploaded_yt_shorts BOOLEAN NOT NULL DEFAULT 0
+        )
+        """)
+        
+        self.conn.commit()
+
+    def close_database(self):
+        """Commit before we close."""
+        if self.conn:
+            self.conn.commit()
+            self.conn.close()
+
+    def __mark_as(self, record_id: str, set_row: str, mark: str = "1" ):
+        self.cursor.execute(
+            f"UPDATE twitch_videos SET {set_row} = ? WHERE id = ?", 
+            (mark, record_id))
+        self.conn.commit()
+
+    def mark_as_uploaded_shorts(self, record_id: str):
+        """Flags a specific row record to uploaded_yt_shorts (1)."""
+        self.__mark_as(record_id, "uploaded_yt_shorts")
+
+    def mark_as_uploaded_chats(self, record_id: str):
+        """Flags a specific row record to uploaded_yt_chats (1)."""
+        self.__mark_as(record_id, "uploaded_yt_chats")
+
+    def mark_as_uploaded_yt(self, record_id: str):
+        """Flags a specific row record to uploaded_yt (1)."""
+        self.__mark_as(record_id, "uploaded_yt")
+
+    def mark_as_downloaded(self, record_id: str):
+        """Flags a specific row record to downloaded (1)."""
+        self.__mark_as(record_id, "downloaded")
+
+    def unmark_as_download(self, record_id: str):
+        """Flags a specific row record to downloaded (0)."""
+        self.__mark_as(record_id, "downloaded", "0")
+
+    def update_datetime(self, record_id: str, created_at: str):
+        """Updates the Datetime for timezone fix"""
+        self.cursor.execute(
+            f"UPDATE twitch_videos SET created_at = ? WHERE id = ?",
+            (created_at, record_id))
+        self.conn.commit()
+
+    def __get_unuploaded(self, set_row: str, also: str = "") -> list[Any]:
+        """Retrieve all rows that were download but not uploaded"""
+        self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE downloaded = 1 AND {set_row} = 0 {also}")
+        return self.cursor.fetchall()
+
+    def get_unuploaded_shorts(self) -> list[Any]:
+        """Retrieve all rows that were download but not uploaded_yt_shorts"""
+        return self.__get_unuploaded("uploaded_yt_shorts", "AND clip_is = 1")
+
+    def get_unuploaded_chats(self) -> list[Any]:
+        """Retrieve all rows that were download but not uploaded_yt_chats"""
+        return self.__get_unuploaded("uploaded_yt_chats", "AND clip_is = 0")
+                
+    def get_unuploaded(self) -> list[Any]:
+        """Retrieve all rows that were download but not uploaded_yt"""
+        return self.__get_unuploaded("uploaded_yt")
+
+    def get_undownloaded(self) -> list[Any]:
+        """Retrieves all rows that were not downloaded"""
+        self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE downloaded = 0")
+        return self.cursor.fetchall()
+
+    def get_download(self) -> list[Any]:
+        """Retrieves all rows that were downloaded"""
+        self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE downloaded = 1")
+        return self.cursor.fetchall()
+
+    def get_clips(self) -> list[Any]:
+        """Retrieves all rows that are clip_is"""
+        self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE clip_is = 1")
+        return self.cursor.fetchall()
+
+    def get_vods(self) -> list[Any]:
+        self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE clip_is = 0")
+        return self.cursor.fetchall()
+
+    def insert_video_record(
+        self, id: str, title: str, created_at: str, view_count: int, duration: str, 
+        url: str, thumbnail_url: str, game_id: int, game_name: str, stream_id: str, 
+        creator_name: str, clip_is: bool
+    ):
+        try:
+            dt_obj = datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
+            clean_datetime = dt_obj.strftime("%Y-%m-%d %H:%M:%S")
+        except (ValueError, TypeError):
+            clean_datetime = created_at
+
+        # Explicitly defining columns removes the security risk and column-count bug
+        query = """
+            INSERT INTO twitch_videos (
+                id, title, created_at, view_count, duration, url, thumbnail_url, 
+                game_id, game_name, stream_id, creator_name, clip_is
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT(id) DO UPDATE SET
+                title = excluded.title,
+                created_at = excluded.created_at,
+                view_count = excluded.view_count,
+                duration = excluded.duration,
+                url = excluded.url,
+                thumbnail_url = excluded.thumbnail_url,
+                game_id = excluded.game_id,
+                game_name = excluded.game_name,
+                stream_id = excluded.stream_id,
+                creator_name = excluded.creator_name,
+                clip_is = excluded.clip_is
+            WHERE excluded.duration != twitch_videos.duration
+        """
+
+        values = (
+            id, title, clean_datetime, view_count, duration, url, thumbnail_url, 
+            game_id, game_name, stream_id, creator_name, clip_is
+        )
+
+        try:
+            self.cursor.execute(query, values)
+            self.conn.commit()
+        except Exception as e:
+            # Prevent silent failures if the database connection drops
+            print(f"❌ Database insertion failed: {e}")
+            self.conn.rollback()

package/transcribe_video.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from faster_whisper import WhisperModel
+from faster_whisper.utils import format_timestamp
+
+# Prevent OpenMP thread conflicts from crashing the script
+os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
+
+# 5 minutes per chunk (300 seconds) keeps RAM usage low and stable
+CHUNK_DURATION_SEC = 300 
+
+print("🔧 Initializing C++ Engine...")
+model = WhisperModel(
+    "base", 
+    device="cpu", 
+    compute_type="int8", 
+    cpu_threads=0, # Let CTranslate2 auto-detect safe core counts
+    num_workers=1         
+)
+print("✅ C++ Model loaded successfully.")
+
+
+def extract_audio_and_chunk(video_path: str, output_dir: Path) -> list:
+    """Extracts and splits audio into 5-minute chunks using a single FFmpeg pass."""
+    print("🚀 Extracting and chunking audio with FFmpeg...")
+    output_dir.mkdir(parents=True, exist_ok=True)
+    
+    # Segment format output: chunk_000.wav, chunk_001.wav, etc.
+    chunk_pattern = str(output_dir / "chunk_%03d.wav")
+    
+    command = [
+        "ffmpeg", "-y", "-i", video_path,
+        "-vn", "-ac", "1", "-ar", "16000",
+        "-acodec", "pcm_s16le", "-sn", "-map_chapters", "-1",
+        "-f", "segment", "-segment_time", str(CHUNK_DURATION_SEC),
+        chunk_pattern
+    ]
+    
+    result = subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+    if result.returncode != 0:
+        print(f"❌ FFmpeg Error Output:\n{result.stderr}")
+        raise RuntimeError("FFmpeg extraction and chunking failed.")
+        
+    # Return sorted list of generated chunk files
+    return sorted(list(output_dir.glob("chunk_*.wav")))
+
+
+def transcribe_to_srt(video_path: str, force: bool = False):
+    video_path_obj = Path(video_path)
+    srt_path = video_path_obj.with_suffix(".srt")
+    temp_dir = video_path_obj.parent / f"temp_chunks_{video_path_obj.stem}"
+
+    if srt_path.exists():
+        if not force:
+            print(f"❌ Error: Transcription file already exists: {srt_path}")
+            return
+        else:
+            srt_path.unlink()
+
+    try:
+        # Step 1: Split audio into bite-sized pieces
+        audio_chunks = extract_audio_and_chunk(str(video_path_obj), temp_dir)
+        if not audio_chunks:
+            print("❌ Error: No audio chunks were generated.")
+            return
+
+        print(f"📦 Successfully split audio into {len(audio_chunks)} chunks.")
+        print("🎙️ Starting safe chunk-by-chunk transcription...")
+
+        global_segment_index = 1
+        
+        with open(srt_path, "w", encoding="utf-8") as srt_file:
+            for chunk_idx, chunk_path in enumerate(audio_chunks):
+                # Calculate the time offset for the current chunk
+                time_offset = chunk_idx * CHUNK_DURATION_SEC
+                print(f"\n⏳ Processing chunk {chunk_idx + 1}/{len(audio_chunks)} ({chunk_path.name})...")
+                
+                segments_generator, info = model.transcribe(
+                    str(chunk_path), 
+                    beam_size=1,
+                    vad_filter=True,
+                    temperature=0.0
+                )
+                
+                # Consume chunk generator and shift timestamps instantly
+                for segment in segments_generator:
+                    # Shift timestamps relative to the original video timeline
+                    actual_start = segment.start + time_offset
+                    actual_end = segment.end + time_offset
+                    
+                    start_str = format_timestamp(actual_start, always_include_hours=True)
+                    end_str = format_timestamp(actual_end, always_include_hours=True)
+                    
+                    srt_file.write(f"{global_segment_index}\n{start_str} --> {end_str}\n{segment.text.strip()}\n\n")
+                    global_segment_index += 1
+                    
+                # Free up space as we go by deleting the processed chunk
+                chunk_path.unlink()
+
+        print(f"\n✅ All chunks combined! SRT subtitle file saved in: {srt_path}")
+
+    except Exception as e:
+        print(f"\n❌ Execution Error: {e}")
+    finally:
+        # Clean up the temporary folder entirely
+        if temp_dir.exists():
+            shutil.rmtree(temp_dir)
+
+
+if __name__ == "__main__":
+    target_video = "download/videos/2813112936/2813112936.mp4"
+    if not os.path.exists(target_video):
+        print(f"❌ System Error: Target video file does not exist at path: {target_video}")
+    else:
+        transcribe_to_srt(target_video, force=True)

package/twitch/__init__.py

@@ -0,0 +1,15 @@
+# package/twitch/__init__.py
+
+# Expose components from individual modules using relative imports
+from .chat_vod import combine_twitch_vod_and_chat
+from .download_thumbnails import main as download_thumbnails
+from .download_videos import main as download_videos
+from .video_info import main as video_info
+
+# Explicitly define what is exposed when importing the subpackage
+__all__ = [
+    "combine_twitch_vod_and_chat",
+    "download_thumbnails",
+    "download_videos",
+    "video_info",
+]

package/twitch/chat_vod.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+import  os
+import cv2
+import package.utils as utils
+
+def get_video_height(video_path: str) -> int:
+    # Open the video file
+    video = cv2.VideoCapture(video_path)
+    
+    # Get the height property
+    height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    
+    # Always release the video object
+    video.release()
+    
+    return height
+
+def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side", force: bool = False) -> bool:
+    """
+    Renders Twitch chat JSON to video and combines it with the source VOD.
+    Provides real-time terminal feedback for all processing steps.
+    """
+    threads = 8
+
+    video_height = 1080
+
+    video_path = vod_path
+    video_chat = ""
+    chat_path = video_path.replace(".mp4", "_chat.json")
+    temp_with_chat = video_path.replace(".mp4", "_temp_with_chat.mp4")
+    temp_chat = video_path.replace(".mp4", "_temp_chat.mp4")
+    video_chat_SS = video_path.replace(".mp4", "_SS_with_chat.mp4")
+    video_chat_over = video_path.replace(".mp4", "_over_with_chat.mp4")
+    mask_path = video_path.replace(".mp4", "_temp_chat_mask.mp4")
+
+    # Step 1: Pre-flight checks
+    if not os.path.exists(video_path):
+        print(f"❌ Error: Source video not found: {video_path}")
+        return False
+    
+    if not os.path.exists(chat_path):
+        print(f"❌ Error: Chat file not found: {chat_path}")
+        return False
+
+    if os.path.exists(temp_with_chat):
+        os.remove(temp_with_chat)
+
+    if layout == "side-by-side":
+        video_chat = video_chat_SS
+    elif layout == "overlay":
+        video_chat = video_chat_over
+    else:
+        return False
+
+    if os.path.exists(video_chat):
+        if not force:
+            print(f"❌ Error: Chat video already exists: {video_chat}")
+            return False
+        else:
+            os.remove(video_chat)
+
+    if os.path.exists(temp_chat):
+        os.remove(temp_chat)
+    if os.path.exists(mask_path):
+        os.remove(mask_path)
+    
+    # Step 2: Render Chat to Video
+    print("====================================================")
+    print("🚀 STEP 1: Rendering Chat JSON to Video Layer")
+    print("====================================================")
+
+    video_height = get_video_height(vod_path)
+    
+    chat_cmd = (
+        f"TwitchDownloaderCLI chatrender "
+        f"-i {chat_path} "
+        f"-w 400 -h {video_height} "
+        f"--collision Overwrite "
+        f"--temp-path download/temp "
+        f"--font-size 20 "           
+        f"--background-color #00000000 "
+        f"-o {temp_chat}"
+    )
+
+    if layout == "overlay":
+        chat_cmd = (f"{chat_cmd} --generate-mask ")
+
+    if not os.path.exists(temp_chat):
+        chat_success = utils.run_command(chat_cmd, look_for=["[STATUS]"])
+        if chat_success:
+            print("✅ Success: TwitchDownloaderCLI render chat video.")
+        else:
+            print("❌ Error: TwitchDownloaderCLI failed to render chat video.")
+            if os.path.exists(temp_chat):
+                os.remove(temp_chat)
+            if os.path.exists(mask_path):
+                os.remove(mask_path)
+            return False
+
+    # Step 3: Combine Video and Chat using FFmpeg
+    print("====================================================")
+    print(f"🚀 STEP 2: Merging VOD and Chat Layout ({layout})")
+    print("====================================================")
+    
+    # -preset superfast speeds up the 3+ hour encoding process significantly
+    # -map 0:a? safely includes audio if it exists, without breaking on silent VODs
+    if layout == "side-by-side":
+        ffmpeg_cmd = (
+            f"ffmpeg -y -i {video_path} -i {temp_chat} "
+            f"-filter_complex '[1:v]scale=-1:ih[scaled_chat];[0:v][scaled_chat]hstack=inputs=2[v]' "
+            f"-map '[v]' -map 0:a? -c:v libx264 -crf 18 -preset slow -c:a copy "
+            f"-threads {threads} {temp_with_chat}"
+        )
+    elif layout == "overlay":
+        ffmpeg_cmd = (
+            f"ffmpeg -y -i {video_path} -i {temp_chat} -i {mask_path} "
+            f"-filter_complex '[1:v][2:v]alphamerge[masked_chat];[0:v][masked_chat]overlay=x=0:y=10[v]' "
+            f"-map '[v]' -map 0:a? -c:v libx264 -crf 18 -preset slow -c:a copy "
+            f"-threads {threads} {temp_with_chat}"
+        )
+    else:
+        raise ValueError("Invalid layout choice. Choose 'side-by-side' or 'overlay'.")
+
+    ffmpeg_success = utils.run_command(ffmpeg_cmd, progress_prefix="FFmpeg Merge")
+
+    # Step 4: Final verification and cleanup
+    if ffmpeg_success and os.path.exists(temp_with_chat):
+        os.rename(temp_with_chat, video_chat)
+        print("====================================================")
+        print(f"🎉 SUCCESS: Video processing complete!")
+        print(f"📁 Output Saved: {video_chat}")
+        print("====================================================")
+        
+        # Clean up the massive temporary chat video to save storage space
+        if os.path.exists(temp_chat):
+            print("🧹 Cleaning up temporary chat render video...")
+            os.remove(temp_chat)
+        if os.path.exists(mask_path):
+            os.remove(mask_path)
+        if os.path.exists(temp_with_chat):
+            os.remove(temp_with_chat)
+        return True
+    else:
+        print("❌ Error: FFmpeg failed to merge the video streams.")
+        if os.path.exists(video_chat):
+            os.remove(video_chat)
+        if os.path.exists(temp_with_chat):
+            os.remove(temp_with_chat)
+        return False
+
+if __name__ == "__main__":
+    combine_twitch_vod_and_chat("download/videos/2813112936/2813112936.mp4", "overlay", True)

package/twitch/config.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+import json
+
+from twitchAPI.twitch import Twitch
+from twitchAPI.helper import first
+
+from package.config import CHANNEL_NAME as CHANNEL_NAME
+
+# 1. Fill in your credentials from the Twitch Developer Console
+SECRETS = None
+TWITCH = None
+USER = None
+
+async def get_twitch():
+    global SECRETS, TWITCH, USER
+    if SECRETS is None:
+        with open('twitch_secrets.json', 'r') as f:
+            SECRETS = json.load(f)
+    if TWITCH is None:
+        TWITCH = await Twitch(SECRETS['client_id'], SECRETS['client_secret'])
+    if USER is None:
+        USER = await first(TWITCH.get_users(logins=[CHANNEL_NAME]))
+        if not USER:
+            print("User not found.")

package/twitch/download_thumbnails.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+import os
+import asyncio
+import requests
+
+from twitchAPI.helper import first
+
+import package.config as config
+import package.database as database
+
+async def download_live_thumbnail(twitch_client, streamer_username: str, w: int = 1920, h: int = 1080):
+    """Fetches and saves the live stream thumbnail for an active broadcast."""
+    print(f"🔎 Checking live status for: {streamer_username}...")
+    
+    # Query the live streams endpoint
+    stream_generator = twitch_client.get_streams(user_logins=[streamer_username])
+    stream_data = await first(stream_generator)
+    
+    if not stream_data:
+        print(f"❌ User '{streamer_username}' is offline. Live thumbnails require an active stream.")
+        return
+
+    # Twitch API live streams use the {width} and {height} format
+    raw_url = stream_data.thumbnail_url
+    clean_url = raw_url.replace('{width}', str(w)).replace('{height}', str(h))
+    
+    filename = f"live_{streamer_username}_{w}x{h}.jpg"
+    save_image(clean_url, filename)
+
+async def download_vod_thumbnail(twitch_client, vod_id: str, w: int = 1920, h: int = 1080):
+    """Fetches and saves a thumbnail from a past broadcast VOD ID."""
+    print(f"🔎 Searching for VOD ID: {vod_id}...")
+    
+    # Query the videos endpoint
+    video_generator = twitch_client.get_videos(vod_id)
+    video_data = await first(video_generator)
+
+    filename = f"download/videos/{vod_id}/{vod_id}_{w}x{h}.jpg"
+    if os.path.exists(filename):
+        return
+    
+    if not video_data:
+        print(f"❌ VOD ID {vod_id} could not be found.")
+        return
+
+    # Twitch VOD endpoints typically format string tokens as %{width} and %{height}
+    raw_url = video_data.thumbnail_url
+    if not raw_url:
+        print("❌ This VOD does not have an available thumbnail.")
+        return
+        
+    clean_url = raw_url.replace('%{width}', str(w)).replace('%{height}', str(h))
+    
+    #filename = f"vod_{vod_id}_{w}x{h}.jpg"
+    save_image(clean_url, filename)
+
+async def download_clip_thumbnail(clip_id: str, url: str):
+    print(f"🔎 Searching for Clip ID: {clip_id}...")
+           
+    filename = f"download/clips/{clip_id}/{clip_id}.jpg"
+    if os.path.exists(filename):
+        return
+           
+    save_image(url, filename, False)
+
+def save_image(url: str, filename: str, stream: bool = True):
+    """Helper function to stream image bytes directly to a file."""
+    try:
+        response = requests.get(url, stream)
+        if response.status_code == 200:
+            with open(filename, 'wb') as file:
+                if stream is True:
+                    for chunk in response.iter_content(1024):
+                        file.write(chunk)
+                else:
+                    file.write(response.content)
+            print(f"✅ Success! Saved as: {filename}")
+        else:
+            print(f"❌ Download failed. HTTP Status: {response.status_code}")
+    except Exception as e:
+        print(f"❌ An error occurred during file writing: {e}")
+
+async def main():
+    # Initialize connection & automatically authorize the App token
+    await config.get_twitch()
+    twitch = config.TWITCH # Twitch(APP_ID, APP_SECRET)
+    
+    # --- OPTION A: Download Live Thumbnail ---
+    # Target user must be streaming live right now
+    target_streamer = config.CHANNEL_NAME
+    #await download_live_thumbnail(twitch, target_streamer, 1920, 1080)
+    
+    # --- OPTION B: Download Past Broadcast VOD Thumbnail ---
+    # Extract the ID sequence from your target video link
+    # target_vod = "2145678901"
+    db = database.Database()
+    vods = db.get_vods()
+    for vod in vods:
+        (
+            id,
+            title,
+            created_at,
+            view_count,
+            duration,
+            url,
+            thumbnail_url,
+            game_id,
+            game_name,
+            stream_id,
+            creator_name,
+            clip_is,
+            downloaded,
+            uploaded_yt,
+            uploaded_yt_chats,
+            uploaded_yt_shorts,
+        ) = vod
+
+        await download_vod_thumbnail(twitch, id, 1920, 1080)
+
+    clips = db.get_clips()
+    for clip in clips:
+        (
+            id,
+            title,
+            created_at,
+            view_count,
+            duration,
+            url,
+            thumbnail_url,
+            game_id,
+            game_name,
+            stream_id,
+            creator_name,
+            clip_is,
+            downloaded,
+            uploaded_yt,
+            uploaded_yt_chats,
+            uploaded_yt_shorts,
+        ) = clip
+    
+        await download_clip_thumbnail(id, thumbnail_url)
+
+if __name__ == '__main__':
+    asyncio.run(main())

package/twitch/download_videos.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+import csv
+import subprocess
+from datetime import datetime, timedelta, timezone
+
+import package.utils as utils
+from package.config import DB as DB
+
+def write_csv(data, file_name):
+    # Open file with newline='' to prevent extra blank rows across platforms
+    with open(file_name, "w", newline="", encoding="utf-8") as file:
+        writer = csv.writer(file)
+        
+        # Write all rows at once
+        writer.writerows(data)
+
+def download():
+    """Find all undownloaded videos and clips, and download them safely."""
+    undownloaded = DB.get_undownloaded()
+
+    print("Starting downloads...")
+    for row in undownloaded:
+        # Unpack variables clearly
+        (
+            id,
+            title,
+            created_at,
+            view_count,
+            duration,
+            url,
+            thumbnail_url,
+            game_id,
+            game_name,
+            stream_id,
+            creator_name,
+            clip_is,
+            downloaded,
+            uploaded_yt,
+            uploaded_yt_chats,
+            uploaded_yt_shorts,
+        ) = row
+
+        # 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM)
+        #datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
+        data_timestamp = datetime.fromisoformat(created_at)
+
+        # 2. Get the exact current date and time
+        current_time = datetime.now(timezone.utc)
+
+        # 3. Calculate the difference between the two times
+        time_difference = current_time - data_timestamp
+
+        # 4. Check if the difference is greater than 24 hours
+        if time_difference > timedelta(hours=24):
+            print("The data is more than 24 hours old.")
+        else:
+            #lets wait 24 hours befor downloading
+            print("The data is less than 24 hours old.")
+            continue
+
+        data = [list(row)]
+        print(
+            f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}"
+        )
+
+        # 1. Define paths and isolate base directory
+        if clip_is:
+            target_dir = f"download/clips/{id}"
+            cmd = f"TwitchDownloaderCLI clipdownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --temp-path download/temp"
+        else:
+            target_dir = f"download/videos/{id}"
+            # FIXED: Removed the duplicated command string combined with '&&'
+            cmd = f"TwitchDownloaderCLI videodownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
+
+        csv_file = f"{target_dir}/{id}.csv"
+
+        # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
+        output = utils.run_command(cmd, look_for=["[STATUS]"])
+        output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
+
+        if output:
+            # Only write CSV and update database if download actually completed
+            write_csv(data, csv_file)
+            print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
+            DB.mark_as_downloaded(id)
+        else:
+            print(f"ID: {id} | Download Process failed.")
+            #print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
+            #if output.get("stderr"):
+            #    print(f"Details: {output['stderr']}")
+
+            # Clear temporary chunk clutter immediately if a VOD crashes out
+            if not clip_is:
+                print("Flushing temporary crash chunks...")
+                subprocess.run("rm -rf download/temp/*", shell=True)
+
+    print("Finished Downloading Pipeline.")
+
+def main():
+    download()
+
+if __name__ == "__main__":
+    main()

package/twitch/video_info.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+import asyncio
+import httpx  # Switched from requests to prevent async loop freezing
+
+from twitchAPI.helper import first
+from twitchAPI.type import VideoType
+
+import package.twitch.config as config
+from package.config import DB as DB
+
+GAME_CACHE = {} # Local cache dictionary to store game_id -> game_name mapping
+
+async def get_game_id_by_name(game_name: str) -> str:
+    if config.SECRETS is None:
+        await config.get_twitch()    
+    # Query the 'get_games' endpoint using the names parameter
+    # The API expects a list, even for a single game name
+    async for game in config.TWITCH.get_games(names=[game_name]):
+        print(f"Game Name: {game.name}")
+        print(f"Game ID:   {game.id}")
+        print(f"IGDB ID:   {game.igdb_id}")
+        return game.id
+
+    print(f"No game found with the exact name: '{game_name}'")
+    return None
+
+async def get_game_name_by_id(game_id: str) -> str:
+    """Helper function to fetch game names and cache them locally."""
+    if not game_id:
+        return "Unknown / No Category"
+    if game_id in GAME_CACHE:
+        return GAME_CACHE[game_id]
+    try:
+        game_generator = config.TWITCH.get_games(game_ids=[game_id])
+        game = await first(game_generator)
+        if game:
+            GAME_CACHE[game_id] = game.name
+            return game.name
+    except Exception:
+        pass
+    return "Unknown Game"
+
+async def get_vod_game_name(vod_id: str):
+    """Asynchronously query Twitch GQL endpoint for VOD game metadata."""
+    game_id = "0"
+    game_name = "Unknown Game"
+    url = "https://gql.twitch.tv/gql"
+
+    headers = {
+        "Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
+        "Content-Type": "application/json"
+    }
+
+    payload = [{
+        "operationName": "VideoMetadata",
+        "variables": {
+            "channelLogin": "",
+            "videoID": str(vod_id)
+        },
+        "extensions": {
+            "persistedQuery": {
+                "version": 1,
+                "sha256Hash": "45111672eea2e507f8ba44d101a61862f9c56b11dee09a15634cb75cb9b9084d"
+            }
+        }
+    }]
+
+    # Using httpx async client to prevent blocking the asyncio event loop
+    async with httpx.AsyncClient() as client:
+        try:
+            response = await client.post(url, headers=headers, json=payload)
+            if response.status_code == 200:
+                data = response.json()
+                video_info = data[0].get('data', {}).get('video')
+                if video_info and video_info.get('game'):
+                    game_id = str(video_info['game']['id'])
+                    game_name = video_info['game']['displayName']
+                    print(f"GQL Found: {game_name} (ID: {game_id})")
+                else:
+                    print(f"No game information found in GQL for VOD {vod_id}.")
+        except Exception as e:
+            print(f"Error fetching GQL metadata for VOD {vod_id}: {e}")
+
+    return game_id, game_name
+
+async def get_streamer_vods():
+    await config.get_twitch()
+    
+    print(f"Starting VOD extraction for {config.USER.display_name}...")
+    vod_generator = config.TWITCH.get_videos(user_id=config.USER.id, first=100, video_type=VideoType.ALL)
+ 
+    all_vods = []
+
+    async for v in vod_generator:
+        # Resolving game category safely without freezing the event loop
+        game_id, game_name = await get_vod_game_name(v.id)
+        
+        # Safely convert game_id to integer if possible, otherwise default to 0
+        try:
+            clean_game_id = game_id
+        except ValueError:
+            clean_game_id = 0
+
+        vod_data = {
+            "id": v.id,
+            "title": v.title,
+            "created_at": str(v.published_at),
+            "view_count": int(v.view_count) if v.view_count else 0,
+            "duration": v.duration,
+            "url": v.url,
+            "thumbnail_url": v.thumbnail_url,
+            "game_id": clean_game_id,
+            "game_name": game_name,
+            "stream_id": str(v.stream_id) if v.stream_id else "0",
+            "creator_name": config.CHANNEL_NAME,
+            "clip_is": False,
+        }
+        all_vods.append(vod_data)
+        print(f"Collected VOD: {v.title} | Category: {game_name} ({v.duration})")
+
+    print(f"\nFinished extracting VODs. Total gathered: {len(all_vods)}")
+    return all_vods
+
+async def get_streamer_clips():
+    await config.get_twitch()
+
+    print(f"Starting clip extraction for {config.USER.display_name}...")
+    clip_generator = config.TWITCH.get_clips(broadcaster_id=config.USER.id, first=100)
+    all_clips = []
+
+    async for c in clip_generator:
+        game_name = await get_game_name_by_id(c.game_id)
+
+        clip_data = {
+            "id": c.id,
+            "title": c.title,
+            "created_at": str(c.created_at),             
+            "view_count": int(c.view_count) if c.view_count else 0,
+            "duration": c.duration,
+            "url": c.url,
+            "thumbnail_url": c.thumbnail_url,
+            "game_id": c.game_id,
+            "game_name": game_name,
+            "stream_id": "0",
+            "creator_name": c.creator_name,
+            "clip_is": True,
+        }
+        all_clips.append(clip_data)
+        print(f"Collected clip: {c.title} | Category: {game_name} ({c.view_count} views)")
+
+    print(f"\nFinished extracting clips. Total gathered: {len(all_clips)}")
+    return all_clips
+
+async def main():
+    print("--- Script Started ---")
+    
+    vods_list = await get_streamer_vods()
+    
+    for v in vods_list:
+        DB.insert_video_record(
+            v['id'], v['title'], v['created_at'], v['view_count'], v['duration'], 
+            v['url'], v['thumbnail_url'], v['game_id'], v['game_name'], 
+            v['stream_id'], v['creator_name'], v['clip_is']
+        )
+
+    if vods_list:
+        print(f"Recent VOD: '{vods_list[0]['title']}'")
+
+    clips_list = await get_streamer_clips()
+        
+    for v in clips_list:
+        DB.insert_video_record(
+            v['id'], v['title'], v['created_at'], v['view_count'], v['duration'], 
+            v['url'], v['thumbnail_url'], v['game_id'], v['game_name'], 
+            v['stream_id'], v['creator_name'], v['clip_is']
+        )
+    
+    if clips_list:
+        print(f"Recent VOD: '{clips_list[0]['title']}'")
+
+if __name__ == "__main__":
+    asyncio.run(main())

package/utils.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+import sys
+import shlex
+import subprocess
+
+def run_command(cmd_str: str, progress_prefix: str = "Progress", look_for: list = ["frame=", "time=", "fps=", "Rendering frame"]) -> bool:
+   """Runs a system command, streams its output live, and reports errors on failure."""
+   args = shlex.split(cmd_str)
+
+   # Redirect stderr to stdout to catch all logging/progress in one stream
+   process = subprocess.Popen(
+        args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1
+    )
+   print(f"Executing: {cmd_str[:90]}...")
+
+   # Maintain a small buffer history to display context if a crash occurs
+   output_history = []
+
+   # Stream the output live to the terminal
+   while True:
+        line = process.stdout.readline()
+        if not line and process.poll() is not None:
+            break
+        if line:
+            clean_line = line.strip()
+            output_history.append(clean_line) # Keep history for error reporting
+            
+            # Keep history slim by only keeping the last 20 lines
+            if len(output_history) > 20:
+                output_history.pop(0)
+
+            # Only print updates that show progress metrics to keep terminal clean
+            if any(metric in clean_line for metric in look_for):
+                sys.stdout.write(f"\r[{progress_prefix}] {clean_line}")
+                sys.stdout.flush()
+            elif "Error" in clean_line or "failed" in clean_line:
+                print(f"\n[Alert] {clean_line}")
+            
+   print("\n") # New line after process finishes
+   
+   # Evaluate success status
+   success = (process.returncode == 0)
+   
+   if not success:
+       print(f"❌ Command failed with exit code: {process.returncode}")
+       print("--- Technical Error Details (Last 5 lines of output) ---")
+       # Print the last 5 captured lines to show the exact point of failure
+       for error_line in output_history[-5:]:
+           print(f"  > {error_line}")
+       print("---------------------------------------------------------")
+
+   return success