exparmental run_command in linux.py migrate_sql1.py

database.py

@@ -5,11 +5,10 @@ from pathlib import Path
 from typing import Any
 
 class Database:
-    def __init__(self):
+    def __init__(self, db_path: str = "database.db"):
         self.columns = "id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts"
 
-        # IMPORTANT Must check if database.db exists before connecting to it.
-        file_exists = Path("database.db").is_file()
+        file_exists = self.db_path.is_file()
 
         self.conn = sqlite3.connect("database.db")
         self.cursor = self.conn.cursor()

linux.py

@@ -1,7 +1,83 @@
 #!/usr/bin/env python3
+import sys
+import shlex
 import subprocess
 
-def run_command(command: str):
+def run_command_ffmpeg(cmd_str: str, progress_prefix: str = "Progress") -> bool:
+    """Runs a system command and streams its output live to the console."""
+    # shlex safely handles quotes and paths inside the command string
+    args = shlex.split(cmd_str)
+    
+    # Redirect stderr to stdout because FFmpeg outputs status updates to stderr
+    process = subprocess.Popen(
+        args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1
+    )
+
+    print(f"Executing: {cmd_str[:90]}...")
+    
+    # Stream the output live to the terminal
+    while True:
+        line = process.stdout.readline()
+        if not line and process.poll() is not None:
+            break
+        if line:
+            clean_line = line.strip()
+            # Only print updates that show progress metrics to keep terminal clean
+            if any(metric in clean_line for metric in ["frame=", "time=", "fps=", "Rendering frame"]):
+                sys.stdout.write(f"\r[{progress_prefix}] {clean_line}")
+                sys.stdout.flush()
+            elif "Error" in clean_line or "failed" in clean_line:
+                print(f"\n[Alert] {clean_line}")
+
+    print("\n") # New line after process finishes
+    return process.returncode == 0
+
+def run_command(command: list[str]) -> dict:
+    """Executes a command, streams stdout only when the line changes, and detects OOM."""
+    try:
+        process = subprocess.Popen(
+            command,
+            shell=False,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,
+        )
+
+        last_line = None  # Track the previous line
+
+        if process.stdout:
+            for line in process.stdout:
+                if line != last_line:  # Only print if the content changed
+                    print(line, end="")
+                    sys.stdout.flush()
+                    last_line = line  # Update the tracking variable
+
+        _, stderr = process.communicate()
+
+        if process.returncode != 0:
+            if process.returncode in [137, -9]:
+                return {
+                    "success": False,
+                    "error": "Process was KILLED by the Linux kernel (Out of Memory).",
+                    "stderr": stderr,
+                }
+            return {
+                "success": False,
+                "error": f"Exit code {process.returncode}",
+                "stderr": stderr,
+            }
+
+        return {"success": True, "error": "", "stderr": stderr}
+
+    except Exception as e:
+        return {"success": False, "error": str(e), "stderr": ""}
+
+def run_command_old(command: str):
     """Executes a Linux command, waits for completion, and returns output."""
     try:
         # shell=True allows running full command strings with pipes/wildcards

migrate_sql1.py

@@ -0,0 +1,66 @@
+import sqlite3
+
+# Connect to your database file
+db_name = "database.db"  # Change to your actual file name
+conn = sqlite3.connect(db_name)
+cursor = conn.cursor()
+
+try:
+  # 1. Create a temporary table with the new text-based game_id
+  cursor.execute(
+      """
+        CREATE TABLE IF NOT EXISTS twitch_videos_temp (
+            id TEXT PRIMARY KEY,
+            title TEXT NOT NULL,
+            created_at TEXT NOT NULL,
+            view_count INTEGER NOT NULL,
+            duration TEXT NOT NULL,
+            url TEXT NOT NULL,
+            thumbnail_url TEXT NOT NULL,
+            game_id TEXT NOT NULL,
+            game_name TEXT NOT NULL,
+            stream_id TEXT NOT NULL,
+            creator_name TEXT NOT NULL,
+            clip_is BOOLEAN NOT NULL DEFAULT 0,
+            downloaded BOOLEAN NOT NULL DEFAULT 0,
+            uploaded_yt BOOLEAN NOT NULL DEFAULT 0,
+            uploaded_yt_chats BOOLEAN NOT NULL DEFAULT 0,
+            uploaded_yt_shorts BOOLEAN NOT NULL DEFAULT 0
+        )
+    """
+  )
+
+  # 2. Copy and convert data to the temporary table
+  cursor.execute(
+      """
+        INSERT INTO twitch_videos_temp (
+            id, title, created_at, view_count, duration, url, thumbnail_url, 
+            game_id, game_name, stream_id, creator_name, clip_is, downloaded, 
+            uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts
+        )
+        SELECT 
+            id, title, created_at, view_count, duration, url, thumbnail_url, 
+            CAST(game_id AS TEXT), game_name, stream_id, creator_name, clip_is, downloaded, 
+            uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts
+        FROM twitch_videos
+    """
+  )
+
+  # 3. Drop the old table configuration
+  cursor.execute("DROP TABLE twitch_videos")
+
+  # 4. Rename the temporary table to your exact original table name
+  cursor.execute("ALTER TABLE twitch_videos_temp RENAME TO twitch_videos")
+
+  # Commit changes if everything succeeded
+  conn.commit()
+  print("Migration successful! 'twitch_videos' table updated.")
+
+except sqlite3.Error as e:
+  # Roll back changes if an error occurs
+  conn.rollback()
+  print(f"An error occurred: {e}")
+
+finally:
+  # Close database connection
+  conn.close()

twitch_chat_vod.py

@@ -0,0 +1,86 @@
+import  os
+
+import linux
+
+def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side") -> bool:
+    """
+    Renders Twitch chat JSON to video and combines it with the source VOD.
+    Provides real-time terminal feedback for all processing steps.
+    """
+    video_path = vod_path
+    chat_path = video_path.replace(".mp4", "_chat.json")
+    temp_chat = video_path.replace(".mp4", "_temp_chat.mp4")
+    video_chat = video_path.replace(".mp4", "_with_chat.mp4")
+    mask_path = video_chat.replace(".mp4", "_mask.mp4")
+
+    # Step 1: Pre-flight checks
+    if not os.path.exists(video_path):
+        print(f"❌ Error: Source video not found: {video_path}")
+        return False
+
+    if not os.path.exists(chat_path):
+        print(f"❌ Error: Chat file not found: {chat_path}")
+        return False
+
+    # Step 2: Render Chat to Video
+    print("====================================================")
+    print("🚀 STEP 1: Rendering Chat JSON to Video Layer")
+    print("====================================================")
+    
+    chat_cmd = (
+        f"TwitchDownloaderCLI chatrender "
+        f"-i {chat_path} "
+        f"-w 400 -h 1080 "
+        f"--collision Overwrite "
+        f"--temp-path download/temp "
+        f"--font-size 25 "
+        f"--generate-mask "
+        f"-o {temp_chat}"
+        f'--output-args="-threads 8 "'
+    )
+    
+    chat_success = linux.run_command(chat_cmd, progress_prefix="Chat Render")
+    if not chat_success:
+        print("❌ Error: TwitchDownloaderCLI failed to render chat video.")
+        return False
+
+    # Step 3: Combine Video and Chat using FFmpeg
+    print("====================================================")
+    print(f"🚀 STEP 2: Merging VOD and Chat Layout ({layout})")
+    print("====================================================")
+    
+    # -preset superfast speeds up the 3+ hour encoding process significantly
+    # -map 0:a? safely includes audio if it exists, without breaking on silent VODs
+    if layout == "side-by-side":
+        ffmpeg_cmd = (
+            f"ffmpeg -y -i {video_path} -i {temp_chat} "
+            f"-filter_complex '[0:v][1:v]hstack=inputs=2[v]' "
+            f"-map '[v]' -map 0:a? -c:a copy -preset superfast {video_chat}"
+        )
+    elif layout == "overlay":
+        ffmpeg_cmd = (
+            f"ffmpeg -y -i {video_path} -i {temp_chat} -i {mask_path} "
+            f"-filter_complex '[1:v][2:v]alphamerge[masked_chat];"
+            f"[0:v][masked_chat]overlay=x=10:y=10[v]' "
+            f"-map '[v]' -map 0:a? -c:a copy -preset superfast {video_chat}"
+        )
+    else:
+        raise ValueError("Invalid layout choice. Choose 'side-by-side' or 'overlay'.")
+
+    ffmpeg_success = linux.run_command_ffmpeg(ffmpeg_cmd, progress_prefix="FFmpeg Merge")
+
+    # Step 4: Final verification and cleanup
+    if ffmpeg_success and os.path.exists(video_chat):
+        print("====================================================")
+        print(f"🎉 SUCCESS: Video processing complete!")
+        print(f"📁 Output Saved: {video_chat}")
+        print("====================================================")
+        
+        # Clean up the massive temporary chat video to save storage space
+        if os.path.exists(temp_chat):
+            print("🧹 Cleaning up temporary chat render video...")
+            #os.remove(temp_chat)
+        return True
+    else:
+        print("❌ Error: FFmpeg failed to merge the video streams.")
+        return False

twitch_download_videos.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python3
-import requests
-import os
-import time
 import csv
+import subprocess
 
 import linux
 from database import Database
@@ -19,6 +17,70 @@ def write_csv(data, file_name):
         writer.writerows(data)
 
 def download():
+    """Find all undownloaded videos and clips, and download them safely."""
+    undownloaded = DB.get_undownloaded()
+
+    print("Starting downloads...")
+    for row in undownloaded:
+        # Unpack variables clearly
+        (
+            id,
+            title,
+            created_at,
+            view_count,
+            duration,
+            url,
+            thumbnail_url,
+            game_id,
+            game_name,
+            stream_id,
+            creator_name,
+            clip_is,
+            downloaded,
+            uploaded_yt,
+            uploaded_yt_chats,
+            uploaded_yt_shorts,
+        ) = row
+
+        data = [list(row)]
+        print(
+            f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}"
+        )
+
+        # 1. Define paths and isolate base directory
+        if clip_is:
+            target_dir = f"download/clips/{id}"
+            cmd = f"TwitchDownloaderCLI clipdownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --temp-path download/temp"
+        else:
+            target_dir = f"download/videos/{id}"
+            # FIXED: Removed the duplicated command string combined with '&&'
+            cmd = f"TwitchDownloaderCLI videodownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
+
+        csv_file = f"{target_dir}/{id}.csv"
+
+        # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
+        output = linux.run_command_streaming(cmd)
+        output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp")
+
+        if output["success"]:
+            # Only write CSV and update database if download actually completed
+            write_csv(data, csv_file)
+            print(f"ID: {id} | Download was successful.")
+            DB.mark_as_downloaded(id)
+        else:
+            print(f"ID: {id} | Download Process failed.")
+            print(f"Reason/Error: {output.get('error', 'Unknown Error')}")
+            if output.get("stderr"):
+                print(f"Details: {output['stderr']}")
+
+            # Clear temporary chunk clutter immediately if a VOD crashes out
+            if not clip_is:
+                print("Flushing temporary crash chunks...")
+                subprocess.run("rm -rf download/temp/*", shell=True)
+
+    print("Finished Downloading Pipeline.")
+
+def download_old():
     """Find all undownload videos and download them."""
     undownloaded = DB.get_undownloaded()
 
@@ -27,15 +89,13 @@ def download():
         data = [[id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts]]
         print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}")
         if not clip_is:
-            output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {id} -o download/videos/{id}/{id}.mp4 --collision Overwrite")
-            output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o download/videos/{id}/{id}_chat.json -E --collision Overwrite")
+            output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {id} -o download/videos/{id}/{id}.mp4 --collision Overwrite --temp-path download/temp")
+            output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o download/videos/{id}/{id}_chat.json -E --collision Overwrite --temp-path download/temp")
             write_csv(data, f"download/videos/{id}/{id}.csv")
         elif clip_is:
             output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {id} -o download/clips/{id}/{id}.mp4 --collision Overwrite")
             write_csv(data, f"download/clips/{id}/{id}.csv")
 
-        time.sleep(1)
-
         if output['success'] is True:
             print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title} | was successful")
             DB.mark_as_downloaded(id)

twitch_video_info.py

@@ -99,7 +99,7 @@ async def get_streamer_vods():
         
         # Safely convert game_id to integer if possible, otherwise default to 0
         try:
-            clean_game_id = int(game_id)
+            clean_game_id = game_id
         except ValueError:
             clean_game_id = 0
 
@@ -133,11 +133,6 @@ async def get_streamer_clips():
     async for c in clip_generator:
         game_name = await get_game_name_by_id(c.game_id)
 
-        try:
-            clean_game_id = int(c.game_id)
-        except (ValueError, TypeError):
-            clean_game_id = 0
-
         clip_data = {
             "id": c.id,
             "title": c.title,
@@ -146,7 +141,7 @@ async def get_streamer_clips():
             "duration": c.duration,
             "url": c.url,
             "thumbnail_url": c.thumbnail_url,
-            "game_id": clean_game_id,
+            "game_id": c.game_id,
             "game_name": game_name,
             "stream_id": "0",
             "creator_name": c.creator_name,