From 9203b0f88c35ebdf45dbdcdbaa3e225474697db4 Mon Sep 17 00:00:00 2001 From: SumGuyV5 Date: Tue, 28 Jul 2026 03:27:26 +0000 Subject: [PATCH] exparmental run_command in linux.py migrate_sql1.py --- database.py | 5 +-- linux.py | 78 ++++++++++++++++++++++++++++++++++- migrate_sql1.py | 66 ++++++++++++++++++++++++++++++ twitch_chat_vod.py | 86 +++++++++++++++++++++++++++++++++++++++ twitch_download_videos.py | 74 +++++++++++++++++++++++++++++---- twitch_video_info.py | 9 +--- 6 files changed, 300 insertions(+), 18 deletions(-) create mode 100644 migrate_sql1.py create mode 100644 twitch_chat_vod.py diff --git a/database.py b/database.py index 6d07697..7d15e2d 100755 --- a/database.py +++ b/database.py @@ -5,11 +5,10 @@ from pathlib import Path from typing import Any class Database: - def __init__(self): + def __init__(self, db_path: str = "database.db"): self.columns = "id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts" - # IMPORTANT Must check if database.db exists before connecting to it. - file_exists = Path("database.db").is_file() + file_exists = self.db_path.is_file() self.conn = sqlite3.connect("database.db") self.cursor = self.conn.cursor() diff --git a/linux.py b/linux.py index 108778a..191668b 100755 --- a/linux.py +++ b/linux.py @@ -1,7 +1,83 @@ #!/usr/bin/env python3 +import sys +import shlex import subprocess -def run_command(command: str): +def run_command_ffmpeg(cmd_str: str, progress_prefix: str = "Progress") -> bool: + """Runs a system command and streams its output live to the console.""" + # shlex safely handles quotes and paths inside the command string + args = shlex.split(cmd_str) + + # Redirect stderr to stdout because FFmpeg outputs status updates to stderr + process = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1 + ) + + print(f"Executing: {cmd_str[:90]}...") + + # Stream the output live to the terminal + while True: + line = process.stdout.readline() + if not line and process.poll() is not None: + break + if line: + clean_line = line.strip() + # Only print updates that show progress metrics to keep terminal clean + if any(metric in clean_line for metric in ["frame=", "time=", "fps=", "Rendering frame"]): + sys.stdout.write(f"\r[{progress_prefix}] {clean_line}") + sys.stdout.flush() + elif "Error" in clean_line or "failed" in clean_line: + print(f"\n[Alert] {clean_line}") + + print("\n") # New line after process finishes + return process.returncode == 0 + +def run_command(command: list[str]) -> dict: + """Executes a command, streams stdout only when the line changes, and detects OOM.""" + try: + process = subprocess.Popen( + command, + shell=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + ) + + last_line = None # Track the previous line + + if process.stdout: + for line in process.stdout: + if line != last_line: # Only print if the content changed + print(line, end="") + sys.stdout.flush() + last_line = line # Update the tracking variable + + _, stderr = process.communicate() + + if process.returncode != 0: + if process.returncode in [137, -9]: + return { + "success": False, + "error": "Process was KILLED by the Linux kernel (Out of Memory).", + "stderr": stderr, + } + return { + "success": False, + "error": f"Exit code {process.returncode}", + "stderr": stderr, + } + + return {"success": True, "error": "", "stderr": stderr} + + except Exception as e: + return {"success": False, "error": str(e), "stderr": ""} + +def run_command_old(command: str): """Executes a Linux command, waits for completion, and returns output.""" try: # shell=True allows running full command strings with pipes/wildcards diff --git a/migrate_sql1.py b/migrate_sql1.py new file mode 100644 index 0000000..d77f9c2 --- /dev/null +++ b/migrate_sql1.py @@ -0,0 +1,66 @@ +import sqlite3 + +# Connect to your database file +db_name = "database.db" # Change to your actual file name +conn = sqlite3.connect(db_name) +cursor = conn.cursor() + +try: + # 1. Create a temporary table with the new text-based game_id + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS twitch_videos_temp ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + created_at TEXT NOT NULL, + view_count INTEGER NOT NULL, + duration TEXT NOT NULL, + url TEXT NOT NULL, + thumbnail_url TEXT NOT NULL, + game_id TEXT NOT NULL, + game_name TEXT NOT NULL, + stream_id TEXT NOT NULL, + creator_name TEXT NOT NULL, + clip_is BOOLEAN NOT NULL DEFAULT 0, + downloaded BOOLEAN NOT NULL DEFAULT 0, + uploaded_yt BOOLEAN NOT NULL DEFAULT 0, + uploaded_yt_chats BOOLEAN NOT NULL DEFAULT 0, + uploaded_yt_shorts BOOLEAN NOT NULL DEFAULT 0 + ) + """ + ) + + # 2. Copy and convert data to the temporary table + cursor.execute( + """ + INSERT INTO twitch_videos_temp ( + id, title, created_at, view_count, duration, url, thumbnail_url, + game_id, game_name, stream_id, creator_name, clip_is, downloaded, + uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts + ) + SELECT + id, title, created_at, view_count, duration, url, thumbnail_url, + CAST(game_id AS TEXT), game_name, stream_id, creator_name, clip_is, downloaded, + uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts + FROM twitch_videos + """ + ) + + # 3. Drop the old table configuration + cursor.execute("DROP TABLE twitch_videos") + + # 4. Rename the temporary table to your exact original table name + cursor.execute("ALTER TABLE twitch_videos_temp RENAME TO twitch_videos") + + # Commit changes if everything succeeded + conn.commit() + print("Migration successful! 'twitch_videos' table updated.") + +except sqlite3.Error as e: + # Roll back changes if an error occurs + conn.rollback() + print(f"An error occurred: {e}") + +finally: + # Close database connection + conn.close() diff --git a/twitch_chat_vod.py b/twitch_chat_vod.py new file mode 100644 index 0000000..723f952 --- /dev/null +++ b/twitch_chat_vod.py @@ -0,0 +1,86 @@ +import os + +import linux + +def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side") -> bool: + """ + Renders Twitch chat JSON to video and combines it with the source VOD. + Provides real-time terminal feedback for all processing steps. + """ + video_path = vod_path + chat_path = video_path.replace(".mp4", "_chat.json") + temp_chat = video_path.replace(".mp4", "_temp_chat.mp4") + video_chat = video_path.replace(".mp4", "_with_chat.mp4") + mask_path = video_chat.replace(".mp4", "_mask.mp4") + + # Step 1: Pre-flight checks + if not os.path.exists(video_path): + print(f"โŒ Error: Source video not found: {video_path}") + return False + + if not os.path.exists(chat_path): + print(f"โŒ Error: Chat file not found: {chat_path}") + return False + + # Step 2: Render Chat to Video + print("====================================================") + print("๐Ÿš€ STEP 1: Rendering Chat JSON to Video Layer") + print("====================================================") + + chat_cmd = ( + f"TwitchDownloaderCLI chatrender " + f"-i {chat_path} " + f"-w 400 -h 1080 " + f"--collision Overwrite " + f"--temp-path download/temp " + f"--font-size 25 " + f"--generate-mask " + f"-o {temp_chat}" + f'--output-args="-threads 8 "' + ) + + chat_success = linux.run_command(chat_cmd, progress_prefix="Chat Render") + if not chat_success: + print("โŒ Error: TwitchDownloaderCLI failed to render chat video.") + return False + + # Step 3: Combine Video and Chat using FFmpeg + print("====================================================") + print(f"๐Ÿš€ STEP 2: Merging VOD and Chat Layout ({layout})") + print("====================================================") + + # -preset superfast speeds up the 3+ hour encoding process significantly + # -map 0:a? safely includes audio if it exists, without breaking on silent VODs + if layout == "side-by-side": + ffmpeg_cmd = ( + f"ffmpeg -y -i {video_path} -i {temp_chat} " + f"-filter_complex '[0:v][1:v]hstack=inputs=2[v]' " + f"-map '[v]' -map 0:a? -c:a copy -preset superfast {video_chat}" + ) + elif layout == "overlay": + ffmpeg_cmd = ( + f"ffmpeg -y -i {video_path} -i {temp_chat} -i {mask_path} " + f"-filter_complex '[1:v][2:v]alphamerge[masked_chat];" + f"[0:v][masked_chat]overlay=x=10:y=10[v]' " + f"-map '[v]' -map 0:a? -c:a copy -preset superfast {video_chat}" + ) + else: + raise ValueError("Invalid layout choice. Choose 'side-by-side' or 'overlay'.") + + ffmpeg_success = linux.run_command_ffmpeg(ffmpeg_cmd, progress_prefix="FFmpeg Merge") + + # Step 4: Final verification and cleanup + if ffmpeg_success and os.path.exists(video_chat): + print("====================================================") + print(f"๐ŸŽ‰ SUCCESS: Video processing complete!") + print(f"๐Ÿ“ Output Saved: {video_chat}") + print("====================================================") + + # Clean up the massive temporary chat video to save storage space + if os.path.exists(temp_chat): + print("๐Ÿงน Cleaning up temporary chat render video...") + #os.remove(temp_chat) + return True + else: + print("โŒ Error: FFmpeg failed to merge the video streams.") + return False \ No newline at end of file diff --git a/twitch_download_videos.py b/twitch_download_videos.py index 6d8ce7e..468779c 100755 --- a/twitch_download_videos.py +++ b/twitch_download_videos.py @@ -1,8 +1,6 @@ #!/usr/bin/env python3 -import requests -import os -import time import csv +import subprocess import linux from database import Database @@ -19,6 +17,70 @@ def write_csv(data, file_name): writer.writerows(data) def download(): + """Find all undownloaded videos and clips, and download them safely.""" + undownloaded = DB.get_undownloaded() + + print("Starting downloads...") + for row in undownloaded: + # Unpack variables clearly + ( + id, + title, + created_at, + view_count, + duration, + url, + thumbnail_url, + game_id, + game_name, + stream_id, + creator_name, + clip_is, + downloaded, + uploaded_yt, + uploaded_yt_chats, + uploaded_yt_shorts, + ) = row + + data = [list(row)] + print( + f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}" + ) + + # 1. Define paths and isolate base directory + if clip_is: + target_dir = f"download/clips/{id}" + cmd = f"TwitchDownloaderCLI clipdownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --temp-path download/temp" + else: + target_dir = f"download/videos/{id}" + # FIXED: Removed the duplicated command string combined with '&&' + cmd = f"TwitchDownloaderCLI videodownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp" + + csv_file = f"{target_dir}/{id}.csv" + + # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes + output = linux.run_command_streaming(cmd) + output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp") + + if output["success"]: + # Only write CSV and update database if download actually completed + write_csv(data, csv_file) + print(f"ID: {id} | Download was successful.") + DB.mark_as_downloaded(id) + else: + print(f"ID: {id} | Download Process failed.") + print(f"Reason/Error: {output.get('error', 'Unknown Error')}") + if output.get("stderr"): + print(f"Details: {output['stderr']}") + + # Clear temporary chunk clutter immediately if a VOD crashes out + if not clip_is: + print("Flushing temporary crash chunks...") + subprocess.run("rm -rf download/temp/*", shell=True) + + print("Finished Downloading Pipeline.") + +def download_old(): """Find all undownload videos and download them.""" undownloaded = DB.get_undownloaded() @@ -27,15 +89,13 @@ def download(): data = [[id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts]] print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}") if not clip_is: - output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {id} -o download/videos/{id}/{id}.mp4 --collision Overwrite") - output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o download/videos/{id}/{id}_chat.json -E --collision Overwrite") + output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {id} -o download/videos/{id}/{id}.mp4 --collision Overwrite --temp-path download/temp") + output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o download/videos/{id}/{id}_chat.json -E --collision Overwrite --temp-path download/temp") write_csv(data, f"download/videos/{id}/{id}.csv") elif clip_is: output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {id} -o download/clips/{id}/{id}.mp4 --collision Overwrite") write_csv(data, f"download/clips/{id}/{id}.csv") - time.sleep(1) - if output['success'] is True: print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title} | was successful") DB.mark_as_downloaded(id) diff --git a/twitch_video_info.py b/twitch_video_info.py index 7ca297d..b0a73f2 100755 --- a/twitch_video_info.py +++ b/twitch_video_info.py @@ -99,7 +99,7 @@ async def get_streamer_vods(): # Safely convert game_id to integer if possible, otherwise default to 0 try: - clean_game_id = int(game_id) + clean_game_id = game_id except ValueError: clean_game_id = 0 @@ -132,11 +132,6 @@ async def get_streamer_clips(): async for c in clip_generator: game_name = await get_game_name_by_id(c.game_id) - - try: - clean_game_id = int(c.game_id) - except (ValueError, TypeError): - clean_game_id = 0 clip_data = { "id": c.id, @@ -146,7 +141,7 @@ async def get_streamer_clips(): "duration": c.duration, "url": c.url, "thumbnail_url": c.thumbnail_url, - "game_id": clean_game_id, + "game_id": c.game_id, "game_name": game_name, "stream_id": "0", "creator_name": c.creator_name,