From c4f501883398c3282c3a049d57908cc4f173360f Mon Sep 17 00:00:00 2001 From: SumGuyV5 Date: Mon, 21 Sep 2026 18:51:13 +0000 Subject: [PATCH] I forgot what I chanagerd --- delete/twitch_down_uploader.py | 4 +- delete/twitch_download_clips.py | 4 +- help.sqlite3-query | 6 +- helper_files/chapters.py | 71 ++++++ helper_files/combine_videos.py | 129 +++++++++++ helper_files/datetime_fix.py | 2 +- helper_files/fix_download.py | 2 +- helper_files/import_vod.py | 2 +- helper_files/migrate_sql2.py | 45 ++++ helper_files/move_vod.py | 2 +- helper_files/redownload_clips.py | 2 +- helper_files/youtube_description_updater.py | 225 ++++++++++++++++++++ package/__init__.py | 4 +- package/config.py | 4 +- package/database/__init__.py | 5 +- package/database/database.py | 98 +++++++-- package/process_uploads.py | 17 +- package/twitch/download_thumbnails.py | 2 +- package/twitch/download_videos.py | 89 ++++---- package/utils.py | 42 +++- 20 files changed, 678 insertions(+), 77 deletions(-) create mode 100644 helper_files/chapters.py create mode 100644 helper_files/combine_videos.py create mode 100644 helper_files/migrate_sql2.py create mode 100644 helper_files/youtube_description_updater.py diff --git a/delete/twitch_down_uploader.py b/delete/twitch_down_uploader.py index e8cc2ea..fb4cbc9 100755 --- a/delete/twitch_down_uploader.py +++ b/delete/twitch_down_uploader.py @@ -4,7 +4,7 @@ import os import package.utils as linux import time -from package.database import Database +from package.database import Repository import package.youtube.uploader as uploader from package.youtube.uploader import CategoryId @@ -319,7 +319,7 @@ def get_vod_ids_simplified(channel_name: str): if __name__ == "__main__": tables = ["clips"] for table in tables: - DB = Database(table) + DB = Repository(table) get_vod_ids_simplified(CHANNEL_NAME) #download() diff --git a/delete/twitch_download_clips.py b/delete/twitch_download_clips.py index 662f557..9eba318 100644 --- a/delete/twitch_download_clips.py +++ b/delete/twitch_download_clips.py @@ -3,13 +3,13 @@ import requests import subprocess from datetime import datetime -from package.database import Database +from package.database import Repository import package.youtube.uploader as uploader from package.youtube.uploader import CategoryId CHANNEL_NAME = 'teampgp' # Replace with the streamer's username -DB = Database("clips") +DB = Repository("clips") def run_linux_command(command: str): """Executes a Linux command, waits for completion, and returns output.""" diff --git a/help.sqlite3-query b/help.sqlite3-query index c81b372..d629142 100644 --- a/help.sqlite3-query +++ b/help.sqlite3-query @@ -1,9 +1,11 @@ -- database: ./database.db -SELECT * FROM twitch_videos WHERE clip_is == "1" and uploaded_yt_shorts == "1"; +SELECT * FROM twitch_videos WHERE clip_is == "1"; UPDATE twitch_videos SET uploaded_yt_shorts = 0 WHERE uploaded_yt_shorts == "1"; SELECT * FROM twitch_videos WHERE title == "Oh Dear"; SELECT * FROM twitch_videos WHERE title = 'Kreestuh FINALLY hits her "Ryan Dark Story" limit. Jesus Big K it''s about time!'; SELECT * FROM twitch_videos WHERE title = "Bolt-Action Bureaucracy"; SELECT * FROM twitch_videos WHERE title = "Just after going through a mass starvation..."; -SELECT * FROM twitch_videos WHERE title = "Semi-Automatic Sanctimony"; \ No newline at end of file +SELECT * FROM twitch_videos WHERE title = "Semi-Automatic Sanctimony"; +SELECT * FROM twitch_videos WHERE id = "2848295283"; +SELECT * FROM twitch_videos WHERE game_name = "Darkwood"; \ No newline at end of file diff --git a/helper_files/chapters.py b/helper_files/chapters.py new file mode 100644 index 0000000..90f97c5 --- /dev/null +++ b/helper_files/chapters.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +import os +import json +import sys + +# Finds the absolute path to 'project_directory' and adds it to Python's search path +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) +sys.path.append(PROJECT_ROOT) + +from package.config import DB as DB +from package.database import Chapter + +def get_chapters(json_path): + """Loads the JSON file and returns all chapters.""" + with open(json_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + # The chapters are under video -> chapters + video = data.get('video', {}) + chapters = video.get('chapters', []) + + if not chapters: + print("No chapters found.") + return [] + + print(f"Found {len(chapters)} chapter(s) in video '{video.get('title')}':\n") + + for i, chapter in enumerate(chapters, 1): + start_sec = chapter.get('startMilliseconds', 0) / 1000 + length_sec = chapter.get('lengthMilliseconds', 0) / 1000 + + print(f"{i}. {chapter.get('description', 'No Title')}") + print(f" - Type: {chapter.get('type')}") + print(f" - Start: {start_sec}s ({chapter.get('startMilliseconds')}ms)") + print(f" - Length: {length_sec}s ({chapter.get('lengthMilliseconds')}ms)") + print(f" - Game: {chapter.get('gameDisplayName')} (ID: {chapter.get('gameId')})") + print() + + return chapters + +def main(): + downloads = DB.get_downloaded() + + for download in downloads: + json_path = f"download/clips/{download.id}/{download.id}_chat.json" if download.clip_is else f"download/videos/{download.id}/{download.id}_chat.json" + if os.path.exists(json_path): + chapters = get_chapters(json_path) + for chapter in chapters: + chapter_record = Chapter( + twitch_video_id=download.id, + start_milliseconds=chapter.get('startMilliseconds'), + length_milliseconds=chapter.get('lengthMilliseconds'), + description=chapter.get('description'), + type=chapter.get('type'), + game_display_name=chapter.get('gameDisplayName'), + game_id=chapter.get('gameId') + ) + DB.insert(chapter_record) + else: + print(f"JSON file not found for video {download.id}: {json_path}") + +if __name__ == "__main__": + main() + # Usage: python get_chapters.py your_file.json + #path = sys.argv[1] if len(sys.argv) > 1 else "/home/richard/python_scripts/download/videos/2873107835/2873107835_chat.json" + #try: + # get_chapters(path) + #except FileNotFoundError: + # print(f"Error: File not found -> {path}") + #except json.JSONDecodeError: + # print(f"Error: File is not valid JSON -> {path}") \ No newline at end of file diff --git a/helper_files/combine_videos.py b/helper_files/combine_videos.py new file mode 100644 index 0000000..f2a981b --- /dev/null +++ b/helper_files/combine_videos.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Combine multiple video files into one file using FFmpeg. + +Example: + python helper_files/combine_videos.py output.mp4 video1.mp4 video2.mp4 video3.mp4 + python helper_files/combine_videos.py --output merged.mp4 --inputs video1.mp4 video2.mp4 +""" + +import argparse +import os +import subprocess +import sys +from pathlib import Path + + +def build_concat_list(input_files, list_path): + with open(list_path, "w", encoding="utf-8") as f: + for path in input_files: + f.write(f"file '{os.path.abspath(path)}'\n") + + +def run_ffmpeg(cmd): + print("Running:", " ".join(cmd)) + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(result.stdout) + print(result.stderr) + raise RuntimeError(f"FFmpeg failed with exit code {result.returncode}") + print("✅ Finished successfully.") + + +def concat_videos_copy(input_files, output_path): + list_path = Path(output_path).with_suffix(".concat.txt") + build_concat_list(input_files, list_path) + + cmd = [ + "ffmpeg", + "-y", + "-f", "concat", + "-safe", "0", + "-i", str(list_path), + "-c", "copy", + str(output_path), + ] + run_ffmpeg(cmd) + list_path.unlink(missing_ok=True) + + +def concat_videos_reencode(input_files, output_path): + inputs = [] + filter_parts = [] + audio_parts = [] + + for i, video_path in enumerate(input_files): + inputs.extend(["-i", str(video_path)]) + filter_parts.append(f"[{i}:v]scale=trunc(iw/2)*2:trunc(ih/2)*2,setpts=PTS-STARTPTS[v{i}]") + audio_parts.append(f"[{i}:a]aresample=async=1[a{i}]") + + concat_filter = ",".join(filter_parts + audio_parts) + map_parts = [] + for i in range(len(input_files)): + map_parts.append(f"[v{i}]") + map_parts.append(f"[a{i}]") + + # Join the video and audio streams in order and concatenate them. + # This works well when the clips are similar enough in format. + filter_complex = f"{','.join(filter_parts)};{';'.join(audio_parts)};" + filter_complex += "".join(f"[{i}:v][{i}:a]" for i in range(len(input_files))) + filter_complex += f"concat=n={len(input_files)}:v=1:a=1[outv][outa]" + + cmd = [ + "ffmpeg", + "-y", + *inputs, + "-filter_complex", + filter_complex, + "-map", + "[outv]", + "-map", + "[outa]", + "-c:v", + "libx264", + "-c:a", + "aac", + "-movflags", + "+faststart", + str(output_path), + ] + run_ffmpeg(cmd) + + +def merge_videos(input_files, output_path): + input_files = [str(Path(p).expanduser().resolve()) for p in input_files] + output_path = str(Path(output_path).expanduser().resolve()) + + if len(input_files) < 2: + raise ValueError("You need at least 2 video files to merge.") + + for path in input_files: + if not os.path.exists(path): + raise FileNotFoundError(f"File not found: {path}") + + output_dir = Path(output_path).parent + output_dir.mkdir(parents=True, exist_ok=True) + + try: + concat_videos_copy(input_files, output_path) + except Exception as exc: + print(f"Fast copy method failed: {exc}") + print("Retrying with re-encode fallback...") + concat_videos_reencode(input_files, output_path) + + +def main(): + parser = argparse.ArgumentParser(description="Join two or more video files together with FFmpeg.") + parser.add_argument("output_file", help="Path to the merged output video") + parser.add_argument("input_files", nargs="+", help="Two or more videos to concatenate") + args = parser.parse_args() + + try: + merge_videos(args.input_files, args.output_file) + print(f"Merged video saved to: {args.output_file}") + except Exception as exc: + print(f"Error: {exc}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/helper_files/datetime_fix.py b/helper_files/datetime_fix.py index 1b1df86..75d91e2 100644 --- a/helper_files/datetime_fix.py +++ b/helper_files/datetime_fix.py @@ -55,7 +55,7 @@ def fix_time(): def main(): global DB - DB = database.Database() + DB = database.Repository() fix_time() diff --git a/helper_files/fix_download.py b/helper_files/fix_download.py index 0b0ea3c..caf6ede 100644 --- a/helper_files/fix_download.py +++ b/helper_files/fix_download.py @@ -44,7 +44,7 @@ def fix_download(): def main(): global DB - DB = database.Database() + DB = database.Repository() fix_download() diff --git a/helper_files/import_vod.py b/helper_files/import_vod.py index 304aae3..02f458c 100644 --- a/helper_files/import_vod.py +++ b/helper_files/import_vod.py @@ -81,7 +81,7 @@ def read_json(file_path: str): def main(): global DB - DB = database.Database() + DB = database.Repository() diff --git a/helper_files/migrate_sql2.py b/helper_files/migrate_sql2.py new file mode 100644 index 0000000..71302d7 --- /dev/null +++ b/helper_files/migrate_sql2.py @@ -0,0 +1,45 @@ +import sqlite3 +DB_PATH = "database.db" + +def migrate_to_many(db_path): + con = sqlite3.connect(db_path) + cur = con.cursor() + cur.execute("PRAGMA foreign_keys=OFF;") + try: + con.execute("BEGIN;") + cur.execute("ALTER TABLE youtube_info RENAME TO youtube_info_old;") + + cur.execute(""" + CREATE TABLE youtube_info ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + twitch_video_id VARCHAR NOT NULL, + yt_id VARCHAR, + yt_date VARCHAR, + yt_short_id VARCHAR, + yt_short_date VARCHAR, + yt_chat_SS_id VARCHAR, + yt_chat_SS_date VARCHAR, + yt_chat_over_id VARCHAR, + yt_chat_over_date VARCHAR, + FOREIGN KEY(twitch_video_id) REFERENCES twitch_videos(id) ON DELETE CASCADE + ); + """) + cur.execute("CREATE INDEX ix_youtube_info_twitch_video_id ON youtube_info (twitch_video_id);") + + cur.execute(""" + INSERT INTO youtube_info (twitch_video_id, yt_id, yt_date, yt_short_id, yt_short_date, yt_chat_SS_id, yt_chat_SS_date, yt_chat_over_id, yt_chat_over_date) + SELECT id, yt_id, yt_date, yt_short_id, yt_short_date, yt_chat_SS_id, yt_chat_SS_date, yt_chat_over_id, yt_chat_over_date + FROM youtube_info_old; + """) + + cur.execute("DROP TABLE youtube_info_old;") + con.commit() + print("Done - now 1-to-Many") + except Exception as e: + con.rollback() + raise e + finally: + cur.execute("PRAGMA foreign_keys=ON;") + con.close() + +migrate_to_many(DB_PATH) \ No newline at end of file diff --git a/helper_files/move_vod.py b/helper_files/move_vod.py index f1c165b..7f8e1f1 100644 --- a/helper_files/move_vod.py +++ b/helper_files/move_vod.py @@ -49,7 +49,7 @@ def remove_vods(): def main(): global DB - DB = database.Database() + DB = database.Repository() remove_vods() diff --git a/helper_files/redownload_clips.py b/helper_files/redownload_clips.py index ae9b782..ef11d4c 100644 --- a/helper_files/redownload_clips.py +++ b/helper_files/redownload_clips.py @@ -31,5 +31,5 @@ def redownload_clips(): if __name__ == "__main__": - DB = database.Database() + DB = database.Repository() redownload_clips() \ No newline at end of file diff --git a/helper_files/youtube_description_updater.py b/helper_files/youtube_description_updater.py new file mode 100644 index 0000000..7a73cdc --- /dev/null +++ b/helper_files/youtube_description_updater.py @@ -0,0 +1,225 @@ +""" +YouTube Description Updater +Requires: pip install google-api-python-client google-auth-oauthlib google-auth + +1. Enable YouTube Data API v3 in Google Cloud Console +2. Create OAuth 2.0 credentials (Desktop App) +3. Download as client_secret.json +""" + +import os +import sys +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials +from google_auth_oauthlib.flow import InstalledAppFlow +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +# Finds the absolute path to 'project_directory' and adds it to Python's search path +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) +sys.path.append(PROJECT_ROOT) + +from package.config import DB as DB +from package.youtube.config import CREDENTIALS as CREDENTIALS +from package.database import YoutubeInfo as YoutubeInfo + +""" +SCOPES = ["https://www.googleapis.com/auth/youtube.force-ssl"] +TOKEN_FILE = "token.json" +CLIENT_SECRET_FILE = "client_secret.json" + +def get_authenticated_service(): + ""Authenticates and returns a YouTube service object."" + creds = None + if os.path.exists(TOKEN_FILE): + creds = Credentials.from_authorized_user_file(TOKEN_FILE, SCOPES) + + if not creds or not creds.valid: + if creds and creds.expired and creds.refresh_token: + creds.refresh(Request()) + else: + if not os.path.exists(CLIENT_SECRET_FILE): + raise FileNotFoundError( + f"{CLIENT_SECRET_FILE} not found. Download it from Google Cloud Console." + ) + flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET_FILE, SCOPES) + creds = flow.run_local_server(port=0) + + with open(TOKEN_FILE, "w") as token: + token.write(creds.to_json()) + + return build("youtube", "v3", credentials=creds) +""" + +def get_video_details(youtube, video_id: str): + """Finds the video and returns its current snippet.""" + try: + response = youtube.videos().list( + part="snippet,status", + id=video_id + ).execute() + + if not response["items"]: + print(f"❌ Video not found: {video_id}") + return None + + return response["items"][0] + except HttpError as e: + print(f"❌ API Error while fetching video: {e}") + return None + +def update_video_description(video_id: str, new_description: str, append: bool = False): + """ + Takes a YouTube video ID, finds the video, and changes its description. + + Args: + video_id (str): The YouTube video ID (e.g., 'dQw4w9WgXcQ') + new_description (str): The new description to set + append (bool): If True, appends new_description to existing description instead of replacing + + Returns: + dict | None: Updated video snippet or None on failure + """ + # Load credentials + if not CREDENTIALS: + print("❌ Error: YouTube API credentials not configured.") + return None + + # Create YouTube API client + youtube = build('youtube', 'v3', credentials=CREDENTIALS) + + video = get_video_details(youtube, video_id) + if not video: + return None + + snippet = video["snippet"] + current_title = snippet["title"] + current_category = snippet.get("categoryId", "22") + + print(f"Found video: '{current_title}' [{video_id}]") + print(f"Current description length: {len(snippet.get('description', ''))}") + current_description = snippet.get("description", "") or "" + + # Decide final description + if append: + if new_description.strip().lower() in current_description.lower(): + print("ℹ️ Description already contains that string, skipping") + return snippet + final_description = (snippet.get("description", "") + "\n\n" + new_description).strip() + else: + final_description = new_description + + # YouTube requires you to send title + categoryId + description when updating snippet + body = { + "id": video_id, + "snippet": { + "title": current_title, + "description": final_description, + "categoryId": current_category, + # Keep existing tags, etc if you want + "tags": snippet.get("tags", []), + "defaultLanguage": snippet.get("defaultLanguage"), + } + } + + try: + update_response = youtube.videos().update( + part="snippet", + body=body + ).execute() + + print(f"✅ Description updated for {video_id}") + print(f"New length: {len(final_description)}") + return update_response["snippet"] + + except HttpError as e: + print(f"❌ Failed to update video {video_id}: {e}") + if "quotaExceeded" in str(e): + print("You hit the YouTube API quota.") + return None + +def append_tag_if_not_exists(video_id: str, new_tag: str): + """ + Appends a new tag to a YouTube video if it doesn't already exist. + Case-insensitive check. + """ + if not new_tag or not new_tag.strip(): + print("❌ Tag is empty") + return None + + new_tag_clean = new_tag.strip() + + # Load credentials + if not CREDENTIALS: + print("❌ Error: YouTube API credentials not configured.") + return None + + # Create YouTube API client + youtube = build('youtube', 'v3', credentials=CREDENTIALS) + + video = get_video_details(youtube, video_id) + if not video: + return None + + snippet = video["snippet"] + existing_tags = snippet.get("tags", []) or [] + existing_lower = {t.lower().strip() for t in existing_tags} + + if new_tag_clean.lower() in existing_lower: + print(f"ℹ️ Tag '{new_tag_clean}' already exists on {video_id}, skipping") + return existing_tags + + updated_tags = existing_tags + [new_tag_clean] + + total_chars = sum(len(t) for t in updated_tags) + if total_chars > 450: + print(f"⚠️ Tags approaching 500 char limit ({total_chars})") + if len(updated_tags) > 15: + print(f"⚠️ {len(updated_tags)} tags - YouTube ignores after ~15-20") + + print(f"Current tags: {existing_tags}") + print(f"Adding: '{new_tag_clean}'") + + body = { + "id": video_id, + "snippet": { + "title": snippet["title"], + "description": snippet.get("description", ""), + "categoryId": snippet.get("categoryId", "22"), + "tags": updated_tags, + } + } + try: + resp = youtube.videos().update(part="snippet", body=body).execute() + print(f"✅ Tag '{new_tag_clean}' added to {video_id}") + return resp["snippet"].get("tags", []) + except HttpError as e: + print(f"❌ Failed to add tag: {e}") + return None + +def main(): + videos = DB.get_youtube_info_all() + for video in videos: + if video.yt_id: + update_video_description(video.yt_id, f"({video.id})", True) + append_tag_if_not_exists(video.yt_id, video.id) + if video.yt_short_id: + update_video_description(video.yt_short_id, f"({video.id})", True) + append_tag_if_not_exists(video.yt_short_id, video.id) + + +# --- Example Usage --- +if __name__ == "__main__": + main() + + # Example 1: Replace description + # update_video_description("YOUR_VIDEO_ID_HERE", "This is my new description!\nGame: Valorant") + + # Example 2: Append to description + # update_video_description("YOUR_VIDEO_ID_HERE", "\n\nCheck out https://twitch.tv/teampgp", append=True) + + # For integration with your existing code: + #vid = input("Enter YouTube Video ID: ").strip() + #if vid: + # new_desc = input("Enter new description: ").strip() + # update_video_description(vid, new_desc) diff --git a/package/__init__.py b/package/__init__.py index 12013af..eecf374 100644 --- a/package/__init__.py +++ b/package/__init__.py @@ -20,7 +20,7 @@ from .config import CHANNEL_NAME, TZ, DB from .hash_tags import get_top_nouns from .process_uploads import process_uploads from .transcribe_video import transcribe_to_srt -from .utils import write_csv, run_command +from .utils import get_chapters, get_video_id_from_chat, write_csv, run_command # 4. Expose Subpackages from . import twitch @@ -39,6 +39,8 @@ __all__ = [ "transcribe_to_srt", "write_csv", "run_command", + "get_chapters", + "get_video_id_from_chat" "twitch", "youtube" ] diff --git a/package/config.py b/package/config.py index 991d7ac..aefe33d 100644 --- a/package/config.py +++ b/package/config.py @@ -2,9 +2,9 @@ from zoneinfo import ZoneInfo #import package.database as database -from package.database.database import Database +from package.database.database import Repository CHANNEL_NAME = "teampgp" TZ = ZoneInfo("America/Toronto") -DB = Database() \ No newline at end of file +DB = Repository() \ No newline at end of file diff --git a/package/database/__init__.py b/package/database/__init__.py index 2d52fcb..4fca978 100644 --- a/package/database/__init__.py +++ b/package/database/__init__.py @@ -1,12 +1,13 @@ # package/database/__init__.py # Expose components from individual modules using relative imports -from .database import TwitchVideo, Data, YoutubeInfo, Database +from .database import TwitchVideo, Data, YoutubeInfo, Chapter, Repository # Explicitly define what is exposed when importing the subpackage __all__ = [ "TwitchVideo", "Data", "YoutubeInfo", - "Database", + "Chapter", + "Repository", ] \ No newline at end of file diff --git a/package/database/database.py b/package/database/database.py index 85a7340..def0c49 100644 --- a/package/database/database.py +++ b/package/database/database.py @@ -26,8 +26,12 @@ class TwitchVideo(Base): uploaded_yt_chats = Column(Boolean, nullable=False, default=False) uploaded_yt_shorts = Column(Boolean, nullable=False, default=False) - # Relationship back to YoutubeInfo (uselist=False enforces a 1-to-1 structure) - youtube_info = relationship("YoutubeInfo", back_populates="twitch_video", uselist=False) + # This now actually points to twitch_videos.id + related_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=True) + + # Relationship back to YoutubeInfo (uselist=True enforces a 1-to-many structure) + youtube_info = relationship("YoutubeInfo", back_populates="twitch_video", uselist=True) + chapters = relationship("Chapter", back_populates="twitch_video", cascade="all, delete-orphan") # 3. Define the data table class Data(Base): @@ -43,7 +47,8 @@ class Data(Base): class YoutubeInfo(Base): __tablename__ = "youtube_info" - id = Column(String, ForeignKey("twitch_videos.id"), primary_key=True) + id = Column(Integer, primary_key=True, autoincrement=True) + twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False) yt_id = Column(String, nullable=True) yt_date = Column(String, nullable=True) yt_short_id = Column(String, nullable=True) @@ -56,8 +61,22 @@ class YoutubeInfo(Base): # Relationship linking back to the twitch video record twitch_video = relationship("TwitchVideo", back_populates="youtube_info") +class Chapter(Base): + __tablename__ = "chapters" -class Database: + id = Column(Integer, primary_key=True, autoincrement=True) + twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False) + start_milliseconds = Column(Integer, nullable=False) + length_milliseconds = Column(Integer, nullable=False) + description = Column(String, nullable=True) + type = Column(String, nullable=True) + game_display_name = Column(String, nullable=True) + game_id = Column(String, nullable=True) + + # Relationship linking back to the twitch video record + twitch_video = relationship("TwitchVideo", back_populates="chapters") + +class Repository: def __init__(self, db_path: str = "database.db"): # 4. Connect to the SQLite file and generate the tables # This will create 'database.db' in the current working directory if it doesn't exist. @@ -68,6 +87,15 @@ class Database: Session = sessionmaker(bind=engine) self.session = Session() + def set_related_video(self, record_id: str, related_video_id: str): + record = self.session.get(TwitchVideo, record_id) + + if record: + setattr(record, "related_video", related_video_id) + self.session.commit() + else: + print(f"❌ Record {record_id} not found.") + def __mark_as(self, record_id: str, field_name: str, value: bool = True): record = self.session.get(TwitchVideo, record_id) @@ -178,33 +206,67 @@ class Database: stmt = select(Data) return self.session.scalars(stmt).all() - def get_youtube_info(self, record_id) -> YoutubeInfo | None: - return self.session.get(YoutubeInfo, record_id) + def get_youtube_info_for_video(self, video_id: str) -> list[YoutubeInfo]: + """Retrieve all YouTubeInfo records associated with a specific TwitchVideo ID.""" + stmt = select(YoutubeInfo).where(YoutubeInfo.twitch_video_id == video_id) + return self.session.scalars(stmt).all() - def insert(self, data: TwitchVideo | Data | YoutubeInfo): - # SQLAlchemy 2.0 context manager handles commit/rollback automatically + def get_youtube_info_all(self) -> list[YoutubeInfo]: + return self.session.scalars(select(YoutubeInfo)).all() + + def insert(self, data: TwitchVideo | Data | YoutubeInfo | Chapter): + def insert(self, data: TwitchVideo | Data | YoutubeInfo | Chapter | list[YoutubeInfo] | list[Chapter]): + """ + Handles insertion/updating for any of the defined models, + using session merge/add for SQLAlchemy 2.0 best practices. + Accepts single objects or lists of objects for collection handling. + """ try: if isinstance(data, TwitchVideo): - # 1. Query for the existing row using SQLAlchemy 2.0 scalar execution + # 1. Check if the record exists stmt = select(TwitchVideo).where(TwitchVideo.id == data.id) existing_record = self.session.scalar(stmt) if existing_record: + # 2. Update relevant fields on the managed instance if existing_record.view_count != data.view_count or existing_record.duration != data.duration: - # 2. Update the columns on the managed instance existing_record.view_count = data.view_count existing_record.duration = data.duration - print(f"🔄 Record {data.id} updated successfully!") + print(f"🔄 TwitchVideo record {data.id} updated successfully!") + # Note: Add more field updates here as needed else: - # 3. Add new instance if it doesn't exist + # 3. Add new instance self.session.add(data) - print("➕ New TwitchVideo record inserted successfully!") - else: + print(f"➕ New TwitchVideo record with ID {data.id} inserted successfully!") + elif isinstance(data, Data): self.session.merge(data) - print(f"➕ {type(data).__name__} record inserted successfully!") - # 4. Flush changes and commit the transaction - self.session.commit() + print(f"➕ Data record merged/inserted successfully!") + elif isinstance(data, YoutubeInfo): + # Handle single YoutubeInfo object + self.session.merge(data) + print(f"➕ YoutubeInfo record merged/inserted successfully!") + elif isinstance(data, Chapter): + # Handle single Chapter object + self.session.merge(data) + print(f"➕ Chapter record merged/inserted successfully!") + elif isinstance(data, list): + # Handle lists of records (YoutubeInfo or Chapter) + record_type = type(data[0]) + if record_type in (YoutubeInfo, Chapter): + for item in data: + self.session.merge(item) + print(f"➕ Successfully processed {len(data)} records of type {record_type.__name__}.") + else: + raise TypeError(f"Unsupported list content type: {record_type.__name__}") + else: + raise TypeError(f"Unsupported data type for insertion: {type(data).__name__}") + + # 4. Commit the transaction + self.session.flush() # Flush first to ensure IDs are populated except Exception as e: self.session.rollback() - print(f"❌ An error occurred, transaction rolled back: {e}") + print(f"❌ A critical error occurred during record insertion/update: {e}") + raise + + self.session.commit() diff --git a/package/process_uploads.py b/package/process_uploads.py index 6b2b123..53aeecb 100644 --- a/package/process_uploads.py +++ b/package/process_uploads.py @@ -14,9 +14,10 @@ from package.youtube.uploader import CategoryId as CategoryId from package.database import TwitchVideo, YoutubeInfo, Data -def process_videos(video_type: bool = True): +def process_videos(video_type: bool = True, limit: int = 22): + count = 0 # count how many videos we have upload and stop at var: limit videos = DB.get_unuploaded_chats() if video_type else DB.get_unuploaded_shorts() - file_type = "download/clips" if video_type else "download/video" + file_type = "download/video" if video_type else "download/clips" meta_data: Data = DB.get_data()[0] current_time = datetime.now(TZ) @@ -29,6 +30,10 @@ def process_videos(video_type: bool = True): release_time += timedelta(days=1) for video in videos: + #We reached limit stop uploading videos + if count >= limit: + break + base_path = Path(f"{file_type}/{video.id}/{video.id}") original_path = base_path.with_suffix(".mp4") @@ -66,6 +71,7 @@ def process_videos(video_type: bool = True): tags = ['gaming', 'TeamPGP', f'{video.game_name}', f'{video.creator_name}', 'Level1Techs', 'twitch'] if not video_type: tags.extend(['shorts', 'twitch_clips', 'clips']) + tags.extend([video.id]) if srt_path.exists(): try: @@ -78,7 +84,8 @@ def process_videos(video_type: bool = True): if youtube_info.yt_id: main_video_id = youtube_info.yt_id else: - main_video_id = upload_video(str(original_path), video.title, category_id, description, 'unlisted', tags) + main_video_id = upload_video(str(original_path), video.title, category_id, f"{description} \n\n({video.id})", 'unlisted', tags) + count += 1 if not main_video_id: print(f"❌ Error: main video upload failed for {original_path}") continue @@ -86,7 +93,7 @@ def process_videos(video_type: bool = True): youtube_info.yt_date = datetime.now(TZ) # Upload secondary - secondary_desc = f"{description} Original Full size @ https://youtu.be/{main_video_id}" + secondary_desc = f"{description} Original Full size @ https://youtu.be/{main_video_id} \n\n({video.id})" secondary_video_id = upload_video(str(file_path), video.title, category_id, secondary_desc, 'private', tags, release_time) if not secondary_video_id: @@ -94,6 +101,8 @@ def process_videos(video_type: bool = True): DB.insert(youtube_info) # save main id at least continue + count += 1 + # Success - write CSV and DB data_csv = [main_video_id, secondary_video_id, release_time.isoformat()] utils.write_csv([data_csv], str(file_csv)) diff --git a/package/twitch/download_thumbnails.py b/package/twitch/download_thumbnails.py index 4a8833c..38ad8c7 100644 --- a/package/twitch/download_thumbnails.py +++ b/package/twitch/download_thumbnails.py @@ -93,7 +93,7 @@ async def download_thumbnails(): # --- OPTION B: Download Past Broadcast VOD Thumbnail --- # Extract the ID sequence from your target video link # target_vod = "2145678901" - db = database.Database() + db = database.Repository() vods = db.get_vods() for vod in vods: ( diff --git a/package/twitch/download_videos.py b/package/twitch/download_videos.py index 8c09c8d..a841b75 100644 --- a/package/twitch/download_videos.py +++ b/package/twitch/download_videos.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 import subprocess +from pathlib import Path from datetime import datetime, timedelta, timezone import package.utils as utils from package.config import DB as DB -from package.database import TwitchVideo +from package.config import TZ as TZ +from package.database import TwitchVideo, Chapter def download_videos(): """Find all undownloaded videos and clips, and download them safely.""" @@ -12,59 +14,72 @@ def download_videos(): print("Starting downloads...") for row in undownloaded: - # 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM) - #datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ") - data_timestamp = datetime.fromisoformat(row.created_at) + try: + # 2026-08-16T23:52:10Z -> replace Z with +00:00 + data_timestamp = datetime.fromisoformat(row.created_at.replace("Z", "+00:00")) + except Exception: + data_timestamp = datetime.strptime(row.created_at, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) - # 2. Get the exact current date and time - current_time = datetime.now(timezone.utc) - - # 3. Calculate the difference between the two times - time_difference = current_time - data_timestamp - - # 4. Check if the difference is greater than 24 hours - if time_difference > timedelta(hours=24): - print("The data is more than 24 hours old.") - else: - #lets wait 24 hours befor downloading - print("The data is less than 24 hours old.") + if datetime.now(timezone.utc) - data_timestamp < timedelta(hours=24): + print(f"Skipping {row.id} - less than 24h old") continue - data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')] print(f"ID: {row.id} | Date: {row.created_at} | Game: {row.game_name} | Title: {row.title}") - # 1. Define paths and isolate base directory + base_dir = Path(f"download/clips/{row.id}" if row.clip_is else f"download/videos/{row.id}") + base_dir.mkdir(parents=True, exist_ok=True) + + temp_path = Path("download/temp") + temp_path.mkdir(parents=True, exist_ok=True) + + target_video = base_dir / f"{row.id}.mp4" + json_file = base_dir / f"{row.id}_chat.json" + csv_file = base_dir / f"{row.id}.csv" + if row.clip_is: - target_dir = f"download/clips/{row.id}" - cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --temp-path download/temp" + cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_video} --collision Overwrite --temp-path {temp_path}" else: - target_dir = f"download/videos/{row.id}" - # FIXED: Removed the duplicated command string combined with '&&' - cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp" + cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_video} --collision Overwrite --threads 2 --temp-path {temp_path}" - csv_file = f"{target_dir}/{row.id}.csv" - - # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes output = utils.run_command(cmd, look_for=["[STATUS]"]) - output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {row.id} -o {target_dir}/{row.id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"]) + output2 = utils.run_command( + f"TwitchDownloaderCLI chatdownload --id {row.id} -o {json_file} -E --collision Overwrite --temp-path {temp_path}", + look_for=["[STATUS]"] + ) + + if output2 and json_file.exists(): + related_id = utils.get_video_id_from_chat(json_file) + if related_id: + # This will set related_video_id = 2848295283 for a clip + DB.set_related_video(row.id, related_id) + print(f" -> Linked {row.id} -> related VOD {related_id}") + if row.clip_is: + chapters = utils.get_chapters(json_file) + for chapter in chapters: + chapter_record = Chapter( + twitch_video_id=row.id, + start_ms=chapter.get('startMilliseconds'), + length_ms=chapter.get('lengthMilliseconds'), + description=chapter.get('description'), + type=chapter.get('type'), + game_display_name=chapter.get('gameDisplayName'), + game_id=chapter.get('gameId') + ) + DB.insert(chapter_record) + print(f" ➕ Chapter added: {chapter_record.description} ({chapter_record.start_ms}ms)") if output: - # Only write CSV and update database if download actually completed - data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')] - utils.write_csv(data, csv_file) - print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}") + # FIX 4: write_csv needs actual row data, not column names + utils.write_csv([row], str(csv_file)) # pass the object/dict, not vars(TwitchVideo) + print(f"✅ Success: {base_dir}") DB.mark_as_downloaded(row.id) else: - print(f"ID: {row.id} | Download Process failed.") - #print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}") - #if output.get("stderr"): - # print(f"Details: {output['stderr']}") - - # Clear temporary chunk clutter immediately if a VOD crashes out + print(f"❌ ID: {row.id} | Download failed.") if not row.clip_is: print("Flushing temporary crash chunks...") subprocess.run("rm -rf download/temp/*", shell=True) + print("Finished Downloading Pipeline.") if __name__ == "__main__": diff --git a/package/utils.py b/package/utils.py index cb34689..10a93ee 100755 --- a/package/utils.py +++ b/package/utils.py @@ -1,10 +1,50 @@ #!/usr/bin/env python3 import csv +from pathlib import Path import sys +import json import shlex import subprocess -def write_csv(data, file_name): +def get_chapters(json_path): + """Loads the JSON file and returns all chapters.""" + with open(json_path, 'r', encoding='utf-8') as f: + data = json.load(f) + + # The chapters are under video -> chapters + video = data.get('video', {}) + chapters = video.get('chapters', []) + + if not chapters: + print("No chapters found.") + return [] + + print(f"Found {len(chapters)} chapter(s) in video '{video.get('title')}':\n") + + for i, chapter in enumerate(chapters, 1): + start_sec = chapter.get('startMilliseconds', 0) / 1000 + length_sec = chapter.get('lengthMilliseconds', 0) / 1000 + + #print(f"{i}. {chapter.get('description', 'No Title')}") + #print(f" - Type: {chapter.get('type')}") + #print(f" - Start: {start_sec}s ({chapter.get('startMilliseconds')}ms)") + #print(f" - Length: {length_sec}s ({chapter.get('lengthMilliseconds')}ms)") + #print(f" - Game: {chapter.get('gameDisplayName')} (ID: {chapter.get('gameId')})") + #print() + + return chapters + +def get_video_id_from_chat(json_path: str | Path) -> str | None: + """Gets the id inside video:{ } - from the chat json you download""" + try: + with open(json_path, 'r', encoding='utf-8') as f: + data = json.load(f) + return data.get("video", {}).get("id") + except Exception as e: + print(f"Could not read video.id from {json_path}: {e}") + return None + +def write_csv(data: list, file_name: str): # Open file with newline='' to prevent extra blank rows across platforms with open(file_name, "w", newline="", encoding="utf-8") as file: writer = csv.writer(file)