I forgot what I chanagerd

delete/twitch_down_uploader.py

@@ -4,7 +4,7 @@ import os
 import package.utils as linux
 import time
 
-from package.database import Database
+from package.database import Repository
 
 import package.youtube.uploader as uploader
 from package.youtube.uploader import CategoryId
@@ -319,7 +319,7 @@ def get_vod_ids_simplified(channel_name: str):
 if __name__ == "__main__":
     tables = ["clips"]
     for table in tables:
-        DB = Database(table)
+        DB = Repository(table)
 
         get_vod_ids_simplified(CHANNEL_NAME)
         #download()

delete/twitch_download_clips.py

@@ -3,13 +3,13 @@ import requests
 import subprocess
 from datetime import datetime
 
-from package.database import Database
+from package.database import Repository
 
 import package.youtube.uploader as uploader
 from package.youtube.uploader import CategoryId
 
 CHANNEL_NAME = 'teampgp'  # Replace with the streamer's username
-DB = Database("clips")
+DB = Repository("clips")
 
 def run_linux_command(command: str):
     """Executes a Linux command, waits for completion, and returns output."""

help.sqlite3-query

@@ -1,9 +1,11 @@
 -- database: ./database.db
 
-SELECT * FROM twitch_videos WHERE clip_is == "1" and uploaded_yt_shorts == "1";
+SELECT * FROM twitch_videos WHERE clip_is == "1";
 UPDATE twitch_videos SET uploaded_yt_shorts = 0 WHERE uploaded_yt_shorts == "1";
 SELECT * FROM twitch_videos WHERE title == "Oh Dear";
 SELECT * FROM twitch_videos WHERE title = 'Kreestuh FINALLY hits her "Ryan Dark Story" limit.  Jesus Big K it''s about time!';
 SELECT * FROM twitch_videos WHERE title = "Bolt-Action Bureaucracy";
 SELECT * FROM twitch_videos WHERE title = "Just after going through a mass starvation...";
 SELECT * FROM twitch_videos WHERE title = "Semi-Automatic Sanctimony";
+SELECT * FROM twitch_videos WHERE id = "2848295283";
+SELECT * FROM twitch_videos WHERE game_name = "Darkwood";

helper_files/chapters.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+import os
+import json
+import sys
+
+# Finds the absolute path to 'project_directory' and adds it to Python's search path
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+sys.path.append(PROJECT_ROOT)
+
+from package.config import DB as DB
+from package.database import Chapter
+
+def get_chapters(json_path):
+    """Loads the JSON file and returns all chapters."""
+    with open(json_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+
+    # The chapters are under video -> chapters
+    video = data.get('video', {})
+    chapters = video.get('chapters', [])
+
+    if not chapters:
+        print("No chapters found.")
+        return []
+
+    print(f"Found {len(chapters)} chapter(s) in video '{video.get('title')}':\n")
+
+    for i, chapter in enumerate(chapters, 1):
+        start_sec = chapter.get('startMilliseconds', 0) / 1000
+        length_sec = chapter.get('lengthMilliseconds', 0) / 1000
+
+        print(f"{i}. {chapter.get('description', 'No Title')}")
+        print(f" - Type: {chapter.get('type')}")
+        print(f" - Start: {start_sec}s ({chapter.get('startMilliseconds')}ms)")
+        print(f" - Length: {length_sec}s ({chapter.get('lengthMilliseconds')}ms)")
+        print(f" - Game: {chapter.get('gameDisplayName')} (ID: {chapter.get('gameId')})")
+        print()
+
+    return chapters
+
+def main():
+    downloads = DB.get_downloaded()
+
+    for download in downloads:
+        json_path = f"download/clips/{download.id}/{download.id}_chat.json" if download.clip_is else f"download/videos/{download.id}/{download.id}_chat.json"
+        if os.path.exists(json_path):
+            chapters = get_chapters(json_path)
+            for chapter in chapters:
+                chapter_record = Chapter(
+                    twitch_video_id=download.id,
+                    start_milliseconds=chapter.get('startMilliseconds'),
+                    length_milliseconds=chapter.get('lengthMilliseconds'),
+                    description=chapter.get('description'),
+                    type=chapter.get('type'),
+                    game_display_name=chapter.get('gameDisplayName'),
+                    game_id=chapter.get('gameId')
+                )
+                DB.insert(chapter_record)
+        else:
+            print(f"JSON file not found for video {download.id}: {json_path}")
+
+if __name__ == "__main__":
+    main()
+    # Usage: python get_chapters.py your_file.json
+    #path = sys.argv[1] if len(sys.argv) > 1 else "/home/richard/python_scripts/download/videos/2873107835/2873107835_chat.json"
+    #try:
+    #    get_chapters(path)
+    #except FileNotFoundError:
+    #    print(f"Error: File not found -> {path}")
+    #except json.JSONDecodeError:
+    #    print(f"Error: File is not valid JSON -> {path}")

helper_files/combine_videos.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""Combine multiple video files into one file using FFmpeg.
+
+Example:
+    python helper_files/combine_videos.py output.mp4 video1.mp4 video2.mp4 video3.mp4
+    python helper_files/combine_videos.py --output merged.mp4 --inputs video1.mp4 video2.mp4
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def build_concat_list(input_files, list_path):
+    with open(list_path, "w", encoding="utf-8") as f:
+        for path in input_files:
+            f.write(f"file '{os.path.abspath(path)}'\n")
+
+
+def run_ffmpeg(cmd):
+    print("Running:", " ".join(cmd))
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.returncode != 0:
+        print(result.stdout)
+        print(result.stderr)
+        raise RuntimeError(f"FFmpeg failed with exit code {result.returncode}")
+    print("✅ Finished successfully.")
+
+
+def concat_videos_copy(input_files, output_path):
+    list_path = Path(output_path).with_suffix(".concat.txt")
+    build_concat_list(input_files, list_path)
+
+    cmd = [
+        "ffmpeg",
+        "-y",
+        "-f", "concat",
+        "-safe", "0",
+        "-i", str(list_path),
+        "-c", "copy",
+        str(output_path),
+    ]
+    run_ffmpeg(cmd)
+    list_path.unlink(missing_ok=True)
+
+
+def concat_videos_reencode(input_files, output_path):
+    inputs = []
+    filter_parts = []
+    audio_parts = []
+
+    for i, video_path in enumerate(input_files):
+        inputs.extend(["-i", str(video_path)])
+        filter_parts.append(f"[{i}:v]scale=trunc(iw/2)*2:trunc(ih/2)*2,setpts=PTS-STARTPTS[v{i}]")
+        audio_parts.append(f"[{i}:a]aresample=async=1[a{i}]")
+
+    concat_filter = ",".join(filter_parts + audio_parts)
+    map_parts = []
+    for i in range(len(input_files)):
+        map_parts.append(f"[v{i}]")
+        map_parts.append(f"[a{i}]")
+
+    # Join the video and audio streams in order and concatenate them.
+    # This works well when the clips are similar enough in format.
+    filter_complex = f"{','.join(filter_parts)};{';'.join(audio_parts)};"
+    filter_complex += "".join(f"[{i}:v][{i}:a]" for i in range(len(input_files)))
+    filter_complex += f"concat=n={len(input_files)}:v=1:a=1[outv][outa]"
+
+    cmd = [
+        "ffmpeg",
+        "-y",
+        *inputs,
+        "-filter_complex",
+        filter_complex,
+        "-map",
+        "[outv]",
+        "-map",
+        "[outa]",
+        "-c:v",
+        "libx264",
+        "-c:a",
+        "aac",
+        "-movflags",
+        "+faststart",
+        str(output_path),
+    ]
+    run_ffmpeg(cmd)
+
+
+def merge_videos(input_files, output_path):
+    input_files = [str(Path(p).expanduser().resolve()) for p in input_files]
+    output_path = str(Path(output_path).expanduser().resolve())
+
+    if len(input_files) < 2:
+        raise ValueError("You need at least 2 video files to merge.")
+
+    for path in input_files:
+        if not os.path.exists(path):
+            raise FileNotFoundError(f"File not found: {path}")
+
+    output_dir = Path(output_path).parent
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    try:
+        concat_videos_copy(input_files, output_path)
+    except Exception as exc:
+        print(f"Fast copy method failed: {exc}")
+        print("Retrying with re-encode fallback...")
+        concat_videos_reencode(input_files, output_path)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Join two or more video files together with FFmpeg.")
+    parser.add_argument("output_file", help="Path to the merged output video")
+    parser.add_argument("input_files", nargs="+", help="Two or more videos to concatenate")
+    args = parser.parse_args()
+
+    try:
+        merge_videos(args.input_files, args.output_file)
+        print(f"Merged video saved to: {args.output_file}")
+    except Exception as exc:
+        print(f"Error: {exc}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

helper_files/datetime_fix.py

@@ -55,7 +55,7 @@ def fix_time():
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
     fix_time()
 

helper_files/fix_download.py

@@ -44,7 +44,7 @@ def fix_download():
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
     fix_download()
 

helper_files/import_vod.py

@@ -81,7 +81,7 @@ def read_json(file_path: str):
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
 
 

helper_files/migrate_sql2.py

@@ -0,0 +1,45 @@
+import sqlite3
+DB_PATH = "database.db"
+
+def migrate_to_many(db_path):
+    con = sqlite3.connect(db_path)
+    cur = con.cursor()
+    cur.execute("PRAGMA foreign_keys=OFF;")
+    try:
+        con.execute("BEGIN;")
+        cur.execute("ALTER TABLE youtube_info RENAME TO youtube_info_old;")
+        
+        cur.execute("""
+        CREATE TABLE youtube_info (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            twitch_video_id VARCHAR NOT NULL,
+            yt_id VARCHAR,
+            yt_date VARCHAR,
+            yt_short_id VARCHAR,
+            yt_short_date VARCHAR,
+            yt_chat_SS_id VARCHAR,
+            yt_chat_SS_date VARCHAR,
+            yt_chat_over_id VARCHAR,
+            yt_chat_over_date VARCHAR,
+            FOREIGN KEY(twitch_video_id) REFERENCES twitch_videos(id) ON DELETE CASCADE
+        );
+        """)
+        cur.execute("CREATE INDEX ix_youtube_info_twitch_video_id ON youtube_info (twitch_video_id);")
+
+        cur.execute("""
+        INSERT INTO youtube_info (twitch_video_id, yt_id, yt_date, yt_short_id, yt_short_date, yt_chat_SS_id, yt_chat_SS_date, yt_chat_over_id, yt_chat_over_date)
+        SELECT id, yt_id, yt_date, yt_short_id, yt_short_date, yt_chat_SS_id, yt_chat_SS_date, yt_chat_over_id, yt_chat_over_date
+        FROM youtube_info_old;
+        """)
+        
+        cur.execute("DROP TABLE youtube_info_old;")
+        con.commit()
+        print("Done - now 1-to-Many")
+    except Exception as e:
+        con.rollback()
+        raise e
+    finally:
+        cur.execute("PRAGMA foreign_keys=ON;")
+        con.close()
+
+migrate_to_many(DB_PATH)

helper_files/move_vod.py

@@ -49,7 +49,7 @@ def remove_vods():
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
     remove_vods()
 

helper_files/redownload_clips.py

@@ -31,5 +31,5 @@ def redownload_clips():
     
 
 if __name__ == "__main__":
-    DB = database.Database()
+    DB = database.Repository()
     redownload_clips()

helper_files/youtube_description_updater.py

@@ -0,0 +1,225 @@
+"""
+YouTube Description Updater
+Requires: pip install google-api-python-client google-auth-oauthlib google-auth
+
+1. Enable YouTube Data API v3 in Google Cloud Console
+2. Create OAuth 2.0 credentials (Desktop App)
+3. Download as client_secret.json
+"""
+
+import os
+import sys
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+# Finds the absolute path to 'project_directory' and adds it to Python's search path
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+sys.path.append(PROJECT_ROOT)
+
+from package.config import DB as DB
+from package.youtube.config import CREDENTIALS as CREDENTIALS
+from package.database import YoutubeInfo as YoutubeInfo
+
+"""
+SCOPES = ["https://www.googleapis.com/auth/youtube.force-ssl"]
+TOKEN_FILE = "token.json"
+CLIENT_SECRET_FILE = "client_secret.json"
+
+def get_authenticated_service():
+    ""Authenticates and returns a YouTube service object.""
+    creds = None
+    if os.path.exists(TOKEN_FILE):
+        creds = Credentials.from_authorized_user_file(TOKEN_FILE, SCOPES)
+    
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            if not os.path.exists(CLIENT_SECRET_FILE):
+                raise FileNotFoundError(
+                    f"{CLIENT_SECRET_FILE} not found. Download it from Google Cloud Console."
+                )
+            flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET_FILE, SCOPES)
+            creds = flow.run_local_server(port=0)
+        
+        with open(TOKEN_FILE, "w") as token:
+            token.write(creds.to_json())
+
+    return build("youtube", "v3", credentials=creds)
+"""
+
+def get_video_details(youtube, video_id: str):
+    """Finds the video and returns its current snippet."""
+    try:
+        response = youtube.videos().list(
+            part="snippet,status",
+            id=video_id
+        ).execute()
+
+        if not response["items"]:
+            print(f"❌ Video not found: {video_id}")
+            return None
+        
+        return response["items"][0]
+    except HttpError as e:
+        print(f"❌ API Error while fetching video: {e}")
+        return None
+
+def update_video_description(video_id: str, new_description: str, append: bool = False):
+    """
+    Takes a YouTube video ID, finds the video, and changes its description.
+
+    Args:
+        video_id (str): The YouTube video ID (e.g., 'dQw4w9WgXcQ')
+        new_description (str): The new description to set
+        append (bool): If True, appends new_description to existing description instead of replacing
+
+    Returns:
+        dict | None: Updated video snippet or None on failure
+    """
+    # Load credentials
+    if not CREDENTIALS:
+        print("❌ Error: YouTube API credentials not configured.")
+        return None
+    
+    # Create YouTube API client
+    youtube = build('youtube', 'v3', credentials=CREDENTIALS)
+
+    video = get_video_details(youtube, video_id)
+    if not video:
+        return None
+
+    snippet = video["snippet"]
+    current_title = snippet["title"]
+    current_category = snippet.get("categoryId", "22")
+    
+    print(f"Found video: '{current_title}' [{video_id}]")
+    print(f"Current description length: {len(snippet.get('description', ''))}")
+    current_description = snippet.get("description", "") or ""
+
+    # Decide final description
+    if append:
+        if new_description.strip().lower() in current_description.lower():
+            print("ℹ️ Description already contains that string, skipping")
+            return snippet
+        final_description = (snippet.get("description", "") + "\n\n" + new_description).strip()
+    else:
+        final_description = new_description
+
+    # YouTube requires you to send title + categoryId + description when updating snippet
+    body = {
+        "id": video_id,
+        "snippet": {
+            "title": current_title,
+            "description": final_description,
+            "categoryId": current_category,
+            # Keep existing tags, etc if you want
+            "tags": snippet.get("tags", []),
+            "defaultLanguage": snippet.get("defaultLanguage"),
+        }
+    }
+
+    try:
+        update_response = youtube.videos().update(
+            part="snippet",
+            body=body
+        ).execute()
+
+        print(f"✅ Description updated for {video_id}")
+        print(f"New length: {len(final_description)}")
+        return update_response["snippet"]
+
+    except HttpError as e:
+        print(f"❌ Failed to update video {video_id}: {e}")
+        if "quotaExceeded" in str(e):
+            print("You hit the YouTube API quota.")
+        return None
+
+def append_tag_if_not_exists(video_id: str, new_tag: str):
+    """
+    Appends a new tag to a YouTube video if it doesn't already exist.
+    Case-insensitive check.
+    """
+    if not new_tag or not new_tag.strip():
+        print("❌ Tag is empty")
+        return None
+
+    new_tag_clean = new_tag.strip()
+
+    # Load credentials
+    if not CREDENTIALS:
+        print("❌ Error: YouTube API credentials not configured.")
+        return None
+        
+    # Create YouTube API client
+    youtube = build('youtube', 'v3', credentials=CREDENTIALS)
+    
+    video = get_video_details(youtube, video_id)
+    if not video:
+        return None
+    
+    snippet = video["snippet"]
+    existing_tags = snippet.get("tags", []) or []
+    existing_lower = {t.lower().strip() for t in existing_tags}
+
+    if new_tag_clean.lower() in existing_lower:
+        print(f"ℹ️ Tag '{new_tag_clean}' already exists on {video_id}, skipping")
+        return existing_tags
+
+    updated_tags = existing_tags + [new_tag_clean]
+
+    total_chars = sum(len(t) for t in updated_tags)
+    if total_chars > 450:
+        print(f"⚠️ Tags approaching 500 char limit ({total_chars})")
+    if len(updated_tags) > 15:
+        print(f"⚠️ {len(updated_tags)} tags - YouTube ignores after ~15-20")
+
+    print(f"Current tags: {existing_tags}")
+    print(f"Adding: '{new_tag_clean}'")
+
+    body = {
+        "id": video_id,
+        "snippet": {
+            "title": snippet["title"],
+            "description": snippet.get("description", ""),
+            "categoryId": snippet.get("categoryId", "22"),
+            "tags": updated_tags,
+        }
+    }
+    try:
+        resp = youtube.videos().update(part="snippet", body=body).execute()
+        print(f"✅ Tag '{new_tag_clean}' added to {video_id}")
+        return resp["snippet"].get("tags", [])
+    except HttpError as e:
+        print(f"❌ Failed to add tag: {e}")
+        return None
+
+def main():
+    videos = DB.get_youtube_info_all()
+    for video in videos:
+        if video.yt_id:
+            update_video_description(video.yt_id, f"({video.id})", True)
+            append_tag_if_not_exists(video.yt_id, video.id)
+        if video.yt_short_id:
+            update_video_description(video.yt_short_id, f"({video.id})", True)
+            append_tag_if_not_exists(video.yt_short_id, video.id)
+
+
+# --- Example Usage ---
+if __name__ == "__main__":
+    main()
+    
+    # Example 1: Replace description
+    # update_video_description("YOUR_VIDEO_ID_HERE", "This is my new description!\nGame: Valorant")
+
+    # Example 2: Append to description
+    # update_video_description("YOUR_VIDEO_ID_HERE", "\n\nCheck out https://twitch.tv/teampgp", append=True)
+
+    # For integration with your existing code:
+    #vid = input("Enter YouTube Video ID: ").strip()
+    #if vid:
+    #    new_desc = input("Enter new description: ").strip()
+    #    update_video_description(vid, new_desc)

package/__init__.py

@@ -20,7 +20,7 @@ from .config import CHANNEL_NAME, TZ, DB
 from .hash_tags import get_top_nouns
 from .process_uploads import process_uploads
 from .transcribe_video import transcribe_to_srt
-from .utils import write_csv, run_command
+from .utils import get_chapters, get_video_id_from_chat, write_csv, run_command
 
 # 4. Expose Subpackages
 from . import twitch
@@ -39,6 +39,8 @@ __all__ = [
     "transcribe_to_srt", 
     "write_csv",
     "run_command",
+    "get_chapters",
+    "get_video_id_from_chat"
     "twitch",
     "youtube"
 ]

package/config.py

@@ -2,9 +2,9 @@
 from zoneinfo import ZoneInfo
 
 #import package.database as database
-from package.database.database import Database
+from package.database.database import Repository
 
 CHANNEL_NAME = "teampgp"
 TZ = ZoneInfo("America/Toronto")
 
-DB = Database()
+DB = Repository()

package/database/__init__.py

@@ -1,12 +1,13 @@
 # package/database/__init__.py
 
 # Expose components from individual modules using relative imports
-from .database import TwitchVideo, Data, YoutubeInfo, Database
+from .database import TwitchVideo, Data, YoutubeInfo, Chapter, Repository
 
 # Explicitly define what is exposed when importing the subpackage
 __all__ = [
     "TwitchVideo",
     "Data",
     "YoutubeInfo",
-    "Database",
+    "Chapter",
+    "Repository",
 ]

package/database/database.py

@@ -26,8 +26,12 @@ class TwitchVideo(Base):
     uploaded_yt_chats = Column(Boolean, nullable=False, default=False)
     uploaded_yt_shorts = Column(Boolean, nullable=False, default=False)
 
-    # Relationship back to YoutubeInfo (uselist=False enforces a 1-to-1 structure)
+    # This now actually points to twitch_videos.id
-    youtube_info = relationship("YoutubeInfo", back_populates="twitch_video", uselist=False)
+    related_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=True)
+
+    # Relationship back to YoutubeInfo (uselist=True enforces a 1-to-many structure)
+    youtube_info = relationship("YoutubeInfo", back_populates="twitch_video", uselist=True)
+    chapters = relationship("Chapter", back_populates="twitch_video", cascade="all, delete-orphan")
 
 # 3. Define the data table
 class Data(Base):
@@ -43,7 +47,8 @@ class Data(Base):
 class YoutubeInfo(Base):
     __tablename__ = "youtube_info"
 
-    id = Column(String, ForeignKey("twitch_videos.id"), primary_key=True)
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
     yt_id = Column(String, nullable=True)
     yt_date = Column(String, nullable=True)
     yt_short_id = Column(String, nullable=True)
@@ -56,8 +61,22 @@ class YoutubeInfo(Base):
     # Relationship linking back to the twitch video record
     twitch_video = relationship("TwitchVideo", back_populates="youtube_info")
 
+class Chapter(Base):
+    __tablename__ = "chapters"
 
-class Database:
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
+    start_milliseconds = Column(Integer, nullable=False)
+    length_milliseconds = Column(Integer, nullable=False)
+    description = Column(String, nullable=True)
+    type = Column(String, nullable=True)
+    game_display_name = Column(String, nullable=True)
+    game_id = Column(String, nullable=True)
+
+    # Relationship linking back to the twitch video record
+    twitch_video = relationship("TwitchVideo", back_populates="chapters")    
+
+class Repository:
     def __init__(self, db_path: str = "database.db"):
         # 4. Connect to the SQLite file and generate the tables
         # This will create 'database.db' in the current working directory if it doesn't exist.
@@ -68,6 +87,15 @@ class Database:
         Session = sessionmaker(bind=engine)
         self.session =  Session()
 
+    def set_related_video(self, record_id: str, related_video_id: str):
+        record = self.session.get(TwitchVideo, record_id)
+
+        if record:
+            setattr(record, "related_video", related_video_id)
+            self.session.commit()
+        else:
+            print(f"❌ Record {record_id} not found.")
+
     def __mark_as(self, record_id: str, field_name: str, value: bool = True):
         record = self.session.get(TwitchVideo, record_id)
         
@@ -178,33 +206,67 @@ class Database:
         stmt = select(Data)
         return self.session.scalars(stmt).all()
 
-    def get_youtube_info(self, record_id) -> YoutubeInfo | None:
+    def get_youtube_info_for_video(self, video_id: str) -> list[YoutubeInfo]:
-        return self.session.get(YoutubeInfo, record_id)
+        """Retrieve all YouTubeInfo records associated with a specific TwitchVideo ID."""
+        stmt = select(YoutubeInfo).where(YoutubeInfo.twitch_video_id == video_id)
+        return self.session.scalars(stmt).all()
 
-    def insert(self, data: TwitchVideo | Data | YoutubeInfo):
+    def get_youtube_info_all(self) -> list[YoutubeInfo]:
-        # SQLAlchemy 2.0 context manager handles commit/rollback automatically
+        return self.session.scalars(select(YoutubeInfo)).all()
+
+    def insert(self, data: TwitchVideo | Data | YoutubeInfo | Chapter):
+    def insert(self, data: TwitchVideo | Data | YoutubeInfo | Chapter | list[YoutubeInfo] | list[Chapter]):
+        """
+        Handles insertion/updating for any of the defined models,
+        using session merge/add for SQLAlchemy 2.0 best practices.
+        Accepts single objects or lists of objects for collection handling.
+        """
         try:
             if isinstance(data, TwitchVideo):
-                # 1. Query for the existing row using SQLAlchemy 2.0 scalar execution
+                # 1. Check if the record exists
                 stmt = select(TwitchVideo).where(TwitchVideo.id == data.id)
                 existing_record = self.session.scalar(stmt)
                 
                 if existing_record:
+                    # 2. Update relevant fields on the managed instance
                     if existing_record.view_count != data.view_count or existing_record.duration != data.duration:
-                        # 2. Update the columns on the managed instance
                         existing_record.view_count = data.view_count
                         existing_record.duration = data.duration
-                        print(f"🔄 Record {data.id} updated successfully!")
+                        print(f"🔄 TwitchVideo record {data.id} updated successfully!")
+                    # Note: Add more field updates here as needed
                 else:
-                    # 3. Add new instance if it doesn't exist
+                    # 3. Add new instance
                     self.session.add(data)
-                    print("➕ New TwitchVideo record inserted successfully!")
+                    print(f"➕ New TwitchVideo record with ID {data.id} inserted successfully!")
-            else:                   
+            elif isinstance(data, Data):
                 self.session.merge(data)
-                print(f"➕ {type(data).__name__} record inserted successfully!")
+                print(f"➕ Data record merged/inserted successfully!")
-            # 4. Flush changes and commit the transaction
+            elif isinstance(data, YoutubeInfo):
-            self.session.commit()
+                # Handle single YoutubeInfo object
+                self.session.merge(data)
+                print(f"➕ YoutubeInfo record merged/inserted successfully!")
+            elif isinstance(data, Chapter):
+                # Handle single Chapter object
+                self.session.merge(data)
+                print(f"➕ Chapter record merged/inserted successfully!")
+            elif isinstance(data, list):
+                # Handle lists of records (YoutubeInfo or Chapter)
+                record_type = type(data[0])
+                if record_type in (YoutubeInfo, Chapter):
+                    for item in data:
+                        self.session.merge(item)
+                    print(f"➕ Successfully processed {len(data)} records of type {record_type.__name__}.")
+                else:
+                    raise TypeError(f"Unsupported list content type: {record_type.__name__}")
+            else:
+                raise TypeError(f"Unsupported data type for insertion: {type(data).__name__}")
+            
+            # 4. Commit the transaction
+            self.session.flush() # Flush first to ensure IDs are populated
             
         except Exception as e:
             self.session.rollback()
-            print(f"❌ An error occurred, transaction rolled back: {e}")
+            print(f"❌ A critical error occurred during record insertion/update: {e}")
+            raise
+
+        self.session.commit()

package/process_uploads.py

@@ -14,9 +14,10 @@ from package.youtube.uploader import CategoryId as CategoryId
 
 from package.database import TwitchVideo, YoutubeInfo, Data
 
-def process_videos(video_type: bool = True):
+def process_videos(video_type: bool = True, limit: int = 22):
+    count = 0 # count how many videos we have upload and stop at var: limit
     videos = DB.get_unuploaded_chats() if video_type else DB.get_unuploaded_shorts()
-    file_type = "download/clips" if video_type else "download/video"
+    file_type = "download/video" if video_type else "download/clips"
     meta_data: Data = DB.get_data()[0]
 
     current_time = datetime.now(TZ)
@@ -29,6 +30,10 @@ def process_videos(video_type: bool = True):
             release_time += timedelta(days=1)
 
     for video in videos:
+        #We reached limit stop uploading videos
+        if count >= limit:
+            break
+
         base_path = Path(f"{file_type}/{video.id}/{video.id}")
         original_path = base_path.with_suffix(".mp4")
 
@@ -66,6 +71,7 @@ def process_videos(video_type: bool = True):
         tags = ['gaming', 'TeamPGP', f'{video.game_name}', f'{video.creator_name}', 'Level1Techs', 'twitch']
         if not video_type:
             tags.extend(['shorts', 'twitch_clips', 'clips'])
+        tags.extend([video.id])
 
         if srt_path.exists():
             try:
@@ -78,7 +84,8 @@ def process_videos(video_type: bool = True):
         if youtube_info.yt_id:
             main_video_id = youtube_info.yt_id
         else:
-            main_video_id = upload_video(str(original_path), video.title, category_id, description, 'unlisted', tags)
+            main_video_id = upload_video(str(original_path), video.title, category_id, f"{description} \n\n({video.id})", 'unlisted', tags)
+            count += 1
             if not main_video_id:
                 print(f"❌ Error: main video upload failed for {original_path}")
                 continue
@@ -86,7 +93,7 @@ def process_videos(video_type: bool = True):
             youtube_info.yt_date = datetime.now(TZ)
 
         # Upload secondary
-        secondary_desc = f"{description} Original Full size @ https://youtu.be/{main_video_id}"
+        secondary_desc = f"{description} Original Full size @ https://youtu.be/{main_video_id} \n\n({video.id})"
         secondary_video_id = upload_video(str(file_path), video.title, category_id, secondary_desc, 'private', tags, release_time)
 
         if not secondary_video_id:
@@ -94,6 +101,8 @@ def process_videos(video_type: bool = True):
             DB.insert(youtube_info) # save main id at least
             continue
 
+        count += 1
+
         # Success - write CSV and DB
         data_csv = [main_video_id, secondary_video_id, release_time.isoformat()]
         utils.write_csv([data_csv], str(file_csv))

package/twitch/download_thumbnails.py

@@ -93,7 +93,7 @@ async def download_thumbnails():
     # --- OPTION B: Download Past Broadcast VOD Thumbnail ---
     # Extract the ID sequence from your target video link
     # target_vod = "2145678901"
-    db = database.Database()
+    db = database.Repository()
     vods = db.get_vods()
     for vod in vods:
         (

package/twitch/download_videos.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 import subprocess
+from pathlib import Path
 from datetime import datetime, timedelta, timezone
 
 import package.utils as utils
 from package.config import DB as DB
-from package.database import TwitchVideo
+from package.config import TZ as TZ
+from package.database import TwitchVideo, Chapter
 
 def download_videos():
     """Find all undownloaded videos and clips, and download them safely."""
@@ -12,59 +14,72 @@ def download_videos():
 
     print("Starting downloads...")
     for row in undownloaded:
-        # 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM)
+        try:
-        #datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
+            # 2026-08-16T23:52:10Z -> replace Z with +00:00
-        data_timestamp = datetime.fromisoformat(row.created_at)
+            data_timestamp = datetime.fromisoformat(row.created_at.replace("Z", "+00:00"))
+        except Exception:
+            data_timestamp = datetime.strptime(row.created_at, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
 
-        # 2. Get the exact current date and time
+        if datetime.now(timezone.utc) - data_timestamp < timedelta(hours=24):
-        current_time = datetime.now(timezone.utc)
+            print(f"Skipping {row.id} - less than 24h old")
-
-        # 3. Calculate the difference between the two times
-        time_difference = current_time - data_timestamp
-
-        # 4. Check if the difference is greater than 24 hours
-        if time_difference > timedelta(hours=24):
-            print("The data is more than 24 hours old.")
-        else:
-            #lets wait 24 hours befor downloading
-            print("The data is less than 24 hours old.")
             continue
 
-        data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
         print(f"ID: {row.id} | Date: {row.created_at} | Game: {row.game_name} | Title: {row.title}")
 
-        # 1. Define paths and isolate base directory
+        base_dir = Path(f"download/clips/{row.id}" if row.clip_is else f"download/videos/{row.id}")
+        base_dir.mkdir(parents=True, exist_ok=True)
+
+        temp_path = Path("download/temp")
+        temp_path.mkdir(parents=True, exist_ok=True)
+
+        target_video = base_dir / f"{row.id}.mp4"
+        json_file = base_dir / f"{row.id}_chat.json"
+        csv_file = base_dir / f"{row.id}.csv"
+
         if row.clip_is:
-            target_dir = f"download/clips/{row.id}"
+            cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_video} --collision Overwrite --temp-path {temp_path}"
-            cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --temp-path download/temp"
         else:
-            target_dir = f"download/videos/{row.id}"
+            cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_video} --collision Overwrite --threads 2 --temp-path {temp_path}"
-            # FIXED: Removed the duplicated command string combined with '&&'
-            cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
 
-        csv_file = f"{target_dir}/{row.id}.csv"
-
-        # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
         output = utils.run_command(cmd, look_for=["[STATUS]"])
-        output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {row.id} -o {target_dir}/{row.id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
+        output2 = utils.run_command(
+            f"TwitchDownloaderCLI chatdownload --id {row.id} -o {json_file} -E --collision Overwrite --temp-path {temp_path}",
+            look_for=["[STATUS]"]
+        )
+
+        if output2 and json_file.exists():
+            related_id = utils.get_video_id_from_chat(json_file)
+            if related_id:
+                # This will set related_video_id = 2848295283 for a clip
+                DB.set_related_video(row.id, related_id)
+                print(f" -> Linked {row.id} -> related VOD {related_id}")
+            if row.clip_is:
+                chapters = utils.get_chapters(json_file)
+                for chapter in chapters:
+                    chapter_record = Chapter(
+                        twitch_video_id=row.id,
+                        start_ms=chapter.get('startMilliseconds'),
+                        length_ms=chapter.get('lengthMilliseconds'),
+                        description=chapter.get('description'),
+                        type=chapter.get('type'),
+                        game_display_name=chapter.get('gameDisplayName'),
+                        game_id=chapter.get('gameId')
+                    )
+                    DB.insert(chapter_record)
+                    print(f" ➕ Chapter added: {chapter_record.description} ({chapter_record.start_ms}ms)")
 
         if output:
-            # Only write CSV and update database if download actually completed
+            # FIX 4: write_csv needs actual row data, not column names
-            data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
+            utils.write_csv([row], str(csv_file)) # pass the object/dict, not vars(TwitchVideo)
-            utils.write_csv(data, csv_file)
+            print(f"✅ Success: {base_dir}")
-            print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
             DB.mark_as_downloaded(row.id)
         else:
-            print(f"ID: {row.id} | Download Process failed.")
+            print(f"❌ ID: {row.id} | Download failed.")
-            #print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
-            #if output.get("stderr"):
-            #    print(f"Details: {output['stderr']}")
-
-            # Clear temporary chunk clutter immediately if a VOD crashes out
             if not row.clip_is:
                 print("Flushing temporary crash chunks...")
                 subprocess.run("rm -rf download/temp/*", shell=True)
 
+
     print("Finished Downloading Pipeline.")
 
 if __name__ == "__main__":

package/utils.py

@@ -1,10 +1,50 @@
 #!/usr/bin/env python3
 import csv
+from pathlib import Path
 import sys
+import json
 import shlex
 import subprocess
 
-def write_csv(data, file_name):
+def get_chapters(json_path):
+    """Loads the JSON file and returns all chapters."""
+    with open(json_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+
+    # The chapters are under video -> chapters
+    video = data.get('video', {})
+    chapters = video.get('chapters', [])
+
+    if not chapters:
+        print("No chapters found.")
+        return []
+
+    print(f"Found {len(chapters)} chapter(s) in video '{video.get('title')}':\n")
+
+    for i, chapter in enumerate(chapters, 1):
+        start_sec = chapter.get('startMilliseconds', 0) / 1000
+        length_sec = chapter.get('lengthMilliseconds', 0) / 1000
+
+        #print(f"{i}. {chapter.get('description', 'No Title')}")
+        #print(f" - Type: {chapter.get('type')}")
+        #print(f" - Start: {start_sec}s ({chapter.get('startMilliseconds')}ms)")
+        #print(f" - Length: {length_sec}s ({chapter.get('lengthMilliseconds')}ms)")
+        #print(f" - Game: {chapter.get('gameDisplayName')} (ID: {chapter.get('gameId')})")
+        #print()
+
+    return chapters
+
+def get_video_id_from_chat(json_path: str | Path) -> str | None:
+    """Gets the id inside video:{ } - from the chat json you download"""
+    try:
+        with open(json_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return data.get("video", {}).get("id")
+    except Exception as e:
+        print(f"Could not read video.id from {json_path}: {e}")
+        return None
+
+def write_csv(data: list, file_name: str):
     # Open file with newline='' to prevent extra blank rows across platforms
     with open(file_name, "w", newline="", encoding="utf-8") as file:
         writer = csv.writer(file)