update system to use find Highlights *not tested

.gitignore

@@ -9,3 +9,6 @@ twitch_secrets.json
 download
 output.log
 *.mp4
+database copy.db
+tiktok_secrets.json
+cookies.txt

delete/database.py

@@ -7,6 +7,7 @@ from typing import Any
 class Database:
     def __init__(self, db_path: str = "database.db"):
         self.columns = "id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts"
+        self.data = "id, name, date_shorts, date_videos, date_videos_chat"
 
         file_exists = Path(db_path).is_file()
 
@@ -46,6 +47,16 @@ class Database:
         )
         """)
 
+        self.cursor.execute("""
+        CREATE TABLE IF NOT EXISTS data (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL,
+            date_shorts TEXT NOT NULL,
+            date_videos TEXT NOT NULL,
+            date_videos_chat TEXT NOT NULL
+        )
+        """)
+        
         self.conn.commit()
 
     def close_database(self):
@@ -167,3 +178,55 @@ class Database:
             # Prevent silent failures if the database connection drops
             print(f"❌ Database insertion failed: {e}")
             self.conn.rollback()
+
+    def __update_date(self, record_id: str, set_row: str, date: str):
+        """Updates the Datetime for timezone fix"""
+        self.cursor.execute(
+            f"UPDATE data SET {set_row} = ? WHERE id = ?",
+            (date, record_id))
+        self.conn.commit()
+
+    def update_date_shorts(self, record_id: str, date_shorts: str):
+        self.__update_date(record_id, "date_shorts", date_shorts)
+
+    def get_data(self) -> list[Any]:
+        self.cursor.execute(f"SELECT {self.data} FROM data")
+        return self.cursor.fetchall()
+
+    def insert_data_record(self, name: str, date_shorts: str, date_videos: str, date_videos_chat: str):
+        try:
+            dt_obj = datetime.strptime(date_shorts, "%Y-%m-%dT%H:%M:%SZ")
+            clean_date_shorts = dt_obj.strftime("%Y-%m-%d %H:%M:%S")
+        except (ValueError, TypeError):
+            clean_date_shorts = date_shorts
+
+        try:
+            dt_obj = datetime.strptime(date_videos, "%Y-%m-%dT%H:%M:%SZ")
+            clean_date_videos = dt_obj.strftime("%Y-%m-%d %H:%M:%S")
+        except (ValueError, TypeError):
+            clean_date_videos = date_videos
+
+        try:
+            dt_obj = datetime.strptime(date_videos_chat, "%Y-%m-%dT%H:%M:%SZ")
+            clean_date_videos_chat = dt_obj.strftime("%Y-%m-%d %H:%M:%S")
+        except (ValueError, TypeError):
+            clean_date_videos_chat = date_videos_chat
+
+        # Explicitly defining columns removes the security risk and column-count bug
+        query = """
+            INSERT INTO data (
+                name, date_shorts, date_videos, date_videos_chat
+            ) VALUES (?, ?, ?, ?)
+        """
+
+        values = (
+            name, clean_date_shorts, clean_date_videos, clean_date_videos_chat
+        )
+
+        try:
+            self.cursor.execute(query, values)
+            self.conn.commit()
+        except Exception as e:
+            # Prevent silent failures if the database connection drops
+            print(f"❌ Database insertion failed: {e}")
+            self.conn.rollback()

delete/twitch_down_uploader.py

@@ -1,15 +1,15 @@
 #!/usr/bin/env python3
 import requests
 import os
-import package.linux as linux
+import package.utils as linux
 import time
 
-from package.database import Database
+from package.database import Repository
 
-import uploader
-from uploader import CategoryId
+import package.youtube.uploader as uploader
+from package.youtube.uploader import CategoryId
 
-import youtube_hashtags
+import package.hash_tags as hash_tags
 import package.transcribe_video as transcribe_video
 
 CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
@@ -43,7 +43,7 @@ def top_hashtags(id: str):
         print(f"Transcribe file not found {file_srt}")
         return []
 
-    tags = youtube_hashtags.get_top_hashtags(file_srt)
+    tags = hash_tags.get_top_hashtags(file_srt)
     return tags
 
 def download():
@@ -319,7 +319,7 @@ def get_vod_ids_simplified(channel_name: str):
 if __name__ == "__main__":
     tables = ["clips"]
     for table in tables:
-        DB = Database(table)
+        DB = Repository(table)
 
         get_vod_ids_simplified(CHANNEL_NAME)
         #download()

delete/twitch_download_clips.py

@@ -3,13 +3,13 @@ import requests
 import subprocess
 from datetime import datetime
 
-from package.database import Database
+from package.database import Repository
 
-import uploader
-from uploader import CategoryId
+import package.youtube.uploader as uploader
+from package.youtube.uploader import CategoryId
 
 CHANNEL_NAME = 'teampgp'  # Replace with the streamer's username
-DB = Database("clips")
+DB = Repository("clips")
 
 def run_linux_command(command: str):
     """Executes a Linux command, waits for completion, and returns output."""
@@ -35,7 +35,7 @@ def transcribe(slug: str):
         print(f"video already transcribed:")
         return True
 
-    import transcribe_video
+    import package.transcribe_video as transcribe_video
     transcribe_video.extract_audio(video_file, f"save/clips/{slug}/temp_{slug}_audio.wav")
     transcribe_video.transcribe_to_srt(f"save/clips/{slug}/temp_{slug}_audio.wav", f"save/clips/{slug}/", f"transcribe_{slug}")
 
@@ -49,9 +49,9 @@ def top_hashtags(slug: str):
         print(f"Transcribe file not found {file_srt}")
         return []
 
-    import youtube_hashtags
+    import package.hash_tags as hash_tags
 
-    return youtube_hashtags.get_top_hashtags(file_srt)
+    return hash_tags.get_top_hashtags(file_srt)
 
 
 def download_clips():

help.sqlite3-query

@@ -0,0 +1,12 @@
+-- database: ./database.db
+
+SELECT * FROM twitch_videos WHERE clip_is == "1";
+UPDATE twitch_videos SET uploaded_yt_shorts = 0 WHERE uploaded_yt_shorts == "1";
+SELECT * FROM twitch_videos WHERE title == "Oh Dear";
+SELECT * FROM twitch_videos WHERE title = 'Kreestuh FINALLY hits her "Ryan Dark Story" limit.  Jesus Big K it''s about time!';
+SELECT * FROM twitch_videos WHERE title = "Bolt-Action Bureaucracy";
+SELECT * FROM twitch_videos WHERE title = "Just after going through a mass starvation...";
+SELECT * FROM twitch_videos WHERE title = "Semi-Automatic Sanctimony";
+SELECT * FROM twitch_videos WHERE id = "2848295283";
+SELECT * FROM twitch_videos WHERE game_name = "Darkwood";
+SELECT * FROM data;

helper_files/chapters.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+import os
+import json
+import sys
+
+# Finds the absolute path to 'project_directory' and adds it to Python's search path
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+sys.path.append(PROJECT_ROOT)
+
+from package.config import DB as DB
+from package.database import Chapter
+
+def get_chapters(json_path):
+    """Loads the JSON file and returns all chapters."""
+    with open(json_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+
+    # The chapters are under video -> chapters
+    video = data.get('video', {})
+    chapters = video.get('chapters', [])
+
+    if not chapters:
+        print("No chapters found.")
+        return []
+
+    print(f"Found {len(chapters)} chapter(s) in video '{video.get('title')}':\n")
+
+    for i, chapter in enumerate(chapters, 1):
+        start_sec = chapter.get('startMilliseconds', 0) / 1000
+        length_sec = chapter.get('lengthMilliseconds', 0) / 1000
+
+        print(f"{i}. {chapter.get('description', 'No Title')}")
+        print(f" - Type: {chapter.get('type')}")
+        print(f" - Start: {start_sec}s ({chapter.get('startMilliseconds')}ms)")
+        print(f" - Length: {length_sec}s ({chapter.get('lengthMilliseconds')}ms)")
+        print(f" - Game: {chapter.get('gameDisplayName')} (ID: {chapter.get('gameId')})")
+        print()
+
+    return chapters
+
+def main():
+    downloads = DB.get_downloaded()
+
+    for download in downloads:
+        json_path = f"download/clips/{download.id}/{download.id}_chat.json" if download.clip_is else f"download/videos/{download.id}/{download.id}_chat.json"
+        if os.path.exists(json_path):
+            chapters = get_chapters(json_path)
+            for chapter in chapters:
+                chapter_record = Chapter(
+                    twitch_video_id=download.id,
+                    start_milliseconds=chapter.get('startMilliseconds'),
+                    length_milliseconds=chapter.get('lengthMilliseconds'),
+                    description=chapter.get('description'),
+                    type=chapter.get('type'),
+                    game_display_name=chapter.get('gameDisplayName'),
+                    game_id=chapter.get('gameId')
+                )
+                DB.insert(chapter_record)
+        else:
+            print(f"JSON file not found for video {download.id}: {json_path}")
+
+if __name__ == "__main__":
+    main()
+    # Usage: python get_chapters.py your_file.json
+    #path = sys.argv[1] if len(sys.argv) > 1 else "/home/richard/python_scripts/download/videos/2873107835/2873107835_chat.json"
+    #try:
+    #    get_chapters(path)
+    #except FileNotFoundError:
+    #    print(f"Error: File not found -> {path}")
+    #except json.JSONDecodeError:
+    #    print(f"Error: File is not valid JSON -> {path}")

helper_files/datetime_fix.py

@@ -55,7 +55,7 @@ def fix_time():
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
     fix_time()
 

helper_files/fix_download.py

@@ -44,7 +44,7 @@ def fix_download():
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
     fix_download()
 

helper_files/import_vod.py

@@ -81,7 +81,7 @@ def read_json(file_path: str):
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
 
 

helper_files/migrate_sql2.py

@@ -0,0 +1,45 @@
+import sqlite3
+DB_PATH = "database.db"
+
+def migrate_to_many(db_path):
+    con = sqlite3.connect(db_path)
+    cur = con.cursor()
+    cur.execute("PRAGMA foreign_keys=OFF;")
+    try:
+        con.execute("BEGIN;")
+        cur.execute("ALTER TABLE youtube_info RENAME TO youtube_info_old;")
+        
+        cur.execute("""
+        CREATE TABLE youtube_info (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            twitch_video_id VARCHAR NOT NULL,
+            yt_id VARCHAR,
+            yt_date VARCHAR,
+            yt_short_id VARCHAR,
+            yt_short_date VARCHAR,
+            yt_chat_SS_id VARCHAR,
+            yt_chat_SS_date VARCHAR,
+            yt_chat_over_id VARCHAR,
+            yt_chat_over_date VARCHAR,
+            FOREIGN KEY(twitch_video_id) REFERENCES twitch_videos(id) ON DELETE CASCADE
+        );
+        """)
+        cur.execute("CREATE INDEX ix_youtube_info_twitch_video_id ON youtube_info (twitch_video_id);")
+
+        cur.execute("""
+        INSERT INTO youtube_info (twitch_video_id, yt_id, yt_date, yt_short_id, yt_short_date, yt_chat_SS_id, yt_chat_SS_date, yt_chat_over_id, yt_chat_over_date)
+        SELECT id, yt_id, yt_date, yt_short_id, yt_short_date, yt_chat_SS_id, yt_chat_SS_date, yt_chat_over_id, yt_chat_over_date
+        FROM youtube_info_old;
+        """)
+        
+        cur.execute("DROP TABLE youtube_info_old;")
+        con.commit()
+        print("Done - now 1-to-Many")
+    except Exception as e:
+        con.rollback()
+        raise e
+    finally:
+        cur.execute("PRAGMA foreign_keys=ON;")
+        con.close()
+
+migrate_to_many(DB_PATH)

helper_files/move_vod.py

@@ -49,7 +49,7 @@ def remove_vods():
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
     remove_vods()
 

helper_files/redownload_clips.py

@@ -31,5 +31,5 @@ def redownload_clips():
     
 
 if __name__ == "__main__":
-    DB = database.Database()
+    DB = database.Repository()
     redownload_clips()

helper_files/remove_uneed_files.py

@@ -1,50 +1,36 @@
 import os
+import sys
 
-import package.database as database
+# Finds the absolute path to 'project_directory' and adds it to Python's search path
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+sys.path.append(PROJECT_ROOT)
+
+# Your original import will now work
+from package.config import DB as DB
 
-DB = None
 
 def remove_files():
-    datas = DB.get_download()
-
-    top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
+    datas = DB.get_downloaded()
 
     for data in datas:
-        (
-            id,
-            title,
-            created_at,
-            view_count,
-            duration,
-            url,
-            thumbnail_url,
-            game_id,
-            game_name,
-            stream_id,
-            creator_name,
-            clip_is,
-            downloaded,
-            uploaded_yt,
-            uploaded_yt_chats,
-            uploaded_yt_shorts,
-        ) = data
         
-        if clip_is == 0:
-            video_path = f"download/videos/{id}/{id}.mp4"
+        if data.clip_is is False:
+            #continue
+            video_path = f"download/videos/{data.id}/{data.id}.mp4"
         else:
-            video_path = f"download/clips/{id}/{id}.mp4"
+            video_path = f"download/clips/{data.id}/{data.id}.mp4"
         
-        temp_chat = video_path.replace(".mp4", "_temp_chat.mp4")
-        temp_with_chat = video_path.replace(".mp4", "_temp_with_chat.mp4")
-        video_chat = video_path.replace(".mp4", "_with_chat.mp4")
-        video_chatSS = video_path.replace(".mp4", "_SS_with_chat.mp4")
-        video_chat_over = video_path.replace(".mp4", "_over_with_chat.mp4")
-        mask_path = video_path.replace(".mp4", "_temp_chat_mask.mp4")
+        temp_chat = ""#video_path.replace(".mp4", "_temp_chat.mp4")
+        temp_with_chat = ""#video_path.replace(".mp4", "_temp_with_chat.mp4")
+        video_chat = ""#video_path.replace(".mp4", "_with_chat.mp4")
+        video_chatSS = ""#video_path.replace(".mp4", "_SS_with_chat.mp4")
+        video_chat_over = ""#video_path.replace(".mp4", "_over_with_chat.mp4")
+        mask_path = ""#video_path.replace(".mp4", "_temp_chat_mask.mp4")
 
         shorts_gaussian_9_16 = ""#video_path.replace(".mp4", "_gaussian_9_16.mp4")
         shorts_9_16 = ""#video_path.replace(".mp4", "_9_16.mp4")
 
-        srt_file = ""#video_path.replace(".mp4", ".srt")
+        srt_file = video_path.replace(".mp4", ".srt")
         
 
         files = [temp_chat, temp_with_chat, video_chatSS, video_chat_over, mask_path, shorts_gaussian_9_16, shorts_9_16, srt_file, video_chat]
@@ -62,11 +48,9 @@ def redownload():
 
 
 def main():
-    global DB
-    DB = database.Database()
-
     remove_files()
-    redownload()
+    print("done")
+    #redownload()
 
 if __name__ == "__main__":
     main()

helper_files/youtube_description_updater.py

@@ -0,0 +1,225 @@
+"""
+YouTube Description Updater
+Requires: pip install google-api-python-client google-auth-oauthlib google-auth
+
+1. Enable YouTube Data API v3 in Google Cloud Console
+2. Create OAuth 2.0 credentials (Desktop App)
+3. Download as client_secret.json
+"""
+
+import os
+import sys
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+# Finds the absolute path to 'project_directory' and adds it to Python's search path
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+sys.path.append(PROJECT_ROOT)
+
+from package.config import DB as DB
+from package.youtube.config import CREDENTIALS as CREDENTIALS
+from package.database import YoutubeInfo as YoutubeInfo
+
+"""
+SCOPES = ["https://www.googleapis.com/auth/youtube.force-ssl"]
+TOKEN_FILE = "token.json"
+CLIENT_SECRET_FILE = "client_secret.json"
+
+def get_authenticated_service():
+    ""Authenticates and returns a YouTube service object.""
+    creds = None
+    if os.path.exists(TOKEN_FILE):
+        creds = Credentials.from_authorized_user_file(TOKEN_FILE, SCOPES)
+    
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            if not os.path.exists(CLIENT_SECRET_FILE):
+                raise FileNotFoundError(
+                    f"{CLIENT_SECRET_FILE} not found. Download it from Google Cloud Console."
+                )
+            flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET_FILE, SCOPES)
+            creds = flow.run_local_server(port=0)
+        
+        with open(TOKEN_FILE, "w") as token:
+            token.write(creds.to_json())
+
+    return build("youtube", "v3", credentials=creds)
+"""
+
+def get_video_details(youtube, video_id: str):
+    """Finds the video and returns its current snippet."""
+    try:
+        response = youtube.videos().list(
+            part="snippet,status",
+            id=video_id
+        ).execute()
+
+        if not response["items"]:
+            print(f"❌ Video not found: {video_id}")
+            return None
+        
+        return response["items"][0]
+    except HttpError as e:
+        print(f"❌ API Error while fetching video: {e}")
+        return None
+
+def update_video_description(video_id: str, new_description: str, append: bool = False):
+    """
+    Takes a YouTube video ID, finds the video, and changes its description.
+
+    Args:
+        video_id (str): The YouTube video ID (e.g., 'dQw4w9WgXcQ')
+        new_description (str): The new description to set
+        append (bool): If True, appends new_description to existing description instead of replacing
+
+    Returns:
+        dict | None: Updated video snippet or None on failure
+    """
+    # Load credentials
+    if not CREDENTIALS:
+        print("❌ Error: YouTube API credentials not configured.")
+        return None
+    
+    # Create YouTube API client
+    youtube = build('youtube', 'v3', credentials=CREDENTIALS)
+
+    video = get_video_details(youtube, video_id)
+    if not video:
+        return None
+
+    snippet = video["snippet"]
+    current_title = snippet["title"]
+    current_category = snippet.get("categoryId", "22")
+    
+    print(f"Found video: '{current_title}' [{video_id}]")
+    print(f"Current description length: {len(snippet.get('description', ''))}")
+    current_description = snippet.get("description", "") or ""
+
+    # Decide final description
+    if append:
+        if new_description.strip().lower() in current_description.lower():
+            print("ℹ️ Description already contains that string, skipping")
+            return snippet
+        final_description = (snippet.get("description", "") + "\n\n" + new_description).strip()
+    else:
+        final_description = new_description
+
+    # YouTube requires you to send title + categoryId + description when updating snippet
+    body = {
+        "id": video_id,
+        "snippet": {
+            "title": current_title,
+            "description": final_description,
+            "categoryId": current_category,
+            # Keep existing tags, etc if you want
+            "tags": snippet.get("tags", []),
+            "defaultLanguage": snippet.get("defaultLanguage"),
+        }
+    }
+
+    try:
+        update_response = youtube.videos().update(
+            part="snippet",
+            body=body
+        ).execute()
+
+        print(f"✅ Description updated for {video_id}")
+        print(f"New length: {len(final_description)}")
+        return update_response["snippet"]
+
+    except HttpError as e:
+        print(f"❌ Failed to update video {video_id}: {e}")
+        if "quotaExceeded" in str(e):
+            print("You hit the YouTube API quota.")
+        return None
+
+def append_tag_if_not_exists(video_id: str, new_tag: str):
+    """
+    Appends a new tag to a YouTube video if it doesn't already exist.
+    Case-insensitive check.
+    """
+    if not new_tag or not new_tag.strip():
+        print("❌ Tag is empty")
+        return None
+
+    new_tag_clean = new_tag.strip()
+
+    # Load credentials
+    if not CREDENTIALS:
+        print("❌ Error: YouTube API credentials not configured.")
+        return None
+        
+    # Create YouTube API client
+    youtube = build('youtube', 'v3', credentials=CREDENTIALS)
+    
+    video = get_video_details(youtube, video_id)
+    if not video:
+        return None
+    
+    snippet = video["snippet"]
+    existing_tags = snippet.get("tags", []) or []
+    existing_lower = {t.lower().strip() for t in existing_tags}
+
+    if new_tag_clean.lower() in existing_lower:
+        print(f"ℹ️ Tag '{new_tag_clean}' already exists on {video_id}, skipping")
+        return existing_tags
+
+    updated_tags = existing_tags + [new_tag_clean]
+
+    total_chars = sum(len(t) for t in updated_tags)
+    if total_chars > 450:
+        print(f"⚠️ Tags approaching 500 char limit ({total_chars})")
+    if len(updated_tags) > 15:
+        print(f"⚠️ {len(updated_tags)} tags - YouTube ignores after ~15-20")
+
+    print(f"Current tags: {existing_tags}")
+    print(f"Adding: '{new_tag_clean}'")
+
+    body = {
+        "id": video_id,
+        "snippet": {
+            "title": snippet["title"],
+            "description": snippet.get("description", ""),
+            "categoryId": snippet.get("categoryId", "22"),
+            "tags": updated_tags,
+        }
+    }
+    try:
+        resp = youtube.videos().update(part="snippet", body=body).execute()
+        print(f"✅ Tag '{new_tag_clean}' added to {video_id}")
+        return resp["snippet"].get("tags", [])
+    except HttpError as e:
+        print(f"❌ Failed to add tag: {e}")
+        return None
+
+def main():
+    videos = DB.get_youtube_info_all()
+    for video in videos:
+        if video.yt_id:
+            update_video_description(video.yt_id, f"({video.id})", True)
+            append_tag_if_not_exists(video.yt_id, video.id)
+        if video.yt_short_id:
+            update_video_description(video.yt_short_id, f"({video.id})", True)
+            append_tag_if_not_exists(video.yt_short_id, video.id)
+
+
+# --- Example Usage ---
+if __name__ == "__main__":
+    main()
+    
+    # Example 1: Replace description
+    # update_video_description("YOUR_VIDEO_ID_HERE", "This is my new description!\nGame: Valorant")
+
+    # Example 2: Append to description
+    # update_video_description("YOUR_VIDEO_ID_HERE", "\n\nCheck out https://twitch.tv/teampgp", append=True)
+
+    # For integration with your existing code:
+    #vid = input("Enter YouTube Video ID: ").strip()
+    #if vid:
+    #    new_desc = input("Enter new description: ").strip()
+    #    update_video_description(vid, new_desc)

main.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+import os
+import sys
+import fcntl
 import asyncio
 
 import package.twitch.video_info as video_info
@@ -6,18 +9,40 @@ import package.twitch.download_videos as download_videos
 import package.twitch.download_thumbnails as download_thumbnails
 
 import package.build_videos as build_videos
+import package.process_uploads as process_uploads
+
+def lock_file():
+    # Open a lock file
+    lock_file_path = "/tmp/python_script.lock"
+    fp = open(lock_file_path, 'w')
+
+    try:
+        # Try to grab an exclusive lock without blocking (LOCK_NB)
+        fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except IOError:
+        print("Another instance is already running. Exiting.")
+        sys.exit(1)
+
+    # Your script logic goes here
+    print("Running exclusively on Linux/macOS...")
 
 if __name__ == "__main__":
-    # Get Twitch Video Info
-    asyncio.run(video_info())
+    lock_file()
+    upload = True
 
-    # Download Twitch Videos
-    download_videos()
+    if upload is False:
+        # Get Twitch Video Info
+        asyncio.run(video_info())
 
-    download_thumbnails()
+        # Download Twitch Videos
+        download_videos()
 
-    #Build
-    build_videos()
+        download_thumbnails()
+
+        #Build
+        build_videos()
+    else:
+         process_uploads()
 
     print("====================================================")
     print(f"✅ Done.")

package/__init__.py

@@ -14,22 +14,35 @@ os.makedirs(CACHE_DIR, exist_ok=True)
 
 # 3. Expose Database Tools (Added)
 # Replace 'DatabaseClient' and 'connect_to_db' with your actual class/function names
-from .build_videos import main as build_videos
-from .config import CHANNEL_NAME, DB
-from .database import Database
+from .video_editing.build_shorts import fit_to_9_16_letterbox
+from .build_videos import build_videos
+from .config import CHANNEL_NAME, TZ, DB
+from .hash_tags import get_top_nouns
+from .process_uploads import process_uploads
 from .transcribe_video import transcribe_to_srt
-from .utils import run_command
+from .transcribe_client import transcribe_to_srt_remote
+from .utils import get_chapters, get_video_id_from_chat, write_csv, run_command
 
 # 4. Expose Subpackages
 from . import twitch
+from . import youtube
+from . import database
 
 # (Optional) Defines what is exposed during 'from package import *'
 __all__ = [
+    "fit_to_9_16_letterbox"
     "build_videos",
     "CHANNEL_NAME",
+    "TZ",
     "DB",
-    "Database",
+    "get_top_nouns",
+    "process_uploads",
     "transcribe_to_srt",
-    "utils", 
+    "transcribe_to_srt_remote",
+    "write_csv",
+    "run_command",
+    "get_chapters",
+    "get_video_id_from_chat"
     "twitch",
+    "youtube"
 ]

package/build_videos.py

@@ -1,123 +1,103 @@
 #!/usr/bin/env python3
-import youtube_short
+from datetime import datetime
+
+import package.video_editing.build_shorts as yt_shorts
 import package.transcribe_video as transcribe_video
+import package.transcribe_client as transcribe_client
 import package.twitch.chat_vod as chat_vod
+import package.video_editing.highlights as highlights
 from package.config import DB as DB
+from package.config import TZ as TZ
 
 def build_chat_video():
     chats = DB.get_unuploaded_chats()
     
     for chat in chats:    
-        (
-            id,
-            title,
-            created_at,
-            view_count,
-            duration,
-            url,
-            thumbnail_url,
-            game_id,
-            game_name,
-            stream_id,
-            creator_name,
-            clip_is,
-            downloaded,
-            uploaded_yt,
-            uploaded_yt_chats,
-            uploaded_yt_shorts,
-        ) = chat
-        
         print("====================================================")
-        print(f"🚀 Chat Video: {title}")
+        print(f"🚀 Chat Video: {chat.title}")
         print("====================================================")
 
-        chat_vod.combine_twitch_vod_and_chat(f"download/videos/{id}/{id}.mp4", "side-by-side")
+        chat_vod.combine_twitch_vod_and_chat(f"download/videos/{chat.id}/{chat.id}.mp4", "side-by-side")
 
     print("====================================================")
     print(f"✅ Processing: Chat Video Done.")
     print("====================================================")
 
-
 def build_transcribe():
-    unuploadeds = DB.get_unuploaded()
+    downloads = DB.get_downloaded()
 
-    for unuploaded in unuploadeds:
-        (
-            id,
-            title,
-            created_at,
-            view_count,
-            duration,
-            url,
-            thumbnail_url,
-            game_id,
-            game_name,
-            stream_id,
-            creator_name,
-            clip_is,
-            downloaded,
-            uploaded_yt,
-            uploaded_yt_chats,
-            uploaded_yt_shorts,
-        ) = unuploaded
-
-        if clip_is:
+    for download in downloads:
+        if download.clip_is:
             target_dir = f"download/clips"
         else:
             target_dir = f"download/videos"
 
         print("====================================================")
-        print(f"🚀 Transcribe: {title}")
+        print(f"🚀 Transcribe: {download.title}")
         print("====================================================")
         
-        transcribe_video.transcribe_to_srt(f"{target_dir}/{id}/{id}.mp4")
+        #transcribe_video.transcribe_to_srt(f"{target_dir}/{unuploaded.id}/{unuploaded.id}.mp4", True)
+        transcribe_client.transcribe_to_srt_remote(f"{target_dir}/{download.id}/{download.id}.mp4")
 
     print("====================================================")
     print(f"✅ Processing: Transcribing Done.")
     print("====================================================")
 
-
 def build_shorts():
     shorts = DB.get_unuploaded_shorts()
 
     top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
+    count = 1
 
     for short in shorts:
-        (
-            id,
-            title,
-            created_at,
-            view_count,
-            duration,
-            url,
-            thumbnail_url,
-            game_id,
-            game_name,
-            stream_id,
-            creator_name,
-            clip_is,
-            downloaded,
-            uploaded_yt,
-            uploaded_yt_chats,
-            uploaded_yt_shorts,
-        ) = short
-
-        target_dir = f"download/clips/{id}/{id}.mp4"
+        target_dir = f"download/clips/{short.id}/{short.id}.mp4"
 
         print("====================================================")
-        print(f"🚀 Processing: Clip to Youtube Short {title}")
+        print(f"🚀 Processing: Clip to Youtube Short {short.title}")
         print("====================================================")
+        print(f"{count}/{len(shorts)}")
 
-        youtube_short.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {creator_name}.")
+        created_at = datetime.strptime(short.created_at, "%Y-%m-%d %H:%M:%S%z")
+        created_at = created_at.astimezone(TZ)    
+
+        #ToDo
+        yt_shorts.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {short.creator_name},\n on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z")}")
+        count += 1
 
     print("====================================================")
     print(f"✅ Processing: Clip to Youtube Short Done.")
     print("====================================================")
 
-def main():
+def build_highlights():
+    lights = DB.get_downloaded()
+
+    top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
+    count = 1
+
+    for light in lights:
+        target_dir = f"download/clips/{light.id}/{light.id}.mp4"
+
+        print("====================================================")
+        print(f"🚀 Processing: Highlights {light.title}")
+        print("====================================================")
+        print(f"{count}/{len(lights)}")
+
+        created_at = datetime.strptime(light.created_at, "%Y-%m-%d %H:%M:%S%z")
+        created_at = created_at.astimezone(TZ)    
+
+        #ToDo
+        highlights.highlights(target_dir, [], 5, 8)
+        #yt_shorts.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {light.creator_name},\n on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z")}")
+        count += 1
+
+    print("====================================================")
+    print(f"✅ Processing: Clip to Youtube Short Done.")
+    print("====================================================")
+
+def build_videos():
     build_shorts()
     build_transcribe()
     build_chat_video()
 
 if __name__ == "__main__":
-    main()
+    build_videos()

package/config.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
-import package.database as database
+from zoneinfo import ZoneInfo
+
+#import package.database as database
+from package.database.database import Repository
 
 CHANNEL_NAME = "teampgp"
+TZ = ZoneInfo("America/Toronto")
 
-DB = database.Database()
+DB = Repository()

package/database/__init__.py

@@ -0,0 +1,15 @@
+# package/database/__init__.py
+
+# Expose components from individual modules using relative imports
+from .database import TwitchVideo, Data, YoutubeInfo, TiktokInfo, Highlight, Chapter, Repository
+
+# Explicitly define what is exposed when importing the subpackage
+__all__ = [
+    "TwitchVideo",
+    "Data",
+    "YoutubeInfo",
+    "TiktokInfo",
+    "Highlight",
+    "Chapter",
+    "Repository",
+]

package/database/database.py

@@ -0,0 +1,319 @@
+from sqlalchemy import create_engine, Column, String, Integer, Boolean, Float, ForeignKey, select
+from sqlalchemy.orm import declarative_base, relationship, sessionmaker
+from typing import Any
+
+# 1. Define the base class for declarative models
+Base = declarative_base()
+
+# 2. Define the twitch_videos table
+class TwitchVideo(Base):
+    __tablename__ = "twitch_videos"
+
+    id = Column(String, primary_key=True)
+    title = Column(String, nullable=False)
+    created_at = Column(String, nullable=False)
+    view_count = Column(Integer, nullable=False)
+    duration = Column(String, nullable=False)
+    url = Column(String, nullable=False)
+    thumbnail_url = Column(String, nullable=False)
+    game_id = Column(String, nullable=False)
+    game_name = Column(String, nullable=False)
+    stream_id = Column(String, nullable=False)
+    creator_name = Column(String, nullable=False)
+    clip_is = Column(Boolean, nullable=False, default=False)
+    downloaded = Column(Boolean, nullable=False, default=False)
+    uploaded_yt = Column(Boolean, nullable=False, default=False)
+    uploaded_yt_chats = Column(Boolean, nullable=False, default=False)
+    uploaded_yt_shorts = Column(Boolean, nullable=False, default=False)
+    uploaded_tiktok = Column(Boolean, nullable=False, default=False)
+
+    # This now actually points to twitch_videos.id
+    related_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=True)
+
+    # Relationship back to YoutubeInfo (uselist=True enforces a 1-to-many structure)
+    youtube_info = relationship("YoutubeInfo", back_populates="twitch_video", uselist=True)
+    tiktok_info = relationship("TiktokInfo", back_populates="twitch_video", uselist=True)
+    chapters = relationship("Chapter", back_populates="twitch_video", cascade="all, delete-orphan")
+
+    # THIS WAS MISSING - this is what Highlight is looking for
+    highlights = relationship("Highlight", back_populates="twitch_video", cascade="all, delete-orphan")
+
+# 3. Define the data table
+class Data(Base):
+    __tablename__ = "data"
+
+    # In SQLAlchemy, an Integer primary key automatically acts as AUTOINCREMENT in SQLite
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    name = Column(String, nullable=False)
+    date_shorts = Column(String, nullable=False)
+    date_videos = Column(String, nullable=False)
+    date_videos_chats = Column(String, nullable=False)
+    date_tiktok = Column(String, nullable=False)
+
+class YoutubeInfo(Base):
+    __tablename__ = "youtube_info"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
+    yt_id = Column(String, nullable=True)
+    yt_date = Column(String, nullable=True)
+    yt_short_id = Column(String, nullable=True)
+    yt_short_date = Column(String, nullable=True)
+    yt_chat_SS_id = Column(String, nullable=True)
+    yt_chat_SS_date = Column(String, nullable=True)
+    yt_chat_over_id = Column(String, nullable=True)
+    yt_chat_over_date = Column(String, nullable=True)
+
+    # Relationship linking back to the twitch video record
+    twitch_video = relationship("TwitchVideo", back_populates="youtube_info")
+
+class TiktokInfo(Base):
+    __tablename__ = "tiktok_info"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
+    tiktok_id = Column(String, nullable=True)
+    tiktok_date = Column(String, nullable=True)
+
+    # Relationship linking back to the twitch video record
+    twitch_video = relationship("TwitchVideo", back_populates="tiktok_info")
+
+class Highlight(Base):
+    __tablename__ = "highlights"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
+    start_milliseconds = Column(Float, nullable=False)
+    length_milliseconds = Column(Float, nullable=False)
+    start_timestamp = Column(String, nullable=False)
+    end_timestamp = Column(String, nullable=False)
+    description = Column(String, nullable=False)
+    score = Column(Float, nullable=False)
+
+    # Relationship linking back to the twitch video record
+    twitch_video = relationship("TwitchVideo", back_populates="highlights")
+
+class Chapter(Base):
+    __tablename__ = "chapters"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
+    start_milliseconds = Column(Integer, nullable=False)
+    length_milliseconds = Column(Integer, nullable=False)
+    description = Column(String, nullable=True)
+    type = Column(String, nullable=True)
+    game_display_name = Column(String, nullable=True)
+    game_id = Column(String, nullable=True)
+
+    # Relationship linking back to the twitch video record
+    twitch_video = relationship("TwitchVideo", back_populates="chapters")    
+
+class Repository:
+    def __init__(self, db_path: str = "database.db"):
+        # 4. Connect to the SQLite file and generate the tables
+        # This will create 'database.db' in the current working directory if it doesn't exist.
+        engine = create_engine(f"sqlite:///{db_path}", echo=True)
+        Base.metadata.create_all(engine)
+        
+        # 5. Optional: Set up a session factory for interacting with the database
+        Session = sessionmaker(bind=engine)
+        self.session =  Session()
+
+    def set_related_video(self, record_id: str, related_video_id: str):
+        record = self.session.get(TwitchVideo, record_id)
+
+        if record:
+            setattr(record, "related_video", related_video_id)
+            self.session.commit()
+        else:
+            print(f"❌ Record {record_id} not found.")
+
+    def __mark_as(self, record_id: str, field_name: str, value: bool = True):
+        record = self.session.get(TwitchVideo, record_id)
+        
+        if record:
+            setattr(record, field_name, value)
+            self.session.commit()
+        else:
+            print(f"❌ Record {record_id} not found in __mark_as.")
+
+    def mark_as_uploaded_shorts(self, record_id: str):
+        """Flags a specific row record to uploaded_yt_shorts (1)."""
+        self.__mark_as(record_id, "uploaded_yt_shorts")
+
+    def mark_as_uploaded_chats(self, record_id: str):
+        """Flags a specific row record to uploaded_yt_chats (1)."""
+        self.__mark_as(record_id, "uploaded_yt_chats")
+
+    def mark_as_uploaded_yt(self, record_id: str):
+        """Flags a specific row record to uploaded_yt (1)."""
+        self.__mark_as(record_id, "uploaded_yt")
+
+    def mark_as_downloaded(self, record_id: str):
+        """Flags a specific row record to downloaded (1)."""
+        self.__mark_as(record_id, "downloaded")
+
+    def mark_as_uploaded_tiktok(self, record_id: str):
+        """Flags a specific row record to uploaded_tiktok (1)."""
+        self.__mark_as(record_id, "uploaded_tiktok")
+
+    def unmark_as_download(self, record_id: str):
+        """Flags a specific row record to downloaded (0)."""
+        self.__mark_as(record_id, "downloaded", False)
+
+    def update_datetime(self, record_id: str, created_at: str):
+        record = self.session.get(TwitchVideo, record_id)
+
+        if record:
+            record.created_at = created_at
+            self.session.commit()
+        else:
+            print(f"❌ Record {record_id} not found in __mark_as.")
+
+    def __get_unuploaded(self, field_name: str, also_clause: Any = None) -> list[TwitchVideo]:
+        """Retrieve all rows that were downloaded but not uploaded using SQLAlchemy."""
+    
+        # 1. Validation: Verify the string corresponds to an actual column
+        if not hasattr(TwitchVideo, field_name):
+            raise ValueError(f"🚨 Invalid column name: '{field_name}' does not exist on TwitchVideo.")
+            
+        # Get the actual SQLAlchemy column object dynamically
+        status_column = getattr(TwitchVideo, field_name)
+        
+        # 2. Build the query programmatically
+        stmt = select(TwitchVideo).where(
+            TwitchVideo.downloaded == True,
+            status_column == False
+        )
+        
+        # 3. Safely append additional filters if provided
+        if also_clause is not None:
+            stmt = stmt.where(also_clause)
+            
+        # 4. Execute and return results
+        return self.session.scalars(stmt).all()
+
+    def get_unuploaded_shorts(self) -> list[TwitchVideo]:
+        """Retrieve all rows that were download but not uploaded_yt_shorts"""
+        return self.__get_unuploaded("uploaded_yt_shorts", (TwitchVideo.clip_is == True))
+
+    def get_unuploaded_chats(self) -> list[TwitchVideo]:
+        """Retrieve all rows that were download but not uploaded_yt_chats"""
+        return self.__get_unuploaded("uploaded_yt_chats", (TwitchVideo.clip_is == False))
+                
+    def get_unuploaded(self) -> list[TwitchVideo]:
+        """Retrieve all rows that were download but not uploaded_yt"""
+        return self.__get_unuploaded("uploaded_yt")
+
+    def get_undownloaded(self) -> list[TwitchVideo]:
+        stmt = select(TwitchVideo).where(TwitchVideo.downloaded == False)
+        return self.session.scalars(stmt).all()
+
+    def get_unuploaded_tiktok(self) -> list[TwitchVideo]:
+        """Retrieve all rows that were download but not uploaded_tiktok"""
+        return self.__get_unuploaded("uploaded_tiktok", (TwitchVideo.clip_is == True))
+
+    def get_downloaded(self) -> list[TwitchVideo]:
+        stmt = select(TwitchVideo).where(TwitchVideo.downloaded == True)
+        return self.session.scalars(stmt).all()
+
+    def get_clips(self) -> list[TwitchVideo]:
+        """Retrieves all rows that are clip_is"""
+        stmt = select(TwitchVideo).where(TwitchVideo.clip_is == True)
+        return self.session.scalars(stmt).all()
+
+    def get_vods(self) -> list[TwitchVideo]:
+        stmt = select(TwitchVideo).where(TwitchVideo.clip_is == False)
+        return self.session.scalars(stmt).all()
+
+    def __update_date(self, field_name: str, date: str):
+        record_id = 1
+        record = self.session.get(Data, record_id)
+
+        if record:
+            setattr(record, field_name, date)
+            self.session.commit()
+        else:
+            print(f"❌ Record {record_id} not found in __update_date.")
+
+    def update_date_shorts(self, date: str):
+        self.__update_date("date_shorts", date)
+
+    def update_date_chats(self, date: str):
+        self.__update_date("date_videos_chat", date)
+
+    def get_data(self) -> Data:
+        stmt = select(Data)
+        return self.session.scalars(stmt).first()
+
+    def get_youtube_info_for_video(self, video_id: str) -> YoutubeInfo | None:
+        """Retrieve all YouTubeInfo records associated with a specific TwitchVideo ID."""
+        stmt = select(YoutubeInfo).where(YoutubeInfo.twitch_video_id == video_id)
+        return self.session.scalars(stmt).first()
+
+    def get_youtube_info_all(self) -> list[YoutubeInfo]:
+        return self.session.scalars(select(YoutubeInfo)).all()
+
+    def insert(self, data: TwitchVideo | Data | YoutubeInfo | Chapter | list[YoutubeInfo] | list[Chapter] | TiktokInfo | Highlight):
+        """
+        Handles insertion/updating for any of the defined models,
+        using session merge/add for SQLAlchemy 2.0 best practices.
+        Accepts single objects or lists of objects for collection handling.
+        """
+        try:
+            if isinstance(data, TwitchVideo):
+                # 1. Check if the record exists
+                stmt = select(TwitchVideo).where(TwitchVideo.id == data.id)
+                existing_record = self.session.scalar(stmt)
+                
+                if existing_record:
+                    # 2. Update relevant fields on the managed instance
+                    if existing_record.view_count != data.view_count or existing_record.duration != data.duration:
+                        existing_record.view_count = data.view_count
+                        existing_record.duration = data.duration
+                        print(f"🔄 TwitchVideo record {data.id} updated successfully!")
+                    # Note: Add more field updates here as needed
+                else:
+                    # 3. Add new instance
+                    self.session.add(data)
+                    print(f"➕ New TwitchVideo record with ID {data.id} inserted successfully!")
+            elif isinstance(data, Data):
+                self.session.merge(data)
+                print(f"➕ Data record merged/inserted successfully!")
+            elif isinstance(data, YoutubeInfo):
+                # Handle single YoutubeInfo object
+                self.session.merge(data)
+                print(f"➕ YoutubeInfo record merged/inserted successfully!")
+            elif isinstance(data, Chapter):
+                # Handle single Chapter object
+                self.session.merge(data)
+                print(f"➕ Chapter record merged/inserted successfully!")
+            elif isinstance(data, list):
+                # Handle lists of records (YoutubeInfo or Chapter)
+                record_type = type(data[0])
+                if record_type in (YoutubeInfo, Chapter):
+                    for item in data:
+                        self.session.merge(item)
+                    print(f"➕ Successfully processed {len(data)} records of type {record_type.__name__}.")
+                else:
+                    raise TypeError(f"Unsupported list content type: {record_type.__name__}")
+            elif isinstance(data, TiktokInfo):
+                # Handle single TiktokInfo object
+                self.session.merge(data)
+                print(f"➕ TiktokInfo record merged/inserted successfully!")
+            elif isinstance(data, Highlight):
+                # Handle single Highlight object
+                self.session.merge(data)
+                print(f"➕ Highlight record merged/inserted successfully!")
+            else:
+                raise TypeError(f"Unsupported data type for insertion: {type(data).__name__}")
+            
+            # 4. Commit the transaction
+            self.session.flush() # Flush first to ensure IDs are populated
+            
+        except Exception as e:
+            self.session.rollback()
+            print(f"❌ A critical error occurred during record insertion/update: {e}")
+            raise
+
+        self.session.commit()

package/hash_tags.py

@@ -14,7 +14,16 @@ nltk.download('punkt_tab', quiet=True)
 nltk.download('averaged_perceptron_tagger', quiet=True) # Required for POS tagging
 nltk.download('averaged_perceptron_tagger_eng', quiet=True)
 
-def extract_text_from_srt(file_path: str):
+def extract_text_from_srt(file_path: str) -> str:
+    """
+    Extracts the text from the srt file.
+
+    Args:
+        file_path (str): is the path to the str file.
+
+    Returns:
+        returns (str): returns the cleaned up text.
+    """
     with open(file_path, 'r', encoding='utf-8') as file:
         content = file.read()
     # Remove SRT timestamps and sequence numbers
@@ -22,7 +31,16 @@ def extract_text_from_srt(file_path: str):
     clean_text = re.sub(r'\d+', '', clean_text)
     return clean_text
 
-def extract_text_from_json(file_path: str):
+def extract_text_from_json(file_path: str) -> str:
+    """
+    Extracts the text from the json file.
+    
+    Args:
+        file_path (str): is the path to the json file.
+    
+    Returns:
+        returns (str): returns the cleaned up text.
+    """
     with open(file_path, "r", encoding="utf-8") as file:
         data = json.load(file)
 
@@ -34,7 +52,17 @@ def extract_text_from_json(file_path: str):
     # Return a single merged string of all chat text
     return " ".join(messages)
 
-def get_top_nouns(file_path: str, top_n: int = 10):
+def get_top_nouns(file_path: str, top_n: int = 10) -> list:
+    """
+    Gets the top number of nouns based on past (top_n) arg.
+    
+    Args:
+        file_path (str): is the path to the text file.
+        top_n (int): is the number of nouns to return.
+    
+    Returns:
+        returns (list): returns a list of nouns.
+    """
     # 1. Extract raw text
     if file_path.endswith(".srt"):
         raw_text = extract_text_from_srt(file_path)

package/process_uploads.py

@@ -0,0 +1,337 @@
+#!/usr/bin/env python3
+import os
+
+from pathlib import Path
+from datetime import datetime, timedelta
+
+import package.utils as utils
+
+from package.config import TZ as TZ
+from package.config import DB as DB
+from package.hash_tags import get_top_nouns as get_top_nouns
+from package.youtube.uploader import upload_video as upload_video
+from package.youtube.uploader import CategoryId as CategoryId
+from package.tiktok.uploader import upload_to_tiktok as upload_to_tiktok
+
+from package.database import TwitchVideo, YoutubeInfo, TiktokInfo, Data, TiktokInfo
+
+def process_videos(video_type: bool = True, limit: int = 22):
+    count = 0 # count how many videos we have upload and stop at var: limit
+    videos = DB.get_unuploaded_chats() if video_type else DB.get_unuploaded_shorts()
+    file_type = "download/video" if video_type else "download/clips"
+    meta_data: Data = DB.get_data()
+
+    #current_time = datetime.now(TZ)
+    #current_time = current_time.replace(hour=0, minute=0, second=0, microsecond=0)
+
+    #meta_data.date_videos = current_time
+    #meta_data.date_shorts = current_time
+    #meta_data.date_tiktok = current_time
+    #meta_data.date_videos_chats = current_time
+
+    #DB.insert(meta_data) # ensure we have a record in the database
+
+    current_time = datetime.now(TZ)
+    next_time = meta_data.date_videos if video_type else meta_data.date_shorts
+    release_time = datetime.strptime(next_time, "%Y-%m-%d %H:%M:%S%z")
+
+    if current_time > release_time:
+        release_time = release_time.replace(hour=12, minute=0, second=0, microsecond=0)
+        # if noon today is already past, push to tomorrow
+        if release_time <= current_time:
+            release_time += timedelta(days=1)
+
+    for video in videos:
+        #We reached limit stop uploading videos
+        if count >= limit:
+            break
+
+        base_path = Path(f"{file_type}/{video.id}/{video.id}")
+        original_path = base_path.with_suffix(".mp4")
+
+        if not original_path.exists():
+            print(f"❌ Error: no video at {original_path}")
+            continue
+
+        # Build candidate edited files
+        if video_type:
+            primary_edited = base_path.parent / f"{base_path.name}_SS_with_chat.mp4"
+            fallback_edited = base_path.parent / f"{base_path.name}_over_with_chat.mp4"
+        else:
+            primary_edited = base_path.parent / f"{base_path.name}_gaussian_9_16.mp4"
+            fallback_edited = base_path.parent / f"{base_path.name}_black_9_16.mp4"
+
+        file_path = primary_edited if primary_edited.exists() else fallback_edited
+        if not file_path.exists():
+            continue
+
+        file_csv = base_path.parent / f"{base_path.name}_{meta_data.name}.csv"
+        srt_path = base_path.with_suffix(".srt")
+
+        created_at = datetime.strptime(video.created_at, "%Y-%m-%d %H:%M:%S%z").astimezone(TZ)
+        category_id = CategoryId.GAMING
+
+        if video_type:
+            origin_clipped = "Originally Streamed on"
+        else:
+            origin_clipped = f"Clipped By: {video.creator_name}, on"
+
+        privatcyStatus = 'private'
+        tags = ['gaming', 'TeamPGP', f'{video.game_name}', f'{video.creator_name}', 'Level1Techs', 'twitch']
+        if not video_type:
+            tags.extend(['shorts', 'twitch_clips', 'clips'])
+        tags.extend([video.id])       
+
+        if srt_path.exists():
+            try:
+                tags.extend(get_top_nouns(str(srt_path), (15 - len(tags))))
+            except Exception as e:
+                print(f"⚠️ Could not get nouns from {srt_path}: {e}")
+
+        hashs = " ".join([f"#{t}" for t in tags])
+
+        description = f"Game: {video.game_name}, {origin_clipped} {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, {hashs} Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp"
+
+        youtube_info = DB.get_youtube_info_for_video(video.id) or YoutubeInfo(twitch_video_id=video.id)
+
+        if youtube_info.yt_id:
+            main_video_id = youtube_info.yt_id
+        else:
+            main_video_id = upload_video(str(original_path), video.title, category_id, f"{description} \n\n({video.id})", 'unlisted', tags)
+            count += 1
+            if not main_video_id:
+                print(f"❌ Error: main video upload failed for {original_path}")
+                continue
+            youtube_info.yt_id = main_video_id
+            youtube_info.yt_date = datetime.now(TZ)
+
+        # Upload secondary
+        secondary_desc = f"{description} Original Full size @ https://youtu.be/{main_video_id} \n\n({video.id})"
+        secondary_video_id = upload_video(str(file_path), video.title, category_id, secondary_desc, 'private', tags, release_time)
+
+        if not secondary_video_id:
+            print(f"❌ Error: secondary upload failed for {file_path}")
+            DB.insert(youtube_info) # save main id at least
+            continue
+
+        count += 1
+
+        # Success - write CSV and DB
+        data_csv = [main_video_id, secondary_video_id, release_time.isoformat()]
+        utils.write_csv([data_csv], str(file_csv))
+
+        if video_type:
+            DB.mark_as_uploaded_chats(video.id)
+            youtube_info.yt_chat_SS_id = secondary_video_id
+            youtube_info.yt_chat_SS_date = datetime.now(TZ)
+        else:
+            DB.mark_as_uploaded_shorts(video.id)
+            youtube_info.yt_short_id = secondary_video_id
+            youtube_info.yt_short_date = datetime.now(TZ)
+
+        DB.insert(youtube_info)
+        release_time += timedelta(days=0.5)
+
+    # Update global release pointer once at end
+    if video_type:
+        DB.update_date_chats(release_time)
+    else:
+        DB.update_date_shorts(release_time)
+
+def process_tiktoks():
+    tiktoks = DB.get_unuploaded_tiktok()
+    data = DB.get_data()
+    meta_data: Data = data[0]
+
+    release_time = meta_data.date_tiktok
+    
+    current_time = datetime.now(TZ)
+    release_time = datetime.strptime(release_time, "%Y-%m-%d %H:%M:%S%z")
+
+    count = 0
+
+    if current_time > release_time:
+        release_time = current_time
+        release_time = release_time.replace(hour=12, minute=0, second=0, microsecond=0)
+
+    for tiktok in tiktoks:
+        if count >= 10:
+            break
+        created_at = datetime.strptime(tiktok.created_at, "%Y-%m-%d %H:%M:%S%z").astimezone(TZ)
+        tags = ["Twitch", "TeamPGP", f"{tiktok.game_name}", f"{tiktok.creator_name}", "Level1Techs", "clips"]
+        origin_clipped = f"Clipped By: {tiktok.creator_name}, on"
+        description = f"Game: {tiktok.game_name}, {origin_clipped} {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp"
+        original_path = f"download/clips/{tiktok.id}/{tiktok.id}.mp4"
+        file_path = f"download/clips/{tiktok.id}/{tiktok.id}_gaussian_9_16.mp4"
+        file_csv = f"download/clips/{tiktok.id}/{tiktok.id}_{meta_data.name}.csv"
+
+        srt_path = f"download/clips/{tiktok.id}/{tiktok.id}.srt"
+
+        try:
+            tags.extend(get_top_nouns(str(srt_path), (15 - len(tags))))
+        except Exception as e:
+            print(f"⚠️ Could not get nouns from {srt_path}: {e}")
+                
+        if not os.path.exists(file_path):
+            file_path = f"download/clips/{tiktok.id}/{tiktok.id}_black_9_16.mp4"
+            if not os.path.exists(file_path):
+                continue
+        if tiktok_video_id := upload_to_tiktok(file_path, description, tags, "teampgpunofficialarchive"):
+            tiktok_info = TiktokInfo(twitch_video_id=tiktok.id, tiktok_id=tiktok_video_id, tiktok_date=datetime.now(TZ))
+            DB.insert(tiktok_info)
+        count += 1
+
+def process_shorts():
+    process_videos(False)
+    return
+
+    shorts = DB.get_unuploaded_shorts()
+    data = DB.get_data()
+    meta_data: Data = data[0]
+
+    release_time = meta_data.date_shorts
+    
+
+    current_time = datetime.now(TZ)
+    release_time = datetime.strptime(release_time, "%Y-%m-%d %H:%M:%S%z")
+
+    if current_time > release_time:
+        release_time = current_time
+        release_time = release_time.replace(hour=12, minute=0, second=0, microsecond=0)
+
+    for short in shorts:
+        original_path = f"download/clips/{short.id}/{short.id}.mp4"
+        file_path = f"download/clips/{short.id}/{short.id}_gaussian_9_16.mp4"
+        file_csv = f"download/clips/{short.id}/{short.id}_{meta_data.name}.csv"
+        if not os.path.exists(file_path):
+            file_path = f"download/clips/{short.id}/{short.id}_black_9_16.mp4"
+            if not os.path.exists(file_path):
+                continue
+
+        #short.title = short.title
+        created_at = datetime.strptime(short.created_at, "%Y-%m-%d %H:%M:%S%z")
+        created_at = created_at.astimezone(TZ)
+        categoryId = CategoryId.GAMING
+        description = f"Game: {short.game_name}, Clipped By: {short.creator_name}, on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, #Shorts #Clips #Twitch Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp"
+        privatcyStatus = 'private'
+        tags = ['shorts', 'gaming', 'TeamPGP', f'{short.game_name}', f'{short.creator_name}', 'twitch_clips', 'clips', 'Level1Techs', 'twitch']
+        
+        tags.extend(get_top_nouns(f"download/clips/{short.id}/{short.id}.srt"))
+
+        youtube_info = DB.get_youtube_info(short.id)
+        
+        if not youtube_info:
+            youtube_info = YoutubeInfo()
+
+        if os.path.exists(original_path):
+            yt_video_id = None
+            if (youtube_info.id == short.id and youtube_info.yt_id is not None) or (yt_video_id := upload_video(original_path, short.title, categoryId, description, 'unlisted', tags)):
+                if youtube_info.id == short.id:
+                    yt_video_id = youtube_info.yt_short_id
+
+                data_csv = [yt_video_id, current_time]
+                release_time += timedelta(days=1)
+
+                # Assign / Update values on the object
+                youtube_info.id = short.id
+                youtube_info.yt_id = yt_video_id
+                youtube_info.yt_date = datetime.now(TZ)
+
+                if yt_video_id := upload_video(file_path, short.title, categoryId, f"{description} Original Full size @ https://youtu.be/{yt_video_id}", privatcyStatus, tags, release_time):
+                    data_csv.extend([yt_video_id, release_time])
+                    DB.mark_as_uploaded_yt(short.id)
+                    DB.mark_as_uploaded_shorts(short.id)
+                    utils.write_csv([data_csv], file_csv)
+
+                    youtube_info.yt_short_id = yt_video_id
+                    youtube_info.yt_short_date = datetime.now(TZ)
+                else:
+                    print(f"Why is this short allreay update")
+
+                DB.insert(youtube_info)
+            else:
+                break
+
+    DB.update_date_shorts(release_time)
+
+def process_chats():
+    process_videos(True)
+    return
+    chats = DB.get_unuploaded_chats()
+    data = DB.get_data()
+    meta_data: Data = data[0]
+
+    release_time = meta_data.date_shorts
+    
+    current_time = datetime.now(TZ)
+    release_time = datetime.strptime(release_time, "%Y-%m-%d %H:%M:%S%z")
+
+    if current_time > release_time:
+        release_time = current_time
+        release_time = release_time.replace(hour=12, minute=0, second=0, microsecond=0)
+
+    for chat in chats:
+        original_path = f"download/videos/{chat.id}/{chat.id}.mp4"
+        file_path = f"download/videos/{chat.id}/{chat.id}_SS_with_chat.mp4"
+        file_csv = f"download/videos/{chat.id}/{chat.id}_{meta_data.name}.csv"
+
+        if not os.path.exists(file_path):
+            file_path = f"download/videos/{chat.id}/{chat.id}_over_with_chat.mp4"
+            if not os.path.exists(file_path):
+                continue
+
+            #chat.title = chat.title
+            created_at = datetime.strptime(chat.created_at, "%Y-%m-%d %H:%M:%S%z")
+            created_at = created_at.astimezone(TZ)
+            categoryId = CategoryId.GAMING
+            description = f"Game: {chat.game_name}, Originally Streamed on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, #Twitch Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp"
+            privatcyStatus = 'private'
+            tags = ['gaming', 'TeamPGP', f'{chat.game_name}', f'{chat.creator_name}', 'Level1Techs', 'twitch']
+            
+            tags.extend(get_top_nouns(f"download/video/{chat.id}/{chat.id}.srt"))
+    
+            youtube_info = DB.get_youtube_info(chat.id)
+            
+            if not youtube_info:
+                youtube_info = YoutubeInfo()
+
+            if os.path.exists(original_path):
+                yt_video_id = None
+                if (youtube_info.id == chat.id and youtube_info.yt_id is not None) or (yt_video_id := upload_video(original_path, chat.title, categoryId, description, 'unlisted', tags)):
+                    if youtube_info.id == chat.id:
+                        yt_video_id = youtube_info.yt_short_id
+        
+                    data_csv = [yt_video_id, current_time]
+                    release_time += timedelta(days=1)
+        
+                    # Assign / Update values on the object
+                    youtube_info.id = chat.id
+                    youtube_info.yt_id = yt_video_id
+                    youtube_info.yt_date = datetime.now(TZ)
+
+                    if yt_video_id := upload_video(file_path, chat.title, categoryId, f"{description} Original no chat @ https://youtu.be/{yt_video_id}", privatcyStatus, tags, release_time):
+                        data_csv.extend([yt_video_id, release_time])
+                        DB.mark_as_uploaded_yt(chat.id)
+                        DB.mark_as_uploaded_shorts(chat.id)
+                        utils.write_csv([data_csv], file_csv)
+        
+                        youtube_info.yt_short_id = yt_video_id
+                        youtube_info.yt_short_date = datetime.now(TZ)
+                    else:
+                        print(f"Why is this short allreay update")
+        
+                    DB.insert(youtube_info)
+                else:
+                    break
+
+    DB.update_date_shorts(release_time)
+
+
+
+def process_uploads():
+    #process_tiktoks()
+    process_shorts()
+    #process_chats()
+
+if __name__ == "__main__":
+    pass

package/tiktok/__init__.py

@@ -0,0 +1,9 @@
+# package/tiktok/__init__.py
+
+# Expose components from individual modules using relative imports
+from .uploader import upload_to_tiktok
+
+# Explicitly define what is exposed when importing the subpackage
+__all__ = [
+    "upload_to_tiktok"
+]

package/tiktok/uploader.py

@@ -0,0 +1,74 @@
+import json
+import requests
+from pathlib import Path
+
+SECRETS_FILE = Path(__file__).parent.parent / "tiktok_secrets.json"
+# if your file is at /home/richard/python_scripts/tiktok_secrets.json
+# use: Path("/home/richard/python_scripts/tiktok_secrets.json")
+
+def load_secrets():
+    with open(SECRETS_FILE) as f:
+        secrets = json.load(f)
+    return secrets["client_key"], secrets["client_secret"], secrets["access_token"], secrets.get("refresh_token", "")
+
+API_BASE = "https://open.tiktokapis.com/v2"
+
+def upload_to_tiktok_api(video_path: str, title: str, privacy="PUBLIC_TO_EVERYONE"):
+    client_key, client_secret, access_token, refresh_token = load_secrets()
+    
+    video_path = Path(video_path)
+    file_size = video_path.stat().st_size
+    
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "Content-Type": "application/json; charset=UTF-8"
+    }
+
+    # 1. INIT
+    init_payload = {
+        "source_info": {
+            "source": "FILE_UPLOAD",
+            "video_size": file_size,
+            "chunk_size": file_size,
+            "total_chunk_count": 1
+        }
+    }
+    r = requests.post(f"{API_BASE}/post/publish/video/init/", headers=headers, json=init_payload)
+    r.raise_for_status()
+    data = r.json()
+    if data.get("error", {}).get("code") != "ok":
+        raise Exception(f"Init failed: {data}")
+
+    publish_id = data["data"]["publish_id"]
+    upload_url = data["data"]["upload_url"]
+
+    # 2. UPLOAD
+    with open(video_path, "rb") as f:
+        put_headers = {
+            "Content-Range": f"bytes 0-{file_size-1}/{file_size}",
+            "Content-Type": "video/mp4"
+        }
+        requests.put(upload_url, headers=put_headers, data=f).raise_for_status()
+
+    # 3. PUBLISH
+    publish_payload = {
+        "publish_id": publish_id,
+        "post_info": {
+            "title": title[:2200],
+            "privacy_level": privacy,
+            "disable_duet": False,
+            "disable_comment": False,
+            "disable_stitch": False,
+            "video_cover_timestamp_ms": 1000
+        }
+    }
+    pr = requests.post(f"{API_BASE}/post/publish/", headers=headers, json=publish_payload)
+    pr.raise_for_status()
+    return pr.json()
+
+if __name__ == "__main__":
+    result = upload_to_tiktok_api(
+        "/home/richard/python_scripts/download/clips/ProductiveConfidentWombatThunBeast/ProductiveConfidentWombatThunBeast_gaussian_9_16.mp4",
+        "Testing upload #Twitch #TeamPGP #Stellaris"
+    )
+    print(result)

package/transcribe_client.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+
+#!/usr/bin/env python3
+import threading
+# --- HOTFIX for Python 3.13.5 _DeleteDummyThreadOnDel spam ---
+# The CPython fix in 3.13.4 didn't fully kill it for debugpy/asyncio cases
+if hasattr(threading, '_DeleteDummyThreadOnDel'):
+    threading._DeleteDummyThreadOnDel.__del__ = lambda self: None
+
+import os
+import shutil
+# ... rest of your imports
+import os
+import shutil
+import subprocess
+import json
+from pathlib import Path
+import wave
+import requests
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
+
+CHUNK_DURATION_SEC = 600 # target chunk size
+OVERLAP_SEC = 8 # overlap to prevent cut words
+VAD_SEARCH_WINDOW = 15 # look for silence within 15s of target
+WORKER_URL = "http://192.168.5.55:8000/transcribe"
+MAX_WORKERS = 4 # perfect for 4070 Ti Super 16GB
+
+def get_duration(video_path: str) -> float:
+    cmd = ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "json", video_path]
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    return float(json.loads(result.stdout)["format"]["duration"])
+
+def find_silence_near(video_path: str, target_time: float) -> float:
+    """Find nearest silence within VAD_SEARCH_WINDOW of target_time"""
+    start_search = max(0, target_time - VAD_SEARCH_WINDOW/2)
+    cmd = [
+        "ffmpeg", "-ss", str(start_search), "-t", str(VAD_SEARCH_WINDOW),
+        "-i", video_path, "-af", "silencedetect=noise=-30dB:d=0.5",
+        "-f", "null", "-"
+    ]
+    result = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+    # Parse last silence end before target
+    best = target_time
+    for line in result.stderr.splitlines():
+        if "silence_end:" in line:
+            try:
+                t = float(line.split("silence_end:")[1].split("|")[0].strip())
+                abs_t = start_search + t
+                if abs(abs_t - target_time) < abs(best - target_time):
+                    best = abs_t
+            except: pass
+    return best
+
+def extract_audio_and_chunk_smart(video_path: str, output_dir: Path):
+    print(f"🚀 Smart chunking: target {CHUNK_DURATION_SEC}s + {OVERLAP_SEC}s overlap")
+    output_dir.mkdir(parents=True, exist_ok=True)
+    total_dur = get_duration(video_path)
+    print(f"📏 Total duration: {total_dur/60:.1f} min")
+
+    chunks = []
+    start = 0.0
+    idx = 0
+
+    while start < total_dur:
+        # Find smart cut point
+        target_end = start + CHUNK_DURATION_SEC
+        if target_end >= total_dur:
+            end = total_dur
+        else:
+            end = find_silence_near(video_path, target_end)
+            # clamp
+            end = max(start + CHUNK_DURATION_SEC * 0.7, min(end, start + CHUNK_DURATION_SEC * 1.3))
+
+        chunk_path = output_dir / f"chunk_{idx:03d}.wav"
+        # Extract with overlap for next chunk
+        extract_end = end + (OVERLAP_SEC if end < total_dur else 0)
+
+        cmd = [
+            "ffmpeg", "-y", "-ss", str(start), "-to", str(extract_end),
+            "-i", video_path, "-vn", "-ac", "1", "-ar", "16000", "-acodec", "pcm_s16le",
+            str(chunk_path)
+        ]
+        subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
+
+        chunks.append((chunk_path, start, end)) # path, true_start, true_end (without overlap)
+        print(f" Chunk {idx}: {start:.1f}s -> {end:.1f}s (file to {extract_end:.1f}s)")
+
+        if end >= total_dur:
+            break
+        start = end - OVERLAP_SEC/2 # start next chunk with half overlap
+        idx += 1
+
+    return chunks
+
+def format_srt_time(seconds: float):
+    h = int(seconds // 3600)
+    m = int((seconds % 3600) // 60)
+    s = int(seconds % 60)
+    ms = int((seconds % 1) * 1000)
+    return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}"
+
+def send_chunk_for_transcription(args):
+    chunk_path, true_start, true_end, chunk_idx = args
+    try:
+        with open(chunk_path, 'rb') as f:
+            files = {'file': (chunk_path.name, f, 'audio/wav')}
+            resp = requests.post(WORKER_URL, files=files, timeout=1200)
+            resp.raise_for_status()
+            data = resp.json()
+            # Filter to only keep segments within true window (dedupe overlap)
+            filtered = []
+            for seg in data['segments']:
+                abs_start = true_start + seg['start']
+                # keep if it starts before the true_end (allow small spill)
+                if abs_start < true_end + OVERLAP_SEC/2:
+                    # if it's in overlap zone of NEXT chunk, only keep first half
+                    if abs_start < true_end - OVERLAP_SEC/2 or chunk_idx == 0:
+                        filtered.append(seg)
+                    elif seg['start'] < (true_end - true_start) - OVERLAP_SEC/2 + 1.0:
+                        filtered.append(seg)
+            return chunk_idx, true_start, filtered, None
+    except Exception as e:
+        return chunk_idx, true_start, None, str(e)
+
+def transcribe_to_srt_remote(video_path: str, force: bool = False):
+    video_path_obj = Path(video_path)
+    srt_path = video_path_obj.with_suffix(".srt")
+    temp_dir = video_path_obj.parent / f"temp_chunks_{video_path_obj.stem}"
+
+    if srt_path.exists() and not force:
+        print(f"❌ SRT already exists: {srt_path}")
+        return
+    if srt_path.exists():
+        srt_path.unlink()
+
+    audio_chunks = extract_audio_and_chunk_smart(str(video_path_obj), temp_dir)
+    jobs = [(path, s, e, idx) for idx, (path, s, e) in enumerate(audio_chunks)]
+
+    results = []
+    executor = ThreadPoolExecutor(max_workers=MAX_WORKERS)
+    try:
+        futures = {executor.submit(send_chunk_for_transcription, job): job for job in jobs}
+        for future in as_completed(futures):
+            idx, offset, segments, error = future.result()
+            if error:
+                print(f"❌ Chunk {idx} failed: {error}")
+            else:
+                print(f"✅ Chunk {idx} done: {len(segments)} segs (offset {offset:.1f}s)")
+                results.append((idx, offset, segments))
+    finally:
+        executor.shutdown(wait=True)
+
+    results.sort(key=lambda x: x[0])
+
+    print("📝 Writing final SRT with smart overlap dedupe...")
+    global_idx = 1
+    with open(srt_path, "w", encoding="utf-8") as srt_file:
+        for _, time_offset, segments in results:
+            for seg in segments:
+                actual_start = seg['start'] + time_offset
+                actual_end = seg['end'] + time_offset
+                srt_file.write(f"{global_idx}\n{format_srt_time(actual_start)} --> {format_srt_time(actual_end)}\n{seg['text'].strip()}\n\n")
+                global_idx += 1
+
+    print(f"\n✅ SRT saved: {srt_path} - {global_idx-1} lines")
+    if temp_dir.exists():
+        shutil.rmtree(temp_dir)
+
+if __name__ == "__main__":
+    target_video = "download/videos/2813112936/2813112936.mp4"
+    transcribe_to_srt_remote(target_video, force=True)

package/transcribe_video.py

@@ -3,116 +3,117 @@ import os
 import shutil
 import subprocess
 from pathlib import Path
+import wave
 from faster_whisper import WhisperModel
-from faster_whisper.utils import format_timestamp
 
-# Prevent OpenMP thread conflicts from crashing the script
 os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
 
-# 5 minutes per chunk (300 seconds) keeps RAM usage low and stable
 CHUNK_DURATION_SEC = 300 
 
-print("🔧 Initializing C++ Engine...")
-model = WhisperModel(
-    "base", 
-    device="cpu", 
-    compute_type="int8", 
-    cpu_threads=0, # Let CTranslate2 auto-detect safe core counts
-    num_workers=1         
-)
-print("✅ C++ Model loaded successfully.")
-
-
-def extract_audio_and_chunk(video_path: str, output_dir: Path) -> list:
-    """Extracts and splits audio into 5-minute chunks using a single FFmpeg pass."""
+def extract_audio_and_chunk(video_path: str, output_dir: Path) -> list[tuple[Path, float]]:
+    """Returns list of (chunk_path, start_offset_seconds)"""
     print("🚀 Extracting and chunking audio with FFmpeg...")
     output_dir.mkdir(parents=True, exist_ok=True)
     
-    # Segment format output: chunk_000.wav, chunk_001.wav, etc.
     chunk_pattern = str(output_dir / "chunk_%03d.wav")
     
     command = [
         "ffmpeg", "-y", "-i", video_path,
         "-vn", "-ac", "1", "-ar", "16000",
-        "-acodec", "pcm_s16le", "-sn", "-map_chapters", "-1",
+        "-acodec", "pcm_s16le",
         "-f", "segment", "-segment_time", str(CHUNK_DURATION_SEC),
+        "-reset_timestamps", "0", # keep each chunk starting at 0
         chunk_pattern
     ]
     
     result = subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
     if result.returncode != 0:
-        print(f"❌ FFmpeg Error Output:\n{result.stderr}")
-        raise RuntimeError("FFmpeg extraction and chunking failed.")
+        print(f"❌ FFmpeg Error:\n{result.stderr}")
+        raise RuntimeError("FFmpeg failed")
 
-    # Return sorted list of generated chunk files
-    return sorted(list(output_dir.glob("chunk_*.wav")))
+    chunks = sorted(output_dir.glob("chunk_*.wav"))
     
+    # FIX: Calculate real offset from actual file durations, not idx*300
+    offsets = []
+    current_offset = 0.0
+    for chunk_path in chunks:
+        offsets.append((chunk_path, current_offset))
+        with wave.open(str(chunk_path), 'rb') as wf:
+            duration = wf.getnframes() / wf.getframerate()
+            current_offset += duration
+            
+    return offsets
+
+def format_srt_time(seconds: float):
+    h = int(seconds // 3600)
+    m = int((seconds % 3600) // 60)
+    s = int(seconds % 60)
+    ms = int((seconds % 1) * 1000)
+    return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}"
 
 def transcribe_to_srt(video_path: str, force: bool = False):
+    if not hasattr(transcribe_to_srt, "model"):
+        print("🔧 Initializing C++ Engine...")
+        transcribe_to_srt.model = WhisperModel(
+            "distil-small.en",
+            device="cpu",
+            compute_type="int8",
+            cpu_threads=4, # 4 = physical cores on E3-1280 v5 = fastest
+            num_workers=1
+        )
+        print("✅ Model loaded.")
+
     video_path_obj = Path(video_path)
     srt_path = video_path_obj.with_suffix(".srt")
     temp_dir = video_path_obj.parent / f"temp_chunks_{video_path_obj.stem}"
 
+    if srt_path.exists() and not force:
+        print(f"❌ SRT already exists: {srt_path}")
+        return
     if srt_path.exists():
-        if not force:
-            print(f"❌ Error: Transcription file already exists: {srt_path}")
-            return
-        else:
-            srt_path.unlink()
+        srt_path.unlink()
 
     try:
-        # Step 1: Split audio into bite-sized pieces
         audio_chunks = extract_audio_and_chunk(str(video_path_obj), temp_dir)
-        if not audio_chunks:
-            print("❌ Error: No audio chunks were generated.")
-            return
-
-        print(f"📦 Successfully split audio into {len(audio_chunks)} chunks.")
-        print("🎙️ Starting safe chunk-by-chunk transcription...")
-
-        global_segment_index = 1
+        print(f"📦 {len(audio_chunks)} chunks.")
+        print("🎙 Transcribing...")
 
+        global_idx = 1
         with open(srt_path, "w", encoding="utf-8") as srt_file:
-            for chunk_idx, chunk_path in enumerate(audio_chunks):
-                # Calculate the time offset for the current chunk
-                time_offset = chunk_idx * CHUNK_DURATION_SEC
-                print(f"\n⏳ Processing chunk {chunk_idx + 1}/{len(audio_chunks)} ({chunk_path.name})...")
+            for chunk_idx, (chunk_path, time_offset) in enumerate(audio_chunks):
+                print(f"\n⏳ Chunk {chunk_idx+1}/{len(audio_chunks)} offset={time_offset:.1f}s")
                 
-                segments_generator, info = model.transcribe(
+                segments, info = transcribe_to_srt.model.transcribe(
                     str(chunk_path), 
+                    language="en",
                     beam_size=1,
                     vad_filter=True,
-                    temperature=0.0
+                    vad_parameters=dict(min_silence_duration_ms=500),
+                    condition_on_previous_text=False # important for chunked mode
                 )
                 
-                # Consume chunk generator and shift timestamps instantly
-                for segment in segments_generator:
-                    # Shift timestamps relative to the original video timeline
-                    actual_start = segment.start + time_offset
-                    actual_end = segment.end + time_offset
+                for seg in segments:
+                    actual_start = seg.start + time_offset
+                    actual_end = seg.end + time_offset
                     
-                    start_str = format_timestamp(actual_start, always_include_hours=True)
-                    end_str = format_timestamp(actual_end, always_include_hours=True)
+                    # Use comma for SRT, not dot
+                    srt_file.write(f"{global_idx}\n{format_srt_time(actual_start)} --> {format_srt_time(actual_end)}\n{seg.text.strip()}\n\n")
+                    global_idx += 1
                     
-                    srt_file.write(f"{global_segment_index}\n{start_str} --> {end_str}\n{segment.text.strip()}\n\n")
-                    global_segment_index += 1
-                    
-                # Free up space as we go by deleting the processed chunk
                 chunk_path.unlink()
 
-        print(f"\n✅ All chunks combined! SRT subtitle file saved in: {srt_path}")
+        print(f"\n✅ SRT saved: {srt_path}")
 
     except Exception as e:
-        print(f"\n❌ Execution Error: {e}")
+        print(f"\n❌ Error: {e}")
+        import traceback; traceback.print_exc()
     finally:
-        # Clean up the temporary folder entirely
         if temp_dir.exists():
             shutil.rmtree(temp_dir)
 
-
 if __name__ == "__main__":
     target_video = "download/videos/2813112936/2813112936.mp4"
     if not os.path.exists(target_video):
-        print(f"❌ System Error: Target video file does not exist at path: {target_video}")
+        print(f"❌ Not found: {target_video}")
     else:
         transcribe_to_srt(target_video, force=True)

package/twitch/__init__.py

@@ -2,9 +2,9 @@
 
 # Expose components from individual modules using relative imports
 from .chat_vod import combine_twitch_vod_and_chat
-from .download_thumbnails import main as download_thumbnails
-from .download_videos import main as download_videos
-from .video_info import main as video_info
+from .download_thumbnails import download_thumbnails
+from .download_videos import download_videos
+from .video_info import video_info
 
 # Explicitly define what is exposed when importing the subpackage
 __all__ = [

package/twitch/chat_vod.py

@@ -4,6 +4,9 @@ import cv2
 import package.utils as utils
 
 def get_video_height(video_path: str) -> int:
+    """
+    Gets the videos height
+    """
     # Open the video file
     video = cv2.VideoCapture(video_path)
     

package/twitch/download_thumbnails.py

@@ -80,7 +80,7 @@ def save_image(url: str, filename: str, stream: bool = True):
     except Exception as e:
         print(f"❌ An error occurred during file writing: {e}")
 
-async def main():
+async def download_thumbnails():
     # Initialize connection & automatically authorize the App token
     await config.get_twitch()
     twitch = config.TWITCH # Twitch(APP_ID, APP_SECRET)
@@ -93,7 +93,7 @@ async def main():
     # --- OPTION B: Download Past Broadcast VOD Thumbnail ---
     # Extract the ID sequence from your target video link
     # target_vod = "2145678901"
-    db = database.Database()
+    db = database.Repository()
     vods = db.get_vods()
     for vod in vods:
         (
@@ -141,4 +141,4 @@ async def main():
         await download_clip_thumbnail(id, thumbnail_url)
 
 if __name__ == '__main__':
-    asyncio.run(main())
+    asyncio.run(download_thumbnails())

package/twitch/download_videos.py

@@ -1,103 +1,86 @@
 #!/usr/bin/env python3
-import csv
 import subprocess
+from pathlib import Path
 from datetime import datetime, timedelta, timezone
 
 import package.utils as utils
 from package.config import DB as DB
+from package.config import TZ as TZ
+from package.database import TwitchVideo, Chapter
 
-def write_csv(data, file_name):
-    # Open file with newline='' to prevent extra blank rows across platforms
-    with open(file_name, "w", newline="", encoding="utf-8") as file:
-        writer = csv.writer(file)
-        
-        # Write all rows at once
-        writer.writerows(data)
-
-def download():
+def download_videos():
     """Find all undownloaded videos and clips, and download them safely."""
     undownloaded = DB.get_undownloaded()
 
     print("Starting downloads...")
     for row in undownloaded:
-        # Unpack variables clearly
-        (
-            id,
-            title,
-            created_at,
-            view_count,
-            duration,
-            url,
-            thumbnail_url,
-            game_id,
-            game_name,
-            stream_id,
-            creator_name,
-            clip_is,
-            downloaded,
-            uploaded_yt,
-            uploaded_yt_chats,
-            uploaded_yt_shorts,
-        ) = row
+        try:
+            # 2026-08-16T23:52:10Z -> replace Z with +00:00
+            data_timestamp = datetime.fromisoformat(row.created_at.replace("Z", "+00:00"))
+        except Exception:
+            data_timestamp = datetime.strptime(row.created_at, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
 
-        # 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM)
-        #datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
-        data_timestamp = datetime.fromisoformat(created_at)
-
-        # 2. Get the exact current date and time
-        current_time = datetime.now(timezone.utc)
-
-        # 3. Calculate the difference between the two times
-        time_difference = current_time - data_timestamp
-
-        # 4. Check if the difference is greater than 24 hours
-        if time_difference > timedelta(hours=24):
-            print("The data is more than 24 hours old.")
-        else:
-            #lets wait 24 hours befor downloading
-            print("The data is less than 24 hours old.")
+        if datetime.now(timezone.utc) - data_timestamp < timedelta(hours=24):
+            print(f"Skipping {row.id} - less than 24h old")
             continue
 
-        data = [list(row)]
-        print(
-            f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}"
+        print(f"ID: {row.id} | Date: {row.created_at} | Game: {row.game_name} | Title: {row.title}")
+
+        base_dir = Path(f"download/clips/{row.id}" if row.clip_is else f"download/videos/{row.id}")
+        base_dir.mkdir(parents=True, exist_ok=True)
+
+        temp_path = Path("download/temp")
+        temp_path.mkdir(parents=True, exist_ok=True)
+
+        target_video = base_dir / f"{row.id}.mp4"
+        json_file = base_dir / f"{row.id}_chat.json"
+        csv_file = base_dir / f"{row.id}.csv"
+
+        if row.clip_is:
+            cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_video} --collision Overwrite --temp-path {temp_path}"
+        else:
+            cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_video} --collision Overwrite --threads 2 --temp-path {temp_path}"
+
+        output = utils.run_command(cmd, look_for=["[STATUS]"])
+        output2 = utils.run_command(
+            f"TwitchDownloaderCLI chatdownload --id {row.id} -o {json_file} -E --collision Overwrite --temp-path {temp_path}",
+            look_for=["[STATUS]"]
         )
 
-        # 1. Define paths and isolate base directory
-        if clip_is:
-            target_dir = f"download/clips/{id}"
-            cmd = f"TwitchDownloaderCLI clipdownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --temp-path download/temp"
-        else:
-            target_dir = f"download/videos/{id}"
-            # FIXED: Removed the duplicated command string combined with '&&'
-            cmd = f"TwitchDownloaderCLI videodownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
-
-        csv_file = f"{target_dir}/{id}.csv"
-
-        # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
-        output = utils.run_command(cmd, look_for=["[STATUS]"])
-        output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
+        if output2 and json_file.exists():
+            related_id = utils.get_video_id_from_chat(json_file)
+            if related_id:
+                # This will set related_video_id = 2848295283 for a clip
+                DB.set_related_video(row.id, related_id)
+                print(f" -> Linked {row.id} -> related VOD {related_id}")
+            if row.clip_is:
+                chapters = utils.get_chapters(json_file)
+                for chapter in chapters:
+                    chapter_record = Chapter(
+                        twitch_video_id=row.id,
+                        start_ms=chapter.get('startMilliseconds'),
+                        length_ms=chapter.get('lengthMilliseconds'),
+                        description=chapter.get('description'),
+                        type=chapter.get('type'),
+                        game_display_name=chapter.get('gameDisplayName'),
+                        game_id=chapter.get('gameId')
+                    )
+                    DB.insert(chapter_record)
+                    print(f" ➕ Chapter added: {chapter_record.description} ({chapter_record.start_ms}ms)")
 
         if output:
-            # Only write CSV and update database if download actually completed
-            write_csv(data, csv_file)
-            print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
-            DB.mark_as_downloaded(id)
+            # FIX 4: write_csv needs actual row data, not column names
+            utils.write_csv([row], str(csv_file)) # pass the object/dict, not vars(TwitchVideo)
+            print(f"✅ Success: {base_dir}")
+            DB.mark_as_downloaded(row.id)
         else:
-            print(f"ID: {id} | Download Process failed.")
-            #print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
-            #if output.get("stderr"):
-            #    print(f"Details: {output['stderr']}")
-
-            # Clear temporary chunk clutter immediately if a VOD crashes out
-            if not clip_is:
+            print(f"❌ ID: {row.id} | Download failed.")
+            if not row.clip_is:
                 print("Flushing temporary crash chunks...")
                 subprocess.run("rm -rf download/temp/*", shell=True)
 
+
     print("Finished Downloading Pipeline.")
 
-def main():
-    download()
-
 if __name__ == "__main__":
-    main()
+    download_videos()

package/twitch/video_info.py

@@ -7,6 +7,7 @@ from twitchAPI.type import VideoType
 
 import package.twitch.config as config
 from package.config import DB as DB
+from package.database.database import TwitchVideo
 
 GAME_CACHE = {} # Local cache dictionary to store game_id -> game_name mapping
 
@@ -40,7 +41,7 @@ async def get_game_name_by_id(game_id: str) -> str:
         pass
     return "Unknown Game"
 
-async def get_vod_game_name(vod_id: str):
+async def get_vod_game_name(vod_id: str) -> tuple:
     """Asynchronously query Twitch GQL endpoint for VOD game metadata."""
     game_id = "0"
     game_name = "Unknown Game"
@@ -83,7 +84,7 @@ async def get_vod_game_name(vod_id: str):
 
     return game_id, game_name
 
-async def get_streamer_vods():
+async def get_streamer_vods() -> list:
     await config.get_twitch()
     
     print(f"Starting VOD extraction for {config.USER.display_name}...")
@@ -101,27 +102,27 @@ async def get_streamer_vods():
         except ValueError:
             clean_game_id = 0
 
-        vod_data = {
-            "id": v.id,
-            "title": v.title,
-            "created_at": str(v.published_at),
-            "view_count": int(v.view_count) if v.view_count else 0,
-            "duration": v.duration,
-            "url": v.url,
-            "thumbnail_url": v.thumbnail_url,
-            "game_id": clean_game_id,
-            "game_name": game_name,
-            "stream_id": str(v.stream_id) if v.stream_id else "0",
-            "creator_name": config.CHANNEL_NAME,
-            "clip_is": False,
-        }
+        vod_data = TwitchVideo()
+        vod_data.id = v.id
+        vod_data.title = v.title
+        vod_data.created_at = str(v.published_at)
+        vod_data.view_count = int(v.view_count) if v.view_count else 0
+        vod_data.duration = v.duration
+        vod_data.url = v.url
+        vod_data.thumbnail_url = v.thumbnail_url
+        vod_data.game_id = clean_game_id
+        vod_data.game_name = game_name
+        vod_data.stream_id = str(v.stream_id) if v.stream_id else "0"
+        vod_data.creator_name = config.CHANNEL_NAME
+        vod_data.clip_is = False
+  
         all_vods.append(vod_data)
         print(f"Collected VOD: {v.title} | Category: {game_name} ({v.duration})")
 
     print(f"\nFinished extracting VODs. Total gathered: {len(all_vods)}")
     return all_vods
 
-async def get_streamer_clips():
+async def get_streamer_clips() -> list:
     await config.get_twitch()
 
     print(f"Starting clip extraction for {config.USER.display_name}...")
@@ -131,52 +132,48 @@ async def get_streamer_clips():
     async for c in clip_generator:
         game_name = await get_game_name_by_id(c.game_id)
 
-        clip_data = {
-            "id": c.id,
-            "title": c.title,
-            "created_at": str(c.created_at),             
-            "view_count": int(c.view_count) if c.view_count else 0,
-            "duration": c.duration,
-            "url": c.url,
-            "thumbnail_url": c.thumbnail_url,
-            "game_id": c.game_id,
-            "game_name": game_name,
-            "stream_id": "0",
-            "creator_name": c.creator_name,
-            "clip_is": True,
-        }
+        clip_data = TwitchVideo()
+        clip_data.id = c.id
+        clip_data.title = c.title
+        clip_data.created_at = str(c.created_at)
+        clip_data.view_count = int(c.view_count) if c.view_count else 0
+        clip_data.duration = c.duration
+        clip_data.url = c.url
+        clip_data.thumbnail_url = c.thumbnail_url
+        clip_data.game_id = c.game_id
+        clip_data.game_name = game_name
+        clip_data.stream_id = "0"
+        clip_data.creator_name = c.creator_name
+        clip_data.clip_is = True
+
         all_clips.append(clip_data)
         print(f"Collected clip: {c.title} | Category: {game_name} ({c.view_count} views)")
 
     print(f"\nFinished extracting clips. Total gathered: {len(all_clips)}")
     return all_clips
 
-async def main():
+async def video_info():
     print("--- Script Started ---")
     
     vods_list = await get_streamer_vods()
     
     for v in vods_list:
-        DB.insert_video_record(
-            v['id'], v['title'], v['created_at'], v['view_count'], v['duration'], 
-            v['url'], v['thumbnail_url'], v['game_id'], v['game_name'], 
-            v['stream_id'], v['creator_name'], v['clip_is']
-        )
+        DB.insert(v)
 
     if vods_list:
-        print(f"Recent VOD: '{vods_list[0]['title']}'")
+        print("====================================================")
+        print(f"Recent VOD: '{vods_list[0].title}'")
+        print("====================================================")
 
     clips_list = await get_streamer_clips()
         
     for v in clips_list:
-        DB.insert_video_record(
-            v['id'], v['title'], v['created_at'], v['view_count'], v['duration'], 
-            v['url'], v['thumbnail_url'], v['game_id'], v['game_name'], 
-            v['stream_id'], v['creator_name'], v['clip_is']
-        )
+        DB.insert(v)
 
     if clips_list:
-        print(f"Recent VOD: '{clips_list[0]['title']}'")
+        print("====================================================")
+        print(f"Recent VOD: '{clips_list[0].title}'")
+        print("====================================================")
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    asyncio.run(video_info())

package/utils.py

@@ -1,8 +1,90 @@
 #!/usr/bin/env python3
+import csv
+from pathlib import Path
 import sys
+import json
 import shlex
 import subprocess
 
+def get_chapters(json_path):
+    """Loads the JSON file and returns all chapters."""
+    with open(json_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+
+    # The chapters are under video -> chapters
+    video = data.get('video', {})
+    chapters = video.get('chapters', [])
+
+    if not chapters:
+        print("No chapters found.")
+        return []
+
+    print(f"Found {len(chapters)} chapter(s) in video '{video.get('title')}':\n")
+
+    for i, chapter in enumerate(chapters, 1):
+        start_sec = chapter.get('startMilliseconds', 0) / 1000
+        length_sec = chapter.get('lengthMilliseconds', 0) / 1000
+
+        #print(f"{i}. {chapter.get('description', 'No Title')}")
+        #print(f" - Type: {chapter.get('type')}")
+        #print(f" - Start: {start_sec}s ({chapter.get('startMilliseconds')}ms)")
+        #print(f" - Length: {length_sec}s ({chapter.get('lengthMilliseconds')}ms)")
+        #print(f" - Game: {chapter.get('gameDisplayName')} (ID: {chapter.get('gameId')})")
+        #print()
+
+    return chapters
+
+def get_video_id_from_chat(json_path: str | Path) -> str | None:
+    """Gets the id inside video:{ } - from the chat json you download"""
+    try:
+        with open(json_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return data.get("video", {}).get("id")
+    except Exception as e:
+        print(f"Could not read video.id from {json_path}: {e}")
+        return None
+
+def write_csv(data: list, file_name: str):
+    """Writes a list of dictionaries or objects to a CSV file."""
+    
+    if not data:
+        print(f"⚠️ Warning: No data provided to write to {file_name}.")
+        return
+
+    # Determine if the data contains objects that need serialization
+    first_item = data[0]
+    
+    if hasattr(first_item, '__dict__') and not isinstance(first_item, (str, int, float, bool)):
+        print("ℹ️ Detected object types. Converting objects to dictionary rows for CSV writing.")
+        
+        # Extract attributes/properties into a list of dictionaries
+        data_to_write = []
+        # We assume all objects in the list have the same structure for simplicity
+        # A more robust solution would require knowing the schema.
+        for item in data:
+            row_dict = {}
+            # Export all instance attributes to the dictionary
+            for key, value in item.__dict__.items():
+                row_dict[key] = value
+            data_to_write.append(row_dict)
+            
+        # Since csv.writer doesn't handle dicts directly, we list the keys from the first dict
+        fieldnames = list(data_to_write[0].keys())
+        
+        with open(file_name, "w", newline="", encoding="utf-8") as file:
+            writer = csv.DictWriter(file, fieldnames=fieldnames)
+            
+            writer.writeheader()
+            writer.writerows(data_to_write)
+            
+    else:
+        # Assume data is already a list of lists/tuples
+        with open(file_name, "w", newline="", encoding="utf-8") as file:
+            writer = csv.writer(file)
+            
+            # Write all rows at once
+            writer.writerows(data)
+
 def run_command(cmd_str: str, progress_prefix: str = "Progress", look_for: list = ["frame=", "time=", "fps=", "Rendering frame"]) -> bool:
    """Runs a system command, streams its output live, and reports errors on failure."""
    args = shlex.split(cmd_str)

package/video_editing/__init__.py

@@ -0,0 +1,13 @@
+# package/video_editing/__init__.py
+
+# Expose components from individual modules using relative imports
+from .combine_videos import merge_videos
+from .split_video_by_timestamps import split_video
+from .build_shorts import fit_to_9_16_letterbox
+
+# Explicitly define what is exposed when importing the subpackage
+__all__ = [
+    "merge_videos",
+    "split_video",
+    "fit_to_9_16_letterbox"
+]

package/video_editing/build_shorts.py

@@ -4,12 +4,9 @@ import json
 import package.utils as utils
 import tempfile
 from pathlib import Path
-from moviepy import VideoFileClip, ColorClip, CompositeVideoClip
 from moviepy.video.VideoClip import TextClip
 
 from pathlib import Path
-import numpy as np
-from PIL import Image, ImageFilter
 
 def get_video_info(input_path: Path) -> tuple:
     """Uses ffprobe to instantly read input video dimensions and frame rate."""
@@ -126,7 +123,6 @@ def fit_to_9_16_letterbox(input_path: str, top_text: str = "TOP TEXT", bottom_te
             if os.path.exists(path):
                 os.unlink(path)
 
-
 if __name__ == "__main__":
     import time
 

package/video_editing/combine_videos.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""Combine multiple video files into one file using FFmpeg.
+
+Example:
+    python helper_files/combine_videos.py output.mp4 video1.mp4 video2.mp4 video3.mp4
+    python helper_files/combine_videos.py --output merged.mp4 --inputs video1.mp4 video2.mp4
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def build_concat_list(input_files, list_path):
+    with open(list_path, "w", encoding="utf-8") as f:
+        for path in input_files:
+            f.write(f"file '{os.path.abspath(path)}'\n")
+
+
+def run_ffmpeg(cmd):
+    print("Running:", " ".join(cmd))
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.returncode != 0:
+        print(result.stdout)
+        print(result.stderr)
+        raise RuntimeError(f"FFmpeg failed with exit code {result.returncode}")
+    print("✅ Finished successfully.")
+
+
+def concat_videos_copy(input_files, output_path):
+    list_path = Path(output_path).with_suffix(".concat.txt")
+    build_concat_list(input_files, list_path)
+
+    cmd = [
+        "ffmpeg",
+        "-y",
+        "-f", "concat",
+        "-safe", "0",
+        "-i", str(list_path),
+        "-c", "copy",
+        str(output_path),
+    ]
+    run_ffmpeg(cmd)
+    list_path.unlink(missing_ok=True)
+
+
+def concat_videos_reencode(input_files, output_path):
+    inputs = []
+    filter_parts = []
+    audio_parts = []
+
+    for i, video_path in enumerate(input_files):
+        inputs.extend(["-i", str(video_path)])
+        # Use semicolons to separate different input streams in the filter graph
+        filter_parts.append(f"[{i}:v]scale=trunc(iw/2)*2:trunc(ih/2)*2,setpts=PTS-STARTPTS[v{i}]")
+        audio_parts.append(f"[{i}:a]aresample=async=1[a{i}]")
+
+    # Correctly separate individual stream filters with semicolons
+    filter_complex = f"{';'.join(filter_parts)};{';'.join(audio_parts)};"
+    filter_complex += "".join(f"[v{i}][a{i}]" for i in range(len(input_files)))
+    filter_complex += f"concat=n={len(input_files)}:v=1:a=1[outv][outa]"
+
+    cmd = [
+        "ffmpeg",
+        "-y",
+        *inputs,
+        "-filter_complex", filter_complex,
+        "-map", "[outv]",
+        "-map", "[outa]",
+        "-c:v", "libx264",
+        "-crf", "18",         # High quality video
+        "-preset", "slow",    # High efficiency
+        "-c:a", "aac",
+        "-b:a", "256k",       # High quality audio
+        "-movflags", "+faststart",
+        str(output_path),
+    ]
+    run_ffmpeg(cmd)
+
+
+def merge_videos(input_files, output_path):
+    input_files = [str(Path(p).expanduser().resolve()) for p in input_files]
+    output_path = str(Path(output_path).expanduser().resolve())
+
+    if len(input_files) < 2:
+        raise ValueError("You need at least 2 video files to merge.")
+
+    for path in input_files:
+        if not os.path.exists(path):
+            raise FileNotFoundError(f"File not found: {path}")
+
+    output_dir = Path(output_path).parent
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    try:
+        concat_videos_copy(input_files, output_path)
+    except Exception as exc:
+        print(f"Fast copy method failed: {exc}")
+        print("Retrying with re-encode fallback...")
+        concat_videos_reencode(input_files, output_path)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Join two or more video files together with FFmpeg.")
+    parser.add_argument("output_file", help="Path to the merged output video")
+    parser.add_argument("input_files", nargs="+", help="Two or more videos to concatenate")
+    args = parser.parse_args()
+
+    try:
+        merge_videos(args.input_files, args.output_file)
+        print(f"Merged video saved to: {args.output_file}")
+    except Exception as exc:
+        print(f"Error: {exc}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/video_editing/highlights.py

@@ -0,0 +1,378 @@
+import re
+import cv2
+import numpy as np
+import argparse
+import json
+from pathlib import Path
+from moviepy import VideoFileClip
+
+# ---- CONFIG ----
+DEFAULT_KEYWORDS = [
+    # Pure Hype - the #1 highlight signal
+    "pog", "poggers", "pogchamp", "let's go", "lets go", "huge", "insane", "crazy",
+    "unbelievable", "no way", "way too good", "oh my god", "omg", "wow", "holy",
+    "holy shit", "holy fuck", "what the fuck", "wtf", "lfg", "clip that", "clip it",
+    "someone clip", "chat clip", "world record", "pb", "personal best",
+
+    # Chat interaction = something happened
+    "chat", "look at chat", "chat is going crazy", "we are so back", "we're so back",
+    "it's over", "ggs", "gg", "not like this", "like this", "AYAYA", "KEKW",
+    "monka", "pepega", "copium",
+
+    # Gaming clutch moments
+    "clutch", "ace", "1v4", "1v3", "1v2", "quadra", "penta", "pentakill",
+    "triple kill", "rampage", "unstoppable", "godlike", "wipe", "team wipe",
+    "no scope", "collat", "one shot", "one tap", "200 pumped", "knocked", "cracked",
+
+    # Reaction / laugh / fail
+    "bro", "dude", "what is happening", "what just happened", "there is no way",
+    "you're kidding", "stop", "no no no", "yes yes yes", "i can't", "i cant believe",
+    "this is insane", "this guy", "look at this", "wait wait wait"
+]
+
+# You can add your own: --keywords "goal, fight, knock out"
+import subprocess
+import tempfile
+import os
+from scipy.io import wavfile
+
+def get_av_scores(video_path, sample_fps=2):
+    print(f"-> Opening: {video_path}", flush=True)
+    if not os.path.exists(video_path):
+        raise FileNotFoundError(video_path)
+
+    with VideoFileClip(video_path) as clip:
+        duration = clip.duration
+        fps = clip.fps or 30
+        has_audio = clip.audio is not None
+
+    print(f" duration={duration:.2f}s fps={fps} has_audio={has_audio}", flush=True)
+    n_secs = int(duration)
+
+     # ---- 1. AUDIO (streaming, constant RAM) ----
+    print(f"[1/3] Audio energy...", flush=True)
+    audio_per_sec = np.zeros(n_secs)
+
+    if has_audio:
+        tmp_wav = tempfile.mktemp(suffix=".wav")
+        try:
+            print(f" extracting mono 16k wav with ffmpeg...", flush=True)
+            cmd = [
+                "ffmpeg", "-y",
+                "-i", video_path,
+                "-ar", "16000",
+                "-ac", "1",
+                "-c:a", "pcm_s16le",
+                tmp_wav
+            ]
+            result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=300)
+            if result.returncode!= 0:
+                print(f" ffmpeg failed: {result.stderr.decode()[:1000]}")
+            else:
+                wav_size = os.path.getsize(tmp_wav)
+                print(f" wav size: {wav_size} bytes, computing RMS streaming...", flush=True)
+
+                # STREAM instead of loading all at once
+                import wave
+                with wave.open(tmp_wav, 'rb') as wf:
+                    sr = wf.getframerate() # should be 16000
+                    n_frames = wf.getnframes()
+                    # sr frames = 1 second
+                    for i in range(n_secs):
+                        if wf.tell() >= n_frames:
+                            break
+                        # read 1 second
+                        frames = wf.readframes(sr)
+                        if not frames:
+                            break
+                        # int16 -> float without extra copy
+                        chunk = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
+                        if len(chunk) > 0:
+                            audio_per_sec[i] = np.sqrt(np.mean(chunk**2))
+
+                        if i % 300 == 0: # log every 5 min
+                            print(f" audio {i}/{n_secs}s", flush=True)
+
+                if audio_per_sec.max() > 0:
+                    audio_per_sec = (audio_per_sec - audio_per_sec.min()) / (audio_per_sec.max() - audio_per_sec.min() + 1e-6)
+                print(f" audio done", flush=True)
+
+        except Exception as e:
+            print(f" audio error: {e}")
+            import traceback; traceback.print_exc()
+        finally:
+            if os.path.exists(tmp_wav):
+                os.remove(tmp_wav)
+    else:
+        print(" no audio track", flush=True)
+
+    # ---- 2. MOTION ----
+    print(f"[2/3] Motion analysis sample_fps={sample_fps}...", flush=True)
+    cap = cv2.VideoCapture(video_path)
+    motion_scores = []
+    prev_gray = None
+    frame_interval = int(fps / sample_fps) if fps > 0 else 15
+    num_samples = int(duration * sample_fps)
+
+    for i in range(num_samples):
+        cap.set(cv2.CAP_PROP_POS_FRAMES, i * frame_interval)
+        ret, frame = cap.read()
+        if not ret:
+            continue
+        gray = cv2.cvtColor(cv2.resize(frame, (320, 180)), cv2.COLOR_BGR2GRAY)
+        if prev_gray is not None:
+            motion_scores.append(np.mean(cv2.absdiff(gray, prev_gray)) / 255.0)
+        else:
+            motion_scores.append(0)
+        prev_gray = gray
+        if i % 50 == 0:
+            print(f" motion {i}/{num_samples}", flush=True)
+    cap.release()
+
+    if len(motion_scores) > 1:
+        x = np.linspace(0, duration-1, len(motion_scores))
+        motion_per_sec = np.interp(np.arange(n_secs), x, motion_scores)
+        if motion_per_sec.max() > 0:
+            motion_per_sec = (motion_per_sec - motion_per_sec.min()) / (motion_per_sec.max() - motion_per_sec.min() + 1e-6)
+    else:
+        motion_per_sec = np.zeros(n_secs)
+
+    combined_av = 0.6 * audio_per_sec + 0.4 * motion_per_sec
+    return combined_av, duration
+
+def srt_time_to_seconds(time_str):
+    # 00:01:23,456 or 00:01:23.456
+    time_str = time_str.strip().replace('.', ',')
+    h, m, s_ms = time_str.split(':')
+    s, ms = s_ms.split(',')
+    return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000.0
+
+def load_text_from_srt(srt_path):
+    print(f"[3/3] Loading transcript from SRT: {srt_path}")
+    if not Path(srt_path).exists():
+        raise FileNotFoundError(f"SRT not found: {srt_path}")
+
+    # utf-8-sig strips BOM that Whisper sometimes adds
+    with open(srt_path, 'r', encoding='utf-8-sig') as f:
+        content = f.read().strip()
+
+    if not content:
+        raise ValueError(f"SRT file is empty: {srt_path}")
+
+    # Split by blank line - works for \n\n and \r\n\r\n
+    blocks = re.split(r'\r?\n\r?\n', content)
+    
+    segments = []
+    # Pattern inside each block only
+    time_pattern = re.compile(r'(\d{2}:\d{2}:\d{2}[,\.]\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}[,\.]\d{3})')
+
+    for block in blocks:
+        block = block.strip()
+        if not block:
+            continue
+        
+        lines = block.splitlines()
+        if len(lines) < 2:
+            continue
+
+        # Find time line (line 0 or 1 - index may be missing)
+        time_match = None
+        text_start_idx = 0
+        for idx, line in enumerate(lines[:3]): # check first 3 lines for time
+            m = time_pattern.search(line)
+            if m:
+                time_match = m
+                text_start_idx = idx + 1
+                break
+        
+        if not time_match:
+            continue
+
+        start_str, end_str = time_match.groups()
+        text = ' '.join(lines[text_start_idx:]).strip()
+        
+        # Clean Whisper / Twitch artifacts
+        text = re.sub(r'<[^>]+>', '', text) # remove <i>, <font> etc
+        text = text.replace('\n', ' ').strip()
+        
+        if text:
+            segments.append({
+                "start": srt_time_to_seconds(start_str),
+                "end": srt_time_to_seconds(end_str),
+                "text": text
+            })
+    
+    print(f"Loaded {len(segments)} segments from SRT")
+    if len(segments) == 0:
+        print(f"WARNING: Parsed 0 segments. First 500 chars of file:\n{content[:500]}")
+    return segments
+
+def score_keywords(segments, duration, keywords):
+    """Returns per-second keyword boost score + keyword hits"""
+    keyword_score = np.zeros(int(duration))
+    hits = []
+
+    # Build regex with word boundaries for each keyword
+    # \bace\b won't match "place", \bgg\b won't match "ggs"
+    patterns = []
+    for kw in keywords:
+        kw = kw.strip()
+        if not kw:
+            continue
+        # \b at start/end, but handle multi-word phrases
+        # re.escape handles "let's go", "1v4", etc
+        pat = re.compile(r'\b' + re.escape(kw.lower()) + r'\b', re.IGNORECASE)
+        patterns.append((kw.lower(), pat))
+
+    for seg in segments:
+        text = seg['text']
+        text_lower = text.lower()
+        for kw_lower, pat in patterns:
+            if pat.search(text_lower):
+                start = int(max(0, seg['start'] - 1))
+                end = int(min(duration-1, seg['end'] + 2))
+                keyword_score[start:end+1] += 1.0
+                hits.append({
+                    "timestamp": round(seg['start'], 2),
+                    "end": round(seg['end'], 2),
+                    "keyword": kw_lower,
+                    "text": text.strip()
+                })
+                break
+
+    if keyword_score.max() > 0:
+        keyword_score = keyword_score / keyword_score.max()
+
+    return keyword_score, hits
+
+def find_highlights(final_scores, duration, top_n=5, clip_len=8, min_gap=12):
+    candidates = []
+    for i in range(len(final_scores) - clip_len):
+        candidates.append((np.mean(final_scores[i:i+clip_len]), i))
+
+    candidates.sort(reverse=True, key=lambda x: x[0])
+    picked = []
+    for score, start in candidates:
+        if all(abs(start - p[1]) >= min_gap for p in picked):
+            picked.append((score, start))
+        if len(picked) >= top_n:
+            break
+
+    picked.sort(key=lambda x: x[1])
+    return [(s, e, sc) for sc, s in [(sc, st) for sc, st in picked] for e in [s+clip_len]]
+
+def format_timestamp(seconds):
+    m, s = divmod(int(seconds), 60)
+    h, m = divmod(m, 60)
+    return f"{h:02d}:{m:02d}:{s:02d}" if h > 0 else f"{m:02d}:{s:02d}"
+
+def highlights(video_path, keywords=None, top_n=5, clip_len=8, min_gap=12):
+    av_score, duration = get_av_scores(video_path, sample_fps=0.5)
+    
+    segments = load_text_from_srt(video_path.replace('.mp4', '.srt')) 
+    kw_score, kw_hits = score_keywords(segments, duration, keywords)
+
+    # FINAL SCORE: AV + keyword boost (keyword is huge if found)
+    final_score = av_score + (kw_score * 1.5)
+    final_score = (final_score - final_score.min()) / (final_score.max() - final_score.min() + 1e-6)
+    final_score = np.convolve(final_score, np.ones(3)/3, mode='same') # smooth
+
+    highlights = find_highlights(final_score, duration, top_n=top_n, clip_len=clip_len)
+
+    # --- OUTPUT ---
+    print("\n" + "="*60)
+    print("KEYWORD HITS:")
+    for h in kw_hits:
+        print(f" [{format_timestamp(h['timestamp'])}] ({h['keyword']}) -> {h['text']}")
+
+    print("\n" + "="*60)
+    print("FINAL HIGHLIGHTS:")
+    output_data = []
+    for i, (start, end, score) in enumerate(highlights, 1):
+        # find transcript in that window
+        texts_in_window = [s['text'].strip() for s in segments if not (s['end'] < start or s['start'] > end)]
+        transcript = " ".join(texts_in_window)[:120]
+
+        print(f"{i}. {format_timestamp(start)} - {format_timestamp(end)} (score {score:.2f}) | {transcript}")
+        output_data.append({
+            "id": i,
+            "start_sec": float(start),
+            "end_sec": float(end),
+            "start_timestamp": format_timestamp(start),
+            "end_timestamp": format_timestamp(end),
+            "score": float(score),
+            "transcript": transcript
+        })
+
+    # Save timestamps to JSON
+    with open("highlights.json", "w") as f:
+        json.dump({"video": video_path, "keywords": keywords, "highlights": output_data, "keyword_hits": kw_hits}, f, indent=2)
+    print(f"\nTimestamps saved to highlights.json")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Find highlights with Whisper keywords + timestamps")
+    parser.add_argument("video", help="Input video file")
+    parser.add_argument("--keywords", type=str, default=",".join(DEFAULT_KEYWORDS), help="Comma-separated keywords to hunt for")
+    parser.add_argument("--model", type=str, default="base", help="Whisper model: tiny, base, small, medium")
+    parser.add_argument("--top", type=int, default=5)
+    parser.add_argument("--len", type=int, default=30, help="Highlight length in seconds")
+    parser.add_argument("--no-cut", action="store_true", help="Only print timestamps, don't cut video files")
+    args = parser.parse_args()
+
+    keywords = [k.strip() for k in args.keywords.split(",") if k.strip()]
+
+    av_score, duration = get_av_scores(args.video, sample_fps=0.5)
+    
+    segments = load_text_from_srt(args.video.replace('.mp4', '.srt')) 
+    kw_score, kw_hits = score_keywords(segments, duration, keywords)
+
+    # FINAL SCORE: AV + keyword boost (keyword is huge if found)
+    final_score = av_score + (kw_score * 1.5)
+    final_score = (final_score - final_score.min()) / (final_score.max() - final_score.min() + 1e-6)
+    final_score = np.convolve(final_score, np.ones(3)/3, mode='same') # smooth
+
+    highlights = find_highlights(final_score, duration, top_n=args.top, clip_len=args.len)
+
+    # --- OUTPUT ---
+    print("\n" + "="*60)
+    print("KEYWORD HITS:")
+    for h in kw_hits:
+        print(f" [{format_timestamp(h['timestamp'])}] ({h['keyword']}) -> {h['text']}")
+
+    print("\n" + "="*60)
+    print("FINAL HIGHLIGHTS:")
+    output_data = []
+    for i, (start, end, score) in enumerate(highlights, 1):
+        # find transcript in that window
+        texts_in_window = [s['text'].strip() for s in segments if not (s['end'] < start or s['start'] > end)]
+        transcript = " ".join(texts_in_window)[:120]
+
+        print(f"{i}. {format_timestamp(start)} - {format_timestamp(end)} (score {score:.2f}) | {transcript}")
+        output_data.append({
+            "id": i,
+            "start_sec": float(start),
+            "end_sec": float(end),
+            "start_timestamp": format_timestamp(start),
+            "end_timestamp": format_timestamp(end),
+            "score": float(score),
+            "transcript": transcript
+        })
+
+    # Save timestamps to JSON
+    with open("highlights.json", "w") as f:
+        json.dump({"video": args.video, "keywords": keywords, "highlights": output_data, "keyword_hits": kw_hits}, f, indent=2)
+    print(f"\nTimestamps saved to highlights.json")
+
+    # Cut videos if requested
+    #if not args.no_cut:
+    if args.no_cut:
+        print("\nCutting clips...")
+        Path("highlights").mkdir(exist_ok=True)
+        clip = VideoFileClip(args.video)
+        for h in output_data:
+            sub = clip.subclip(h['start_sec'], h['end_sec'])
+            out = f"highlights/highlight_{h['id']}_{h['start_timestamp'].replace(':','-')}.mp4"
+            sub.write_videofile(out, codec="libx264", audio_codec="aac", logger=None)
+        clip.close()
+        print("Done ->./highlights/")

package/video_editing/split_video_by_timestamps.py

@@ -0,0 +1,112 @@
+import subprocess
+import os
+from typing import List, Tuple
+
+def split_video(input_file: str, output_dir: str, timestamps: List[Tuple[float, float]], frame_perfect: bool = False) -> None:
+    """
+    Splits a video file into multiple segments based on specified time ranges.
+    
+    Args:
+        input_file (str): The path to the video file to be split.
+        output_dir (str): The directory where the split segments will be saved.
+        timestamps (List[Tuple[float, float]]): A list of tuples representing (start_time, end_time).
+        frame_perfect (bool): If True, skips fast-copy and forces high-quality re-encoding 
+                              to ensure cuts happen exactly on the requested frame.
+    """
+    if not os.path.exists(input_file):
+        print(f"Error: Input file not found at {input_file}")
+        return
+
+    os.makedirs(output_dir, exist_ok=True)
+    print(f"Starting video split process for {input_file}...")
+
+    for i, (start, end) in enumerate(timestamps):
+        segment_name = f"segment_{i+1:02d}_{start:.2f}s_to_{end:.2f}s.mp4"
+        output_file = os.path.join(output_dir, segment_name)
+
+        print(f"\n-> Processing segment {i+1}/{len(timestamps)}: {segment_name}")
+        
+        # Method 1: Fast Copy (Lossless, no re-encoding)
+        # Moving -ss BEFORE -i fixes the frozen/corrupted initial frames.
+        fast_command = [
+            "ffmpeg", "-y",
+            "-ss", str(start),
+            "-i", input_file,
+            "-to", str(end),
+            "-c", "copy",
+            output_file
+        ]
+        
+        # Method 2: High-Quality Re-encode (Frame-perfect fallback)
+        # Used if fast copy fails or if frame-accuracy is explicitly requested.
+        reencode_command = [
+            "ffmpeg", "-y",
+            "-ss", str(start),
+            "-i", input_file,
+            "-to", str(end),
+            "-c:v", "libx264",
+            "-crf", "18",         # High visual quality
+            "-preset", "slow",    # Better compression efficiency
+            "-c:a", "aac",
+            "-b:a", "256k",       # High-fidelity audio
+            output_file
+        ]
+
+        try:
+            if frame_perfect:
+                print("Running in frame-perfect mode (re-encoding)...")
+                subprocess.run(reencode_command, check=True, capture_output=True, text=True)
+            else:
+                try:
+                    print("Attempting lightning-fast stream copy...")
+                    subprocess.run(fast_command, check=True, capture_output=True, text=True)
+                except subprocess.CalledProcessError:
+                    print("⚠️ Fast copy failed. Falling back to high-quality re-encode...")
+                    subprocess.run(reencode_command, check=True, capture_output=True, text=True)
+            
+            print(f"Successfully created segment: {output_file}")
+
+        except subprocess.CalledProcessError as e:
+            print(f"❌ Error splitting video segment {i+1}:")
+            print(f"Command failed with return code {e.returncode}")
+            print(f"STDERR: {e.stderr}")
+        except FileNotFoundError:
+            print("-" * 50)
+            print("CRITICAL ERROR: FFmpeg executable not found.")
+            print("Please ensure 'ffmpeg' is installed and accessible in your system's PATH.")
+            print("-" * 50)
+            return
+
+def main():
+    """
+    Example usage: 
+    Assumes 'input.mp4' exists and a 'split_videos' directory is ready.
+    """
+    # --- CONFIGURATION ---
+    INPUT_VIDEO_PATH = "path/to/your/input_video.mp4"  # <<< CHANGE THIS
+    OUTPUT_SEGMENTS_DIR = "output_splits"              # <<< CHANGE THIS
+    
+    # Define time segments: [(start_time, end_time), ...]
+    # Times should be floats representing seconds.
+    # Example 1: 0s to 10s, then 10s to 20s
+    TIMESTAMPS: List[Tuple[float, float]] = [
+        (0.0, 10.0),
+        (15.0, 25.0),
+        (30.0, 35.0)
+    ]
+    # ---------------------
+
+    if not os.path.exists(INPUT_VIDEO_PATH):
+        print("!!! IMPORTANT !!!")
+        print(f"Please update the 'INPUT_VIDEO_PATH' variable in the main() function with the correct path to your video.")
+        print("!!! Script execution aborted. !!!")
+        return
+
+    split_video(
+        input_file=INPUT_VIDEO_PATH,
+        output_dir=OUTPUT_SEGMENTS_DIR,
+        timestamps=TIMESTAMPS
+    )
+
+if __name__ == "__main__":
+    main()

package/youtube/__init__.py

@@ -0,0 +1,13 @@
+# package/youtube/__init__.py
+
+# Expose components from individual modules using relative imports
+from .find_tags import find_tags
+from .uploader import upload_video, CategoryId
+
+# Explicitly define what is exposed when importing the subpackage
+__all__ = [
+    "find_tags",
+    "get_top_nouns",
+    "upload_video",
+    "CategoryId",
+]

package/youtube/config.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+import json
+
+from google.oauth2.credentials import Credentials
+from google.auth.transport.requests import Request
+from googleapiclient.discovery import build
+
+def get_youtube_service():
+    """Get the YouTube service using the loaded credentials."""
+    try:
+        with open('secrets.json', 'r') as f:
+            creds_data = json.load(f)
+                
+        credentials = Credentials(
+            token=creds_data['token'],
+            refresh_token=creds_data['refresh_token'],
+            token_uri=creds_data['token_uri'],
+            client_id=creds_data['client_id'],
+            client_secret=creds_data['client_secret'],
+            scopes=creds_data['scopes']
+        )
+            
+        # Refresh token if expired
+        if credentials.expired:
+            credentials.refresh(Request())
+                
+    except FileNotFoundError:
+        print("Error: secrets.json not found! Please run setup.py first.")
+        return None
+    except Exception as e:
+        print(f"Error loading credentials: {str(e)}")
+        return None
+    
+    if credentials is None:
+        print("Error: Credentials not loaded. Please check secrets.json.")
+        return None
+    
+    try:
+        youtube_service = build('youtube', 'v3', credentials=credentials)
+        return youtube_service
+    except Exception as e:
+        print(f"Error creating YouTube service: {str(e)}")
+        return None

package/youtube/find_tags.py

@@ -1,37 +1,12 @@
 #!/usr/bin/env python3
-import json
-from google.oauth2.credentials import Credentials
-from google.auth.transport.requests import Request
+from typing import Any
+
 from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 
-def load_credentials():
-    """Load credentials from secrets.json (Reused from your previous flow)"""
-    try:
-        with open('secrets.json', 'r') as f:
-            creds_data = json.load(f)
+import package.youtube.config as config
 
-        credentials = Credentials(
-            token=creds_data['token'],
-            refresh_token=creds_data['refresh_token'],
-            token_uri=creds_data['token_uri'],
-            client_id=creds_data['client_id'],
-            client_secret=creds_data['client_secret'],
-            scopes=creds_data['scopes']
-        )
-        
-        if credentials.expired:
-            credentials.refresh(Request())
-            
-        return credentials
-    except FileNotFoundError:
-        print("Error: secrets.json not found!")
-        return None
-    except Exception as e:
-        print(f"Error loading credentials: {str(e)}")
-        return None
-
-def get_my_uploads_playlist_id(youtube):
+def get_my_uploads_playlist_id(youtube: build) -> Any:
     """Retrieves the system upload playlist ID for the authenticated user's channel."""
     try:
         # mine=True automatically references the authorized account
@@ -47,7 +22,7 @@ def get_my_uploads_playlist_id(youtube):
         print(f"API Error retrieving channel details: {str(e)}")
         return None
 
-def scan_channel_videos_for_tag(youtube, uploads_playlist_id: str, target_tag: str):
+def scan_channel_videos_for_tag(youtube: build, uploads_playlist_id: str, target_tag: str) -> list:
     """
     Loops through all channel video uploads and filters those possessing the target tag.
     """
@@ -120,12 +95,10 @@ def scan_channel_videos_for_tag(youtube, uploads_playlist_id: str, target_tag: s
     
     return matched_videos
 
-def main():
-    credentials = load_credentials()
-    if not credentials:
-        return
+def find_tags():
     
-    youtube = build('youtube', 'v3', credentials=credentials)
+  
+    youtube = config.get_youtube_service()
     
     # 1. Fetch your dynamic uploads playlist pointer
     uploads_id = get_my_uploads_playlist_id(youtube)
@@ -144,4 +117,4 @@ def main():
                 print(f"{index}. {item['title']} -> https://youtu.be{item['id']}")
 
 if __name__ == "__main__":
-    main()
+    find_tags()

package/youtube/uploader.py

@@ -1,16 +1,15 @@
 #!/usr/bin/env python3
 import os
-import json
 import argparse
 from enum import Enum
-from datetime import datetime
-from google.oauth2.credentials import Credentials
-from google_auth_oauthlib.flow import InstalledAppFlow
-from google.auth.transport.requests import Request
+from datetime import datetime, timezone
+
 from googleapiclient.discovery import build
 from googleapiclient.http import MediaFileUpload
 from googleapiclient.errors import HttpError
 
+import package.youtube.config as config
+
 class CategoryId(Enum):
     """Official YouTube Category IDs for API Uploads."""
 
@@ -30,34 +29,7 @@ class CategoryId(Enum):
     SCIENCE_AND_TECHNOLOGY = "28"
     NONPROFITS_AND_ACTIVISM = "29"
 
-def load_credentials():
-    """Load credentials from secrets.json"""
-    try:
-        with open('secrets.json', 'r') as f:
-            creds_data = json.load(f)
-            
-        credentials = Credentials(
-            token=creds_data['token'],
-            refresh_token=creds_data['refresh_token'],
-            token_uri=creds_data['token_uri'],
-            client_id=creds_data['client_id'],
-            client_secret=creds_data['client_secret'],
-            scopes=creds_data['scopes']
-        )
-        
-        # Refresh token if expired
-        if credentials.expired:
-            credentials.refresh(Request())
-            
-        return credentials
-    except FileNotFoundError:
-        print("Error: secrets.json not found! Please run setup.py first.")
-        return None
-    except Exception as e:
-        print(f"Error loading credentials: {str(e)}")
-        return None
-
-def upload_video(file_path: str, title: str, category: CategoryId, description: str = "", privacyStatus: str = 'private', tags: list = None, release_time: datetime = None):
+def upload_video(file_path: str, title: str, category: CategoryId, description: str = "", privacyStatus: str = 'private', tags: list = None, release_time: datetime = None) -> str | None:
     """
     Upload a video to YouTube
     
@@ -71,6 +43,8 @@ def upload_video(file_path: str, title: str, category: CategoryId, description:
         release_time (datetime): Optional timezone-aware UTC datetime object for timed release
 
         schedule_date = datetime.now(timezone.utc) + timedelta(days=2)
+    Returns:
+        a String(str) of with the youtube video id or None if somthing fails
     """
     if tags is None:
         tags = []
@@ -79,15 +53,11 @@ def upload_video(file_path: str, title: str, category: CategoryId, description:
         # Check if file exists
         if not os.path.exists(file_path):
             print(f"❌ Error: File not found: {file_path}")
-            return False
+            return None
 
-        # Load credentials
-        credentials = load_credentials()
-        if not credentials:
-            return False
   
         # Create YouTube API client
-        youtube = build('youtube', 'v3', credentials=credentials)
+        youtube = config.get_youtube_service()
 
         # Configure the status object dynamically
         status_body = {
@@ -100,7 +70,8 @@ def upload_video(file_path: str, title: str, category: CategoryId, description:
 
             # If release_time is passed, YouTube forces privacyStatus to 'private'
             status_body['privacyStatus'] = 'private'
-            status_body['publishAt'] = release_time.strftime('%Y-%m-%dT%H:%M:%S.000Z')
+            #status_body['publishAt'] = release_time.strftime('%Y-%m-%dT%H:%M:%S.000Z')
+            status_body['publishAt'] = release_time.isoformat() 
             print(f"Configuring timed release for: {status_body['publishAt']}")
         else:
             # Standard immediate upload
@@ -150,14 +121,14 @@ def upload_video(file_path: str, title: str, category: CategoryId, description:
         else:
             print(f"Current Privacy Status: {response['status']['privacyStatus']}")
             
-        return True
+        return response['id']
 
     except HttpError as e:
         print(f"An HTTP error occurred: {str(e)}")
-        return False
+        return None
     except Exception as e:
         print(f"An error occurred: {str(e)}")
-        return False
+        return None
 
 def main():
     parser = argparse.ArgumentParser(description='Upload a video to YouTube')
@@ -197,5 +168,6 @@ def main():
         tags=parsed_tags,
         release_time=release_datetime
     )
+
 if __name__ == "__main__":
     main()

requirements.txt

@@ -5,43 +5,68 @@ anyio==4.15.1
 attrs==26.1.0
 av==18.1.0
 certifi==2026.7.22
+cffi==2.1.1
 charset-normalizer==3.5.1
 click==8.5.0
+cloudpickle==3.1.2
+cryptography==50.0.1
 ctranslate2==4.8.2
 decorator==5.3.1
+defusedxml==0.7.1
 enum-tools==0.13.0
 faster-whisper==1.2.1
 filelock==3.32.6
 flatbuffers==25.12.19
 frozenlist==1.8.0
 fsspec==2026.7.0
+google-api-core==2.36.0
+google-api-python-client==2.200.0
+google-auth==2.58.0
+google-auth-httplib2==0.4.2
+google-auth-oauthlib==1.4.1
+googleapis-common-protos==1.75.3
+greenlet==3.5.6
 h11==0.16.0
 hf-xet==1.6.0
 httpcore==1.0.9
+httplib2==0.32.0
 httpx==0.28.1
 huggingface_hub==1.31.0
 idna==3.19
 ImageIO==2.37.4
 imageio-ffmpeg==0.6.0
+joblib==1.6.0
 moviepy==2.2.1
 multidict==6.8.0
+nltk==3.10.3
 numpy==2.5.3
+oauthlib==3.3.1
 onnxruntime==1.30.0
 opencv-python==5.0.0.93
+opentelemetry-api==1.44.0
 packaging==26.3
 pillow==11.3.0
 proglog==0.1.12
 propcache==0.5.2
+proto-plus==1.28.4
 protobuf==7.36.1
+pyasn1==0.6.4
+pyasn1_modules==0.4.2
+pycparser==3.0
 Pygments==2.21.0
+pyparsing==3.3.2
 python-dateutil==2.9.0.post0
 python-dotenv==1.2.3
 PyYAML==6.0.3
+regex==2026.9.10
 requests==2.34.2
+requests-oauthlib==2.0.0
 six==1.17.0
+SQLAlchemy==2.0.54
 tokenizers==0.23.2
 tqdm==4.70.1
 twitchAPI==4.5.0
 typing_extensions==4.16.0
+uritemplate==4.2.0
 urllib3==2.7.0
 yarl==1.24.5