update system to use find Highlights *not tested

.gitignore

@@ -10,3 +10,5 @@ download
 output.log
 *.mp4
 database copy.db
+tiktok_secrets.json
+cookies.txt

delete/twitch_down_uploader.py

@@ -4,7 +4,7 @@ import os
 import package.utils as linux
 import time
 
-from package.database import Database
+from package.database import Repository
 
 import package.youtube.uploader as uploader
 from package.youtube.uploader import CategoryId
@@ -319,7 +319,7 @@ def get_vod_ids_simplified(channel_name: str):
 if __name__ == "__main__":
     tables = ["clips"]
     for table in tables:
-        DB = Database(table)
+        DB = Repository(table)
 
         get_vod_ids_simplified(CHANNEL_NAME)
         #download()

delete/twitch_download_clips.py

@@ -3,13 +3,13 @@ import requests
 import subprocess
 from datetime import datetime
 
-from package.database import Database
+from package.database import Repository
 
 import package.youtube.uploader as uploader
 from package.youtube.uploader import CategoryId
 
 CHANNEL_NAME = 'teampgp'  # Replace with the streamer's username
-DB = Database("clips")
+DB = Repository("clips")
 
 def run_linux_command(command: str):
     """Executes a Linux command, waits for completion, and returns output."""

help.sqlite3-query

@@ -1,9 +1,12 @@
 -- database: ./database.db
 
-SELECT * FROM twitch_videos WHERE clip_is == "1" and uploaded_yt_shorts == "1";
+SELECT * FROM twitch_videos WHERE clip_is == "1";
 UPDATE twitch_videos SET uploaded_yt_shorts = 0 WHERE uploaded_yt_shorts == "1";
 SELECT * FROM twitch_videos WHERE title == "Oh Dear";
 SELECT * FROM twitch_videos WHERE title = 'Kreestuh FINALLY hits her "Ryan Dark Story" limit.  Jesus Big K it''s about time!';
 SELECT * FROM twitch_videos WHERE title = "Bolt-Action Bureaucracy";
 SELECT * FROM twitch_videos WHERE title = "Just after going through a mass starvation...";
 SELECT * FROM twitch_videos WHERE title = "Semi-Automatic Sanctimony";
+SELECT * FROM twitch_videos WHERE id = "2848295283";
+SELECT * FROM twitch_videos WHERE game_name = "Darkwood";
+SELECT * FROM data;

helper_files/chapters.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+import os
+import json
+import sys
+
+# Finds the absolute path to 'project_directory' and adds it to Python's search path
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+sys.path.append(PROJECT_ROOT)
+
+from package.config import DB as DB
+from package.database import Chapter
+
+def get_chapters(json_path):
+    """Loads the JSON file and returns all chapters."""
+    with open(json_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+
+    # The chapters are under video -> chapters
+    video = data.get('video', {})
+    chapters = video.get('chapters', [])
+
+    if not chapters:
+        print("No chapters found.")
+        return []
+
+    print(f"Found {len(chapters)} chapter(s) in video '{video.get('title')}':\n")
+
+    for i, chapter in enumerate(chapters, 1):
+        start_sec = chapter.get('startMilliseconds', 0) / 1000
+        length_sec = chapter.get('lengthMilliseconds', 0) / 1000
+
+        print(f"{i}. {chapter.get('description', 'No Title')}")
+        print(f" - Type: {chapter.get('type')}")
+        print(f" - Start: {start_sec}s ({chapter.get('startMilliseconds')}ms)")
+        print(f" - Length: {length_sec}s ({chapter.get('lengthMilliseconds')}ms)")
+        print(f" - Game: {chapter.get('gameDisplayName')} (ID: {chapter.get('gameId')})")
+        print()
+
+    return chapters
+
+def main():
+    downloads = DB.get_downloaded()
+
+    for download in downloads:
+        json_path = f"download/clips/{download.id}/{download.id}_chat.json" if download.clip_is else f"download/videos/{download.id}/{download.id}_chat.json"
+        if os.path.exists(json_path):
+            chapters = get_chapters(json_path)
+            for chapter in chapters:
+                chapter_record = Chapter(
+                    twitch_video_id=download.id,
+                    start_milliseconds=chapter.get('startMilliseconds'),
+                    length_milliseconds=chapter.get('lengthMilliseconds'),
+                    description=chapter.get('description'),
+                    type=chapter.get('type'),
+                    game_display_name=chapter.get('gameDisplayName'),
+                    game_id=chapter.get('gameId')
+                )
+                DB.insert(chapter_record)
+        else:
+            print(f"JSON file not found for video {download.id}: {json_path}")
+
+if __name__ == "__main__":
+    main()
+    # Usage: python get_chapters.py your_file.json
+    #path = sys.argv[1] if len(sys.argv) > 1 else "/home/richard/python_scripts/download/videos/2873107835/2873107835_chat.json"
+    #try:
+    #    get_chapters(path)
+    #except FileNotFoundError:
+    #    print(f"Error: File not found -> {path}")
+    #except json.JSONDecodeError:
+    #    print(f"Error: File is not valid JSON -> {path}")

helper_files/datetime_fix.py

@@ -55,7 +55,7 @@ def fix_time():
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
     fix_time()
 

helper_files/fix_download.py

@@ -44,7 +44,7 @@ def fix_download():
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
     fix_download()
 

helper_files/import_vod.py

@@ -81,7 +81,7 @@ def read_json(file_path: str):
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
 
 

helper_files/migrate_sql2.py

@@ -0,0 +1,45 @@
+import sqlite3
+DB_PATH = "database.db"
+
+def migrate_to_many(db_path):
+    con = sqlite3.connect(db_path)
+    cur = con.cursor()
+    cur.execute("PRAGMA foreign_keys=OFF;")
+    try:
+        con.execute("BEGIN;")
+        cur.execute("ALTER TABLE youtube_info RENAME TO youtube_info_old;")
+        
+        cur.execute("""
+        CREATE TABLE youtube_info (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            twitch_video_id VARCHAR NOT NULL,
+            yt_id VARCHAR,
+            yt_date VARCHAR,
+            yt_short_id VARCHAR,
+            yt_short_date VARCHAR,
+            yt_chat_SS_id VARCHAR,
+            yt_chat_SS_date VARCHAR,
+            yt_chat_over_id VARCHAR,
+            yt_chat_over_date VARCHAR,
+            FOREIGN KEY(twitch_video_id) REFERENCES twitch_videos(id) ON DELETE CASCADE
+        );
+        """)
+        cur.execute("CREATE INDEX ix_youtube_info_twitch_video_id ON youtube_info (twitch_video_id);")
+
+        cur.execute("""
+        INSERT INTO youtube_info (twitch_video_id, yt_id, yt_date, yt_short_id, yt_short_date, yt_chat_SS_id, yt_chat_SS_date, yt_chat_over_id, yt_chat_over_date)
+        SELECT id, yt_id, yt_date, yt_short_id, yt_short_date, yt_chat_SS_id, yt_chat_SS_date, yt_chat_over_id, yt_chat_over_date
+        FROM youtube_info_old;
+        """)
+        
+        cur.execute("DROP TABLE youtube_info_old;")
+        con.commit()
+        print("Done - now 1-to-Many")
+    except Exception as e:
+        con.rollback()
+        raise e
+    finally:
+        cur.execute("PRAGMA foreign_keys=ON;")
+        con.close()
+
+migrate_to_many(DB_PATH)

helper_files/move_vod.py

@@ -49,7 +49,7 @@ def remove_vods():
 
 def main():
     global DB
-    DB = database.Database()
+    DB = database.Repository()
 
     remove_vods()
 

helper_files/redownload_clips.py

@@ -31,5 +31,5 @@ def redownload_clips():
     
 
 if __name__ == "__main__":
-    DB = database.Database()
+    DB = database.Repository()
     redownload_clips()

helper_files/remove_uneed_files.py

@@ -15,7 +15,7 @@ def remove_files():
     for data in datas:
         
         if data.clip_is is False:
-            continue
+            #continue
             video_path = f"download/videos/{data.id}/{data.id}.mp4"
         else:
             video_path = f"download/clips/{data.id}/{data.id}.mp4"
@@ -27,10 +27,10 @@ def remove_files():
         video_chat_over = ""#video_path.replace(".mp4", "_over_with_chat.mp4")
         mask_path = ""#video_path.replace(".mp4", "_temp_chat_mask.mp4")
 
-        shorts_gaussian_9_16 = video_path.replace(".mp4", "_gaussian_9_16.mp4")
+        shorts_gaussian_9_16 = ""#video_path.replace(".mp4", "_gaussian_9_16.mp4")
-        shorts_9_16 = video_path.replace(".mp4", "_9_16.mp4")
+        shorts_9_16 = ""#video_path.replace(".mp4", "_9_16.mp4")
 
-        srt_file = ""#video_path.replace(".mp4", ".srt")
+        srt_file = video_path.replace(".mp4", ".srt")
         
 
         files = [temp_chat, temp_with_chat, video_chatSS, video_chat_over, mask_path, shorts_gaussian_9_16, shorts_9_16, srt_file, video_chat]

helper_files/youtube_description_updater.py

@@ -0,0 +1,225 @@
+"""
+YouTube Description Updater
+Requires: pip install google-api-python-client google-auth-oauthlib google-auth
+
+1. Enable YouTube Data API v3 in Google Cloud Console
+2. Create OAuth 2.0 credentials (Desktop App)
+3. Download as client_secret.json
+"""
+
+import os
+import sys
+from google.auth.transport.requests import Request
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import InstalledAppFlow
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+# Finds the absolute path to 'project_directory' and adds it to Python's search path
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+sys.path.append(PROJECT_ROOT)
+
+from package.config import DB as DB
+from package.youtube.config import CREDENTIALS as CREDENTIALS
+from package.database import YoutubeInfo as YoutubeInfo
+
+"""
+SCOPES = ["https://www.googleapis.com/auth/youtube.force-ssl"]
+TOKEN_FILE = "token.json"
+CLIENT_SECRET_FILE = "client_secret.json"
+
+def get_authenticated_service():
+    ""Authenticates and returns a YouTube service object.""
+    creds = None
+    if os.path.exists(TOKEN_FILE):
+        creds = Credentials.from_authorized_user_file(TOKEN_FILE, SCOPES)
+    
+    if not creds or not creds.valid:
+        if creds and creds.expired and creds.refresh_token:
+            creds.refresh(Request())
+        else:
+            if not os.path.exists(CLIENT_SECRET_FILE):
+                raise FileNotFoundError(
+                    f"{CLIENT_SECRET_FILE} not found. Download it from Google Cloud Console."
+                )
+            flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRET_FILE, SCOPES)
+            creds = flow.run_local_server(port=0)
+        
+        with open(TOKEN_FILE, "w") as token:
+            token.write(creds.to_json())
+
+    return build("youtube", "v3", credentials=creds)
+"""
+
+def get_video_details(youtube, video_id: str):
+    """Finds the video and returns its current snippet."""
+    try:
+        response = youtube.videos().list(
+            part="snippet,status",
+            id=video_id
+        ).execute()
+
+        if not response["items"]:
+            print(f"❌ Video not found: {video_id}")
+            return None
+        
+        return response["items"][0]
+    except HttpError as e:
+        print(f"❌ API Error while fetching video: {e}")
+        return None
+
+def update_video_description(video_id: str, new_description: str, append: bool = False):
+    """
+    Takes a YouTube video ID, finds the video, and changes its description.
+
+    Args:
+        video_id (str): The YouTube video ID (e.g., 'dQw4w9WgXcQ')
+        new_description (str): The new description to set
+        append (bool): If True, appends new_description to existing description instead of replacing
+
+    Returns:
+        dict | None: Updated video snippet or None on failure
+    """
+    # Load credentials
+    if not CREDENTIALS:
+        print("❌ Error: YouTube API credentials not configured.")
+        return None
+    
+    # Create YouTube API client
+    youtube = build('youtube', 'v3', credentials=CREDENTIALS)
+
+    video = get_video_details(youtube, video_id)
+    if not video:
+        return None
+
+    snippet = video["snippet"]
+    current_title = snippet["title"]
+    current_category = snippet.get("categoryId", "22")
+    
+    print(f"Found video: '{current_title}' [{video_id}]")
+    print(f"Current description length: {len(snippet.get('description', ''))}")
+    current_description = snippet.get("description", "") or ""
+
+    # Decide final description
+    if append:
+        if new_description.strip().lower() in current_description.lower():
+            print("ℹ️ Description already contains that string, skipping")
+            return snippet
+        final_description = (snippet.get("description", "") + "\n\n" + new_description).strip()
+    else:
+        final_description = new_description
+
+    # YouTube requires you to send title + categoryId + description when updating snippet
+    body = {
+        "id": video_id,
+        "snippet": {
+            "title": current_title,
+            "description": final_description,
+            "categoryId": current_category,
+            # Keep existing tags, etc if you want
+            "tags": snippet.get("tags", []),
+            "defaultLanguage": snippet.get("defaultLanguage"),
+        }
+    }
+
+    try:
+        update_response = youtube.videos().update(
+            part="snippet",
+            body=body
+        ).execute()
+
+        print(f"✅ Description updated for {video_id}")
+        print(f"New length: {len(final_description)}")
+        return update_response["snippet"]
+
+    except HttpError as e:
+        print(f"❌ Failed to update video {video_id}: {e}")
+        if "quotaExceeded" in str(e):
+            print("You hit the YouTube API quota.")
+        return None
+
+def append_tag_if_not_exists(video_id: str, new_tag: str):
+    """
+    Appends a new tag to a YouTube video if it doesn't already exist.
+    Case-insensitive check.
+    """
+    if not new_tag or not new_tag.strip():
+        print("❌ Tag is empty")
+        return None
+
+    new_tag_clean = new_tag.strip()
+
+    # Load credentials
+    if not CREDENTIALS:
+        print("❌ Error: YouTube API credentials not configured.")
+        return None
+        
+    # Create YouTube API client
+    youtube = build('youtube', 'v3', credentials=CREDENTIALS)
+    
+    video = get_video_details(youtube, video_id)
+    if not video:
+        return None
+    
+    snippet = video["snippet"]
+    existing_tags = snippet.get("tags", []) or []
+    existing_lower = {t.lower().strip() for t in existing_tags}
+
+    if new_tag_clean.lower() in existing_lower:
+        print(f"ℹ️ Tag '{new_tag_clean}' already exists on {video_id}, skipping")
+        return existing_tags
+
+    updated_tags = existing_tags + [new_tag_clean]
+
+    total_chars = sum(len(t) for t in updated_tags)
+    if total_chars > 450:
+        print(f"⚠️ Tags approaching 500 char limit ({total_chars})")
+    if len(updated_tags) > 15:
+        print(f"⚠️ {len(updated_tags)} tags - YouTube ignores after ~15-20")
+
+    print(f"Current tags: {existing_tags}")
+    print(f"Adding: '{new_tag_clean}'")
+
+    body = {
+        "id": video_id,
+        "snippet": {
+            "title": snippet["title"],
+            "description": snippet.get("description", ""),
+            "categoryId": snippet.get("categoryId", "22"),
+            "tags": updated_tags,
+        }
+    }
+    try:
+        resp = youtube.videos().update(part="snippet", body=body).execute()
+        print(f"✅ Tag '{new_tag_clean}' added to {video_id}")
+        return resp["snippet"].get("tags", [])
+    except HttpError as e:
+        print(f"❌ Failed to add tag: {e}")
+        return None
+
+def main():
+    videos = DB.get_youtube_info_all()
+    for video in videos:
+        if video.yt_id:
+            update_video_description(video.yt_id, f"({video.id})", True)
+            append_tag_if_not_exists(video.yt_id, video.id)
+        if video.yt_short_id:
+            update_video_description(video.yt_short_id, f"({video.id})", True)
+            append_tag_if_not_exists(video.yt_short_id, video.id)
+
+
+# --- Example Usage ---
+if __name__ == "__main__":
+    main()
+    
+    # Example 1: Replace description
+    # update_video_description("YOUR_VIDEO_ID_HERE", "This is my new description!\nGame: Valorant")
+
+    # Example 2: Append to description
+    # update_video_description("YOUR_VIDEO_ID_HERE", "\n\nCheck out https://twitch.tv/teampgp", append=True)
+
+    # For integration with your existing code:
+    #vid = input("Enter YouTube Video ID: ").strip()
+    #if vid:
+    #    new_desc = input("Enter new description: ").strip()
+    #    update_video_description(vid, new_desc)

main.py

@@ -1,4 +1,7 @@
 #!/usr/bin/env python3
+import os
+import sys
+import fcntl
 import asyncio
 
 import package.twitch.video_info as video_info
@@ -8,7 +11,23 @@ import package.twitch.download_thumbnails as download_thumbnails
 import package.build_videos as build_videos
 import package.process_uploads as process_uploads
 
+def lock_file():
+    # Open a lock file
+    lock_file_path = "/tmp/python_script.lock"
+    fp = open(lock_file_path, 'w')
+
+    try:
+        # Try to grab an exclusive lock without blocking (LOCK_NB)
+        fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except IOError:
+        print("Another instance is already running. Exiting.")
+        sys.exit(1)
+
+    # Your script logic goes here
+    print("Running exclusively on Linux/macOS...")
+
 if __name__ == "__main__":
+    lock_file()
     upload = True
 
     if upload is False:
@@ -23,7 +42,7 @@ if __name__ == "__main__":
         #Build
         build_videos()
     else:
-        process_uploads()
+         process_uploads()
 
     print("====================================================")
     print(f"✅ Done.")

package/__init__.py

@@ -14,13 +14,14 @@ os.makedirs(CACHE_DIR, exist_ok=True)
 
 # 3. Expose Database Tools (Added)
 # Replace 'DatabaseClient' and 'connect_to_db' with your actual class/function names
-from .build_shorts import fit_to_9_16_letterbox
+from .video_editing.build_shorts import fit_to_9_16_letterbox
 from .build_videos import build_videos
 from .config import CHANNEL_NAME, TZ, DB
 from .hash_tags import get_top_nouns
 from .process_uploads import process_uploads
 from .transcribe_video import transcribe_to_srt
-from .utils import write_csv, run_command
+from .transcribe_client import transcribe_to_srt_remote
+from .utils import get_chapters, get_video_id_from_chat, write_csv, run_command
 
 # 4. Expose Subpackages
 from . import twitch
@@ -37,8 +38,11 @@ __all__ = [
     "get_top_nouns",
     "process_uploads",
     "transcribe_to_srt",
+    "transcribe_to_srt_remote",
     "write_csv",
     "run_command",
+    "get_chapters",
+    "get_video_id_from_chat"
     "twitch",
     "youtube"
 ]

package/build_videos.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 from datetime import datetime
 
-import package.build_shorts as yt_shorts
+import package.video_editing.build_shorts as yt_shorts
 import package.transcribe_video as transcribe_video
+import package.transcribe_client as transcribe_client
 import package.twitch.chat_vod as chat_vod
+import package.video_editing.highlights as highlights
 from package.config import DB as DB
 from package.config import TZ as TZ
 
@@ -22,19 +24,20 @@ def build_chat_video():
     print("====================================================")
 
 def build_transcribe():
-    unuploadeds = DB.get_unuploaded()
+    downloads = DB.get_downloaded()
 
-    for unuploaded in unuploadeds:
+    for download in downloads:
-        if unuploaded.clip_is:
+        if download.clip_is:
             target_dir = f"download/clips"
         else:
             target_dir = f"download/videos"
 
         print("====================================================")
-        print(f"🚀 Transcribe: {unuploaded.title}")
+        print(f"🚀 Transcribe: {download.title}")
         print("====================================================")
         
-        transcribe_video.transcribe_to_srt(f"{target_dir}/{unuploaded.id}/{unuploaded.id}.mp4")
+        #transcribe_video.transcribe_to_srt(f"{target_dir}/{unuploaded.id}/{unuploaded.id}.mp4", True)
+        transcribe_client.transcribe_to_srt_remote(f"{target_dir}/{download.id}/{download.id}.mp4")
 
     print("====================================================")
     print(f"✅ Processing: Transcribing Done.")
@@ -65,6 +68,32 @@ def build_shorts():
     print(f"✅ Processing: Clip to Youtube Short Done.")
     print("====================================================")
 
+def build_highlights():
+    lights = DB.get_downloaded()
+
+    top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
+    count = 1
+
+    for light in lights:
+        target_dir = f"download/clips/{light.id}/{light.id}.mp4"
+
+        print("====================================================")
+        print(f"🚀 Processing: Highlights {light.title}")
+        print("====================================================")
+        print(f"{count}/{len(lights)}")
+
+        created_at = datetime.strptime(light.created_at, "%Y-%m-%d %H:%M:%S%z")
+        created_at = created_at.astimezone(TZ)    
+
+        #ToDo
+        highlights.highlights(target_dir, [], 5, 8)
+        #yt_shorts.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {light.creator_name},\n on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z")}")
+        count += 1
+
+    print("====================================================")
+    print(f"✅ Processing: Clip to Youtube Short Done.")
+    print("====================================================")
+
 def build_videos():
     build_shorts()
     build_transcribe()

package/config.py

@@ -2,9 +2,9 @@
 from zoneinfo import ZoneInfo
 
 #import package.database as database
-from package.database.database import Database
+from package.database.database import Repository
 
 CHANNEL_NAME = "teampgp"
 TZ = ZoneInfo("America/Toronto")
 
-DB = Database()
+DB = Repository()

package/database/__init__.py

@@ -1,12 +1,15 @@
 # package/database/__init__.py
 
 # Expose components from individual modules using relative imports
-from .database import TwitchVideo, Data, YoutubeInfo, Database
+from .database import TwitchVideo, Data, YoutubeInfo, TiktokInfo, Highlight, Chapter, Repository
 
 # Explicitly define what is exposed when importing the subpackage
 __all__ = [
     "TwitchVideo",
     "Data",
     "YoutubeInfo",
-    "Database",
+    "TiktokInfo",
+    "Highlight",
+    "Chapter",
+    "Repository",
 ]

package/database/database.py

@@ -1,4 +1,4 @@
-from sqlalchemy import create_engine, Column, String, Integer, Boolean, ForeignKey, select
+from sqlalchemy import create_engine, Column, String, Integer, Boolean, Float, ForeignKey, select
 from sqlalchemy.orm import declarative_base, relationship, sessionmaker
 from typing import Any
 
@@ -25,9 +25,18 @@ class TwitchVideo(Base):
     uploaded_yt = Column(Boolean, nullable=False, default=False)
     uploaded_yt_chats = Column(Boolean, nullable=False, default=False)
     uploaded_yt_shorts = Column(Boolean, nullable=False, default=False)
+    uploaded_tiktok = Column(Boolean, nullable=False, default=False)
 
-    # Relationship back to YoutubeInfo (uselist=False enforces a 1-to-1 structure)
+    # This now actually points to twitch_videos.id
-    youtube_info = relationship("YoutubeInfo", back_populates="twitch_video", uselist=False)
+    related_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=True)
+
+    # Relationship back to YoutubeInfo (uselist=True enforces a 1-to-many structure)
+    youtube_info = relationship("YoutubeInfo", back_populates="twitch_video", uselist=True)
+    tiktok_info = relationship("TiktokInfo", back_populates="twitch_video", uselist=True)
+    chapters = relationship("Chapter", back_populates="twitch_video", cascade="all, delete-orphan")
+
+    # THIS WAS MISSING - this is what Highlight is looking for
+    highlights = relationship("Highlight", back_populates="twitch_video", cascade="all, delete-orphan")
 
 # 3. Define the data table
 class Data(Base):
@@ -39,11 +48,13 @@ class Data(Base):
     date_shorts = Column(String, nullable=False)
     date_videos = Column(String, nullable=False)
     date_videos_chats = Column(String, nullable=False)
+    date_tiktok = Column(String, nullable=False)
 
 class YoutubeInfo(Base):
     __tablename__ = "youtube_info"
 
-    id = Column(String, ForeignKey("twitch_videos.id"), primary_key=True)
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
     yt_id = Column(String, nullable=True)
     yt_date = Column(String, nullable=True)
     yt_short_id = Column(String, nullable=True)
@@ -56,8 +67,48 @@ class YoutubeInfo(Base):
     # Relationship linking back to the twitch video record
     twitch_video = relationship("TwitchVideo", back_populates="youtube_info")
 
+class TiktokInfo(Base):
+    __tablename__ = "tiktok_info"
 
-class Database:
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
+    tiktok_id = Column(String, nullable=True)
+    tiktok_date = Column(String, nullable=True)
+
+    # Relationship linking back to the twitch video record
+    twitch_video = relationship("TwitchVideo", back_populates="tiktok_info")
+
+class Highlight(Base):
+    __tablename__ = "highlights"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
+    start_milliseconds = Column(Float, nullable=False)
+    length_milliseconds = Column(Float, nullable=False)
+    start_timestamp = Column(String, nullable=False)
+    end_timestamp = Column(String, nullable=False)
+    description = Column(String, nullable=False)
+    score = Column(Float, nullable=False)
+
+    # Relationship linking back to the twitch video record
+    twitch_video = relationship("TwitchVideo", back_populates="highlights")
+
+class Chapter(Base):
+    __tablename__ = "chapters"
+
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
+    start_milliseconds = Column(Integer, nullable=False)
+    length_milliseconds = Column(Integer, nullable=False)
+    description = Column(String, nullable=True)
+    type = Column(String, nullable=True)
+    game_display_name = Column(String, nullable=True)
+    game_id = Column(String, nullable=True)
+
+    # Relationship linking back to the twitch video record
+    twitch_video = relationship("TwitchVideo", back_populates="chapters")    
+
+class Repository:
     def __init__(self, db_path: str = "database.db"):
         # 4. Connect to the SQLite file and generate the tables
         # This will create 'database.db' in the current working directory if it doesn't exist.
@@ -68,6 +119,15 @@ class Database:
         Session = sessionmaker(bind=engine)
         self.session =  Session()
 
+    def set_related_video(self, record_id: str, related_video_id: str):
+        record = self.session.get(TwitchVideo, record_id)
+
+        if record:
+            setattr(record, "related_video", related_video_id)
+            self.session.commit()
+        else:
+            print(f"❌ Record {record_id} not found.")
+
     def __mark_as(self, record_id: str, field_name: str, value: bool = True):
         record = self.session.get(TwitchVideo, record_id)
         
@@ -93,6 +153,10 @@ class Database:
         """Flags a specific row record to downloaded (1)."""
         self.__mark_as(record_id, "downloaded")
 
+    def mark_as_uploaded_tiktok(self, record_id: str):
+        """Flags a specific row record to uploaded_tiktok (1)."""
+        self.__mark_as(record_id, "uploaded_tiktok")
+
     def unmark_as_download(self, record_id: str):
         """Flags a specific row record to downloaded (0)."""
         self.__mark_as(record_id, "downloaded", False)
@@ -145,6 +209,10 @@ class Database:
         stmt = select(TwitchVideo).where(TwitchVideo.downloaded == False)
         return self.session.scalars(stmt).all()
 
+    def get_unuploaded_tiktok(self) -> list[TwitchVideo]:
+        """Retrieve all rows that were download but not uploaded_tiktok"""
+        return self.__get_unuploaded("uploaded_tiktok", (TwitchVideo.clip_is == True))
+
     def get_downloaded(self) -> list[TwitchVideo]:
         stmt = select(TwitchVideo).where(TwitchVideo.downloaded == True)
         return self.session.scalars(stmt).all()
@@ -174,37 +242,78 @@ class Database:
     def update_date_chats(self, date: str):
         self.__update_date("date_videos_chat", date)
 
-    def get_data(self) -> list[Data]:
+    def get_data(self) -> Data:
         stmt = select(Data)
-        return self.session.scalars(stmt).all()
+        return self.session.scalars(stmt).first()
 
-    def get_youtube_info(self, record_id) -> YoutubeInfo | None:
+    def get_youtube_info_for_video(self, video_id: str) -> YoutubeInfo | None:
-        return self.session.get(YoutubeInfo, record_id)
+        """Retrieve all YouTubeInfo records associated with a specific TwitchVideo ID."""
+        stmt = select(YoutubeInfo).where(YoutubeInfo.twitch_video_id == video_id)
+        return self.session.scalars(stmt).first()
 
-    def insert(self, data: TwitchVideo | Data | YoutubeInfo):
+    def get_youtube_info_all(self) -> list[YoutubeInfo]:
-        # SQLAlchemy 2.0 context manager handles commit/rollback automatically
+        return self.session.scalars(select(YoutubeInfo)).all()
+
+    def insert(self, data: TwitchVideo | Data | YoutubeInfo | Chapter | list[YoutubeInfo] | list[Chapter] | TiktokInfo | Highlight):
+        """
+        Handles insertion/updating for any of the defined models,
+        using session merge/add for SQLAlchemy 2.0 best practices.
+        Accepts single objects or lists of objects for collection handling.
+        """
         try:
             if isinstance(data, TwitchVideo):
-                # 1. Query for the existing row using SQLAlchemy 2.0 scalar execution
+                # 1. Check if the record exists
                 stmt = select(TwitchVideo).where(TwitchVideo.id == data.id)
                 existing_record = self.session.scalar(stmt)
                 
                 if existing_record:
+                    # 2. Update relevant fields on the managed instance
                     if existing_record.view_count != data.view_count or existing_record.duration != data.duration:
-                        # 2. Update the columns on the managed instance
                         existing_record.view_count = data.view_count
                         existing_record.duration = data.duration
-                        print(f"🔄 Record {data.id} updated successfully!")
+                        print(f"🔄 TwitchVideo record {data.id} updated successfully!")
+                    # Note: Add more field updates here as needed
                 else:
-                    # 3. Add new instance if it doesn't exist
+                    # 3. Add new instance
                     self.session.add(data)
-                    print("➕ New TwitchVideo record inserted successfully!")
+                    print(f"➕ New TwitchVideo record with ID {data.id} inserted successfully!")
-            else:                   
+            elif isinstance(data, Data):
                 self.session.merge(data)
-                print(f"➕ {type(data).__name__} record inserted successfully!")
+                print(f"➕ Data record merged/inserted successfully!")
-            # 4. Flush changes and commit the transaction
+            elif isinstance(data, YoutubeInfo):
-            self.session.commit()
+                # Handle single YoutubeInfo object
+                self.session.merge(data)
+                print(f"➕ YoutubeInfo record merged/inserted successfully!")
+            elif isinstance(data, Chapter):
+                # Handle single Chapter object
+                self.session.merge(data)
+                print(f"➕ Chapter record merged/inserted successfully!")
+            elif isinstance(data, list):
+                # Handle lists of records (YoutubeInfo or Chapter)
+                record_type = type(data[0])
+                if record_type in (YoutubeInfo, Chapter):
+                    for item in data:
+                        self.session.merge(item)
+                    print(f"➕ Successfully processed {len(data)} records of type {record_type.__name__}.")
+                else:
+                    raise TypeError(f"Unsupported list content type: {record_type.__name__}")
+            elif isinstance(data, TiktokInfo):
+                # Handle single TiktokInfo object
+                self.session.merge(data)
+                print(f"➕ TiktokInfo record merged/inserted successfully!")
+            elif isinstance(data, Highlight):
+                # Handle single Highlight object
+                self.session.merge(data)
+                print(f"➕ Highlight record merged/inserted successfully!")
+            else:
+                raise TypeError(f"Unsupported data type for insertion: {type(data).__name__}")
+            
+            # 4. Commit the transaction
+            self.session.flush() # Flush first to ensure IDs are populated
             
         except Exception as e:
             self.session.rollback()
-            print(f"❌ An error occurred, transaction rolled back: {e}")
+            print(f"❌ A critical error occurred during record insertion/update: {e}")
+            raise
+
+        self.session.commit()

package/process_uploads.py

@@ -11,16 +11,29 @@ from package.config import DB as DB
 from package.hash_tags import get_top_nouns as get_top_nouns
 from package.youtube.uploader import upload_video as upload_video
 from package.youtube.uploader import CategoryId as CategoryId
+from package.tiktok.uploader import upload_to_tiktok as upload_to_tiktok
 
-from package.database import TwitchVideo, YoutubeInfo, Data
+from package.database import TwitchVideo, YoutubeInfo, TiktokInfo, Data, TiktokInfo
 
-def process_videos(video_type: bool = True):
+def process_videos(video_type: bool = True, limit: int = 22):
+    count = 0 # count how many videos we have upload and stop at var: limit
     videos = DB.get_unuploaded_chats() if video_type else DB.get_unuploaded_shorts()
-    file_type = "download/clips" if video_type else "download/video"
+    file_type = "download/video" if video_type else "download/clips"
-    meta_data: Data = DB.get_data()[0]
+    meta_data: Data = DB.get_data()
+
+    #current_time = datetime.now(TZ)
+    #current_time = current_time.replace(hour=0, minute=0, second=0, microsecond=0)
+
+    #meta_data.date_videos = current_time
+    #meta_data.date_shorts = current_time
+    #meta_data.date_tiktok = current_time
+    #meta_data.date_videos_chats = current_time
+
+    #DB.insert(meta_data) # ensure we have a record in the database
 
     current_time = datetime.now(TZ)
-    release_time = datetime.strptime(meta_data.date_shorts, "%Y-%m-%d %H:%M:%S%z")
+    next_time = meta_data.date_videos if video_type else meta_data.date_shorts
+    release_time = datetime.strptime(next_time, "%Y-%m-%d %H:%M:%S%z")
 
     if current_time > release_time:
         release_time = release_time.replace(hour=12, minute=0, second=0, microsecond=0)
@@ -29,6 +42,10 @@ def process_videos(video_type: bool = True):
             release_time += timedelta(days=1)
 
     for video in videos:
+        #We reached limit stop uploading videos
+        if count >= limit:
+            break
+
         base_path = Path(f"{file_type}/{video.id}/{video.id}")
         original_path = base_path.with_suffix(".mp4")
 
@@ -56,29 +73,32 @@ def process_videos(video_type: bool = True):
 
         if video_type:
             origin_clipped = "Originally Streamed on"
-            hashs = "#Twitch"
         else:
             origin_clipped = f"Clipped By: {video.creator_name}, on"
-            hashs = "#Shorts #Clips #Twitch"
 
-        description = f"Game: {video.game_name}, {origin_clipped} {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, {hashs} Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp"
         privatcyStatus = 'private'
         tags = ['gaming', 'TeamPGP', f'{video.game_name}', f'{video.creator_name}', 'Level1Techs', 'twitch']
         if not video_type:
             tags.extend(['shorts', 'twitch_clips', 'clips'])
+        tags.extend([video.id])       
 
         if srt_path.exists():
             try:
-                tags.extend(get_top_nouns(str(srt_path)))
+                tags.extend(get_top_nouns(str(srt_path), (15 - len(tags))))
             except Exception as e:
                 print(f"⚠️ Could not get nouns from {srt_path}: {e}")
 
-        youtube_info = DB.get_youtube_info(video.id) or YoutubeInfo(id=video.id)
+        hashs = " ".join([f"#{t}" for t in tags])
+
+        description = f"Game: {video.game_name}, {origin_clipped} {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, {hashs} Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp"
+
+        youtube_info = DB.get_youtube_info_for_video(video.id) or YoutubeInfo(twitch_video_id=video.id)
 
         if youtube_info.yt_id:
             main_video_id = youtube_info.yt_id
         else:
-            main_video_id = upload_video(str(original_path), video.title, category_id, description, 'unlisted', tags)
+            main_video_id = upload_video(str(original_path), video.title, category_id, f"{description} \n\n({video.id})", 'unlisted', tags)
+            count += 1
             if not main_video_id:
                 print(f"❌ Error: main video upload failed for {original_path}")
                 continue
@@ -86,7 +106,7 @@ def process_videos(video_type: bool = True):
             youtube_info.yt_date = datetime.now(TZ)
 
         # Upload secondary
-        secondary_desc = f"{description} Original Full size @ https://youtu.be/{main_video_id}"
+        secondary_desc = f"{description} Original Full size @ https://youtu.be/{main_video_id} \n\n({video.id})"
         secondary_video_id = upload_video(str(file_path), video.title, category_id, secondary_desc, 'private', tags, release_time)
 
         if not secondary_video_id:
@@ -94,6 +114,8 @@ def process_videos(video_type: bool = True):
             DB.insert(youtube_info) # save main id at least
             continue
 
+        count += 1
+
         # Success - write CSV and DB
         data_csv = [main_video_id, secondary_video_id, release_time.isoformat()]
         utils.write_csv([data_csv], str(file_csv))
@@ -108,7 +130,7 @@ def process_videos(video_type: bool = True):
             youtube_info.yt_short_date = datetime.now(TZ)
 
         DB.insert(youtube_info)
-        release_time += timedelta(days=1)
+        release_time += timedelta(days=0.5)
 
     # Update global release pointer once at end
     if video_type:
@@ -116,6 +138,49 @@ def process_videos(video_type: bool = True):
     else:
         DB.update_date_shorts(release_time)
 
+def process_tiktoks():
+    tiktoks = DB.get_unuploaded_tiktok()
+    data = DB.get_data()
+    meta_data: Data = data[0]
+
+    release_time = meta_data.date_tiktok
+    
+    current_time = datetime.now(TZ)
+    release_time = datetime.strptime(release_time, "%Y-%m-%d %H:%M:%S%z")
+
+    count = 0
+
+    if current_time > release_time:
+        release_time = current_time
+        release_time = release_time.replace(hour=12, minute=0, second=0, microsecond=0)
+
+    for tiktok in tiktoks:
+        if count >= 10:
+            break
+        created_at = datetime.strptime(tiktok.created_at, "%Y-%m-%d %H:%M:%S%z").astimezone(TZ)
+        tags = ["Twitch", "TeamPGP", f"{tiktok.game_name}", f"{tiktok.creator_name}", "Level1Techs", "clips"]
+        origin_clipped = f"Clipped By: {tiktok.creator_name}, on"
+        description = f"Game: {tiktok.game_name}, {origin_clipped} {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp"
+        original_path = f"download/clips/{tiktok.id}/{tiktok.id}.mp4"
+        file_path = f"download/clips/{tiktok.id}/{tiktok.id}_gaussian_9_16.mp4"
+        file_csv = f"download/clips/{tiktok.id}/{tiktok.id}_{meta_data.name}.csv"
+
+        srt_path = f"download/clips/{tiktok.id}/{tiktok.id}.srt"
+
+        try:
+            tags.extend(get_top_nouns(str(srt_path), (15 - len(tags))))
+        except Exception as e:
+            print(f"⚠️ Could not get nouns from {srt_path}: {e}")
+                
+        if not os.path.exists(file_path):
+            file_path = f"download/clips/{tiktok.id}/{tiktok.id}_black_9_16.mp4"
+            if not os.path.exists(file_path):
+                continue
+        if tiktok_video_id := upload_to_tiktok(file_path, description, tags, "teampgpunofficialarchive"):
+            tiktok_info = TiktokInfo(twitch_video_id=tiktok.id, tiktok_id=tiktok_video_id, tiktok_date=datetime.now(TZ))
+            DB.insert(tiktok_info)
+        count += 1
+
 def process_shorts():
     process_videos(False)
     return
@@ -264,6 +329,7 @@ def process_chats():
 
 
 def process_uploads():
+    #process_tiktoks()
     process_shorts()
     #process_chats()
 

package/tiktok/__init__.py

@@ -0,0 +1,9 @@
+# package/tiktok/__init__.py
+
+# Expose components from individual modules using relative imports
+from .uploader import upload_to_tiktok
+
+# Explicitly define what is exposed when importing the subpackage
+__all__ = [
+    "upload_to_tiktok"
+]

package/tiktok/uploader.py

@@ -0,0 +1,74 @@
+import json
+import requests
+from pathlib import Path
+
+SECRETS_FILE = Path(__file__).parent.parent / "tiktok_secrets.json"
+# if your file is at /home/richard/python_scripts/tiktok_secrets.json
+# use: Path("/home/richard/python_scripts/tiktok_secrets.json")
+
+def load_secrets():
+    with open(SECRETS_FILE) as f:
+        secrets = json.load(f)
+    return secrets["client_key"], secrets["client_secret"], secrets["access_token"], secrets.get("refresh_token", "")
+
+API_BASE = "https://open.tiktokapis.com/v2"
+
+def upload_to_tiktok_api(video_path: str, title: str, privacy="PUBLIC_TO_EVERYONE"):
+    client_key, client_secret, access_token, refresh_token = load_secrets()
+    
+    video_path = Path(video_path)
+    file_size = video_path.stat().st_size
+    
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "Content-Type": "application/json; charset=UTF-8"
+    }
+
+    # 1. INIT
+    init_payload = {
+        "source_info": {
+            "source": "FILE_UPLOAD",
+            "video_size": file_size,
+            "chunk_size": file_size,
+            "total_chunk_count": 1
+        }
+    }
+    r = requests.post(f"{API_BASE}/post/publish/video/init/", headers=headers, json=init_payload)
+    r.raise_for_status()
+    data = r.json()
+    if data.get("error", {}).get("code") != "ok":
+        raise Exception(f"Init failed: {data}")
+
+    publish_id = data["data"]["publish_id"]
+    upload_url = data["data"]["upload_url"]
+
+    # 2. UPLOAD
+    with open(video_path, "rb") as f:
+        put_headers = {
+            "Content-Range": f"bytes 0-{file_size-1}/{file_size}",
+            "Content-Type": "video/mp4"
+        }
+        requests.put(upload_url, headers=put_headers, data=f).raise_for_status()
+
+    # 3. PUBLISH
+    publish_payload = {
+        "publish_id": publish_id,
+        "post_info": {
+            "title": title[:2200],
+            "privacy_level": privacy,
+            "disable_duet": False,
+            "disable_comment": False,
+            "disable_stitch": False,
+            "video_cover_timestamp_ms": 1000
+        }
+    }
+    pr = requests.post(f"{API_BASE}/post/publish/", headers=headers, json=publish_payload)
+    pr.raise_for_status()
+    return pr.json()
+
+if __name__ == "__main__":
+    result = upload_to_tiktok_api(
+        "/home/richard/python_scripts/download/clips/ProductiveConfidentWombatThunBeast/ProductiveConfidentWombatThunBeast_gaussian_9_16.mp4",
+        "Testing upload #Twitch #TeamPGP #Stellaris"
+    )
+    print(result)

package/transcribe_client.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+
+#!/usr/bin/env python3
+import threading
+# --- HOTFIX for Python 3.13.5 _DeleteDummyThreadOnDel spam ---
+# The CPython fix in 3.13.4 didn't fully kill it for debugpy/asyncio cases
+if hasattr(threading, '_DeleteDummyThreadOnDel'):
+    threading._DeleteDummyThreadOnDel.__del__ = lambda self: None
+
+import os
+import shutil
+# ... rest of your imports
+import os
+import shutil
+import subprocess
+import json
+from pathlib import Path
+import wave
+import requests
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
+
+CHUNK_DURATION_SEC = 600 # target chunk size
+OVERLAP_SEC = 8 # overlap to prevent cut words
+VAD_SEARCH_WINDOW = 15 # look for silence within 15s of target
+WORKER_URL = "http://192.168.5.55:8000/transcribe"
+MAX_WORKERS = 4 # perfect for 4070 Ti Super 16GB
+
+def get_duration(video_path: str) -> float:
+    cmd = ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "json", video_path]
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    return float(json.loads(result.stdout)["format"]["duration"])
+
+def find_silence_near(video_path: str, target_time: float) -> float:
+    """Find nearest silence within VAD_SEARCH_WINDOW of target_time"""
+    start_search = max(0, target_time - VAD_SEARCH_WINDOW/2)
+    cmd = [
+        "ffmpeg", "-ss", str(start_search), "-t", str(VAD_SEARCH_WINDOW),
+        "-i", video_path, "-af", "silencedetect=noise=-30dB:d=0.5",
+        "-f", "null", "-"
+    ]
+    result = subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+    # Parse last silence end before target
+    best = target_time
+    for line in result.stderr.splitlines():
+        if "silence_end:" in line:
+            try:
+                t = float(line.split("silence_end:")[1].split("|")[0].strip())
+                abs_t = start_search + t
+                if abs(abs_t - target_time) < abs(best - target_time):
+                    best = abs_t
+            except: pass
+    return best
+
+def extract_audio_and_chunk_smart(video_path: str, output_dir: Path):
+    print(f"🚀 Smart chunking: target {CHUNK_DURATION_SEC}s + {OVERLAP_SEC}s overlap")
+    output_dir.mkdir(parents=True, exist_ok=True)
+    total_dur = get_duration(video_path)
+    print(f"📏 Total duration: {total_dur/60:.1f} min")
+
+    chunks = []
+    start = 0.0
+    idx = 0
+
+    while start < total_dur:
+        # Find smart cut point
+        target_end = start + CHUNK_DURATION_SEC
+        if target_end >= total_dur:
+            end = total_dur
+        else:
+            end = find_silence_near(video_path, target_end)
+            # clamp
+            end = max(start + CHUNK_DURATION_SEC * 0.7, min(end, start + CHUNK_DURATION_SEC * 1.3))
+
+        chunk_path = output_dir / f"chunk_{idx:03d}.wav"
+        # Extract with overlap for next chunk
+        extract_end = end + (OVERLAP_SEC if end < total_dur else 0)
+
+        cmd = [
+            "ffmpeg", "-y", "-ss", str(start), "-to", str(extract_end),
+            "-i", video_path, "-vn", "-ac", "1", "-ar", "16000", "-acodec", "pcm_s16le",
+            str(chunk_path)
+        ]
+        subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=True)
+
+        chunks.append((chunk_path, start, end)) # path, true_start, true_end (without overlap)
+        print(f" Chunk {idx}: {start:.1f}s -> {end:.1f}s (file to {extract_end:.1f}s)")
+
+        if end >= total_dur:
+            break
+        start = end - OVERLAP_SEC/2 # start next chunk with half overlap
+        idx += 1
+
+    return chunks
+
+def format_srt_time(seconds: float):
+    h = int(seconds // 3600)
+    m = int((seconds % 3600) // 60)
+    s = int(seconds % 60)
+    ms = int((seconds % 1) * 1000)
+    return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}"
+
+def send_chunk_for_transcription(args):
+    chunk_path, true_start, true_end, chunk_idx = args
+    try:
+        with open(chunk_path, 'rb') as f:
+            files = {'file': (chunk_path.name, f, 'audio/wav')}
+            resp = requests.post(WORKER_URL, files=files, timeout=1200)
+            resp.raise_for_status()
+            data = resp.json()
+            # Filter to only keep segments within true window (dedupe overlap)
+            filtered = []
+            for seg in data['segments']:
+                abs_start = true_start + seg['start']
+                # keep if it starts before the true_end (allow small spill)
+                if abs_start < true_end + OVERLAP_SEC/2:
+                    # if it's in overlap zone of NEXT chunk, only keep first half
+                    if abs_start < true_end - OVERLAP_SEC/2 or chunk_idx == 0:
+                        filtered.append(seg)
+                    elif seg['start'] < (true_end - true_start) - OVERLAP_SEC/2 + 1.0:
+                        filtered.append(seg)
+            return chunk_idx, true_start, filtered, None
+    except Exception as e:
+        return chunk_idx, true_start, None, str(e)
+
+def transcribe_to_srt_remote(video_path: str, force: bool = False):
+    video_path_obj = Path(video_path)
+    srt_path = video_path_obj.with_suffix(".srt")
+    temp_dir = video_path_obj.parent / f"temp_chunks_{video_path_obj.stem}"
+
+    if srt_path.exists() and not force:
+        print(f"❌ SRT already exists: {srt_path}")
+        return
+    if srt_path.exists():
+        srt_path.unlink()
+
+    audio_chunks = extract_audio_and_chunk_smart(str(video_path_obj), temp_dir)
+    jobs = [(path, s, e, idx) for idx, (path, s, e) in enumerate(audio_chunks)]
+
+    results = []
+    executor = ThreadPoolExecutor(max_workers=MAX_WORKERS)
+    try:
+        futures = {executor.submit(send_chunk_for_transcription, job): job for job in jobs}
+        for future in as_completed(futures):
+            idx, offset, segments, error = future.result()
+            if error:
+                print(f"❌ Chunk {idx} failed: {error}")
+            else:
+                print(f"✅ Chunk {idx} done: {len(segments)} segs (offset {offset:.1f}s)")
+                results.append((idx, offset, segments))
+    finally:
+        executor.shutdown(wait=True)
+
+    results.sort(key=lambda x: x[0])
+
+    print("📝 Writing final SRT with smart overlap dedupe...")
+    global_idx = 1
+    with open(srt_path, "w", encoding="utf-8") as srt_file:
+        for _, time_offset, segments in results:
+            for seg in segments:
+                actual_start = seg['start'] + time_offset
+                actual_end = seg['end'] + time_offset
+                srt_file.write(f"{global_idx}\n{format_srt_time(actual_start)} --> {format_srt_time(actual_end)}\n{seg['text'].strip()}\n\n")
+                global_idx += 1
+
+    print(f"\n✅ SRT saved: {srt_path} - {global_idx-1} lines")
+    if temp_dir.exists():
+        shutil.rmtree(temp_dir)
+
+if __name__ == "__main__":
+    target_video = "download/videos/2813112936/2813112936.mp4"
+    transcribe_to_srt_remote(target_video, force=True)

package/transcribe_video.py

@@ -3,117 +3,117 @@ import os
 import shutil
 import subprocess
 from pathlib import Path
+import wave
 from faster_whisper import WhisperModel
-from faster_whisper.utils import format_timestamp
 
-# Prevent OpenMP thread conflicts from crashing the script
 os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
 
-# 5 minutes per chunk (300 seconds) keeps RAM usage low and stable
 CHUNK_DURATION_SEC = 300 
 
-def extract_audio_and_chunk(video_path: str, output_dir: Path) -> list:
+def extract_audio_and_chunk(video_path: str, output_dir: Path) -> list[tuple[Path, float]]:
-    """Extracts and splits audio into 5-minute chunks using a single FFmpeg pass."""
+    """Returns list of (chunk_path, start_offset_seconds)"""
     print("🚀 Extracting and chunking audio with FFmpeg...")
     output_dir.mkdir(parents=True, exist_ok=True)
     
-    # Segment format output: chunk_000.wav, chunk_001.wav, etc.
     chunk_pattern = str(output_dir / "chunk_%03d.wav")
     
     command = [
         "ffmpeg", "-y", "-i", video_path,
         "-vn", "-ac", "1", "-ar", "16000",
-        "-acodec", "pcm_s16le", "-sn", "-map_chapters", "-1",
+        "-acodec", "pcm_s16le",
         "-f", "segment", "-segment_time", str(CHUNK_DURATION_SEC),
+        "-reset_timestamps", "0", # keep each chunk starting at 0
         chunk_pattern
     ]
     
     result = subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
     if result.returncode != 0:
-        print(f"❌ FFmpeg Error Output:\n{result.stderr}")
+        print(f"❌ FFmpeg Error:\n{result.stderr}")
-        raise RuntimeError("FFmpeg extraction and chunking failed.")
+        raise RuntimeError("FFmpeg failed")
 
-    # Return sorted list of generated chunk files
+    chunks = sorted(output_dir.glob("chunk_*.wav"))
-    return sorted(list(output_dir.glob("chunk_*.wav")))
     
+    # FIX: Calculate real offset from actual file durations, not idx*300
+    offsets = []
+    current_offset = 0.0
+    for chunk_path in chunks:
+        offsets.append((chunk_path, current_offset))
+        with wave.open(str(chunk_path), 'rb') as wf:
+            duration = wf.getnframes() / wf.getframerate()
+            current_offset += duration
+            
+    return offsets
+
+def format_srt_time(seconds: float):
+    h = int(seconds // 3600)
+    m = int((seconds % 3600) // 60)
+    s = int(seconds % 60)
+    ms = int((seconds % 1) * 1000)
+    return f"{h:02d}:{m:02d}:{s:02d},{ms:03d}"
 
 def transcribe_to_srt(video_path: str, force: bool = False):
     if not hasattr(transcribe_to_srt, "model"):
         print("🔧 Initializing C++ Engine...")
         transcribe_to_srt.model = WhisperModel(
-            "base", 
+            "distil-small.en",
             device="cpu",
             compute_type="int8",
-            cpu_threads=0, # Let CTranslate2 auto-detect safe core counts
+            cpu_threads=4, # 4 = physical cores on E3-1280 v5 = fastest
             num_workers=1
         )
-        print("✅ C++ Model loaded successfully.")
+        print("✅ Model loaded.")
-
 
     video_path_obj = Path(video_path)
     srt_path = video_path_obj.with_suffix(".srt")
     temp_dir = video_path_obj.parent / f"temp_chunks_{video_path_obj.stem}"
 
+    if srt_path.exists() and not force:
+        print(f"❌ SRT already exists: {srt_path}")
+        return
     if srt_path.exists():
-        if not force:
+        srt_path.unlink()
-            print(f"❌ Error: Transcription file already exists: {srt_path}")
-            return
-        else:
-            srt_path.unlink()
 
     try:
-        # Step 1: Split audio into bite-sized pieces
         audio_chunks = extract_audio_and_chunk(str(video_path_obj), temp_dir)
-        if not audio_chunks:
+        print(f"📦 {len(audio_chunks)} chunks.")
-            print("❌ Error: No audio chunks were generated.")
+        print("🎙 Transcribing...")
-            return
-
-        print(f"📦 Successfully split audio into {len(audio_chunks)} chunks.")
-        print("🎙️ Starting safe chunk-by-chunk transcription...")
-
-        global_segment_index = 1
 
+        global_idx = 1
         with open(srt_path, "w", encoding="utf-8") as srt_file:
-            for chunk_idx, chunk_path in enumerate(audio_chunks):
+            for chunk_idx, (chunk_path, time_offset) in enumerate(audio_chunks):
-                # Calculate the time offset for the current chunk
+                print(f"\n⏳ Chunk {chunk_idx+1}/{len(audio_chunks)} offset={time_offset:.1f}s")
-                time_offset = chunk_idx * CHUNK_DURATION_SEC
-                print(f"\n⏳ Processing chunk {chunk_idx + 1}/{len(audio_chunks)} ({chunk_path.name})...")
                 
-                segments_generator, info = transcribe_to_srt.model.transcribe(
+                segments, info = transcribe_to_srt.model.transcribe(
                     str(chunk_path), 
+                    language="en",
                     beam_size=1,
                     vad_filter=True,
-                    temperature=0.0
+                    vad_parameters=dict(min_silence_duration_ms=500),
+                    condition_on_previous_text=False # important for chunked mode
                 )
                 
-                # Consume chunk generator and shift timestamps instantly
+                for seg in segments:
-                for segment in segments_generator:
+                    actual_start = seg.start + time_offset
-                    # Shift timestamps relative to the original video timeline
+                    actual_end = seg.end + time_offset
-                    actual_start = segment.start + time_offset
-                    actual_end = segment.end + time_offset
                     
-                    start_str = format_timestamp(actual_start, always_include_hours=True)
+                    # Use comma for SRT, not dot
-                    end_str = format_timestamp(actual_end, always_include_hours=True)
+                    srt_file.write(f"{global_idx}\n{format_srt_time(actual_start)} --> {format_srt_time(actual_end)}\n{seg.text.strip()}\n\n")
+                    global_idx += 1
                     
-                    srt_file.write(f"{global_segment_index}\n{start_str} --> {end_str}\n{segment.text.strip()}\n\n")
-                    global_segment_index += 1
-                    
-                # Free up space as we go by deleting the processed chunk
                 chunk_path.unlink()
 
-        print(f"\n✅ All chunks combined! SRT subtitle file saved in: {srt_path}")
+        print(f"\n✅ SRT saved: {srt_path}")
 
     except Exception as e:
-        print(f"\n❌ Execution Error: {e}")
+        print(f"\n❌ Error: {e}")
+        import traceback; traceback.print_exc()
     finally:
-        # Clean up the temporary folder entirely
         if temp_dir.exists():
             shutil.rmtree(temp_dir)
 
-
 if __name__ == "__main__":
     target_video = "download/videos/2813112936/2813112936.mp4"
     if not os.path.exists(target_video):
-        print(f"❌ System Error: Target video file does not exist at path: {target_video}")
+        print(f"❌ Not found: {target_video}")
     else:
         transcribe_to_srt(target_video, force=True)

package/twitch/download_thumbnails.py

@@ -93,7 +93,7 @@ async def download_thumbnails():
     # --- OPTION B: Download Past Broadcast VOD Thumbnail ---
     # Extract the ID sequence from your target video link
     # target_vod = "2145678901"
-    db = database.Database()
+    db = database.Repository()
     vods = db.get_vods()
     for vod in vods:
         (

package/twitch/download_videos.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 import subprocess
+from pathlib import Path
 from datetime import datetime, timedelta, timezone
 
 import package.utils as utils
 from package.config import DB as DB
-from package.database import TwitchVideo
+from package.config import TZ as TZ
+from package.database import TwitchVideo, Chapter
 
 def download_videos():
     """Find all undownloaded videos and clips, and download them safely."""
@@ -12,59 +14,72 @@ def download_videos():
 
     print("Starting downloads...")
     for row in undownloaded:
-        # 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM)
+        try:
-        #datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
+            # 2026-08-16T23:52:10Z -> replace Z with +00:00
-        data_timestamp = datetime.fromisoformat(row.created_at)
+            data_timestamp = datetime.fromisoformat(row.created_at.replace("Z", "+00:00"))
+        except Exception:
+            data_timestamp = datetime.strptime(row.created_at, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
 
-        # 2. Get the exact current date and time
+        if datetime.now(timezone.utc) - data_timestamp < timedelta(hours=24):
-        current_time = datetime.now(timezone.utc)
+            print(f"Skipping {row.id} - less than 24h old")
-
-        # 3. Calculate the difference between the two times
-        time_difference = current_time - data_timestamp
-
-        # 4. Check if the difference is greater than 24 hours
-        if time_difference > timedelta(hours=24):
-            print("The data is more than 24 hours old.")
-        else:
-            #lets wait 24 hours befor downloading
-            print("The data is less than 24 hours old.")
             continue
 
-        data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
         print(f"ID: {row.id} | Date: {row.created_at} | Game: {row.game_name} | Title: {row.title}")
 
-        # 1. Define paths and isolate base directory
+        base_dir = Path(f"download/clips/{row.id}" if row.clip_is else f"download/videos/{row.id}")
+        base_dir.mkdir(parents=True, exist_ok=True)
+
+        temp_path = Path("download/temp")
+        temp_path.mkdir(parents=True, exist_ok=True)
+
+        target_video = base_dir / f"{row.id}.mp4"
+        json_file = base_dir / f"{row.id}_chat.json"
+        csv_file = base_dir / f"{row.id}.csv"
+
         if row.clip_is:
-            target_dir = f"download/clips/{row.id}"
+            cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_video} --collision Overwrite --temp-path {temp_path}"
-            cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --temp-path download/temp"
         else:
-            target_dir = f"download/videos/{row.id}"
+            cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_video} --collision Overwrite --threads 2 --temp-path {temp_path}"
-            # FIXED: Removed the duplicated command string combined with '&&'
-            cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
 
-        csv_file = f"{target_dir}/{row.id}.csv"
-
-        # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
         output = utils.run_command(cmd, look_for=["[STATUS]"])
-        output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {row.id} -o {target_dir}/{row.id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
+        output2 = utils.run_command(
+            f"TwitchDownloaderCLI chatdownload --id {row.id} -o {json_file} -E --collision Overwrite --temp-path {temp_path}",
+            look_for=["[STATUS]"]
+        )
+
+        if output2 and json_file.exists():
+            related_id = utils.get_video_id_from_chat(json_file)
+            if related_id:
+                # This will set related_video_id = 2848295283 for a clip
+                DB.set_related_video(row.id, related_id)
+                print(f" -> Linked {row.id} -> related VOD {related_id}")
+            if row.clip_is:
+                chapters = utils.get_chapters(json_file)
+                for chapter in chapters:
+                    chapter_record = Chapter(
+                        twitch_video_id=row.id,
+                        start_ms=chapter.get('startMilliseconds'),
+                        length_ms=chapter.get('lengthMilliseconds'),
+                        description=chapter.get('description'),
+                        type=chapter.get('type'),
+                        game_display_name=chapter.get('gameDisplayName'),
+                        game_id=chapter.get('gameId')
+                    )
+                    DB.insert(chapter_record)
+                    print(f" ➕ Chapter added: {chapter_record.description} ({chapter_record.start_ms}ms)")
 
         if output:
-            # Only write CSV and update database if download actually completed
+            # FIX 4: write_csv needs actual row data, not column names
-            data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
+            utils.write_csv([row], str(csv_file)) # pass the object/dict, not vars(TwitchVideo)
-            utils.write_csv(data, csv_file)
+            print(f"✅ Success: {base_dir}")
-            print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
             DB.mark_as_downloaded(row.id)
         else:
-            print(f"ID: {row.id} | Download Process failed.")
+            print(f"❌ ID: {row.id} | Download failed.")
-            #print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
-            #if output.get("stderr"):
-            #    print(f"Details: {output['stderr']}")
-
-            # Clear temporary chunk clutter immediately if a VOD crashes out
             if not row.clip_is:
                 print("Flushing temporary crash chunks...")
                 subprocess.run("rm -rf download/temp/*", shell=True)
 
+
     print("Finished Downloading Pipeline.")
 
 if __name__ == "__main__":

package/utils.py

@@ -1,16 +1,89 @@
 #!/usr/bin/env python3
 import csv
+from pathlib import Path
 import sys
+import json
 import shlex
 import subprocess
 
-def write_csv(data, file_name):
+def get_chapters(json_path):
-    # Open file with newline='' to prevent extra blank rows across platforms
+    """Loads the JSON file and returns all chapters."""
-    with open(file_name, "w", newline="", encoding="utf-8") as file:
+    with open(json_path, 'r', encoding='utf-8') as f:
-        writer = csv.writer(file)
+        data = json.load(f)
 
-        # Write all rows at once
+    # The chapters are under video -> chapters
-        writer.writerows(data)
+    video = data.get('video', {})
+    chapters = video.get('chapters', [])
+
+    if not chapters:
+        print("No chapters found.")
+        return []
+
+    print(f"Found {len(chapters)} chapter(s) in video '{video.get('title')}':\n")
+
+    for i, chapter in enumerate(chapters, 1):
+        start_sec = chapter.get('startMilliseconds', 0) / 1000
+        length_sec = chapter.get('lengthMilliseconds', 0) / 1000
+
+        #print(f"{i}. {chapter.get('description', 'No Title')}")
+        #print(f" - Type: {chapter.get('type')}")
+        #print(f" - Start: {start_sec}s ({chapter.get('startMilliseconds')}ms)")
+        #print(f" - Length: {length_sec}s ({chapter.get('lengthMilliseconds')}ms)")
+        #print(f" - Game: {chapter.get('gameDisplayName')} (ID: {chapter.get('gameId')})")
+        #print()
+
+    return chapters
+
+def get_video_id_from_chat(json_path: str | Path) -> str | None:
+    """Gets the id inside video:{ } - from the chat json you download"""
+    try:
+        with open(json_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return data.get("video", {}).get("id")
+    except Exception as e:
+        print(f"Could not read video.id from {json_path}: {e}")
+        return None
+
+def write_csv(data: list, file_name: str):
+    """Writes a list of dictionaries or objects to a CSV file."""
+    
+    if not data:
+        print(f"⚠️ Warning: No data provided to write to {file_name}.")
+        return
+
+    # Determine if the data contains objects that need serialization
+    first_item = data[0]
+    
+    if hasattr(first_item, '__dict__') and not isinstance(first_item, (str, int, float, bool)):
+        print("ℹ️ Detected object types. Converting objects to dictionary rows for CSV writing.")
+        
+        # Extract attributes/properties into a list of dictionaries
+        data_to_write = []
+        # We assume all objects in the list have the same structure for simplicity
+        # A more robust solution would require knowing the schema.
+        for item in data:
+            row_dict = {}
+            # Export all instance attributes to the dictionary
+            for key, value in item.__dict__.items():
+                row_dict[key] = value
+            data_to_write.append(row_dict)
+            
+        # Since csv.writer doesn't handle dicts directly, we list the keys from the first dict
+        fieldnames = list(data_to_write[0].keys())
+        
+        with open(file_name, "w", newline="", encoding="utf-8") as file:
+            writer = csv.DictWriter(file, fieldnames=fieldnames)
+            
+            writer.writeheader()
+            writer.writerows(data_to_write)
+            
+    else:
+        # Assume data is already a list of lists/tuples
+        with open(file_name, "w", newline="", encoding="utf-8") as file:
+            writer = csv.writer(file)
+            
+            # Write all rows at once
+            writer.writerows(data)
 
 def run_command(cmd_str: str, progress_prefix: str = "Progress", look_for: list = ["frame=", "time=", "fps=", "Rendering frame"]) -> bool:
    """Runs a system command, streams its output live, and reports errors on failure."""

package/video_editing/__init__.py

@@ -0,0 +1,13 @@
+# package/video_editing/__init__.py
+
+# Expose components from individual modules using relative imports
+from .combine_videos import merge_videos
+from .split_video_by_timestamps import split_video
+from .build_shorts import fit_to_9_16_letterbox
+
+# Explicitly define what is exposed when importing the subpackage
+__all__ = [
+    "merge_videos",
+    "split_video",
+    "fit_to_9_16_letterbox"
+]

package/video_editing/combine_videos.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""Combine multiple video files into one file using FFmpeg.
+
+Example:
+    python helper_files/combine_videos.py output.mp4 video1.mp4 video2.mp4 video3.mp4
+    python helper_files/combine_videos.py --output merged.mp4 --inputs video1.mp4 video2.mp4
+"""
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+def build_concat_list(input_files, list_path):
+    with open(list_path, "w", encoding="utf-8") as f:
+        for path in input_files:
+            f.write(f"file '{os.path.abspath(path)}'\n")
+
+
+def run_ffmpeg(cmd):
+    print("Running:", " ".join(cmd))
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.returncode != 0:
+        print(result.stdout)
+        print(result.stderr)
+        raise RuntimeError(f"FFmpeg failed with exit code {result.returncode}")
+    print("✅ Finished successfully.")
+
+
+def concat_videos_copy(input_files, output_path):
+    list_path = Path(output_path).with_suffix(".concat.txt")
+    build_concat_list(input_files, list_path)
+
+    cmd = [
+        "ffmpeg",
+        "-y",
+        "-f", "concat",
+        "-safe", "0",
+        "-i", str(list_path),
+        "-c", "copy",
+        str(output_path),
+    ]
+    run_ffmpeg(cmd)
+    list_path.unlink(missing_ok=True)
+
+
+def concat_videos_reencode(input_files, output_path):
+    inputs = []
+    filter_parts = []
+    audio_parts = []
+
+    for i, video_path in enumerate(input_files):
+        inputs.extend(["-i", str(video_path)])
+        # Use semicolons to separate different input streams in the filter graph
+        filter_parts.append(f"[{i}:v]scale=trunc(iw/2)*2:trunc(ih/2)*2,setpts=PTS-STARTPTS[v{i}]")
+        audio_parts.append(f"[{i}:a]aresample=async=1[a{i}]")
+
+    # Correctly separate individual stream filters with semicolons
+    filter_complex = f"{';'.join(filter_parts)};{';'.join(audio_parts)};"
+    filter_complex += "".join(f"[v{i}][a{i}]" for i in range(len(input_files)))
+    filter_complex += f"concat=n={len(input_files)}:v=1:a=1[outv][outa]"
+
+    cmd = [
+        "ffmpeg",
+        "-y",
+        *inputs,
+        "-filter_complex", filter_complex,
+        "-map", "[outv]",
+        "-map", "[outa]",
+        "-c:v", "libx264",
+        "-crf", "18",         # High quality video
+        "-preset", "slow",    # High efficiency
+        "-c:a", "aac",
+        "-b:a", "256k",       # High quality audio
+        "-movflags", "+faststart",
+        str(output_path),
+    ]
+    run_ffmpeg(cmd)
+
+
+def merge_videos(input_files, output_path):
+    input_files = [str(Path(p).expanduser().resolve()) for p in input_files]
+    output_path = str(Path(output_path).expanduser().resolve())
+
+    if len(input_files) < 2:
+        raise ValueError("You need at least 2 video files to merge.")
+
+    for path in input_files:
+        if not os.path.exists(path):
+            raise FileNotFoundError(f"File not found: {path}")
+
+    output_dir = Path(output_path).parent
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    try:
+        concat_videos_copy(input_files, output_path)
+    except Exception as exc:
+        print(f"Fast copy method failed: {exc}")
+        print("Retrying with re-encode fallback...")
+        concat_videos_reencode(input_files, output_path)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Join two or more video files together with FFmpeg.")
+    parser.add_argument("output_file", help="Path to the merged output video")
+    parser.add_argument("input_files", nargs="+", help="Two or more videos to concatenate")
+    args = parser.parse_args()
+
+    try:
+        merge_videos(args.input_files, args.output_file)
+        print(f"Merged video saved to: {args.output_file}")
+    except Exception as exc:
+        print(f"Error: {exc}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/video_editing/highlights.py

@@ -0,0 +1,378 @@
+import re
+import cv2
+import numpy as np
+import argparse
+import json
+from pathlib import Path
+from moviepy import VideoFileClip
+
+# ---- CONFIG ----
+DEFAULT_KEYWORDS = [
+    # Pure Hype - the #1 highlight signal
+    "pog", "poggers", "pogchamp", "let's go", "lets go", "huge", "insane", "crazy",
+    "unbelievable", "no way", "way too good", "oh my god", "omg", "wow", "holy",
+    "holy shit", "holy fuck", "what the fuck", "wtf", "lfg", "clip that", "clip it",
+    "someone clip", "chat clip", "world record", "pb", "personal best",
+
+    # Chat interaction = something happened
+    "chat", "look at chat", "chat is going crazy", "we are so back", "we're so back",
+    "it's over", "ggs", "gg", "not like this", "like this", "AYAYA", "KEKW",
+    "monka", "pepega", "copium",
+
+    # Gaming clutch moments
+    "clutch", "ace", "1v4", "1v3", "1v2", "quadra", "penta", "pentakill",
+    "triple kill", "rampage", "unstoppable", "godlike", "wipe", "team wipe",
+    "no scope", "collat", "one shot", "one tap", "200 pumped", "knocked", "cracked",
+
+    # Reaction / laugh / fail
+    "bro", "dude", "what is happening", "what just happened", "there is no way",
+    "you're kidding", "stop", "no no no", "yes yes yes", "i can't", "i cant believe",
+    "this is insane", "this guy", "look at this", "wait wait wait"
+]
+
+# You can add your own: --keywords "goal, fight, knock out"
+import subprocess
+import tempfile
+import os
+from scipy.io import wavfile
+
+def get_av_scores(video_path, sample_fps=2):
+    print(f"-> Opening: {video_path}", flush=True)
+    if not os.path.exists(video_path):
+        raise FileNotFoundError(video_path)
+
+    with VideoFileClip(video_path) as clip:
+        duration = clip.duration
+        fps = clip.fps or 30
+        has_audio = clip.audio is not None
+
+    print(f" duration={duration:.2f}s fps={fps} has_audio={has_audio}", flush=True)
+    n_secs = int(duration)
+
+     # ---- 1. AUDIO (streaming, constant RAM) ----
+    print(f"[1/3] Audio energy...", flush=True)
+    audio_per_sec = np.zeros(n_secs)
+
+    if has_audio:
+        tmp_wav = tempfile.mktemp(suffix=".wav")
+        try:
+            print(f" extracting mono 16k wav with ffmpeg...", flush=True)
+            cmd = [
+                "ffmpeg", "-y",
+                "-i", video_path,
+                "-ar", "16000",
+                "-ac", "1",
+                "-c:a", "pcm_s16le",
+                tmp_wav
+            ]
+            result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=300)
+            if result.returncode!= 0:
+                print(f" ffmpeg failed: {result.stderr.decode()[:1000]}")
+            else:
+                wav_size = os.path.getsize(tmp_wav)
+                print(f" wav size: {wav_size} bytes, computing RMS streaming...", flush=True)
+
+                # STREAM instead of loading all at once
+                import wave
+                with wave.open(tmp_wav, 'rb') as wf:
+                    sr = wf.getframerate() # should be 16000
+                    n_frames = wf.getnframes()
+                    # sr frames = 1 second
+                    for i in range(n_secs):
+                        if wf.tell() >= n_frames:
+                            break
+                        # read 1 second
+                        frames = wf.readframes(sr)
+                        if not frames:
+                            break
+                        # int16 -> float without extra copy
+                        chunk = np.frombuffer(frames, dtype=np.int16).astype(np.float32) / 32768.0
+                        if len(chunk) > 0:
+                            audio_per_sec[i] = np.sqrt(np.mean(chunk**2))
+
+                        if i % 300 == 0: # log every 5 min
+                            print(f" audio {i}/{n_secs}s", flush=True)
+
+                if audio_per_sec.max() > 0:
+                    audio_per_sec = (audio_per_sec - audio_per_sec.min()) / (audio_per_sec.max() - audio_per_sec.min() + 1e-6)
+                print(f" audio done", flush=True)
+
+        except Exception as e:
+            print(f" audio error: {e}")
+            import traceback; traceback.print_exc()
+        finally:
+            if os.path.exists(tmp_wav):
+                os.remove(tmp_wav)
+    else:
+        print(" no audio track", flush=True)
+
+    # ---- 2. MOTION ----
+    print(f"[2/3] Motion analysis sample_fps={sample_fps}...", flush=True)
+    cap = cv2.VideoCapture(video_path)
+    motion_scores = []
+    prev_gray = None
+    frame_interval = int(fps / sample_fps) if fps > 0 else 15
+    num_samples = int(duration * sample_fps)
+
+    for i in range(num_samples):
+        cap.set(cv2.CAP_PROP_POS_FRAMES, i * frame_interval)
+        ret, frame = cap.read()
+        if not ret:
+            continue
+        gray = cv2.cvtColor(cv2.resize(frame, (320, 180)), cv2.COLOR_BGR2GRAY)
+        if prev_gray is not None:
+            motion_scores.append(np.mean(cv2.absdiff(gray, prev_gray)) / 255.0)
+        else:
+            motion_scores.append(0)
+        prev_gray = gray
+        if i % 50 == 0:
+            print(f" motion {i}/{num_samples}", flush=True)
+    cap.release()
+
+    if len(motion_scores) > 1:
+        x = np.linspace(0, duration-1, len(motion_scores))
+        motion_per_sec = np.interp(np.arange(n_secs), x, motion_scores)
+        if motion_per_sec.max() > 0:
+            motion_per_sec = (motion_per_sec - motion_per_sec.min()) / (motion_per_sec.max() - motion_per_sec.min() + 1e-6)
+    else:
+        motion_per_sec = np.zeros(n_secs)
+
+    combined_av = 0.6 * audio_per_sec + 0.4 * motion_per_sec
+    return combined_av, duration
+
+def srt_time_to_seconds(time_str):
+    # 00:01:23,456 or 00:01:23.456
+    time_str = time_str.strip().replace('.', ',')
+    h, m, s_ms = time_str.split(':')
+    s, ms = s_ms.split(',')
+    return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000.0
+
+def load_text_from_srt(srt_path):
+    print(f"[3/3] Loading transcript from SRT: {srt_path}")
+    if not Path(srt_path).exists():
+        raise FileNotFoundError(f"SRT not found: {srt_path}")
+
+    # utf-8-sig strips BOM that Whisper sometimes adds
+    with open(srt_path, 'r', encoding='utf-8-sig') as f:
+        content = f.read().strip()
+
+    if not content:
+        raise ValueError(f"SRT file is empty: {srt_path}")
+
+    # Split by blank line - works for \n\n and \r\n\r\n
+    blocks = re.split(r'\r?\n\r?\n', content)
+    
+    segments = []
+    # Pattern inside each block only
+    time_pattern = re.compile(r'(\d{2}:\d{2}:\d{2}[,\.]\d{3})\s*-->\s*(\d{2}:\d{2}:\d{2}[,\.]\d{3})')
+
+    for block in blocks:
+        block = block.strip()
+        if not block:
+            continue
+        
+        lines = block.splitlines()
+        if len(lines) < 2:
+            continue
+
+        # Find time line (line 0 or 1 - index may be missing)
+        time_match = None
+        text_start_idx = 0
+        for idx, line in enumerate(lines[:3]): # check first 3 lines for time
+            m = time_pattern.search(line)
+            if m:
+                time_match = m
+                text_start_idx = idx + 1
+                break
+        
+        if not time_match:
+            continue
+
+        start_str, end_str = time_match.groups()
+        text = ' '.join(lines[text_start_idx:]).strip()
+        
+        # Clean Whisper / Twitch artifacts
+        text = re.sub(r'<[^>]+>', '', text) # remove <i>, <font> etc
+        text = text.replace('\n', ' ').strip()
+        
+        if text:
+            segments.append({
+                "start": srt_time_to_seconds(start_str),
+                "end": srt_time_to_seconds(end_str),
+                "text": text
+            })
+    
+    print(f"Loaded {len(segments)} segments from SRT")
+    if len(segments) == 0:
+        print(f"WARNING: Parsed 0 segments. First 500 chars of file:\n{content[:500]}")
+    return segments
+
+def score_keywords(segments, duration, keywords):
+    """Returns per-second keyword boost score + keyword hits"""
+    keyword_score = np.zeros(int(duration))
+    hits = []
+
+    # Build regex with word boundaries for each keyword
+    # \bace\b won't match "place", \bgg\b won't match "ggs"
+    patterns = []
+    for kw in keywords:
+        kw = kw.strip()
+        if not kw:
+            continue
+        # \b at start/end, but handle multi-word phrases
+        # re.escape handles "let's go", "1v4", etc
+        pat = re.compile(r'\b' + re.escape(kw.lower()) + r'\b', re.IGNORECASE)
+        patterns.append((kw.lower(), pat))
+
+    for seg in segments:
+        text = seg['text']
+        text_lower = text.lower()
+        for kw_lower, pat in patterns:
+            if pat.search(text_lower):
+                start = int(max(0, seg['start'] - 1))
+                end = int(min(duration-1, seg['end'] + 2))
+                keyword_score[start:end+1] += 1.0
+                hits.append({
+                    "timestamp": round(seg['start'], 2),
+                    "end": round(seg['end'], 2),
+                    "keyword": kw_lower,
+                    "text": text.strip()
+                })
+                break
+
+    if keyword_score.max() > 0:
+        keyword_score = keyword_score / keyword_score.max()
+
+    return keyword_score, hits
+
+def find_highlights(final_scores, duration, top_n=5, clip_len=8, min_gap=12):
+    candidates = []
+    for i in range(len(final_scores) - clip_len):
+        candidates.append((np.mean(final_scores[i:i+clip_len]), i))
+
+    candidates.sort(reverse=True, key=lambda x: x[0])
+    picked = []
+    for score, start in candidates:
+        if all(abs(start - p[1]) >= min_gap for p in picked):
+            picked.append((score, start))
+        if len(picked) >= top_n:
+            break
+
+    picked.sort(key=lambda x: x[1])
+    return [(s, e, sc) for sc, s in [(sc, st) for sc, st in picked] for e in [s+clip_len]]
+
+def format_timestamp(seconds):
+    m, s = divmod(int(seconds), 60)
+    h, m = divmod(m, 60)
+    return f"{h:02d}:{m:02d}:{s:02d}" if h > 0 else f"{m:02d}:{s:02d}"
+
+def highlights(video_path, keywords=None, top_n=5, clip_len=8, min_gap=12):
+    av_score, duration = get_av_scores(video_path, sample_fps=0.5)
+    
+    segments = load_text_from_srt(video_path.replace('.mp4', '.srt')) 
+    kw_score, kw_hits = score_keywords(segments, duration, keywords)
+
+    # FINAL SCORE: AV + keyword boost (keyword is huge if found)
+    final_score = av_score + (kw_score * 1.5)
+    final_score = (final_score - final_score.min()) / (final_score.max() - final_score.min() + 1e-6)
+    final_score = np.convolve(final_score, np.ones(3)/3, mode='same') # smooth
+
+    highlights = find_highlights(final_score, duration, top_n=top_n, clip_len=clip_len)
+
+    # --- OUTPUT ---
+    print("\n" + "="*60)
+    print("KEYWORD HITS:")
+    for h in kw_hits:
+        print(f" [{format_timestamp(h['timestamp'])}] ({h['keyword']}) -> {h['text']}")
+
+    print("\n" + "="*60)
+    print("FINAL HIGHLIGHTS:")
+    output_data = []
+    for i, (start, end, score) in enumerate(highlights, 1):
+        # find transcript in that window
+        texts_in_window = [s['text'].strip() for s in segments if not (s['end'] < start or s['start'] > end)]
+        transcript = " ".join(texts_in_window)[:120]
+
+        print(f"{i}. {format_timestamp(start)} - {format_timestamp(end)} (score {score:.2f}) | {transcript}")
+        output_data.append({
+            "id": i,
+            "start_sec": float(start),
+            "end_sec": float(end),
+            "start_timestamp": format_timestamp(start),
+            "end_timestamp": format_timestamp(end),
+            "score": float(score),
+            "transcript": transcript
+        })
+
+    # Save timestamps to JSON
+    with open("highlights.json", "w") as f:
+        json.dump({"video": video_path, "keywords": keywords, "highlights": output_data, "keyword_hits": kw_hits}, f, indent=2)
+    print(f"\nTimestamps saved to highlights.json")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Find highlights with Whisper keywords + timestamps")
+    parser.add_argument("video", help="Input video file")
+    parser.add_argument("--keywords", type=str, default=",".join(DEFAULT_KEYWORDS), help="Comma-separated keywords to hunt for")
+    parser.add_argument("--model", type=str, default="base", help="Whisper model: tiny, base, small, medium")
+    parser.add_argument("--top", type=int, default=5)
+    parser.add_argument("--len", type=int, default=30, help="Highlight length in seconds")
+    parser.add_argument("--no-cut", action="store_true", help="Only print timestamps, don't cut video files")
+    args = parser.parse_args()
+
+    keywords = [k.strip() for k in args.keywords.split(",") if k.strip()]
+
+    av_score, duration = get_av_scores(args.video, sample_fps=0.5)
+    
+    segments = load_text_from_srt(args.video.replace('.mp4', '.srt')) 
+    kw_score, kw_hits = score_keywords(segments, duration, keywords)
+
+    # FINAL SCORE: AV + keyword boost (keyword is huge if found)
+    final_score = av_score + (kw_score * 1.5)
+    final_score = (final_score - final_score.min()) / (final_score.max() - final_score.min() + 1e-6)
+    final_score = np.convolve(final_score, np.ones(3)/3, mode='same') # smooth
+
+    highlights = find_highlights(final_score, duration, top_n=args.top, clip_len=args.len)
+
+    # --- OUTPUT ---
+    print("\n" + "="*60)
+    print("KEYWORD HITS:")
+    for h in kw_hits:
+        print(f" [{format_timestamp(h['timestamp'])}] ({h['keyword']}) -> {h['text']}")
+
+    print("\n" + "="*60)
+    print("FINAL HIGHLIGHTS:")
+    output_data = []
+    for i, (start, end, score) in enumerate(highlights, 1):
+        # find transcript in that window
+        texts_in_window = [s['text'].strip() for s in segments if not (s['end'] < start or s['start'] > end)]
+        transcript = " ".join(texts_in_window)[:120]
+
+        print(f"{i}. {format_timestamp(start)} - {format_timestamp(end)} (score {score:.2f}) | {transcript}")
+        output_data.append({
+            "id": i,
+            "start_sec": float(start),
+            "end_sec": float(end),
+            "start_timestamp": format_timestamp(start),
+            "end_timestamp": format_timestamp(end),
+            "score": float(score),
+            "transcript": transcript
+        })
+
+    # Save timestamps to JSON
+    with open("highlights.json", "w") as f:
+        json.dump({"video": args.video, "keywords": keywords, "highlights": output_data, "keyword_hits": kw_hits}, f, indent=2)
+    print(f"\nTimestamps saved to highlights.json")
+
+    # Cut videos if requested
+    #if not args.no_cut:
+    if args.no_cut:
+        print("\nCutting clips...")
+        Path("highlights").mkdir(exist_ok=True)
+        clip = VideoFileClip(args.video)
+        for h in output_data:
+            sub = clip.subclip(h['start_sec'], h['end_sec'])
+            out = f"highlights/highlight_{h['id']}_{h['start_timestamp'].replace(':','-')}.mp4"
+            sub.write_videofile(out, codec="libx264", audio_codec="aac", logger=None)
+        clip.close()
+        print("Done ->./highlights/")

package/video_editing/split_video_by_timestamps.py

@@ -0,0 +1,112 @@
+import subprocess
+import os
+from typing import List, Tuple
+
+def split_video(input_file: str, output_dir: str, timestamps: List[Tuple[float, float]], frame_perfect: bool = False) -> None:
+    """
+    Splits a video file into multiple segments based on specified time ranges.
+    
+    Args:
+        input_file (str): The path to the video file to be split.
+        output_dir (str): The directory where the split segments will be saved.
+        timestamps (List[Tuple[float, float]]): A list of tuples representing (start_time, end_time).
+        frame_perfect (bool): If True, skips fast-copy and forces high-quality re-encoding 
+                              to ensure cuts happen exactly on the requested frame.
+    """
+    if not os.path.exists(input_file):
+        print(f"Error: Input file not found at {input_file}")
+        return
+
+    os.makedirs(output_dir, exist_ok=True)
+    print(f"Starting video split process for {input_file}...")
+
+    for i, (start, end) in enumerate(timestamps):
+        segment_name = f"segment_{i+1:02d}_{start:.2f}s_to_{end:.2f}s.mp4"
+        output_file = os.path.join(output_dir, segment_name)
+
+        print(f"\n-> Processing segment {i+1}/{len(timestamps)}: {segment_name}")
+        
+        # Method 1: Fast Copy (Lossless, no re-encoding)
+        # Moving -ss BEFORE -i fixes the frozen/corrupted initial frames.
+        fast_command = [
+            "ffmpeg", "-y",
+            "-ss", str(start),
+            "-i", input_file,
+            "-to", str(end),
+            "-c", "copy",
+            output_file
+        ]
+        
+        # Method 2: High-Quality Re-encode (Frame-perfect fallback)
+        # Used if fast copy fails or if frame-accuracy is explicitly requested.
+        reencode_command = [
+            "ffmpeg", "-y",
+            "-ss", str(start),
+            "-i", input_file,
+            "-to", str(end),
+            "-c:v", "libx264",
+            "-crf", "18",         # High visual quality
+            "-preset", "slow",    # Better compression efficiency
+            "-c:a", "aac",
+            "-b:a", "256k",       # High-fidelity audio
+            output_file
+        ]
+
+        try:
+            if frame_perfect:
+                print("Running in frame-perfect mode (re-encoding)...")
+                subprocess.run(reencode_command, check=True, capture_output=True, text=True)
+            else:
+                try:
+                    print("Attempting lightning-fast stream copy...")
+                    subprocess.run(fast_command, check=True, capture_output=True, text=True)
+                except subprocess.CalledProcessError:
+                    print("⚠️ Fast copy failed. Falling back to high-quality re-encode...")
+                    subprocess.run(reencode_command, check=True, capture_output=True, text=True)
+            
+            print(f"Successfully created segment: {output_file}")
+
+        except subprocess.CalledProcessError as e:
+            print(f"❌ Error splitting video segment {i+1}:")
+            print(f"Command failed with return code {e.returncode}")
+            print(f"STDERR: {e.stderr}")
+        except FileNotFoundError:
+            print("-" * 50)
+            print("CRITICAL ERROR: FFmpeg executable not found.")
+            print("Please ensure 'ffmpeg' is installed and accessible in your system's PATH.")
+            print("-" * 50)
+            return
+
+def main():
+    """
+    Example usage: 
+    Assumes 'input.mp4' exists and a 'split_videos' directory is ready.
+    """
+    # --- CONFIGURATION ---
+    INPUT_VIDEO_PATH = "path/to/your/input_video.mp4"  # <<< CHANGE THIS
+    OUTPUT_SEGMENTS_DIR = "output_splits"              # <<< CHANGE THIS
+    
+    # Define time segments: [(start_time, end_time), ...]
+    # Times should be floats representing seconds.
+    # Example 1: 0s to 10s, then 10s to 20s
+    TIMESTAMPS: List[Tuple[float, float]] = [
+        (0.0, 10.0),
+        (15.0, 25.0),
+        (30.0, 35.0)
+    ]
+    # ---------------------
+
+    if not os.path.exists(INPUT_VIDEO_PATH):
+        print("!!! IMPORTANT !!!")
+        print(f"Please update the 'INPUT_VIDEO_PATH' variable in the main() function with the correct path to your video.")
+        print("!!! Script execution aborted. !!!")
+        return
+
+    split_video(
+        input_file=INPUT_VIDEO_PATH,
+        output_dir=OUTPUT_SEGMENTS_DIR,
+        timestamps=TIMESTAMPS
+    )
+
+if __name__ == "__main__":
+    main()

package/youtube/config.py

@@ -3,11 +3,10 @@ import json
 
 from google.oauth2.credentials import Credentials
 from google.auth.transport.requests import Request
+from googleapiclient.discovery import build
 
-CREDENTIALS = None
+def get_youtube_service():
-
+    """Get the YouTube service using the loaded credentials."""
-def load_credentials() -> Credentials:
-    """Load credentials from secrets.json"""
     try:
         with open('secrets.json', 'r') as f:
             creds_data = json.load(f)
@@ -25,7 +24,6 @@ def load_credentials() -> Credentials:
         if credentials.expired:
             credentials.refresh(Request())
                 
-        return credentials
     except FileNotFoundError:
         print("Error: secrets.json not found! Please run setup.py first.")
         return None
@@ -33,4 +31,13 @@ def load_credentials() -> Credentials:
         print(f"Error loading credentials: {str(e)}")
         return None
     
-CREDENTIALS = load_credentials()
+    if credentials is None:
+        print("Error: Credentials not loaded. Please check secrets.json.")
+        return None
+    
+    try:
+        youtube_service = build('youtube', 'v3', credentials=credentials)
+        return youtube_service
+    except Exception as e:
+        print(f"Error creating YouTube service: {str(e)}")
+        return None

package/youtube/find_tags.py

@@ -97,10 +97,8 @@ def scan_channel_videos_for_tag(youtube: build, uploads_playlist_id: str, target
 
 def find_tags():
     
-    if not config.CREDENTIALS:
-        return
   
-    youtube = build('youtube', 'v3', credentials=config.CREDENTIALS)
+    youtube = config.get_youtube_service()
     
     # 1. Fetch your dynamic uploads playlist pointer
     uploads_id = get_my_uploads_playlist_id(youtube)

package/youtube/uploader.py

@@ -55,13 +55,9 @@ def upload_video(file_path: str, title: str, category: CategoryId, description:
             print(f"❌ Error: File not found: {file_path}")
             return None
 
-        # Load credentials
-        if not config.CREDENTIALS:
-            print("❌ Error: YouTube API credentials not configured.")
-            return None
   
         # Create YouTube API client
-        youtube = build('youtube', 'v3', credentials=config.CREDENTIALS)
+        youtube = config.get_youtube_service()
 
         # Configure the status object dynamically
         status_body = {