I forgot what I chanagerd

package/__init__.py

@@ -20,7 +20,7 @@ from .config import CHANNEL_NAME, TZ, DB
 from .hash_tags import get_top_nouns
 from .process_uploads import process_uploads
 from .transcribe_video import transcribe_to_srt
-from .utils import write_csv, run_command
+from .utils import get_chapters, get_video_id_from_chat, write_csv, run_command
 
 # 4. Expose Subpackages
 from . import twitch
@@ -39,6 +39,8 @@ __all__ = [
     "transcribe_to_srt", 
     "write_csv",
     "run_command",
+    "get_chapters",
+    "get_video_id_from_chat"
     "twitch",
     "youtube"
 ]

package/config.py

@@ -2,9 +2,9 @@
 from zoneinfo import ZoneInfo
 
 #import package.database as database
-from package.database.database import Database
+from package.database.database import Repository
 
 CHANNEL_NAME = "teampgp"
 TZ = ZoneInfo("America/Toronto")
 
-DB = Database()
+DB = Repository()

package/database/__init__.py

@@ -1,12 +1,13 @@
 # package/database/__init__.py
 
 # Expose components from individual modules using relative imports
-from .database import TwitchVideo, Data, YoutubeInfo, Database
+from .database import TwitchVideo, Data, YoutubeInfo, Chapter, Repository
 
 # Explicitly define what is exposed when importing the subpackage
 __all__ = [
     "TwitchVideo",
     "Data",
     "YoutubeInfo",
-    "Database",
+    "Chapter",
+    "Repository",
 ]

package/database/database.py

@@ -26,8 +26,12 @@ class TwitchVideo(Base):
     uploaded_yt_chats = Column(Boolean, nullable=False, default=False)
     uploaded_yt_shorts = Column(Boolean, nullable=False, default=False)
 
-    # Relationship back to YoutubeInfo (uselist=False enforces a 1-to-1 structure)
-    youtube_info = relationship("YoutubeInfo", back_populates="twitch_video", uselist=False)
+    # This now actually points to twitch_videos.id
+    related_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=True)
+
+    # Relationship back to YoutubeInfo (uselist=True enforces a 1-to-many structure)
+    youtube_info = relationship("YoutubeInfo", back_populates="twitch_video", uselist=True)
+    chapters = relationship("Chapter", back_populates="twitch_video", cascade="all, delete-orphan")
 
 # 3. Define the data table
 class Data(Base):
@@ -43,7 +47,8 @@ class Data(Base):
 class YoutubeInfo(Base):
     __tablename__ = "youtube_info"
 
-    id = Column(String, ForeignKey("twitch_videos.id"), primary_key=True)
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
     yt_id = Column(String, nullable=True)
     yt_date = Column(String, nullable=True)
     yt_short_id = Column(String, nullable=True)
@@ -56,8 +61,22 @@ class YoutubeInfo(Base):
     # Relationship linking back to the twitch video record
     twitch_video = relationship("TwitchVideo", back_populates="youtube_info")
 
+class Chapter(Base):
+    __tablename__ = "chapters"
 
-class Database:
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False)
+    start_milliseconds = Column(Integer, nullable=False)
+    length_milliseconds = Column(Integer, nullable=False)
+    description = Column(String, nullable=True)
+    type = Column(String, nullable=True)
+    game_display_name = Column(String, nullable=True)
+    game_id = Column(String, nullable=True)
+
+    # Relationship linking back to the twitch video record
+    twitch_video = relationship("TwitchVideo", back_populates="chapters")    
+
+class Repository:
     def __init__(self, db_path: str = "database.db"):
         # 4. Connect to the SQLite file and generate the tables
         # This will create 'database.db' in the current working directory if it doesn't exist.
@@ -68,6 +87,15 @@ class Database:
         Session = sessionmaker(bind=engine)
         self.session =  Session()
 
+    def set_related_video(self, record_id: str, related_video_id: str):
+        record = self.session.get(TwitchVideo, record_id)
+
+        if record:
+            setattr(record, "related_video", related_video_id)
+            self.session.commit()
+        else:
+            print(f"❌ Record {record_id} not found.")
+
     def __mark_as(self, record_id: str, field_name: str, value: bool = True):
         record = self.session.get(TwitchVideo, record_id)
         
@@ -178,33 +206,67 @@ class Database:
         stmt = select(Data)
         return self.session.scalars(stmt).all()
 
-    def get_youtube_info(self, record_id) -> YoutubeInfo | None:
-        return self.session.get(YoutubeInfo, record_id)
+    def get_youtube_info_for_video(self, video_id: str) -> list[YoutubeInfo]:
+        """Retrieve all YouTubeInfo records associated with a specific TwitchVideo ID."""
+        stmt = select(YoutubeInfo).where(YoutubeInfo.twitch_video_id == video_id)
+        return self.session.scalars(stmt).all()
 
-    def insert(self, data: TwitchVideo | Data | YoutubeInfo):
-        # SQLAlchemy 2.0 context manager handles commit/rollback automatically
+    def get_youtube_info_all(self) -> list[YoutubeInfo]:
+        return self.session.scalars(select(YoutubeInfo)).all()
+
+    def insert(self, data: TwitchVideo | Data | YoutubeInfo | Chapter):
+    def insert(self, data: TwitchVideo | Data | YoutubeInfo | Chapter | list[YoutubeInfo] | list[Chapter]):
+        """
+        Handles insertion/updating for any of the defined models,
+        using session merge/add for SQLAlchemy 2.0 best practices.
+        Accepts single objects or lists of objects for collection handling.
+        """
         try:
             if isinstance(data, TwitchVideo):
-                # 1. Query for the existing row using SQLAlchemy 2.0 scalar execution
+                # 1. Check if the record exists
                 stmt = select(TwitchVideo).where(TwitchVideo.id == data.id)
                 existing_record = self.session.scalar(stmt)
                 
                 if existing_record:
+                    # 2. Update relevant fields on the managed instance
                     if existing_record.view_count != data.view_count or existing_record.duration != data.duration:
-                        # 2. Update the columns on the managed instance
                         existing_record.view_count = data.view_count
                         existing_record.duration = data.duration
-                        print(f"🔄 Record {data.id} updated successfully!")
+                        print(f"🔄 TwitchVideo record {data.id} updated successfully!")
+                    # Note: Add more field updates here as needed
                 else:
-                    # 3. Add new instance if it doesn't exist
+                    # 3. Add new instance
                     self.session.add(data)
-                    print("➕ New TwitchVideo record inserted successfully!")
-            else:                   
+                    print(f"➕ New TwitchVideo record with ID {data.id} inserted successfully!")
+            elif isinstance(data, Data):
                 self.session.merge(data)
-                print(f"➕ {type(data).__name__} record inserted successfully!")
-            # 4. Flush changes and commit the transaction
-            self.session.commit()
+                print(f"➕ Data record merged/inserted successfully!")
+            elif isinstance(data, YoutubeInfo):
+                # Handle single YoutubeInfo object
+                self.session.merge(data)
+                print(f"➕ YoutubeInfo record merged/inserted successfully!")
+            elif isinstance(data, Chapter):
+                # Handle single Chapter object
+                self.session.merge(data)
+                print(f"➕ Chapter record merged/inserted successfully!")
+            elif isinstance(data, list):
+                # Handle lists of records (YoutubeInfo or Chapter)
+                record_type = type(data[0])
+                if record_type in (YoutubeInfo, Chapter):
+                    for item in data:
+                        self.session.merge(item)
+                    print(f"➕ Successfully processed {len(data)} records of type {record_type.__name__}.")
+                else:
+                    raise TypeError(f"Unsupported list content type: {record_type.__name__}")
+            else:
+                raise TypeError(f"Unsupported data type for insertion: {type(data).__name__}")
+            
+            # 4. Commit the transaction
+            self.session.flush() # Flush first to ensure IDs are populated
             
         except Exception as e:
             self.session.rollback()
-            print(f"❌ An error occurred, transaction rolled back: {e}")
+            print(f"❌ A critical error occurred during record insertion/update: {e}")
+            raise
+
+        self.session.commit()

package/process_uploads.py

@@ -14,9 +14,10 @@ from package.youtube.uploader import CategoryId as CategoryId
 
 from package.database import TwitchVideo, YoutubeInfo, Data
 
-def process_videos(video_type: bool = True):
+def process_videos(video_type: bool = True, limit: int = 22):
+    count = 0 # count how many videos we have upload and stop at var: limit
     videos = DB.get_unuploaded_chats() if video_type else DB.get_unuploaded_shorts()
-    file_type = "download/clips" if video_type else "download/video"
+    file_type = "download/video" if video_type else "download/clips"
     meta_data: Data = DB.get_data()[0]
 
     current_time = datetime.now(TZ)
@@ -29,6 +30,10 @@ def process_videos(video_type: bool = True):
             release_time += timedelta(days=1)
 
     for video in videos:
+        #We reached limit stop uploading videos
+        if count >= limit:
+            break
+
         base_path = Path(f"{file_type}/{video.id}/{video.id}")
         original_path = base_path.with_suffix(".mp4")
 
@@ -66,6 +71,7 @@ def process_videos(video_type: bool = True):
         tags = ['gaming', 'TeamPGP', f'{video.game_name}', f'{video.creator_name}', 'Level1Techs', 'twitch']
         if not video_type:
             tags.extend(['shorts', 'twitch_clips', 'clips'])
+        tags.extend([video.id])
 
         if srt_path.exists():
             try:
@@ -78,7 +84,8 @@ def process_videos(video_type: bool = True):
         if youtube_info.yt_id:
             main_video_id = youtube_info.yt_id
         else:
-            main_video_id = upload_video(str(original_path), video.title, category_id, description, 'unlisted', tags)
+            main_video_id = upload_video(str(original_path), video.title, category_id, f"{description} \n\n({video.id})", 'unlisted', tags)
+            count += 1
             if not main_video_id:
                 print(f"❌ Error: main video upload failed for {original_path}")
                 continue
@@ -86,7 +93,7 @@ def process_videos(video_type: bool = True):
             youtube_info.yt_date = datetime.now(TZ)
 
         # Upload secondary
-        secondary_desc = f"{description} Original Full size @ https://youtu.be/{main_video_id}"
+        secondary_desc = f"{description} Original Full size @ https://youtu.be/{main_video_id} \n\n({video.id})"
         secondary_video_id = upload_video(str(file_path), video.title, category_id, secondary_desc, 'private', tags, release_time)
 
         if not secondary_video_id:
@@ -94,6 +101,8 @@ def process_videos(video_type: bool = True):
             DB.insert(youtube_info) # save main id at least
             continue
 
+        count += 1
+
         # Success - write CSV and DB
         data_csv = [main_video_id, secondary_video_id, release_time.isoformat()]
         utils.write_csv([data_csv], str(file_csv))

package/twitch/download_thumbnails.py

@@ -93,7 +93,7 @@ async def download_thumbnails():
     # --- OPTION B: Download Past Broadcast VOD Thumbnail ---
     # Extract the ID sequence from your target video link
     # target_vod = "2145678901"
-    db = database.Database()
+    db = database.Repository()
     vods = db.get_vods()
     for vod in vods:
         (

package/twitch/download_videos.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python3
 import subprocess
+from pathlib import Path
 from datetime import datetime, timedelta, timezone
 
 import package.utils as utils
 from package.config import DB as DB
-from package.database import TwitchVideo
+from package.config import TZ as TZ
+from package.database import TwitchVideo, Chapter
 
 def download_videos():
     """Find all undownloaded videos and clips, and download them safely."""
@@ -12,59 +14,72 @@ def download_videos():
 
     print("Starting downloads...")
     for row in undownloaded:
-        # 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM)
-        #datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
-        data_timestamp = datetime.fromisoformat(row.created_at)
+        try:
+            # 2026-08-16T23:52:10Z -> replace Z with +00:00
+            data_timestamp = datetime.fromisoformat(row.created_at.replace("Z", "+00:00"))
+        except Exception:
+            data_timestamp = datetime.strptime(row.created_at, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
 
-        # 2. Get the exact current date and time
-        current_time = datetime.now(timezone.utc)
-
-        # 3. Calculate the difference between the two times
-        time_difference = current_time - data_timestamp
-
-        # 4. Check if the difference is greater than 24 hours
-        if time_difference > timedelta(hours=24):
-            print("The data is more than 24 hours old.")
-        else:
-            #lets wait 24 hours befor downloading
-            print("The data is less than 24 hours old.")
+        if datetime.now(timezone.utc) - data_timestamp < timedelta(hours=24):
+            print(f"Skipping {row.id} - less than 24h old")
             continue
 
-        data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
         print(f"ID: {row.id} | Date: {row.created_at} | Game: {row.game_name} | Title: {row.title}")
 
-        # 1. Define paths and isolate base directory
+        base_dir = Path(f"download/clips/{row.id}" if row.clip_is else f"download/videos/{row.id}")
+        base_dir.mkdir(parents=True, exist_ok=True)
+
+        temp_path = Path("download/temp")
+        temp_path.mkdir(parents=True, exist_ok=True)
+
+        target_video = base_dir / f"{row.id}.mp4"
+        json_file = base_dir / f"{row.id}_chat.json"
+        csv_file = base_dir / f"{row.id}.csv"
+
         if row.clip_is:
-            target_dir = f"download/clips/{row.id}"
-            cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --temp-path download/temp"
+            cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_video} --collision Overwrite --temp-path {temp_path}"
         else:
-            target_dir = f"download/videos/{row.id}"
-            # FIXED: Removed the duplicated command string combined with '&&'
-            cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
+            cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_video} --collision Overwrite --threads 2 --temp-path {temp_path}"
 
-        csv_file = f"{target_dir}/{row.id}.csv"
-
-        # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
         output = utils.run_command(cmd, look_for=["[STATUS]"])
-        output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {row.id} -o {target_dir}/{row.id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
+        output2 = utils.run_command(
+            f"TwitchDownloaderCLI chatdownload --id {row.id} -o {json_file} -E --collision Overwrite --temp-path {temp_path}",
+            look_for=["[STATUS]"]
+        )
+
+        if output2 and json_file.exists():
+            related_id = utils.get_video_id_from_chat(json_file)
+            if related_id:
+                # This will set related_video_id = 2848295283 for a clip
+                DB.set_related_video(row.id, related_id)
+                print(f" -> Linked {row.id} -> related VOD {related_id}")
+            if row.clip_is:
+                chapters = utils.get_chapters(json_file)
+                for chapter in chapters:
+                    chapter_record = Chapter(
+                        twitch_video_id=row.id,
+                        start_ms=chapter.get('startMilliseconds'),
+                        length_ms=chapter.get('lengthMilliseconds'),
+                        description=chapter.get('description'),
+                        type=chapter.get('type'),
+                        game_display_name=chapter.get('gameDisplayName'),
+                        game_id=chapter.get('gameId')
+                    )
+                    DB.insert(chapter_record)
+                    print(f" ➕ Chapter added: {chapter_record.description} ({chapter_record.start_ms}ms)")
 
         if output:
-            # Only write CSV and update database if download actually completed
-            data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
-            utils.write_csv(data, csv_file)
-            print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
+            # FIX 4: write_csv needs actual row data, not column names
+            utils.write_csv([row], str(csv_file)) # pass the object/dict, not vars(TwitchVideo)
+            print(f"✅ Success: {base_dir}")
             DB.mark_as_downloaded(row.id)
         else:
-            print(f"ID: {row.id} | Download Process failed.")
-            #print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
-            #if output.get("stderr"):
-            #    print(f"Details: {output['stderr']}")
-
-            # Clear temporary chunk clutter immediately if a VOD crashes out
+            print(f"❌ ID: {row.id} | Download failed.")
             if not row.clip_is:
                 print("Flushing temporary crash chunks...")
                 subprocess.run("rm -rf download/temp/*", shell=True)
 
+
     print("Finished Downloading Pipeline.")
 
 if __name__ == "__main__":

package/utils.py

@@ -1,10 +1,50 @@
 #!/usr/bin/env python3
 import csv
+from pathlib import Path
 import sys
+import json
 import shlex
 import subprocess
 
-def write_csv(data, file_name):
+def get_chapters(json_path):
+    """Loads the JSON file and returns all chapters."""
+    with open(json_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+
+    # The chapters are under video -> chapters
+    video = data.get('video', {})
+    chapters = video.get('chapters', [])
+
+    if not chapters:
+        print("No chapters found.")
+        return []
+
+    print(f"Found {len(chapters)} chapter(s) in video '{video.get('title')}':\n")
+
+    for i, chapter in enumerate(chapters, 1):
+        start_sec = chapter.get('startMilliseconds', 0) / 1000
+        length_sec = chapter.get('lengthMilliseconds', 0) / 1000
+
+        #print(f"{i}. {chapter.get('description', 'No Title')}")
+        #print(f" - Type: {chapter.get('type')}")
+        #print(f" - Start: {start_sec}s ({chapter.get('startMilliseconds')}ms)")
+        #print(f" - Length: {length_sec}s ({chapter.get('lengthMilliseconds')}ms)")
+        #print(f" - Game: {chapter.get('gameDisplayName')} (ID: {chapter.get('gameId')})")
+        #print()
+
+    return chapters
+
+def get_video_id_from_chat(json_path: str | Path) -> str | None:
+    """Gets the id inside video:{ } - from the chat json you download"""
+    try:
+        with open(json_path, 'r', encoding='utf-8') as f:
+            data = json.load(f)
+        return data.get("video", {}).get("id")
+    except Exception as e:
+        print(f"Could not read video.id from {json_path}: {e}")
+        return None
+
+def write_csv(data: list, file_name: str):
     # Open file with newline='' to prevent extra blank rows across platforms
     with open(file_name, "w", newline="", encoding="utf-8") as file:
         writer = csv.writer(file)