update database to use sqlalchemy

helper_files/remove_uneed_files.py

@@ -10,33 +10,15 @@ from package.config import DB as DB
 
 
 def remove_files():
-    datas = DB.get_download()
+    datas = DB.get_downloaded()
 
     for data in datas:
-        (
-            id,
-            title,
-            created_at,
-            view_count,
-            duration,
-            url,
-            thumbnail_url,
-            game_id,
-            game_name,
-            stream_id,
-            creator_name,
-            clip_is,
-            downloaded,
-            uploaded_yt,
-            uploaded_yt_chats,
-            uploaded_yt_shorts,
-        ) = data
-
-        if clip_is == 0:
+        
+        if data.clip_is is False:
             continue
-            video_path = f"download/videos/{id}/{id}.mp4"
+            video_path = f"download/videos/{data.id}/{data.id}.mp4"
         else:
-            video_path = f"download/clips/{id}/{id}.mp4"
+            video_path = f"download/clips/{data.id}/{data.id}.mp4"
         
         temp_chat = ""#video_path.replace(".mp4", "_temp_chat.mp4")
         temp_with_chat = ""#video_path.replace(".mp4", "_temp_with_chat.mp4")

package/build_videos.py

@@ -10,76 +10,36 @@ from package.config import TZ as TZ
 def build_chat_video():
     chats = DB.get_unuploaded_chats()
     
-    for chat in chats:
-        (
-            id,
-            title,
-            created_at,
-            view_count,
-            duration,
-            url,
-            thumbnail_url,
-            game_id,
-            game_name,
-            stream_id,
-            creator_name,
-            clip_is,
-            downloaded,
-            uploaded_yt,
-            uploaded_yt_chats,
-            uploaded_yt_shorts,
-        ) = chat
-        
+    for chat in chats:    
         print("====================================================")
-        print(f"🚀 Chat Video: {title}")
+        print(f"🚀 Chat Video: {chat.title}")
         print("====================================================")
 
-        chat_vod.combine_twitch_vod_and_chat(f"download/videos/{id}/{id}.mp4", "side-by-side")
+        chat_vod.combine_twitch_vod_and_chat(f"download/videos/{chat.id}/{chat.id}.mp4", "side-by-side")
 
     print("====================================================")
     print(f"✅ Processing: Chat Video Done.")
     print("====================================================")
 
-
 def build_transcribe():
     unuploadeds = DB.get_unuploaded()
 
     for unuploaded in unuploadeds:
-        (
-            id,
-            title,
-            created_at,
-            view_count,
-            duration,
-            url,
-            thumbnail_url,
-            game_id,
-            game_name,
-            stream_id,
-            creator_name,
-            clip_is,
-            downloaded,
-            uploaded_yt,
-            uploaded_yt_chats,
-            uploaded_yt_shorts,
-        ) = unuploaded
-
-        if clip_is:
+        if unuploaded.clip_is:
             target_dir = f"download/clips"
         else:
             target_dir = f"download/videos"
 
         print("====================================================")
-        print(f"🚀 Transcribe: {title}")
+        print(f"🚀 Transcribe: {unuploaded.title}")
         print("====================================================")
         
-        transcribe_video.transcribe_to_srt(f"{target_dir}/{id}/{id}.mp4")
+        transcribe_video.transcribe_to_srt(f"{target_dir}/{unuploaded.id}/{unuploaded.id}.mp4")
 
     print("====================================================")
     print(f"✅ Processing: Transcribing Done.")
     print("====================================================")
 
-
 def build_shorts():
     shorts = DB.get_unuploaded_shorts()
 
@@ -87,37 +47,18 @@ def build_shorts():
     count = 1
 
     for short in shorts:
-        (
-            id,
-            title,
-            created_at,
-            view_count,
-            duration,
-            url,
-            thumbnail_url,
-            game_id,
-            game_name,
-            stream_id,
-            creator_name,
-            clip_is,
-            downloaded,
-            uploaded_yt,
-            uploaded_yt_chats,
-            uploaded_yt_shorts,
-        ) = short
-
-        target_dir = f"download/clips/{id}/{id}.mp4"
+        target_dir = f"download/clips/{short.id}/{short.id}.mp4"
 
         print("====================================================")
-        print(f"🚀 Processing: Clip to Youtube Short {title}")
+        print(f"🚀 Processing: Clip to Youtube Short {short.title}")
         print("====================================================")
         print(f"{count}/{len(shorts)}")
 
-        created_at = datetime.strptime(created_at, "%Y-%m-%d %H:%M:%S%z")
+        created_at = datetime.strptime(short.created_at, "%Y-%m-%d %H:%M:%S%z")
         created_at = created_at.astimezone(TZ)    
 
         #ToDo
-        yt_shorts.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {creator_name},\n on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z")}", True, True)
+        yt_shorts.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {short.creator_name},\n on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z")}")
         count += 1
 
     print("====================================================")

package/config.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 from zoneinfo import ZoneInfo
 
-import package.database as database
+#import package.database as database
+from package.database.database import Database
 
 CHANNEL_NAME = "teampgp"
 TZ = ZoneInfo("America/Toronto")
 
-DB = database.Database()
+DB = Database()

package/database/__init__.py

@@ -0,0 +1,11 @@
+# package/database/__init__.py
+
+# Expose components from individual modules using relative imports
+from .database import TwitchVideo, Data, Database
+
+# Explicitly define what is exposed when importing the subpackage
+__all__ = [
+    "TwitchVideo",
+    "Data",
+    "Database",
+]

package/database/database.py

@@ -0,0 +1,183 @@
+from sqlalchemy import create_engine, Column, String, Integer, Boolean, select
+from sqlalchemy.orm import declarative_base, sessionmaker
+from typing import Any
+
+# 1. Define the base class for declarative models
+Base = declarative_base()
+
+# 2. Define the twitch_videos table
+class TwitchVideo(Base):
+    __tablename__ = "twitch_videos"
+
+    id = Column(String, primary_key=True)
+    title = Column(String, nullable=False)
+    created_at = Column(String, nullable=False)
+    view_count = Column(Integer, nullable=False)
+    duration = Column(String, nullable=False)
+    url = Column(String, nullable=False)
+    thumbnail_url = Column(String, nullable=False)
+    game_id = Column(String, nullable=False)
+    game_name = Column(String, nullable=False)
+    stream_id = Column(String, nullable=False)
+    creator_name = Column(String, nullable=False)
+    clip_is = Column(Boolean, nullable=False, default=False)
+    downloaded = Column(Boolean, nullable=False, default=False)
+    uploaded_yt = Column(Boolean, nullable=False, default=False)
+    uploaded_yt_chats = Column(Boolean, nullable=False, default=False)
+    uploaded_yt_shorts = Column(Boolean, nullable=False, default=False)
+
+
+# 3. Define the data table
+class Data(Base):
+    __tablename__ = "data"
+
+    # In SQLAlchemy, an Integer primary key automatically acts as AUTOINCREMENT in SQLite
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    name = Column(String, nullable=False)
+    date_shorts = Column(String, nullable=False)
+    date_videos = Column(String, nullable=False)
+    date_videos_chat = Column(String, nullable=False)
+
+class Database:
+    def __init__(self, db_path: str = "database.db"):
+        # 4. Connect to the SQLite file and generate the tables
+        # This will create 'database.db' in the current working directory if it doesn't exist.
+        engine = create_engine(f"sqlite:///{db_path}", echo=True)
+        Base.metadata.create_all(engine)
+        
+        # 5. Optional: Set up a session factory for interacting with the database
+        Session = sessionmaker(bind=engine)
+        self.session =  Session()
+
+    def __mark_as(self, record_id: str, field_name: str, value: bool = True):
+        record = self.session.get(TwitchVideo, record_id)
+        
+        if record:
+            setattr(record, field_name, value)
+            self.session.commit()
+        else:
+            print(f"❌ Record {record_id} not found in __mark_as.")
+
+    def mark_as_uploaded_shorts(self, record_id: str):
+        """Flags a specific row record to uploaded_yt_shorts (1)."""
+        self.__mark_as(record_id, "uploaded_yt_shorts")
+
+    def mark_as_uploaded_chats(self, record_id: str):
+        """Flags a specific row record to uploaded_yt_chats (1)."""
+        self.__mark_as(record_id, "uploaded_yt_chats")
+
+    def mark_as_uploaded_yt(self, record_id: str):
+        """Flags a specific row record to uploaded_yt (1)."""
+        self.__mark_as(record_id, "uploaded_yt")
+
+    def mark_as_downloaded(self, record_id: str):
+        """Flags a specific row record to downloaded (1)."""
+        self.__mark_as(record_id, "downloaded")
+
+    def unmark_as_download(self, record_id: str):
+        """Flags a specific row record to downloaded (0)."""
+        self.__mark_as(record_id, "downloaded", False)
+
+    def update_datetime(self, record_id: str, created_at: str):
+        record = self.session.get(TwitchVideo, record_id)
+
+        if record:
+            record.created_at = created_at
+            self.session.commit()
+        else:
+            print(f"❌ Record {record_id} not found in __mark_as.")
+
+    def __get_unuploaded(self, field_name: str, also_clause: Any = None) -> list[TwitchVideo]:
+        """Retrieve all rows that were downloaded but not uploaded using SQLAlchemy."""
+    
+        # 1. Validation: Verify the string corresponds to an actual column
+        if not hasattr(TwitchVideo, field_name):
+            raise ValueError(f"🚨 Invalid column name: '{field_name}' does not exist on TwitchVideo.")
+            
+        # Get the actual SQLAlchemy column object dynamically
+        status_column = getattr(TwitchVideo, field_name)
+        
+        # 2. Build the query programmatically
+        stmt = select(TwitchVideo).where(
+            TwitchVideo.downloaded == True,
+            status_column == False
+        )
+        
+        # 3. Safely append additional filters if provided
+        if also_clause is not None:
+            stmt = stmt.where(also_clause)
+            
+        # 4. Execute and return results
+        return self.session.scalars(stmt).all()
+
+    def get_unuploaded_shorts(self) -> list[TwitchVideo]:
+        """Retrieve all rows that were download but not uploaded_yt_shorts"""
+        return self.__get_unuploaded("uploaded_yt_shorts", (TwitchVideo.clip_is == True))
+
+    def get_unuploaded_chats(self) -> list[TwitchVideo]:
+        """Retrieve all rows that were download but not uploaded_yt_chats"""
+        return self.__get_unuploaded("uploaded_yt_chats", (TwitchVideo.clip_is == False))
+                
+    def get_unuploaded(self) -> list[TwitchVideo]:
+        """Retrieve all rows that were download but not uploaded_yt"""
+        return self.__get_unuploaded("uploaded_yt")
+
+    def get_undownloaded(self) -> list[TwitchVideo]:
+        stmt = select(TwitchVideo).where(TwitchVideo.downloaded == False)
+        return self.session.scalars(stmt).all()
+
+    def get_downloaded(self) -> list[TwitchVideo]:
+        stmt = select(TwitchVideo).where(TwitchVideo.downloaded == True)
+        return self.session.scalars(stmt).all()
+
+    def get_clips(self) -> list[TwitchVideo]:
+        """Retrieves all rows that are clip_is"""
+        stmt = select(TwitchVideo).where(TwitchVideo.clip_is == True)
+        return self.session.scalars(stmt).all()
+
+    def get_vods(self) -> list[TwitchVideo]:
+        stmt = select(TwitchVideo).where(TwitchVideo.clip_is == False)
+        return self.session.scalars(stmt).all()
+
+    def __update_date(self, record_id: str, field_name: str, date: str):
+        record = self.session.get(Data, record_id)
+
+        if record:
+            setattr(record, field_name, date)
+            self.session.commit()
+        else:
+            print(f"❌ Record {record_id} not found in __update_date.")
+
+    def get_data(self) -> list[Data]:
+        stmt = select(Data)
+        return self.session.scalars(stmt).all()
+
+    def insert(self, data: TwitchVideo | Data):
+        # SQLAlchemy 2.0 context manager handles commit/rollback automatically
+        try:
+            if isinstance(data, TwitchVideo):
+                # 1. Query for the existing row using SQLAlchemy 2.0 scalar execution
+                stmt = select(TwitchVideo).where(TwitchVideo.id == data.id)
+                existing_record = self.session.scalar(stmt)
+                
+                if existing_record:
+                    if existing_record.view_count != data.view_count or existing_record.duration != data.duration:
+                        # 2. Update the columns on the managed instance
+                        existing_record.view_count = data.view_count
+                        existing_record.duration = data.duration
+                        print(f"🔄 Record {data.id} updated successfully!")
+                else:
+                    # 3. Add new instance if it doesn't exist
+                    self.session.add(data)
+                    print("➕ New TwitchVideo record inserted successfully!")
+            else:
+                # Handle standard insertion for the Data model
+                self.session.add(data)
+                print("➕ Data record inserted successfully!")
+            
+            # 4. Flush changes and commit the transaction
+            self.session.commit()
+            
+        except Exception as e:
+            self.session.rollback()
+            print(f"❌ An error occurred, transaction rolled back: {e}")

package/twitch/download_videos.py

@@ -4,6 +4,7 @@ from datetime import datetime, timedelta, timezone
 
 import package.utils as utils
 from package.config import DB as DB
+from package.database import TwitchVideo
 
 def download_videos():
     """Find all undownloaded videos and clips, and download them safely."""
@@ -11,29 +12,9 @@ def download_videos():
 
     print("Starting downloads...")
     for row in undownloaded:
-        # Unpack variables clearly
-        (
-            id,
-            title,
-            created_at,
-            view_count,
-            duration,
-            url,
-            thumbnail_url,
-            game_id,
-            game_name,
-            stream_id,
-            creator_name,
-            clip_is,
-            downloaded,
-            uploaded_yt,
-            uploaded_yt_chats,
-            uploaded_yt_shorts,
-        ) = row
-
         # 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM)
         #datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
-        data_timestamp = datetime.fromisoformat(created_at)
+        data_timestamp = datetime.fromisoformat(row.created_at)
 
         # 2. Get the exact current date and time
         current_time = datetime.now(timezone.utc)
@@ -49,39 +30,38 @@ def download_videos():
             print("The data is less than 24 hours old.")
             continue
 
-        data = [list(row)]
-        print(
-            f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}"
-        )
+        data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
+        print(f"ID: {row.id} | Date: {row.created_at} | Game: {row.game_name} | Title: {row.title}")
 
         # 1. Define paths and isolate base directory
-        if clip_is:
-            target_dir = f"download/clips/{id}"
-            cmd = f"TwitchDownloaderCLI clipdownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --temp-path download/temp"
+        if row.clip_is:
+            target_dir = f"download/clips/{row.id}"
+            cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --temp-path download/temp"
         else:
-            target_dir = f"download/videos/{id}"
+            target_dir = f"download/videos/{row.id}"
             # FIXED: Removed the duplicated command string combined with '&&'
-            cmd = f"TwitchDownloaderCLI videodownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
+            cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
 
-        csv_file = f"{target_dir}/{id}.csv"
+        csv_file = f"{target_dir}/{row.id}.csv"
 
         # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
         output = utils.run_command(cmd, look_for=["[STATUS]"])
-        output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
+        output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {row.id} -o {target_dir}/{row.id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
 
         if output:
             # Only write CSV and update database if download actually completed
+            data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
             utils.write_csv(data, csv_file)
             print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
-            DB.mark_as_downloaded(id)
+            DB.mark_as_downloaded(row.id)
         else:
-            print(f"ID: {id} | Download Process failed.")
+            print(f"ID: {row.id} | Download Process failed.")
             #print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
             #if output.get("stderr"):
             #    print(f"Details: {output['stderr']}")
 
             # Clear temporary chunk clutter immediately if a VOD crashes out
-            if not clip_is:
+            if not row.clip_is:
                 print("Flushing temporary crash chunks...")
                 subprocess.run("rm -rf download/temp/*", shell=True)
 

package/twitch/video_info.py

@@ -7,6 +7,7 @@ from twitchAPI.type import VideoType
 
 import package.twitch.config as config
 from package.config import DB as DB
+from package.database.database import TwitchVideo
 
 GAME_CACHE = {} # Local cache dictionary to store game_id -> game_name mapping
 
@@ -101,20 +102,20 @@ async def get_streamer_vods() -> list:
         except ValueError:
             clean_game_id = 0
 
-        vod_data = {
-            "id": v.id,
-            "title": v.title,
-            "created_at": str(v.published_at),
-            "view_count": int(v.view_count) if v.view_count else 0,
-            "duration": v.duration,
-            "url": v.url,
-            "thumbnail_url": v.thumbnail_url,
-            "game_id": clean_game_id,
-            "game_name": game_name,
-            "stream_id": str(v.stream_id) if v.stream_id else "0",
-            "creator_name": config.CHANNEL_NAME,
-            "clip_is": False,
-        }
+        vod_data = TwitchVideo()
+        vod_data.id = v.id
+        vod_data.title = v.title
+        vod_data.created_at = str(v.published_at)
+        vod_data.view_count = int(v.view_count) if v.view_count else 0
+        vod_data.duration = v.duration
+        vod_data.url = v.url
+        vod_data.thumbnail_url = v.thumbnail_url
+        vod_data.game_id = clean_game_id
+        vod_data.game_name = game_name
+        vod_data.stream_id = str(v.stream_id) if v.stream_id else "0"
+        vod_data.creator_name = config.CHANNEL_NAME
+        vod_data.clip_is = False
+  
         all_vods.append(vod_data)
         print(f"Collected VOD: {v.title} | Category: {game_name} ({v.duration})")
 
@@ -131,20 +132,20 @@ async def get_streamer_clips() -> list:
     async for c in clip_generator:
         game_name = await get_game_name_by_id(c.game_id)
 
-        clip_data = {
-            "id": c.id,
-            "title": c.title,
-            "created_at": str(c.created_at),             
-            "view_count": int(c.view_count) if c.view_count else 0,
-            "duration": c.duration,
-            "url": c.url,
-            "thumbnail_url": c.thumbnail_url,
-            "game_id": c.game_id,
-            "game_name": game_name,
-            "stream_id": "0",
-            "creator_name": c.creator_name,
-            "clip_is": True,
-        }
+        clip_data = TwitchVideo()
+        clip_data.id = c.id
+        clip_data.title = c.title
+        clip_data.created_at = str(c.created_at)
+        clip_data.view_count = int(c.view_count) if c.view_count else 0
+        clip_data.duration = c.duration
+        clip_data.url = c.url
+        clip_data.thumbnail_url = c.thumbnail_url
+        clip_data.game_id = c.game_id
+        clip_data.game_name = game_name
+        clip_data.stream_id = "0"
+        clip_data.creator_name = c.creator_name
+        clip_data.clip_is = True
+
         all_clips.append(clip_data)
         print(f"Collected clip: {c.title} | Category: {game_name} ({c.view_count} views)")
 
@@ -157,26 +158,22 @@ async def video_info():
     vods_list = await get_streamer_vods()
     
     for v in vods_list:
-        DB.insert_video_record(
-            v['id'], v['title'], v['created_at'], v['view_count'], v['duration'], 
-            v['url'], v['thumbnail_url'], v['game_id'], v['game_name'], 
-            v['stream_id'], v['creator_name'], v['clip_is']
-        )
+        DB.insert(v)
 
     if vods_list:
-        print(f"Recent VOD: '{vods_list[0]['title']}'")
+        print("====================================================")
+        print(f"Recent VOD: '{vods_list[0].title}'")
+        print("====================================================")
 
     clips_list = await get_streamer_clips()
         
     for v in clips_list:
-        DB.insert_video_record(
-            v['id'], v['title'], v['created_at'], v['view_count'], v['duration'], 
-            v['url'], v['thumbnail_url'], v['game_id'], v['game_name'], 
-            v['stream_id'], v['creator_name'], v['clip_is']
-        )
-    
+        DB.insert(v)
+
     if clips_list:
-        print(f"Recent VOD: '{clips_list[0]['title']}'")
+        print("====================================================")
+        print(f"Recent VOD: '{clips_list[0].title}'")
+        print("====================================================")
 
 if __name__ == "__main__":
     asyncio.run(video_info())

requirements.txt

@@ -25,6 +25,7 @@ google-auth==2.58.0
 google-auth-httplib2==0.4.2
 google-auth-oauthlib==1.4.1
 googleapis-common-protos==1.75.3
+greenlet==3.5.6
 h11==0.16.0
 hf-xet==1.6.0
 httpcore==1.0.9
@@ -61,6 +62,7 @@ regex==2026.9.10
 requests==2.34.2
 requests-oauthlib==2.0.0
 six==1.17.0
+SQLAlchemy==2.0.54
 tokenizers==0.23.2
 tqdm==4.70.1
 twitchAPI==4.5.0