From 61fe9eca25b55c3d2a3d9f6b63547d69f8786a5d Mon Sep 17 00:00:00 2001 From: SumGuyV5 Date: Thu, 17 Sep 2026 03:57:44 +0000 Subject: [PATCH] update database to use sqlalchemy --- {package => delete}/database.py | 0 helper_files/remove_uneed_files.py | 28 +---- package/build_videos.py | 79 ++----------- package/config.py | 5 +- package/database/__init__.py | 11 ++ package/database/database.py | 183 +++++++++++++++++++++++++++++ package/twitch/download_videos.py | 50 +++----- package/twitch/video_info.py | 79 ++++++------- requirements.txt | 2 + 9 files changed, 267 insertions(+), 170 deletions(-) rename {package => delete}/database.py (100%) create mode 100644 package/database/__init__.py create mode 100644 package/database/database.py diff --git a/package/database.py b/delete/database.py similarity index 100% rename from package/database.py rename to delete/database.py diff --git a/helper_files/remove_uneed_files.py b/helper_files/remove_uneed_files.py index 139a070..7ba6a3c 100644 --- a/helper_files/remove_uneed_files.py +++ b/helper_files/remove_uneed_files.py @@ -10,33 +10,15 @@ from package.config import DB as DB def remove_files(): - datas = DB.get_download() + datas = DB.get_downloaded() for data in datas: - ( - id, - title, - created_at, - view_count, - duration, - url, - thumbnail_url, - game_id, - game_name, - stream_id, - creator_name, - clip_is, - downloaded, - uploaded_yt, - uploaded_yt_chats, - uploaded_yt_shorts, - ) = data - - if clip_is == 0: + + if data.clip_is is False: continue - video_path = f"download/videos/{id}/{id}.mp4" + video_path = f"download/videos/{data.id}/{data.id}.mp4" else: - video_path = f"download/clips/{id}/{id}.mp4" + video_path = f"download/clips/{data.id}/{data.id}.mp4" temp_chat = ""#video_path.replace(".mp4", "_temp_chat.mp4") temp_with_chat = ""#video_path.replace(".mp4", "_temp_with_chat.mp4") diff --git a/package/build_videos.py b/package/build_videos.py index 70885a6..4b991e2 100644 --- a/package/build_videos.py +++ b/package/build_videos.py @@ -10,76 +10,36 @@ from package.config import TZ as TZ def build_chat_video(): chats = DB.get_unuploaded_chats() - for chat in chats: - ( - id, - title, - created_at, - view_count, - duration, - url, - thumbnail_url, - game_id, - game_name, - stream_id, - creator_name, - clip_is, - downloaded, - uploaded_yt, - uploaded_yt_chats, - uploaded_yt_shorts, - ) = chat - + for chat in chats: print("====================================================") - print(f"🚀 Chat Video: {title}") + print(f"🚀 Chat Video: {chat.title}") print("====================================================") - chat_vod.combine_twitch_vod_and_chat(f"download/videos/{id}/{id}.mp4", "side-by-side") + chat_vod.combine_twitch_vod_and_chat(f"download/videos/{chat.id}/{chat.id}.mp4", "side-by-side") print("====================================================") print(f"✅ Processing: Chat Video Done.") print("====================================================") - def build_transcribe(): unuploadeds = DB.get_unuploaded() for unuploaded in unuploadeds: - ( - id, - title, - created_at, - view_count, - duration, - url, - thumbnail_url, - game_id, - game_name, - stream_id, - creator_name, - clip_is, - downloaded, - uploaded_yt, - uploaded_yt_chats, - uploaded_yt_shorts, - ) = unuploaded - - if clip_is: + if unuploaded.clip_is: target_dir = f"download/clips" else: target_dir = f"download/videos" print("====================================================") - print(f"🚀 Transcribe: {title}") + print(f"🚀 Transcribe: {unuploaded.title}") print("====================================================") - transcribe_video.transcribe_to_srt(f"{target_dir}/{id}/{id}.mp4") + transcribe_video.transcribe_to_srt(f"{target_dir}/{unuploaded.id}/{unuploaded.id}.mp4") print("====================================================") print(f"✅ Processing: Transcribing Done.") print("====================================================") - def build_shorts(): shorts = DB.get_unuploaded_shorts() @@ -87,37 +47,18 @@ def build_shorts(): count = 1 for short in shorts: - ( - id, - title, - created_at, - view_count, - duration, - url, - thumbnail_url, - game_id, - game_name, - stream_id, - creator_name, - clip_is, - downloaded, - uploaded_yt, - uploaded_yt_chats, - uploaded_yt_shorts, - ) = short - - target_dir = f"download/clips/{id}/{id}.mp4" + target_dir = f"download/clips/{short.id}/{short.id}.mp4" print("====================================================") - print(f"🚀 Processing: Clip to Youtube Short {title}") + print(f"🚀 Processing: Clip to Youtube Short {short.title}") print("====================================================") print(f"{count}/{len(shorts)}") - created_at = datetime.strptime(created_at, "%Y-%m-%d %H:%M:%S%z") + created_at = datetime.strptime(short.created_at, "%Y-%m-%d %H:%M:%S%z") created_at = created_at.astimezone(TZ) #ToDo - yt_shorts.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {creator_name},\n on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z")}", True, True) + yt_shorts.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {short.creator_name},\n on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z")}") count += 1 print("====================================================") diff --git a/package/config.py b/package/config.py index 4423c0d..991d7ac 100644 --- a/package/config.py +++ b/package/config.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 from zoneinfo import ZoneInfo -import package.database as database +#import package.database as database +from package.database.database import Database CHANNEL_NAME = "teampgp" TZ = ZoneInfo("America/Toronto") -DB = database.Database() \ No newline at end of file +DB = Database() \ No newline at end of file diff --git a/package/database/__init__.py b/package/database/__init__.py new file mode 100644 index 0000000..1d39528 --- /dev/null +++ b/package/database/__init__.py @@ -0,0 +1,11 @@ +# package/database/__init__.py + +# Expose components from individual modules using relative imports +from .database import TwitchVideo, Data, Database + +# Explicitly define what is exposed when importing the subpackage +__all__ = [ + "TwitchVideo", + "Data", + "Database", +] diff --git a/package/database/database.py b/package/database/database.py new file mode 100644 index 0000000..491564e --- /dev/null +++ b/package/database/database.py @@ -0,0 +1,183 @@ +from sqlalchemy import create_engine, Column, String, Integer, Boolean, select +from sqlalchemy.orm import declarative_base, sessionmaker +from typing import Any + +# 1. Define the base class for declarative models +Base = declarative_base() + +# 2. Define the twitch_videos table +class TwitchVideo(Base): + __tablename__ = "twitch_videos" + + id = Column(String, primary_key=True) + title = Column(String, nullable=False) + created_at = Column(String, nullable=False) + view_count = Column(Integer, nullable=False) + duration = Column(String, nullable=False) + url = Column(String, nullable=False) + thumbnail_url = Column(String, nullable=False) + game_id = Column(String, nullable=False) + game_name = Column(String, nullable=False) + stream_id = Column(String, nullable=False) + creator_name = Column(String, nullable=False) + clip_is = Column(Boolean, nullable=False, default=False) + downloaded = Column(Boolean, nullable=False, default=False) + uploaded_yt = Column(Boolean, nullable=False, default=False) + uploaded_yt_chats = Column(Boolean, nullable=False, default=False) + uploaded_yt_shorts = Column(Boolean, nullable=False, default=False) + + +# 3. Define the data table +class Data(Base): + __tablename__ = "data" + + # In SQLAlchemy, an Integer primary key automatically acts as AUTOINCREMENT in SQLite + id = Column(Integer, primary_key=True, autoincrement=True) + name = Column(String, nullable=False) + date_shorts = Column(String, nullable=False) + date_videos = Column(String, nullable=False) + date_videos_chat = Column(String, nullable=False) + +class Database: + def __init__(self, db_path: str = "database.db"): + # 4. Connect to the SQLite file and generate the tables + # This will create 'database.db' in the current working directory if it doesn't exist. + engine = create_engine(f"sqlite:///{db_path}", echo=True) + Base.metadata.create_all(engine) + + # 5. Optional: Set up a session factory for interacting with the database + Session = sessionmaker(bind=engine) + self.session = Session() + + def __mark_as(self, record_id: str, field_name: str, value: bool = True): + record = self.session.get(TwitchVideo, record_id) + + if record: + setattr(record, field_name, value) + self.session.commit() + else: + print(f"❌ Record {record_id} not found in __mark_as.") + + def mark_as_uploaded_shorts(self, record_id: str): + """Flags a specific row record to uploaded_yt_shorts (1).""" + self.__mark_as(record_id, "uploaded_yt_shorts") + + def mark_as_uploaded_chats(self, record_id: str): + """Flags a specific row record to uploaded_yt_chats (1).""" + self.__mark_as(record_id, "uploaded_yt_chats") + + def mark_as_uploaded_yt(self, record_id: str): + """Flags a specific row record to uploaded_yt (1).""" + self.__mark_as(record_id, "uploaded_yt") + + def mark_as_downloaded(self, record_id: str): + """Flags a specific row record to downloaded (1).""" + self.__mark_as(record_id, "downloaded") + + def unmark_as_download(self, record_id: str): + """Flags a specific row record to downloaded (0).""" + self.__mark_as(record_id, "downloaded", False) + + def update_datetime(self, record_id: str, created_at: str): + record = self.session.get(TwitchVideo, record_id) + + if record: + record.created_at = created_at + self.session.commit() + else: + print(f"❌ Record {record_id} not found in __mark_as.") + + def __get_unuploaded(self, field_name: str, also_clause: Any = None) -> list[TwitchVideo]: + """Retrieve all rows that were downloaded but not uploaded using SQLAlchemy.""" + + # 1. Validation: Verify the string corresponds to an actual column + if not hasattr(TwitchVideo, field_name): + raise ValueError(f"🚨 Invalid column name: '{field_name}' does not exist on TwitchVideo.") + + # Get the actual SQLAlchemy column object dynamically + status_column = getattr(TwitchVideo, field_name) + + # 2. Build the query programmatically + stmt = select(TwitchVideo).where( + TwitchVideo.downloaded == True, + status_column == False + ) + + # 3. Safely append additional filters if provided + if also_clause is not None: + stmt = stmt.where(also_clause) + + # 4. Execute and return results + return self.session.scalars(stmt).all() + + def get_unuploaded_shorts(self) -> list[TwitchVideo]: + """Retrieve all rows that were download but not uploaded_yt_shorts""" + return self.__get_unuploaded("uploaded_yt_shorts", (TwitchVideo.clip_is == True)) + + def get_unuploaded_chats(self) -> list[TwitchVideo]: + """Retrieve all rows that were download but not uploaded_yt_chats""" + return self.__get_unuploaded("uploaded_yt_chats", (TwitchVideo.clip_is == False)) + + def get_unuploaded(self) -> list[TwitchVideo]: + """Retrieve all rows that were download but not uploaded_yt""" + return self.__get_unuploaded("uploaded_yt") + + def get_undownloaded(self) -> list[TwitchVideo]: + stmt = select(TwitchVideo).where(TwitchVideo.downloaded == False) + return self.session.scalars(stmt).all() + + def get_downloaded(self) -> list[TwitchVideo]: + stmt = select(TwitchVideo).where(TwitchVideo.downloaded == True) + return self.session.scalars(stmt).all() + + def get_clips(self) -> list[TwitchVideo]: + """Retrieves all rows that are clip_is""" + stmt = select(TwitchVideo).where(TwitchVideo.clip_is == True) + return self.session.scalars(stmt).all() + + def get_vods(self) -> list[TwitchVideo]: + stmt = select(TwitchVideo).where(TwitchVideo.clip_is == False) + return self.session.scalars(stmt).all() + + def __update_date(self, record_id: str, field_name: str, date: str): + record = self.session.get(Data, record_id) + + if record: + setattr(record, field_name, date) + self.session.commit() + else: + print(f"❌ Record {record_id} not found in __update_date.") + + def get_data(self) -> list[Data]: + stmt = select(Data) + return self.session.scalars(stmt).all() + + def insert(self, data: TwitchVideo | Data): + # SQLAlchemy 2.0 context manager handles commit/rollback automatically + try: + if isinstance(data, TwitchVideo): + # 1. Query for the existing row using SQLAlchemy 2.0 scalar execution + stmt = select(TwitchVideo).where(TwitchVideo.id == data.id) + existing_record = self.session.scalar(stmt) + + if existing_record: + if existing_record.view_count != data.view_count or existing_record.duration != data.duration: + # 2. Update the columns on the managed instance + existing_record.view_count = data.view_count + existing_record.duration = data.duration + print(f"🔄 Record {data.id} updated successfully!") + else: + # 3. Add new instance if it doesn't exist + self.session.add(data) + print("➕ New TwitchVideo record inserted successfully!") + else: + # Handle standard insertion for the Data model + self.session.add(data) + print("➕ Data record inserted successfully!") + + # 4. Flush changes and commit the transaction + self.session.commit() + + except Exception as e: + self.session.rollback() + print(f"❌ An error occurred, transaction rolled back: {e}") diff --git a/package/twitch/download_videos.py b/package/twitch/download_videos.py index 0682f36..8c09c8d 100644 --- a/package/twitch/download_videos.py +++ b/package/twitch/download_videos.py @@ -4,6 +4,7 @@ from datetime import datetime, timedelta, timezone import package.utils as utils from package.config import DB as DB +from package.database import TwitchVideo def download_videos(): """Find all undownloaded videos and clips, and download them safely.""" @@ -11,29 +12,9 @@ def download_videos(): print("Starting downloads...") for row in undownloaded: - # Unpack variables clearly - ( - id, - title, - created_at, - view_count, - duration, - url, - thumbnail_url, - game_id, - game_name, - stream_id, - creator_name, - clip_is, - downloaded, - uploaded_yt, - uploaded_yt_chats, - uploaded_yt_shorts, - ) = row - # 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM) #datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ") - data_timestamp = datetime.fromisoformat(created_at) + data_timestamp = datetime.fromisoformat(row.created_at) # 2. Get the exact current date and time current_time = datetime.now(timezone.utc) @@ -49,39 +30,38 @@ def download_videos(): print("The data is less than 24 hours old.") continue - data = [list(row)] - print( - f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}" - ) + data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')] + print(f"ID: {row.id} | Date: {row.created_at} | Game: {row.game_name} | Title: {row.title}") # 1. Define paths and isolate base directory - if clip_is: - target_dir = f"download/clips/{id}" - cmd = f"TwitchDownloaderCLI clipdownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --temp-path download/temp" + if row.clip_is: + target_dir = f"download/clips/{row.id}" + cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --temp-path download/temp" else: - target_dir = f"download/videos/{id}" + target_dir = f"download/videos/{row.id}" # FIXED: Removed the duplicated command string combined with '&&' - cmd = f"TwitchDownloaderCLI videodownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp" + cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp" - csv_file = f"{target_dir}/{id}.csv" + csv_file = f"{target_dir}/{row.id}.csv" # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes output = utils.run_command(cmd, look_for=["[STATUS]"]) - output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"]) + output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {row.id} -o {target_dir}/{row.id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"]) if output: # Only write CSV and update database if download actually completed + data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')] utils.write_csv(data, csv_file) print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}") - DB.mark_as_downloaded(id) + DB.mark_as_downloaded(row.id) else: - print(f"ID: {id} | Download Process failed.") + print(f"ID: {row.id} | Download Process failed.") #print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}") #if output.get("stderr"): # print(f"Details: {output['stderr']}") # Clear temporary chunk clutter immediately if a VOD crashes out - if not clip_is: + if not row.clip_is: print("Flushing temporary crash chunks...") subprocess.run("rm -rf download/temp/*", shell=True) diff --git a/package/twitch/video_info.py b/package/twitch/video_info.py index 9b2b8e4..f391d22 100755 --- a/package/twitch/video_info.py +++ b/package/twitch/video_info.py @@ -7,6 +7,7 @@ from twitchAPI.type import VideoType import package.twitch.config as config from package.config import DB as DB +from package.database.database import TwitchVideo GAME_CACHE = {} # Local cache dictionary to store game_id -> game_name mapping @@ -101,20 +102,20 @@ async def get_streamer_vods() -> list: except ValueError: clean_game_id = 0 - vod_data = { - "id": v.id, - "title": v.title, - "created_at": str(v.published_at), - "view_count": int(v.view_count) if v.view_count else 0, - "duration": v.duration, - "url": v.url, - "thumbnail_url": v.thumbnail_url, - "game_id": clean_game_id, - "game_name": game_name, - "stream_id": str(v.stream_id) if v.stream_id else "0", - "creator_name": config.CHANNEL_NAME, - "clip_is": False, - } + vod_data = TwitchVideo() + vod_data.id = v.id + vod_data.title = v.title + vod_data.created_at = str(v.published_at) + vod_data.view_count = int(v.view_count) if v.view_count else 0 + vod_data.duration = v.duration + vod_data.url = v.url + vod_data.thumbnail_url = v.thumbnail_url + vod_data.game_id = clean_game_id + vod_data.game_name = game_name + vod_data.stream_id = str(v.stream_id) if v.stream_id else "0" + vod_data.creator_name = config.CHANNEL_NAME + vod_data.clip_is = False + all_vods.append(vod_data) print(f"Collected VOD: {v.title} | Category: {game_name} ({v.duration})") @@ -131,20 +132,20 @@ async def get_streamer_clips() -> list: async for c in clip_generator: game_name = await get_game_name_by_id(c.game_id) - clip_data = { - "id": c.id, - "title": c.title, - "created_at": str(c.created_at), - "view_count": int(c.view_count) if c.view_count else 0, - "duration": c.duration, - "url": c.url, - "thumbnail_url": c.thumbnail_url, - "game_id": c.game_id, - "game_name": game_name, - "stream_id": "0", - "creator_name": c.creator_name, - "clip_is": True, - } + clip_data = TwitchVideo() + clip_data.id = c.id + clip_data.title = c.title + clip_data.created_at = str(c.created_at) + clip_data.view_count = int(c.view_count) if c.view_count else 0 + clip_data.duration = c.duration + clip_data.url = c.url + clip_data.thumbnail_url = c.thumbnail_url + clip_data.game_id = c.game_id + clip_data.game_name = game_name + clip_data.stream_id = "0" + clip_data.creator_name = c.creator_name + clip_data.clip_is = True + all_clips.append(clip_data) print(f"Collected clip: {c.title} | Category: {game_name} ({c.view_count} views)") @@ -157,26 +158,22 @@ async def video_info(): vods_list = await get_streamer_vods() for v in vods_list: - DB.insert_video_record( - v['id'], v['title'], v['created_at'], v['view_count'], v['duration'], - v['url'], v['thumbnail_url'], v['game_id'], v['game_name'], - v['stream_id'], v['creator_name'], v['clip_is'] - ) + DB.insert(v) if vods_list: - print(f"Recent VOD: '{vods_list[0]['title']}'") + print("====================================================") + print(f"Recent VOD: '{vods_list[0].title}'") + print("====================================================") clips_list = await get_streamer_clips() for v in clips_list: - DB.insert_video_record( - v['id'], v['title'], v['created_at'], v['view_count'], v['duration'], - v['url'], v['thumbnail_url'], v['game_id'], v['game_name'], - v['stream_id'], v['creator_name'], v['clip_is'] - ) - + DB.insert(v) + if clips_list: - print(f"Recent VOD: '{clips_list[0]['title']}'") + print("====================================================") + print(f"Recent VOD: '{clips_list[0].title}'") + print("====================================================") if __name__ == "__main__": asyncio.run(video_info()) diff --git a/requirements.txt b/requirements.txt index 1695f73..f82bd62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,7 @@ google-auth==2.58.0 google-auth-httplib2==0.4.2 google-auth-oauthlib==1.4.1 googleapis-common-protos==1.75.3 +greenlet==3.5.6 h11==0.16.0 hf-xet==1.6.0 httpcore==1.0.9 @@ -61,6 +62,7 @@ regex==2026.9.10 requests==2.34.2 requests-oauthlib==2.0.0 six==1.17.0 +SQLAlchemy==2.0.54 tokenizers==0.23.2 tqdm==4.70.1 twitchAPI==4.5.0