diff --git a/.gitignore b/.gitignore index 2490866..48f3717 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ /client_secrets.json /clips_database.db __pycache__/uploader.cpython-313.pyc +database.db +__pycache__/database.cpython-313.pyc diff --git a/database.py b/database.py index 0cd16da..0d85fa8 100644 --- a/database.py +++ b/database.py @@ -1,19 +1,33 @@ import sqlite3 +from datetime import datetime +from datetime import date as datetime_date +from pathlib import Path -# Stores data locally -CONN = sqlite3.connect("database.db") -CURSOR = CONN.cursor() +class Database: + def __init__(self, table): + self.table = table + self.columns = "" -class Database(): - def __init__(): - file_path = Path("database.db") + # IMPORTANT Must check if database.db exists before connecting to it. + file_exists = Path("database.db").is_file() - if file_path.is_file(): + self.CONN = sqlite3.connect("database.db") + self.CURSOR = self.CONN.cursor() + + if self.table == "vods": + self.columns = "id, date, title, gamename, downloaded, uploaded_yt, chat_upload_yt" + elif self.table == "clips": + self.columns = "slug, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt" + + if not file_exists: self.create_database() - def create_database(): + def __del__(self): + self.close_database() + + def create_database(self): """Creates a table structured explicitly for Twitch clip properties.""" - CURSOR.execute( + self.CURSOR.execute( """ CREATE TABLE IF NOT EXISTS clips ( slug TEXT PRIMARY KEY, @@ -28,7 +42,8 @@ class Database(): """ ) - CURSOR.execute( + """Creates a table structured explicitly for Twitch vods properties.""" + self.CURSOR.execute( """ CREATE TABLE IF NOT EXISTS vods ( id INTEGER PRIMARY KEY, @@ -42,9 +57,60 @@ class Database(): """ ) - CONN.commit() + self.CONN.commit() - def close_database(): + def close_database(self): """Commit before we close.""" - CONN.commit() - CONN.close() \ No newline at end of file + self.CONN.commit() + self.CONN.close() + + def mark_as_downloaded(self, record_id: int): + """Updates the downloaded status to True (1) for a specific record ID.""" + + # Updates the row matching the specific ID + # TODO clips uses slug + self.CURSOR.execute( + f"UPDATE {self.table} SET downloaded = 1 WHERE id = ?", + (record_id,) + ) + self.CONN.commit() + + def get_unuploaded(self): + """Retrieves all clip rows remaining to be chat uploaded.""" + self.CURSOR.execute(f"SELECT {self.columns} FROM {self.table} WHERE downloaded = 1 AND uploaded_yt = 0") + return self.CURSOR.fetchall() + + def get_undownloaded(self): + """Retrieves all rows where downloaded status is False (0).""" + self.CURSOR.execute(f"SELECT {self.columns} FROM {self.table} WHERE downloaded = 0") + return self.CURSOR.fetchall() + + def insert_vods_record(self, record_id: int, record_date_str: datetime_date, title: str, gamename: str): + """Inserts a record with ID, date, gamename, and title into a SQLite database.""" + try: + clean_date = datetime.strptime(record_date_str, "%Y-%m-%dT%H:%M:%SZ").date() + except ValueError: + clean_date = record_date_str + # Connects to database file (creates it if missing) + + # Inserts data using parameterized queries to prevent SQL injection + self.CURSOR.execute( + f"INSERT OR IGNORE INTO {self.table} ({self.columns}) VALUES (?, ?, ?, ?, ?, ?, ?)", + (record_id, str(clean_date), title, gamename, False, False, False), + ) + + # Saves changes and closes the connection + self.CONN.commit() + + def insert_clips_record(slug: str, record_date_str: str, title: str, gamename: str, clip_by: str, views: int): + """Cleans up ISO-8601 strings into unified date structures for the database.""" + try: + clean_date = datetime.strptime(record_date_str, "%Y-%m-%dT%H:%M:%SZ").date() + except ValueError: + clean_date = record_date_str + + self.CURSOR.execute( + f"INSERT OR IGNORE INTO {self.table} ({self.columns}) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (slug, str(clean_date), title, gamename, clip_by, views, False, False), + ) + self.CONN.commit() \ No newline at end of file diff --git a/twitch_download_clips.py b/twitch_download_clips.py index 2899d54..278be71 100644 --- a/twitch_download_clips.py +++ b/twitch_download_clips.py @@ -9,33 +9,6 @@ from uploader import CategoryId CHANNEL_NAME = 'teampgp' # Replace with the streamer's username -# Stores data locally -CONN = sqlite3.connect("clips_database.db") -CURSOR = CONN.cursor() - -def create_database(): - """Creates a table structured explicitly for Twitch clip properties.""" - CURSOR.execute( - """ - CREATE TABLE IF NOT EXISTS clips ( - slug TEXT PRIMARY KEY, - date TEXT NOT NULL, - title TEXT NOT NULL, - gamename TEXT NOT NULL, - clip_by TEXT NOT NULL, - view_count INTEGER NOT NULL, - downloaded INTEGER NOT NULL, - uploaded_yt INTEGER NOT NULL - ) - """ - ) - CONN.commit() - -def close_database(): - """Commits queries before ending connection context.""" - CONN.commit() - CONN.close() - def download_clips(): """Loops over undownloaded metadata entries to write files down locally.""" undownloaded_clips = get_undownloaded_clips() @@ -109,13 +82,6 @@ def mark_as_downloaded(slug: str): ) CONN.commit() -def get_unuploaded_clips(): - """Retrieves all clip rows remaining to be uploaded.""" - CURSOR.execute( - "SELECT slug, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt FROM clips WHERE downloaded = 1 AND uploaded_yt = 0" - ) - return CURSOR.fetchall() - def get_undownloaded_clips(): """Retrieves all clip rows remaining to be captured.""" CURSOR.execute( diff --git a/twitch_download_vod.py b/twitch_download_vod.py index 10a4727..1b8fa0a 100644 --- a/twitch_download_vod.py +++ b/twitch_download_vod.py @@ -2,55 +2,15 @@ import requests import sqlite3 -from datetime import date as datetime_date - import subprocess +from database import Database + +import uploader +from uploader import CategoryId + CHANNEL_NAME = 'teampgp' # Replace with the streamer's username - -CONN = sqlite3.connect("database.db") -CURSOR = CONN.cursor() - -def create_database(): - # Creates table safely using multi-line string - CURSOR.execute( - """ - CREATE TABLE IF NOT EXISTS vods ( - id INTEGER PRIMARY KEY, - date TEXT NOT NULL, - title TEXT NOT NULL, - gamename TEXT NOT NULL, - downloaded INTEGER NOT NULL, - uploaded_yt INTEGER NOT NULL, - chat_upload_yt INTEGER NOT NULL - ) - """ - ) - - CONN.commit() - -def close_database(): - """Commit before we close.""" - CONN.commit() - CONN.close() - -def download_vods(): - """Find all undownload vods and download them.""" - undownload_vods = get_undownloaded_vods() - - print("Download VODs...") - - for record_id, record_date, title, game_name, downloaded in undownload_vods: - print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}") - output = run_linux_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/vod/{record_id}/{record_id}.mp4") - output2 = run_linux_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/vod/{record_id}/{record_id}_chat.json -E") - if output["success"] is True: - print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was successful") - mark_as_downloaded(record_id) - else: - print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was Failed") - - print("Finished Downloading VODs...") +DB = Database("vods") def run_linux_command(command: str): """Executes a Linux command, waits for completion, and returns output.""" @@ -67,52 +27,50 @@ def run_linux_command(command: str): # Handles errors if the Linux command returns a non-zero exit code return {"success": False, "stdout": e.stdout, "stderr": e.stderr} -def mark_as_downloaded(record_id: int): - """Updates the downloaded status to True (1) for a specific record ID.""" - - # Updates the row matching the specific ID - CURSOR.execute( - "UPDATE vods SET downloaded = 1 WHERE id = ?", - (record_id,) - ) +def download_vods(): + """Find all undownload vods and download them.""" + undownloaded_vods = DB.get_undownloaded() - CONN.commit() + print("Download VODs...") -def get_unuploaded_clips(): - """Retrieves all clip rows remaining to be chat uploaded.""" - CURSOR.execute( - "SELECT id, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt, chat_uploaded_yt FROM vods WHERE downloaded = 1 AND chat_uploaded_yt = 0" - ) - return CURSOR.fetchall() + for record_id, record_date, title, game_name, downloaded, uploaded_yt, chat_upload_yt in undownloaded_vods: + print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}") + output = run_linux_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/vods/{record_id}/{title}.mp4") + output2 = run_linux_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/vods/{record_id}/{title}_chat.json -E") + if output["success"] is True: + print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was successful") + DB.mark_as_downloaded(record_id) + else: + print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was Failed") -def get_unuploaded_clips(): - """Retrieves all clip rows remaining to be uploaded.""" - CURSOR.execute( - "SELECT id, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt FROM vods WHERE downloaded = 1 AND uploaded_yt = 0" - ) - return CURSOR.fetchall() + print("Finished Downloading VODs...") -def get_undownloaded_vods(): - """Retrieves all rows where downloaded status is False (0).""" - - # Query filters by 0 because SQLite stores booleans as integers - CURSOR.execute( - "SELECT id, date, title, gamename, downloaded FROM vods WHERE downloaded = 0" - ) - return CURSOR.fetchall() +def upload_vods(): + """Loops over the downloaded videos entries and uploaded them to youtube.""" + print("Uploading Clips...") -def insert_record(record_id: int, record_date: datetime_date, title: str, gamename: str): - """Inserts a record with ID, date, gamename, and title into a SQLite database.""" - # Connects to database file (creates it if missing) + unuploaded_vods = DB.get_unuploaded() - # Inserts data using parameterized queries to prevent SQL injection - CURSOR.execute( - "INSERT OR IGNORE INTO vods (id, date, title, gamename, downloaded, uploaded_yt, chat_uploaded_yt) VALUES (?, ?, ?, ?, ?, ?, ?)", - (record_id, str(record_date), title, gamename, False, False, False), - ) + for record_id, record_date, title, game_name, downloaded, uploaded_yt, chat_upload_yt in unuploaded_vods: + print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}") - # Saves changes and closes the connection - CONN.commit() + file_path = f"save/vods/{record_id}/{title}.mp4" + title = title + description = f"Game: {game_name}, on {record_date}, #Shorts" + categoryId = CategoryId.GAMING + privatcyStatus = 'private' + tags = ['gaming', 'TeamPGP', f'{game_name}'] + + output = uploader.upload_video(file_path, title, description, categoryId, privatcyStatus, tags) + + if output is True: + print(f"ID: {record_id} | Was successfully uploaded.") + DB.mark_as_uploaded(record_id) + else: + print(f"ID: {record_id} | Upload Process failed.") + +def create_chat_vods(): + pass def get_vod_ids_simplified(channel_name: str): session = requests.Session() @@ -190,7 +148,8 @@ def get_vod_ids_simplified(channel_name: str): print(f"ID: {node['id']} | Date: {node['publishedAt']} | Game: {game_name} | Title: {node['title']}") # Pass game_name to your database logic - insert_record(node['id'], node['publishedAt'], node['title'], game_name) + DB.insert_vods_record(node['id'], node['publishedAt'], node['title'], game_name) + #insert_record(node['id'], node['publishedAt'], node['title'], game_name) vod_ids.append(node['id']) return vod_ids @@ -202,7 +161,7 @@ def get_vod_ids_simplified(channel_name: str): return [] if __name__ == "__main__": - create_database() get_vod_ids_simplified(CHANNEL_NAME) download_vods() - close_database() + upload_vods() + #close_database() \ No newline at end of file