diff --git a/.gitignore b/.gitignore index 82af27f..a5bceec 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ twitch_secrets.json download output.log *.mp4 +database copy.db diff --git a/delete/twitch_down_uploader.py b/delete/twitch_down_uploader.py index 8fd9bbb..e8cc2ea 100755 --- a/delete/twitch_down_uploader.py +++ b/delete/twitch_down_uploader.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import requests import os -import package.linux as linux +import package.utils as linux import time from package.database import Database @@ -9,7 +9,7 @@ from package.database import Database import package.youtube.uploader as uploader from package.youtube.uploader import CategoryId -import package.youtube.hashtags as hashtags +import package.hash_tags as hash_tags import package.transcribe_video as transcribe_video CHANNEL_NAME = 'teampgp' # Replace with the streamer's username @@ -43,7 +43,7 @@ def top_hashtags(id: str): print(f"Transcribe file not found {file_srt}") return [] - tags = hashtags.get_top_hashtags(file_srt) + tags = hash_tags.get_top_hashtags(file_srt) return tags def download(): diff --git a/delete/twitch_download_clips.py b/delete/twitch_download_clips.py index ec8a0fa..662f557 100644 --- a/delete/twitch_download_clips.py +++ b/delete/twitch_download_clips.py @@ -35,7 +35,7 @@ def transcribe(slug: str): print(f"video already transcribed:") return True - import transcribe_video + import package.transcribe_video as transcribe_video transcribe_video.extract_audio(video_file, f"save/clips/{slug}/temp_{slug}_audio.wav") transcribe_video.transcribe_to_srt(f"save/clips/{slug}/temp_{slug}_audio.wav", f"save/clips/{slug}/", f"transcribe_{slug}") @@ -49,9 +49,9 @@ def top_hashtags(slug: str): print(f"Transcribe file not found {file_srt}") return [] - import package.youtube.hashtags as hashtags + import package.hash_tags as hash_tags - return hashtags.get_top_hashtags(file_srt) + return hash_tags.get_top_hashtags(file_srt) def download_clips(): diff --git a/helper_files/remove_uneed_files.py b/helper_files/remove_uneed_files.py index 02a4f6a..139a070 100644 --- a/helper_files/remove_uneed_files.py +++ b/helper_files/remove_uneed_files.py @@ -1,14 +1,17 @@ import os +import sys -import package.database as database +# Finds the absolute path to 'project_directory' and adds it to Python's search path +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) +sys.path.append(PROJECT_ROOT) + +# Your original import will now work +from package.config import DB as DB -DB = None def remove_files(): datas = DB.get_download() - top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp" - for data in datas: ( id, @@ -30,19 +33,20 @@ def remove_files(): ) = data if clip_is == 0: + continue video_path = f"download/videos/{id}/{id}.mp4" else: video_path = f"download/clips/{id}/{id}.mp4" - temp_chat = video_path.replace(".mp4", "_temp_chat.mp4") - temp_with_chat = video_path.replace(".mp4", "_temp_with_chat.mp4") - video_chat = video_path.replace(".mp4", "_with_chat.mp4") - video_chatSS = video_path.replace(".mp4", "_SS_with_chat.mp4") - video_chat_over = video_path.replace(".mp4", "_over_with_chat.mp4") - mask_path = video_path.replace(".mp4", "_temp_chat_mask.mp4") + temp_chat = ""#video_path.replace(".mp4", "_temp_chat.mp4") + temp_with_chat = ""#video_path.replace(".mp4", "_temp_with_chat.mp4") + video_chat = ""#video_path.replace(".mp4", "_with_chat.mp4") + video_chatSS = ""#video_path.replace(".mp4", "_SS_with_chat.mp4") + video_chat_over = ""#video_path.replace(".mp4", "_over_with_chat.mp4") + mask_path = ""#video_path.replace(".mp4", "_temp_chat_mask.mp4") - shorts_gaussian_9_16 = ""#video_path.replace(".mp4", "_gaussian_9_16.mp4") - shorts_9_16 = ""#video_path.replace(".mp4", "_9_16.mp4") + shorts_gaussian_9_16 = video_path.replace(".mp4", "_gaussian_9_16.mp4") + shorts_9_16 = video_path.replace(".mp4", "_9_16.mp4") srt_file = ""#video_path.replace(".mp4", ".srt") @@ -62,11 +66,9 @@ def redownload(): def main(): - global DB - DB = database.Database() - remove_files() - redownload() + print("done") + #redownload() if __name__ == "__main__": main() \ No newline at end of file diff --git a/main.py b/main.py index 7c46cda..0e826dd 100644 --- a/main.py +++ b/main.py @@ -6,18 +6,24 @@ import package.twitch.download_videos as download_videos import package.twitch.download_thumbnails as download_thumbnails import package.build_videos as build_videos +import package.process_uploads as process_uploads if __name__ == "__main__": - # Get Twitch Video Info - asyncio.run(video_info()) + upload = False - # Download Twitch Videos - download_videos() + if upload is False: + # Get Twitch Video Info + asyncio.run(video_info()) - download_thumbnails() + # Download Twitch Videos + download_videos() - #Build - build_videos() + download_thumbnails() + + #Build + build_videos() + else: + process_uploads() print("====================================================") print(f"āœ… Done.") diff --git a/package/__init__.py b/package/__init__.py index 11e3d4d..1cb2d20 100644 --- a/package/__init__.py +++ b/package/__init__.py @@ -14,22 +14,32 @@ os.makedirs(CACHE_DIR, exist_ok=True) # 3. Expose Database Tools (Added) # Replace 'DatabaseClient' and 'connect_to_db' with your actual class/function names +from .build_shorts import fit_to_9_16_letterbox from .build_videos import build_videos -from .config import CHANNEL_NAME, DB +from .config import CHANNEL_NAME, TZ, DB from .database import Database +from .hash_tags import get_top_nouns +from .process_uploads import process_uploads from .transcribe_video import transcribe_to_srt -from .utils import run_command +from .utils import write_csv, run_command # 4. Expose Subpackages from . import twitch +from . import youtube # (Optional) Defines what is exposed during 'from package import *' __all__ = [ + "fit_to_9_16_letterbox" "build_videos", "CHANNEL_NAME", + "TZ", "DB", "Database", + "get_top_nouns", + "process_uploads", "transcribe_to_srt", - "utils", + "write_csv", + "run_command", "twitch", + "youtube" ] diff --git a/package/youtube/shorts.py b/package/build_shorts.py similarity index 97% rename from package/youtube/shorts.py rename to package/build_shorts.py index d690120..b9f8192 100755 --- a/package/youtube/shorts.py +++ b/package/build_shorts.py @@ -4,12 +4,9 @@ import json import package.utils as utils import tempfile from pathlib import Path -from moviepy import VideoFileClip, ColorClip, CompositeVideoClip from moviepy.video.VideoClip import TextClip from pathlib import Path -import numpy as np -from PIL import Image, ImageFilter def get_video_info(input_path: Path) -> tuple: """Uses ffprobe to instantly read input video dimensions and frame rate.""" diff --git a/package/build_videos.py b/package/build_videos.py index 4b8b48e..70885a6 100644 --- a/package/build_videos.py +++ b/package/build_videos.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 -import package.youtube.shorts as yt_shorts +from datetime import datetime + +import package.build_shorts as yt_shorts import package.transcribe_video as transcribe_video import package.twitch.chat_vod as chat_vod from package.config import DB as DB +from package.config import TZ as TZ def build_chat_video(): chats = DB.get_unuploaded_chats() @@ -81,6 +84,7 @@ def build_shorts(): shorts = DB.get_unuploaded_shorts() top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp" + count = 1 for short in shorts: ( @@ -107,8 +111,14 @@ def build_shorts(): print("====================================================") print(f"šŸš€ Processing: Clip to Youtube Short {title}") print("====================================================") + print(f"{count}/{len(shorts)}") - yt_shorts.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {creator_name}.") + created_at = datetime.strptime(created_at, "%Y-%m-%d %H:%M:%S%z") + created_at = created_at.astimezone(TZ) + + #ToDo + yt_shorts.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {creator_name},\n on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z")}", True, True) + count += 1 print("====================================================") print(f"āœ… Processing: Clip to Youtube Short Done.") diff --git a/package/config.py b/package/config.py index 12214d3..4423c0d 100644 --- a/package/config.py +++ b/package/config.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 +from zoneinfo import ZoneInfo + import package.database as database CHANNEL_NAME = "teampgp" +TZ = ZoneInfo("America/Toronto") DB = database.Database() \ No newline at end of file diff --git a/package/database.py b/package/database.py index fb7c308..073f85c 100755 --- a/package/database.py +++ b/package/database.py @@ -7,6 +7,7 @@ from typing import Any class Database: def __init__(self, db_path: str = "database.db"): self.columns = "id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts" + self.data = "id, name, date_shorts, date_videos, date_videos_chat" file_exists = Path(db_path).is_file() @@ -45,6 +46,16 @@ class Database: uploaded_yt_shorts BOOLEAN NOT NULL DEFAULT 0 ) """) + + self.cursor.execute(""" + CREATE TABLE IF NOT EXISTS data ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + date_shorts TEXT NOT NULL, + date_videos TEXT NOT NULL, + date_videos_chat TEXT NOT NULL + ) + """) self.conn.commit() @@ -167,3 +178,55 @@ class Database: # Prevent silent failures if the database connection drops print(f"āŒ Database insertion failed: {e}") self.conn.rollback() + + def __update_date(self, record_id: str, set_row: str, date: str): + """Updates the Datetime for timezone fix""" + self.cursor.execute( + f"UPDATE data SET {set_row} = ? WHERE id = ?", + (date, record_id)) + self.conn.commit() + + def update_date_shorts(self, record_id: str, date_shorts: str): + self.__update_date(record_id, "date_shorts", date_shorts) + + def get_data(self) -> list[Any]: + self.cursor.execute(f"SELECT {self.data} FROM data") + return self.cursor.fetchall() + + def insert_data_record(self, name: str, date_shorts: str, date_videos: str, date_videos_chat: str): + try: + dt_obj = datetime.strptime(date_shorts, "%Y-%m-%dT%H:%M:%SZ") + clean_date_shorts = dt_obj.strftime("%Y-%m-%d %H:%M:%S") + except (ValueError, TypeError): + clean_date_shorts = date_shorts + + try: + dt_obj = datetime.strptime(date_videos, "%Y-%m-%dT%H:%M:%SZ") + clean_date_videos = dt_obj.strftime("%Y-%m-%d %H:%M:%S") + except (ValueError, TypeError): + clean_date_videos = date_videos + + try: + dt_obj = datetime.strptime(date_videos_chat, "%Y-%m-%dT%H:%M:%SZ") + clean_date_videos_chat = dt_obj.strftime("%Y-%m-%d %H:%M:%S") + except (ValueError, TypeError): + clean_date_videos_chat = date_videos_chat + + # Explicitly defining columns removes the security risk and column-count bug + query = """ + INSERT INTO data ( + name, date_shorts, date_videos, date_videos_chat + ) VALUES (?, ?, ?, ?) + """ + + values = ( + name, clean_date_shorts, clean_date_videos, clean_date_videos_chat + ) + + try: + self.cursor.execute(query, values) + self.conn.commit() + except Exception as e: + # Prevent silent failures if the database connection drops + print(f"āŒ Database insertion failed: {e}") + self.conn.rollback() \ No newline at end of file diff --git a/package/youtube/hashtags.py b/package/hash_tags.py similarity index 100% rename from package/youtube/hashtags.py rename to package/hash_tags.py diff --git a/package/process_uploads.py b/package/process_uploads.py new file mode 100644 index 0000000..1754621 --- /dev/null +++ b/package/process_uploads.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +import os + +from datetime import datetime, timedelta + +import package.utils as utils + +from package.config import TZ as TZ +from package.config import DB as DB +from package.hash_tags import get_top_nouns as get_top_nouns +from package.youtube.uploader import upload_video as upload_video +from package.youtube.uploader import CategoryId as CategoryId + +def process_shorts(): + shorts = DB.get_unuploaded_shorts() + data = DB.get_data() + ( data_id, name, release_time, _, _) = data[0] + + current_time = datetime.now(TZ) + release_time = datetime.strptime(release_time, "%Y-%m-%d %H:%M:%S%z") + + if current_time > release_time: + pass + else: + release_time = current_time + release_time = release_time.replace(hour=12, minute=0, second=0, microsecond=0) + + for short in shorts: + ( + id, + title, + created_at, + view_count, + duration, + url, + thumbnail_url, + game_id, + game_name, + stream_id, + creator_name, + clip_is, + downloaded, + uploaded_yt, + uploaded_yt_chats, + uploaded_yt_shorts, + ) = short + + original_path = f"download/clips/{id}/{id}.mp4" + file_path = f"download/clips/{id}/{id}_gaussian_9_16.mp4" + file_csv = f"download/clips/{id}/{id}_{name}.csv" + if not os.path.exists(file_path): + file_path = f"download/clips/{id}/{id}_black_9_16.mp4" + if not os.path.exists(file_path): + continue + + title = title + created_at = datetime.strptime(created_at, "%Y-%m-%d %H:%M:%S%z") + created_at = created_at.astimezone(TZ) + categoryId = CategoryId.GAMING + description = f"Game: {game_name}, Clipped By: {creator_name}, on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, #Shorts #Clips #Twitch Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp" + privatcyStatus = 'private' + tags = ['shorts', 'gaming', 'TeamPGP', f'{game_name}', f'{creator_name}', 'twitch_clips', 'clips', 'Level1Techs', 'twitch'] + + tags.extend(get_top_nouns(f"download/clips/{id}/{id}.srt")) + + data_csv = [] + + if os.path.exists(original_path): + if video_id := upload_video(original_path, title, categoryId, description, 'unlisted', tags): + data_csv.extend(video_id, current_time) + release_time += timedelta(days=1) + if video_id := upload_video(file_path, title, categoryId, f"{description} Original Full size @ https://youtu.be/{video_id}", privatcyStatus, tags, release_time): + data_csv.extend(video_id, release_time) + DB.mark_as_uploaded_shorts(id) + utils.write_csv(data_csv, file_csv) + + DB.update_date_shorts(data_id, release_time) + +def process_uploads(): + process_shorts() + +if __name__ == "__main__": + pass \ No newline at end of file diff --git a/package/transcribe_video.py b/package/transcribe_video.py index 829acd2..858d149 100755 --- a/package/transcribe_video.py +++ b/package/transcribe_video.py @@ -12,17 +12,6 @@ os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" # 5 minutes per chunk (300 seconds) keeps RAM usage low and stable CHUNK_DURATION_SEC = 300 -print("šŸ”§ Initializing C++ Engine...") -model = WhisperModel( - "base", - device="cpu", - compute_type="int8", - cpu_threads=0, # Let CTranslate2 auto-detect safe core counts - num_workers=1 -) -print("āœ… C++ Model loaded successfully.") - - def extract_audio_and_chunk(video_path: str, output_dir: Path) -> list: """Extracts and splits audio into 5-minute chunks using a single FFmpeg pass.""" print("šŸš€ Extracting and chunking audio with FFmpeg...") @@ -49,6 +38,18 @@ def extract_audio_and_chunk(video_path: str, output_dir: Path) -> list: def transcribe_to_srt(video_path: str, force: bool = False): + if not hasattr(transcribe_to_srt, "model"): + print("šŸ”§ Initializing C++ Engine...") + transcribe_to_srt.model = WhisperModel( + "base", + device="cpu", + compute_type="int8", + cpu_threads=0, # Let CTranslate2 auto-detect safe core counts + num_workers=1 + ) + print("āœ… C++ Model loaded successfully.") + + video_path_obj = Path(video_path) srt_path = video_path_obj.with_suffix(".srt") temp_dir = video_path_obj.parent / f"temp_chunks_{video_path_obj.stem}" @@ -78,7 +79,7 @@ def transcribe_to_srt(video_path: str, force: bool = False): time_offset = chunk_idx * CHUNK_DURATION_SEC print(f"\nā³ Processing chunk {chunk_idx + 1}/{len(audio_chunks)} ({chunk_path.name})...") - segments_generator, info = model.transcribe( + segments_generator, info = transcribe_to_srt.model.transcribe( str(chunk_path), beam_size=1, vad_filter=True, diff --git a/package/twitch/download_videos.py b/package/twitch/download_videos.py old mode 100755 new mode 100644 index 4b57cfc..0682f36 --- a/package/twitch/download_videos.py +++ b/package/twitch/download_videos.py @@ -1,19 +1,10 @@ #!/usr/bin/env python3 -import csv import subprocess from datetime import datetime, timedelta, timezone import package.utils as utils from package.config import DB as DB -def write_csv(data, file_name): - # Open file with newline='' to prevent extra blank rows across platforms - with open(file_name, "w", newline="", encoding="utf-8") as file: - writer = csv.writer(file) - - # Write all rows at once - writer.writerows(data) - def download_videos(): """Find all undownloaded videos and clips, and download them safely.""" undownloaded = DB.get_undownloaded() @@ -80,7 +71,7 @@ def download_videos(): if output: # Only write CSV and update database if download actually completed - write_csv(data, csv_file) + utils.write_csv(data, csv_file) print(f"āœ… Success: TwitchDownloaderCLI video. {target_dir}") DB.mark_as_downloaded(id) else: diff --git a/package/utils.py b/package/utils.py index 38869e6..cb34689 100755 --- a/package/utils.py +++ b/package/utils.py @@ -1,8 +1,17 @@ #!/usr/bin/env python3 +import csv import sys import shlex import subprocess +def write_csv(data, file_name): + # Open file with newline='' to prevent extra blank rows across platforms + with open(file_name, "w", newline="", encoding="utf-8") as file: + writer = csv.writer(file) + + # Write all rows at once + writer.writerows(data) + def run_command(cmd_str: str, progress_prefix: str = "Progress", look_for: list = ["frame=", "time=", "fps=", "Rendering frame"]) -> bool: """Runs a system command, streams its output live, and reports errors on failure.""" args = shlex.split(cmd_str) diff --git a/package/youtube/__init__.py b/package/youtube/__init__.py index 1b8c2e9..5749796 100644 --- a/package/youtube/__init__.py +++ b/package/youtube/__init__.py @@ -2,14 +2,11 @@ # Expose components from individual modules using relative imports from .find_tags import find_tags -from .hashtags import get_top_nouns -from .shorts import fit_to_9_16_letterbox from .uploader import upload_video, CategoryId # Explicitly define what is exposed when importing the subpackage __all__ = [ "find_tags", "get_top_nouns", - "fit_to_9_16_letterbox", "upload_video", ] diff --git a/package/youtube/uploader.py b/package/youtube/uploader.py index fd03ab0..174861c 100755 --- a/package/youtube/uploader.py +++ b/package/youtube/uploader.py @@ -2,7 +2,7 @@ import os import argparse from enum import Enum -from datetime import datetime +from datetime import datetime, timezone from googleapiclient.discovery import build from googleapiclient.http import MediaFileUpload @@ -29,7 +29,7 @@ class CategoryId(Enum): SCIENCE_AND_TECHNOLOGY = "28" NONPROFITS_AND_ACTIVISM = "29" -def upload_video(file_path: str, title: str, category: CategoryId, description: str = "", privacyStatus: str = 'private', tags: list = None, release_time: datetime = None) -> bool: +def upload_video(file_path: str, title: str, category: CategoryId, description: str = "", privacyStatus: str = 'private', tags: list = None, release_time: datetime = None) -> str | None: """ Upload a video to YouTube @@ -51,11 +51,12 @@ def upload_video(file_path: str, title: str, category: CategoryId, description: # Check if file exists if not os.path.exists(file_path): print(f"āŒ Error: File not found: {file_path}") - return False + return None # Load credentials if not config.CREDENTIALS: - return False + print("āŒ Error: YouTube API credentials not configured.") + return None # Create YouTube API client youtube = build('youtube', 'v3', credentials=config.CREDENTIALS) @@ -71,7 +72,8 @@ def upload_video(file_path: str, title: str, category: CategoryId, description: # If release_time is passed, YouTube forces privacyStatus to 'private' status_body['privacyStatus'] = 'private' - status_body['publishAt'] = release_time.strftime('%Y-%m-%dT%H:%M:%S.000Z') + #status_body['publishAt'] = release_time.strftime('%Y-%m-%dT%H:%M:%S.000Z') + status_body['publishAt'] = release_time.isoformat() print(f"Configuring timed release for: {status_body['publishAt']}") else: # Standard immediate upload @@ -121,14 +123,14 @@ def upload_video(file_path: str, title: str, category: CategoryId, description: else: print(f"Current Privacy Status: {response['status']['privacyStatus']}") - return True + return response['id'] except HttpError as e: print(f"An HTTP error occurred: {str(e)}") - return False + return None except Exception as e: print(f"An error occurred: {str(e)}") - return False + return None def main(): parser = argparse.ArgumentParser(description='Upload a video to YouTube')