update to the process_uploads upload.py and new database table called data

.gitignore

@@ -9,3 +9,4 @@ twitch_secrets.json
 download
 output.log
 *.mp4
+database copy.db

delete/twitch_down_uploader.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import requests
 import os
-import package.linux as linux
+import package.utils as linux
 import time
 
 from package.database import Database
@@ -9,7 +9,7 @@ from package.database import Database
 import package.youtube.uploader as uploader
 from package.youtube.uploader import CategoryId
 
-import package.youtube.hashtags as hashtags
+import package.hash_tags as hash_tags
 import package.transcribe_video as transcribe_video
 
 CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
@@ -43,7 +43,7 @@ def top_hashtags(id: str):
         print(f"Transcribe file not found {file_srt}")
         return []
 
-    tags = hashtags.get_top_hashtags(file_srt)
+    tags = hash_tags.get_top_hashtags(file_srt)
     return tags
 
 def download():

delete/twitch_download_clips.py

@@ -35,7 +35,7 @@ def transcribe(slug: str):
         print(f"video already transcribed:")
         return True
 
-    import transcribe_video
+    import package.transcribe_video as transcribe_video
     transcribe_video.extract_audio(video_file, f"save/clips/{slug}/temp_{slug}_audio.wav")
     transcribe_video.transcribe_to_srt(f"save/clips/{slug}/temp_{slug}_audio.wav", f"save/clips/{slug}/", f"transcribe_{slug}")
 
@@ -49,9 +49,9 @@ def top_hashtags(slug: str):
         print(f"Transcribe file not found {file_srt}")
         return []
 
-    import package.youtube.hashtags as hashtags
+    import package.hash_tags as hash_tags
 
-    return hashtags.get_top_hashtags(file_srt)
+    return hash_tags.get_top_hashtags(file_srt)
 
 
 def download_clips():

helper_files/remove_uneed_files.py

@@ -1,14 +1,17 @@
 import os
+import sys
 
-import package.database as database
+# Finds the absolute path to 'project_directory' and adds it to Python's search path
+PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../"))
+sys.path.append(PROJECT_ROOT)
+
+# Your original import will now work
+from package.config import DB as DB
 
-DB = None
 
 def remove_files():
     datas = DB.get_download()
 
-    top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
-
     for data in datas:
         (
             id,
@@ -30,19 +33,20 @@ def remove_files():
         ) = data
 
         if clip_is == 0:
+            continue
             video_path = f"download/videos/{id}/{id}.mp4"
         else:
             video_path = f"download/clips/{id}/{id}.mp4"
         
-        temp_chat = video_path.replace(".mp4", "_temp_chat.mp4")
+        temp_chat = ""#video_path.replace(".mp4", "_temp_chat.mp4")
-        temp_with_chat = video_path.replace(".mp4", "_temp_with_chat.mp4")
+        temp_with_chat = ""#video_path.replace(".mp4", "_temp_with_chat.mp4")
-        video_chat = video_path.replace(".mp4", "_with_chat.mp4")
+        video_chat = ""#video_path.replace(".mp4", "_with_chat.mp4")
-        video_chatSS = video_path.replace(".mp4", "_SS_with_chat.mp4")
+        video_chatSS = ""#video_path.replace(".mp4", "_SS_with_chat.mp4")
-        video_chat_over = video_path.replace(".mp4", "_over_with_chat.mp4")
+        video_chat_over = ""#video_path.replace(".mp4", "_over_with_chat.mp4")
-        mask_path = video_path.replace(".mp4", "_temp_chat_mask.mp4")
+        mask_path = ""#video_path.replace(".mp4", "_temp_chat_mask.mp4")
 
-        shorts_gaussian_9_16 = ""#video_path.replace(".mp4", "_gaussian_9_16.mp4")
+        shorts_gaussian_9_16 = video_path.replace(".mp4", "_gaussian_9_16.mp4")
-        shorts_9_16 = ""#video_path.replace(".mp4", "_9_16.mp4")
+        shorts_9_16 = video_path.replace(".mp4", "_9_16.mp4")
 
         srt_file = ""#video_path.replace(".mp4", ".srt")
         
@@ -62,11 +66,9 @@ def redownload():
 
 
 def main():
-    global DB
-    DB = database.Database()
-
     remove_files()
-    redownload()
+    print("done")
+    #redownload()
 
 if __name__ == "__main__":
     main()

main.py

@@ -6,18 +6,24 @@ import package.twitch.download_videos as download_videos
 import package.twitch.download_thumbnails as download_thumbnails
 
 import package.build_videos as build_videos
+import package.process_uploads as process_uploads
 
 if __name__ == "__main__":
-    # Get Twitch Video Info
+    upload = False
-    asyncio.run(video_info())
 
-    # Download Twitch Videos
+    if upload is False:
-    download_videos()
+        # Get Twitch Video Info
+        asyncio.run(video_info())
 
-    download_thumbnails()
+        # Download Twitch Videos
+        download_videos()
 
-    #Build
+        download_thumbnails()
-    build_videos()
+
+        #Build
+        build_videos()
+    else:
+        process_uploads()
 
     print("====================================================")
     print(f"✅ Done.")

package/__init__.py

@@ -14,22 +14,32 @@ os.makedirs(CACHE_DIR, exist_ok=True)
 
 # 3. Expose Database Tools (Added)
 # Replace 'DatabaseClient' and 'connect_to_db' with your actual class/function names
+from .build_shorts import fit_to_9_16_letterbox
 from .build_videos import build_videos
-from .config import CHANNEL_NAME, DB
+from .config import CHANNEL_NAME, TZ, DB
 from .database import Database
+from .hash_tags import get_top_nouns
+from .process_uploads import process_uploads
 from .transcribe_video import transcribe_to_srt
-from .utils import run_command
+from .utils import write_csv, run_command
 
 # 4. Expose Subpackages
 from . import twitch
+from . import youtube
 
 # (Optional) Defines what is exposed during 'from package import *'
 __all__ = [
+    "fit_to_9_16_letterbox"
     "build_videos",
     "CHANNEL_NAME",
+    "TZ",
     "DB",
     "Database",
+    "get_top_nouns",
+    "process_uploads",
     "transcribe_to_srt", 
-    "utils", 
+    "write_csv",
+    "run_command",
     "twitch",
+    "youtube"
 ]

package/build_shorts.py

@@ -4,12 +4,9 @@ import json
 import package.utils as utils
 import tempfile
 from pathlib import Path
-from moviepy import VideoFileClip, ColorClip, CompositeVideoClip
 from moviepy.video.VideoClip import TextClip
 
 from pathlib import Path
-import numpy as np
-from PIL import Image, ImageFilter
 
 def get_video_info(input_path: Path) -> tuple:
     """Uses ffprobe to instantly read input video dimensions and frame rate."""

package/build_videos.py

@@ -1,8 +1,11 @@
 #!/usr/bin/env python3
-import package.youtube.shorts as yt_shorts
+from datetime import datetime
+
+import package.build_shorts as yt_shorts
 import package.transcribe_video as transcribe_video
 import package.twitch.chat_vod as chat_vod
 from package.config import DB as DB
+from package.config import TZ as TZ
 
 def build_chat_video():
     chats = DB.get_unuploaded_chats()
@@ -81,6 +84,7 @@ def build_shorts():
     shorts = DB.get_unuploaded_shorts()
 
     top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
+    count = 1
 
     for short in shorts:
         (
@@ -107,8 +111,14 @@ def build_shorts():
         print("====================================================")
         print(f"🚀 Processing: Clip to Youtube Short {title}")
         print("====================================================")
+        print(f"{count}/{len(shorts)}")
 
-        yt_shorts.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {creator_name}.")
+        created_at = datetime.strptime(created_at, "%Y-%m-%d %H:%M:%S%z")
+        created_at = created_at.astimezone(TZ)    
+
+        #ToDo
+        yt_shorts.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {creator_name},\n on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z")}", True, True)
+        count += 1
 
     print("====================================================")
     print(f"✅ Processing: Clip to Youtube Short Done.")

package/config.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python3
+from zoneinfo import ZoneInfo
+
 import package.database as database
 
 CHANNEL_NAME = "teampgp"
+TZ = ZoneInfo("America/Toronto")
 
 DB = database.Database()

package/database.py

@@ -7,6 +7,7 @@ from typing import Any
 class Database:
     def __init__(self, db_path: str = "database.db"):
         self.columns = "id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts"
+        self.data = "id, name, date_shorts, date_videos, date_videos_chat"
 
         file_exists = Path(db_path).is_file()
 
@@ -46,6 +47,16 @@ class Database:
         )
         """)
 
+        self.cursor.execute("""
+        CREATE TABLE IF NOT EXISTS data (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT NOT NULL,
+            date_shorts TEXT NOT NULL,
+            date_videos TEXT NOT NULL,
+            date_videos_chat TEXT NOT NULL
+        )
+        """)
+        
         self.conn.commit()
 
     def close_database(self):
@@ -167,3 +178,55 @@ class Database:
             # Prevent silent failures if the database connection drops
             print(f"❌ Database insertion failed: {e}")
             self.conn.rollback()
+
+    def __update_date(self, record_id: str, set_row: str, date: str):
+        """Updates the Datetime for timezone fix"""
+        self.cursor.execute(
+            f"UPDATE data SET {set_row} = ? WHERE id = ?",
+            (date, record_id))
+        self.conn.commit()
+
+    def update_date_shorts(self, record_id: str, date_shorts: str):
+        self.__update_date(record_id, "date_shorts", date_shorts)
+
+    def get_data(self) -> list[Any]:
+        self.cursor.execute(f"SELECT {self.data} FROM data")
+        return self.cursor.fetchall()
+
+    def insert_data_record(self, name: str, date_shorts: str, date_videos: str, date_videos_chat: str):
+        try:
+            dt_obj = datetime.strptime(date_shorts, "%Y-%m-%dT%H:%M:%SZ")
+            clean_date_shorts = dt_obj.strftime("%Y-%m-%d %H:%M:%S")
+        except (ValueError, TypeError):
+            clean_date_shorts = date_shorts
+
+        try:
+            dt_obj = datetime.strptime(date_videos, "%Y-%m-%dT%H:%M:%SZ")
+            clean_date_videos = dt_obj.strftime("%Y-%m-%d %H:%M:%S")
+        except (ValueError, TypeError):
+            clean_date_videos = date_videos
+
+        try:
+            dt_obj = datetime.strptime(date_videos_chat, "%Y-%m-%dT%H:%M:%SZ")
+            clean_date_videos_chat = dt_obj.strftime("%Y-%m-%d %H:%M:%S")
+        except (ValueError, TypeError):
+            clean_date_videos_chat = date_videos_chat
+
+        # Explicitly defining columns removes the security risk and column-count bug
+        query = """
+            INSERT INTO data (
+                name, date_shorts, date_videos, date_videos_chat
+            ) VALUES (?, ?, ?, ?)
+        """
+
+        values = (
+            name, clean_date_shorts, clean_date_videos, clean_date_videos_chat
+        )
+
+        try:
+            self.cursor.execute(query, values)
+            self.conn.commit()
+        except Exception as e:
+            # Prevent silent failures if the database connection drops
+            print(f"❌ Database insertion failed: {e}")
+            self.conn.rollback()

package/process_uploads.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+import os
+
+from datetime import datetime, timedelta
+
+import package.utils as utils
+
+from package.config import TZ as TZ
+from package.config import DB as DB
+from package.hash_tags import get_top_nouns as get_top_nouns
+from package.youtube.uploader import upload_video as upload_video
+from package.youtube.uploader import CategoryId as CategoryId
+
+def process_shorts():
+    shorts = DB.get_unuploaded_shorts()
+    data = DB.get_data()
+    ( data_id, name, release_time, _, _)  = data[0]
+
+    current_time = datetime.now(TZ)
+    release_time = datetime.strptime(release_time, "%Y-%m-%d %H:%M:%S%z")
+
+    if current_time > release_time:
+        pass
+    else:
+        release_time = current_time
+        release_time = release_time.replace(hour=12, minute=0, second=0, microsecond=0)
+
+    for short in shorts:
+        (
+            id,
+            title,
+            created_at,
+            view_count,
+            duration,
+            url,
+            thumbnail_url,
+            game_id,
+            game_name,
+            stream_id,
+            creator_name,
+            clip_is,
+            downloaded,
+            uploaded_yt,
+            uploaded_yt_chats,
+            uploaded_yt_shorts,
+        ) = short
+
+        original_path = f"download/clips/{id}/{id}.mp4"
+        file_path = f"download/clips/{id}/{id}_gaussian_9_16.mp4"
+        file_csv = f"download/clips/{id}/{id}_{name}.csv"
+        if not os.path.exists(file_path):
+            file_path = f"download/clips/{id}/{id}_black_9_16.mp4"
+            if not os.path.exists(file_path):
+                continue
+
+        title = title
+        created_at = datetime.strptime(created_at, "%Y-%m-%d %H:%M:%S%z")
+        created_at = created_at.astimezone(TZ)
+        categoryId = CategoryId.GAMING
+        description = f"Game: {game_name}, Clipped By: {creator_name}, on {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, #Shorts #Clips #Twitch Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp"
+        privatcyStatus = 'private'
+        tags = ['shorts', 'gaming', 'TeamPGP', f'{game_name}', f'{creator_name}', 'twitch_clips', 'clips', 'Level1Techs', 'twitch']
+        
+        tags.extend(get_top_nouns(f"download/clips/{id}/{id}.srt"))
+
+        data_csv = []
+
+        if os.path.exists(original_path):
+            if video_id := upload_video(original_path, title, categoryId, description, 'unlisted', tags):
+                data_csv.extend(video_id, current_time)
+                release_time += timedelta(days=1)       
+                if video_id := upload_video(file_path, title, categoryId, f"{description} Original Full size @ https://youtu.be/{video_id}", privatcyStatus, tags, release_time):
+                    data_csv.extend(video_id, release_time)
+                    DB.mark_as_uploaded_shorts(id)
+                    utils.write_csv(data_csv, file_csv)
+
+    DB.update_date_shorts(data_id, release_time)
+
+def process_uploads():
+    process_shorts()
+
+if __name__ == "__main__":
+    pass

package/transcribe_video.py

@@ -12,17 +12,6 @@ os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
 # 5 minutes per chunk (300 seconds) keeps RAM usage low and stable
 CHUNK_DURATION_SEC = 300 
 
-print("🔧 Initializing C++ Engine...")
-model = WhisperModel(
-    "base", 
-    device="cpu", 
-    compute_type="int8", 
-    cpu_threads=0, # Let CTranslate2 auto-detect safe core counts
-    num_workers=1         
-)
-print("✅ C++ Model loaded successfully.")
-
-
 def extract_audio_and_chunk(video_path: str, output_dir: Path) -> list:
     """Extracts and splits audio into 5-minute chunks using a single FFmpeg pass."""
     print("🚀 Extracting and chunking audio with FFmpeg...")
@@ -49,6 +38,18 @@ def extract_audio_and_chunk(video_path: str, output_dir: Path) -> list:
 
 
 def transcribe_to_srt(video_path: str, force: bool = False):
+    if not hasattr(transcribe_to_srt, "model"):
+        print("🔧 Initializing C++ Engine...")
+        transcribe_to_srt.model = WhisperModel(
+            "base", 
+            device="cpu", 
+            compute_type="int8", 
+            cpu_threads=0, # Let CTranslate2 auto-detect safe core counts
+            num_workers=1         
+        )
+        print("✅ C++ Model loaded successfully.")
+
+
     video_path_obj = Path(video_path)
     srt_path = video_path_obj.with_suffix(".srt")
     temp_dir = video_path_obj.parent / f"temp_chunks_{video_path_obj.stem}"
@@ -78,7 +79,7 @@ def transcribe_to_srt(video_path: str, force: bool = False):
                 time_offset = chunk_idx * CHUNK_DURATION_SEC
                 print(f"\n⏳ Processing chunk {chunk_idx + 1}/{len(audio_chunks)} ({chunk_path.name})...")
                 
-                segments_generator, info = model.transcribe(
+                segments_generator, info = transcribe_to_srt.model.transcribe(
                     str(chunk_path), 
                     beam_size=1,
                     vad_filter=True,

package/twitch/download_videos.py

@@ -1,19 +1,10 @@
 #!/usr/bin/env python3
-import csv
 import subprocess
 from datetime import datetime, timedelta, timezone
 
 import package.utils as utils
 from package.config import DB as DB
 
-def write_csv(data, file_name):
-    # Open file with newline='' to prevent extra blank rows across platforms
-    with open(file_name, "w", newline="", encoding="utf-8") as file:
-        writer = csv.writer(file)
-        
-        # Write all rows at once
-        writer.writerows(data)
-
 def download_videos():
     """Find all undownloaded videos and clips, and download them safely."""
     undownloaded = DB.get_undownloaded()
@@ -80,7 +71,7 @@ def download_videos():
 
         if output:
             # Only write CSV and update database if download actually completed
-            write_csv(data, csv_file)
+            utils.write_csv(data, csv_file)
             print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
             DB.mark_as_downloaded(id)
         else:

package/utils.py

@@ -1,8 +1,17 @@
 #!/usr/bin/env python3
+import csv
 import sys
 import shlex
 import subprocess
 
+def write_csv(data, file_name):
+    # Open file with newline='' to prevent extra blank rows across platforms
+    with open(file_name, "w", newline="", encoding="utf-8") as file:
+        writer = csv.writer(file)
+        
+        # Write all rows at once
+        writer.writerows(data)
+
 def run_command(cmd_str: str, progress_prefix: str = "Progress", look_for: list = ["frame=", "time=", "fps=", "Rendering frame"]) -> bool:
    """Runs a system command, streams its output live, and reports errors on failure."""
    args = shlex.split(cmd_str)

package/youtube/__init__.py

@@ -2,14 +2,11 @@
 
 # Expose components from individual modules using relative imports
 from .find_tags import find_tags
-from .hashtags import get_top_nouns
-from .shorts import fit_to_9_16_letterbox
 from .uploader import upload_video, CategoryId
 
 # Explicitly define what is exposed when importing the subpackage
 __all__ = [
     "find_tags",
     "get_top_nouns",
-    "fit_to_9_16_letterbox",
     "upload_video",
 ]

package/youtube/uploader.py

@@ -2,7 +2,7 @@
 import os
 import argparse
 from enum import Enum
-from datetime import datetime
+from datetime import datetime, timezone
 
 from googleapiclient.discovery import build
 from googleapiclient.http import MediaFileUpload
@@ -29,7 +29,7 @@ class CategoryId(Enum):
     SCIENCE_AND_TECHNOLOGY = "28"
     NONPROFITS_AND_ACTIVISM = "29"
 
-def upload_video(file_path: str, title: str, category: CategoryId, description: str = "", privacyStatus: str = 'private', tags: list = None, release_time: datetime = None) -> bool:
+def upload_video(file_path: str, title: str, category: CategoryId, description: str = "", privacyStatus: str = 'private', tags: list = None, release_time: datetime = None) -> str | None:
     """
     Upload a video to YouTube
     
@@ -51,11 +51,12 @@ def upload_video(file_path: str, title: str, category: CategoryId, description:
         # Check if file exists
         if not os.path.exists(file_path):
             print(f"❌ Error: File not found: {file_path}")
-            return False
+            return None
 
         # Load credentials
         if not config.CREDENTIALS:
-            return False
+            print("❌ Error: YouTube API credentials not configured.")
+            return None
 
         # Create YouTube API client
         youtube = build('youtube', 'v3', credentials=config.CREDENTIALS)
@@ -71,7 +72,8 @@ def upload_video(file_path: str, title: str, category: CategoryId, description:
 
             # If release_time is passed, YouTube forces privacyStatus to 'private'
             status_body['privacyStatus'] = 'private'
-            status_body['publishAt'] = release_time.strftime('%Y-%m-%dT%H:%M:%S.000Z')
+            #status_body['publishAt'] = release_time.strftime('%Y-%m-%dT%H:%M:%S.000Z')
+            status_body['publishAt'] = release_time.isoformat() 
             print(f"Configuring timed release for: {status_body['publishAt']}")
         else:
             # Standard immediate upload
@@ -121,14 +123,14 @@ def upload_video(file_path: str, title: str, category: CategoryId, description:
         else:
             print(f"Current Privacy Status: {response['status']['privacyStatus']}")
             
-        return True
+        return response['id']
 
     except HttpError as e:
         print(f"An HTTP error occurred: {str(e)}")
-        return False
+        return None
     except Exception as e:
         print(f"An error occurred: {str(e)}")
-        return False
+        return None
 
 def main():
     parser = argparse.ArgumentParser(description='Upload a video to YouTube')