Compare commits
30
Commits
f4c366f9aa
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7f1f1e7b4f | ||
|
|
33ea2b7fcf | ||
|
|
33d78759fe | ||
|
|
d6503b7d92 | ||
|
|
736fb47e7a | ||
|
|
93b480ae8d | ||
|
|
b6d9b56848 | ||
|
|
500fe87614 | ||
|
|
bbac53d15d | ||
|
|
9203b0f88c | ||
|
|
6882b7ae04 | ||
|
|
0f96eedc54 | ||
|
|
83f7bbead8 | ||
|
|
1257a15eb9 | ||
|
|
765eaf5026 | ||
|
|
4934ddb3f8 | ||
|
|
c8d12469e3 | ||
|
|
81e76b25d0 | ||
|
|
c8d645e9bd | ||
|
|
8e4a34dce1 | ||
|
|
19e525d64c | ||
|
|
ff15ded0c9 | ||
|
|
0505d2a581 | ||
|
|
483a771575 | ||
|
|
8be00d16d1 | ||
|
|
8d6e29a161 | ||
|
|
b573ca05aa | ||
|
|
228cf37eba | ||
|
|
26bb38c2aa | ||
|
|
efba28a7b5 |
@@ -1,3 +1,11 @@
|
|||||||
/secrets.json
|
/secrets.json
|
||||||
/client_secrets.json
|
/client_secrets.json
|
||||||
/clips_database.db
|
/clips_database.db
|
||||||
|
database.db
|
||||||
|
save
|
||||||
|
__pycache__
|
||||||
|
twitch_secrets.json
|
||||||
|
.vscode/settings.json
|
||||||
|
download
|
||||||
|
output.log
|
||||||
|
*.mp4
|
||||||
|
|||||||
Vendored
+15
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
// Use IntelliSense to learn about possible attributes.
|
||||||
|
// Hover to view descriptions of existing attributes.
|
||||||
|
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||||
|
"version": "0.2.0",
|
||||||
|
"configurations": [
|
||||||
|
{
|
||||||
|
"name": "Python Debugger: Python File",
|
||||||
|
"type": "debugpy",
|
||||||
|
"request": "launch",
|
||||||
|
"program": "${file}",
|
||||||
|
"args": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Executable
+74
@@ -0,0 +1,74 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import webbrowser
|
||||||
|
from twitchAPI.twitch import Twitch
|
||||||
|
from twitchAPI.oauth import UserAuthenticator
|
||||||
|
from twitchAPI.type import AuthScope
|
||||||
|
|
||||||
|
SECRETS_FILE = "twitch_secrets.json"
|
||||||
|
|
||||||
|
def load_credentials():
|
||||||
|
"""Loads existing Client ID and Secret from your JSON file."""
|
||||||
|
if not os.path.exists(SECRETS_FILE):
|
||||||
|
raise FileNotFoundError(f"Could not find {SECRETS_FILE} in this directory.")
|
||||||
|
with open(SECRETS_FILE, "r") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
return data.get("client_id"), data.get("client_secret")
|
||||||
|
|
||||||
|
def save_token_to_json(token):
|
||||||
|
"""Saves the generated token into twitch_secrets.json under 'manual_token'."""
|
||||||
|
with open(SECRETS_FILE, "r") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
# Inject the new token
|
||||||
|
data["manual_token"] = token
|
||||||
|
|
||||||
|
with open(SECRETS_FILE, "w") as f:
|
||||||
|
json.dump(data, f, indent=4)
|
||||||
|
print(f"\n[SUCCESS] Token saved inside '{SECRETS_FILE}' under 'manual_token'!")
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
try:
|
||||||
|
client_id, client_secret = load_credentials()
|
||||||
|
if not client_id or not client_secret:
|
||||||
|
print("[ERROR] Please add your client_id and client_secret to the JSON file first.")
|
||||||
|
return
|
||||||
|
|
||||||
|
print("Initializing local connection loop...")
|
||||||
|
# Initialize official Twitch connection interface
|
||||||
|
twitch = await Twitch(client_id, client_secret)
|
||||||
|
|
||||||
|
# Scopes: We leave this empty [] since VOD collection only requires basic public clearance
|
||||||
|
scopes = []
|
||||||
|
|
||||||
|
# Create an authenticator that automatically sets up http://localhost:17563
|
||||||
|
auth = UserAuthenticator(twitch, scopes, url="http://localhost:17563")
|
||||||
|
|
||||||
|
# Request authentication URL
|
||||||
|
auth_url = auth.return_auth_url()
|
||||||
|
print(f"\nIf your browser does not open automatically, copy and paste this URL into your browser:\n{auth_url}\n")
|
||||||
|
|
||||||
|
# Open your system default browser to let you manually click "Authorize"
|
||||||
|
webbrowser.open(auth_url)
|
||||||
|
|
||||||
|
print("Waiting for you to click 'Authorize' in your web browser...")
|
||||||
|
# The script halts here, running a local background server until you click authorize
|
||||||
|
token, refresh_token = await auth.authenticate()
|
||||||
|
|
||||||
|
print(f"\nSuccessfully generated Token: {token}")
|
||||||
|
|
||||||
|
# Save it right back into your configuration file
|
||||||
|
save_token_to_json(token)
|
||||||
|
|
||||||
|
# Gracefully shut down the library connection
|
||||||
|
await twitch.close()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"\n[ERROR] An error occurred: {e}")
|
||||||
|
print("Double-check that http://localhost:17563 is added to your Twitch Dev Console.")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# Run the asynchronous loop
|
||||||
|
asyncio.run(main())
|
||||||
+127
@@ -0,0 +1,127 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import database
|
||||||
|
import youtube_short
|
||||||
|
import transcribe_video
|
||||||
|
import twitch_chat_vod
|
||||||
|
|
||||||
|
DB = None
|
||||||
|
|
||||||
|
def build_chat_video():
|
||||||
|
chats = DB.get_unuploaded_chats()
|
||||||
|
|
||||||
|
for chat in chats:
|
||||||
|
(
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
created_at,
|
||||||
|
view_count,
|
||||||
|
duration,
|
||||||
|
url,
|
||||||
|
thumbnail_url,
|
||||||
|
game_id,
|
||||||
|
game_name,
|
||||||
|
stream_id,
|
||||||
|
creator_name,
|
||||||
|
clip_is,
|
||||||
|
downloaded,
|
||||||
|
uploaded_yt,
|
||||||
|
uploaded_yt_chats,
|
||||||
|
uploaded_yt_shorts,
|
||||||
|
) = chat
|
||||||
|
|
||||||
|
print("====================================================")
|
||||||
|
print(f"🚀 Chat Video: {title}")
|
||||||
|
print("====================================================")
|
||||||
|
|
||||||
|
twitch_chat_vod.combine_twitch_vod_and_chat(f"download/videos/{id}/{id}.mp4", "side-by-side")
|
||||||
|
|
||||||
|
print("====================================================")
|
||||||
|
print(f"✅ Processing: Chat Video Done.")
|
||||||
|
print("====================================================")
|
||||||
|
|
||||||
|
|
||||||
|
def build_transcribe():
|
||||||
|
unuploadeds = DB.get_unuploaded()
|
||||||
|
|
||||||
|
for unuploaded in unuploadeds:
|
||||||
|
(
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
created_at,
|
||||||
|
view_count,
|
||||||
|
duration,
|
||||||
|
url,
|
||||||
|
thumbnail_url,
|
||||||
|
game_id,
|
||||||
|
game_name,
|
||||||
|
stream_id,
|
||||||
|
creator_name,
|
||||||
|
clip_is,
|
||||||
|
downloaded,
|
||||||
|
uploaded_yt,
|
||||||
|
uploaded_yt_chats,
|
||||||
|
uploaded_yt_shorts,
|
||||||
|
) = unuploaded
|
||||||
|
|
||||||
|
if clip_is:
|
||||||
|
target_dir = f"download/clips"
|
||||||
|
else:
|
||||||
|
target_dir = f"download/videos"
|
||||||
|
|
||||||
|
print("====================================================")
|
||||||
|
print(f"🚀 Transcribe: {title}")
|
||||||
|
print("====================================================")
|
||||||
|
|
||||||
|
transcribe_video.transcribe_to_srt(f"{target_dir}/{id}/{id}.mp4")
|
||||||
|
|
||||||
|
print("====================================================")
|
||||||
|
print(f"✅ Processing: Transcribing Done.")
|
||||||
|
print("====================================================")
|
||||||
|
|
||||||
|
|
||||||
|
def build_shorts():
|
||||||
|
shorts = DB.get_unuploaded_shorts()
|
||||||
|
|
||||||
|
top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
|
||||||
|
|
||||||
|
for short in shorts:
|
||||||
|
(
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
created_at,
|
||||||
|
view_count,
|
||||||
|
duration,
|
||||||
|
url,
|
||||||
|
thumbnail_url,
|
||||||
|
game_id,
|
||||||
|
game_name,
|
||||||
|
stream_id,
|
||||||
|
creator_name,
|
||||||
|
clip_is,
|
||||||
|
downloaded,
|
||||||
|
uploaded_yt,
|
||||||
|
uploaded_yt_chats,
|
||||||
|
uploaded_yt_shorts,
|
||||||
|
) = short
|
||||||
|
|
||||||
|
target_dir = f"download/clips/{id}/{id}.mp4"
|
||||||
|
|
||||||
|
print("====================================================")
|
||||||
|
print(f"🚀 Processing: Clip to Youtube Short {title}")
|
||||||
|
print("====================================================")
|
||||||
|
|
||||||
|
youtube_short.fit_to_9_16_letterbox(target_dir, top_txt, f"Clipped By: {creator_name}.")
|
||||||
|
|
||||||
|
print("====================================================")
|
||||||
|
print(f"✅ Processing: Clip to Youtube Short Done.")
|
||||||
|
print("====================================================")
|
||||||
|
|
||||||
|
def main():
|
||||||
|
global DB
|
||||||
|
DB = database.Database()
|
||||||
|
build_shorts()
|
||||||
|
build_transcribe()
|
||||||
|
build_chat_video()
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+162
@@ -0,0 +1,162 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import sqlite3
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
class Database:
|
||||||
|
def __init__(self, db_path: str = "database.db"):
|
||||||
|
self.columns = "id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts"
|
||||||
|
|
||||||
|
file_exists = Path(db_path).is_file()
|
||||||
|
|
||||||
|
self.conn = sqlite3.connect(db_path)
|
||||||
|
self.cursor = self.conn.cursor()
|
||||||
|
|
||||||
|
if not file_exists:
|
||||||
|
self.create_database()
|
||||||
|
|
||||||
|
def __exit__(self):
|
||||||
|
# Destructors are unpredictable in Python; explicitly close when done instead
|
||||||
|
try:
|
||||||
|
self.close_database()
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def create_database(self):
|
||||||
|
"""Creates a table structured explicitly for Twitch videos properties."""
|
||||||
|
self.cursor.execute("""
|
||||||
|
CREATE TABLE IF NOT EXISTS twitch_videos (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
view_count INTEGER NOT NULL,
|
||||||
|
duration TEXT NOT NULL,
|
||||||
|
url TEXT NOT NULL,
|
||||||
|
thumbnail_url TEXT NOT NULL,
|
||||||
|
game_id TEXT NOT NULL,
|
||||||
|
game_name TEXT NOT NULL,
|
||||||
|
stream_id TEXT NOT NULL,
|
||||||
|
creator_name TEXT NOT NULL,
|
||||||
|
clip_is BOOLEAN NOT NULL DEFAULT 0,
|
||||||
|
downloaded BOOLEAN NOT NULL DEFAULT 0,
|
||||||
|
uploaded_yt BOOLEAN NOT NULL DEFAULT 0,
|
||||||
|
uploaded_yt_chats BOOLEAN NOT NULL DEFAULT 0,
|
||||||
|
uploaded_yt_shorts BOOLEAN NOT NULL DEFAULT 0
|
||||||
|
)
|
||||||
|
""")
|
||||||
|
|
||||||
|
self.conn.commit()
|
||||||
|
|
||||||
|
def close_database(self):
|
||||||
|
"""Commit before we close."""
|
||||||
|
if self.conn:
|
||||||
|
self.conn.commit()
|
||||||
|
self.conn.close()
|
||||||
|
|
||||||
|
def __mark_as(self, record_id: str, set_row: str, mark: str = "1" ):
|
||||||
|
self.cursor.execute(
|
||||||
|
f"UPDATE twitch_videos SET {set_row} = ? WHERE id = ?",
|
||||||
|
(mark, record_id))
|
||||||
|
self.conn.commit()
|
||||||
|
|
||||||
|
def mark_as_uploaded_shorts(self, record_id: str):
|
||||||
|
"""Flags a specific row record to uploaded_yt_shorts (1)."""
|
||||||
|
self.__mark_as(record_id, "uploaded_yt_shorts")
|
||||||
|
|
||||||
|
def mark_as_uploaded_chats(self, record_id: str):
|
||||||
|
"""Flags a specific row record to uploaded_yt_chats (1)."""
|
||||||
|
self.__mark_as(record_id, "uploaded_yt_chats")
|
||||||
|
|
||||||
|
def mark_as_uploaded_yt(self, record_id: str):
|
||||||
|
"""Flags a specific row record to uploaded_yt (1)."""
|
||||||
|
self.__mark_as(record_id, "uploaded_yt")
|
||||||
|
|
||||||
|
def mark_as_downloaded(self, record_id: str):
|
||||||
|
"""Flags a specific row record to downloaded (1)."""
|
||||||
|
self.__mark_as(record_id, "downloaded")
|
||||||
|
|
||||||
|
def unmark_as_download(self, record_id: str):
|
||||||
|
"""Flags a specific row record to downloaded (0)."""
|
||||||
|
self.__mark_as(record_id, "downloaded", "0")
|
||||||
|
|
||||||
|
def __get_unuploaded(self, set_row: str, also: str = "") -> list[Any]:
|
||||||
|
"""Retrieve all rows that were download but not uploaded"""
|
||||||
|
self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE downloaded = 1 AND {set_row} = 0 {also}")
|
||||||
|
return self.cursor.fetchall()
|
||||||
|
|
||||||
|
def get_unuploaded_shorts(self) -> list[Any]:
|
||||||
|
"""Retrieve all rows that were download but not uploaded_yt_shorts"""
|
||||||
|
return self.__get_unuploaded("uploaded_yt_shorts", "AND clip_is = 1")
|
||||||
|
|
||||||
|
def get_unuploaded_chats(self) -> list[Any]:
|
||||||
|
"""Retrieve all rows that were download but not uploaded_yt_chats"""
|
||||||
|
return self.__get_unuploaded("uploaded_yt_chats", "AND clip_is = 0")
|
||||||
|
|
||||||
|
def get_unuploaded(self) -> list[Any]:
|
||||||
|
"""Retrieve all rows that were download but not uploaded_yt"""
|
||||||
|
return self.__get_unuploaded("uploaded_yt")
|
||||||
|
|
||||||
|
def get_undownloaded(self) -> list[Any]:
|
||||||
|
"""Retrieves all rows that were not downloaded"""
|
||||||
|
self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE downloaded = 0")
|
||||||
|
return self.cursor.fetchall()
|
||||||
|
|
||||||
|
def get_download(self) -> list[Any]:
|
||||||
|
"""Retrieves all rows that were downloaded"""
|
||||||
|
self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE downloaded = 1")
|
||||||
|
return self.cursor.fetchall()
|
||||||
|
|
||||||
|
def get_clips(self) -> list[Any]:
|
||||||
|
"""Retrieves all rows that are clip_is"""
|
||||||
|
self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE clip_is = 1")
|
||||||
|
return self.cursor.fetchall()
|
||||||
|
|
||||||
|
def get_vods(self) -> list[Any]:
|
||||||
|
self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE clip_is = 0")
|
||||||
|
return self.cursor.fetchall()
|
||||||
|
|
||||||
|
def insert_video_record(
|
||||||
|
self, id: str, title: str, created_at: str, view_count: int, duration: str,
|
||||||
|
url: str, thumbnail_url: str, game_id: int, game_name: str, stream_id: str,
|
||||||
|
creator_name: str, clip_is: bool
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
dt_obj = datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
clean_datetime = dt_obj.strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
clean_datetime = created_at
|
||||||
|
|
||||||
|
# Explicitly defining columns removes the security risk and column-count bug
|
||||||
|
query = """
|
||||||
|
INSERT INTO twitch_videos (
|
||||||
|
id, title, created_at, view_count, duration, url, thumbnail_url,
|
||||||
|
game_id, game_name, stream_id, creator_name, clip_is
|
||||||
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
|
ON CONFLICT(id) DO UPDATE SET
|
||||||
|
title = excluded.title,
|
||||||
|
created_at = excluded.created_at,
|
||||||
|
view_count = excluded.view_count,
|
||||||
|
duration = excluded.duration,
|
||||||
|
url = excluded.url,
|
||||||
|
thumbnail_url = excluded.thumbnail_url,
|
||||||
|
game_id = excluded.game_id,
|
||||||
|
game_name = excluded.game_name,
|
||||||
|
stream_id = excluded.stream_id,
|
||||||
|
creator_name = excluded.creator_name,
|
||||||
|
clip_is = excluded.clip_is
|
||||||
|
WHERE excluded.duration != twitch_videos.duration
|
||||||
|
"""
|
||||||
|
|
||||||
|
values = (
|
||||||
|
id, title, clean_datetime, view_count, duration, url, thumbnail_url,
|
||||||
|
game_id, game_name, stream_id, creator_name, clip_is
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.cursor.execute(query, values)
|
||||||
|
self.conn.commit()
|
||||||
|
except Exception as e:
|
||||||
|
# Prevent silent failures if the database connection drops
|
||||||
|
print(f"❌ Database insertion failed: {e}")
|
||||||
|
self.conn.rollback()
|
||||||
Executable
+147
@@ -0,0 +1,147 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import json
|
||||||
|
from google.oauth2.credentials import Credentials
|
||||||
|
from google.auth.transport.requests import Request
|
||||||
|
from googleapiclient.discovery import build
|
||||||
|
from googleapiclient.errors import HttpError
|
||||||
|
|
||||||
|
def load_credentials():
|
||||||
|
"""Load credentials from secrets.json (Reused from your previous flow)"""
|
||||||
|
try:
|
||||||
|
with open('secrets.json', 'r') as f:
|
||||||
|
creds_data = json.load(f)
|
||||||
|
|
||||||
|
credentials = Credentials(
|
||||||
|
token=creds_data['token'],
|
||||||
|
refresh_token=creds_data['refresh_token'],
|
||||||
|
token_uri=creds_data['token_uri'],
|
||||||
|
client_id=creds_data['client_id'],
|
||||||
|
client_secret=creds_data['client_secret'],
|
||||||
|
scopes=creds_data['scopes']
|
||||||
|
)
|
||||||
|
|
||||||
|
if credentials.expired:
|
||||||
|
credentials.refresh(Request())
|
||||||
|
|
||||||
|
return credentials
|
||||||
|
except FileNotFoundError:
|
||||||
|
print("Error: secrets.json not found!")
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error loading credentials: {str(e)}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_my_uploads_playlist_id(youtube):
|
||||||
|
"""Retrieves the system upload playlist ID for the authenticated user's channel."""
|
||||||
|
try:
|
||||||
|
# mine=True automatically references the authorized account
|
||||||
|
request = youtube.channels().list(part="contentDetails", mine=True)
|
||||||
|
response = request.execute()
|
||||||
|
|
||||||
|
if "items" in response and len(response['items']) > 0:
|
||||||
|
return response['items'][0]['contentDetails']['relatedPlaylists']['uploads']
|
||||||
|
else:
|
||||||
|
print("No channel found for these credentials.")
|
||||||
|
return None
|
||||||
|
except HttpError as e:
|
||||||
|
print(f"API Error retrieving channel details: {str(e)}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def scan_channel_videos_for_tag(youtube, uploads_playlist_id: str, target_tag: str):
|
||||||
|
"""
|
||||||
|
Loops through all channel video uploads and filters those possessing the target tag.
|
||||||
|
"""
|
||||||
|
target_tag_lower = target_tag.lower()
|
||||||
|
all_videos_count = 0
|
||||||
|
matched_videos = []
|
||||||
|
next_page_token = None
|
||||||
|
|
||||||
|
print("Beginning channel scan...")
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
# Step A: Retrieve a batch of video IDs from the uploads playlist container
|
||||||
|
playlist_request = youtube.playlistItems().list(
|
||||||
|
part="snippet",
|
||||||
|
playlistId=uploads_playlist_id,
|
||||||
|
maxResults=50,
|
||||||
|
pageToken=next_page_token
|
||||||
|
)
|
||||||
|
playlist_response = playlist_request.execute()
|
||||||
|
|
||||||
|
video_ids_batch = [
|
||||||
|
item['snippet']['resourceId']['videoId']
|
||||||
|
for item in playlist_response.get("items", [])
|
||||||
|
]
|
||||||
|
|
||||||
|
if not video_ids_batch:
|
||||||
|
break
|
||||||
|
|
||||||
|
all_videos_count += len(video_ids_batch)
|
||||||
|
|
||||||
|
# Step B: Pass batch to videos().list to extract metadata details (including tags)
|
||||||
|
video_request = youtube.videos().list(
|
||||||
|
part="snippet",
|
||||||
|
id=",".join(video_ids_batch)
|
||||||
|
)
|
||||||
|
video_response = video_request.execute()
|
||||||
|
|
||||||
|
for video in video_response.get("items", []):
|
||||||
|
title = video['snippet']['title']
|
||||||
|
video_id = video['id']
|
||||||
|
# Tags are optional fields on YouTube; default to an empty list if absent
|
||||||
|
tags = video['snippet'].get("tags", [])
|
||||||
|
|
||||||
|
# Normalize tags to lowercase for clean matching evaluation
|
||||||
|
tags_lower = [tag.lower() for tag in tags]
|
||||||
|
|
||||||
|
if target_tag_lower in tags_lower:
|
||||||
|
matched_videos.append({
|
||||||
|
"id": video_id,
|
||||||
|
"title": title,
|
||||||
|
"tags": tags
|
||||||
|
})
|
||||||
|
print(f"🔍 Found Match: '{title}' (ID: {video_id})")
|
||||||
|
|
||||||
|
# Check if another page token exists, if not break the pagination loop
|
||||||
|
next_page_token = playlist_response.get("nextPageToken")
|
||||||
|
if not next_page_token:
|
||||||
|
break
|
||||||
|
|
||||||
|
except HttpError as e:
|
||||||
|
print(f"An error occurred while fetching video batches: {str(e)}")
|
||||||
|
break
|
||||||
|
|
||||||
|
# Summary reporting
|
||||||
|
print("\n" + "="*40)
|
||||||
|
print(f"Scan complete. Analyzed {all_videos_count} total videos.")
|
||||||
|
print(f"Found {len(matched_videos)} videos containing the '{target_tag}' tag.")
|
||||||
|
print("="*40)
|
||||||
|
|
||||||
|
return matched_videos
|
||||||
|
|
||||||
|
def main():
|
||||||
|
credentials = load_credentials()
|
||||||
|
if not credentials:
|
||||||
|
return
|
||||||
|
|
||||||
|
youtube = build('youtube', 'v3', credentials=credentials)
|
||||||
|
|
||||||
|
# 1. Fetch your dynamic uploads playlist pointer
|
||||||
|
uploads_id = get_my_uploads_playlist_id(youtube)
|
||||||
|
|
||||||
|
if uploads_id:
|
||||||
|
print(f"Target Uploads Playlist ID: {uploads_id}")
|
||||||
|
|
||||||
|
# 2. Run the iterative match parser targeting the 'clips' keyword tag
|
||||||
|
target_keyword = "clips"
|
||||||
|
results = scan_channel_videos_for_tag(youtube, uploads_id, target_keyword)
|
||||||
|
|
||||||
|
# 3. Print out a neat clean list of matches
|
||||||
|
if results:
|
||||||
|
print(f"\n--- List of matching videos for tag '{target_keyword}': ---")
|
||||||
|
for index, item in enumerate(results, start=1):
|
||||||
|
print(f"{index}. {item['title']} -> https://youtu.be{item['id']}")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import sys
|
||||||
|
import shlex
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
def run_command(cmd_str: str, progress_prefix: str = "Progress", look_for: list = ["frame=", "time=", "fps=", "Rendering frame"]) -> bool:
|
||||||
|
"""Runs a system command, streams its output live, and reports errors on failure."""
|
||||||
|
args = shlex.split(cmd_str)
|
||||||
|
|
||||||
|
# Redirect stderr to stdout to catch all logging/progress in one stream
|
||||||
|
process = subprocess.Popen(
|
||||||
|
args,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
text=True,
|
||||||
|
bufsize=1
|
||||||
|
)
|
||||||
|
print(f"Executing: {cmd_str[:90]}...")
|
||||||
|
|
||||||
|
# Maintain a small buffer history to display context if a crash occurs
|
||||||
|
output_history = []
|
||||||
|
|
||||||
|
# Stream the output live to the terminal
|
||||||
|
while True:
|
||||||
|
line = process.stdout.readline()
|
||||||
|
if not line and process.poll() is not None:
|
||||||
|
break
|
||||||
|
if line:
|
||||||
|
clean_line = line.strip()
|
||||||
|
output_history.append(clean_line) # Keep history for error reporting
|
||||||
|
|
||||||
|
# Keep history slim by only keeping the last 20 lines
|
||||||
|
if len(output_history) > 20:
|
||||||
|
output_history.pop(0)
|
||||||
|
|
||||||
|
# Only print updates that show progress metrics to keep terminal clean
|
||||||
|
if any(metric in clean_line for metric in look_for):
|
||||||
|
sys.stdout.write(f"\r[{progress_prefix}] {clean_line}")
|
||||||
|
sys.stdout.flush()
|
||||||
|
elif "Error" in clean_line or "failed" in clean_line:
|
||||||
|
print(f"\n[Alert] {clean_line}")
|
||||||
|
|
||||||
|
print("\n") # New line after process finishes
|
||||||
|
|
||||||
|
# Evaluate success status
|
||||||
|
success = (process.returncode == 0)
|
||||||
|
|
||||||
|
if not success:
|
||||||
|
print(f"❌ Command failed with exit code: {process.returncode}")
|
||||||
|
print("--- Technical Error Details (Last 5 lines of output) ---")
|
||||||
|
# Print the last 5 captured lines to show the exact point of failure
|
||||||
|
for error_line in output_history[-5:]:
|
||||||
|
print(f" > {error_line}")
|
||||||
|
print("---------------------------------------------------------")
|
||||||
|
|
||||||
|
return success
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
import twitch_video_info
|
||||||
|
import twitch_download_videos
|
||||||
|
import twitch_download_thumbnails
|
||||||
|
|
||||||
|
import build_videos
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# Get Twitch Video Info
|
||||||
|
asyncio.run(twitch_video_info.main())
|
||||||
|
|
||||||
|
# Download Twitch Videos
|
||||||
|
twitch_download_videos.main()
|
||||||
|
|
||||||
|
twitch_download_thumbnails.main()
|
||||||
|
|
||||||
|
#Build
|
||||||
|
build_videos.main()
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
# Connect to your database file
|
||||||
|
db_name = "database.db" # Change to your actual file name
|
||||||
|
conn = sqlite3.connect(db_name)
|
||||||
|
cursor = conn.cursor()
|
||||||
|
|
||||||
|
try:
|
||||||
|
# 1. Create a temporary table with the new text-based game_id
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS twitch_videos_temp (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
created_at TEXT NOT NULL,
|
||||||
|
view_count INTEGER NOT NULL,
|
||||||
|
duration TEXT NOT NULL,
|
||||||
|
url TEXT NOT NULL,
|
||||||
|
thumbnail_url TEXT NOT NULL,
|
||||||
|
game_id TEXT NOT NULL,
|
||||||
|
game_name TEXT NOT NULL,
|
||||||
|
stream_id TEXT NOT NULL,
|
||||||
|
creator_name TEXT NOT NULL,
|
||||||
|
clip_is BOOLEAN NOT NULL DEFAULT 0,
|
||||||
|
downloaded BOOLEAN NOT NULL DEFAULT 0,
|
||||||
|
uploaded_yt BOOLEAN NOT NULL DEFAULT 0,
|
||||||
|
uploaded_yt_chats BOOLEAN NOT NULL DEFAULT 0,
|
||||||
|
uploaded_yt_shorts BOOLEAN NOT NULL DEFAULT 0
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
# 2. Copy and convert data to the temporary table
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
INSERT INTO twitch_videos_temp (
|
||||||
|
id, title, created_at, view_count, duration, url, thumbnail_url,
|
||||||
|
game_id, game_name, stream_id, creator_name, clip_is, downloaded,
|
||||||
|
uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
id, title, created_at, view_count, duration, url, thumbnail_url,
|
||||||
|
CAST(game_id AS TEXT), game_name, stream_id, creator_name, clip_is, downloaded,
|
||||||
|
uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts
|
||||||
|
FROM twitch_videos
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
# 3. Drop the old table configuration
|
||||||
|
cursor.execute("DROP TABLE twitch_videos")
|
||||||
|
|
||||||
|
# 4. Rename the temporary table to your exact original table name
|
||||||
|
cursor.execute("ALTER TABLE twitch_videos_temp RENAME TO twitch_videos")
|
||||||
|
|
||||||
|
# Commit changes if everything succeeded
|
||||||
|
conn.commit()
|
||||||
|
print("Migration successful! 'twitch_videos' table updated.")
|
||||||
|
|
||||||
|
except sqlite3.Error as e:
|
||||||
|
# Roll back changes if an error occurs
|
||||||
|
conn.rollback()
|
||||||
|
print(f"An error occurred: {e}")
|
||||||
|
|
||||||
|
finally:
|
||||||
|
# Close database connection
|
||||||
|
conn.close()
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import database
|
||||||
|
|
||||||
|
DB = None
|
||||||
|
|
||||||
|
def redownload_clips():
|
||||||
|
clips = DB.get_clips()
|
||||||
|
|
||||||
|
for clip in clips:
|
||||||
|
# Unpack variables clearly
|
||||||
|
(
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
created_at,
|
||||||
|
view_count,
|
||||||
|
duration,
|
||||||
|
url,
|
||||||
|
thumbnail_url,
|
||||||
|
game_id,
|
||||||
|
game_name,
|
||||||
|
stream_id,
|
||||||
|
creator_name,
|
||||||
|
clip_is,
|
||||||
|
downloaded,
|
||||||
|
uploaded_yt,
|
||||||
|
uploaded_yt_chats,
|
||||||
|
uploaded_yt_shorts,
|
||||||
|
) = clip
|
||||||
|
|
||||||
|
DB.unmark_as_download(id)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
DB = database.Database()
|
||||||
|
redownload_clips()
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
import os
|
||||||
|
|
||||||
|
import database
|
||||||
|
|
||||||
|
DB = None
|
||||||
|
|
||||||
|
def remove_files():
|
||||||
|
datas = DB.get_download()
|
||||||
|
|
||||||
|
top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
|
||||||
|
|
||||||
|
for data in datas:
|
||||||
|
(
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
created_at,
|
||||||
|
view_count,
|
||||||
|
duration,
|
||||||
|
url,
|
||||||
|
thumbnail_url,
|
||||||
|
game_id,
|
||||||
|
game_name,
|
||||||
|
stream_id,
|
||||||
|
creator_name,
|
||||||
|
clip_is,
|
||||||
|
downloaded,
|
||||||
|
uploaded_yt,
|
||||||
|
uploaded_yt_chats,
|
||||||
|
uploaded_yt_shorts,
|
||||||
|
) = data
|
||||||
|
|
||||||
|
if clip_is == 0:
|
||||||
|
video_path = f"download/videos/{id}/{id}.mp4"
|
||||||
|
else:
|
||||||
|
video_path = f"download/clips/{id}/{id}.mp4"
|
||||||
|
|
||||||
|
temp_chat = video_path.replace(".mp4", "_temp_chat.mp4")
|
||||||
|
temp_with_chat = video_path.replace(".mp4", "_temp_with_chat.mp4")
|
||||||
|
video_chat = video_path.replace(".mp4", "_with_chat.mp4")
|
||||||
|
video_chatSS = video_path.replace(".mp4", "_SS_with_chat.mp4")
|
||||||
|
video_chat_over = video_path.replace(".mp4", "_over_with_chat.mp4")
|
||||||
|
mask_path = video_path.replace(".mp4", "_temp_chat_mask.mp4")
|
||||||
|
|
||||||
|
shorts_gaussian_9_16 = ""#video_path.replace(".mp4", "_gaussian_9_16.mp4")
|
||||||
|
shorts_9_16 = ""#video_path.replace(".mp4", "_9_16.mp4")
|
||||||
|
|
||||||
|
srt_file = ""#video_path.replace(".mp4", ".srt")
|
||||||
|
|
||||||
|
|
||||||
|
files = [temp_chat, temp_with_chat, video_chatSS, video_chat_over, mask_path, shorts_gaussian_9_16, shorts_9_16, srt_file, video_chat]
|
||||||
|
|
||||||
|
for file in files:
|
||||||
|
if os.path.exists(file):
|
||||||
|
if file == video_path:
|
||||||
|
print("❌ Error: trying to delete import file.")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
os.remove(file)
|
||||||
|
|
||||||
|
def redownload():
|
||||||
|
DB.unmark_as_download("2840110867")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
global DB
|
||||||
|
DB = database.Database()
|
||||||
|
|
||||||
|
remove_files()
|
||||||
|
redownload()
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
aiohappyeyeballs==2.7.1
|
||||||
|
aiohttp==3.14.2
|
||||||
|
aiosignal==1.4.0
|
||||||
|
anyio==4.14.2
|
||||||
|
attrs==26.1.0
|
||||||
|
beautifulsoup4==4.15.0
|
||||||
|
certifi==2026.7.22
|
||||||
|
cffi==2.1.0
|
||||||
|
charset-normalizer==3.4.9
|
||||||
|
chat-downloader==0.2.8
|
||||||
|
click==8.4.2
|
||||||
|
colorlog==6.12.0
|
||||||
|
cryptography==49.0.0
|
||||||
|
cuda-bindings==13.3.1
|
||||||
|
cuda-pathfinder==1.6.0
|
||||||
|
cuda-toolkit==13.0.3.0
|
||||||
|
decorator==5.3.1
|
||||||
|
defusedxml==0.7.1
|
||||||
|
docstring_parser==0.18.0
|
||||||
|
enum-tools==0.13.0
|
||||||
|
filelock==3.32.0
|
||||||
|
frozenlist==1.8.0
|
||||||
|
fsspec==2026.6.0
|
||||||
|
git-filter-repo==2.47.0
|
||||||
|
google==3.0.0
|
||||||
|
google-api-core==2.32.0
|
||||||
|
google-api-python-client==2.198.0
|
||||||
|
google-auth==2.56.2
|
||||||
|
google-auth-httplib2==0.4.0
|
||||||
|
google-auth-oauthlib==1.4.0
|
||||||
|
googleapis-common-protos==1.75.0
|
||||||
|
h11==0.16.0
|
||||||
|
httpcore==1.0.9
|
||||||
|
httplib2==0.32.0
|
||||||
|
httpx==0.28.1
|
||||||
|
idna==3.18
|
||||||
|
ImageIO==2.37.4
|
||||||
|
imageio-ffmpeg==0.6.0
|
||||||
|
isodate==0.7.2
|
||||||
|
Jinja2==3.1.6
|
||||||
|
joblib==1.5.3
|
||||||
|
llvmlite==0.48.0
|
||||||
|
lxml==6.1.1
|
||||||
|
MarkupSafe==3.0.3
|
||||||
|
more-itertools==11.1.0
|
||||||
|
moviepy==2.2.1
|
||||||
|
mpmath==1.3.0
|
||||||
|
multidict==6.7.1
|
||||||
|
networkx==3.6.1
|
||||||
|
nltk==3.10.0
|
||||||
|
numba==0.66.0
|
||||||
|
numpy==2.4.6
|
||||||
|
nvidia-cublas==13.1.1.3
|
||||||
|
nvidia-cuda-cupti==13.0.85
|
||||||
|
nvidia-cuda-nvrtc==13.0.88
|
||||||
|
nvidia-cuda-runtime==13.0.96
|
||||||
|
nvidia-cudnn-cu13==9.20.0.48
|
||||||
|
nvidia-cufft==12.0.0.61
|
||||||
|
nvidia-cufile==1.15.1.6
|
||||||
|
nvidia-curand==10.4.0.35
|
||||||
|
nvidia-cusolver==12.0.4.66
|
||||||
|
nvidia-cusparse==12.6.3.3
|
||||||
|
nvidia-cusparselt-cu13==0.8.1
|
||||||
|
nvidia-nccl-cu13==2.29.7
|
||||||
|
nvidia-nvjitlink==13.3.33
|
||||||
|
nvidia-nvshmem-cu13==3.4.5
|
||||||
|
nvidia-nvtx==13.0.85
|
||||||
|
oauthlib==3.3.1
|
||||||
|
openai-whisper==20250625
|
||||||
|
outcome==1.3.0.post0
|
||||||
|
pillow==11.3.0
|
||||||
|
pip_system_certs==5.3
|
||||||
|
proglog==0.1.12
|
||||||
|
propcache==0.5.2
|
||||||
|
proto-plus==1.28.1
|
||||||
|
protobuf==7.35.1
|
||||||
|
pyasn1==0.6.4
|
||||||
|
pyasn1_modules==0.4.2
|
||||||
|
pycountry==26.2.16
|
||||||
|
pycparser==3.0
|
||||||
|
pycryptodome==3.23.0
|
||||||
|
Pygments==2.20.0
|
||||||
|
pyparsing==3.3.2
|
||||||
|
PySocks==1.7.1
|
||||||
|
python-dateutil==2.9.0.post0
|
||||||
|
python-dotenv==1.2.2
|
||||||
|
regex==2026.7.19
|
||||||
|
requests==2.34.2
|
||||||
|
requests-oauthlib==2.0.0
|
||||||
|
setuptools==83.0.0
|
||||||
|
six==1.17.0
|
||||||
|
sniffio==1.3.1
|
||||||
|
sortedcontainers==2.4.0
|
||||||
|
soupsieve==2.9.1
|
||||||
|
streamlink==8.4.0
|
||||||
|
sympy==1.14.0
|
||||||
|
tiktoken==0.13.0
|
||||||
|
torch==2.13.0
|
||||||
|
tqdm==4.69.0
|
||||||
|
trio==0.33.0
|
||||||
|
trio-websocket==0.12.2
|
||||||
|
triton==3.7.1
|
||||||
|
twitchAPI==4.5.0
|
||||||
|
typing_extensions==4.16.0
|
||||||
|
uritemplate==4.2.0
|
||||||
|
urllib3==2.7.0
|
||||||
|
websocket-client==1.9.0
|
||||||
|
wsproto==1.3.2
|
||||||
|
yarl==1.24.5
|
||||||
Executable
+119
@@ -0,0 +1,119 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
from faster_whisper import WhisperModel
|
||||||
|
from faster_whisper.utils import format_timestamp
|
||||||
|
|
||||||
|
# Prevent OpenMP thread conflicts from crashing the script
|
||||||
|
os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
|
||||||
|
|
||||||
|
# 5 minutes per chunk (300 seconds) keeps RAM usage low and stable
|
||||||
|
CHUNK_DURATION_SEC = 300
|
||||||
|
|
||||||
|
print("🔧 Initializing C++ Engine...")
|
||||||
|
model = WhisperModel(
|
||||||
|
"base",
|
||||||
|
device="cpu",
|
||||||
|
compute_type="int8",
|
||||||
|
cpu_threads=0, # Let CTranslate2 auto-detect safe core counts
|
||||||
|
num_workers=1
|
||||||
|
)
|
||||||
|
print("✅ C++ Model loaded successfully.")
|
||||||
|
|
||||||
|
|
||||||
|
def extract_audio_and_chunk(video_path: str, output_dir: Path) -> list:
|
||||||
|
"""Extracts and splits audio into 5-minute chunks using a single FFmpeg pass."""
|
||||||
|
print("🚀 Extracting and chunking audio with FFmpeg...")
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Segment format output: chunk_000.wav, chunk_001.wav, etc.
|
||||||
|
chunk_pattern = str(output_dir / "chunk_%03d.wav")
|
||||||
|
|
||||||
|
command = [
|
||||||
|
"ffmpeg", "-y", "-i", video_path,
|
||||||
|
"-vn", "-ac", "1", "-ar", "16000",
|
||||||
|
"-acodec", "pcm_s16le", "-sn", "-map_chapters", "-1",
|
||||||
|
"-f", "segment", "-segment_time", str(CHUNK_DURATION_SEC),
|
||||||
|
chunk_pattern
|
||||||
|
]
|
||||||
|
|
||||||
|
result = subprocess.run(command, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
|
||||||
|
if result.returncode != 0:
|
||||||
|
print(f"❌ FFmpeg Error Output:\n{result.stderr}")
|
||||||
|
raise RuntimeError("FFmpeg extraction and chunking failed.")
|
||||||
|
|
||||||
|
# Return sorted list of generated chunk files
|
||||||
|
return sorted(list(output_dir.glob("chunk_*.wav")))
|
||||||
|
|
||||||
|
|
||||||
|
def transcribe_to_srt(video_path: str, force: bool = False):
|
||||||
|
video_path_obj = Path(video_path)
|
||||||
|
srt_path = video_path_obj.with_suffix(".srt")
|
||||||
|
temp_dir = video_path_obj.parent / f"temp_chunks_{video_path_obj.stem}"
|
||||||
|
|
||||||
|
if srt_path.exists():
|
||||||
|
if not force:
|
||||||
|
print(f"❌ Error: Transcription file already exists: {srt_path}")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
srt_path.unlink()
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Step 1: Split audio into bite-sized pieces
|
||||||
|
audio_chunks = extract_audio_and_chunk(str(video_path_obj), temp_dir)
|
||||||
|
if not audio_chunks:
|
||||||
|
print("❌ Error: No audio chunks were generated.")
|
||||||
|
return
|
||||||
|
|
||||||
|
print(f"📦 Successfully split audio into {len(audio_chunks)} chunks.")
|
||||||
|
print("🎙️ Starting safe chunk-by-chunk transcription...")
|
||||||
|
|
||||||
|
global_segment_index = 1
|
||||||
|
|
||||||
|
with open(srt_path, "w", encoding="utf-8") as srt_file:
|
||||||
|
for chunk_idx, chunk_path in enumerate(audio_chunks):
|
||||||
|
# Calculate the time offset for the current chunk
|
||||||
|
time_offset = chunk_idx * CHUNK_DURATION_SEC
|
||||||
|
print(f"\n⏳ Processing chunk {chunk_idx + 1}/{len(audio_chunks)} ({chunk_path.name})...")
|
||||||
|
|
||||||
|
segments_generator, info = model.transcribe(
|
||||||
|
str(chunk_path),
|
||||||
|
beam_size=1,
|
||||||
|
vad_filter=True,
|
||||||
|
temperature=0.0
|
||||||
|
)
|
||||||
|
|
||||||
|
# Consume chunk generator and shift timestamps instantly
|
||||||
|
for segment in segments_generator:
|
||||||
|
# Shift timestamps relative to the original video timeline
|
||||||
|
actual_start = segment.start + time_offset
|
||||||
|
actual_end = segment.end + time_offset
|
||||||
|
|
||||||
|
start_str = format_timestamp(actual_start, always_include_hours=True)
|
||||||
|
end_str = format_timestamp(actual_end, always_include_hours=True)
|
||||||
|
|
||||||
|
srt_file.write(f"{global_segment_index}\n{start_str} --> {end_str}\n{segment.text.strip()}\n\n")
|
||||||
|
global_segment_index += 1
|
||||||
|
|
||||||
|
# Free up space as we go by deleting the processed chunk
|
||||||
|
chunk_path.unlink()
|
||||||
|
|
||||||
|
print(f"\n✅ All chunks combined! SRT subtitle file saved in: {srt_path}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"\n❌ Execution Error: {e}")
|
||||||
|
finally:
|
||||||
|
# Clean up the temporary folder entirely
|
||||||
|
if temp_dir.exists():
|
||||||
|
shutil.rmtree(temp_dir)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
target_video = "download/videos/2813112936/2813112936.mp4"
|
||||||
|
if not os.path.exists(target_video):
|
||||||
|
print(f"❌ System Error: Target video file does not exist at path: {target_video}")
|
||||||
|
else:
|
||||||
|
transcribe_to_srt(target_video, force=True)
|
||||||
@@ -0,0 +1,150 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import cv2
|
||||||
|
import linux
|
||||||
|
|
||||||
|
def get_video_height(video_path: str) -> int:
|
||||||
|
# Open the video file
|
||||||
|
video = cv2.VideoCapture(video_path)
|
||||||
|
|
||||||
|
# Get the height property
|
||||||
|
height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
||||||
|
|
||||||
|
# Always release the video object
|
||||||
|
video.release()
|
||||||
|
|
||||||
|
return height
|
||||||
|
|
||||||
|
def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side", force: bool = False) -> bool:
|
||||||
|
"""
|
||||||
|
Renders Twitch chat JSON to video and combines it with the source VOD.
|
||||||
|
Provides real-time terminal feedback for all processing steps.
|
||||||
|
"""
|
||||||
|
threads = 8
|
||||||
|
|
||||||
|
video_height = 1080
|
||||||
|
|
||||||
|
video_path = vod_path
|
||||||
|
video_chat = ""
|
||||||
|
chat_path = video_path.replace(".mp4", "_chat.json")
|
||||||
|
temp_with_chat = video_path.replace(".mp4", "_temp_with_chat.mp4")
|
||||||
|
temp_chat = video_path.replace(".mp4", "_temp_chat.mp4")
|
||||||
|
video_chat_SS = video_path.replace(".mp4", "_SS_with_chat.mp4")
|
||||||
|
video_chat_over = video_path.replace(".mp4", "_over_with_chat.mp4")
|
||||||
|
mask_path = video_path.replace(".mp4", "_temp_chat_mask.mp4")
|
||||||
|
|
||||||
|
# Step 1: Pre-flight checks
|
||||||
|
if not os.path.exists(video_path):
|
||||||
|
print(f"❌ Error: Source video not found: {video_path}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
if not os.path.exists(chat_path):
|
||||||
|
print(f"❌ Error: Chat file not found: {chat_path}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
if os.path.exists(temp_with_chat):
|
||||||
|
os.remove(temp_with_chat)
|
||||||
|
|
||||||
|
if layout == "side-by-side":
|
||||||
|
video_chat = video_chat_SS
|
||||||
|
elif layout == "overlay":
|
||||||
|
video_chat = video_chat_over
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if os.path.exists(video_chat):
|
||||||
|
if not force:
|
||||||
|
print(f"❌ Error: Chat video already exists: {video_chat}")
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
os.remove(video_chat)
|
||||||
|
|
||||||
|
if os.path.exists(temp_chat):
|
||||||
|
os.remove(temp_chat)
|
||||||
|
if os.path.exists(mask_path):
|
||||||
|
os.remove(mask_path)
|
||||||
|
|
||||||
|
# Step 2: Render Chat to Video
|
||||||
|
print("====================================================")
|
||||||
|
print("🚀 STEP 1: Rendering Chat JSON to Video Layer")
|
||||||
|
print("====================================================")
|
||||||
|
|
||||||
|
video_height = get_video_height(vod_path)
|
||||||
|
|
||||||
|
chat_cmd = (
|
||||||
|
f"TwitchDownloaderCLI chatrender "
|
||||||
|
f"-i {chat_path} "
|
||||||
|
f"-w 400 -h {video_height} "
|
||||||
|
f"--collision Overwrite "
|
||||||
|
f"--temp-path download/temp "
|
||||||
|
f"--font-size 20 "
|
||||||
|
f"--background-color #00000000 "
|
||||||
|
f"-o {temp_chat}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if layout == "overlay":
|
||||||
|
chat_cmd = (f"{chat_cmd} --generate-mask ")
|
||||||
|
|
||||||
|
if not os.path.exists(temp_chat):
|
||||||
|
chat_success = linux.run_command(chat_cmd, look_for=["[STATUS]"])
|
||||||
|
if chat_success:
|
||||||
|
print("✅ Success: TwitchDownloaderCLI render chat video.")
|
||||||
|
else:
|
||||||
|
print("❌ Error: TwitchDownloaderCLI failed to render chat video.")
|
||||||
|
os.remove(temp_chat)
|
||||||
|
os.remove(mask_path)
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Step 3: Combine Video and Chat using FFmpeg
|
||||||
|
print("====================================================")
|
||||||
|
print(f"🚀 STEP 2: Merging VOD and Chat Layout ({layout})")
|
||||||
|
print("====================================================")
|
||||||
|
|
||||||
|
# -preset superfast speeds up the 3+ hour encoding process significantly
|
||||||
|
# -map 0:a? safely includes audio if it exists, without breaking on silent VODs
|
||||||
|
if layout == "side-by-side":
|
||||||
|
ffmpeg_cmd = (
|
||||||
|
f"ffmpeg -y -i {video_path} -i {temp_chat} "
|
||||||
|
f"-filter_complex '[1:v]scale=-1:ih[scaled_chat];[0:v][scaled_chat]hstack=inputs=2[v]' "
|
||||||
|
f"-map '[v]' -map 0:a? -c:v libx264 -crf 18 -preset slow -c:a copy "
|
||||||
|
f"-threads {threads} {temp_with_chat}"
|
||||||
|
)
|
||||||
|
elif layout == "overlay":
|
||||||
|
ffmpeg_cmd = (
|
||||||
|
f"ffmpeg -y -i {video_path} -i {temp_chat} -i {mask_path} "
|
||||||
|
f"-filter_complex '[1:v][2:v]alphamerge[masked_chat];[0:v][masked_chat]overlay=x=0:y=10[v]' "
|
||||||
|
f"-map '[v]' -map 0:a? -c:v libx264 -crf 18 -preset slow -c:a copy "
|
||||||
|
f"-threads {threads} {temp_with_chat}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError("Invalid layout choice. Choose 'side-by-side' or 'overlay'.")
|
||||||
|
|
||||||
|
ffmpeg_success = linux.run_command(ffmpeg_cmd, progress_prefix="FFmpeg Merge")
|
||||||
|
|
||||||
|
# Step 4: Final verification and cleanup
|
||||||
|
if ffmpeg_success and os.path.exists(temp_with_chat):
|
||||||
|
os.rename(temp_with_chat, video_chat)
|
||||||
|
print("====================================================")
|
||||||
|
print(f"🎉 SUCCESS: Video processing complete!")
|
||||||
|
print(f"📁 Output Saved: {video_chat}")
|
||||||
|
print("====================================================")
|
||||||
|
|
||||||
|
# Clean up the massive temporary chat video to save storage space
|
||||||
|
if os.path.exists(temp_chat):
|
||||||
|
print("🧹 Cleaning up temporary chat render video...")
|
||||||
|
os.remove(temp_chat)
|
||||||
|
if os.path.exists(mask_path):
|
||||||
|
os.remove(mask_path)
|
||||||
|
if os.path.exists(temp_with_chat):
|
||||||
|
os.remove(temp_with_chat)
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
print("❌ Error: FFmpeg failed to merge the video streams.")
|
||||||
|
if os.path.exists(video_chat):
|
||||||
|
os.remove(video_chat)
|
||||||
|
if os.path.exists(temp_with_chat):
|
||||||
|
os.remove(temp_with_chat)
|
||||||
|
return False
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
combine_twitch_vod_and_chat("download/videos/2813112936/2813112936.mp4", "overlay", True)
|
||||||
Executable
+327
@@ -0,0 +1,327 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import requests
|
||||||
|
import os
|
||||||
|
import linux
|
||||||
|
import time
|
||||||
|
|
||||||
|
from database import Database
|
||||||
|
|
||||||
|
import uploader
|
||||||
|
from uploader import CategoryId
|
||||||
|
|
||||||
|
import youtube_hashtags
|
||||||
|
import transcribe_video
|
||||||
|
|
||||||
|
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
||||||
|
DB = None
|
||||||
|
|
||||||
|
def transcribe(id: str):
|
||||||
|
"""Transcribes the Video File."""
|
||||||
|
video_file = f"save/{DB.table}/{id}/{id}.mp4"
|
||||||
|
|
||||||
|
# Check if the video file exists
|
||||||
|
if not os.path.exists(video_file):
|
||||||
|
print(f"Error: File not found: {video_file}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# no need to continue if srt transcribe file already exists
|
||||||
|
if os.path.exists(f"save/{DB.table}/{id}/transcribe_{id}.srt"):
|
||||||
|
print(f"video already transcribed:")
|
||||||
|
return True
|
||||||
|
|
||||||
|
transcribe_video.extract_audio(video_file, f"save/{DB.table}/{id}/temp_{id}_audio.wav")
|
||||||
|
transcribe_video.transcribe_to_srt(f"save/{DB.table}/{id}/temp_{id}_audio.wav", f"save/{DB.table}/{id}/", f"transcribe_{id}")
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def top_hashtags(id: str):
|
||||||
|
""""Hashtags from transcribed SRT file."""
|
||||||
|
file_srt = f"save/{DB.table}/{id}/transcribe_{id}.srt"
|
||||||
|
|
||||||
|
# if srt transcribe file not exists
|
||||||
|
if not os.path.exists(file_srt):
|
||||||
|
print(f"Transcribe file not found {file_srt}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
tags = youtube_hashtags.get_top_hashtags(file_srt)
|
||||||
|
return tags
|
||||||
|
|
||||||
|
def download():
|
||||||
|
"""Find all undownload videos and download them."""
|
||||||
|
undownloaded = DB.get_undownloaded()
|
||||||
|
|
||||||
|
print(f"Download {DB.table}...")
|
||||||
|
|
||||||
|
if DB.table == "videos":
|
||||||
|
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chat_upload_yt in undownloaded:
|
||||||
|
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
|
||||||
|
output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/videos/{record_id}/{record_id}.mp4 --collision Overwrite")
|
||||||
|
output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/videos/{record_id}/{record_id}_chat.json -E --collision Overwrite")
|
||||||
|
time.sleep(1)
|
||||||
|
if output['success'] is True:
|
||||||
|
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was successful")
|
||||||
|
DB.mark_as_downloaded(record_id)
|
||||||
|
else:
|
||||||
|
print(f"ID: {record_id} | Download Process failed. {output['stdout']}. Error: {output['stderr']}")
|
||||||
|
|
||||||
|
elif DB.table == "clips":
|
||||||
|
for slug, record_date, title, game_name, clip_by, views, downloaded, uploaded_yt, uploaded_shorts_yt in undownloaded:
|
||||||
|
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
|
||||||
|
|
||||||
|
# Uses standard clipdownload directive
|
||||||
|
output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{slug}.mp4 --collision Overwrite")
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
if output['success'] is True:
|
||||||
|
print(f"Slug: {slug} | Was successfully downloaded.")
|
||||||
|
DB.mark_as_downloaded(slug)
|
||||||
|
else:
|
||||||
|
print(f"Slug: {slug} | Download Process failed. {output['stdout']}. Error: {output['stderr']}")
|
||||||
|
|
||||||
|
|
||||||
|
print(f"Finished Downloading {DB.table}...")
|
||||||
|
|
||||||
|
def upload():
|
||||||
|
"""Loops over the downloaded videos entries and uploaded them to youtube."""
|
||||||
|
print(f"Uploading {DB.table}...")
|
||||||
|
|
||||||
|
unuploaded = DB.get_unuploaded()
|
||||||
|
|
||||||
|
twitch_datetime = " Live on Twitch Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp"
|
||||||
|
file_path = ""
|
||||||
|
title = ""
|
||||||
|
description = ""
|
||||||
|
categoryId = CategoryId.GAMING
|
||||||
|
privatcyStatus = 'private'
|
||||||
|
base_tags = ['gaming', 'TeamPGP', 'twitch', 'Level1Techs']
|
||||||
|
|
||||||
|
upload_queue = []
|
||||||
|
|
||||||
|
if DB.table == "videos":
|
||||||
|
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chats_upload_yt in unuploaded:
|
||||||
|
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
|
||||||
|
|
||||||
|
file_path = f"save/videos/{record_id}/{record_id}.mp4"
|
||||||
|
description = f"Game: {game_name}, on {record_date}, #VODS {twitch_datetime}"
|
||||||
|
tags = list(base_tags)
|
||||||
|
tags.extend([f'{game_name}', 'twitch_vods', 'vods'])
|
||||||
|
tags.extend(top_hashtags(record_id))
|
||||||
|
|
||||||
|
upload_queue.append([record_id, file_path, title, categoryId, description, privatcyStatus, tags])
|
||||||
|
|
||||||
|
elif DB.table == "clips":
|
||||||
|
for slug, record_date, title, game_name, clip_by, views, downloaded, uploaded_yt, shorts_uploaded_yt in unuploaded:
|
||||||
|
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
|
||||||
|
|
||||||
|
transcribe(slug)
|
||||||
|
|
||||||
|
file_path = f"save/clips/{slug}/{slug}.mp4"
|
||||||
|
description = f"Game: {game_name}, Cliped By: {clip_by}, on {record_date}, #Shorts #Clips {twitch_datetime}"
|
||||||
|
tags = list(base_tags)
|
||||||
|
tags.extend([f'{game_name}', 'twitch_clips', 'clips', f'{clip_by}', 'shorts'])
|
||||||
|
tags.extend(top_hashtags(slug))
|
||||||
|
|
||||||
|
upload_queue.append([slug, file_path, title, categoryId, description, privatcyStatus, tags])
|
||||||
|
|
||||||
|
for db_id, file_path, title, categoryId, description, privatcyStatus, tags in upload_queue:
|
||||||
|
output = uploader.upload_video(file_path, title, categoryId, description, privatcyStatus, tags)
|
||||||
|
if output is True:
|
||||||
|
print(f"Title: {title} | Was successfully uploaded.")
|
||||||
|
DB.mark_as_uploaded(db_id)
|
||||||
|
else:
|
||||||
|
print(f"Title: {title} | Download Process failed.")
|
||||||
|
|
||||||
|
def create_chats():
|
||||||
|
pass
|
||||||
|
|
||||||
|
def create_shorts():
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get_vod_ids_simplified(channel_name: str):
|
||||||
|
"""Queries Twitch's public endpoint directly for trending clips."""
|
||||||
|
session = requests.Session()
|
||||||
|
url = "https://gql.twitch.tv/gql"
|
||||||
|
|
||||||
|
# Case-preserved headers to prevent 400 Bad Request errors
|
||||||
|
session.headers = {
|
||||||
|
"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
|
"Content-Type": "text/plain"
|
||||||
|
}
|
||||||
|
|
||||||
|
videos_query_string = """
|
||||||
|
query GetChannelVideos($login: String!, $limit: Int!, $after: Cursor) {
|
||||||
|
user(login: $login) {
|
||||||
|
videos(first: $limit, types: [ARCHIVE], after: $after) {
|
||||||
|
pageInfo {
|
||||||
|
hasNextPage
|
||||||
|
endCursor
|
||||||
|
}
|
||||||
|
edges {
|
||||||
|
node {
|
||||||
|
id
|
||||||
|
title
|
||||||
|
publishedAt
|
||||||
|
game {
|
||||||
|
displayName
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
clips_query_string = """
|
||||||
|
query GetChannelClips($login: String!, $limit: Int!, $after: Cursor) {
|
||||||
|
user(login: $login) {
|
||||||
|
clips(first: $limit, criteria: { period: ALL_TIME }, after: $after) {
|
||||||
|
pageInfo {
|
||||||
|
hasNextPage
|
||||||
|
endCursor
|
||||||
|
}
|
||||||
|
edges {
|
||||||
|
cursor
|
||||||
|
node {
|
||||||
|
slug
|
||||||
|
title
|
||||||
|
createdAt
|
||||||
|
viewCount
|
||||||
|
game {
|
||||||
|
displayName
|
||||||
|
}
|
||||||
|
curator {
|
||||||
|
login
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
video_ids = []
|
||||||
|
has_next_page = True
|
||||||
|
cursor = None
|
||||||
|
|
||||||
|
limit = 50
|
||||||
|
query_string = ""
|
||||||
|
operation_name = ""
|
||||||
|
if DB.table == "videos":
|
||||||
|
query_string = videos_query_string
|
||||||
|
limit = 100
|
||||||
|
operation_name = "GetChannelVideos"
|
||||||
|
elif DB.table == "clips":
|
||||||
|
query_string = clips_query_string
|
||||||
|
limit = 40
|
||||||
|
operation_name = "GetChannelClips"
|
||||||
|
|
||||||
|
while has_next_page:
|
||||||
|
time.sleep(1)
|
||||||
|
# A simplified query string that hardcodes the ARCHIVE type filter into the request structure
|
||||||
|
payload = [{
|
||||||
|
"operationName": operation_name,
|
||||||
|
"query": query_string,
|
||||||
|
"variables": {
|
||||||
|
"login": channel_name.lower(),
|
||||||
|
"limit": limit,
|
||||||
|
"after": cursor
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Prepping ensures Python does not rewrite the Client-ID header case
|
||||||
|
req = requests.Request('POST', url, json=payload)
|
||||||
|
#req = requests.Request('POST', url, data=json.dumps(payload))
|
||||||
|
prepped = session.prepare_request(req)
|
||||||
|
|
||||||
|
response = session.send(prepped)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
# Pull out the target index array dictionary object
|
||||||
|
result = data[0] if isinstance(data, list) else data
|
||||||
|
|
||||||
|
if "errors" in result:
|
||||||
|
print(f"Twitch GraphQL Error: {result['errors']}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
user_data = result.get('data', {}).get('user', {})
|
||||||
|
if not user_data:
|
||||||
|
print(f"Channel '{channel_name}' not found.")
|
||||||
|
return []
|
||||||
|
|
||||||
|
edges = user_data.get(DB.table, {}).get('edges', [])
|
||||||
|
|
||||||
|
# --- BREAK CONDITION 1: Stop if Twitch returns no more data items ---
|
||||||
|
if not edges or len(edges) == 0:
|
||||||
|
print("No more items returned by the server. Ending pagination loop.")
|
||||||
|
break
|
||||||
|
|
||||||
|
last_edge_cursor = None
|
||||||
|
print(f"--- Processing {DB.table} for {channel_name} ---")
|
||||||
|
for edge in edges:
|
||||||
|
last_edge_cursor = edge.get("cursor")
|
||||||
|
node = edge.get('node', {})
|
||||||
|
if not node:
|
||||||
|
continue
|
||||||
|
|
||||||
|
game_info = node.get('game')
|
||||||
|
game_name = game_info.get('displayName') if game_info else "Unknown/No Category"
|
||||||
|
|
||||||
|
# FIXED: Switched fields to use safe .get() metrics to completely prevent KeyErrors
|
||||||
|
node_id = node.get('id')
|
||||||
|
node_title = node.get('title', 'No Title')
|
||||||
|
|
||||||
|
if DB.table == "videos":
|
||||||
|
published_at = node.get('publishedAt')
|
||||||
|
print(f"ID: {node_id} | Date: {published_at} | Game: {game_name} | Title: {node_title}")
|
||||||
|
|
||||||
|
DB.insert_videos_record(node_id, published_at, node_title, game_name)
|
||||||
|
if node_id:
|
||||||
|
video_ids.append(node_id)
|
||||||
|
|
||||||
|
elif DB.table == "clips":
|
||||||
|
slug = node.get('slug')
|
||||||
|
created_at = node.get('createdAt')
|
||||||
|
view_count = node.get('viewCount', 0)
|
||||||
|
|
||||||
|
curator_info = node.get('curator')
|
||||||
|
clip_by = curator_info.get('login') if curator_info else "Unknown Creator"
|
||||||
|
|
||||||
|
print(f"Slug: {slug} | Date: {created_at} | Game: {game_name} | By: {clip_by} | Views: {view_count} | Title: {node_title}")
|
||||||
|
|
||||||
|
DB.insert_clips_record(slug, created_at, node_title, game_name, clip_by, int(view_count))
|
||||||
|
if slug:
|
||||||
|
video_ids.append(slug)
|
||||||
|
|
||||||
|
# --- CORRECTED PAGINATION ENGINE FOR BOTH TABLES ---
|
||||||
|
page_info = user_data.get(DB.table, {}).get('pageInfo', {})
|
||||||
|
has_next_page = page_info.get("hasNextPage", False)
|
||||||
|
|
||||||
|
next_cursor = page_info.get("endCursor") or last_edge_cursor
|
||||||
|
|
||||||
|
if not next_cursor or next_cursor == cursor:
|
||||||
|
print("Cursor did not advance or is null. Safely terminating loop.")
|
||||||
|
break
|
||||||
|
|
||||||
|
cursor = next_cursor
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"An unexpected error occurred: {e}")
|
||||||
|
if 'response' in locals():
|
||||||
|
print(f"Server Response Text: {response.text}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
return video_ids
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
tables = ["clips"]
|
||||||
|
for table in tables:
|
||||||
|
DB = Database(table)
|
||||||
|
|
||||||
|
get_vod_ids_simplified(CHANNEL_NAME)
|
||||||
|
#download()
|
||||||
|
#upload()
|
||||||
|
#DB.close_database()
|
||||||
Regular → Executable
+85
-78
@@ -1,60 +1,15 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import requests
|
import requests
|
||||||
import sqlite3
|
|
||||||
import subprocess
|
import subprocess
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
|
from database import Database
|
||||||
|
|
||||||
|
import uploader
|
||||||
|
from uploader import CategoryId
|
||||||
|
|
||||||
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
||||||
|
DB = Database("clips")
|
||||||
# Stores data locally
|
|
||||||
CONN = sqlite3.connect("clips_database.db")
|
|
||||||
CURSOR = CONN.cursor()
|
|
||||||
|
|
||||||
def create_database():
|
|
||||||
"""Creates a table structured explicitly for Twitch clip properties."""
|
|
||||||
CURSOR.execute(
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS clips (
|
|
||||||
slug TEXT PRIMARY KEY,
|
|
||||||
date TEXT NOT NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
gamename TEXT NOT NULL,
|
|
||||||
clip_by TEXT NOT NULL,
|
|
||||||
view_count INTEGER NOT NULL,
|
|
||||||
downloaded INTEGER NOT NULL,
|
|
||||||
uploaded_yt INTEGER NOT NULL
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
CONN.commit()
|
|
||||||
|
|
||||||
def close_database():
|
|
||||||
"""Commits queries before ending connection context."""
|
|
||||||
CONN.commit()
|
|
||||||
CONN.close()
|
|
||||||
|
|
||||||
def download_clips():
|
|
||||||
"""Loops over undownloaded metadata entries to write files down locally."""
|
|
||||||
undownloaded_clips = get_undownloaded_clips()
|
|
||||||
|
|
||||||
print("Downloading Clips...")
|
|
||||||
|
|
||||||
for slug, record_date, title, game_name, clip_by, views, downloaded in undownloaded_clips:
|
|
||||||
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
|
|
||||||
|
|
||||||
# Ensures destination folder structures exist before executing CLI tool
|
|
||||||
run_linux_command(f"mkdir -p save/clips/{slug}")
|
|
||||||
|
|
||||||
# Uses standard clipdownload directive
|
|
||||||
output = run_linux_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{slug}.mp4")
|
|
||||||
|
|
||||||
if output["success"] is True:
|
|
||||||
print(f"Slug: {slug} | Was successfully downloaded.")
|
|
||||||
mark_as_downloaded(slug)
|
|
||||||
else:
|
|
||||||
print(f"Slug: {slug} | Process failed.")
|
|
||||||
|
|
||||||
print("Finished Downloading Clips...")
|
|
||||||
|
|
||||||
def run_linux_command(command: str):
|
def run_linux_command(command: str):
|
||||||
"""Executes a Linux command, waits for completion, and returns output."""
|
"""Executes a Linux command, waits for completion, and returns output."""
|
||||||
@@ -66,34 +21,87 @@ def run_linux_command(command: str):
|
|||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
|
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
|
||||||
|
|
||||||
def mark_as_downloaded(slug: str):
|
def transcribe(slug: str):
|
||||||
"""Flags a specific clip row record to downloaded (1)."""
|
"""Transcribes the Video File."""
|
||||||
CURSOR.execute(
|
video_file = f"save/clips/{slug}/{slug}.mp4"
|
||||||
"UPDATE clips SET downloaded = 1 WHERE slug = ?",
|
|
||||||
(slug,)
|
|
||||||
)
|
|
||||||
CONN.commit()
|
|
||||||
|
|
||||||
def get_undownloaded_clips():
|
# Check if the video file exists
|
||||||
"""Retrieves all clip rows remaining to be captured."""
|
if not os.path.exists(video_file):
|
||||||
CURSOR.execute(
|
print(f"Error: File not found: {video_file}")
|
||||||
"SELECT slug, date, title, gamename, clip_by, view_count, downloaded FROM clips WHERE downloaded = 0"
|
return False
|
||||||
)
|
|
||||||
return CURSOR.fetchall()
|
|
||||||
|
|
||||||
def insert_record(slug: str, record_date_str: str, title: str, gamename: str, clip_by: str, views: int):
|
# no need to continue if srt transcribe file already exists
|
||||||
"""Cleans up ISO-8601 strings into unified date structures for the database."""
|
if os.path.exists(f"save/clips/{slug}/transcribe_{slug}.srt"):
|
||||||
try:
|
print(f"video already transcribed:")
|
||||||
clean_date = datetime.strptime(record_date_str, "%Y-%m-%dT%H:%M:%SZ").date()
|
return True
|
||||||
except ValueError:
|
|
||||||
clean_date = record_date_str
|
|
||||||
|
|
||||||
CURSOR.execute(
|
import transcribe_video
|
||||||
"INSERT OR IGNORE INTO clips (slug, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
transcribe_video.extract_audio(video_file, f"save/clips/{slug}/temp_{slug}_audio.wav")
|
||||||
(slug, str(clean_date), title, gamename, clip_by, views, False, False),
|
transcribe_video.transcribe_to_srt(f"save/clips/{slug}/temp_{slug}_audio.wav", f"save/clips/{slug}/", f"transcribe_{slug}")
|
||||||
)
|
|
||||||
CONN.commit()
|
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
def top_hashtags(slug: str):
|
||||||
|
file_srt = f"save/clips/{slug}/transcribe_{slug}.srt"
|
||||||
|
|
||||||
|
# if srt transcribe file not exists
|
||||||
|
if not os.path.exists(file_srt):
|
||||||
|
print(f"Transcribe file not found {file_srt}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
import youtube_hashtags
|
||||||
|
|
||||||
|
return youtube_hashtags.get_top_hashtags(file_srt)
|
||||||
|
|
||||||
|
|
||||||
|
def download_clips():
|
||||||
|
"""Loops over undownloaded metadata entries to write files down locally."""
|
||||||
|
undownloaded_clips = DB.get_undownloaded()
|
||||||
|
|
||||||
|
print("Downloading Clips...")
|
||||||
|
|
||||||
|
for slug, record_date, title, game_name, clip_by, views, downloaded, uploaded_yt, uploaded_shorts_yt in undownloaded_clips:
|
||||||
|
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
|
||||||
|
|
||||||
|
# Uses standard clipdownload directive
|
||||||
|
output = run_linux_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{slug}.mp4")
|
||||||
|
|
||||||
|
if output['success'] is True:
|
||||||
|
print(f"Slug: {slug} | Was successfully downloaded.")
|
||||||
|
DB.mark_as_downloaded(slug)
|
||||||
|
else:
|
||||||
|
print(f"Slug: {slug} | Download Process failed. {output['stdout']}. Error: {output['stderr']}")
|
||||||
|
|
||||||
|
print("Finished Downloading Clips...")
|
||||||
|
|
||||||
|
def upload_clips():
|
||||||
|
"""Loops over the downloaded videos entries and uploaded them to youtube."""
|
||||||
|
print(f"Uploading Clips...")
|
||||||
|
|
||||||
|
unuploaded_clips = DB.get_unuploaded()
|
||||||
|
|
||||||
|
categoryId = CategoryId.GAMING
|
||||||
|
|
||||||
|
for slug, record_date, title, game_name, clip_by, views, downloaded, uploaded_yt in unuploaded_clips:
|
||||||
|
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
|
||||||
|
|
||||||
|
file_path = f"save/clips/{slug}/{slug}.mp4"
|
||||||
|
title = title
|
||||||
|
description = f"Game: {game_name}, Cliped By: {clip_by}, on {record_date}, #Shorts #Clips #Twitch Every Friday and Sunday @7:30 EST https://twitch.tv/teampgp"
|
||||||
|
categoryId = CategoryId.GAMING
|
||||||
|
privatcyStatus = 'private'
|
||||||
|
tags = ['shorts', 'gaming', 'TeamPGP', f'{game_name}', f'{clip_by}', 'twitch_clips', 'clips', 'Level1Techs', 'twitch']
|
||||||
|
|
||||||
|
tags.extend(top_hashtags({slug}))
|
||||||
|
|
||||||
|
output = uploader.upload_video(file_path, title, categoryId, description, privatcyStatus, tags)
|
||||||
|
|
||||||
|
if output is True:
|
||||||
|
print(f"Slug: {slug} | Was successfully uploaded.")
|
||||||
|
DB.mark_as_uploaded(slug)
|
||||||
|
else:
|
||||||
|
print(f"Slug: {slug} | Upload Process failed.")
|
||||||
|
|
||||||
def get_channel_clips(channel_name: str):
|
def get_channel_clips(channel_name: str):
|
||||||
"""Queries Twitch's public endpoint directly for trending clips."""
|
"""Queries Twitch's public endpoint directly for trending clips."""
|
||||||
session = requests.Session()
|
session = requests.Session()
|
||||||
@@ -174,7 +182,7 @@ def get_channel_clips(channel_name: str):
|
|||||||
|
|
||||||
print(f"Slug: {node['slug']} | Date: {node['createdAt']} | Game: {game_name} | By: {clip_by} | Views: {node['viewCount']} | Title: {node['title']}")
|
print(f"Slug: {node['slug']} | Date: {node['createdAt']} | Game: {game_name} | By: {clip_by} | Views: {node['viewCount']} | Title: {node['title']}")
|
||||||
|
|
||||||
insert_record(node['slug'], node['createdAt'], node['title'], game_name, clip_by, int(node['viewCount']))
|
DB.insert_clips_record(node['slug'], node['createdAt'], node['title'], game_name, clip_by, int(node['viewCount']))
|
||||||
slugs.append(slug_id)
|
slugs.append(slug_id)
|
||||||
|
|
||||||
return slugs
|
return slugs
|
||||||
@@ -184,7 +192,6 @@ def get_channel_clips(channel_name: str):
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
create_database()
|
|
||||||
get_channel_clips(CHANNEL_NAME)
|
get_channel_clips(CHANNEL_NAME)
|
||||||
download_clips()
|
download_clips()
|
||||||
close_database()
|
upload_clips()
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
import os
|
||||||
|
import json
|
||||||
|
import asyncio
|
||||||
|
import requests
|
||||||
|
from twitchAPI.twitch import Twitch
|
||||||
|
from twitchAPI.helper import first
|
||||||
|
|
||||||
|
import database
|
||||||
|
|
||||||
|
# 1. Fill in your credentials from the Twitch Developer Console
|
||||||
|
SECRETS = None
|
||||||
|
TWITCH = None
|
||||||
|
USER = None
|
||||||
|
CHANNEL_NAME = "teampgp"
|
||||||
|
|
||||||
|
async def get_twitch():
|
||||||
|
global SECRETS, TWITCH, USER
|
||||||
|
if SECRETS is None:
|
||||||
|
with open('twitch_secrets.json', 'r') as f:
|
||||||
|
SECRETS = json.load(f)
|
||||||
|
if TWITCH is None:
|
||||||
|
TWITCH = await Twitch(SECRETS['client_id'], SECRETS['client_secret'])
|
||||||
|
if USER is None:
|
||||||
|
USER = await first(TWITCH.get_users(logins=[CHANNEL_NAME]))
|
||||||
|
if not USER:
|
||||||
|
print("User not found.")
|
||||||
|
|
||||||
|
|
||||||
|
async def download_live_thumbnail(twitch_client, streamer_username: str, w: int = 1920, h: int = 1080):
|
||||||
|
"""Fetches and saves the live stream thumbnail for an active broadcast."""
|
||||||
|
print(f"🔎 Checking live status for: {streamer_username}...")
|
||||||
|
|
||||||
|
# Query the live streams endpoint
|
||||||
|
stream_generator = twitch_client.get_streams(user_logins=[streamer_username])
|
||||||
|
stream_data = await first(stream_generator)
|
||||||
|
|
||||||
|
if not stream_data:
|
||||||
|
print(f"❌ User '{streamer_username}' is offline. Live thumbnails require an active stream.")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Twitch API live streams use the {width} and {height} format
|
||||||
|
raw_url = stream_data.thumbnail_url
|
||||||
|
clean_url = raw_url.replace('{width}', str(w)).replace('{height}', str(h))
|
||||||
|
|
||||||
|
filename = f"live_{streamer_username}_{w}x{h}.jpg"
|
||||||
|
save_image(clean_url, filename)
|
||||||
|
|
||||||
|
async def download_vod_thumbnail(twitch_client, vod_id: str, w: int = 1920, h: int = 1080):
|
||||||
|
"""Fetches and saves a thumbnail from a past broadcast VOD ID."""
|
||||||
|
print(f"🔎 Searching for VOD ID: {vod_id}...")
|
||||||
|
|
||||||
|
# Query the videos endpoint
|
||||||
|
video_generator = twitch_client.get_videos(vod_id)
|
||||||
|
video_data = await first(video_generator)
|
||||||
|
|
||||||
|
filename = f"download/videos/{vod_id}/{vod_id}_{w}x{h}.jpg"
|
||||||
|
if os.path.exists(filename):
|
||||||
|
return
|
||||||
|
|
||||||
|
if not video_data:
|
||||||
|
print(f"❌ VOD ID {vod_id} could not be found.")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Twitch VOD endpoints typically format string tokens as %{width} and %{height}
|
||||||
|
raw_url = video_data.thumbnail_url
|
||||||
|
if not raw_url:
|
||||||
|
print("❌ This VOD does not have an available thumbnail.")
|
||||||
|
return
|
||||||
|
|
||||||
|
clean_url = raw_url.replace('%{width}', str(w)).replace('%{height}', str(h))
|
||||||
|
|
||||||
|
#filename = f"vod_{vod_id}_{w}x{h}.jpg"
|
||||||
|
save_image(clean_url, filename)
|
||||||
|
|
||||||
|
async def download_clip_thumbnail(clip_id: str, url: str):
|
||||||
|
print(f"🔎 Searching for Clip ID: {clip_id}...")
|
||||||
|
|
||||||
|
filename = f"download/clips/{clip_id}/{clip_id}.jpg"
|
||||||
|
if os.path.exists(filename):
|
||||||
|
return
|
||||||
|
|
||||||
|
save_image(url, filename, False)
|
||||||
|
|
||||||
|
def save_image(url: str, filename: str, stream: bool = True):
|
||||||
|
"""Helper function to stream image bytes directly to a file."""
|
||||||
|
try:
|
||||||
|
response = requests.get(url, stream)
|
||||||
|
if response.status_code == 200:
|
||||||
|
with open(filename, 'wb') as file:
|
||||||
|
if stream is True:
|
||||||
|
for chunk in response.iter_content(1024):
|
||||||
|
file.write(chunk)
|
||||||
|
else:
|
||||||
|
file.write(response.content)
|
||||||
|
print(f"✅ Success! Saved as: {filename}")
|
||||||
|
else:
|
||||||
|
print(f"❌ Download failed. HTTP Status: {response.status_code}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"❌ An error occurred during file writing: {e}")
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
# Initialize connection & automatically authorize the App token
|
||||||
|
await get_twitch()
|
||||||
|
twitch = TWITCH # Twitch(APP_ID, APP_SECRET)
|
||||||
|
|
||||||
|
# --- OPTION A: Download Live Thumbnail ---
|
||||||
|
# Target user must be streaming live right now
|
||||||
|
target_streamer = CHANNEL_NAME
|
||||||
|
#await download_live_thumbnail(twitch, target_streamer, 1920, 1080)
|
||||||
|
|
||||||
|
# --- OPTION B: Download Past Broadcast VOD Thumbnail ---
|
||||||
|
# Extract the ID sequence from your target video link
|
||||||
|
# target_vod = "2145678901"
|
||||||
|
db = database.Database()
|
||||||
|
vods = db.get_vods()
|
||||||
|
for vod in vods:
|
||||||
|
(
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
created_at,
|
||||||
|
view_count,
|
||||||
|
duration,
|
||||||
|
url,
|
||||||
|
thumbnail_url,
|
||||||
|
game_id,
|
||||||
|
game_name,
|
||||||
|
stream_id,
|
||||||
|
creator_name,
|
||||||
|
clip_is,
|
||||||
|
downloaded,
|
||||||
|
uploaded_yt,
|
||||||
|
uploaded_yt_chats,
|
||||||
|
uploaded_yt_shorts,
|
||||||
|
) = vod
|
||||||
|
|
||||||
|
await download_vod_thumbnail(twitch, id, 1920, 1080)
|
||||||
|
|
||||||
|
clips = db.get_clips()
|
||||||
|
for clip in clips:
|
||||||
|
(
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
created_at,
|
||||||
|
view_count,
|
||||||
|
duration,
|
||||||
|
url,
|
||||||
|
thumbnail_url,
|
||||||
|
game_id,
|
||||||
|
game_name,
|
||||||
|
stream_id,
|
||||||
|
creator_name,
|
||||||
|
clip_is,
|
||||||
|
downloaded,
|
||||||
|
uploaded_yt,
|
||||||
|
uploaded_yt_chats,
|
||||||
|
uploaded_yt_shorts,
|
||||||
|
) = clip
|
||||||
|
|
||||||
|
await download_clip_thumbnail(id, thumbnail_url)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
asyncio.run(main())
|
||||||
Executable
+108
@@ -0,0 +1,108 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import csv
|
||||||
|
import subprocess
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import linux
|
||||||
|
from database import Database
|
||||||
|
|
||||||
|
CHANNEL_NAME = 'teampgp'
|
||||||
|
DB = None
|
||||||
|
|
||||||
|
def write_csv(data, file_name):
|
||||||
|
# Open file with newline='' to prevent extra blank rows across platforms
|
||||||
|
with open(file_name, "w", newline="", encoding="utf-8") as file:
|
||||||
|
writer = csv.writer(file)
|
||||||
|
|
||||||
|
# Write all rows at once
|
||||||
|
writer.writerows(data)
|
||||||
|
|
||||||
|
def download():
|
||||||
|
"""Find all undownloaded videos and clips, and download them safely."""
|
||||||
|
undownloaded = DB.get_undownloaded()
|
||||||
|
|
||||||
|
print("Starting downloads...")
|
||||||
|
for row in undownloaded:
|
||||||
|
# Unpack variables clearly
|
||||||
|
(
|
||||||
|
id,
|
||||||
|
title,
|
||||||
|
created_at,
|
||||||
|
view_count,
|
||||||
|
duration,
|
||||||
|
url,
|
||||||
|
thumbnail_url,
|
||||||
|
game_id,
|
||||||
|
game_name,
|
||||||
|
stream_id,
|
||||||
|
creator_name,
|
||||||
|
clip_is,
|
||||||
|
downloaded,
|
||||||
|
uploaded_yt,
|
||||||
|
uploaded_yt_chats,
|
||||||
|
uploaded_yt_shorts,
|
||||||
|
) = row
|
||||||
|
|
||||||
|
# 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM)
|
||||||
|
#datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
data_timestamp = datetime.fromisoformat(created_at)
|
||||||
|
|
||||||
|
# 2. Get the exact current date and time
|
||||||
|
current_time = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
# 3. Calculate the difference between the two times
|
||||||
|
time_difference = current_time - data_timestamp
|
||||||
|
|
||||||
|
# 4. Check if the difference is greater than 24 hours
|
||||||
|
if time_difference > timedelta(hours=24):
|
||||||
|
print("The data is more than 24 hours old.")
|
||||||
|
else:
|
||||||
|
#lets wait 24 hours befor downloading
|
||||||
|
print("The data is less than 24 hours old.")
|
||||||
|
continue
|
||||||
|
|
||||||
|
data = [list(row)]
|
||||||
|
print(
|
||||||
|
f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# 1. Define paths and isolate base directory
|
||||||
|
if clip_is:
|
||||||
|
target_dir = f"download/clips/{id}"
|
||||||
|
cmd = f"TwitchDownloaderCLI clipdownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --temp-path download/temp"
|
||||||
|
else:
|
||||||
|
target_dir = f"download/videos/{id}"
|
||||||
|
# FIXED: Removed the duplicated command string combined with '&&'
|
||||||
|
cmd = f"TwitchDownloaderCLI videodownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
|
||||||
|
|
||||||
|
csv_file = f"{target_dir}/{id}.csv"
|
||||||
|
|
||||||
|
# 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
|
||||||
|
output = linux.run_command(cmd, look_for=["[STATUS]"])
|
||||||
|
output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
|
||||||
|
|
||||||
|
if output:
|
||||||
|
# Only write CSV and update database if download actually completed
|
||||||
|
write_csv(data, csv_file)
|
||||||
|
print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
|
||||||
|
DB.mark_as_downloaded(id)
|
||||||
|
else:
|
||||||
|
print(f"ID: {id} | Download Process failed.")
|
||||||
|
#print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
|
||||||
|
#if output.get("stderr"):
|
||||||
|
# print(f"Details: {output['stderr']}")
|
||||||
|
|
||||||
|
# Clear temporary chunk clutter immediately if a VOD crashes out
|
||||||
|
if not clip_is:
|
||||||
|
print("Flushing temporary crash chunks...")
|
||||||
|
subprocess.run("rm -rf download/temp/*", shell=True)
|
||||||
|
|
||||||
|
print("Finished Downloading Pipeline.")
|
||||||
|
|
||||||
|
def main():
|
||||||
|
global DB
|
||||||
|
DB = Database()
|
||||||
|
download()
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+102
@@ -0,0 +1,102 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
|
||||||
|
# The target Twitch streamer username
|
||||||
|
TWITCH_USERNAME = "SumGuyV5"
|
||||||
|
|
||||||
|
def load_secrets(filepath="twitch_secrets.json"):
|
||||||
|
"""Loads client credentials and potential manual token from JSON file."""
|
||||||
|
if not os.path.exists(filepath):
|
||||||
|
raise FileNotFoundError(f"Missing credential file: '{filepath}'")
|
||||||
|
|
||||||
|
with open(filepath, "r") as file:
|
||||||
|
secrets = json.load(file)
|
||||||
|
|
||||||
|
if "client_id" not in secrets or "client_secret" not in secrets:
|
||||||
|
raise KeyError("JSON file must contain 'client_id' and 'client_secret'.")
|
||||||
|
|
||||||
|
return secrets["client_id"], secrets["client_secret"], secrets.get("manual_token")
|
||||||
|
|
||||||
|
def get_app_access_token(client_id, client_secret):
|
||||||
|
"""Generates an App Access Token using the correct Twitch ID server."""
|
||||||
|
auth_url = "https://twitch.tv" # FIXED: Correct auth endpoint
|
||||||
|
payload = {
|
||||||
|
"client_id": client_id,
|
||||||
|
"client_secret": client_secret,
|
||||||
|
"grant_type": "client_credentials"
|
||||||
|
}
|
||||||
|
headers = {"Content-Type": "application/x-www-form-urlencoded"}
|
||||||
|
|
||||||
|
response = requests.post(auth_url, data=payload, headers=headers)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json()["access_token"]
|
||||||
|
|
||||||
|
def get_user_id(username, headers):
|
||||||
|
"""Retrieves the unique numerical Twitch User ID from Helix."""
|
||||||
|
url = f"https://twitch.tv{username}" # FIXED: Endpoint & parameter
|
||||||
|
response = requests.get(url, headers=headers)
|
||||||
|
response.raise_for_status()
|
||||||
|
data = response.json().get("data")
|
||||||
|
if data and len(data) > 0:
|
||||||
|
return data[0]["id"] # FIXED: Helix data array returns user dictionaries
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Twitch user '{username}' not found.")
|
||||||
|
|
||||||
|
def get_channel_vods(user_id, headers, limit=10):
|
||||||
|
"""Fetches past broadcasts (VODs) using valid Helix syntax."""
|
||||||
|
# FIXED: Restructured URL to use correct endpoint and standard query parameters
|
||||||
|
url = f"https://twitch.tv{user_id}&type=archive&first={limit}"
|
||||||
|
response = requests.get(url, headers=headers)
|
||||||
|
response.raise_for_status()
|
||||||
|
return response.json().get("data", [])
|
||||||
|
|
||||||
|
def main():
|
||||||
|
try:
|
||||||
|
# 1. Load credentials from external JSON file
|
||||||
|
client_id, client_secret, manual_token = load_secrets("twitch_secrets.json")
|
||||||
|
|
||||||
|
# 2. Assign or generate OAuth Access Token
|
||||||
|
if manual_token:
|
||||||
|
print("Using manual access token from JSON config file...")
|
||||||
|
access_token = manual_token
|
||||||
|
else:
|
||||||
|
print("No manual token found. Attempting to contact Twitch Auth Server...")
|
||||||
|
access_token = get_app_access_token(client_id, client_secret)
|
||||||
|
|
||||||
|
# 3. Setup Headers required by Twitch Helix API
|
||||||
|
headers = {
|
||||||
|
"Client-ID": client_id,
|
||||||
|
"Authorization": f"Bearer {access_token}"
|
||||||
|
}
|
||||||
|
|
||||||
|
# 4. Translate Username to User ID
|
||||||
|
user_id = get_user_id(TWITCH_USERNAME, headers)
|
||||||
|
print(f"Successfully retrieved ID for {TWITCH_USERNAME}: {user_id}\n")
|
||||||
|
|
||||||
|
# 5. Fetch and Print VOD details
|
||||||
|
vods = get_channel_vods(user_id, headers, limit=5)
|
||||||
|
|
||||||
|
if not vods:
|
||||||
|
print(f"No VODs found for {TWITCH_USERNAME}.")
|
||||||
|
return
|
||||||
|
|
||||||
|
print(f"--- Latest VODs for {TWITCH_USERNAME} ---")
|
||||||
|
for vod in vods:
|
||||||
|
print(f"Title: {vod['title']}")
|
||||||
|
print(f"URL: {vod['url']}")
|
||||||
|
print(f"Published At: {vod['published_at']}")
|
||||||
|
print(f"Duration: {vod['duration']}")
|
||||||
|
print(f"Views: {vod['view_count']}")
|
||||||
|
print("-" * 40)
|
||||||
|
|
||||||
|
except (FileNotFoundError, KeyError) as config_err:
|
||||||
|
print(f"Configuration Error: {config_err}")
|
||||||
|
except requests.exceptions.HTTPError as err:
|
||||||
|
print(f"HTTP Error detail: {err.response.text if err.response else err}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"An error occurred: {e}")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Executable
+185
@@ -0,0 +1,185 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import httpx # Switched from requests to prevent async loop freezing
|
||||||
|
import database
|
||||||
|
from twitchAPI.twitch import Twitch
|
||||||
|
from twitchAPI.helper import first
|
||||||
|
from twitchAPI.type import VideoType
|
||||||
|
|
||||||
|
SECRETS = None
|
||||||
|
TWITCH = None
|
||||||
|
USER = None
|
||||||
|
GAME_CACHE = {} # Local cache dictionary to store game_id -> game_name mapping
|
||||||
|
|
||||||
|
CHANNEL_NAME = "teampgp"
|
||||||
|
|
||||||
|
async def get_twitch():
|
||||||
|
global SECRETS, TWITCH, USER
|
||||||
|
if SECRETS is None:
|
||||||
|
with open('twitch_secrets.json', 'r') as f:
|
||||||
|
SECRETS = json.load(f)
|
||||||
|
if TWITCH is None:
|
||||||
|
TWITCH = await Twitch(SECRETS['client_id'], SECRETS['client_secret'])
|
||||||
|
if USER is None:
|
||||||
|
USER = await first(TWITCH.get_users(logins=[CHANNEL_NAME]))
|
||||||
|
if not USER:
|
||||||
|
print("User not found.")
|
||||||
|
|
||||||
|
async def get_game_name_by_id(game_id: str) -> str:
|
||||||
|
"""Helper function to fetch game names and cache them locally."""
|
||||||
|
if not game_id:
|
||||||
|
return "Unknown / No Category"
|
||||||
|
if game_id in GAME_CACHE:
|
||||||
|
return GAME_CACHE[game_id]
|
||||||
|
try:
|
||||||
|
game_generator = TWITCH.get_games(game_ids=[game_id])
|
||||||
|
game = await first(game_generator)
|
||||||
|
if game:
|
||||||
|
GAME_CACHE[game_id] = game.name
|
||||||
|
return game.name
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return "Unknown Game"
|
||||||
|
|
||||||
|
async def get_vod_game_name(vod_id: str):
|
||||||
|
"""Asynchronously query Twitch GQL endpoint for VOD game metadata."""
|
||||||
|
game_id = "0"
|
||||||
|
game_name = "Unknown Game"
|
||||||
|
url = "https://gql.twitch.tv/gql"
|
||||||
|
|
||||||
|
headers = {
|
||||||
|
"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
|
||||||
|
"Content-Type": "application/json"
|
||||||
|
}
|
||||||
|
|
||||||
|
payload = [{
|
||||||
|
"operationName": "VideoMetadata",
|
||||||
|
"variables": {
|
||||||
|
"channelLogin": "",
|
||||||
|
"videoID": str(vod_id)
|
||||||
|
},
|
||||||
|
"extensions": {
|
||||||
|
"persistedQuery": {
|
||||||
|
"version": 1,
|
||||||
|
"sha256Hash": "45111672eea2e507f8ba44d101a61862f9c56b11dee09a15634cb75cb9b9084d"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
|
||||||
|
# Using httpx async client to prevent blocking the asyncio event loop
|
||||||
|
async with httpx.AsyncClient() as client:
|
||||||
|
try:
|
||||||
|
response = await client.post(url, headers=headers, json=payload)
|
||||||
|
if response.status_code == 200:
|
||||||
|
data = response.json()
|
||||||
|
video_info = data[0].get('data', {}).get('video')
|
||||||
|
if video_info and video_info.get('game'):
|
||||||
|
game_id = str(video_info['game']['id'])
|
||||||
|
game_name = video_info['game']['displayName']
|
||||||
|
print(f"GQL Found: {game_name} (ID: {game_id})")
|
||||||
|
else:
|
||||||
|
print(f"No game information found in GQL for VOD {vod_id}.")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error fetching GQL metadata for VOD {vod_id}: {e}")
|
||||||
|
|
||||||
|
return game_id, game_name
|
||||||
|
|
||||||
|
async def get_streamer_vods():
|
||||||
|
await get_twitch()
|
||||||
|
|
||||||
|
print(f"Starting VOD extraction for {USER.display_name}...")
|
||||||
|
vod_generator = TWITCH.get_videos(user_id=USER.id, first=100, video_type=VideoType.ALL)
|
||||||
|
|
||||||
|
all_vods = []
|
||||||
|
|
||||||
|
async for v in vod_generator:
|
||||||
|
# Resolving game category safely without freezing the event loop
|
||||||
|
game_id, game_name = await get_vod_game_name(v.id)
|
||||||
|
|
||||||
|
# Safely convert game_id to integer if possible, otherwise default to 0
|
||||||
|
try:
|
||||||
|
clean_game_id = game_id
|
||||||
|
except ValueError:
|
||||||
|
clean_game_id = 0
|
||||||
|
|
||||||
|
vod_data = {
|
||||||
|
"id": v.id,
|
||||||
|
"title": v.title,
|
||||||
|
"created_at": str(v.published_at),
|
||||||
|
"view_count": int(v.view_count) if v.view_count else 0,
|
||||||
|
"duration": v.duration,
|
||||||
|
"url": v.url,
|
||||||
|
"thumbnail_url": v.thumbnail_url,
|
||||||
|
"game_id": clean_game_id,
|
||||||
|
"game_name": game_name,
|
||||||
|
"stream_id": str(v.stream_id) if v.stream_id else "0",
|
||||||
|
"creator_name": CHANNEL_NAME,
|
||||||
|
"clip_is": False,
|
||||||
|
}
|
||||||
|
all_vods.append(vod_data)
|
||||||
|
print(f"Collected VOD: {v.title} | Category: {game_name} ({v.duration})")
|
||||||
|
|
||||||
|
print(f"\nFinished extracting VODs. Total gathered: {len(all_vods)}")
|
||||||
|
return all_vods
|
||||||
|
|
||||||
|
async def get_streamer_clips():
|
||||||
|
await get_twitch()
|
||||||
|
|
||||||
|
print(f"Starting clip extraction for {USER.display_name}...")
|
||||||
|
clip_generator = TWITCH.get_clips(broadcaster_id=USER.id, first=100)
|
||||||
|
all_clips = []
|
||||||
|
|
||||||
|
async for c in clip_generator:
|
||||||
|
game_name = await get_game_name_by_id(c.game_id)
|
||||||
|
|
||||||
|
clip_data = {
|
||||||
|
"id": c.id,
|
||||||
|
"title": c.title,
|
||||||
|
"created_at": str(c.created_at),
|
||||||
|
"view_count": int(c.view_count) if c.view_count else 0,
|
||||||
|
"duration": c.duration,
|
||||||
|
"url": c.url,
|
||||||
|
"thumbnail_url": c.thumbnail_url,
|
||||||
|
"game_id": c.game_id,
|
||||||
|
"game_name": game_name,
|
||||||
|
"stream_id": "0",
|
||||||
|
"creator_name": c.creator_name,
|
||||||
|
"clip_is": True,
|
||||||
|
}
|
||||||
|
all_clips.append(clip_data)
|
||||||
|
print(f"Collected clip: {c.title} | Category: {game_name} ({c.view_count} views)")
|
||||||
|
|
||||||
|
print(f"\nFinished extracting clips. Total gathered: {len(all_clips)}")
|
||||||
|
return all_clips
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
print("--- Script Started ---")
|
||||||
|
|
||||||
|
db = database.Database()
|
||||||
|
vods_list = await get_streamer_vods()
|
||||||
|
|
||||||
|
for v in vods_list:
|
||||||
|
db.insert_video_record(
|
||||||
|
v['id'], v['title'], v['created_at'], v['view_count'], v['duration'],
|
||||||
|
v['url'], v['thumbnail_url'], v['game_id'], v['game_name'],
|
||||||
|
v['stream_id'], v['creator_name'], v['clip_is']
|
||||||
|
)
|
||||||
|
|
||||||
|
if vods_list:
|
||||||
|
print(f"Recent VOD: '{vods_list[0]['title']}'")
|
||||||
|
|
||||||
|
clips_list = await get_streamer_clips()
|
||||||
|
|
||||||
|
for v in clips_list:
|
||||||
|
db.insert_video_record(
|
||||||
|
v['id'], v['title'], v['created_at'], v['view_count'], v['duration'],
|
||||||
|
v['url'], v['thumbnail_url'], v['game_id'], v['game_name'],
|
||||||
|
v['stream_id'], v['creator_name'], v['clip_is']
|
||||||
|
)
|
||||||
|
|
||||||
|
if clips_list:
|
||||||
|
print(f"Recent VOD: '{clips_list[0]['title']}'")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(main())
|
||||||
Regular → Executable
+95
-15
@@ -2,6 +2,8 @@
|
|||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import argparse
|
import argparse
|
||||||
|
from enum import Enum
|
||||||
|
from datetime import datetime
|
||||||
from google.oauth2.credentials import Credentials
|
from google.oauth2.credentials import Credentials
|
||||||
from google_auth_oauthlib.flow import InstalledAppFlow
|
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||||
from google.auth.transport.requests import Request
|
from google.auth.transport.requests import Request
|
||||||
@@ -9,6 +11,25 @@ from googleapiclient.discovery import build
|
|||||||
from googleapiclient.http import MediaFileUpload
|
from googleapiclient.http import MediaFileUpload
|
||||||
from googleapiclient.errors import HttpError
|
from googleapiclient.errors import HttpError
|
||||||
|
|
||||||
|
class CategoryId(Enum):
|
||||||
|
"""Official YouTube Category IDs for API Uploads."""
|
||||||
|
|
||||||
|
FILM_AND_ANIMATION = "1"
|
||||||
|
AUTOS_AND_VEHICLES = "2"
|
||||||
|
MUSIC = "10"
|
||||||
|
PETS_AND_ANIMALS = "15"
|
||||||
|
SPORTS = "17"
|
||||||
|
TRAVEL_AND_EVENTS = "19"
|
||||||
|
GAMING = "20"
|
||||||
|
PEOPLE_AND_BLOGS = "22"
|
||||||
|
COMEDY = "23"
|
||||||
|
ENTERTAINMENT = "24"
|
||||||
|
NEWS_AND_POLITICS = "25"
|
||||||
|
HOWTO_AND_STYLE = "26"
|
||||||
|
EDUCATION = "27"
|
||||||
|
SCIENCE_AND_TECHNOLOGY = "28"
|
||||||
|
NONPROFITS_AND_ACTIVISM = "29"
|
||||||
|
|
||||||
def load_credentials():
|
def load_credentials():
|
||||||
"""Load credentials from secrets.json"""
|
"""Load credentials from secrets.json"""
|
||||||
try:
|
try:
|
||||||
@@ -36,18 +57,28 @@ def load_credentials():
|
|||||||
print(f"Error loading credentials: {str(e)}")
|
print(f"Error loading credentials: {str(e)}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def upload_video(file_path, description):
|
def upload_video(file_path: str, title: str, category: CategoryId, description: str = "", privacyStatus: str = 'private', tags: list = None, release_time: datetime = None):
|
||||||
"""
|
"""
|
||||||
Upload a video to YouTube
|
Upload a video to YouTube
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
file_path (str): Path to the video file
|
file_path (str): Path to the video file
|
||||||
|
title (str): Title of the Video
|
||||||
|
categoryId (str): categoryId of the video
|
||||||
description (str): Video description
|
description (str): Video description
|
||||||
|
privacyStatus (str): privacyStatus of the video
|
||||||
|
tags (str): tags to be use on the video
|
||||||
|
release_time (datetime): Optional timezone-aware UTC datetime object for timed release
|
||||||
|
|
||||||
|
schedule_date = datetime.now(timezone.utc) + timedelta(days=2)
|
||||||
"""
|
"""
|
||||||
|
if tags is None:
|
||||||
|
tags = []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Check if file exists
|
# Check if file exists
|
||||||
if not os.path.exists(file_path):
|
if not os.path.exists(file_path):
|
||||||
print(f"Error: File not found: {file_path}")
|
print(f"❌ Error: File not found: {file_path}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Load credentials
|
# Load credentials
|
||||||
@@ -58,21 +89,33 @@ def upload_video(file_path, description):
|
|||||||
# Create YouTube API client
|
# Create YouTube API client
|
||||||
youtube = build('youtube', 'v3', credentials=credentials)
|
youtube = build('youtube', 'v3', credentials=credentials)
|
||||||
|
|
||||||
# Get the filename without extension as default title
|
# Configure the status object dynamically
|
||||||
title = os.path.splitext(os.path.basename(file_path))[0]
|
status_body = {
|
||||||
|
'selfDeclaredMadeForKids': False
|
||||||
|
}
|
||||||
|
|
||||||
|
if release_time is not None:
|
||||||
|
if release_time.tzinfo is None:
|
||||||
|
release_time = release_time.replace(tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
# If release_time is passed, YouTube forces privacyStatus to 'private'
|
||||||
|
status_body['privacyStatus'] = 'private'
|
||||||
|
status_body['publishAt'] = release_time.strftime('%Y-%m-%dT%H:%M:%S.000Z')
|
||||||
|
print(f"Configuring timed release for: {status_body['publishAt']}")
|
||||||
|
else:
|
||||||
|
# Standard immediate upload
|
||||||
|
status_body['privacyStatus'] = privacyStatus
|
||||||
|
print(f"Configuring immediate upload with status: {privacyStatus}")
|
||||||
|
|
||||||
# Prepare the video upload request
|
# Prepare the video upload request
|
||||||
body = {
|
body = {
|
||||||
'snippet': {
|
'snippet': {
|
||||||
'title': title,
|
'title': title,
|
||||||
'description': description,
|
'description': description,
|
||||||
'tags': [],
|
'tags': tags,
|
||||||
'categoryId': '22' # Default to 'People & Blogs' category
|
'categoryId': category.value
|
||||||
},
|
},
|
||||||
'status': {
|
'status': status_body
|
||||||
'privacyStatus': 'private', # Default to private
|
|
||||||
'selfDeclaredMadeForKids': False
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# Create media file upload
|
# Create media file upload
|
||||||
@@ -89,7 +132,7 @@ def upload_video(file_path, description):
|
|||||||
media_body=media
|
media_body=media
|
||||||
)
|
)
|
||||||
|
|
||||||
print("Starting upload...")
|
print(f"Starting upload for '{title}'...")
|
||||||
response = None
|
response = None
|
||||||
while response is None:
|
while response is None:
|
||||||
status, response = insert_request.next_chunk()
|
status, response = insert_request.next_chunk()
|
||||||
@@ -100,6 +143,13 @@ def upload_video(file_path, description):
|
|||||||
print(f"Video ID: {response['id']}")
|
print(f"Video ID: {response['id']}")
|
||||||
print(f"Title: {response['snippet']['title']}")
|
print(f"Title: {response['snippet']['title']}")
|
||||||
print(f"URL: https://youtu.be/{response['id']}")
|
print(f"URL: https://youtu.be/{response['id']}")
|
||||||
|
|
||||||
|
# Output confirmation based on what was chosen
|
||||||
|
if 'publishAt' in response['status']:
|
||||||
|
print(f"Scheduled Release Time: {response['status']['publishAt']}")
|
||||||
|
else:
|
||||||
|
print(f"Current Privacy Status: {response['status']['privacyStatus']}")
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
except HttpError as e:
|
except HttpError as e:
|
||||||
@@ -111,11 +161,41 @@ def upload_video(file_path, description):
|
|||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(description='Upload a video to YouTube')
|
parser = argparse.ArgumentParser(description='Upload a video to YouTube')
|
||||||
parser.add_argument('file', help='Path to the video file')
|
parser.add_argument('--file', required=True, help='Path to the video file')
|
||||||
parser.add_argument('description', help='Video description')
|
parser.add_argument('--title', required=True, help='Title of the video')
|
||||||
|
parser.add_argument('--category', default='PEOPLE_AND_BLOGS', choices=[c.name for c in CategoryId], help='Video category genre')
|
||||||
|
parser.add_argument('--description', default='', help='Video description text')
|
||||||
|
parser.add_argument('--privacy', default='private', choices=['public', 'private', 'unlisted'], help='Video privacy settings')
|
||||||
|
|
||||||
|
# ADDED: Feature parsing to easily pass tags from the CLI split by commas
|
||||||
|
parser.add_argument('--tags', default='', help='Comma-separated tags list (e.g. "python,coding,api")')
|
||||||
|
|
||||||
|
# ADDED: Option to provide a scheduled upload timestamp natively from CLI
|
||||||
|
parser.add_argument('--schedule', default=None, help='UTC Release date/time in ISO format: YYYY-MM-DDTHH:MM:SS (e.g. 2026-08-15T14:30:00)')
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
upload_video(args.file, args.description)
|
|
||||||
|
chosen_category = CategoryId[args.category]
|
||||||
|
parsed_tags = [t.strip() for t in args.tags.split(',')] if args.tags else []
|
||||||
|
|
||||||
|
# ADDED: Parse schedule string into datetime object dynamically
|
||||||
|
release_datetime = None
|
||||||
|
if args.schedule:
|
||||||
|
try:
|
||||||
|
# Assumes format matches CLI help instruction text
|
||||||
|
release_datetime = datetime.strptime(args.schedule, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc)
|
||||||
|
except ValueError:
|
||||||
|
print("Error: Schedule date must format strictly as YYYY-MM-DDTHH:MM:SS")
|
||||||
|
return
|
||||||
|
|
||||||
|
upload_video(
|
||||||
|
file_path=args.file,
|
||||||
|
title=args.title,
|
||||||
|
category=chosen_category,
|
||||||
|
description=args.description,
|
||||||
|
privacyStatus=args.privacy,
|
||||||
|
tags=parsed_tags,
|
||||||
|
release_time=release_datetime
|
||||||
|
)
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
Regular → Executable
+1
-1
@@ -183,7 +183,7 @@ def get_clip_slugs(channel_name):
|
|||||||
return clip_slugs
|
return clip_slugs
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"An unexpected error occurred: {e}")
|
print(f"❌ An unexpected error occurred: {e}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
Regular → Executable
+1
@@ -1,3 +1,4 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import requests
|
import requests
|
||||||
|
|||||||
-195
@@ -1,195 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
import requests
|
|
||||||
|
|
||||||
import sqlite3
|
|
||||||
from datetime import date as datetime_date
|
|
||||||
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
|
||||||
|
|
||||||
CONN = sqlite3.connect("database.db")
|
|
||||||
CURSOR = CONN.cursor()
|
|
||||||
|
|
||||||
def create_database():
|
|
||||||
# Creates table safely using multi-line string
|
|
||||||
CURSOR.execute(
|
|
||||||
"""
|
|
||||||
CREATE TABLE IF NOT EXISTS vods (
|
|
||||||
id INTEGER PRIMARY KEY,
|
|
||||||
date TEXT NOT NULL,
|
|
||||||
title TEXT NOT NULL,
|
|
||||||
gamename TEXT NOT NULL,
|
|
||||||
downloaded INTEGER NOT NULL,
|
|
||||||
uploaded_yt INTEGER NOT NULL
|
|
||||||
)
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
|
|
||||||
CONN.commit()
|
|
||||||
|
|
||||||
def close_database():
|
|
||||||
"""Commit before we close."""
|
|
||||||
CONN.commit()
|
|
||||||
CONN.close()
|
|
||||||
|
|
||||||
def download_vods():
|
|
||||||
"""Find all undownload vods and download them."""
|
|
||||||
undownload_vods = get_undownloaded_vods()
|
|
||||||
|
|
||||||
print("Download VODs...")
|
|
||||||
|
|
||||||
for record_id, record_date, title, game_name, downloaded in undownload_vods:
|
|
||||||
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
|
|
||||||
output = run_linux_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/{record_id}/{record_id}.mp4")
|
|
||||||
output2 = run_linux_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/{record_id}/{record_id}_chat.json -E")
|
|
||||||
if output["success"] is True:
|
|
||||||
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was successful")
|
|
||||||
mark_as_downloaded(record_id)
|
|
||||||
else:
|
|
||||||
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was Failed")
|
|
||||||
|
|
||||||
print("Finished Downloading VODs...")
|
|
||||||
|
|
||||||
def run_linux_command(command: str):
|
|
||||||
"""Executes a Linux command, waits for completion, and returns output."""
|
|
||||||
try:
|
|
||||||
# shell=True allows running full command strings with pipes/wildcards
|
|
||||||
# text=True returns strings instead of bytes
|
|
||||||
result = subprocess.run(
|
|
||||||
command, shell=True, check=True, capture_output=True, text=True
|
|
||||||
)
|
|
||||||
|
|
||||||
return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
|
|
||||||
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
# Handles errors if the Linux command returns a non-zero exit code
|
|
||||||
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
|
|
||||||
|
|
||||||
def mark_as_downloaded(record_id: int):
|
|
||||||
"""Updates the downloaded status to True (1) for a specific record ID."""
|
|
||||||
|
|
||||||
# Updates the row matching the specific ID
|
|
||||||
CURSOR.execute(
|
|
||||||
"UPDATE vods SET downloaded = 1 WHERE id = ?",
|
|
||||||
(record_id,)
|
|
||||||
)
|
|
||||||
|
|
||||||
CONN.commit()
|
|
||||||
|
|
||||||
def get_undownloaded_vods():
|
|
||||||
"""Retrieves all rows where downloaded status is False (0)."""
|
|
||||||
|
|
||||||
# Query filters by 0 because SQLite stores booleans as integers
|
|
||||||
CURSOR.execute(
|
|
||||||
"SELECT id, date, title, gamename, downloaded FROM vods WHERE downloaded = 0"
|
|
||||||
)
|
|
||||||
records = CURSOR.fetchall()
|
|
||||||
|
|
||||||
return records
|
|
||||||
|
|
||||||
def insert_record(record_id: int, record_date: datetime_date, title: str, gamename: str):
|
|
||||||
"""Inserts a record with ID, date, gamename, and title into a SQLite database."""
|
|
||||||
# Connects to database file (creates it if missing)
|
|
||||||
|
|
||||||
# Inserts data using parameterized queries to prevent SQL injection
|
|
||||||
CURSOR.execute(
|
|
||||||
"INSERT OR IGNORE INTO vods (id, date, title, gamename, downloaded, uploaded_yt) VALUES (?, ?, ?, ?, ?, ?)",
|
|
||||||
(record_id, str(record_date), title, gamename, False, False),
|
|
||||||
)
|
|
||||||
|
|
||||||
# Saves changes and closes the connection
|
|
||||||
CONN.commit()
|
|
||||||
|
|
||||||
def get_vod_ids_simplified(channel_name: str):
|
|
||||||
session = requests.Session()
|
|
||||||
url = "https://gql.twitch.tv/gql"
|
|
||||||
|
|
||||||
# Case-preserved headers to prevent 400 Bad Request errors
|
|
||||||
session.headers = {
|
|
||||||
"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
|
|
||||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
||||||
"Content-Type": "text/plain"
|
|
||||||
}
|
|
||||||
|
|
||||||
# A simplified query string that hardcodes the ARCHIVE type filter into the request structure
|
|
||||||
query_string = """
|
|
||||||
query GetChannelVideos($login: String!, $limit: Int!) {
|
|
||||||
user(login: $login) {
|
|
||||||
videos(first: $limit, types: [ARCHIVE]) {
|
|
||||||
edges {
|
|
||||||
node {
|
|
||||||
id
|
|
||||||
title
|
|
||||||
publishedAt
|
|
||||||
game {
|
|
||||||
displayName
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
payload = [{
|
|
||||||
"operationName": "GetChannelVideos",
|
|
||||||
"query": query_string,
|
|
||||||
"variables": {
|
|
||||||
"login": channel_name.lower(),
|
|
||||||
"limit": 50
|
|
||||||
}
|
|
||||||
}]
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Prepping ensures Python does not rewrite the Client-ID header case
|
|
||||||
req = requests.Request('POST', url, json=payload)
|
|
||||||
prepped = session.prepare_request(req)
|
|
||||||
|
|
||||||
response = session.send(prepped)
|
|
||||||
response.raise_for_status()
|
|
||||||
|
|
||||||
data = response.json()
|
|
||||||
|
|
||||||
# Pull out the target index array dictionary object
|
|
||||||
result = data[0] if isinstance(data, list) else data
|
|
||||||
|
|
||||||
if "errors" in result:
|
|
||||||
print(f"Twitch GraphQL Error: {result['errors']}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
user_data = result['data']['user']
|
|
||||||
if not user_data:
|
|
||||||
print(f"Channel '{channel_name}' not found.")
|
|
||||||
return []
|
|
||||||
|
|
||||||
edges = user_data['videos']['edges']
|
|
||||||
vod_ids = []
|
|
||||||
|
|
||||||
print(f"--- Latest VODs for {channel_name} ---")
|
|
||||||
for edge in edges:
|
|
||||||
node = edge['node']
|
|
||||||
|
|
||||||
# Safe extraction in case a VOD has no category set (Just Chatting, Uncategorized, etc.)
|
|
||||||
game_info = node.get('game')
|
|
||||||
game_name = game_info.get('displayName') if game_info else "Unknown/No Category"
|
|
||||||
|
|
||||||
print(f"ID: {node['id']} | Date: {node['publishedAt']} | Game: {game_name} | Title: {node['title']}")
|
|
||||||
|
|
||||||
# Pass game_name to your database logic
|
|
||||||
insert_record(node['id'], node['publishedAt'], node['title'], game_name)
|
|
||||||
vod_ids.append(node['id'])
|
|
||||||
|
|
||||||
return vod_ids
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
print(f"An unexpected error occurred: {e}")
|
|
||||||
if 'response' in locals():
|
|
||||||
print(f"Server Response Text: {response.text}")
|
|
||||||
return []
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
create_database()
|
|
||||||
get_vod_ids_simplified(CHANNEL_NAME)
|
|
||||||
download_vods()
|
|
||||||
close_database()
|
|
||||||
Executable
+76
@@ -0,0 +1,76 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import re
|
||||||
|
import json
|
||||||
|
import nltk
|
||||||
|
import string
|
||||||
|
from collections import Counter
|
||||||
|
from nltk.corpus import stopwords
|
||||||
|
from nltk.tokenize import word_tokenize
|
||||||
|
|
||||||
|
# Download necessary NLTK data modules
|
||||||
|
nltk.download('punkt', quiet=True)
|
||||||
|
nltk.download('stopwords', quiet=True)
|
||||||
|
nltk.download('punkt_tab', quiet=True)
|
||||||
|
nltk.download('averaged_perceptron_tagger', quiet=True) # Required for POS tagging
|
||||||
|
nltk.download('averaged_perceptron_tagger_eng', quiet=True)
|
||||||
|
|
||||||
|
def extract_text_from_srt(file_path: str):
|
||||||
|
with open(file_path, 'r', encoding='utf-8') as file:
|
||||||
|
content = file.read()
|
||||||
|
# Remove SRT timestamps and sequence numbers
|
||||||
|
clean_text = re.sub(r'\d+\n\d{2}:\d{2}:\d{2},\d{3} --> \d{2}:\d{2}:\d{2},\d{3}\n', '', content)
|
||||||
|
clean_text = re.sub(r'\d+', '', clean_text)
|
||||||
|
return clean_text
|
||||||
|
|
||||||
|
def extract_text_from_json(file_path: str):
|
||||||
|
with open(file_path, "r", encoding="utf-8") as file:
|
||||||
|
data = json.load(file)
|
||||||
|
|
||||||
|
messages = []
|
||||||
|
for comment in data.get("comments", []):
|
||||||
|
message_text = comment.get("message", {}).get("body", "")
|
||||||
|
messages.append(message_text)
|
||||||
|
|
||||||
|
# Return a single merged string of all chat text
|
||||||
|
return " ".join(messages)
|
||||||
|
|
||||||
|
def get_top_nouns(file_path: str, top_n: int = 10):
|
||||||
|
# 1. Extract raw text
|
||||||
|
if file_path.endswith(".srt"):
|
||||||
|
raw_text = extract_text_from_srt(file_path)
|
||||||
|
else:
|
||||||
|
raw_text = extract_text_from_json(file_path)
|
||||||
|
|
||||||
|
# 2. Basic cleanup (Keep original case for proper noun accuracy)
|
||||||
|
# Strip basic punctuation but leave words intact
|
||||||
|
clean_text = raw_text.translate(str.maketrans('', '', string.punctuation))
|
||||||
|
|
||||||
|
# 3. Tokenize words
|
||||||
|
words = word_tokenize(clean_text)
|
||||||
|
|
||||||
|
# 4. Part-of-Speech Tagging
|
||||||
|
tagged_words = nltk.pos_tag(words)
|
||||||
|
|
||||||
|
# 5. Filter for Nouns (NN = Singular Noun, NNP = Proper Noun, NNS = Plural Noun)
|
||||||
|
stop_words = set(stopwords.words('english'))
|
||||||
|
nouns = []
|
||||||
|
|
||||||
|
for word, tag in tagged_words:
|
||||||
|
word_lower = word.lower()
|
||||||
|
# Filter out short fragments and standard stopwords
|
||||||
|
if tag in ['NN', 'NNP', 'NNS'] and len(word_lower) > 2 and word_lower not in stop_words:
|
||||||
|
nouns.append(word_lower)
|
||||||
|
|
||||||
|
# 6. Count frequencies
|
||||||
|
noun_counts = Counter(nouns)
|
||||||
|
return [noun for noun, count in noun_counts.most_common(top_n)]
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
# Example usage
|
||||||
|
# Replace 'your_video.srt' with the path to your file
|
||||||
|
results = get_top_nouns('download/videos/2813112936/2813112936.srt', top_n=10)
|
||||||
|
print("Trending Hashtags SRT:", results)
|
||||||
|
|
||||||
|
results = get_top_nouns("download/videos/2813112936/2813112936_chat.json", top_n=10)
|
||||||
|
print("Trending Hashtags JSON:", results)
|
||||||
Executable
+143
@@ -0,0 +1,143 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import linux
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
from moviepy import VideoFileClip, ColorClip, CompositeVideoClip
|
||||||
|
from moviepy.video.VideoClip import TextClip
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import numpy as np
|
||||||
|
from PIL import Image, ImageFilter
|
||||||
|
|
||||||
|
def get_video_info(input_path: Path) -> tuple:
|
||||||
|
"""Uses ffprobe to instantly read input video dimensions and frame rate."""
|
||||||
|
cmd = f"ffprobe -v error -select_streams v:0 -show_entries stream=width,height,r_frame_rate -of json {input_path}"
|
||||||
|
# Run command and capture output (assumes linux.run_command prints or you use subprocess)
|
||||||
|
import subprocess
|
||||||
|
result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||||
|
#linux.run_command(cmd)
|
||||||
|
try:
|
||||||
|
data = json.loads(result.stdout)
|
||||||
|
stream = data['streams'][0]
|
||||||
|
w = int(stream['width'])
|
||||||
|
h = int(stream['height'])
|
||||||
|
# Convert fractional FPS string (e.g. "60/1" or "30000/1001") to float
|
||||||
|
fps_parts = stream['r_frame_rate'].split('/')
|
||||||
|
fps = float(fps_parts[0]) / float(fps_parts[1]) if len(fps_parts) > 1 else float(fps_parts[0])
|
||||||
|
return w, h, fps
|
||||||
|
except Exception:
|
||||||
|
return 1920, 1080, 60.0 # Safe defaults if probe fails
|
||||||
|
|
||||||
|
def fit_to_9_16_letterbox(input_path: str, top_text: str = "TOP TEXT", bottom_text: str = "BOTTOM TEXT", use_blur: bool = True, force: bool = False):
|
||||||
|
threads = "8"
|
||||||
|
input_file = Path(input_path)
|
||||||
|
output_suffix = "gaussian_9_16" if use_blur else "black_9_16"
|
||||||
|
output_path = input_file.parent / f"{input_file.stem}_{output_suffix}{input_file.suffix}"
|
||||||
|
|
||||||
|
if os.path.exists(output_path):
|
||||||
|
if not force:
|
||||||
|
print(f"❌ Error: Short video already exists: {output_path}")
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
os.remove(output_path)
|
||||||
|
|
||||||
|
# 1. Probe input metadata instantly
|
||||||
|
orig_w, orig_h, fps = get_video_info(input_file)
|
||||||
|
canvas_w = 1080
|
||||||
|
canvas_h = 1920
|
||||||
|
|
||||||
|
print("✍️ Generating text overlay graphics via MoviePy...")
|
||||||
|
# Render static images for text instead of running a video context
|
||||||
|
title_clip = TextClip(
|
||||||
|
text=top_text, font_size=55, color="white", font="DejaVuSans-Bold",
|
||||||
|
text_align="center", size=(canvas_w - 100, 300), method="caption"
|
||||||
|
)
|
||||||
|
bottom_clip = TextClip(
|
||||||
|
text=bottom_text, font_size=55, color="white", font="DejaVuSans-Bold",
|
||||||
|
text_align="center", size=(canvas_w - 100, 300), method="caption"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Save text layers to temporary PNGs
|
||||||
|
top_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
|
||||||
|
bottom_png = tempfile.NamedTemporaryFile(suffix=".png", delete=False).name
|
||||||
|
title_clip.save_frame(top_png)
|
||||||
|
bottom_clip.save_frame(bottom_png)
|
||||||
|
|
||||||
|
title_clip.close()
|
||||||
|
bottom_clip.close()
|
||||||
|
|
||||||
|
print("🎬 Dispatching compilation workload to FFmpeg filtergraph...")
|
||||||
|
|
||||||
|
# 2. Build the complex FFmpeg filtergraph
|
||||||
|
# [0:v] is the raw input video stream
|
||||||
|
filter_complex = []
|
||||||
|
|
||||||
|
if use_blur:
|
||||||
|
# Scale height to 1920, crop center 1080x1920, apply fast boxblur (power of 3 approximates Gaussian)
|
||||||
|
filter_complex.append(
|
||||||
|
f"[0:v]scale=-1:{canvas_h},crop={canvas_w}:{canvas_h}:(iw-{canvas_w})/2:0,boxblur=luma_radius=35:luma_power=3[bg];"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# Generate a pure black background canvas matching video frame specs
|
||||||
|
filter_complex.append(
|
||||||
|
f"color=c=black:s={canvas_w}x{canvas_h}:r={fps}[bg];"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Scale the foreground video to a clean 1080 width, keeping aspect ratio
|
||||||
|
filter_complex.append(
|
||||||
|
f"[0:v]scale={canvas_w}:-1[fg];"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Layer composition chain:
|
||||||
|
# Overlay 1: Put scaled foreground onto background (centered vertically)
|
||||||
|
filter_complex.append(
|
||||||
|
f"[bg][fg]overlay=0:(H-h)/2[tmp1];"
|
||||||
|
)
|
||||||
|
# Overlay 2: Drop top text asset onto position Y=180
|
||||||
|
filter_complex.append(
|
||||||
|
f"[tmp1][1:v]overlay=(W-w)/2:180[tmp2];"
|
||||||
|
)
|
||||||
|
# Overlay 3: Drop bottom text asset onto position Y=1430
|
||||||
|
filter_complex.append(
|
||||||
|
f"[tmp2][2:v]overlay=(W-w)/2:1430[finalv]"
|
||||||
|
)
|
||||||
|
|
||||||
|
filter_graph = "".join(filter_complex)
|
||||||
|
|
||||||
|
# 3. Execute the native assembly command
|
||||||
|
# -map_chapters -1 -sn: Strips unnecessary metadata chunks instantly
|
||||||
|
# -c:a copy: Safely pulls original digital audio directly without decompression cycles
|
||||||
|
# -threads 0: Forces FFmpeg to auto-consume all available processing cores
|
||||||
|
ffmpeg_cmd = (
|
||||||
|
f'ffmpeg -y -v error -i "{input_file}" -i "{top_png}" -i "{bottom_png}" '
|
||||||
|
f'-filter_complex "{filter_graph}" '
|
||||||
|
f'-map "[finalv]" -map 0:a? -c:v libx264 -crf 18 -preset slow -pix_fmt yuv420p '
|
||||||
|
f'-c:a copy -map_chapters -1 -sn -threads {threads} "{output_path}"'
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
linux.run_command(ffmpeg_cmd)
|
||||||
|
print(f"🎉 High-speed processing complete! Video saved to: {output_path}")
|
||||||
|
finally:
|
||||||
|
# Clean up temporary PNG picture files safely
|
||||||
|
for path in (top_png, bottom_png):
|
||||||
|
if os.path.exists(path):
|
||||||
|
os.unlink(path)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
import time
|
||||||
|
|
||||||
|
start_time = time.perf_counter()
|
||||||
|
|
||||||
|
creator_name = "greenskiesbluegrass"
|
||||||
|
top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
|
||||||
|
bottom_txt = f"Clipped By: {creator_name}."
|
||||||
|
fit_to_9_16_letterbox("download/clips/AbnegateAgitatedGrassPJSalt/AbnegateAgitatedGrassPJSalt.mp4", top_txt, bottom_txt, True, True)
|
||||||
|
|
||||||
|
end_time = time.perf_counter()
|
||||||
|
execution_time = end_time - start_time
|
||||||
|
|
||||||
|
print(f"The function took {execution_time:.6f} seconds to complete.")
|
||||||
Reference in New Issue
Block a user