Compare commits

...
6 Commits
18 changed files with 658 additions and 501 deletions
+2
View File
@@ -4,3 +4,5 @@
database.db
save
__pycache__
twitch_secrets.json
.vscode/settings.json
Executable
+74
View File
@@ -0,0 +1,74 @@
#!/usr/bin/env python3
import asyncio
import json
import os
import webbrowser
from twitchAPI.twitch import Twitch
from twitchAPI.oauth import UserAuthenticator
from twitchAPI.type import AuthScope
SECRETS_FILE = "twitch_secrets.json"
def load_credentials():
"""Loads existing Client ID and Secret from your JSON file."""
if not os.path.exists(SECRETS_FILE):
raise FileNotFoundError(f"Could not find {SECRETS_FILE} in this directory.")
with open(SECRETS_FILE, "r") as f:
data = json.load(f)
return data.get("client_id"), data.get("client_secret")
def save_token_to_json(token):
"""Saves the generated token into twitch_secrets.json under 'manual_token'."""
with open(SECRETS_FILE, "r") as f:
data = json.load(f)
# Inject the new token
data["manual_token"] = token
with open(SECRETS_FILE, "w") as f:
json.dump(data, f, indent=4)
print(f"\n[SUCCESS] Token saved inside '{SECRETS_FILE}' under 'manual_token'!")
async def main():
try:
client_id, client_secret = load_credentials()
if not client_id or not client_secret:
print("[ERROR] Please add your client_id and client_secret to the JSON file first.")
return
print("Initializing local connection loop...")
# Initialize official Twitch connection interface
twitch = await Twitch(client_id, client_secret)
# Scopes: We leave this empty [] since VOD collection only requires basic public clearance
scopes = []
# Create an authenticator that automatically sets up http://localhost:17563
auth = UserAuthenticator(twitch, scopes, url="http://localhost:17563")
# Request authentication URL
auth_url = auth.return_auth_url()
print(f"\nIf your browser does not open automatically, copy and paste this URL into your browser:\n{auth_url}\n")
# Open your system default browser to let you manually click "Authorize"
webbrowser.open(auth_url)
print("Waiting for you to click 'Authorize' in your web browser...")
# The script halts here, running a local background server until you click authorize
token, refresh_token = await auth.authenticate()
print(f"\nSuccessfully generated Token: {token}")
# Save it right back into your configuration file
save_token_to_json(token)
# Gracefully shut down the library connection
await twitch.close()
except Exception as e:
print(f"\n[ERROR] An error occurred: {e}")
print("Double-check that http://localhost:17563 is added to your Twitch Dev Console.")
if __name__ == "__main__":
# Run the asynchronous loop
asyncio.run(main())
Regular → Executable
+89 -105
View File
@@ -1,148 +1,132 @@
#!/usr/bin/env python3
import sqlite3
from datetime import datetime
from datetime import date as datetime_date
from pathlib import Path
from typing import Any
class Database:
def __init__(self, table: str):
self.table = table
self.columns = ""
def __init__(self):
self.columns = "id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts"
# IMPORTANT Must check if database.db exists before connecting to it.
file_exists = Path("database.db").is_file()
self.CONN = sqlite3.connect("database.db")
self.CURSOR = self.CONN.cursor()
if self.table == "vods":
self.columns = "id, date, title, gamename, downloaded, uploaded_yt, chats_upload_yt"
elif self.table == "clips":
self.columns = "slug, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt, shorts_upload_yt"
self.conn = sqlite3.connect("database.db")
self.cursor = self.conn.cursor()
if not file_exists:
self.create_database()
def __del__(self):
# Destructors are unpredictable in Python; explicitly close when done instead
try:
self.close_database()
except:
pass
def create_database(self):
"""Creates a table structured explicitly for Twitch clip properties."""
self.CURSOR.execute(
"""
CREATE TABLE IF NOT EXISTS clips (
slug TEXT PRIMARY KEY,
date TEXT NOT NULL,
"""Creates a table structured explicitly for Twitch videos properties."""
self.cursor.execute("""
CREATE TABLE IF NOT EXISTS twitch_videos (
id TEXT PRIMARY KEY,
title TEXT NOT NULL,
gamename TEXT NOT NULL,
clip_by TEXT NOT NULL,
created_at TEXT NOT NULL,
view_count INTEGER NOT NULL,
downloaded INTEGER NOT NULL,
uploaded_yt INTEGER NOT NULL,
shorts_upload_yt INTEGER NOT NULL
)
"""
duration TEXT NOT NULL,
url TEXT NOT NULL,
thumbnail_url TEXT NOT NULL,
game_id INTEGER NOT NULL,
game_name TEXT NOT NULL,
stream_id TEXT NOT NULL,
creator_name TEXT NOT NULL,
clip_is BOOLEAN NOT NULL DEFAULT 0,
downloaded BOOLEAN NOT NULL DEFAULT 0,
uploaded_yt BOOLEAN NOT NULL DEFAULT 0,
uploaded_yt_chats BOOLEAN NOT NULL DEFAULT 0,
uploaded_yt_shorts BOOLEAN NOT NULL DEFAULT 0
)
""")
"""Creates a table structured explicitly for Twitch vods properties."""
self.CURSOR.execute(
"""
CREATE TABLE IF NOT EXISTS vods (
id INTEGER PRIMARY KEY,
date TEXT NOT NULL,
title TEXT NOT NULL,
gamename TEXT NOT NULL,
downloaded INTEGER NOT NULL,
uploaded_yt INTEGER NOT NULL,
chats_upload_yt INTEGER NOT NULL
)
"""
)
self.CONN.commit()
self.conn.commit()
def close_database(self):
"""Commit before we close."""
self.CONN.commit()
self.CONN.close()
if self.conn:
self.conn.commit()
self.conn.close()
def __mark_as(self, record_id: int, set_sql: str):
id = "id"
def __mark_as(self, record_id: str, set_sql: str):
self.cursor.execute(
f"UPDATE twitch_videos SET {set_sql} = 1 WHERE id = ?",
(record_id,))
self.conn.commit()
if self.table == "clips":
id = "slug"
def mark_as_uploaded_shorts(self, record_id: str):
"""Flags a specific row record to uploaded_yt_shorts (1)."""
self.__mark_as(record_id, "uploaded_yt_shorts")
self.CURSOR.execute(
f"UPDATE {self.table} SET {set_sql} = 1 WHERE {id} = ?",
(record_id,)
)
self.CONN.commit()
def mark_as_uploaded_shorts_chats(self, record_id: int):
"""Flags a specific row record to uploaded (1)."""
set_sql = "chats_upload_yt"
if self.table == "clips":
set_sql = "shorts_uploaded_yt"
self.__mark_as(record_id, set_sql)
def mark_as_uploaded(self, record_id: int):
"""Flags a specific row record to uploaded (1)."""
def mark_as_uploaded_chats(self, record_id: str):
"""Flags a specific row record to uploaded_yt_chats (1)."""
self.__mark_as(record_id, "uploaded_yt_chats")
def mark_as_uploaded_yt(self, record_id: str):
"""Flags a specific row record to uploaded_yt (1)."""
self.__mark_as(record_id, "uploaded_yt")
def mark_as_downloaded(self, record_id: int):
"""Updates the downloaded status to True (1) for a specific record ID."""
def mark_as_downloaded(self, record_id: str):
"""Flags a specific row record to downloaded (1)."""
self.__mark_as(record_id, "downloaded")
def get_unuploaded_shorts_chats(self):
"""Retrieves all clip rows remaining to be chat uploaded."""
def __get_unuploaded(self, set_sql: str) -> list[Any]:
"""Retrieve all rows that were download but not uploaded"""
self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE downloaded = 1 AND {set_sql} = 0")
return self.cursor.fetchall()
where_sql = "chats_upload_yt"
if self.table == "clips":
where_sql = "shorts_uploaded_yt"
def get_unuploaded_shorts(self) -> list[Any]:
"""Retrieve all rows that were download but not uploaded_yt_shorts"""
return self.__get_unuploaded("uploaded_yt_shorts")
self.CURSOR.execute(f"SELECT {self.columns} FROM {self.table} WHERE downloaded = 1 AND uploaded_yt = 1 AND {where_sql} = 0")
return self.CURSOR.fetchall()
def get_unuploaded_chats(self) -> list[Any]:
"""Retrieve all rows that were download but not uploaded_yt_chats"""
return self.__get_unuploaded("uploaded_yt_chats")
def get_unuploaded(self):
"""Retrieves all clip rows remaining to be uploaded."""
self.CURSOR.execute(f"SELECT {self.columns} FROM {self.table} WHERE downloaded = 1 AND uploaded_yt = 0")
return self.CURSOR.fetchall()
def get_unuploaded(self) -> list[Any]:
"""Retrieve all rows that were download but not uploaded_yt"""
return self.__get_unuploaded("uploaded_yt")
def get_undownloaded(self):
"""Retrieves all rows where downloaded status is False (0)."""
self.CURSOR.execute(f"SELECT {self.columns} FROM {self.table} WHERE downloaded = 0")
return self.CURSOR.fetchall()
def get_undownloaded(self) -> list[Any]:
"""Retrieves all rows that were not downloaded"""
self.cursor.execute(f"SELECT {self.columns} FROM twitch_videos WHERE downloaded = 0")
return self.cursor.fetchall()
def insert_vods_record(self, record_id: int, record_date_str: datetime_date, title: str, gamename: str):
"""Inserts a record with ID, date, gamename, and title into a SQLite database."""
def insert_video_record(
self, id: str, title: str, created_at: str, view_count: int, duration: str,
url: str, thumbnail_url: str, game_id: int, game_name: str, stream_id: str,
creator_name: str, clip_is: bool
):
try:
clean_date = datetime.strptime(record_date_str, "%Y-%m-%dT%H:%M:%SZ").date()
except ValueError:
clean_date = record_date_str
# Connects to database file (creates it if missing)
dt_obj = datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
clean_datetime = dt_obj.strftime("%Y-%m-%d %H:%M:%S")
except (ValueError, TypeError):
clean_datetime = created_at
# Inserts data using parameterized queries to prevent SQL injection
self.CURSOR.execute(
f"INSERT OR IGNORE INTO {self.table} ({self.columns}) VALUES (?, ?, ?, ?, ?, ?, ?)",
(record_id, str(clean_date), title, gamename, False, False, False),
# Explicitly defining columns removes the security risk and column-count bug
query = """
INSERT OR IGNORE INTO twitch_videos (
id, title, created_at, view_count, duration, url, thumbnail_url,
game_id, game_name, stream_id, creator_name, clip_is
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
"""
values = (
id, title, clean_datetime, view_count, duration, url, thumbnail_url,
game_id, game_name, stream_id, creator_name, clip_is
)
# Saves changes and closes the connection
self.CONN.commit()
def insert_clips_record(self, slug: str, record_date_str: str, title: str, gamename: str, clip_by: str, views: int):
"""Cleans up ISO-8601 strings into unified date structures for the database."""
try:
clean_date = datetime.strptime(record_date_str, "%Y-%m-%dT%H:%M:%SZ").date()
except ValueError:
clean_date = record_date_str
self.CURSOR.execute(
f"INSERT OR IGNORE INTO {self.table} ({self.columns}) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(slug, str(clean_date), title, gamename, clip_by, views, False, False, False),
)
self.CONN.commit()
self.cursor.execute(query, values)
self.conn.commit()
except Exception as e:
# Prevent silent failures if the database connection drops
print(f"Database insertion failed: {e}")
self.conn.rollback()
Regular → Executable
View File
Executable
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env python3
import subprocess
def run_command(command: str):
"""Executes a Linux command, waits for completion, and returns output."""
try:
# shell=True allows running full command strings with pipes/wildcards
# text=True returns strings instead of bytes
result = subprocess.run(
command, shell=True, check=True, capture_output=True, text=True
)
return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
except subprocess.CalledProcessError as e:
# Handles errors if the Linux command returns a non-zero exit code
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
-1
View File
@@ -1,4 +1,3 @@
apt-listchanges==4.8
attrs==26.1.0
beautifulsoup4==4.14.3
certifi==2026.2.25
Regular → Executable
+2 -8
View File
@@ -2,6 +2,7 @@
import os
import subprocess
import whisper
import linux
from moviepy import VideoFileClip
from whisper.utils import get_writer
@@ -15,14 +16,7 @@ def extract_audio(video_path: str, audio_temp_path: str):
# -map_chapters -1 removes chapter layouts that break the parser.
# -sn strips text/subtitle streams that crash MoviePy.
# -c copy copies video and audio instantly without quality loss.
cleanup_cmd = [
"ffmpeg", "-y", "-i", video_path,
"-map_chapters", "-1", "-sn",
"-c", "copy", sanitized_video_path
]
# Run the sanitization process silently
subprocess.run(cleanup_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
linux.run_command(f"ffmpeg -y -i {video_path} -map_chapters -1 -sn -c copy {sanitized_video_path}")
print("Extracting uncompressed WAV audio...")
try:
Regular → Executable
+89 -56
View File
@@ -1,7 +1,8 @@
#!/usr/bin/env python3
import requests
import os
import subprocess
import linux
import time
from database import Database
@@ -14,21 +15,6 @@ import transcribe_video
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
DB = None
def run_linux_command(command: str):
"""Executes a Linux command, waits for completion, and returns output."""
try:
# shell=True allows running full command strings with pipes/wildcards
# text=True returns strings instead of bytes
result = subprocess.run(
command, shell=True, check=True, capture_output=True, text=True
)
return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
except subprocess.CalledProcessError as e:
# Handles errors if the Linux command returns a non-zero exit code
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
def transcribe(id: str):
"""Transcribes the Video File."""
video_file = f"save/{DB.table}/{id}/{id}.mp4"
@@ -61,16 +47,17 @@ def top_hashtags(id: str):
return tags
def download():
"""Find all undownload vods and download them."""
"""Find all undownload videos and download them."""
undownloaded = DB.get_undownloaded()
print(f"Download {DB.table}...")
if DB.table == "vods":
if DB.table == "videos":
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chat_upload_yt in undownloaded:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
output = run_linux_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/vods/{record_id}/{record_id}.mp4 --collision Overwrite")
output2 = run_linux_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/vods/{record_id}/{record_id}_chat.json -E --collision Overwrite")
output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/videos/{record_id}/{record_id}.mp4 --collision Overwrite")
output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/videos/{record_id}/{record_id}_chat.json -E --collision Overwrite")
time.sleep(1)
if output["success"] is True:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was successful")
DB.mark_as_downloaded(record_id)
@@ -82,7 +69,8 @@ def download():
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
# Uses standard clipdownload directive
output = run_linux_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{slug}.mp4 --collision Overwrite")
output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{slug}.mp4 --collision Overwrite")
time.sleep(1)
if output["success"] is True:
print(f"Slug: {slug} | Was successfully downloaded.")
@@ -109,11 +97,11 @@ def upload():
upload_queue = []
if DB.table == "vods":
if DB.table == "videos":
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chats_upload_yt in unuploaded:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
file_path = f"save/vods/{record_id}/{record_id}.mp4"
file_path = f"save/videos/{record_id}/{record_id}.mp4"
description = f"Game: {game_name}, on {record_date}, #VODS {twitch_datetime}"
tags = list(base_tags)
tags.extend([f'{game_name}', 'twitch_vods', 'vods'])
@@ -161,10 +149,14 @@ def get_vod_ids_simplified(channel_name: str):
"Content-Type": "text/plain"
}
vods_query_string = """
query GetChannelVideos($login: String!, $limit: Int!) {
videos_query_string = """
query GetChannelVideos($login: String!, $limit: Int!, $after: Cursor) {
user(login: $login) {
videos(first: $limit, types: [ARCHIVE]) {
videos(first: $limit, types: [ARCHIVE], after: $after) {
pageInfo {
hasNextPage
endCursor
}
edges {
node {
id
@@ -181,10 +173,15 @@ def get_vod_ids_simplified(channel_name: str):
"""
clips_query_string = """
query GetChannelClips($login: String!, $limit: Int!) {
query GetChannelClips($login: String!, $limit: Int!, $after: Cursor) {
user(login: $login) {
clips(first: $limit, criteria: { period: ALL_TIME }) {
clips(first: $limit, criteria: { period: ALL_TIME }, after: $after) {
pageInfo {
hasNextPage
endCursor
}
edges {
cursor
node {
slug
title
@@ -203,30 +200,39 @@ def get_vod_ids_simplified(channel_name: str):
}
"""
video_ids = []
has_next_page = True
cursor = None
limit = 50
query_string = ""
operation_name = ""
if DB.table == "vods":
query_string = vods_query_string
limit = 50
if DB.table == "videos":
query_string = videos_query_string
limit = 100
operation_name = "GetChannelVideos"
elif DB.table == "clips":
query_string = clips_query_string
limit = 40
operation_name = "GetChannelClips"
while has_next_page:
time.sleep(1)
# A simplified query string that hardcodes the ARCHIVE type filter into the request structure
payload = [{
"operationName": operation_name,
"query": query_string,
"variables": {
"login": channel_name.lower(),
"limit": limit
"limit": limit,
"after": cursor
}
}]
try:
# Prepping ensures Python does not rewrite the Client-ID header case
req = requests.Request('POST', url, json=payload)
#req = requests.Request('POST', url, data=json.dumps(payload))
prepped = session.prepare_request(req)
response = session.send(prepped)
@@ -241,42 +247,66 @@ def get_vod_ids_simplified(channel_name: str):
print(f"Twitch GraphQL Error: {result['errors']}")
return []
user_data = result['data']['user']
user_data = result.get('data', {}).get('user', {})
if not user_data:
print(f"Channel '{channel_name}' not found.")
return []
edges = None
if DB.table == "vods":
edges = user_data['videos']['edges']
elif DB.table == "clips":
edges = user_data['clips']['edges']
video_ids = []
edges = user_data.get(DB.table, {}).get('edges', [])
print(f"--- Latest VODs for {channel_name} ---")
# --- BREAK CONDITION 1: Stop if Twitch returns no more data items ---
if not edges or len(edges) == 0:
print("No more items returned by the server. Ending pagination loop.")
break
last_edge_cursor = None
print(f"--- Processing {DB.table} for {channel_name} ---")
for edge in edges:
node = edge['node']
last_edge_cursor = edge.get("cursor")
node = edge.get('node', {})
if not node:
continue
# Safe extraction in case a VOD has no category set (Just Chatting, Uncategorized, etc.)
game_info = node.get('game')
game_name = game_info.get('displayName') if game_info else "Unknown/No Category"
if DB.table == "vods":
print(f"ID: {node['id']} | Date: {node['publishedAt']} | Game: {game_name} | Title: {node['title']}")
# Pass game_name to your database logic
DB.insert_vods_record(node['id'], node['publishedAt'], node['title'], game_name)
video_ids.append(node['id'])
# FIXED: Switched fields to use safe .get() metrics to completely prevent KeyErrors
node_id = node.get('id')
node_title = node.get('title', 'No Title')
if DB.table == "videos":
published_at = node.get('publishedAt')
print(f"ID: {node_id} | Date: {published_at} | Game: {game_name} | Title: {node_title}")
DB.insert_videos_record(node_id, published_at, node_title, game_name)
if node_id:
video_ids.append(node_id)
elif DB.table == "clips":
# Safe extraction in case the curator account was deleted/missing
slug = node.get('slug')
created_at = node.get('createdAt')
view_count = node.get('viewCount', 0)
curator_info = node.get('curator')
clip_by = curator_info.get('login') if curator_info else "Unknown Creator"
print(f"Slug: {node['slug']} | Date: {node['createdAt']} | Game: {game_name} | By: {clip_by} | Views: {node['viewCount']} | Title: {node['title']}")
print(f"Slug: {slug} | Date: {created_at} | Game: {game_name} | By: {clip_by} | Views: {view_count} | Title: {node_title}")
DB.insert_clips_record(node['slug'], node['createdAt'], node['title'], game_name, clip_by, int(node['viewCount']))
video_ids.append(node['slug'])
DB.insert_clips_record(slug, created_at, node_title, game_name, clip_by, int(view_count))
if slug:
video_ids.append(slug)
return video_ids
# --- CORRECTED PAGINATION ENGINE FOR BOTH TABLES ---
page_info = user_data.get(DB.table, {}).get('pageInfo', {})
has_next_page = page_info.get("hasNextPage", False)
next_cursor = page_info.get("endCursor") or last_edge_cursor
if not next_cursor or next_cursor == cursor:
print("Cursor did not advance or is null. Safely terminating loop.")
break
cursor = next_cursor
except Exception as e:
print(f"An unexpected error occurred: {e}")
@@ -284,11 +314,14 @@ def get_vod_ids_simplified(channel_name: str):
print(f"Server Response Text: {response.text}")
return []
return video_ids
if __name__ == "__main__":
tables = ["vods", "clips"]
tables = ["clips"]
for table in tables:
DB = Database(table)
get_vod_ids_simplified(CHANNEL_NAME)
download()
upload()
DB.close_database()
#download()
#upload()
#DB.close_database()
Regular → Executable
-1
View File
@@ -1,6 +1,5 @@
#!/usr/bin/env python3
import requests
import sqlite3
import subprocess
from datetime import datetime
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
import requests
import os
import time
import csv
import linux
from database import Database
CHANNEL_NAME = 'teampgp'
DB = None
def write_csv(data, file_name):
# Open file with newline='' to prevent extra blank rows across platforms
with open(file_name, "w", newline="", encoding="utf-8") as file:
writer = csv.writer(file)
# Write all rows at once
writer.writerows(data)
def download():
"""Find all undownload videos and download them."""
undownloaded = DB.get_undownloaded()
print(f"Download...")
for id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts in undownloaded:
data = [[id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts]]
print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}")
if not clip_is:
output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {id} -o download/videos/{id}/{id}.mp4 --collision Overwrite")
output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o download/videos/{id}/{id}_chat.json -E --collision Overwrite")
write_csv(data, f"download/videos/{id}/{id}.csv")
elif clip_is:
output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {id} -o download/clips/{id}/{id}.mp4 --collision Overwrite")
write_csv(data, f"download/clips/{id}/{id}.csv")
time.sleep(1)
if output["success"] is True:
print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title} | was successful")
DB.mark_as_downloaded(id)
else:
print(f"ID: {id} | Download Process failed. {output["stdout"]}. Error: {output["stderr"]}")
print(f"Finished Downloading...")
if __name__ == "__main__":
DB = Database()
download()
Regular → Executable
+78 -262
View File
@@ -1,286 +1,102 @@
#!/usr/bin/env python3
import json
import os
import requests
import sqlite3
import subprocess
# The target Twitch streamer username
TWITCH_USERNAME = "SumGuyV5"
from database import Database
def load_secrets(filepath="twitch_secrets.json"):
"""Loads client credentials and potential manual token from JSON file."""
if not os.path.exists(filepath):
raise FileNotFoundError(f"Missing credential file: '{filepath}'")
import uploader
from uploader import CategoryId
with open(filepath, "r") as file:
secrets = json.load(file)
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
DB = Database("vods")
if "client_id" not in secrets or "client_secret" not in secrets:
raise KeyError("JSON file must contain 'client_id' and 'client_secret'.")
def run_linux_command(command: str):
"""Executes a Linux command, waits for completion, and returns output."""
try:
# shell=True allows running full command strings with pipes/wildcards
# text=True returns strings instead of bytes
result = subprocess.run(
command, shell=True, check=True, capture_output=True, text=True
)
return secrets["client_id"], secrets["client_secret"], secrets.get("manual_token")
return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
except subprocess.CalledProcessError as e:
# Handles errors if the Linux command returns a non-zero exit code
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
def transcribe(slug: str):
"""Transcribes the Video File."""
video_file = f"save/clips/{slug}/{slug}.mp4"
# Check if the video file exists
if not os.path.exists(video_file):
print(f"Error: File not found: {video_file}")
return False
# no need to continue if srt transcribe file already exists
if os.path.exists(f"save/clips/{slug}/transcribe_{slug}.srt"):
print(f"video already transcribed:")
return True
import transcribe_video
transcribe_video.extract_audio(video_file, f"save/clips/{slug}/temp_{slug}_audio.wav")
transcribe_video.transcribe_to_srt(f"save/clips/{slug}/temp_{slug}_audio.wav", f"save/clips/{slug}/", f"transcribe_{slug}")
return True
def top_hashtags(slug: str):
file_srt = f"save/clips/{slug}/transcribe_{slug}.srt"
# if srt transcribe file not exists
if not os.path.exists(file_srt):
print(f"Transcribe file not found {file_srt}")
return []
import youtube_hashtags
return youtube_hashtags.get_top_hashtags(file_srt)
def download():
"""Find all undownload vods and download them."""
undownloaded = DB.get_undownloaded()
print(f"Download {DB.table}...")
if DB.table == "vods":
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chat_upload_yt in undownloaded:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
output = run_linux_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/vods/{record_id}/{title}.mp4")
output2 = run_linux_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/vods/{record_id}/{title}_chat.json -E")
if output["success"] is True:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was successful")
DB.mark_as_downloaded(record_id)
else:
print(f"ID: {record_id} | Download Process failed. {output["stdout"]}. Error: {output["stderr"]}")
elif DB.table == "clips":
for slug, record_date, title, game_name, clip_by, views, downloaded, uploaded_yt, uploaded_shorts_yt in undownloaded:
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
# Uses standard clipdownload directive
output = run_linux_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{title}.mp4")
if output["success"] is True:
print(f"Slug: {slug} | Was successfully downloaded.")
DB.mark_as_downloaded(slug)
else:
print(f"Slug: {slug} | Download Process failed. {output["stdout"]}. Error: {output["stderr"]}")
print(f"Finished Downloading {DB.table}...")
def upload():
"""Loops over the downloaded videos entries and uploaded them to youtube."""
print(f"Uploading {DB.table}...")
unuploaded = DB.get_unuploaded()
twitch_datetime = " #Twitch Every Friday and Sunday @7:30 EST https://twitch.tv/teampgp"
file_path = ""
title = ""
description = ""
categoryId = CategoryId.GAMING
privatcyStatus = 'private'
base_tags = ['gaming', 'TeamPGP', 'twitch', 'Level1Techs', 'twitch']
upload_queue = []
if DB.table == "vods":
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chats_upload_yt in unuploaded:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
file_path = f"save/vods/{record_id}/{title}.mp4"
description = f"Game: {game_name}, on {record_date}, #VODS {twitch_datetime}"
tags = list(base_tags)
tags.extend([f'{game_name}', 'twitch_vods', 'vods'])
tags.extend(top_hashtags(record_id))
upload_queue.append([record_id, file_path, title, categoryId, description, privatcyStatus, tags])
elif DB.table == "clips":
for slug, record_date, title, game_name, clip_by, views, downloaded, uploaded_yt in unuploaded:
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
file_path = f"save/clips/{slug}/{title}.mp4"
description = f"Game: {game_name}, Cliped By: {clip_by}, on {record_date}, #Shorts #Clips {twitch_datetime}"
tags = list(base_tags)
tags.extend([f'{game_name}', 'twitch_clips', 'clips', f'{clip_by}', 'shorts'])
tags.extend(top_hashtags(slug))
upload_queue.append([slug, file_path, title, categoryId, description, privatcyStatus, tags])
for db_id, file_path, title, categoryId, description, privatcyStatus, tags in upload_queue:
output = uploader.upload_video(file_path, title, categoryId, description, privatcyStatus, tags)
if output is True:
print(f"Title: {title} | Was successfully uploaded.")
DB.mark_as_uploaded(db_id)
else:
print(f"Title: {title} | Download Process failed.")
def create_chats():
pass
def create_shorts():
pass
def get_vod_ids_simplified(channel_name: str):
"""Queries Twitch's public endpoint directly for trending clips."""
session = requests.Session()
url = "https://gql.twitch.tv/gql"
# Case-preserved headers to prevent 400 Bad Request errors
session.headers = {
"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Content-Type": "text/plain"
def get_app_access_token(client_id, client_secret):
"""Generates an App Access Token using the correct Twitch ID server."""
auth_url = "https://twitch.tv" # FIXED: Correct auth endpoint
payload = {
"client_id": client_id,
"client_secret": client_secret,
"grant_type": "client_credentials"
}
headers = {"Content-Type": "application/x-www-form-urlencoded"}
vods_query_string = """
query GetChannelVideos($login: String!, $limit: Int!) {
user(login: $login) {
videos(first: $limit, types: [ARCHIVE]) {
edges {
node {
id
title
publishedAt
game {
displayName
}
}
}
}
}
}
"""
clips_query_string = """
query GetChannelClips($login: String!, $limit: Int!) {
user(login: $login) {
clips(first: $limit, criteria: { period: ALL_TIME }) {
edges {
node {
slug
title
createdAt
viewCount
game {
displayName
}
curator {
login
}
}
}
}
}
}
"""
limit = 50
query_string = ""
if DB.table == "vods":
query_string = vods_query_string
limit = 50
elif DB.table == "clips":
query_string = clips_query_string
limit = 40
# A simplified query string that hardcodes the ARCHIVE type filter into the request structure
payload = [{
"operationName": "GetChannelVideos",
"query": query_string,
"variables": {
"login": channel_name.lower(),
"limit": limit
}
}]
try:
# Prepping ensures Python does not rewrite the Client-ID header case
req = requests.Request('POST', url, json=payload)
prepped = session.prepare_request(req)
response = session.send(prepped)
response = requests.post(auth_url, data=payload, headers=headers)
response.raise_for_status()
return response.json()["access_token"]
data = response.json()
def get_user_id(username, headers):
"""Retrieves the unique numerical Twitch User ID from Helix."""
url = f"https://twitch.tv{username}" # FIXED: Endpoint & parameter
response = requests.get(url, headers=headers)
response.raise_for_status()
data = response.json().get("data")
if data and len(data) > 0:
return data[0]["id"] # FIXED: Helix data array returns user dictionaries
else:
raise ValueError(f"Twitch user '{username}' not found.")
# Pull out the target index array dictionary object
result = data[0] if isinstance(data, list) else data
def get_channel_vods(user_id, headers, limit=10):
"""Fetches past broadcasts (VODs) using valid Helix syntax."""
# FIXED: Restructured URL to use correct endpoint and standard query parameters
url = f"https://twitch.tv{user_id}&type=archive&first={limit}"
response = requests.get(url, headers=headers)
response.raise_for_status()
return response.json().get("data", [])
if "errors" in result:
print(f"Twitch GraphQL Error: {result['errors']}")
return []
def main():
try:
# 1. Load credentials from external JSON file
client_id, client_secret, manual_token = load_secrets("twitch_secrets.json")
user_data = result['data']['user']
if not user_data:
print(f"Channel '{channel_name}' not found.")
return []
# 2. Assign or generate OAuth Access Token
if manual_token:
print("Using manual access token from JSON config file...")
access_token = manual_token
else:
print("No manual token found. Attempting to contact Twitch Auth Server...")
access_token = get_app_access_token(client_id, client_secret)
edges = None
if DB.table == "vods":
edges = user_data['videos']['edges']
elif DB.table == "clips":
edges = user_data['clips']['edges']
video_ids = []
# 3. Setup Headers required by Twitch Helix API
headers = {
"Client-ID": client_id,
"Authorization": f"Bearer {access_token}"
}
print(f"--- Latest VODs for {channel_name} ---")
for edge in edges:
node = edge['node']
# 4. Translate Username to User ID
user_id = get_user_id(TWITCH_USERNAME, headers)
print(f"Successfully retrieved ID for {TWITCH_USERNAME}: {user_id}\n")
# Safe extraction in case a VOD has no category set (Just Chatting, Uncategorized, etc.)
game_info = node.get('game')
game_name = game_info.get('displayName') if game_info else "Unknown/No Category"
if DB.table == "vods":
print(f"ID: {node['id']} | Date: {node['publishedAt']} | Game: {game_name} | Title: {node['title']}")
# 5. Fetch and Print VOD details
vods = get_channel_vods(user_id, headers, limit=5)
# Pass game_name to your database logic
DB.insert_vods_record(node['id'], node['publishedAt'], node['title'], game_name)
#insert_record(node['id'], node['publishedAt'], node['title'], game_name)
video_ids.append(node['id'])
elif DB.table == "clips":
# Safe extraction in case the curator account was deleted/missing
curator_info = node.get('curator')
clip_by = curator_info.get('login') if curator_info else "Unknown Creator"
if not vods:
print(f"No VODs found for {TWITCH_USERNAME}.")
return
print(f"Slug: {node['slug']} | Date: {node['createdAt']} | Game: {game_name} | By: {clip_by} | Views: {node['viewCount']} | Title: {node['title']}")
DB.insert_clips_record(node['slug'], node['createdAt'], node['title'], game_name, clip_by, int(node['viewCount']))
video_ids.append(node['slug'])
return video_ids
print(f"--- Latest VODs for {TWITCH_USERNAME} ---")
for vod in vods:
print(f"Title: {vod['title']}")
print(f"URL: {vod['url']}")
print(f"Published At: {vod['published_at']}")
print(f"Duration: {vod['duration']}")
print(f"Views: {vod['view_count']}")
print("-" * 40)
except (FileNotFoundError, KeyError) as config_err:
print(f"Configuration Error: {config_err}")
except requests.exceptions.HTTPError as err:
print(f"HTTP Error detail: {err.response.text if err.response else err}")
except Exception as e:
print(f"An unexpected error occurred: {e}")
if 'response' in locals():
print(f"Server Response Text: {response.text}")
return []
print(f"An error occurred: {e}")
if __name__ == "__main__":
get_vod_ids_simplified(CHANNEL_NAME)
download_vods()
upload_vods()
DB.close_database()
main()
+5
View File
@@ -0,0 +1,5 @@
{
"client_id": "yr610ucde5vlae3zqniv23eps4ky7j",
"client_secret": "1m8bopo5hwtnv0mox9wql8i33fqrgr",
"manual_token": "dkmhv4f6k0mr1yyzof54xqll5pf7l3"
}
+190
View File
@@ -0,0 +1,190 @@
#!/usr/bin/env python3
import asyncio
import json
import httpx # Switched from requests to prevent async loop freezing
import database
from twitchAPI.twitch import Twitch
from twitchAPI.helper import first
from twitchAPI.type import VideoType
SECRETS = None
TWITCH = None
USER = None
GAME_CACHE = {} # Local cache dictionary to store game_id -> game_name mapping
CHANNEL_NAME = "teampgp"
async def get_twitch():
global SECRETS, TWITCH, USER
if SECRETS is None:
with open('twitch_secrets.json', 'r') as f:
SECRETS = json.load(f)
if TWITCH is None:
TWITCH = await Twitch(SECRETS['client_id'], SECRETS['client_secret'])
if USER is None:
USER = await first(TWITCH.get_users(logins=[CHANNEL_NAME]))
if not USER:
print("User not found.")
async def get_game_name_by_id(game_id: str) -> str:
"""Helper function to fetch game names and cache them locally."""
if not game_id:
return "Unknown / No Category"
if game_id in GAME_CACHE:
return GAME_CACHE[game_id]
try:
game_generator = TWITCH.get_games(game_ids=[game_id])
game = await first(game_generator)
if game:
GAME_CACHE[game_id] = game.name
return game.name
except Exception:
pass
return "Unknown Game"
async def get_vod_game_name(vod_id: str):
"""Asynchronously query Twitch GQL endpoint for VOD game metadata."""
game_id = "0"
game_name = "Unknown Game"
url = "https://gql.twitch.tv/gql"
headers = {
"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
"Content-Type": "application/json"
}
payload = [{
"operationName": "VideoMetadata",
"variables": {
"channelLogin": "",
"videoID": str(vod_id)
},
"extensions": {
"persistedQuery": {
"version": 1,
"sha256Hash": "45111672eea2e507f8ba44d101a61862f9c56b11dee09a15634cb75cb9b9084d"
}
}
}]
# Using httpx async client to prevent blocking the asyncio event loop
async with httpx.AsyncClient() as client:
try:
response = await client.post(url, headers=headers, json=payload)
if response.status_code == 200:
data = response.json()
video_info = data[0].get('data', {}).get('video')
if video_info and video_info.get('game'):
game_id = str(video_info['game']['id'])
game_name = video_info['game']['displayName']
print(f"GQL Found: {game_name} (ID: {game_id})")
else:
print(f"No game information found in GQL for VOD {vod_id}.")
except Exception as e:
print(f"Error fetching GQL metadata for VOD {vod_id}: {e}")
return game_id, game_name
async def get_streamer_vods():
await get_twitch()
print(f"Starting VOD extraction for {USER.display_name}...")
vod_generator = TWITCH.get_videos(user_id=USER.id, first=100, video_type=VideoType.ALL)
all_vods = []
async for v in vod_generator:
# Resolving game category safely without freezing the event loop
game_id, game_name = await get_vod_game_name(v.id)
# Safely convert game_id to integer if possible, otherwise default to 0
try:
clean_game_id = int(game_id)
except ValueError:
clean_game_id = 0
vod_data = {
"id": v.id,
"title": v.title,
"created_at": str(v.published_at),
"view_count": int(v.view_count) if v.view_count else 0,
"duration": v.duration,
"url": v.url,
"thumbnail_url": v.thumbnail_url,
"game_id": clean_game_id,
"game_name": game_name,
"stream_id": str(v.stream_id) if v.stream_id else "0",
"creator_name": CHANNEL_NAME,
"clip_is": False,
}
all_vods.append(vod_data)
print(f"Collected VOD: {v.title} | Category: {game_name} ({v.duration})")
print(f"\nFinished extracting VODs. Total gathered: {len(all_vods)}")
return all_vods
async def get_streamer_clips():
await get_twitch()
print(f"Starting clip extraction for {USER.display_name}...")
clip_generator = TWITCH.get_clips(broadcaster_id=USER.id, first=100)
all_clips = []
async for c in clip_generator:
game_name = await get_game_name_by_id(c.game_id)
try:
clean_game_id = int(c.game_id)
except (ValueError, TypeError):
clean_game_id = 0
clip_data = {
"id": c.id,
"title": c.title,
"created_at": str(c.created_at),
"view_count": int(c.view_count) if c.view_count else 0,
"duration": c.duration,
"url": c.url,
"thumbnail_url": c.thumbnail_url,
"game_id": clean_game_id,
"game_name": game_name,
"stream_id": "0",
"creator_name": c.creator_name,
"clip_is": True,
}
all_clips.append(clip_data)
print(f"Collected clip: {c.title} | Category: {game_name} ({c.view_count} views)")
print(f"\nFinished extracting clips. Total gathered: {len(all_clips)}")
return all_clips
async def main():
print("--- Script Started ---")
db = database.Database()
vods_list = await get_streamer_vods()
for v in vods_list:
db.insert_video_record(
v['id'], v['title'], v['created_at'], v['view_count'], v['duration'],
v['url'], v['thumbnail_url'], v['game_id'], v['game_name'],
v['stream_id'], v['creator_name'], v['clip_is']
)
if vods_list:
print(f"Recent VOD: '{vods_list[0]['title']}'")
clips_list = await get_streamer_clips()
for v in clips_list:
db.insert_video_record(
v['id'], v['title'], v['created_at'], v['view_count'], v['duration'],
v['url'], v['thumbnail_url'], v['game_id'], v['game_name'],
v['stream_id'], v['creator_name'], v['clip_is']
)
if clips_list:
print(f"Recent VOD: '{clips_list[0]['title']}'")
if __name__ == "__main__":
asyncio.run(main())
Regular → Executable
View File
Regular → Executable
View File
Regular → Executable
View File
Regular → Executable
View File
Regular → Executable
+2 -7
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
import argparse
import sys
import subprocess
import linux
import tempfile
from pathlib import Path
from moviepy import VideoFileClip, ColorClip, CompositeVideoClip
@@ -36,12 +36,7 @@ def fit_to_9_16_letterbox(input_path: str, top_text: str = "TOP TEXT", bottom_te
background_layer = None
try:
cleanup_cmd = [
"ffmpeg", "-y", "-i", str(input_file),
"-map_chapters", "-1", "-sn",
"-c", "copy", temp_path
]
subprocess.run(cleanup_cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
linux.run_command(f"ffmpeg -y -i {input_file} -map_chapters -1 -sn -c copy {temp_path}")
# Load video
clip = VideoFileClip(temp_path)