first commit
This commit is contained in:
@@ -0,0 +1,3 @@
|
|||||||
|
/secrets.json
|
||||||
|
/client_secrets.json
|
||||||
|
/clips_database.db
|
||||||
+121
@@ -0,0 +1,121 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import argparse
|
||||||
|
from google.oauth2.credentials import Credentials
|
||||||
|
from google_auth_oauthlib.flow import InstalledAppFlow
|
||||||
|
from google.auth.transport.requests import Request
|
||||||
|
from googleapiclient.discovery import build
|
||||||
|
from googleapiclient.http import MediaFileUpload
|
||||||
|
from googleapiclient.errors import HttpError
|
||||||
|
|
||||||
|
def load_credentials():
|
||||||
|
"""Load credentials from secrets.json"""
|
||||||
|
try:
|
||||||
|
with open('secrets.json', 'r') as f:
|
||||||
|
creds_data = json.load(f)
|
||||||
|
|
||||||
|
credentials = Credentials(
|
||||||
|
token=creds_data['token'],
|
||||||
|
refresh_token=creds_data['refresh_token'],
|
||||||
|
token_uri=creds_data['token_uri'],
|
||||||
|
client_id=creds_data['client_id'],
|
||||||
|
client_secret=creds_data['client_secret'],
|
||||||
|
scopes=creds_data['scopes']
|
||||||
|
)
|
||||||
|
|
||||||
|
# Refresh token if expired
|
||||||
|
if credentials.expired:
|
||||||
|
credentials.refresh(Request())
|
||||||
|
|
||||||
|
return credentials
|
||||||
|
except FileNotFoundError:
|
||||||
|
print("Error: secrets.json not found! Please run setup.py first.")
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error loading credentials: {str(e)}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
def upload_video(file_path, description):
|
||||||
|
"""
|
||||||
|
Upload a video to YouTube
|
||||||
|
|
||||||
|
Args:
|
||||||
|
file_path (str): Path to the video file
|
||||||
|
description (str): Video description
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Check if file exists
|
||||||
|
if not os.path.exists(file_path):
|
||||||
|
print(f"Error: File not found: {file_path}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Load credentials
|
||||||
|
credentials = load_credentials()
|
||||||
|
if not credentials:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Create YouTube API client
|
||||||
|
youtube = build('youtube', 'v3', credentials=credentials)
|
||||||
|
|
||||||
|
# Get the filename without extension as default title
|
||||||
|
title = os.path.splitext(os.path.basename(file_path))[0]
|
||||||
|
|
||||||
|
# Prepare the video upload request
|
||||||
|
body = {
|
||||||
|
'snippet': {
|
||||||
|
'title': title,
|
||||||
|
'description': description,
|
||||||
|
'tags': [],
|
||||||
|
'categoryId': '22' # Default to 'People & Blogs' category
|
||||||
|
},
|
||||||
|
'status': {
|
||||||
|
'privacyStatus': 'private', # Default to private
|
||||||
|
'selfDeclaredMadeForKids': False
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Create media file upload
|
||||||
|
media = MediaFileUpload(
|
||||||
|
file_path,
|
||||||
|
chunksize=1024*1024,
|
||||||
|
resumable=True
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create the video insert request
|
||||||
|
insert_request = youtube.videos().insert(
|
||||||
|
part=','.join(body.keys()),
|
||||||
|
body=body,
|
||||||
|
media_body=media
|
||||||
|
)
|
||||||
|
|
||||||
|
print("Starting upload...")
|
||||||
|
response = None
|
||||||
|
while response is None:
|
||||||
|
status, response = insert_request.next_chunk()
|
||||||
|
if status:
|
||||||
|
print(f"Uploaded {int(status.progress() * 100)}%")
|
||||||
|
|
||||||
|
print(f"\nUpload Complete!")
|
||||||
|
print(f"Video ID: {response['id']}")
|
||||||
|
print(f"Title: {response['snippet']['title']}")
|
||||||
|
print(f"URL: https://youtu.be/{response['id']}")
|
||||||
|
return True
|
||||||
|
|
||||||
|
except HttpError as e:
|
||||||
|
print(f"An HTTP error occurred: {str(e)}")
|
||||||
|
return False
|
||||||
|
except Exception as e:
|
||||||
|
print(f"An error occurred: {str(e)}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description='Upload a video to YouTube')
|
||||||
|
parser.add_argument('file', help='Path to the video file')
|
||||||
|
parser.add_argument('description', help='Video description')
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
upload_video(args.file, args.description)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
+191
@@ -0,0 +1,191 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
||||||
|
|
||||||
|
def download_clips():
|
||||||
|
undownload_items = get_undownloaded_records()
|
||||||
|
|
||||||
|
print("Download Clips...")
|
||||||
|
|
||||||
|
# Updated unpack loop to include clip_by
|
||||||
|
for record_id, record_date, title, gamename, clip_by, downloaded in undownload_items:
|
||||||
|
print(f"Slug: {record_id} | Date: {record_date} | Game: {gamename} | By: {clip_by} | Title: {title}")
|
||||||
|
|
||||||
|
# Ensure output directory exists before running CLI tools
|
||||||
|
os.makedirs(f"save/clips/{record_id}", exist_ok=True)
|
||||||
|
|
||||||
|
# Uses the 'clipdownload' command
|
||||||
|
output = run_linux_command(f"TwitchDownloaderCLI clipdownload --id {record_id} -o save/clips/{record_id}/{record_id}.mp4")
|
||||||
|
|
||||||
|
if output["success"] is True:
|
||||||
|
print(f"Slug: {record_id} | Date: {record_date} | Game: {gamename} | By: {clip_by} | Title: {title} | was successful")
|
||||||
|
mark_as_downloaded(record_id)
|
||||||
|
else:
|
||||||
|
print(f"Slug: {record_id} | Date: {record_date} | Game: {gamename} | By: {clip_by} | Title: {title} | was Failed")
|
||||||
|
if not output["success"]: print(f"-> Clip Download Error: {output['stderr']}")
|
||||||
|
|
||||||
|
def run_linux_command(command: str):
|
||||||
|
"""Executes a Linux command, waits for completion, and returns output."""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
command, shell=True, check=True, capture_output=True, text=True
|
||||||
|
)
|
||||||
|
return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
|
||||||
|
|
||||||
|
def mark_as_downloaded(record_id: str):
|
||||||
|
"""Updates the downloaded status to True (1) for a specific clip slug."""
|
||||||
|
conn = sqlite3.connect("database.db")
|
||||||
|
cursor = conn.cursor()
|
||||||
|
cursor.execute(
|
||||||
|
"UPDATE clips SET downloaded = 1 WHERE id = ?",
|
||||||
|
(record_id,)
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def get_undownloaded_records():
|
||||||
|
"""Retrieves all rows where downloaded status is False (0)."""
|
||||||
|
conn = sqlite3.connect("database.db")
|
||||||
|
cursor = conn.cursor()
|
||||||
|
# Added clip_by to the SELECT fields
|
||||||
|
cursor.execute(
|
||||||
|
"SELECT id, date, title, gamename, clip_by, downloaded, uploaded FROM clips WHERE downloaded = 0"
|
||||||
|
)
|
||||||
|
records = cursor.fetchall()
|
||||||
|
conn.close()
|
||||||
|
return records
|
||||||
|
|
||||||
|
def insert_record(record_id: str, record_date: str, title: str, gamename: str, clip_by: str):
|
||||||
|
"""Inserts a clip record including its text slug string and creator username."""
|
||||||
|
conn = sqlite3.connect("database.db")
|
||||||
|
cursor = conn.cursor()
|
||||||
|
|
||||||
|
# Added clip_by TEXT NOT NULL field to table definition
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS clips (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
date TEXT NOT NULL,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
gamename TEXT NOT NULL,
|
||||||
|
clip_by TEXT NOT NULL,
|
||||||
|
downloaded INTEGER NOT NULL,
|
||||||
|
uploaded INTEGER NOT NULL
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
# Handled database migration gracefully if running script on an older DB missing the column
|
||||||
|
try:
|
||||||
|
cursor.execute(
|
||||||
|
"INSERT OR IGNORE INTO clips (id, date, title, gamename, clip_by, downloaded, uploaded) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||||
|
(str(record_id), str(record_date), title, gamename, clip_by, 0, 0),
|
||||||
|
)
|
||||||
|
except sqlite3.OperationalError as e:
|
||||||
|
if "has no column named clip_by" in str(e):
|
||||||
|
print("Upgrading database schema to support creator names...")
|
||||||
|
cursor.execute("ALTER TABLE clips ADD COLUMN clip_by TEXT DEFAULT 'Unknown'")
|
||||||
|
cursor.execute(
|
||||||
|
"INSERT OR IGNORE INTO clips (id, date, title, gamename, clip_by, downloaded, uploaded) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||||
|
(str(record_id), str(record_date), title, gamename, clip_by, 0, 0),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise e
|
||||||
|
|
||||||
|
conn.commit()
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def get_clip_slugs(channel_name):
|
||||||
|
session = requests.Session()
|
||||||
|
url = "https://twitch.tv"
|
||||||
|
|
||||||
|
session.headers = {
|
||||||
|
"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
|
"Content-Type": "text/plain"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Query updated to fetch the curator's login (the user who clipped it)
|
||||||
|
query_string = """
|
||||||
|
query GetChannelClips($login: String!, $limit: Int!) {
|
||||||
|
user(login: $login) {
|
||||||
|
clips(first: $limit, criteria: { period: ALL_TIME, sort: VIEWS_DESC }) {
|
||||||
|
edges {
|
||||||
|
node {
|
||||||
|
slug
|
||||||
|
title
|
||||||
|
createdAt
|
||||||
|
game {
|
||||||
|
displayName
|
||||||
|
}
|
||||||
|
curator {
|
||||||
|
login
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
payload = [{
|
||||||
|
"operationName": "GetChannelClips",
|
||||||
|
"query": query_string,
|
||||||
|
"variables": {
|
||||||
|
"login": channel_name.lower(),
|
||||||
|
"limit": 30
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
|
||||||
|
try:
|
||||||
|
req = requests.Request('POST', url, json=payload)
|
||||||
|
prepped = session.prepare_request(req)
|
||||||
|
response = session.send(prepped)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
result = data[0] if isinstance(data, list) else data
|
||||||
|
|
||||||
|
if "errors" in result:
|
||||||
|
print(f"Twitch GraphQL Error: {result['errors']}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
user_data = result['data']['user']
|
||||||
|
if not user_data:
|
||||||
|
print(f"Channel '{channel_name}' not found.")
|
||||||
|
return []
|
||||||
|
|
||||||
|
edges = user_data['clips']['edges']
|
||||||
|
clip_slugs = []
|
||||||
|
|
||||||
|
print(f"--- Top Clips for {channel_name} ---")
|
||||||
|
for edge in edges:
|
||||||
|
node = edge['node']
|
||||||
|
game_info = node.get('game')
|
||||||
|
game_name = game_info.get('displayName') if game_info else "Unknown/No Category"
|
||||||
|
|
||||||
|
# Safe extraction in case the curator account was deleted/missing
|
||||||
|
curator_info = node.get('curator')
|
||||||
|
clip_by = curator_info.get('login') if curator_info else "Unknown Creator"
|
||||||
|
|
||||||
|
print(f"Slug: {node['slug']} | Date: {node['createdAt']} | Game: {game_name} | By: {clip_by} | Title: {node['title']}")
|
||||||
|
|
||||||
|
# Passing clip_by to database writer block
|
||||||
|
insert_record(node['slug'], node['createdAt'], node['title'], game_name, clip_by)
|
||||||
|
clip_slugs.append(node['slug'])
|
||||||
|
|
||||||
|
return clip_slugs
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"An unexpected error occurred: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
get_clip_slugs(CHANNEL_NAME)
|
||||||
|
download_clips()
|
||||||
+190
@@ -0,0 +1,190 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import requests
|
||||||
|
import sqlite3
|
||||||
|
import subprocess
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
||||||
|
|
||||||
|
# Stores data locally
|
||||||
|
CONN = sqlite3.connect("clips_database.db")
|
||||||
|
CURSOR = CONN.cursor()
|
||||||
|
|
||||||
|
def create_database():
|
||||||
|
"""Creates a table structured explicitly for Twitch clip properties."""
|
||||||
|
CURSOR.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS clips (
|
||||||
|
slug TEXT PRIMARY KEY,
|
||||||
|
date TEXT NOT NULL,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
gamename TEXT NOT NULL,
|
||||||
|
clip_by TEXT NOT NULL,
|
||||||
|
view_count INTEGER NOT NULL,
|
||||||
|
downloaded INTEGER NOT NULL,
|
||||||
|
uploaded_yt INTEGER NOT NULL
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
CONN.commit()
|
||||||
|
|
||||||
|
def close_database():
|
||||||
|
"""Commits queries before ending connection context."""
|
||||||
|
CONN.commit()
|
||||||
|
CONN.close()
|
||||||
|
|
||||||
|
def download_clips():
|
||||||
|
"""Loops over undownloaded metadata entries to write files down locally."""
|
||||||
|
undownloaded_clips = get_undownloaded_clips()
|
||||||
|
|
||||||
|
print("Downloading Clips...")
|
||||||
|
|
||||||
|
for slug, record_date, title, game_name, clip_by, views, downloaded in undownloaded_clips:
|
||||||
|
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
|
||||||
|
|
||||||
|
# Ensures destination folder structures exist before executing CLI tool
|
||||||
|
run_linux_command(f"mkdir -p save/clips/{slug}")
|
||||||
|
|
||||||
|
# Uses standard clipdownload directive
|
||||||
|
output = run_linux_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{slug}.mp4")
|
||||||
|
|
||||||
|
if output["success"] is True:
|
||||||
|
print(f"Slug: {slug} | Was successfully downloaded.")
|
||||||
|
mark_as_downloaded(slug)
|
||||||
|
else:
|
||||||
|
print(f"Slug: {slug} | Process failed.")
|
||||||
|
|
||||||
|
print("Finished Downloading Clips...")
|
||||||
|
|
||||||
|
def run_linux_command(command: str):
|
||||||
|
"""Executes a Linux command, waits for completion, and returns output."""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
command, shell=True, check=True, capture_output=True, text=True
|
||||||
|
)
|
||||||
|
return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
|
||||||
|
|
||||||
|
def mark_as_downloaded(slug: str):
|
||||||
|
"""Flags a specific clip row record to downloaded (1)."""
|
||||||
|
CURSOR.execute(
|
||||||
|
"UPDATE clips SET downloaded = 1 WHERE slug = ?",
|
||||||
|
(slug,)
|
||||||
|
)
|
||||||
|
CONN.commit()
|
||||||
|
|
||||||
|
def get_undownloaded_clips():
|
||||||
|
"""Retrieves all clip rows remaining to be captured."""
|
||||||
|
CURSOR.execute(
|
||||||
|
"SELECT slug, date, title, gamename, clip_by, view_count, downloaded FROM clips WHERE downloaded = 0"
|
||||||
|
)
|
||||||
|
return CURSOR.fetchall()
|
||||||
|
|
||||||
|
def insert_record(slug: str, record_date_str: str, title: str, gamename: str, clip_by: str, views: int):
|
||||||
|
"""Cleans up ISO-8601 strings into unified date structures for the database."""
|
||||||
|
try:
|
||||||
|
clean_date = datetime.strptime(record_date_str, "%Y-%m-%dT%H:%M:%SZ").date()
|
||||||
|
except ValueError:
|
||||||
|
clean_date = record_date_str
|
||||||
|
|
||||||
|
CURSOR.execute(
|
||||||
|
"INSERT OR IGNORE INTO clips (slug, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||||
|
(slug, str(clean_date), title, gamename, clip_by, views, False, False),
|
||||||
|
)
|
||||||
|
CONN.commit()
|
||||||
|
|
||||||
|
def get_channel_clips(channel_name: str):
|
||||||
|
"""Queries Twitch's public endpoint directly for trending clips."""
|
||||||
|
session = requests.Session()
|
||||||
|
url = "https://gql.twitch.tv/gql"
|
||||||
|
|
||||||
|
session.headers = {
|
||||||
|
"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
|
"Content-Type": "text/plain"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Twitch GQL schema for channel discovery clips
|
||||||
|
query_string = """
|
||||||
|
query GetChannelClips($login: String!, $limit: Int!) {
|
||||||
|
user(login: $login) {
|
||||||
|
clips(first: $limit, criteria: { period: ALL_TIME }) {
|
||||||
|
edges {
|
||||||
|
node {
|
||||||
|
slug
|
||||||
|
title
|
||||||
|
createdAt
|
||||||
|
viewCount
|
||||||
|
game {
|
||||||
|
displayName
|
||||||
|
}
|
||||||
|
curator {
|
||||||
|
login
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
payload = [{
|
||||||
|
"operationName": "GetChannelClips",
|
||||||
|
"query": query_string,
|
||||||
|
"variables": {
|
||||||
|
"login": channel_name.lower(),
|
||||||
|
"limit": 40
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
|
||||||
|
try:
|
||||||
|
req = requests.Request('POST', url, json=payload)
|
||||||
|
prepped = session.prepare_request(req)
|
||||||
|
|
||||||
|
response = session.send(prepped)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
result = data[0] if isinstance(data, list) else data
|
||||||
|
|
||||||
|
if "errors" in result:
|
||||||
|
print(f"Twitch GraphQL Error: {result['errors']}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
user_data = result['data']['user']
|
||||||
|
if not user_data:
|
||||||
|
print(f"Channel '{channel_name}' not found.")
|
||||||
|
return []
|
||||||
|
|
||||||
|
edges = user_data['clips']['edges']
|
||||||
|
slugs = []
|
||||||
|
|
||||||
|
print(f"--- Top Clips for {channel_name} ---")
|
||||||
|
for edge in edges:
|
||||||
|
node = edge['node']
|
||||||
|
|
||||||
|
game_info = node.get('game')
|
||||||
|
game_name = game_info.get('displayName') if game_info else "Unknown/No Category"
|
||||||
|
slug_id = node['slug']
|
||||||
|
|
||||||
|
# Safe extraction in case the curator account was deleted/missing
|
||||||
|
curator_info = node.get('curator')
|
||||||
|
clip_by = curator_info.get('login') if curator_info else "Unknown Creator"
|
||||||
|
|
||||||
|
print(f"Slug: {node['slug']} | Date: {node['createdAt']} | Game: {game_name} | By: {clip_by} | Views: {node['viewCount']} | Title: {node['title']}")
|
||||||
|
|
||||||
|
insert_record(node['slug'], node['createdAt'], node['title'], game_name, clip_by, int(node['viewCount']))
|
||||||
|
slugs.append(slug_id)
|
||||||
|
|
||||||
|
return slugs
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"An unexpected error occurred: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
create_database()
|
||||||
|
get_channel_clips(CHANNEL_NAME)
|
||||||
|
download_clips()
|
||||||
|
close_database()
|
||||||
+102
@@ -0,0 +1,102 @@
|
|||||||
|
import json
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
|
||||||
|
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
||||||
|
DB_FILE = "twitch_vods.json"
|
||||||
|
|
||||||
|
def get_vod_data(channel_name):
|
||||||
|
session = requests.Session()
|
||||||
|
url = "https://twitch.tv"
|
||||||
|
|
||||||
|
session.headers = {
|
||||||
|
"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
|
"Content-Type": "text/plain"
|
||||||
|
}
|
||||||
|
|
||||||
|
query_string = """
|
||||||
|
query GetChannelVideos($login: String!, $limit: Int!) {
|
||||||
|
user(login: $login) {
|
||||||
|
videos(first: $limit, types: [ARCHIVE]) {
|
||||||
|
edges {
|
||||||
|
node {
|
||||||
|
id
|
||||||
|
title
|
||||||
|
publishedAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
payload = [{
|
||||||
|
"operationName": "GetChannelVideos",
|
||||||
|
"query": query_string,
|
||||||
|
"variables": {
|
||||||
|
"login": channel_name.lower(),
|
||||||
|
"limit": 30
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
|
||||||
|
try:
|
||||||
|
req = requests.Request('POST', url, json=payload)
|
||||||
|
prepped = session.prepare_request(req)
|
||||||
|
response = session.send(prepped)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
result = data if isinstance(data, list) else data
|
||||||
|
|
||||||
|
if "errors" in result:
|
||||||
|
print(f"Twitch GraphQL Error: {result['errors']}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
user_data = result['data']['user']
|
||||||
|
if not user_data:
|
||||||
|
print(f"Channel '{channel_name}' not found.")
|
||||||
|
return []
|
||||||
|
|
||||||
|
return user_data['videos']['edges']
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Failed to fetch data: {e}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def store_in_nosql_format(edges):
|
||||||
|
# Load existing NoSQL database if it exists, otherwise start fresh
|
||||||
|
if os.path.exists(DB_FILE):
|
||||||
|
with open(DB_FILE, 'r', encoding='utf-8') as f:
|
||||||
|
try:
|
||||||
|
db = json.load(f)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
db = {}
|
||||||
|
else:
|
||||||
|
db = {}
|
||||||
|
|
||||||
|
new_records_count = 0
|
||||||
|
|
||||||
|
# Process and structure data using VOD ID as the key
|
||||||
|
for edge in edges:
|
||||||
|
node = edge['node']
|
||||||
|
vod_id = node['id']
|
||||||
|
|
||||||
|
# This structures the document under the specific ID key
|
||||||
|
db[vod_id] = {
|
||||||
|
"title": node['title'],
|
||||||
|
"published_at": node['publishedAt'],
|
||||||
|
"channel": CHANNEL_NAME.lower()
|
||||||
|
}
|
||||||
|
new_records_count += 1
|
||||||
|
|
||||||
|
# Save the updated document store back to the disk
|
||||||
|
with open(DB_FILE, 'w', encoding='utf-8') as f:
|
||||||
|
json.dump(db, f, indent=4, ensure_ascii=False)
|
||||||
|
|
||||||
|
print(f"Successfully processed {new_records_count} VOD documents into '{DB_FILE}'.")
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
vod_edges = get_vod_data(CHANNEL_NAME)
|
||||||
|
if vod_edges:
|
||||||
|
store_in_nosql_format(vod_edges)
|
||||||
+195
@@ -0,0 +1,195 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
import requests
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
from datetime import date as datetime_date
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
||||||
|
|
||||||
|
CONN = sqlite3.connect("database.db")
|
||||||
|
CURSOR = CONN.cursor()
|
||||||
|
|
||||||
|
def create_database():
|
||||||
|
# Creates table safely using multi-line string
|
||||||
|
CURSOR.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS vods (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
date TEXT NOT NULL,
|
||||||
|
title TEXT NOT NULL,
|
||||||
|
gamename TEXT NOT NULL,
|
||||||
|
downloaded INTEGER NOT NULL,
|
||||||
|
uploaded_yt INTEGER NOT NULL
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
CONN.commit()
|
||||||
|
|
||||||
|
def close_database():
|
||||||
|
"""Commit before we close."""
|
||||||
|
CONN.commit()
|
||||||
|
CONN.close()
|
||||||
|
|
||||||
|
def download_vods():
|
||||||
|
"""Find all undownload vods and download them."""
|
||||||
|
undownload_vods = get_undownloaded_vods()
|
||||||
|
|
||||||
|
print("Download VODs...")
|
||||||
|
|
||||||
|
for record_id, record_date, title, game_name, downloaded in undownload_vods:
|
||||||
|
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
|
||||||
|
output = run_linux_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/{record_id}/{record_id}.mp4")
|
||||||
|
output2 = run_linux_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/{record_id}/{record_id}_chat.json -E")
|
||||||
|
if output["success"] is True:
|
||||||
|
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was successful")
|
||||||
|
mark_as_downloaded(record_id)
|
||||||
|
else:
|
||||||
|
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was Failed")
|
||||||
|
|
||||||
|
print("Finished Downloading VODs...")
|
||||||
|
|
||||||
|
def run_linux_command(command: str):
|
||||||
|
"""Executes a Linux command, waits for completion, and returns output."""
|
||||||
|
try:
|
||||||
|
# shell=True allows running full command strings with pipes/wildcards
|
||||||
|
# text=True returns strings instead of bytes
|
||||||
|
result = subprocess.run(
|
||||||
|
command, shell=True, check=True, capture_output=True, text=True
|
||||||
|
)
|
||||||
|
|
||||||
|
return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
|
||||||
|
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
# Handles errors if the Linux command returns a non-zero exit code
|
||||||
|
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
|
||||||
|
|
||||||
|
def mark_as_downloaded(record_id: int):
|
||||||
|
"""Updates the downloaded status to True (1) for a specific record ID."""
|
||||||
|
|
||||||
|
# Updates the row matching the specific ID
|
||||||
|
CURSOR.execute(
|
||||||
|
"UPDATE vods SET downloaded = 1 WHERE id = ?",
|
||||||
|
(record_id,)
|
||||||
|
)
|
||||||
|
|
||||||
|
CONN.commit()
|
||||||
|
|
||||||
|
def get_undownloaded_vods():
|
||||||
|
"""Retrieves all rows where downloaded status is False (0)."""
|
||||||
|
|
||||||
|
# Query filters by 0 because SQLite stores booleans as integers
|
||||||
|
CURSOR.execute(
|
||||||
|
"SELECT id, date, title, gamename, downloaded FROM vods WHERE downloaded = 0"
|
||||||
|
)
|
||||||
|
records = CURSOR.fetchall()
|
||||||
|
|
||||||
|
return records
|
||||||
|
|
||||||
|
def insert_record(record_id: int, record_date: datetime_date, title: str, gamename: str):
|
||||||
|
"""Inserts a record with ID, date, gamename, and title into a SQLite database."""
|
||||||
|
# Connects to database file (creates it if missing)
|
||||||
|
|
||||||
|
# Inserts data using parameterized queries to prevent SQL injection
|
||||||
|
CURSOR.execute(
|
||||||
|
"INSERT OR IGNORE INTO vods (id, date, title, gamename, downloaded, uploaded_yt) VALUES (?, ?, ?, ?, ?, ?)",
|
||||||
|
(record_id, str(record_date), title, gamename, False, False),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Saves changes and closes the connection
|
||||||
|
CONN.commit()
|
||||||
|
|
||||||
|
def get_vod_ids_simplified(channel_name: str):
|
||||||
|
session = requests.Session()
|
||||||
|
url = "https://gql.twitch.tv/gql"
|
||||||
|
|
||||||
|
# Case-preserved headers to prevent 400 Bad Request errors
|
||||||
|
session.headers = {
|
||||||
|
"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
|
||||||
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||||
|
"Content-Type": "text/plain"
|
||||||
|
}
|
||||||
|
|
||||||
|
# A simplified query string that hardcodes the ARCHIVE type filter into the request structure
|
||||||
|
query_string = """
|
||||||
|
query GetChannelVideos($login: String!, $limit: Int!) {
|
||||||
|
user(login: $login) {
|
||||||
|
videos(first: $limit, types: [ARCHIVE]) {
|
||||||
|
edges {
|
||||||
|
node {
|
||||||
|
id
|
||||||
|
title
|
||||||
|
publishedAt
|
||||||
|
game {
|
||||||
|
displayName
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
|
||||||
|
payload = [{
|
||||||
|
"operationName": "GetChannelVideos",
|
||||||
|
"query": query_string,
|
||||||
|
"variables": {
|
||||||
|
"login": channel_name.lower(),
|
||||||
|
"limit": 50
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Prepping ensures Python does not rewrite the Client-ID header case
|
||||||
|
req = requests.Request('POST', url, json=payload)
|
||||||
|
prepped = session.prepare_request(req)
|
||||||
|
|
||||||
|
response = session.send(prepped)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
data = response.json()
|
||||||
|
|
||||||
|
# Pull out the target index array dictionary object
|
||||||
|
result = data[0] if isinstance(data, list) else data
|
||||||
|
|
||||||
|
if "errors" in result:
|
||||||
|
print(f"Twitch GraphQL Error: {result['errors']}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
user_data = result['data']['user']
|
||||||
|
if not user_data:
|
||||||
|
print(f"Channel '{channel_name}' not found.")
|
||||||
|
return []
|
||||||
|
|
||||||
|
edges = user_data['videos']['edges']
|
||||||
|
vod_ids = []
|
||||||
|
|
||||||
|
print(f"--- Latest VODs for {channel_name} ---")
|
||||||
|
for edge in edges:
|
||||||
|
node = edge['node']
|
||||||
|
|
||||||
|
# Safe extraction in case a VOD has no category set (Just Chatting, Uncategorized, etc.)
|
||||||
|
game_info = node.get('game')
|
||||||
|
game_name = game_info.get('displayName') if game_info else "Unknown/No Category"
|
||||||
|
|
||||||
|
print(f"ID: {node['id']} | Date: {node['publishedAt']} | Game: {game_name} | Title: {node['title']}")
|
||||||
|
|
||||||
|
# Pass game_name to your database logic
|
||||||
|
insert_record(node['id'], node['publishedAt'], node['title'], game_name)
|
||||||
|
vod_ids.append(node['id'])
|
||||||
|
|
||||||
|
return vod_ids
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"An unexpected error occurred: {e}")
|
||||||
|
if 'response' in locals():
|
||||||
|
print(f"Server Response Text: {response.text}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
create_database()
|
||||||
|
get_vod_ids_simplified(CHANNEL_NAME)
|
||||||
|
download_vods()
|
||||||
|
close_database()
|
||||||
Reference in New Issue
Block a user