Made some chanages and now Can't remmebr what I did
This commit is contained in:
+136
-103
@@ -1,7 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
import requests
|
||||
import os
|
||||
import subprocess
|
||||
import linux
|
||||
import time
|
||||
|
||||
from database import Database
|
||||
|
||||
@@ -14,21 +15,6 @@ import transcribe_video
|
||||
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
||||
DB = None
|
||||
|
||||
def run_linux_command(command: str):
|
||||
"""Executes a Linux command, waits for completion, and returns output."""
|
||||
try:
|
||||
# shell=True allows running full command strings with pipes/wildcards
|
||||
# text=True returns strings instead of bytes
|
||||
result = subprocess.run(
|
||||
command, shell=True, check=True, capture_output=True, text=True
|
||||
)
|
||||
|
||||
return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
|
||||
|
||||
except subprocess.CalledProcessError as e:
|
||||
# Handles errors if the Linux command returns a non-zero exit code
|
||||
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
|
||||
|
||||
def transcribe(id: str):
|
||||
"""Transcribes the Video File."""
|
||||
video_file = f"save/{DB.table}/{id}/{id}.mp4"
|
||||
@@ -61,16 +47,17 @@ def top_hashtags(id: str):
|
||||
return tags
|
||||
|
||||
def download():
|
||||
"""Find all undownload vods and download them."""
|
||||
"""Find all undownload videos and download them."""
|
||||
undownloaded = DB.get_undownloaded()
|
||||
|
||||
print(f"Download {DB.table}...")
|
||||
|
||||
if DB.table == "vods":
|
||||
if DB.table == "videos":
|
||||
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chat_upload_yt in undownloaded:
|
||||
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
|
||||
output = run_linux_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/vods/{record_id}/{record_id}.mp4 --collision Overwrite")
|
||||
output2 = run_linux_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/vods/{record_id}/{record_id}_chat.json -E --collision Overwrite")
|
||||
output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/videos/{record_id}/{record_id}.mp4 --collision Overwrite")
|
||||
output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/videos/{record_id}/{record_id}_chat.json -E --collision Overwrite")
|
||||
time.sleep(1)
|
||||
if output["success"] is True:
|
||||
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was successful")
|
||||
DB.mark_as_downloaded(record_id)
|
||||
@@ -82,7 +69,8 @@ def download():
|
||||
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
|
||||
|
||||
# Uses standard clipdownload directive
|
||||
output = run_linux_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{slug}.mp4 --collision Overwrite")
|
||||
output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{slug}.mp4 --collision Overwrite")
|
||||
time.sleep(1)
|
||||
|
||||
if output["success"] is True:
|
||||
print(f"Slug: {slug} | Was successfully downloaded.")
|
||||
@@ -109,11 +97,11 @@ def upload():
|
||||
|
||||
upload_queue = []
|
||||
|
||||
if DB.table == "vods":
|
||||
if DB.table == "videos":
|
||||
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chats_upload_yt in unuploaded:
|
||||
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
|
||||
|
||||
file_path = f"save/vods/{record_id}/{record_id}.mp4"
|
||||
file_path = f"save/videos/{record_id}/{record_id}.mp4"
|
||||
description = f"Game: {game_name}, on {record_date}, #VODS {twitch_datetime}"
|
||||
tags = list(base_tags)
|
||||
tags.extend([f'{game_name}', 'twitch_vods', 'vods'])
|
||||
@@ -161,10 +149,14 @@ def get_vod_ids_simplified(channel_name: str):
|
||||
"Content-Type": "text/plain"
|
||||
}
|
||||
|
||||
vods_query_string = """
|
||||
query GetChannelVideos($login: String!, $limit: Int!) {
|
||||
videos_query_string = """
|
||||
query GetChannelVideos($login: String!, $limit: Int!, $after: Cursor) {
|
||||
user(login: $login) {
|
||||
videos(first: $limit, types: [ARCHIVE]) {
|
||||
videos(first: $limit, types: [ARCHIVE], after: $after) {
|
||||
pageInfo {
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
edges {
|
||||
node {
|
||||
id
|
||||
@@ -181,10 +173,15 @@ def get_vod_ids_simplified(channel_name: str):
|
||||
"""
|
||||
|
||||
clips_query_string = """
|
||||
query GetChannelClips($login: String!, $limit: Int!) {
|
||||
query GetChannelClips($login: String!, $limit: Int!, $after: Cursor) {
|
||||
user(login: $login) {
|
||||
clips(first: $limit, criteria: { period: ALL_TIME }) {
|
||||
clips(first: $limit, criteria: { period: ALL_TIME }, after: $after) {
|
||||
pageInfo {
|
||||
hasNextPage
|
||||
endCursor
|
||||
}
|
||||
edges {
|
||||
cursor
|
||||
node {
|
||||
slug
|
||||
title
|
||||
@@ -202,93 +199,129 @@ def get_vod_ids_simplified(channel_name: str):
|
||||
}
|
||||
}
|
||||
"""
|
||||
|
||||
video_ids = []
|
||||
has_next_page = True
|
||||
cursor = None
|
||||
|
||||
limit = 50
|
||||
query_string = ""
|
||||
operation_name = ""
|
||||
if DB.table == "vods":
|
||||
query_string = vods_query_string
|
||||
limit = 50
|
||||
if DB.table == "videos":
|
||||
query_string = videos_query_string
|
||||
limit = 100
|
||||
operation_name = "GetChannelVideos"
|
||||
elif DB.table == "clips":
|
||||
query_string = clips_query_string
|
||||
limit = 40
|
||||
operation_name = "GetChannelClips"
|
||||
# A simplified query string that hardcodes the ARCHIVE type filter into the request structure
|
||||
payload = [{
|
||||
"operationName": operation_name,
|
||||
"query": query_string,
|
||||
"variables": {
|
||||
"login": channel_name.lower(),
|
||||
"limit": limit
|
||||
}
|
||||
}]
|
||||
|
||||
try:
|
||||
# Prepping ensures Python does not rewrite the Client-ID header case
|
||||
req = requests.Request('POST', url, json=payload)
|
||||
prepped = session.prepare_request(req)
|
||||
|
||||
response = session.send(prepped)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
|
||||
# Pull out the target index array dictionary object
|
||||
result = data[0] if isinstance(data, list) else data
|
||||
|
||||
if "errors" in result:
|
||||
print(f"Twitch GraphQL Error: {result['errors']}")
|
||||
return []
|
||||
|
||||
user_data = result['data']['user']
|
||||
if not user_data:
|
||||
print(f"Channel '{channel_name}' not found.")
|
||||
return []
|
||||
|
||||
edges = None
|
||||
if DB.table == "vods":
|
||||
edges = user_data['videos']['edges']
|
||||
elif DB.table == "clips":
|
||||
edges = user_data['clips']['edges']
|
||||
video_ids = []
|
||||
|
||||
print(f"--- Latest VODs for {channel_name} ---")
|
||||
for edge in edges:
|
||||
node = edge['node']
|
||||
|
||||
# Safe extraction in case a VOD has no category set (Just Chatting, Uncategorized, etc.)
|
||||
game_info = node.get('game')
|
||||
game_name = game_info.get('displayName') if game_info else "Unknown/No Category"
|
||||
if DB.table == "vods":
|
||||
print(f"ID: {node['id']} | Date: {node['publishedAt']} | Game: {game_name} | Title: {node['title']}")
|
||||
|
||||
# Pass game_name to your database logic
|
||||
DB.insert_vods_record(node['id'], node['publishedAt'], node['title'], game_name)
|
||||
video_ids.append(node['id'])
|
||||
elif DB.table == "clips":
|
||||
# Safe extraction in case the curator account was deleted/missing
|
||||
curator_info = node.get('curator')
|
||||
clip_by = curator_info.get('login') if curator_info else "Unknown Creator"
|
||||
|
||||
print(f"Slug: {node['slug']} | Date: {node['createdAt']} | Game: {game_name} | By: {clip_by} | Views: {node['viewCount']} | Title: {node['title']}")
|
||||
|
||||
DB.insert_clips_record(node['slug'], node['createdAt'], node['title'], game_name, clip_by, int(node['viewCount']))
|
||||
video_ids.append(node['slug'])
|
||||
|
||||
return video_ids
|
||||
|
||||
except Exception as e:
|
||||
print(f"An unexpected error occurred: {e}")
|
||||
if 'response' in locals():
|
||||
print(f"Server Response Text: {response.text}")
|
||||
return []
|
||||
while has_next_page:
|
||||
time.sleep(1)
|
||||
# A simplified query string that hardcodes the ARCHIVE type filter into the request structure
|
||||
payload = [{
|
||||
"operationName": operation_name,
|
||||
"query": query_string,
|
||||
"variables": {
|
||||
"login": channel_name.lower(),
|
||||
"limit": limit,
|
||||
"after": cursor
|
||||
}
|
||||
}]
|
||||
|
||||
try:
|
||||
# Prepping ensures Python does not rewrite the Client-ID header case
|
||||
req = requests.Request('POST', url, json=payload)
|
||||
#req = requests.Request('POST', url, data=json.dumps(payload))
|
||||
prepped = session.prepare_request(req)
|
||||
|
||||
response = session.send(prepped)
|
||||
response.raise_for_status()
|
||||
|
||||
data = response.json()
|
||||
|
||||
# Pull out the target index array dictionary object
|
||||
result = data[0] if isinstance(data, list) else data
|
||||
|
||||
if "errors" in result:
|
||||
print(f"Twitch GraphQL Error: {result['errors']}")
|
||||
return []
|
||||
|
||||
user_data = result.get('data', {}).get('user', {})
|
||||
if not user_data:
|
||||
print(f"Channel '{channel_name}' not found.")
|
||||
return []
|
||||
|
||||
edges = user_data.get(DB.table, {}).get('edges', [])
|
||||
|
||||
# --- BREAK CONDITION 1: Stop if Twitch returns no more data items ---
|
||||
if not edges or len(edges) == 0:
|
||||
print("No more items returned by the server. Ending pagination loop.")
|
||||
break
|
||||
|
||||
last_edge_cursor = None
|
||||
print(f"--- Processing {DB.table} for {channel_name} ---")
|
||||
for edge in edges:
|
||||
last_edge_cursor = edge.get("cursor")
|
||||
node = edge.get('node', {})
|
||||
if not node:
|
||||
continue
|
||||
|
||||
game_info = node.get('game')
|
||||
game_name = game_info.get('displayName') if game_info else "Unknown/No Category"
|
||||
|
||||
# FIXED: Switched fields to use safe .get() metrics to completely prevent KeyErrors
|
||||
node_id = node.get('id')
|
||||
node_title = node.get('title', 'No Title')
|
||||
|
||||
if DB.table == "videos":
|
||||
published_at = node.get('publishedAt')
|
||||
print(f"ID: {node_id} | Date: {published_at} | Game: {game_name} | Title: {node_title}")
|
||||
|
||||
DB.insert_videos_record(node_id, published_at, node_title, game_name)
|
||||
if node_id:
|
||||
video_ids.append(node_id)
|
||||
|
||||
elif DB.table == "clips":
|
||||
slug = node.get('slug')
|
||||
created_at = node.get('createdAt')
|
||||
view_count = node.get('viewCount', 0)
|
||||
|
||||
curator_info = node.get('curator')
|
||||
clip_by = curator_info.get('login') if curator_info else "Unknown Creator"
|
||||
|
||||
print(f"Slug: {slug} | Date: {created_at} | Game: {game_name} | By: {clip_by} | Views: {view_count} | Title: {node_title}")
|
||||
|
||||
DB.insert_clips_record(slug, created_at, node_title, game_name, clip_by, int(view_count))
|
||||
if slug:
|
||||
video_ids.append(slug)
|
||||
|
||||
# --- CORRECTED PAGINATION ENGINE FOR BOTH TABLES ---
|
||||
page_info = user_data.get(DB.table, {}).get('pageInfo', {})
|
||||
has_next_page = page_info.get("hasNextPage", False)
|
||||
|
||||
next_cursor = page_info.get("endCursor") or last_edge_cursor
|
||||
|
||||
if not next_cursor or next_cursor == cursor:
|
||||
print("Cursor did not advance or is null. Safely terminating loop.")
|
||||
break
|
||||
|
||||
cursor = next_cursor
|
||||
|
||||
except Exception as e:
|
||||
print(f"An unexpected error occurred: {e}")
|
||||
if 'response' in locals():
|
||||
print(f"Server Response Text: {response.text}")
|
||||
return []
|
||||
|
||||
return video_ids
|
||||
|
||||
if __name__ == "__main__":
|
||||
tables = ["vods", "clips"]
|
||||
tables = ["clips"]
|
||||
for table in tables:
|
||||
DB = Database(table)
|
||||
|
||||
get_vod_ids_simplified(CHANNEL_NAME)
|
||||
download()
|
||||
upload()
|
||||
DB.close_database()
|
||||
#download()
|
||||
#upload()
|
||||
#DB.close_database()
|
||||
Reference in New Issue
Block a user