327 lines
12 KiB
Python
Executable File
327 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import requests
|
|
import os
|
|
import linux
|
|
import time
|
|
|
|
from database import Database
|
|
|
|
import uploader
|
|
from uploader import CategoryId
|
|
|
|
import youtube_hashtags
|
|
import transcribe_video
|
|
|
|
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
|
DB = None
|
|
|
|
def transcribe(id: str):
|
|
"""Transcribes the Video File."""
|
|
video_file = f"save/{DB.table}/{id}/{id}.mp4"
|
|
|
|
# Check if the video file exists
|
|
if not os.path.exists(video_file):
|
|
print(f"Error: File not found: {video_file}")
|
|
return False
|
|
|
|
# no need to continue if srt transcribe file already exists
|
|
if os.path.exists(f"save/{DB.table}/{id}/transcribe_{id}.srt"):
|
|
print(f"video already transcribed:")
|
|
return True
|
|
|
|
transcribe_video.extract_audio(video_file, f"save/{DB.table}/{id}/temp_{id}_audio.wav")
|
|
transcribe_video.transcribe_to_srt(f"save/{DB.table}/{id}/temp_{id}_audio.wav", f"save/{DB.table}/{id}/", f"transcribe_{id}")
|
|
|
|
return True
|
|
|
|
def top_hashtags(id: str):
|
|
""""Hashtags from transcribed SRT file."""
|
|
file_srt = f"save/{DB.table}/{id}/transcribe_{id}.srt"
|
|
|
|
# if srt transcribe file not exists
|
|
if not os.path.exists(file_srt):
|
|
print(f"Transcribe file not found {file_srt}")
|
|
return []
|
|
|
|
tags = youtube_hashtags.get_top_hashtags(file_srt)
|
|
return tags
|
|
|
|
def download():
|
|
"""Find all undownload videos and download them."""
|
|
undownloaded = DB.get_undownloaded()
|
|
|
|
print(f"Download {DB.table}...")
|
|
|
|
if DB.table == "videos":
|
|
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chat_upload_yt in undownloaded:
|
|
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
|
|
output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/videos/{record_id}/{record_id}.mp4 --collision Overwrite")
|
|
output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/videos/{record_id}/{record_id}_chat.json -E --collision Overwrite")
|
|
time.sleep(1)
|
|
if output['success'] is True:
|
|
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was successful")
|
|
DB.mark_as_downloaded(record_id)
|
|
else:
|
|
print(f"ID: {record_id} | Download Process failed. {output['stdout']}. Error: {output['stderr']}")
|
|
|
|
elif DB.table == "clips":
|
|
for slug, record_date, title, game_name, clip_by, views, downloaded, uploaded_yt, uploaded_shorts_yt in undownloaded:
|
|
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
|
|
|
|
# Uses standard clipdownload directive
|
|
output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{slug}.mp4 --collision Overwrite")
|
|
time.sleep(1)
|
|
|
|
if output['success'] is True:
|
|
print(f"Slug: {slug} | Was successfully downloaded.")
|
|
DB.mark_as_downloaded(slug)
|
|
else:
|
|
print(f"Slug: {slug} | Download Process failed. {output['stdout']}. Error: {output['stderr']}")
|
|
|
|
|
|
print(f"Finished Downloading {DB.table}...")
|
|
|
|
def upload():
|
|
"""Loops over the downloaded videos entries and uploaded them to youtube."""
|
|
print(f"Uploading {DB.table}...")
|
|
|
|
unuploaded = DB.get_unuploaded()
|
|
|
|
twitch_datetime = " Live on Twitch Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp"
|
|
file_path = ""
|
|
title = ""
|
|
description = ""
|
|
categoryId = CategoryId.GAMING
|
|
privatcyStatus = 'private'
|
|
base_tags = ['gaming', 'TeamPGP', 'twitch', 'Level1Techs']
|
|
|
|
upload_queue = []
|
|
|
|
if DB.table == "videos":
|
|
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chats_upload_yt in unuploaded:
|
|
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
|
|
|
|
file_path = f"save/videos/{record_id}/{record_id}.mp4"
|
|
description = f"Game: {game_name}, on {record_date}, #VODS {twitch_datetime}"
|
|
tags = list(base_tags)
|
|
tags.extend([f'{game_name}', 'twitch_vods', 'vods'])
|
|
tags.extend(top_hashtags(record_id))
|
|
|
|
upload_queue.append([record_id, file_path, title, categoryId, description, privatcyStatus, tags])
|
|
|
|
elif DB.table == "clips":
|
|
for slug, record_date, title, game_name, clip_by, views, downloaded, uploaded_yt, shorts_uploaded_yt in unuploaded:
|
|
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
|
|
|
|
transcribe(slug)
|
|
|
|
file_path = f"save/clips/{slug}/{slug}.mp4"
|
|
description = f"Game: {game_name}, Cliped By: {clip_by}, on {record_date}, #Shorts #Clips {twitch_datetime}"
|
|
tags = list(base_tags)
|
|
tags.extend([f'{game_name}', 'twitch_clips', 'clips', f'{clip_by}', 'shorts'])
|
|
tags.extend(top_hashtags(slug))
|
|
|
|
upload_queue.append([slug, file_path, title, categoryId, description, privatcyStatus, tags])
|
|
|
|
for db_id, file_path, title, categoryId, description, privatcyStatus, tags in upload_queue:
|
|
output = uploader.upload_video(file_path, title, categoryId, description, privatcyStatus, tags)
|
|
if output is True:
|
|
print(f"Title: {title} | Was successfully uploaded.")
|
|
DB.mark_as_uploaded(db_id)
|
|
else:
|
|
print(f"Title: {title} | Download Process failed.")
|
|
|
|
def create_chats():
|
|
pass
|
|
|
|
def create_shorts():
|
|
pass
|
|
|
|
def get_vod_ids_simplified(channel_name: str):
|
|
"""Queries Twitch's public endpoint directly for trending clips."""
|
|
session = requests.Session()
|
|
url = "https://gql.twitch.tv/gql"
|
|
|
|
# Case-preserved headers to prevent 400 Bad Request errors
|
|
session.headers = {
|
|
"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
"Content-Type": "text/plain"
|
|
}
|
|
|
|
videos_query_string = """
|
|
query GetChannelVideos($login: String!, $limit: Int!, $after: Cursor) {
|
|
user(login: $login) {
|
|
videos(first: $limit, types: [ARCHIVE], after: $after) {
|
|
pageInfo {
|
|
hasNextPage
|
|
endCursor
|
|
}
|
|
edges {
|
|
node {
|
|
id
|
|
title
|
|
publishedAt
|
|
game {
|
|
displayName
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
clips_query_string = """
|
|
query GetChannelClips($login: String!, $limit: Int!, $after: Cursor) {
|
|
user(login: $login) {
|
|
clips(first: $limit, criteria: { period: ALL_TIME }, after: $after) {
|
|
pageInfo {
|
|
hasNextPage
|
|
endCursor
|
|
}
|
|
edges {
|
|
cursor
|
|
node {
|
|
slug
|
|
title
|
|
createdAt
|
|
viewCount
|
|
game {
|
|
displayName
|
|
}
|
|
curator {
|
|
login
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
video_ids = []
|
|
has_next_page = True
|
|
cursor = None
|
|
|
|
limit = 50
|
|
query_string = ""
|
|
operation_name = ""
|
|
if DB.table == "videos":
|
|
query_string = videos_query_string
|
|
limit = 100
|
|
operation_name = "GetChannelVideos"
|
|
elif DB.table == "clips":
|
|
query_string = clips_query_string
|
|
limit = 40
|
|
operation_name = "GetChannelClips"
|
|
|
|
while has_next_page:
|
|
time.sleep(1)
|
|
# A simplified query string that hardcodes the ARCHIVE type filter into the request structure
|
|
payload = [{
|
|
"operationName": operation_name,
|
|
"query": query_string,
|
|
"variables": {
|
|
"login": channel_name.lower(),
|
|
"limit": limit,
|
|
"after": cursor
|
|
}
|
|
}]
|
|
|
|
try:
|
|
# Prepping ensures Python does not rewrite the Client-ID header case
|
|
req = requests.Request('POST', url, json=payload)
|
|
#req = requests.Request('POST', url, data=json.dumps(payload))
|
|
prepped = session.prepare_request(req)
|
|
|
|
response = session.send(prepped)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
|
|
# Pull out the target index array dictionary object
|
|
result = data[0] if isinstance(data, list) else data
|
|
|
|
if "errors" in result:
|
|
print(f"Twitch GraphQL Error: {result['errors']}")
|
|
return []
|
|
|
|
user_data = result.get('data', {}).get('user', {})
|
|
if not user_data:
|
|
print(f"Channel '{channel_name}' not found.")
|
|
return []
|
|
|
|
edges = user_data.get(DB.table, {}).get('edges', [])
|
|
|
|
# --- BREAK CONDITION 1: Stop if Twitch returns no more data items ---
|
|
if not edges or len(edges) == 0:
|
|
print("No more items returned by the server. Ending pagination loop.")
|
|
break
|
|
|
|
last_edge_cursor = None
|
|
print(f"--- Processing {DB.table} for {channel_name} ---")
|
|
for edge in edges:
|
|
last_edge_cursor = edge.get("cursor")
|
|
node = edge.get('node', {})
|
|
if not node:
|
|
continue
|
|
|
|
game_info = node.get('game')
|
|
game_name = game_info.get('displayName') if game_info else "Unknown/No Category"
|
|
|
|
# FIXED: Switched fields to use safe .get() metrics to completely prevent KeyErrors
|
|
node_id = node.get('id')
|
|
node_title = node.get('title', 'No Title')
|
|
|
|
if DB.table == "videos":
|
|
published_at = node.get('publishedAt')
|
|
print(f"ID: {node_id} | Date: {published_at} | Game: {game_name} | Title: {node_title}")
|
|
|
|
DB.insert_videos_record(node_id, published_at, node_title, game_name)
|
|
if node_id:
|
|
video_ids.append(node_id)
|
|
|
|
elif DB.table == "clips":
|
|
slug = node.get('slug')
|
|
created_at = node.get('createdAt')
|
|
view_count = node.get('viewCount', 0)
|
|
|
|
curator_info = node.get('curator')
|
|
clip_by = curator_info.get('login') if curator_info else "Unknown Creator"
|
|
|
|
print(f"Slug: {slug} | Date: {created_at} | Game: {game_name} | By: {clip_by} | Views: {view_count} | Title: {node_title}")
|
|
|
|
DB.insert_clips_record(slug, created_at, node_title, game_name, clip_by, int(view_count))
|
|
if slug:
|
|
video_ids.append(slug)
|
|
|
|
# --- CORRECTED PAGINATION ENGINE FOR BOTH TABLES ---
|
|
page_info = user_data.get(DB.table, {}).get('pageInfo', {})
|
|
has_next_page = page_info.get("hasNextPage", False)
|
|
|
|
next_cursor = page_info.get("endCursor") or last_edge_cursor
|
|
|
|
if not next_cursor or next_cursor == cursor:
|
|
print("Cursor did not advance or is null. Safely terminating loop.")
|
|
break
|
|
|
|
cursor = next_cursor
|
|
|
|
except Exception as e:
|
|
print(f"An unexpected error occurred: {e}")
|
|
if 'response' in locals():
|
|
print(f"Server Response Text: {response.text}")
|
|
return []
|
|
|
|
return video_ids
|
|
|
|
if __name__ == "__main__":
|
|
tables = ["clips"]
|
|
for table in tables:
|
|
DB = Database(table)
|
|
|
|
get_vod_ids_simplified(CHANNEL_NAME)
|
|
#download()
|
|
#upload()
|
|
#DB.close_database() |