104 lines
2.9 KiB
Python
104 lines
2.9 KiB
Python
#!/usr/bin/env python3
|
|
import json
|
|
import os
|
|
import requests
|
|
|
|
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
|
DB_FILE = "twitch_vods.json"
|
|
|
|
def get_vod_data(channel_name):
|
|
session = requests.Session()
|
|
url = "https://twitch.tv"
|
|
|
|
session.headers = {
|
|
"Client-ID": "kimne78kx3ncx6brgo4mv6wki5h1ko",
|
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
"Content-Type": "text/plain"
|
|
}
|
|
|
|
query_string = """
|
|
query GetChannelVideos($login: String!, $limit: Int!) {
|
|
user(login: $login) {
|
|
videos(first: $limit, types: [ARCHIVE]) {
|
|
edges {
|
|
node {
|
|
id
|
|
title
|
|
publishedAt
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
payload = [{
|
|
"operationName": "GetChannelVideos",
|
|
"query": query_string,
|
|
"variables": {
|
|
"login": channel_name.lower(),
|
|
"limit": 30
|
|
}
|
|
}]
|
|
|
|
try:
|
|
req = requests.Request('POST', url, json=payload)
|
|
prepped = session.prepare_request(req)
|
|
response = session.send(prepped)
|
|
response.raise_for_status()
|
|
|
|
data = response.json()
|
|
result = data if isinstance(data, list) else data
|
|
|
|
if "errors" in result:
|
|
print(f"Twitch GraphQL Error: {result['errors']}")
|
|
return []
|
|
|
|
user_data = result['data']['user']
|
|
if not user_data:
|
|
print(f"Channel '{channel_name}' not found.")
|
|
return []
|
|
|
|
return user_data['videos']['edges']
|
|
|
|
except Exception as e:
|
|
print(f"Failed to fetch data: {e}")
|
|
return []
|
|
|
|
def store_in_nosql_format(edges):
|
|
# Load existing NoSQL database if it exists, otherwise start fresh
|
|
if os.path.exists(DB_FILE):
|
|
with open(DB_FILE, 'r', encoding='utf-8') as f:
|
|
try:
|
|
db = json.load(f)
|
|
except json.JSONDecodeError:
|
|
db = {}
|
|
else:
|
|
db = {}
|
|
|
|
new_records_count = 0
|
|
|
|
# Process and structure data using VOD ID as the key
|
|
for edge in edges:
|
|
node = edge['node']
|
|
vod_id = node['id']
|
|
|
|
# This structures the document under the specific ID key
|
|
db[vod_id] = {
|
|
"title": node['title'],
|
|
"published_at": node['publishedAt'],
|
|
"channel": CHANNEL_NAME.lower()
|
|
}
|
|
new_records_count += 1
|
|
|
|
# Save the updated document store back to the disk
|
|
with open(DB_FILE, 'w', encoding='utf-8') as f:
|
|
json.dump(db, f, indent=4, ensure_ascii=False)
|
|
|
|
print(f"Successfully processed {new_records_count} VOD documents into '{DB_FILE}'.")
|
|
|
|
if __name__ == "__main__":
|
|
vod_edges = get_vod_data(CHANNEL_NAME)
|
|
if vod_edges:
|
|
store_in_nosql_format(vod_edges)
|