109 lines
4.4 KiB
Python
Executable File
109 lines
4.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import csv
|
|
import subprocess
|
|
|
|
import linux
|
|
from database import Database
|
|
|
|
CHANNEL_NAME = 'teampgp'
|
|
DB = None
|
|
|
|
def write_csv(data, file_name):
|
|
# Open file with newline='' to prevent extra blank rows across platforms
|
|
with open(file_name, "w", newline="", encoding="utf-8") as file:
|
|
writer = csv.writer(file)
|
|
|
|
# Write all rows at once
|
|
writer.writerows(data)
|
|
|
|
def download():
|
|
"""Find all undownloaded videos and clips, and download them safely."""
|
|
undownloaded = DB.get_undownloaded()
|
|
|
|
print("Starting downloads...")
|
|
for row in undownloaded:
|
|
# Unpack variables clearly
|
|
(
|
|
id,
|
|
title,
|
|
created_at,
|
|
view_count,
|
|
duration,
|
|
url,
|
|
thumbnail_url,
|
|
game_id,
|
|
game_name,
|
|
stream_id,
|
|
creator_name,
|
|
clip_is,
|
|
downloaded,
|
|
uploaded_yt,
|
|
uploaded_yt_chats,
|
|
uploaded_yt_shorts,
|
|
) = row
|
|
|
|
data = [list(row)]
|
|
print(
|
|
f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}"
|
|
)
|
|
|
|
# 1. Define paths and isolate base directory
|
|
if clip_is:
|
|
target_dir = f"download/clips/{id}"
|
|
cmd = f"TwitchDownloaderCLI clipdownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --temp-path download/temp"
|
|
else:
|
|
target_dir = f"download/videos/{id}"
|
|
# FIXED: Removed the duplicated command string combined with '&&'
|
|
cmd = f"TwitchDownloaderCLI videodownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
|
|
|
|
csv_file = f"{target_dir}/{id}.csv"
|
|
|
|
# 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
|
|
output = linux.run_command_streaming(cmd)
|
|
output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp")
|
|
|
|
if output["success"]:
|
|
# Only write CSV and update database if download actually completed
|
|
write_csv(data, csv_file)
|
|
print(f"ID: {id} | Download was successful.")
|
|
DB.mark_as_downloaded(id)
|
|
else:
|
|
print(f"ID: {id} | Download Process failed.")
|
|
print(f"Reason/Error: {output.get('error', 'Unknown Error')}")
|
|
if output.get("stderr"):
|
|
print(f"Details: {output['stderr']}")
|
|
|
|
# Clear temporary chunk clutter immediately if a VOD crashes out
|
|
if not clip_is:
|
|
print("Flushing temporary crash chunks...")
|
|
subprocess.run("rm -rf download/temp/*", shell=True)
|
|
|
|
print("Finished Downloading Pipeline.")
|
|
|
|
def download_old():
|
|
"""Find all undownload videos and download them."""
|
|
undownloaded = DB.get_undownloaded()
|
|
|
|
print(f"Download...")
|
|
for id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts in undownloaded:
|
|
data = [[id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts]]
|
|
print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}")
|
|
if not clip_is:
|
|
output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {id} -o download/videos/{id}/{id}.mp4 --collision Overwrite --temp-path download/temp")
|
|
output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o download/videos/{id}/{id}_chat.json -E --collision Overwrite --temp-path download/temp")
|
|
write_csv(data, f"download/videos/{id}/{id}.csv")
|
|
elif clip_is:
|
|
output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {id} -o download/clips/{id}/{id}.mp4 --collision Overwrite")
|
|
write_csv(data, f"download/clips/{id}/{id}.csv")
|
|
|
|
if output['success'] is True:
|
|
print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title} | was successful")
|
|
DB.mark_as_downloaded(id)
|
|
else:
|
|
print(f"ID: {id} | Download Process failed. {output['stdout']}. Error: {output['stderr']}")
|
|
|
|
print(f"Finished Downloading...")
|
|
|
|
if __name__ == "__main__":
|
|
DB = Database()
|
|
download() |