86 lines
3.7 KiB
Python
86 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
||
import subprocess
|
||
from pathlib import Path
|
||
from datetime import datetime, timedelta, timezone
|
||
|
||
import package.utils as utils
|
||
from package.config import DB as DB
|
||
from package.config import TZ as TZ
|
||
from package.database import TwitchVideo, Chapter
|
||
|
||
def download_videos():
|
||
"""Find all undownloaded videos and clips, and download them safely."""
|
||
undownloaded = DB.get_undownloaded()
|
||
|
||
print("Starting downloads...")
|
||
for row in undownloaded:
|
||
try:
|
||
# 2026-08-16T23:52:10Z -> replace Z with +00:00
|
||
data_timestamp = datetime.fromisoformat(row.created_at.replace("Z", "+00:00"))
|
||
except Exception:
|
||
data_timestamp = datetime.strptime(row.created_at, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
|
||
|
||
if datetime.now(timezone.utc) - data_timestamp < timedelta(hours=24):
|
||
print(f"Skipping {row.id} - less than 24h old")
|
||
continue
|
||
|
||
print(f"ID: {row.id} | Date: {row.created_at} | Game: {row.game_name} | Title: {row.title}")
|
||
|
||
base_dir = Path(f"download/clips/{row.id}" if row.clip_is else f"download/videos/{row.id}")
|
||
base_dir.mkdir(parents=True, exist_ok=True)
|
||
|
||
temp_path = Path("download/temp")
|
||
temp_path.mkdir(parents=True, exist_ok=True)
|
||
|
||
target_video = base_dir / f"{row.id}.mp4"
|
||
json_file = base_dir / f"{row.id}_chat.json"
|
||
csv_file = base_dir / f"{row.id}.csv"
|
||
|
||
if row.clip_is:
|
||
cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_video} --collision Overwrite --temp-path {temp_path}"
|
||
else:
|
||
cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_video} --collision Overwrite --threads 2 --temp-path {temp_path}"
|
||
|
||
output = utils.run_command(cmd, look_for=["[STATUS]"])
|
||
output2 = utils.run_command(
|
||
f"TwitchDownloaderCLI chatdownload --id {row.id} -o {json_file} -E --collision Overwrite --temp-path {temp_path}",
|
||
look_for=["[STATUS]"]
|
||
)
|
||
|
||
if output2 and json_file.exists():
|
||
related_id = utils.get_video_id_from_chat(json_file)
|
||
if related_id:
|
||
# This will set related_video_id = 2848295283 for a clip
|
||
DB.set_related_video(row.id, related_id)
|
||
print(f" -> Linked {row.id} -> related VOD {related_id}")
|
||
if row.clip_is:
|
||
chapters = utils.get_chapters(json_file)
|
||
for chapter in chapters:
|
||
chapter_record = Chapter(
|
||
twitch_video_id=row.id,
|
||
start_ms=chapter.get('startMilliseconds'),
|
||
length_ms=chapter.get('lengthMilliseconds'),
|
||
description=chapter.get('description'),
|
||
type=chapter.get('type'),
|
||
game_display_name=chapter.get('gameDisplayName'),
|
||
game_id=chapter.get('gameId')
|
||
)
|
||
DB.insert(chapter_record)
|
||
print(f" ➕ Chapter added: {chapter_record.description} ({chapter_record.start_ms}ms)")
|
||
|
||
if output:
|
||
# FIX 4: write_csv needs actual row data, not column names
|
||
utils.write_csv([row], str(csv_file)) # pass the object/dict, not vars(TwitchVideo)
|
||
print(f"✅ Success: {base_dir}")
|
||
DB.mark_as_downloaded(row.id)
|
||
else:
|
||
print(f"❌ ID: {row.id} | Download failed.")
|
||
if not row.clip_is:
|
||
print("Flushing temporary crash chunks...")
|
||
subprocess.run("rm -rf download/temp/*", shell=True)
|
||
|
||
|
||
print("Finished Downloading Pipeline.")
|
||
|
||
if __name__ == "__main__":
|
||
download_videos() |