I forgot what I chanagerd
This commit is contained in:
SumGuyV5 committed
1 parent
24081e1576
commit
c4f5018833
20 files changed
+678
-77
No files matched your search
@@ -1,10 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import package.utils as utils
|
||||
from package.config import DB as DB
|
||||
from package.database import TwitchVideo
|
||||
from package.config import TZ as TZ
|
||||
from package.database import TwitchVideo, Chapter
|
||||
|
||||
def download_videos():
|
||||
"""Find all undownloaded videos and clips, and download them safely."""
|
||||
@@ -12,59 +14,72 @@ def download_videos():
|
||||
|
||||
print("Starting downloads...")
|
||||
for row in undownloaded:
|
||||
# 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM)
|
||||
#datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
|
||||
data_timestamp = datetime.fromisoformat(row.created_at)
|
||||
try:
|
||||
# 2026-08-16T23:52:10Z -> replace Z with +00:00
|
||||
data_timestamp = datetime.fromisoformat(row.created_at.replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
data_timestamp = datetime.strptime(row.created_at, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc)
|
||||
|
||||
# 2. Get the exact current date and time
|
||||
current_time = datetime.now(timezone.utc)
|
||||
|
||||
# 3. Calculate the difference between the two times
|
||||
time_difference = current_time - data_timestamp
|
||||
|
||||
# 4. Check if the difference is greater than 24 hours
|
||||
if time_difference > timedelta(hours=24):
|
||||
print("The data is more than 24 hours old.")
|
||||
else:
|
||||
#lets wait 24 hours befor downloading
|
||||
print("The data is less than 24 hours old.")
|
||||
if datetime.now(timezone.utc) - data_timestamp < timedelta(hours=24):
|
||||
print(f"Skipping {row.id} - less than 24h old")
|
||||
continue
|
||||
|
||||
data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
|
||||
print(f"ID: {row.id} | Date: {row.created_at} | Game: {row.game_name} | Title: {row.title}")
|
||||
|
||||
# 1. Define paths and isolate base directory
|
||||
base_dir = Path(f"download/clips/{row.id}" if row.clip_is else f"download/videos/{row.id}")
|
||||
base_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
temp_path = Path("download/temp")
|
||||
temp_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
target_video = base_dir / f"{row.id}.mp4"
|
||||
json_file = base_dir / f"{row.id}_chat.json"
|
||||
csv_file = base_dir / f"{row.id}.csv"
|
||||
|
||||
if row.clip_is:
|
||||
target_dir = f"download/clips/{row.id}"
|
||||
cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --temp-path download/temp"
|
||||
cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_video} --collision Overwrite --temp-path {temp_path}"
|
||||
else:
|
||||
target_dir = f"download/videos/{row.id}"
|
||||
# FIXED: Removed the duplicated command string combined with '&&'
|
||||
cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
|
||||
cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_video} --collision Overwrite --threads 2 --temp-path {temp_path}"
|
||||
|
||||
csv_file = f"{target_dir}/{row.id}.csv"
|
||||
|
||||
# 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
|
||||
output = utils.run_command(cmd, look_for=["[STATUS]"])
|
||||
output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {row.id} -o {target_dir}/{row.id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
|
||||
output2 = utils.run_command(
|
||||
f"TwitchDownloaderCLI chatdownload --id {row.id} -o {json_file} -E --collision Overwrite --temp-path {temp_path}",
|
||||
look_for=["[STATUS]"]
|
||||
)
|
||||
|
||||
if output2 and json_file.exists():
|
||||
related_id = utils.get_video_id_from_chat(json_file)
|
||||
if related_id:
|
||||
# This will set related_video_id = 2848295283 for a clip
|
||||
DB.set_related_video(row.id, related_id)
|
||||
print(f" -> Linked {row.id} -> related VOD {related_id}")
|
||||
if row.clip_is:
|
||||
chapters = utils.get_chapters(json_file)
|
||||
for chapter in chapters:
|
||||
chapter_record = Chapter(
|
||||
twitch_video_id=row.id,
|
||||
start_ms=chapter.get('startMilliseconds'),
|
||||
length_ms=chapter.get('lengthMilliseconds'),
|
||||
description=chapter.get('description'),
|
||||
type=chapter.get('type'),
|
||||
game_display_name=chapter.get('gameDisplayName'),
|
||||
game_id=chapter.get('gameId')
|
||||
)
|
||||
DB.insert(chapter_record)
|
||||
print(f" ➕ Chapter added: {chapter_record.description} ({chapter_record.start_ms}ms)")
|
||||
|
||||
if output:
|
||||
# Only write CSV and update database if download actually completed
|
||||
data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
|
||||
utils.write_csv(data, csv_file)
|
||||
print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
|
||||
# FIX 4: write_csv needs actual row data, not column names
|
||||
utils.write_csv([row], str(csv_file)) # pass the object/dict, not vars(TwitchVideo)
|
||||
print(f"✅ Success: {base_dir}")
|
||||
DB.mark_as_downloaded(row.id)
|
||||
else:
|
||||
print(f"ID: {row.id} | Download Process failed.")
|
||||
#print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
|
||||
#if output.get("stderr"):
|
||||
# print(f"Details: {output['stderr']}")
|
||||
|
||||
# Clear temporary chunk clutter immediately if a VOD crashes out
|
||||
print(f"❌ ID: {row.id} | Download failed.")
|
||||
if not row.clip_is:
|
||||
print("Flushing temporary crash chunks...")
|
||||
subprocess.run("rm -rf download/temp/*", shell=True)
|
||||
|
||||
|
||||
print("Finished Downloading Pipeline.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user