exparmental run_command in linux.py migrate_sql1.py

twitch_download_videos.py

@@ -1,8 +1,6 @@
 #!/usr/bin/env python3
-import requests
-import os
-import time
 import csv
+import subprocess
 
 import linux
 from database import Database
@@ -19,6 +17,70 @@ def write_csv(data, file_name):
         writer.writerows(data)
 
 def download():
+    """Find all undownloaded videos and clips, and download them safely."""
+    undownloaded = DB.get_undownloaded()
+
+    print("Starting downloads...")
+    for row in undownloaded:
+        # Unpack variables clearly
+        (
+            id,
+            title,
+            created_at,
+            view_count,
+            duration,
+            url,
+            thumbnail_url,
+            game_id,
+            game_name,
+            stream_id,
+            creator_name,
+            clip_is,
+            downloaded,
+            uploaded_yt,
+            uploaded_yt_chats,
+            uploaded_yt_shorts,
+        ) = row
+
+        data = [list(row)]
+        print(
+            f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}"
+        )
+
+        # 1. Define paths and isolate base directory
+        if clip_is:
+            target_dir = f"download/clips/{id}"
+            cmd = f"TwitchDownloaderCLI clipdownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --temp-path download/temp"
+        else:
+            target_dir = f"download/videos/{id}"
+            # FIXED: Removed the duplicated command string combined with '&&'
+            cmd = f"TwitchDownloaderCLI videodownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
+
+        csv_file = f"{target_dir}/{id}.csv"
+
+        # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
+        output = linux.run_command_streaming(cmd)
+        output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp")
+
+        if output["success"]:
+            # Only write CSV and update database if download actually completed
+            write_csv(data, csv_file)
+            print(f"ID: {id} | Download was successful.")
+            DB.mark_as_downloaded(id)
+        else:
+            print(f"ID: {id} | Download Process failed.")
+            print(f"Reason/Error: {output.get('error', 'Unknown Error')}")
+            if output.get("stderr"):
+                print(f"Details: {output['stderr']}")
+
+            # Clear temporary chunk clutter immediately if a VOD crashes out
+            if not clip_is:
+                print("Flushing temporary crash chunks...")
+                subprocess.run("rm -rf download/temp/*", shell=True)
+
+    print("Finished Downloading Pipeline.")
+
+def download_old():
     """Find all undownload videos and download them."""
     undownloaded = DB.get_undownloaded()
 
@@ -27,15 +89,13 @@ def download():
         data = [[id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts]]
         print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}")
         if not clip_is:
-            output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {id} -o download/videos/{id}/{id}.mp4 --collision Overwrite")
-            output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o download/videos/{id}/{id}_chat.json -E --collision Overwrite")
+            output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {id} -o download/videos/{id}/{id}.mp4 --collision Overwrite --temp-path download/temp")
+            output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o download/videos/{id}/{id}_chat.json -E --collision Overwrite --temp-path download/temp")
             write_csv(data, f"download/videos/{id}/{id}.csv")
         elif clip_is:
             output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {id} -o download/clips/{id}/{id}.mp4 --collision Overwrite")
             write_csv(data, f"download/clips/{id}/{id}.csv")
 
-        time.sleep(1)
-
         if output['success'] is True:
             print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title} | was successful")
             DB.mark_as_downloaded(id)