update and make test for transcribe and chat_vod and shorts

linux.py

@@ -36,46 +36,38 @@ def run_command_ffmpeg(cmd_str: str, progress_prefix: str = "Progress") -> bool:
     print("\n") # New line after process finishes
     return process.returncode == 0
 
-def run_command(command: list[str]) -> dict:
+def run_command(cmd_str: str, progress_prefix: str = "Progress", look_for: list = ["frame=", "time=", "fps=", "Rendering frame"]) -> bool:
-    """Executes a command, streams stdout only when the line changes, and detects OOM."""
+   pass
-    try:
+   """Runs a system command and streams its output live to the console."""
-        process = subprocess.Popen(
+   # shlex safely handles quotes and paths inside the command string
-            command,
+   args = shlex.split(cmd_str)
-            shell=False,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True,
-            bufsize=1,
-        )
 
-        last_line = None  # Track the previous line
+   # Redirect stderr to stdout because FFmpeg outputs status updates to stderr
+   process = subprocess.Popen(
+        args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        bufsize=1
+    )
+   print(f"Executing: {cmd_str[:90]}...")
 
-        if process.stdout:
+   # Stream the output live to the terminal
-            for line in process.stdout:
+   while True:
-                if line != last_line:  # Only print if the content changed
+        line = process.stdout.readline()
-                    print(line, end="")
+        if not line and process.poll() is not None:
-                    sys.stdout.flush()
+            break
-                    last_line = line  # Update the tracking variable
+        if line:
+            clean_line = line.strip()
+            # Only print updates that show progress metrics to keep terminal clean
+            if any(metric in clean_line for metric in look_for):
+                sys.stdout.write(f"\r[{progress_prefix}] {clean_line}")
+                sys.stdout.flush()
+            elif "Error" in clean_line or "failed" in clean_line:
+                print(f"\n[Alert] {clean_line}")
             
-        _, stderr = process.communicate()
+   print("\n") # New line after process finishes
-
+   return process.returncode == 0
-        if process.returncode != 0:
-            if process.returncode in [137, -9]:
-                return {
-                    "success": False,
-                    "error": "Process was KILLED by the Linux kernel (Out of Memory).",
-                    "stderr": stderr,
-                }
-            return {
-                "success": False,
-                "error": f"Exit code {process.returncode}",
-                "stderr": stderr,
-            }
-
-        return {"success": True, "error": "", "stderr": stderr}
-
-    except Exception as e:
-        return {"success": False, "error": str(e), "stderr": ""}
 
 def run_command_old(command: str):
     """Executes a Linux command, waits for completion, and returns output."""

transcribe_video.py

@@ -5,6 +5,7 @@ import whisper
 import linux
 from moviepy import VideoFileClip
 from whisper.utils import get_writer
+from pathlib import Path
 
 model = whisper.load_model("base")
 
@@ -29,6 +30,8 @@ def extract_audio(video_path: str):
                 codec="pcm_s16le", 
                 ffmpeg_params=["-ac", "1"]
             )
+    except Exception as e:
+        print(f"❌ Error: {e}")
     finally:
         # Always clean up the temporary sanitized video on Windows 11
         if os.path.exists(sanitized_video_path):
@@ -45,7 +48,7 @@ def transcribe_to_srt(video_path: str):
     result = model.transcribe(audio_path)
     
     print("Creating SRT file...")
-    srt_writer = get_writer("srt", transcribe_path)
+    srt_writer = get_writer("srt", Path(transcribe_path).parent)
     srt_writer(result, transcribe_path, {})
     
     print(f"SRT subtitle file saved in: {transcribe_path}")
@@ -54,14 +57,4 @@ def transcribe_to_srt(video_path: str):
 
 
 if __name__ == "__main__":
-    video_path = "my_video.mp4" 
+    transcribe_to_srt("download/clips/AbnegateAgitatedGrassPJSalt/AbnegateAgitatedGrassPJSalt.mp4")
-    audio_temp_path = "temp_audio.wav"  # Changed extension to .wav
-    
-    output_dir = os.getcwd()  
-    output_prefix = "my_video_subtitles"  
-    
-    extract_audio(video_path, audio_temp_path)
-    transcribe_to_srt(audio_temp_path, output_dir, output_prefix)
-    
-    if os.path.exists(audio_temp_path):
-        os.remove(audio_temp_path)

twitch_chat_vod.py

@@ -11,7 +11,7 @@ def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side") ->
     chat_path = video_path.replace(".mp4", "_chat.json")
     temp_chat = video_path.replace(".mp4", "_temp_chat.mp4")
     video_chat = video_path.replace(".mp4", "_with_chat.mp4")
-    mask_path = video_chat.replace(".mp4", "_mask.mp4")
+    mask_path = video_path.replace(".mp4", "_temp_chat_mask.mp4")
 
     # Step 1: Pre-flight checks
     if not os.path.exists(video_path):
@@ -33,16 +33,21 @@ def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side") ->
         f"-w 400 -h 1080 "
         f"--collision Overwrite "
         f"--temp-path download/temp "
-        f"--font-size 25 "
+        f"--font-size 20 "
         f"--generate-mask "
-        f"-o {temp_chat}"
+        f"--background-color #00000000 "
-        f'--output-args="-threads 8 "'
+        f"-o {temp_chat} "
     )
 
-    chat_success = linux.run_command(chat_cmd, progress_prefix="Chat Render")
+    if not os.path.exists(temp_chat):
-    if not chat_success:
+        chat_success = linux.run_command(chat_cmd, look_for=["[STATUS]"])
-        print("❌ Error: TwitchDownloaderCLI failed to render chat video.")
+        if chat_success:
-        return False
+            print("✅ Success: TwitchDownloaderCLI render chat video.")
+        else:
+            print(f"❌ Error: TwitchDownloaderCLI failed to render chat video. {chat_success['error']} : {chat_success['stderr']}")
+            os.remove(temp_chat)
+            os.remove(mask_path)
+            return False
 
     # Step 3: Combine Video and Chat using FFmpeg
     print("====================================================")
@@ -67,7 +72,7 @@ def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side") ->
     else:
         raise ValueError("Invalid layout choice. Choose 'side-by-side' or 'overlay'.")
 
-    ffmpeg_success = linux.run_command_ffmpeg(ffmpeg_cmd, progress_prefix="FFmpeg Merge")
+    ffmpeg_success = linux.run_command(ffmpeg_cmd, progress_prefix="FFmpeg Merge")
 
     # Step 4: Final verification and cleanup
     if ffmpeg_success and os.path.exists(video_chat):
@@ -79,8 +84,13 @@ def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side") ->
         # Clean up the massive temporary chat video to save storage space
         if os.path.exists(temp_chat):
             print("🧹 Cleaning up temporary chat render video...")
-            #os.remove(temp_chat)
+            os.remove(temp_chat)
+            os.remove(mask_path)
         return True
     else:
         print("❌ Error: FFmpeg failed to merge the video streams.")
+        os.remove(video_chat)
         return False
+
+if __name__ == "__main__":
+    combine_twitch_vod_and_chat("download/videos/2813112936/2813112936.mp4", "overlay")

twitch_download_videos.py

@@ -59,19 +59,19 @@ def download():
         csv_file = f"{target_dir}/{id}.csv"
 
         # 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
-        output = linux.run_command_streaming(cmd)
+        output = linux.run_command(cmd, look_for=["[STATUS]"])
-        output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp")
+        output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
 
-        if output["success"]:
+        if output:
             # Only write CSV and update database if download actually completed
             write_csv(data, csv_file)
-            print(f"ID: {id} | Download was successful.")
+            print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
             DB.mark_as_downloaded(id)
         else:
             print(f"ID: {id} | Download Process failed.")
-            print(f"Reason/Error: {output.get('error', 'Unknown Error')}")
+            #print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
-            if output.get("stderr"):
+            #if output.get("stderr"):
-                print(f"Details: {output['stderr']}")
+            #    print(f"Details: {output['stderr']}")
 
             # Clear temporary chunk clutter immediately if a VOD crashes out
             if not clip_is:
@@ -80,30 +80,6 @@ def download():
 
     print("Finished Downloading Pipeline.")
 
-def download_old():
-    """Find all undownload videos and download them."""
-    undownloaded = DB.get_undownloaded()
-
-    print(f"Download...")
-    for id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts in undownloaded:
-        data = [[id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts]]
-        print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}")
-        if not clip_is:
-            output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {id} -o download/videos/{id}/{id}.mp4 --collision Overwrite --temp-path download/temp")
-            output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o download/videos/{id}/{id}_chat.json -E --collision Overwrite --temp-path download/temp")
-            write_csv(data, f"download/videos/{id}/{id}.csv")
-        elif clip_is:
-            output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {id} -o download/clips/{id}/{id}.mp4 --collision Overwrite")
-            write_csv(data, f"download/clips/{id}/{id}.csv")
-
-        if output['success'] is True:
-            print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title} | was successful")
-            DB.mark_as_downloaded(id)
-        else:
-            print(f"ID: {id} | Download Process failed. {output['stdout']}. Error: {output['stderr']}")
-
-    print(f"Finished Downloading...")
-
 if __name__ == "__main__":
     DB = Database()
     download()

youtube_short.py

@@ -123,30 +123,8 @@ def fit_to_9_16_letterbox(input_path: str, top_text: str = "TOP TEXT", bottom_te
         
     print(f"🎉 Text overlay video saved to: {output_path}")
 
-def main():
-    # Set up the command-line argument parser
-    parser = argparse.ArgumentParser(description="TeamPGP Clip Processing and Upload Pipeline")
-    
-    # Add optional arguments
-    parser.add_argument('--convert', type=str, metavar='CLIP_PATH', help='Path to a video file to convert to a 9:16 Short')
-    parser.add_argument('--upload', action='store_true', help='Process and upload pending Shorts in the database to YouTube')
-
-    args = parser.parse_args()
-
-    # If no flags are provided, show help text and exit
-    if not args.convert and not args.upload:
-        parser.print_help()
-        sys.exit("\n❌ Error: You must provide at least one action flag (--convert or --upload).")
-
-    # Execute conversion step if path is provided
-    if args.convert:
-        clip_by = "joelmckinney"
-        #convert_to_short(args.convert)
-        fit_to_9_16_letterbox(args.convert, "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp", f"Clipped By: {clip_by}.", True)
-
-    # Execute database upload step if flag is provided
-    if args.upload:
-        upload_shorts()
-
 if __name__ == "__main__":
-    main()
+    creator_name = "greenskiesbluegrass"
+    top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
+    bottom_txt = f"Clipped By: {creator_name}."
+    fit_to_9_16_letterbox("download/clips/AbnegateAgitatedGrassPJSalt/AbnegateAgitatedGrassPJSalt.mp4", top_txt, bottom_txt)