update and make test for transcribe and chat_vod and shorts
This commit is contained in:
@@ -36,46 +36,38 @@ def run_command_ffmpeg(cmd_str: str, progress_prefix: str = "Progress") -> bool:
|
||||
print("\n") # New line after process finishes
|
||||
return process.returncode == 0
|
||||
|
||||
def run_command(command: list[str]) -> dict:
|
||||
"""Executes a command, streams stdout only when the line changes, and detects OOM."""
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
shell=False,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
)
|
||||
def run_command(cmd_str: str, progress_prefix: str = "Progress", look_for: list = ["frame=", "time=", "fps=", "Rendering frame"]) -> bool:
|
||||
pass
|
||||
"""Runs a system command and streams its output live to the console."""
|
||||
# shlex safely handles quotes and paths inside the command string
|
||||
args = shlex.split(cmd_str)
|
||||
|
||||
last_line = None # Track the previous line
|
||||
# Redirect stderr to stdout because FFmpeg outputs status updates to stderr
|
||||
process = subprocess.Popen(
|
||||
args,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
bufsize=1
|
||||
)
|
||||
print(f"Executing: {cmd_str[:90]}...")
|
||||
|
||||
if process.stdout:
|
||||
for line in process.stdout:
|
||||
if line != last_line: # Only print if the content changed
|
||||
print(line, end="")
|
||||
sys.stdout.flush()
|
||||
last_line = line # Update the tracking variable
|
||||
# Stream the output live to the terminal
|
||||
while True:
|
||||
line = process.stdout.readline()
|
||||
if not line and process.poll() is not None:
|
||||
break
|
||||
if line:
|
||||
clean_line = line.strip()
|
||||
# Only print updates that show progress metrics to keep terminal clean
|
||||
if any(metric in clean_line for metric in look_for):
|
||||
sys.stdout.write(f"\r[{progress_prefix}] {clean_line}")
|
||||
sys.stdout.flush()
|
||||
elif "Error" in clean_line or "failed" in clean_line:
|
||||
print(f"\n[Alert] {clean_line}")
|
||||
|
||||
_, stderr = process.communicate()
|
||||
|
||||
if process.returncode != 0:
|
||||
if process.returncode in [137, -9]:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Process was KILLED by the Linux kernel (Out of Memory).",
|
||||
"stderr": stderr,
|
||||
}
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Exit code {process.returncode}",
|
||||
"stderr": stderr,
|
||||
}
|
||||
|
||||
return {"success": True, "error": "", "stderr": stderr}
|
||||
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e), "stderr": ""}
|
||||
print("\n") # New line after process finishes
|
||||
return process.returncode == 0
|
||||
|
||||
def run_command_old(command: str):
|
||||
"""Executes a Linux command, waits for completion, and returns output."""
|
||||
|
||||
+5
-12
@@ -5,6 +5,7 @@ import whisper
|
||||
import linux
|
||||
from moviepy import VideoFileClip
|
||||
from whisper.utils import get_writer
|
||||
from pathlib import Path
|
||||
|
||||
model = whisper.load_model("base")
|
||||
|
||||
@@ -29,6 +30,8 @@ def extract_audio(video_path: str):
|
||||
codec="pcm_s16le",
|
||||
ffmpeg_params=["-ac", "1"]
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
finally:
|
||||
# Always clean up the temporary sanitized video on Windows 11
|
||||
if os.path.exists(sanitized_video_path):
|
||||
@@ -45,7 +48,7 @@ def transcribe_to_srt(video_path: str):
|
||||
result = model.transcribe(audio_path)
|
||||
|
||||
print("Creating SRT file...")
|
||||
srt_writer = get_writer("srt", transcribe_path)
|
||||
srt_writer = get_writer("srt", Path(transcribe_path).parent)
|
||||
srt_writer(result, transcribe_path, {})
|
||||
|
||||
print(f"SRT subtitle file saved in: {transcribe_path}")
|
||||
@@ -54,14 +57,4 @@ def transcribe_to_srt(video_path: str):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
video_path = "my_video.mp4"
|
||||
audio_temp_path = "temp_audio.wav" # Changed extension to .wav
|
||||
|
||||
output_dir = os.getcwd()
|
||||
output_prefix = "my_video_subtitles"
|
||||
|
||||
extract_audio(video_path, audio_temp_path)
|
||||
transcribe_to_srt(audio_temp_path, output_dir, output_prefix)
|
||||
|
||||
if os.path.exists(audio_temp_path):
|
||||
os.remove(audio_temp_path)
|
||||
transcribe_to_srt("download/clips/AbnegateAgitatedGrassPJSalt/AbnegateAgitatedGrassPJSalt.mp4")
|
||||
+20
-10
@@ -11,7 +11,7 @@ def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side") ->
|
||||
chat_path = video_path.replace(".mp4", "_chat.json")
|
||||
temp_chat = video_path.replace(".mp4", "_temp_chat.mp4")
|
||||
video_chat = video_path.replace(".mp4", "_with_chat.mp4")
|
||||
mask_path = video_chat.replace(".mp4", "_mask.mp4")
|
||||
mask_path = video_path.replace(".mp4", "_temp_chat_mask.mp4")
|
||||
|
||||
# Step 1: Pre-flight checks
|
||||
if not os.path.exists(video_path):
|
||||
@@ -33,16 +33,21 @@ def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side") ->
|
||||
f"-w 400 -h 1080 "
|
||||
f"--collision Overwrite "
|
||||
f"--temp-path download/temp "
|
||||
f"--font-size 25 "
|
||||
f"--font-size 20 "
|
||||
f"--generate-mask "
|
||||
f"-o {temp_chat}"
|
||||
f'--output-args="-threads 8 "'
|
||||
f"--background-color #00000000 "
|
||||
f"-o {temp_chat} "
|
||||
)
|
||||
|
||||
chat_success = linux.run_command(chat_cmd, progress_prefix="Chat Render")
|
||||
if not chat_success:
|
||||
print("❌ Error: TwitchDownloaderCLI failed to render chat video.")
|
||||
return False
|
||||
if not os.path.exists(temp_chat):
|
||||
chat_success = linux.run_command(chat_cmd, look_for=["[STATUS]"])
|
||||
if chat_success:
|
||||
print("✅ Success: TwitchDownloaderCLI render chat video.")
|
||||
else:
|
||||
print(f"❌ Error: TwitchDownloaderCLI failed to render chat video. {chat_success['error']} : {chat_success['stderr']}")
|
||||
os.remove(temp_chat)
|
||||
os.remove(mask_path)
|
||||
return False
|
||||
|
||||
# Step 3: Combine Video and Chat using FFmpeg
|
||||
print("====================================================")
|
||||
@@ -67,7 +72,7 @@ def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side") ->
|
||||
else:
|
||||
raise ValueError("Invalid layout choice. Choose 'side-by-side' or 'overlay'.")
|
||||
|
||||
ffmpeg_success = linux.run_command_ffmpeg(ffmpeg_cmd, progress_prefix="FFmpeg Merge")
|
||||
ffmpeg_success = linux.run_command(ffmpeg_cmd, progress_prefix="FFmpeg Merge")
|
||||
|
||||
# Step 4: Final verification and cleanup
|
||||
if ffmpeg_success and os.path.exists(video_chat):
|
||||
@@ -79,8 +84,13 @@ def combine_twitch_vod_and_chat(vod_path: str, layout: str = "side-by-side") ->
|
||||
# Clean up the massive temporary chat video to save storage space
|
||||
if os.path.exists(temp_chat):
|
||||
print("🧹 Cleaning up temporary chat render video...")
|
||||
#os.remove(temp_chat)
|
||||
os.remove(temp_chat)
|
||||
os.remove(mask_path)
|
||||
return True
|
||||
else:
|
||||
print("❌ Error: FFmpeg failed to merge the video streams.")
|
||||
os.remove(video_chat)
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
combine_twitch_vod_and_chat("download/videos/2813112936/2813112936.mp4", "overlay")
|
||||
@@ -59,19 +59,19 @@ def download():
|
||||
csv_file = f"{target_dir}/{id}.csv"
|
||||
|
||||
# 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
|
||||
output = linux.run_command_streaming(cmd)
|
||||
output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp")
|
||||
output = linux.run_command(cmd, look_for=["[STATUS]"])
|
||||
output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
|
||||
|
||||
if output["success"]:
|
||||
if output:
|
||||
# Only write CSV and update database if download actually completed
|
||||
write_csv(data, csv_file)
|
||||
print(f"ID: {id} | Download was successful.")
|
||||
print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
|
||||
DB.mark_as_downloaded(id)
|
||||
else:
|
||||
print(f"ID: {id} | Download Process failed.")
|
||||
print(f"Reason/Error: {output.get('error', 'Unknown Error')}")
|
||||
if output.get("stderr"):
|
||||
print(f"Details: {output['stderr']}")
|
||||
#print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
|
||||
#if output.get("stderr"):
|
||||
# print(f"Details: {output['stderr']}")
|
||||
|
||||
# Clear temporary chunk clutter immediately if a VOD crashes out
|
||||
if not clip_is:
|
||||
@@ -80,30 +80,6 @@ def download():
|
||||
|
||||
print("Finished Downloading Pipeline.")
|
||||
|
||||
def download_old():
|
||||
"""Find all undownload videos and download them."""
|
||||
undownloaded = DB.get_undownloaded()
|
||||
|
||||
print(f"Download...")
|
||||
for id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts in undownloaded:
|
||||
data = [[id, title, created_at, view_count, duration, url, thumbnail_url, game_id, game_name, stream_id, creator_name, clip_is, downloaded, uploaded_yt, uploaded_yt_chats, uploaded_yt_shorts]]
|
||||
print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}")
|
||||
if not clip_is:
|
||||
output = linux.run_command(f"TwitchDownloaderCLI videodownload --id {id} -o download/videos/{id}/{id}.mp4 --collision Overwrite --temp-path download/temp")
|
||||
output2 = linux.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o download/videos/{id}/{id}_chat.json -E --collision Overwrite --temp-path download/temp")
|
||||
write_csv(data, f"download/videos/{id}/{id}.csv")
|
||||
elif clip_is:
|
||||
output = linux.run_command(f"TwitchDownloaderCLI clipdownload --id {id} -o download/clips/{id}/{id}.mp4 --collision Overwrite")
|
||||
write_csv(data, f"download/clips/{id}/{id}.csv")
|
||||
|
||||
if output['success'] is True:
|
||||
print(f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title} | was successful")
|
||||
DB.mark_as_downloaded(id)
|
||||
else:
|
||||
print(f"ID: {id} | Download Process failed. {output['stdout']}. Error: {output['stderr']}")
|
||||
|
||||
print(f"Finished Downloading...")
|
||||
|
||||
if __name__ == "__main__":
|
||||
DB = Database()
|
||||
download()
|
||||
+4
-26
@@ -123,30 +123,8 @@ def fit_to_9_16_letterbox(input_path: str, top_text: str = "TOP TEXT", bottom_te
|
||||
|
||||
print(f"🎉 Text overlay video saved to: {output_path}")
|
||||
|
||||
def main():
|
||||
# Set up the command-line argument parser
|
||||
parser = argparse.ArgumentParser(description="TeamPGP Clip Processing and Upload Pipeline")
|
||||
|
||||
# Add optional arguments
|
||||
parser.add_argument('--convert', type=str, metavar='CLIP_PATH', help='Path to a video file to convert to a 9:16 Short')
|
||||
parser.add_argument('--upload', action='store_true', help='Process and upload pending Shorts in the database to YouTube')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# If no flags are provided, show help text and exit
|
||||
if not args.convert and not args.upload:
|
||||
parser.print_help()
|
||||
sys.exit("\n❌ Error: You must provide at least one action flag (--convert or --upload).")
|
||||
|
||||
# Execute conversion step if path is provided
|
||||
if args.convert:
|
||||
clip_by = "joelmckinney"
|
||||
#convert_to_short(args.convert)
|
||||
fit_to_9_16_letterbox(args.convert, "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp", f"Clipped By: {clip_by}.", True)
|
||||
|
||||
# Execute database upload step if flag is provided
|
||||
if args.upload:
|
||||
upload_shorts()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
creator_name = "greenskiesbluegrass"
|
||||
top_txt = "TeamPGP Live on Twitch...\n Every Friday and Sunday @7:30 ET.\n twitch.tv/teampgp"
|
||||
bottom_txt = f"Clipped By: {creator_name}."
|
||||
fit_to_9_16_letterbox("download/clips/AbnegateAgitatedGrassPJSalt/AbnegateAgitatedGrassPJSalt.mp4", top_txt, bottom_txt)
|
||||
Reference in New Issue
Block a user