update database to use sqlalchemy

This commit is contained in:
SumGuyV5 committed 2026-09-17 03:57:44 +00:00
1 parent 2773200a94
commit 61fe9eca25
9 files changed
+267 -170

No files matched your search

+15 -35
View File
@@ -4,6 +4,7 @@ from datetime import datetime, timedelta, timezone
import package.utils as utils
from package.config import DB as DB
from package.database import TwitchVideo
def download_videos():
"""Find all undownloaded videos and clips, and download them safely."""
@@ -11,29 +12,9 @@ def download_videos():
print("Starting downloads...")
for row in undownloaded:
# Unpack variables clearly
(
id,
title,
created_at,
view_count,
duration,
url,
thumbnail_url,
game_id,
game_name,
stream_id,
creator_name,
clip_is,
downloaded,
uploaded_yt,
uploaded_yt_chats,
uploaded_yt_shorts,
) = row
# 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM)
#datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
data_timestamp = datetime.fromisoformat(created_at)
data_timestamp = datetime.fromisoformat(row.created_at)
# 2. Get the exact current date and time
current_time = datetime.now(timezone.utc)
@@ -49,39 +30,38 @@ def download_videos():
print("The data is less than 24 hours old.")
continue
data = [list(row)]
print(
f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}"
)
data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
print(f"ID: {row.id} | Date: {row.created_at} | Game: {row.game_name} | Title: {row.title}")
# 1. Define paths and isolate base directory
if clip_is:
target_dir = f"download/clips/{id}"
cmd = f"TwitchDownloaderCLI clipdownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --temp-path download/temp"
if row.clip_is:
target_dir = f"download/clips/{row.id}"
cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --temp-path download/temp"
else:
target_dir = f"download/videos/{id}"
target_dir = f"download/videos/{row.id}"
# FIXED: Removed the duplicated command string combined with '&&'
cmd = f"TwitchDownloaderCLI videodownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
csv_file = f"{target_dir}/{id}.csv"
csv_file = f"{target_dir}/{row.id}.csv"
# 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
output = utils.run_command(cmd, look_for=["[STATUS]"])
output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {row.id} -o {target_dir}/{row.id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
if output:
# Only write CSV and update database if download actually completed
data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
utils.write_csv(data, csv_file)
print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
DB.mark_as_downloaded(id)
DB.mark_as_downloaded(row.id)
else:
print(f"ID: {id} | Download Process failed.")
print(f"ID: {row.id} | Download Process failed.")
#print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
#if output.get("stderr"):
# print(f"Details: {output['stderr']}")
# Clear temporary chunk clutter immediately if a VOD crashes out
if not clip_is:
if not row.clip_is:
print("Flushing temporary crash chunks...")
subprocess.run("rm -rf download/temp/*", shell=True)
+38 -41
View File
@@ -7,6 +7,7 @@ from twitchAPI.type import VideoType
import package.twitch.config as config
from package.config import DB as DB
from package.database.database import TwitchVideo
GAME_CACHE = {} # Local cache dictionary to store game_id -> game_name mapping
@@ -101,20 +102,20 @@ async def get_streamer_vods() -> list:
except ValueError:
clean_game_id = 0
vod_data = {
"id": v.id,
"title": v.title,
"created_at": str(v.published_at),
"view_count": int(v.view_count) if v.view_count else 0,
"duration": v.duration,
"url": v.url,
"thumbnail_url": v.thumbnail_url,
"game_id": clean_game_id,
"game_name": game_name,
"stream_id": str(v.stream_id) if v.stream_id else "0",
"creator_name": config.CHANNEL_NAME,
"clip_is": False,
}
vod_data = TwitchVideo()
vod_data.id = v.id
vod_data.title = v.title
vod_data.created_at = str(v.published_at)
vod_data.view_count = int(v.view_count) if v.view_count else 0
vod_data.duration = v.duration
vod_data.url = v.url
vod_data.thumbnail_url = v.thumbnail_url
vod_data.game_id = clean_game_id
vod_data.game_name = game_name
vod_data.stream_id = str(v.stream_id) if v.stream_id else "0"
vod_data.creator_name = config.CHANNEL_NAME
vod_data.clip_is = False
all_vods.append(vod_data)
print(f"Collected VOD: {v.title} | Category: {game_name} ({v.duration})")
@@ -131,20 +132,20 @@ async def get_streamer_clips() -> list:
async for c in clip_generator:
game_name = await get_game_name_by_id(c.game_id)
clip_data = {
"id": c.id,
"title": c.title,
"created_at": str(c.created_at),
"view_count": int(c.view_count) if c.view_count else 0,
"duration": c.duration,
"url": c.url,
"thumbnail_url": c.thumbnail_url,
"game_id": c.game_id,
"game_name": game_name,
"stream_id": "0",
"creator_name": c.creator_name,
"clip_is": True,
}
clip_data = TwitchVideo()
clip_data.id = c.id
clip_data.title = c.title
clip_data.created_at = str(c.created_at)
clip_data.view_count = int(c.view_count) if c.view_count else 0
clip_data.duration = c.duration
clip_data.url = c.url
clip_data.thumbnail_url = c.thumbnail_url
clip_data.game_id = c.game_id
clip_data.game_name = game_name
clip_data.stream_id = "0"
clip_data.creator_name = c.creator_name
clip_data.clip_is = True
all_clips.append(clip_data)
print(f"Collected clip: {c.title} | Category: {game_name} ({c.view_count} views)")
@@ -157,26 +158,22 @@ async def video_info():
vods_list = await get_streamer_vods()
for v in vods_list:
DB.insert_video_record(
v['id'], v['title'], v['created_at'], v['view_count'], v['duration'],
v['url'], v['thumbnail_url'], v['game_id'], v['game_name'],
v['stream_id'], v['creator_name'], v['clip_is']
)
DB.insert(v)
if vods_list:
print(f"Recent VOD: '{vods_list[0]['title']}'")
print("====================================================")
print(f"Recent VOD: '{vods_list[0].title}'")
print("====================================================")
clips_list = await get_streamer_clips()
for v in clips_list:
DB.insert_video_record(
v['id'], v['title'], v['created_at'], v['view_count'], v['duration'],
v['url'], v['thumbnail_url'], v['game_id'], v['game_name'],
v['stream_id'], v['creator_name'], v['clip_is']
)
DB.insert(v)
if clips_list:
print(f"Recent VOD: '{clips_list[0]['title']}'")
print("====================================================")
print(f"Recent VOD: '{clips_list[0].title}'")
print("====================================================")
if __name__ == "__main__":
asyncio.run(video_info())