update database to use sqlalchemy
This commit is contained in:
1 parent
2773200a94
commit
61fe9eca25
9 files changed
+267
-170
No files matched your search
@@ -4,6 +4,7 @@ from datetime import datetime, timedelta, timezone
|
||||
|
||||
import package.utils as utils
|
||||
from package.config import DB as DB
|
||||
from package.database import TwitchVideo
|
||||
|
||||
def download_videos():
|
||||
"""Find all undownloaded videos and clips, and download them safely."""
|
||||
@@ -11,29 +12,9 @@ def download_videos():
|
||||
|
||||
print("Starting downloads...")
|
||||
for row in undownloaded:
|
||||
# Unpack variables clearly
|
||||
(
|
||||
id,
|
||||
title,
|
||||
created_at,
|
||||
view_count,
|
||||
duration,
|
||||
url,
|
||||
thumbnail_url,
|
||||
game_id,
|
||||
game_name,
|
||||
stream_id,
|
||||
creator_name,
|
||||
clip_is,
|
||||
downloaded,
|
||||
uploaded_yt,
|
||||
uploaded_yt_chats,
|
||||
uploaded_yt_shorts,
|
||||
) = row
|
||||
|
||||
# 1. Define your data's timestamp (Example: April 10, 2026, at 10:00 AM)
|
||||
#datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ")
|
||||
data_timestamp = datetime.fromisoformat(created_at)
|
||||
data_timestamp = datetime.fromisoformat(row.created_at)
|
||||
|
||||
# 2. Get the exact current date and time
|
||||
current_time = datetime.now(timezone.utc)
|
||||
@@ -49,39 +30,38 @@ def download_videos():
|
||||
print("The data is less than 24 hours old.")
|
||||
continue
|
||||
|
||||
data = [list(row)]
|
||||
print(
|
||||
f"ID: {id} | Date: {created_at} | Game: {game_name} | Title: {title}"
|
||||
)
|
||||
data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
|
||||
print(f"ID: {row.id} | Date: {row.created_at} | Game: {row.game_name} | Title: {row.title}")
|
||||
|
||||
# 1. Define paths and isolate base directory
|
||||
if clip_is:
|
||||
target_dir = f"download/clips/{id}"
|
||||
cmd = f"TwitchDownloaderCLI clipdownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --temp-path download/temp"
|
||||
if row.clip_is:
|
||||
target_dir = f"download/clips/{row.id}"
|
||||
cmd = f"TwitchDownloaderCLI clipdownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --temp-path download/temp"
|
||||
else:
|
||||
target_dir = f"download/videos/{id}"
|
||||
target_dir = f"download/videos/{row.id}"
|
||||
# FIXED: Removed the duplicated command string combined with '&&'
|
||||
cmd = f"TwitchDownloaderCLI videodownload --id {id} -o {target_dir}/{id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
|
||||
cmd = f"TwitchDownloaderCLI videodownload --id {row.id} -o {target_dir}/{row.id}.mp4 --collision Overwrite --threads 2 --temp-path download/temp"
|
||||
|
||||
csv_file = f"{target_dir}/{id}.csv"
|
||||
csv_file = f"{target_dir}/{row.id}.csv"
|
||||
|
||||
# 3. FIXED: Use the live streaming function to prevent Out-Of-Memory crashes
|
||||
output = utils.run_command(cmd, look_for=["[STATUS]"])
|
||||
output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {id} -o {target_dir}/{id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
|
||||
output2 = utils.run_command(f"TwitchDownloaderCLI chatdownload --id {row.id} -o {target_dir}/{row.id}_chat.json -E --collision Overwrite --temp-path download/temp", look_for=["[STATUS]"])
|
||||
|
||||
if output:
|
||||
# Only write CSV and update database if download actually completed
|
||||
data = [attr for attr in vars(TwitchVideo) if not attr.startswith('__')]
|
||||
utils.write_csv(data, csv_file)
|
||||
print(f"✅ Success: TwitchDownloaderCLI video. {target_dir}")
|
||||
DB.mark_as_downloaded(id)
|
||||
DB.mark_as_downloaded(row.id)
|
||||
else:
|
||||
print(f"ID: {id} | Download Process failed.")
|
||||
print(f"ID: {row.id} | Download Process failed.")
|
||||
#print(f"❌ Reason/Error: {output.get('error', 'Unknown Error')}")
|
||||
#if output.get("stderr"):
|
||||
# print(f"Details: {output['stderr']}")
|
||||
|
||||
# Clear temporary chunk clutter immediately if a VOD crashes out
|
||||
if not clip_is:
|
||||
if not row.clip_is:
|
||||
print("Flushing temporary crash chunks...")
|
||||
subprocess.run("rm -rf download/temp/*", shell=True)
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from twitchAPI.type import VideoType
|
||||
|
||||
import package.twitch.config as config
|
||||
from package.config import DB as DB
|
||||
from package.database.database import TwitchVideo
|
||||
|
||||
GAME_CACHE = {} # Local cache dictionary to store game_id -> game_name mapping
|
||||
|
||||
@@ -101,20 +102,20 @@ async def get_streamer_vods() -> list:
|
||||
except ValueError:
|
||||
clean_game_id = 0
|
||||
|
||||
vod_data = {
|
||||
"id": v.id,
|
||||
"title": v.title,
|
||||
"created_at": str(v.published_at),
|
||||
"view_count": int(v.view_count) if v.view_count else 0,
|
||||
"duration": v.duration,
|
||||
"url": v.url,
|
||||
"thumbnail_url": v.thumbnail_url,
|
||||
"game_id": clean_game_id,
|
||||
"game_name": game_name,
|
||||
"stream_id": str(v.stream_id) if v.stream_id else "0",
|
||||
"creator_name": config.CHANNEL_NAME,
|
||||
"clip_is": False,
|
||||
}
|
||||
vod_data = TwitchVideo()
|
||||
vod_data.id = v.id
|
||||
vod_data.title = v.title
|
||||
vod_data.created_at = str(v.published_at)
|
||||
vod_data.view_count = int(v.view_count) if v.view_count else 0
|
||||
vod_data.duration = v.duration
|
||||
vod_data.url = v.url
|
||||
vod_data.thumbnail_url = v.thumbnail_url
|
||||
vod_data.game_id = clean_game_id
|
||||
vod_data.game_name = game_name
|
||||
vod_data.stream_id = str(v.stream_id) if v.stream_id else "0"
|
||||
vod_data.creator_name = config.CHANNEL_NAME
|
||||
vod_data.clip_is = False
|
||||
|
||||
all_vods.append(vod_data)
|
||||
print(f"Collected VOD: {v.title} | Category: {game_name} ({v.duration})")
|
||||
|
||||
@@ -131,20 +132,20 @@ async def get_streamer_clips() -> list:
|
||||
async for c in clip_generator:
|
||||
game_name = await get_game_name_by_id(c.game_id)
|
||||
|
||||
clip_data = {
|
||||
"id": c.id,
|
||||
"title": c.title,
|
||||
"created_at": str(c.created_at),
|
||||
"view_count": int(c.view_count) if c.view_count else 0,
|
||||
"duration": c.duration,
|
||||
"url": c.url,
|
||||
"thumbnail_url": c.thumbnail_url,
|
||||
"game_id": c.game_id,
|
||||
"game_name": game_name,
|
||||
"stream_id": "0",
|
||||
"creator_name": c.creator_name,
|
||||
"clip_is": True,
|
||||
}
|
||||
clip_data = TwitchVideo()
|
||||
clip_data.id = c.id
|
||||
clip_data.title = c.title
|
||||
clip_data.created_at = str(c.created_at)
|
||||
clip_data.view_count = int(c.view_count) if c.view_count else 0
|
||||
clip_data.duration = c.duration
|
||||
clip_data.url = c.url
|
||||
clip_data.thumbnail_url = c.thumbnail_url
|
||||
clip_data.game_id = c.game_id
|
||||
clip_data.game_name = game_name
|
||||
clip_data.stream_id = "0"
|
||||
clip_data.creator_name = c.creator_name
|
||||
clip_data.clip_is = True
|
||||
|
||||
all_clips.append(clip_data)
|
||||
print(f"Collected clip: {c.title} | Category: {game_name} ({c.view_count} views)")
|
||||
|
||||
@@ -157,26 +158,22 @@ async def video_info():
|
||||
vods_list = await get_streamer_vods()
|
||||
|
||||
for v in vods_list:
|
||||
DB.insert_video_record(
|
||||
v['id'], v['title'], v['created_at'], v['view_count'], v['duration'],
|
||||
v['url'], v['thumbnail_url'], v['game_id'], v['game_name'],
|
||||
v['stream_id'], v['creator_name'], v['clip_is']
|
||||
)
|
||||
DB.insert(v)
|
||||
|
||||
if vods_list:
|
||||
print(f"Recent VOD: '{vods_list[0]['title']}'")
|
||||
print("====================================================")
|
||||
print(f"Recent VOD: '{vods_list[0].title}'")
|
||||
print("====================================================")
|
||||
|
||||
clips_list = await get_streamer_clips()
|
||||
|
||||
for v in clips_list:
|
||||
DB.insert_video_record(
|
||||
v['id'], v['title'], v['created_at'], v['view_count'], v['duration'],
|
||||
v['url'], v['thumbnail_url'], v['game_id'], v['game_name'],
|
||||
v['stream_id'], v['creator_name'], v['clip_is']
|
||||
)
|
||||
|
||||
DB.insert(v)
|
||||
|
||||
if clips_list:
|
||||
print(f"Recent VOD: '{clips_list[0]['title']}'")
|
||||
print("====================================================")
|
||||
print(f"Recent VOD: '{clips_list[0].title}'")
|
||||
print("====================================================")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(video_info())
|
||||
Reference in new issue
Block a user