update clips and database
This commit is contained in:
+19
-2
@@ -64,13 +64,30 @@ class Database:
|
|||||||
self.CONN.commit()
|
self.CONN.commit()
|
||||||
self.CONN.close()
|
self.CONN.close()
|
||||||
|
|
||||||
|
def mark_as_uploaded(self, record_id: int):
|
||||||
|
"""Flags a specific clip row record to uploaded (1)."""
|
||||||
|
|
||||||
|
id = "id"
|
||||||
|
if self.table == "clips":
|
||||||
|
id = "slug"
|
||||||
|
|
||||||
|
CURSOR.execute(
|
||||||
|
f"UPDATE {self.table} SET uploaded_yt = 1 WHERE {id} = ?",
|
||||||
|
(record_id,)
|
||||||
|
)
|
||||||
|
CONN.commit()
|
||||||
|
|
||||||
def mark_as_downloaded(self, record_id: int):
|
def mark_as_downloaded(self, record_id: int):
|
||||||
"""Updates the downloaded status to True (1) for a specific record ID."""
|
"""Updates the downloaded status to True (1) for a specific record ID."""
|
||||||
|
|
||||||
|
id = "id"
|
||||||
|
if self.table == "clips":
|
||||||
|
id = "slug"
|
||||||
|
|
||||||
# Updates the row matching the specific ID
|
# Updates the row matching the specific ID
|
||||||
# TODO clips uses slug
|
# TODO clips uses slug
|
||||||
self.CURSOR.execute(
|
self.CURSOR.execute(
|
||||||
f"UPDATE {self.table} SET downloaded = 1 WHERE id = ?",
|
f"UPDATE {self.table} SET downloaded = 1 WHERE {id} = ?",
|
||||||
(record_id,)
|
(record_id,)
|
||||||
)
|
)
|
||||||
self.CONN.commit()
|
self.CONN.commit()
|
||||||
@@ -102,7 +119,7 @@ class Database:
|
|||||||
# Saves changes and closes the connection
|
# Saves changes and closes the connection
|
||||||
self.CONN.commit()
|
self.CONN.commit()
|
||||||
|
|
||||||
def insert_clips_record(slug: str, record_date_str: str, title: str, gamename: str, clip_by: str, views: int):
|
def insert_clips_record(self, slug: str, record_date_str: str, title: str, gamename: str, clip_by: str, views: int):
|
||||||
"""Cleans up ISO-8601 strings into unified date structures for the database."""
|
"""Cleans up ISO-8601 strings into unified date structures for the database."""
|
||||||
try:
|
try:
|
||||||
clean_date = datetime.strptime(record_date_str, "%Y-%m-%dT%H:%M:%SZ").date()
|
clean_date = datetime.strptime(record_date_str, "%Y-%m-%dT%H:%M:%SZ").date()
|
||||||
|
|||||||
+19
-57
@@ -4,10 +4,23 @@ import sqlite3
|
|||||||
import subprocess
|
import subprocess
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
|
from database import Database
|
||||||
|
|
||||||
import uploader
|
import uploader
|
||||||
from uploader import CategoryId
|
from uploader import CategoryId
|
||||||
|
|
||||||
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
|
||||||
|
DB = Database("clips")
|
||||||
|
|
||||||
|
def run_linux_command(command: str):
|
||||||
|
"""Executes a Linux command, waits for completion, and returns output."""
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
command, shell=True, check=True, capture_output=True, text=True
|
||||||
|
)
|
||||||
|
return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
|
||||||
|
|
||||||
def download_clips():
|
def download_clips():
|
||||||
"""Loops over undownloaded metadata entries to write files down locally."""
|
"""Loops over undownloaded metadata entries to write files down locally."""
|
||||||
@@ -17,16 +30,13 @@ def download_clips():
|
|||||||
|
|
||||||
for slug, record_date, title, game_name, clip_by, views, downloaded in undownloaded_clips:
|
for slug, record_date, title, game_name, clip_by, views, downloaded in undownloaded_clips:
|
||||||
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
|
print(f"Slug: {slug} | Date: {record_date} | Game: {game_name} | By: {clip_by} | Views: {views} | Title: {title}")
|
||||||
|
|
||||||
# Ensures destination folder structures exist before executing CLI tool
|
|
||||||
run_linux_command(f"mkdir -p save/clips/{slug}")
|
|
||||||
|
|
||||||
# Uses standard clipdownload directive
|
# Uses standard clipdownload directive
|
||||||
output = run_linux_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{slug}.mp4")
|
output = run_linux_command(f"TwitchDownloaderCLI clipdownload --id {slug} -o save/clips/{slug}/{title}.mp4")
|
||||||
|
|
||||||
if output["success"] is True:
|
if output["success"] is True:
|
||||||
print(f"Slug: {slug} | Was successfully downloaded.")
|
print(f"Slug: {slug} | Was successfully downloaded.")
|
||||||
mark_as_downloaded(slug)
|
DB.mark_as_downloaded(slug)
|
||||||
else:
|
else:
|
||||||
print(f"Slug: {slug} | Download Process failed.")
|
print(f"Slug: {slug} | Download Process failed.")
|
||||||
|
|
||||||
@@ -52,56 +62,10 @@ def upload_clips():
|
|||||||
|
|
||||||
if output is True:
|
if output is True:
|
||||||
print(f"Slug: {slug} | Was successfully uploaded.")
|
print(f"Slug: {slug} | Was successfully uploaded.")
|
||||||
mark_as_uploaded(slug)
|
DB.mark_as_uploaded(slug)
|
||||||
else:
|
else:
|
||||||
print(f"Slug: {slug} | Upload Process failed.")
|
print(f"Slug: {slug} | Upload Process failed.")
|
||||||
|
|
||||||
def run_linux_command(command: str):
|
|
||||||
"""Executes a Linux command, waits for completion, and returns output."""
|
|
||||||
try:
|
|
||||||
result = subprocess.run(
|
|
||||||
command, shell=True, check=True, capture_output=True, text=True
|
|
||||||
)
|
|
||||||
return {"success": True, "stdout": result.stdout, "stderr": result.stderr}
|
|
||||||
except subprocess.CalledProcessError as e:
|
|
||||||
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
|
|
||||||
|
|
||||||
def mark_as_uploaded(slug: str):
|
|
||||||
"""Flags a specific clip row record to uploaded (1)."""
|
|
||||||
CURSOR.execute(
|
|
||||||
"UPDATE clips SET uploaded_yt = 1 WHERE slug = ?",
|
|
||||||
(slug,)
|
|
||||||
)
|
|
||||||
CONN.commit()
|
|
||||||
|
|
||||||
def mark_as_downloaded(slug: str):
|
|
||||||
"""Flags a specific clip row record to downloaded (1)."""
|
|
||||||
CURSOR.execute(
|
|
||||||
"UPDATE clips SET downloaded = 1 WHERE slug = ?",
|
|
||||||
(slug,)
|
|
||||||
)
|
|
||||||
CONN.commit()
|
|
||||||
|
|
||||||
def get_undownloaded_clips():
|
|
||||||
"""Retrieves all clip rows remaining to be captured."""
|
|
||||||
CURSOR.execute(
|
|
||||||
"SELECT slug, date, title, gamename, clip_by, view_count, downloaded FROM clips WHERE downloaded = 0"
|
|
||||||
)
|
|
||||||
return CURSOR.fetchall()
|
|
||||||
|
|
||||||
def insert_record(slug: str, record_date_str: str, title: str, gamename: str, clip_by: str, views: int):
|
|
||||||
"""Cleans up ISO-8601 strings into unified date structures for the database."""
|
|
||||||
try:
|
|
||||||
clean_date = datetime.strptime(record_date_str, "%Y-%m-%dT%H:%M:%SZ").date()
|
|
||||||
except ValueError:
|
|
||||||
clean_date = record_date_str
|
|
||||||
|
|
||||||
CURSOR.execute(
|
|
||||||
"INSERT OR IGNORE INTO clips (slug, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
|
||||||
(slug, str(clean_date), title, gamename, clip_by, views, False, False),
|
|
||||||
)
|
|
||||||
CONN.commit()
|
|
||||||
|
|
||||||
def get_channel_clips(channel_name: str):
|
def get_channel_clips(channel_name: str):
|
||||||
"""Queries Twitch's public endpoint directly for trending clips."""
|
"""Queries Twitch's public endpoint directly for trending clips."""
|
||||||
session = requests.Session()
|
session = requests.Session()
|
||||||
@@ -182,7 +146,7 @@ def get_channel_clips(channel_name: str):
|
|||||||
|
|
||||||
print(f"Slug: {node['slug']} | Date: {node['createdAt']} | Game: {game_name} | By: {clip_by} | Views: {node['viewCount']} | Title: {node['title']}")
|
print(f"Slug: {node['slug']} | Date: {node['createdAt']} | Game: {game_name} | By: {clip_by} | Views: {node['viewCount']} | Title: {node['title']}")
|
||||||
|
|
||||||
insert_record(node['slug'], node['createdAt'], node['title'], game_name, clip_by, int(node['viewCount']))
|
DB.insert_clips_record(node['slug'], node['createdAt'], node['title'], game_name, clip_by, int(node['viewCount']))
|
||||||
slugs.append(slug_id)
|
slugs.append(slug_id)
|
||||||
|
|
||||||
return slugs
|
return slugs
|
||||||
@@ -192,8 +156,6 @@ def get_channel_clips(channel_name: str):
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
create_database()
|
|
||||||
get_channel_clips(CHANNEL_NAME)
|
get_channel_clips(CHANNEL_NAME)
|
||||||
download_clips()
|
download_clips()
|
||||||
upload_clips()
|
upload_clips()
|
||||||
close_database()
|
|
||||||
Reference in New Issue
Block a user