SumGuyV5/python_scripts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update vod downloader to use the new database.py file

Browse Source
This commit is contained in:
SumGuyV5
2026-07-17 19:39:11 -04:00
parent b573ca05aa
commit 8d6e29a161
4 changed files with 129 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+2
View File
@@ -2,3 +2,5 @@
/client_secrets.json
/clips_database.db
__pycache__/uploader.cpython-313.pyc
database.db
__pycache__/database.cpython-313.pyc
database.py
+80 -14
View File
@@ -1,19 +1,33 @@
import sqlite3
from datetime import datetime
from datetime import date as datetime_date
from pathlib import Path
# Stores data locally
CONN = sqlite3.connect("database.db")
CURSOR = CONN.cursor()
class Database:
def __init__(self, table):
self.table = table
self.columns = ""
class Database():
def __init__():
file_path = Path("database.db")
# IMPORTANT Must check if database.db exists before connecting to it.
file_exists = Path("database.db").is_file()
if file_path.is_file():
self.CONN = sqlite3.connect("database.db")
self.CURSOR = self.CONN.cursor()
if self.table == "vods":
self.columns = "id, date, title, gamename, downloaded, uploaded_yt, chat_upload_yt"
elif self.table == "clips":
self.columns = "slug, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt"
if not file_exists:
self.create_database()
def create_database():
def __del__(self):
self.close_database()
def create_database(self):
"""Creates a table structured explicitly for Twitch clip properties."""
CURSOR.execute(
self.CURSOR.execute(
"""
CREATE TABLE IF NOT EXISTS clips (
slug TEXT PRIMARY KEY,
@@ -28,7 +42,8 @@ class Database():
"""
)
CURSOR.execute(
"""Creates a table structured explicitly for Twitch vods properties."""
self.CURSOR.execute(
"""
CREATE TABLE IF NOT EXISTS vods (
id INTEGER PRIMARY KEY,
@@ -42,9 +57,60 @@ class Database():
"""
)
CONN.commit()
self.CONN.commit()
def close_database():
def close_database(self):
"""Commit before we close."""
CONN.commit()
CONN.close()
self.CONN.commit()
self.CONN.close()
def mark_as_downloaded(self, record_id: int):
"""Updates the downloaded status to True (1) for a specific record ID."""
# Updates the row matching the specific ID
# TODO clips uses slug
self.CURSOR.execute(
f"UPDATE {self.table} SET downloaded = 1 WHERE id = ?",
(record_id,)
)
self.CONN.commit()
def get_unuploaded(self):
"""Retrieves all clip rows remaining to be chat uploaded."""
self.CURSOR.execute(f"SELECT {self.columns} FROM {self.table} WHERE downloaded = 1 AND uploaded_yt = 0")
return self.CURSOR.fetchall()
def get_undownloaded(self):
"""Retrieves all rows where downloaded status is False (0)."""
self.CURSOR.execute(f"SELECT {self.columns} FROM {self.table} WHERE downloaded = 0")
return self.CURSOR.fetchall()
def insert_vods_record(self, record_id: int, record_date_str: datetime_date, title: str, gamename: str):
"""Inserts a record with ID, date, gamename, and title into a SQLite database."""
try:
clean_date = datetime.strptime(record_date_str, "%Y-%m-%dT%H:%M:%SZ").date()
except ValueError:
clean_date = record_date_str
# Connects to database file (creates it if missing)
# Inserts data using parameterized queries to prevent SQL injection
self.CURSOR.execute(
f"INSERT OR IGNORE INTO {self.table} ({self.columns}) VALUES (?, ?, ?, ?, ?, ?, ?)",
(record_id, str(clean_date), title, gamename, False, False, False),
)
# Saves changes and closes the connection
self.CONN.commit()
def insert_clips_record(slug: str, record_date_str: str, title: str, gamename: str, clip_by: str, views: int):
"""Cleans up ISO-8601 strings into unified date structures for the database."""
try:
clean_date = datetime.strptime(record_date_str, "%Y-%m-%dT%H:%M:%SZ").date()
except ValueError:
clean_date = record_date_str
self.CURSOR.execute(
f"INSERT OR IGNORE INTO {self.table} ({self.columns}) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
(slug, str(clean_date), title, gamename, clip_by, views, False, False),
)
self.CONN.commit()
twitch_download_clips.py
-34
View File
@@ -9,33 +9,6 @@ from uploader import CategoryId
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
# Stores data locally
CONN = sqlite3.connect("clips_database.db")
CURSOR = CONN.cursor()
def create_database():
"""Creates a table structured explicitly for Twitch clip properties."""
CURSOR.execute(
"""
CREATE TABLE IF NOT EXISTS clips (
slug TEXT PRIMARY KEY,
date TEXT NOT NULL,
title TEXT NOT NULL,
gamename TEXT NOT NULL,
clip_by TEXT NOT NULL,
view_count INTEGER NOT NULL,
downloaded INTEGER NOT NULL,
uploaded_yt INTEGER NOT NULL
)
"""
)
CONN.commit()
def close_database():
"""Commits queries before ending connection context."""
CONN.commit()
CONN.close()
def download_clips():
"""Loops over undownloaded metadata entries to write files down locally."""
undownloaded_clips = get_undownloaded_clips()
@@ -109,13 +82,6 @@ def mark_as_downloaded(slug: str):
)
CONN.commit()
def get_unuploaded_clips():
"""Retrieves all clip rows remaining to be uploaded."""
CURSOR.execute(
"SELECT slug, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt FROM clips WHERE downloaded = 1 AND uploaded_yt = 0"
)
return CURSOR.fetchall()
def get_undownloaded_clips():
"""Retrieves all clip rows remaining to be captured."""
CURSOR.execute(
twitch_download_vod.py
+47 -88
View File
@@ -2,55 +2,15 @@
import requests
import sqlite3
from datetime import date as datetime_date
import subprocess
from database import Database
import uploader
from uploader import CategoryId
CHANNEL_NAME = 'teampgp' # Replace with the streamer's username
CONN = sqlite3.connect("database.db")
CURSOR = CONN.cursor()
def create_database():
# Creates table safely using multi-line string
CURSOR.execute(
"""
CREATE TABLE IF NOT EXISTS vods (
id INTEGER PRIMARY KEY,
date TEXT NOT NULL,
title TEXT NOT NULL,
gamename TEXT NOT NULL,
downloaded INTEGER NOT NULL,
uploaded_yt INTEGER NOT NULL,
chat_upload_yt INTEGER NOT NULL
)
"""
)
CONN.commit()
def close_database():
"""Commit before we close."""
CONN.commit()
CONN.close()
def download_vods():
"""Find all undownload vods and download them."""
undownload_vods = get_undownloaded_vods()
print("Download VODs...")
for record_id, record_date, title, game_name, downloaded in undownload_vods:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
output = run_linux_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/vod/{record_id}/{record_id}.mp4")
output2 = run_linux_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/vod/{record_id}/{record_id}_chat.json -E")
if output["success"] is True:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was successful")
mark_as_downloaded(record_id)
else:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was Failed")
print("Finished Downloading VODs...")
DB = Database("vods")
def run_linux_command(command: str):
"""Executes a Linux command, waits for completion, and returns output."""
@@ -67,52 +27,50 @@ def run_linux_command(command: str):
# Handles errors if the Linux command returns a non-zero exit code
return {"success": False, "stdout": e.stdout, "stderr": e.stderr}
def mark_as_downloaded(record_id: int):
"""Updates the downloaded status to True (1) for a specific record ID."""
# Updates the row matching the specific ID
CURSOR.execute(
"UPDATE vods SET downloaded = 1 WHERE id = ?",
(record_id,)
)
def download_vods():
"""Find all undownload vods and download them."""
undownloaded_vods = DB.get_undownloaded()
CONN.commit()
print("Download VODs...")
def get_unuploaded_clips():
"""Retrieves all clip rows remaining to be chat uploaded."""
CURSOR.execute(
"SELECT id, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt, chat_uploaded_yt FROM vods WHERE downloaded = 1 AND chat_uploaded_yt = 0"
)
return CURSOR.fetchall()
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chat_upload_yt in undownloaded_vods:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
output = run_linux_command(f"TwitchDownloaderCLI videodownload --id {record_id} -o save/vods/{record_id}/{title}.mp4")
output2 = run_linux_command(f"TwitchDownloaderCLI chatdownload --id {record_id} -o save/vods/{record_id}/{title}_chat.json -E")
if output["success"] is True:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was successful")
DB.mark_as_downloaded(record_id)
else:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title} | was Failed")
def get_unuploaded_clips():
"""Retrieves all clip rows remaining to be uploaded."""
CURSOR.execute(
"SELECT id, date, title, gamename, clip_by, view_count, downloaded, uploaded_yt FROM vods WHERE downloaded = 1 AND uploaded_yt = 0"
)
return CURSOR.fetchall()
print("Finished Downloading VODs...")
def get_undownloaded_vods():
"""Retrieves all rows where downloaded status is False (0)."""
# Query filters by 0 because SQLite stores booleans as integers
CURSOR.execute(
"SELECT id, date, title, gamename, downloaded FROM vods WHERE downloaded = 0"
)
return CURSOR.fetchall()
def upload_vods():
"""Loops over the downloaded videos entries and uploaded them to youtube."""
print("Uploading Clips...")
def insert_record(record_id: int, record_date: datetime_date, title: str, gamename: str):
"""Inserts a record with ID, date, gamename, and title into a SQLite database."""
# Connects to database file (creates it if missing)
unuploaded_vods = DB.get_unuploaded()
# Inserts data using parameterized queries to prevent SQL injection
CURSOR.execute(
"INSERT OR IGNORE INTO vods (id, date, title, gamename, downloaded, uploaded_yt, chat_uploaded_yt) VALUES (?, ?, ?, ?, ?, ?, ?)",
(record_id, str(record_date), title, gamename, False, False, False),
)
for record_id, record_date, title, game_name, downloaded, uploaded_yt, chat_upload_yt in unuploaded_vods:
print(f"ID: {record_id} | Date: {record_date} | Game: {game_name} | Title: {title}")
# Saves changes and closes the connection
CONN.commit()
file_path = f"save/vods/{record_id}/{title}.mp4"
title = title
description = f"Game: {game_name}, on {record_date}, #Shorts"
categoryId = CategoryId.GAMING
privatcyStatus = 'private'
tags = ['gaming', 'TeamPGP', f'{game_name}']
output = uploader.upload_video(file_path, title, description, categoryId, privatcyStatus, tags)
if output is True:
print(f"ID: {record_id} | Was successfully uploaded.")
DB.mark_as_uploaded(record_id)
else:
print(f"ID: {record_id} | Upload Process failed.")
def create_chat_vods():
pass
def get_vod_ids_simplified(channel_name: str):
session = requests.Session()
@@ -190,7 +148,8 @@ def get_vod_ids_simplified(channel_name: str):
print(f"ID: {node['id']} | Date: {node['publishedAt']} | Game: {game_name} | Title: {node['title']}")
# Pass game_name to your database logic
insert_record(node['id'], node['publishedAt'], node['title'], game_name)
DB.insert_vods_record(node['id'], node['publishedAt'], node['title'], game_name)
#insert_record(node['id'], node['publishedAt'], node['title'], game_name)
vod_ids.append(node['id'])
return vod_ids
@@ -202,7 +161,7 @@ def get_vod_ids_simplified(channel_name: str):
return []
if __name__ == "__main__":
create_database()
get_vod_ids_simplified(CHANNEL_NAME)
download_vods()
close_database()
upload_vods()
#close_database()