diff --git a/.gitignore b/.gitignore index a5bceec..767d3b3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ download output.log *.mp4 database copy.db +tiktok_secrets.json +cookies.txt diff --git a/help.sqlite3-query b/help.sqlite3-query index d629142..68ff878 100644 --- a/help.sqlite3-query +++ b/help.sqlite3-query @@ -8,4 +8,5 @@ SELECT * FROM twitch_videos WHERE title = "Bolt-Action Bureaucracy"; SELECT * FROM twitch_videos WHERE title = "Just after going through a mass starvation..."; SELECT * FROM twitch_videos WHERE title = "Semi-Automatic Sanctimony"; SELECT * FROM twitch_videos WHERE id = "2848295283"; -SELECT * FROM twitch_videos WHERE game_name = "Darkwood"; \ No newline at end of file +SELECT * FROM twitch_videos WHERE game_name = "Darkwood"; +SELECT * FROM data; \ No newline at end of file diff --git a/main.py b/main.py index 64757c9..8513b27 100644 --- a/main.py +++ b/main.py @@ -28,7 +28,7 @@ def lock_file(): if __name__ == "__main__": lock_file() - upload = False + upload = True if upload is False: # Get Twitch Video Info @@ -42,7 +42,7 @@ if __name__ == "__main__": #Build build_videos() else: - process_uploads() + process_uploads() print("====================================================") print(f"✅ Done.") diff --git a/package/database/__init__.py b/package/database/__init__.py index 4fca978..c738639 100644 --- a/package/database/__init__.py +++ b/package/database/__init__.py @@ -1,13 +1,15 @@ # package/database/__init__.py # Expose components from individual modules using relative imports -from .database import TwitchVideo, Data, YoutubeInfo, Chapter, Repository +from .database import TwitchVideo, Data, YoutubeInfo, TiktokInfo, Highlight, Chapter, Repository # Explicitly define what is exposed when importing the subpackage __all__ = [ "TwitchVideo", "Data", "YoutubeInfo", + "TiktokInfo", + "Highlight", "Chapter", "Repository", ] \ No newline at end of file diff --git a/package/database/database.py b/package/database/database.py index 17a6223..aeac58e 100644 --- a/package/database/database.py +++ b/package/database/database.py @@ -1,4 +1,4 @@ -from sqlalchemy import create_engine, Column, String, Integer, Boolean, ForeignKey, select +from sqlalchemy import create_engine, Column, String, Integer, Boolean, Float, ForeignKey, select from sqlalchemy.orm import declarative_base, relationship, sessionmaker from typing import Any @@ -35,6 +35,9 @@ class TwitchVideo(Base): tiktok_info = relationship("TiktokInfo", back_populates="twitch_video", uselist=True) chapters = relationship("Chapter", back_populates="twitch_video", cascade="all, delete-orphan") + # THIS WAS MISSING - this is what Highlight is looking for + highlights = relationship("Highlight", back_populates="twitch_video", cascade="all, delete-orphan") + # 3. Define the data table class Data(Base): __tablename__ = "data" @@ -45,6 +48,7 @@ class Data(Base): date_shorts = Column(String, nullable=False) date_videos = Column(String, nullable=False) date_videos_chats = Column(String, nullable=False) + date_tiktok = Column(String, nullable=False) class YoutubeInfo(Base): __tablename__ = "youtube_info" @@ -74,6 +78,21 @@ class TiktokInfo(Base): # Relationship linking back to the twitch video record twitch_video = relationship("TwitchVideo", back_populates="tiktok_info") +class Highlight(Base): + __tablename__ = "highlights" + + id = Column(Integer, primary_key=True, autoincrement=True) + twitch_video_id = Column(String, ForeignKey("twitch_videos.id"), nullable=False) + start_milliseconds = Column(Float, nullable=False) + length_milliseconds = Column(Float, nullable=False) + start_timestamp = Column(String, nullable=False) + end_timestamp = Column(String, nullable=False) + description = Column(String, nullable=False) + score = Column(Float, nullable=False) + + # Relationship linking back to the twitch video record + twitch_video = relationship("TwitchVideo", back_populates="highlights") + class Chapter(Base): __tablename__ = "chapters" @@ -134,6 +153,10 @@ class Repository: """Flags a specific row record to downloaded (1).""" self.__mark_as(record_id, "downloaded") + def mark_as_uploaded_tiktok(self, record_id: str): + """Flags a specific row record to uploaded_tiktok (1).""" + self.__mark_as(record_id, "uploaded_tiktok") + def unmark_as_download(self, record_id: str): """Flags a specific row record to downloaded (0).""" self.__mark_as(record_id, "downloaded", False) @@ -186,6 +209,10 @@ class Repository: stmt = select(TwitchVideo).where(TwitchVideo.downloaded == False) return self.session.scalars(stmt).all() + def get_unuploaded_tiktok(self) -> list[TwitchVideo]: + """Retrieve all rows that were download but not uploaded_tiktok""" + return self.__get_unuploaded("uploaded_tiktok", (TwitchVideo.clip_is == True)) + def get_downloaded(self) -> list[TwitchVideo]: stmt = select(TwitchVideo).where(TwitchVideo.downloaded == True) return self.session.scalars(stmt).all() @@ -215,19 +242,19 @@ class Repository: def update_date_chats(self, date: str): self.__update_date("date_videos_chat", date) - def get_data(self) -> list[Data]: + def get_data(self) -> Data: stmt = select(Data) - return self.session.scalars(stmt).all() + return self.session.scalars(stmt).first() - def get_youtube_info_for_video(self, video_id: str) -> list[YoutubeInfo]: + def get_youtube_info_for_video(self, video_id: str) -> YoutubeInfo | None: """Retrieve all YouTubeInfo records associated with a specific TwitchVideo ID.""" stmt = select(YoutubeInfo).where(YoutubeInfo.twitch_video_id == video_id) - return self.session.scalars(stmt).all() + return self.session.scalars(stmt).first() def get_youtube_info_all(self) -> list[YoutubeInfo]: return self.session.scalars(select(YoutubeInfo)).all() - def insert(self, data: TwitchVideo | Data | YoutubeInfo | Chapter | list[YoutubeInfo] | list[Chapter]): + def insert(self, data: TwitchVideo | Data | YoutubeInfo | Chapter | list[YoutubeInfo] | list[Chapter] | TiktokInfo | Highlight): """ Handles insertion/updating for any of the defined models, using session merge/add for SQLAlchemy 2.0 best practices. @@ -270,6 +297,14 @@ class Repository: print(f"➕ Successfully processed {len(data)} records of type {record_type.__name__}.") else: raise TypeError(f"Unsupported list content type: {record_type.__name__}") + elif isinstance(data, TiktokInfo): + # Handle single TiktokInfo object + self.session.merge(data) + print(f"➕ TiktokInfo record merged/inserted successfully!") + elif isinstance(data, Highlight): + # Handle single Highlight object + self.session.merge(data) + print(f"➕ Highlight record merged/inserted successfully!") else: raise TypeError(f"Unsupported data type for insertion: {type(data).__name__}") diff --git a/package/process_uploads.py b/package/process_uploads.py index 916dd7e..d103c89 100644 --- a/package/process_uploads.py +++ b/package/process_uploads.py @@ -11,14 +11,25 @@ from package.config import DB as DB from package.hash_tags import get_top_nouns as get_top_nouns from package.youtube.uploader import upload_video as upload_video from package.youtube.uploader import CategoryId as CategoryId +from package.tiktok.uploader import upload_to_tiktok as upload_to_tiktok -from package.database import TwitchVideo, YoutubeInfo, Data +from package.database import TwitchVideo, YoutubeInfo, TiktokInfo, Data, TiktokInfo def process_videos(video_type: bool = True, limit: int = 22): count = 0 # count how many videos we have upload and stop at var: limit videos = DB.get_unuploaded_chats() if video_type else DB.get_unuploaded_shorts() file_type = "download/video" if video_type else "download/clips" - meta_data: Data = DB.get_data()[0] + meta_data: Data = DB.get_data() + + #current_time = datetime.now(TZ) + #current_time = current_time.replace(hour=0, minute=0, second=0, microsecond=0) + + #meta_data.date_videos = current_time + #meta_data.date_shorts = current_time + #meta_data.date_tiktok = current_time + #meta_data.date_videos_chats = current_time + + #DB.insert(meta_data) # ensure we have a record in the database current_time = datetime.now(TZ) next_time = meta_data.date_videos if video_type else meta_data.date_shorts @@ -62,17 +73,14 @@ def process_videos(video_type: bool = True, limit: int = 22): if video_type: origin_clipped = "Originally Streamed on" - hashs = "#Twitch" else: origin_clipped = f"Clipped By: {video.creator_name}, on" - hashs = "#Shorts #Clips #Twitch" - description = f"Game: {video.game_name}, {origin_clipped} {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, {hashs} Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp" privatcyStatus = 'private' tags = ['gaming', 'TeamPGP', f'{video.game_name}', f'{video.creator_name}', 'Level1Techs', 'twitch'] if not video_type: tags.extend(['shorts', 'twitch_clips', 'clips']) - tags.extend([video.id]) + tags.extend([video.id]) if srt_path.exists(): try: @@ -80,7 +88,11 @@ def process_videos(video_type: bool = True, limit: int = 22): except Exception as e: print(f"⚠️ Could not get nouns from {srt_path}: {e}") - youtube_info = DB.get_youtube_info(video.id) or YoutubeInfo(id=video.id) + hashs = " ".join([f"#{t}" for t in tags]) + + description = f"Game: {video.game_name}, {origin_clipped} {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, {hashs} Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp" + + youtube_info = DB.get_youtube_info_for_video(video.id) or YoutubeInfo(twitch_video_id=video.id) if youtube_info.yt_id: main_video_id = youtube_info.yt_id @@ -118,8 +130,7 @@ def process_videos(video_type: bool = True, limit: int = 22): youtube_info.yt_short_date = datetime.now(TZ) DB.insert(youtube_info) - #release_time += timedelta(days=1) - release_time += timedelta(hours=12) # release every 12 hours instead of every day + release_time += timedelta(days=0.5) # Update global release pointer once at end if video_type: @@ -127,6 +138,49 @@ def process_videos(video_type: bool = True, limit: int = 22): else: DB.update_date_shorts(release_time) +def process_tiktoks(): + tiktoks = DB.get_unuploaded_tiktok() + data = DB.get_data() + meta_data: Data = data[0] + + release_time = meta_data.date_tiktok + + current_time = datetime.now(TZ) + release_time = datetime.strptime(release_time, "%Y-%m-%d %H:%M:%S%z") + + count = 0 + + if current_time > release_time: + release_time = current_time + release_time = release_time.replace(hour=12, minute=0, second=0, microsecond=0) + + for tiktok in tiktoks: + if count >= 10: + break + created_at = datetime.strptime(tiktok.created_at, "%Y-%m-%d %H:%M:%S%z").astimezone(TZ) + tags = ["Twitch", "TeamPGP", f"{tiktok.game_name}", f"{tiktok.creator_name}", "Level1Techs", "clips"] + origin_clipped = f"Clipped By: {tiktok.creator_name}, on" + description = f"Game: {tiktok.game_name}, {origin_clipped} {created_at.strftime("%Y-%m-%d %H:%M:%S %Z (%:z)")}, Every Friday and Sunday @7:30 ET https://twitch.tv/teampgp" + original_path = f"download/clips/{tiktok.id}/{tiktok.id}.mp4" + file_path = f"download/clips/{tiktok.id}/{tiktok.id}_gaussian_9_16.mp4" + file_csv = f"download/clips/{tiktok.id}/{tiktok.id}_{meta_data.name}.csv" + + srt_path = f"download/clips/{tiktok.id}/{tiktok.id}.srt" + + try: + tags.extend(get_top_nouns(str(srt_path), (15 - len(tags)))) + except Exception as e: + print(f"⚠️ Could not get nouns from {srt_path}: {e}") + + if not os.path.exists(file_path): + file_path = f"download/clips/{tiktok.id}/{tiktok.id}_black_9_16.mp4" + if not os.path.exists(file_path): + continue + if tiktok_video_id := upload_to_tiktok(file_path, description, tags, "teampgpunofficialarchive"): + tiktok_info = TiktokInfo(twitch_video_id=tiktok.id, tiktok_id=tiktok_video_id, tiktok_date=datetime.now(TZ)) + DB.insert(tiktok_info) + count += 1 + def process_shorts(): process_videos(False) return @@ -275,6 +329,7 @@ def process_chats(): def process_uploads(): + #process_tiktoks() process_shorts() #process_chats() diff --git a/package/tiktok/uploader.py b/package/tiktok/uploader.py index a4ae818..27895c3 100644 --- a/package/tiktok/uploader.py +++ b/package/tiktok/uploader.py @@ -1,72 +1,74 @@ -#!/usr/bin/env python3 -from tiktok_uploader.upload import upload_video -from pathlib import Path import json -import time -from playwright.sync_api import sync_playwright +import requests +from pathlib import Path -COOKIES_FILE = "cookies.txt" +SECRETS_FILE = Path(__file__).parent.parent / "tiktok_secrets.json" +# if your file is at /home/richard/python_scripts/tiktok_secrets.json +# use: Path("/home/richard/python_scripts/tiktok_secrets.json") -def get_last_video_link(username): - """Grab newest video link from profile""" - url = f"https://www.tiktok.com/@{username}" - with sync_playwright() as p: - browser = p.chromium.launch(headless=True) - context = browser.new_context(storage_state=COOKIES_FILE) if Path(COOKIES_FILE).exists() else browser.new_context() - page = context.new_page() - page.goto(url, timeout=60000) - page.wait_for_selector('[data-e2e="user-post-item"] a', timeout=15000) - first_video = page.query_selector('[data-e2e="user-post-item"] a') - link = first_video.get_attribute('href') if first_video else None - browser.close() - return link +def load_secrets(): + with open(SECRETS_FILE) as f: + secrets = json.load(f) + return secrets["client_key"], secrets["client_secret"], secrets["access_token"], secrets.get("refresh_token", "") -def upload_to_tiktok(video_path, description, hashtags=[], username="YOUR_USERNAME_HERE"): - tags = " ".join([f"#{t}" for t in hashtags]) - full_desc = f"{description}\n\n{tags}" +API_BASE = "https://open.tiktokapis.com/v2" + +def upload_to_tiktok_api(video_path: str, title: str, privacy="PUBLIC_TO_EVERYONE"): + client_key, client_secret, access_token, refresh_token = load_secrets() - failed = upload_video( - filename=str(video_path), - description=full_desc, - cookies=COOKIES_FILE, - headless=True, - browser="chrome" - ) + video_path = Path(video_path) + file_size = video_path.stat().st_size - if len(failed) != 0: - print(f"❌ Failed {video_path}: {failed}") - return None + headers = { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json; charset=UTF-8" + } - print(f"✅ Uploaded {video_path} - fetching link...") - time.sleep(8) # give TikTok time to process + # 1. INIT + init_payload = { + "source_info": { + "source": "FILE_UPLOAD", + "video_size": file_size, + "chunk_size": file_size, + "total_chunk_count": 1 + } + } + r = requests.post(f"{API_BASE}/post/publish/video/init/", headers=headers, json=init_payload) + r.raise_for_status() + data = r.json() + if data.get("error", {}).get("code") != "ok": + raise Exception(f"Init failed: {data}") - # Try 3 times, TikTok takes ~10-30s to show new video - for i in range(3): - link = get_last_video_link(username) - if link: - print(f"🔗 {link}") - return link - print(f" Waiting for TikTok to publish... retry {i+1}/3") - time.sleep(10) - - return f"https://www.tiktok.com/@{username} (uploaded, but link not ready yet - check profile)" + publish_id = data["data"]["publish_id"] + upload_url = data["data"]["upload_url"] + + # 2. UPLOAD + with open(video_path, "rb") as f: + put_headers = { + "Content-Range": f"bytes 0-{file_size-1}/{file_size}", + "Content-Type": "video/mp4" + } + requests.put(upload_url, headers=put_headers, data=f).raise_for_status() + + # 3. PUBLISH + publish_payload = { + "publish_id": publish_id, + "post_info": { + "title": title[:2200], + "privacy_level": privacy, + "disable_duet": False, + "disable_comment": False, + "disable_stitch": False, + "video_cover_timestamp_ms": 1000 + } + } + pr = requests.post(f"{API_BASE}/post/publish/", headers=headers, json=publish_payload) + pr.raise_for_status() + return pr.json() if __name__ == "__main__": - USERNAME = "your_tiktok_username" # <-- CHANGE THIS, no @ - - with open("highlights.json") as f: - data = json.load(f) - - for h in data['highlights'][:3]: - video_file = f"highlights/highlight_{h['id']}_{h['start_timestamp'].replace(':','-')}.mp4" - if not Path(video_file).exists(): - continue - - desc = h['transcript'][:150] - link = upload_to_tiktok( - video_file, - description=desc, - hashtags=["fyp", "clutch", "highlight", "pog"], - username=USERNAME - ) - print(f"LINK: {link}\n") \ No newline at end of file + result = upload_to_tiktok_api( + "/home/richard/python_scripts/download/clips/ProductiveConfidentWombatThunBeast/ProductiveConfidentWombatThunBeast_gaussian_9_16.mp4", + "Testing upload #Twitch #TeamPGP #Stellaris" + ) + print(result) \ No newline at end of file diff --git a/package/video_editing/highlights.py b/package/video_editing/highlights.py index 025c7f9..682556d 100644 --- a/package/video_editing/highlights.py +++ b/package/video_editing/highlights.py @@ -267,20 +267,48 @@ def format_timestamp(seconds): return f"{h:02d}:{m:02d}:{s:02d}" if h > 0 else f"{m:02d}:{s:02d}" def highlights(video_path, keywords=None, top_n=5, clip_len=8, min_gap=12): - if keywords is None: - keywords = DEFAULT_KEYWORDS - av_score, duration = get_av_scores(video_path, sample_fps=0.5) + segments = load_text_from_srt(video_path.replace('.mp4', '.srt')) kw_score, kw_hits = score_keywords(segments, duration, keywords) + # FINAL SCORE: AV + keyword boost (keyword is huge if found) final_score = av_score + (kw_score * 1.5) final_score = (final_score - final_score.min()) / (final_score.max() - final_score.min() + 1e-6) - final_score = np.convolve(final_score, np.ones(3)/3, mode='same') + final_score = np.convolve(final_score, np.ones(3)/3, mode='same') # smooth highlights = find_highlights(final_score, duration, top_n=top_n, clip_len=clip_len) - return highlights, kw_hits + # --- OUTPUT --- + print("\n" + "="*60) + print("KEYWORD HITS:") + for h in kw_hits: + print(f" [{format_timestamp(h['timestamp'])}] ({h['keyword']}) -> {h['text']}") + + print("\n" + "="*60) + print("FINAL HIGHLIGHTS:") + output_data = [] + for i, (start, end, score) in enumerate(highlights, 1): + # find transcript in that window + texts_in_window = [s['text'].strip() for s in segments if not (s['end'] < start or s['start'] > end)] + transcript = " ".join(texts_in_window)[:120] + + print(f"{i}. {format_timestamp(start)} - {format_timestamp(end)} (score {score:.2f}) | {transcript}") + output_data.append({ + "id": i, + "start_sec": float(start), + "end_sec": float(end), + "start_timestamp": format_timestamp(start), + "end_timestamp": format_timestamp(end), + "score": float(score), + "transcript": transcript + }) + + # Save timestamps to JSON + with open("highlights.json", "w") as f: + json.dump({"video": video_path, "keywords": keywords, "highlights": output_data, "keyword_hits": kw_hits}, f, indent=2) + print(f"\nTimestamps saved to highlights.json") + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Find highlights with Whisper keywords + timestamps")