import os
import time
import requests
from tqdm import tqdm
from selenium import webdriver
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.common.by import By
from webdriver_manager.firefox import GeckoDriverManager
# === CONFIG ===
URL = "
https://civitai.com/videos?tags=5146&view=feed"
SAVE_DIR = "civitai_scraped_videos"
SCROLL_PAUSE = 3
MAX_EMPTY_SCROLLS = 3 # Stop when no new videos after these many scrolls
# === SETUP DOWNLOAD FOLDER ===
os.makedirs(SAVE_DIR, exist_ok=True)
# === SETUP FIREFOX BROWSER ===
def setup_browser():
firefox_options = Options()
firefox_options.add_argument("--width=1200")
firefox_options.add_argument("--height=800")
# firefox_options.add_argument("--headless") # Uncomment for headless mode
return webdriver.Firefox(service=Service(GeckoDriverManager().install()), options=firefox_options)
# === MANUAL LOGIN ===
def manual_login(driver):
print(" Opening Firefox browser for login...")
driver.get("
https://civitai.com/login")
input(" After logging in, press Enter here to continue...")
# === SCROLL PAGE AND COLLECT VIDEO URLs ===
def scroll_and_collect_video_urls(driver):
print(" Navigating to video feed...")
driver.get(URL)
time.sleep(5)
video_urls = set()
last_height = driver.execute_script("return document.body.scrollHeight")
empty_scrolls = 0
while empty_scrolls < MAX_EMPTY_SCROLLS:
print(" Scrolling...")
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(SCROLL_PAUSE)
new_videos = driver.find_elements(By.TAG_NAME, "video")
new_found = 0
for video in new_videos:
src = video.get_attribute("src")
if src and (".mp4" in src or ".webm" in src or ".gif" in src):
if src not in video_urls:
video_urls.add(src)
new_found += 1
