105 lines
3.4 KiB
Python
105 lines
3.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Download Danbooru posts tagged 'holding_teapot' and convert them to WebP.
|
|
|
|
Requires ImageMagick's `mogrify` to be installed and on PATH.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
BASE_URL = "https://danbooru.donmai.us/posts"
|
|
TAGS = "holding_teapot -animated rating:general score:>=4"
|
|
OUT_DIR = "teapots"
|
|
IMAGE_EXTS = {"jpg", "jpeg", "png", "gif", "bmp"}
|
|
HEADERS = {"User-Agent": "teapot-scraper/1.0"}
|
|
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
|
|
|
|
def fetch_page(page: int):
|
|
query = urllib.parse.urlencode(
|
|
{"tags": TAGS, "format": "json", "page": page})
|
|
req = urllib.request.Request(f"{BASE_URL}?{query}", headers=HEADERS)
|
|
with urllib.request.urlopen(req) as resp:
|
|
return json.loads(resp.read())
|
|
|
|
|
|
def download(url: str, dest: str):
|
|
req = urllib.request.Request(url, headers=HEADERS)
|
|
with urllib.request.urlopen(req) as resp, open(dest, "wb") as f:
|
|
f.write(resp.read())
|
|
|
|
|
|
def main():
|
|
page = 64 # last page I got, will figure out how to get only posts we don't have later
|
|
while True:
|
|
print(f"Fetching page {page}...")
|
|
try:
|
|
posts = fetch_page(page)
|
|
except Exception as e:
|
|
print(f"Error fetching page {page}: {e}", file=sys.stderr)
|
|
break
|
|
|
|
if not posts:
|
|
print("Empty page, stopping.")
|
|
break
|
|
|
|
downloaded = []
|
|
for post in posts:
|
|
post_id = post.get("id")
|
|
file_ext = (post.get("file_ext") or "").lower()
|
|
file_url = post.get("file_url")
|
|
|
|
if file_ext not in IMAGE_EXTS:
|
|
continue
|
|
if not file_url:
|
|
print(f" Skipping post {
|
|
post_id}: no file_url (likely gated/login-only)")
|
|
continue
|
|
|
|
dest = os.path.join(OUT_DIR, f"{post_id}.{file_ext}")
|
|
try:
|
|
download(file_url, dest)
|
|
downloaded.append(dest)
|
|
print(f" Saved {dest}")
|
|
except Exception as e:
|
|
print(f" Failed to download post {
|
|
post_id}: {e}", file=sys.stderr)
|
|
|
|
if downloaded:
|
|
print(f"Converting {len(downloaded)} file(s) to webp...")
|
|
# WebP's bitstream format hard-caps width/height at 16383px, so
|
|
# downscale anything bigger before converting (the trailing '>'
|
|
# means "only resize if larger than this", smaller images are
|
|
# left untouched). Don't use check=True: one oversized/corrupt
|
|
# input shouldn't take down the whole batch.
|
|
result = subprocess.run(
|
|
["mogrify", "-resize", "16383x16383>",
|
|
"-format", "webp", *downloaded],
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if result.returncode != 0:
|
|
print(f" mogrify reported errors:\n{
|
|
result.stderr}", file=sys.stderr)
|
|
|
|
for path in downloaded:
|
|
webp_path = os.path.splitext(path)[0] + ".webp"
|
|
if os.path.exists(webp_path):
|
|
os.remove(path)
|
|
else:
|
|
print(f" Keeping original, conversion failed: {
|
|
path}", file=sys.stderr)
|
|
|
|
page += 1
|
|
time.sleep(1) # be polite to the API
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|