From 3a931fe75272608d9d693b95bcc99ee6e50a67c0 Mon Sep 17 00:00:00 2001 From: Zichao Lin Date: Sat, 18 Jul 2026 08:03:18 +0800 Subject: [PATCH] feat(storage): add S3 image storage support with file-based config and migration script --- .gitignore | 5 +- app/routers/api.py | 13 ++- app/routers/web.py | 19 +-- app/schemas.py | 16 ++- app/storage.py | 180 +++++++++++++++++++++++++++++ app/templates/dashboard.html | 2 +- app/templates/index.html | 4 +- app/templates/postcard_detail.html | 4 +- app/templates/postcards.html | 4 +- app/templates/showcase.html | 16 +-- s3_config.example.json | 9 ++ 11 files changed, 242 insertions(+), 30 deletions(-) create mode 100644 app/storage.py create mode 100644 s3_config.example.json diff --git a/.gitignore b/.gitignore index ba1ffc2..f706380 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,7 @@ uploads/* *.db .env server.log -scripts/ +s3_config.json +scripts/fix_*.py +scripts/migrate_*.py +scripts/test_*.py diff --git a/app/routers/api.py b/app/routers/api.py index c9a66ec..cb2d649 100644 --- a/app/routers/api.py +++ b/app/routers/api.py @@ -13,7 +13,7 @@ from app.models import ( PERM_POSTCARDS_READ, PERM_POSTCARDS_WRITE, PERM_IMAGES_UPLOAD, ) -from app.config import UPLOAD_DIR +# config removed: image storage now via app.storage router = APIRouter(prefix="/api/v1", tags=["api"]) @@ -380,15 +380,16 @@ async def upload_image( ak: ApiKey = Depends(require_scopes(PERM_IMAGES_UPLOAD)), ): pc = _postcard_or_404(ak.user, profile_id, postcard_id, db) + from app.storage import get_storage + storage = get_storage() ext = Path(file.filename).suffix if file.filename else ".jpg" - save_name = f"pc_{pc.id}_{side}{ext}" - save_path = UPLOAD_DIR / save_name + key = f"pc_{pc.id}_{side}{ext}" content = await file.read() - save_path.write_bytes(content) + storage.put(key, content) if side == "front": - pc.image_front = f"/uploads/{save_name}" + pc.image_front = key else: - pc.image_back = f"/uploads/{save_name}" + pc.image_back = key db.commit() db.refresh(pc) return pc diff --git a/app/routers/web.py b/app/routers/web.py index 0ab8cc5..d58ab32 100644 --- a/app/routers/web.py +++ b/app/routers/web.py @@ -16,7 +16,7 @@ from app.auth import ( redirect_if_not_logged_in, verify_password, ) -from app.config import SECRET_KEY, SESSION_COOKIE_NAME, UPLOAD_DIR +from app.config import SECRET_KEY, SESSION_COOKIE_NAME from app.database import get_db from random import choice from string import ascii_letters, digits @@ -86,6 +86,10 @@ def _translate_filter(key: str, lang: str = "zh") -> str: templates.env.filters["t"] = _translate_filter +# image URL resolver: bare key → full URL (local /uploads/... or S3 public URL) +from app.storage import resolve_url as _resolve_image_url +templates.env.filters["image_url"] = lambda v: _resolve_image_url(v) if v else "" + from app.i18n import get_languages as _get_languages templates.env.globals["available_languages"] = _get_languages() @@ -188,7 +192,7 @@ def api_public_cards( "page": page, "has_more": start + limit < total, "cards": [{ - "image_front": c.image_front, + "image_front": _resolve_image_url(c.image_front), "country_from": c.country_from, "country_to": c.country_to, "status": c.status, @@ -230,7 +234,7 @@ def api_public_cards_user( "page": page, "has_more": start + limit < total, "cards": [{ - "image_front": c.image_front, + "image_front": _resolve_image_url(c.image_front), "country_from": c.country_from, "country_to": c.country_to, } for c in batch], @@ -959,12 +963,13 @@ async def upload_image( ) if not pc: return _redirect("/profiles") + from app.storage import get_storage + storage = get_storage() ext = Path(file.filename or "upload.jpg").suffix or ".jpg" - filename = f"{uuid4().hex}{ext}" - dest = UPLOAD_DIR / filename + key = f"{uuid4().hex}{ext}" content = await file.read() - dest.write_bytes(content) - setattr(pc, f"image_{side}", f"/uploads/{filename}") + storage.put(key, content) + setattr(pc, f"image_{side}", key) db.commit() return _redirect(f"/postcards/{postcard_id}") diff --git a/app/schemas.py b/app/schemas.py index ecac069..3b31d4e 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -1,6 +1,6 @@ from datetime import datetime -from pydantic import BaseModel +from pydantic import BaseModel, model_validator # ---------- Profile ---------- @@ -23,6 +23,13 @@ class ProfileOut(BaseModel): model_config = {"from_attributes": True} + @model_validator(mode="after") + def _resolve_image_urls(self) -> "PostcardOut": + from app.storage import resolve_url + self.image_front = resolve_url(self.image_front) or None + self.image_back = resolve_url(self.image_back) or None + return self + # ---------- Postcard ---------- @@ -68,3 +75,10 @@ class PostcardOut(BaseModel): updated_at: datetime model_config = {"from_attributes": True} + + @model_validator(mode="after") + def _resolve_image_urls(self) -> "PostcardOut": + from app.storage import resolve_url + self.image_front = resolve_url(self.image_front) or None + self.image_back = resolve_url(self.image_back) or None + return self diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 0000000..42d3560 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,180 @@ +"""Image storage abstraction: local filesystem or S3-compatible object storage. + +S3 config is read from ``s3_config.json`` in the project root:: + + { + "enabled": true, + "endpoint": "https://s3.amazonaws.com", + "bucket": "my-mailova-bucket", + "region": "auto", + "access_key": "AKIA...", + "secret_key": "...", + "public_url": "https://cdn.example.com/mailova" + } + +When the file is missing or ``enabled`` is false, local storage is used. + +URL scheme +~~~~~~~~~~ +Database stores **bare filenames** (``xxx.jpeg``) regardless of backend. +``resolve_url(key)`` converts a bare key to a full URL: + - S3 → ``/xxx.jpeg`` + - local → ``/uploads/xxx.jpeg`` + +Legacy records that still contain ``/uploads/xxx.jpeg`` are auto-stripped +by ``resolve_url``. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Protocol + +from app.config import UPLOAD_DIR + +S3_CONFIG_PATH = Path(__file__).resolve().parent.parent / "s3_config.json" + + +class StorageBackend(Protocol): + def put(self, key: str, data: bytes, content_type: str = "image/jpeg") -> str: + """Store data and return the bare key.""" + ... + + def delete(self, key: str) -> None: ... + + def exists(self, key: str) -> bool: ... + + def resolve_url(self, key: str) -> str: + """Convert a bare key (or legacy /uploads/xxx path) to a full URL.""" + ... + + +# ── Local Storage ────────────────────────────────────────────────── + +class LocalStorage: + def __init__(self) -> None: + UPLOAD_DIR.mkdir(parents=True, exist_ok=True) + + def put(self, key: str, data: bytes, content_type: str = "") -> str: + (UPLOAD_DIR / key).write_bytes(data) + return key + + def delete(self, key: str) -> None: + p = UPLOAD_DIR / key + if p.exists(): + p.unlink() + + def exists(self, key: str) -> bool: + return (UPLOAD_DIR / key).exists() + + def resolve_url(self, key: str) -> str: + return f"/uploads/{_strip_prefix(key)}" + + +# ── S3 Storage ───────────────────────────────────────────────────── + +class S3Storage: + def __init__( + self, + endpoint_url: str, + bucket: str, + access_key: str, + secret_key: str, + region: str = "auto", + public_url: str = "", + ) -> None: + import boto3 + + self.bucket = bucket + self.public_url = public_url.rstrip("/") + self._client = boto3.client( + "s3", + endpoint_url=endpoint_url or None, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name=region or "auto", + ) + + def put(self, key: str, data: bytes, content_type: str = "image/jpeg") -> str: + self._client.put_object( + Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, + ) + return key + + def delete(self, key: str) -> None: + self._client.delete_object(Bucket=self.bucket, Key=key) + + def exists(self, key: str) -> bool: + try: + self._client.head_object(Bucket=self.bucket, Key=key) + return True + except Exception: + return False + + def resolve_url(self, key: str) -> str: + k = _strip_prefix(key) + if self.public_url: + return f"{self.public_url}/{k}" + return k + + +# ── Helpers ──────────────────────────────────────────────────────── + +def _strip_prefix(key: str) -> str: + """Strip ``/uploads/`` prefix from legacy DB values.""" + if key.startswith("/uploads/"): + return key[len("/uploads/"):] + return key + + +# ── Singleton accessor ───────────────────────────────────────────── + +_backend: StorageBackend | None = None + + +def _load_s3_config() -> dict | None: + """Read s3_config.json. Returns None if not configured / disabled.""" + if not S3_CONFIG_PATH.exists(): + return None + try: + cfg = json.loads(S3_CONFIG_PATH.read_text(encoding="utf-8")) + if not cfg.get("enabled") or not cfg.get("bucket"): + return None + return cfg + except Exception: + return None + + +def get_storage() -> StorageBackend: + """Get the active storage backend (S3 if configured, otherwise local).""" + global _backend + if _backend is not None: + return _backend + + s3_cfg = _load_s3_config() + if s3_cfg: + _backend = S3Storage( + endpoint_url=s3_cfg.get("endpoint", ""), + bucket=s3_cfg["bucket"], + access_key=s3_cfg.get("access_key", ""), + secret_key=s3_cfg.get("secret_key", ""), + region=s3_cfg.get("region", "auto"), + public_url=s3_cfg.get("public_url", ""), + ) + else: + _backend = LocalStorage() + return _backend + + +def resolve_url(key: str | None) -> str: + """Convenience: resolve a possibly-None image key to a URL (or empty).""" + if not key: + return "" + return get_storage().resolve_url(key) + + +def reset_storage() -> None: + """Force re-detection on next get_storage() call.""" + global _backend + _backend = None diff --git a/app/templates/dashboard.html b/app/templates/dashboard.html index 18c91be..9bce907 100644 --- a/app/templates/dashboard.html +++ b/app/templates/dashboard.html @@ -108,7 +108,7 @@ new Chart(document.getElementById('chart-received'), {
{% if pc.image_front %} - + {% else %} 📷 {% endif %} diff --git a/app/templates/index.html b/app/templates/index.html index d9dd2ca..181fdd2 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -33,8 +33,8 @@ {% if visible|length > 0 or has_more %}
{% for pc in visible %} -
- +
+
{% if pc.country_from and pc.country_to %}{{ pc.country_from|flag }} {{ pc.country_from }} → {{ pc.country_to|flag }} {{ pc.country_to }} diff --git a/app/templates/postcard_detail.html b/app/templates/postcard_detail.html index f040361..3788eba 100644 --- a/app/templates/postcard_detail.html +++ b/app/templates/postcard_detail.html @@ -28,7 +28,7 @@

{{ 'pc_detail.front'|t(user.language) }}

{% if postcard.image_front %} - + {% else %}
{{ 'pc_detail.no_front'|t(user.language) }}
{% endif %} @@ -40,7 +40,7 @@

{{ 'pc_detail.back'|t(user.language) }}

{% if postcard.image_back %} - + {% else %}
{{ 'pc_detail.no_back'|t(user.language) }}
{% endif %} diff --git a/app/templates/postcards.html b/app/templates/postcards.html index 4ff94ce..d88dba6 100644 --- a/app/templates/postcards.html +++ b/app/templates/postcards.html @@ -36,7 +36,7 @@
{% if pc.image_front %} - + {% else %}
📷
{% endif %} @@ -78,7 +78,7 @@
{% if pc.image_front %} - + {% else %}
📷
{% endif %} diff --git a/app/templates/showcase.html b/app/templates/showcase.html index 0f9b483..dabd5f3 100644 --- a/app/templates/showcase.html +++ b/app/templates/showcase.html @@ -45,8 +45,8 @@ {% if mode == 'flat' %}
{% for pc in sent_visible %} -
- +
+
{% if pc.country_from and pc.country_to %}{{ pc.country_from|flag }} {{ pc.country_from }} → {{ pc.country_to|flag }} {{ pc.country_to }} @@ -70,8 +70,8 @@

{{ g.profile.nickname }}

{% for pc in g_sent %} -
- +
+
{% if pc.country_from and pc.country_to %}{{ pc.country_from|flag }} {{ pc.country_from }} → {{ pc.country_to|flag }} {{ pc.country_to }} @@ -96,8 +96,8 @@ {% if mode == 'flat' %}
{% for pc in received_visible %} -
- +
+
{% if pc.country_from and pc.country_to %}{{ pc.country_from|flag }} {{ pc.country_from }} → {{ pc.country_to|flag }} {{ pc.country_to }} @@ -121,8 +121,8 @@

{{ g.profile.nickname }}

{% for pc in g_received %} -
- +
+
{% if pc.country_from and pc.country_to %}{{ pc.country_from|flag }} {{ pc.country_from }} → {{ pc.country_to|flag }} {{ pc.country_to }} diff --git a/s3_config.example.json b/s3_config.example.json new file mode 100644 index 0000000..d670ef2 --- /dev/null +++ b/s3_config.example.json @@ -0,0 +1,9 @@ +{ + "enabled": false, + "endpoint": "https://s3.amazonaws.com", + "bucket": "my-mailova-bucket", + "region": "auto", + "access_key": "AKIA...", + "secret_key": "...", + "public_url": "https://cdn.example.com/mailova" +}