diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..68d0804 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,48 @@ +name: Build & Publish Docker Image + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +permissions: + contents: read + packages: write + +jobs: + build-and-push: + runs-on: ubuntu-latest + + steps: + # 1. Checkout the repo + - name: Checkout code + uses: actions/checkout@v4 + + # 2. Enable QEMU for cross‑platform + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + # 3. Enable Buildx + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + # 4. Log in to GitHub Container Registry + - name: Log in to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # 5. Build & push multi‑arch image + - name: Build and push Docker image + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ghcr.io/${{ github.repository_owner }}/ffmpeg-api-service:latest + ghcr.io/${{ github.repository_owner }}/ffmpeg-api-service:${{ github.sha }} diff --git a/Dockerfile b/Dockerfile index 7f3be68..718b11f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,69 @@ +# Use Docker BuildKit syntax v1.3+ for multi‐arch args +# syntax=docker/dockerfile:1.3 + +######################################## +# 1. Builder stage: install deps & FFmpeg +######################################## +FROM python:3.10-slim AS builder + +# Declare the build platform variable +ARG TARGETPLATFORM + +# Install tools for download & extraction +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + curl xz-utils ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /home/ffapi/app + +# Python deps +COPY requirements.txt . +RUN python -m pip install --upgrade pip \ + && pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Download & extract correct FFmpeg build +RUN set -eux; \ + case "${TARGETPLATFORM}" in \ + "linux/amd64") ARCH="amd64" ;; \ + "linux/arm64") ARCH="arm64" ;; \ + *) echo "Unsupported platform ${TARGETPLATFORM}" >&2; exit 1 ;; \ + esac; \ + URL="https://github.com/BtbN/FFmpeg-Builds/releases/latest/download/ffmpeg-git-${ARCH}-static.tar.xz"; \ + echo "Downloading FFmpeg from ${URL}"; \ + curl -fL "${URL}" -o /tmp/ffmpeg.tar.xz; \ + mkdir -p /tmp/ffmpeg; \ + tar -xJf /tmp/ffmpeg.tar.xz -C /tmp/ffmpeg --strip-components=1; \ + install -m755 /tmp/ffmpeg/ffmpeg /usr/local/bin/ffmpeg; \ + install -m755 /tmp/ffmpeg/ffprobe /usr/local/bin/ffprobe; \ + install -m755 /tmp/ffmpeg/ffplay /usr/local/bin/ffplay; \ + rm -rf /tmp/ffmpeg /tmp/ffmpeg.tar.xz + +######################################## +# 2. Runtime stage +######################################## FROM python:3.10-slim RUN useradd -m ffapi + WORKDIR /home/ffapi/app -RUN apt-get update && apt-get install -y --no-install-recommends \ - ssh gcc libsm6 libxext6 libxrender1 && rm -rf /var/lib/apt/lists/* +# Copy over static FFmpeg binaries +COPY --from=builder /usr/local/bin/ffmpeg /usr/local/bin/ffmpeg +COPY --from=builder /usr/local/bin/ffprobe /usr/local/bin/ffprobe +COPY --from=builder /usr/local/bin/ffplay /usr/local/bin/ffplay + +# Copy Python packages +COPY --from=builder /usr/local/lib/python3.10/site-packages /usr/local/lib/python3.10/site-packages -COPY requirements.txt ./ -RUN pip install --no-cache-dir -r requirements.txt +# Copy application code COPY . . + +RUN chown -R ffapi:ffapi /home/ffapi/app + USER ffapi EXPOSE 8000 -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] + +ENTRYPOINT ["uvicorn","app.main:app","--host","0.0.0.0","--port","8000"] diff --git a/README.md b/README.md index b9b143a..86bf459 100644 --- a/README.md +++ b/README.md @@ -1 +1,315 @@ -*(See detailed README.md in repo for setup, API docs, examples)* \ No newline at end of file +# FFmpeg API Service + +> **Open‑source**, **production‑grade**, Dockerized REST API exposing FFmpeg, ffprobe and quality‑metrics (VMAF, PSNR, SSIM) for video, image and audio workflows. + + +## Features + +- **Authentication**: JWT tokens via `/api/v1/auth/token` +- **Asynchronous Jobs**: Job IDs, status, logs, errors, elapsed time +- **Location Abstraction**: Local filesystem or AWS S3 URLs for input/output +- **Video** + - Transcode to H.264, HEVC, VP9, AV1 + - Quality metrics: VMAF, PSNR, SSIM + - Scaling, frame‑rate conversion, HDR & color‑space + - HLS & DASH packaging + - Burn‑in subtitles (SRT/VTT) + - Thumbnail and preview sprite generation +- **Image** + - Resize, crop, filters (grayscale, blur, etc.) + - Watermark overlay + - Format conversion: JPEG, PNG, WebP, AVIF +- **Audio** + - Convert to AAC, Opus, MP3, WAV + - Bitrate/sample‑rate adjustments, mono/stereo + - Loudness normalization (EBU R128) +- **Invoke FFmpeg Locally or Remotely** via SSH +- **AWS S3 Integration** for scalable storage + +--- + +## Prerequisites + +- **Docker** (Engine & Compose) or **Python 3.10+** +- **FFmpeg**, **ffprobe**, **ffmpeg‑quality‑metrics** installed on host or remote server +- (Optional) AWS credentials with S3 read/write permissions + +--- + +## One‑Time-Setup + +Instead of manually editing `.env`, run the interactive installer: + +```bash +chmod +x setup.py +./setup.py +``` + +You’ll be prompted for: + +- **FFmpeg** binary paths or SSH details +- **AWS S3** access key, secret, region (if using S3) +- **JWT** secret key and expiry +- **API Host**, **Port**, **Worker count** + +This generates a `.env` file consumed by the service. + +--- + +## Configuration + +Copy the example and verify: + +```bash +cp config/example.env .env +# Or edit the generated .env from setup.py +``` + +Key environment variables: + +```ini +# API server +HOST=0.0.0.0 +PORT=8000 +WORKERS=4 + +# FFmpeg binaries +FFMPEG_PATH=/usr/bin/ffmpeg +FFPROBE_PATH=/usr/bin/ffprobe +VMAF_PATH=/usr/local/bin/ffmpeg-quality-metrics +MODE=local # or ssh +# SSH_HOST=... +# SSH_USER=... +# SSH_KEY_PATH=... + +# AWS S3 (optional) +AWS_ACCESS_KEY_ID=... +AWS_SECRET_ACCESS_KEY=... +AWS_REGION=us-east-1 + +# JWT auth +SECRET_KEY=your_secure_key +ALGORITHM=HS256 +ACCESS_TOKEN_EXPIRE_MINUTES=60 +``` + +--- + +## Docker Deployment + +Build and run the container: + +```bash +docker build -t ffmpeg-api-service:latest . +docker run -d --name ffapi \ + --env-file .env \ + -v /usr/bin/ffmpeg:/usr/bin/ffmpeg:ro \ + -p 8000:8000 \ + ffmpeg-api-service:latest +``` + +> **Note**: Mount your ffmpeg binaries into the container if installed on host. + +--- + +## Local Development + +```bash +pip install --upgrade pip +pip install -r requirements.txt +uvicorn app.main:app \ + --host 0.0.0.0 --port 8000 --workers 4 +``` + +--- + +## Running Tests + +Place sample fixtures in `tests/fixtures/`: + +- `test.mp4` (small video) +- `test.jpg` +- `test.wav` + +Then: + +```bash +pytest --maxfail=1 --disable-warnings -q +``` + +--- + +## Authentication + +Obtain a bearer token: + +```bash +curl -X POST http://localhost:8000/api/v1/auth/token \ + -F "username=user@example.com" \ + -F "password=password" +``` + +Response: + +```json +{ "access_token":"", "token_type":"bearer" } +``` + +Include in subsequent requests: + +``` +Authorization: Bearer +``` + +--- + +## API Endpoints + +### Video Endpoints + +#### POST /api/v1/video/transcode + +Async video transcode job. + +**Body**: + +```json +{ + "input": { "local_path": "/tmp/in.mp4", "s3_path": null }, + "output": { "local_path": "/tmp/out.mp4", "s3_path": null }, + "codec": "h264", + "crf": 23, + "preset": "medium" +} +``` + +**Response**: + +```json +{ "job_id": "abcdef123456" } +``` + +#### GET /api/v1/video/jobs/{job_id} + +Check job status. + +**Response**: + +```json +{ + "id": "abcdef123456", + "status": "RUNNING|SUCCESS|FAILED", + "log": "...", + "error": "...", + "time_taken": 12.345 +} +``` + +#### POST /api/v1/video/quality + +Compute quality metrics. + +**Body**: + +```json +{ + "reference": { "local_path":"ref.mp4", "s3_path": null }, + "distorted": { "local_path":"dist.mp4", "s3_path": null }, + "metrics": ["vmaf","psnr","ssim"] +} +``` + +**Response**: + +```json +{ "vmaf":"...", "psnr":"...", "ssim":"..." } +``` + +### Image Endpoints + +#### POST /api/v1/image/process + +Async image processing job. + +**Body**: + +```json +{ + "input": { + "local_path": "tests/fixtures/test.jpg", + "s3_path": null + }, + "output": { + "local_path": "tests/fixtures/out.jpg", + "s3_path": null + }, + "operations": [ + { "type":"resize", "params":{ "width":100, "height":100 } }, + { "type":"filter", "params":{ "name":"grayscale" } } + ] +} +``` + +**Response**: + +```json +{ "job_id":"abcdef123456" } +``` + +#### GET /api/v1/image/jobs/{job_id} + +Check image job status. + +### Audio Endpoints + +#### POST /api/v1/audio/convert + +Async audio conversion job. + +**Body**: + +```json +{ + "input": { "local_path":"in.wav", "s3_path":null }, + "output": { "local_path":"out.aac", "s3_path":null }, + "target_codec":"aac", + "bitrate":"64k", + "sample_rate":44100, + "channels":2 +} +``` + +**Response**: + +```json +{ "job_id":"abcdef123456" } +``` + +#### GET /api/v1/audio/jobs/{job_id} + +Check audio job status. + +--- + +## Continuous Integration + +The repository includes a GitHub Actions workflow (`.github/workflows/ci.yml`) that: + +1. Checks out code +2. Sets up Python 3.10 +3. Installs dependencies & runs tests +4. Builds a multi-arch Docker image +5. Optionally pushes to GitHub Container Registry + +--- + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines. + +--- + +## License + +This project is licensed under the **MIT License**. + diff --git a/app/config.py b/app/config.py index 1a393e7..b45b97c 100644 --- a/app/config.py +++ b/app/config.py @@ -1,35 +1,30 @@ -from pydantic import BaseSettings, Field +from pydantic_settings import BaseSettings +from pydantic import Field from pathlib import Path from typing import Optional class Settings(BaseSettings): - HOST: str - PORT: int - WORKERS: int - - FFMPEG_PATH: Path - FFPROBE_PATH: Path - VMAF_PATH: Path - MODE: str + HOST: str = Field("0.0.0.0") + PORT: int = Field(8000) + WORKERS: int = Field(4) + + FFMPEG_PATH: Path = Field(Path("/usr/bin/ffmpeg")) + FFPROBE_PATH: Path = Field(Path("/usr/bin/ffprobe")) + VMAF_PATH: Path = Field(Path("/usr/local/bin/ffmpeg-quality-metrics")) + MODE: str = Field("local") SSH_HOST: Optional[str] SSH_USER: Optional[str] SSH_KEY_PATH: Optional[Path] AWS_ACCESS_KEY_ID: Optional[str] AWS_SECRET_ACCESS_KEY: Optional[str] - AWS_REGION: str + AWS_REGION: str = Field("us-east-1") - SECRET_KEY: str - ALGORITHM: str - ACCESS_TOKEN_EXPIRE_MINUTES: int + SECRET_KEY: str = Field(...) + ALGORITHM: str = Field("HS256") + ACCESS_TOKEN_EXPIRE_MINUTES: int = Field(60) class Config: env_file = ".env" settings = Settings() - - - - - - diff --git a/app/engine.py b/app/engine.py index e69de29..55a3203 100644 --- a/app/engine.py +++ b/app/engine.py @@ -0,0 +1,56 @@ +from fastapi import HTTPException +from app.utils.ffmpeg_client import run_ffmpeg_command +from app.config import settings + + +def transcode(input_path, output_path, codec, crf, preset): + cmd = ['-y', '-i', input_path, '-c:v', codec, '-preset', preset, '-crf', str(crf), output_path] + res = run_ffmpeg_command(cmd) + if res.returncode != 0: + raise HTTPException(status_code=500, detail=res.stderr.decode()) + return output_path + + +def measure_quality(reference_path, distorted_path, metrics: list): + result = {} + if 'vmaf' in metrics: + cmd = ['-i', distorted_path, '-i', reference_path, '-lavfi', f"libvmaf=model_path={settings.VMAF_PATH}", '-f', 'null', '-'] + r = run_ffmpeg_command(cmd) + if r.returncode != 0: + raise HTTPException(status_code=500, detail=r.stderr.decode()) + result['vmaf'] = r.stderr.decode() + if 'psnr' in metrics: + cmd = ['-i', distorted_path, '-i', reference_path, '-lavfi', 'psnr', '-f', 'null', '-'] + r = run_ffmpeg_command(cmd) + result['psnr'] = r.stderr.decode() + if 'ssim' in metrics: + cmd = ['-i', distorted_path, '-i', reference_path, '-lavfi', 'ssim', '-f', 'null', '-'] + r = run_ffmpeg_command(cmd) + result['ssim'] = r.stderr.decode() + return result + + +def process_image(req, input_path, output_path): + filters = [] + for op in req.operations: + if op.type == 'resize': + filters.append(f"scale={op.params['width']}:{op.params['height']}") + if op.type == 'crop': + filters.append(f"crop={op.params['width']}:{op.params['height']}:{op.params.get('x',0)}:{op.params.get('y',0)}") + if op.type == 'filter': + filters.append(f"{op.params['name']}={op.params.get('args','')}") + if op.type == 'watermark': + filters.append(f"movie={op.params['file']}[wm];[in][wm]overlay={op.params.get('x',10)}:{op.params.get('y',10)}") + cmd = ['-y', '-i', input_path, '-vf', ','.join(filters), output_path] + res = run_ffmpeg_command(cmd) + if res.returncode != 0: + raise HTTPException(status_code=500, detail=res.stderr.decode()) + return output_path + + +def convert_audio(req, input_path, output_path): + cmd = ['-y', '-i', input_path, '-c:a', req.target_codec, '-b:a', req.bitrate, '-ar', str(req.sample_rate), '-ac', str(req.channels), output_path] + res = run_ffmpeg_command(cmd) + if res.returncode != 0: + raise HTTPException(status_code=500, detail=res.stderr.decode()) + return output_path \ No newline at end of file diff --git a/app/routers/audio.py b/app/routers/audio.py index e69de29..9157776 100644 --- a/app/routers/audio.py +++ b/app/routers/audio.py @@ -0,0 +1,36 @@ +from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks +from app.schemas import ImageReq, JobResp, JobStatusResp +from app.engine import process_image +from app.jobs import jobs, Job, JobStatus +from app.auth import get_current_user +from app.utils.s3_utils import fetch, upload +import uuid, time + +router = APIRouter(prefix="/api/v1/image", dependencies=[Depends(get_current_user)]) + +@router.get("/health") +def health(): return {"status":"ok"} + +@router.post("/process", response_model=JobResp) +def api_process_image(req: ImageReq, background_tasks: BackgroundTasks): + job_id=uuid.uuid4().hex; job=Job(job_id); jobs[job_id]=job + def run_job(): + job.status=JobStatus.RUNNING; job.start_time=time.time() + try: + in_path=fetch(req.input) + out_path=req.output.local_path or f"/tmp/{job_id}_out.png" + process_image(req, in_path, out_path) + upload(out_path, req.output) + job.log=f"Image processed to {out_path}"; job.status=JobStatus.SUCCESS + except Exception as e: + job.error=str(e); job.status=JobStatus.FAILED + finally: + job.end_time=time.time() + background_tasks.add_task(run_job) + return {"job_id":job_id} + +@router.get("/jobs/{job_id}", response_model=JobStatusResp) +def get_image_job(job_id): + job=jobs.get(job_id) + if not job: raise HTTPException(404) + return job.to_dict() \ No newline at end of file diff --git a/app/routers/auth.py b/app/routers/auth.py index e69de29..2fb81b7 100644 --- a/app/routers/auth.py +++ b/app/routers/auth.py @@ -0,0 +1,20 @@ +from fastapi import APIRouter, Depends, HTTPException +from fastapi.security import OAuth2PasswordRequestForm +from app.auth import authenticate_user, create_access_token +from app.schemas import Token + +router = APIRouter(prefix="/api/v1/auth") + +@router.post("/token", response_model=Token) +def login(form_data: OAuth2PasswordRequestForm = Depends()): + user = authenticate_user(form_data.username, form_data.password) + if not user: + raise HTTPException(status_code=400, detail="Incorrect username or password") + access_token = create_access_token(data={"sub": user["username"]}) + return {"access_token": access_token, "token_type": "bearer"} + + + + +NEW + diff --git a/app/routers/image.py b/app/routers/image.py index e69de29..9157776 100644 --- a/app/routers/image.py +++ b/app/routers/image.py @@ -0,0 +1,36 @@ +from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks +from app.schemas import ImageReq, JobResp, JobStatusResp +from app.engine import process_image +from app.jobs import jobs, Job, JobStatus +from app.auth import get_current_user +from app.utils.s3_utils import fetch, upload +import uuid, time + +router = APIRouter(prefix="/api/v1/image", dependencies=[Depends(get_current_user)]) + +@router.get("/health") +def health(): return {"status":"ok"} + +@router.post("/process", response_model=JobResp) +def api_process_image(req: ImageReq, background_tasks: BackgroundTasks): + job_id=uuid.uuid4().hex; job=Job(job_id); jobs[job_id]=job + def run_job(): + job.status=JobStatus.RUNNING; job.start_time=time.time() + try: + in_path=fetch(req.input) + out_path=req.output.local_path or f"/tmp/{job_id}_out.png" + process_image(req, in_path, out_path) + upload(out_path, req.output) + job.log=f"Image processed to {out_path}"; job.status=JobStatus.SUCCESS + except Exception as e: + job.error=str(e); job.status=JobStatus.FAILED + finally: + job.end_time=time.time() + background_tasks.add_task(run_job) + return {"job_id":job_id} + +@router.get("/jobs/{job_id}", response_model=JobStatusResp) +def get_image_job(job_id): + job=jobs.get(job_id) + if not job: raise HTTPException(404) + return job.to_dict() \ No newline at end of file diff --git a/app/routers/video.py b/app/routers/video.py index e69de29..a4d38cf 100644 --- a/app/routers/video.py +++ b/app/routers/video.py @@ -0,0 +1,43 @@ +from fastapi import APIRouter, Depends, HTTPException, BackgroundTasks +from app.schemas import TranscodeReq, QualityReq, JobResp, JobStatusResp +from app.engine import transcode, measure_quality +from app.jobs import jobs, Job, JobStatus +from app.auth import get_current_user +from app.utils.s3_utils import fetch, upload +import uuid, time + +router = APIRouter(prefix="/api/v1/video", dependencies=[Depends(get_current_user)]) + +@router.get("/health") +def health(): + return {"status": "ok"} + +@router.post("/transcode", response_model=JobResp) +def api_transcode(req: TranscodeReq, background_tasks: BackgroundTasks): + job_id = uuid.uuid4().hex + job = Job(job_id); jobs[job_id]=job + def run_job(): + job.status = JobStatus.RUNNING; job.start_time = time.time() + try: + in_path = fetch(req.input) + out_path = req.output.local_path or f"/tmp/{job_id}_out.{req.output.s3_path.split('.')[-1]}" + transcode(in_path, out_path, req.codec, req.crf, req.preset) + upload(out_path, req.output) + job.log = f"Encoded to {out_path}"; job.status = JobStatus.SUCCESS + except Exception as e: + job.error = str(e); job.status = JobStatus.FAILED + finally: + job.end_time = time.time() + background_tasks.add_task(run_job) + return {"job_id": job_id} + +@router.get("/jobs/{job_id}", response_model=JobStatusResp) +def get_job(job_id: str): + job = jobs.get(job_id) + if not job: raise HTTPException(404, "Job not found") + return job.to_dict() + +@router.post("/quality") +def api_quality(req: QualityReq): + ref = fetch(req.reference); dist = fetch(req.distorted) + return measure_quality(ref, dist, req.metrics) \ No newline at end of file diff --git a/app/utils/s3_utils.py b/app/utils/s3_utils.py index e69de29..3520ca0 100644 --- a/app/utils/s3_utils.py +++ b/app/utils/s3_utils.py @@ -0,0 +1,25 @@ +import boto3 +import os +from urllib.parse import urlparse +from app.config import settings + +s3 = boto3.client('s3', + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + region_name=settings.AWS_REGION +) + +def fetch(loc): + if loc.local_path: + return loc.local_path + if loc.s3_path: + p = urlparse(loc.s3_path) + out = f"/tmp/{os.path.basename(p.path)}" + s3.download_file(p.netloc, p.path.lstrip('/'), out) + return out + raise ValueError("Invalid location") + +def upload(local, loc): + if loc.s3_path: + p = urlparse(loc.s3_path) + s3.upload_file(local, p.netloc, p.path.lstrip('/')) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 704795f..b594175 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,10 @@ -fastapi==0.95.1 +fastapi==0.115.12 uvicorn[standard]==0.23.2 -pydantic==2.1.1 +pydantic==2.8.1 +pydantic-settings==2.9.1 python-multipart==0.0.6 requests==2.31.0 boto3==1.26.145 python-jose==3.3.0 +httpx==0.24.1 +pytest==7.4.0 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/fixtures/test.jpg b/tests/fixtures/test.jpg new file mode 100644 index 0000000..454d350 Binary files /dev/null and b/tests/fixtures/test.jpg differ diff --git a/tests/fixtures/test.mp4 b/tests/fixtures/test.mp4 new file mode 100644 index 0000000..7936dc0 Binary files /dev/null and b/tests/fixtures/test.mp4 differ diff --git a/tests/fixtures/test.wav b/tests/fixtures/test.wav new file mode 100644 index 0000000..e643750 Binary files /dev/null and b/tests/fixtures/test.wav differ