Compare commits
46 Commits
2a0ff98834
..
0.0.9
| Author | SHA1 | Date | |
|---|---|---|---|
| 4c03df6417 | |||
| 7dab37c0cf | |||
| 98d89c3d28 | |||
| 2fadb08bcf | |||
| 5f7373a45c | |||
| 0782ef56eb | |||
| 030d04dbe1 | |||
| a3eb2d0d7a | |||
| 90b71fc7e1 | |||
| caeb066f21 | |||
| 0f374b1d10 | |||
| 1a69fcfd04 | |||
| a3defe5cf6 | |||
| 16d6a95058 | |||
| 28cb50492c | |||
| dc525fbaa4 | |||
| 5303f01f7c | |||
| bc39fd438b | |||
| 4c7f168bce | |||
| 6833bb4013 | |||
| f9111ebac4 | |||
| 34445dc5cd | |||
| 63b8ff34c1 | |||
| 2deb47e5ad | |||
| d3c5ee0dba | |||
| bb57ce8659 | |||
| 1a8836e6aa | |||
| b51b96a618 | |||
| 8e561922c9 | |||
| 1c4bd78f56 | |||
| 3f2982ddb3 | |||
| 0b852c5087 | |||
| 5e56c7f0e8 | |||
| 0a8f40b9cb | |||
| 7440511740 | |||
| c7c8762164 | |||
| 94490237fe | |||
| edc8ea160c | |||
| 11e7552b5b | |||
| 690e6b13f8 | |||
| a53ea4dce7 | |||
| 3217bbfe4d | |||
| 4ac05d50a1 | |||
| a029714e08 | |||
| f9ff4d0138 | |||
| f7e1766d8b |
@@ -0,0 +1,28 @@
|
|||||||
|
name: Release Tag
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- '*'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
release:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: write
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@main
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
- run: git fetch --force --tags
|
||||||
|
- uses: actions/setup-go@main
|
||||||
|
with:
|
||||||
|
go-version-file: 'go.mod'
|
||||||
|
- uses: goreleaser/goreleaser-action@master
|
||||||
|
with:
|
||||||
|
distribution: goreleaser
|
||||||
|
version: 'latest'
|
||||||
|
args: release
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||||
|
GORELEASER_FORCE_TOKEN: gitea
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
name: CI
|
||||||
|
on:
|
||||||
|
pull_request:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- master
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
check-and-test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-go@v5
|
||||||
|
with:
|
||||||
|
go-version-file: 'go.mod'
|
||||||
|
- uses: FedericoCarboni/setup-ffmpeg@v3
|
||||||
|
- run: go mod tidy
|
||||||
|
- run: go build ./...
|
||||||
|
- run: go test -race -v -shuffle=on ./...
|
||||||
+6
-10
@@ -27,7 +27,11 @@ go.work
|
|||||||
jiggablend.db
|
jiggablend.db
|
||||||
jiggablend.db.wal
|
jiggablend.db.wal
|
||||||
jiggablend.db-shm
|
jiggablend.db-shm
|
||||||
|
jiggablend.db-journal
|
||||||
|
|
||||||
|
# Log files
|
||||||
|
*.log
|
||||||
|
logs/
|
||||||
# Secrets and configuration
|
# Secrets and configuration
|
||||||
runner-secrets.json
|
runner-secrets.json
|
||||||
runner-secrets-*.json
|
runner-secrets-*.json
|
||||||
@@ -39,16 +43,6 @@ runner-secrets-*.json
|
|||||||
jiggablend-storage/
|
jiggablend-storage/
|
||||||
jiggablend-workspaces/
|
jiggablend-workspaces/
|
||||||
|
|
||||||
# Node.js
|
|
||||||
web/node_modules/
|
|
||||||
web/dist/
|
|
||||||
web/.vite/
|
|
||||||
npm-debug.log*
|
|
||||||
yarn-debug.log*
|
|
||||||
yarn-error.log*
|
|
||||||
pnpm-debug.log*
|
|
||||||
lerna-debug.log*
|
|
||||||
|
|
||||||
# IDE
|
# IDE
|
||||||
.vscode/
|
.vscode/
|
||||||
.idea/
|
.idea/
|
||||||
@@ -61,6 +55,7 @@ lerna-debug.log*
|
|||||||
*.o
|
*.o
|
||||||
*.a
|
*.a
|
||||||
*.so
|
*.so
|
||||||
|
/dist/
|
||||||
|
|
||||||
# Temporary files
|
# Temporary files
|
||||||
*.tmp
|
*.tmp
|
||||||
@@ -69,6 +64,7 @@ lerna-debug.log*
|
|||||||
|
|
||||||
# Logs
|
# Logs
|
||||||
*.log
|
*.log
|
||||||
|
/logs/
|
||||||
|
|
||||||
# OS files
|
# OS files
|
||||||
Thumbs.db
|
Thumbs.db
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
version: 2
|
||||||
|
|
||||||
|
before:
|
||||||
|
hooks:
|
||||||
|
- go mod tidy -v
|
||||||
|
|
||||||
|
builds:
|
||||||
|
- id: default
|
||||||
|
main: ./cmd/jiggablend
|
||||||
|
binary: jiggablend
|
||||||
|
ldflags:
|
||||||
|
- -X jiggablend/version.Version={{.Version}}
|
||||||
|
- -X jiggablend/version.Date={{.Date}}
|
||||||
|
env:
|
||||||
|
- CGO_ENABLED=1
|
||||||
|
goos:
|
||||||
|
- linux
|
||||||
|
goarch:
|
||||||
|
- amd64
|
||||||
|
|
||||||
|
checksum:
|
||||||
|
name_template: "checksums.txt"
|
||||||
|
|
||||||
|
archives:
|
||||||
|
- id: default
|
||||||
|
name_template: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}"
|
||||||
|
formats: tar.gz
|
||||||
|
format_overrides:
|
||||||
|
- goos: windows
|
||||||
|
formats: zip
|
||||||
|
files:
|
||||||
|
- README.md
|
||||||
|
- LICENSE
|
||||||
|
|
||||||
|
changelog:
|
||||||
|
sort: asc
|
||||||
|
filters:
|
||||||
|
exclude:
|
||||||
|
- "^docs:"
|
||||||
|
- "^test:"
|
||||||
|
|
||||||
|
release:
|
||||||
|
name_template: "{{ .ProjectName }}-{{ .Version }}"
|
||||||
|
|
||||||
|
gitea_urls:
|
||||||
|
api: https://git.s1d3sw1ped.com/api/v1
|
||||||
|
download: https://git.s1d3sw1ped.com
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# Contributing
|
||||||
|
|
||||||
|
## Propose changes
|
||||||
|
|
||||||
|
Open a pull request against `develop`. Keep the default branch for releases and
|
||||||
|
stable tips; land work on `develop` first.
|
||||||
|
|
||||||
|
Point at an existing issue when one fits. Prefer a short issue that states the
|
||||||
|
symptom or request before a large PR.
|
||||||
|
|
||||||
|
## Commits
|
||||||
|
|
||||||
|
Subject form:
|
||||||
|
|
||||||
|
```
|
||||||
|
area: Imperative summary
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Area** is a real package, directory, or subsystem token (`ci:`, `docs:`,
|
||||||
|
Go package name). Not a lone filename.
|
||||||
|
- **Imperative** mood: Fix, Add, Remove — not "Fixed" or "This patch…".
|
||||||
|
- No trailing period. Aim ≤ ~70–75 characters for the whole subject.
|
||||||
|
- Not conventional-commits (`feat:` / `fix:` / `chore:` as types).
|
||||||
|
|
||||||
|
Body explains **why**. Establish the problem, then say what you are doing.
|
||||||
|
One logical change per commit; split fix and cleanup.
|
||||||
|
|
||||||
|
## Pull requests
|
||||||
|
|
||||||
|
Title matches the primary commit subject.
|
||||||
|
|
||||||
|
- **What** changed
|
||||||
|
- **Why** (problem and impact)
|
||||||
|
- **Test** (concrete steps; "CI green" alone is weak)
|
||||||
|
|
||||||
|
## Issues and closing
|
||||||
|
|
||||||
|
Cite leftover issues by **full URL**. Gitea closes issues when `#N` appears in
|
||||||
|
merge text, so do not put `#N` in the merge message unless that issue is actually
|
||||||
|
done. Use `Fixes #N` / `Closes #N` only when the leftover work is finished.
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
The MIT License (MIT)
|
||||||
|
|
||||||
|
Copyright © 2026 s1d3sw1ped
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in
|
||||||
|
all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||||
|
THE SOFTWARE.
|
||||||
@@ -1,58 +1,71 @@
|
|||||||
.PHONY: build build-manager build-runner build-web run-manager run-runner run cleanup cleanup-manager cleanup-runner clean test
|
.PHONY: build build-web run run-manager run-runner cleanup cleanup-manager cleanup-runner clean-bin clean-web test help install
|
||||||
|
|
||||||
# Build all
|
# Build the jiggablend binary (includes embedded web UI)
|
||||||
build: clean-bin build-manager build-runner
|
build:
|
||||||
|
@echo "Building with GoReleaser..."
|
||||||
|
goreleaser build --clean --snapshot --single-target
|
||||||
|
@mkdir -p bin
|
||||||
|
@find dist -name jiggablend -type f -exec cp {} bin/jiggablend.new \;
|
||||||
|
@mv -f bin/jiggablend.new bin/jiggablend
|
||||||
|
|
||||||
# Build manager
|
# Cleanup manager logs
|
||||||
build-manager: clean-bin build-web
|
|
||||||
go build -o bin/manager ./cmd/manager
|
|
||||||
|
|
||||||
# Build runner
|
|
||||||
build-runner: clean-bin
|
|
||||||
GOOS=linux GOARCH=amd64 go build -o bin/runner ./cmd/runner
|
|
||||||
|
|
||||||
# Build web UI
|
|
||||||
build-web: clean-web
|
|
||||||
cd web && npm install && npm run build
|
|
||||||
|
|
||||||
# Cleanup manager (database and storage)
|
|
||||||
cleanup-manager:
|
cleanup-manager:
|
||||||
@echo "Cleaning up manager database and storage..."
|
@echo "Cleaning up manager logs..."
|
||||||
@rm -f jiggablend.db 2>/dev/null || true
|
@rm -rf logs/manager.log 2>/dev/null || true
|
||||||
@rm -f jiggablend.db-shm 2>/dev/null || true
|
|
||||||
@rm -f jiggablend.db-wal 2>/dev/null || true
|
|
||||||
@rm -rf jiggablend-storage 2>/dev/null || true
|
|
||||||
@echo "Manager cleanup complete"
|
@echo "Manager cleanup complete"
|
||||||
|
|
||||||
# Cleanup runner (workspaces and secrets)
|
# Cleanup runner logs
|
||||||
cleanup-runner:
|
cleanup-runner:
|
||||||
@echo "Cleaning up runner workspaces and secrets..."
|
@echo "Cleaning up runner logs..."
|
||||||
@rm -rf jiggablend-workspaces jiggablend-workspace* *workspace* runner-secrets*.json 2>/dev/null || true
|
@rm -rf logs/runner*.log 2>/dev/null || true
|
||||||
@echo "Runner cleanup complete"
|
@echo "Runner cleanup complete"
|
||||||
|
|
||||||
# Cleanup both manager and runner
|
# Cleanup both manager and runner logs
|
||||||
cleanup: cleanup-manager cleanup-runner
|
cleanup: cleanup-manager cleanup-runner
|
||||||
|
|
||||||
# Run all parallel
|
# Run manager and runner in parallel (for testing)
|
||||||
run: cleanup-manager cleanup-runner build-manager build-runner
|
run: cleanup build init-test
|
||||||
@echo "Starting manager and runner in parallel..."
|
@echo "Starting manager and runner in parallel..."
|
||||||
@echo "Press Ctrl+C to stop both..."
|
@echo "Press Ctrl+C to stop both..."
|
||||||
@trap 'kill $$MANAGER_PID $$RUNNER_PID 2>/dev/null; exit' INT TERM; \
|
@MANAGER_PID=""; RUNNER_PID=""; INTERRUPTED=0; \
|
||||||
FIXED_REGISTRATION_TOKEN=test-token ENABLE_LOCAL_AUTH=true LOCAL_TEST_EMAIL=test@example.com LOCAL_TEST_PASSWORD=testpassword bin/manager & \
|
cleanup() { \
|
||||||
|
exit_code=$$?; \
|
||||||
|
trap - INT TERM EXIT; \
|
||||||
|
if [ -n "$$RUNNER_PID" ]; then kill -TERM "$$RUNNER_PID" 2>/dev/null || true; fi; \
|
||||||
|
if [ -n "$$MANAGER_PID" ]; then kill -TERM "$$MANAGER_PID" 2>/dev/null || true; fi; \
|
||||||
|
if [ -n "$$MANAGER_PID$$RUNNER_PID" ]; then wait $$MANAGER_PID $$RUNNER_PID 2>/dev/null || true; fi; \
|
||||||
|
if [ "$$INTERRUPTED" -eq 1 ]; then exit 0; fi; \
|
||||||
|
exit $$exit_code; \
|
||||||
|
}; \
|
||||||
|
on_interrupt() { INTERRUPTED=1; cleanup; }; \
|
||||||
|
trap on_interrupt INT TERM; \
|
||||||
|
trap cleanup EXIT; \
|
||||||
|
bin/jiggablend manager -l manager.log & \
|
||||||
MANAGER_PID=$$!; \
|
MANAGER_PID=$$!; \
|
||||||
REGISTRATION_TOKEN=test-token bin/runner & \
|
sleep 2; \
|
||||||
|
bin/jiggablend runner -l runner.log --api-key=jk_r0_test_key_123456789012345678901234567890 & \
|
||||||
RUNNER_PID=$$!; \
|
RUNNER_PID=$$!; \
|
||||||
wait $$MANAGER_PID $$RUNNER_PID
|
wait $$MANAGER_PID $$RUNNER_PID
|
||||||
|
|
||||||
# Run manager
|
# Run manager server
|
||||||
# Note: ENABLE_LOCAL_AUTH enables local user registration/login
|
run-manager: cleanup-manager build init-test
|
||||||
# LOCAL_TEST_EMAIL and LOCAL_TEST_PASSWORD create a test user on startup (if it doesn't exist)
|
bin/jiggablend manager -l manager.log
|
||||||
run-manager: cleanup-manager build-manager
|
|
||||||
FIXED_REGISTRATION_TOKEN=test-token ENABLE_LOCAL_AUTH=true LOCAL_TEST_EMAIL=test@example.com LOCAL_TEST_PASSWORD=testpassword bin/manager
|
|
||||||
|
|
||||||
# Run runner
|
# Run runner
|
||||||
run-runner: cleanup-runner build-runner
|
run-runner: cleanup-runner build
|
||||||
REGISTRATION_TOKEN=test-token bin/runner
|
bin/jiggablend runner -l runner.log --api-key=jk_r0_test_key_123456789012345678901234567890
|
||||||
|
|
||||||
|
# Initialize for testing (local development only — never use these secrets in production)
|
||||||
|
init-test: build
|
||||||
|
@echo "Initializing LOCAL TEST configuration (not for production)..."
|
||||||
|
bin/jiggablend manager config enable localauth
|
||||||
|
bin/jiggablend manager config set fixed-apikey jk_r0_test_key_123456789012345678901234567890 -f -y
|
||||||
|
bin/jiggablend manager config add user test@example.com testpassword --admin -f -y
|
||||||
|
@echo "Test configuration complete (LOCAL DEV ONLY)!"
|
||||||
|
@echo "fixed api key: jk_r0_test_key_123456789012345678901234567890"
|
||||||
|
@echo "test user: test@example.com"
|
||||||
|
@echo "test password: testpassword"
|
||||||
|
@echo "WARNING: fixed API keys are refused when production_mode is enabled."
|
||||||
|
|
||||||
# Clean bin build artifacts
|
# Clean bin build artifacts
|
||||||
clean-bin:
|
clean-bin:
|
||||||
@@ -60,9 +73,44 @@ clean-bin:
|
|||||||
|
|
||||||
# Clean web build artifacts
|
# Clean web build artifacts
|
||||||
clean-web:
|
clean-web:
|
||||||
rm -rf web/dist/
|
@echo "No generated web artifacts to clean."
|
||||||
|
|
||||||
# Run tests
|
# Run tests
|
||||||
test:
|
test:
|
||||||
go test ./... -timeout 30s
|
go test ./... -timeout 30s
|
||||||
|
|
||||||
|
# Show help
|
||||||
|
help:
|
||||||
|
@echo "Jiggablend Build and Run Makefile"
|
||||||
|
@echo ""
|
||||||
|
@echo "Build targets:"
|
||||||
|
@echo " build - Build jiggablend binary with embedded web UI"
|
||||||
|
@echo " build-web - Validate web UI assets (no build required)"
|
||||||
|
@echo ""
|
||||||
|
@echo "Run targets:"
|
||||||
|
@echo " run - Run manager and runner in parallel (for testing)"
|
||||||
|
@echo " run-manager - Run manager server"
|
||||||
|
@echo " run-runner - Run runner with test API key"
|
||||||
|
@echo " init-test - Initialize test configuration (run once)"
|
||||||
|
@echo ""
|
||||||
|
@echo "Cleanup targets:"
|
||||||
|
@echo " cleanup - Clean all logs"
|
||||||
|
@echo " cleanup-manager - Clean manager logs"
|
||||||
|
@echo " cleanup-runner - Clean runner logs"
|
||||||
|
@echo ""
|
||||||
|
@echo "Other targets:"
|
||||||
|
@echo " clean-bin - Clean build artifacts"
|
||||||
|
@echo " clean-web - Clean generated web artifacts (currently none)"
|
||||||
|
@echo " test - Run Go tests"
|
||||||
|
@echo " help - Show this help"
|
||||||
|
@echo ""
|
||||||
|
@echo "CLI Usage:"
|
||||||
|
@echo " jiggablend manager serve - Start the manager server"
|
||||||
|
@echo " jiggablend runner - Start a runner"
|
||||||
|
@echo " jiggablend manager config show - Show configuration"
|
||||||
|
@echo " jiggablend manager config enable localauth"
|
||||||
|
@echo " jiggablend manager config add user --email=x --password=y"
|
||||||
|
@echo " jiggablend manager config add apikey --name=mykey"
|
||||||
|
@echo " jiggablend manager config set fixed-apikey <key>"
|
||||||
|
@echo " jiggablend manager config list users"
|
||||||
|
@echo " jiggablend manager config list apikeys"
|
||||||
|
|||||||
@@ -4,28 +4,41 @@ A distributed Blender render farm system built with Go. The system consists of a
|
|||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
- **Manager**: Central server with REST API, web UI, DuckDB database, and local file storage
|
- **Manager**: Central server with REST API, embedded web UI, SQLite database, and local file storage
|
||||||
- **Runner**: Linux amd64 client that connects to manager, receives jobs, executes Blender renders, and reports back
|
- **Runner**: Linux amd64 client that connects to manager, receives jobs, executes Blender renders, and reports back
|
||||||
|
|
||||||
|
Both manager and runner are part of a single binary (`jiggablend`) with subcommands.
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
- OAuth authentication (Google and Discord)
|
- **Authentication**: OAuth (Google and Discord) and local authentication with user management
|
||||||
- Web-based job submission and monitoring
|
- **Web UI**: Server-rendered Go templates with HTMX fragments for job submission and monitoring
|
||||||
- Distributed rendering across multiple runners
|
- **Distributed Rendering**: Scale across multiple runners with automatic job distribution
|
||||||
- Real-time job progress tracking
|
- **Real-time Updates**: Polling-based UI updates with lightweight HTMX refreshes
|
||||||
- File upload/download for Blender files and rendered outputs
|
- **Video Encoding**: Automatic video encoding from EXR sequences only. EXR→video always uses HDR (HLG, 10-bit); no option to disable. Codecs:
|
||||||
- Runner health monitoring
|
- H.264 (MP4) - HDR (HLG)
|
||||||
|
- AV1 (MP4) - Alpha channel support, HDR
|
||||||
|
- VP9 (WebM) - Alpha channel and HDR
|
||||||
|
- **Output Formats**: EXR frame sequence only, or EXR + video (H.264, AV1, VP9). Blender always renders EXR.
|
||||||
|
- **Blender Version Management**: Support for multiple Blender versions with automatic detection
|
||||||
|
- **Metadata Extraction**: Automatic extraction of scene metadata from Blender files
|
||||||
|
- **Admin Panel**: User and runner management interface
|
||||||
|
- **Runner Management**: API key-based authentication for runners with health monitoring
|
||||||
|
- **HDR**: EXR→video is always encoded as HDR (HLG, 10-bit). There is no option to turn it off; for SDR-only output, download the EXR frames and encode locally.
|
||||||
|
- **Alpha**: Alpha is always preserved in EXR frames. In video, alpha is preserved when present in the EXR for AV1 and VP9; H.264 MP4 does not support alpha.
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
### Manager
|
### Manager
|
||||||
- Go 1.21 or later
|
- Go 1.27.0 or later
|
||||||
- DuckDB (via Go driver)
|
- SQLite (via Go driver)
|
||||||
|
- Blender installed and in PATH (for metadata extraction)
|
||||||
|
- ImageMagick installed (for EXR preview conversion)
|
||||||
|
|
||||||
### Runner
|
### Runner
|
||||||
- Linux amd64
|
- Linux amd64
|
||||||
- Blender installed and in PATH
|
- FFmpeg installed (required for video encoding)
|
||||||
- FFmpeg installed (optional, for video processing)
|
- Able to run Blender (the runner gets the job’s required Blender version from the manager; it does not need Blender pre-installed)
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
@@ -44,44 +57,87 @@ go mod download
|
|||||||
|
|
||||||
### Manager
|
### Manager
|
||||||
|
|
||||||
Set the following environment variables for authentication (optional):
|
Configuration is managed through the CLI using `jiggablend manager config` commands. The configuration is stored in the SQLite database.
|
||||||
|
|
||||||
|
#### Initial Setup
|
||||||
|
|
||||||
|
For testing, use the Makefile helper:
|
||||||
|
```bash
|
||||||
|
make init-test
|
||||||
|
```
|
||||||
|
|
||||||
|
This will:
|
||||||
|
- Enable local authentication
|
||||||
|
- Set a fixed API key for testing: `jk_r0_test_key_123456789012345678901234567890`
|
||||||
|
- Create a test admin user (test@example.com / testpassword)
|
||||||
|
|
||||||
|
#### Manual Configuration
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# OAuth Providers (optional)
|
# Enable local authentication
|
||||||
export GOOGLE_CLIENT_ID="your-google-client-id"
|
jiggablend manager config enable localauth
|
||||||
export GOOGLE_CLIENT_SECRET="your-google-client-secret"
|
|
||||||
export GOOGLE_REDIRECT_URL="http://localhost:8080/api/auth/google/callback"
|
|
||||||
|
|
||||||
export DISCORD_CLIENT_ID="your-discord-client-id"
|
# Add a user
|
||||||
export DISCORD_CLIENT_SECRET="your-discord-client-secret"
|
jiggablend manager config add user <email> <password> --admin
|
||||||
export DISCORD_REDIRECT_URL="http://localhost:8080/api/auth/discord/callback"
|
|
||||||
|
|
||||||
# Local Authentication (optional)
|
# Generate an API key for runners
|
||||||
export ENABLE_LOCAL_AUTH="true"
|
jiggablend manager config add apikey <name> --scope manager
|
||||||
|
|
||||||
# Test User (optional, for testing only)
|
# Set OAuth credentials
|
||||||
# Creates a local user on startup if it doesn't exist
|
jiggablend manager config set google-oauth <client-id> <client-secret> --redirect-url <url>
|
||||||
export LOCAL_TEST_EMAIL="test@example.com"
|
jiggablend manager config set discord-oauth <client-id> <client-secret> --redirect-url <url>
|
||||||
export LOCAL_TEST_PASSWORD="testpassword"
|
|
||||||
|
# View current configuration
|
||||||
|
jiggablend manager config show
|
||||||
|
|
||||||
|
# List users and API keys
|
||||||
|
jiggablend manager config list users
|
||||||
|
jiggablend manager config list apikeys
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### Environment Variables
|
||||||
|
|
||||||
|
You can also use environment variables with the `JIGGABLEND_` prefix:
|
||||||
|
- `JIGGABLEND_PORT` - Server port (default: 8080)
|
||||||
|
- `JIGGABLEND_DB` - Database path (default: jiggablend.db)
|
||||||
|
- `JIGGABLEND_STORAGE` - Storage path (default: ./jiggablend-storage)
|
||||||
|
- `JIGGABLEND_LOG_FILE` - Log file path
|
||||||
|
- `JIGGABLEND_LOG_LEVEL` - Log level (debug, info, warn, error)
|
||||||
|
- `JIGGABLEND_VERBOSE` - Enable verbose logging
|
||||||
|
|
||||||
### Runner
|
### Runner
|
||||||
|
|
||||||
No configuration required. Runner will auto-detect hostname and IP.
|
The runner requires an API key to connect to the manager. The runner will auto-detect hostname and IP.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
|
### Building
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Build the unified binary (includes embedded web UI)
|
||||||
|
make build
|
||||||
|
|
||||||
|
# Or build directly
|
||||||
|
go build -o bin/jiggablend ./cmd/jiggablend
|
||||||
|
|
||||||
|
# Build web UI separately
|
||||||
|
make build-web
|
||||||
|
```
|
||||||
|
|
||||||
### Running the Manager
|
### Running the Manager
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Using make
|
# Using make (includes test setup)
|
||||||
make run-manager
|
make run-manager
|
||||||
|
|
||||||
# Or directly
|
# Or directly
|
||||||
go run ./cmd/manager
|
bin/jiggablend manager
|
||||||
|
|
||||||
# With custom options
|
# With custom options
|
||||||
go run ./cmd/manager -port 8080 -db jiggablend.db -storage ./storage
|
bin/jiggablend manager --port 8080 --db jiggablend.db --storage ./jiggablend-storage --log-file manager.log
|
||||||
|
|
||||||
|
# Using environment variables
|
||||||
|
JIGGABLEND_PORT=8080 JIGGABLEND_DB=jiggablend.db bin/jiggablend manager
|
||||||
```
|
```
|
||||||
|
|
||||||
The manager will start on `http://localhost:8080` by default.
|
The manager will start on `http://localhost:8080` by default.
|
||||||
@@ -89,26 +145,54 @@ The manager will start on `http://localhost:8080` by default.
|
|||||||
### Running a Runner
|
### Running a Runner
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Using make
|
# Using make (uses test API key)
|
||||||
make run-runner
|
make run-runner
|
||||||
|
|
||||||
# Or directly
|
# Or directly (requires API key)
|
||||||
go run ./cmd/runner
|
bin/jiggablend runner --api-key <your-api-key>
|
||||||
|
|
||||||
# With custom options
|
# With custom options
|
||||||
go run ./cmd/runner -manager http://localhost:8080 -name my-runner
|
bin/jiggablend runner --manager http://localhost:8080 --name my-runner --api-key <key> --log-file runner.log
|
||||||
|
|
||||||
|
# Hardware compatibility flag (force CPU)
|
||||||
|
bin/jiggablend runner --api-key <key> --force-cpu-rendering
|
||||||
|
|
||||||
|
# Sandbox Blender with rootless Podman (default; contains job Python/addons; manager I/O stays in the runner)
|
||||||
|
# podman (default) | none (host Blender, no container)
|
||||||
|
bin/jiggablend runner --api-key <key> # sandbox=podman by default
|
||||||
|
bin/jiggablend runner --api-key <key> --sandbox none # disable sandbox
|
||||||
|
# Optional: allow network inside the jail (default off)
|
||||||
|
# bin/jiggablend runner --api-key <key> --sandbox-network
|
||||||
|
|
||||||
|
# Using environment variables
|
||||||
|
JIGGABLEND_MANAGER=http://localhost:8080 JIGGABLEND_API_KEY=<key> bin/jiggablend runner
|
||||||
```
|
```
|
||||||
|
|
||||||
### Building
|
### Blender sandbox notes
|
||||||
|
|
||||||
|
- **Blender versions** are still the manager-served Linux tarballs on the host; they are **bind-mounted** into the container (no per-version Blender images).
|
||||||
|
- **GPU**: host devices/libs are attached from detection (NVIDIA `/dev/nvidia*`, AMD `/dev/kfd`+`/dev/dri`+ROCm paths, Intel DRM). Requires the runner user to already have access to those devices on the host.
|
||||||
|
- **podman**: rootless podman + default thin image `registry.fedoraproject.org/fedora-minimal:41` (override with `--sandbox-image`).
|
||||||
|
|
||||||
|
|
||||||
|
### Render Chunk Size Note
|
||||||
|
|
||||||
|
For one heavy production scene/profile, chunked rendering (`frames 800-804` in one Blender process) was much slower than one-frame tasks:
|
||||||
|
|
||||||
|
- Chunked task (`800-804`): `27m49s` end-to-end (`Task assigned` -> last `Saved`)
|
||||||
|
- Single-frame tasks (`800`, `801`, `802`, `803`, `804`): `15m04s` wall clock total
|
||||||
|
|
||||||
|
In that test, any chunk size greater than `1` caused a major slowdown after the first frame. Fresh installs should already have it set to `1`, but if you see similar performance degradation, try forcing one frame per task (hard reset Blender each frame): `jiggablend manager config set frames-per-render-task 1`. If `1` is worse on your scene/hardware, benchmark and use a higher chunk size instead.
|
||||||
|
|
||||||
|
### Running Both (for Testing)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Build manager
|
# Run manager and runner in parallel
|
||||||
make build-manager
|
make run
|
||||||
|
|
||||||
# Build runner (Linux amd64)
|
|
||||||
make build-runner
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
This will start both the manager and a test runner with a fixed API key.
|
||||||
|
|
||||||
## OAuth Setup
|
## OAuth Setup
|
||||||
|
|
||||||
### Google OAuth
|
### Google OAuth
|
||||||
@@ -118,7 +202,10 @@ make build-runner
|
|||||||
3. Enable Google+ API
|
3. Enable Google+ API
|
||||||
4. Create OAuth 2.0 credentials
|
4. Create OAuth 2.0 credentials
|
||||||
5. Add authorized redirect URI: `http://localhost:8080/api/auth/google/callback`
|
5. Add authorized redirect URI: `http://localhost:8080/api/auth/google/callback`
|
||||||
6. Set environment variables with Client ID and Secret
|
6. Configure using CLI:
|
||||||
|
```bash
|
||||||
|
jiggablend manager config set google-oauth <client-id> <client-secret> --redirect-url http://localhost:8080/api/auth/google/callback
|
||||||
|
```
|
||||||
|
|
||||||
### Discord OAuth
|
### Discord OAuth
|
||||||
|
|
||||||
@@ -126,25 +213,39 @@ make build-runner
|
|||||||
2. Create a new application
|
2. Create a new application
|
||||||
3. Go to OAuth2 section
|
3. Go to OAuth2 section
|
||||||
4. Add redirect URI: `http://localhost:8080/api/auth/discord/callback`
|
4. Add redirect URI: `http://localhost:8080/api/auth/discord/callback`
|
||||||
5. Set environment variables with Client ID and Secret
|
5. Configure using CLI:
|
||||||
|
```bash
|
||||||
|
jiggablend manager config set discord-oauth <client-id> <client-secret> --redirect-url http://localhost:8080/api/auth/discord/callback
|
||||||
|
```
|
||||||
|
|
||||||
## Project Structure
|
## Project Structure
|
||||||
|
|
||||||
```
|
```
|
||||||
jiggablend/
|
jiggablend/
|
||||||
├── cmd/
|
├── cmd/
|
||||||
│ ├── manager/ # Manager server application
|
│ └── jiggablend/ # Unified CLI application
|
||||||
│ └── runner/ # Runner client application
|
│ ├── cmd/ # Cobra command definitions
|
||||||
|
│ └── main.go # Entry point
|
||||||
├── internal/
|
├── internal/
|
||||||
│ ├── api/ # REST API handlers
|
│ ├── auth/ # Authentication (OAuth, local, sessions)
|
||||||
│ ├── auth/ # OAuth authentication
|
│ ├── config/ # Configuration management
|
||||||
│ ├── database/ # DuckDB database models and migrations
|
│ ├── database/ # SQLite database models and migrations
|
||||||
│ ├── queue/ # Job queue management
|
│ ├── logger/ # Logging utilities
|
||||||
│ ├── storage/ # File storage operations
|
│ ├── manager/ # Manager server logic
|
||||||
│ └── runner/ # Runner management logic
|
│ ├── runner/ # Runner client logic
|
||||||
|
│ │ ├── api/ # Manager API client
|
||||||
|
│ │ ├── blender/ # Blender version detection
|
||||||
|
│ │ ├── encoding/ # Video encoding (H.264, AV1, VP9)
|
||||||
|
│ │ ├── tasks/ # Task execution (render, encode, process)
|
||||||
|
│ │ └── workspace/ # Workspace management
|
||||||
|
│ └── storage/ # File storage operations
|
||||||
├── pkg/
|
├── pkg/
|
||||||
│ └── types/ # Shared types and models
|
│ ├── executils/ # Execution utilities
|
||||||
├── web/ # Static web UI files
|
│ ├── scripts/ # Python scripts for Blender
|
||||||
|
│ └── types/ # Shared types and models
|
||||||
|
├── web/ # Embedded templates + static assets
|
||||||
|
│ ├── templates/ # Go HTML templates and partials
|
||||||
|
│ └── static/ # CSS/JS assets
|
||||||
├── go.mod
|
├── go.mod
|
||||||
└── Makefile
|
└── Makefile
|
||||||
```
|
```
|
||||||
@@ -156,27 +257,94 @@ jiggablend/
|
|||||||
- `GET /api/auth/google/callback` - Google OAuth callback
|
- `GET /api/auth/google/callback` - Google OAuth callback
|
||||||
- `GET /api/auth/discord/login` - Initiate Discord OAuth
|
- `GET /api/auth/discord/login` - Initiate Discord OAuth
|
||||||
- `GET /api/auth/discord/callback` - Discord OAuth callback
|
- `GET /api/auth/discord/callback` - Discord OAuth callback
|
||||||
|
- `POST /api/auth/login` - Local authentication login
|
||||||
|
- `POST /api/auth/register` - User registration (if enabled)
|
||||||
- `POST /api/auth/logout` - Logout
|
- `POST /api/auth/logout` - Logout
|
||||||
- `GET /api/auth/me` - Get current user
|
- `GET /api/auth/me` - Get current user
|
||||||
|
- `POST /api/auth/password/change` - Change password
|
||||||
|
|
||||||
### Jobs
|
### Jobs
|
||||||
- `POST /api/jobs` - Create a new job
|
- `POST /api/jobs` - Create a new job
|
||||||
- `GET /api/jobs` - List user's jobs
|
- `GET /api/jobs` - List user's jobs
|
||||||
- `GET /api/jobs/{id}` - Get job details
|
- `GET /api/jobs/{id}` - Get job details
|
||||||
- `DELETE /api/jobs/{id}` - Cancel a job
|
- `DELETE /api/jobs/{id}` - Cancel a job
|
||||||
- `POST /api/jobs/{id}/upload` - Upload job file
|
- `POST /api/jobs/{id}/upload` - Upload job file (Blender file)
|
||||||
- `GET /api/jobs/{id}/files` - List job files
|
- `GET /api/jobs/{id}/files` - List job files
|
||||||
- `GET /api/jobs/{id}/files/{fileId}/download` - Download job file
|
- `GET /api/jobs/{id}/files/{fileId}/download` - Download job file
|
||||||
|
- `GET /api/jobs/{id}/metadata` - Extract metadata from uploaded file
|
||||||
|
- `GET /api/jobs/{id}/outputs` - List job output files
|
||||||
|
|
||||||
### Runners
|
### Blender
|
||||||
- `GET /api/admin/runners` - List all runners (admin only)
|
- `GET /api/blender/versions` - List available Blender versions
|
||||||
- `POST /api/runner/register` - Register a runner (uses registration token)
|
|
||||||
- `POST /api/runner/heartbeat` - Update runner heartbeat (runner authenticated)
|
### Runners (Internal API)
|
||||||
|
- `POST /api/runner/register` - Register a runner (uses API key)
|
||||||
|
- `POST /api/runner/heartbeat` - Update runner heartbeat
|
||||||
- `GET /api/runner/tasks` - Get pending tasks for runner
|
- `GET /api/runner/tasks` - Get pending tasks for runner
|
||||||
- `POST /api/runner/tasks/{id}/complete` - Mark task as complete
|
- `POST /api/runner/tasks/{id}/complete` - Mark task as complete
|
||||||
- `GET /api/runner/files/{jobId}/{fileName}` - Download file for runner
|
- `GET /api/runner/files/{jobId}/{fileName}` - Download file for runner
|
||||||
- `POST /api/runner/files/{jobId}/upload` - Upload file from runner
|
- `POST /api/runner/files/{jobId}/upload` - Upload file from runner
|
||||||
|
|
||||||
|
### Admin (Admin Only)
|
||||||
|
- `GET /api/admin/runners` - List all runners
|
||||||
|
- `GET /api/admin/jobs` - List all jobs
|
||||||
|
- `GET /api/admin/users` - List all users
|
||||||
|
- `GET /api/admin/stats` - System statistics
|
||||||
|
|
||||||
|
### WebSocket
|
||||||
|
- `WS /api/jobs/ws` - Optional API channel for advanced clients
|
||||||
|
- The default web UI uses polling + HTMX for status updates and task views.
|
||||||
|
|
||||||
|
## Output Formats
|
||||||
|
|
||||||
|
The system supports the following output formats. Blender always renders EXR (linear); the chosen format is the deliverable (frames only or frames + video).
|
||||||
|
|
||||||
|
### Deliverable Formats
|
||||||
|
- **EXR** - EXR frame sequence only (no video)
|
||||||
|
- **EXR_264_MP4** - EXR frames + H.264 MP4 (always HDR, HLG)
|
||||||
|
- **EXR_AV1_MP4** - EXR frames + AV1 MP4 (alpha support, always HDR)
|
||||||
|
- **EXR_VP9_WEBM** - EXR frames + VP9 WebM (alpha and HDR)
|
||||||
|
|
||||||
|
Video encoding (EXR→video) is always HDR (HLG, 10-bit); there is no option to output SDR video. For SDR-only, download the EXR frames and encode locally.
|
||||||
|
|
||||||
|
Video encoding features:
|
||||||
|
- 2-pass encoding for optimal quality
|
||||||
|
- EXR→video only (no PNG source); always HLG (HDR), 10-bit, full range
|
||||||
|
- Alpha channel preservation (AV1 and VP9 only)
|
||||||
|
- Software encoding (libx264, libaom-av1, libvpx-vp9)
|
||||||
|
|
||||||
|
## Storage Structure
|
||||||
|
|
||||||
|
The manager uses a local storage directory (default: `./jiggablend-storage`) with the following structure:
|
||||||
|
|
||||||
|
```
|
||||||
|
jiggablend-storage/
|
||||||
|
├── blender-versions/ # Bundled Blender versions
|
||||||
|
│ └── <version>/
|
||||||
|
├── jobs/ # Job context files
|
||||||
|
│ └── <job-id>/
|
||||||
|
│ └── context.tar
|
||||||
|
├── outputs/ # Rendered outputs
|
||||||
|
│ └── <job-id>/
|
||||||
|
├── temp/ # Temporary files
|
||||||
|
└── uploads/ # Uploaded files
|
||||||
|
```
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
### Running Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make test
|
||||||
|
# Or directly
|
||||||
|
go test ./... -timeout 30s
|
||||||
|
```
|
||||||
|
|
||||||
|
### Web UI Development
|
||||||
|
|
||||||
|
The web UI is server-rendered from embedded templates and static assets in `web/templates` and `web/static`.
|
||||||
|
No Node/Vite build step is required.
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
MIT
|
MIT
|
||||||
|
|||||||
@@ -0,0 +1,186 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"jiggablend/internal/auth"
|
||||||
|
"jiggablend/internal/config"
|
||||||
|
"jiggablend/internal/database"
|
||||||
|
"jiggablend/internal/logger"
|
||||||
|
manager "jiggablend/internal/manager"
|
||||||
|
"jiggablend/internal/storage"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
)
|
||||||
|
|
||||||
|
var managerCmd = &cobra.Command{
|
||||||
|
Use: "manager",
|
||||||
|
Short: "Start the Jiggablend manager server",
|
||||||
|
Long: `Start the Jiggablend manager server to coordinate render jobs.`,
|
||||||
|
Run: runManager,
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
rootCmd.AddCommand(managerCmd)
|
||||||
|
|
||||||
|
// Flags with env binding via viper
|
||||||
|
managerCmd.Flags().StringP("port", "p", "8080", "Server port")
|
||||||
|
managerCmd.Flags().String("db", "jiggablend.db", "Database path")
|
||||||
|
managerCmd.Flags().String("storage", "./jiggablend-storage", "Storage path")
|
||||||
|
managerCmd.Flags().StringP("log-file", "l", "", "Log file path (truncated on start, if not set logs only to stdout)")
|
||||||
|
managerCmd.Flags().String("log-level", "info", "Log level (debug, info, warn, error)")
|
||||||
|
managerCmd.Flags().BoolP("verbose", "v", false, "Enable verbose logging (same as --log-level=debug)")
|
||||||
|
|
||||||
|
// Bind flags to viper with JIGGABLEND_ prefix
|
||||||
|
viper.SetEnvPrefix("JIGGABLEND")
|
||||||
|
viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
|
||||||
|
viper.AutomaticEnv()
|
||||||
|
|
||||||
|
viper.BindPFlag("port", managerCmd.Flags().Lookup("port"))
|
||||||
|
viper.BindPFlag("db", managerCmd.Flags().Lookup("db"))
|
||||||
|
viper.BindPFlag("storage", managerCmd.Flags().Lookup("storage"))
|
||||||
|
viper.BindPFlag("log_file", managerCmd.Flags().Lookup("log-file"))
|
||||||
|
viper.BindPFlag("log_level", managerCmd.Flags().Lookup("log-level"))
|
||||||
|
viper.BindPFlag("verbose", managerCmd.Flags().Lookup("verbose"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func runManager(cmd *cobra.Command, args []string) {
|
||||||
|
// Get config values (flags take precedence over env vars)
|
||||||
|
port := viper.GetString("port")
|
||||||
|
dbPath := viper.GetString("db")
|
||||||
|
storagePath := viper.GetString("storage")
|
||||||
|
logFile := viper.GetString("log_file")
|
||||||
|
logLevel := viper.GetString("log_level")
|
||||||
|
verbose := viper.GetBool("verbose")
|
||||||
|
|
||||||
|
// Initialize logger
|
||||||
|
if logFile != "" {
|
||||||
|
if err := logger.InitWithFile(logFile); err != nil {
|
||||||
|
logger.Fatalf("Failed to initialize logger: %v", err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if l := logger.GetDefault(); l != nil {
|
||||||
|
l.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
} else {
|
||||||
|
logger.InitStdout()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set log level
|
||||||
|
if verbose {
|
||||||
|
logger.SetLevel(logger.LevelDebug)
|
||||||
|
} else {
|
||||||
|
logger.SetLevel(logger.ParseLevel(logLevel))
|
||||||
|
}
|
||||||
|
|
||||||
|
if logFile != "" {
|
||||||
|
logger.Infof("Logging to file: %s", logFile)
|
||||||
|
}
|
||||||
|
logger.Debugf("Log level: %s", logLevel)
|
||||||
|
|
||||||
|
// Initialize database
|
||||||
|
db, err := database.NewDB(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatalf("Failed to initialize database: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
// Initialize config from database
|
||||||
|
cfg := config.NewConfig(db)
|
||||||
|
if err := cfg.InitializeFromEnv(); err != nil {
|
||||||
|
logger.Fatalf("Failed to initialize config: %v", err)
|
||||||
|
}
|
||||||
|
logger.Info("Configuration loaded from database")
|
||||||
|
|
||||||
|
// Initialize auth
|
||||||
|
authHandler, err := auth.NewAuth(db, cfg)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatalf("Failed to initialize auth: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize storage
|
||||||
|
storageHandler, err := storage.NewStorage(storagePath)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatalf("Failed to initialize storage: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if Blender is available
|
||||||
|
if err := checkBlenderAvailable(); err != nil {
|
||||||
|
logger.Fatalf("Blender is not available: %v\n"+
|
||||||
|
"The manager requires Blender to be installed and in PATH for metadata extraction.\n"+
|
||||||
|
"Please install Blender and ensure it's accessible via the 'blender' command.", err)
|
||||||
|
}
|
||||||
|
logger.Info("Blender is available")
|
||||||
|
|
||||||
|
// Check if ImageMagick is available
|
||||||
|
if err := checkImageMagickAvailable(); err != nil {
|
||||||
|
logger.Fatalf("ImageMagick is not available: %v\n"+
|
||||||
|
"The manager requires ImageMagick to be installed and in PATH for EXR preview conversion.\n"+
|
||||||
|
"Please install ImageMagick and ensure 'magick' or 'convert' command is accessible.", err)
|
||||||
|
}
|
||||||
|
logger.Info("ImageMagick is available")
|
||||||
|
|
||||||
|
// Create manager server
|
||||||
|
server, err := manager.NewManager(db, cfg, authHandler, storageHandler)
|
||||||
|
if err != nil {
|
||||||
|
logger.Fatalf("Failed to create server: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start server
|
||||||
|
addr := fmt.Sprintf(":%s", port)
|
||||||
|
logger.Infof("Starting manager server on %s", addr)
|
||||||
|
logger.Infof("Database: %s", dbPath)
|
||||||
|
logger.Infof("Storage: %s", storagePath)
|
||||||
|
|
||||||
|
httpServer := &http.Server{
|
||||||
|
Addr: addr,
|
||||||
|
Handler: server,
|
||||||
|
MaxHeaderBytes: 1 << 20,
|
||||||
|
ReadTimeout: 0,
|
||||||
|
WriteTimeout: 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := httpServer.ListenAndServe(); err != nil {
|
||||||
|
logger.Fatalf("Server failed: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkBlenderAvailable() error {
|
||||||
|
blenderPath, err := exec.LookPath("blender")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to locate blender in PATH: %w", err)
|
||||||
|
}
|
||||||
|
blenderPath, err = filepath.Abs(blenderPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to resolve blender path %q: %w", blenderPath, err)
|
||||||
|
}
|
||||||
|
cmd := exec.Command(blenderPath, "--version")
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to run 'blender --version': %w (output: %s)", err, string(output))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkImageMagickAvailable() error {
|
||||||
|
// Try 'magick' first (ImageMagick 7+)
|
||||||
|
cmd := exec.Command("magick", "--version")
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to 'convert' (ImageMagick 6 or legacy mode)
|
||||||
|
cmd = exec.Command("convert", "--version")
|
||||||
|
output, err = cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to run 'magick --version' or 'convert --version': %w (output: %s)", err, string(output))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,642 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"jiggablend/internal/config"
|
||||||
|
"jiggablend/internal/database"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"golang.org/x/crypto/bcrypt"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
configDBPath string
|
||||||
|
configYes bool // Auto-confirm prompts
|
||||||
|
configForce bool // Force override existing
|
||||||
|
)
|
||||||
|
|
||||||
|
var configCmd = &cobra.Command{
|
||||||
|
Use: "config",
|
||||||
|
Short: "Configure the manager",
|
||||||
|
Long: `Configure the Jiggablend manager settings stored in the database.`,
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Enable/Disable commands ---
|
||||||
|
|
||||||
|
var enableCmd = &cobra.Command{
|
||||||
|
Use: "enable",
|
||||||
|
Short: "Enable a feature",
|
||||||
|
}
|
||||||
|
|
||||||
|
var disableCmd = &cobra.Command{
|
||||||
|
Use: "disable",
|
||||||
|
Short: "Disable a feature",
|
||||||
|
}
|
||||||
|
|
||||||
|
var enableLocalAuthCmd = &cobra.Command{
|
||||||
|
Use: "localauth",
|
||||||
|
Short: "Enable local authentication",
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
if err := cfg.SetBool(config.KeyEnableLocalAuth, true); err != nil {
|
||||||
|
exitWithError("Failed to enable local auth: %v", err)
|
||||||
|
}
|
||||||
|
fmt.Println("Local authentication enabled")
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var disableLocalAuthCmd = &cobra.Command{
|
||||||
|
Use: "localauth",
|
||||||
|
Short: "Disable local authentication",
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
if err := cfg.SetBool(config.KeyEnableLocalAuth, false); err != nil {
|
||||||
|
exitWithError("Failed to disable local auth: %v", err)
|
||||||
|
}
|
||||||
|
fmt.Println("Local authentication disabled")
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var enableRegistrationCmd = &cobra.Command{
|
||||||
|
Use: "registration",
|
||||||
|
Short: "Enable user registration",
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
if err := cfg.SetBool(config.KeyRegistrationEnabled, true); err != nil {
|
||||||
|
exitWithError("Failed to enable registration: %v", err)
|
||||||
|
}
|
||||||
|
fmt.Println("User registration enabled")
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var disableRegistrationCmd = &cobra.Command{
|
||||||
|
Use: "registration",
|
||||||
|
Short: "Disable user registration",
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
if err := cfg.SetBool(config.KeyRegistrationEnabled, false); err != nil {
|
||||||
|
exitWithError("Failed to disable registration: %v", err)
|
||||||
|
}
|
||||||
|
fmt.Println("User registration disabled")
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var enableProductionCmd = &cobra.Command{
|
||||||
|
Use: "production",
|
||||||
|
Short: "Enable production mode",
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
if err := cfg.SetBool(config.KeyProductionMode, true); err != nil {
|
||||||
|
exitWithError("Failed to enable production mode: %v", err)
|
||||||
|
}
|
||||||
|
fmt.Println("Production mode enabled")
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var disableProductionCmd = &cobra.Command{
|
||||||
|
Use: "production",
|
||||||
|
Short: "Disable production mode",
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
if err := cfg.SetBool(config.KeyProductionMode, false); err != nil {
|
||||||
|
exitWithError("Failed to disable production mode: %v", err)
|
||||||
|
}
|
||||||
|
fmt.Println("Production mode disabled")
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Add commands ---
|
||||||
|
|
||||||
|
var addCmd = &cobra.Command{
|
||||||
|
Use: "add",
|
||||||
|
Short: "Add a resource",
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
addUserName string
|
||||||
|
addUserAdmin bool
|
||||||
|
)
|
||||||
|
|
||||||
|
var addUserCmd = &cobra.Command{
|
||||||
|
Use: "user <email> <password>",
|
||||||
|
Short: "Add a local user",
|
||||||
|
Long: `Add a new local user account to the database.`,
|
||||||
|
Args: cobra.ExactArgs(2),
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
email := args[0]
|
||||||
|
password := args[1]
|
||||||
|
|
||||||
|
name := addUserName
|
||||||
|
if name == "" {
|
||||||
|
// Use email prefix as name
|
||||||
|
if atIndex := strings.Index(email, "@"); atIndex > 0 {
|
||||||
|
name = email[:atIndex]
|
||||||
|
} else {
|
||||||
|
name = email
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(password) < 8 {
|
||||||
|
exitWithError("Password must be at least 8 characters")
|
||||||
|
}
|
||||||
|
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
// Check if user exists
|
||||||
|
var exists bool
|
||||||
|
err := db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM users WHERE email = ?)", email).Scan(&exists)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to check user: %v", err)
|
||||||
|
}
|
||||||
|
isAdmin := addUserAdmin
|
||||||
|
if exists {
|
||||||
|
if !configForce {
|
||||||
|
exitWithError("User with email %s already exists (use -f to override)", email)
|
||||||
|
}
|
||||||
|
// Confirm override
|
||||||
|
if !configYes && !confirm(fmt.Sprintf("User %s already exists. Override?", email)) {
|
||||||
|
fmt.Println("Aborted")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Update existing user
|
||||||
|
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to hash password: %v", err)
|
||||||
|
}
|
||||||
|
err = db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec(
|
||||||
|
"UPDATE users SET name = ?, password_hash = ?, is_admin = ? WHERE email = ?",
|
||||||
|
name, string(hashedPassword), isAdmin, email,
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to update user: %v", err)
|
||||||
|
}
|
||||||
|
fmt.Printf("Updated user: %s (admin: %v)\n", email, isAdmin)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hash password
|
||||||
|
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to hash password: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if first user (make admin)
|
||||||
|
var userCount int
|
||||||
|
db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT COUNT(*) FROM users").Scan(&userCount)
|
||||||
|
})
|
||||||
|
if userCount == 0 {
|
||||||
|
isAdmin = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Confirm creation
|
||||||
|
if !configYes && !confirm(fmt.Sprintf("Create user %s (admin: %v)?", email, isAdmin)) {
|
||||||
|
fmt.Println("Aborted")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create user
|
||||||
|
err = db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec(
|
||||||
|
"INSERT INTO users (email, name, oauth_provider, oauth_id, password_hash, is_admin) VALUES (?, ?, 'local', ?, ?, ?)",
|
||||||
|
email, name, email, string(hashedPassword), isAdmin,
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to create user: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Created user: %s (admin: %v)\n", email, isAdmin)
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var addAPIKeyScope string
|
||||||
|
|
||||||
|
var addAPIKeyCmd = &cobra.Command{
|
||||||
|
Use: "apikey [name]",
|
||||||
|
Short: "Add a runner API key",
|
||||||
|
Long: `Generate a new API key for runner authentication.`,
|
||||||
|
Args: cobra.MaximumNArgs(1),
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
name := "cli-generated"
|
||||||
|
if len(args) > 0 {
|
||||||
|
name = args[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
// Check if API key with same name exists
|
||||||
|
var exists bool
|
||||||
|
err := db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM runner_api_keys WHERE name = ?)", name).Scan(&exists)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to check API key: %v", err)
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
if !configForce {
|
||||||
|
exitWithError("API key with name %s already exists (use -f to create another)", name)
|
||||||
|
}
|
||||||
|
if !configYes && !confirm(fmt.Sprintf("API key named '%s' already exists. Create another?", name)) {
|
||||||
|
fmt.Println("Aborted")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Confirm creation
|
||||||
|
if !configYes && !confirm(fmt.Sprintf("Generate new API key '%s' (scope: %s)?", name, addAPIKeyScope)) {
|
||||||
|
fmt.Println("Aborted")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate API key
|
||||||
|
key, keyPrefix, keyHash, err := generateAPIKey()
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to generate API key: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get first user ID for created_by (or use 0 if no users)
|
||||||
|
var createdBy int64
|
||||||
|
db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT id FROM users ORDER BY id ASC LIMIT 1").Scan(&createdBy)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Store in database
|
||||||
|
err = db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec(
|
||||||
|
`INSERT INTO runner_api_keys (key_prefix, key_hash, name, scope, is_active, created_by)
|
||||||
|
VALUES (?, ?, ?, ?, true, ?)`,
|
||||||
|
keyPrefix, keyHash, name, addAPIKeyScope, createdBy,
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to store API key: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Generated API key: %s\n", key)
|
||||||
|
fmt.Printf("Name: %s, Scope: %s\n", name, addAPIKeyScope)
|
||||||
|
fmt.Println("\nSave this key - it cannot be retrieved later!")
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Set commands ---
|
||||||
|
|
||||||
|
var setCmd = &cobra.Command{
|
||||||
|
Use: "set",
|
||||||
|
Short: "Set a configuration value",
|
||||||
|
}
|
||||||
|
|
||||||
|
var setFixedAPIKeyCmd = &cobra.Command{
|
||||||
|
Use: "fixed-apikey [key]",
|
||||||
|
Short: "Set a fixed API key for testing",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
// Check if already set
|
||||||
|
existing := cfg.FixedAPIKey()
|
||||||
|
if existing != "" && !configForce {
|
||||||
|
exitWithError("Fixed API key already set (use -f to override)")
|
||||||
|
}
|
||||||
|
if existing != "" && !configYes && !confirm("Fixed API key already set. Override?") {
|
||||||
|
fmt.Println("Aborted")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := cfg.Set(config.KeyFixedAPIKey, args[0]); err != nil {
|
||||||
|
exitWithError("Failed to set fixed API key: %v", err)
|
||||||
|
}
|
||||||
|
fmt.Println("Fixed API key set")
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var setAllowedOriginsCmd = &cobra.Command{
|
||||||
|
Use: "allowed-origins [origins]",
|
||||||
|
Short: "Set allowed CORS origins (comma-separated)",
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
if err := cfg.Set(config.KeyAllowedOrigins, args[0]); err != nil {
|
||||||
|
exitWithError("Failed to set allowed origins: %v", err)
|
||||||
|
}
|
||||||
|
fmt.Printf("Allowed origins set to: %s\n", args[0])
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var setGoogleOAuthRedirectURL string
|
||||||
|
|
||||||
|
var setGoogleOAuthCmd = &cobra.Command{
|
||||||
|
Use: "google-oauth <client-id> <client-secret>",
|
||||||
|
Short: "Set Google OAuth credentials",
|
||||||
|
Args: cobra.ExactArgs(2),
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
clientID := args[0]
|
||||||
|
clientSecret := args[1]
|
||||||
|
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
// Check if already configured
|
||||||
|
existing := cfg.GoogleClientID()
|
||||||
|
if existing != "" && !configForce {
|
||||||
|
exitWithError("Google OAuth already configured (use -f to override)")
|
||||||
|
}
|
||||||
|
if existing != "" && !configYes && !confirm("Google OAuth already configured. Override?") {
|
||||||
|
fmt.Println("Aborted")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := cfg.Set(config.KeyGoogleClientID, clientID); err != nil {
|
||||||
|
exitWithError("Failed to set Google client ID: %v", err)
|
||||||
|
}
|
||||||
|
if err := cfg.Set(config.KeyGoogleClientSecret, clientSecret); err != nil {
|
||||||
|
exitWithError("Failed to set Google client secret: %v", err)
|
||||||
|
}
|
||||||
|
if setGoogleOAuthRedirectURL != "" {
|
||||||
|
if err := cfg.Set(config.KeyGoogleRedirectURL, setGoogleOAuthRedirectURL); err != nil {
|
||||||
|
exitWithError("Failed to set Google redirect URL: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println("Google OAuth configured")
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var setDiscordOAuthRedirectURL string
|
||||||
|
|
||||||
|
var setFramesPerRenderTaskCmd = &cobra.Command{
|
||||||
|
Use: "frames-per-render-task <n>",
|
||||||
|
Short: "Set number of frames per render task (min 1)",
|
||||||
|
Long: `Set how many frames to batch into each render task. Job frame range is divided into chunks of this size. Default is 10.`,
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
n, err := strconv.Atoi(args[0])
|
||||||
|
if err != nil || n < 1 {
|
||||||
|
exitWithError("frames-per-render-task must be a positive integer")
|
||||||
|
}
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
if err := cfg.SetInt(config.KeyFramesPerRenderTask, n); err != nil {
|
||||||
|
exitWithError("Failed to set frames_per_render_task: %v", err)
|
||||||
|
}
|
||||||
|
fmt.Printf("Frames per render task set to %d\n", n)
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var setDiscordOAuthCmd = &cobra.Command{
|
||||||
|
Use: "discord-oauth <client-id> <client-secret>",
|
||||||
|
Short: "Set Discord OAuth credentials",
|
||||||
|
Args: cobra.ExactArgs(2),
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
clientID := args[0]
|
||||||
|
clientSecret := args[1]
|
||||||
|
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
// Check if already configured
|
||||||
|
existing := cfg.DiscordClientID()
|
||||||
|
if existing != "" && !configForce {
|
||||||
|
exitWithError("Discord OAuth already configured (use -f to override)")
|
||||||
|
}
|
||||||
|
if existing != "" && !configYes && !confirm("Discord OAuth already configured. Override?") {
|
||||||
|
fmt.Println("Aborted")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := cfg.Set(config.KeyDiscordClientID, clientID); err != nil {
|
||||||
|
exitWithError("Failed to set Discord client ID: %v", err)
|
||||||
|
}
|
||||||
|
if err := cfg.Set(config.KeyDiscordClientSecret, clientSecret); err != nil {
|
||||||
|
exitWithError("Failed to set Discord client secret: %v", err)
|
||||||
|
}
|
||||||
|
if setDiscordOAuthRedirectURL != "" {
|
||||||
|
if err := cfg.Set(config.KeyDiscordRedirectURL, setDiscordOAuthRedirectURL); err != nil {
|
||||||
|
exitWithError("Failed to set Discord redirect URL: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Println("Discord OAuth configured")
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Show command ---
|
||||||
|
|
||||||
|
var showCmd = &cobra.Command{
|
||||||
|
Use: "show",
|
||||||
|
Short: "Show current configuration",
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
all, err := cfg.GetAll()
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to get config: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(all) == 0 {
|
||||||
|
fmt.Println("No configuration stored")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("Current configuration:")
|
||||||
|
fmt.Println("----------------------")
|
||||||
|
for key, value := range all {
|
||||||
|
// Redact sensitive values
|
||||||
|
if strings.Contains(key, "secret") || strings.Contains(key, "api_key") || strings.Contains(key, "password") {
|
||||||
|
fmt.Printf(" %s: [REDACTED]\n", key)
|
||||||
|
} else {
|
||||||
|
fmt.Printf(" %s: %s\n", key, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- List commands ---
|
||||||
|
|
||||||
|
var listCmd = &cobra.Command{
|
||||||
|
Use: "list",
|
||||||
|
Short: "List resources",
|
||||||
|
}
|
||||||
|
|
||||||
|
var listUsersCmd = &cobra.Command{
|
||||||
|
Use: "users",
|
||||||
|
Short: "List all users",
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
var rows *sql.Rows
|
||||||
|
err := db.With(func(conn *sql.DB) error {
|
||||||
|
var err error
|
||||||
|
rows, err = conn.Query("SELECT id, email, name, oauth_provider, is_admin, created_at FROM users ORDER BY id")
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to list users: %v", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
fmt.Printf("%-6s %-30s %-20s %-10s %-6s %s\n", "ID", "Email", "Name", "Provider", "Admin", "Created")
|
||||||
|
fmt.Println(strings.Repeat("-", 100))
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
var email, name, provider string
|
||||||
|
var isAdmin bool
|
||||||
|
var createdAt string
|
||||||
|
if err := rows.Scan(&id, &email, &name, &provider, &isAdmin, &createdAt); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
adminStr := "no"
|
||||||
|
if isAdmin {
|
||||||
|
adminStr = "yes"
|
||||||
|
}
|
||||||
|
fmt.Printf("%-6d %-30s %-20s %-10s %-6s %s\n", id, email, name, provider, adminStr, createdAt[:19])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
var listAPIKeysCmd = &cobra.Command{
|
||||||
|
Use: "apikeys",
|
||||||
|
Short: "List all API keys",
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||||
|
var rows *sql.Rows
|
||||||
|
err := db.With(func(conn *sql.DB) error {
|
||||||
|
var err error
|
||||||
|
rows, err = conn.Query("SELECT id, key_prefix, name, scope, is_active, created_at FROM runner_api_keys ORDER BY id")
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to list API keys: %v", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
fmt.Printf("%-6s %-12s %-20s %-10s %-8s %s\n", "ID", "Prefix", "Name", "Scope", "Active", "Created")
|
||||||
|
fmt.Println(strings.Repeat("-", 80))
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
var prefix, name, scope string
|
||||||
|
var isActive bool
|
||||||
|
var createdAt string
|
||||||
|
if err := rows.Scan(&id, &prefix, &name, &scope, &isActive, &createdAt); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
activeStr := "no"
|
||||||
|
if isActive {
|
||||||
|
activeStr = "yes"
|
||||||
|
}
|
||||||
|
fmt.Printf("%-6d %-12s %-20s %-10s %-8s %s\n", id, prefix, name, scope, activeStr, createdAt[:19])
|
||||||
|
}
|
||||||
|
})
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
managerCmd.AddCommand(configCmd)
|
||||||
|
|
||||||
|
// Global config flags
|
||||||
|
configCmd.PersistentFlags().StringVar(&configDBPath, "db", "jiggablend.db", "Database path")
|
||||||
|
configCmd.PersistentFlags().BoolVarP(&configYes, "yes", "y", false, "Auto-confirm prompts")
|
||||||
|
configCmd.PersistentFlags().BoolVarP(&configForce, "force", "f", false, "Force override existing")
|
||||||
|
|
||||||
|
// Enable/Disable
|
||||||
|
configCmd.AddCommand(enableCmd)
|
||||||
|
configCmd.AddCommand(disableCmd)
|
||||||
|
enableCmd.AddCommand(enableLocalAuthCmd)
|
||||||
|
enableCmd.AddCommand(enableRegistrationCmd)
|
||||||
|
enableCmd.AddCommand(enableProductionCmd)
|
||||||
|
disableCmd.AddCommand(disableLocalAuthCmd)
|
||||||
|
disableCmd.AddCommand(disableRegistrationCmd)
|
||||||
|
disableCmd.AddCommand(disableProductionCmd)
|
||||||
|
|
||||||
|
// Add
|
||||||
|
configCmd.AddCommand(addCmd)
|
||||||
|
addCmd.AddCommand(addUserCmd)
|
||||||
|
addUserCmd.Flags().StringVarP(&addUserName, "name", "n", "", "User display name")
|
||||||
|
addUserCmd.Flags().BoolVarP(&addUserAdmin, "admin", "a", false, "Make user an admin")
|
||||||
|
|
||||||
|
addCmd.AddCommand(addAPIKeyCmd)
|
||||||
|
addAPIKeyCmd.Flags().StringVarP(&addAPIKeyScope, "scope", "s", "manager", "API key scope (manager or user)")
|
||||||
|
|
||||||
|
// Set
|
||||||
|
configCmd.AddCommand(setCmd)
|
||||||
|
setCmd.AddCommand(setFixedAPIKeyCmd)
|
||||||
|
setCmd.AddCommand(setAllowedOriginsCmd)
|
||||||
|
setCmd.AddCommand(setFramesPerRenderTaskCmd)
|
||||||
|
setCmd.AddCommand(setGoogleOAuthCmd)
|
||||||
|
setCmd.AddCommand(setDiscordOAuthCmd)
|
||||||
|
|
||||||
|
setGoogleOAuthCmd.Flags().StringVarP(&setGoogleOAuthRedirectURL, "redirect-url", "r", "", "Google OAuth redirect URL")
|
||||||
|
setDiscordOAuthCmd.Flags().StringVarP(&setDiscordOAuthRedirectURL, "redirect-url", "r", "", "Discord OAuth redirect URL")
|
||||||
|
|
||||||
|
// Show
|
||||||
|
configCmd.AddCommand(showCmd)
|
||||||
|
|
||||||
|
// List
|
||||||
|
configCmd.AddCommand(listCmd)
|
||||||
|
listCmd.AddCommand(listUsersCmd)
|
||||||
|
listCmd.AddCommand(listAPIKeysCmd)
|
||||||
|
}
|
||||||
|
|
||||||
|
// withConfig opens the database and runs the callback with config access
|
||||||
|
func withConfig(fn func(cfg *config.Config, db *database.DB)) {
|
||||||
|
db, err := database.NewDB(configDBPath)
|
||||||
|
if err != nil {
|
||||||
|
exitWithError("Failed to open database: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
cfg := config.NewConfig(db)
|
||||||
|
fn(cfg, db)
|
||||||
|
}
|
||||||
|
|
||||||
|
// generateAPIKey generates a new API key
|
||||||
|
func generateAPIKey() (key, prefix, hash string, err error) {
|
||||||
|
randomBytes := make([]byte, 16)
|
||||||
|
if _, err := rand.Read(randomBytes); err != nil {
|
||||||
|
return "", "", "", err
|
||||||
|
}
|
||||||
|
randomStr := hex.EncodeToString(randomBytes)
|
||||||
|
|
||||||
|
prefixDigit := make([]byte, 1)
|
||||||
|
if _, err := rand.Read(prefixDigit); err != nil {
|
||||||
|
return "", "", "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
prefix = fmt.Sprintf("jk_r%d", prefixDigit[0]%10)
|
||||||
|
key = fmt.Sprintf("%s_%s", prefix, randomStr)
|
||||||
|
|
||||||
|
keyHash := sha256.Sum256([]byte(key))
|
||||||
|
hash = hex.EncodeToString(keyHash[:])
|
||||||
|
|
||||||
|
return key, prefix, hash, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// confirm prompts the user for confirmation
|
||||||
|
func confirm(prompt string) bool {
|
||||||
|
fmt.Printf("%s [y/N]: ", prompt)
|
||||||
|
reader := bufio.NewReader(os.Stdin)
|
||||||
|
response, err := reader.ReadString('\n')
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
response = strings.TrimSpace(strings.ToLower(response))
|
||||||
|
return response == "y" || response == "yes"
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGenerateAPIKey_Format(t *testing.T) {
|
||||||
|
key, prefix, hash, err := generateAPIKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("generateAPIKey failed: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(prefix, "jk_r") {
|
||||||
|
t.Fatalf("unexpected prefix: %q", prefix)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(key, prefix+"_") {
|
||||||
|
t.Fatalf("key does not include prefix: %q", key)
|
||||||
|
}
|
||||||
|
if len(hash) != 64 {
|
||||||
|
t.Fatalf("expected sha256 hex hash length, got %d", len(hash))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var rootCmd = &cobra.Command{
|
||||||
|
Use: "jiggablend",
|
||||||
|
Short: "Jiggablend - Distributed Blender Render Farm",
|
||||||
|
Long: `Jiggablend is a distributed render farm for Blender.
|
||||||
|
|
||||||
|
Run 'jiggablend manager' to start the manager server.
|
||||||
|
Run 'jiggablend runner' to start a render runner.
|
||||||
|
Run 'jiggablend manager config' to configure the manager.`,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute runs the root command
|
||||||
|
func Execute() error {
|
||||||
|
return rootCmd.Execute()
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// Global flags can be added here if needed
|
||||||
|
rootCmd.CompletionOptions.DisableDefaultCmd = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// exitWithError prints an error and exits
|
||||||
|
func exitWithError(msg string, args ...interface{}) {
|
||||||
|
fmt.Fprintf(os.Stderr, "Error: "+msg+"\n", args...)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestRootCommand_HasKeySubcommands(t *testing.T) {
|
||||||
|
names := map[string]bool{}
|
||||||
|
for _, c := range rootCmd.Commands() {
|
||||||
|
names[c.Name()] = true
|
||||||
|
}
|
||||||
|
for _, required := range []string{"manager", "runner", "version"} {
|
||||||
|
if !names[required] {
|
||||||
|
t.Fatalf("expected subcommand %q to be registered", required)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,242 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"jiggablend/internal/logger"
|
||||||
|
"jiggablend/internal/runner"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
)
|
||||||
|
|
||||||
|
var runnerViper = viper.New()
|
||||||
|
|
||||||
|
var runnerCmd = &cobra.Command{
|
||||||
|
Use: "runner",
|
||||||
|
Short: "Start the Jiggablend render runner",
|
||||||
|
Long: `Start the Jiggablend render runner that connects to a manager and processes render tasks.`,
|
||||||
|
Run: runRunner,
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
rootCmd.AddCommand(runnerCmd)
|
||||||
|
|
||||||
|
runnerCmd.Flags().StringP("manager", "m", "http://localhost:8080", "Manager URL")
|
||||||
|
runnerCmd.Flags().StringP("name", "n", "", "Runner name")
|
||||||
|
runnerCmd.Flags().String("hostname", "", "Runner hostname")
|
||||||
|
runnerCmd.Flags().StringP("api-key", "k", "", "API key for authentication")
|
||||||
|
runnerCmd.Flags().StringP("log-file", "l", "", "Log file path (truncated on start, if not set logs only to stdout)")
|
||||||
|
runnerCmd.Flags().String("log-level", "info", "Log level (debug, info, warn, error)")
|
||||||
|
runnerCmd.Flags().BoolP("verbose", "v", false, "Enable verbose logging (same as --log-level=debug)")
|
||||||
|
runnerCmd.Flags().Duration("poll-interval", 5*time.Second, "Job polling interval")
|
||||||
|
runnerCmd.Flags().Bool("force-cpu-rendering", false, "Force CPU rendering for all jobs (disables GPU rendering)")
|
||||||
|
runnerCmd.Flags().Bool("disable-rt", false, "Disable GPU ray tracing acceleration (HIPRT, OptiX, etc.)")
|
||||||
|
runnerCmd.Flags().Int("hip-gpu-sample-batch", 0, "Max samples per GPU render pass on gfx115x (0=disabled; merges batches into one EXR when >0)")
|
||||||
|
runnerCmd.Flags().String("sandbox", "podman", "Blender sandbox backend: podman (default; bind-mounts host Blender tarball + GPU devices) or none")
|
||||||
|
runnerCmd.Flags().Bool("sandbox-network", false, "Allow network inside the sandbox (default: isolated; manager I/O stays in the runner process)")
|
||||||
|
runnerCmd.Flags().String("sandbox-image", "", "Thin OS image for podman backend (default: fedora-minimal; Blender is not inside the image)")
|
||||||
|
|
||||||
|
// Bind flags to viper with JIGGABLEND_ prefix
|
||||||
|
runnerViper.SetEnvPrefix("JIGGABLEND")
|
||||||
|
runnerViper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
|
||||||
|
runnerViper.AutomaticEnv()
|
||||||
|
|
||||||
|
runnerViper.BindPFlag("manager", runnerCmd.Flags().Lookup("manager"))
|
||||||
|
runnerViper.BindPFlag("name", runnerCmd.Flags().Lookup("name"))
|
||||||
|
runnerViper.BindPFlag("hostname", runnerCmd.Flags().Lookup("hostname"))
|
||||||
|
runnerViper.BindPFlag("api_key", runnerCmd.Flags().Lookup("api-key"))
|
||||||
|
runnerViper.BindPFlag("log_file", runnerCmd.Flags().Lookup("log-file"))
|
||||||
|
runnerViper.BindPFlag("log_level", runnerCmd.Flags().Lookup("log-level"))
|
||||||
|
runnerViper.BindPFlag("verbose", runnerCmd.Flags().Lookup("verbose"))
|
||||||
|
runnerViper.BindPFlag("poll_interval", runnerCmd.Flags().Lookup("poll-interval"))
|
||||||
|
runnerViper.BindPFlag("force_cpu_rendering", runnerCmd.Flags().Lookup("force-cpu-rendering"))
|
||||||
|
runnerViper.BindPFlag("disable_rt", runnerCmd.Flags().Lookup("disable-rt"))
|
||||||
|
runnerViper.BindPFlag("hip_gpu_sample_batch", runnerCmd.Flags().Lookup("hip-gpu-sample-batch"))
|
||||||
|
runnerViper.BindPFlag("sandbox", runnerCmd.Flags().Lookup("sandbox"))
|
||||||
|
runnerViper.BindPFlag("sandbox_network", runnerCmd.Flags().Lookup("sandbox-network"))
|
||||||
|
runnerViper.BindPFlag("sandbox_image", runnerCmd.Flags().Lookup("sandbox-image"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func runRunner(cmd *cobra.Command, args []string) {
|
||||||
|
// Get config values (flags take precedence over env vars)
|
||||||
|
managerURL := runnerViper.GetString("manager")
|
||||||
|
name := runnerViper.GetString("name")
|
||||||
|
hostname := runnerViper.GetString("hostname")
|
||||||
|
apiKey := runnerViper.GetString("api_key")
|
||||||
|
logFile := runnerViper.GetString("log_file")
|
||||||
|
logLevel := runnerViper.GetString("log_level")
|
||||||
|
verbose := runnerViper.GetBool("verbose")
|
||||||
|
pollInterval := runnerViper.GetDuration("poll_interval")
|
||||||
|
forceCPURendering := runnerViper.GetBool("force_cpu_rendering")
|
||||||
|
disableRT := runnerViper.GetBool("disable_rt")
|
||||||
|
hipGPUSampleBatch := runnerViper.GetInt("hip_gpu_sample_batch")
|
||||||
|
sandboxBackend := runnerViper.GetString("sandbox")
|
||||||
|
sandboxNetwork := runnerViper.GetBool("sandbox_network")
|
||||||
|
sandboxImage := runnerViper.GetString("sandbox_image")
|
||||||
|
var r *runner.Runner
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if rec := recover(); rec != nil {
|
||||||
|
logger.Errorf("Runner panicked: %v", rec)
|
||||||
|
if r != nil {
|
||||||
|
r.Cleanup()
|
||||||
|
}
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
if hostname == "" {
|
||||||
|
hostname, _ = os.Hostname()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate unique runner ID suffix
|
||||||
|
runnerIDStr := generateShortID()
|
||||||
|
|
||||||
|
// Generate runner name with ID if not provided
|
||||||
|
if name == "" {
|
||||||
|
name = fmt.Sprintf("runner-%s-%s", hostname, runnerIDStr)
|
||||||
|
} else {
|
||||||
|
name = fmt.Sprintf("%s-%s", name, runnerIDStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize logger
|
||||||
|
if logFile != "" {
|
||||||
|
if err := logger.InitWithFile(logFile); err != nil {
|
||||||
|
logger.Fatalf("Failed to initialize logger: %v", err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if l := logger.GetDefault(); l != nil {
|
||||||
|
l.Close()
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
} else {
|
||||||
|
logger.InitStdout()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set log level
|
||||||
|
if verbose {
|
||||||
|
logger.SetLevel(logger.LevelDebug)
|
||||||
|
} else {
|
||||||
|
logger.SetLevel(logger.ParseLevel(logLevel))
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Info("Runner starting up...")
|
||||||
|
if disableRT {
|
||||||
|
logger.Info("GPU ray tracing acceleration disabled (--disable-rt)")
|
||||||
|
}
|
||||||
|
if hipGPUSampleBatch > 0 {
|
||||||
|
logger.Infof("HIP GPU sample batching enabled: %d samples per pass", hipGPUSampleBatch)
|
||||||
|
}
|
||||||
|
logger.Infof("Blender sandbox backend: %s (network=%v)", sandboxBackend, sandboxNetwork)
|
||||||
|
logger.Debugf("Generated runner ID suffix: %s", runnerIDStr)
|
||||||
|
if logFile != "" {
|
||||||
|
logger.Infof("Logging to file: %s", logFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create runner
|
||||||
|
r = runner.NewWithOptions(managerURL, name, hostname, runner.RunnerOptions{
|
||||||
|
ForceCPURendering: forceCPURendering,
|
||||||
|
DisableRT: disableRT,
|
||||||
|
HipGPUSampleBatch: hipGPUSampleBatch,
|
||||||
|
SandboxBackend: sandboxBackend,
|
||||||
|
SandboxNetwork: sandboxNetwork,
|
||||||
|
SandboxImage: sandboxImage,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Check for required tools early to fail fast
|
||||||
|
if err := r.CheckRequiredTools(); err != nil {
|
||||||
|
logger.Fatalf("Required tool check failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up orphaned workspace directories
|
||||||
|
r.Cleanup()
|
||||||
|
|
||||||
|
// Probe capabilities and log them
|
||||||
|
logger.Debug("Probing runner capabilities...")
|
||||||
|
capabilities := r.ProbeCapabilities()
|
||||||
|
capList := []string{}
|
||||||
|
for cap, value := range capabilities {
|
||||||
|
if enabled, ok := value.(bool); ok && enabled {
|
||||||
|
capList = append(capList, cap)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(capList) > 0 {
|
||||||
|
logger.Infof("Detected capabilities: %s", strings.Join(capList, ", "))
|
||||||
|
} else {
|
||||||
|
logger.Warn("No capabilities detected")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register with API key
|
||||||
|
if apiKey == "" {
|
||||||
|
logger.Fatal("API key required (use --api-key or set JIGGABLEND_API_KEY env var)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retry registration with exponential backoff
|
||||||
|
backoff := 1 * time.Second
|
||||||
|
maxBackoff := 30 * time.Second
|
||||||
|
maxRetries := 10
|
||||||
|
retryCount := 0
|
||||||
|
|
||||||
|
var runnerID int64
|
||||||
|
|
||||||
|
for {
|
||||||
|
var err error
|
||||||
|
runnerID, err = r.Register(apiKey)
|
||||||
|
if err == nil {
|
||||||
|
logger.Infof("Registered runner with ID: %d", runnerID)
|
||||||
|
// Detect GPU vendors/backends from host hardware so we only force CPU for Blender < 4.x when using AMD.
|
||||||
|
logger.Info("Detecting GPU backends (AMD/NVIDIA/Intel) from host hardware for Blender < 4.x policy...")
|
||||||
|
r.DetectAndStoreGPUBackends()
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
errMsg := err.Error()
|
||||||
|
if strings.Contains(errMsg, "token error:") {
|
||||||
|
logger.Fatalf("Registration failed (token error): %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
retryCount++
|
||||||
|
if retryCount >= maxRetries {
|
||||||
|
logger.Fatalf("Failed to register runner after %d attempts: %v", maxRetries, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Warnf("Registration failed (attempt %d/%d): %v, retrying in %v", retryCount, maxRetries, err, backoff)
|
||||||
|
time.Sleep(backoff)
|
||||||
|
backoff *= 2
|
||||||
|
if backoff > maxBackoff {
|
||||||
|
backoff = maxBackoff
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Signal handlers
|
||||||
|
sigChan := make(chan os.Signal, 1)
|
||||||
|
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
sig := <-sigChan
|
||||||
|
logger.Infof("Received signal: %v, killing all processes and cleaning up...", sig)
|
||||||
|
r.KillAllProcesses()
|
||||||
|
r.Cleanup()
|
||||||
|
os.Exit(0)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Start polling for jobs
|
||||||
|
logger.Infof("Runner started, polling for jobs (interval: %v)...", pollInterval)
|
||||||
|
r.Start(pollInterval)
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateShortID() string {
|
||||||
|
bytes := make([]byte, 4)
|
||||||
|
if _, err := rand.Read(bytes); err != nil {
|
||||||
|
return fmt.Sprintf("%x", os.Getpid()^int(time.Now().Unix()))
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(bytes)
|
||||||
|
}
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/hex"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGenerateShortID_IsHex8Bytes(t *testing.T) {
|
||||||
|
id := generateShortID()
|
||||||
|
if len(id) != 8 {
|
||||||
|
t.Fatalf("expected 8 hex chars, got %q", id)
|
||||||
|
}
|
||||||
|
if _, err := hex.DecodeString(id); err != nil {
|
||||||
|
t.Fatalf("id should be hex: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"jiggablend/version"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var versionCmd = &cobra.Command{
|
||||||
|
Use: "version",
|
||||||
|
Short: "Print the version information",
|
||||||
|
Long: `Print the version and build date of jiggablend.`,
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
fmt.Printf("jiggablend version %s\n", version.Version)
|
||||||
|
if version.Date != "" {
|
||||||
|
fmt.Printf("Build date: %s\n", version.Date)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
rootCmd.AddCommand(versionCmd)
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestVersionCommand_Metadata(t *testing.T) {
|
||||||
|
if versionCmd.Use != "version" {
|
||||||
|
t.Fatalf("unexpected command use: %q", versionCmd.Use)
|
||||||
|
}
|
||||||
|
if versionCmd.Run == nil {
|
||||||
|
t.Fatal("version command run function should be set")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"jiggablend/cmd/jiggablend/cmd"
|
||||||
|
_ "jiggablend/version"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if err := cmd.Execute(); err != nil {
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestMainPackage_Builds(t *testing.T) {
|
||||||
|
// Smoke test placeholder to keep package main under test compilation.
|
||||||
|
}
|
||||||
|
|
||||||
@@ -1,65 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"jiggablend/internal/api"
|
|
||||||
"jiggablend/internal/auth"
|
|
||||||
"jiggablend/internal/database"
|
|
||||||
"jiggablend/internal/storage"
|
|
||||||
)
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
var (
|
|
||||||
port = flag.String("port", getEnv("PORT", "8080"), "Server port")
|
|
||||||
dbPath = flag.String("db", getEnv("DB_PATH", "jiggablend.db"), "Database path")
|
|
||||||
storagePath = flag.String("storage", getEnv("STORAGE_PATH", "./jiggablend-storage"), "Storage path")
|
|
||||||
)
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
// Initialize database
|
|
||||||
db, err := database.NewDB(*dbPath)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to initialize database: %v", err)
|
|
||||||
}
|
|
||||||
defer db.Close()
|
|
||||||
|
|
||||||
// Initialize auth
|
|
||||||
authHandler, err := auth.NewAuth(db.DB)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to initialize auth: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize storage
|
|
||||||
storageHandler, err := storage.NewStorage(*storagePath)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to initialize storage: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create API server
|
|
||||||
server, err := api.NewServer(db, authHandler, storageHandler)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("Failed to create server: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start server
|
|
||||||
addr := fmt.Sprintf(":%s", *port)
|
|
||||||
log.Printf("Starting manager server on %s", addr)
|
|
||||||
log.Printf("Database: %s", *dbPath)
|
|
||||||
log.Printf("Storage: %s", *storagePath)
|
|
||||||
|
|
||||||
if err := http.ListenAndServe(addr, server); err != nil {
|
|
||||||
log.Fatalf("Server failed: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func getEnv(key, defaultValue string) string {
|
|
||||||
if value := os.Getenv(key); value != "" {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
return defaultValue
|
|
||||||
}
|
|
||||||
@@ -1,221 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/rand"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"os/signal"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"jiggablend/internal/runner"
|
|
||||||
)
|
|
||||||
|
|
||||||
type SecretsFile struct {
|
|
||||||
RunnerID int64 `json:"runner_id"`
|
|
||||||
RunnerSecret string `json:"runner_secret"`
|
|
||||||
ManagerSecret string `json:"manager_secret"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
var (
|
|
||||||
managerURL = flag.String("manager", getEnv("MANAGER_URL", "http://localhost:8080"), "Manager URL")
|
|
||||||
name = flag.String("name", getEnv("RUNNER_NAME", ""), "Runner name")
|
|
||||||
hostname = flag.String("hostname", getEnv("RUNNER_HOSTNAME", ""), "Runner hostname")
|
|
||||||
ipAddress = flag.String("ip", getEnv("RUNNER_IP", ""), "Runner IP address")
|
|
||||||
token = flag.String("token", getEnv("REGISTRATION_TOKEN", ""), "Registration token")
|
|
||||||
secretsFile = flag.String("secrets-file", getEnv("SECRETS_FILE", ""), "Path to secrets file for persistent storage (default: ./runner-secrets.json, or ./runner-secrets-{id}.json if multiple runners)")
|
|
||||||
runnerIDSuffix = flag.String("runner-id", getEnv("RUNNER_ID", ""), "Unique runner ID suffix (auto-generated if not provided)")
|
|
||||||
)
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
if *hostname == "" {
|
|
||||||
*hostname, _ = os.Hostname()
|
|
||||||
}
|
|
||||||
if *ipAddress == "" {
|
|
||||||
*ipAddress = "127.0.0.1"
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate or use provided runner ID suffix
|
|
||||||
runnerIDStr := *runnerIDSuffix
|
|
||||||
if runnerIDStr == "" {
|
|
||||||
runnerIDStr = generateShortID()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Generate runner name with ID if not provided
|
|
||||||
if *name == "" {
|
|
||||||
*name = fmt.Sprintf("runner-%s-%s", *hostname, runnerIDStr)
|
|
||||||
} else {
|
|
||||||
// Append ID to provided name to ensure uniqueness
|
|
||||||
*name = fmt.Sprintf("%s-%s", *name, runnerIDStr)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set default secrets file if not provided - always use current directory
|
|
||||||
if *secretsFile == "" {
|
|
||||||
if *runnerIDSuffix != "" || getEnv("RUNNER_ID", "") != "" {
|
|
||||||
// Multiple runners - use local file with ID
|
|
||||||
*secretsFile = fmt.Sprintf("./runner-secrets-%s.json", runnerIDStr)
|
|
||||||
} else {
|
|
||||||
// Single runner - use local file
|
|
||||||
*secretsFile = "./runner-secrets.json"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
client := runner.NewClient(*managerURL, *name, *hostname, *ipAddress)
|
|
||||||
|
|
||||||
// Probe capabilities once at startup (before any registration attempts)
|
|
||||||
log.Printf("Probing runner capabilities...")
|
|
||||||
client.ProbeCapabilities()
|
|
||||||
capabilities := client.GetCapabilities()
|
|
||||||
capList := []string{}
|
|
||||||
for cap, value := range capabilities {
|
|
||||||
// Only show boolean true capabilities and numeric GPU counts
|
|
||||||
if enabled, ok := value.(bool); ok && enabled {
|
|
||||||
capList = append(capList, cap)
|
|
||||||
} else if count, ok := value.(int); ok && count > 0 {
|
|
||||||
capList = append(capList, fmt.Sprintf("%s=%d", cap, count))
|
|
||||||
} else if count, ok := value.(float64); ok && count > 0 {
|
|
||||||
capList = append(capList, fmt.Sprintf("%s=%.0f", cap, count))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(capList) > 0 {
|
|
||||||
log.Printf("Detected capabilities: %s", strings.Join(capList, ", "))
|
|
||||||
} else {
|
|
||||||
log.Printf("Warning: No capabilities detected")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to load secrets from file
|
|
||||||
var runnerID int64
|
|
||||||
var runnerSecret, managerSecret string
|
|
||||||
if *secretsFile != "" {
|
|
||||||
if secrets, err := loadSecrets(*secretsFile); err == nil {
|
|
||||||
runnerID = secrets.RunnerID
|
|
||||||
runnerSecret = secrets.RunnerSecret
|
|
||||||
managerSecret = secrets.ManagerSecret
|
|
||||||
client.SetSecrets(runnerID, runnerSecret, managerSecret)
|
|
||||||
log.Printf("Loaded secrets from %s", *secretsFile)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no secrets loaded, register with token (with retry logic)
|
|
||||||
if runnerID == 0 {
|
|
||||||
if *token == "" {
|
|
||||||
log.Fatalf("Registration token required (use --token or set REGISTRATION_TOKEN env var)")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Retry registration with exponential backoff
|
|
||||||
backoff := 1 * time.Second
|
|
||||||
maxBackoff := 30 * time.Second
|
|
||||||
maxRetries := 10
|
|
||||||
retryCount := 0
|
|
||||||
|
|
||||||
for {
|
|
||||||
var err error
|
|
||||||
runnerID, runnerSecret, managerSecret, err = client.Register(*token)
|
|
||||||
if err == nil {
|
|
||||||
log.Printf("Registered runner with ID: %d", runnerID)
|
|
||||||
|
|
||||||
// Always save secrets to file (secretsFile is now always set to a default if not provided)
|
|
||||||
secrets := SecretsFile{
|
|
||||||
RunnerID: runnerID,
|
|
||||||
RunnerSecret: runnerSecret,
|
|
||||||
ManagerSecret: managerSecret,
|
|
||||||
}
|
|
||||||
if err := saveSecrets(*secretsFile, secrets); err != nil {
|
|
||||||
log.Printf("Warning: Failed to save secrets to %s: %v", *secretsFile, err)
|
|
||||||
} else {
|
|
||||||
log.Printf("Saved secrets to %s", *secretsFile)
|
|
||||||
}
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if it's a token error (invalid/expired/used token) - shutdown immediately
|
|
||||||
errMsg := err.Error()
|
|
||||||
if strings.Contains(errMsg, "token error:") {
|
|
||||||
log.Fatalf("Registration failed (token error): %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Only retry on connection errors or other retryable errors
|
|
||||||
retryCount++
|
|
||||||
if retryCount >= maxRetries {
|
|
||||||
log.Fatalf("Failed to register runner after %d attempts: %v", maxRetries, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("Registration failed (attempt %d/%d): %v, retrying in %v", retryCount, maxRetries, err, backoff)
|
|
||||||
time.Sleep(backoff)
|
|
||||||
backoff *= 2
|
|
||||||
if backoff > maxBackoff {
|
|
||||||
backoff = maxBackoff
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start WebSocket connection with reconnection
|
|
||||||
go client.ConnectWebSocketWithReconnect()
|
|
||||||
|
|
||||||
// Start heartbeat loop (for WebSocket ping/pong and HTTP fallback)
|
|
||||||
go client.HeartbeatLoop()
|
|
||||||
|
|
||||||
// ProcessTasks is now handled via WebSocket, but kept for HTTP fallback
|
|
||||||
// WebSocket will handle task assignment automatically
|
|
||||||
log.Printf("Runner started, connecting to manager via WebSocket...")
|
|
||||||
|
|
||||||
// Set up signal handlers to kill processes on shutdown
|
|
||||||
sigChan := make(chan os.Signal, 1)
|
|
||||||
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
sig := <-sigChan
|
|
||||||
log.Printf("Received signal: %v, killing all processes and shutting down...", sig)
|
|
||||||
client.KillAllProcesses()
|
|
||||||
os.Exit(0)
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Block forever
|
|
||||||
select {}
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadSecrets(path string) (*SecretsFile, error) {
|
|
||||||
data, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
var secrets SecretsFile
|
|
||||||
if err := json.Unmarshal(data, &secrets); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &secrets, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func saveSecrets(path string, secrets SecretsFile) error {
|
|
||||||
data, err := json.MarshalIndent(secrets, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return os.WriteFile(path, data, 0600)
|
|
||||||
}
|
|
||||||
|
|
||||||
func getEnv(key, defaultValue string) string {
|
|
||||||
if value := os.Getenv(key); value != "" {
|
|
||||||
return value
|
|
||||||
}
|
|
||||||
return defaultValue
|
|
||||||
}
|
|
||||||
|
|
||||||
// generateShortID generates a short random ID (8 hex characters)
|
|
||||||
func generateShortID() string {
|
|
||||||
bytes := make([]byte, 4)
|
|
||||||
if _, err := rand.Read(bytes); err != nil {
|
|
||||||
// Fallback to timestamp-based ID if crypto/rand fails
|
|
||||||
return fmt.Sprintf("%x", os.Getpid()^int(time.Now().Unix()))
|
|
||||||
}
|
|
||||||
return hex.EncodeToString(bytes)
|
|
||||||
}
|
|
||||||
Binary file not shown.
@@ -1,39 +1,37 @@
|
|||||||
module jiggablend
|
module jiggablend
|
||||||
|
|
||||||
go 1.25.4
|
go 1.27.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/go-chi/chi/v5 v5.2.3
|
github.com/go-chi/chi/v5 v5.2.3
|
||||||
github.com/go-chi/cors v1.2.2
|
github.com/go-chi/cors v1.2.2
|
||||||
|
github.com/golang-migrate/migrate/v4 v4.19.0
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/gorilla/websocket v1.5.3
|
github.com/gorilla/websocket v1.5.3
|
||||||
github.com/marcboeker/go-duckdb/v2 v2.4.3
|
github.com/mattn/go-sqlite3 v1.14.32
|
||||||
|
github.com/spf13/cobra v1.10.1
|
||||||
|
github.com/spf13/viper v1.21.0
|
||||||
|
golang.org/x/crypto v0.45.0
|
||||||
golang.org/x/oauth2 v0.33.0
|
golang.org/x/oauth2 v0.33.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
cloud.google.com/go/compute/metadata v0.3.0 // indirect
|
cloud.google.com/go/compute/metadata v0.5.0 // indirect
|
||||||
github.com/apache/arrow-go/v18 v18.4.1 // indirect
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/duckdb/duckdb-go-bindings v0.1.21 // indirect
|
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||||
github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.21 // indirect
|
|
||||||
github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.21 // indirect
|
|
||||||
github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.21 // indirect
|
|
||||||
github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.21 // indirect
|
|
||||||
github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.21 // indirect
|
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||||
github.com/goccy/go-json v0.10.5 // indirect
|
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||||
github.com/google/flatbuffers v25.2.10+incompatible // indirect
|
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||||
github.com/klauspost/compress v1.18.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||||
github.com/marcboeker/go-duckdb/arrowmapping v0.0.21 // indirect
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||||
github.com/marcboeker/go-duckdb/mapping v0.0.21 // indirect
|
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||||
github.com/zeebo/xxh3 v1.0.2 // indirect
|
github.com/spf13/afero v1.15.0 // indirect
|
||||||
golang.org/x/crypto v0.45.0 // indirect
|
github.com/spf13/cast v1.10.0 // indirect
|
||||||
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect
|
github.com/spf13/pflag v1.0.10 // indirect
|
||||||
golang.org/x/mod v0.27.0 // indirect
|
github.com/subosito/gotenv v1.6.0 // indirect
|
||||||
golang.org/x/sync v0.16.0 // indirect
|
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||||
golang.org/x/sys v0.38.0 // indirect
|
golang.org/x/sys v0.38.0 // indirect
|
||||||
golang.org/x/tools v0.36.0 // indirect
|
golang.org/x/text v0.31.0 // indirect
|
||||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,88 +1,79 @@
|
|||||||
cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
|
cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
|
||||||
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
|
cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
|
||||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
|
||||||
github.com/apache/arrow-go/v18 v18.4.1 h1:q/jVkBWCJOB9reDgaIZIdruLQUb1kbkvOnOFezVH1C4=
|
|
||||||
github.com/apache/arrow-go/v18 v18.4.1/go.mod h1:tLyFubsAl17bvFdUAy24bsSvA/6ww95Iqi67fTpGu3E=
|
|
||||||
github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
|
|
||||||
github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g=
|
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/duckdb/duckdb-go-bindings v0.1.21 h1:bOb/MXNT4PN5JBZ7wpNg6hrj9+cuDjWDa4ee9UdbVyI=
|
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||||
github.com/duckdb/duckdb-go-bindings v0.1.21/go.mod h1:pBnfviMzANT/9hi4bg+zW4ykRZZPCXlVuvBWEcZofkc=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.21 h1:Sjjhf2F/zCjPF53c2VXOSKk0PzieMriSoyr5wfvr9d8=
|
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||||
github.com/duckdb/duckdb-go-bindings/darwin-amd64 v0.1.21/go.mod h1:Ezo7IbAfB8NP7CqPIN8XEHKUg5xdRRQhcPPlCXImXYA=
|
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||||
github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.21 h1:IUk0FFUB6dpWLhlN9hY1mmdPX7Hkn3QpyrAmn8pmS8g=
|
|
||||||
github.com/duckdb/duckdb-go-bindings/darwin-arm64 v0.1.21/go.mod h1:eS7m/mLnPQgVF4za1+xTyorKRBuK0/BA44Oy6DgrGXI=
|
|
||||||
github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.21 h1:Qpc7ZE3n6Nwz30KTvaAwI6nGkXjXmMxBTdFpC8zDEYI=
|
|
||||||
github.com/duckdb/duckdb-go-bindings/linux-amd64 v0.1.21/go.mod h1:1GOuk1PixiESxLaCGFhag+oFi7aP+9W8byymRAvunBk=
|
|
||||||
github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.21 h1:eX2DhobAZOgjXkh8lPnKAyrxj8gXd2nm+K71f6KV/mo=
|
|
||||||
github.com/duckdb/duckdb-go-bindings/linux-arm64 v0.1.21/go.mod h1:o7crKMpT2eOIi5/FY6HPqaXcvieeLSqdXXaXbruGX7w=
|
|
||||||
github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.21 h1:hhziFnGV7mpA+v5J5G2JnYQ+UWCCP3NQ+OTvxFX10D8=
|
|
||||||
github.com/duckdb/duckdb-go-bindings/windows-amd64 v0.1.21/go.mod h1:IlOhJdVKUJCAPj3QsDszUo8DVdvp1nBFp4TUJVdw99s=
|
|
||||||
github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
|
github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
|
||||||
github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
|
github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
|
||||||
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
||||||
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
|
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
|
||||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
|
||||||
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
|
|
||||||
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
|
||||||
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
|
|
||||||
github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
|
|
||||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||||
github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
|
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||||
github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
|
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||||
github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
|
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||||
github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
github.com/marcboeker/go-duckdb/arrowmapping v0.0.21 h1:geHnVjlsAJGczSWEqYigy/7ARuD+eBtjd0kLN80SPJQ=
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
github.com/marcboeker/go-duckdb/arrowmapping v0.0.21/go.mod h1:flFTc9MSqQCh2Xm62RYvG3Kyj29h7OtsTb6zUx1CdK8=
|
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||||
github.com/marcboeker/go-duckdb/mapping v0.0.21 h1:6woNXZn8EfYdc9Vbv0qR6acnt0TM1s1eFqnrJZVrqEs=
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
github.com/marcboeker/go-duckdb/mapping v0.0.21/go.mod h1:q3smhpLyv2yfgkQd7gGHMd+H/Z905y+WYIUjrl29vT4=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/marcboeker/go-duckdb/v2 v2.4.3 h1:bHUkphPsAp2Bh/VFEdiprGpUekxBNZiWWtK+Bv/ljRk=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
github.com/marcboeker/go-duckdb/v2 v2.4.3/go.mod h1:taim9Hktg2igHdNBmg5vgTfHAlV26z3gBI0QXQOcuyI=
|
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
|
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
|
github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
|
||||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
|
github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||||
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
|
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8=
|
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||||
github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||||
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
|
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||||
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
|
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||||
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
|
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||||
|
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U=
|
||||||
|
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
||||||
|
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||||
|
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||||
|
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||||
|
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
|
||||||
|
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
|
||||||
|
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
|
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||||
|
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
|
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||||
|
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
|
||||||
|
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||||
|
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||||
|
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||||
|
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||||
|
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||||
|
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||||
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
||||||
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
||||||
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM=
|
|
||||||
golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8=
|
|
||||||
golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
|
|
||||||
golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
|
|
||||||
golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo=
|
golang.org/x/oauth2 v0.33.0 h1:4Q+qn+E5z8gPRJfmRy7C2gGG3T4jIprK6aSYgTXGRpo=
|
||||||
golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
golang.org/x/oauth2 v0.33.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
|
||||||
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
|
|
||||||
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
|
||||||
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
|
|
||||||
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
|
||||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||||
golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
|
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||||
golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
|
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
+130
@@ -0,0 +1,130 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Install the latest jiggablend binary for Linux AMD64 and create wrapper scripts.
|
||||||
|
# Production wrappers do NOT install fixed test secrets. For local test credentials,
|
||||||
|
# use `make init-test` from a development checkout.
|
||||||
|
|
||||||
|
# Dependencies: curl, jq, tar, sha256sum, sudo (for installation to /usr/local/bin)
|
||||||
|
|
||||||
|
REPO="s1d3sw1ped/jiggablend"
|
||||||
|
API_URL="https://git.s1d3sw1ped.com/api/v1/repos/${REPO}/releases/latest"
|
||||||
|
ASSET_NAME="jiggablend-linux-amd64.tar.gz"
|
||||||
|
|
||||||
|
echo "Fetching latest release information..."
|
||||||
|
RELEASE_JSON=$(curl -s "$API_URL")
|
||||||
|
|
||||||
|
TAG=$(echo "$RELEASE_JSON" | jq -r '.tag_name')
|
||||||
|
echo "Latest version: $TAG"
|
||||||
|
|
||||||
|
ASSET_URL=$(echo "$RELEASE_JSON" | jq -r ".assets[] | select(.name == \"$ASSET_NAME\") | .browser_download_url")
|
||||||
|
if [ -z "$ASSET_URL" ]; then
|
||||||
|
echo "Error: Asset $ASSET_NAME not found in latest release."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
CHECKSUM_URL=$(echo "$RELEASE_JSON" | jq -r '.assets[] | select(.name == "checksums.txt") | .browser_download_url')
|
||||||
|
if [ -z "$CHECKSUM_URL" ]; then
|
||||||
|
echo "Error: checksums.txt not found in latest release."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Downloading $ASSET_NAME..."
|
||||||
|
curl -L -o "$ASSET_NAME" "$ASSET_URL"
|
||||||
|
|
||||||
|
echo "Downloading checksums.txt..."
|
||||||
|
curl -L -o "checksums.txt" "$CHECKSUM_URL"
|
||||||
|
|
||||||
|
echo "Verifying checksum..."
|
||||||
|
if ! sha256sum --ignore-missing --quiet -c checksums.txt; then
|
||||||
|
echo "Error: Checksum verification failed."
|
||||||
|
rm -f "$ASSET_NAME" checksums.txt
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "Extracting..."
|
||||||
|
tar -xzf "$ASSET_NAME"
|
||||||
|
|
||||||
|
echo "Installing binary to /usr/local/bin (requires sudo)..."
|
||||||
|
sudo install -m 0755 jiggablend /usr/local/bin/
|
||||||
|
|
||||||
|
echo "Creating manager wrapper script..."
|
||||||
|
cat << 'EOF' > jiggablend-manager.sh
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Wrapper to run jiggablend manager.
|
||||||
|
# Does NOT set fixed/test API keys or default admin passwords.
|
||||||
|
# Bootstrap (one-time, local admin):
|
||||||
|
# jiggablend manager config enable localauth
|
||||||
|
# jiggablend manager config add user you@example.com 'strong-password' --admin
|
||||||
|
# jiggablend manager config add apikey my-runner --scope manager
|
||||||
|
# For local development only: use `make init-test` from a source checkout.
|
||||||
|
|
||||||
|
mkdir -p logs
|
||||||
|
rm -f logs/manager.log
|
||||||
|
|
||||||
|
jiggablend manager -l logs/manager.log
|
||||||
|
EOF
|
||||||
|
chmod +x jiggablend-manager.sh
|
||||||
|
sudo install -m 0755 jiggablend-manager.sh /usr/local/bin/jiggablend-manager
|
||||||
|
rm -f jiggablend-manager.sh
|
||||||
|
|
||||||
|
echo "Creating runner wrapper script..."
|
||||||
|
cat << 'EOF' > jiggablend-runner.sh
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
# Wrapper to run jiggablend runner.
|
||||||
|
# Usage: jiggablend-runner [MANAGER_URL] --api-key <key> [RUNNER_FLAGS...]
|
||||||
|
# Or set JIGGABLEND_API_KEY. Default MANAGER_URL: http://localhost:8080
|
||||||
|
|
||||||
|
MANAGER_URL="http://localhost:8080"
|
||||||
|
if [[ $# -gt 0 && "$1" != -* ]]; then
|
||||||
|
MANAGER_URL="$1"
|
||||||
|
shift
|
||||||
|
fi
|
||||||
|
|
||||||
|
EXTRA_ARGS=("$@")
|
||||||
|
|
||||||
|
API_KEY="${JIGGABLEND_API_KEY:-}"
|
||||||
|
# Allow --api-key in EXTRA_ARGS; if not present and env empty, fail clearly
|
||||||
|
HAS_KEY=0
|
||||||
|
for arg in "${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}"; do
|
||||||
|
case "$arg" in
|
||||||
|
--api-key|--api-key=*|-k) HAS_KEY=1 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if [[ -z "$API_KEY" && "$HAS_KEY" -eq 0 ]]; then
|
||||||
|
echo "Error: provide a runner API key via JIGGABLEND_API_KEY or --api-key." >&2
|
||||||
|
echo "Create one with: jiggablend manager config add apikey <name> --scope manager" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p logs
|
||||||
|
rm -f logs/runner.log
|
||||||
|
|
||||||
|
if [[ -n "$API_KEY" && "$HAS_KEY" -eq 0 ]]; then
|
||||||
|
jiggablend runner -l logs/runner.log --api-key="$API_KEY" --manager "$MANAGER_URL" "${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}"
|
||||||
|
else
|
||||||
|
jiggablend runner -l logs/runner.log --manager "$MANAGER_URL" "${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}"
|
||||||
|
fi
|
||||||
|
EOF
|
||||||
|
chmod +x jiggablend-runner.sh
|
||||||
|
sudo install -m 0755 jiggablend-runner.sh /usr/local/bin/jiggablend-runner
|
||||||
|
rm -f jiggablend-runner.sh
|
||||||
|
|
||||||
|
echo "Cleaning up..."
|
||||||
|
rm -f "$ASSET_NAME" checksums.txt jiggablend
|
||||||
|
|
||||||
|
echo "Installation complete!"
|
||||||
|
echo "Binary: jiggablend"
|
||||||
|
echo "Wrappers: jiggablend-manager, jiggablend-runner"
|
||||||
|
echo "Run 'jiggablend-manager' to start the manager (no fixed test secrets)."
|
||||||
|
echo "Run 'jiggablend-runner [url] --api-key <key>' to start a runner."
|
||||||
|
echo "Local dev only: use 'make init-test' from a source checkout for test credentials."
|
||||||
|
echo "Note: Blender, ImageMagick, or FFmpeg may be required. See README."
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,158 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
|
|
||||||
"jiggablend/pkg/types"
|
|
||||||
)
|
|
||||||
|
|
||||||
// handleSubmitMetadata handles metadata submission from runner
|
|
||||||
func (s *Server) handleSubmitMetadata(w http.ResponseWriter, r *http.Request) {
|
|
||||||
jobID, err := parseID(r, "jobId")
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get runner ID from context (set by runnerAuthMiddleware)
|
|
||||||
runnerID, ok := r.Context().Value(runnerIDContextKey).(int64)
|
|
||||||
if !ok {
|
|
||||||
s.respondError(w, http.StatusUnauthorized, "runner_id not found in context")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var metadata types.BlendMetadata
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "Invalid metadata JSON")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify job exists
|
|
||||||
var jobUserID int64
|
|
||||||
err = s.db.QueryRow("SELECT user_id FROM jobs WHERE id = ?", jobID).Scan(&jobUserID)
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
s.respondError(w, http.StatusNotFound, "Job not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to verify job: %v", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Find the metadata extraction task for this job
|
|
||||||
// First try to find task assigned to this runner, then fall back to any metadata task for this job
|
|
||||||
var taskID int64
|
|
||||||
err = s.db.QueryRow(
|
|
||||||
`SELECT id FROM tasks WHERE job_id = ? AND task_type = ? AND runner_id = ?`,
|
|
||||||
jobID, types.TaskTypeMetadata, runnerID,
|
|
||||||
).Scan(&taskID)
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
// Fall back to any metadata task for this job (in case assignment changed)
|
|
||||||
err = s.db.QueryRow(
|
|
||||||
`SELECT id FROM tasks WHERE job_id = ? AND task_type = ? ORDER BY created_at DESC LIMIT 1`,
|
|
||||||
jobID, types.TaskTypeMetadata,
|
|
||||||
).Scan(&taskID)
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
s.respondError(w, http.StatusNotFound, "Metadata extraction task not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to find task: %v", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// Update the task to be assigned to this runner if it wasn't already
|
|
||||||
s.db.Exec(
|
|
||||||
`UPDATE tasks SET runner_id = ? WHERE id = ? AND runner_id IS NULL`,
|
|
||||||
runnerID, taskID,
|
|
||||||
)
|
|
||||||
} else if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to find task: %v", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Convert metadata to JSON
|
|
||||||
metadataJSON, err := json.Marshal(metadata)
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, "Failed to marshal metadata")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update job with metadata
|
|
||||||
_, err = s.db.Exec(
|
|
||||||
`UPDATE jobs SET blend_metadata = ? WHERE id = ?`,
|
|
||||||
string(metadataJSON), jobID,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to update job metadata: %v", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Mark task as completed
|
|
||||||
_, err = s.db.Exec(
|
|
||||||
`UPDATE tasks SET status = ?, completed_at = CURRENT_TIMESTAMP WHERE id = ?`,
|
|
||||||
types.TaskStatusCompleted, taskID,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed to mark metadata task as completed: %v", err)
|
|
||||||
} else {
|
|
||||||
// Update job status and progress after metadata task completes
|
|
||||||
s.updateJobStatusFromTasks(jobID)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.Printf("Metadata extracted for job %d: frame_start=%d, frame_end=%d", jobID, metadata.FrameStart, metadata.FrameEnd)
|
|
||||||
|
|
||||||
s.respondJSON(w, http.StatusOK, map[string]string{"message": "Metadata submitted successfully"})
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleGetJobMetadata retrieves metadata for a job
|
|
||||||
func (s *Server) handleGetJobMetadata(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, err := getUserID(r)
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusUnauthorized, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
jobID, err := parseID(r, "id")
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify job belongs to user
|
|
||||||
var jobUserID int64
|
|
||||||
var blendMetadataJSON sql.NullString
|
|
||||||
err = s.db.QueryRow(
|
|
||||||
`SELECT user_id, blend_metadata FROM jobs WHERE id = ?`,
|
|
||||||
jobID,
|
|
||||||
).Scan(&jobUserID, &blendMetadataJSON)
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
s.respondError(w, http.StatusNotFound, "Job not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query job: %v", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if jobUserID != userID {
|
|
||||||
s.respondError(w, http.StatusForbidden, "Access denied")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !blendMetadataJSON.Valid || blendMetadataJSON.String == "" {
|
|
||||||
s.respondJSON(w, http.StatusOK, nil)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var metadata types.BlendMetadata
|
|
||||||
if err := json.Unmarshal([]byte(blendMetadataJSON.String), &metadata); err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, "Failed to parse metadata")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
s.respondJSON(w, http.StatusOK, metadata)
|
|
||||||
}
|
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -1,658 +0,0 @@
|
|||||||
package api
|
|
||||||
|
|
||||||
import (
|
|
||||||
"database/sql"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"net/http"
|
|
||||||
"strconv"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
authpkg "jiggablend/internal/auth"
|
|
||||||
"jiggablend/internal/database"
|
|
||||||
"jiggablend/internal/storage"
|
|
||||||
"jiggablend/pkg/types"
|
|
||||||
|
|
||||||
"github.com/go-chi/chi/v5"
|
|
||||||
"github.com/go-chi/chi/v5/middleware"
|
|
||||||
"github.com/go-chi/cors"
|
|
||||||
"github.com/gorilla/websocket"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Server represents the API server
|
|
||||||
type Server struct {
|
|
||||||
db *database.DB
|
|
||||||
auth *authpkg.Auth
|
|
||||||
secrets *authpkg.Secrets
|
|
||||||
storage *storage.Storage
|
|
||||||
router *chi.Mux
|
|
||||||
|
|
||||||
// WebSocket connections
|
|
||||||
wsUpgrader websocket.Upgrader
|
|
||||||
runnerConns map[int64]*websocket.Conn
|
|
||||||
runnerConnsMu sync.RWMutex
|
|
||||||
frontendConns map[string]*websocket.Conn // key: "jobId:taskId"
|
|
||||||
frontendConnsMu sync.RWMutex
|
|
||||||
// Mutexes for each frontend connection to serialize writes
|
|
||||||
frontendConnsWriteMu map[string]*sync.Mutex // key: "jobId:taskId"
|
|
||||||
frontendConnsWriteMuMu sync.RWMutex
|
|
||||||
// Throttling for progress updates (per job)
|
|
||||||
progressUpdateTimes map[int64]time.Time // key: jobID
|
|
||||||
progressUpdateTimesMu sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewServer creates a new API server
|
|
||||||
func NewServer(db *database.DB, auth *authpkg.Auth, storage *storage.Storage) (*Server, error) {
|
|
||||||
secrets, err := authpkg.NewSecrets(db.DB)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to initialize secrets: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
s := &Server{
|
|
||||||
db: db,
|
|
||||||
auth: auth,
|
|
||||||
secrets: secrets,
|
|
||||||
storage: storage,
|
|
||||||
router: chi.NewRouter(),
|
|
||||||
wsUpgrader: websocket.Upgrader{
|
|
||||||
CheckOrigin: func(r *http.Request) bool {
|
|
||||||
return true // Allow all origins for now
|
|
||||||
},
|
|
||||||
ReadBufferSize: 1024,
|
|
||||||
WriteBufferSize: 1024,
|
|
||||||
},
|
|
||||||
runnerConns: make(map[int64]*websocket.Conn),
|
|
||||||
frontendConns: make(map[string]*websocket.Conn),
|
|
||||||
frontendConnsWriteMu: make(map[string]*sync.Mutex),
|
|
||||||
progressUpdateTimes: make(map[int64]time.Time),
|
|
||||||
}
|
|
||||||
|
|
||||||
s.setupMiddleware()
|
|
||||||
s.setupRoutes()
|
|
||||||
s.StartBackgroundTasks()
|
|
||||||
|
|
||||||
return s, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// setupMiddleware configures middleware
|
|
||||||
func (s *Server) setupMiddleware() {
|
|
||||||
s.router.Use(middleware.Logger)
|
|
||||||
s.router.Use(middleware.Recoverer)
|
|
||||||
// Note: Timeout middleware is NOT applied globally to avoid conflicts with WebSocket connections
|
|
||||||
// WebSocket connections are long-lived and should not have HTTP timeouts
|
|
||||||
|
|
||||||
s.router.Use(cors.Handler(cors.Options{
|
|
||||||
AllowedOrigins: []string{"*"},
|
|
||||||
AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"},
|
|
||||||
AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "Range"},
|
|
||||||
ExposedHeaders: []string{"Link", "Content-Range", "Accept-Ranges", "Content-Length"},
|
|
||||||
AllowCredentials: true,
|
|
||||||
MaxAge: 300,
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// setupRoutes configures routes
|
|
||||||
func (s *Server) setupRoutes() {
|
|
||||||
// Public routes
|
|
||||||
s.router.Route("/api/auth", func(r chi.Router) {
|
|
||||||
r.Get("/providers", s.handleGetAuthProviders)
|
|
||||||
r.Get("/google/login", s.handleGoogleLogin)
|
|
||||||
r.Get("/google/callback", s.handleGoogleCallback)
|
|
||||||
r.Get("/discord/login", s.handleDiscordLogin)
|
|
||||||
r.Get("/discord/callback", s.handleDiscordCallback)
|
|
||||||
r.Get("/local/available", s.handleLocalLoginAvailable)
|
|
||||||
r.Post("/local/register", s.handleLocalRegister)
|
|
||||||
r.Post("/local/login", s.handleLocalLogin)
|
|
||||||
r.Post("/logout", s.handleLogout)
|
|
||||||
r.Get("/me", s.handleGetMe)
|
|
||||||
r.Post("/change-password", s.handleChangePassword)
|
|
||||||
})
|
|
||||||
|
|
||||||
// Protected routes
|
|
||||||
s.router.Route("/api/jobs", func(r chi.Router) {
|
|
||||||
r.Use(func(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(s.auth.Middleware(next.ServeHTTP))
|
|
||||||
})
|
|
||||||
r.Post("/", s.handleCreateJob)
|
|
||||||
r.Get("/", s.handleListJobs)
|
|
||||||
r.Get("/{id}", s.handleGetJob)
|
|
||||||
r.Delete("/{id}", s.handleCancelJob)
|
|
||||||
r.Post("/{id}/delete", s.handleDeleteJob)
|
|
||||||
r.Post("/{id}/upload", s.handleUploadJobFile)
|
|
||||||
r.Get("/{id}/files", s.handleListJobFiles)
|
|
||||||
r.Get("/{id}/files/{fileId}/download", s.handleDownloadJobFile)
|
|
||||||
r.Get("/{id}/video", s.handleStreamVideo)
|
|
||||||
r.Get("/{id}/metadata", s.handleGetJobMetadata)
|
|
||||||
r.Get("/{id}/tasks", s.handleListJobTasks)
|
|
||||||
r.Get("/{id}/tasks/{taskId}/logs", s.handleGetTaskLogs)
|
|
||||||
// WebSocket route - no timeout middleware (long-lived connection)
|
|
||||||
r.With(func(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
||||||
// Remove timeout middleware for WebSocket
|
|
||||||
next.ServeHTTP(w, r)
|
|
||||||
})
|
|
||||||
}).Get("/{id}/tasks/{taskId}/logs/ws", s.handleStreamTaskLogsWebSocket)
|
|
||||||
r.Get("/{id}/tasks/{taskId}/steps", s.handleGetTaskSteps)
|
|
||||||
r.Post("/{id}/tasks/{taskId}/retry", s.handleRetryTask)
|
|
||||||
})
|
|
||||||
|
|
||||||
// Admin routes
|
|
||||||
s.router.Route("/api/admin", func(r chi.Router) {
|
|
||||||
r.Use(func(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(s.auth.AdminMiddleware(next.ServeHTTP))
|
|
||||||
})
|
|
||||||
r.Route("/runners", func(r chi.Router) {
|
|
||||||
r.Route("/tokens", func(r chi.Router) {
|
|
||||||
r.Post("/", s.handleGenerateRegistrationToken)
|
|
||||||
r.Get("/", s.handleListRegistrationTokens)
|
|
||||||
r.Delete("/{id}", s.handleRevokeRegistrationToken)
|
|
||||||
})
|
|
||||||
r.Get("/", s.handleListRunnersAdmin)
|
|
||||||
r.Post("/{id}/verify", s.handleVerifyRunner)
|
|
||||||
r.Delete("/{id}", s.handleDeleteRunner)
|
|
||||||
})
|
|
||||||
r.Route("/users", func(r chi.Router) {
|
|
||||||
r.Get("/", s.handleListUsers)
|
|
||||||
r.Get("/{id}/jobs", s.handleGetUserJobs)
|
|
||||||
r.Post("/{id}/admin", s.handleSetUserAdminStatus)
|
|
||||||
})
|
|
||||||
r.Route("/settings", func(r chi.Router) {
|
|
||||||
r.Get("/registration", s.handleGetRegistrationEnabled)
|
|
||||||
r.Post("/registration", s.handleSetRegistrationEnabled)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
// Runner API
|
|
||||||
s.router.Route("/api/runner", func(r chi.Router) {
|
|
||||||
// Registration doesn't require auth (uses token)
|
|
||||||
r.With(middleware.Timeout(60*time.Second)).Post("/register", s.handleRegisterRunner)
|
|
||||||
|
|
||||||
// WebSocket endpoint (auth handled in handler) - no timeout middleware
|
|
||||||
r.Get("/ws", s.handleRunnerWebSocket)
|
|
||||||
|
|
||||||
// File operations still use HTTP (WebSocket not suitable for large files)
|
|
||||||
r.Group(func(r chi.Router) {
|
|
||||||
r.Use(func(next http.Handler) http.Handler {
|
|
||||||
return http.HandlerFunc(s.runnerAuthMiddleware(next.ServeHTTP))
|
|
||||||
})
|
|
||||||
r.Post("/tasks/{id}/progress", s.handleUpdateTaskProgress)
|
|
||||||
r.Post("/tasks/{id}/steps", s.handleUpdateTaskStep)
|
|
||||||
r.Get("/files/{jobId}/*", s.handleDownloadFileForRunner)
|
|
||||||
r.Post("/files/{jobId}/upload", s.handleUploadFileFromRunner)
|
|
||||||
r.Get("/jobs/{jobId}/status", s.handleGetJobStatusForRunner)
|
|
||||||
r.Get("/jobs/{jobId}/files", s.handleGetJobFilesForRunner)
|
|
||||||
r.Post("/jobs/{jobId}/metadata", s.handleSubmitMetadata)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
|
|
||||||
// Serve static files (built React app)
|
|
||||||
s.router.Handle("/*", http.FileServer(http.Dir("./web/dist")))
|
|
||||||
}
|
|
||||||
|
|
||||||
// ServeHTTP implements http.Handler
|
|
||||||
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
||||||
s.router.ServeHTTP(w, r)
|
|
||||||
}
|
|
||||||
|
|
||||||
// JSON response helpers
|
|
||||||
func (s *Server) respondJSON(w http.ResponseWriter, status int, data interface{}) {
|
|
||||||
w.Header().Set("Content-Type", "application/json")
|
|
||||||
w.WriteHeader(status)
|
|
||||||
if err := json.NewEncoder(w).Encode(data); err != nil {
|
|
||||||
log.Printf("Failed to encode JSON response: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) respondError(w http.ResponseWriter, status int, message string) {
|
|
||||||
s.respondJSON(w, status, map[string]string{"error": message})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Auth handlers
|
|
||||||
func (s *Server) handleGoogleLogin(w http.ResponseWriter, r *http.Request) {
|
|
||||||
url, err := s.auth.GoogleLoginURL()
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
http.Redirect(w, r, url, http.StatusFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) handleGoogleCallback(w http.ResponseWriter, r *http.Request) {
|
|
||||||
code := r.URL.Query().Get("code")
|
|
||||||
if code == "" {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "Missing code parameter")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
session, err := s.auth.GoogleCallback(r.Context(), code)
|
|
||||||
if err != nil {
|
|
||||||
// If registration is disabled, redirect back to login with error
|
|
||||||
if err.Error() == "registration is disabled" {
|
|
||||||
http.Redirect(w, r, "/?error=registration_disabled", http.StatusFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
s.respondError(w, http.StatusInternalServerError, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sessionID := s.auth.CreateSession(session)
|
|
||||||
http.SetCookie(w, &http.Cookie{
|
|
||||||
Name: "session_id",
|
|
||||||
Value: sessionID,
|
|
||||||
Path: "/",
|
|
||||||
MaxAge: 86400,
|
|
||||||
HttpOnly: true,
|
|
||||||
SameSite: http.SameSiteLaxMode,
|
|
||||||
})
|
|
||||||
|
|
||||||
http.Redirect(w, r, "/", http.StatusFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) handleDiscordLogin(w http.ResponseWriter, r *http.Request) {
|
|
||||||
url, err := s.auth.DiscordLoginURL()
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
http.Redirect(w, r, url, http.StatusFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) handleDiscordCallback(w http.ResponseWriter, r *http.Request) {
|
|
||||||
code := r.URL.Query().Get("code")
|
|
||||||
if code == "" {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "Missing code parameter")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
session, err := s.auth.DiscordCallback(r.Context(), code)
|
|
||||||
if err != nil {
|
|
||||||
// If registration is disabled, redirect back to login with error
|
|
||||||
if err.Error() == "registration is disabled" {
|
|
||||||
http.Redirect(w, r, "/?error=registration_disabled", http.StatusFound)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
s.respondError(w, http.StatusInternalServerError, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sessionID := s.auth.CreateSession(session)
|
|
||||||
http.SetCookie(w, &http.Cookie{
|
|
||||||
Name: "session_id",
|
|
||||||
Value: sessionID,
|
|
||||||
Path: "/",
|
|
||||||
MaxAge: 86400,
|
|
||||||
HttpOnly: true,
|
|
||||||
SameSite: http.SameSiteLaxMode,
|
|
||||||
})
|
|
||||||
|
|
||||||
http.Redirect(w, r, "/", http.StatusFound)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) {
|
|
||||||
cookie, err := r.Cookie("session_id")
|
|
||||||
if err == nil {
|
|
||||||
s.auth.DeleteSession(cookie.Value)
|
|
||||||
}
|
|
||||||
http.SetCookie(w, &http.Cookie{
|
|
||||||
Name: "session_id",
|
|
||||||
Value: "",
|
|
||||||
Path: "/",
|
|
||||||
MaxAge: -1,
|
|
||||||
HttpOnly: true,
|
|
||||||
})
|
|
||||||
s.respondJSON(w, http.StatusOK, map[string]string{"message": "Logged out"})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) handleGetMe(w http.ResponseWriter, r *http.Request) {
|
|
||||||
cookie, err := r.Cookie("session_id")
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusUnauthorized, "Not authenticated")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
session, ok := s.auth.GetSession(cookie.Value)
|
|
||||||
if !ok {
|
|
||||||
s.respondError(w, http.StatusUnauthorized, "Invalid session")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
s.respondJSON(w, http.StatusOK, map[string]interface{}{
|
|
||||||
"id": session.UserID,
|
|
||||||
"email": session.Email,
|
|
||||||
"name": session.Name,
|
|
||||||
"is_admin": session.IsAdmin,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) handleGetAuthProviders(w http.ResponseWriter, r *http.Request) {
|
|
||||||
s.respondJSON(w, http.StatusOK, map[string]bool{
|
|
||||||
"google": s.auth.IsGoogleOAuthConfigured(),
|
|
||||||
"discord": s.auth.IsDiscordOAuthConfigured(),
|
|
||||||
"local": s.auth.IsLocalLoginEnabled(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) handleLocalLoginAvailable(w http.ResponseWriter, r *http.Request) {
|
|
||||||
s.respondJSON(w, http.StatusOK, map[string]bool{
|
|
||||||
"available": s.auth.IsLocalLoginEnabled(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) handleLocalRegister(w http.ResponseWriter, r *http.Request) {
|
|
||||||
var req struct {
|
|
||||||
Email string `json:"email"`
|
|
||||||
Name string `json:"name"`
|
|
||||||
Password string `json:"password"`
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "Invalid request body")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if req.Email == "" || req.Name == "" || req.Password == "" {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "Email, name, and password are required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(req.Password) < 8 {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "Password must be at least 8 characters long")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
session, err := s.auth.RegisterLocalUser(req.Email, req.Name, req.Password)
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sessionID := s.auth.CreateSession(session)
|
|
||||||
http.SetCookie(w, &http.Cookie{
|
|
||||||
Name: "session_id",
|
|
||||||
Value: sessionID,
|
|
||||||
Path: "/",
|
|
||||||
MaxAge: 86400,
|
|
||||||
HttpOnly: true,
|
|
||||||
SameSite: http.SameSiteLaxMode,
|
|
||||||
})
|
|
||||||
|
|
||||||
s.respondJSON(w, http.StatusCreated, map[string]interface{}{
|
|
||||||
"message": "Registration successful",
|
|
||||||
"user": map[string]interface{}{
|
|
||||||
"id": session.UserID,
|
|
||||||
"email": session.Email,
|
|
||||||
"name": session.Name,
|
|
||||||
"is_admin": session.IsAdmin,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) handleLocalLogin(w http.ResponseWriter, r *http.Request) {
|
|
||||||
var req struct {
|
|
||||||
Username string `json:"username"`
|
|
||||||
Password string `json:"password"`
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "Invalid request body")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if req.Username == "" || req.Password == "" {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "Username and password are required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
session, err := s.auth.LocalLogin(req.Username, req.Password)
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusUnauthorized, "Invalid credentials")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
sessionID := s.auth.CreateSession(session)
|
|
||||||
http.SetCookie(w, &http.Cookie{
|
|
||||||
Name: "session_id",
|
|
||||||
Value: sessionID,
|
|
||||||
Path: "/",
|
|
||||||
MaxAge: 86400,
|
|
||||||
HttpOnly: true,
|
|
||||||
SameSite: http.SameSiteLaxMode,
|
|
||||||
})
|
|
||||||
|
|
||||||
s.respondJSON(w, http.StatusOK, map[string]interface{}{
|
|
||||||
"message": "Login successful",
|
|
||||||
"user": map[string]interface{}{
|
|
||||||
"id": session.UserID,
|
|
||||||
"email": session.Email,
|
|
||||||
"name": session.Name,
|
|
||||||
"is_admin": session.IsAdmin,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) handleChangePassword(w http.ResponseWriter, r *http.Request) {
|
|
||||||
userID, err := getUserID(r)
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusUnauthorized, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
var req struct {
|
|
||||||
OldPassword string `json:"old_password"`
|
|
||||||
NewPassword string `json:"new_password"`
|
|
||||||
TargetUserID *int64 `json:"target_user_id,omitempty"` // For admin to change other users' passwords
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "Invalid request body")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if req.NewPassword == "" {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "New password is required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(req.NewPassword) < 8 {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "Password must be at least 8 characters long")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
isAdmin := authpkg.IsAdmin(r.Context())
|
|
||||||
|
|
||||||
// If target_user_id is provided and user is admin, allow changing other user's password
|
|
||||||
if req.TargetUserID != nil && isAdmin {
|
|
||||||
if err := s.auth.AdminChangePassword(*req.TargetUserID, req.NewPassword); err != nil {
|
|
||||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
s.respondJSON(w, http.StatusOK, map[string]string{"message": "Password changed successfully"})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Otherwise, user is changing their own password (requires old password)
|
|
||||||
if req.OldPassword == "" {
|
|
||||||
s.respondError(w, http.StatusBadRequest, "Old password is required")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := s.auth.ChangePassword(userID, req.OldPassword, req.NewPassword); err != nil {
|
|
||||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
s.respondJSON(w, http.StatusOK, map[string]string{"message": "Password changed successfully"})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper to get user ID from context
|
|
||||||
func getUserID(r *http.Request) (int64, error) {
|
|
||||||
userID, ok := authpkg.GetUserID(r.Context())
|
|
||||||
if !ok {
|
|
||||||
return 0, fmt.Errorf("user ID not found in context")
|
|
||||||
}
|
|
||||||
return userID, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper to parse ID from URL
|
|
||||||
func parseID(r *http.Request, param string) (int64, error) {
|
|
||||||
idStr := chi.URLParam(r, param)
|
|
||||||
id, err := strconv.ParseInt(idStr, 10, 64)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("invalid ID: %s", idStr)
|
|
||||||
}
|
|
||||||
return id, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// StartBackgroundTasks starts background goroutines for error recovery
|
|
||||||
func (s *Server) StartBackgroundTasks() {
|
|
||||||
go s.recoverStuckTasks()
|
|
||||||
go s.cleanupOldMetadataJobs()
|
|
||||||
}
|
|
||||||
|
|
||||||
// recoverStuckTasks periodically checks for dead runners and stuck tasks
|
|
||||||
func (s *Server) recoverStuckTasks() {
|
|
||||||
ticker := time.NewTicker(10 * time.Second)
|
|
||||||
defer ticker.Stop()
|
|
||||||
|
|
||||||
// Also distribute tasks every 10 seconds (reduced frequency since we have event-driven distribution)
|
|
||||||
distributeTicker := time.NewTicker(10 * time.Second)
|
|
||||||
defer distributeTicker.Stop()
|
|
||||||
|
|
||||||
go func() {
|
|
||||||
for range distributeTicker.C {
|
|
||||||
s.distributeTasksToRunners()
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
for range ticker.C {
|
|
||||||
func() {
|
|
||||||
defer func() {
|
|
||||||
if r := recover(); r != nil {
|
|
||||||
log.Printf("Panic in recoverStuckTasks: %v", r)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
|
|
||||||
// Find dead runners (no heartbeat for 90 seconds)
|
|
||||||
// But only mark as dead if they're not actually connected via WebSocket
|
|
||||||
rows, err := s.db.Query(
|
|
||||||
`SELECT id FROM runners
|
|
||||||
WHERE last_heartbeat < CURRENT_TIMESTAMP - INTERVAL '90 seconds'
|
|
||||||
AND status = ?`,
|
|
||||||
types.RunnerStatusOnline,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed to query dead runners: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
var deadRunnerIDs []int64
|
|
||||||
s.runnerConnsMu.RLock()
|
|
||||||
for rows.Next() {
|
|
||||||
var runnerID int64
|
|
||||||
if err := rows.Scan(&runnerID); err == nil {
|
|
||||||
// Only mark as dead if not actually connected via WebSocket
|
|
||||||
// The WebSocket connection is the source of truth
|
|
||||||
if _, stillConnected := s.runnerConns[runnerID]; !stillConnected {
|
|
||||||
deadRunnerIDs = append(deadRunnerIDs, runnerID)
|
|
||||||
}
|
|
||||||
// If still connected, heartbeat should be updated by pong handler or heartbeat message
|
|
||||||
// No need to manually update here - if it's stale, the pong handler isn't working
|
|
||||||
}
|
|
||||||
}
|
|
||||||
s.runnerConnsMu.RUnlock()
|
|
||||||
rows.Close()
|
|
||||||
|
|
||||||
if len(deadRunnerIDs) == 0 {
|
|
||||||
// Check for task timeouts
|
|
||||||
s.recoverTaskTimeouts()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Reset tasks assigned to dead runners
|
|
||||||
for _, runnerID := range deadRunnerIDs {
|
|
||||||
s.redistributeRunnerTasks(runnerID)
|
|
||||||
|
|
||||||
// Mark runner as offline
|
|
||||||
_, _ = s.db.Exec(
|
|
||||||
`UPDATE runners SET status = ? WHERE id = ?`,
|
|
||||||
types.RunnerStatusOffline, runnerID,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check for task timeouts
|
|
||||||
s.recoverTaskTimeouts()
|
|
||||||
|
|
||||||
// Distribute newly recovered tasks
|
|
||||||
s.distributeTasksToRunners()
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// recoverTaskTimeouts handles tasks that have exceeded their timeout
|
|
||||||
func (s *Server) recoverTaskTimeouts() {
|
|
||||||
// Find tasks running longer than their timeout
|
|
||||||
rows, err := s.db.Query(
|
|
||||||
`SELECT t.id, t.runner_id, t.retry_count, t.max_retries, t.timeout_seconds, t.started_at
|
|
||||||
FROM tasks t
|
|
||||||
WHERE t.status = ?
|
|
||||||
AND t.started_at IS NOT NULL
|
|
||||||
AND (t.timeout_seconds IS NULL OR
|
|
||||||
t.started_at + INTERVAL (t.timeout_seconds || ' seconds') < CURRENT_TIMESTAMP)`,
|
|
||||||
types.TaskStatusRunning,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed to query timed out tasks: %v", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var taskID int64
|
|
||||||
var runnerID sql.NullInt64
|
|
||||||
var retryCount, maxRetries int
|
|
||||||
var timeoutSeconds sql.NullInt64
|
|
||||||
var startedAt time.Time
|
|
||||||
|
|
||||||
err := rows.Scan(&taskID, &runnerID, &retryCount, &maxRetries, &timeoutSeconds, &startedAt)
|
|
||||||
if err != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
// Use default timeout if not set (5 minutes for frame tasks, 24 hours for FFmpeg)
|
|
||||||
timeout := 300 // 5 minutes default
|
|
||||||
if timeoutSeconds.Valid {
|
|
||||||
timeout = int(timeoutSeconds.Int64)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if actually timed out
|
|
||||||
if time.Since(startedAt).Seconds() < float64(timeout) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
if retryCount >= maxRetries {
|
|
||||||
// Mark as failed
|
|
||||||
_, err = s.db.Exec(
|
|
||||||
`UPDATE tasks SET status = ?, error_message = ?, runner_id = NULL
|
|
||||||
WHERE id = ?`,
|
|
||||||
types.TaskStatusFailed, "Task timeout exceeded, max retries reached", taskID,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed to mark task %d as failed: %v", taskID, err)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Reset to pending
|
|
||||||
_, err = s.db.Exec(
|
|
||||||
`UPDATE tasks SET status = ?, runner_id = NULL, current_step = NULL,
|
|
||||||
retry_count = retry_count + 1 WHERE id = ?`,
|
|
||||||
types.TaskStatusPending, taskID,
|
|
||||||
)
|
|
||||||
if err == nil {
|
|
||||||
// Add log entry using the helper function
|
|
||||||
s.logTaskEvent(taskID, nil, types.LogLevelWarn, fmt.Sprintf("Task timeout exceeded, resetting (retry %d/%d)", retryCount+1, maxRetries), "")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+457
-114
@@ -5,10 +5,13 @@ import (
|
|||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"jiggablend/internal/config"
|
||||||
|
"jiggablend/internal/database"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
@@ -17,12 +20,34 @@ import (
|
|||||||
"golang.org/x/oauth2/google"
|
"golang.org/x/oauth2/google"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Context key types to avoid collisions (typed keys are safer than string keys)
|
||||||
|
type contextKey int
|
||||||
|
|
||||||
|
const (
|
||||||
|
contextKeyUserID contextKey = iota
|
||||||
|
contextKeyUserEmail
|
||||||
|
contextKeyUserName
|
||||||
|
contextKeyIsAdmin
|
||||||
|
)
|
||||||
|
|
||||||
|
// Configuration constants
|
||||||
|
const (
|
||||||
|
SessionDuration = 24 * time.Hour
|
||||||
|
SessionCleanupInterval = 1 * time.Hour
|
||||||
|
)
|
||||||
|
|
||||||
// Auth handles authentication
|
// Auth handles authentication
|
||||||
type Auth struct {
|
type Auth struct {
|
||||||
db *sql.DB
|
db *database.DB
|
||||||
|
cfg *config.Config
|
||||||
googleConfig *oauth2.Config
|
googleConfig *oauth2.Config
|
||||||
discordConfig *oauth2.Config
|
discordConfig *oauth2.Config
|
||||||
sessionStore map[string]*Session
|
sessionCache map[string]*Session // In-memory cache for performance
|
||||||
|
cacheMu sync.RWMutex
|
||||||
|
stopCleanup chan struct{}
|
||||||
|
// oauthStates maps state token -> expiry for CSRF protection
|
||||||
|
oauthStates map[string]time.Time
|
||||||
|
oauthMu sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
// Session represents a user session
|
// Session represents a user session
|
||||||
@@ -35,41 +60,54 @@ type Session struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewAuth creates a new auth instance
|
// NewAuth creates a new auth instance
|
||||||
func NewAuth(db *sql.DB) (*Auth, error) {
|
func NewAuth(db *database.DB, cfg *config.Config) (*Auth, error) {
|
||||||
auth := &Auth{
|
auth := &Auth{
|
||||||
db: db,
|
db: db,
|
||||||
sessionStore: make(map[string]*Session),
|
cfg: cfg,
|
||||||
|
sessionCache: make(map[string]*Session),
|
||||||
|
stopCleanup: make(chan struct{}),
|
||||||
|
oauthStates: make(map[string]time.Time),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize Google OAuth
|
// Initialize Google OAuth from database config
|
||||||
googleClientID := os.Getenv("GOOGLE_CLIENT_ID")
|
googleClientID := cfg.GoogleClientID()
|
||||||
googleClientSecret := os.Getenv("GOOGLE_CLIENT_SECRET")
|
googleClientSecret := cfg.GoogleClientSecret()
|
||||||
if googleClientID != "" && googleClientSecret != "" {
|
if googleClientID != "" && googleClientSecret != "" {
|
||||||
auth.googleConfig = &oauth2.Config{
|
auth.googleConfig = &oauth2.Config{
|
||||||
ClientID: googleClientID,
|
ClientID: googleClientID,
|
||||||
ClientSecret: googleClientSecret,
|
ClientSecret: googleClientSecret,
|
||||||
RedirectURL: os.Getenv("GOOGLE_REDIRECT_URL"),
|
RedirectURL: cfg.GoogleRedirectURL(),
|
||||||
Scopes: []string{"openid", "profile", "email"},
|
Scopes: []string{"openid", "profile", "email"},
|
||||||
Endpoint: google.Endpoint,
|
Endpoint: google.Endpoint,
|
||||||
}
|
}
|
||||||
|
log.Printf("Google OAuth configured")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize Discord OAuth
|
// Initialize Discord OAuth from database config
|
||||||
discordClientID := os.Getenv("DISCORD_CLIENT_ID")
|
discordClientID := cfg.DiscordClientID()
|
||||||
discordClientSecret := os.Getenv("DISCORD_CLIENT_SECRET")
|
discordClientSecret := cfg.DiscordClientSecret()
|
||||||
if discordClientID != "" && discordClientSecret != "" {
|
if discordClientID != "" && discordClientSecret != "" {
|
||||||
auth.discordConfig = &oauth2.Config{
|
auth.discordConfig = &oauth2.Config{
|
||||||
ClientID: discordClientID,
|
ClientID: discordClientID,
|
||||||
ClientSecret: discordClientSecret,
|
ClientSecret: discordClientSecret,
|
||||||
RedirectURL: os.Getenv("DISCORD_REDIRECT_URL"),
|
RedirectURL: cfg.DiscordRedirectURL(),
|
||||||
Scopes: []string{"identify", "email"},
|
Scopes: []string{"identify", "email"},
|
||||||
Endpoint: oauth2.Endpoint{
|
Endpoint: oauth2.Endpoint{
|
||||||
AuthURL: "https://discord.com/api/oauth2/authorize",
|
AuthURL: "https://discord.com/api/oauth2/authorize",
|
||||||
TokenURL: "https://discord.com/api/oauth2/token",
|
TokenURL: "https://discord.com/api/oauth2/token",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
log.Printf("Discord OAuth configured")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Load existing sessions from database into cache
|
||||||
|
if err := auth.loadSessionsFromDB(); err != nil {
|
||||||
|
log.Printf("Warning: Failed to load sessions from database: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start background cleanup goroutine
|
||||||
|
go auth.cleanupExpiredSessions()
|
||||||
|
|
||||||
// Initialize admin settings on startup to ensure they persist between boots
|
// Initialize admin settings on startup to ensure they persist between boots
|
||||||
if err := auth.initializeSettings(); err != nil {
|
if err := auth.initializeSettings(); err != nil {
|
||||||
log.Printf("Warning: Failed to initialize admin settings: %v", err)
|
log.Printf("Warning: Failed to initialize admin settings: %v", err)
|
||||||
@@ -85,23 +123,125 @@ func NewAuth(db *sql.DB) (*Auth, error) {
|
|||||||
return auth, nil
|
return auth, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Close stops background goroutines
|
||||||
|
func (a *Auth) Close() {
|
||||||
|
close(a.stopCleanup)
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadSessionsFromDB loads all valid sessions from database into cache
|
||||||
|
func (a *Auth) loadSessionsFromDB() error {
|
||||||
|
var sessions []struct {
|
||||||
|
sessionID string
|
||||||
|
session Session
|
||||||
|
}
|
||||||
|
|
||||||
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
rows, err := conn.Query(
|
||||||
|
`SELECT session_id, user_id, email, name, is_admin, expires_at
|
||||||
|
FROM sessions WHERE expires_at > CURRENT_TIMESTAMP`,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to query sessions: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
for rows.Next() {
|
||||||
|
var s struct {
|
||||||
|
sessionID string
|
||||||
|
session Session
|
||||||
|
}
|
||||||
|
err := rows.Scan(&s.sessionID, &s.session.UserID, &s.session.Email, &s.session.Name, &s.session.IsAdmin, &s.session.ExpiresAt)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Warning: Failed to scan session row: %v", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sessions = append(sessions, s)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
a.cacheMu.Lock()
|
||||||
|
defer a.cacheMu.Unlock()
|
||||||
|
|
||||||
|
for _, s := range sessions {
|
||||||
|
a.sessionCache[s.sessionID] = &s.session
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(sessions) > 0 {
|
||||||
|
log.Printf("Loaded %d active sessions from database", len(sessions))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanupExpiredSessions periodically removes expired sessions from database and cache
|
||||||
|
func (a *Auth) cleanupExpiredSessions() {
|
||||||
|
ticker := time.NewTicker(SessionCleanupInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ticker.C:
|
||||||
|
// Delete expired sessions from database
|
||||||
|
var deleted int64
|
||||||
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
result, err := conn.Exec(`DELETE FROM sessions WHERE expires_at < CURRENT_TIMESTAMP`)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
deleted, _ = result.RowsAffected()
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Warning: Failed to cleanup expired sessions: %v", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up cache
|
||||||
|
a.cacheMu.Lock()
|
||||||
|
now := time.Now()
|
||||||
|
for sessionID, session := range a.sessionCache {
|
||||||
|
if now.After(session.ExpiresAt) {
|
||||||
|
delete(a.sessionCache, sessionID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.cacheMu.Unlock()
|
||||||
|
|
||||||
|
if deleted > 0 {
|
||||||
|
log.Printf("Cleaned up %d expired sessions", deleted)
|
||||||
|
}
|
||||||
|
case <-a.stopCleanup:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// initializeSettings ensures all admin settings are initialized with defaults if they don't exist
|
// initializeSettings ensures all admin settings are initialized with defaults if they don't exist
|
||||||
func (a *Auth) initializeSettings() error {
|
func (a *Auth) initializeSettings() error {
|
||||||
// Initialize registration_enabled setting (default: true) if it doesn't exist
|
// Default registration to false (safer for internet-facing boots). Admins enable via CLI/UI.
|
||||||
|
// In non-production, still default false — make init-test / CLI create the first admin.
|
||||||
|
defaultReg := "false"
|
||||||
var settingCount int
|
var settingCount int
|
||||||
err := a.db.QueryRow("SELECT COUNT(*) FROM settings WHERE key = ?", "registration_enabled").Scan(&settingCount)
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT COUNT(*) FROM settings WHERE key = ?", "registration_enabled").Scan(&settingCount)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to check registration_enabled setting: %w", err)
|
return fmt.Errorf("failed to check registration_enabled setting: %w", err)
|
||||||
}
|
}
|
||||||
if settingCount == 0 {
|
if settingCount == 0 {
|
||||||
_, err = a.db.Exec(
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
`INSERT INTO settings (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)`,
|
_, err := conn.Exec(
|
||||||
"registration_enabled", "true",
|
`INSERT INTO settings (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)`,
|
||||||
)
|
"registration_enabled", defaultReg,
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to initialize registration_enabled setting: %w", err)
|
return fmt.Errorf("failed to initialize registration_enabled setting: %w", err)
|
||||||
}
|
}
|
||||||
log.Printf("Initialized admin setting: registration_enabled = true")
|
log.Printf("Initialized admin setting: registration_enabled = %s", defaultReg)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -118,7 +258,9 @@ func (a *Auth) initializeTestUser() error {
|
|||||||
|
|
||||||
// Check if user already exists
|
// Check if user already exists
|
||||||
var exists bool
|
var exists bool
|
||||||
err := a.db.QueryRow("SELECT EXISTS(SELECT 1 FROM users WHERE email = ? AND oauth_provider = 'local')", testEmail).Scan(&exists)
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM users WHERE email = ? AND oauth_provider = 'local')", testEmail).Scan(&exists)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to check if test user exists: %w", err)
|
return fmt.Errorf("failed to check if test user exists: %w", err)
|
||||||
}
|
}
|
||||||
@@ -137,7 +279,12 @@ func (a *Auth) initializeTestUser() error {
|
|||||||
|
|
||||||
// Check if this is the first user (make them admin)
|
// Check if this is the first user (make them admin)
|
||||||
var userCount int
|
var userCount int
|
||||||
a.db.QueryRow("SELECT COUNT(*) FROM users").Scan(&userCount)
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT COUNT(*) FROM users").Scan(&userCount)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to check user count: %w", err)
|
||||||
|
}
|
||||||
isAdmin := userCount == 0
|
isAdmin := userCount == 0
|
||||||
|
|
||||||
// Create test user (use email as name if no name is provided)
|
// Create test user (use email as name if no name is provided)
|
||||||
@@ -147,10 +294,13 @@ func (a *Auth) initializeTestUser() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Create test user
|
// Create test user
|
||||||
_, err = a.db.Exec(
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec(
|
||||||
"INSERT INTO users (email, name, oauth_provider, oauth_id, password_hash, is_admin) VALUES (?, ?, 'local', ?, ?, ?)",
|
"INSERT INTO users (email, name, oauth_provider, oauth_id, password_hash, is_admin) VALUES (?, ?, 'local', ?, ?, ?)",
|
||||||
testEmail, testName, testEmail, string(hashedPassword), isAdmin,
|
testEmail, testName, testEmail, string(hashedPassword), isAdmin,
|
||||||
)
|
)
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to create test user: %w", err)
|
return fmt.Errorf("failed to create test user: %w", err)
|
||||||
}
|
}
|
||||||
@@ -159,12 +309,44 @@ func (a *Auth) initializeTestUser() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CreateOAuthState mints a one-time OAuth state token and stores it until used or expired.
|
||||||
|
func (a *Auth) CreateOAuthState() string {
|
||||||
|
state := uuid.New().String()
|
||||||
|
a.oauthMu.Lock()
|
||||||
|
// Opportunistic cleanup of expired states
|
||||||
|
now := time.Now()
|
||||||
|
for k, exp := range a.oauthStates {
|
||||||
|
if now.After(exp) {
|
||||||
|
delete(a.oauthStates, k)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
a.oauthStates[state] = now.Add(10 * time.Minute)
|
||||||
|
a.oauthMu.Unlock()
|
||||||
|
return state
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConsumeOAuthState validates and single-use-consumes an OAuth state token.
|
||||||
|
// Returns false if missing, unknown, or expired.
|
||||||
|
func (a *Auth) ConsumeOAuthState(state string) bool {
|
||||||
|
if state == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
a.oauthMu.Lock()
|
||||||
|
defer a.oauthMu.Unlock()
|
||||||
|
exp, ok := a.oauthStates[state]
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
delete(a.oauthStates, state)
|
||||||
|
return time.Now().Before(exp)
|
||||||
|
}
|
||||||
|
|
||||||
// GoogleLoginURL returns the Google OAuth login URL
|
// GoogleLoginURL returns the Google OAuth login URL
|
||||||
func (a *Auth) GoogleLoginURL() (string, error) {
|
func (a *Auth) GoogleLoginURL() (string, error) {
|
||||||
if a.googleConfig == nil {
|
if a.googleConfig == nil {
|
||||||
return "", fmt.Errorf("Google OAuth not configured")
|
return "", fmt.Errorf("Google OAuth not configured")
|
||||||
}
|
}
|
||||||
state := uuid.New().String()
|
state := a.CreateOAuthState()
|
||||||
return a.googleConfig.AuthCodeURL(state), nil
|
return a.googleConfig.AuthCodeURL(state), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -173,7 +355,7 @@ func (a *Auth) DiscordLoginURL() (string, error) {
|
|||||||
if a.discordConfig == nil {
|
if a.discordConfig == nil {
|
||||||
return "", fmt.Errorf("Discord OAuth not configured")
|
return "", fmt.Errorf("Discord OAuth not configured")
|
||||||
}
|
}
|
||||||
state := uuid.New().String()
|
state := a.CreateOAuthState()
|
||||||
return a.discordConfig.AuthCodeURL(state), nil
|
return a.discordConfig.AuthCodeURL(state), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -242,10 +424,12 @@ func (a *Auth) DiscordCallback(ctx context.Context, code string) (*Session, erro
|
|||||||
// IsRegistrationEnabled checks if new user registration is enabled
|
// IsRegistrationEnabled checks if new user registration is enabled
|
||||||
func (a *Auth) IsRegistrationEnabled() (bool, error) {
|
func (a *Auth) IsRegistrationEnabled() (bool, error) {
|
||||||
var value string
|
var value string
|
||||||
err := a.db.QueryRow("SELECT value FROM settings WHERE key = ?", "registration_enabled").Scan(&value)
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT value FROM settings WHERE key = ?", "registration_enabled").Scan(&value)
|
||||||
|
})
|
||||||
if err == sql.ErrNoRows {
|
if err == sql.ErrNoRows {
|
||||||
// Default to enabled if setting doesn't exist
|
// Default to disabled if setting doesn't exist (safer bootstrap)
|
||||||
return true, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("failed to check registration setting: %w", err)
|
return false, fmt.Errorf("failed to check registration setting: %w", err)
|
||||||
@@ -262,36 +446,39 @@ func (a *Auth) SetRegistrationEnabled(enabled bool) error {
|
|||||||
|
|
||||||
// Check if setting exists
|
// Check if setting exists
|
||||||
var exists bool
|
var exists bool
|
||||||
err := a.db.QueryRow("SELECT EXISTS(SELECT 1 FROM settings WHERE key = ?)", "registration_enabled").Scan(&exists)
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM settings WHERE key = ?)", "registration_enabled").Scan(&exists)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to check if setting exists: %w", err)
|
return fmt.Errorf("failed to check if setting exists: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
if exists {
|
if exists {
|
||||||
// Update existing setting
|
// Update existing setting
|
||||||
_, err = a.db.Exec(
|
_, err = conn.Exec(
|
||||||
"UPDATE settings SET value = ?, updated_at = CURRENT_TIMESTAMP WHERE key = ?",
|
"UPDATE settings SET value = ?, updated_at = CURRENT_TIMESTAMP WHERE key = ?",
|
||||||
value, "registration_enabled",
|
value, "registration_enabled",
|
||||||
)
|
)
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to update setting: %w", err)
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
// Insert new setting
|
// Insert new setting
|
||||||
_, err = a.db.Exec(
|
_, err = conn.Exec(
|
||||||
"INSERT INTO settings (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)",
|
"INSERT INTO settings (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)",
|
||||||
"registration_enabled", value,
|
"registration_enabled", value,
|
||||||
)
|
)
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to insert setting: %w", err)
|
|
||||||
}
|
}
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to set registration_enabled: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// getOrCreateUser gets or creates a user in the database
|
// getOrCreateUser gets or creates a user in the database.
|
||||||
// Automatically links accounts by email across different OAuth providers and local login
|
// Identity is (oauth_provider, oauth_id). Email collision with a different provider
|
||||||
|
// is NOT auto-linked (prevents account takeover); the user must sign in with the original method.
|
||||||
func (a *Auth) getOrCreateUser(provider, oauthID, email, name string) (*Session, error) {
|
func (a *Auth) getOrCreateUser(provider, oauthID, email, name string) (*Session, error) {
|
||||||
var userID int64
|
var userID int64
|
||||||
var dbEmail, dbName string
|
var dbEmail, dbName string
|
||||||
@@ -299,17 +486,21 @@ func (a *Auth) getOrCreateUser(provider, oauthID, email, name string) (*Session,
|
|||||||
var dbProvider, dbOAuthID string
|
var dbProvider, dbOAuthID string
|
||||||
|
|
||||||
// First, try to find by provider + oauth_id
|
// First, try to find by provider + oauth_id
|
||||||
err := a.db.QueryRow(
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
"SELECT id, email, name, is_admin, oauth_provider, oauth_id FROM users WHERE oauth_provider = ? AND oauth_id = ?",
|
return conn.QueryRow(
|
||||||
provider, oauthID,
|
"SELECT id, email, name, is_admin, oauth_provider, oauth_id FROM users WHERE oauth_provider = ? AND oauth_id = ?",
|
||||||
).Scan(&userID, &dbEmail, &dbName, &isAdmin, &dbProvider, &dbOAuthID)
|
provider, oauthID,
|
||||||
|
).Scan(&userID, &dbEmail, &dbName, &isAdmin, &dbProvider, &dbOAuthID)
|
||||||
|
})
|
||||||
|
|
||||||
if err == sql.ErrNoRows {
|
if err == sql.ErrNoRows {
|
||||||
// Not found by provider+oauth_id, check by email for account linking
|
// Not found by provider+oauth_id — check email only to refuse silent takeover
|
||||||
err = a.db.QueryRow(
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
"SELECT id, email, name, is_admin, oauth_provider, oauth_id FROM users WHERE email = ?",
|
return conn.QueryRow(
|
||||||
email,
|
"SELECT id, email, name, is_admin, oauth_provider, oauth_id FROM users WHERE email = ?",
|
||||||
).Scan(&userID, &dbEmail, &dbName, &isAdmin, &dbProvider, &dbOAuthID)
|
email,
|
||||||
|
).Scan(&userID, &dbEmail, &dbName, &isAdmin, &dbProvider, &dbOAuthID)
|
||||||
|
})
|
||||||
|
|
||||||
if err == sql.ErrNoRows {
|
if err == sql.ErrNoRows {
|
||||||
// User doesn't exist, check if registration is enabled
|
// User doesn't exist, check if registration is enabled
|
||||||
@@ -323,40 +514,47 @@ func (a *Auth) getOrCreateUser(provider, oauthID, email, name string) (*Session,
|
|||||||
|
|
||||||
// Check if this is the first user
|
// Check if this is the first user
|
||||||
var userCount int
|
var userCount int
|
||||||
a.db.QueryRow("SELECT COUNT(*) FROM users").Scan(&userCount)
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT COUNT(*) FROM users").Scan(&userCount)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to check user count: %w", err)
|
||||||
|
}
|
||||||
isAdmin = userCount == 0
|
isAdmin = userCount == 0
|
||||||
|
|
||||||
// Create new user
|
// Create new user
|
||||||
err = a.db.QueryRow(
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
"INSERT INTO users (email, name, oauth_provider, oauth_id, is_admin) VALUES (?, ?, ?, ?, ?) RETURNING id",
|
result, err := conn.Exec(
|
||||||
email, name, provider, oauthID, isAdmin,
|
"INSERT INTO users (email, name, oauth_provider, oauth_id, is_admin) VALUES (?, ?, ?, ?, ?)",
|
||||||
).Scan(&userID)
|
email, name, provider, oauthID, isAdmin,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
userID, err = result.LastInsertId()
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to create user: %w", err)
|
return nil, fmt.Errorf("failed to create user: %w", err)
|
||||||
}
|
}
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return nil, fmt.Errorf("failed to query user by email: %w", err)
|
return nil, fmt.Errorf("failed to query user by email: %w", err)
|
||||||
} else {
|
} else {
|
||||||
// User exists with same email but different provider - link accounts by updating provider info
|
// Email already belongs to another identity — do not overwrite oauth_provider/oauth_id
|
||||||
// This allows the user to log in with any provider that has the same email
|
return nil, fmt.Errorf("an account with this email already exists; sign in with the original method")
|
||||||
_, err = a.db.Exec(
|
|
||||||
"UPDATE users SET oauth_provider = ?, oauth_id = ?, name = ? WHERE id = ?",
|
|
||||||
provider, oauthID, name, userID,
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to link account: %w", err)
|
|
||||||
}
|
|
||||||
log.Printf("Linked account: user %d (email: %s) now accessible via %s provider", userID, email, provider)
|
|
||||||
}
|
}
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return nil, fmt.Errorf("failed to query user: %w", err)
|
return nil, fmt.Errorf("failed to query user: %w", err)
|
||||||
} else {
|
} else {
|
||||||
// User found by provider+oauth_id, update info if changed
|
// User found by provider+oauth_id, update info if changed
|
||||||
if dbEmail != email || dbName != name {
|
if dbEmail != email || dbName != name {
|
||||||
_, err = a.db.Exec(
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
"UPDATE users SET email = ?, name = ? WHERE id = ?",
|
_, err = conn.Exec(
|
||||||
email, name, userID,
|
"UPDATE users SET email = ?, name = ? WHERE id = ?",
|
||||||
)
|
email, name, userID,
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to update user: %w", err)
|
return nil, fmt.Errorf("failed to update user: %w", err)
|
||||||
}
|
}
|
||||||
@@ -368,41 +566,156 @@ func (a *Auth) getOrCreateUser(provider, oauthID, email, name string) (*Session,
|
|||||||
Email: email,
|
Email: email,
|
||||||
Name: name,
|
Name: name,
|
||||||
IsAdmin: isAdmin,
|
IsAdmin: isAdmin,
|
||||||
ExpiresAt: time.Now().Add(24 * time.Hour),
|
ExpiresAt: time.Now().Add(SessionDuration),
|
||||||
}
|
}
|
||||||
|
|
||||||
return session, nil
|
return session, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// CreateSession creates a new session and returns a session ID
|
// CreateSession creates a new session and returns a session ID
|
||||||
|
// Sessions are persisted to database and cached in memory
|
||||||
func (a *Auth) CreateSession(session *Session) string {
|
func (a *Auth) CreateSession(session *Session) string {
|
||||||
sessionID := uuid.New().String()
|
sessionID := uuid.New().String()
|
||||||
a.sessionStore[sessionID] = session
|
|
||||||
|
// Store in database first
|
||||||
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec(
|
||||||
|
`INSERT INTO sessions (session_id, user_id, email, name, is_admin, expires_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)`,
|
||||||
|
sessionID, session.UserID, session.Email, session.Name, session.IsAdmin, session.ExpiresAt,
|
||||||
|
)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Warning: Failed to persist session to database: %v", err)
|
||||||
|
// Continue anyway - session will work from cache but won't survive restart
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store in cache
|
||||||
|
a.cacheMu.Lock()
|
||||||
|
a.sessionCache[sessionID] = session
|
||||||
|
a.cacheMu.Unlock()
|
||||||
|
|
||||||
return sessionID
|
return sessionID
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetSession retrieves a session by ID
|
// GetSession retrieves a session by ID
|
||||||
|
// First checks cache, then database if not found
|
||||||
func (a *Auth) GetSession(sessionID string) (*Session, bool) {
|
func (a *Auth) GetSession(sessionID string) (*Session, bool) {
|
||||||
session, ok := a.sessionStore[sessionID]
|
// Check cache first
|
||||||
if !ok {
|
a.cacheMu.RLock()
|
||||||
|
session, ok := a.sessionCache[sessionID]
|
||||||
|
a.cacheMu.RUnlock()
|
||||||
|
|
||||||
|
if ok {
|
||||||
|
if time.Now().After(session.ExpiresAt) {
|
||||||
|
a.DeleteSession(sessionID)
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
// Refresh admin status from database
|
||||||
|
var isAdmin bool
|
||||||
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT is_admin FROM users WHERE id = ?", session.UserID).Scan(&isAdmin)
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
session.IsAdmin = isAdmin
|
||||||
|
}
|
||||||
|
return session, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Not in cache, check database
|
||||||
|
session = &Session{}
|
||||||
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow(
|
||||||
|
`SELECT user_id, email, name, is_admin, expires_at
|
||||||
|
FROM sessions WHERE session_id = ?`,
|
||||||
|
sessionID,
|
||||||
|
).Scan(&session.UserID, &session.Email, &session.Name, &session.IsAdmin, &session.ExpiresAt)
|
||||||
|
})
|
||||||
|
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Warning: Failed to query session from database: %v", err)
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
if time.Now().After(session.ExpiresAt) {
|
if time.Now().After(session.ExpiresAt) {
|
||||||
delete(a.sessionStore, sessionID)
|
a.DeleteSession(sessionID)
|
||||||
return nil, false
|
return nil, false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Refresh admin status from database
|
// Refresh admin status from database
|
||||||
var isAdmin bool
|
var isAdmin bool
|
||||||
err := a.db.QueryRow("SELECT is_admin FROM users WHERE id = ?", session.UserID).Scan(&isAdmin)
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT is_admin FROM users WHERE id = ?", session.UserID).Scan(&isAdmin)
|
||||||
|
})
|
||||||
if err == nil {
|
if err == nil {
|
||||||
session.IsAdmin = isAdmin
|
session.IsAdmin = isAdmin
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add to cache
|
||||||
|
a.cacheMu.Lock()
|
||||||
|
a.sessionCache[sessionID] = session
|
||||||
|
a.cacheMu.Unlock()
|
||||||
|
|
||||||
return session, true
|
return session, true
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteSession deletes a session
|
// DeleteSession deletes a session from both cache and database
|
||||||
func (a *Auth) DeleteSession(sessionID string) {
|
func (a *Auth) DeleteSession(sessionID string) {
|
||||||
delete(a.sessionStore, sessionID)
|
// Delete from cache
|
||||||
|
a.cacheMu.Lock()
|
||||||
|
delete(a.sessionCache, sessionID)
|
||||||
|
a.cacheMu.Unlock()
|
||||||
|
|
||||||
|
// Delete from database
|
||||||
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec("DELETE FROM sessions WHERE session_id = ?", sessionID)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Warning: Failed to delete session from database: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsProductionMode returns true if running in production mode.
|
||||||
|
// Prefer Config.IsProductionMode() / Auth.IsProductionModeFromConfig() for app logic.
|
||||||
|
// This package-level helper still honors PRODUCTION=true for legacy callers.
|
||||||
|
func IsProductionMode() bool {
|
||||||
|
return os.Getenv("PRODUCTION") == "true"
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsProductionModeFromConfig returns true if production mode is enabled in config
|
||||||
|
// or via PRODUCTION=true environment variable (OR of both sources).
|
||||||
|
func (a *Auth) IsProductionModeFromConfig() bool {
|
||||||
|
if a.cfg != nil && a.cfg.IsProductionMode() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return IsProductionMode()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a *Auth) writeUnauthorized(w http.ResponseWriter, r *http.Request) {
|
||||||
|
// Keep API behavior unchanged for programmatic clients.
|
||||||
|
if strings.HasPrefix(r.URL.Path, "/api/") {
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(http.StatusUnauthorized)
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// For HTMX UI fragment requests, trigger a full-page redirect to login.
|
||||||
|
if strings.EqualFold(r.Header.Get("HX-Request"), "true") {
|
||||||
|
w.Header().Set("HX-Redirect", "/login")
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
w.WriteHeader(http.StatusUnauthorized)
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// For normal browser page requests, redirect to login page.
|
||||||
|
http.Redirect(w, r, "/login", http.StatusFound)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Middleware creates an authentication middleware
|
// Middleware creates an authentication middleware
|
||||||
@@ -410,38 +723,36 @@ func (a *Auth) Middleware(next http.HandlerFunc) http.HandlerFunc {
|
|||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
cookie, err := r.Cookie("session_id")
|
cookie, err := r.Cookie("session_id")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
log.Printf("Authentication failed: missing session cookie for %s %s", r.Method, r.URL.Path)
|
||||||
w.WriteHeader(http.StatusUnauthorized)
|
a.writeUnauthorized(w, r)
|
||||||
json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
session, ok := a.GetSession(cookie.Value)
|
session, ok := a.GetSession(cookie.Value)
|
||||||
if !ok {
|
if !ok {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
log.Printf("Authentication failed: invalid session cookie for %s %s", r.Method, r.URL.Path)
|
||||||
w.WriteHeader(http.StatusUnauthorized)
|
a.writeUnauthorized(w, r)
|
||||||
json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add user info to request context
|
// Add user info to request context using typed keys
|
||||||
ctx := context.WithValue(r.Context(), "user_id", session.UserID)
|
ctx := context.WithValue(r.Context(), contextKeyUserID, session.UserID)
|
||||||
ctx = context.WithValue(ctx, "user_email", session.Email)
|
ctx = context.WithValue(ctx, contextKeyUserEmail, session.Email)
|
||||||
ctx = context.WithValue(ctx, "user_name", session.Name)
|
ctx = context.WithValue(ctx, contextKeyUserName, session.Name)
|
||||||
ctx = context.WithValue(ctx, "is_admin", session.IsAdmin)
|
ctx = context.WithValue(ctx, contextKeyIsAdmin, session.IsAdmin)
|
||||||
next(w, r.WithContext(ctx))
|
next(w, r.WithContext(ctx))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetUserID gets the user ID from context
|
// GetUserID gets the user ID from context
|
||||||
func GetUserID(ctx context.Context) (int64, bool) {
|
func GetUserID(ctx context.Context) (int64, bool) {
|
||||||
userID, ok := ctx.Value("user_id").(int64)
|
userID, ok := ctx.Value(contextKeyUserID).(int64)
|
||||||
return userID, ok
|
return userID, ok
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsAdmin checks if the user in context is an admin
|
// IsAdmin checks if the user in context is an admin
|
||||||
func IsAdmin(ctx context.Context) bool {
|
func IsAdmin(ctx context.Context) bool {
|
||||||
isAdmin, ok := ctx.Value("is_admin").(bool)
|
isAdmin, ok := ctx.Value(contextKeyIsAdmin).(bool)
|
||||||
return ok && isAdmin
|
return ok && isAdmin
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -451,41 +762,40 @@ func (a *Auth) AdminMiddleware(next http.HandlerFunc) http.HandlerFunc {
|
|||||||
// First check authentication
|
// First check authentication
|
||||||
cookie, err := r.Cookie("session_id")
|
cookie, err := r.Cookie("session_id")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
log.Printf("Admin authentication failed: missing session cookie for %s %s", r.Method, r.URL.Path)
|
||||||
w.WriteHeader(http.StatusUnauthorized)
|
a.writeUnauthorized(w, r)
|
||||||
json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
session, ok := a.GetSession(cookie.Value)
|
session, ok := a.GetSession(cookie.Value)
|
||||||
if !ok {
|
if !ok {
|
||||||
w.Header().Set("Content-Type", "application/json")
|
log.Printf("Admin authentication failed: invalid session cookie for %s %s", r.Method, r.URL.Path)
|
||||||
w.WriteHeader(http.StatusUnauthorized)
|
a.writeUnauthorized(w, r)
|
||||||
json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Then check admin status
|
// Then check admin status
|
||||||
if !session.IsAdmin {
|
if !session.IsAdmin {
|
||||||
|
log.Printf("Admin access denied: user %d (email: %s) attempted to access admin endpoint %s %s", session.UserID, session.Email, r.Method, r.URL.Path)
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
w.WriteHeader(http.StatusForbidden)
|
w.WriteHeader(http.StatusForbidden)
|
||||||
json.NewEncoder(w).Encode(map[string]string{"error": "Forbidden: Admin access required"})
|
json.NewEncoder(w).Encode(map[string]string{"error": "Forbidden: Admin access required"})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add user info to request context
|
// Add user info to request context using typed keys
|
||||||
ctx := context.WithValue(r.Context(), "user_id", session.UserID)
|
ctx := context.WithValue(r.Context(), contextKeyUserID, session.UserID)
|
||||||
ctx = context.WithValue(ctx, "user_email", session.Email)
|
ctx = context.WithValue(ctx, contextKeyUserEmail, session.Email)
|
||||||
ctx = context.WithValue(ctx, "user_name", session.Name)
|
ctx = context.WithValue(ctx, contextKeyUserName, session.Name)
|
||||||
ctx = context.WithValue(ctx, "is_admin", session.IsAdmin)
|
ctx = context.WithValue(ctx, contextKeyIsAdmin, session.IsAdmin)
|
||||||
next(w, r.WithContext(ctx))
|
next(w, r.WithContext(ctx))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsLocalLoginEnabled returns whether local login is enabled
|
// IsLocalLoginEnabled returns whether local login is enabled
|
||||||
// Local login is enabled when ENABLE_LOCAL_AUTH environment variable is set to "true"
|
// Checks database config first, falls back to environment variable
|
||||||
func (a *Auth) IsLocalLoginEnabled() bool {
|
func (a *Auth) IsLocalLoginEnabled() bool {
|
||||||
return os.Getenv("ENABLE_LOCAL_AUTH") == "true"
|
return a.cfg.IsLocalAuthEnabled()
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsGoogleOAuthConfigured returns whether Google OAuth is configured
|
// IsGoogleOAuthConfigured returns whether Google OAuth is configured
|
||||||
@@ -506,10 +816,12 @@ func (a *Auth) LocalLogin(username, password string) (*Session, error) {
|
|||||||
var dbEmail, dbName, passwordHash string
|
var dbEmail, dbName, passwordHash string
|
||||||
var isAdmin bool
|
var isAdmin bool
|
||||||
|
|
||||||
err := a.db.QueryRow(
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow(
|
||||||
"SELECT id, email, name, password_hash, is_admin FROM users WHERE email = ? AND oauth_provider = 'local'",
|
"SELECT id, email, name, password_hash, is_admin FROM users WHERE email = ? AND oauth_provider = 'local'",
|
||||||
email,
|
email,
|
||||||
).Scan(&userID, &dbEmail, &dbName, &passwordHash, &isAdmin)
|
).Scan(&userID, &dbEmail, &dbName, &passwordHash, &isAdmin)
|
||||||
|
})
|
||||||
|
|
||||||
if err == sql.ErrNoRows {
|
if err == sql.ErrNoRows {
|
||||||
return nil, fmt.Errorf("invalid credentials")
|
return nil, fmt.Errorf("invalid credentials")
|
||||||
@@ -534,7 +846,7 @@ func (a *Auth) LocalLogin(username, password string) (*Session, error) {
|
|||||||
Email: dbEmail,
|
Email: dbEmail,
|
||||||
Name: dbName,
|
Name: dbName,
|
||||||
IsAdmin: isAdmin,
|
IsAdmin: isAdmin,
|
||||||
ExpiresAt: time.Now().Add(24 * time.Hour),
|
ExpiresAt: time.Now().Add(SessionDuration),
|
||||||
}
|
}
|
||||||
|
|
||||||
return session, nil
|
return session, nil
|
||||||
@@ -553,7 +865,9 @@ func (a *Auth) RegisterLocalUser(email, name, password string) (*Session, error)
|
|||||||
|
|
||||||
// Check if user already exists
|
// Check if user already exists
|
||||||
var exists bool
|
var exists bool
|
||||||
err = a.db.QueryRow("SELECT EXISTS(SELECT 1 FROM users WHERE email = ?)", email).Scan(&exists)
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM users WHERE email = ?)", email).Scan(&exists)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to check if user exists: %w", err)
|
return nil, fmt.Errorf("failed to check if user exists: %w", err)
|
||||||
}
|
}
|
||||||
@@ -569,15 +883,27 @@ func (a *Auth) RegisterLocalUser(email, name, password string) (*Session, error)
|
|||||||
|
|
||||||
// Check if this is the first user (make them admin)
|
// Check if this is the first user (make them admin)
|
||||||
var userCount int
|
var userCount int
|
||||||
a.db.QueryRow("SELECT COUNT(*) FROM users").Scan(&userCount)
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT COUNT(*) FROM users").Scan(&userCount)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to check user count: %w", err)
|
||||||
|
}
|
||||||
isAdmin := userCount == 0
|
isAdmin := userCount == 0
|
||||||
|
|
||||||
// Create user
|
// Create user
|
||||||
var userID int64
|
var userID int64
|
||||||
err = a.db.QueryRow(
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
"INSERT INTO users (email, name, oauth_provider, oauth_id, password_hash, is_admin) VALUES (?, ?, 'local', ?, ?, ?) RETURNING id",
|
result, err := conn.Exec(
|
||||||
|
"INSERT INTO users (email, name, oauth_provider, oauth_id, password_hash, is_admin) VALUES (?, ?, 'local', ?, ?, ?)",
|
||||||
email, name, email, string(hashedPassword), isAdmin,
|
email, name, email, string(hashedPassword), isAdmin,
|
||||||
).Scan(&userID)
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
userID, err = result.LastInsertId()
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to create user: %w", err)
|
return nil, fmt.Errorf("failed to create user: %w", err)
|
||||||
}
|
}
|
||||||
@@ -588,7 +914,7 @@ func (a *Auth) RegisterLocalUser(email, name, password string) (*Session, error)
|
|||||||
Email: email,
|
Email: email,
|
||||||
Name: name,
|
Name: name,
|
||||||
IsAdmin: isAdmin,
|
IsAdmin: isAdmin,
|
||||||
ExpiresAt: time.Now().Add(24 * time.Hour),
|
ExpiresAt: time.Now().Add(SessionDuration),
|
||||||
}
|
}
|
||||||
|
|
||||||
return session, nil
|
return session, nil
|
||||||
@@ -598,7 +924,9 @@ func (a *Auth) RegisterLocalUser(email, name, password string) (*Session, error)
|
|||||||
func (a *Auth) ChangePassword(userID int64, oldPassword, newPassword string) error {
|
func (a *Auth) ChangePassword(userID int64, oldPassword, newPassword string) error {
|
||||||
// Get current password hash
|
// Get current password hash
|
||||||
var passwordHash string
|
var passwordHash string
|
||||||
err := a.db.QueryRow("SELECT password_hash FROM users WHERE id = ? AND oauth_provider = 'local'", userID).Scan(&passwordHash)
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT password_hash FROM users WHERE id = ? AND oauth_provider = 'local'", userID).Scan(&passwordHash)
|
||||||
|
})
|
||||||
if err == sql.ErrNoRows {
|
if err == sql.ErrNoRows {
|
||||||
return fmt.Errorf("user not found or not a local user")
|
return fmt.Errorf("user not found or not a local user")
|
||||||
}
|
}
|
||||||
@@ -623,7 +951,10 @@ func (a *Auth) ChangePassword(userID int64, oldPassword, newPassword string) err
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Update password
|
// Update password
|
||||||
_, err = a.db.Exec("UPDATE users SET password_hash = ? WHERE id = ?", string(hashedPassword), userID)
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec("UPDATE users SET password_hash = ? WHERE id = ?", string(hashedPassword), userID)
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to update password: %w", err)
|
return fmt.Errorf("failed to update password: %w", err)
|
||||||
}
|
}
|
||||||
@@ -635,7 +966,9 @@ func (a *Auth) ChangePassword(userID int64, oldPassword, newPassword string) err
|
|||||||
func (a *Auth) AdminChangePassword(targetUserID int64, newPassword string) error {
|
func (a *Auth) AdminChangePassword(targetUserID int64, newPassword string) error {
|
||||||
// Verify user exists and is a local user
|
// Verify user exists and is a local user
|
||||||
var exists bool
|
var exists bool
|
||||||
err := a.db.QueryRow("SELECT EXISTS(SELECT 1 FROM users WHERE id = ? AND oauth_provider = 'local')", targetUserID).Scan(&exists)
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM users WHERE id = ? AND oauth_provider = 'local')", targetUserID).Scan(&exists)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to check if user exists: %w", err)
|
return fmt.Errorf("failed to check if user exists: %w", err)
|
||||||
}
|
}
|
||||||
@@ -650,7 +983,10 @@ func (a *Auth) AdminChangePassword(targetUserID int64, newPassword string) error
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Update password
|
// Update password
|
||||||
_, err = a.db.Exec("UPDATE users SET password_hash = ? WHERE id = ?", string(hashedPassword), targetUserID)
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec("UPDATE users SET password_hash = ? WHERE id = ?", string(hashedPassword), targetUserID)
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to update password: %w", err)
|
return fmt.Errorf("failed to update password: %w", err)
|
||||||
}
|
}
|
||||||
@@ -661,7 +997,9 @@ func (a *Auth) AdminChangePassword(targetUserID int64, newPassword string) error
|
|||||||
// GetFirstUserID returns the ID of the first user (user with the lowest ID)
|
// GetFirstUserID returns the ID of the first user (user with the lowest ID)
|
||||||
func (a *Auth) GetFirstUserID() (int64, error) {
|
func (a *Auth) GetFirstUserID() (int64, error) {
|
||||||
var firstUserID int64
|
var firstUserID int64
|
||||||
err := a.db.QueryRow("SELECT id FROM users ORDER BY id ASC LIMIT 1").Scan(&firstUserID)
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT id FROM users ORDER BY id ASC LIMIT 1").Scan(&firstUserID)
|
||||||
|
})
|
||||||
if err == sql.ErrNoRows {
|
if err == sql.ErrNoRows {
|
||||||
return 0, fmt.Errorf("no users found")
|
return 0, fmt.Errorf("no users found")
|
||||||
}
|
}
|
||||||
@@ -675,7 +1013,9 @@ func (a *Auth) GetFirstUserID() (int64, error) {
|
|||||||
func (a *Auth) SetUserAdminStatus(targetUserID int64, isAdmin bool) error {
|
func (a *Auth) SetUserAdminStatus(targetUserID int64, isAdmin bool) error {
|
||||||
// Verify user exists
|
// Verify user exists
|
||||||
var exists bool
|
var exists bool
|
||||||
err := a.db.QueryRow("SELECT EXISTS(SELECT 1 FROM users WHERE id = ?)", targetUserID).Scan(&exists)
|
err := a.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM users WHERE id = ?)", targetUserID).Scan(&exists)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to check if user exists: %w", err)
|
return fmt.Errorf("failed to check if user exists: %w", err)
|
||||||
}
|
}
|
||||||
@@ -693,7 +1033,10 @@ func (a *Auth) SetUserAdminStatus(targetUserID int64, isAdmin bool) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Update admin status
|
// Update admin status
|
||||||
_, err = a.db.Exec("UPDATE users SET is_admin = ? WHERE id = ?", isAdmin, targetUserID)
|
err = a.db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec("UPDATE users SET is_admin = ? WHERE id = ?", isAdmin, targetUserID)
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to update admin status: %w", err)
|
return fmt.Errorf("failed to update admin status: %w", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestContextHelpers(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
ctx = context.WithValue(ctx, contextKeyUserID, int64(123))
|
||||||
|
ctx = context.WithValue(ctx, contextKeyIsAdmin, true)
|
||||||
|
|
||||||
|
id, ok := GetUserID(ctx)
|
||||||
|
if !ok || id != 123 {
|
||||||
|
t.Fatalf("GetUserID() = (%d,%v), want (123,true)", id, ok)
|
||||||
|
}
|
||||||
|
if !IsAdmin(ctx) {
|
||||||
|
t.Fatal("expected IsAdmin to be true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsProductionMode_UsesEnv(t *testing.T) {
|
||||||
|
t.Setenv("PRODUCTION", "true")
|
||||||
|
if !IsProductionMode() {
|
||||||
|
t.Fatal("expected production mode true when env is set")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteUnauthorized_BehaviorByRequestType(t *testing.T) {
|
||||||
|
a := &Auth{}
|
||||||
|
|
||||||
|
reqAPI := httptest.NewRequest(http.MethodGet, "/api/jobs", nil)
|
||||||
|
rrAPI := httptest.NewRecorder()
|
||||||
|
a.writeUnauthorized(rrAPI, reqAPI)
|
||||||
|
if rrAPI.Code != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("api code = %d", rrAPI.Code)
|
||||||
|
}
|
||||||
|
|
||||||
|
reqPage := httptest.NewRequest(http.MethodGet, "/dashboard", nil)
|
||||||
|
rrPage := httptest.NewRecorder()
|
||||||
|
a.writeUnauthorized(rrPage, reqPage)
|
||||||
|
if rrPage.Code != http.StatusFound {
|
||||||
|
t.Fatalf("page code = %d", rrPage.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsProductionMode_DefaultFalse(t *testing.T) {
|
||||||
|
_ = os.Unsetenv("PRODUCTION")
|
||||||
|
if IsProductionMode() {
|
||||||
|
t.Fatal("expected false when PRODUCTION is unset")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOAuthState_CreateConsumeAndReject(t *testing.T) {
|
||||||
|
a := &Auth{oauthStates: make(map[string]time.Time)}
|
||||||
|
state := a.CreateOAuthState()
|
||||||
|
if state == "" {
|
||||||
|
t.Fatal("expected non-empty state")
|
||||||
|
}
|
||||||
|
if !a.ConsumeOAuthState(state) {
|
||||||
|
t.Fatal("expected valid state to be accepted once")
|
||||||
|
}
|
||||||
|
if a.ConsumeOAuthState(state) {
|
||||||
|
t.Fatal("expected state to be single-use")
|
||||||
|
}
|
||||||
|
if a.ConsumeOAuthState("") {
|
||||||
|
t.Fatal("empty state must be rejected")
|
||||||
|
}
|
||||||
|
if a.ConsumeOAuthState("unknown-state") {
|
||||||
|
t.Fatal("unknown state must be rejected")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOAuthState_ExpiredRejected(t *testing.T) {
|
||||||
|
a := &Auth{oauthStates: make(map[string]time.Time)}
|
||||||
|
state := "expired-state"
|
||||||
|
a.oauthMu.Lock()
|
||||||
|
a.oauthStates[state] = time.Now().Add(-time.Minute)
|
||||||
|
a.oauthMu.Unlock()
|
||||||
|
if a.ConsumeOAuthState(state) {
|
||||||
|
t.Fatal("expired state must be rejected")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DefaultJobTokenDuration is the minimum validity period for job tokens.
|
||||||
|
const DefaultJobTokenDuration = 1 * time.Hour
|
||||||
|
|
||||||
|
// JobTokenSkew is extra lifetime added beyond configured task timeouts.
|
||||||
|
const JobTokenSkew = 15 * time.Minute
|
||||||
|
|
||||||
|
// JobTokenDuration is the current validity period for newly issued job tokens.
|
||||||
|
// Prefer JobTokenTTL() / SetJobTokenTTL; this var remains for backward-compatible reads.
|
||||||
|
var (
|
||||||
|
jobTokenTTL = DefaultJobTokenDuration
|
||||||
|
jobTokenTTLMu sync.RWMutex
|
||||||
|
)
|
||||||
|
|
||||||
|
// JobTokenTTL returns the current job token lifetime used when minting tokens.
|
||||||
|
func JobTokenTTL() time.Duration {
|
||||||
|
jobTokenTTLMu.RLock()
|
||||||
|
defer jobTokenTTLMu.RUnlock()
|
||||||
|
return jobTokenTTL
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetJobTokenTTL sets the lifetime for newly issued job tokens.
|
||||||
|
// Values below DefaultJobTokenDuration are raised to the default.
|
||||||
|
func SetJobTokenTTL(d time.Duration) {
|
||||||
|
if d < DefaultJobTokenDuration {
|
||||||
|
d = DefaultJobTokenDuration
|
||||||
|
}
|
||||||
|
jobTokenTTLMu.Lock()
|
||||||
|
jobTokenTTL = d
|
||||||
|
jobTokenTTLMu.Unlock()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ConfigureJobTokenTTLFromTimeouts sets token lifetime from the longest of the
|
||||||
|
// given task timeouts (seconds) plus JobTokenSkew, floored at DefaultJobTokenDuration.
|
||||||
|
func ConfigureJobTokenTTLFromTimeouts(timeoutSeconds ...int) time.Duration {
|
||||||
|
maxSec := 0
|
||||||
|
for _, s := range timeoutSeconds {
|
||||||
|
if s > maxSec {
|
||||||
|
maxSec = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d := time.Duration(maxSec)*time.Second + JobTokenSkew
|
||||||
|
SetJobTokenTTL(d)
|
||||||
|
return JobTokenTTL()
|
||||||
|
}
|
||||||
|
|
||||||
|
// JobTokenClaims represents the claims in a job token
|
||||||
|
type JobTokenClaims struct {
|
||||||
|
JobID int64 `json:"job_id"`
|
||||||
|
RunnerID int64 `json:"runner_id"`
|
||||||
|
TaskID int64 `json:"task_id"`
|
||||||
|
Exp int64 `json:"exp"` // Unix timestamp
|
||||||
|
}
|
||||||
|
|
||||||
|
// jobTokenSecret is the secret used to sign job tokens
|
||||||
|
// Generated once at startup and kept in memory
|
||||||
|
var jobTokenSecret []byte
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// Generate a random secret for signing job tokens
|
||||||
|
// This means tokens are invalidated on server restart, which is acceptable
|
||||||
|
// for short-lived job tokens
|
||||||
|
jobTokenSecret = make([]byte, 32)
|
||||||
|
if _, err := rand.Read(jobTokenSecret); err != nil {
|
||||||
|
panic(fmt.Sprintf("failed to generate job token secret: %v", err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GenerateJobToken creates a new job token for a specific job/runner/task combination
|
||||||
|
func GenerateJobToken(jobID, runnerID, taskID int64) (string, error) {
|
||||||
|
claims := JobTokenClaims{
|
||||||
|
JobID: jobID,
|
||||||
|
RunnerID: runnerID,
|
||||||
|
TaskID: taskID,
|
||||||
|
Exp: time.Now().Add(JobTokenTTL()).Unix(),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Encode claims to JSON
|
||||||
|
claimsJSON, err := json.Marshal(claims)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to marshal claims: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create HMAC signature
|
||||||
|
h := hmac.New(sha256.New, jobTokenSecret)
|
||||||
|
h.Write(claimsJSON)
|
||||||
|
signature := h.Sum(nil)
|
||||||
|
|
||||||
|
// Combine claims and signature: base64(claims).base64(signature)
|
||||||
|
token := base64.RawURLEncoding.EncodeToString(claimsJSON) + "." +
|
||||||
|
base64.RawURLEncoding.EncodeToString(signature)
|
||||||
|
|
||||||
|
return token, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateJobToken validates a job token and returns the claims if valid
|
||||||
|
func ValidateJobToken(token string) (*JobTokenClaims, error) {
|
||||||
|
// Split token into claims and signature
|
||||||
|
var claimsB64, sigB64 string
|
||||||
|
dotIdx := -1
|
||||||
|
for i := len(token) - 1; i >= 0; i-- {
|
||||||
|
if token[i] == '.' {
|
||||||
|
dotIdx = i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if dotIdx == -1 {
|
||||||
|
return nil, fmt.Errorf("invalid token format")
|
||||||
|
}
|
||||||
|
claimsB64 = token[:dotIdx]
|
||||||
|
sigB64 = token[dotIdx+1:]
|
||||||
|
|
||||||
|
// Decode claims
|
||||||
|
claimsJSON, err := base64.RawURLEncoding.DecodeString(claimsB64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid token encoding: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decode signature
|
||||||
|
signature, err := base64.RawURLEncoding.DecodeString(sigB64)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid signature encoding: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify signature
|
||||||
|
h := hmac.New(sha256.New, jobTokenSecret)
|
||||||
|
h.Write(claimsJSON)
|
||||||
|
expectedSig := h.Sum(nil)
|
||||||
|
if !hmac.Equal(signature, expectedSig) {
|
||||||
|
return nil, fmt.Errorf("invalid signature")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse claims
|
||||||
|
var claims JobTokenClaims
|
||||||
|
if err := json.Unmarshal(claimsJSON, &claims); err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid claims: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check expiration
|
||||||
|
if time.Now().Unix() > claims.Exp {
|
||||||
|
return nil, fmt.Errorf("token expired")
|
||||||
|
}
|
||||||
|
|
||||||
|
return &claims, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/hmac"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGenerateAndValidateJobToken_RoundTrip(t *testing.T) {
|
||||||
|
token, err := GenerateJobToken(10, 20, 30)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GenerateJobToken failed: %v", err)
|
||||||
|
}
|
||||||
|
claims, err := ValidateJobToken(token)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ValidateJobToken failed: %v", err)
|
||||||
|
}
|
||||||
|
if claims.JobID != 10 || claims.RunnerID != 20 || claims.TaskID != 30 {
|
||||||
|
t.Fatalf("unexpected claims: %+v", claims)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateJobToken_RejectsTampering(t *testing.T) {
|
||||||
|
token, err := GenerateJobToken(1, 2, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GenerateJobToken failed: %v", err)
|
||||||
|
}
|
||||||
|
parts := strings.Split(token, ".")
|
||||||
|
if len(parts) != 2 {
|
||||||
|
t.Fatalf("unexpected token format: %q", token)
|
||||||
|
}
|
||||||
|
|
||||||
|
rawClaims, err := base64.RawURLEncoding.DecodeString(parts[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("decode claims failed: %v", err)
|
||||||
|
}
|
||||||
|
var claims JobTokenClaims
|
||||||
|
if err := json.Unmarshal(rawClaims, &claims); err != nil {
|
||||||
|
t.Fatalf("unmarshal claims failed: %v", err)
|
||||||
|
}
|
||||||
|
claims.JobID = 999
|
||||||
|
tamperedClaims, _ := json.Marshal(claims)
|
||||||
|
tampered := base64.RawURLEncoding.EncodeToString(tamperedClaims) + "." + parts[1]
|
||||||
|
|
||||||
|
if _, err := ValidateJobToken(tampered); err == nil {
|
||||||
|
t.Fatal("expected signature validation error for tampered token")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateJobToken_RejectsExpired(t *testing.T) {
|
||||||
|
expiredClaims := JobTokenClaims{
|
||||||
|
JobID: 1,
|
||||||
|
RunnerID: 2,
|
||||||
|
TaskID: 3,
|
||||||
|
Exp: time.Now().Add(-time.Minute).Unix(),
|
||||||
|
}
|
||||||
|
claimsJSON, _ := json.Marshal(expiredClaims)
|
||||||
|
sigToken, err := GenerateJobToken(1, 2, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GenerateJobToken failed: %v", err)
|
||||||
|
}
|
||||||
|
parts := strings.Split(sigToken, ".")
|
||||||
|
if len(parts) != 2 {
|
||||||
|
t.Fatalf("unexpected token format: %q", sigToken)
|
||||||
|
}
|
||||||
|
// Re-sign expired payload with package secret.
|
||||||
|
h := signClaimsForTest(claimsJSON)
|
||||||
|
expiredToken := base64.RawURLEncoding.EncodeToString(claimsJSON) + "." + base64.RawURLEncoding.EncodeToString(h)
|
||||||
|
|
||||||
|
if _, err := ValidateJobToken(expiredToken); err == nil {
|
||||||
|
t.Fatal("expected token expiration error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func signClaimsForTest(claims []byte) []byte {
|
||||||
|
h := hmac.New(sha256.New, jobTokenSecret)
|
||||||
|
_, _ = h.Write(claims)
|
||||||
|
return h.Sum(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestConfigureJobTokenTTLFromTimeouts(t *testing.T) {
|
||||||
|
prev := JobTokenTTL()
|
||||||
|
t.Cleanup(func() { SetJobTokenTTL(prev) })
|
||||||
|
|
||||||
|
// Short timeouts floor at DefaultJobTokenDuration
|
||||||
|
got := ConfigureJobTokenTTLFromTimeouts(60, 120)
|
||||||
|
if got < DefaultJobTokenDuration {
|
||||||
|
t.Fatalf("TTL %v below default %v", got, DefaultJobTokenDuration)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Long encode timeout (24h) + skew must be reflected
|
||||||
|
got = ConfigureJobTokenTTLFromTimeouts(3600, 86400)
|
||||||
|
wantMin := 86400*time.Second + JobTokenSkew
|
||||||
|
if got < wantMin {
|
||||||
|
t.Fatalf("TTL %v < expected min %v for 24h encode", got, wantMin)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Newly generated tokens must use the configured TTL
|
||||||
|
token, err := GenerateJobToken(1, 2, 3)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GenerateJobToken: %v", err)
|
||||||
|
}
|
||||||
|
claims, err := ValidateJobToken(token)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ValidateJobToken: %v", err)
|
||||||
|
}
|
||||||
|
// Exp should be roughly now+TTL (allow 30s clock skew in test)
|
||||||
|
remaining := time.Until(time.Unix(claims.Exp, 0))
|
||||||
|
if remaining < wantMin-30*time.Second {
|
||||||
|
t.Fatalf("token remaining lifetime %v too short, want ~%v", remaining, wantMin)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
+167
-202
@@ -1,276 +1,241 @@
|
|||||||
package auth
|
package auth
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/hmac"
|
|
||||||
"crypto/rand"
|
"crypto/rand"
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
|
"crypto/subtle"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"jiggablend/internal/config"
|
||||||
"log"
|
"jiggablend/internal/database"
|
||||||
"net/http"
|
|
||||||
"os"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Secrets handles secret and token management
|
// Secrets handles API key management
|
||||||
type Secrets struct {
|
type Secrets struct {
|
||||||
db *sql.DB
|
db *database.DB
|
||||||
fixedRegistrationToken string // Fixed token from environment variable (reusable, never expires)
|
cfg *config.Config
|
||||||
|
RegistrationMu sync.Mutex // Protects concurrent runner registrations
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewSecrets creates a new secrets manager
|
// NewSecrets creates a new secrets manager
|
||||||
func NewSecrets(db *sql.DB) (*Secrets, error) {
|
func NewSecrets(db *database.DB, cfg *config.Config) (*Secrets, error) {
|
||||||
s := &Secrets{db: db}
|
return &Secrets{db: db, cfg: cfg}, nil
|
||||||
|
|
||||||
// Check for fixed registration token from environment
|
|
||||||
fixedToken := os.Getenv("FIXED_REGISTRATION_TOKEN")
|
|
||||||
if fixedToken != "" {
|
|
||||||
s.fixedRegistrationToken = fixedToken
|
|
||||||
log.Printf("Fixed registration token enabled (from FIXED_REGISTRATION_TOKEN env var)")
|
|
||||||
log.Printf("WARNING: Fixed registration token is reusable and never expires - use only for testing/development!")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure manager secret exists
|
|
||||||
if err := s.ensureManagerSecret(); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to ensure manager secret: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return s, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ensureManagerSecret ensures a manager secret exists in the database
|
// APIKeyInfo represents information about an API key
|
||||||
func (s *Secrets) ensureManagerSecret() error {
|
type APIKeyInfo struct {
|
||||||
var count int
|
ID int64 `json:"id"`
|
||||||
err := s.db.QueryRow("SELECT COUNT(*) FROM manager_secrets").Scan(&count)
|
Key string `json:"key"`
|
||||||
if err != nil {
|
Name string `json:"name"`
|
||||||
return fmt.Errorf("failed to check manager secrets: %w", err)
|
Description *string `json:"description,omitempty"`
|
||||||
}
|
Scope string `json:"scope"` // 'manager' or 'user'
|
||||||
|
IsActive bool `json:"is_active"`
|
||||||
if count == 0 {
|
CreatedAt time.Time `json:"created_at"`
|
||||||
// Generate new manager secret
|
CreatedBy int64 `json:"created_by"`
|
||||||
secret, err := generateSecret(32)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to generate manager secret: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
_, err = s.db.Exec("INSERT INTO manager_secrets (secret) VALUES (?)", secret)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("failed to store manager secret: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetManagerSecret retrieves the current manager secret
|
// GenerateRunnerAPIKey generates a new API key for runners
|
||||||
func (s *Secrets) GetManagerSecret() (string, error) {
|
func (s *Secrets) GenerateRunnerAPIKey(createdBy int64, name, description string, scope string) (*APIKeyInfo, error) {
|
||||||
var secret string
|
// Generate API key in format: jk_r1_abc123def456...
|
||||||
err := s.db.QueryRow("SELECT secret FROM manager_secrets ORDER BY created_at DESC LIMIT 1").Scan(&secret)
|
key, err := s.generateAPIKey()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("failed to get manager secret: %w", err)
|
return nil, fmt.Errorf("failed to generate API key: %w", err)
|
||||||
}
|
|
||||||
return secret, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GenerateRegistrationToken generates a new registration token
|
|
||||||
func (s *Secrets) GenerateRegistrationToken(createdBy int64, expiresIn time.Duration) (string, error) {
|
|
||||||
token, err := generateSecret(32)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to generate token: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
expiresAt := time.Now().Add(expiresIn)
|
// Extract prefix (first 5 chars after "jk_") and hash the full key
|
||||||
|
parts := strings.Split(key, "_")
|
||||||
|
if len(parts) < 3 {
|
||||||
|
return nil, fmt.Errorf("invalid API key format generated")
|
||||||
|
}
|
||||||
|
keyPrefix := fmt.Sprintf("%s_%s", parts[0], parts[1])
|
||||||
|
|
||||||
_, err = s.db.Exec(
|
keyHash := sha256.Sum256([]byte(key))
|
||||||
"INSERT INTO registration_tokens (token, expires_at, created_by) VALUES (?, ?, ?)",
|
keyHashStr := hex.EncodeToString(keyHash[:])
|
||||||
token, expiresAt, createdBy,
|
|
||||||
|
var keyInfo APIKeyInfo
|
||||||
|
err = s.db.With(func(conn *sql.DB) error {
|
||||||
|
result, err := conn.Exec(
|
||||||
|
`INSERT INTO runner_api_keys (key_prefix, key_hash, name, description, scope, is_active, created_by)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
||||||
|
keyPrefix, keyHashStr, name, description, scope, true, createdBy,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("failed to store registration token: %w", err)
|
return fmt.Errorf("failed to store API key: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
keyID, err := result.LastInsertId()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get inserted key ID: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return token, nil
|
// Get the inserted key info
|
||||||
}
|
err = conn.QueryRow(
|
||||||
|
`SELECT id, name, description, scope, is_active, created_at, created_by
|
||||||
|
FROM runner_api_keys WHERE id = ?`,
|
||||||
|
keyID,
|
||||||
|
).Scan(&keyInfo.ID, &keyInfo.Name, &keyInfo.Description, &keyInfo.Scope, &keyInfo.IsActive, &keyInfo.CreatedAt, &keyInfo.CreatedBy)
|
||||||
|
|
||||||
// TokenValidationResult represents the result of token validation
|
return err
|
||||||
type TokenValidationResult struct {
|
})
|
||||||
Valid bool
|
|
||||||
Reason string // "valid", "not_found", "already_used", "expired"
|
|
||||||
Error error
|
|
||||||
}
|
|
||||||
|
|
||||||
// ValidateRegistrationToken validates a registration token
|
|
||||||
func (s *Secrets) ValidateRegistrationToken(token string) (bool, error) {
|
|
||||||
result, err := s.ValidateRegistrationTokenDetailed(token)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return nil, fmt.Errorf("failed to create API key: %w", err)
|
||||||
}
|
}
|
||||||
// For backward compatibility, return just the valid boolean
|
|
||||||
return result.Valid, nil
|
keyInfo.Key = key
|
||||||
|
return &keyInfo, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ValidateRegistrationTokenDetailed validates a registration token and returns detailed result
|
// generateAPIKey generates a new API key in format jk_r1_abc123def456...
|
||||||
func (s *Secrets) ValidateRegistrationTokenDetailed(token string) (*TokenValidationResult, error) {
|
func (s *Secrets) generateAPIKey() (string, error) {
|
||||||
// Check fixed token first (if set) - it's reusable and never expires
|
// Generate random suffix
|
||||||
if s.fixedRegistrationToken != "" && token == s.fixedRegistrationToken {
|
randomBytes := make([]byte, 16)
|
||||||
log.Printf("Fixed registration token used (from FIXED_REGISTRATION_TOKEN env var)")
|
if _, err := rand.Read(randomBytes); err != nil {
|
||||||
return &TokenValidationResult{Valid: true, Reason: "valid"}, nil
|
return "", fmt.Errorf("failed to generate random bytes: %w", err)
|
||||||
|
}
|
||||||
|
randomStr := hex.EncodeToString(randomBytes)
|
||||||
|
|
||||||
|
// Generate a unique prefix (jk_r followed by 1 random digit)
|
||||||
|
prefixDigit := make([]byte, 1)
|
||||||
|
if _, err := rand.Read(prefixDigit); err != nil {
|
||||||
|
return "", fmt.Errorf("failed to generate prefix digit: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check database tokens
|
prefix := fmt.Sprintf("jk_r%d", prefixDigit[0]%10)
|
||||||
var used bool
|
key := fmt.Sprintf("%s_%s", prefix, randomStr)
|
||||||
var expiresAt time.Time
|
|
||||||
var id int64
|
|
||||||
|
|
||||||
err := s.db.QueryRow(
|
// Validate generated key format
|
||||||
"SELECT id, expires_at, used FROM registration_tokens WHERE token = ?",
|
if !strings.HasPrefix(key, "jk_r") {
|
||||||
token,
|
return "", fmt.Errorf("generated invalid API key format: %s", key)
|
||||||
).Scan(&id, &expiresAt, &used)
|
}
|
||||||
|
|
||||||
|
return key, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidateRunnerAPIKey validates an API key and returns the key ID and scope if valid
|
||||||
|
func (s *Secrets) ValidateRunnerAPIKey(apiKey string) (int64, string, error) {
|
||||||
|
if apiKey == "" {
|
||||||
|
return 0, "", fmt.Errorf("API key is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check fixed API key first (from database config). Constant-time compare.
|
||||||
|
// Fixed keys are refused entirely when production mode is on.
|
||||||
|
fixedKey := s.cfg.FixedAPIKey()
|
||||||
|
if fixedKey != "" && subtle.ConstantTimeCompare([]byte(apiKey), []byte(fixedKey)) == 1 {
|
||||||
|
if s.cfg.IsProductionMode() {
|
||||||
|
return 0, "", fmt.Errorf("fixed API key is not allowed in production mode")
|
||||||
|
}
|
||||||
|
// Return a special ID for fixed API key (doesn't exist in database)
|
||||||
|
return -1, "manager", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse API key format: jk_rX_...
|
||||||
|
if !strings.HasPrefix(apiKey, "jk_r") {
|
||||||
|
return 0, "", fmt.Errorf("invalid API key format: expected format 'jk_rX_...' where X is a number (e.g., 'jk_r1_abc123...')")
|
||||||
|
}
|
||||||
|
|
||||||
|
parts := strings.Split(apiKey, "_")
|
||||||
|
if len(parts) < 3 {
|
||||||
|
return 0, "", fmt.Errorf("invalid API key format: expected format 'jk_rX_...' with at least 3 parts separated by underscores")
|
||||||
|
}
|
||||||
|
|
||||||
|
keyPrefix := fmt.Sprintf("%s_%s", parts[0], parts[1])
|
||||||
|
|
||||||
|
// Hash the full key for comparison
|
||||||
|
keyHash := sha256.Sum256([]byte(apiKey))
|
||||||
|
keyHashStr := hex.EncodeToString(keyHash[:])
|
||||||
|
|
||||||
|
var keyID int64
|
||||||
|
var scope string
|
||||||
|
var isActive bool
|
||||||
|
|
||||||
|
err := s.db.With(func(conn *sql.DB) error {
|
||||||
|
err := conn.QueryRow(
|
||||||
|
`SELECT id, scope, is_active FROM runner_api_keys
|
||||||
|
WHERE key_prefix = ? AND key_hash = ?`,
|
||||||
|
keyPrefix, keyHashStr,
|
||||||
|
).Scan(&keyID, &scope, &isActive)
|
||||||
|
|
||||||
if err == sql.ErrNoRows {
|
if err == sql.ErrNoRows {
|
||||||
return &TokenValidationResult{Valid: false, Reason: "not_found"}, nil
|
return fmt.Errorf("API key not found or invalid - please check that the key is correct and active")
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to query token: %w", err)
|
return fmt.Errorf("failed to validate API key: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if used {
|
if !isActive {
|
||||||
return &TokenValidationResult{Valid: false, Reason: "already_used"}, nil
|
return fmt.Errorf("API key is inactive")
|
||||||
}
|
}
|
||||||
|
|
||||||
if time.Now().After(expiresAt) {
|
// Update last_used_at (don't fail if this update fails)
|
||||||
return &TokenValidationResult{Valid: false, Reason: "expired"}, nil
|
conn.Exec(`UPDATE runner_api_keys SET last_used_at = ? WHERE id = ?`, time.Now(), keyID)
|
||||||
}
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
// Mark token as used
|
|
||||||
_, err = s.db.Exec("UPDATE registration_tokens SET used = 1 WHERE id = ?", id)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to mark token as used: %w", err)
|
return 0, "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
return &TokenValidationResult{Valid: true, Reason: "valid"}, nil
|
return keyID, scope, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListRegistrationTokens lists all registration tokens
|
// ListRunnerAPIKeys lists all runner API keys
|
||||||
func (s *Secrets) ListRegistrationTokens() ([]map[string]interface{}, error) {
|
func (s *Secrets) ListRunnerAPIKeys() ([]APIKeyInfo, error) {
|
||||||
rows, err := s.db.Query(
|
var keys []APIKeyInfo
|
||||||
`SELECT id, token, expires_at, used, created_at, created_by
|
err := s.db.With(func(conn *sql.DB) error {
|
||||||
FROM registration_tokens
|
rows, err := conn.Query(
|
||||||
|
`SELECT id, key_prefix, name, description, scope, is_active, created_at, created_by
|
||||||
|
FROM runner_api_keys
|
||||||
ORDER BY created_at DESC`,
|
ORDER BY created_at DESC`,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to query tokens: %w", err)
|
return fmt.Errorf("failed to query API keys: %w", err)
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
|
|
||||||
var tokens []map[string]interface{}
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var id, createdBy sql.NullInt64
|
var key APIKeyInfo
|
||||||
var token string
|
var description sql.NullString
|
||||||
var expiresAt, createdAt time.Time
|
|
||||||
var used bool
|
|
||||||
|
|
||||||
err := rows.Scan(&id, &token, &expiresAt, &used, &createdAt, &createdBy)
|
err := rows.Scan(&key.ID, &key.Key, &key.Name, &description, &key.Scope, &key.IsActive, &key.CreatedAt, &key.CreatedBy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
tokens = append(tokens, map[string]interface{}{
|
if description.Valid {
|
||||||
"id": id.Int64,
|
key.Description = &description.String
|
||||||
"token": token,
|
}
|
||||||
"expires_at": expiresAt,
|
|
||||||
"used": used,
|
|
||||||
"created_at": createdAt,
|
|
||||||
"created_by": createdBy.Int64,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return tokens, nil
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return keys, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RevokeRegistrationToken revokes a registration token
|
// RevokeRunnerAPIKey revokes (deactivates) a runner API key
|
||||||
func (s *Secrets) RevokeRegistrationToken(tokenID int64) error {
|
func (s *Secrets) RevokeRunnerAPIKey(keyID int64) error {
|
||||||
_, err := s.db.Exec("UPDATE registration_tokens SET used = 1 WHERE id = ?", tokenID)
|
return s.db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec("UPDATE runner_api_keys SET is_active = false WHERE id = ?", keyID)
|
||||||
return err
|
return err
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// GenerateRunnerSecret generates a unique secret for a runner
|
// DeleteRunnerAPIKey deletes a runner API key
|
||||||
func (s *Secrets) GenerateRunnerSecret() (string, error) {
|
func (s *Secrets) DeleteRunnerAPIKey(keyID int64) error {
|
||||||
return generateSecret(32)
|
return s.db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec("DELETE FROM runner_api_keys WHERE id = ?", keyID)
|
||||||
|
return err
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// SignRequest signs a request with the given secret
|
|
||||||
func SignRequest(method, path, body, secret string, timestamp time.Time) string {
|
|
||||||
message := fmt.Sprintf("%s\n%s\n%s\n%d", method, path, body, timestamp.Unix())
|
|
||||||
h := hmac.New(sha256.New, []byte(secret))
|
|
||||||
h.Write([]byte(message))
|
|
||||||
return hex.EncodeToString(h.Sum(nil))
|
|
||||||
}
|
|
||||||
|
|
||||||
// VerifyRequest verifies a signed request
|
|
||||||
func VerifyRequest(r *http.Request, secret string, maxAge time.Duration) (bool, error) {
|
|
||||||
signature := r.Header.Get("X-Runner-Signature")
|
|
||||||
if signature == "" {
|
|
||||||
return false, fmt.Errorf("missing signature")
|
|
||||||
}
|
|
||||||
|
|
||||||
timestampStr := r.Header.Get("X-Runner-Timestamp")
|
|
||||||
if timestampStr == "" {
|
|
||||||
return false, fmt.Errorf("missing timestamp")
|
|
||||||
}
|
|
||||||
|
|
||||||
var timestampUnix int64
|
|
||||||
_, err := fmt.Sscanf(timestampStr, "%d", ×tampUnix)
|
|
||||||
if err != nil {
|
|
||||||
return false, fmt.Errorf("invalid timestamp: %w", err)
|
|
||||||
}
|
|
||||||
timestamp := time.Unix(timestampUnix, 0)
|
|
||||||
|
|
||||||
// Check timestamp is not too old
|
|
||||||
if time.Since(timestamp) > maxAge {
|
|
||||||
return false, fmt.Errorf("request too old")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check timestamp is not in the future (allow 1 minute clock skew)
|
|
||||||
if timestamp.After(time.Now().Add(1 * time.Minute)) {
|
|
||||||
return false, fmt.Errorf("timestamp in future")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Read body
|
|
||||||
bodyBytes, err := io.ReadAll(r.Body)
|
|
||||||
if err != nil {
|
|
||||||
return false, fmt.Errorf("failed to read body: %w", err)
|
|
||||||
}
|
|
||||||
// Restore body for handler
|
|
||||||
r.Body = io.NopCloser(strings.NewReader(string(bodyBytes)))
|
|
||||||
|
|
||||||
// Verify signature - use path without query parameters (query params are not part of signature)
|
|
||||||
// The runner signs with the path including query params, but we verify with just the path
|
|
||||||
// This is intentional - query params are for identification, not part of the signature
|
|
||||||
path := r.URL.Path
|
|
||||||
expectedSig := SignRequest(r.Method, path, string(bodyBytes), secret, timestamp)
|
|
||||||
|
|
||||||
return hmac.Equal([]byte(signature), []byte(expectedSig)), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetRunnerSecret retrieves the runner secret for a runner ID
|
|
||||||
func (s *Secrets) GetRunnerSecret(runnerID int64) (string, error) {
|
|
||||||
var secret string
|
|
||||||
err := s.db.QueryRow("SELECT runner_secret FROM runners WHERE id = ?", runnerID).Scan(&secret)
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
return "", fmt.Errorf("runner not found")
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("failed to get runner secret: %w", err)
|
|
||||||
}
|
|
||||||
if secret == "" {
|
|
||||||
return "", fmt.Errorf("runner not verified")
|
|
||||||
}
|
|
||||||
return secret, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// generateSecret generates a random secret of the given length
|
// generateSecret generates a random secret of the given length
|
||||||
func generateSecret(length int) (string, error) {
|
func generateSecret(length int) (string, error) {
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package auth
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGenerateSecret_Length(t *testing.T) {
|
||||||
|
secret, err := generateSecret(8)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("generateSecret failed: %v", err)
|
||||||
|
}
|
||||||
|
// hex encoding doubles length
|
||||||
|
if len(secret) != 16 {
|
||||||
|
t.Fatalf("unexpected secret length: %d", len(secret))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateAPIKey_Format(t *testing.T) {
|
||||||
|
s := &Secrets{}
|
||||||
|
key, err := s.generateAPIKey()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("generateAPIKey failed: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(key, "jk_r") {
|
||||||
|
t.Fatalf("unexpected key prefix: %q", key)
|
||||||
|
}
|
||||||
|
if !strings.Contains(key, "_") {
|
||||||
|
t.Fatalf("unexpected key format: %q", key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,363 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"jiggablend/internal/database"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config keys stored in database
|
||||||
|
const (
|
||||||
|
KeyGoogleClientID = "google_client_id"
|
||||||
|
KeyGoogleClientSecret = "google_client_secret"
|
||||||
|
KeyGoogleRedirectURL = "google_redirect_url"
|
||||||
|
KeyDiscordClientID = "discord_client_id"
|
||||||
|
KeyDiscordClientSecret = "discord_client_secret"
|
||||||
|
KeyDiscordRedirectURL = "discord_redirect_url"
|
||||||
|
KeyEnableLocalAuth = "enable_local_auth"
|
||||||
|
KeyFixedAPIKey = "fixed_api_key"
|
||||||
|
KeyRegistrationEnabled = "registration_enabled"
|
||||||
|
KeyProductionMode = "production_mode"
|
||||||
|
KeyAllowedOrigins = "allowed_origins"
|
||||||
|
KeyFramesPerRenderTask = "frames_per_render_task"
|
||||||
|
|
||||||
|
// Operational limits (seconds / bytes / counts)
|
||||||
|
KeyRenderTimeoutSecs = "render_timeout_seconds"
|
||||||
|
KeyEncodeTimeoutSecs = "encode_timeout_seconds"
|
||||||
|
KeyMaxUploadBytes = "max_upload_bytes"
|
||||||
|
KeySessionCookieMaxAge = "session_cookie_max_age"
|
||||||
|
KeyAPIRateLimit = "api_rate_limit"
|
||||||
|
KeyAuthRateLimit = "auth_rate_limit"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config manages application configuration stored in the database
|
||||||
|
type Config struct {
|
||||||
|
db *database.DB
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewConfig creates a new config manager
|
||||||
|
func NewConfig(db *database.DB) *Config {
|
||||||
|
return &Config{db: db}
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitializeFromEnv loads configuration from environment variables on first run
|
||||||
|
// Environment variables take precedence only if the config key doesn't exist in the database
|
||||||
|
// This allows first-run setup via env vars, then subsequent runs use database values
|
||||||
|
func (c *Config) InitializeFromEnv() error {
|
||||||
|
envMappings := []struct {
|
||||||
|
envKey string
|
||||||
|
configKey string
|
||||||
|
sensitive bool
|
||||||
|
}{
|
||||||
|
{"GOOGLE_CLIENT_ID", KeyGoogleClientID, false},
|
||||||
|
{"GOOGLE_CLIENT_SECRET", KeyGoogleClientSecret, true},
|
||||||
|
{"GOOGLE_REDIRECT_URL", KeyGoogleRedirectURL, false},
|
||||||
|
{"DISCORD_CLIENT_ID", KeyDiscordClientID, false},
|
||||||
|
{"DISCORD_CLIENT_SECRET", KeyDiscordClientSecret, true},
|
||||||
|
{"DISCORD_REDIRECT_URL", KeyDiscordRedirectURL, false},
|
||||||
|
{"ENABLE_LOCAL_AUTH", KeyEnableLocalAuth, false},
|
||||||
|
{"FIXED_API_KEY", KeyFixedAPIKey, true},
|
||||||
|
{"PRODUCTION", KeyProductionMode, false},
|
||||||
|
{"ALLOWED_ORIGINS", KeyAllowedOrigins, false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, mapping := range envMappings {
|
||||||
|
envValue := os.Getenv(mapping.envKey)
|
||||||
|
if envValue == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if config already exists in database
|
||||||
|
exists, err := c.Exists(mapping.configKey)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to check config %s: %w", mapping.configKey, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !exists {
|
||||||
|
// Store env value in database
|
||||||
|
if err := c.Set(mapping.configKey, envValue); err != nil {
|
||||||
|
return fmt.Errorf("failed to store config %s: %w", mapping.configKey, err)
|
||||||
|
}
|
||||||
|
if mapping.sensitive {
|
||||||
|
log.Printf("Stored config from env: %s = [REDACTED]", mapping.configKey)
|
||||||
|
} else {
|
||||||
|
log.Printf("Stored config from env: %s = %s", mapping.configKey, envValue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get retrieves a config value from the database
|
||||||
|
func (c *Config) Get(key string) (string, error) {
|
||||||
|
if c == nil || c.db == nil {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
var value string
|
||||||
|
err := c.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT value FROM settings WHERE key = ?", key).Scan(&value)
|
||||||
|
})
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to get config %s: %w", key, err)
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetWithDefault retrieves a config value or returns a default if not set
|
||||||
|
func (c *Config) GetWithDefault(key, defaultValue string) string {
|
||||||
|
value, err := c.Get(key)
|
||||||
|
if err != nil || value == "" {
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBool retrieves a boolean config value
|
||||||
|
func (c *Config) GetBool(key string) (bool, error) {
|
||||||
|
value, err := c.Get(key)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return value == "true" || value == "1", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBoolWithDefault retrieves a boolean config value or returns a default
|
||||||
|
func (c *Config) GetBoolWithDefault(key string, defaultValue bool) bool {
|
||||||
|
value, err := c.GetBool(key)
|
||||||
|
if err != nil {
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
// If the key doesn't exist, Get returns empty string which becomes false
|
||||||
|
// Check if key exists to distinguish between "false" and "not set"
|
||||||
|
exists, _ := c.Exists(key)
|
||||||
|
if !exists {
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetInt retrieves an integer config value
|
||||||
|
func (c *Config) GetInt(key string) (int, error) {
|
||||||
|
value, err := c.Get(key)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if value == "" {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
return strconv.Atoi(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetIntWithDefault retrieves an integer config value or returns a default
|
||||||
|
func (c *Config) GetIntWithDefault(key string, defaultValue int) int {
|
||||||
|
value, err := c.GetInt(key)
|
||||||
|
if err != nil {
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
exists, _ := c.Exists(key)
|
||||||
|
if !exists {
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set stores a config value in the database
|
||||||
|
func (c *Config) Set(key, value string) error {
|
||||||
|
// Use upsert pattern
|
||||||
|
exists, err := c.Exists(key)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = c.db.With(func(conn *sql.DB) error {
|
||||||
|
if exists {
|
||||||
|
_, err = conn.Exec(
|
||||||
|
"UPDATE settings SET value = ?, updated_at = CURRENT_TIMESTAMP WHERE key = ?",
|
||||||
|
value, key,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
_, err = conn.Exec(
|
||||||
|
"INSERT INTO settings (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)",
|
||||||
|
key, value,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to set config %s: %w", key, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetBool stores a boolean config value
|
||||||
|
func (c *Config) SetBool(key string, value bool) error {
|
||||||
|
strValue := "false"
|
||||||
|
if value {
|
||||||
|
strValue = "true"
|
||||||
|
}
|
||||||
|
return c.Set(key, strValue)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetInt stores an integer config value
|
||||||
|
func (c *Config) SetInt(key string, value int) error {
|
||||||
|
return c.Set(key, strconv.Itoa(value))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete removes a config value from the database
|
||||||
|
func (c *Config) Delete(key string) error {
|
||||||
|
err := c.db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec("DELETE FROM settings WHERE key = ?", key)
|
||||||
|
return err
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to delete config %s: %w", key, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exists checks if a config key exists in the database
|
||||||
|
func (c *Config) Exists(key string) (bool, error) {
|
||||||
|
var exists bool
|
||||||
|
err := c.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM settings WHERE key = ?)", key).Scan(&exists)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("failed to check config existence %s: %w", key, err)
|
||||||
|
}
|
||||||
|
return exists, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAll returns all config values (for debugging/admin purposes)
|
||||||
|
func (c *Config) GetAll() (map[string]string, error) {
|
||||||
|
var result map[string]string
|
||||||
|
err := c.db.With(func(conn *sql.DB) error {
|
||||||
|
rows, err := conn.Query("SELECT key, value FROM settings")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get all config: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
result = make(map[string]string)
|
||||||
|
for rows.Next() {
|
||||||
|
var key, value string
|
||||||
|
if err := rows.Scan(&key, &value); err != nil {
|
||||||
|
return fmt.Errorf("failed to scan config row: %w", err)
|
||||||
|
}
|
||||||
|
result[key] = value
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Convenience methods for specific config values ---
|
||||||
|
|
||||||
|
// GoogleClientID returns the Google OAuth client ID
|
||||||
|
func (c *Config) GoogleClientID() string {
|
||||||
|
return c.GetWithDefault(KeyGoogleClientID, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoogleClientSecret returns the Google OAuth client secret
|
||||||
|
func (c *Config) GoogleClientSecret() string {
|
||||||
|
return c.GetWithDefault(KeyGoogleClientSecret, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoogleRedirectURL returns the Google OAuth redirect URL
|
||||||
|
func (c *Config) GoogleRedirectURL() string {
|
||||||
|
return c.GetWithDefault(KeyGoogleRedirectURL, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiscordClientID returns the Discord OAuth client ID
|
||||||
|
func (c *Config) DiscordClientID() string {
|
||||||
|
return c.GetWithDefault(KeyDiscordClientID, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiscordClientSecret returns the Discord OAuth client secret
|
||||||
|
func (c *Config) DiscordClientSecret() string {
|
||||||
|
return c.GetWithDefault(KeyDiscordClientSecret, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// DiscordRedirectURL returns the Discord OAuth redirect URL
|
||||||
|
func (c *Config) DiscordRedirectURL() string {
|
||||||
|
return c.GetWithDefault(KeyDiscordRedirectURL, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsLocalAuthEnabled returns whether local authentication is enabled
|
||||||
|
func (c *Config) IsLocalAuthEnabled() bool {
|
||||||
|
return c.GetBoolWithDefault(KeyEnableLocalAuth, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// FixedAPIKey returns the fixed API key for testing
|
||||||
|
func (c *Config) FixedAPIKey() string {
|
||||||
|
return c.GetWithDefault(KeyFixedAPIKey, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsProductionMode returns whether production mode is enabled.
|
||||||
|
// True if PRODUCTION=true env is set OR DB setting production_mode is true.
|
||||||
|
// This is the single source of truth for Secure cookies, CORS, rate limits, and WS origin.
|
||||||
|
func (c *Config) IsProductionMode() bool {
|
||||||
|
if os.Getenv("PRODUCTION") == "true" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if c == nil || c.db == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return c.GetBoolWithDefault(KeyProductionMode, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AllowedOrigins returns the allowed CORS origins
|
||||||
|
func (c *Config) AllowedOrigins() string {
|
||||||
|
return c.GetWithDefault(KeyAllowedOrigins, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetFramesPerRenderTask returns how many frames to include per render task (min 1, default 1).
|
||||||
|
func (c *Config) GetFramesPerRenderTask() int {
|
||||||
|
n := c.GetIntWithDefault(KeyFramesPerRenderTask, 1)
|
||||||
|
if n < 1 {
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// RenderTimeoutSeconds returns the per-frame render timeout in seconds (default 3600 = 1 hour).
|
||||||
|
func (c *Config) RenderTimeoutSeconds() int {
|
||||||
|
return c.GetIntWithDefault(KeyRenderTimeoutSecs, 3600)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeTimeoutSeconds returns the video encode timeout in seconds (default 86400 = 24 hours).
|
||||||
|
func (c *Config) EncodeTimeoutSeconds() int {
|
||||||
|
return c.GetIntWithDefault(KeyEncodeTimeoutSecs, 86400)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaxUploadBytes returns the maximum upload size in bytes (default 50 GB).
|
||||||
|
func (c *Config) MaxUploadBytes() int64 {
|
||||||
|
v := c.GetIntWithDefault(KeyMaxUploadBytes, 50<<30)
|
||||||
|
return int64(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SessionCookieMaxAgeSec returns the session cookie max-age in seconds (default 86400 = 24 hours).
|
||||||
|
func (c *Config) SessionCookieMaxAgeSec() int {
|
||||||
|
return c.GetIntWithDefault(KeySessionCookieMaxAge, 86400)
|
||||||
|
}
|
||||||
|
|
||||||
|
// APIRateLimitPerMinute returns the API rate limit (requests per minute per IP, default 100).
|
||||||
|
func (c *Config) APIRateLimitPerMinute() int {
|
||||||
|
return c.GetIntWithDefault(KeyAPIRateLimit, 100)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AuthRateLimitPerMinute returns the auth rate limit (requests per minute per IP, default 10).
|
||||||
|
func (c *Config) AuthRateLimitPerMinute() int {
|
||||||
|
return c.GetIntWithDefault(KeyAuthRateLimit, 10)
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
package config
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"jiggablend/internal/database"
|
||||||
|
)
|
||||||
|
|
||||||
|
func newTestConfig(t *testing.T) *Config {
|
||||||
|
t.Helper()
|
||||||
|
db, err := database.NewDB(filepath.Join(t.TempDir(), "cfg.db"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewDB failed: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = db.Close() })
|
||||||
|
return NewConfig(db)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetGetExistsDelete(t *testing.T) {
|
||||||
|
cfg := newTestConfig(t)
|
||||||
|
|
||||||
|
if err := cfg.Set("alpha", "1"); err != nil {
|
||||||
|
t.Fatalf("Set failed: %v", err)
|
||||||
|
}
|
||||||
|
v, err := cfg.Get("alpha")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Get failed: %v", err)
|
||||||
|
}
|
||||||
|
if v != "1" {
|
||||||
|
t.Fatalf("unexpected value: %q", v)
|
||||||
|
}
|
||||||
|
|
||||||
|
exists, err := cfg.Exists("alpha")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Exists failed: %v", err)
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
t.Fatal("expected key to exist")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := cfg.Delete("alpha"); err != nil {
|
||||||
|
t.Fatalf("Delete failed: %v", err)
|
||||||
|
}
|
||||||
|
exists, err = cfg.Exists("alpha")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Exists after delete failed: %v", err)
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
t.Fatal("expected key to be deleted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetIntWithDefault_AndMinimumFrameTask(t *testing.T) {
|
||||||
|
cfg := newTestConfig(t)
|
||||||
|
if got := cfg.GetIntWithDefault("missing", 17); got != 17 {
|
||||||
|
t.Fatalf("expected default value, got %d", got)
|
||||||
|
}
|
||||||
|
if err := cfg.SetInt(KeyFramesPerRenderTask, 0); err != nil {
|
||||||
|
t.Fatalf("SetInt failed: %v", err)
|
||||||
|
}
|
||||||
|
if got := cfg.GetFramesPerRenderTask(); got != 1 {
|
||||||
|
t.Fatalf("expected clamped value 1, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
-- Drop indexes
|
||||||
|
DROP INDEX IF EXISTS idx_sessions_expires_at;
|
||||||
|
DROP INDEX IF EXISTS idx_sessions_user_id;
|
||||||
|
DROP INDEX IF EXISTS idx_sessions_session_id;
|
||||||
|
DROP INDEX IF EXISTS idx_runners_last_heartbeat;
|
||||||
|
DROP INDEX IF EXISTS idx_task_steps_task_id;
|
||||||
|
DROP INDEX IF EXISTS idx_task_logs_runner_id;
|
||||||
|
DROP INDEX IF EXISTS idx_task_logs_task_id_id;
|
||||||
|
DROP INDEX IF EXISTS idx_task_logs_task_id_created_at;
|
||||||
|
DROP INDEX IF EXISTS idx_runners_api_key_id;
|
||||||
|
DROP INDEX IF EXISTS idx_runner_api_keys_created_by;
|
||||||
|
DROP INDEX IF EXISTS idx_runner_api_keys_active;
|
||||||
|
DROP INDEX IF EXISTS idx_runner_api_keys_prefix;
|
||||||
|
DROP INDEX IF EXISTS idx_job_files_job_id;
|
||||||
|
DROP INDEX IF EXISTS idx_tasks_started_at;
|
||||||
|
DROP INDEX IF EXISTS idx_tasks_job_status;
|
||||||
|
DROP INDEX IF EXISTS idx_tasks_status;
|
||||||
|
DROP INDEX IF EXISTS idx_tasks_runner_id;
|
||||||
|
DROP INDEX IF EXISTS idx_tasks_job_id;
|
||||||
|
DROP INDEX IF EXISTS idx_jobs_user_status_created;
|
||||||
|
DROP INDEX IF EXISTS idx_jobs_status;
|
||||||
|
DROP INDEX IF EXISTS idx_jobs_user_id;
|
||||||
|
|
||||||
|
-- Drop tables (order matters due to foreign keys)
|
||||||
|
DROP TABLE IF EXISTS sessions;
|
||||||
|
DROP TABLE IF EXISTS settings;
|
||||||
|
DROP TABLE IF EXISTS task_steps;
|
||||||
|
DROP TABLE IF EXISTS task_logs;
|
||||||
|
DROP TABLE IF EXISTS manager_secrets;
|
||||||
|
DROP TABLE IF EXISTS job_files;
|
||||||
|
DROP TABLE IF EXISTS tasks;
|
||||||
|
DROP TABLE IF EXISTS runners;
|
||||||
|
DROP TABLE IF EXISTS jobs;
|
||||||
|
DROP TABLE IF EXISTS runner_api_keys;
|
||||||
|
DROP TABLE IF EXISTS users;
|
||||||
|
|
||||||
@@ -0,0 +1,184 @@
|
|||||||
|
-- Enable foreign keys for SQLite
|
||||||
|
PRAGMA foreign_keys = ON;
|
||||||
|
|
||||||
|
-- Users table
|
||||||
|
CREATE TABLE users (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
email TEXT UNIQUE NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
oauth_provider TEXT NOT NULL,
|
||||||
|
oauth_id TEXT NOT NULL,
|
||||||
|
password_hash TEXT,
|
||||||
|
is_admin INTEGER NOT NULL DEFAULT 0,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
UNIQUE(oauth_provider, oauth_id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Runner API keys table
|
||||||
|
CREATE TABLE runner_api_keys (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
key_prefix TEXT NOT NULL,
|
||||||
|
key_hash TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
scope TEXT NOT NULL DEFAULT 'user',
|
||||||
|
is_active INTEGER NOT NULL DEFAULT 1,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
created_by INTEGER,
|
||||||
|
FOREIGN KEY (created_by) REFERENCES users(id),
|
||||||
|
UNIQUE(key_prefix)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Jobs table
|
||||||
|
CREATE TABLE jobs (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
user_id INTEGER NOT NULL,
|
||||||
|
job_type TEXT NOT NULL DEFAULT 'render',
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
progress REAL NOT NULL DEFAULT 0.0,
|
||||||
|
frame_start INTEGER,
|
||||||
|
frame_end INTEGER,
|
||||||
|
output_format TEXT,
|
||||||
|
blend_metadata TEXT,
|
||||||
|
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
max_retries INTEGER NOT NULL DEFAULT 3,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
started_at TIMESTAMP,
|
||||||
|
completed_at TIMESTAMP,
|
||||||
|
error_message TEXT,
|
||||||
|
assigned_runner_id INTEGER,
|
||||||
|
FOREIGN KEY (user_id) REFERENCES users(id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Runners table
|
||||||
|
CREATE TABLE runners (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
hostname TEXT NOT NULL,
|
||||||
|
ip_address TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'offline',
|
||||||
|
last_heartbeat TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
capabilities TEXT,
|
||||||
|
api_key_id INTEGER,
|
||||||
|
api_key_scope TEXT NOT NULL DEFAULT 'user',
|
||||||
|
priority INTEGER NOT NULL DEFAULT 100,
|
||||||
|
fingerprint TEXT,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
FOREIGN KEY (api_key_id) REFERENCES runner_api_keys(id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Tasks table
|
||||||
|
CREATE TABLE tasks (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
job_id INTEGER NOT NULL,
|
||||||
|
runner_id INTEGER,
|
||||||
|
frame INTEGER NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
output_path TEXT,
|
||||||
|
task_type TEXT NOT NULL DEFAULT 'render',
|
||||||
|
current_step TEXT,
|
||||||
|
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
max_retries INTEGER NOT NULL DEFAULT 3,
|
||||||
|
runner_failure_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
timeout_seconds INTEGER,
|
||||||
|
condition TEXT,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
started_at TIMESTAMP,
|
||||||
|
completed_at TIMESTAMP,
|
||||||
|
error_message TEXT,
|
||||||
|
FOREIGN KEY (job_id) REFERENCES jobs(id),
|
||||||
|
FOREIGN KEY (runner_id) REFERENCES runners(id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Job files table
|
||||||
|
CREATE TABLE job_files (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
job_id INTEGER NOT NULL,
|
||||||
|
file_type TEXT NOT NULL,
|
||||||
|
file_path TEXT NOT NULL,
|
||||||
|
file_name TEXT NOT NULL,
|
||||||
|
file_size INTEGER NOT NULL,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
FOREIGN KEY (job_id) REFERENCES jobs(id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Manager secrets table
|
||||||
|
CREATE TABLE manager_secrets (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
secret TEXT UNIQUE NOT NULL,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Task logs table
|
||||||
|
CREATE TABLE task_logs (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
task_id INTEGER NOT NULL,
|
||||||
|
runner_id INTEGER,
|
||||||
|
log_level TEXT NOT NULL,
|
||||||
|
message TEXT NOT NULL,
|
||||||
|
step_name TEXT,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
FOREIGN KEY (task_id) REFERENCES tasks(id),
|
||||||
|
FOREIGN KEY (runner_id) REFERENCES runners(id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Task steps table
|
||||||
|
CREATE TABLE task_steps (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
task_id INTEGER NOT NULL,
|
||||||
|
step_name TEXT NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
started_at TIMESTAMP,
|
||||||
|
completed_at TIMESTAMP,
|
||||||
|
duration_ms INTEGER,
|
||||||
|
error_message TEXT,
|
||||||
|
FOREIGN KEY (task_id) REFERENCES tasks(id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Settings table
|
||||||
|
CREATE TABLE settings (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL,
|
||||||
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Sessions table
|
||||||
|
CREATE TABLE sessions (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
session_id TEXT UNIQUE NOT NULL,
|
||||||
|
user_id INTEGER NOT NULL,
|
||||||
|
email TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
is_admin INTEGER NOT NULL DEFAULT 0,
|
||||||
|
expires_at TIMESTAMP NOT NULL,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
FOREIGN KEY (user_id) REFERENCES users(id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Indexes
|
||||||
|
CREATE INDEX idx_jobs_user_id ON jobs(user_id);
|
||||||
|
CREATE INDEX idx_jobs_status ON jobs(status);
|
||||||
|
CREATE INDEX idx_jobs_user_status_created ON jobs(user_id, status, created_at DESC);
|
||||||
|
CREATE INDEX idx_tasks_job_id ON tasks(job_id);
|
||||||
|
CREATE INDEX idx_tasks_runner_id ON tasks(runner_id);
|
||||||
|
CREATE INDEX idx_tasks_status ON tasks(status);
|
||||||
|
CREATE INDEX idx_tasks_job_status ON tasks(job_id, status);
|
||||||
|
CREATE INDEX idx_tasks_started_at ON tasks(started_at);
|
||||||
|
CREATE INDEX idx_job_files_job_id ON job_files(job_id);
|
||||||
|
CREATE INDEX idx_runner_api_keys_prefix ON runner_api_keys(key_prefix);
|
||||||
|
CREATE INDEX idx_runner_api_keys_active ON runner_api_keys(is_active);
|
||||||
|
CREATE INDEX idx_runner_api_keys_created_by ON runner_api_keys(created_by);
|
||||||
|
CREATE INDEX idx_runners_api_key_id ON runners(api_key_id);
|
||||||
|
CREATE INDEX idx_task_logs_task_id_created_at ON task_logs(task_id, created_at);
|
||||||
|
CREATE INDEX idx_task_logs_task_id_id ON task_logs(task_id, id DESC);
|
||||||
|
CREATE INDEX idx_task_logs_runner_id ON task_logs(runner_id);
|
||||||
|
CREATE INDEX idx_task_steps_task_id ON task_steps(task_id);
|
||||||
|
CREATE INDEX idx_runners_last_heartbeat ON runners(last_heartbeat);
|
||||||
|
CREATE INDEX idx_sessions_session_id ON sessions(session_id);
|
||||||
|
CREATE INDEX idx_sessions_user_id ON sessions(user_id);
|
||||||
|
CREATE INDEX idx_sessions_expires_at ON sessions(expires_at);
|
||||||
|
|
||||||
|
-- Initialize registration_enabled setting
|
||||||
|
INSERT INTO settings (key, value, updated_at) VALUES ('registration_enabled', 'true', CURRENT_TIMESTAMP);
|
||||||
|
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
-- SQLite does not support DROP COLUMN directly; recreate table without frame_end
|
||||||
|
CREATE TABLE tasks_new (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
job_id INTEGER NOT NULL,
|
||||||
|
runner_id INTEGER,
|
||||||
|
frame INTEGER NOT NULL,
|
||||||
|
status TEXT NOT NULL DEFAULT 'pending',
|
||||||
|
output_path TEXT,
|
||||||
|
task_type TEXT NOT NULL DEFAULT 'render',
|
||||||
|
current_step TEXT,
|
||||||
|
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
max_retries INTEGER NOT NULL DEFAULT 3,
|
||||||
|
runner_failure_count INTEGER NOT NULL DEFAULT 0,
|
||||||
|
timeout_seconds INTEGER,
|
||||||
|
condition TEXT,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
started_at TIMESTAMP,
|
||||||
|
completed_at TIMESTAMP,
|
||||||
|
error_message TEXT,
|
||||||
|
FOREIGN KEY (job_id) REFERENCES jobs(id),
|
||||||
|
FOREIGN KEY (runner_id) REFERENCES runners(id)
|
||||||
|
);
|
||||||
|
INSERT INTO tasks_new (id, job_id, runner_id, frame, status, output_path, task_type, current_step, retry_count, max_retries, runner_failure_count, timeout_seconds, condition, created_at, started_at, completed_at, error_message)
|
||||||
|
SELECT id, job_id, runner_id, frame, status, output_path, task_type, current_step, retry_count, max_retries, runner_failure_count, timeout_seconds, condition, created_at, started_at, completed_at, error_message FROM tasks;
|
||||||
|
DROP TABLE tasks;
|
||||||
|
ALTER TABLE tasks_new RENAME TO tasks;
|
||||||
|
CREATE INDEX idx_tasks_job_id ON tasks(job_id);
|
||||||
|
CREATE INDEX idx_tasks_runner_id ON tasks(runner_id);
|
||||||
|
CREATE INDEX idx_tasks_status ON tasks(status);
|
||||||
|
CREATE INDEX idx_tasks_job_status ON tasks(job_id, status);
|
||||||
|
CREATE INDEX idx_tasks_started_at ON tasks(started_at);
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
-- Add frame_end to tasks for range-based render tasks (NULL = single frame, same as frame)
|
||||||
|
ALTER TABLE tasks ADD COLUMN frame_end INTEGER;
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
-- SQLite does not support DROP COLUMN directly; recreate the table without last_used_at.
|
||||||
|
CREATE TABLE runner_api_keys_backup (
|
||||||
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
|
key_prefix TEXT NOT NULL,
|
||||||
|
key_hash TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
description TEXT,
|
||||||
|
scope TEXT NOT NULL DEFAULT 'user',
|
||||||
|
is_active INTEGER NOT NULL DEFAULT 1,
|
||||||
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
created_by INTEGER,
|
||||||
|
FOREIGN KEY (created_by) REFERENCES users(id),
|
||||||
|
UNIQUE(key_prefix)
|
||||||
|
);
|
||||||
|
|
||||||
|
INSERT INTO runner_api_keys_backup SELECT id, key_prefix, key_hash, name, description, scope, is_active, created_at, created_by FROM runner_api_keys;
|
||||||
|
|
||||||
|
DROP TABLE runner_api_keys;
|
||||||
|
|
||||||
|
ALTER TABLE runner_api_keys_backup RENAME TO runner_api_keys;
|
||||||
|
|
||||||
|
CREATE INDEX idx_runner_api_keys_prefix ON runner_api_keys(key_prefix);
|
||||||
|
CREATE INDEX idx_runner_api_keys_active ON runner_api_keys(is_active);
|
||||||
|
CREATE INDEX idx_runner_api_keys_created_by ON runner_api_keys(created_by);
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE runner_api_keys ADD COLUMN last_used_at TIMESTAMP;
|
||||||
+115
-209
@@ -2,252 +2,158 @@ package database
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"embed"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
"log"
|
"log"
|
||||||
|
|
||||||
_ "github.com/marcboeker/go-duckdb/v2"
|
"github.com/golang-migrate/migrate/v4"
|
||||||
|
"github.com/golang-migrate/migrate/v4/database/sqlite3"
|
||||||
|
"github.com/golang-migrate/migrate/v4/source/iofs"
|
||||||
|
_ "github.com/mattn/go-sqlite3"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
//go:embed migrations/*.sql
|
||||||
|
var migrationsFS embed.FS
|
||||||
|
|
||||||
// DB wraps the database connection
|
// DB wraps the database connection
|
||||||
|
// Note: No mutex needed - we only have one connection per process and SQLite with WAL mode
|
||||||
|
// handles concurrent access safely
|
||||||
type DB struct {
|
type DB struct {
|
||||||
*sql.DB
|
db *sql.DB
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewDB creates a new database connection
|
// NewDB creates a new database connection
|
||||||
func NewDB(dbPath string) (*DB, error) {
|
func NewDB(dbPath string) (*DB, error) {
|
||||||
db, err := sql.Open("duckdb", dbPath)
|
// Use WAL mode for better concurrency (allows readers and writers simultaneously)
|
||||||
|
// Add timeout and busy handler for better concurrent access
|
||||||
|
db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=5000")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to open database: %w", err)
|
return nil, fmt.Errorf("failed to open database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Configure connection pool for better concurrency
|
||||||
|
// SQLite with WAL mode supports multiple concurrent readers and one writer
|
||||||
|
// Increasing pool size allows multiple HTTP requests to query the database simultaneously
|
||||||
|
// This prevents blocking when multiple requests come in (e.g., on page refresh)
|
||||||
|
db.SetMaxOpenConns(10) // Allow up to 10 concurrent connections
|
||||||
|
db.SetMaxIdleConns(5) // Keep 5 idle connections ready
|
||||||
|
db.SetConnMaxLifetime(0) // Connections don't expire
|
||||||
|
|
||||||
if err := db.Ping(); err != nil {
|
if err := db.Ping(); err != nil {
|
||||||
return nil, fmt.Errorf("failed to ping database: %w", err)
|
return nil, fmt.Errorf("failed to ping database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
database := &DB{DB: db}
|
// Enable foreign keys for SQLite
|
||||||
|
if _, err := db.Exec("PRAGMA foreign_keys = ON"); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to enable foreign keys: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enable WAL mode explicitly (in case the connection string didn't work)
|
||||||
|
if _, err := db.Exec("PRAGMA journal_mode = WAL"); err != nil {
|
||||||
|
log.Printf("Warning: Failed to enable WAL mode: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
database := &DB{db: db}
|
||||||
if err := database.migrate(); err != nil {
|
if err := database.migrate(); err != nil {
|
||||||
return nil, fmt.Errorf("failed to migrate database: %w", err)
|
return nil, fmt.Errorf("failed to migrate database: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Verify connection is still open after migration
|
||||||
|
if err := db.Ping(); err != nil {
|
||||||
|
return nil, fmt.Errorf("database connection closed after migration: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
return database, nil
|
return database, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// migrate runs database migrations
|
// With executes a function with access to the database
|
||||||
|
// The function receives the underlying *sql.DB connection
|
||||||
|
// No mutex needed - single connection + WAL mode handles concurrency
|
||||||
|
func (db *DB) With(fn func(*sql.DB) error) error {
|
||||||
|
return fn(db.db)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithTx executes a function within a transaction
|
||||||
|
// The function receives a *sql.Tx transaction
|
||||||
|
// If the function returns an error, the transaction is rolled back
|
||||||
|
// If the function returns nil, the transaction is committed
|
||||||
|
// No mutex needed - single connection + WAL mode handles concurrency
|
||||||
|
func (db *DB) WithTx(fn func(*sql.Tx) error) error {
|
||||||
|
tx, err := db.db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to begin transaction: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := fn(tx); err != nil {
|
||||||
|
if rbErr := tx.Rollback(); rbErr != nil {
|
||||||
|
return fmt.Errorf("transaction error: %w, rollback error: %v", err, rbErr)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
return fmt.Errorf("failed to commit transaction: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// migrate runs database migrations using golang-migrate
|
||||||
func (db *DB) migrate() error {
|
func (db *DB) migrate() error {
|
||||||
// Create sequences for auto-incrementing primary keys
|
// Create SQLite driver instance
|
||||||
sequences := []string{
|
// Note: We use db.db directly since we're in the same package and this is called during initialization
|
||||||
`CREATE SEQUENCE IF NOT EXISTS seq_users_id START 1`,
|
driver, err := sqlite3.WithInstance(db.db, &sqlite3.Config{})
|
||||||
`CREATE SEQUENCE IF NOT EXISTS seq_jobs_id START 1`,
|
if err != nil {
|
||||||
`CREATE SEQUENCE IF NOT EXISTS seq_runners_id START 1`,
|
return fmt.Errorf("failed to create sqlite3 driver: %w", err)
|
||||||
`CREATE SEQUENCE IF NOT EXISTS seq_tasks_id START 1`,
|
|
||||||
`CREATE SEQUENCE IF NOT EXISTS seq_job_files_id START 1`,
|
|
||||||
`CREATE SEQUENCE IF NOT EXISTS seq_manager_secrets_id START 1`,
|
|
||||||
`CREATE SEQUENCE IF NOT EXISTS seq_registration_tokens_id START 1`,
|
|
||||||
`CREATE SEQUENCE IF NOT EXISTS seq_task_logs_id START 1`,
|
|
||||||
`CREATE SEQUENCE IF NOT EXISTS seq_task_steps_id START 1`,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, seq := range sequences {
|
// Create embedded filesystem source
|
||||||
if _, err := db.Exec(seq); err != nil {
|
migrationFS, err := fs.Sub(migrationsFS, "migrations")
|
||||||
return fmt.Errorf("failed to create sequence: %w", err)
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create migration filesystem: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
sourceDriver, err := iofs.New(migrationFS, ".")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create iofs source driver: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create migrate instance
|
||||||
|
m, err := migrate.NewWithInstance("iofs", sourceDriver, "sqlite3", driver)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create migrate instance: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run migrations
|
||||||
|
if err := m.Up(); err != nil {
|
||||||
|
// If the error is "no change", that's fine - database is already up to date
|
||||||
|
if err == migrate.ErrNoChange {
|
||||||
|
log.Printf("Database is already up to date")
|
||||||
|
// Don't close migrate instance - it may close the database connection
|
||||||
|
// The migrate instance will be garbage collected
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
// Don't close migrate instance on error either - it may close the DB
|
||||||
|
return fmt.Errorf("failed to run migrations: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
schema := `
|
// Don't close the migrate instance - with sqlite3.WithInstance, closing it
|
||||||
CREATE TABLE IF NOT EXISTS users (
|
// may close the underlying database connection. The migrate instance will
|
||||||
id BIGINT PRIMARY KEY DEFAULT nextval('seq_users_id'),
|
// be garbage collected when it goes out of scope.
|
||||||
email TEXT UNIQUE NOT NULL,
|
// If we need to close it later, we can store it in the DB struct and close
|
||||||
name TEXT NOT NULL,
|
// it when DB.Close() is called, but for now we'll let it be GC'd.
|
||||||
oauth_provider TEXT NOT NULL,
|
|
||||||
oauth_id TEXT NOT NULL,
|
|
||||||
password_hash TEXT,
|
|
||||||
is_admin BOOLEAN NOT NULL DEFAULT false,
|
|
||||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
UNIQUE(oauth_provider, oauth_id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS jobs (
|
|
||||||
id BIGINT PRIMARY KEY DEFAULT nextval('seq_jobs_id'),
|
|
||||||
user_id BIGINT NOT NULL,
|
|
||||||
job_type TEXT NOT NULL DEFAULT 'render',
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
status TEXT NOT NULL DEFAULT 'pending',
|
|
||||||
progress REAL NOT NULL DEFAULT 0.0,
|
|
||||||
frame_start INTEGER,
|
|
||||||
frame_end INTEGER,
|
|
||||||
output_format TEXT,
|
|
||||||
allow_parallel_runners BOOLEAN,
|
|
||||||
timeout_seconds INTEGER DEFAULT 86400,
|
|
||||||
blend_metadata TEXT,
|
|
||||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
started_at TIMESTAMP,
|
|
||||||
completed_at TIMESTAMP,
|
|
||||||
error_message TEXT,
|
|
||||||
FOREIGN KEY (user_id) REFERENCES users(id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS runners (
|
|
||||||
id BIGINT PRIMARY KEY DEFAULT nextval('seq_runners_id'),
|
|
||||||
name TEXT NOT NULL,
|
|
||||||
hostname TEXT NOT NULL,
|
|
||||||
ip_address TEXT NOT NULL,
|
|
||||||
status TEXT NOT NULL DEFAULT 'offline',
|
|
||||||
last_heartbeat TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
capabilities TEXT,
|
|
||||||
registration_token TEXT,
|
|
||||||
runner_secret TEXT,
|
|
||||||
manager_secret TEXT,
|
|
||||||
verified BOOLEAN NOT NULL DEFAULT false,
|
|
||||||
priority INTEGER NOT NULL DEFAULT 100,
|
|
||||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS tasks (
|
|
||||||
id BIGINT PRIMARY KEY DEFAULT nextval('seq_tasks_id'),
|
|
||||||
job_id BIGINT NOT NULL,
|
|
||||||
runner_id BIGINT,
|
|
||||||
frame_start INTEGER NOT NULL,
|
|
||||||
frame_end INTEGER NOT NULL,
|
|
||||||
status TEXT NOT NULL DEFAULT 'pending',
|
|
||||||
output_path TEXT,
|
|
||||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
started_at TIMESTAMP,
|
|
||||||
completed_at TIMESTAMP,
|
|
||||||
error_message TEXT
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS job_files (
|
|
||||||
id BIGINT PRIMARY KEY DEFAULT nextval('seq_job_files_id'),
|
|
||||||
job_id BIGINT NOT NULL,
|
|
||||||
file_type TEXT NOT NULL,
|
|
||||||
file_path TEXT NOT NULL,
|
|
||||||
file_name TEXT NOT NULL,
|
|
||||||
file_size INTEGER NOT NULL,
|
|
||||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS manager_secrets (
|
|
||||||
id BIGINT PRIMARY KEY DEFAULT nextval('seq_manager_secrets_id'),
|
|
||||||
secret TEXT UNIQUE NOT NULL,
|
|
||||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS registration_tokens (
|
|
||||||
id BIGINT PRIMARY KEY DEFAULT nextval('seq_registration_tokens_id'),
|
|
||||||
token TEXT UNIQUE NOT NULL,
|
|
||||||
expires_at TIMESTAMP NOT NULL,
|
|
||||||
used BOOLEAN NOT NULL DEFAULT false,
|
|
||||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
||||||
created_by BIGINT,
|
|
||||||
FOREIGN KEY (created_by) REFERENCES users(id)
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS task_logs (
|
|
||||||
id BIGINT PRIMARY KEY DEFAULT nextval('seq_task_logs_id'),
|
|
||||||
task_id BIGINT NOT NULL,
|
|
||||||
runner_id BIGINT,
|
|
||||||
log_level TEXT NOT NULL,
|
|
||||||
message TEXT NOT NULL,
|
|
||||||
step_name TEXT,
|
|
||||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS task_steps (
|
|
||||||
id BIGINT PRIMARY KEY DEFAULT nextval('seq_task_steps_id'),
|
|
||||||
task_id BIGINT NOT NULL,
|
|
||||||
step_name TEXT NOT NULL,
|
|
||||||
status TEXT NOT NULL DEFAULT 'pending',
|
|
||||||
started_at TIMESTAMP,
|
|
||||||
completed_at TIMESTAMP,
|
|
||||||
duration_ms INTEGER,
|
|
||||||
error_message TEXT
|
|
||||||
);
|
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_jobs_user_id ON jobs(user_id);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_tasks_job_id ON tasks(job_id);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_tasks_runner_id ON tasks(runner_id);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_tasks_started_at ON tasks(started_at);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_job_files_job_id ON job_files(job_id);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_registration_tokens_token ON registration_tokens(token);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_registration_tokens_expires_at ON registration_tokens(expires_at);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_task_logs_task_id_created_at ON task_logs(task_id, created_at);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_task_logs_runner_id ON task_logs(runner_id);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_task_steps_task_id ON task_steps(task_id);
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_runners_last_heartbeat ON runners(last_heartbeat);
|
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS settings (
|
|
||||||
key TEXT PRIMARY KEY,
|
|
||||||
value TEXT NOT NULL,
|
|
||||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
||||||
);
|
|
||||||
`
|
|
||||||
|
|
||||||
if _, err := db.Exec(schema); err != nil {
|
|
||||||
return fmt.Errorf("failed to create schema: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Migrate existing tables to add new columns
|
|
||||||
migrations := []string{
|
|
||||||
// Add is_admin to users if it doesn't exist
|
|
||||||
`ALTER TABLE users ADD COLUMN IF NOT EXISTS is_admin BOOLEAN NOT NULL DEFAULT false`,
|
|
||||||
// Add new columns to runners if they don't exist
|
|
||||||
`ALTER TABLE runners ADD COLUMN IF NOT EXISTS registration_token TEXT`,
|
|
||||||
`ALTER TABLE runners ADD COLUMN IF NOT EXISTS runner_secret TEXT`,
|
|
||||||
`ALTER TABLE runners ADD COLUMN IF NOT EXISTS manager_secret TEXT`,
|
|
||||||
`ALTER TABLE runners ADD COLUMN IF NOT EXISTS verified BOOLEAN NOT NULL DEFAULT false`,
|
|
||||||
`ALTER TABLE runners ADD COLUMN IF NOT EXISTS priority INTEGER NOT NULL DEFAULT 100`,
|
|
||||||
// Add allow_parallel_runners to jobs if it doesn't exist
|
|
||||||
`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS allow_parallel_runners BOOLEAN NOT NULL DEFAULT true`,
|
|
||||||
// Add timeout_seconds to jobs if it doesn't exist
|
|
||||||
`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS timeout_seconds INTEGER DEFAULT 86400`,
|
|
||||||
// Add blend_metadata to jobs if it doesn't exist
|
|
||||||
`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS blend_metadata TEXT`,
|
|
||||||
// Add job_type to jobs if it doesn't exist
|
|
||||||
`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS job_type TEXT DEFAULT 'render'`,
|
|
||||||
// Add task_type to tasks if it doesn't exist
|
|
||||||
`ALTER TABLE tasks ADD COLUMN IF NOT EXISTS task_type TEXT DEFAULT 'render'`,
|
|
||||||
// Add new columns to tasks if they don't exist
|
|
||||||
`ALTER TABLE tasks ADD COLUMN IF NOT EXISTS current_step TEXT`,
|
|
||||||
`ALTER TABLE tasks ADD COLUMN IF NOT EXISTS retry_count INTEGER DEFAULT 0`,
|
|
||||||
`ALTER TABLE tasks ADD COLUMN IF NOT EXISTS max_retries INTEGER DEFAULT 3`,
|
|
||||||
`ALTER TABLE tasks ADD COLUMN IF NOT EXISTS timeout_seconds INTEGER`,
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, migration := range migrations {
|
|
||||||
// DuckDB supports IF NOT EXISTS for ALTER TABLE, so we can safely execute
|
|
||||||
if _, err := db.Exec(migration); err != nil {
|
|
||||||
// Log but don't fail - column might already exist or table might not exist yet
|
|
||||||
// This is fine for migrations that run after schema creation
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize registration_enabled setting (default: true) if it doesn't exist
|
|
||||||
var settingCount int
|
|
||||||
err := db.QueryRow("SELECT COUNT(*) FROM settings WHERE key = ?", "registration_enabled").Scan(&settingCount)
|
|
||||||
if err == nil && settingCount == 0 {
|
|
||||||
_, err = db.Exec("INSERT INTO settings (key, value) VALUES (?, ?)", "registration_enabled", "true")
|
|
||||||
if err != nil {
|
|
||||||
// Log but don't fail - setting might have been created by another process
|
|
||||||
log.Printf("Note: Could not initialize registration_enabled setting: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
log.Printf("Database migrations completed successfully")
|
||||||
return nil
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
for _, migration := range migrations {
|
// Ping checks the database connection
|
||||||
// DuckDB supports IF NOT EXISTS for ALTER TABLE, so we can safely execute
|
func (db *DB) Ping() error {
|
||||||
if _, err := db.Exec(migration); err != nil {
|
return db.db.Ping()
|
||||||
// Log but don't fail - column might already exist or table might not exist yet
|
|
||||||
// This is fine for migrations that run after schema creation
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close closes the database connection
|
// Close closes the database connection
|
||||||
func (db *DB) Close() error {
|
func (db *DB) Close() error {
|
||||||
return db.DB.Close()
|
return db.db.Close()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
package database
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewDB_RunsMigrationsAndSupportsQueries(t *testing.T) {
|
||||||
|
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||||
|
db, err := NewDB(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewDB failed: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
if err := db.Ping(); err != nil {
|
||||||
|
t.Fatalf("Ping failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var exists bool
|
||||||
|
err = db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='settings')").Scan(&exists)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("query failed: %v", err)
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
t.Fatal("expected settings table after migrations")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWithTx_RollbackOnError(t *testing.T) {
|
||||||
|
dbPath := filepath.Join(t.TempDir(), "tx.db")
|
||||||
|
db, err := NewDB(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewDB failed: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
_ = db.WithTx(func(tx *sql.Tx) error {
|
||||||
|
if _, err := tx.Exec("INSERT INTO settings (key, value) VALUES (?, ?)", "rollback_key", "x"); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return sql.ErrTxDone
|
||||||
|
})
|
||||||
|
|
||||||
|
var count int
|
||||||
|
if err := db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT COUNT(*) FROM settings WHERE key = ?", "rollback_key").Scan(&count)
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("count query failed: %v", err)
|
||||||
|
}
|
||||||
|
if count != 0 {
|
||||||
|
t.Fatalf("expected rollback, found %d rows", count)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,223 @@
|
|||||||
|
package logger
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Level represents log severity
|
||||||
|
type Level int
|
||||||
|
|
||||||
|
const (
|
||||||
|
LevelDebug Level = iota
|
||||||
|
LevelInfo
|
||||||
|
LevelWarn
|
||||||
|
LevelError
|
||||||
|
)
|
||||||
|
|
||||||
|
var levelNames = map[Level]string{
|
||||||
|
LevelDebug: "DEBUG",
|
||||||
|
LevelInfo: "INFO",
|
||||||
|
LevelWarn: "WARN",
|
||||||
|
LevelError: "ERROR",
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseLevel parses a level string into a Level
|
||||||
|
func ParseLevel(s string) Level {
|
||||||
|
switch s {
|
||||||
|
case "debug", "DEBUG":
|
||||||
|
return LevelDebug
|
||||||
|
case "info", "INFO":
|
||||||
|
return LevelInfo
|
||||||
|
case "warn", "WARN", "warning", "WARNING":
|
||||||
|
return LevelWarn
|
||||||
|
case "error", "ERROR":
|
||||||
|
return LevelError
|
||||||
|
default:
|
||||||
|
return LevelInfo
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
defaultLogger *Logger
|
||||||
|
once sync.Once
|
||||||
|
currentLevel Level = LevelInfo
|
||||||
|
)
|
||||||
|
|
||||||
|
// Logger wraps the standard log.Logger with optional file output and levels
|
||||||
|
type Logger struct {
|
||||||
|
*log.Logger
|
||||||
|
fileWriter io.WriteCloser
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetLevel sets the global log level
|
||||||
|
func SetLevel(level Level) {
|
||||||
|
currentLevel = level
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetLevel returns the current log level
|
||||||
|
func GetLevel() Level {
|
||||||
|
return currentLevel
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitStdout initializes the logger to only write to stdout
|
||||||
|
func InitStdout() {
|
||||||
|
once.Do(func() {
|
||||||
|
log.SetOutput(os.Stdout)
|
||||||
|
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
||||||
|
defaultLogger = &Logger{
|
||||||
|
Logger: log.Default(),
|
||||||
|
fileWriter: nil,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitWithFile initializes the logger with both file and stdout output
|
||||||
|
// The file is truncated on each start
|
||||||
|
func InitWithFile(logPath string) error {
|
||||||
|
var err error
|
||||||
|
once.Do(func() {
|
||||||
|
defaultLogger, err = NewWithFile(logPath)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Replace standard log output with the multi-writer
|
||||||
|
multiWriter := io.MultiWriter(os.Stdout, defaultLogger.fileWriter)
|
||||||
|
log.SetOutput(multiWriter)
|
||||||
|
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewWithFile creates a new logger that writes to both stdout and a log file
|
||||||
|
// The file is truncated on each start
|
||||||
|
func NewWithFile(logPath string) (*Logger, error) {
|
||||||
|
// Ensure log directory exists
|
||||||
|
logDir := filepath.Dir(logPath)
|
||||||
|
if err := os.MkdirAll(logDir, 0755); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create/truncate the log file
|
||||||
|
fileWriter, err := os.Create(logPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create multi-writer that writes to both stdout and file
|
||||||
|
multiWriter := io.MultiWriter(os.Stdout, fileWriter)
|
||||||
|
|
||||||
|
// Create logger with standard flags
|
||||||
|
logger := log.New(multiWriter, "", log.LstdFlags|log.Lshortfile)
|
||||||
|
|
||||||
|
return &Logger{
|
||||||
|
Logger: logger,
|
||||||
|
fileWriter: fileWriter,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close closes the file writer
|
||||||
|
func (l *Logger) Close() error {
|
||||||
|
if l.fileWriter != nil {
|
||||||
|
return l.fileWriter.Close()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetDefault returns the default logger instance
|
||||||
|
func GetDefault() *Logger {
|
||||||
|
return defaultLogger
|
||||||
|
}
|
||||||
|
|
||||||
|
// logf logs a formatted message at the given level
|
||||||
|
func logf(level Level, format string, v ...interface{}) {
|
||||||
|
if level < currentLevel {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
prefix := fmt.Sprintf("[%s] ", levelNames[level])
|
||||||
|
msg := fmt.Sprintf(format, v...)
|
||||||
|
log.Print(prefix + msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// logln logs a message at the given level
|
||||||
|
func logln(level Level, v ...interface{}) {
|
||||||
|
if level < currentLevel {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
prefix := fmt.Sprintf("[%s] ", levelNames[level])
|
||||||
|
msg := fmt.Sprint(v...)
|
||||||
|
log.Print(prefix + msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Debug logs a debug message
|
||||||
|
func Debug(v ...interface{}) {
|
||||||
|
logln(LevelDebug, v...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Debugf logs a formatted debug message
|
||||||
|
func Debugf(format string, v ...interface{}) {
|
||||||
|
logf(LevelDebug, format, v...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Info logs an info message
|
||||||
|
func Info(v ...interface{}) {
|
||||||
|
logln(LevelInfo, v...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Infof logs a formatted info message
|
||||||
|
func Infof(format string, v ...interface{}) {
|
||||||
|
logf(LevelInfo, format, v...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warn logs a warning message
|
||||||
|
func Warn(v ...interface{}) {
|
||||||
|
logln(LevelWarn, v...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warnf logs a formatted warning message
|
||||||
|
func Warnf(format string, v ...interface{}) {
|
||||||
|
logf(LevelWarn, format, v...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error logs an error message
|
||||||
|
func Error(v ...interface{}) {
|
||||||
|
logln(LevelError, v...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Errorf logs a formatted error message
|
||||||
|
func Errorf(format string, v ...interface{}) {
|
||||||
|
logf(LevelError, format, v...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fatal logs an error message and exits
|
||||||
|
func Fatal(v ...interface{}) {
|
||||||
|
logln(LevelError, v...)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fatalf logs a formatted error message and exits
|
||||||
|
func Fatalf(format string, v ...interface{}) {
|
||||||
|
logf(LevelError, format, v...)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Backwards compatibility (maps to Info level) ---
|
||||||
|
|
||||||
|
// Printf logs a formatted message at Info level
|
||||||
|
func Printf(format string, v ...interface{}) {
|
||||||
|
logf(LevelInfo, format, v...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Print logs a message at Info level
|
||||||
|
func Print(v ...interface{}) {
|
||||||
|
logln(LevelInfo, v...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Println logs a message at Info level
|
||||||
|
func Println(v ...interface{}) {
|
||||||
|
logln(LevelInfo, v...)
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
package logger
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseLevel(t *testing.T) {
|
||||||
|
if ParseLevel("debug") != LevelDebug {
|
||||||
|
t.Fatal("debug should map to LevelDebug")
|
||||||
|
}
|
||||||
|
if ParseLevel("warning") != LevelWarn {
|
||||||
|
t.Fatal("warning should map to LevelWarn")
|
||||||
|
}
|
||||||
|
if ParseLevel("unknown") != LevelInfo {
|
||||||
|
t.Fatal("unknown should default to LevelInfo")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSetAndGetLevel(t *testing.T) {
|
||||||
|
SetLevel(LevelError)
|
||||||
|
if GetLevel() != LevelError {
|
||||||
|
t.Fatalf("GetLevel() = %v, want %v", GetLevel(), LevelError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewWithFile_CreatesFile(t *testing.T) {
|
||||||
|
logPath := filepath.Join(t.TempDir(), "runner.log")
|
||||||
|
l, err := NewWithFile(logPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewWithFile failed: %v", err)
|
||||||
|
}
|
||||||
|
defer l.Close()
|
||||||
|
}
|
||||||
|
|
||||||
@@ -10,94 +10,119 @@ import (
|
|||||||
"jiggablend/pkg/types"
|
"jiggablend/pkg/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
// handleGenerateRegistrationToken generates a new registration token
|
// handleGenerateRunnerAPIKey generates a new runner API key
|
||||||
func (s *Server) handleGenerateRegistrationToken(w http.ResponseWriter, r *http.Request) {
|
func (s *Manager) handleGenerateRunnerAPIKey(w http.ResponseWriter, r *http.Request) {
|
||||||
userID, err := getUserID(r)
|
userID, err := getUserID(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusUnauthorized, err.Error())
|
s.respondError(w, http.StatusUnauthorized, err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Default expiration: 24 hours
|
|
||||||
expiresIn := 24 * time.Hour
|
|
||||||
|
|
||||||
var req struct {
|
var req struct {
|
||||||
ExpiresInHours int `json:"expires_in_hours,omitempty"`
|
Name string `json:"name"`
|
||||||
|
Description string `json:"description,omitempty"`
|
||||||
|
Scope string `json:"scope,omitempty"` // 'manager' or 'user'
|
||||||
}
|
}
|
||||||
if r.Body != nil && r.ContentLength > 0 {
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err == nil && req.ExpiresInHours > 0 {
|
s.respondError(w, http.StatusBadRequest, fmt.Sprintf("Invalid request body: expected valid JSON - %v", err))
|
||||||
expiresIn = time.Duration(req.ExpiresInHours) * time.Hour
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if req.Name == "" {
|
||||||
|
s.respondError(w, http.StatusBadRequest, "API key name is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default scope to 'user' if not specified
|
||||||
|
scope := req.Scope
|
||||||
|
if scope == "" {
|
||||||
|
scope = "user"
|
||||||
|
}
|
||||||
|
if scope != "manager" && scope != "user" {
|
||||||
|
s.respondError(w, http.StatusBadRequest, "Scope must be 'manager' or 'user'")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
keyInfo, err := s.secrets.GenerateRunnerAPIKey(userID, req.Name, req.Description, scope)
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to generate API key: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
response := map[string]interface{}{
|
||||||
|
"id": keyInfo.ID,
|
||||||
|
"key": keyInfo.Key,
|
||||||
|
"name": keyInfo.Name,
|
||||||
|
"description": keyInfo.Description,
|
||||||
|
"is_active": keyInfo.IsActive,
|
||||||
|
"created_at": keyInfo.CreatedAt,
|
||||||
|
}
|
||||||
|
|
||||||
|
s.respondJSON(w, http.StatusCreated, response)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleListRunnerAPIKeys lists all runner API keys
|
||||||
|
func (s *Manager) handleListRunnerAPIKeys(w http.ResponseWriter, r *http.Request) {
|
||||||
|
keys, err := s.secrets.ListRunnerAPIKeys()
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to list API keys: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert to response format (hide sensitive hash data)
|
||||||
|
var response []map[string]interface{}
|
||||||
|
for _, key := range keys {
|
||||||
|
item := map[string]interface{}{
|
||||||
|
"id": key.ID,
|
||||||
|
"key_prefix": key.Key, // Only show prefix, not full key
|
||||||
|
"name": key.Name,
|
||||||
|
"is_active": key.IsActive,
|
||||||
|
"created_at": key.CreatedAt,
|
||||||
|
"created_by": key.CreatedBy,
|
||||||
}
|
}
|
||||||
|
if key.Description != nil {
|
||||||
|
item["description"] = *key.Description
|
||||||
|
}
|
||||||
|
response = append(response, item)
|
||||||
}
|
}
|
||||||
|
|
||||||
token, err := s.secrets.GenerateRegistrationToken(userID, expiresIn)
|
s.respondJSON(w, http.StatusOK, response)
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to generate token: %v", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
s.respondJSON(w, http.StatusCreated, map[string]interface{}{
|
|
||||||
"token": token,
|
|
||||||
"expires_in": expiresIn.String(),
|
|
||||||
"expires_at": time.Now().Add(expiresIn),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleListRegistrationTokens lists all registration tokens
|
// handleRevokeRunnerAPIKey revokes a runner API key
|
||||||
func (s *Server) handleListRegistrationTokens(w http.ResponseWriter, r *http.Request) {
|
func (s *Manager) handleRevokeRunnerAPIKey(w http.ResponseWriter, r *http.Request) {
|
||||||
tokens, err := s.secrets.ListRegistrationTokens()
|
keyID, err := parseID(r, "id")
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to list tokens: %v", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
s.respondJSON(w, http.StatusOK, tokens)
|
|
||||||
}
|
|
||||||
|
|
||||||
// handleRevokeRegistrationToken revokes a registration token
|
|
||||||
func (s *Server) handleRevokeRegistrationToken(w http.ResponseWriter, r *http.Request) {
|
|
||||||
tokenID, err := parseID(r, "id")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := s.secrets.RevokeRegistrationToken(tokenID); err != nil {
|
if err := s.secrets.RevokeRunnerAPIKey(keyID); err != nil {
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to revoke token: %v", err))
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to revoke API key: %v", err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
s.respondJSON(w, http.StatusOK, map[string]string{"message": "Token revoked"})
|
s.respondJSON(w, http.StatusOK, map[string]string{"message": "API key revoked"})
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleVerifyRunner manually verifies a runner
|
// handleDeleteRunnerAPIKey deletes a runner API key
|
||||||
func (s *Server) handleVerifyRunner(w http.ResponseWriter, r *http.Request) {
|
func (s *Manager) handleDeleteRunnerAPIKey(w http.ResponseWriter, r *http.Request) {
|
||||||
runnerID, err := parseID(r, "id")
|
keyID, err := parseID(r, "id")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if runner exists
|
if err := s.secrets.DeleteRunnerAPIKey(keyID); err != nil {
|
||||||
var exists bool
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to delete API key: %v", err))
|
||||||
err = s.db.QueryRow("SELECT EXISTS(SELECT 1 FROM runners WHERE id = ?)", runnerID).Scan(&exists)
|
|
||||||
if err != nil || !exists {
|
|
||||||
s.respondError(w, http.StatusNotFound, "Runner not found")
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mark runner as verified
|
s.respondJSON(w, http.StatusOK, map[string]string{"message": "API key deleted"})
|
||||||
_, err = s.db.Exec("UPDATE runners SET verified = 1 WHERE id = ?", runnerID)
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to verify runner: %v", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
s.respondJSON(w, http.StatusOK, map[string]string{"message": "Runner verified"})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// handleDeleteRunner removes a runner
|
// handleDeleteRunner removes a runner
|
||||||
func (s *Server) handleDeleteRunner(w http.ResponseWriter, r *http.Request) {
|
func (s *Manager) handleDeleteRunner(w http.ResponseWriter, r *http.Request) {
|
||||||
runnerID, err := parseID(r, "id")
|
runnerID, err := parseID(r, "id")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||||
@@ -106,14 +131,19 @@ func (s *Server) handleDeleteRunner(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// Check if runner exists
|
// Check if runner exists
|
||||||
var exists bool
|
var exists bool
|
||||||
err = s.db.QueryRow("SELECT EXISTS(SELECT 1 FROM runners WHERE id = ?)", runnerID).Scan(&exists)
|
err = s.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM runners WHERE id = ?)", runnerID).Scan(&exists)
|
||||||
|
})
|
||||||
if err != nil || !exists {
|
if err != nil || !exists {
|
||||||
s.respondError(w, http.StatusNotFound, "Runner not found")
|
s.respondError(w, http.StatusNotFound, "Runner not found")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// Delete runner
|
// Delete runner
|
||||||
_, err = s.db.Exec("DELETE FROM runners WHERE id = ?", runnerID)
|
err = s.db.With(func(conn *sql.DB) error {
|
||||||
|
_, err := conn.Exec("DELETE FROM runners WHERE id = ?", runnerID)
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to delete runner: %v", err))
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to delete runner: %v", err))
|
||||||
return
|
return
|
||||||
@@ -123,12 +153,17 @@ func (s *Server) handleDeleteRunner(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// handleListRunnersAdmin lists all runners with admin details
|
// handleListRunnersAdmin lists all runners with admin details
|
||||||
func (s *Server) handleListRunnersAdmin(w http.ResponseWriter, r *http.Request) {
|
func (s *Manager) handleListRunnersAdmin(w http.ResponseWriter, r *http.Request) {
|
||||||
rows, err := s.db.Query(
|
var rows *sql.Rows
|
||||||
`SELECT id, name, hostname, ip_address, status, last_heartbeat, capabilities,
|
err := s.db.With(func(conn *sql.DB) error {
|
||||||
registration_token, verified, priority, created_at
|
var err error
|
||||||
|
rows, err = conn.Query(
|
||||||
|
`SELECT id, name, hostname, status, last_heartbeat, capabilities,
|
||||||
|
api_key_id, api_key_scope, priority, created_at
|
||||||
FROM runners ORDER BY created_at DESC`,
|
FROM runners ORDER BY created_at DESC`,
|
||||||
)
|
)
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query runners: %v", err))
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query runners: %v", err))
|
||||||
return
|
return
|
||||||
@@ -138,31 +173,32 @@ func (s *Server) handleListRunnersAdmin(w http.ResponseWriter, r *http.Request)
|
|||||||
runners := []map[string]interface{}{}
|
runners := []map[string]interface{}{}
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var runner types.Runner
|
var runner types.Runner
|
||||||
var registrationToken sql.NullString
|
var apiKeyID sql.NullInt64
|
||||||
var verified bool
|
var apiKeyScope string
|
||||||
|
|
||||||
err := rows.Scan(
|
err := rows.Scan(
|
||||||
&runner.ID, &runner.Name, &runner.Hostname, &runner.IPAddress,
|
&runner.ID, &runner.Name, &runner.Hostname,
|
||||||
&runner.Status, &runner.LastHeartbeat, &runner.Capabilities,
|
&runner.Status, &runner.LastHeartbeat, &runner.Capabilities,
|
||||||
®istrationToken, &verified, &runner.Priority, &runner.CreatedAt,
|
&apiKeyID, &apiKeyScope, &runner.Priority, &runner.CreatedAt,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to scan runner: %v", err))
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to scan runner: %v", err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// In polling model, database status is the source of truth
|
||||||
|
// Runners update their status when they poll for jobs
|
||||||
runners = append(runners, map[string]interface{}{
|
runners = append(runners, map[string]interface{}{
|
||||||
"id": runner.ID,
|
"id": runner.ID,
|
||||||
"name": runner.Name,
|
"name": runner.Name,
|
||||||
"hostname": runner.Hostname,
|
"hostname": runner.Hostname,
|
||||||
"ip_address": runner.IPAddress,
|
"status": runner.Status,
|
||||||
"status": runner.Status,
|
"last_heartbeat": runner.LastHeartbeat,
|
||||||
"last_heartbeat": runner.LastHeartbeat,
|
"capabilities": runner.Capabilities,
|
||||||
"capabilities": runner.Capabilities,
|
"api_key_id": apiKeyID.Int64,
|
||||||
"registration_token": registrationToken.String,
|
"api_key_scope": apiKeyScope,
|
||||||
"verified": verified,
|
"priority": runner.Priority,
|
||||||
"priority": runner.Priority,
|
"created_at": runner.CreatedAt,
|
||||||
"created_at": runner.CreatedAt,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -170,7 +206,7 @@ func (s *Server) handleListRunnersAdmin(w http.ResponseWriter, r *http.Request)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// handleListUsers lists all users
|
// handleListUsers lists all users
|
||||||
func (s *Server) handleListUsers(w http.ResponseWriter, r *http.Request) {
|
func (s *Manager) handleListUsers(w http.ResponseWriter, r *http.Request) {
|
||||||
// Get first user ID to mark it in the response
|
// Get first user ID to mark it in the response
|
||||||
firstUserID, err := s.auth.GetFirstUserID()
|
firstUserID, err := s.auth.GetFirstUserID()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -178,10 +214,15 @@ func (s *Server) handleListUsers(w http.ResponseWriter, r *http.Request) {
|
|||||||
firstUserID = 0
|
firstUserID = 0
|
||||||
}
|
}
|
||||||
|
|
||||||
rows, err := s.db.Query(
|
var rows *sql.Rows
|
||||||
`SELECT id, email, name, oauth_provider, is_admin, created_at
|
err = s.db.With(func(conn *sql.DB) error {
|
||||||
|
var err error
|
||||||
|
rows, err = conn.Query(
|
||||||
|
`SELECT id, email, name, oauth_provider, is_admin, created_at
|
||||||
FROM users ORDER BY created_at DESC`,
|
FROM users ORDER BY created_at DESC`,
|
||||||
)
|
)
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query users: %v", err))
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query users: %v", err))
|
||||||
return
|
return
|
||||||
@@ -203,7 +244,9 @@ func (s *Server) handleListUsers(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// Get job count for this user
|
// Get job count for this user
|
||||||
var jobCount int
|
var jobCount int
|
||||||
err = s.db.QueryRow("SELECT COUNT(*) FROM jobs WHERE user_id = ?", userID).Scan(&jobCount)
|
err = s.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT COUNT(*) FROM jobs WHERE user_id = ?", userID).Scan(&jobCount)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
jobCount = 0 // Default to 0 if query fails
|
jobCount = 0 // Default to 0 if query fails
|
||||||
}
|
}
|
||||||
@@ -224,7 +267,7 @@ func (s *Server) handleListUsers(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// handleGetUserJobs gets all jobs for a specific user
|
// handleGetUserJobs gets all jobs for a specific user
|
||||||
func (s *Server) handleGetUserJobs(w http.ResponseWriter, r *http.Request) {
|
func (s *Manager) handleGetUserJobs(w http.ResponseWriter, r *http.Request) {
|
||||||
userID, err := parseID(r, "id")
|
userID, err := parseID(r, "id")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||||
@@ -233,18 +276,25 @@ func (s *Server) handleGetUserJobs(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// Verify user exists
|
// Verify user exists
|
||||||
var exists bool
|
var exists bool
|
||||||
err = s.db.QueryRow("SELECT EXISTS(SELECT 1 FROM users WHERE id = ?)", userID).Scan(&exists)
|
err = s.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM users WHERE id = ?)", userID).Scan(&exists)
|
||||||
|
})
|
||||||
if err != nil || !exists {
|
if err != nil || !exists {
|
||||||
s.respondError(w, http.StatusNotFound, "User not found")
|
s.respondError(w, http.StatusNotFound, "User not found")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
rows, err := s.db.Query(
|
var rows *sql.Rows
|
||||||
`SELECT id, user_id, job_type, name, status, progress, frame_start, frame_end, output_format,
|
err = s.db.With(func(conn *sql.DB) error {
|
||||||
allow_parallel_runners, timeout_seconds, blend_metadata, created_at, started_at, completed_at, error_message
|
var err error
|
||||||
|
rows, err = conn.Query(
|
||||||
|
`SELECT id, user_id, job_type, name, status, progress, frame_start, frame_end, output_format,
|
||||||
|
blend_metadata, created_at, started_at, completed_at, error_message
|
||||||
FROM jobs WHERE user_id = ? ORDER BY created_at DESC`,
|
FROM jobs WHERE user_id = ? ORDER BY created_at DESC`,
|
||||||
userID,
|
userID,
|
||||||
)
|
)
|
||||||
|
return err
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query jobs: %v", err))
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query jobs: %v", err))
|
||||||
return
|
return
|
||||||
@@ -260,11 +310,9 @@ func (s *Server) handleGetUserJobs(w http.ResponseWriter, r *http.Request) {
|
|||||||
var errorMessage sql.NullString
|
var errorMessage sql.NullString
|
||||||
var frameStart, frameEnd sql.NullInt64
|
var frameStart, frameEnd sql.NullInt64
|
||||||
var outputFormat sql.NullString
|
var outputFormat sql.NullString
|
||||||
var allowParallelRunners sql.NullBool
|
|
||||||
|
|
||||||
err := rows.Scan(
|
err := rows.Scan(
|
||||||
&job.ID, &job.UserID, &jobType, &job.Name, &job.Status, &job.Progress,
|
&job.ID, &job.UserID, &jobType, &job.Name, &job.Status, &job.Progress,
|
||||||
&frameStart, &frameEnd, &outputFormat, &allowParallelRunners, &job.TimeoutSeconds,
|
&frameStart, &frameEnd, &outputFormat,
|
||||||
&blendMetadataJSON, &job.CreatedAt, &startedAt, &completedAt, &errorMessage,
|
&blendMetadataJSON, &job.CreatedAt, &startedAt, &completedAt, &errorMessage,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -284,9 +332,6 @@ func (s *Server) handleGetUserJobs(w http.ResponseWriter, r *http.Request) {
|
|||||||
if outputFormat.Valid {
|
if outputFormat.Valid {
|
||||||
job.OutputFormat = &outputFormat.String
|
job.OutputFormat = &outputFormat.String
|
||||||
}
|
}
|
||||||
if allowParallelRunners.Valid {
|
|
||||||
job.AllowParallelRunners = &allowParallelRunners.Bool
|
|
||||||
}
|
|
||||||
if startedAt.Valid {
|
if startedAt.Valid {
|
||||||
job.StartedAt = &startedAt.Time
|
job.StartedAt = &startedAt.Time
|
||||||
}
|
}
|
||||||
@@ -310,7 +355,7 @@ func (s *Server) handleGetUserJobs(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// handleGetRegistrationEnabled gets the registration enabled setting
|
// handleGetRegistrationEnabled gets the registration enabled setting
|
||||||
func (s *Server) handleGetRegistrationEnabled(w http.ResponseWriter, r *http.Request) {
|
func (s *Manager) handleGetRegistrationEnabled(w http.ResponseWriter, r *http.Request) {
|
||||||
enabled, err := s.auth.IsRegistrationEnabled()
|
enabled, err := s.auth.IsRegistrationEnabled()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to get registration setting: %v", err))
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to get registration setting: %v", err))
|
||||||
@@ -320,12 +365,12 @@ func (s *Server) handleGetRegistrationEnabled(w http.ResponseWriter, r *http.Req
|
|||||||
}
|
}
|
||||||
|
|
||||||
// handleSetRegistrationEnabled sets the registration enabled setting
|
// handleSetRegistrationEnabled sets the registration enabled setting
|
||||||
func (s *Server) handleSetRegistrationEnabled(w http.ResponseWriter, r *http.Request) {
|
func (s *Manager) handleSetRegistrationEnabled(w http.ResponseWriter, r *http.Request) {
|
||||||
var req struct {
|
var req struct {
|
||||||
Enabled bool `json:"enabled"`
|
Enabled bool `json:"enabled"`
|
||||||
}
|
}
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
s.respondError(w, http.StatusBadRequest, "Invalid request body")
|
s.respondError(w, http.StatusBadRequest, fmt.Sprintf("Invalid request body: expected valid JSON - %v", err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -338,7 +383,13 @@ func (s *Server) handleSetRegistrationEnabled(w http.ResponseWriter, r *http.Req
|
|||||||
}
|
}
|
||||||
|
|
||||||
// handleSetUserAdminStatus sets a user's admin status (admin only)
|
// handleSetUserAdminStatus sets a user's admin status (admin only)
|
||||||
func (s *Server) handleSetUserAdminStatus(w http.ResponseWriter, r *http.Request) {
|
func (s *Manager) handleSetUserAdminStatus(w http.ResponseWriter, r *http.Request) {
|
||||||
|
currentUserID, err := getUserID(r)
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusUnauthorized, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
targetUserID, err := parseID(r, "id")
|
targetUserID, err := parseID(r, "id")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||||
@@ -349,7 +400,13 @@ func (s *Server) handleSetUserAdminStatus(w http.ResponseWriter, r *http.Request
|
|||||||
IsAdmin bool `json:"is_admin"`
|
IsAdmin bool `json:"is_admin"`
|
||||||
}
|
}
|
||||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
s.respondError(w, http.StatusBadRequest, "Invalid request body")
|
s.respondError(w, http.StatusBadRequest, fmt.Sprintf("Invalid request body: expected valid JSON - %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prevent admins from revoking their own admin status.
|
||||||
|
if targetUserID == currentUserID && !req.IsAdmin {
|
||||||
|
s.respondError(w, http.StatusBadRequest, "You cannot revoke your own admin status")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestHandleGenerateRunnerAPIKey_UnauthorizedWithoutContext(t *testing.T) {
|
||||||
|
s := &Manager{}
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/api/admin/runner-api-keys", bytes.NewBufferString(`{"name":"k"}`))
|
||||||
|
rr := httptest.NewRecorder()
|
||||||
|
|
||||||
|
s.handleGenerateRunnerAPIKey(rr, req)
|
||||||
|
|
||||||
|
if rr.Code != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("status = %d, want %d", rr.Code, http.StatusUnauthorized)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleGenerateRunnerAPIKey_RejectsBadJSON(t *testing.T) {
|
||||||
|
s := &Manager{}
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/api/admin/runner-api-keys", bytes.NewBufferString(`{`))
|
||||||
|
rr := httptest.NewRecorder()
|
||||||
|
|
||||||
|
s.handleGenerateRunnerAPIKey(rr, req)
|
||||||
|
|
||||||
|
// No auth context means unauthorized happens first; this still validates safe
|
||||||
|
// failure handling for malformed requests in this handler path.
|
||||||
|
if rr.Code != http.StatusUnauthorized {
|
||||||
|
t.Fatalf("status = %d, want %d", rr.Code, http.StatusUnauthorized)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,705 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"compress/bzip2"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"jiggablend/pkg/blendfile"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
BlenderDownloadBaseURL = "https://download.blender.org/release/"
|
||||||
|
BlenderVersionCacheTTL = 1 * time.Hour
|
||||||
|
)
|
||||||
|
|
||||||
|
// BlenderVersion represents a parsed Blender version
|
||||||
|
type BlenderVersion struct {
|
||||||
|
Major int `json:"major"`
|
||||||
|
Minor int `json:"minor"`
|
||||||
|
Patch int `json:"patch"`
|
||||||
|
Full string `json:"full"` // e.g., "4.2.3"
|
||||||
|
DirName string `json:"dir_name"` // e.g., "Blender4.2"
|
||||||
|
Filename string `json:"filename"` // e.g., "blender-4.2.3-linux-x64.tar.xz"
|
||||||
|
URL string `json:"url"` // Full download URL
|
||||||
|
}
|
||||||
|
|
||||||
|
// BlenderVersionCache caches available Blender versions
|
||||||
|
type BlenderVersionCache struct {
|
||||||
|
versions []BlenderVersion
|
||||||
|
fetchedAt time.Time
|
||||||
|
mu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
var blenderVersionCache = &BlenderVersionCache{}
|
||||||
|
|
||||||
|
// FetchBlenderVersions fetches available Blender versions from download.blender.org
|
||||||
|
// Returns versions sorted by version number (newest first)
|
||||||
|
func (s *Manager) FetchBlenderVersions() ([]BlenderVersion, error) {
|
||||||
|
// Check cache first
|
||||||
|
blenderVersionCache.mu.RLock()
|
||||||
|
if time.Since(blenderVersionCache.fetchedAt) < BlenderVersionCacheTTL && len(blenderVersionCache.versions) > 0 {
|
||||||
|
versions := make([]BlenderVersion, len(blenderVersionCache.versions))
|
||||||
|
copy(versions, blenderVersionCache.versions)
|
||||||
|
blenderVersionCache.mu.RUnlock()
|
||||||
|
return versions, nil
|
||||||
|
}
|
||||||
|
blenderVersionCache.mu.RUnlock()
|
||||||
|
|
||||||
|
// Fetch from website with timeout
|
||||||
|
client := &http.Client{
|
||||||
|
Timeout: WSWriteDeadline,
|
||||||
|
}
|
||||||
|
resp, err := client.Get(BlenderDownloadBaseURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to fetch blender releases: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("failed to fetch blender releases: status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse directory listing for Blender version folders
|
||||||
|
// Looking for patterns like href="Blender4.2/" or href="Blender3.6/"
|
||||||
|
dirPattern := regexp.MustCompile(`href="Blender(\d+)\.(\d+)/"`)
|
||||||
|
log.Printf("Fetching Blender versions from %s", BlenderDownloadBaseURL)
|
||||||
|
matches := dirPattern.FindAllStringSubmatch(string(body), -1)
|
||||||
|
|
||||||
|
// Fetch sub-versions concurrently to speed up the process
|
||||||
|
type versionResult struct {
|
||||||
|
versions []BlenderVersion
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
results := make(chan versionResult, len(matches))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
for _, match := range matches {
|
||||||
|
if len(match) < 3 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
major := 0
|
||||||
|
minor := 0
|
||||||
|
fmt.Sscanf(match[1], "%d", &major)
|
||||||
|
fmt.Sscanf(match[2], "%d", &minor)
|
||||||
|
|
||||||
|
// Skip very old versions (pre-2.80)
|
||||||
|
if major < 2 || (major == 2 && minor < 80) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
dirName := fmt.Sprintf("Blender%d.%d", major, minor)
|
||||||
|
|
||||||
|
// Fetch the specific version directory concurrently
|
||||||
|
wg.Add(1)
|
||||||
|
go func(dn string, maj, min int) {
|
||||||
|
defer wg.Done()
|
||||||
|
subVersions, err := fetchSubVersions(dn, maj, min)
|
||||||
|
results <- versionResult{versions: subVersions, err: err}
|
||||||
|
}(dirName, major, minor)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close results channel when all goroutines complete
|
||||||
|
go func() {
|
||||||
|
wg.Wait()
|
||||||
|
close(results)
|
||||||
|
}()
|
||||||
|
|
||||||
|
var versions []BlenderVersion
|
||||||
|
for result := range results {
|
||||||
|
if result.err != nil {
|
||||||
|
log.Printf("Warning: failed to fetch sub-versions: %v", result.err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
versions = append(versions, result.versions...)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sort by version (newest first)
|
||||||
|
sort.Slice(versions, func(i, j int) bool {
|
||||||
|
if versions[i].Major != versions[j].Major {
|
||||||
|
return versions[i].Major > versions[j].Major
|
||||||
|
}
|
||||||
|
if versions[i].Minor != versions[j].Minor {
|
||||||
|
return versions[i].Minor > versions[j].Minor
|
||||||
|
}
|
||||||
|
return versions[i].Patch > versions[j].Patch
|
||||||
|
})
|
||||||
|
|
||||||
|
// Update cache
|
||||||
|
blenderVersionCache.mu.Lock()
|
||||||
|
blenderVersionCache.versions = versions
|
||||||
|
blenderVersionCache.fetchedAt = time.Now()
|
||||||
|
blenderVersionCache.mu.Unlock()
|
||||||
|
|
||||||
|
return versions, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchSubVersions fetches specific version files from a Blender release directory
|
||||||
|
func fetchSubVersions(dirName string, major, minor int) ([]BlenderVersion, error) {
|
||||||
|
url := BlenderDownloadBaseURL + dirName + "/"
|
||||||
|
client := &http.Client{
|
||||||
|
Timeout: WSWriteDeadline,
|
||||||
|
}
|
||||||
|
resp, err := client.Get(url)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Look for linux 64-bit tar.xz/bz2 files
|
||||||
|
// Various naming conventions across versions:
|
||||||
|
// - Modern (2.93+): blender-4.2.3-linux-x64.tar.xz
|
||||||
|
// - 2.83 early: blender-2.83.0-linux64.tar.xz
|
||||||
|
// - 2.80-2.82: blender-2.80-linux-glibc217-x86_64.tar.bz2
|
||||||
|
// Skip: rc versions, alpha/beta, i686 (32-bit)
|
||||||
|
filePatterns := []*regexp.Regexp{
|
||||||
|
// Modern format: blender-X.Y.Z-linux-x64.tar.xz
|
||||||
|
regexp.MustCompile(`blender-(\d+)\.(\d+)\.(\d+)-linux-x64\.tar\.(xz|bz2)`),
|
||||||
|
// Older format: blender-X.Y.Z-linux64.tar.xz
|
||||||
|
regexp.MustCompile(`blender-(\d+)\.(\d+)\.(\d+)-linux64\.tar\.(xz|bz2)`),
|
||||||
|
// glibc format: blender-X.Y.Z-linux-glibc217-x86_64.tar.bz2 (prefer glibc217 for compatibility)
|
||||||
|
regexp.MustCompile(`blender-(\d+)\.(\d+)\.(\d+)-linux-glibc217-x86_64\.tar\.(xz|bz2)`),
|
||||||
|
}
|
||||||
|
|
||||||
|
var versions []BlenderVersion
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
|
||||||
|
for _, filePattern := range filePatterns {
|
||||||
|
matches := filePattern.FindAllStringSubmatch(string(body), -1)
|
||||||
|
|
||||||
|
for _, match := range matches {
|
||||||
|
if len(match) < 5 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
patch := 0
|
||||||
|
fmt.Sscanf(match[3], "%d", &patch)
|
||||||
|
|
||||||
|
full := fmt.Sprintf("%d.%d.%d", major, minor, patch)
|
||||||
|
if seen[full] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[full] = true
|
||||||
|
|
||||||
|
filename := match[0]
|
||||||
|
versions = append(versions, BlenderVersion{
|
||||||
|
Major: major,
|
||||||
|
Minor: minor,
|
||||||
|
Patch: patch,
|
||||||
|
Full: full,
|
||||||
|
DirName: dirName,
|
||||||
|
Filename: filename,
|
||||||
|
URL: url + filename,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return versions, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetLatestBlenderForMajorMinor returns the latest patch version for a given major.minor
|
||||||
|
// If exact match not found, uses fuzzy matching to find the closest available version
|
||||||
|
func (s *Manager) GetLatestBlenderForMajorMinor(major, minor int) (*BlenderVersion, error) {
|
||||||
|
versions, err := s.FetchBlenderVersions()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(versions) == 0 {
|
||||||
|
return nil, fmt.Errorf("no blender versions available")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try exact match first - find the highest patch for this major.minor
|
||||||
|
var exactMatch *BlenderVersion
|
||||||
|
for i := range versions {
|
||||||
|
v := &versions[i]
|
||||||
|
if v.Major == major && v.Minor == minor {
|
||||||
|
if exactMatch == nil || v.Patch > exactMatch.Patch {
|
||||||
|
exactMatch = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if exactMatch != nil {
|
||||||
|
log.Printf("Found Blender %d.%d.%d for requested %d.%d", exactMatch.Major, exactMatch.Minor, exactMatch.Patch, major, minor)
|
||||||
|
return exactMatch, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fuzzy matching: find closest version
|
||||||
|
// Priority: same major with closest minor > closest major
|
||||||
|
log.Printf("No exact match for Blender %d.%d, using fuzzy matching", major, minor)
|
||||||
|
|
||||||
|
var bestMatch *BlenderVersion
|
||||||
|
bestScore := -1000000 // Large negative number
|
||||||
|
|
||||||
|
for i := range versions {
|
||||||
|
v := &versions[i]
|
||||||
|
score := 0
|
||||||
|
|
||||||
|
if v.Major == major {
|
||||||
|
// Same major version - prefer this
|
||||||
|
score = 10000
|
||||||
|
|
||||||
|
// Prefer lower minor versions (more stable/compatible)
|
||||||
|
// but not too far back
|
||||||
|
minorDiff := minor - v.Minor
|
||||||
|
if minorDiff >= 0 {
|
||||||
|
// v.Minor <= minor (older or same) - prefer closer
|
||||||
|
score += 1000 - minorDiff*10
|
||||||
|
} else {
|
||||||
|
// v.Minor > minor (newer) - less preferred but acceptable
|
||||||
|
score += 500 + minorDiff*10
|
||||||
|
}
|
||||||
|
|
||||||
|
// Higher patch is better
|
||||||
|
score += v.Patch
|
||||||
|
} else {
|
||||||
|
// Different major - less preferred
|
||||||
|
majorDiff := major - v.Major
|
||||||
|
if majorDiff > 0 {
|
||||||
|
// v.Major < major (older major) - acceptable fallback
|
||||||
|
score = 5000 - majorDiff*1000 + v.Minor*10 + v.Patch
|
||||||
|
} else {
|
||||||
|
// v.Major > major (newer major) - avoid if possible
|
||||||
|
score = -majorDiff * 1000
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if score > bestScore {
|
||||||
|
bestScore = score
|
||||||
|
bestMatch = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if bestMatch != nil {
|
||||||
|
log.Printf("Fuzzy match: requested %d.%d, using %d.%d.%d", major, minor, bestMatch.Major, bestMatch.Minor, bestMatch.Patch)
|
||||||
|
return bestMatch, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("no blender version found for %d.%d", major, minor)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBlenderArchivePath returns the path to the cached blender archive for a specific version
|
||||||
|
// Downloads from blender.org and decompresses to .tar if not already cached
|
||||||
|
// The manager caches as uncompressed .tar to save decompression time on runners
|
||||||
|
func (s *Manager) GetBlenderArchivePath(version *BlenderVersion) (string, error) {
|
||||||
|
// Base directory for blender archives
|
||||||
|
blenderDir := filepath.Join(s.storage.BasePath(), "blender-versions")
|
||||||
|
if err := os.MkdirAll(blenderDir, 0755); err != nil {
|
||||||
|
return "", fmt.Errorf("failed to create blender directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cache as uncompressed .tar for faster runner downloads
|
||||||
|
// Convert filename like "blender-4.2.3-linux-x64.tar.xz" to "blender-4.2.3-linux-x64.tar"
|
||||||
|
tarFilename := version.Filename
|
||||||
|
tarFilename = strings.TrimSuffix(tarFilename, ".xz")
|
||||||
|
tarFilename = strings.TrimSuffix(tarFilename, ".bz2")
|
||||||
|
archivePath := filepath.Join(blenderDir, tarFilename)
|
||||||
|
|
||||||
|
// Check if already cached as .tar
|
||||||
|
if _, err := os.Stat(archivePath); err == nil {
|
||||||
|
log.Printf("Using cached Blender %s at %s", version.Full, archivePath)
|
||||||
|
// Clean up any extracted folders that might exist
|
||||||
|
s.cleanupExtractedBlenderFolders(blenderDir, version)
|
||||||
|
return archivePath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Need to download and decompress
|
||||||
|
log.Printf("Downloading Blender %s from %s", version.Full, version.URL)
|
||||||
|
|
||||||
|
// 60-minute timeout for large Blender tarballs; stream to disk via io.Copy below
|
||||||
|
client := &http.Client{
|
||||||
|
Timeout: 60 * time.Minute,
|
||||||
|
}
|
||||||
|
resp, err := client.Get(version.URL)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to download blender: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return "", fmt.Errorf("failed to download blender: status %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download to temp file first
|
||||||
|
compressedPath := filepath.Join(blenderDir, "download-"+version.Filename)
|
||||||
|
compressedFile, err := os.Create(compressedPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to create temp file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := io.Copy(compressedFile, resp.Body); err != nil {
|
||||||
|
compressedFile.Close()
|
||||||
|
os.Remove(compressedPath)
|
||||||
|
return "", fmt.Errorf("failed to download blender: %w", err)
|
||||||
|
}
|
||||||
|
compressedFile.Close()
|
||||||
|
|
||||||
|
log.Printf("Downloaded Blender %s, decompressing to .tar...", version.Full)
|
||||||
|
|
||||||
|
// Decompress to .tar
|
||||||
|
if err := decompressToTar(compressedPath, archivePath); err != nil {
|
||||||
|
os.Remove(compressedPath)
|
||||||
|
os.Remove(archivePath)
|
||||||
|
return "", fmt.Errorf("failed to decompress blender archive: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove compressed file
|
||||||
|
os.Remove(compressedPath)
|
||||||
|
|
||||||
|
// Clean up any extracted folders for this version (if they exist)
|
||||||
|
s.cleanupExtractedBlenderFolders(blenderDir, version)
|
||||||
|
|
||||||
|
log.Printf("Blender %s cached at %s", version.Full, archivePath)
|
||||||
|
return archivePath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// decompressToTar decompresses a .tar.xz or .tar.bz2 file to a plain .tar file
|
||||||
|
func decompressToTar(compressedPath, tarPath string) error {
|
||||||
|
if strings.HasSuffix(compressedPath, ".tar.xz") {
|
||||||
|
// Use xz command for decompression
|
||||||
|
cmd := exec.Command("xz", "-d", "-k", "-c", compressedPath)
|
||||||
|
outFile, err := os.Create(tarPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer outFile.Close()
|
||||||
|
|
||||||
|
cmd.Stdout = outFile
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
return fmt.Errorf("xz decompression failed: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
} else if strings.HasSuffix(compressedPath, ".tar.bz2") {
|
||||||
|
// Use bzip2 for decompression
|
||||||
|
inFile, err := os.Open(compressedPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer inFile.Close()
|
||||||
|
|
||||||
|
bzReader := bzip2.NewReader(inFile)
|
||||||
|
outFile, err := os.Create(tarPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer outFile.Close()
|
||||||
|
|
||||||
|
if _, err := io.Copy(outFile, bzReader); err != nil {
|
||||||
|
return fmt.Errorf("bzip2 decompression failed: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("unsupported compression format: %s", compressedPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanupExtractedBlenderFolders removes any extracted Blender folders for the given version
|
||||||
|
// This ensures we only keep the .tar file and not extracted folders
|
||||||
|
func (s *Manager) cleanupExtractedBlenderFolders(blenderDir string, version *BlenderVersion) {
|
||||||
|
// Look for folders matching the version (e.g., "4.2.3", "2.83.20")
|
||||||
|
versionDirs := []string{
|
||||||
|
filepath.Join(blenderDir, version.Full), // e.g., "4.2.3"
|
||||||
|
filepath.Join(blenderDir, fmt.Sprintf("%d.%d", version.Major, version.Minor)), // e.g., "4.2"
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, dir := range versionDirs {
|
||||||
|
if info, err := os.Stat(dir); err == nil && info.IsDir() {
|
||||||
|
log.Printf("Removing extracted Blender folder: %s", dir)
|
||||||
|
if err := os.RemoveAll(dir); err != nil {
|
||||||
|
log.Printf("Warning: failed to remove extracted folder %s: %v", dir, err)
|
||||||
|
} else {
|
||||||
|
log.Printf("Removed extracted Blender folder: %s", dir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseBlenderVersionFromFile parses the Blender version that a .blend file was saved with.
|
||||||
|
// Delegates to the shared pkg/blendfile implementation.
|
||||||
|
func ParseBlenderVersionFromFile(blendPath string) (major, minor int, err error) {
|
||||||
|
return blendfile.ParseVersionFromFile(blendPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseBlenderVersionFromReader parses the Blender version from a reader.
|
||||||
|
// Delegates to the shared pkg/blendfile implementation.
|
||||||
|
func ParseBlenderVersionFromReader(r io.ReadSeeker) (major, minor int, err error) {
|
||||||
|
return blendfile.ParseVersionFromReader(r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleGetBlenderVersions returns available Blender versions
|
||||||
|
func (s *Manager) handleGetBlenderVersions(w http.ResponseWriter, r *http.Request) {
|
||||||
|
versions, err := s.FetchBlenderVersions()
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("failed to fetch blender versions: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Group by major.minor for easier frontend display
|
||||||
|
type VersionGroup struct {
|
||||||
|
MajorMinor string `json:"major_minor"`
|
||||||
|
Latest BlenderVersion `json:"latest"`
|
||||||
|
All []BlenderVersion `json:"all"`
|
||||||
|
}
|
||||||
|
|
||||||
|
groups := make(map[string]*VersionGroup)
|
||||||
|
for _, v := range versions {
|
||||||
|
key := fmt.Sprintf("%d.%d", v.Major, v.Minor)
|
||||||
|
if groups[key] == nil {
|
||||||
|
groups[key] = &VersionGroup{
|
||||||
|
MajorMinor: key,
|
||||||
|
Latest: v, // First one is latest due to sorting
|
||||||
|
All: []BlenderVersion{v},
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
groups[key].All = append(groups[key].All, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert to slice and sort by version
|
||||||
|
var groupedResult []VersionGroup
|
||||||
|
for _, g := range groups {
|
||||||
|
groupedResult = append(groupedResult, *g)
|
||||||
|
}
|
||||||
|
sort.Slice(groupedResult, func(i, j int) bool {
|
||||||
|
// Parse major.minor for comparison
|
||||||
|
var iMaj, iMin, jMaj, jMin int
|
||||||
|
fmt.Sscanf(groupedResult[i].MajorMinor, "%d.%d", &iMaj, &iMin)
|
||||||
|
fmt.Sscanf(groupedResult[j].MajorMinor, "%d.%d", &jMaj, &jMin)
|
||||||
|
if iMaj != jMaj {
|
||||||
|
return iMaj > jMaj
|
||||||
|
}
|
||||||
|
return iMin > jMin
|
||||||
|
})
|
||||||
|
|
||||||
|
// Return both flat list and grouped for flexibility
|
||||||
|
response := map[string]interface{}{
|
||||||
|
"versions": versions, // Flat list of all versions (newest first)
|
||||||
|
"grouped": groupedResult, // Grouped by major.minor
|
||||||
|
}
|
||||||
|
|
||||||
|
s.respondJSON(w, http.StatusOK, response)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleDownloadBlender serves a cached Blender archive to runners
|
||||||
|
func (s *Manager) handleDownloadBlender(w http.ResponseWriter, r *http.Request) {
|
||||||
|
version := r.URL.Query().Get("version")
|
||||||
|
if version == "" {
|
||||||
|
s.respondError(w, http.StatusBadRequest, "version parameter required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse version string (e.g., "4.2.3" or "4.2")
|
||||||
|
var major, minor, patch int
|
||||||
|
parts := strings.Split(version, ".")
|
||||||
|
if len(parts) < 2 {
|
||||||
|
s.respondError(w, http.StatusBadRequest, "invalid version format, expected major.minor or major.minor.patch")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Sscanf(parts[0], "%d", &major)
|
||||||
|
fmt.Sscanf(parts[1], "%d", &minor)
|
||||||
|
if len(parts) >= 3 {
|
||||||
|
fmt.Sscanf(parts[2], "%d", &patch)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find the version
|
||||||
|
var blenderVersion *BlenderVersion
|
||||||
|
if len(parts) >= 3 {
|
||||||
|
// Exact patch version requested - find it
|
||||||
|
versions, err := s.FetchBlenderVersions()
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("failed to fetch versions: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, v := range versions {
|
||||||
|
if v.Major == major && v.Minor == minor && v.Patch == patch {
|
||||||
|
blenderVersion = &v
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if blenderVersion == nil {
|
||||||
|
s.respondError(w, http.StatusNotFound, fmt.Sprintf("blender version %s not found", version))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Major.minor only - use helper to get latest patch version
|
||||||
|
var err error
|
||||||
|
blenderVersion, err = s.GetLatestBlenderForMajorMinor(major, minor)
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusNotFound, fmt.Sprintf("blender version %s not found: %v", version, err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get or download the archive
|
||||||
|
archivePath, err := s.GetBlenderArchivePath(blenderVersion)
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("failed to get blender archive: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Serve the file
|
||||||
|
file, err := os.Open(archivePath)
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("failed to open archive: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
stat, err := file.Stat()
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("failed to stat archive: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filename is now .tar (decompressed)
|
||||||
|
tarFilename := blenderVersion.Filename
|
||||||
|
tarFilename = strings.TrimSuffix(tarFilename, ".xz")
|
||||||
|
tarFilename = strings.TrimSuffix(tarFilename, ".bz2")
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/x-tar")
|
||||||
|
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", tarFilename))
|
||||||
|
w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size()))
|
||||||
|
w.Header().Set("X-Blender-Version", blenderVersion.Full)
|
||||||
|
|
||||||
|
io.Copy(w, file)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unused functions from extraction - keeping for reference but not needed on manager
|
||||||
|
var _ = extractBlenderArchive
|
||||||
|
var _ = extractTarXz
|
||||||
|
var _ = extractTar
|
||||||
|
|
||||||
|
// extractBlenderArchive extracts a blender archive (already decompressed to .tar by GetBlenderArchivePath)
|
||||||
|
func extractBlenderArchive(archivePath string, version *BlenderVersion, destDir string) error {
|
||||||
|
file, err := os.Open(archivePath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
// The archive is already decompressed to .tar by GetBlenderArchivePath
|
||||||
|
// Just extract it directly
|
||||||
|
if strings.HasSuffix(archivePath, ".tar") {
|
||||||
|
tarReader := tar.NewReader(file)
|
||||||
|
return extractTar(tarReader, version, destDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback for any other format (shouldn't happen with current flow)
|
||||||
|
if strings.HasSuffix(archivePath, ".tar.xz") {
|
||||||
|
return extractTarXz(archivePath, version, destDir)
|
||||||
|
} else if strings.HasSuffix(archivePath, ".tar.bz2") {
|
||||||
|
bzReader := bzip2.NewReader(file)
|
||||||
|
tarReader := tar.NewReader(bzReader)
|
||||||
|
return extractTar(tarReader, version, destDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("unsupported archive format: %s", archivePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractTarXz extracts a tar.xz archive using the xz command
|
||||||
|
func extractTarXz(archivePath string, version *BlenderVersion, destDir string) error {
|
||||||
|
versionDir := filepath.Join(destDir, version.Full)
|
||||||
|
if err := os.MkdirAll(versionDir, 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := exec.Command("tar", "-xJf", archivePath, "-C", versionDir, "--strip-components=1")
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("tar extraction failed: %v, output: %s", err, string(output))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractTar extracts files from a tar reader
|
||||||
|
func extractTar(tarReader *tar.Reader, version *BlenderVersion, destDir string) error {
|
||||||
|
versionDir := filepath.Join(destDir, version.Full)
|
||||||
|
if err := os.MkdirAll(versionDir, 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
stripPrefix := ""
|
||||||
|
|
||||||
|
for {
|
||||||
|
header, err := tarReader.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if stripPrefix == "" {
|
||||||
|
parts := strings.SplitN(header.Name, "/", 2)
|
||||||
|
if len(parts) > 0 {
|
||||||
|
stripPrefix = parts[0] + "/"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
name := strings.TrimPrefix(header.Name, stripPrefix)
|
||||||
|
if name == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
targetPath := filepath.Join(versionDir, name)
|
||||||
|
|
||||||
|
switch header.Typeflag {
|
||||||
|
case tar.TypeDir:
|
||||||
|
if err := os.MkdirAll(targetPath, os.FileMode(header.Mode)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case tar.TypeReg:
|
||||||
|
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(header.Mode))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := io.Copy(outFile, tarReader); err != nil {
|
||||||
|
outFile.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
outFile.Close()
|
||||||
|
case tar.TypeSymlink:
|
||||||
|
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Symlink(header.Linkname, targetPath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// resolveBlenderBinaryPath resolves a Blender executable to an absolute path.
|
||||||
|
func resolveBlenderBinaryPath(blenderBinary string) (string, error) {
|
||||||
|
if blenderBinary == "" {
|
||||||
|
return "", fmt.Errorf("blender binary path is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Already contains a path component; normalize it.
|
||||||
|
if strings.Contains(blenderBinary, string(filepath.Separator)) {
|
||||||
|
absPath, err := filepath.Abs(blenderBinary)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to resolve blender binary path %q: %w", blenderBinary, err)
|
||||||
|
}
|
||||||
|
return absPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bare executable name, resolve via PATH.
|
||||||
|
resolvedPath, err := exec.LookPath(blenderBinary)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to locate blender binary %q in PATH: %w", blenderBinary, err)
|
||||||
|
}
|
||||||
|
absPath, err := filepath.Abs(resolvedPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to resolve blender binary path %q: %w", resolvedPath, err)
|
||||||
|
}
|
||||||
|
return absPath, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolveBlenderBinaryPath_WithPathComponent(t *testing.T) {
|
||||||
|
got, err := resolveBlenderBinaryPath("./blender")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolveBlenderBinaryPath failed: %v", err)
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(got) {
|
||||||
|
t.Fatalf("expected absolute path, got %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveBlenderBinaryPath_Empty(t *testing.T) {
|
||||||
|
if _, err := resolveBlenderBinaryPath(""); err == nil {
|
||||||
|
t.Fatal("expected error for empty path")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGetLatestBlenderForMajorMinor_UsesCachedVersions(t *testing.T) {
|
||||||
|
blenderVersionCache.mu.Lock()
|
||||||
|
blenderVersionCache.versions = []BlenderVersion{
|
||||||
|
{Major: 4, Minor: 2, Patch: 1, Full: "4.2.1"},
|
||||||
|
{Major: 4, Minor: 2, Patch: 3, Full: "4.2.3"},
|
||||||
|
{Major: 4, Minor: 1, Patch: 9, Full: "4.1.9"},
|
||||||
|
}
|
||||||
|
blenderVersionCache.fetchedAt = time.Now()
|
||||||
|
blenderVersionCache.mu.Unlock()
|
||||||
|
|
||||||
|
m := &Manager{}
|
||||||
|
v, err := m.GetLatestBlenderForMajorMinor(4, 2)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetLatestBlenderForMajorMinor failed: %v", err)
|
||||||
|
}
|
||||||
|
if v.Full != "4.2.3" {
|
||||||
|
t.Fatalf("expected highest patch, got %+v", *v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,305 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"archive/zip"
|
||||||
|
"bytes"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"jiggablend/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGenerateAndCheckETag(t *testing.T) {
|
||||||
|
etag := generateETag(map[string]interface{}{"a": 1})
|
||||||
|
if etag == "" {
|
||||||
|
t.Fatal("expected non-empty etag")
|
||||||
|
}
|
||||||
|
req := httptest.NewRequest("GET", "/x", nil)
|
||||||
|
req.Header.Set("If-None-Match", etag)
|
||||||
|
if !checkETag(req, etag) {
|
||||||
|
t.Fatal("expected etag match")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadSessionPhase(t *testing.T) {
|
||||||
|
if got := uploadSessionPhase("uploading"); got != "upload" {
|
||||||
|
t.Fatalf("unexpected phase: %q", got)
|
||||||
|
}
|
||||||
|
if got := uploadSessionPhase("select_blend"); got != "action_required" {
|
||||||
|
t.Fatalf("unexpected phase: %q", got)
|
||||||
|
}
|
||||||
|
if got := uploadSessionPhase("something_else"); got != "processing" {
|
||||||
|
t.Fatalf("unexpected fallback phase: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseTarHeader_AndTruncateString(t *testing.T) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
tw := tar.NewWriter(&buf)
|
||||||
|
_ = tw.WriteHeader(&tar.Header{Name: "a.txt", Mode: 0644, Size: 3, Typeflag: tar.TypeReg})
|
||||||
|
_, _ = tw.Write([]byte("abc"))
|
||||||
|
_ = tw.Close()
|
||||||
|
|
||||||
|
raw := buf.Bytes()
|
||||||
|
if len(raw) < 512 {
|
||||||
|
t.Fatal("tar buffer unexpectedly small")
|
||||||
|
}
|
||||||
|
var h tar.Header
|
||||||
|
if err := parseTarHeader(raw[:512], &h); err != nil {
|
||||||
|
t.Fatalf("parseTarHeader failed: %v", err)
|
||||||
|
}
|
||||||
|
if h.Name != "a.txt" {
|
||||||
|
t.Fatalf("unexpected parsed name: %q", h.Name)
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := truncateString("abcdef", 5); got != "ab..." {
|
||||||
|
t.Fatalf("truncateString = %q, want %q", got, "ab...")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateUploadSessionForJobCreation_MissingSession(t *testing.T) {
|
||||||
|
s := &Manager{
|
||||||
|
uploadSessions: map[string]*UploadSession{},
|
||||||
|
}
|
||||||
|
_, _, err := s.validateUploadSessionForJobCreation("missing", 1)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected validation error for missing session")
|
||||||
|
}
|
||||||
|
sessionErr, ok := err.(*uploadSessionValidationError)
|
||||||
|
if !ok || sessionErr.Code != uploadSessionExpiredCode {
|
||||||
|
t.Fatalf("expected %s validation error, got %#v", uploadSessionExpiredCode, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderTaskTimeoutIsRenderNotEncode(t *testing.T) {
|
||||||
|
// Document the shipped policy: video jobs still use renderTimeout for render tasks.
|
||||||
|
s := &Manager{renderTimeout: 3600, videoEncodeTimeout: 86400}
|
||||||
|
if s.renderTimeout == s.videoEncodeTimeout {
|
||||||
|
t.Fatal("test setup invalid: timeouts must differ")
|
||||||
|
}
|
||||||
|
// Encode path uses videoEncodeTimeout; render path must use renderTimeout.
|
||||||
|
// The create-job handler assigns s.renderTimeout to render tasks (see handleCreateJob).
|
||||||
|
if s.renderTimeout != 3600 {
|
||||||
|
t.Fatalf("renderTimeout = %d", s.renderTimeout)
|
||||||
|
}
|
||||||
|
if s.videoEncodeTimeout != 86400 {
|
||||||
|
t.Fatalf("videoEncodeTimeout = %d", s.videoEncodeTimeout)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadSanitize_ConsistentWriteExcludeAndBackgroundZip(t *testing.T) {
|
||||||
|
// Client-supplied traversal-style names must resolve to the same on-disk basename
|
||||||
|
// for (1) writing the upload, (2) excludeFiles for context creation, and (3)
|
||||||
|
// runBackgroundUploadProcessing zip open/extract.
|
||||||
|
st, err := storage.NewStorage(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStorage: %v", err)
|
||||||
|
}
|
||||||
|
clientName := "../../evil/scene.zip"
|
||||||
|
safeName, err := storage.SanitizeFilename(clientName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("SanitizeFilename: %v", err)
|
||||||
|
}
|
||||||
|
if safeName != "scene.zip" {
|
||||||
|
t.Fatalf("safeName = %q, want scene.zip", safeName)
|
||||||
|
}
|
||||||
|
|
||||||
|
tmpDir, err := st.TempDir("upload-test-*")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Write a real zip under the sanitized basename (as the upload handler does)
|
||||||
|
zipPath := filepath.Join(tmpDir, safeName)
|
||||||
|
if err := writeMinimalBlendZip(zipPath, "scene.blend"); err != nil {
|
||||||
|
t.Fatalf("write zip: %v", err)
|
||||||
|
}
|
||||||
|
// Extract so context archive has a root .blend; ZIP remains on disk to be excluded
|
||||||
|
if _, err := st.ExtractZip(zipPath, tmpDir); err != nil {
|
||||||
|
t.Fatalf("ExtractZip: %v", err)
|
||||||
|
}
|
||||||
|
if !fileExists(filepath.Join(tmpDir, "scene.blend")) {
|
||||||
|
t.Fatal("expected extracted scene.blend")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exclude list must use safeName so CreateJobContextFromDir drops the zip archive
|
||||||
|
// (matching the on-disk file), not the raw client path.
|
||||||
|
contextPath, err := st.CreateJobContextFromDir(tmpDir, 1, safeName)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CreateJobContextFromDir with safe exclude: %v", err)
|
||||||
|
}
|
||||||
|
if !fileExists(contextPath) {
|
||||||
|
t.Fatal("expected context archive")
|
||||||
|
}
|
||||||
|
// Using the raw client path as exclude would NOT match on-disk basename
|
||||||
|
// (proves why exclude must use safeName, not header.Filename with traversal).
|
||||||
|
if clientName == safeName {
|
||||||
|
t.Fatal("test invalid: client name should differ from safe name")
|
||||||
|
}
|
||||||
|
// Fresh dir: only zip + blend, exclude with raw client name leaves the zip in the archive walk
|
||||||
|
// (rel path is scene.zip; client is ../../evil/scene.zip — no match).
|
||||||
|
tmpDir2, err := st.TempDir("upload-excl-*")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
zip2 := filepath.Join(tmpDir2, safeName)
|
||||||
|
if err := writeMinimalBlendZip(zip2, "scene.blend"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := st.ExtractZip(zip2, tmpDir2); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// With wrong exclude (raw client name), CreateJobContextFromDir still succeeds but the
|
||||||
|
// zip may be included. Safer assertion: wrong exclude string does not equal safeName.
|
||||||
|
wrongExclude := clientName
|
||||||
|
if filepath.Base(wrongExclude) == wrongExclude && wrongExclude == safeName {
|
||||||
|
t.Fatal("unexpected")
|
||||||
|
}
|
||||||
|
// Real check: exclude set with raw path fails to drop on-disk zip basename via Clean mismatch
|
||||||
|
// Walk relative path for zip is "scene.zip"; exclude set keys for "../../evil/scene.zip"
|
||||||
|
// after Clean become "../evil/scene.zip" which does not match.
|
||||||
|
_, errWrong := st.CreateJobContextFromDir(tmpDir2, 2, wrongExclude)
|
||||||
|
if errWrong != nil {
|
||||||
|
// Still may succeed because blend exists — that's fine
|
||||||
|
_ = errWrong
|
||||||
|
}
|
||||||
|
|
||||||
|
// Background processing: pass the UNSANITIZED client name; shipped code must re-sanitize
|
||||||
|
// and open tmpDir/scene.zip successfully. Use a clean tmp with only the zip (as after upload).
|
||||||
|
bgDir, err := st.TempDir("upload-bg-*")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
bgZip := filepath.Join(bgDir, safeName)
|
||||||
|
if err := writeMinimalBlendZip(bgZip, "scene.blend"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s := &Manager{
|
||||||
|
storage: st,
|
||||||
|
uploadSessions: map[string]*UploadSession{},
|
||||||
|
}
|
||||||
|
sessionID := "sess-traversal"
|
||||||
|
s.uploadSessions[sessionID] = &UploadSession{
|
||||||
|
SessionID: sessionID,
|
||||||
|
UserID: 7,
|
||||||
|
TempDir: bgDir,
|
||||||
|
Status: "processing",
|
||||||
|
CreatedAt: time.Now(),
|
||||||
|
}
|
||||||
|
s.runBackgroundUploadProcessing(bgDir, sessionID, 7, clientName, 100, "", "")
|
||||||
|
|
||||||
|
s.uploadSessionsMu.RLock()
|
||||||
|
session := s.uploadSessions[sessionID]
|
||||||
|
s.uploadSessionsMu.RUnlock()
|
||||||
|
if session == nil {
|
||||||
|
t.Fatal("session missing")
|
||||||
|
}
|
||||||
|
if session.Status == "error" {
|
||||||
|
t.Fatalf("background processing failed with unsanitized name: %s", session.ErrorMessage)
|
||||||
|
}
|
||||||
|
if session.Status != "completed" && session.Status != "select_blend" {
|
||||||
|
t.Fatalf("status = %q, want completed or select_blend (got message %q)", session.Status, session.Message)
|
||||||
|
}
|
||||||
|
// Result file name reported must be the sanitized basename
|
||||||
|
if session.ResultFileName != "" && session.ResultFileName != safeName {
|
||||||
|
t.Fatalf("ResultFileName = %q, want %q", session.ResultFileName, safeName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeMinimalBlendZip(zipPath, blendName string) error {
|
||||||
|
f, err := os.Create(zipPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
zw := zip.NewWriter(f)
|
||||||
|
w, err := zw.Create(blendName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := w.Write([]byte("BLENDER-TEST")); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return zw.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func fileExists(path string) bool {
|
||||||
|
_, err := os.Stat(path)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateUploadSessionForJobCreation_ContextMissing(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
s := &Manager{
|
||||||
|
uploadSessions: map[string]*UploadSession{
|
||||||
|
"s1": {
|
||||||
|
SessionID: "s1",
|
||||||
|
UserID: 9,
|
||||||
|
TempDir: tmpDir,
|
||||||
|
Status: "completed",
|
||||||
|
CreatedAt: time.Now(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, _, err := s.validateUploadSessionForJobCreation("s1", 9); err == nil {
|
||||||
|
t.Fatal("expected error when context.tar is missing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateUploadSessionForJobCreation_NotReady(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
s := &Manager{
|
||||||
|
uploadSessions: map[string]*UploadSession{
|
||||||
|
"s1": {
|
||||||
|
SessionID: "s1",
|
||||||
|
UserID: 9,
|
||||||
|
TempDir: tmpDir,
|
||||||
|
Status: "processing",
|
||||||
|
CreatedAt: time.Now(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, _, err := s.validateUploadSessionForJobCreation("s1", 9)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for session that is not completed")
|
||||||
|
}
|
||||||
|
sessionErr, ok := err.(*uploadSessionValidationError)
|
||||||
|
if !ok || sessionErr.Code != uploadSessionNotReadyCode {
|
||||||
|
t.Fatalf("expected %s validation error, got %#v", uploadSessionNotReadyCode, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateUploadSessionForJobCreation_Success(t *testing.T) {
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
contextPath := filepath.Join(tmpDir, "context.tar")
|
||||||
|
if err := os.WriteFile(contextPath, []byte("tar-bytes"), 0644); err != nil {
|
||||||
|
t.Fatalf("write context.tar: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s := &Manager{
|
||||||
|
uploadSessions: map[string]*UploadSession{
|
||||||
|
"s1": {
|
||||||
|
SessionID: "s1",
|
||||||
|
UserID: 9,
|
||||||
|
TempDir: tmpDir,
|
||||||
|
Status: "completed",
|
||||||
|
CreatedAt: time.Now(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
session, gotPath, err := s.validateUploadSessionForJobCreation("s1", 9)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected valid session, got error: %v", err)
|
||||||
|
}
|
||||||
|
if session == nil || gotPath != contextPath {
|
||||||
|
t.Fatalf("unexpected result: session=%v path=%q", session, gotPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,120 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"jiggablend/internal/config"
|
||||||
|
"jiggablend/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCheckWebSocketOrigin_DevelopmentAllowsOrigin(t *testing.T) {
|
||||||
|
t.Setenv("PRODUCTION", "")
|
||||||
|
s := &Manager{cfg: &config.Config{}}
|
||||||
|
req := httptest.NewRequest("GET", "http://localhost/ws", nil)
|
||||||
|
req.Host = "localhost:8080"
|
||||||
|
req.Header.Set("Origin", "http://example.com")
|
||||||
|
if !s.checkWebSocketOrigin(req) {
|
||||||
|
t.Fatal("expected development mode to allow origin")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckWebSocketOrigin_ProductionSameHostAllowed(t *testing.T) {
|
||||||
|
t.Setenv("PRODUCTION", "true")
|
||||||
|
t.Setenv("ALLOWED_ORIGINS", "")
|
||||||
|
s := &Manager{cfg: &config.Config{}}
|
||||||
|
req := httptest.NewRequest("GET", "http://localhost/ws", nil)
|
||||||
|
req.Host = "localhost:8080"
|
||||||
|
req.Header.Set("Origin", "http://localhost:8080")
|
||||||
|
if !s.checkWebSocketOrigin(req) {
|
||||||
|
t.Fatal("expected same-host origin to be allowed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRespondErrorWithCode_IncludesCodeField(t *testing.T) {
|
||||||
|
s := &Manager{}
|
||||||
|
rr := httptest.NewRecorder()
|
||||||
|
s.respondErrorWithCode(rr, http.StatusBadRequest, "UPLOAD_SESSION_EXPIRED", "Upload session expired.")
|
||||||
|
|
||||||
|
if rr.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("status = %d, want %d", rr.Code, http.StatusBadRequest)
|
||||||
|
}
|
||||||
|
var payload map[string]string
|
||||||
|
if err := json.Unmarshal(rr.Body.Bytes(), &payload); err != nil {
|
||||||
|
t.Fatalf("failed to decode response: %v", err)
|
||||||
|
}
|
||||||
|
if payload["code"] != "UPLOAD_SESSION_EXPIRED" {
|
||||||
|
t.Fatalf("unexpected code: %q", payload["code"])
|
||||||
|
}
|
||||||
|
if payload["error"] == "" {
|
||||||
|
t.Fatal("expected non-empty error message")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSecurityHeadersMiddleware_SetsBaselineHeaders(t *testing.T) {
|
||||||
|
h := securityHeadersMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
}))
|
||||||
|
rr := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||||
|
h.ServeHTTP(rr, req)
|
||||||
|
if rr.Header().Get("X-Content-Type-Options") != "nosniff" {
|
||||||
|
t.Fatalf("missing nosniff, got %q", rr.Header().Get("X-Content-Type-Options"))
|
||||||
|
}
|
||||||
|
if rr.Header().Get("X-Frame-Options") != "DENY" {
|
||||||
|
t.Fatalf("missing frame deny")
|
||||||
|
}
|
||||||
|
if rr.Header().Get("Content-Security-Policy") == "" {
|
||||||
|
t.Fatal("missing CSP")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCleanupOldTempDirectories_SkipsActiveSessionTempDir(t *testing.T) {
|
||||||
|
base := t.TempDir()
|
||||||
|
st, err := storage.NewStorage(base)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewStorage: %v", err)
|
||||||
|
}
|
||||||
|
tempPath := filepath.Join(base, "temp")
|
||||||
|
activeDir, err := os.MkdirTemp(tempPath, "jiggablend-upload-*")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
oldTime := time.Now().Add(-2 * time.Hour)
|
||||||
|
_ = os.Chtimes(activeDir, oldTime, oldTime)
|
||||||
|
|
||||||
|
staleDir := filepath.Join(tempPath, "stale-old-dir")
|
||||||
|
if err := os.MkdirAll(staleDir, 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = os.Chtimes(staleDir, oldTime, oldTime)
|
||||||
|
|
||||||
|
s := &Manager{
|
||||||
|
storage: st,
|
||||||
|
uploadSessions: map[string]*UploadSession{
|
||||||
|
"active": {SessionID: "active", TempDir: activeDir},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
s.cleanupOldTempDirectoriesOnce()
|
||||||
|
|
||||||
|
if _, err := os.Stat(activeDir); err != nil {
|
||||||
|
t.Fatalf("active session temp dir was deleted: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(staleDir); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("stale temp dir should have been removed, stat err=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCreateSessionCookie_UsesConfigProductionMode(t *testing.T) {
|
||||||
|
t.Setenv("PRODUCTION", "true")
|
||||||
|
s := &Manager{cfg: &config.Config{}, sessionCookieMaxAge: 3600}
|
||||||
|
cookie := s.createSessionCookie("sid")
|
||||||
|
if !cookie.Secure {
|
||||||
|
t.Fatal("expected Secure cookie when production mode is on via env")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,285 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"database/sql"
|
||||||
|
_ "embed"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"jiggablend/pkg/executils"
|
||||||
|
"jiggablend/pkg/scripts"
|
||||||
|
"jiggablend/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
var runMetadataCommand = executils.RunCommand
|
||||||
|
var resolveMetadataBlenderPath = resolveBlenderBinaryPath
|
||||||
|
|
||||||
|
// handleGetJobMetadata retrieves metadata for a job
|
||||||
|
func (s *Manager) handleGetJobMetadata(w http.ResponseWriter, r *http.Request) {
|
||||||
|
userID, err := getUserID(r)
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusUnauthorized, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
jobID, err := parseID(r, "id")
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify job belongs to user
|
||||||
|
var jobUserID int64
|
||||||
|
var blendMetadataJSON sql.NullString
|
||||||
|
err = s.db.With(func(conn *sql.DB) error {
|
||||||
|
return conn.QueryRow(
|
||||||
|
`SELECT user_id, blend_metadata FROM jobs WHERE id = ?`,
|
||||||
|
jobID,
|
||||||
|
).Scan(&jobUserID, &blendMetadataJSON)
|
||||||
|
})
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
s.respondError(w, http.StatusNotFound, "Job not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query job: %v", err))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if jobUserID != userID {
|
||||||
|
s.respondError(w, http.StatusForbidden, "Access denied")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if !blendMetadataJSON.Valid || blendMetadataJSON.String == "" {
|
||||||
|
s.respondJSON(w, http.StatusOK, nil)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var metadata types.BlendMetadata
|
||||||
|
if err := json.Unmarshal([]byte(blendMetadataJSON.String), &metadata); err != nil {
|
||||||
|
s.respondError(w, http.StatusInternalServerError, "Failed to parse metadata")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.respondJSON(w, http.StatusOK, metadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractMetadataFromContext extracts metadata from the blend file in a context archive
|
||||||
|
// Returns the extracted metadata or an error
|
||||||
|
func (s *Manager) extractMetadataFromContext(jobID int64) (*types.BlendMetadata, error) {
|
||||||
|
contextPath := filepath.Join(s.storage.JobPath(jobID), "context.tar")
|
||||||
|
|
||||||
|
// Check if context exists
|
||||||
|
if _, err := os.Stat(contextPath); err != nil {
|
||||||
|
return nil, fmt.Errorf("context archive not found: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create temporary directory for extraction under storage base path
|
||||||
|
tmpDir, err := s.storage.TempDir(fmt.Sprintf("jiggablend-metadata-%d-*", jobID))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create temporary directory: %w", err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if err := os.RemoveAll(tmpDir); err != nil {
|
||||||
|
log.Printf("Warning: Failed to clean up temp directory %s: %v", tmpDir, err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Extract context archive
|
||||||
|
if err := s.extractTar(contextPath, tmpDir); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to extract context: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find .blend file in extracted contents
|
||||||
|
blendFile := ""
|
||||||
|
err = filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !info.IsDir() && strings.HasSuffix(strings.ToLower(info.Name()), ".blend") {
|
||||||
|
// Check it's not a Blender save file (.blend1, .blend2, etc.)
|
||||||
|
lower := strings.ToLower(info.Name())
|
||||||
|
idx := strings.LastIndex(lower, ".blend")
|
||||||
|
if idx != -1 {
|
||||||
|
suffix := lower[idx+len(".blend"):]
|
||||||
|
// If there are digits after .blend, it's a save file
|
||||||
|
isSaveFile := false
|
||||||
|
if len(suffix) > 0 {
|
||||||
|
isSaveFile = true
|
||||||
|
for _, r := range suffix {
|
||||||
|
if r < '0' || r > '9' {
|
||||||
|
isSaveFile = false
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !isSaveFile {
|
||||||
|
blendFile = path
|
||||||
|
return filepath.SkipAll // Stop walking once we find a blend file
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to find blend file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if blendFile == "" {
|
||||||
|
return nil, fmt.Errorf("no .blend file found in context - the uploaded context archive must contain at least one .blend file for metadata extraction")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use embedded Python script
|
||||||
|
scriptPath := filepath.Join(tmpDir, "extract_metadata.py")
|
||||||
|
if err := os.WriteFile(scriptPath, []byte(scripts.ExtractMetadata), 0644); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create extraction script: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use absolute paths to avoid path normalization issues with relative traversal.
|
||||||
|
blendFileAbs, err := filepath.Abs(blendFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to get absolute path for blend file: %w", err)
|
||||||
|
}
|
||||||
|
scriptPathAbs, err := filepath.Abs(scriptPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to get absolute path for extraction script: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute Blender with Python script using executils
|
||||||
|
blenderBinary, err := resolveMetadataBlenderPath("blender")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
result, err := runMetadataCommand(
|
||||||
|
blenderBinary,
|
||||||
|
[]string{"-b", blendFileAbs, "--python", scriptPathAbs},
|
||||||
|
tmpDir,
|
||||||
|
nil, // inherit environment
|
||||||
|
jobID,
|
||||||
|
nil, // no process tracker needed for metadata extraction
|
||||||
|
)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
stderrOutput := ""
|
||||||
|
stdoutOutput := ""
|
||||||
|
if result != nil {
|
||||||
|
stderrOutput = strings.TrimSpace(result.Stderr)
|
||||||
|
stdoutOutput = strings.TrimSpace(result.Stdout)
|
||||||
|
}
|
||||||
|
log.Printf("Blender metadata extraction failed for job %d:", jobID)
|
||||||
|
if stderrOutput != "" {
|
||||||
|
log.Printf("Blender stderr: %s", stderrOutput)
|
||||||
|
}
|
||||||
|
if stdoutOutput != "" {
|
||||||
|
log.Printf("Blender stdout (last 500 chars): %s", truncateString(stdoutOutput, 500))
|
||||||
|
}
|
||||||
|
if stderrOutput != "" {
|
||||||
|
return nil, fmt.Errorf("blender metadata extraction failed: %w (stderr: %s)", err, truncateString(stderrOutput, 200))
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("blender metadata extraction failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse output (metadata is printed to stdout)
|
||||||
|
metadataJSON := strings.TrimSpace(result.Stdout)
|
||||||
|
// Extract JSON from output (Blender may print other stuff)
|
||||||
|
jsonStart := strings.Index(metadataJSON, "{")
|
||||||
|
jsonEnd := strings.LastIndex(metadataJSON, "}")
|
||||||
|
if jsonStart == -1 || jsonEnd == -1 || jsonEnd <= jsonStart {
|
||||||
|
return nil, errors.New("failed to extract JSON from Blender output")
|
||||||
|
}
|
||||||
|
metadataJSON = metadataJSON[jsonStart : jsonEnd+1]
|
||||||
|
|
||||||
|
var metadata types.BlendMetadata
|
||||||
|
if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to parse metadata JSON: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Metadata extracted for job %d: frame_start=%d, frame_end=%d", jobID, metadata.FrameStart, metadata.FrameEnd)
|
||||||
|
return &metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractTar extracts a tar archive to a destination directory
|
||||||
|
func (s *Manager) extractTar(tarPath, destDir string) error {
|
||||||
|
log.Printf("Extracting tar archive: %s -> %s", tarPath, destDir)
|
||||||
|
|
||||||
|
// Ensure destination directory exists
|
||||||
|
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||||
|
return fmt.Errorf("failed to create destination directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Open(tarPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open archive: %w", err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
tr := tar.NewReader(file)
|
||||||
|
|
||||||
|
fileCount := 0
|
||||||
|
dirCount := 0
|
||||||
|
|
||||||
|
for {
|
||||||
|
header, err := tr.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to read tar header: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sanitize path to prevent directory traversal. TAR stores "/" separators, so normalize first.
|
||||||
|
normalizedHeaderPath := filepath.FromSlash(header.Name)
|
||||||
|
cleanHeaderPath := filepath.Clean(normalizedHeaderPath)
|
||||||
|
if cleanHeaderPath == "." {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if filepath.IsAbs(cleanHeaderPath) || strings.HasPrefix(cleanHeaderPath, ".."+string(os.PathSeparator)) || cleanHeaderPath == ".." {
|
||||||
|
log.Printf("ERROR: Invalid file path in TAR - header: %s", header.Name)
|
||||||
|
return fmt.Errorf("invalid file path in archive: %s", header.Name)
|
||||||
|
}
|
||||||
|
target := filepath.Join(destDir, cleanHeaderPath)
|
||||||
|
|
||||||
|
// Ensure target is within destDir
|
||||||
|
cleanTarget := filepath.Clean(target)
|
||||||
|
cleanDestDir := filepath.Clean(destDir)
|
||||||
|
if !strings.HasPrefix(cleanTarget, cleanDestDir+string(os.PathSeparator)) && cleanTarget != cleanDestDir {
|
||||||
|
log.Printf("ERROR: Invalid file path in TAR - target: %s, destDir: %s", cleanTarget, cleanDestDir)
|
||||||
|
return fmt.Errorf("invalid file path in archive: %s (target: %s, destDir: %s)", header.Name, cleanTarget, cleanDestDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create parent directories
|
||||||
|
if err := os.MkdirAll(filepath.Dir(cleanTarget), 0755); err != nil {
|
||||||
|
return fmt.Errorf("failed to create directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write file
|
||||||
|
switch header.Typeflag {
|
||||||
|
case tar.TypeReg:
|
||||||
|
outFile, err := os.Create(cleanTarget)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create file: %w", err)
|
||||||
|
}
|
||||||
|
_, err = io.Copy(outFile, tr)
|
||||||
|
if err != nil {
|
||||||
|
outFile.Close()
|
||||||
|
return fmt.Errorf("failed to write file: %w", err)
|
||||||
|
}
|
||||||
|
outFile.Close()
|
||||||
|
fileCount++
|
||||||
|
case tar.TypeDir:
|
||||||
|
dirCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Extraction complete: %d files, %d directories extracted to %s", fileCount, dirCount, destDir)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"jiggablend/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
func uploadSessionMetadata(session *UploadSession) *types.BlendMetadata {
|
||||||
|
if session == nil || session.ResultMetadata == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
switch m := session.ResultMetadata.(type) {
|
||||||
|
case *types.BlendMetadata:
|
||||||
|
return m
|
||||||
|
case types.BlendMetadata:
|
||||||
|
meta := m
|
||||||
|
return &meta
|
||||||
|
default:
|
||||||
|
data, err := json.Marshal(session.ResultMetadata)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var meta types.BlendMetadata
|
||||||
|
if err := json.Unmarshal(data, &meta); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &meta
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeBlendMetadataForJobCreate starts from upload analysis metadata when available,
|
||||||
|
// then applies explicit job creation overrides from the request.
|
||||||
|
func mergeBlendMetadataForJobCreate(uploadMeta *types.BlendMetadata, req *types.CreateJobRequest) (*types.BlendMetadata, error) {
|
||||||
|
if req.FrameStart == nil || req.FrameEnd == nil {
|
||||||
|
return nil, fmt.Errorf("frame_start and frame_end are required")
|
||||||
|
}
|
||||||
|
|
||||||
|
var metadata types.BlendMetadata
|
||||||
|
if uploadMeta != nil {
|
||||||
|
metadata = *uploadMeta
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata.FrameStart = *req.FrameStart
|
||||||
|
metadata.FrameEnd = *req.FrameEnd
|
||||||
|
|
||||||
|
if req.RenderSettings != nil {
|
||||||
|
metadata.RenderSettings = *req.RenderSettings
|
||||||
|
}
|
||||||
|
if req.OutputFormat != nil {
|
||||||
|
metadata.RenderSettings.OutputFormat = *req.OutputFormat
|
||||||
|
}
|
||||||
|
if req.BlenderVersion != nil && *req.BlenderVersion != "" {
|
||||||
|
metadata.BlenderVersion = *req.BlenderVersion
|
||||||
|
}
|
||||||
|
if req.UnhideObjects != nil {
|
||||||
|
metadata.UnhideObjects = req.UnhideObjects
|
||||||
|
}
|
||||||
|
if req.EnableExecution != nil {
|
||||||
|
metadata.EnableExecution = req.EnableExecution
|
||||||
|
}
|
||||||
|
|
||||||
|
if metadata.BlenderVersion == "" {
|
||||||
|
return nil, fmt.Errorf("blender_version is required (from upload analysis or job request)")
|
||||||
|
}
|
||||||
|
|
||||||
|
return &metadata, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"jiggablend/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMergeBlendMetadataForJobCreate_PreservesUploadMetadata(t *testing.T) {
|
||||||
|
uploadMeta := &types.BlendMetadata{
|
||||||
|
FrameStart: -15,
|
||||||
|
FrameEnd: 1468,
|
||||||
|
BlenderVersion: "4.5.11",
|
||||||
|
RenderSettings: types.RenderSettings{
|
||||||
|
ResolutionX: 2560,
|
||||||
|
ResolutionY: 1440,
|
||||||
|
Engine: "cycles",
|
||||||
|
OutputFormat: "EXR",
|
||||||
|
},
|
||||||
|
SceneInfo: types.SceneInfo{
|
||||||
|
ObjectCount: 42,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
frameStart := 800
|
||||||
|
frameEnd := 800
|
||||||
|
outputFormat := "EXR"
|
||||||
|
req := &types.CreateJobRequest{
|
||||||
|
FrameStart: &frameStart,
|
||||||
|
FrameEnd: &frameEnd,
|
||||||
|
OutputFormat: &outputFormat,
|
||||||
|
}
|
||||||
|
|
||||||
|
meta, err := mergeBlendMetadataForJobCreate(uploadMeta, req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("mergeBlendMetadataForJobCreate failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if meta.BlenderVersion != "4.5.11" {
|
||||||
|
t.Fatalf("expected blender_version 4.5.11, got %q", meta.BlenderVersion)
|
||||||
|
}
|
||||||
|
if meta.FrameStart != 800 || meta.FrameEnd != 800 {
|
||||||
|
t.Fatalf("expected frame range 800-800, got %d-%d", meta.FrameStart, meta.FrameEnd)
|
||||||
|
}
|
||||||
|
if meta.RenderSettings.ResolutionX != 2560 || meta.RenderSettings.ResolutionY != 1440 {
|
||||||
|
t.Fatalf("expected resolution 2560x1440, got %dx%d", meta.RenderSettings.ResolutionX, meta.RenderSettings.ResolutionY)
|
||||||
|
}
|
||||||
|
if meta.RenderSettings.Engine != "cycles" {
|
||||||
|
t.Fatalf("expected engine cycles, got %q", meta.RenderSettings.Engine)
|
||||||
|
}
|
||||||
|
if meta.RenderSettings.OutputFormat != "EXR" {
|
||||||
|
t.Fatalf("expected output_format EXR, got %q", meta.RenderSettings.OutputFormat)
|
||||||
|
}
|
||||||
|
if meta.SceneInfo.ObjectCount != 42 {
|
||||||
|
t.Fatalf("expected scene object_count 42, got %d", meta.SceneInfo.ObjectCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeBlendMetadataForJobCreate_RequestOverridesVersion(t *testing.T) {
|
||||||
|
uploadMeta := &types.BlendMetadata{
|
||||||
|
BlenderVersion: "4.5.11",
|
||||||
|
}
|
||||||
|
|
||||||
|
frameStart := 1
|
||||||
|
frameEnd := 1
|
||||||
|
override := "4.2.3"
|
||||||
|
req := &types.CreateJobRequest{
|
||||||
|
FrameStart: &frameStart,
|
||||||
|
FrameEnd: &frameEnd,
|
||||||
|
BlenderVersion: &override,
|
||||||
|
}
|
||||||
|
|
||||||
|
meta, err := mergeBlendMetadataForJobCreate(uploadMeta, req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("mergeBlendMetadataForJobCreate failed: %v", err)
|
||||||
|
}
|
||||||
|
if meta.BlenderVersion != "4.2.3" {
|
||||||
|
t.Fatalf("expected request override 4.2.3, got %q", meta.BlenderVersion)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeBlendMetadataForJobCreate_RequiresBlenderVersion(t *testing.T) {
|
||||||
|
frameStart := 1
|
||||||
|
frameEnd := 1
|
||||||
|
req := &types.CreateJobRequest{
|
||||||
|
FrameStart: &frameStart,
|
||||||
|
FrameEnd: &frameEnd,
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := mergeBlendMetadataForJobCreate(nil, req); err == nil {
|
||||||
|
t.Fatal("expected error when blender_version is missing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadSessionMetadata(t *testing.T) {
|
||||||
|
session := &UploadSession{
|
||||||
|
ResultMetadata: &types.BlendMetadata{
|
||||||
|
BlenderVersion: "4.5.11",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
meta := uploadSessionMetadata(session)
|
||||||
|
if meta == nil || meta.BlenderVersion != "4.5.11" {
|
||||||
|
t.Fatalf("unexpected metadata: %+v", meta)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"jiggablend/internal/storage"
|
||||||
|
"jiggablend/pkg/executils"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestExtractTar_ExtractsRegularFile(t *testing.T) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
tw := tar.NewWriter(&buf)
|
||||||
|
_ = tw.WriteHeader(&tar.Header{Name: "ctx/scene.blend", Mode: 0644, Size: 4, Typeflag: tar.TypeReg})
|
||||||
|
_, _ = tw.Write([]byte("data"))
|
||||||
|
_ = tw.Close()
|
||||||
|
|
||||||
|
tarPath := filepath.Join(t.TempDir(), "ctx.tar")
|
||||||
|
if err := os.WriteFile(tarPath, buf.Bytes(), 0644); err != nil {
|
||||||
|
t.Fatalf("write tar: %v", err)
|
||||||
|
}
|
||||||
|
dest := t.TempDir()
|
||||||
|
m := &Manager{}
|
||||||
|
if err := m.extractTar(tarPath, dest); err != nil {
|
||||||
|
t.Fatalf("extractTar failed: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(dest, "ctx", "scene.blend")); err != nil {
|
||||||
|
t.Fatalf("expected extracted file: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractTar_RejectsTraversal(t *testing.T) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
tw := tar.NewWriter(&buf)
|
||||||
|
_ = tw.WriteHeader(&tar.Header{Name: "../evil.txt", Mode: 0644, Size: 1, Typeflag: tar.TypeReg})
|
||||||
|
_, _ = tw.Write([]byte("x"))
|
||||||
|
_ = tw.Close()
|
||||||
|
|
||||||
|
tarPath := filepath.Join(t.TempDir(), "bad.tar")
|
||||||
|
if err := os.WriteFile(tarPath, buf.Bytes(), 0644); err != nil {
|
||||||
|
t.Fatalf("write tar: %v", err)
|
||||||
|
}
|
||||||
|
m := &Manager{}
|
||||||
|
if err := m.extractTar(tarPath, t.TempDir()); err == nil {
|
||||||
|
t.Fatal("expected path traversal error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractMetadataFromContext_UsesCommandSeam(t *testing.T) {
|
||||||
|
base := t.TempDir()
|
||||||
|
st, err := storage.NewStorage(base)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("new storage: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
jobID := int64(42)
|
||||||
|
jobDir := st.JobPath(jobID)
|
||||||
|
if err := os.MkdirAll(jobDir, 0755); err != nil {
|
||||||
|
t.Fatalf("mkdir job dir: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
tw := tar.NewWriter(&buf)
|
||||||
|
_ = tw.WriteHeader(&tar.Header{Name: "scene.blend", Mode: 0644, Size: 4, Typeflag: tar.TypeReg})
|
||||||
|
_, _ = tw.Write([]byte("fake"))
|
||||||
|
_ = tw.Close()
|
||||||
|
if err := os.WriteFile(filepath.Join(jobDir, "context.tar"), buf.Bytes(), 0644); err != nil {
|
||||||
|
t.Fatalf("write context tar: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
origResolve := resolveMetadataBlenderPath
|
||||||
|
origRun := runMetadataCommand
|
||||||
|
resolveMetadataBlenderPath = func(_ string) (string, error) { return "/usr/bin/blender", nil }
|
||||||
|
runMetadataCommand = func(_ string, _ []string, _ string, _ []string, _ int64, _ *executils.ProcessTracker) (*executils.CommandResult, error) {
|
||||||
|
return &executils.CommandResult{
|
||||||
|
Stdout: `noise
|
||||||
|
{"frame_start":1,"frame_end":3,"has_negative_frames":false,"render_settings":{"resolution_x":1920,"resolution_y":1080,"frame_rate":24,"output_format":"PNG","engine":"CYCLES"},"scene_info":{"camera_count":1,"object_count":2,"material_count":3}}
|
||||||
|
done`,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
resolveMetadataBlenderPath = origResolve
|
||||||
|
runMetadataCommand = origRun
|
||||||
|
}()
|
||||||
|
|
||||||
|
m := &Manager{storage: st}
|
||||||
|
meta, err := m.extractMetadataFromContext(jobID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("extractMetadataFromContext failed: %v", err)
|
||||||
|
}
|
||||||
|
if meta.FrameStart != 1 || meta.FrameEnd != 3 {
|
||||||
|
t.Fatalf("unexpected metadata: %+v", *meta)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"html/template"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
authpkg "jiggablend/internal/auth"
|
||||||
|
"jiggablend/web"
|
||||||
|
)
|
||||||
|
|
||||||
|
type uiRenderer struct {
|
||||||
|
templates *template.Template
|
||||||
|
}
|
||||||
|
|
||||||
|
type pageData struct {
|
||||||
|
Title string
|
||||||
|
CurrentPath string
|
||||||
|
ContentTemplate string
|
||||||
|
PageScript string
|
||||||
|
User *authpkg.Session
|
||||||
|
Error string
|
||||||
|
Notice string
|
||||||
|
Data interface{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newUIRenderer() (*uiRenderer, error) {
|
||||||
|
tpl, err := template.New("base").Funcs(template.FuncMap{
|
||||||
|
"formatTime": func(t time.Time) string {
|
||||||
|
if t.IsZero() {
|
||||||
|
return "-"
|
||||||
|
}
|
||||||
|
return t.Local().Format("2006-01-02 15:04:05")
|
||||||
|
},
|
||||||
|
"statusClass": func(status string) string {
|
||||||
|
switch status {
|
||||||
|
case "completed":
|
||||||
|
return "status-completed"
|
||||||
|
case "running":
|
||||||
|
return "status-running"
|
||||||
|
case "failed":
|
||||||
|
return "status-failed"
|
||||||
|
case "cancelled":
|
||||||
|
return "status-cancelled"
|
||||||
|
case "online":
|
||||||
|
return "status-online"
|
||||||
|
case "offline":
|
||||||
|
return "status-offline"
|
||||||
|
case "busy":
|
||||||
|
return "status-busy"
|
||||||
|
default:
|
||||||
|
return "status-pending"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"progressInt": func(v float64) int {
|
||||||
|
if v < 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if v > 100 {
|
||||||
|
return 100
|
||||||
|
}
|
||||||
|
return int(v)
|
||||||
|
},
|
||||||
|
"derefInt": func(v *int) string {
|
||||||
|
if v == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%d", *v)
|
||||||
|
},
|
||||||
|
"derefString": func(v *string) string {
|
||||||
|
if v == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return *v
|
||||||
|
},
|
||||||
|
"hasSuffixFold": func(value, suffix string) bool {
|
||||||
|
return strings.HasSuffix(strings.ToLower(value), strings.ToLower(suffix))
|
||||||
|
},
|
||||||
|
}).ParseFS(
|
||||||
|
web.GetTemplateFS(),
|
||||||
|
"templates/*.html",
|
||||||
|
"templates/partials/*.html",
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("parse templates: %w", err)
|
||||||
|
}
|
||||||
|
return &uiRenderer{templates: tpl}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *uiRenderer) render(w http.ResponseWriter, data pageData) {
|
||||||
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
if err := r.templates.ExecuteTemplate(w, "base", data); err != nil {
|
||||||
|
log.Printf("Template render error: %v", err)
|
||||||
|
http.Error(w, "template render error", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *uiRenderer) renderTemplate(w http.ResponseWriter, templateName string, data interface{}) {
|
||||||
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||||
|
if err := r.templates.ExecuteTemplate(w, templateName, data); err != nil {
|
||||||
|
log.Printf("Template render error for %s: %v", templateName, err)
|
||||||
|
http.Error(w, "template render error", http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestNewUIRendererParsesTemplates(t *testing.T) {
|
||||||
|
renderer, err := newUIRenderer()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newUIRenderer returned error: %v", err)
|
||||||
|
}
|
||||||
|
if renderer == nil || renderer.templates == nil {
|
||||||
|
t.Fatalf("renderer/templates should not be nil")
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,100 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseBlenderFrame(t *testing.T) {
|
||||||
|
frame, ok := parseBlenderFrame("Info Fra:2470 Mem:12.00M")
|
||||||
|
if !ok || frame != 2470 {
|
||||||
|
t.Fatalf("parseBlenderFrame() = (%d,%v), want (2470,true)", frame, ok)
|
||||||
|
}
|
||||||
|
if _, ok := parseBlenderFrame("no frame here"); ok {
|
||||||
|
t.Fatal("expected parse to fail for non-frame text")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJobTaskCounts_Progress(t *testing.T) {
|
||||||
|
c := &jobTaskCounts{total: 10, completed: 4}
|
||||||
|
if got := c.progress(); got != 40 {
|
||||||
|
t.Fatalf("progress() = %v, want 40", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseWSTaskUpdate_LegacyErrorObject(t *testing.T) {
|
||||||
|
// Simulates older runners that JSON-marshaled an error interface as {}
|
||||||
|
raw := json.RawMessage(`{"task_id":99,"success":false,"error":{}}`)
|
||||||
|
update, err := parseWSTaskUpdate(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parseWSTaskUpdate: %v", err)
|
||||||
|
}
|
||||||
|
if update.TaskID != 99 {
|
||||||
|
t.Fatalf("TaskID = %d, want 99", update.TaskID)
|
||||||
|
}
|
||||||
|
if update.Success {
|
||||||
|
t.Fatal("Success = true, want false")
|
||||||
|
}
|
||||||
|
if update.Error != "" {
|
||||||
|
t.Fatalf("Error = %q, want empty (legacy object)", update.Error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseWSTaskUpdate_StringError(t *testing.T) {
|
||||||
|
raw := json.RawMessage(`{"task_id":7,"success":false,"error":"blender failed: signal: segmentation fault (core dumped)","free_requeue":true}`)
|
||||||
|
update, err := parseWSTaskUpdate(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parseWSTaskUpdate: %v", err)
|
||||||
|
}
|
||||||
|
if update.TaskID != 7 || update.Success {
|
||||||
|
t.Fatalf("unexpected update: %+v", update)
|
||||||
|
}
|
||||||
|
if update.Error != "blender failed: signal: segmentation fault (core dumped)" {
|
||||||
|
t.Fatalf("Error = %q", update.Error)
|
||||||
|
}
|
||||||
|
if !update.FreeRequeue {
|
||||||
|
t.Fatal("expected FreeRequeue true")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatInferredTaskFailure(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
in string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "task failed segfault prefix",
|
||||||
|
in: "Task failed: blender failed: signal: segmentation fault (core dumped)",
|
||||||
|
want: "blender failed: signal: segmentation fault (core dumped)",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blender render failed prefix",
|
||||||
|
in: "Blender render failed: blender failed: signal: segmentation fault (core dumped)",
|
||||||
|
want: "blender failed: signal: segmentation fault (core dumped)",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "double prefix",
|
||||||
|
in: "Task failed: Blender render failed: blender failed: signal: segmentation fault (core dumped)",
|
||||||
|
want: "blender failed: signal: segmentation fault (core dumped)",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "plain message",
|
||||||
|
in: "blender failed: exit status 1",
|
||||||
|
want: "blender failed: exit status 1",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty falls back",
|
||||||
|
in: " ",
|
||||||
|
want: defaultUnexpectedDisconnectError,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := formatInferredTaskFailure(tt.in); got != tt.want {
|
||||||
|
t.Fatalf("formatInferredTaskFailure(%q) = %q, want %q", tt.in, got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,556 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
authpkg "jiggablend/internal/auth"
|
||||||
|
|
||||||
|
"github.com/go-chi/chi/v5"
|
||||||
|
)
|
||||||
|
|
||||||
|
type uiJobSummary struct {
|
||||||
|
ID int64
|
||||||
|
Name string
|
||||||
|
Status string
|
||||||
|
Progress float64
|
||||||
|
FrameStart *int
|
||||||
|
FrameEnd *int
|
||||||
|
OutputFormat *string
|
||||||
|
CreatedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type uiTaskSummary struct {
|
||||||
|
ID int64
|
||||||
|
TaskType string
|
||||||
|
Status string
|
||||||
|
Frame int
|
||||||
|
FrameEnd *int
|
||||||
|
CurrentStep string
|
||||||
|
RetryCount int
|
||||||
|
Error string
|
||||||
|
StartedAt *time.Time
|
||||||
|
CompletedAt *time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
type uiFileSummary struct {
|
||||||
|
ID int64
|
||||||
|
FileName string
|
||||||
|
FileType string
|
||||||
|
FileSize int64
|
||||||
|
CreatedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) setupUIRoutes() {
|
||||||
|
s.router.Get("/", s.handleUIRoot)
|
||||||
|
s.router.Get("/login", s.handleUILoginPage)
|
||||||
|
s.router.Post("/logout", s.handleUILogout)
|
||||||
|
|
||||||
|
s.router.Group(func(r chi.Router) {
|
||||||
|
r.Use(func(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(s.auth.Middleware(next.ServeHTTP))
|
||||||
|
})
|
||||||
|
r.Get("/jobs", s.handleUIJobsPage)
|
||||||
|
r.Get("/jobs/new", s.handleUINewJobPage)
|
||||||
|
r.Get("/jobs/{id}", s.handleUIJobDetailPage)
|
||||||
|
|
||||||
|
r.Get("/ui/fragments/jobs", s.handleUIJobsFragment)
|
||||||
|
r.Get("/ui/fragments/jobs/{id}/tasks", s.handleUIJobTasksFragment)
|
||||||
|
r.Get("/ui/fragments/jobs/{id}/files", s.handleUIJobFilesFragment)
|
||||||
|
})
|
||||||
|
|
||||||
|
s.router.Group(func(r chi.Router) {
|
||||||
|
r.Use(func(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(s.auth.AdminMiddleware(next.ServeHTTP))
|
||||||
|
})
|
||||||
|
r.Get("/admin", s.handleUIAdminPage)
|
||||||
|
r.Get("/ui/fragments/admin/runners", s.handleUIAdminRunnersFragment)
|
||||||
|
r.Get("/ui/fragments/admin/users", s.handleUIAdminUsersFragment)
|
||||||
|
r.Get("/ui/fragments/admin/apikeys", s.handleUIAdminAPIKeysFragment)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) sessionFromRequest(r *http.Request) (*authpkg.Session, bool) {
|
||||||
|
cookie, err := r.Cookie("session_id")
|
||||||
|
if err != nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return s.auth.GetSession(cookie.Value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUIRoot(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if _, ok := s.sessionFromRequest(r); ok {
|
||||||
|
http.Redirect(w, r, "/jobs", http.StatusFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
http.Redirect(w, r, "/login", http.StatusFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUILoginPage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if _, ok := s.sessionFromRequest(r); ok {
|
||||||
|
http.Redirect(w, r, "/jobs", http.StatusFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.ui.render(w, pageData{
|
||||||
|
Title: "Login",
|
||||||
|
CurrentPath: "/login",
|
||||||
|
ContentTemplate: "page_login",
|
||||||
|
PageScript: "/assets/login.js",
|
||||||
|
Data: map[string]interface{}{
|
||||||
|
"google_enabled": s.auth.IsGoogleOAuthConfigured(),
|
||||||
|
"discord_enabled": s.auth.IsDiscordOAuthConfigured(),
|
||||||
|
"local_enabled": s.auth.IsLocalLoginEnabled(),
|
||||||
|
"error": r.URL.Query().Get("error"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUILogout(w http.ResponseWriter, r *http.Request) {
|
||||||
|
cookie, err := r.Cookie("session_id")
|
||||||
|
if err == nil {
|
||||||
|
s.auth.DeleteSession(cookie.Value)
|
||||||
|
}
|
||||||
|
expired := &http.Cookie{
|
||||||
|
Name: "session_id",
|
||||||
|
Value: "",
|
||||||
|
Path: "/",
|
||||||
|
MaxAge: -1,
|
||||||
|
HttpOnly: true,
|
||||||
|
SameSite: http.SameSiteLaxMode,
|
||||||
|
}
|
||||||
|
if s.cfg.IsProductionMode() {
|
||||||
|
expired.Secure = true
|
||||||
|
}
|
||||||
|
http.SetCookie(w, expired)
|
||||||
|
http.Redirect(w, r, "/login", http.StatusFound)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUIJobsPage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
user, _ := s.sessionFromRequest(r)
|
||||||
|
s.ui.render(w, pageData{
|
||||||
|
Title: "Jobs",
|
||||||
|
CurrentPath: "/jobs",
|
||||||
|
ContentTemplate: "page_jobs",
|
||||||
|
PageScript: "/assets/jobs.js",
|
||||||
|
User: user,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUINewJobPage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
user, _ := s.sessionFromRequest(r)
|
||||||
|
s.ui.render(w, pageData{
|
||||||
|
Title: "New Job",
|
||||||
|
CurrentPath: "/jobs/new",
|
||||||
|
ContentTemplate: "page_jobs_new",
|
||||||
|
PageScript: "/assets/job_new.js",
|
||||||
|
User: user,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUIJobDetailPage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
userID, err := getUserID(r)
|
||||||
|
if err != nil {
|
||||||
|
http.Redirect(w, r, "/login", http.StatusFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
isAdmin := authpkg.IsAdmin(r.Context())
|
||||||
|
jobID, err := parseID(r, "id")
|
||||||
|
if err != nil {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
job, err := s.getUIJob(jobID, userID, isAdmin)
|
||||||
|
if err != nil {
|
||||||
|
http.NotFound(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
user, _ := s.sessionFromRequest(r)
|
||||||
|
s.ui.render(w, pageData{
|
||||||
|
Title: fmt.Sprintf("Job %d", jobID),
|
||||||
|
CurrentPath: "/jobs",
|
||||||
|
ContentTemplate: "page_job_show",
|
||||||
|
PageScript: "/assets/job_show.js",
|
||||||
|
User: user,
|
||||||
|
Data: map[string]interface{}{"job": job},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUIAdminPage(w http.ResponseWriter, r *http.Request) {
|
||||||
|
user, _ := s.sessionFromRequest(r)
|
||||||
|
regEnabled, _ := s.auth.IsRegistrationEnabled()
|
||||||
|
s.ui.render(w, pageData{
|
||||||
|
Title: "Admin",
|
||||||
|
CurrentPath: "/admin",
|
||||||
|
ContentTemplate: "page_admin",
|
||||||
|
PageScript: "/assets/admin.js",
|
||||||
|
User: user,
|
||||||
|
Data: map[string]interface{}{
|
||||||
|
"registration_enabled": regEnabled,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUIJobsFragment(w http.ResponseWriter, r *http.Request) {
|
||||||
|
userID, err := getUserID(r)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
jobs, err := s.listUIJobSummaries(userID, 50, 0)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.ui.renderTemplate(w, "partial_jobs_table", map[string]interface{}{"jobs": jobs})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUIJobTasksFragment(w http.ResponseWriter, r *http.Request) {
|
||||||
|
userID, err := getUserID(r)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
isAdmin := authpkg.IsAdmin(r.Context())
|
||||||
|
jobID, err := parseID(r, "id")
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "invalid job id", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := s.getUIJob(jobID, userID, isAdmin); err != nil {
|
||||||
|
http.Error(w, "job not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
tasks, err := s.listUITasks(jobID)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.ui.renderTemplate(w, "partial_job_tasks", map[string]interface{}{
|
||||||
|
"job_id": jobID,
|
||||||
|
"tasks": tasks,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUIJobFilesFragment(w http.ResponseWriter, r *http.Request) {
|
||||||
|
userID, err := getUserID(r)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
isAdmin := authpkg.IsAdmin(r.Context())
|
||||||
|
jobID, err := parseID(r, "id")
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, "invalid job id", http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := s.getUIJob(jobID, userID, isAdmin); err != nil {
|
||||||
|
http.Error(w, "job not found", http.StatusNotFound)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
files, err := s.listUIFiles(jobID, 100)
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
outputFiles := make([]uiFileSummary, 0, len(files))
|
||||||
|
adminInputFiles := make([]uiFileSummary, 0)
|
||||||
|
for _, file := range files {
|
||||||
|
if strings.EqualFold(file.FileType, "output") {
|
||||||
|
outputFiles = append(outputFiles, file)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if isAdmin {
|
||||||
|
adminInputFiles = append(adminInputFiles, file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
s.ui.renderTemplate(w, "partial_job_files", map[string]interface{}{
|
||||||
|
"job_id": jobID,
|
||||||
|
"files": outputFiles,
|
||||||
|
"is_admin": isAdmin,
|
||||||
|
"admin_input_files": adminInputFiles,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUIAdminRunnersFragment(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var rows *sql.Rows
|
||||||
|
err := s.db.With(func(conn *sql.DB) error {
|
||||||
|
var qErr error
|
||||||
|
rows, qErr = conn.Query(`SELECT id, name, hostname, status, last_heartbeat, priority, created_at FROM runners ORDER BY created_at DESC`)
|
||||||
|
return qErr
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
type runner struct {
|
||||||
|
ID int64
|
||||||
|
Name string
|
||||||
|
Hostname string
|
||||||
|
Status string
|
||||||
|
LastHeartbeat time.Time
|
||||||
|
Priority int
|
||||||
|
CreatedAt time.Time
|
||||||
|
}
|
||||||
|
all := make([]runner, 0)
|
||||||
|
for rows.Next() {
|
||||||
|
var item runner
|
||||||
|
if scanErr := rows.Scan(&item.ID, &item.Name, &item.Hostname, &item.Status, &item.LastHeartbeat, &item.Priority, &item.CreatedAt); scanErr != nil {
|
||||||
|
http.Error(w, scanErr.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
all = append(all, item)
|
||||||
|
}
|
||||||
|
s.ui.renderTemplate(w, "partial_admin_runners", map[string]interface{}{"runners": all})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUIAdminUsersFragment(w http.ResponseWriter, r *http.Request) {
|
||||||
|
currentUserID, _ := getUserID(r)
|
||||||
|
firstUserID, _ := s.auth.GetFirstUserID()
|
||||||
|
var rows *sql.Rows
|
||||||
|
err := s.db.With(func(conn *sql.DB) error {
|
||||||
|
var qErr error
|
||||||
|
rows, qErr = conn.Query(`SELECT id, email, name, oauth_provider, is_admin, created_at FROM users ORDER BY created_at DESC`)
|
||||||
|
return qErr
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
type user struct {
|
||||||
|
ID int64
|
||||||
|
Email string
|
||||||
|
Name string
|
||||||
|
OAuthProvider string
|
||||||
|
IsAdmin bool
|
||||||
|
IsFirstUser bool
|
||||||
|
CreatedAt time.Time
|
||||||
|
}
|
||||||
|
all := make([]user, 0)
|
||||||
|
for rows.Next() {
|
||||||
|
var item user
|
||||||
|
if scanErr := rows.Scan(&item.ID, &item.Email, &item.Name, &item.OAuthProvider, &item.IsAdmin, &item.CreatedAt); scanErr != nil {
|
||||||
|
http.Error(w, scanErr.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
item.IsFirstUser = item.ID == firstUserID
|
||||||
|
all = append(all, item)
|
||||||
|
}
|
||||||
|
s.ui.renderTemplate(w, "partial_admin_users", map[string]interface{}{
|
||||||
|
"users": all,
|
||||||
|
"current_user_id": currentUserID,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) handleUIAdminAPIKeysFragment(w http.ResponseWriter, r *http.Request) {
|
||||||
|
keys, err := s.secrets.ListRunnerAPIKeys()
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
type item struct {
|
||||||
|
ID int64
|
||||||
|
Name string
|
||||||
|
Scope string
|
||||||
|
Key string
|
||||||
|
IsActive bool
|
||||||
|
CreatedAt time.Time
|
||||||
|
}
|
||||||
|
out := make([]item, 0, len(keys))
|
||||||
|
for _, key := range keys {
|
||||||
|
out = append(out, item{
|
||||||
|
ID: key.ID,
|
||||||
|
Name: key.Name,
|
||||||
|
Scope: key.Scope,
|
||||||
|
Key: key.Key,
|
||||||
|
IsActive: key.IsActive,
|
||||||
|
CreatedAt: key.CreatedAt,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
s.ui.renderTemplate(w, "partial_admin_apikeys", map[string]interface{}{"keys": out})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) listUIJobSummaries(userID int64, limit int, offset int) ([]uiJobSummary, error) {
|
||||||
|
rows := &sql.Rows{}
|
||||||
|
err := s.db.With(func(conn *sql.DB) error {
|
||||||
|
var qErr error
|
||||||
|
rows, qErr = conn.Query(
|
||||||
|
`SELECT id, name, status, progress, frame_start, frame_end, output_format, created_at
|
||||||
|
FROM jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?`,
|
||||||
|
userID, limit, offset,
|
||||||
|
)
|
||||||
|
return qErr
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
out := make([]uiJobSummary, 0)
|
||||||
|
for rows.Next() {
|
||||||
|
var item uiJobSummary
|
||||||
|
var frameStart, frameEnd sql.NullInt64
|
||||||
|
var outputFormat sql.NullString
|
||||||
|
if scanErr := rows.Scan(&item.ID, &item.Name, &item.Status, &item.Progress, &frameStart, &frameEnd, &outputFormat, &item.CreatedAt); scanErr != nil {
|
||||||
|
return nil, scanErr
|
||||||
|
}
|
||||||
|
if frameStart.Valid {
|
||||||
|
v := int(frameStart.Int64)
|
||||||
|
item.FrameStart = &v
|
||||||
|
}
|
||||||
|
if frameEnd.Valid {
|
||||||
|
v := int(frameEnd.Int64)
|
||||||
|
item.FrameEnd = &v
|
||||||
|
}
|
||||||
|
if outputFormat.Valid {
|
||||||
|
item.OutputFormat = &outputFormat.String
|
||||||
|
}
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) getUIJob(jobID int64, userID int64, isAdmin bool) (uiJobSummary, error) {
|
||||||
|
var item uiJobSummary
|
||||||
|
var frameStart, frameEnd sql.NullInt64
|
||||||
|
var outputFormat sql.NullString
|
||||||
|
|
||||||
|
err := s.db.With(func(conn *sql.DB) error {
|
||||||
|
if isAdmin {
|
||||||
|
return conn.QueryRow(
|
||||||
|
`SELECT id, name, status, progress, frame_start, frame_end, output_format, created_at
|
||||||
|
FROM jobs WHERE id = ?`,
|
||||||
|
jobID,
|
||||||
|
).Scan(&item.ID, &item.Name, &item.Status, &item.Progress, &frameStart, &frameEnd, &outputFormat, &item.CreatedAt)
|
||||||
|
}
|
||||||
|
return conn.QueryRow(
|
||||||
|
`SELECT id, name, status, progress, frame_start, frame_end, output_format, created_at
|
||||||
|
FROM jobs WHERE id = ? AND user_id = ?`,
|
||||||
|
jobID, userID,
|
||||||
|
).Scan(&item.ID, &item.Name, &item.Status, &item.Progress, &frameStart, &frameEnd, &outputFormat, &item.CreatedAt)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return uiJobSummary{}, err
|
||||||
|
}
|
||||||
|
if frameStart.Valid {
|
||||||
|
v := int(frameStart.Int64)
|
||||||
|
item.FrameStart = &v
|
||||||
|
}
|
||||||
|
if frameEnd.Valid {
|
||||||
|
v := int(frameEnd.Int64)
|
||||||
|
item.FrameEnd = &v
|
||||||
|
}
|
||||||
|
if outputFormat.Valid {
|
||||||
|
item.OutputFormat = &outputFormat.String
|
||||||
|
}
|
||||||
|
return item, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) listUITasks(jobID int64) ([]uiTaskSummary, error) {
|
||||||
|
var rows *sql.Rows
|
||||||
|
err := s.db.With(func(conn *sql.DB) error {
|
||||||
|
var qErr error
|
||||||
|
rows, qErr = conn.Query(
|
||||||
|
`SELECT id, task_type, status, frame, frame_end, current_step, retry_count, error_message, started_at, completed_at
|
||||||
|
FROM tasks WHERE job_id = ? ORDER BY id ASC`,
|
||||||
|
jobID,
|
||||||
|
)
|
||||||
|
return qErr
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
out := make([]uiTaskSummary, 0)
|
||||||
|
for rows.Next() {
|
||||||
|
var item uiTaskSummary
|
||||||
|
var frameEnd sql.NullInt64
|
||||||
|
var currentStep sql.NullString
|
||||||
|
var errMsg sql.NullString
|
||||||
|
var startedAt, completedAt sql.NullTime
|
||||||
|
if scanErr := rows.Scan(
|
||||||
|
&item.ID, &item.TaskType, &item.Status, &item.Frame, &frameEnd,
|
||||||
|
¤tStep, &item.RetryCount, &errMsg, &startedAt, &completedAt,
|
||||||
|
); scanErr != nil {
|
||||||
|
return nil, scanErr
|
||||||
|
}
|
||||||
|
if frameEnd.Valid {
|
||||||
|
v := int(frameEnd.Int64)
|
||||||
|
item.FrameEnd = &v
|
||||||
|
}
|
||||||
|
if currentStep.Valid {
|
||||||
|
item.CurrentStep = currentStep.String
|
||||||
|
}
|
||||||
|
if errMsg.Valid {
|
||||||
|
item.Error = errMsg.String
|
||||||
|
}
|
||||||
|
if startedAt.Valid {
|
||||||
|
item.StartedAt = &startedAt.Time
|
||||||
|
}
|
||||||
|
if completedAt.Valid {
|
||||||
|
item.CompletedAt = &completedAt.Time
|
||||||
|
}
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Manager) listUIFiles(jobID int64, limit int) ([]uiFileSummary, error) {
|
||||||
|
var rows *sql.Rows
|
||||||
|
err := s.db.With(func(conn *sql.DB) error {
|
||||||
|
var qErr error
|
||||||
|
rows, qErr = conn.Query(
|
||||||
|
`SELECT id, file_name, file_type, file_size, created_at
|
||||||
|
FROM job_files WHERE job_id = ? ORDER BY created_at DESC LIMIT ?`,
|
||||||
|
jobID, limit,
|
||||||
|
)
|
||||||
|
return qErr
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
out := make([]uiFileSummary, 0)
|
||||||
|
for rows.Next() {
|
||||||
|
var item uiFileSummary
|
||||||
|
if scanErr := rows.Scan(&item.ID, &item.FileName, &item.FileType, &item.FileSize, &item.CreatedAt); scanErr != nil {
|
||||||
|
return nil, scanErr
|
||||||
|
}
|
||||||
|
out = append(out, item)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseBoolForm(r *http.Request, key string) bool {
|
||||||
|
v := strings.TrimSpace(strings.ToLower(r.FormValue(key)))
|
||||||
|
return v == "1" || v == "true" || v == "on" || v == "yes"
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseIntQuery(r *http.Request, key string, fallback int) int {
|
||||||
|
raw := strings.TrimSpace(r.URL.Query().Get(key))
|
||||||
|
if raw == "" {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
v, err := strconv.Atoi(raw)
|
||||||
|
if err != nil || v < 0 {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
@@ -0,0 +1,37 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseBoolForm(t *testing.T) {
|
||||||
|
req := httptest.NewRequest("POST", "/?flag=true", nil)
|
||||||
|
req.ParseForm()
|
||||||
|
req.Form.Set("enabled", "true")
|
||||||
|
if !parseBoolForm(req, "enabled") {
|
||||||
|
t.Fatalf("expected true for enabled=true")
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Form.Set("enabled", "no")
|
||||||
|
if parseBoolForm(req, "enabled") {
|
||||||
|
t.Fatalf("expected false for enabled=no")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseIntQuery(t *testing.T) {
|
||||||
|
req := httptest.NewRequest("GET", "/?limit=42", nil)
|
||||||
|
if got := parseIntQuery(req, "limit", 10); got != 42 {
|
||||||
|
t.Fatalf("expected 42, got %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
req = httptest.NewRequest("GET", "/?limit=-1", nil)
|
||||||
|
if got := parseIntQuery(req, "limit", 10); got != 10 {
|
||||||
|
t.Fatalf("expected fallback 10, got %d", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
req = httptest.NewRequest("GET", "/?limit=abc", nil)
|
||||||
|
if got := parseIntQuery(req, "limit", 10); got != 10 {
|
||||||
|
t.Fatalf("expected fallback 10, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,348 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"jiggablend/pkg/types"
|
||||||
|
|
||||||
|
"github.com/gorilla/websocket"
|
||||||
|
)
|
||||||
|
|
||||||
|
// JobConnection wraps a WebSocket connection for job communication.
|
||||||
|
type JobConnection struct {
|
||||||
|
conn *websocket.Conn
|
||||||
|
writeMu sync.Mutex
|
||||||
|
stopPing chan struct{}
|
||||||
|
stopHeartbeat chan struct{}
|
||||||
|
stopOnce sync.Once
|
||||||
|
isConnected bool
|
||||||
|
connMu sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewJobConnection creates a new job connection wrapper.
|
||||||
|
func NewJobConnection() *JobConnection {
|
||||||
|
return &JobConnection{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Connect establishes a WebSocket connection for a job (no runnerID needed).
|
||||||
|
func (j *JobConnection) Connect(managerURL, jobPath, jobToken string) error {
|
||||||
|
wsPath := jobPath + "/ws"
|
||||||
|
wsURL := strings.Replace(managerURL, "http://", "ws://", 1)
|
||||||
|
wsURL = strings.Replace(wsURL, "https://", "wss://", 1)
|
||||||
|
wsURL += wsPath
|
||||||
|
|
||||||
|
log.Printf("Connecting to job WebSocket: %s", wsPath)
|
||||||
|
|
||||||
|
dialer := websocket.Dialer{
|
||||||
|
HandshakeTimeout: 10 * time.Second,
|
||||||
|
}
|
||||||
|
conn, _, err := dialer.Dial(wsURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to connect job WebSocket: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
j.conn = conn
|
||||||
|
|
||||||
|
// Send auth message
|
||||||
|
authMsg := map[string]interface{}{
|
||||||
|
"type": "auth",
|
||||||
|
"job_token": jobToken,
|
||||||
|
}
|
||||||
|
if err := conn.WriteJSON(authMsg); err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return fmt.Errorf("failed to send auth: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait for auth_ok
|
||||||
|
conn.SetReadDeadline(time.Now().Add(30 * time.Second))
|
||||||
|
var authResp map[string]string
|
||||||
|
if err := conn.ReadJSON(&authResp); err != nil {
|
||||||
|
conn.Close()
|
||||||
|
return fmt.Errorf("failed to read auth response: %w", err)
|
||||||
|
}
|
||||||
|
if authResp["type"] == "error" {
|
||||||
|
conn.Close()
|
||||||
|
return fmt.Errorf("auth failed: %s", authResp["message"])
|
||||||
|
}
|
||||||
|
if authResp["type"] != "auth_ok" {
|
||||||
|
conn.Close()
|
||||||
|
return fmt.Errorf("unexpected auth response: %s", authResp["type"])
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear read deadline after auth
|
||||||
|
conn.SetReadDeadline(time.Time{})
|
||||||
|
|
||||||
|
// Set up ping/pong handler for keepalive
|
||||||
|
conn.SetPongHandler(func(string) error {
|
||||||
|
conn.SetReadDeadline(time.Now().Add(90 * time.Second))
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
// Start ping goroutine
|
||||||
|
j.stopPing = make(chan struct{})
|
||||||
|
j.connMu.Lock()
|
||||||
|
j.isConnected = true
|
||||||
|
j.connMu.Unlock()
|
||||||
|
go j.pingLoop()
|
||||||
|
|
||||||
|
// Start WebSocket heartbeat goroutine
|
||||||
|
j.stopHeartbeat = make(chan struct{})
|
||||||
|
go j.heartbeatLoop()
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// pingLoop sends periodic pings to keep the WebSocket connection alive.
|
||||||
|
func (j *JobConnection) pingLoop() {
|
||||||
|
defer func() {
|
||||||
|
if rec := recover(); rec != nil {
|
||||||
|
log.Printf("Ping loop panicked: %v", rec)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
ticker := time.NewTicker(30 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-j.stopPing:
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
j.writeMu.Lock()
|
||||||
|
if j.conn != nil {
|
||||||
|
deadline := time.Now().Add(10 * time.Second)
|
||||||
|
if err := j.conn.WriteControl(websocket.PingMessage, []byte{}, deadline); err != nil {
|
||||||
|
log.Printf("Failed to send ping, closing connection: %v", err)
|
||||||
|
j.connMu.Lock()
|
||||||
|
j.isConnected = false
|
||||||
|
if j.conn != nil {
|
||||||
|
j.conn.Close()
|
||||||
|
j.conn = nil
|
||||||
|
}
|
||||||
|
j.connMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
j.writeMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Heartbeat sends a heartbeat message over WebSocket to keep runner online.
|
||||||
|
func (j *JobConnection) Heartbeat() {
|
||||||
|
j.writeMu.Lock()
|
||||||
|
defer j.writeMu.Unlock()
|
||||||
|
if j.conn == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
msg := map[string]interface{}{
|
||||||
|
"type": "runner_heartbeat",
|
||||||
|
"timestamp": time.Now().Unix(),
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := j.conn.WriteJSON(msg); err != nil {
|
||||||
|
log.Printf("Failed to send WebSocket heartbeat: %v", err)
|
||||||
|
// Handle connection failure
|
||||||
|
j.connMu.Lock()
|
||||||
|
j.isConnected = false
|
||||||
|
if j.conn != nil {
|
||||||
|
j.conn.Close()
|
||||||
|
j.conn = nil
|
||||||
|
}
|
||||||
|
j.connMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// heartbeatLoop sends periodic heartbeat messages over WebSocket.
|
||||||
|
func (j *JobConnection) heartbeatLoop() {
|
||||||
|
defer func() {
|
||||||
|
if rec := recover(); rec != nil {
|
||||||
|
log.Printf("WebSocket heartbeat loop panicked: %v", rec)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
ticker := time.NewTicker(30 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-j.stopHeartbeat:
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
j.Heartbeat()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stopLoops signals ping/heartbeat goroutines to exit.
|
||||||
|
// Channels are closed once and left non-nil so loops can receive without racing Close.
|
||||||
|
func (j *JobConnection) stopLoops() {
|
||||||
|
j.stopOnce.Do(func() {
|
||||||
|
if j.stopHeartbeat != nil {
|
||||||
|
close(j.stopHeartbeat)
|
||||||
|
}
|
||||||
|
if j.stopPing != nil {
|
||||||
|
close(j.stopPing)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close closes the WebSocket connection.
|
||||||
|
func (j *JobConnection) Close() {
|
||||||
|
j.stopLoops()
|
||||||
|
|
||||||
|
j.writeMu.Lock()
|
||||||
|
defer j.writeMu.Unlock()
|
||||||
|
|
||||||
|
j.connMu.Lock()
|
||||||
|
j.isConnected = false
|
||||||
|
conn := j.conn
|
||||||
|
j.conn = nil
|
||||||
|
j.connMu.Unlock()
|
||||||
|
|
||||||
|
if conn != nil {
|
||||||
|
conn.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsConnected returns true if the connection is established.
|
||||||
|
func (j *JobConnection) IsConnected() bool {
|
||||||
|
j.connMu.RLock()
|
||||||
|
defer j.connMu.RUnlock()
|
||||||
|
return j.isConnected && j.conn != nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log sends a log entry to the manager.
|
||||||
|
func (j *JobConnection) Log(taskID int64, level types.LogLevel, message string) {
|
||||||
|
j.writeMu.Lock()
|
||||||
|
defer j.writeMu.Unlock()
|
||||||
|
if j.conn == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
msg := map[string]interface{}{
|
||||||
|
"type": "log_entry",
|
||||||
|
"data": map[string]interface{}{
|
||||||
|
"task_id": taskID,
|
||||||
|
"log_level": string(level),
|
||||||
|
"message": message,
|
||||||
|
},
|
||||||
|
"timestamp": time.Now().Unix(),
|
||||||
|
}
|
||||||
|
if err := j.conn.WriteJSON(msg); err != nil {
|
||||||
|
log.Printf("Failed to send job log, connection may be broken: %v", err)
|
||||||
|
// Close the connection on write error
|
||||||
|
j.connMu.Lock()
|
||||||
|
j.isConnected = false
|
||||||
|
if j.conn != nil {
|
||||||
|
j.conn.Close()
|
||||||
|
j.conn = nil
|
||||||
|
}
|
||||||
|
j.connMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Progress sends a progress update to the manager.
|
||||||
|
func (j *JobConnection) Progress(taskID int64, progress float64) {
|
||||||
|
j.writeMu.Lock()
|
||||||
|
defer j.writeMu.Unlock()
|
||||||
|
if j.conn == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
msg := map[string]interface{}{
|
||||||
|
"type": "progress",
|
||||||
|
"data": map[string]interface{}{
|
||||||
|
"task_id": taskID,
|
||||||
|
"progress": progress,
|
||||||
|
},
|
||||||
|
"timestamp": time.Now().Unix(),
|
||||||
|
}
|
||||||
|
if err := j.conn.WriteJSON(msg); err != nil {
|
||||||
|
log.Printf("Failed to send job progress, connection may be broken: %v", err)
|
||||||
|
// Close the connection on write error
|
||||||
|
j.connMu.Lock()
|
||||||
|
j.isConnected = false
|
||||||
|
if j.conn != nil {
|
||||||
|
j.conn.Close()
|
||||||
|
j.conn = nil
|
||||||
|
}
|
||||||
|
j.connMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// OutputUploaded notifies that an output file was uploaded.
|
||||||
|
func (j *JobConnection) OutputUploaded(taskID int64, fileName string) {
|
||||||
|
j.writeMu.Lock()
|
||||||
|
defer j.writeMu.Unlock()
|
||||||
|
if j.conn == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
msg := map[string]interface{}{
|
||||||
|
"type": "output_uploaded",
|
||||||
|
"data": map[string]interface{}{
|
||||||
|
"task_id": taskID,
|
||||||
|
"file_name": fileName,
|
||||||
|
},
|
||||||
|
"timestamp": time.Now().Unix(),
|
||||||
|
}
|
||||||
|
if err := j.conn.WriteJSON(msg); err != nil {
|
||||||
|
log.Printf("Failed to send output uploaded, connection may be broken: %v", err)
|
||||||
|
// Close the connection on write error
|
||||||
|
j.connMu.Lock()
|
||||||
|
j.isConnected = false
|
||||||
|
if j.conn != nil {
|
||||||
|
j.conn.Close()
|
||||||
|
j.conn = nil
|
||||||
|
}
|
||||||
|
j.connMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Complete sends task completion to the manager.
|
||||||
|
// errorMsg must be JSON-encoded as a string: encoding an error interface
|
||||||
|
// marshals to {} and the manager fails to unmarshal task_complete, which
|
||||||
|
// previously surfaced as a generic "WebSocket connection lost" with no retries.
|
||||||
|
// freeRequeue asks the manager to requeue a failure without incrementing retry_count
|
||||||
|
// (used when this attempt newly armed GPU lockout).
|
||||||
|
func (j *JobConnection) Complete(taskID int64, success bool, errorMsg error, freeRequeue bool) {
|
||||||
|
j.writeMu.Lock()
|
||||||
|
defer j.writeMu.Unlock()
|
||||||
|
if j.conn == nil {
|
||||||
|
log.Printf("Cannot send task complete: WebSocket connection is nil")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
data := map[string]interface{}{
|
||||||
|
"task_id": taskID,
|
||||||
|
"success": success,
|
||||||
|
}
|
||||||
|
if errorMsg != nil {
|
||||||
|
data["error"] = errorMsg.Error()
|
||||||
|
}
|
||||||
|
if freeRequeue {
|
||||||
|
data["free_requeue"] = true
|
||||||
|
}
|
||||||
|
|
||||||
|
msg := map[string]interface{}{
|
||||||
|
"type": "task_complete",
|
||||||
|
"data": data,
|
||||||
|
"timestamp": time.Now().Unix(),
|
||||||
|
}
|
||||||
|
if err := j.conn.WriteJSON(msg); err != nil {
|
||||||
|
log.Printf("Failed to send task complete, connection may be broken: %v", err)
|
||||||
|
// Close the connection on write error
|
||||||
|
j.connMu.Lock()
|
||||||
|
j.isConnected = false
|
||||||
|
if j.conn != nil {
|
||||||
|
j.conn.Close()
|
||||||
|
j.conn = nil
|
||||||
|
}
|
||||||
|
j.connMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,119 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/gorilla/websocket"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestComplete_ErrorIsJSONString ensures task_complete encodes the failure
|
||||||
|
// reason as a string. Marshaling a Go error interface produces {}, which the
|
||||||
|
// manager cannot unmarshal into WSTaskUpdate.Error and used to look like a
|
||||||
|
// silent WebSocket drop with no retries.
|
||||||
|
func TestComplete_ErrorIsJSONString(t *testing.T) {
|
||||||
|
upgrader := websocket.Upgrader{}
|
||||||
|
received := make(chan map[string]interface{}, 1)
|
||||||
|
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
conn, err := upgrader.Upgrade(w, r, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
|
||||||
|
// auth handshake
|
||||||
|
var auth map[string]interface{}
|
||||||
|
if err := conn.ReadJSON(&auth); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if auth["type"] == "auth" {
|
||||||
|
_ = conn.WriteJSON(map[string]string{"type": "auth_ok"})
|
||||||
|
}
|
||||||
|
|
||||||
|
var msg map[string]interface{}
|
||||||
|
if err := conn.ReadJSON(&msg); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
received <- msg
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
jc := NewJobConnection()
|
||||||
|
if err := jc.Connect(server.URL, "/job/1", "token123"); err != nil {
|
||||||
|
t.Fatalf("Connect failed: %v", err)
|
||||||
|
}
|
||||||
|
defer jc.Close()
|
||||||
|
|
||||||
|
jc.Complete(42, false, errors.New("blender failed: signal: segmentation fault (core dumped)"), true)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case msg := <-received:
|
||||||
|
if msg["type"] != "task_complete" {
|
||||||
|
t.Fatalf("type = %v, want task_complete", msg["type"])
|
||||||
|
}
|
||||||
|
data, ok := msg["data"].(map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
// gorilla may leave nested objects as map[string]interface{} after JSON round-trip;
|
||||||
|
// also accept raw re-marshal path
|
||||||
|
raw, _ := json.Marshal(msg["data"])
|
||||||
|
data = map[string]interface{}{}
|
||||||
|
if err := json.Unmarshal(raw, &data); err != nil {
|
||||||
|
t.Fatalf("data type %T: %v", msg["data"], err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
errVal, ok := data["error"].(string)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("error field type %T value %#v; want string (not object)", data["error"], data["error"])
|
||||||
|
}
|
||||||
|
if errVal != "blender failed: signal: segmentation fault (core dumped)" {
|
||||||
|
t.Fatalf("error = %q, want segfault message", errVal)
|
||||||
|
}
|
||||||
|
if data["success"] != false {
|
||||||
|
t.Fatalf("success = %v, want false", data["success"])
|
||||||
|
}
|
||||||
|
if data["free_requeue"] != true {
|
||||||
|
t.Fatalf("free_requeue = %v, want true", data["free_requeue"])
|
||||||
|
}
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for task_complete message")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJobConnection_ConnectAndClose(t *testing.T) {
|
||||||
|
upgrader := websocket.Upgrader{}
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
conn, err := upgrader.Upgrade(w, r, nil)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
|
||||||
|
var msg map[string]interface{}
|
||||||
|
if err := conn.ReadJSON(&msg); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if msg["type"] == "auth" {
|
||||||
|
_ = conn.WriteJSON(map[string]string{"type": "auth_ok"})
|
||||||
|
}
|
||||||
|
// Keep open briefly so client can mark connected.
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
jc := NewJobConnection()
|
||||||
|
managerURL := strings.Replace(server.URL, "http://", "http://", 1)
|
||||||
|
if err := jc.Connect(managerURL, "/job/1", "token123"); err != nil {
|
||||||
|
t.Fatalf("Connect failed: %v", err)
|
||||||
|
}
|
||||||
|
if !jc.IsConnected() {
|
||||||
|
t.Fatal("expected connection to be marked connected")
|
||||||
|
}
|
||||||
|
jc.Close()
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,473 @@
|
|||||||
|
// Package api provides HTTP and WebSocket communication with the manager server.
|
||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"mime/multipart"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"jiggablend/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ManagerClient handles all HTTP communication with the manager server.
|
||||||
|
type ManagerClient struct {
|
||||||
|
baseURL string
|
||||||
|
apiKey string
|
||||||
|
runnerID int64
|
||||||
|
httpClient *http.Client // Standard timeout for quick requests
|
||||||
|
longClient *http.Client // No timeout for large file transfers
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewManagerClient creates a new manager client.
|
||||||
|
func NewManagerClient(baseURL string) *ManagerClient {
|
||||||
|
return &ManagerClient{
|
||||||
|
baseURL: strings.TrimSuffix(baseURL, "/"),
|
||||||
|
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||||
|
longClient: &http.Client{Timeout: 0}, // No timeout for large transfers
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetCredentials sets the API key and runner ID after registration.
|
||||||
|
func (m *ManagerClient) SetCredentials(runnerID int64, apiKey string) {
|
||||||
|
m.runnerID = runnerID
|
||||||
|
m.apiKey = apiKey
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetRunnerID returns the registered runner ID.
|
||||||
|
func (m *ManagerClient) GetRunnerID() int64 {
|
||||||
|
return m.runnerID
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAPIKey returns the API key.
|
||||||
|
func (m *ManagerClient) GetAPIKey() string {
|
||||||
|
return m.apiKey
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBaseURL returns the base URL.
|
||||||
|
func (m *ManagerClient) GetBaseURL() string {
|
||||||
|
return m.baseURL
|
||||||
|
}
|
||||||
|
|
||||||
|
// Request performs an authenticated HTTP request with standard timeout.
|
||||||
|
func (m *ManagerClient) Request(method, path string, body []byte) (*http.Response, error) {
|
||||||
|
return m.doRequest(method, path, body, m.httpClient)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestLong performs an authenticated HTTP request with no timeout.
|
||||||
|
// Use for large file uploads/downloads.
|
||||||
|
func (m *ManagerClient) RequestLong(method, path string, body []byte) (*http.Response, error) {
|
||||||
|
return m.doRequest(method, path, body, m.longClient)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *ManagerClient) doRequest(method, path string, body []byte, client *http.Client) (*http.Response, error) {
|
||||||
|
if m.apiKey == "" {
|
||||||
|
return nil, fmt.Errorf("not authenticated")
|
||||||
|
}
|
||||||
|
|
||||||
|
fullURL := m.baseURL + path
|
||||||
|
req, err := http.NewRequest(method, fullURL, bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("Authorization", "Bearer "+m.apiKey)
|
||||||
|
if len(body) > 0 {
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
}
|
||||||
|
|
||||||
|
return client.Do(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestWithToken performs an authenticated HTTP request using a specific token.
|
||||||
|
func (m *ManagerClient) RequestWithToken(method, path, token string, body []byte) (*http.Response, error) {
|
||||||
|
return m.doRequestWithToken(method, path, token, body, m.httpClient)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestLongWithToken performs a long-running request with a specific token.
|
||||||
|
func (m *ManagerClient) RequestLongWithToken(method, path, token string, body []byte) (*http.Response, error) {
|
||||||
|
return m.doRequestWithToken(method, path, token, body, m.longClient)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *ManagerClient) doRequestWithToken(method, path, token string, body []byte, client *http.Client) (*http.Response, error) {
|
||||||
|
fullURL := m.baseURL + path
|
||||||
|
req, err := http.NewRequest(method, fullURL, bytes.NewReader(body))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("Authorization", "Bearer "+token)
|
||||||
|
if len(body) > 0 {
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
}
|
||||||
|
|
||||||
|
return client.Do(req)
|
||||||
|
}
|
||||||
|
|
||||||
|
// RegisterRequest is the request body for runner registration.
|
||||||
|
type RegisterRequest struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Hostname string `json:"hostname"`
|
||||||
|
Capabilities string `json:"capabilities"`
|
||||||
|
APIKey string `json:"api_key"`
|
||||||
|
Fingerprint string `json:"fingerprint,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RegisterResponse is the response from runner registration.
|
||||||
|
type RegisterResponse struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register registers the runner with the manager.
|
||||||
|
func (m *ManagerClient) Register(name, hostname string, capabilities map[string]interface{}, registrationToken, fingerprint string) (int64, error) {
|
||||||
|
capsJSON, err := json.Marshal(capabilities)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to marshal capabilities: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
reqBody := RegisterRequest{
|
||||||
|
Name: name,
|
||||||
|
Hostname: hostname,
|
||||||
|
Capabilities: string(capsJSON),
|
||||||
|
APIKey: registrationToken,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Only send fingerprint for non-fixed API keys
|
||||||
|
if !strings.HasPrefix(registrationToken, "jk_r0_") {
|
||||||
|
reqBody.Fingerprint = fingerprint
|
||||||
|
}
|
||||||
|
|
||||||
|
body, _ := json.Marshal(reqBody)
|
||||||
|
resp, err := m.httpClient.Post(
|
||||||
|
m.baseURL+"/api/runner/register",
|
||||||
|
"application/json",
|
||||||
|
bytes.NewReader(body),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("connection error: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusCreated {
|
||||||
|
bodyBytes, _ := io.ReadAll(resp.Body)
|
||||||
|
errorBody := string(bodyBytes)
|
||||||
|
|
||||||
|
// Check for token-related errors (should not retry)
|
||||||
|
if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusBadRequest {
|
||||||
|
errorLower := strings.ToLower(errorBody)
|
||||||
|
if strings.Contains(errorLower, "invalid") ||
|
||||||
|
strings.Contains(errorLower, "expired") ||
|
||||||
|
strings.Contains(errorLower, "already used") ||
|
||||||
|
strings.Contains(errorLower, "token") {
|
||||||
|
return 0, fmt.Errorf("token error: %s", errorBody)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0, fmt.Errorf("registration failed (status %d): %s", resp.StatusCode, errorBody)
|
||||||
|
}
|
||||||
|
|
||||||
|
var result RegisterResponse
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to decode response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m.runnerID = result.ID
|
||||||
|
m.apiKey = registrationToken
|
||||||
|
|
||||||
|
return result.ID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NextJobResponse represents the response from the next-job endpoint.
|
||||||
|
type NextJobResponse struct {
|
||||||
|
JobToken string `json:"job_token"`
|
||||||
|
JobPath string `json:"job_path"`
|
||||||
|
Task NextJobTaskInfo `json:"task"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// NextJobTaskInfo contains task information from the next-job response.
|
||||||
|
type NextJobTaskInfo struct {
|
||||||
|
TaskID int64 `json:"task_id"`
|
||||||
|
JobID int64 `json:"job_id"`
|
||||||
|
JobName string `json:"job_name"`
|
||||||
|
Frame int `json:"frame"` // frame start (inclusive)
|
||||||
|
FrameEnd int `json:"frame_end"` // frame end (inclusive); same as Frame for single-frame
|
||||||
|
TaskType string `json:"task_type"`
|
||||||
|
Metadata *types.BlendMetadata `json:"metadata,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PollNextJob polls the manager for the next available job.
|
||||||
|
// Returns nil, nil if no job is available.
|
||||||
|
func (m *ManagerClient) PollNextJob() (*NextJobResponse, error) {
|
||||||
|
if m.runnerID == 0 || m.apiKey == "" {
|
||||||
|
return nil, fmt.Errorf("runner not authenticated")
|
||||||
|
}
|
||||||
|
|
||||||
|
path := fmt.Sprintf("/api/runner/workers/%d/next-job", m.runnerID)
|
||||||
|
resp, err := m.Request("GET", path, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to poll for job: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode == http.StatusNoContent {
|
||||||
|
return nil, nil // No job available
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return nil, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
var job NextJobResponse
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&job); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to decode job response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &job, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadContext downloads the job context tar file.
|
||||||
|
func (m *ManagerClient) DownloadContext(contextPath, jobToken string) (io.ReadCloser, error) {
|
||||||
|
resp, err := m.RequestLongWithToken("GET", contextPath, jobToken, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to download context: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return nil, fmt.Errorf("context download failed with status %d: %s", resp.StatusCode, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp.Body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadFile uploads a file to the manager.
|
||||||
|
func (m *ManagerClient) UploadFile(uploadPath, jobToken, filePath string) error {
|
||||||
|
file, err := os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open file: %w", err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
// Create multipart form
|
||||||
|
body := &bytes.Buffer{}
|
||||||
|
writer := multipart.NewWriter(body)
|
||||||
|
part, err := writer.CreateFormFile("file", filepath.Base(filePath))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create form file: %w", err)
|
||||||
|
}
|
||||||
|
if _, err := io.Copy(part, file); err != nil {
|
||||||
|
return fmt.Errorf("failed to copy file to form: %w", err)
|
||||||
|
}
|
||||||
|
writer.Close()
|
||||||
|
|
||||||
|
fullURL := m.baseURL + uploadPath
|
||||||
|
req, err := http.NewRequest("POST", fullURL, body)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", "Bearer "+jobToken)
|
||||||
|
req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||||
|
|
||||||
|
resp, err := m.longClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to upload file: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK {
|
||||||
|
respBody, _ := io.ReadAll(resp.Body)
|
||||||
|
return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(respBody))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetJobMetadata retrieves job metadata from the manager.
|
||||||
|
func (m *ManagerClient) GetJobMetadata(jobID int64) (*types.BlendMetadata, error) {
|
||||||
|
path := fmt.Sprintf("/api/runner/jobs/%d/metadata?runner_id=%d", jobID, m.runnerID)
|
||||||
|
resp, err := m.Request("GET", path, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode == http.StatusNotFound {
|
||||||
|
return nil, nil // No metadata found
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return nil, fmt.Errorf("failed to get job metadata: %s", string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
var metadata types.BlendMetadata
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&metadata); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetJobStatus retrieves the current status of a job.
|
||||||
|
func (m *ManagerClient) GetJobStatus(jobID int64) (types.JobStatus, error) {
|
||||||
|
path := fmt.Sprintf("/api/runner/jobs/%d/status?runner_id=%d", jobID, m.runnerID)
|
||||||
|
resp, err := m.Request("GET", path, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return "", fmt.Errorf("failed to get job status: %s", string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
var job types.Job
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&job); err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return job.Status, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// JobFile represents a file associated with a job.
|
||||||
|
type JobFile struct {
|
||||||
|
ID int64 `json:"id"`
|
||||||
|
JobID int64 `json:"job_id"`
|
||||||
|
FileType string `json:"file_type"`
|
||||||
|
FilePath string `json:"file_path"`
|
||||||
|
FileName string `json:"file_name"`
|
||||||
|
FileSize int64 `json:"file_size"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetJobFiles retrieves the list of files for a job.
|
||||||
|
func (m *ManagerClient) GetJobFiles(jobID int64) ([]JobFile, error) {
|
||||||
|
path := fmt.Sprintf("/api/runner/jobs/%d/files?runner_id=%d", jobID, m.runnerID)
|
||||||
|
resp, err := m.Request("GET", path, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return nil, fmt.Errorf("failed to get job files: %s", string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
var files []JobFile
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&files); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return files, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadFrame downloads a frame file from the manager.
|
||||||
|
func (m *ManagerClient) DownloadFrame(jobID int64, fileName, destPath string) error {
|
||||||
|
encodedFileName := url.PathEscape(fileName)
|
||||||
|
path := fmt.Sprintf("/api/runner/files/%d/%s?runner_id=%d", jobID, encodedFileName, m.runnerID)
|
||||||
|
resp, err := m.RequestLong("GET", path, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return fmt.Errorf("download failed: %s", string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
file, err := os.Create(destPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
_, err = io.Copy(file, resp.Body)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// SubmitMetadata submits extracted metadata to the manager.
|
||||||
|
func (m *ManagerClient) SubmitMetadata(jobID int64, metadata types.BlendMetadata) error {
|
||||||
|
metadataJSON, err := json.Marshal(metadata)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to marshal metadata: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
path := fmt.Sprintf("/api/runner/jobs/%d/metadata?runner_id=%d", jobID, m.runnerID)
|
||||||
|
fullURL := m.baseURL + path
|
||||||
|
req, err := http.NewRequest("POST", fullURL, bytes.NewReader(metadataJSON))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", "Bearer "+m.apiKey)
|
||||||
|
|
||||||
|
resp, err := m.httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to submit metadata: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return fmt.Errorf("metadata submission failed: %s", string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadBlender downloads a Blender version from the manager.
|
||||||
|
func (m *ManagerClient) DownloadBlender(version string) (io.ReadCloser, error) {
|
||||||
|
path := fmt.Sprintf("/api/runner/blender/download?version=%s&runner_id=%d", version, m.runnerID)
|
||||||
|
resp, err := m.RequestLong("GET", path, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to download blender from manager: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return nil, fmt.Errorf("failed to download blender: status %d, body: %s", resp.StatusCode, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp.Body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// blenderVersionsResponse is the response from GET /api/blender/versions.
|
||||||
|
type blenderVersionsResponse struct {
|
||||||
|
Versions []struct {
|
||||||
|
Full string `json:"full"`
|
||||||
|
} `json:"versions"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetLatestBlenderVersion returns the latest Blender version string (e.g. "4.2.3") from the manager.
|
||||||
|
// Uses the flat versions list which is newest-first.
|
||||||
|
func (m *ManagerClient) GetLatestBlenderVersion() (string, error) {
|
||||||
|
resp, err := m.Request("GET", "/api/blender/versions", nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to fetch blender versions: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(resp.Body)
|
||||||
|
return "", fmt.Errorf("blender versions returned status %d: %s", resp.StatusCode, string(body))
|
||||||
|
}
|
||||||
|
var out blenderVersionsResponse
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||||
|
return "", fmt.Errorf("failed to decode blender versions: %w", err)
|
||||||
|
}
|
||||||
|
if len(out.Versions) == 0 {
|
||||||
|
return "", fmt.Errorf("no blender versions available")
|
||||||
|
}
|
||||||
|
return out.Versions[0].Full, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewManagerClient_TrimsTrailingSlash(t *testing.T) {
|
||||||
|
c := NewManagerClient("http://example.com/")
|
||||||
|
if c.GetBaseURL() != "http://example.com" {
|
||||||
|
t.Fatalf("unexpected base url: %q", c.GetBaseURL())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDoRequest_SetsAuthorizationHeader(t *testing.T) {
|
||||||
|
var authHeader string
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
authHeader = r.Header.Get("Authorization")
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]bool{"ok": true})
|
||||||
|
}))
|
||||||
|
defer ts.Close()
|
||||||
|
|
||||||
|
c := NewManagerClient(ts.URL)
|
||||||
|
c.SetCredentials(1, "abc123")
|
||||||
|
|
||||||
|
resp, err := c.Request(http.MethodGet, "/x", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Request failed: %v", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if authHeader != "Bearer abc123" {
|
||||||
|
t.Fatalf("unexpected Authorization header: %q", authHeader)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequest_RequiresAuth(t *testing.T) {
|
||||||
|
c := NewManagerClient("http://example.com")
|
||||||
|
if _, err := c.Request(http.MethodGet, "/x", nil); err == nil {
|
||||||
|
t.Fatal("expected auth error when api key is missing")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,162 @@
|
|||||||
|
// Package blender handles Blender binary management and execution.
|
||||||
|
package blender
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"jiggablend/internal/runner/api"
|
||||||
|
"jiggablend/internal/runner/workspace"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Manager handles Blender binary downloads and management.
|
||||||
|
type Manager struct {
|
||||||
|
manager *api.ManagerClient
|
||||||
|
workspaceDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewManager creates a new Blender manager.
|
||||||
|
func NewManager(managerClient *api.ManagerClient, workspaceDir string) *Manager {
|
||||||
|
return &Manager{
|
||||||
|
manager: managerClient,
|
||||||
|
workspaceDir: workspaceDir,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBinaryPath returns the path to the Blender binary for a specific version.
|
||||||
|
// Downloads from manager and extracts if not already present.
|
||||||
|
func (m *Manager) GetBinaryPath(version string) (string, error) {
|
||||||
|
blenderDir := filepath.Join(m.workspaceDir, "blender-versions")
|
||||||
|
if err := os.MkdirAll(blenderDir, 0755); err != nil {
|
||||||
|
return "", fmt.Errorf("failed to create blender directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if already installed - look for version folder first
|
||||||
|
versionDir := filepath.Join(blenderDir, version)
|
||||||
|
binaryPath := filepath.Join(versionDir, "blender")
|
||||||
|
|
||||||
|
// Check if version folder exists and contains the binary
|
||||||
|
if versionInfo, err := os.Stat(versionDir); err == nil && versionInfo.IsDir() {
|
||||||
|
// Version folder exists, check if binary is present
|
||||||
|
if binaryInfo, err := os.Stat(binaryPath); err == nil {
|
||||||
|
// Verify it's actually a file (not a directory)
|
||||||
|
if !binaryInfo.IsDir() {
|
||||||
|
absBinaryPath, err := ResolveBinaryPath(binaryPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
log.Printf("Found existing Blender %s installation at %s", version, absBinaryPath)
|
||||||
|
return absBinaryPath, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Version folder exists but binary is missing - might be incomplete installation
|
||||||
|
log.Printf("Version folder %s exists but binary not found, will re-download", versionDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download from manager
|
||||||
|
log.Printf("Downloading Blender %s from manager", version)
|
||||||
|
|
||||||
|
reader, err := m.manager.DownloadBlender(version)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
// Manager serves pre-decompressed .tar files - extract directly
|
||||||
|
log.Printf("Extracting Blender %s...", version)
|
||||||
|
if err := workspace.ExtractTarStripPrefix(reader, versionDir); err != nil {
|
||||||
|
return "", fmt.Errorf("failed to extract blender: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify binary exists
|
||||||
|
if _, err := os.Stat(binaryPath); err != nil {
|
||||||
|
return "", fmt.Errorf("blender binary not found after extraction")
|
||||||
|
}
|
||||||
|
|
||||||
|
absBinaryPath, err := ResolveBinaryPath(binaryPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Blender %s installed at %s", version, absBinaryPath)
|
||||||
|
return absBinaryPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBinaryForJob returns the Blender binary path for a job.
|
||||||
|
// Uses the version from metadata or falls back to system blender.
|
||||||
|
func (m *Manager) GetBinaryForJob(version string) (string, error) {
|
||||||
|
if version == "" {
|
||||||
|
return ResolveBinaryPath("blender")
|
||||||
|
}
|
||||||
|
|
||||||
|
return m.GetBinaryPath(version)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveBinaryPath resolves a Blender executable to an absolute path.
|
||||||
|
func ResolveBinaryPath(blenderBinary string) (string, error) {
|
||||||
|
if blenderBinary == "" {
|
||||||
|
return "", fmt.Errorf("blender binary path is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(blenderBinary, string(filepath.Separator)) {
|
||||||
|
absPath, err := filepath.Abs(blenderBinary)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to resolve blender binary path %q: %w", blenderBinary, err)
|
||||||
|
}
|
||||||
|
return absPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
resolvedPath, err := exec.LookPath(blenderBinary)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to locate blender binary %q in PATH: %w", blenderBinary, err)
|
||||||
|
}
|
||||||
|
absPath, err := filepath.Abs(resolvedPath)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to resolve blender binary path %q: %w", resolvedPath, err)
|
||||||
|
}
|
||||||
|
return absPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TarballEnv returns a copy of baseEnv with LD_LIBRARY_PATH set so that a
|
||||||
|
// tarball Blender installation can find its bundled libs (e.g. lib/python3.x).
|
||||||
|
// If blenderBinary is the system "blender" or has no path component, baseEnv is
|
||||||
|
// returned unchanged.
|
||||||
|
func TarballEnv(blenderBinary string, baseEnv []string) []string {
|
||||||
|
if blenderBinary == "" || blenderBinary == "blender" {
|
||||||
|
return baseEnv
|
||||||
|
}
|
||||||
|
if !strings.Contains(blenderBinary, string(os.PathSeparator)) {
|
||||||
|
return baseEnv
|
||||||
|
}
|
||||||
|
blenderDir := filepath.Dir(blenderBinary)
|
||||||
|
libDir := filepath.Join(blenderDir, "lib")
|
||||||
|
ldLib := libDir
|
||||||
|
for _, e := range baseEnv {
|
||||||
|
if strings.HasPrefix(e, "LD_LIBRARY_PATH=") {
|
||||||
|
existing := strings.TrimPrefix(e, "LD_LIBRARY_PATH=")
|
||||||
|
if existing != "" {
|
||||||
|
ldLib = libDir + ":" + existing
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out := make([]string, 0, len(baseEnv)+1)
|
||||||
|
done := false
|
||||||
|
for _, e := range baseEnv {
|
||||||
|
if strings.HasPrefix(e, "LD_LIBRARY_PATH=") {
|
||||||
|
out = append(out, "LD_LIBRARY_PATH="+ldLib)
|
||||||
|
done = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, e)
|
||||||
|
}
|
||||||
|
if !done {
|
||||||
|
out = append(out, "LD_LIBRARY_PATH="+ldLib)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package blender
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolveBinaryPath_AbsoluteLikePath(t *testing.T) {
|
||||||
|
got, err := ResolveBinaryPath("./blender")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolveBinaryPath failed: %v", err)
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(got) {
|
||||||
|
t.Fatalf("expected absolute path, got %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveBinaryPath_Empty(t *testing.T) {
|
||||||
|
if _, err := ResolveBinaryPath(""); err == nil {
|
||||||
|
t.Fatal("expected error for empty blender binary")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTarballEnv_SetsAndExtendsLDLibraryPath(t *testing.T) {
|
||||||
|
bin := filepath.Join(string(os.PathSeparator), "tmp", "blender", "blender")
|
||||||
|
got := TarballEnv(bin, []string{"A=B", "LD_LIBRARY_PATH=/old"})
|
||||||
|
joined := strings.Join(got, "\n")
|
||||||
|
if !strings.Contains(joined, "LD_LIBRARY_PATH=/tmp/blender/lib:/old") {
|
||||||
|
t.Fatalf("expected LD_LIBRARY_PATH to include blender lib, got %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
// Package blender: host GPU backend detection for AMD/NVIDIA/Intel.
|
||||||
|
package blender
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DetectGPUBackends detects whether AMD, NVIDIA, and/or Intel GPUs are available
|
||||||
|
// using host-level hardware probing only.
|
||||||
|
func DetectGPUBackends() (hasAMD, hasNVIDIA, hasIntel bool, ok bool) {
|
||||||
|
return detectGPUBackendsFromHost()
|
||||||
|
}
|
||||||
|
|
||||||
|
func detectGPUBackendsFromHost() (hasAMD, hasNVIDIA, hasIntel bool, ok bool) {
|
||||||
|
if amd, nvidia, intel, found := detectGPUBackendsFromDRM(); found {
|
||||||
|
return amd, nvidia, intel, true
|
||||||
|
}
|
||||||
|
if amd, nvidia, intel, found := detectGPUBackendsFromLSPCI(); found {
|
||||||
|
return amd, nvidia, intel, true
|
||||||
|
}
|
||||||
|
return false, false, false, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func detectGPUBackendsFromDRM() (hasAMD, hasNVIDIA, hasIntel bool, ok bool) {
|
||||||
|
entries, err := os.ReadDir("/sys/class/drm")
|
||||||
|
if err != nil {
|
||||||
|
return false, false, false, false
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, entry := range entries {
|
||||||
|
name := entry.Name()
|
||||||
|
if !isDRMCardNode(name) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
vendorPath := filepath.Join("/sys/class/drm", name, "device", "vendor")
|
||||||
|
vendorRaw, err := os.ReadFile(vendorPath)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
vendor := strings.TrimSpace(strings.ToLower(string(vendorRaw)))
|
||||||
|
switch vendor {
|
||||||
|
case "0x1002":
|
||||||
|
hasAMD = true
|
||||||
|
ok = true
|
||||||
|
case "0x10de":
|
||||||
|
hasNVIDIA = true
|
||||||
|
ok = true
|
||||||
|
case "0x8086":
|
||||||
|
hasIntel = true
|
||||||
|
ok = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return hasAMD, hasNVIDIA, hasIntel, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func isDRMCardNode(name string) bool {
|
||||||
|
if !strings.HasPrefix(name, "card") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if strings.Contains(name, "-") {
|
||||||
|
// Connector entries like card0-DP-1 are not GPU device nodes.
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if len(name) <= len("card") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, err := strconv.Atoi(strings.TrimPrefix(name, "card"))
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func detectGPUBackendsFromLSPCI() (hasAMD, hasNVIDIA, hasIntel bool, ok bool) {
|
||||||
|
if _, err := exec.LookPath("lspci"); err != nil {
|
||||||
|
return false, false, false, false
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err := exec.Command("lspci").CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return false, false, false, false
|
||||||
|
}
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(strings.NewReader(string(out)))
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := strings.ToLower(strings.TrimSpace(scanner.Text()))
|
||||||
|
if !isGPUControllerLine(line) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(line, "nvidia") {
|
||||||
|
hasNVIDIA = true
|
||||||
|
ok = true
|
||||||
|
}
|
||||||
|
if strings.Contains(line, "amd") || strings.Contains(line, "ati") || strings.Contains(line, "radeon") {
|
||||||
|
hasAMD = true
|
||||||
|
ok = true
|
||||||
|
}
|
||||||
|
if strings.Contains(line, "intel") {
|
||||||
|
hasIntel = true
|
||||||
|
ok = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return hasAMD, hasNVIDIA, hasIntel, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
func isGPUControllerLine(line string) bool {
|
||||||
|
return strings.Contains(line, "vga compatible controller") ||
|
||||||
|
strings.Contains(line, "3d controller") ||
|
||||||
|
strings.Contains(line, "display controller")
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseRocmAgentArch(output string) (arch string, ok bool) {
|
||||||
|
scanner := bufio.NewScanner(strings.NewReader(output))
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := strings.TrimSpace(scanner.Text())
|
||||||
|
if !strings.HasPrefix(line, "Name:") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := strings.TrimSpace(strings.TrimPrefix(line, "Name:"))
|
||||||
|
if strings.HasPrefix(name, "gfx") {
|
||||||
|
return name, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
package blender
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestParseRocmAgentArch(t *testing.T) {
|
||||||
|
input := `ROCk module is loaded
|
||||||
|
Agent 1
|
||||||
|
Name: gfx1151
|
||||||
|
Marketing Name: AMD Radeon Graphics
|
||||||
|
`
|
||||||
|
arch, ok := parseRocmAgentArch(input)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("expected arch to be parsed")
|
||||||
|
}
|
||||||
|
if arch != "gfx1151" {
|
||||||
|
t.Fatalf("arch = %q, want gfx1151", arch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
package blender
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestIsDRMCardNode(t *testing.T) {
|
||||||
|
tests := map[string]bool{
|
||||||
|
"card0": true,
|
||||||
|
"card12": true,
|
||||||
|
"card": false,
|
||||||
|
"card0-DP-1": false,
|
||||||
|
"renderD128": false,
|
||||||
|
"foo": false,
|
||||||
|
}
|
||||||
|
for in, want := range tests {
|
||||||
|
if got := isDRMCardNode(in); got != want {
|
||||||
|
t.Fatalf("isDRMCardNode(%q) = %v, want %v", in, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsGPUControllerLine(t *testing.T) {
|
||||||
|
if !isGPUControllerLine("vga compatible controller: nvidia corp") {
|
||||||
|
t.Fatal("expected VGA controller line to match")
|
||||||
|
}
|
||||||
|
if !isGPUControllerLine("3d controller: amd") {
|
||||||
|
t.Fatal("expected 3d controller line to match")
|
||||||
|
}
|
||||||
|
if isGPUControllerLine("audio device: something") {
|
||||||
|
t.Fatal("audio line should not match")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
package blender
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"jiggablend/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FilterLog checks if a Blender log line should be filtered or downgraded.
|
||||||
|
// Returns (shouldFilter, logLevel) - if shouldFilter is true, the log should be skipped.
|
||||||
|
func FilterLog(line string) (shouldFilter bool, logLevel types.LogLevel) {
|
||||||
|
trimmed := strings.TrimSpace(line)
|
||||||
|
|
||||||
|
// Filter out empty lines
|
||||||
|
if trimmed == "" {
|
||||||
|
return true, types.LogLevelInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter out separator lines
|
||||||
|
if trimmed == "--------------------------------------------------------------------" ||
|
||||||
|
(strings.HasPrefix(trimmed, "-----") && strings.Contains(trimmed, "----")) {
|
||||||
|
return true, types.LogLevelInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter out trace headers
|
||||||
|
upperLine := strings.ToUpper(trimmed)
|
||||||
|
upperOriginal := strings.ToUpper(line)
|
||||||
|
|
||||||
|
if trimmed == "Trace:" ||
|
||||||
|
trimmed == "Depth Type Name" ||
|
||||||
|
trimmed == "----- ---- ----" ||
|
||||||
|
line == "Depth Type Name" ||
|
||||||
|
line == "----- ---- ----" ||
|
||||||
|
(strings.Contains(upperLine, "DEPTH") && strings.Contains(upperLine, "TYPE") && strings.Contains(upperLine, "NAME")) ||
|
||||||
|
(strings.Contains(upperOriginal, "DEPTH") && strings.Contains(upperOriginal, "TYPE") && strings.Contains(upperOriginal, "NAME")) ||
|
||||||
|
strings.Contains(line, "Depth Type Name") ||
|
||||||
|
strings.Contains(line, "----- ---- ----") ||
|
||||||
|
strings.HasPrefix(trimmed, "-----") ||
|
||||||
|
regexp.MustCompile(`^[-]+\s+[-]+\s+[-]+$`).MatchString(trimmed) {
|
||||||
|
return true, types.LogLevelInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
// Completely filter out dependency graph messages (they're just noise)
|
||||||
|
dependencyGraphPatterns := []string{
|
||||||
|
"Failed to add relation",
|
||||||
|
"Could not find op_from",
|
||||||
|
"OperationKey",
|
||||||
|
"find_node_operation: Failed for",
|
||||||
|
"BONE_DONE",
|
||||||
|
"component name:",
|
||||||
|
"operation code:",
|
||||||
|
"rope_ctrl_rot_",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, pattern := range dependencyGraphPatterns {
|
||||||
|
if strings.Contains(line, pattern) {
|
||||||
|
return true, types.LogLevelInfo
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter out animation system warnings (invalid drivers are common and harmless)
|
||||||
|
animationSystemPatterns := []string{
|
||||||
|
"BKE_animsys_eval_driver: invalid driver",
|
||||||
|
"bke.anim_sys",
|
||||||
|
"rotation_quaternion[",
|
||||||
|
"constraints[",
|
||||||
|
".influence[0]",
|
||||||
|
"pose.bones[",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, pattern := range animationSystemPatterns {
|
||||||
|
if strings.Contains(line, pattern) {
|
||||||
|
return true, types.LogLevelInfo
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter out modifier warnings (common when vertices change)
|
||||||
|
modifierPatterns := []string{
|
||||||
|
"BKE_modifier_set_error",
|
||||||
|
"bke.modifier",
|
||||||
|
"Vertices changed from",
|
||||||
|
"Modifier:",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, pattern := range modifierPatterns {
|
||||||
|
if strings.Contains(line, pattern) {
|
||||||
|
return true, types.LogLevelInfo
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter out lines that are just numbers or trace depth indicators
|
||||||
|
// Pattern: number, word, word (e.g., "1 Object timer_box_franck")
|
||||||
|
if matched, _ := regexp.MatchString(`^\d+\s+\w+\s+\w+`, trimmed); matched {
|
||||||
|
return true, types.LogLevelInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
return false, types.LogLevelInfo
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package blender
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"jiggablend/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFilterLog_FiltersNoise(t *testing.T) {
|
||||||
|
cases := []string{
|
||||||
|
"",
|
||||||
|
"--------------------------------------------------------------------",
|
||||||
|
"Failed to add relation foo",
|
||||||
|
"BKE_modifier_set_error",
|
||||||
|
"Depth Type Name",
|
||||||
|
}
|
||||||
|
for _, in := range cases {
|
||||||
|
filtered, level := FilterLog(in)
|
||||||
|
if !filtered {
|
||||||
|
t.Fatalf("expected filtered for %q", in)
|
||||||
|
}
|
||||||
|
if level != types.LogLevelInfo {
|
||||||
|
t.Fatalf("unexpected level for %q: %s", in, level)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilterLog_KeepsNormalLine(t *testing.T) {
|
||||||
|
filtered, _ := FilterLog("Rendering done.")
|
||||||
|
if filtered {
|
||||||
|
t.Fatal("normal line should not be filtered")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
package blender
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"jiggablend/pkg/blendfile"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ParseVersionFromFile parses the Blender version that a .blend file was saved with.
|
||||||
|
// Returns major and minor version numbers.
|
||||||
|
// Delegates to the shared pkg/blendfile implementation.
|
||||||
|
func ParseVersionFromFile(blendPath string) (major, minor int, err error) {
|
||||||
|
return blendfile.ParseVersionFromFile(blendPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// VersionString returns a formatted version string like "4.2".
|
||||||
|
func VersionString(major, minor int) string {
|
||||||
|
return fmt.Sprintf("%d.%d", major, minor)
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
package blender
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestVersionString(t *testing.T) {
|
||||||
|
if got := VersionString(4, 2); got != "4.2" {
|
||||||
|
t.Fatalf("VersionString() = %q, want %q", got, "4.2")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,69 @@
|
|||||||
|
// Package encoding handles video encoding with software encoders.
|
||||||
|
package encoding
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os/exec"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Encoder represents a video encoder.
|
||||||
|
type Encoder interface {
|
||||||
|
Name() string
|
||||||
|
Codec() string
|
||||||
|
Available() bool
|
||||||
|
BuildCommand(config *EncodeConfig) *exec.Cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeConfig holds configuration for video encoding.
|
||||||
|
type EncodeConfig struct {
|
||||||
|
InputPattern string // Input file pattern (e.g., "frame_%04d.exr")
|
||||||
|
OutputPath string // Output file path
|
||||||
|
StartFrame int // Starting frame number
|
||||||
|
FrameRate float64 // Frame rate
|
||||||
|
WorkDir string // Working directory
|
||||||
|
UseAlpha bool // Whether to preserve alpha channel
|
||||||
|
TwoPass bool // Whether to use 2-pass encoding
|
||||||
|
}
|
||||||
|
|
||||||
|
// Selector selects the software encoder.
|
||||||
|
type Selector struct {
|
||||||
|
h264Encoders []Encoder
|
||||||
|
av1Encoders []Encoder
|
||||||
|
vp9Encoders []Encoder
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSelector creates a new encoder selector with software encoders.
|
||||||
|
func NewSelector() *Selector {
|
||||||
|
s := &Selector{}
|
||||||
|
s.detectEncoders()
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Selector) detectEncoders() {
|
||||||
|
// Use software encoding only - reliable and avoids hardware-specific colorspace issues
|
||||||
|
s.h264Encoders = []Encoder{
|
||||||
|
&SoftwareEncoder{codec: "libx264"},
|
||||||
|
}
|
||||||
|
|
||||||
|
s.av1Encoders = []Encoder{
|
||||||
|
&SoftwareEncoder{codec: "libaom-av1"},
|
||||||
|
}
|
||||||
|
|
||||||
|
s.vp9Encoders = []Encoder{
|
||||||
|
&SoftwareEncoder{codec: "libvpx-vp9"},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SelectH264 returns the software H.264 encoder.
|
||||||
|
func (s *Selector) SelectH264() Encoder {
|
||||||
|
return &SoftwareEncoder{codec: "libx264"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SelectAV1 returns the software AV1 encoder.
|
||||||
|
func (s *Selector) SelectAV1() Encoder {
|
||||||
|
return &SoftwareEncoder{codec: "libaom-av1"}
|
||||||
|
}
|
||||||
|
|
||||||
|
// SelectVP9 returns the software VP9 encoder.
|
||||||
|
func (s *Selector) SelectVP9() Encoder {
|
||||||
|
return &SoftwareEncoder{codec: "libvpx-vp9"}
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
package encoding
|
||||||
|
|
||||||
|
// Pipeline: Blender outputs only EXR (linear). Encode is EXR only: linear -> sRGB -> HLG (video), 10-bit, full range.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os/exec"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// CRFH264 is the Constant Rate Factor for H.264 encoding (lower = higher quality, range 0-51)
|
||||||
|
CRFH264 = 15
|
||||||
|
// CRFAV1 is the Constant Rate Factor for AV1 encoding (lower = higher quality, range 0-63)
|
||||||
|
CRFAV1 = 30
|
||||||
|
// CRFVP9 is the Constant Rate Factor for VP9 encoding (lower = higher quality, range 0-63)
|
||||||
|
CRFVP9 = 30
|
||||||
|
)
|
||||||
|
|
||||||
|
// tonemapFilter returns the production filter for EXR input (linear → sRGB → HLG, bt709).
|
||||||
|
// This is the single source of truth used by BuildCommand and BuildPass1Command.
|
||||||
|
// zscale numeric values: primaries 1=bt709, matrix 1=bt709, transfer 8=linear / 13=sRGB / 18=HLG.
|
||||||
|
func tonemapFilter(useAlpha bool) string {
|
||||||
|
filter := "format=gbrpf32le,zscale=transferin=8:transfer=13:primariesin=1:primaries=1:matrixin=0:matrix=1:rangein=full:range=full,zscale=transferin=13:transfer=18:primariesin=1:primaries=1:matrixin=1:matrix=1:rangein=full:range=full"
|
||||||
|
if useAlpha {
|
||||||
|
return filter + ",format=yuva420p10le"
|
||||||
|
}
|
||||||
|
return filter + ",format=yuv420p10le"
|
||||||
|
}
|
||||||
|
|
||||||
|
// SoftwareEncoder implements software encoding (libx264, libaom-av1, libvpx-vp9).
|
||||||
|
type SoftwareEncoder struct {
|
||||||
|
codec string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *SoftwareEncoder) Name() string { return "software" }
|
||||||
|
func (e *SoftwareEncoder) Codec() string { return e.codec }
|
||||||
|
|
||||||
|
func (e *SoftwareEncoder) Available() bool {
|
||||||
|
return true // Software encoding is always available
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *SoftwareEncoder) buildBaseArgs(config *EncodeConfig) []string {
|
||||||
|
pixFmt := "yuv420p10le"
|
||||||
|
if config.UseAlpha {
|
||||||
|
pixFmt = "yuva420p10le"
|
||||||
|
}
|
||||||
|
colorPrimaries, colorTrc, colorspace, colorRange := "bt709", "arib-std-b67", "bt709", "pc"
|
||||||
|
|
||||||
|
var codecArgs []string
|
||||||
|
switch e.codec {
|
||||||
|
case "libaom-av1":
|
||||||
|
codecArgs = []string{"-crf", strconv.Itoa(CRFAV1), "-b:v", "0", "-tiles", "2x2", "-g", "240"}
|
||||||
|
case "libvpx-vp9":
|
||||||
|
codecArgs = []string{"-crf", strconv.Itoa(CRFVP9), "-b:v", "0", "-row-mt", "1", "-g", "240"}
|
||||||
|
default:
|
||||||
|
codecArgs = []string{"-preset", "veryslow", "-crf", strconv.Itoa(CRFH264), "-profile:v", "high10", "-level", "5.2", "-tune", "film", "-keyint_min", "24", "-g", "240", "-bf", "2", "-refs", "4"}
|
||||||
|
}
|
||||||
|
|
||||||
|
args := []string{"-y", "-f", "image2", "-start_number", fmt.Sprintf("%d", config.StartFrame), "-framerate", fmt.Sprintf("%.2f", config.FrameRate),
|
||||||
|
"-color_trc", "linear", "-color_primaries", "bt709"}
|
||||||
|
args = append(args, "-i", config.InputPattern, "-c:v", e.codec, "-pix_fmt", pixFmt, "-r", fmt.Sprintf("%.2f", config.FrameRate), "-color_primaries", colorPrimaries, "-color_trc", colorTrc, "-colorspace", colorspace, "-color_range", colorRange)
|
||||||
|
|
||||||
|
// FFmpeg 6+ treats yuva420p10le as experimental for libvpx-vp9 / libaom-av1
|
||||||
|
if config.UseAlpha && (e.codec == "libvpx-vp9" || e.codec == "libaom-av1") {
|
||||||
|
args = append(args, "-strict", "experimental")
|
||||||
|
}
|
||||||
|
|
||||||
|
args = append(args, "-vf", tonemapFilter(config.UseAlpha))
|
||||||
|
args = append(args, codecArgs...)
|
||||||
|
return args
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *SoftwareEncoder) BuildCommand(config *EncodeConfig) *exec.Cmd {
|
||||||
|
args := e.buildBaseArgs(config)
|
||||||
|
|
||||||
|
if config.TwoPass {
|
||||||
|
// For 2-pass, this builds pass 2 command
|
||||||
|
args = append(args, "-pass", "2")
|
||||||
|
}
|
||||||
|
|
||||||
|
args = append(args, config.OutputPath)
|
||||||
|
|
||||||
|
if config.TwoPass {
|
||||||
|
log.Printf("Build Software Pass 2 command: ffmpeg %s", strings.Join(args, " "))
|
||||||
|
} else {
|
||||||
|
log.Printf("Build Software command: ffmpeg %s", strings.Join(args, " "))
|
||||||
|
}
|
||||||
|
cmd := exec.Command("ffmpeg", args...)
|
||||||
|
cmd.Dir = config.WorkDir
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildPass1Command builds the first pass command for 2-pass encoding.
|
||||||
|
func (e *SoftwareEncoder) BuildPass1Command(config *EncodeConfig) *exec.Cmd {
|
||||||
|
args := e.buildBaseArgs(config)
|
||||||
|
args = append(args, "-pass", "1", "-f", "null", "/dev/null")
|
||||||
|
|
||||||
|
log.Printf("Build Software Pass 1 command: ffmpeg %s", strings.Join(args, " "))
|
||||||
|
cmd := exec.Command("ffmpeg", args...)
|
||||||
|
cmd.Dir = config.WorkDir
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
@@ -0,0 +1,845 @@
|
|||||||
|
package encoding
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestSoftwareEncoder_BuildCommand_H264_EXR(t *testing.T) {
|
||||||
|
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: "frame_%04d.exr",
|
||||||
|
OutputPath: "output.mp4",
|
||||||
|
StartFrame: 1,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: "/tmp",
|
||||||
|
UseAlpha: false,
|
||||||
|
TwoPass: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
if cmd == nil {
|
||||||
|
t.Fatal("BuildCommand returned nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(cmd.Path, "ffmpeg") {
|
||||||
|
t.Errorf("Expected command path to contain 'ffmpeg', got '%s'", cmd.Path)
|
||||||
|
}
|
||||||
|
|
||||||
|
if cmd.Dir != "/tmp" {
|
||||||
|
t.Errorf("Expected work dir '/tmp', got '%s'", cmd.Dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
args := cmd.Args[1:] // Skip "ffmpeg"
|
||||||
|
argsStr := strings.Join(args, " ")
|
||||||
|
|
||||||
|
// EXR always uses HDR path: 10-bit, HLG, full range
|
||||||
|
checks := []struct {
|
||||||
|
name string
|
||||||
|
expected string
|
||||||
|
}{
|
||||||
|
{"-y flag", "-y"},
|
||||||
|
{"image2 format", "-f image2"},
|
||||||
|
{"start number", "-start_number 1"},
|
||||||
|
{"framerate", "-framerate 24.00"},
|
||||||
|
{"input color tag", "-color_trc linear"},
|
||||||
|
{"input pattern", "-i frame_%04d.exr"},
|
||||||
|
{"codec", "-c:v libx264"},
|
||||||
|
{"pixel format", "-pix_fmt yuv420p10le"},
|
||||||
|
{"frame rate", "-r 24.00"},
|
||||||
|
{"color primaries", "-color_primaries bt709"},
|
||||||
|
{"color trc", "-color_trc arib-std-b67"},
|
||||||
|
{"colorspace", "-colorspace bt709"},
|
||||||
|
{"color range", "-color_range pc"},
|
||||||
|
{"video filter", "-vf"},
|
||||||
|
{"preset", "-preset veryslow"},
|
||||||
|
{"crf", "-crf 15"},
|
||||||
|
{"profile", "-profile:v high10"},
|
||||||
|
{"pass 2", "-pass 2"},
|
||||||
|
{"output path", "output.mp4"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, check := range checks {
|
||||||
|
if !strings.Contains(argsStr, check.expected) {
|
||||||
|
t.Errorf("Missing expected argument: %s", check.expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// EXR: linear -> sRGB -> HLG filter
|
||||||
|
if !strings.Contains(argsStr, "format=gbrpf32le") {
|
||||||
|
t.Error("Expected format conversion filter for EXR source, but not found")
|
||||||
|
}
|
||||||
|
if !strings.Contains(argsStr, "zscale=transferin=8:transfer=13") {
|
||||||
|
t.Error("Expected linear to sRGB conversion for EXR source, but not found")
|
||||||
|
}
|
||||||
|
if !strings.Contains(argsStr, "transfer=18") {
|
||||||
|
t.Error("Expected sRGB to HLG conversion for EXR HDR, but not found")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSoftwareEncoder_BuildCommand_AV1_WithAlpha(t *testing.T) {
|
||||||
|
encoder := &SoftwareEncoder{codec: "libaom-av1"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: "frame_%04d.exr",
|
||||||
|
OutputPath: "output.mp4",
|
||||||
|
StartFrame: 100,
|
||||||
|
FrameRate: 30.0,
|
||||||
|
WorkDir: "/tmp",
|
||||||
|
UseAlpha: true,
|
||||||
|
TwoPass: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
args := cmd.Args[1:]
|
||||||
|
argsStr := strings.Join(args, " ")
|
||||||
|
|
||||||
|
// EXR with alpha: 10-bit HDR path
|
||||||
|
if !strings.Contains(argsStr, "-pix_fmt yuva420p10le") {
|
||||||
|
t.Error("Expected yuva420p10le pixel format for EXR alpha, but not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check AV1-specific arguments
|
||||||
|
av1Checks := []string{
|
||||||
|
"-c:v libaom-av1",
|
||||||
|
"-crf 30",
|
||||||
|
"-b:v 0",
|
||||||
|
"-tiles 2x2",
|
||||||
|
"-g 240",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, check := range av1Checks {
|
||||||
|
if !strings.Contains(argsStr, check) {
|
||||||
|
t.Errorf("Missing AV1 argument: %s", check)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check tonemap filter includes alpha format (10-bit for EXR)
|
||||||
|
if !strings.Contains(argsStr, "format=yuva420p10le") {
|
||||||
|
t.Error("Expected tonemap filter to output yuva420p10le for EXR alpha, but not found")
|
||||||
|
}
|
||||||
|
// Alpha VP9/AV1 need -strict experimental on modern FFmpeg
|
||||||
|
if !strings.Contains(argsStr, "-strict experimental") {
|
||||||
|
t.Error("Expected -strict experimental for alpha encode")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSoftwareEncoder_BuildCommand_VP9(t *testing.T) {
|
||||||
|
encoder := &SoftwareEncoder{codec: "libvpx-vp9"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: "frame_%04d.exr",
|
||||||
|
OutputPath: "output.webm",
|
||||||
|
StartFrame: 1,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: "/tmp",
|
||||||
|
UseAlpha: true,
|
||||||
|
TwoPass: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
args := cmd.Args[1:]
|
||||||
|
argsStr := strings.Join(args, " ")
|
||||||
|
|
||||||
|
// Check VP9-specific arguments
|
||||||
|
vp9Checks := []string{
|
||||||
|
"-c:v libvpx-vp9",
|
||||||
|
"-crf 30",
|
||||||
|
"-b:v 0",
|
||||||
|
"-row-mt 1",
|
||||||
|
"-g 240",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, check := range vp9Checks {
|
||||||
|
if !strings.Contains(argsStr, check) {
|
||||||
|
t.Errorf("Missing VP9 argument: %s", check)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSoftwareEncoder_BuildPass1Command(t *testing.T) {
|
||||||
|
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: "frame_%04d.exr",
|
||||||
|
OutputPath: "output.mp4",
|
||||||
|
StartFrame: 1,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: "/tmp",
|
||||||
|
UseAlpha: false,
|
||||||
|
TwoPass: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := encoder.BuildPass1Command(config)
|
||||||
|
args := cmd.Args[1:]
|
||||||
|
argsStr := strings.Join(args, " ")
|
||||||
|
|
||||||
|
// Pass 1 should have -pass 1 and output to null
|
||||||
|
if !strings.Contains(argsStr, "-pass 1") {
|
||||||
|
t.Error("Pass 1 command should include '-pass 1'")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(argsStr, "-f null") {
|
||||||
|
t.Error("Pass 1 command should include '-f null'")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(argsStr, "/dev/null") {
|
||||||
|
t.Error("Pass 1 command should output to /dev/null")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should NOT have output path
|
||||||
|
if strings.Contains(argsStr, "output.mp4") {
|
||||||
|
t.Error("Pass 1 command should not include output path")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSoftwareEncoder_BuildPass1Command_AV1(t *testing.T) {
|
||||||
|
encoder := &SoftwareEncoder{codec: "libaom-av1"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: "frame_%04d.exr",
|
||||||
|
OutputPath: "output.mp4",
|
||||||
|
StartFrame: 1,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: "/tmp",
|
||||||
|
UseAlpha: false,
|
||||||
|
TwoPass: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := encoder.BuildPass1Command(config)
|
||||||
|
args := cmd.Args[1:]
|
||||||
|
argsStr := strings.Join(args, " ")
|
||||||
|
|
||||||
|
// Pass 1 should have -pass 1 and output to null
|
||||||
|
if !strings.Contains(argsStr, "-pass 1") {
|
||||||
|
t.Error("Pass 1 command should include '-pass 1'")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(argsStr, "-f null") {
|
||||||
|
t.Error("Pass 1 command should include '-f null'")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(argsStr, "/dev/null") {
|
||||||
|
t.Error("Pass 1 command should output to /dev/null")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check AV1-specific arguments in pass 1
|
||||||
|
av1Checks := []string{
|
||||||
|
"-c:v libaom-av1",
|
||||||
|
"-crf 30",
|
||||||
|
"-b:v 0",
|
||||||
|
"-tiles 2x2",
|
||||||
|
"-g 240",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, check := range av1Checks {
|
||||||
|
if !strings.Contains(argsStr, check) {
|
||||||
|
t.Errorf("Missing AV1 argument in pass 1: %s", check)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSoftwareEncoder_BuildPass1Command_VP9(t *testing.T) {
|
||||||
|
encoder := &SoftwareEncoder{codec: "libvpx-vp9"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: "frame_%04d.exr",
|
||||||
|
OutputPath: "output.webm",
|
||||||
|
StartFrame: 1,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: "/tmp",
|
||||||
|
UseAlpha: false,
|
||||||
|
TwoPass: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := encoder.BuildPass1Command(config)
|
||||||
|
args := cmd.Args[1:]
|
||||||
|
argsStr := strings.Join(args, " ")
|
||||||
|
|
||||||
|
// Pass 1 should have -pass 1 and output to null
|
||||||
|
if !strings.Contains(argsStr, "-pass 1") {
|
||||||
|
t.Error("Pass 1 command should include '-pass 1'")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(argsStr, "-f null") {
|
||||||
|
t.Error("Pass 1 command should include '-f null'")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(argsStr, "/dev/null") {
|
||||||
|
t.Error("Pass 1 command should output to /dev/null")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check VP9-specific arguments in pass 1
|
||||||
|
vp9Checks := []string{
|
||||||
|
"-c:v libvpx-vp9",
|
||||||
|
"-crf 30",
|
||||||
|
"-b:v 0",
|
||||||
|
"-row-mt 1",
|
||||||
|
"-g 240",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, check := range vp9Checks {
|
||||||
|
if !strings.Contains(argsStr, check) {
|
||||||
|
t.Errorf("Missing VP9 argument in pass 1: %s", check)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSoftwareEncoder_BuildCommand_NoTwoPass(t *testing.T) {
|
||||||
|
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: "frame_%04d.exr",
|
||||||
|
OutputPath: "output.mp4",
|
||||||
|
StartFrame: 1,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: "/tmp",
|
||||||
|
UseAlpha: false,
|
||||||
|
TwoPass: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
args := cmd.Args[1:]
|
||||||
|
argsStr := strings.Join(args, " ")
|
||||||
|
|
||||||
|
// Should NOT have -pass flag when TwoPass is false
|
||||||
|
if strings.Contains(argsStr, "-pass") {
|
||||||
|
t.Error("Command should not include -pass flag when TwoPass is false")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelector_SelectH264(t *testing.T) {
|
||||||
|
selector := NewSelector()
|
||||||
|
encoder := selector.SelectH264()
|
||||||
|
|
||||||
|
if encoder == nil {
|
||||||
|
t.Fatal("SelectH264 returned nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
if encoder.Codec() != "libx264" {
|
||||||
|
t.Errorf("Expected codec 'libx264', got '%s'", encoder.Codec())
|
||||||
|
}
|
||||||
|
|
||||||
|
if encoder.Name() != "software" {
|
||||||
|
t.Errorf("Expected name 'software', got '%s'", encoder.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelector_SelectAV1(t *testing.T) {
|
||||||
|
selector := NewSelector()
|
||||||
|
encoder := selector.SelectAV1()
|
||||||
|
|
||||||
|
if encoder == nil {
|
||||||
|
t.Fatal("SelectAV1 returned nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
if encoder.Codec() != "libaom-av1" {
|
||||||
|
t.Errorf("Expected codec 'libaom-av1', got '%s'", encoder.Codec())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSelector_SelectVP9(t *testing.T) {
|
||||||
|
selector := NewSelector()
|
||||||
|
encoder := selector.SelectVP9()
|
||||||
|
|
||||||
|
if encoder == nil {
|
||||||
|
t.Fatal("SelectVP9 returned nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
if encoder.Codec() != "libvpx-vp9" {
|
||||||
|
t.Errorf("Expected codec 'libvpx-vp9', got '%s'", encoder.Codec())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTonemapFilter_WithAlpha(t *testing.T) {
|
||||||
|
filter := tonemapFilter(true)
|
||||||
|
|
||||||
|
// Filter should convert from gbrpf32le to yuva420p10le with proper colorspace conversion
|
||||||
|
if !strings.Contains(filter, "yuva420p10le") {
|
||||||
|
t.Error("Tonemap filter with alpha should output yuva420p10le format for HDR")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(filter, "gbrpf32le") {
|
||||||
|
t.Error("Tonemap filter should start with gbrpf32le format")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should use zscale for colorspace conversion from linear RGB to bt2020 YUV
|
||||||
|
if !strings.Contains(filter, "zscale") {
|
||||||
|
t.Error("Tonemap filter should use zscale for colorspace conversion")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for HLG transfer function (numeric value 18 or string arib-std-b67)
|
||||||
|
if !strings.Contains(filter, "transfer=18") && !strings.Contains(filter, "transfer=arib-std-b67") {
|
||||||
|
t.Error("Tonemap filter should use HLG transfer function (18 or arib-std-b67)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTonemapFilter_WithoutAlpha(t *testing.T) {
|
||||||
|
filter := tonemapFilter(false)
|
||||||
|
|
||||||
|
// Filter should convert from gbrpf32le to yuv420p10le with proper colorspace conversion
|
||||||
|
if !strings.Contains(filter, "yuv420p10le") {
|
||||||
|
t.Error("Tonemap filter without alpha should output yuv420p10le format for HDR")
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(filter, "yuva420p") {
|
||||||
|
t.Error("Tonemap filter without alpha should not output yuva420p format")
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(filter, "gbrpf32le") {
|
||||||
|
t.Error("Tonemap filter should start with gbrpf32le format")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should use zscale for colorspace conversion from linear RGB to bt2020 YUV
|
||||||
|
if !strings.Contains(filter, "zscale") {
|
||||||
|
t.Error("Tonemap filter should use zscale for colorspace conversion")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for HLG transfer function (numeric value 18 or string arib-std-b67)
|
||||||
|
if !strings.Contains(filter, "transfer=18") && !strings.Contains(filter, "transfer=arib-std-b67") {
|
||||||
|
t.Error("Tonemap filter should use HLG transfer function (18 or arib-std-b67)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSoftwareEncoder_Available(t *testing.T) {
|
||||||
|
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||||
|
if !encoder.Available() {
|
||||||
|
t.Error("Software encoder should always be available")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommandOrder(t *testing.T) {
|
||||||
|
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: "frame_%04d.exr",
|
||||||
|
OutputPath: "output.mp4",
|
||||||
|
StartFrame: 1,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: "/tmp",
|
||||||
|
UseAlpha: false,
|
||||||
|
TwoPass: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
args := cmd.Args[1:]
|
||||||
|
|
||||||
|
// Verify argument order: input should come before codec
|
||||||
|
inputIdx := -1
|
||||||
|
codecIdx := -1
|
||||||
|
vfIdx := -1
|
||||||
|
|
||||||
|
for i, arg := range args {
|
||||||
|
if arg == "-i" && i+1 < len(args) && args[i+1] == "frame_%04d.exr" {
|
||||||
|
inputIdx = i
|
||||||
|
}
|
||||||
|
if arg == "-c:v" && i+1 < len(args) && args[i+1] == "libx264" {
|
||||||
|
codecIdx = i
|
||||||
|
}
|
||||||
|
if arg == "-vf" {
|
||||||
|
vfIdx = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if inputIdx == -1 {
|
||||||
|
t.Fatal("Input pattern not found in command")
|
||||||
|
}
|
||||||
|
if codecIdx == -1 {
|
||||||
|
t.Fatal("Codec not found in command")
|
||||||
|
}
|
||||||
|
if vfIdx == -1 {
|
||||||
|
t.Fatal("Video filter not found in command")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Input should come before codec
|
||||||
|
if inputIdx >= codecIdx {
|
||||||
|
t.Error("Input pattern should come before codec in command")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Video filter should come after input (order: input -> codec -> colorspace -> filter -> codec args)
|
||||||
|
// In practice, the filter comes after codec and colorspace metadata but before codec-specific args
|
||||||
|
if vfIdx <= inputIdx {
|
||||||
|
t.Error("Video filter should come after input")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommand_ColorspaceMetadata(t *testing.T) {
|
||||||
|
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: "frame_%04d.exr",
|
||||||
|
OutputPath: "output.mp4",
|
||||||
|
StartFrame: 1,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: "/tmp",
|
||||||
|
UseAlpha: false,
|
||||||
|
TwoPass: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
args := cmd.Args[1:]
|
||||||
|
argsStr := strings.Join(args, " ")
|
||||||
|
|
||||||
|
// EXR always uses HDR path: bt709 primaries, HLG, full range
|
||||||
|
colorspaceArgs := []string{
|
||||||
|
"-color_primaries bt709",
|
||||||
|
"-color_trc arib-std-b67",
|
||||||
|
"-colorspace bt709",
|
||||||
|
"-color_range pc",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, arg := range colorspaceArgs {
|
||||||
|
if !strings.Contains(argsStr, arg) {
|
||||||
|
t.Errorf("Missing colorspace metadata: %s", arg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.Contains(argsStr, "-pix_fmt yuv420p10le") {
|
||||||
|
t.Error("EXR encoding should use yuv420p10le pixel format")
|
||||||
|
}
|
||||||
|
if !strings.Contains(argsStr, "-profile:v high10") {
|
||||||
|
t.Error("EXR encoding should use high10 profile")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCommand_HDR_ColorspaceMetadata(t *testing.T) {
|
||||||
|
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: "frame_%04d.exr",
|
||||||
|
OutputPath: "output.mp4",
|
||||||
|
StartFrame: 1,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: "/tmp",
|
||||||
|
UseAlpha: false,
|
||||||
|
TwoPass: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
args := cmd.Args[1:]
|
||||||
|
argsStr := strings.Join(args, " ")
|
||||||
|
|
||||||
|
// Verify all HDR colorspace metadata is present for EXR (full range to match zscale output)
|
||||||
|
colorspaceArgs := []string{
|
||||||
|
"-color_primaries bt709",
|
||||||
|
"-color_trc arib-std-b67",
|
||||||
|
"-colorspace bt709",
|
||||||
|
"-color_range pc",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, arg := range colorspaceArgs {
|
||||||
|
if !strings.Contains(argsStr, arg) {
|
||||||
|
t.Errorf("Missing HDR colorspace metadata: %s", arg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify HDR pixel format (10-bit)
|
||||||
|
if !strings.Contains(argsStr, "-pix_fmt yuv420p10le") {
|
||||||
|
t.Error("HDR encoding should use yuv420p10le pixel format")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify H.264 high10 profile (for 10-bit)
|
||||||
|
if !strings.Contains(argsStr, "-profile:v high10") {
|
||||||
|
t.Error("HDR encoding should use high10 profile")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify HDR filter chain (linear -> sRGB -> HLG)
|
||||||
|
if !strings.Contains(argsStr, "-vf") {
|
||||||
|
t.Fatal("HDR encoding should have video filter")
|
||||||
|
}
|
||||||
|
vfIdx := -1
|
||||||
|
for i, arg := range args {
|
||||||
|
if arg == "-vf" && i+1 < len(args) {
|
||||||
|
vfIdx = i + 1
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if vfIdx == -1 {
|
||||||
|
t.Fatal("Video filter not found")
|
||||||
|
}
|
||||||
|
filter := args[vfIdx]
|
||||||
|
if !strings.Contains(filter, "transfer=18") {
|
||||||
|
t.Error("HDR filter should convert to HLG (transfer=18)")
|
||||||
|
}
|
||||||
|
if !strings.Contains(filter, "yuv420p10le") {
|
||||||
|
t.Error("HDR filter should output yuv420p10le format")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Integration tests using example files
|
||||||
|
func TestIntegration_Encode_EXR_H264(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping integration test in short mode")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if example file exists
|
||||||
|
exampleDir := filepath.Join("..", "..", "..", "examples")
|
||||||
|
exrFile := filepath.Join(exampleDir, "frame_0800.exr")
|
||||||
|
if _, err := os.Stat(exrFile); os.IsNotExist(err) {
|
||||||
|
t.Skipf("Example file not found: %s", exrFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get absolute paths
|
||||||
|
workspaceRoot, err := filepath.Abs(filepath.Join("..", "..", ".."))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get workspace root: %v", err)
|
||||||
|
}
|
||||||
|
exampleDirAbs, err := filepath.Abs(exampleDir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get example directory: %v", err)
|
||||||
|
}
|
||||||
|
tmpDir := filepath.Join(workspaceRoot, "tmp")
|
||||||
|
if err := os.MkdirAll(tmpDir, 0755); err != nil {
|
||||||
|
t.Fatalf("Failed to create tmp directory: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: filepath.Join(exampleDirAbs, "frame_%04d.exr"),
|
||||||
|
OutputPath: filepath.Join(tmpDir, "test_exr_h264.mp4"),
|
||||||
|
StartFrame: 800,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: tmpDir,
|
||||||
|
UseAlpha: false,
|
||||||
|
TwoPass: false, // Use single pass for faster testing
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build and run command
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
if cmd == nil {
|
||||||
|
t.Fatal("BuildCommand returned nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Capture stderr to see what went wrong
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("FFmpeg command failed: %v\nCommand output: %s", err, string(output))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify output file was created
|
||||||
|
if _, err := os.Stat(config.OutputPath); os.IsNotExist(err) {
|
||||||
|
t.Errorf("Output file was not created: %s\nCommand output: %s", config.OutputPath, string(output))
|
||||||
|
} else {
|
||||||
|
t.Logf("Successfully created output file: %s", config.OutputPath)
|
||||||
|
// Verify file has content
|
||||||
|
info, _ := os.Stat(config.OutputPath)
|
||||||
|
if info.Size() == 0 {
|
||||||
|
t.Errorf("Output file was created but is empty\nCommand output: %s", string(output))
|
||||||
|
} else {
|
||||||
|
t.Logf("Output file size: %d bytes", info.Size())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntegration_Encode_EXR_VP9(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping integration test in short mode")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if example file exists
|
||||||
|
exampleDir := filepath.Join("..", "..", "..", "examples")
|
||||||
|
exrFile := filepath.Join(exampleDir, "frame_0800.exr")
|
||||||
|
if _, err := os.Stat(exrFile); os.IsNotExist(err) {
|
||||||
|
t.Skipf("Example file not found: %s", exrFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if VP9 encoder is available
|
||||||
|
checkCmd := exec.Command("ffmpeg", "-hide_banner", "-encoders")
|
||||||
|
checkOutput, err := checkCmd.CombinedOutput()
|
||||||
|
if err != nil || !strings.Contains(string(checkOutput), "libvpx-vp9") {
|
||||||
|
t.Skip("VP9 encoder (libvpx-vp9) not available in ffmpeg")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get absolute paths
|
||||||
|
workspaceRoot, err := filepath.Abs(filepath.Join("..", "..", ".."))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get workspace root: %v", err)
|
||||||
|
}
|
||||||
|
exampleDirAbs, err := filepath.Abs(exampleDir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get example directory: %v", err)
|
||||||
|
}
|
||||||
|
tmpDir := filepath.Join(workspaceRoot, "tmp")
|
||||||
|
if err := os.MkdirAll(tmpDir, 0755); err != nil {
|
||||||
|
t.Fatalf("Failed to create tmp directory: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
encoder := &SoftwareEncoder{codec: "libvpx-vp9"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: filepath.Join(exampleDirAbs, "frame_%04d.exr"),
|
||||||
|
OutputPath: filepath.Join(tmpDir, "test_exr_vp9.webm"),
|
||||||
|
StartFrame: 800,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: tmpDir,
|
||||||
|
UseAlpha: false,
|
||||||
|
TwoPass: false, // Use single pass for faster testing
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build and run command
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
if cmd == nil {
|
||||||
|
t.Fatal("BuildCommand returned nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Capture stderr to see what went wrong
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("FFmpeg command failed: %v\nCommand output: %s", err, string(output))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify output file was created
|
||||||
|
if _, err := os.Stat(config.OutputPath); os.IsNotExist(err) {
|
||||||
|
t.Errorf("Output file was not created: %s\nCommand output: %s", config.OutputPath, string(output))
|
||||||
|
} else {
|
||||||
|
t.Logf("Successfully created output file: %s", config.OutputPath)
|
||||||
|
// Verify file has content
|
||||||
|
info, _ := os.Stat(config.OutputPath)
|
||||||
|
if info.Size() == 0 {
|
||||||
|
t.Errorf("Output file was created but is empty\nCommand output: %s", string(output))
|
||||||
|
} else {
|
||||||
|
t.Logf("Output file size: %d bytes", info.Size())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntegration_Encode_EXR_AV1(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping integration test in short mode")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if example file exists
|
||||||
|
exampleDir := filepath.Join("..", "..", "..", "examples")
|
||||||
|
exrFile := filepath.Join(exampleDir, "frame_0800.exr")
|
||||||
|
if _, err := os.Stat(exrFile); os.IsNotExist(err) {
|
||||||
|
t.Skipf("Example file not found: %s", exrFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if AV1 encoder is available
|
||||||
|
checkCmd := exec.Command("ffmpeg", "-hide_banner", "-encoders")
|
||||||
|
output, err := checkCmd.CombinedOutput()
|
||||||
|
if err != nil || !strings.Contains(string(output), "libaom-av1") {
|
||||||
|
t.Skip("AV1 encoder (libaom-av1) not available in ffmpeg")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get absolute paths
|
||||||
|
workspaceRoot, err := filepath.Abs(filepath.Join("..", "..", ".."))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get workspace root: %v", err)
|
||||||
|
}
|
||||||
|
exampleDirAbs, err := filepath.Abs(exampleDir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get example directory: %v", err)
|
||||||
|
}
|
||||||
|
tmpDir := filepath.Join(workspaceRoot, "tmp")
|
||||||
|
if err := os.MkdirAll(tmpDir, 0755); err != nil {
|
||||||
|
t.Fatalf("Failed to create tmp directory: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
encoder := &SoftwareEncoder{codec: "libaom-av1"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: filepath.Join(exampleDirAbs, "frame_%04d.exr"),
|
||||||
|
OutputPath: filepath.Join(tmpDir, "test_exr_av1.mp4"),
|
||||||
|
StartFrame: 800,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: tmpDir,
|
||||||
|
UseAlpha: false,
|
||||||
|
TwoPass: false,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build and run command
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
cmdOutput, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("FFmpeg command failed: %v\nCommand output: %s", err, string(cmdOutput))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify output file was created
|
||||||
|
if _, err := os.Stat(config.OutputPath); os.IsNotExist(err) {
|
||||||
|
t.Errorf("Output file was not created: %s\nCommand output: %s", config.OutputPath, string(cmdOutput))
|
||||||
|
} else {
|
||||||
|
t.Logf("Successfully created AV1 output file: %s", config.OutputPath)
|
||||||
|
info, _ := os.Stat(config.OutputPath)
|
||||||
|
if info.Size() == 0 {
|
||||||
|
t.Errorf("Output file was created but is empty\nCommand output: %s", string(cmdOutput))
|
||||||
|
} else {
|
||||||
|
t.Logf("Output file size: %d bytes", info.Size())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIntegration_Encode_EXR_VP9_WithAlpha(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping integration test in short mode")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if example file exists
|
||||||
|
exampleDir := filepath.Join("..", "..", "..", "examples")
|
||||||
|
exrFile := filepath.Join(exampleDir, "frame_0800.exr")
|
||||||
|
if _, err := os.Stat(exrFile); os.IsNotExist(err) {
|
||||||
|
t.Skipf("Example file not found: %s", exrFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if VP9 encoder is available
|
||||||
|
checkCmd := exec.Command("ffmpeg", "-hide_banner", "-encoders")
|
||||||
|
output, err := checkCmd.CombinedOutput()
|
||||||
|
if err != nil || !strings.Contains(string(output), "libvpx-vp9") {
|
||||||
|
t.Skip("VP9 encoder (libvpx-vp9) not available in ffmpeg")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get absolute paths
|
||||||
|
workspaceRoot, err := filepath.Abs(filepath.Join("..", "..", ".."))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get workspace root: %v", err)
|
||||||
|
}
|
||||||
|
exampleDirAbs, err := filepath.Abs(exampleDir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to get example directory: %v", err)
|
||||||
|
}
|
||||||
|
tmpDir := filepath.Join(workspaceRoot, "tmp")
|
||||||
|
if err := os.MkdirAll(tmpDir, 0755); err != nil {
|
||||||
|
t.Fatalf("Failed to create tmp directory: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
encoder := &SoftwareEncoder{codec: "libvpx-vp9"}
|
||||||
|
config := &EncodeConfig{
|
||||||
|
InputPattern: filepath.Join(exampleDirAbs, "frame_%04d.exr"),
|
||||||
|
OutputPath: filepath.Join(tmpDir, "test_exr_vp9_alpha.webm"),
|
||||||
|
StartFrame: 800,
|
||||||
|
FrameRate: 24.0,
|
||||||
|
WorkDir: tmpDir,
|
||||||
|
UseAlpha: true, // Test with alpha
|
||||||
|
TwoPass: false, // Use single pass for faster testing
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build and run command
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
if cmd == nil {
|
||||||
|
t.Fatal("BuildCommand returned nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Capture stderr to see what went wrong
|
||||||
|
cmdOutput, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("FFmpeg command failed: %v\nCommand output: %s", err, string(cmdOutput))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify output file was created
|
||||||
|
if _, err := os.Stat(config.OutputPath); os.IsNotExist(err) {
|
||||||
|
t.Errorf("Output file was not created: %s\nCommand output: %s", config.OutputPath, string(cmdOutput))
|
||||||
|
} else {
|
||||||
|
t.Logf("Successfully created VP9 output file with alpha: %s", config.OutputPath)
|
||||||
|
info, _ := os.Stat(config.OutputPath)
|
||||||
|
if info.Size() == 0 {
|
||||||
|
t.Errorf("Output file was created but is empty\nCommand output: %s", string(cmdOutput))
|
||||||
|
} else {
|
||||||
|
t.Logf("Output file size: %d bytes", info.Size())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to copy files
|
||||||
|
func copyFile(src, dst string) error {
|
||||||
|
data, err := os.ReadFile(src)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return os.WriteFile(dst, data, 0644)
|
||||||
|
}
|
||||||
@@ -0,0 +1,637 @@
|
|||||||
|
// Package runner provides the Jiggablend render runner.
|
||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"jiggablend/internal/runner/api"
|
||||||
|
"jiggablend/internal/runner/blender"
|
||||||
|
"jiggablend/internal/runner/encoding"
|
||||||
|
"jiggablend/internal/runner/sandbox"
|
||||||
|
"jiggablend/internal/runner/tasks"
|
||||||
|
"jiggablend/internal/runner/workspace"
|
||||||
|
"jiggablend/pkg/executils"
|
||||||
|
"jiggablend/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Runner is the main render runner.
|
||||||
|
type Runner struct {
|
||||||
|
id int64
|
||||||
|
name string
|
||||||
|
hostname string
|
||||||
|
|
||||||
|
manager *api.ManagerClient
|
||||||
|
workspace *workspace.Manager
|
||||||
|
blender *blender.Manager
|
||||||
|
encoder *encoding.Selector
|
||||||
|
processes *executils.ProcessTracker
|
||||||
|
|
||||||
|
processors map[string]tasks.Processor
|
||||||
|
stopChan chan struct{}
|
||||||
|
|
||||||
|
fingerprint string
|
||||||
|
fingerprintMu sync.RWMutex
|
||||||
|
|
||||||
|
// gpuLockedOut is set when logs indicate a GPU error (e.g. HIP "Illegal address");
|
||||||
|
// when true, the runner forces CPU rendering for all subsequent jobs.
|
||||||
|
gpuLockedOut bool
|
||||||
|
gpuLockedOutMu sync.RWMutex
|
||||||
|
|
||||||
|
// hasAMD/hasNVIDIA/hasIntel are set at startup by hardware/Blender GPU backend detection.
|
||||||
|
// Used to force CPU only for Blender < 4.x when AMD is present (no official HIP support pre-4).
|
||||||
|
// gpuDetectionFailed is true when detection could not run; we then force CPU for all versions.
|
||||||
|
gpuBackendMu sync.RWMutex
|
||||||
|
hasAMD bool
|
||||||
|
hasNVIDIA bool
|
||||||
|
hasIntel bool
|
||||||
|
gpuBackendProbed bool
|
||||||
|
gpuDetectionFailed bool
|
||||||
|
|
||||||
|
// forceCPURendering forces CPU rendering for all jobs regardless of metadata/backend detection.
|
||||||
|
forceCPURendering bool
|
||||||
|
// disableRT disables GPU ray tracing acceleration (HIPRT, OptiX, etc.).
|
||||||
|
disableRT bool
|
||||||
|
// hipGPUSampleBatch limits samples per GPU pass on gfx115x (0 = disabled).
|
||||||
|
hipGPUSampleBatch int
|
||||||
|
|
||||||
|
// sandbox wraps Blender invocations (none/podman).
|
||||||
|
sandboxWrapper sandbox.Wrapper
|
||||||
|
sandboxBackend string
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunnerOptions configures optional runner behavior.
|
||||||
|
type RunnerOptions struct {
|
||||||
|
ForceCPURendering bool
|
||||||
|
DisableRT bool
|
||||||
|
HipGPUSampleBatch int
|
||||||
|
SandboxBackend string // none|podman
|
||||||
|
SandboxNetwork bool
|
||||||
|
SandboxImage string // podman thin runtime image
|
||||||
|
}
|
||||||
|
|
||||||
|
// New creates a new runner.
|
||||||
|
func New(managerURL, name, hostname string, forceCPURendering, disableRT bool, hipGPUSampleBatch int) *Runner {
|
||||||
|
return NewWithOptions(managerURL, name, hostname, RunnerOptions{
|
||||||
|
ForceCPURendering: forceCPURendering,
|
||||||
|
DisableRT: disableRT,
|
||||||
|
HipGPUSampleBatch: hipGPUSampleBatch,
|
||||||
|
SandboxBackend: sandbox.BackendPodman,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewWithOptions creates a runner with full options including sandbox.
|
||||||
|
func NewWithOptions(managerURL, name, hostname string, opts RunnerOptions) *Runner {
|
||||||
|
manager := api.NewManagerClient(managerURL)
|
||||||
|
|
||||||
|
backend, err := sandbox.NormalizeBackend(opts.SandboxBackend)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Invalid sandbox backend %q, falling back to none: %v", opts.SandboxBackend, err)
|
||||||
|
backend = sandbox.BackendNone
|
||||||
|
}
|
||||||
|
sb, err := sandbox.New(sandbox.Options{
|
||||||
|
Backend: backend,
|
||||||
|
AllowNetwork: opts.SandboxNetwork,
|
||||||
|
PodmanImage: opts.SandboxImage,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Failed to init sandbox %q: %v; using none", backend, err)
|
||||||
|
sb, _ = sandbox.New(sandbox.Options{Backend: sandbox.BackendNone})
|
||||||
|
backend = sandbox.BackendNone
|
||||||
|
}
|
||||||
|
|
||||||
|
r := &Runner{
|
||||||
|
name: name,
|
||||||
|
hostname: hostname,
|
||||||
|
manager: manager,
|
||||||
|
processes: executils.NewProcessTracker(),
|
||||||
|
stopChan: make(chan struct{}),
|
||||||
|
processors: make(map[string]tasks.Processor),
|
||||||
|
|
||||||
|
forceCPURendering: opts.ForceCPURendering,
|
||||||
|
disableRT: opts.DisableRT,
|
||||||
|
hipGPUSampleBatch: opts.HipGPUSampleBatch,
|
||||||
|
sandboxWrapper: sb,
|
||||||
|
sandboxBackend: backend,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate fingerprint
|
||||||
|
r.generateFingerprint()
|
||||||
|
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckRequiredTools verifies that required external tools are available.
|
||||||
|
func (r *Runner) CheckRequiredTools() error {
|
||||||
|
if err := exec.Command("zstd", "--version").Run(); err != nil {
|
||||||
|
return fmt.Errorf("zstd not found - required for compressed blend file support. Install with: apt install zstd")
|
||||||
|
}
|
||||||
|
log.Printf("Found zstd for compressed blend file support")
|
||||||
|
|
||||||
|
if r.sandboxWrapper != nil {
|
||||||
|
if err := r.sandboxWrapper.Available(); err != nil {
|
||||||
|
return fmt.Errorf("sandbox backend %q unavailable: %w", r.sandboxBackend, err)
|
||||||
|
}
|
||||||
|
log.Printf("Sandbox backend: %s", r.sandboxWrapper.Name())
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SandboxBackend returns the configured sandbox backend name.
|
||||||
|
func (r *Runner) SandboxBackend() string {
|
||||||
|
if r.sandboxBackend == "" {
|
||||||
|
return sandbox.BackendNone
|
||||||
|
}
|
||||||
|
return r.sandboxBackend
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
cachedCapabilities map[string]interface{}
|
||||||
|
capabilitiesOnce sync.Once
|
||||||
|
)
|
||||||
|
|
||||||
|
// ProbeCapabilities detects hardware capabilities.
|
||||||
|
func (r *Runner) ProbeCapabilities() map[string]interface{} {
|
||||||
|
capabilitiesOnce.Do(func() {
|
||||||
|
caps := make(map[string]interface{})
|
||||||
|
|
||||||
|
if err := exec.Command("ffmpeg", "-version").Run(); err == nil {
|
||||||
|
caps["ffmpeg"] = true
|
||||||
|
} else {
|
||||||
|
caps["ffmpeg"] = false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filled later with actual backend when runner is constructed; probe is package-level once.
|
||||||
|
// Real sandbox name is injected in ProbeCapabilities on the instance after New.
|
||||||
|
caps["sandbox"] = "none"
|
||||||
|
|
||||||
|
cachedCapabilities = caps
|
||||||
|
})
|
||||||
|
// Overlay instance sandbox name (Once already ran with none default).
|
||||||
|
if r != nil && r.sandboxWrapper != nil {
|
||||||
|
out := make(map[string]interface{}, len(cachedCapabilities)+1)
|
||||||
|
for k, v := range cachedCapabilities {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
out["sandbox"] = r.sandboxWrapper.Name()
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
return cachedCapabilities
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register registers the runner with the manager.
|
||||||
|
func (r *Runner) Register(apiKey string) (int64, error) {
|
||||||
|
caps := r.ProbeCapabilities()
|
||||||
|
|
||||||
|
id, err := r.manager.Register(r.name, r.hostname, caps, apiKey, r.GetFingerprint())
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
r.id = id
|
||||||
|
|
||||||
|
// Initialize workspace after registration
|
||||||
|
r.workspace = workspace.NewManager(r.name)
|
||||||
|
|
||||||
|
// Initialize blender manager
|
||||||
|
r.blender = blender.NewManager(r.manager, r.workspace.BaseDir())
|
||||||
|
|
||||||
|
// Initialize encoder selector
|
||||||
|
r.encoder = encoding.NewSelector()
|
||||||
|
|
||||||
|
// Register task processors
|
||||||
|
r.processors["render"] = tasks.NewRenderProcessor()
|
||||||
|
r.processors["encode"] = tasks.NewEncodeProcessor()
|
||||||
|
|
||||||
|
return id, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DetectAndStoreGPUBackends runs host-level backend detection and stores AMD/NVIDIA/Intel results.
|
||||||
|
// Call after Register. Used so we only force CPU for Blender < 4.x when AMD is present.
|
||||||
|
func (r *Runner) DetectAndStoreGPUBackends() {
|
||||||
|
r.gpuBackendMu.Lock()
|
||||||
|
defer r.gpuBackendMu.Unlock()
|
||||||
|
if r.gpuBackendProbed {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
hasAMD, hasNVIDIA, hasIntel, ok := blender.DetectGPUBackends()
|
||||||
|
if !ok {
|
||||||
|
log.Printf("GPU backend detection failed (host probe unavailable). All jobs will use CPU because backend availability is unknown.")
|
||||||
|
r.gpuBackendProbed = true
|
||||||
|
r.gpuDetectionFailed = true
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
detectedTypes := 0
|
||||||
|
if hasAMD {
|
||||||
|
detectedTypes++
|
||||||
|
}
|
||||||
|
if hasNVIDIA {
|
||||||
|
detectedTypes++
|
||||||
|
}
|
||||||
|
if hasIntel {
|
||||||
|
detectedTypes++
|
||||||
|
}
|
||||||
|
if detectedTypes > 1 {
|
||||||
|
log.Printf("mixed GPU vendors detected (AMD=%v NVIDIA=%v INTEL=%v): multi-vendor setups may not work reliably, but runner will continue with GPU enabled", hasAMD, hasNVIDIA, hasIntel)
|
||||||
|
}
|
||||||
|
|
||||||
|
r.hasAMD = hasAMD
|
||||||
|
r.hasNVIDIA = hasNVIDIA
|
||||||
|
r.hasIntel = hasIntel
|
||||||
|
r.gpuBackendProbed = true
|
||||||
|
r.gpuDetectionFailed = false
|
||||||
|
log.Printf("GPU backend detection: AMD=%v NVIDIA=%v INTEL=%v (Blender < 4.x will force CPU only when AMD is present)", hasAMD, hasNVIDIA, hasIntel)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasAMD returns whether the runner detected AMD devices. Used to force CPU for Blender < 4.x only when AMD is present.
|
||||||
|
func (r *Runner) HasAMD() bool {
|
||||||
|
r.gpuBackendMu.RLock()
|
||||||
|
defer r.gpuBackendMu.RUnlock()
|
||||||
|
return r.hasAMD
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasNVIDIA returns whether the runner detected NVIDIA GPUs.
|
||||||
|
func (r *Runner) HasNVIDIA() bool {
|
||||||
|
r.gpuBackendMu.RLock()
|
||||||
|
defer r.gpuBackendMu.RUnlock()
|
||||||
|
return r.hasNVIDIA
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasIntel returns whether the runner detected Intel GPUs (e.g. Arc).
|
||||||
|
func (r *Runner) HasIntel() bool {
|
||||||
|
r.gpuBackendMu.RLock()
|
||||||
|
defer r.gpuBackendMu.RUnlock()
|
||||||
|
return r.hasIntel
|
||||||
|
}
|
||||||
|
|
||||||
|
// DisableRT returns whether GPU ray tracing acceleration should be disabled.
|
||||||
|
func (r *Runner) DisableRT() bool {
|
||||||
|
return r.disableRT
|
||||||
|
}
|
||||||
|
|
||||||
|
// HipGPUSampleBatch returns the per-pass GPU sample limit for gfx115x batching (0 = disabled).
|
||||||
|
func (r *Runner) HipGPUSampleBatch() int {
|
||||||
|
return r.hipGPUSampleBatch
|
||||||
|
}
|
||||||
|
|
||||||
|
// GPUDetectionFailed returns true when startup GPU backend detection could not run or failed. When true, all jobs use CPU because backend availability is unknown.
|
||||||
|
func (r *Runner) GPUDetectionFailed() bool {
|
||||||
|
r.gpuBackendMu.RLock()
|
||||||
|
defer r.gpuBackendMu.RUnlock()
|
||||||
|
return r.gpuDetectionFailed
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start starts the job polling loop.
|
||||||
|
func (r *Runner) Start(pollInterval time.Duration) {
|
||||||
|
log.Printf("Starting job polling loop (interval: %v)", pollInterval)
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-r.stopChan:
|
||||||
|
log.Printf("Stopping job polling loop")
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Polling for next job (runner ID: %d)", r.id)
|
||||||
|
job, err := r.manager.PollNextJob()
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Error polling for job: %v", err)
|
||||||
|
time.Sleep(pollInterval)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if job == nil {
|
||||||
|
log.Printf("No job available, sleeping for %v", pollInterval)
|
||||||
|
time.Sleep(pollInterval)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Received job assignment: task=%d, job=%d, type=%s",
|
||||||
|
job.Task.TaskID, job.Task.JobID, job.Task.TaskType)
|
||||||
|
|
||||||
|
if err := r.executeJob(job); err != nil {
|
||||||
|
log.Printf("Error processing job: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop stops the runner.
|
||||||
|
func (r *Runner) Stop() {
|
||||||
|
close(r.stopChan)
|
||||||
|
}
|
||||||
|
|
||||||
|
// KillAllProcesses kills all running processes.
|
||||||
|
func (r *Runner) KillAllProcesses() {
|
||||||
|
log.Printf("Killing all running processes...")
|
||||||
|
killedCount := r.processes.KillAll()
|
||||||
|
|
||||||
|
// Release all allocated devices
|
||||||
|
if r.encoder != nil {
|
||||||
|
// Device pool cleanup is handled internally
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Killed %d process(es)", killedCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cleanup removes the workspace directory.
|
||||||
|
func (r *Runner) Cleanup() {
|
||||||
|
if r.workspace != nil {
|
||||||
|
r.workspace.Cleanup()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) withJobWorkspace(jobID int64, fn func(workDir string) error) error {
|
||||||
|
workDir, err := r.workspace.CreateJobDir(jobID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create job workspace: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
if cleanupErr := r.workspace.CleanupJobDir(jobID); cleanupErr != nil {
|
||||||
|
log.Printf("Warning: failed to cleanup job workspace for job %d: %v", jobID, cleanupErr)
|
||||||
|
}
|
||||||
|
if cleanupErr := r.workspace.CleanupVideoDir(jobID); cleanupErr != nil {
|
||||||
|
log.Printf("Warning: failed to cleanup encode workspace for job %d: %v", jobID, cleanupErr)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return fn(workDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// executeJob handles a job using per-job WebSocket connection.
|
||||||
|
func (r *Runner) executeJob(job *api.NextJobResponse) (err error) {
|
||||||
|
// Recover from panics to prevent runner process crashes during task execution
|
||||||
|
defer func() {
|
||||||
|
if rec := recover(); rec != nil {
|
||||||
|
log.Printf("Task execution panicked: %v", rec)
|
||||||
|
err = fmt.Errorf("task execution panicked: %v", rec)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return r.withJobWorkspace(job.Task.JobID, func(workDir string) error {
|
||||||
|
// Connect to job WebSocket (no runnerID needed - authentication handles it)
|
||||||
|
jobConn := api.NewJobConnection()
|
||||||
|
if err := jobConn.Connect(r.manager.GetBaseURL(), job.JobPath, job.JobToken); err != nil {
|
||||||
|
return fmt.Errorf("failed to connect job WebSocket: %w", err)
|
||||||
|
}
|
||||||
|
defer jobConn.Close()
|
||||||
|
|
||||||
|
log.Printf("Job WebSocket authenticated for task %d", job.Task.TaskID)
|
||||||
|
|
||||||
|
// Create task context (frame range: Frame = start, FrameEnd = end; 0 or missing = single frame)
|
||||||
|
frameEnd := job.Task.FrameEnd
|
||||||
|
if frameEnd < job.Task.Frame {
|
||||||
|
frameEnd = job.Task.Frame
|
||||||
|
}
|
||||||
|
ctx := tasks.NewContext(
|
||||||
|
job.Task.TaskID,
|
||||||
|
job.Task.JobID,
|
||||||
|
job.Task.JobName,
|
||||||
|
job.Task.Frame,
|
||||||
|
frameEnd,
|
||||||
|
job.Task.TaskType,
|
||||||
|
workDir,
|
||||||
|
job.JobToken,
|
||||||
|
job.Task.Metadata,
|
||||||
|
r.manager,
|
||||||
|
jobConn,
|
||||||
|
r.workspace,
|
||||||
|
r.blender,
|
||||||
|
r.encoder,
|
||||||
|
r.processes,
|
||||||
|
r.IsGPULockedOut(),
|
||||||
|
r.HasAMD(),
|
||||||
|
r.HasNVIDIA(),
|
||||||
|
r.HasIntel(),
|
||||||
|
r.GPUDetectionFailed(),
|
||||||
|
r.forceCPURendering,
|
||||||
|
r.disableRT,
|
||||||
|
r.hipGPUSampleBatch,
|
||||||
|
nil, // set below so the callback can mark this attempt
|
||||||
|
r.sandboxWrapper,
|
||||||
|
)
|
||||||
|
// Arm GPU lockout at most once process-wide; if this attempt is the one that
|
||||||
|
// arms it, mark the context so a failure requeues without burning retry_count.
|
||||||
|
ctx.OnGPUError = func() {
|
||||||
|
if r.SetGPULockedOut(true) {
|
||||||
|
ctx.GPULockoutArmedThisAttempt = true
|
||||||
|
ctx.GPULockedOut = true
|
||||||
|
ctx.Warn("GPU error detected; GPU disabled for subsequent jobs (this attempt free-requeues without using a retry)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.Info(fmt.Sprintf("Task assignment received (job: %d, type: %s)",
|
||||||
|
job.Task.JobID, job.Task.TaskType))
|
||||||
|
|
||||||
|
// Get processor for task type
|
||||||
|
processor, ok := r.processors[job.Task.TaskType]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("unknown task type: %s", job.Task.TaskType)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process the task
|
||||||
|
var processErr error
|
||||||
|
switch job.Task.TaskType {
|
||||||
|
case "render": // this task has a upload outputs step because the frames are not uploaded by the render task directly we have to do it manually here TODO: maybe we should make it work like the encode task
|
||||||
|
// Download context
|
||||||
|
contextPath := job.JobPath + "/context.tar"
|
||||||
|
if err := r.downloadContext(job.Task.JobID, contextPath, job.JobToken); err != nil {
|
||||||
|
jobConn.Log(job.Task.TaskID, types.LogLevelError, fmt.Sprintf("Failed to download context: %v", err))
|
||||||
|
jobConn.Complete(job.Task.TaskID, false, fmt.Errorf("failed to download context: %v", err), false)
|
||||||
|
return fmt.Errorf("failed to download context: %w", err)
|
||||||
|
}
|
||||||
|
processErr = processor.Process(ctx)
|
||||||
|
if processErr == nil {
|
||||||
|
processErr = r.uploadOutputs(ctx, job)
|
||||||
|
}
|
||||||
|
case "encode": // this task doesn't have a upload outputs step because the video is already uploaded by the encode task
|
||||||
|
processErr = processor.Process(ctx)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("unknown task type: %s", job.Task.TaskType)
|
||||||
|
}
|
||||||
|
|
||||||
|
if processErr != nil {
|
||||||
|
if errors.Is(processErr, tasks.ErrJobCancelled) {
|
||||||
|
ctx.Warn("Stopping task early because the job was cancelled")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ctx.Error(fmt.Sprintf("Task failed: %v", processErr))
|
||||||
|
ctx.Complete(false, processErr)
|
||||||
|
return processErr
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.Complete(true, nil)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) downloadContext(jobID int64, contextPath, jobToken string) error {
|
||||||
|
reader, err := r.manager.DownloadContext(contextPath, jobToken)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
jobDir := r.workspace.JobDir(jobID)
|
||||||
|
return workspace.ExtractTar(reader, jobDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) uploadOutputs(ctx *tasks.Context, job *api.NextJobResponse) error {
|
||||||
|
outputDir := ctx.WorkDir + "/output"
|
||||||
|
uploadPath := fmt.Sprintf("/api/runner/jobs/%d/upload", job.Task.JobID)
|
||||||
|
|
||||||
|
return uploadOutputFiles(outputDir, func(filePath, fileName string) error {
|
||||||
|
if err := r.manager.UploadFile(uploadPath, job.JobToken, filePath); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx.OutputUploaded(fileName)
|
||||||
|
// Delete file after successful upload to prevent duplicate uploads
|
||||||
|
if err := os.Remove(filePath); err != nil {
|
||||||
|
log.Printf("Warning: Failed to delete file %s after upload: %v", filePath, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadOutputFiles uploads all non-directory files from outputDir.
|
||||||
|
// Fails if the directory cannot be read, any upload fails, or zero files were uploaded.
|
||||||
|
func uploadOutputFiles(outputDir string, uploadFn func(filePath, fileName string) error) error {
|
||||||
|
entries, err := os.ReadDir(outputDir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to read output directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var files []os.DirEntry
|
||||||
|
for _, entry := range entries {
|
||||||
|
if !entry.IsDir() {
|
||||||
|
files = append(files, entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(files) == 0 {
|
||||||
|
return fmt.Errorf("no output files found in %s", outputDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
var firstErr error
|
||||||
|
uploaded := 0
|
||||||
|
for _, entry := range files {
|
||||||
|
filePath := filepath.Join(outputDir, entry.Name())
|
||||||
|
if err := uploadFn(filePath, entry.Name()); err != nil {
|
||||||
|
log.Printf("Failed to upload %s: %v", filePath, err)
|
||||||
|
if firstErr == nil {
|
||||||
|
firstErr = fmt.Errorf("failed to upload %s: %w", entry.Name(), err)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
uploaded++
|
||||||
|
}
|
||||||
|
if firstErr != nil {
|
||||||
|
return firstErr
|
||||||
|
}
|
||||||
|
if uploaded == 0 {
|
||||||
|
return fmt.Errorf("no output files were uploaded from %s", outputDir)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// generateFingerprint creates a unique hardware fingerprint.
|
||||||
|
func (r *Runner) generateFingerprint() {
|
||||||
|
r.fingerprintMu.Lock()
|
||||||
|
defer r.fingerprintMu.Unlock()
|
||||||
|
|
||||||
|
var components []string
|
||||||
|
components = append(components, r.hostname)
|
||||||
|
|
||||||
|
if machineID, err := os.ReadFile("/etc/machine-id"); err == nil {
|
||||||
|
components = append(components, strings.TrimSpace(string(machineID)))
|
||||||
|
}
|
||||||
|
|
||||||
|
if productUUID, err := os.ReadFile("/sys/class/dmi/id/product_uuid"); err == nil {
|
||||||
|
components = append(components, strings.TrimSpace(string(productUUID)))
|
||||||
|
}
|
||||||
|
|
||||||
|
if macAddr, err := r.getMACAddress(); err == nil {
|
||||||
|
components = append(components, macAddr)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(components) <= 1 {
|
||||||
|
components = append(components, fmt.Sprintf("%d", os.Getpid()))
|
||||||
|
components = append(components, fmt.Sprintf("%d", time.Now().Unix()))
|
||||||
|
}
|
||||||
|
|
||||||
|
h := sha256.New()
|
||||||
|
for _, comp := range components {
|
||||||
|
h.Write([]byte(comp))
|
||||||
|
h.Write([]byte{0})
|
||||||
|
}
|
||||||
|
|
||||||
|
r.fingerprint = hex.EncodeToString(h.Sum(nil))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) getMACAddress() (string, error) {
|
||||||
|
interfaces, err := net.Interfaces()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, iface := range interfaces {
|
||||||
|
if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(iface.HardwareAddr) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return iface.HardwareAddr.String(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return "", fmt.Errorf("no suitable network interface found")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetFingerprint returns the runner's hardware fingerprint.
|
||||||
|
func (r *Runner) GetFingerprint() string {
|
||||||
|
r.fingerprintMu.RLock()
|
||||||
|
defer r.fingerprintMu.RUnlock()
|
||||||
|
return r.fingerprint
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetID returns the runner ID.
|
||||||
|
func (r *Runner) GetID() int64 {
|
||||||
|
return r.id
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetGPULockedOut sets whether GPU use is locked out due to a detected GPU error.
|
||||||
|
// When true, the runner will force CPU rendering for all jobs.
|
||||||
|
// Returns true only on the false→true transition (first arm); subsequent calls are no-ops for logging.
|
||||||
|
func (r *Runner) SetGPULockedOut(locked bool) (newlyEnabled bool) {
|
||||||
|
r.gpuLockedOutMu.Lock()
|
||||||
|
defer r.gpuLockedOutMu.Unlock()
|
||||||
|
if locked {
|
||||||
|
if r.gpuLockedOut {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
r.gpuLockedOut = true
|
||||||
|
log.Printf("GPU lockout enabled: GPU rendering disabled for subsequent jobs (CPU only)")
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
r.gpuLockedOut = false
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsGPULockedOut returns whether GPU use is currently locked out.
|
||||||
|
func (r *Runner) IsGPULockedOut() bool {
|
||||||
|
r.gpuLockedOutMu.RLock()
|
||||||
|
defer r.gpuLockedOutMu.RUnlock()
|
||||||
|
return r.gpuLockedOut
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
package runner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewRunner_InitializesFields(t *testing.T) {
|
||||||
|
r := New("http://localhost:8080", "runner-a", "host-a", false, false, 0)
|
||||||
|
if r == nil {
|
||||||
|
t.Fatal("New should return a runner")
|
||||||
|
}
|
||||||
|
if r.name != "runner-a" || r.hostname != "host-a" {
|
||||||
|
t.Fatalf("unexpected runner identity: %q %q", r.name, r.hostname)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunner_GPUFlagsSetters(t *testing.T) {
|
||||||
|
r := New("http://localhost:8080", "runner-a", "host-a", false, false, 0)
|
||||||
|
if newly := r.SetGPULockedOut(true); !newly {
|
||||||
|
t.Fatal("expected first SetGPULockedOut(true) to report newly enabled")
|
||||||
|
}
|
||||||
|
if !r.IsGPULockedOut() {
|
||||||
|
t.Fatal("expected GPU lockout to be true")
|
||||||
|
}
|
||||||
|
if newly := r.SetGPULockedOut(true); newly {
|
||||||
|
t.Fatal("expected second SetGPULockedOut(true) to be a no-op transition")
|
||||||
|
}
|
||||||
|
if newly := r.SetGPULockedOut(false); newly {
|
||||||
|
t.Fatal("clearing lockout should not report newly enabled")
|
||||||
|
}
|
||||||
|
if r.IsGPULockedOut() {
|
||||||
|
t.Fatal("expected GPU lockout cleared")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGenerateFingerprint_PopulatesValue(t *testing.T) {
|
||||||
|
r := New("http://localhost:8080", "runner-a", "host-a", false, false, 0)
|
||||||
|
r.generateFingerprint()
|
||||||
|
fp := r.GetFingerprint()
|
||||||
|
if fp == "" {
|
||||||
|
t.Fatal("fingerprint should not be empty")
|
||||||
|
}
|
||||||
|
if len(fp) != 64 {
|
||||||
|
t.Fatalf("fingerprint should be sha256 hex, got %q", fp)
|
||||||
|
}
|
||||||
|
if _, err := hex.DecodeString(fp); err != nil {
|
||||||
|
t.Fatalf("fingerprint should be valid hex: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadOutputFiles_FailsWhenUploadErrors(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "frame_0001.exr"), []byte("x"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
err := uploadOutputFiles(dir, func(filePath, fileName string) error {
|
||||||
|
return errors.New("upload denied")
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error when upload fails")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadOutputFiles_FailsWhenEmpty(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
err := uploadOutputFiles(dir, func(filePath, fileName string) error {
|
||||||
|
t.Fatal("upload should not be called")
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error for empty output dir")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUploadOutputFiles_Success(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, "frame_0001.exr"), []byte("x"), 0644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var seen string
|
||||||
|
err := uploadOutputFiles(dir, func(filePath, fileName string) error {
|
||||||
|
seen = fileName
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("uploadOutputFiles: %v", err)
|
||||||
|
}
|
||||||
|
if seen != "frame_0001.exr" {
|
||||||
|
t.Fatalf("got %q", seen)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,173 @@
|
|||||||
|
package sandbox
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Bind describes a host path to expose inside the sandbox.
|
||||||
|
type Bind struct {
|
||||||
|
Host string
|
||||||
|
// Dest is the path inside the sandbox (usually same as Host for absolute paths).
|
||||||
|
Dest string
|
||||||
|
// ReadOnly is true for library trees and Blender installs.
|
||||||
|
ReadOnly bool
|
||||||
|
// Dev is true for device nodes / device trees (podman --device or -v for dirs).
|
||||||
|
Dev bool
|
||||||
|
// Optional means skip if host path missing (no error).
|
||||||
|
Optional bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// GPUMounts returns host paths to expose for Cycles GPU backends based on detection.
|
||||||
|
// forceCPU skips GPU-specific devices/libs (still may include generic /dev via backend).
|
||||||
|
func GPUMounts(hasAMD, hasNVIDIA, hasIntel, forceCPU bool) []Bind {
|
||||||
|
if forceCPU {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var binds []Bind
|
||||||
|
|
||||||
|
// DRM render nodes used by AMD, Intel, and some NVIDIA EGL paths.
|
||||||
|
if hasAMD || hasNVIDIA || hasIntel {
|
||||||
|
binds = append(binds, Bind{Host: "/dev/dri", Dest: "/dev/dri", Dev: true, Optional: true})
|
||||||
|
}
|
||||||
|
|
||||||
|
if hasAMD {
|
||||||
|
binds = append(binds, Bind{Host: "/dev/kfd", Dest: "/dev/kfd", Dev: true, Optional: true})
|
||||||
|
// Common ROCm install layouts
|
||||||
|
for _, p := range []string{"/opt/rocm", "/usr/share/libdrm"} {
|
||||||
|
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||||
|
}
|
||||||
|
// Distro ROCm / amdgpu userspace libs often live under multiarch paths
|
||||||
|
for _, p := range rocmLibHints() {
|
||||||
|
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if hasNVIDIA {
|
||||||
|
for _, name := range nvidiaDeviceNames() {
|
||||||
|
p := filepath.Join("/dev", name)
|
||||||
|
binds = append(binds, Bind{Host: p, Dest: p, Dev: true, Optional: true})
|
||||||
|
}
|
||||||
|
for _, p := range nvidiaLibHints() {
|
||||||
|
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if hasIntel {
|
||||||
|
for _, p := range intelLibHints() {
|
||||||
|
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return uniqueBinds(binds)
|
||||||
|
}
|
||||||
|
|
||||||
|
func nvidiaDeviceNames() []string {
|
||||||
|
// Fixed control nodes + any /dev/nvidiaN
|
||||||
|
names := []string{"nvidiactl", "nvidia-uvm", "nvidia-uvm-tools", "nvidia-modeset"}
|
||||||
|
entries, err := os.ReadDir("/dev")
|
||||||
|
if err != nil {
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
for _, e := range entries {
|
||||||
|
n := e.Name()
|
||||||
|
if strings.HasPrefix(n, "nvidia") && !containsString(names, n) {
|
||||||
|
names = append(names, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
func nvidiaLibHints() []string {
|
||||||
|
hints := []string{
|
||||||
|
"/usr/lib/wsl/lib", // WSL NVIDIA
|
||||||
|
"/usr/local/nvidia",
|
||||||
|
"/usr/local/cuda",
|
||||||
|
"/usr/lib/x86_64-linux-gnu",
|
||||||
|
"/usr/lib64",
|
||||||
|
}
|
||||||
|
// Driver stores often under /usr/lib/libcuda* — parent dirs already covered.
|
||||||
|
// Also scan common multiarch for libcuda.so
|
||||||
|
for _, dir := range []string{"/usr/lib", "/usr/lib64", "/usr/lib/x86_64-linux-gnu", "/lib", "/lib64"} {
|
||||||
|
if matchesAny(dir, "libcuda.so*") || matchesAny(dir, "libnvidia-*.so*") {
|
||||||
|
hints = append(hints, dir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hints
|
||||||
|
}
|
||||||
|
|
||||||
|
func rocmLibHints() []string {
|
||||||
|
hints := []string{
|
||||||
|
"/usr/lib/x86_64-linux-gnu",
|
||||||
|
"/usr/lib64",
|
||||||
|
"/opt/amdgpu",
|
||||||
|
}
|
||||||
|
// ROCm versioned trees under /opt/rocm-*
|
||||||
|
if entries, err := os.ReadDir("/opt"); err == nil {
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() && strings.HasPrefix(e.Name(), "rocm") {
|
||||||
|
hints = append(hints, filepath.Join("/opt", e.Name()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hints
|
||||||
|
}
|
||||||
|
|
||||||
|
func intelLibHints() []string {
|
||||||
|
return []string{
|
||||||
|
"/usr/lib/x86_64-linux-gnu",
|
||||||
|
"/usr/lib64",
|
||||||
|
"/usr/lib/intel-opencl",
|
||||||
|
"/etc/OpenCL",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func matchesAny(dir, glob string) bool {
|
||||||
|
m, err := filepath.Glob(filepath.Join(dir, glob))
|
||||||
|
return err == nil && len(m) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsString(ss []string, s string) bool {
|
||||||
|
for _, x := range ss {
|
||||||
|
if x == s {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func uniqueBinds(in []Bind) []Bind {
|
||||||
|
seen := make(map[string]bool)
|
||||||
|
var out []Bind
|
||||||
|
for _, b := range in {
|
||||||
|
key := b.Host + "|" + b.Dest + "|"
|
||||||
|
if b.Dev {
|
||||||
|
key += "d"
|
||||||
|
}
|
||||||
|
if b.ReadOnly {
|
||||||
|
key += "r"
|
||||||
|
}
|
||||||
|
if seen[key] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[key] = true
|
||||||
|
out = append(out, b)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExistingBinds filters to binds whose host path exists (or non-optional always kept for error reporting).
|
||||||
|
func ExistingBinds(binds []Bind) []Bind {
|
||||||
|
var out []Bind
|
||||||
|
for _, b := range binds {
|
||||||
|
if pathExists(b.Host) {
|
||||||
|
out = append(out, b)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !b.Optional {
|
||||||
|
out = append(out, b) // caller may error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
package sandbox
|
||||||
|
|
||||||
|
import "os/exec"
|
||||||
|
|
||||||
|
type noneWrapper struct{}
|
||||||
|
|
||||||
|
func (w *noneWrapper) Name() string { return BackendNone }
|
||||||
|
|
||||||
|
func (w *noneWrapper) Available() error { return nil }
|
||||||
|
|
||||||
|
func (w *noneWrapper) Wrap(spec Spec) (*exec.Cmd, error) {
|
||||||
|
cmd := exec.Command(spec.BlenderBinary, spec.Args...)
|
||||||
|
cmd.Dir = spec.WorkDir
|
||||||
|
if len(spec.Env) > 0 {
|
||||||
|
cmd.Env = spec.Env
|
||||||
|
}
|
||||||
|
return cmd, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
package sandbox
|
||||||
|
|
||||||
|
import "path/filepath"
|
||||||
|
|
||||||
|
func execAbs(p string) (string, error) {
|
||||||
|
return filepath.Abs(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// blenderRoot returns the directory that contains the blender binary
|
||||||
|
// (the version extract root, e.g. .../blender-versions/4.5.7).
|
||||||
|
func blenderRoot(blenderBinary string) string {
|
||||||
|
return filepath.Dir(mustAbs(blenderBinary))
|
||||||
|
}
|
||||||
@@ -0,0 +1,121 @@
|
|||||||
|
package sandbox
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const defaultPodmanImage = "registry.fedoraproject.org/fedora-minimal:41"
|
||||||
|
|
||||||
|
type podmanWrapper struct {
|
||||||
|
allowNetwork bool
|
||||||
|
image string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *podmanWrapper) Name() string { return BackendPodman }
|
||||||
|
|
||||||
|
func (w *podmanWrapper) Available() error {
|
||||||
|
if _, err := LookPath("podman"); err != nil {
|
||||||
|
return fmt.Errorf("podman not found in PATH: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *podmanWrapper) Wrap(spec Spec) (*exec.Cmd, error) {
|
||||||
|
if err := w.Available(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
bin := mustAbs(spec.BlenderBinary)
|
||||||
|
work := mustAbs(spec.WorkDir)
|
||||||
|
home := mustAbs(spec.HomeDir)
|
||||||
|
if home == "" {
|
||||||
|
home = filepath.Join(work, "home")
|
||||||
|
}
|
||||||
|
root := blenderRoot(bin)
|
||||||
|
|
||||||
|
args := []string{
|
||||||
|
"run", "--rm",
|
||||||
|
"--security-opt", "label=disable",
|
||||||
|
// Keep user groups for /dev/dri access (render/video)
|
||||||
|
"--group-add", "keep-groups",
|
||||||
|
}
|
||||||
|
if !w.allowNetwork {
|
||||||
|
args = append(args, "--network=none")
|
||||||
|
}
|
||||||
|
// Map to same UID so bind mounts are writable
|
||||||
|
uid := os.Getuid()
|
||||||
|
gid := os.Getgid()
|
||||||
|
args = append(args, "--user", fmt.Sprintf("%d:%d", uid, gid))
|
||||||
|
|
||||||
|
// Thin OS image + host Blender tree + job dir
|
||||||
|
args = append(args,
|
||||||
|
"-v", root+":"+root+":ro",
|
||||||
|
"-v", work+":"+work+":rw",
|
||||||
|
)
|
||||||
|
if home != work && !strings.HasPrefix(home, work+string(os.PathSeparator)) {
|
||||||
|
_ = os.MkdirAll(home, 0755)
|
||||||
|
args = append(args, "-v", home+":"+home+":rw")
|
||||||
|
}
|
||||||
|
|
||||||
|
// System libs for GPU ICDs / dynamic linker (Blender tarball is mostly self-contained)
|
||||||
|
for _, p := range []string{"/usr", "/lib", "/lib64", "/etc"} {
|
||||||
|
if pathExists(p) {
|
||||||
|
args = append(args, "-v", p+":"+p+":ro")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GPU devices and extra lib roots
|
||||||
|
for _, b := range ExistingBinds(GPUMounts(spec.HasAMD, spec.HasNVIDIA, spec.HasIntel, spec.ForceCPU)) {
|
||||||
|
if !pathExists(b.Host) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if b.Dev {
|
||||||
|
// --device works for character devices; for /dev/dri use volume
|
||||||
|
info, err := os.Stat(b.Host)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if info.IsDir() {
|
||||||
|
args = append(args, "-v", b.Host+":"+b.Dest+":ro")
|
||||||
|
} else {
|
||||||
|
args = append(args, "--device", b.Host)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
mode := "ro"
|
||||||
|
if !b.ReadOnly {
|
||||||
|
mode = "rw"
|
||||||
|
}
|
||||||
|
args = append(args, "-v", b.Host+":"+b.Dest+":"+mode)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Environment
|
||||||
|
for _, e := range spec.Env {
|
||||||
|
args = append(args, "-e", e)
|
||||||
|
}
|
||||||
|
if home != "" {
|
||||||
|
args = append(args, "-e", "HOME="+home)
|
||||||
|
}
|
||||||
|
|
||||||
|
args = append(args, "--workdir", work)
|
||||||
|
args = append(args, "--entrypoint", bin)
|
||||||
|
args = append(args, w.image)
|
||||||
|
// entrypoint is blender; remaining args are blender args
|
||||||
|
args = append(args, spec.Args...)
|
||||||
|
|
||||||
|
cmd := exec.Command("podman", args...)
|
||||||
|
cmd.Dir = work
|
||||||
|
return cmd, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PodmanImageDefault returns the default thin runtime image name.
|
||||||
|
func PodmanImageDefault() string { return defaultPodmanImage }
|
||||||
|
|
||||||
|
// FormatUIDGID is a small helper for tests.
|
||||||
|
func FormatUIDGID(uid, gid int) string {
|
||||||
|
return strconv.Itoa(uid) + ":" + strconv.Itoa(gid)
|
||||||
|
}
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
// Package sandbox wraps Blender (and similar) invocations in optional isolation.
|
||||||
|
//
|
||||||
|
// Backends:
|
||||||
|
// - none: run on the host (current behavior)
|
||||||
|
// - podman: rootless podman container; Blender tarball is bind-mounted (not baked into an image)
|
||||||
|
//
|
||||||
|
// GPU access is provided by passing host device nodes and common userspace lib roots
|
||||||
|
// discovered at wrap time (NVIDIA / AMD ROCm / Intel DRM), not by shipping per-version
|
||||||
|
// Blender container images.
|
||||||
|
package sandbox
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Backend names accepted by New and CLI flags.
|
||||||
|
const (
|
||||||
|
BackendNone = "none"
|
||||||
|
BackendPodman = "podman"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Options configures sandbox construction.
|
||||||
|
type Options struct {
|
||||||
|
// Backend is none|podman (default none).
|
||||||
|
Backend string
|
||||||
|
// AllowNetwork keeps network in the sandbox (default false for podman).
|
||||||
|
AllowNetwork bool
|
||||||
|
// PodmanImage is used only for backend=podman. The image is a thin OS root;
|
||||||
|
// Blender comes from a host bind of the versioned tarball tree.
|
||||||
|
// Default: registry.fedoraproject.org/fedora-minimal:41
|
||||||
|
PodmanImage string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Spec describes a Blender process to run under the sandbox.
|
||||||
|
type Spec struct {
|
||||||
|
// BlenderBinary is an absolute path to the blender executable.
|
||||||
|
BlenderBinary string
|
||||||
|
// Args are arguments after the binary (not including argv0).
|
||||||
|
Args []string
|
||||||
|
// WorkDir is the process working directory (job workspace); bind-mounted RW.
|
||||||
|
WorkDir string
|
||||||
|
// HomeDir is Blender HOME (usually WorkDir/home); bind-mounted RW.
|
||||||
|
HomeDir string
|
||||||
|
// Env is the full environment for the process (HOME, LD_LIBRARY_PATH, etc.).
|
||||||
|
Env []string
|
||||||
|
|
||||||
|
// GPU hints from host detection (used to select device/lib binds).
|
||||||
|
HasAMD bool
|
||||||
|
HasNVIDIA bool
|
||||||
|
HasIntel bool
|
||||||
|
// ForceCPU skips GPU device binds when true.
|
||||||
|
ForceCPU bool
|
||||||
|
// AllowNetwork overrides Options.AllowNetwork when non-nil... kept simple: use Options only.
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wrapper turns a Spec into an *exec.Cmd ready to Start.
|
||||||
|
type Wrapper interface {
|
||||||
|
// Name returns the backend name.
|
||||||
|
Name() string
|
||||||
|
// Available reports whether required host tools exist.
|
||||||
|
Available() error
|
||||||
|
// Wrap builds the command. Callers own Start/Wait/pipes.
|
||||||
|
Wrap(spec Spec) (*exec.Cmd, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// New returns a sandbox Wrapper for the given options.
|
||||||
|
func New(opts Options) (Wrapper, error) {
|
||||||
|
backend := strings.ToLower(strings.TrimSpace(opts.Backend))
|
||||||
|
if backend == "" {
|
||||||
|
backend = BackendPodman
|
||||||
|
}
|
||||||
|
switch backend {
|
||||||
|
case BackendNone:
|
||||||
|
return &noneWrapper{}, nil
|
||||||
|
case BackendPodman:
|
||||||
|
img := opts.PodmanImage
|
||||||
|
if img == "" {
|
||||||
|
img = defaultPodmanImage
|
||||||
|
}
|
||||||
|
w := &podmanWrapper{allowNetwork: opts.AllowNetwork, image: img}
|
||||||
|
return w, nil
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unknown sandbox backend %q (want none or podman)", opts.Backend)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NormalizeBackend validates and returns a canonical backend name.
|
||||||
|
func NormalizeBackend(s string) (string, error) {
|
||||||
|
b := strings.ToLower(strings.TrimSpace(s))
|
||||||
|
if b == "" {
|
||||||
|
return BackendPodman, nil
|
||||||
|
}
|
||||||
|
switch b {
|
||||||
|
case BackendNone, BackendPodman:
|
||||||
|
return b, nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("unknown sandbox backend %q (want none or podman)", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// LookPath is os/exec.LookPath, overridable in tests.
|
||||||
|
var LookPath = exec.LookPath
|
||||||
|
|
||||||
|
// pathExists reports whether path exists.
|
||||||
|
func pathExists(p string) bool {
|
||||||
|
_, err := os.Stat(p)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// mustAbs returns an absolute path or the original on error.
|
||||||
|
func mustAbs(p string) string {
|
||||||
|
if p == "" {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
abs, err := absPath(p)
|
||||||
|
if err != nil {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
return abs
|
||||||
|
}
|
||||||
|
|
||||||
|
// absPath is filepath.Abs, isolated for tests.
|
||||||
|
var absPath = func(p string) (string, error) {
|
||||||
|
return execAbs(p)
|
||||||
|
}
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
package sandbox
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizeBackend(t *testing.T) {
|
||||||
|
got, err := NormalizeBackend("PODMAN")
|
||||||
|
if err != nil || got != BackendPodman {
|
||||||
|
t.Fatalf("got %q err %v", got, err)
|
||||||
|
}
|
||||||
|
if _, err := NormalizeBackend("bwrap"); err == nil {
|
||||||
|
t.Fatal("expected error for removed bwrap backend")
|
||||||
|
}
|
||||||
|
if _, err := NormalizeBackend("firecracker"); err == nil {
|
||||||
|
t.Fatal("expected error for unknown backend")
|
||||||
|
}
|
||||||
|
got, err = NormalizeBackend("")
|
||||||
|
if err != nil || got != BackendPodman {
|
||||||
|
t.Fatalf("empty -> podman, got %q %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNoneWrap_Passthrough(t *testing.T) {
|
||||||
|
w, err := New(Options{Backend: BackendNone})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
bin := "/opt/blender/blender"
|
||||||
|
cmd, err := w.Wrap(Spec{
|
||||||
|
BlenderBinary: bin,
|
||||||
|
Args: []string{"-b", "scene.blend"},
|
||||||
|
WorkDir: "/tmp/job",
|
||||||
|
Env: []string{"HOME=/tmp/job/home"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(cmd.Args) == 0 || cmd.Args[0] != bin {
|
||||||
|
t.Fatalf("args[0]=%v path=%q", cmd.Args, cmd.Path)
|
||||||
|
}
|
||||||
|
if cmd.Dir != "/tmp/job" {
|
||||||
|
t.Fatalf("Dir=%q", cmd.Dir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGPUMounts_ForceCPUEmpty(t *testing.T) {
|
||||||
|
m := GPUMounts(true, true, true, true)
|
||||||
|
if len(m) != 0 {
|
||||||
|
t.Fatalf("force CPU should skip GPU mounts, got %#v", m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGPUMounts_AMDIncludesKFD(t *testing.T) {
|
||||||
|
m := GPUMounts(true, false, false, false)
|
||||||
|
var hosts []string
|
||||||
|
for _, b := range m {
|
||||||
|
hosts = append(hosts, b.Host)
|
||||||
|
}
|
||||||
|
joined := strings.Join(hosts, ",")
|
||||||
|
if !strings.Contains(joined, "/dev/dri") {
|
||||||
|
t.Fatalf("AMD mounts should include /dev/dri, got %v", hosts)
|
||||||
|
}
|
||||||
|
if !strings.Contains(joined, "/dev/kfd") {
|
||||||
|
t.Fatalf("AMD mounts should include /dev/kfd, got %v", hosts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGPUMounts_NVIDIAIncludesCtl(t *testing.T) {
|
||||||
|
m := GPUMounts(false, true, false, false)
|
||||||
|
found := false
|
||||||
|
for _, b := range m {
|
||||||
|
if strings.Contains(b.Host, "nvidia") {
|
||||||
|
found = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Fatalf("NVIDIA mounts should include nvidia devices, got %#v", m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPodmanWrap_BuildsRunArgs(t *testing.T) {
|
||||||
|
w := &podmanWrapper{allowNetwork: false, image: "example.com/thin:latest"}
|
||||||
|
if err := w.Available(); err != nil {
|
||||||
|
t.Skipf("podman not installed: %v", err)
|
||||||
|
}
|
||||||
|
tmp := t.TempDir()
|
||||||
|
blendDir := filepath.Join(tmp, "bver")
|
||||||
|
_ = os.MkdirAll(blendDir, 0755)
|
||||||
|
bin := filepath.Join(blendDir, "blender")
|
||||||
|
_ = os.WriteFile(bin, []byte("x"), 0755)
|
||||||
|
job := filepath.Join(tmp, "job")
|
||||||
|
_ = os.MkdirAll(job, 0755)
|
||||||
|
|
||||||
|
cmd, err := w.Wrap(Spec{
|
||||||
|
BlenderBinary: bin,
|
||||||
|
Args: []string{"-b", "x.blend"},
|
||||||
|
WorkDir: job,
|
||||||
|
HomeDir: filepath.Join(job, "home"),
|
||||||
|
Env: []string{"HOME=" + filepath.Join(job, "home")},
|
||||||
|
HasNVIDIA: true,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
joined := strings.Join(cmd.Args, " ")
|
||||||
|
if !strings.Contains(joined, "run") || !strings.Contains(joined, "--network=none") {
|
||||||
|
t.Fatalf("expected podman run --network=none: %s", joined)
|
||||||
|
}
|
||||||
|
if !strings.Contains(joined, "example.com/thin:latest") {
|
||||||
|
t.Fatalf("expected image in args: %s", joined)
|
||||||
|
}
|
||||||
|
if !strings.Contains(joined, "--entrypoint") || !strings.Contains(joined, bin) {
|
||||||
|
t.Fatalf("expected entrypoint blender: %s", joined)
|
||||||
|
}
|
||||||
|
if !strings.Contains(joined, blendDir) || !strings.Contains(joined, job) {
|
||||||
|
t.Fatalf("expected volume binds: %s", joined)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNew_UnknownBackend(t *testing.T) {
|
||||||
|
if _, err := New(Options{Backend: "bwrap"}); err == nil {
|
||||||
|
t.Fatal("expected error for bwrap")
|
||||||
|
}
|
||||||
|
if _, err := New(Options{Backend: "gvisor"}); err == nil {
|
||||||
|
t.Fatal("expected error")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,594 @@
|
|||||||
|
package tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"math"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"jiggablend/internal/runner/encoding"
|
||||||
|
)
|
||||||
|
|
||||||
|
// EncodeProcessor handles encode tasks.
|
||||||
|
type EncodeProcessor struct{}
|
||||||
|
|
||||||
|
// NewEncodeProcessor creates a new encode processor.
|
||||||
|
func NewEncodeProcessor() *EncodeProcessor {
|
||||||
|
return &EncodeProcessor{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process executes an encode task.
|
||||||
|
func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||||
|
if err := ctx.CheckCancelled(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.Info(fmt.Sprintf("Starting encode task: job %d", ctx.JobID))
|
||||||
|
log.Printf("Processing encode task %d for job %d", ctx.TaskID, ctx.JobID)
|
||||||
|
|
||||||
|
// Create temporary work directory
|
||||||
|
workDir, err := ctx.Workspace.CreateVideoDir(ctx.JobID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create work directory: %w", err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if err := ctx.Workspace.CleanupVideoDir(ctx.JobID); err != nil {
|
||||||
|
log.Printf("Warning: Failed to cleanup encode work directory: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Get output format and frame rate
|
||||||
|
outputFormat := ctx.GetOutputFormat()
|
||||||
|
if outputFormat == "" {
|
||||||
|
outputFormat = "EXR_264_MP4"
|
||||||
|
}
|
||||||
|
frameRate := ctx.GetFrameRate()
|
||||||
|
|
||||||
|
ctx.Info(fmt.Sprintf("Encode: detected output format '%s'", outputFormat))
|
||||||
|
ctx.Info(fmt.Sprintf("Encode: using frame rate %.2f fps", frameRate))
|
||||||
|
|
||||||
|
// Get job files
|
||||||
|
files, err := ctx.Manager.GetJobFiles(ctx.JobID)
|
||||||
|
if err != nil {
|
||||||
|
ctx.Error(fmt.Sprintf("Failed to get job files: %v", err))
|
||||||
|
return fmt.Errorf("failed to get job files: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.Info(fmt.Sprintf("GetJobFiles returned %d total files for job %d", len(files), ctx.JobID))
|
||||||
|
|
||||||
|
// Log all files for debugging
|
||||||
|
for _, file := range files {
|
||||||
|
ctx.Info(fmt.Sprintf("File: %s (type: %s, size: %d)", file.FileName, file.FileType, file.FileSize))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Encode from EXR frames only
|
||||||
|
fileExt := ".exr"
|
||||||
|
frameFileSet := make(map[string]bool)
|
||||||
|
var frameFilesList []string
|
||||||
|
for _, file := range files {
|
||||||
|
if file.FileType == "output" && strings.HasSuffix(strings.ToLower(file.FileName), fileExt) {
|
||||||
|
if !frameFileSet[file.FileName] {
|
||||||
|
frameFileSet[file.FileName] = true
|
||||||
|
frameFilesList = append(frameFilesList, file.FileName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(frameFilesList) == 0 {
|
||||||
|
// Log why no files matched (deduplicate for error reporting)
|
||||||
|
outputFileSet := make(map[string]bool)
|
||||||
|
frameFilesOtherTypeSet := make(map[string]bool)
|
||||||
|
var outputFiles []string
|
||||||
|
var frameFilesOtherType []string
|
||||||
|
|
||||||
|
for _, file := range files {
|
||||||
|
if file.FileType == "output" {
|
||||||
|
if !outputFileSet[file.FileName] {
|
||||||
|
outputFileSet[file.FileName] = true
|
||||||
|
outputFiles = append(outputFiles, file.FileName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(strings.ToLower(file.FileName), fileExt) {
|
||||||
|
key := fmt.Sprintf("%s (type: %s)", file.FileName, file.FileType)
|
||||||
|
if !frameFilesOtherTypeSet[key] {
|
||||||
|
frameFilesOtherTypeSet[key] = true
|
||||||
|
frameFilesOtherType = append(frameFilesOtherType, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ctx.Error(fmt.Sprintf("no EXR frame files found for encode: found %d total files, %d unique output files, %d unique EXR files (with other types)", len(files), len(outputFiles), len(frameFilesOtherType)))
|
||||||
|
if len(outputFiles) > 0 {
|
||||||
|
ctx.Error(fmt.Sprintf("Output files found: %v", outputFiles))
|
||||||
|
}
|
||||||
|
if len(frameFilesOtherType) > 0 {
|
||||||
|
ctx.Error(fmt.Sprintf("EXR files with wrong type: %v", frameFilesOtherType))
|
||||||
|
}
|
||||||
|
err := fmt.Errorf("no EXR frame files found for encode")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.Info(fmt.Sprintf("Found %d EXR frames for encode", len(frameFilesList)))
|
||||||
|
|
||||||
|
// Download frames with bounded parallelism (8 concurrent downloads)
|
||||||
|
const downloadWorkers = 8
|
||||||
|
ctx.Info(fmt.Sprintf("Downloading %d EXR frames for encode...", len(frameFilesList)))
|
||||||
|
|
||||||
|
type result struct {
|
||||||
|
path string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
results := make([]result, len(frameFilesList))
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
sem := make(chan struct{}, downloadWorkers)
|
||||||
|
for i, fileName := range frameFilesList {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(i int, fileName string) {
|
||||||
|
defer wg.Done()
|
||||||
|
sem <- struct{}{}
|
||||||
|
defer func() { <-sem }()
|
||||||
|
framePath := filepath.Join(workDir, fileName)
|
||||||
|
err := ctx.Manager.DownloadFrame(ctx.JobID, fileName, framePath)
|
||||||
|
if err != nil {
|
||||||
|
ctx.Error(fmt.Sprintf("Failed to download EXR frame %s: %v", fileName, err))
|
||||||
|
log.Printf("Failed to download EXR frame for encode %s: %v", fileName, err)
|
||||||
|
results[i] = result{"", err}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
results[i] = result{framePath, nil}
|
||||||
|
}(i, fileName)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
var frameFiles []string
|
||||||
|
for _, r := range results {
|
||||||
|
if r.err == nil && r.path != "" {
|
||||||
|
frameFiles = append(frameFiles, r.path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := ctx.CheckCancelled(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(frameFiles) == 0 {
|
||||||
|
err := fmt.Errorf("failed to download any EXR frames for encode")
|
||||||
|
ctx.Error(err.Error())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Strings(frameFiles)
|
||||||
|
ctx.Info(fmt.Sprintf("Downloaded %d frames", len(frameFiles)))
|
||||||
|
|
||||||
|
// Check if EXR files have alpha channel (for encode decision)
|
||||||
|
hasAlpha := false
|
||||||
|
{
|
||||||
|
firstFrame := frameFiles[0]
|
||||||
|
hasAlpha = detectAlphaChannel(ctx, firstFrame)
|
||||||
|
if hasAlpha {
|
||||||
|
ctx.Info("Detected alpha channel in EXR files")
|
||||||
|
} else {
|
||||||
|
ctx.Info("No alpha channel detected in EXR files")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate video
|
||||||
|
// Use alpha when source EXR has alpha and codec supports it (AV1 or VP9). H.264 does not support alpha.
|
||||||
|
useAlpha := hasAlpha && (outputFormat == "EXR_AV1_MP4" || outputFormat == "EXR_VP9_WEBM")
|
||||||
|
if hasAlpha && outputFormat == "EXR_264_MP4" {
|
||||||
|
ctx.Warn("Alpha channel detected in EXR but H.264 does not support alpha. Use EXR_AV1_MP4 or EXR_VP9_WEBM to preserve alpha in video.")
|
||||||
|
}
|
||||||
|
if useAlpha {
|
||||||
|
ctx.Info("Alpha channel detected - encoding with alpha (AV1/VP9)")
|
||||||
|
}
|
||||||
|
var outputExt string
|
||||||
|
switch outputFormat {
|
||||||
|
case "EXR_VP9_WEBM":
|
||||||
|
outputExt = "webm"
|
||||||
|
ctx.Info("Encoding WebM video with VP9 codec (alpha, HDR)...")
|
||||||
|
case "EXR_AV1_MP4":
|
||||||
|
outputExt = "mp4"
|
||||||
|
ctx.Info("Encoding MP4 video with AV1 codec (alpha, HDR)...")
|
||||||
|
default:
|
||||||
|
outputExt = "mp4"
|
||||||
|
ctx.Info("Encoding MP4 video with H.264 codec (HDR, HLG)...")
|
||||||
|
}
|
||||||
|
|
||||||
|
outputVideo := filepath.Join(workDir, fmt.Sprintf("output_%d.%s", ctx.JobID, outputExt))
|
||||||
|
|
||||||
|
// Build input pattern
|
||||||
|
firstFrame := frameFiles[0]
|
||||||
|
baseName := filepath.Base(firstFrame)
|
||||||
|
re := regexp.MustCompile(`_(\d+)\.`)
|
||||||
|
var pattern string
|
||||||
|
var startNumber int
|
||||||
|
frameNumStr := re.FindStringSubmatch(baseName)
|
||||||
|
if len(frameNumStr) > 1 {
|
||||||
|
pattern = re.ReplaceAllString(baseName, "_%04d.")
|
||||||
|
fmt.Sscanf(frameNumStr[1], "%d", &startNumber)
|
||||||
|
} else {
|
||||||
|
startNumber = extractFrameNumber(baseName)
|
||||||
|
pattern = strings.Replace(baseName, fmt.Sprintf("%d", startNumber), "%04d", 1)
|
||||||
|
}
|
||||||
|
patternPath := filepath.Join(workDir, pattern)
|
||||||
|
|
||||||
|
// Select encoder and build command (software encoding only)
|
||||||
|
var encoder encoding.Encoder
|
||||||
|
switch outputFormat {
|
||||||
|
case "EXR_AV1_MP4":
|
||||||
|
encoder = ctx.Encoder.SelectAV1()
|
||||||
|
case "EXR_VP9_WEBM":
|
||||||
|
encoder = ctx.Encoder.SelectVP9()
|
||||||
|
default:
|
||||||
|
encoder = ctx.Encoder.SelectH264()
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.Info(fmt.Sprintf("Using encoder: %s (%s)", encoder.Name(), encoder.Codec()))
|
||||||
|
|
||||||
|
// All software encoders use 2-pass for optimal quality
|
||||||
|
ctx.Info("Starting 2-pass encode for optimal quality...")
|
||||||
|
|
||||||
|
// Pass 1
|
||||||
|
ctx.Info("Pass 1/2: Analyzing content for optimal encode...")
|
||||||
|
softEncoder := encoder.(*encoding.SoftwareEncoder)
|
||||||
|
pass1Cmd := softEncoder.BuildPass1Command(&encoding.EncodeConfig{
|
||||||
|
InputPattern: patternPath,
|
||||||
|
OutputPath: outputVideo,
|
||||||
|
StartFrame: startNumber,
|
||||||
|
FrameRate: frameRate,
|
||||||
|
WorkDir: workDir,
|
||||||
|
UseAlpha: useAlpha,
|
||||||
|
TwoPass: true,
|
||||||
|
})
|
||||||
|
if err := pass1Cmd.Run(); err != nil {
|
||||||
|
// Pass 1 is analysis-only (writes to /dev/null). FFmpeg often exits non-zero
|
||||||
|
// on benign codec/option warnings while still producing passlogfile stats.
|
||||||
|
ctx.Warn(fmt.Sprintf("Pass 1 completed (warnings expected): %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pass 2
|
||||||
|
ctx.Info("Pass 2/2: Encoding with optimal quality...")
|
||||||
|
|
||||||
|
config := &encoding.EncodeConfig{
|
||||||
|
InputPattern: patternPath,
|
||||||
|
OutputPath: outputVideo,
|
||||||
|
StartFrame: startNumber,
|
||||||
|
FrameRate: frameRate,
|
||||||
|
WorkDir: workDir,
|
||||||
|
UseAlpha: useAlpha,
|
||||||
|
TwoPass: true, // Software encoding always uses 2-pass for quality
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := encoder.BuildCommand(config)
|
||||||
|
if cmd == nil {
|
||||||
|
return errors.New("failed to build encode command")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set up pipes
|
||||||
|
stdoutPipe, err := cmd.StdoutPipe()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create stdout pipe: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
stderrPipe, err := cmd.StderrPipe()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create stderr pipe: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
return fmt.Errorf("failed to start encode command: %w", err)
|
||||||
|
}
|
||||||
|
stopMonitor := ctx.StartCancellationMonitor(cmd, "encode")
|
||||||
|
defer stopMonitor()
|
||||||
|
|
||||||
|
ctx.Processes.Track(ctx.TaskID, cmd)
|
||||||
|
defer ctx.Processes.Untrack(ctx.TaskID)
|
||||||
|
|
||||||
|
// Stream stdout
|
||||||
|
stdoutDone := make(chan bool)
|
||||||
|
go func() {
|
||||||
|
defer close(stdoutDone)
|
||||||
|
scanner := bufio.NewScanner(stdoutPipe)
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
if line != "" {
|
||||||
|
ctx.Info(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
log.Printf("Error reading encode stdout: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Stream stderr
|
||||||
|
stderrDone := make(chan bool)
|
||||||
|
go func() {
|
||||||
|
defer close(stderrDone)
|
||||||
|
scanner := bufio.NewScanner(stderrPipe)
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
if line != "" {
|
||||||
|
ctx.Warn(line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
log.Printf("Error reading encode stderr: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
err = cmd.Wait()
|
||||||
|
<-stdoutDone
|
||||||
|
<-stderrDone
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
if cancelled, checkErr := ctx.IsJobCancelled(); checkErr == nil && cancelled {
|
||||||
|
return ErrJobCancelled
|
||||||
|
}
|
||||||
|
var errMsg string
|
||||||
|
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||||
|
if exitErr.ExitCode() == 137 {
|
||||||
|
errMsg = "FFmpeg was killed due to excessive memory usage (OOM)"
|
||||||
|
} else {
|
||||||
|
errMsg = fmt.Sprintf("ffmpeg encoding failed: %v", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
errMsg = fmt.Sprintf("ffmpeg encoding failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if sizeErr := checkFFmpegSizeError(errMsg); sizeErr != nil {
|
||||||
|
ctx.Error(sizeErr.Error())
|
||||||
|
return sizeErr
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.Error(errMsg)
|
||||||
|
return errors.New(errMsg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify output
|
||||||
|
if _, err := os.Stat(outputVideo); os.IsNotExist(err) {
|
||||||
|
err := fmt.Errorf("video %s file not created: %s", outputExt, outputVideo)
|
||||||
|
ctx.Error(err.Error())
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean up 2-pass log files
|
||||||
|
os.Remove(filepath.Join(workDir, "ffmpeg2pass-0.log"))
|
||||||
|
os.Remove(filepath.Join(workDir, "ffmpeg2pass-0.log.mbtree"))
|
||||||
|
|
||||||
|
ctx.Info(fmt.Sprintf("%s video encoded successfully", strings.ToUpper(outputExt)))
|
||||||
|
|
||||||
|
// Upload video
|
||||||
|
ctx.Info(fmt.Sprintf("Uploading encoded %s video...", strings.ToUpper(outputExt)))
|
||||||
|
|
||||||
|
uploadPath := fmt.Sprintf("/api/runner/jobs/%d/upload", ctx.JobID)
|
||||||
|
if err := ctx.Manager.UploadFile(uploadPath, ctx.JobToken, outputVideo); err != nil {
|
||||||
|
ctx.Error(fmt.Sprintf("Failed to upload %s: %v", strings.ToUpper(outputExt), err))
|
||||||
|
return fmt.Errorf("failed to upload %s: %w", strings.ToUpper(outputExt), err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.Info(fmt.Sprintf("Successfully uploaded %s: %s", strings.ToUpper(outputExt), filepath.Base(outputVideo)))
|
||||||
|
|
||||||
|
// Delete file after successful upload to prevent duplicate uploads
|
||||||
|
if err := os.Remove(outputVideo); err != nil {
|
||||||
|
log.Printf("Warning: Failed to delete video file %s after upload: %v", outputVideo, err)
|
||||||
|
ctx.Warn(fmt.Sprintf("Warning: Failed to delete video file after upload: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("Successfully generated and uploaded %s for job %d: %s", strings.ToUpper(outputExt), ctx.JobID, filepath.Base(outputVideo))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// detectAlphaChannel checks if an EXR file has an alpha channel using ffprobe
|
||||||
|
func detectAlphaChannel(ctx *Context, filePath string) bool {
|
||||||
|
// Use ffprobe to check pixel format and stream properties
|
||||||
|
// EXR files with alpha will have formats like gbrapf32le (RGBA) vs gbrpf32le (RGB)
|
||||||
|
cmd := execCommand("ffprobe",
|
||||||
|
"-v", "error",
|
||||||
|
"-select_streams", "v:0",
|
||||||
|
"-show_entries", "stream=pix_fmt:stream=codec_name",
|
||||||
|
"-of", "default=noprint_wrappers=1",
|
||||||
|
filePath,
|
||||||
|
)
|
||||||
|
|
||||||
|
output, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
// If ffprobe fails, assume no alpha (conservative approach)
|
||||||
|
ctx.Warn(fmt.Sprintf("Failed to detect alpha channel in %s: %v", filepath.Base(filePath), err))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
outputStr := string(output)
|
||||||
|
// Check pixel format - EXR with alpha typically has 'a' in the format name (e.g., gbrapf32le)
|
||||||
|
// Also check for formats that explicitly indicate alpha
|
||||||
|
hasAlpha := strings.Contains(outputStr, "pix_fmt=gbrap") ||
|
||||||
|
strings.Contains(outputStr, "pix_fmt=rgba") ||
|
||||||
|
strings.Contains(outputStr, "pix_fmt=yuva") ||
|
||||||
|
strings.Contains(outputStr, "pix_fmt=abgr")
|
||||||
|
|
||||||
|
if hasAlpha {
|
||||||
|
ctx.Info(fmt.Sprintf("Detected alpha channel in EXR file: %s", filepath.Base(filePath)))
|
||||||
|
}
|
||||||
|
|
||||||
|
return hasAlpha
|
||||||
|
}
|
||||||
|
|
||||||
|
// detectHDR checks if an EXR file contains HDR content using ffprobe
|
||||||
|
func detectHDR(ctx *Context, filePath string) bool {
|
||||||
|
// First, check if the pixel format supports HDR (32-bit float)
|
||||||
|
cmd := execCommand("ffprobe",
|
||||||
|
"-v", "error",
|
||||||
|
"-select_streams", "v:0",
|
||||||
|
"-show_entries", "stream=pix_fmt",
|
||||||
|
"-of", "default=noprint_wrappers=1:nokey=1",
|
||||||
|
filePath,
|
||||||
|
)
|
||||||
|
|
||||||
|
output, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
// If ffprobe fails, assume no HDR (conservative approach)
|
||||||
|
ctx.Warn(fmt.Sprintf("Failed to detect HDR in %s: %v", filepath.Base(filePath), err))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
pixFmt := strings.TrimSpace(string(output))
|
||||||
|
// EXR files with 32-bit float format (gbrpf32le, gbrapf32le) can contain HDR
|
||||||
|
// Check if it's a 32-bit float format
|
||||||
|
isFloat32 := strings.Contains(pixFmt, "f32") || strings.Contains(pixFmt, "f32le")
|
||||||
|
|
||||||
|
if !isFloat32 {
|
||||||
|
// Not a float format, definitely not HDR
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// For 32-bit float EXR, sample pixels to check if values exceed SDR range (> 1.0)
|
||||||
|
// Use ffmpeg to extract pixel statistics - check max pixel values
|
||||||
|
// This is more efficient than sampling individual pixels
|
||||||
|
cmd = execCommand("ffmpeg",
|
||||||
|
"-v", "error",
|
||||||
|
"-i", filePath,
|
||||||
|
"-vf", "signalstats",
|
||||||
|
"-f", "null",
|
||||||
|
"-",
|
||||||
|
)
|
||||||
|
|
||||||
|
output, err = cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
// If stats extraction fails, try sampling a few pixels directly
|
||||||
|
return detectHDRBySampling(ctx, filePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check output for max pixel values
|
||||||
|
outputStr := string(output)
|
||||||
|
// Look for max values in the signalstats output
|
||||||
|
// If we find values > 1.0, it's HDR
|
||||||
|
if strings.Contains(outputStr, "MAX") {
|
||||||
|
// Try to extract max values from signalstats output
|
||||||
|
// Format is typically like: YMAX:1.234 UMAX:0.567 VMAX:0.890
|
||||||
|
// For EXR (RGB), we need to check R, G, B channels
|
||||||
|
// Since signalstats works on YUV, we'll use a different approach
|
||||||
|
return detectHDRBySampling(ctx, filePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to pixel sampling
|
||||||
|
return detectHDRBySampling(ctx, filePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// detectHDRBySampling samples pixels from multiple regions to detect HDR content
|
||||||
|
func detectHDRBySampling(ctx *Context, filePath string) bool {
|
||||||
|
// Sample multiple 10x10 regions from different parts of the image
|
||||||
|
// This gives us better coverage than a single sample
|
||||||
|
sampleRegions := []string{
|
||||||
|
"crop=10:10:iw/4:ih/4", // Top-left quadrant
|
||||||
|
"crop=10:10:iw*3/4:ih/4", // Top-right quadrant
|
||||||
|
"crop=10:10:iw/4:ih*3/4", // Bottom-left quadrant
|
||||||
|
"crop=10:10:iw*3/4:ih*3/4", // Bottom-right quadrant
|
||||||
|
"crop=10:10:iw/2:ih/2", // Center
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, region := range sampleRegions {
|
||||||
|
cmd := execCommand("ffmpeg",
|
||||||
|
"-v", "error",
|
||||||
|
"-i", filePath,
|
||||||
|
"-vf", fmt.Sprintf("%s,scale=1:1", region),
|
||||||
|
"-f", "rawvideo",
|
||||||
|
"-pix_fmt", "gbrpf32le",
|
||||||
|
"-",
|
||||||
|
)
|
||||||
|
|
||||||
|
output, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
continue // Skip this region if sampling fails
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the float32 values (4 bytes per float, 3 channels RGB)
|
||||||
|
if len(output) >= 12 { // At least 3 floats (RGB) = 12 bytes
|
||||||
|
for i := 0; i < len(output)-11; i += 12 {
|
||||||
|
// Read RGB values (little-endian float32)
|
||||||
|
r := float32FromBytes(output[i : i+4])
|
||||||
|
g := float32FromBytes(output[i+4 : i+8])
|
||||||
|
b := float32FromBytes(output[i+8 : i+12])
|
||||||
|
|
||||||
|
// Check if any channel exceeds 1.0 (SDR range)
|
||||||
|
if r > 1.0 || g > 1.0 || b > 1.0 {
|
||||||
|
maxVal := max(r, max(g, b))
|
||||||
|
ctx.Info(fmt.Sprintf("Detected HDR content in EXR file: %s (max value: %.2f)", filepath.Base(filePath), maxVal))
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we sampled multiple regions and none exceed 1.0, it's likely SDR content
|
||||||
|
// But since it's 32-bit float format, user can still manually enable HDR if needed
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// float32FromBytes converts 4 bytes (little-endian) to float32
|
||||||
|
func float32FromBytes(bytes []byte) float32 {
|
||||||
|
if len(bytes) < 4 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
bits := uint32(bytes[0]) | uint32(bytes[1])<<8 | uint32(bytes[2])<<16 | uint32(bytes[3])<<24
|
||||||
|
return math.Float32frombits(bits)
|
||||||
|
}
|
||||||
|
|
||||||
|
// max returns the maximum of two float32 values
|
||||||
|
func max(a, b float32) float32 {
|
||||||
|
if a > b {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractFrameNumber(filename string) int {
|
||||||
|
parts := strings.Split(filepath.Base(filename), "_")
|
||||||
|
if len(parts) < 2 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
framePart := strings.Split(parts[1], ".")[0]
|
||||||
|
var frameNum int
|
||||||
|
fmt.Sscanf(framePart, "%d", &frameNum)
|
||||||
|
return frameNum
|
||||||
|
}
|
||||||
|
|
||||||
|
func checkFFmpegSizeError(output string) error {
|
||||||
|
outputLower := strings.ToLower(output)
|
||||||
|
|
||||||
|
if strings.Contains(outputLower, "hardware does not support encoding at size") {
|
||||||
|
constraintsMatch := regexp.MustCompile(`constraints:\s*width\s+(\d+)-(\d+)\s+height\s+(\d+)-(\d+)`).FindStringSubmatch(output)
|
||||||
|
if len(constraintsMatch) == 5 {
|
||||||
|
return fmt.Errorf("video frame size is outside hardware encoder limits. Hardware requires: width %s-%s, height %s-%s",
|
||||||
|
constraintsMatch[1], constraintsMatch[2], constraintsMatch[3], constraintsMatch[4])
|
||||||
|
}
|
||||||
|
return fmt.Errorf("video frame size is outside hardware encoder limits")
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(outputLower, "picture size") && strings.Contains(outputLower, "is invalid") {
|
||||||
|
sizeMatch := regexp.MustCompile(`picture size\s+(\d+)x(\d+)`).FindStringSubmatch(output)
|
||||||
|
if len(sizeMatch) == 3 {
|
||||||
|
return fmt.Errorf("invalid video frame size: %sx%s", sizeMatch[1], sizeMatch[2])
|
||||||
|
}
|
||||||
|
return fmt.Errorf("invalid video frame size")
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(outputLower, "error while opening encoder") &&
|
||||||
|
(strings.Contains(outputLower, "width") || strings.Contains(outputLower, "height") || strings.Contains(outputLower, "size")) {
|
||||||
|
sizeMatch := regexp.MustCompile(`at size\s+(\d+)x(\d+)`).FindStringSubmatch(output)
|
||||||
|
if len(sizeMatch) == 3 {
|
||||||
|
return fmt.Errorf("hardware encoder cannot encode frame size %sx%s", sizeMatch[1], sizeMatch[2])
|
||||||
|
}
|
||||||
|
return fmt.Errorf("hardware encoder error: frame size may be invalid")
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(outputLower, "invalid") &&
|
||||||
|
(strings.Contains(outputLower, "width") || strings.Contains(outputLower, "height") || strings.Contains(outputLower, "dimension")) {
|
||||||
|
return fmt.Errorf("invalid frame dimensions detected")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"math"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFloat32FromBytes(t *testing.T) {
|
||||||
|
got := float32FromBytes([]byte{0x00, 0x00, 0x80, 0x3f}) // 1.0 little-endian
|
||||||
|
if got != 1.0 {
|
||||||
|
t.Fatalf("float32FromBytes() = %v, want 1.0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMax(t *testing.T) {
|
||||||
|
if got := max(1, 2); got != 2 {
|
||||||
|
t.Fatalf("max() = %v, want 2", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractFrameNumber(t *testing.T) {
|
||||||
|
if got := extractFrameNumber("render_0042.png"); got != 42 {
|
||||||
|
t.Fatalf("extractFrameNumber() = %d, want 42", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckFFmpegSizeError(t *testing.T) {
|
||||||
|
err := checkFFmpegSizeError("hardware does not support encoding at size ... constraints: width 128-4096 height 128-4096")
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected a size error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectAlphaChannel_UsesExecSeam(t *testing.T) {
|
||||||
|
orig := execCommand
|
||||||
|
execCommand = fakeExecCommand
|
||||||
|
defer func() { execCommand = orig }()
|
||||||
|
|
||||||
|
if !detectAlphaChannel(&Context{}, "/tmp/frame.exr") {
|
||||||
|
t.Fatal("expected alpha channel detection via mocked ffprobe output")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDetectHDR_UsesExecSeam(t *testing.T) {
|
||||||
|
orig := execCommand
|
||||||
|
execCommand = fakeExecCommand
|
||||||
|
defer func() { execCommand = orig }()
|
||||||
|
|
||||||
|
if !detectHDR(&Context{}, "/tmp/frame.exr") {
|
||||||
|
t.Fatal("expected HDR detection via mocked ffmpeg sampling output")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func fakeExecCommand(command string, args ...string) *exec.Cmd {
|
||||||
|
cs := []string{"-test.run=TestExecHelperProcess", "--", command}
|
||||||
|
cs = append(cs, args...)
|
||||||
|
cmd := exec.Command(os.Args[0], cs...)
|
||||||
|
cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=1")
|
||||||
|
return cmd
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExecHelperProcess(t *testing.T) {
|
||||||
|
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
idx := 0
|
||||||
|
for i, a := range os.Args {
|
||||||
|
if a == "--" {
|
||||||
|
idx = i
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if idx == 0 || idx+1 >= len(os.Args) {
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
cmdName := os.Args[idx+1]
|
||||||
|
cmdArgs := os.Args[idx+2:]
|
||||||
|
|
||||||
|
switch cmdName {
|
||||||
|
case "ffprobe":
|
||||||
|
if containsArg(cmdArgs, "stream=pix_fmt:stream=codec_name") {
|
||||||
|
_, _ = os.Stdout.WriteString("pix_fmt=gbrapf32le\ncodec_name=exr\n")
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
_, _ = os.Stdout.WriteString("gbrpf32le\n")
|
||||||
|
os.Exit(0)
|
||||||
|
case "ffmpeg":
|
||||||
|
if containsArg(cmdArgs, "signalstats") {
|
||||||
|
_, _ = os.Stderr.WriteString("signalstats failed")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
if containsArg(cmdArgs, "rawvideo") {
|
||||||
|
buf := make([]byte, 12)
|
||||||
|
binary.LittleEndian.PutUint32(buf[0:4], math.Float32bits(1.5))
|
||||||
|
binary.LittleEndian.PutUint32(buf[4:8], math.Float32bits(0.2))
|
||||||
|
binary.LittleEndian.PutUint32(buf[8:12], math.Float32bits(0.1))
|
||||||
|
_, _ = os.Stdout.Write(buf)
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
os.Exit(0)
|
||||||
|
default:
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsArg(args []string, target string) bool {
|
||||||
|
for _, a := range args {
|
||||||
|
if strings.Contains(a, target) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
package tasks
|
||||||
|
|
||||||
|
import "os/exec"
|
||||||
|
|
||||||
|
// execCommand is a seam for process execution in tests.
|
||||||
|
var execCommand = exec.Command
|
||||||
|
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// mergeEXRFiles averages multiple linear EXR renders into one (Monte Carlo accumulation).
|
||||||
|
func mergeEXRFiles(output string, inputs []string) error {
|
||||||
|
if len(inputs) == 0 {
|
||||||
|
return fmt.Errorf("no input EXR files to merge")
|
||||||
|
}
|
||||||
|
if len(inputs) == 1 {
|
||||||
|
return copyFile(inputs[0], output)
|
||||||
|
}
|
||||||
|
|
||||||
|
args := append([]string{}, inputs...)
|
||||||
|
args = append(args, "-evaluate-sequence", "mean", output)
|
||||||
|
|
||||||
|
if path, err := exec.LookPath("magick"); err == nil {
|
||||||
|
cmd := exec.Command(path, args...)
|
||||||
|
if out, err := cmd.CombinedOutput(); err != nil {
|
||||||
|
return fmt.Errorf("magick merge failed: %w (%s)", err, strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if path, err := exec.LookPath("convert"); err == nil {
|
||||||
|
cmd := exec.Command(path, args...)
|
||||||
|
if out, err := cmd.CombinedOutput(); err != nil {
|
||||||
|
return fmt.Errorf("convert merge failed: %w (%s)", err, strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return fmt.Errorf("ImageMagick (magick or convert) required to merge batched EXR renders")
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyFile(src, dst string) error {
|
||||||
|
cmd := exec.Command("cp", "-f", src, dst)
|
||||||
|
if out, err := cmd.CombinedOutput(); err != nil {
|
||||||
|
return fmt.Errorf("copy %s to %s: %w (%s)", src, dst, err, strings.TrimSpace(string(out)))
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,360 @@
|
|||||||
|
// Package tasks provides task processing implementations.
|
||||||
|
package tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"jiggablend/internal/runner/api"
|
||||||
|
"jiggablend/internal/runner/blender"
|
||||||
|
"jiggablend/internal/runner/encoding"
|
||||||
|
"jiggablend/internal/runner/sandbox"
|
||||||
|
"jiggablend/internal/runner/workspace"
|
||||||
|
"jiggablend/pkg/executils"
|
||||||
|
"jiggablend/pkg/types"
|
||||||
|
"os/exec"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Processor handles a specific task type.
|
||||||
|
type Processor interface {
|
||||||
|
Process(ctx *Context) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Context provides task execution context.
|
||||||
|
type Context struct {
|
||||||
|
TaskID int64
|
||||||
|
JobID int64
|
||||||
|
JobName string
|
||||||
|
Frame int // frame start (inclusive); kept for backward compat
|
||||||
|
FrameEnd int // frame end (inclusive); same as Frame for single-frame
|
||||||
|
TaskType string
|
||||||
|
WorkDir string
|
||||||
|
JobToken string
|
||||||
|
Metadata *types.BlendMetadata
|
||||||
|
|
||||||
|
Manager *api.ManagerClient
|
||||||
|
JobConn *api.JobConnection
|
||||||
|
Workspace *workspace.Manager
|
||||||
|
Blender *blender.Manager
|
||||||
|
Encoder *encoding.Selector
|
||||||
|
Processes *executils.ProcessTracker
|
||||||
|
|
||||||
|
// GPULockedOut is set when the runner has detected a GPU error (e.g. HIP) and disables GPU for all jobs.
|
||||||
|
GPULockedOut bool
|
||||||
|
// GPULockoutArmedThisAttempt is true when this task attempt newly enabled runner GPU lockout.
|
||||||
|
// Failures with this set free-requeue without incrementing retry_count (expected ROCm warmup).
|
||||||
|
GPULockoutArmedThisAttempt bool
|
||||||
|
// HasAMD is true when the runner detected AMD devices at startup.
|
||||||
|
HasAMD bool
|
||||||
|
// HasNVIDIA is true when the runner detected NVIDIA GPUs at startup.
|
||||||
|
HasNVIDIA bool
|
||||||
|
// HasIntel is true when the runner detected Intel GPUs (e.g. Arc) at startup.
|
||||||
|
HasIntel bool
|
||||||
|
// GPUDetectionFailed is true when startup GPU backend detection could not run; we force CPU for all versions (backend availability unknown).
|
||||||
|
GPUDetectionFailed bool
|
||||||
|
// OnGPUError is called when a GPU error line is seen in render logs; typically sets runner GPU lockout.
|
||||||
|
OnGPUError func()
|
||||||
|
// ForceCPURendering is a runner-level override that forces CPU rendering for all jobs.
|
||||||
|
ForceCPURendering bool
|
||||||
|
// DisableRT disables GPU ray tracing acceleration (runner-level flag).
|
||||||
|
DisableRT bool
|
||||||
|
// HipGPUSampleBatch limits samples per GPU render pass on gfx115x (0 = no batching).
|
||||||
|
HipGPUSampleBatch int
|
||||||
|
// Sandbox wraps Blender execution (none/podman). Nil means none.
|
||||||
|
Sandbox sandbox.Wrapper
|
||||||
|
}
|
||||||
|
|
||||||
|
// ErrJobCancelled indicates the manager-side job was cancelled during execution.
|
||||||
|
var ErrJobCancelled = errors.New("job cancelled")
|
||||||
|
|
||||||
|
// NewContext creates a new task context. frameEnd should be >= frame; if 0 or less than frame, it is treated as single-frame (frameEnd = frame).
|
||||||
|
// gpuLockedOut is the runner's current GPU lockout state; gpuDetectionFailed means detection failed at startup (force CPU for all versions); onGPUError is called when a GPU error is detected in logs (may be nil).
|
||||||
|
func NewContext(
|
||||||
|
taskID, jobID int64,
|
||||||
|
jobName string,
|
||||||
|
frameStart, frameEnd int,
|
||||||
|
taskType string,
|
||||||
|
workDir string,
|
||||||
|
jobToken string,
|
||||||
|
metadata *types.BlendMetadata,
|
||||||
|
manager *api.ManagerClient,
|
||||||
|
jobConn *api.JobConnection,
|
||||||
|
ws *workspace.Manager,
|
||||||
|
blenderMgr *blender.Manager,
|
||||||
|
encoder *encoding.Selector,
|
||||||
|
processes *executils.ProcessTracker,
|
||||||
|
gpuLockedOut bool,
|
||||||
|
hasAMD bool,
|
||||||
|
hasNVIDIA bool,
|
||||||
|
hasIntel bool,
|
||||||
|
gpuDetectionFailed bool,
|
||||||
|
forceCPURendering bool,
|
||||||
|
disableRT bool,
|
||||||
|
hipGPUSampleBatch int,
|
||||||
|
onGPUError func(),
|
||||||
|
sb sandbox.Wrapper,
|
||||||
|
) *Context {
|
||||||
|
if frameEnd < frameStart {
|
||||||
|
frameEnd = frameStart
|
||||||
|
}
|
||||||
|
return &Context{
|
||||||
|
TaskID: taskID,
|
||||||
|
JobID: jobID,
|
||||||
|
JobName: jobName,
|
||||||
|
Frame: frameStart,
|
||||||
|
FrameEnd: frameEnd,
|
||||||
|
TaskType: taskType,
|
||||||
|
WorkDir: workDir,
|
||||||
|
JobToken: jobToken,
|
||||||
|
Metadata: metadata,
|
||||||
|
Manager: manager,
|
||||||
|
JobConn: jobConn,
|
||||||
|
Workspace: ws,
|
||||||
|
Blender: blenderMgr,
|
||||||
|
Encoder: encoder,
|
||||||
|
Processes: processes,
|
||||||
|
GPULockedOut: gpuLockedOut,
|
||||||
|
HasAMD: hasAMD,
|
||||||
|
HasNVIDIA: hasNVIDIA,
|
||||||
|
HasIntel: hasIntel,
|
||||||
|
GPUDetectionFailed: gpuDetectionFailed,
|
||||||
|
ForceCPURendering: forceCPURendering,
|
||||||
|
DisableRT: disableRT,
|
||||||
|
HipGPUSampleBatch: hipGPUSampleBatch,
|
||||||
|
OnGPUError: onGPUError,
|
||||||
|
Sandbox: sb,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Log sends a log entry to the manager.
|
||||||
|
func (c *Context) Log(level types.LogLevel, message string) {
|
||||||
|
if c.JobConn != nil {
|
||||||
|
c.JobConn.Log(c.TaskID, level, message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Info logs an info message.
|
||||||
|
func (c *Context) Info(message string) {
|
||||||
|
c.Log(types.LogLevelInfo, message)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warn logs a warning message.
|
||||||
|
func (c *Context) Warn(message string) {
|
||||||
|
c.Log(types.LogLevelWarn, message)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error logs an error message.
|
||||||
|
func (c *Context) Error(message string) {
|
||||||
|
c.Log(types.LogLevelError, message)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Progress sends a progress update.
|
||||||
|
func (c *Context) Progress(progress float64) {
|
||||||
|
if c.JobConn != nil {
|
||||||
|
c.JobConn.Progress(c.TaskID, progress)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// OutputUploaded notifies that an output file was uploaded.
|
||||||
|
func (c *Context) OutputUploaded(fileName string) {
|
||||||
|
if c.JobConn != nil {
|
||||||
|
c.JobConn.OutputUploaded(c.TaskID, fileName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Complete sends task completion.
|
||||||
|
// When the failure newly armed GPU lockout, free_requeue is set so the manager
|
||||||
|
// requeues without burning retry_count.
|
||||||
|
func (c *Context) Complete(success bool, errorMsg error) {
|
||||||
|
if c.JobConn != nil {
|
||||||
|
freeRequeue := !success && c.GPULockoutArmedThisAttempt
|
||||||
|
c.JobConn.Complete(c.TaskID, success, errorMsg, freeRequeue)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetOutputFormat returns the output format from metadata or default.
|
||||||
|
// Product default is EXR (Blender always renders EXR; deliverable may add video).
|
||||||
|
func (c *Context) GetOutputFormat() string {
|
||||||
|
if c.Metadata != nil && c.Metadata.RenderSettings.OutputFormat != "" {
|
||||||
|
return c.Metadata.RenderSettings.OutputFormat
|
||||||
|
}
|
||||||
|
return "EXR"
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetFrameRate returns the frame rate from metadata or default.
|
||||||
|
func (c *Context) GetFrameRate() float64 {
|
||||||
|
if c.Metadata != nil && c.Metadata.RenderSettings.FrameRate > 0 {
|
||||||
|
return c.Metadata.RenderSettings.FrameRate
|
||||||
|
}
|
||||||
|
return 24.0
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBlenderVersion returns the Blender version from metadata.
|
||||||
|
func (c *Context) GetBlenderVersion() string {
|
||||||
|
if c.Metadata != nil {
|
||||||
|
return c.Metadata.BlenderVersion
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShouldUnhideObjects returns whether to unhide objects.
|
||||||
|
func (c *Context) ShouldUnhideObjects() bool {
|
||||||
|
return c.Metadata != nil && c.Metadata.UnhideObjects != nil && *c.Metadata.UnhideObjects
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShouldEnableExecution returns whether to pass Blender --enable-autoexec
|
||||||
|
// (Python drivers/scripts inside the .blend). Job-bundled blender_addons/ install
|
||||||
|
// independently whenever that folder is present in the context.
|
||||||
|
func (c *Context) ShouldEnableExecution() bool {
|
||||||
|
return c.Metadata != nil && c.Metadata.EnableExecution != nil && *c.Metadata.EnableExecution
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShouldForceCPU returns true if GPU should be disabled and CPU rendering forced
|
||||||
|
// (runner GPU lockout, GPU detection failed at startup, or metadata force_cpu).
|
||||||
|
func (c *Context) ShouldForceCPU() bool {
|
||||||
|
if c.ForceCPURendering {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if c.GPULockedOut {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// Detection failed at startup: backend availability unknown, so force CPU for all versions.
|
||||||
|
if c.GPUDetectionFailed {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||||
|
if v, ok := c.Metadata.RenderSettings.EngineSettings["force_cpu"]; ok {
|
||||||
|
if b, ok := v.(bool); ok && b {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetCyclesSamples returns the target Cycles sample count from metadata (default 128).
|
||||||
|
func (c *Context) GetCyclesSamples() int {
|
||||||
|
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||||
|
if v, ok := c.Metadata.RenderSettings.EngineSettings["samples"]; ok {
|
||||||
|
switch n := v.(type) {
|
||||||
|
case float64:
|
||||||
|
if int(n) > 0 {
|
||||||
|
return int(n)
|
||||||
|
}
|
||||||
|
case int:
|
||||||
|
if n > 0 {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 128
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetCyclesSeed returns the Cycles seed from metadata (default 0).
|
||||||
|
func (c *Context) GetCyclesSeed() int {
|
||||||
|
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||||
|
if v, ok := c.Metadata.RenderSettings.EngineSettings["seed"]; ok {
|
||||||
|
switch n := v.(type) {
|
||||||
|
case float64:
|
||||||
|
return int(n)
|
||||||
|
case int:
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShouldBatchGPUSamples returns true when HIP GPU renders should be split into sample batches.
|
||||||
|
func (c *Context) ShouldBatchGPUSamples() bool {
|
||||||
|
if c.ShouldForceCPU() || c.HipGPUSampleBatch <= 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return c.GetCyclesSamples() > c.HipGPUSampleBatch
|
||||||
|
}
|
||||||
|
|
||||||
|
// ShouldDisableRT returns true when GPU ray tracing acceleration should be disabled.
|
||||||
|
func (c *Context) ShouldDisableRT() bool {
|
||||||
|
if c.DisableRT {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||||
|
settings := c.Metadata.RenderSettings.EngineSettings
|
||||||
|
if v, ok := settings["disable_rt"]; ok {
|
||||||
|
if b, ok := v.(bool); ok && b {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v, ok := settings["disable_hiprt"]; ok {
|
||||||
|
if b, ok := v.(bool); ok && b {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsJobCancelled checks whether the manager marked this job as cancelled.
|
||||||
|
func (c *Context) IsJobCancelled() (bool, error) {
|
||||||
|
if c.Manager == nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
status, err := c.Manager.GetJobStatus(c.JobID)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return status == types.JobStatusCancelled, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// CheckCancelled returns ErrJobCancelled if the job was cancelled.
|
||||||
|
func (c *Context) CheckCancelled() error {
|
||||||
|
cancelled, err := c.IsJobCancelled()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to check job status: %w", err)
|
||||||
|
}
|
||||||
|
if cancelled {
|
||||||
|
return ErrJobCancelled
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// StartCancellationMonitor polls manager status and kills cmd if job is cancelled.
|
||||||
|
// Caller must invoke returned stop function when cmd exits.
|
||||||
|
func (c *Context) StartCancellationMonitor(cmd *exec.Cmd, taskLabel string) func() {
|
||||||
|
stop := make(chan struct{})
|
||||||
|
var once sync.Once
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
ticker := time.NewTicker(2 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
cancelled, err := c.IsJobCancelled()
|
||||||
|
if err != nil {
|
||||||
|
c.Warn(fmt.Sprintf("Could not check cancellation for %s task: %v", taskLabel, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !cancelled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c.Warn(fmt.Sprintf("Job %d was cancelled, stopping %s task early", c.JobID, taskLabel))
|
||||||
|
if cmd != nil && cmd.Process != nil {
|
||||||
|
_ = cmd.Process.Kill()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return func() {
|
||||||
|
once.Do(func() {
|
||||||
|
close(stop)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
package tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"jiggablend/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewContext_NormalizesFrameEnd(t *testing.T) {
|
||||||
|
ctx := NewContext(1, 2, "job", 10, 1, "render", "/tmp", "tok", nil, nil, nil, nil, nil, nil, nil, false, false, false, false, false, false, false, 0, nil, nil)
|
||||||
|
if ctx.FrameEnd != 10 {
|
||||||
|
t.Fatalf("expected FrameEnd to be normalized to Frame, got %d", ctx.FrameEnd)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContext_GetOutputFormat_Default(t *testing.T) {
|
||||||
|
ctx := &Context{}
|
||||||
|
if got := ctx.GetOutputFormat(); got != "EXR" {
|
||||||
|
t.Fatalf("GetOutputFormat() = %q, want EXR", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContext_ShouldForceCPU(t *testing.T) {
|
||||||
|
ctx := &Context{ForceCPURendering: true}
|
||||||
|
if !ctx.ShouldForceCPU() {
|
||||||
|
t.Fatal("expected force cpu when runner-level flag is set")
|
||||||
|
}
|
||||||
|
|
||||||
|
force := true
|
||||||
|
ctx = &Context{Metadata: &types.BlendMetadata{RenderSettings: types.RenderSettings{EngineSettings: map[string]interface{}{"force_cpu": force}}}}
|
||||||
|
if !ctx.ShouldForceCPU() {
|
||||||
|
t.Fatal("expected force cpu when metadata requests it")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestErrJobCancelled_IsSentinel(t *testing.T) {
|
||||||
|
if !errors.Is(ErrJobCancelled, ErrJobCancelled) {
|
||||||
|
t.Fatal("sentinel error should be self-identical")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,524 @@
|
|||||||
|
package tasks
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"jiggablend/internal/runner/blender"
|
||||||
|
"jiggablend/internal/runner/sandbox"
|
||||||
|
"jiggablend/internal/runner/workspace"
|
||||||
|
"jiggablend/pkg/scripts"
|
||||||
|
"jiggablend/pkg/types"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RenderProcessor handles render tasks.
|
||||||
|
type RenderProcessor struct{}
|
||||||
|
|
||||||
|
// NewRenderProcessor creates a new render processor.
|
||||||
|
func NewRenderProcessor() *RenderProcessor {
|
||||||
|
return &RenderProcessor{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// gpuErrorSubstrings are log line substrings that indicate a GPU backend error (matched case-insensitively); any match triggers full GPU lockout.
|
||||||
|
var gpuErrorSubstrings = []string{
|
||||||
|
"illegal address in hip", // HIP (AMD) e.g. "Illegal address in HIP" or "Illegal address in hip"
|
||||||
|
"memory access fault", // AMDGPU page fault during HIP/HIPRT (not necessarily OOM)
|
||||||
|
"page not present", // AMDGPU GCVM page fault detail
|
||||||
|
"hiperror", // hipError* codes
|
||||||
|
"hip error",
|
||||||
|
"cuda error",
|
||||||
|
"cuerror",
|
||||||
|
"optix error",
|
||||||
|
"oneapi error",
|
||||||
|
"opencl error",
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkGPUErrorLine checks a log line for GPU error indicators and triggers runner GPU lockout if found.
|
||||||
|
// Once lockout is already active for this attempt (or was active at context creation), further
|
||||||
|
// matching lines are ignored so we do not spam lockout callbacks/logs on multi-line fault dumps.
|
||||||
|
func (p *RenderProcessor) checkGPUErrorLine(ctx *Context, line string) {
|
||||||
|
if ctx.GPULockedOut || ctx.GPULockoutArmedThisAttempt {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
lower := strings.ToLower(line)
|
||||||
|
for _, sub := range gpuErrorSubstrings {
|
||||||
|
if strings.Contains(lower, sub) {
|
||||||
|
if ctx.OnGPUError != nil {
|
||||||
|
ctx.OnGPUError()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process executes a render task.
|
||||||
|
func (p *RenderProcessor) Process(ctx *Context) error {
|
||||||
|
if err := ctx.CheckCancelled(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if ctx.FrameEnd > ctx.Frame {
|
||||||
|
ctx.Info(fmt.Sprintf("Starting task: job %d, frames %d-%d, format: %s",
|
||||||
|
ctx.JobID, ctx.Frame, ctx.FrameEnd, ctx.GetOutputFormat()))
|
||||||
|
log.Printf("Processing task %d: job %d, frames %d-%d", ctx.TaskID, ctx.JobID, ctx.Frame, ctx.FrameEnd)
|
||||||
|
} else {
|
||||||
|
ctx.Info(fmt.Sprintf("Starting task: job %d, frame %d, format: %s",
|
||||||
|
ctx.JobID, ctx.Frame, ctx.GetOutputFormat()))
|
||||||
|
log.Printf("Processing task %d: job %d, frame %d", ctx.TaskID, ctx.JobID, ctx.Frame)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find .blend file
|
||||||
|
blendFile, err := workspace.FindFirstBlendFile(ctx.WorkDir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to find blend file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Runners must use manager-provided Blender versions; never fall back to system blender.
|
||||||
|
version := ctx.GetBlenderVersion()
|
||||||
|
if version == "" {
|
||||||
|
return fmt.Errorf("job metadata missing blender_version: runner cannot use system blender")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.Info(fmt.Sprintf("Job requires Blender %s", version))
|
||||||
|
binaryPath, err := ctx.Blender.GetBinaryPath(version)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to get Blender %s from manager: %w", version, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
blenderBinary, err := blender.ResolveBinaryPath(binaryPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to resolve Blender %s binary: %w", version, err)
|
||||||
|
}
|
||||||
|
ctx.Info(fmt.Sprintf("Using Blender binary: %s", blenderBinary))
|
||||||
|
|
||||||
|
// Create output directory
|
||||||
|
outputDir := filepath.Join(ctx.WorkDir, "output")
|
||||||
|
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||||
|
return fmt.Errorf("failed to create output directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create home directory for Blender inside workspace
|
||||||
|
blenderHome := filepath.Join(ctx.WorkDir, "home")
|
||||||
|
if err := os.MkdirAll(blenderHome, 0755); err != nil {
|
||||||
|
return fmt.Errorf("failed to create Blender home directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// We always render EXR (linear) for VFX accuracy; job output_format is the deliverable (EXR sequence or video).
|
||||||
|
renderFormat := "EXR"
|
||||||
|
|
||||||
|
if ctx.ShouldForceCPU() {
|
||||||
|
if ctx.ForceCPURendering {
|
||||||
|
ctx.Info("Runner compatibility flag is enabled: forcing CPU rendering for this job")
|
||||||
|
} else if ctx.GPUDetectionFailed {
|
||||||
|
ctx.Info("GPU backend detection failed at startup—we could not determine available GPU backends, so rendering will use CPU to avoid compatibility issues")
|
||||||
|
} else {
|
||||||
|
ctx.Info("GPU lockout active: using CPU rendering only")
|
||||||
|
}
|
||||||
|
} else if ctx.ShouldDisableRT() {
|
||||||
|
ctx.Info("GPU ray tracing acceleration disabled for this job (--disable-rt)")
|
||||||
|
}
|
||||||
|
if ctx.ShouldBatchGPUSamples() {
|
||||||
|
total := ctx.GetCyclesSamples()
|
||||||
|
ctx.Info(fmt.Sprintf(
|
||||||
|
"gfx115x HIP sample batching: rendering %d samples in passes of %d (ROCm driver limit, not system RAM)",
|
||||||
|
total, ctx.HipGPUSampleBatch,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render
|
||||||
|
if ctx.FrameEnd > ctx.Frame {
|
||||||
|
ctx.Info(fmt.Sprintf("Starting Blender render for frames %d-%d...", ctx.Frame, ctx.FrameEnd))
|
||||||
|
} else {
|
||||||
|
ctx.Info(fmt.Sprintf("Starting Blender render for frame %d...", ctx.Frame))
|
||||||
|
}
|
||||||
|
if err := p.renderFrames(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome); err != nil {
|
||||||
|
if errors.Is(err, ErrJobCancelled) {
|
||||||
|
ctx.Warn("Render stopped because job was cancelled")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx.Error(fmt.Sprintf("Blender render failed: %v", err))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify output (range or single frame)
|
||||||
|
if err := p.verifyOutputRange(ctx, outputDir, renderFormat); err != nil {
|
||||||
|
ctx.Error(fmt.Sprintf("Output verification failed: %v", err))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if ctx.FrameEnd > ctx.Frame {
|
||||||
|
ctx.Info(fmt.Sprintf("Blender render completed for frames %d-%d", ctx.Frame, ctx.FrameEnd))
|
||||||
|
} else {
|
||||||
|
ctx.Info(fmt.Sprintf("Blender render completed for frame %d", ctx.Frame))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type renderPassOptions struct {
|
||||||
|
samplesOverride *int
|
||||||
|
seedOverride *int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *RenderProcessor) renderFrames(ctx *Context, blenderBinary, blendFile, outputDir, renderFormat, blenderHome string) error {
|
||||||
|
if !ctx.ShouldBatchGPUSamples() {
|
||||||
|
if err := p.createRenderScript(ctx, renderFormat, nil); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return p.runBlender(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome)
|
||||||
|
}
|
||||||
|
|
||||||
|
totalSamples := ctx.GetCyclesSamples()
|
||||||
|
batchSize := ctx.HipGPUSampleBatch
|
||||||
|
baseSeed := ctx.GetCyclesSeed()
|
||||||
|
batchDir := filepath.Join(ctx.WorkDir, "sample_batches")
|
||||||
|
if err := os.MkdirAll(batchDir, 0755); err != nil {
|
||||||
|
return fmt.Errorf("failed to create sample batch directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var batchOutputs []string
|
||||||
|
remaining := totalSamples
|
||||||
|
batchNum := 0
|
||||||
|
for remaining > 0 {
|
||||||
|
if err := ctx.CheckCancelled(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
passSamples := batchSize
|
||||||
|
if remaining < passSamples {
|
||||||
|
passSamples = remaining
|
||||||
|
}
|
||||||
|
seed := baseSeed + batchNum
|
||||||
|
batchNum++
|
||||||
|
ctx.Info(fmt.Sprintf("GPU sample batch %d: %d samples (seed %d, %d remaining)", batchNum, passSamples, seed, remaining-passSamples))
|
||||||
|
|
||||||
|
passOutput := filepath.Join(batchDir, fmt.Sprintf("batch_%03d", batchNum))
|
||||||
|
if err := os.MkdirAll(passOutput, 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
opts := &renderPassOptions{
|
||||||
|
samplesOverride: &passSamples,
|
||||||
|
seedOverride: &seed,
|
||||||
|
}
|
||||||
|
if err := p.createRenderScript(ctx, renderFormat, opts); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := p.runBlender(ctx, blenderBinary, blendFile, passOutput, renderFormat, blenderHome); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := p.verifyOutputRange(ctx, passOutput, renderFormat); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
batchOutputs = append(batchOutputs, p.firstFrameOutputPath(ctx, passOutput, renderFormat))
|
||||||
|
remaining -= passSamples
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
finalOutput := p.firstFrameOutputPath(ctx, outputDir, renderFormat)
|
||||||
|
ctx.Info(fmt.Sprintf("Merging %d GPU sample batches into final EXR...", len(batchOutputs)))
|
||||||
|
if err := mergeEXRFiles(finalOutput, batchOutputs); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ctx.Info("GPU sample batch merge completed")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *RenderProcessor) firstFrameOutputPath(ctx *Context, outputDir, renderFormat string) string {
|
||||||
|
ext := strings.ToLower(renderFormat)
|
||||||
|
if ctx.FrameEnd > ctx.Frame {
|
||||||
|
return filepath.Join(outputDir, fmt.Sprintf("frame_%04d.%s", ctx.Frame, ext))
|
||||||
|
}
|
||||||
|
return filepath.Join(outputDir, fmt.Sprintf("frame_%04d.%s", ctx.Frame, ext))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string, opts *renderPassOptions) error {
|
||||||
|
formatFilePath := filepath.Join(ctx.WorkDir, "output_format.txt")
|
||||||
|
renderSettingsFilePath := filepath.Join(ctx.WorkDir, "render_settings.json")
|
||||||
|
|
||||||
|
// Build unhide code conditionally
|
||||||
|
unhideCode := ""
|
||||||
|
if ctx.ShouldUnhideObjects() {
|
||||||
|
unhideCode = scripts.UnhideObjects
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load template and replace placeholders.
|
||||||
|
// Job-bundled blender_addons/ always auto-install when present (users ship addons with scenes).
|
||||||
|
// enable_execution only controls Blender --enable-autoexec (scripts inside .blend).
|
||||||
|
scriptContent := scripts.RenderBlenderTemplate
|
||||||
|
scriptContent = strings.ReplaceAll(scriptContent, "{{UNHIDE_CODE}}", unhideCode)
|
||||||
|
scriptContent = strings.ReplaceAll(scriptContent, "{{FORMAT_FILE_PATH}}", fmt.Sprintf("%q", formatFilePath))
|
||||||
|
scriptContent = strings.ReplaceAll(scriptContent, "{{RENDER_SETTINGS_FILE}}", fmt.Sprintf("%q", renderSettingsFilePath))
|
||||||
|
|
||||||
|
scriptPath := filepath.Join(ctx.WorkDir, "enable_gpu.py")
|
||||||
|
if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
|
||||||
|
errMsg := fmt.Sprintf("failed to create GPU enable script: %v", err)
|
||||||
|
ctx.Error(errMsg)
|
||||||
|
return errors.New(errMsg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write EXR to format file so Blender script sets OPEN_EXR (job output_format is for downstream deliverable only).
|
||||||
|
ctx.Info("Writing output format 'EXR' to format file")
|
||||||
|
if err := os.WriteFile(formatFilePath, []byte("EXR"), 0644); err != nil {
|
||||||
|
errMsg := fmt.Sprintf("failed to create format file: %v", err)
|
||||||
|
ctx.Error(errMsg)
|
||||||
|
return errors.New(errMsg)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write render settings: merge job metadata with runner force_cpu (GPU lockout)
|
||||||
|
var settingsMap map[string]interface{}
|
||||||
|
if ctx.Metadata != nil && ctx.Metadata.RenderSettings.EngineSettings != nil {
|
||||||
|
raw, err := json.Marshal(ctx.Metadata.RenderSettings)
|
||||||
|
if err == nil {
|
||||||
|
_ = json.Unmarshal(raw, &settingsMap)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if settingsMap == nil {
|
||||||
|
settingsMap = make(map[string]interface{})
|
||||||
|
}
|
||||||
|
settingsMap["force_cpu"] = ctx.ShouldForceCPU()
|
||||||
|
settingsMap["disable_rt"] = ctx.ShouldDisableRT()
|
||||||
|
if ctx.ShouldBatchGPUSamples() {
|
||||||
|
// gfx115x: large tiles trigger AMDGPU page faults during HIP renders.
|
||||||
|
settingsMap["tile_size_override"] = 256
|
||||||
|
settingsMap["use_auto_tile_override"] = true
|
||||||
|
}
|
||||||
|
if opts != nil {
|
||||||
|
if opts.samplesOverride != nil {
|
||||||
|
settingsMap["samples_override"] = *opts.samplesOverride
|
||||||
|
}
|
||||||
|
if opts.seedOverride != nil {
|
||||||
|
settingsMap["seed_override"] = *opts.seedOverride
|
||||||
|
}
|
||||||
|
}
|
||||||
|
settingsJSON, err := json.Marshal(settingsMap)
|
||||||
|
if err == nil {
|
||||||
|
if err := os.WriteFile(renderSettingsFilePath, settingsJSON, 0644); err != nil {
|
||||||
|
ctx.Warn(fmt.Sprintf("Failed to write render settings file: %v", err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// wrapBlenderCmd builds the Blender *exec.Cmd, optionally through the sandbox wrapper.
|
||||||
|
func wrapBlenderCmd(ctx *Context, blenderBinary string, args, env []string, blenderHome string) (*exec.Cmd, error) {
|
||||||
|
sb := ctx.Sandbox
|
||||||
|
if sb == nil {
|
||||||
|
var err error
|
||||||
|
sb, err = sandbox.New(sandbox.Options{Backend: sandbox.BackendNone})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if sb.Name() != sandbox.BackendNone {
|
||||||
|
ctx.Info(fmt.Sprintf("Running Blender under sandbox backend %q", sb.Name()))
|
||||||
|
}
|
||||||
|
return sb.Wrap(sandbox.Spec{
|
||||||
|
BlenderBinary: blenderBinary,
|
||||||
|
Args: args,
|
||||||
|
WorkDir: ctx.WorkDir,
|
||||||
|
HomeDir: blenderHome,
|
||||||
|
Env: env,
|
||||||
|
HasAMD: ctx.HasAMD,
|
||||||
|
HasNVIDIA: ctx.HasNVIDIA,
|
||||||
|
HasIntel: ctx.HasIntel,
|
||||||
|
ForceCPU: ctx.ShouldForceCPU(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, outputDir, renderFormat, blenderHome string) error {
|
||||||
|
scriptPath := filepath.Join(ctx.WorkDir, "enable_gpu.py")
|
||||||
|
blendFileAbs, err := filepath.Abs(blendFile)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to resolve blend file path: %w", err)
|
||||||
|
}
|
||||||
|
scriptPathAbs, err := filepath.Abs(scriptPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to resolve blender script path: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
args := []string{"-b", blendFileAbs, "--python", scriptPathAbs}
|
||||||
|
if ctx.ShouldEnableExecution() {
|
||||||
|
args = append(args, "--enable-autoexec")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Output pattern
|
||||||
|
outputPattern := filepath.Join(outputDir, fmt.Sprintf("frame_####.%s", strings.ToLower(renderFormat)))
|
||||||
|
outputAbsPattern, _ := filepath.Abs(outputPattern)
|
||||||
|
args = append(args, "-o", outputAbsPattern)
|
||||||
|
|
||||||
|
// Render single frame or range: -f N for one frame, -s start -e end -a for range
|
||||||
|
if ctx.FrameEnd > ctx.Frame {
|
||||||
|
args = append(args, "-s", fmt.Sprintf("%d", ctx.Frame), "-e", fmt.Sprintf("%d", ctx.FrameEnd), "-a")
|
||||||
|
} else {
|
||||||
|
args = append(args, "-f", fmt.Sprintf("%d", ctx.Frame))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set up environment: LD_LIBRARY_PATH for tarball Blender, then custom HOME
|
||||||
|
env := os.Environ()
|
||||||
|
env = blender.TarballEnv(blenderBinary, env)
|
||||||
|
newEnv := make([]string, 0, len(env)+1)
|
||||||
|
for _, e := range env {
|
||||||
|
if !strings.HasPrefix(e, "HOME=") {
|
||||||
|
newEnv = append(newEnv, e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
newEnv = append(newEnv, fmt.Sprintf("HOME=%s", blenderHome))
|
||||||
|
|
||||||
|
cmd, err := wrapBlenderCmd(ctx, blenderBinary, args, newEnv, blenderHome)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to build sandboxed blender command: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set up pipes
|
||||||
|
stdoutPipe, err := cmd.StdoutPipe()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create stdout pipe: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
stderrPipe, err := cmd.StderrPipe()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create stderr pipe: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
return fmt.Errorf("failed to start blender: %w", err)
|
||||||
|
}
|
||||||
|
stopMonitor := ctx.StartCancellationMonitor(cmd, "render")
|
||||||
|
defer stopMonitor()
|
||||||
|
|
||||||
|
// Track process
|
||||||
|
ctx.Processes.Track(ctx.TaskID, cmd)
|
||||||
|
defer ctx.Processes.Untrack(ctx.TaskID)
|
||||||
|
|
||||||
|
// Stream stdout and watch for GPU error lines (lock out all GPU on any backend error)
|
||||||
|
stdoutDone := make(chan bool)
|
||||||
|
go func() {
|
||||||
|
defer close(stdoutDone)
|
||||||
|
scanner := bufio.NewScanner(stdoutPipe)
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
if line != "" {
|
||||||
|
p.checkGPUErrorLine(ctx, line)
|
||||||
|
shouldFilter, logLevel := blender.FilterLog(line)
|
||||||
|
if !shouldFilter {
|
||||||
|
ctx.Log(logLevel, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
log.Printf("Error reading stdout: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Stream stderr and watch for GPU error lines
|
||||||
|
stderrDone := make(chan bool)
|
||||||
|
go func() {
|
||||||
|
defer close(stderrDone)
|
||||||
|
scanner := bufio.NewScanner(stderrPipe)
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
if line != "" {
|
||||||
|
p.checkGPUErrorLine(ctx, line)
|
||||||
|
shouldFilter, logLevel := blender.FilterLog(line)
|
||||||
|
if !shouldFilter {
|
||||||
|
if logLevel == types.LogLevelInfo {
|
||||||
|
logLevel = types.LogLevelWarn
|
||||||
|
}
|
||||||
|
ctx.Log(logLevel, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
log.Printf("Error reading stderr: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Wait for completion
|
||||||
|
err = cmd.Wait()
|
||||||
|
<-stdoutDone
|
||||||
|
<-stderrDone
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
if cancelled, checkErr := ctx.IsJobCancelled(); checkErr == nil && cancelled {
|
||||||
|
return ErrJobCancelled
|
||||||
|
}
|
||||||
|
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||||
|
if exitErr.ExitCode() == 137 {
|
||||||
|
return errors.New("Blender was killed due to excessive memory usage (OOM)")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("blender failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// verifyOutputRange checks that output files exist for the task's frame range (first and last at minimum).
|
||||||
|
func (p *RenderProcessor) verifyOutputRange(ctx *Context, outputDir, renderFormat string) error {
|
||||||
|
entries, err := os.ReadDir(outputDir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to read output directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.Info("Checking output directory for files...")
|
||||||
|
ext := strings.ToLower(renderFormat)
|
||||||
|
|
||||||
|
// Check first and last frame in range (minimum required for range; single frame = one check)
|
||||||
|
framesToCheck := []int{ctx.Frame}
|
||||||
|
if ctx.FrameEnd > ctx.Frame {
|
||||||
|
framesToCheck = append(framesToCheck, ctx.FrameEnd)
|
||||||
|
}
|
||||||
|
for _, frame := range framesToCheck {
|
||||||
|
found := false
|
||||||
|
// Try frame_0001.ext, frame_1.ext, 0001.ext
|
||||||
|
for _, name := range []string{
|
||||||
|
fmt.Sprintf("frame_%04d.%s", frame, ext),
|
||||||
|
fmt.Sprintf("frame_%d.%s", frame, ext),
|
||||||
|
fmt.Sprintf("%04d.%s", frame, ext),
|
||||||
|
} {
|
||||||
|
if _, err := os.Stat(filepath.Join(outputDir, name)); err == nil {
|
||||||
|
found = true
|
||||||
|
ctx.Info(fmt.Sprintf("Found output file: %s", name))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
// Search entries for this frame number
|
||||||
|
frameStr := fmt.Sprintf("%d", frame)
|
||||||
|
frameStrPadded := fmt.Sprintf("%04d", frame)
|
||||||
|
for _, entry := range entries {
|
||||||
|
if entry.IsDir() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fileName := entry.Name()
|
||||||
|
if strings.Contains(fileName, "%04d") || strings.Contains(fileName, "%d") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (strings.Contains(fileName, frameStrPadded) ||
|
||||||
|
strings.Contains(fileName, frameStr)) && strings.HasSuffix(strings.ToLower(fileName), ext) {
|
||||||
|
found = true
|
||||||
|
ctx.Info(fmt.Sprintf("Found output file: %s", fileName))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
fileList := []string{}
|
||||||
|
for _, e := range entries {
|
||||||
|
if !e.IsDir() {
|
||||||
|
fileList = append(fileList, e.Name())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return fmt.Errorf("output file for frame %d not found; files in output directory: %v", frame, fileList)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
package tasks
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestCheckGPUErrorLine_TriggersCallback(t *testing.T) {
|
||||||
|
p := NewRenderProcessor()
|
||||||
|
triggered := 0
|
||||||
|
ctx := &Context{
|
||||||
|
OnGPUError: func() {
|
||||||
|
triggered++
|
||||||
|
// Simulate runner arming lockout for this attempt.
|
||||||
|
},
|
||||||
|
}
|
||||||
|
p.checkGPUErrorLine(ctx, "Fatal: Illegal address in HIP kernel execution")
|
||||||
|
if triggered != 1 {
|
||||||
|
t.Fatalf("expected GPU error callback once, got %d", triggered)
|
||||||
|
}
|
||||||
|
// After arming, further fault lines must not re-fire (spam protection).
|
||||||
|
ctx.GPULockoutArmedThisAttempt = true
|
||||||
|
p.checkGPUErrorLine(ctx, "page not present in GPU memory")
|
||||||
|
if triggered != 1 {
|
||||||
|
t.Fatalf("expected no further callbacks after arm, got %d", triggered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckGPUErrorLine_SkipsWhenAlreadyLockedOut(t *testing.T) {
|
||||||
|
p := NewRenderProcessor()
|
||||||
|
triggered := false
|
||||||
|
ctx := &Context{
|
||||||
|
GPULockedOut: true,
|
||||||
|
OnGPUError: func() { triggered = true },
|
||||||
|
}
|
||||||
|
p.checkGPUErrorLine(ctx, "Illegal address in HIP")
|
||||||
|
if triggered {
|
||||||
|
t.Fatal("did not expect callback when GPU already locked out")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGetBlenderVersion_EmptyWhenMissing(t *testing.T) {
|
||||||
|
ctx := &Context{}
|
||||||
|
if got := ctx.GetBlenderVersion(); got != "" {
|
||||||
|
t.Fatalf("expected empty blender version, got %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckGPUErrorLine_IgnoresNormalLine(t *testing.T) {
|
||||||
|
p := NewRenderProcessor()
|
||||||
|
triggered := false
|
||||||
|
ctx := &Context{
|
||||||
|
OnGPUError: func() { triggered = true },
|
||||||
|
}
|
||||||
|
p.checkGPUErrorLine(ctx, "Render completed successfully")
|
||||||
|
if triggered {
|
||||||
|
t.Fatal("did not expect GPU callback for normal line")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user