Compare commits
21 Commits
0.0.3
..
17685e7de5
| Author | SHA1 | Date | |
|---|---|---|---|
| 17685e7de5 | |||
| 2fadb08bcf | |||
| 5f7373a45c | |||
| 030d04dbe1 | |||
| a3eb2d0d7a | |||
| 90b71fc7e1 | |||
| caeb066f21 | |||
| 0f374b1d10 | |||
| 1a69fcfd04 | |||
| a3defe5cf6 | |||
| 16d6a95058 | |||
| 28cb50492c | |||
| dc525fbaa4 | |||
| 5303f01f7c | |||
| bc39fd438b | |||
| 4c7f168bce | |||
| 6833bb4013 | |||
| f9111ebac4 | |||
| 34445dc5cd | |||
| 63b8ff34c1 | |||
| 2deb47e5ad |
@@ -7,6 +7,8 @@ on:
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@main
|
||||
with:
|
||||
@@ -21,4 +23,6 @@ jobs:
|
||||
version: 'latest'
|
||||
args: release
|
||||
env:
|
||||
GITEA_TOKEN: ${{secrets.RELEASE_TOKEN}}
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
GORELEASER_FORCE_TOKEN: gitea
|
||||
|
||||
@@ -1,17 +1,19 @@
|
||||
name: PR Check
|
||||
name: CI
|
||||
on:
|
||||
- pull_request
|
||||
pull_request:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
|
||||
jobs:
|
||||
check-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@main
|
||||
- uses: actions/setup-go@main
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
- uses: FedericoCarboni/setup-ffmpeg@v3
|
||||
- run: go mod tidy
|
||||
- run: cd web && npm install && npm run build
|
||||
- run: go build ./...
|
||||
- run: go test -race -v -shuffle=on ./...
|
||||
- run: go test -race -v -shuffle=on ./...
|
||||
|
||||
-10
@@ -43,16 +43,6 @@ runner-secrets-*.json
|
||||
jiggablend-storage/
|
||||
jiggablend-workspaces/
|
||||
|
||||
# Node.js
|
||||
web/node_modules/
|
||||
web/dist/
|
||||
web/.vite/
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
lerna-debug.log*
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
|
||||
@@ -3,7 +3,6 @@ version: 2
|
||||
before:
|
||||
hooks:
|
||||
- go mod tidy -v
|
||||
- sh -c "cd web && npm install && npm run build"
|
||||
|
||||
builds:
|
||||
- id: default
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# Contributing
|
||||
|
||||
## Propose changes
|
||||
|
||||
Open a pull request against `develop`. Keep the default branch for releases and
|
||||
stable tips; land work on `develop` first.
|
||||
|
||||
Point at an existing issue when one fits. Prefer a short issue that states the
|
||||
symptom or request before a large PR.
|
||||
|
||||
## Commits
|
||||
|
||||
Subject form:
|
||||
|
||||
```
|
||||
area: Imperative summary
|
||||
```
|
||||
|
||||
- **Area** is a real package, directory, or subsystem token (`ci:`, `docs:`,
|
||||
Go package name). Not a lone filename.
|
||||
- **Imperative** mood: Fix, Add, Remove — not "Fixed" or "This patch…".
|
||||
- No trailing period. Aim ≤ ~70–75 characters for the whole subject.
|
||||
- Not conventional-commits (`feat:` / `fix:` / `chore:` as types).
|
||||
|
||||
Body explains **why**. Establish the problem, then say what you are doing.
|
||||
One logical change per commit; split fix and cleanup.
|
||||
|
||||
## Pull requests
|
||||
|
||||
Title matches the primary commit subject.
|
||||
|
||||
- **What** changed
|
||||
- **Why** (problem and impact)
|
||||
- **Test** (concrete steps; "CI green" alone is weak)
|
||||
|
||||
## Issues and closing
|
||||
|
||||
Cite leftover issues by **full URL**. Gitea closes issues when `#N` appears in
|
||||
merge text, so do not put `#N` in the merge message unless that issue is actually
|
||||
done. Use `Fixes #N` / `Closes #N` only when the leftover work is finished.
|
||||
@@ -5,11 +5,8 @@ build:
|
||||
@echo "Building with GoReleaser..."
|
||||
goreleaser build --clean --snapshot --single-target
|
||||
@mkdir -p bin
|
||||
@find dist -name jiggablend -type f -exec cp {} bin/jiggablend \;
|
||||
|
||||
# Build web UI
|
||||
build-web: clean-web
|
||||
cd web && npm install && npm run build
|
||||
@find dist -name jiggablend -type f -exec cp {} bin/jiggablend.new \;
|
||||
@mv -f bin/jiggablend.new bin/jiggablend
|
||||
|
||||
# Cleanup manager logs
|
||||
cleanup-manager:
|
||||
@@ -30,7 +27,19 @@ cleanup: cleanup-manager cleanup-runner
|
||||
run: cleanup build init-test
|
||||
@echo "Starting manager and runner in parallel..."
|
||||
@echo "Press Ctrl+C to stop both..."
|
||||
@trap 'kill $$MANAGER_PID $$RUNNER_PID 2>/dev/null; exit' INT TERM; \
|
||||
@MANAGER_PID=""; RUNNER_PID=""; INTERRUPTED=0; \
|
||||
cleanup() { \
|
||||
exit_code=$$?; \
|
||||
trap - INT TERM EXIT; \
|
||||
if [ -n "$$RUNNER_PID" ]; then kill -TERM "$$RUNNER_PID" 2>/dev/null || true; fi; \
|
||||
if [ -n "$$MANAGER_PID" ]; then kill -TERM "$$MANAGER_PID" 2>/dev/null || true; fi; \
|
||||
if [ -n "$$MANAGER_PID$$RUNNER_PID" ]; then wait $$MANAGER_PID $$RUNNER_PID 2>/dev/null || true; fi; \
|
||||
if [ "$$INTERRUPTED" -eq 1 ]; then exit 0; fi; \
|
||||
exit $$exit_code; \
|
||||
}; \
|
||||
on_interrupt() { INTERRUPTED=1; cleanup; }; \
|
||||
trap on_interrupt INT TERM; \
|
||||
trap cleanup EXIT; \
|
||||
bin/jiggablend manager -l manager.log & \
|
||||
MANAGER_PID=$$!; \
|
||||
sleep 2; \
|
||||
@@ -46,16 +55,17 @@ run-manager: cleanup-manager build init-test
|
||||
run-runner: cleanup-runner build
|
||||
bin/jiggablend runner -l runner.log --api-key=jk_r0_test_key_123456789012345678901234567890
|
||||
|
||||
# Initialize for testing (first run setup)
|
||||
# Initialize for testing (local development only — never use these secrets in production)
|
||||
init-test: build
|
||||
@echo "Initializing test configuration..."
|
||||
@echo "Initializing LOCAL TEST configuration (not for production)..."
|
||||
bin/jiggablend manager config enable localauth
|
||||
bin/jiggablend manager config set fixed-apikey jk_r0_test_key_123456789012345678901234567890 -f -y
|
||||
bin/jiggablend manager config add user test@example.com testpassword --admin -f -y
|
||||
@echo "Test configuration complete!"
|
||||
@echo "Test configuration complete (LOCAL DEV ONLY)!"
|
||||
@echo "fixed api key: jk_r0_test_key_123456789012345678901234567890"
|
||||
@echo "test user: test@example.com"
|
||||
@echo "test password: testpassword"
|
||||
@echo "WARNING: fixed API keys are refused when production_mode is enabled."
|
||||
|
||||
# Clean bin build artifacts
|
||||
clean-bin:
|
||||
@@ -63,7 +73,7 @@ clean-bin:
|
||||
|
||||
# Clean web build artifacts
|
||||
clean-web:
|
||||
rm -rf web/dist/
|
||||
@echo "No generated web artifacts to clean."
|
||||
|
||||
# Run tests
|
||||
test:
|
||||
@@ -75,7 +85,7 @@ help:
|
||||
@echo ""
|
||||
@echo "Build targets:"
|
||||
@echo " build - Build jiggablend binary with embedded web UI"
|
||||
@echo " build-web - Build web UI only"
|
||||
@echo " build-web - Validate web UI assets (no build required)"
|
||||
@echo ""
|
||||
@echo "Run targets:"
|
||||
@echo " run - Run manager and runner in parallel (for testing)"
|
||||
@@ -90,7 +100,7 @@ help:
|
||||
@echo ""
|
||||
@echo "Other targets:"
|
||||
@echo " clean-bin - Clean build artifacts"
|
||||
@echo " clean-web - Clean web build artifacts"
|
||||
@echo " clean-web - Clean generated web artifacts (currently none)"
|
||||
@echo " test - Run Go tests"
|
||||
@echo " help - Show this help"
|
||||
@echo ""
|
||||
|
||||
@@ -12,33 +12,33 @@ Both manager and runner are part of a single binary (`jiggablend`) with subcomma
|
||||
## Features
|
||||
|
||||
- **Authentication**: OAuth (Google and Discord) and local authentication with user management
|
||||
- **Web UI**: Modern React-based interface for job submission and monitoring
|
||||
- **Web UI**: Server-rendered Go templates with HTMX fragments for job submission and monitoring
|
||||
- **Distributed Rendering**: Scale across multiple runners with automatic job distribution
|
||||
- **Real-time Updates**: WebSocket-based progress tracking and job status updates
|
||||
- **Video Encoding**: Automatic video encoding from EXR/PNG sequences with multiple codec support:
|
||||
- H.264 (MP4) - SDR and HDR support
|
||||
- AV1 (MP4) - With alpha channel support
|
||||
- VP9 (WebM) - With alpha channel and HDR support
|
||||
- **Output Formats**: PNG, JPEG, EXR, and video formats (MP4, WebM)
|
||||
- **Real-time Updates**: Polling-based UI updates with lightweight HTMX refreshes
|
||||
- **Video Encoding**: Automatic video encoding from EXR sequences only. EXR→video always uses HDR (HLG, 10-bit); no option to disable. Codecs:
|
||||
- H.264 (MP4) - HDR (HLG)
|
||||
- AV1 (MP4) - Alpha channel support, HDR
|
||||
- VP9 (WebM) - Alpha channel and HDR
|
||||
- **Output Formats**: EXR frame sequence only, or EXR + video (H.264, AV1, VP9). Blender always renders EXR.
|
||||
- **Blender Version Management**: Support for multiple Blender versions with automatic detection
|
||||
- **Metadata Extraction**: Automatic extraction of scene metadata from Blender files
|
||||
- **Admin Panel**: User and runner management interface
|
||||
- **Runner Management**: API key-based authentication for runners with health monitoring
|
||||
- **HDR Support**: Preserve HDR range in video encoding with HLG transfer function
|
||||
- **Alpha Channel**: Preserve alpha channel in video encoding (AV1 and VP9)
|
||||
- **HDR**: EXR→video is always encoded as HDR (HLG, 10-bit). There is no option to turn it off; for SDR-only output, download the EXR frames and encode locally.
|
||||
- **Alpha**: Alpha is always preserved in EXR frames. In video, alpha is preserved when present in the EXR for AV1 and VP9; H.264 MP4 does not support alpha.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### Manager
|
||||
- Go 1.25.4 or later
|
||||
- Go 1.27.0 or later
|
||||
- SQLite (via Go driver)
|
||||
- Blender installed and in PATH (for metadata extraction)
|
||||
- ImageMagick installed (for EXR preview conversion)
|
||||
|
||||
### Runner
|
||||
- Linux amd64
|
||||
- Blender installed (can use bundled versions from storage)
|
||||
- FFmpeg installed (required for video encoding)
|
||||
- Able to run Blender (the runner gets the job’s required Blender version from the manager; it does not need Blender pre-installed)
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -68,7 +68,7 @@ make init-test
|
||||
|
||||
This will:
|
||||
- Enable local authentication
|
||||
- Set a fixed API key for testing
|
||||
- Set a fixed API key for testing: `jk_r0_test_key_123456789012345678901234567890`
|
||||
- Create a test admin user (test@example.com / testpassword)
|
||||
|
||||
#### Manual Configuration
|
||||
@@ -154,10 +154,36 @@ bin/jiggablend runner --api-key <your-api-key>
|
||||
# With custom options
|
||||
bin/jiggablend runner --manager http://localhost:8080 --name my-runner --api-key <key> --log-file runner.log
|
||||
|
||||
# Hardware compatibility flag (force CPU)
|
||||
bin/jiggablend runner --api-key <key> --force-cpu-rendering
|
||||
|
||||
# Sandbox Blender with rootless Podman (default; contains job Python/addons; manager I/O stays in the runner)
|
||||
# podman (default) | none (host Blender, no container)
|
||||
bin/jiggablend runner --api-key <key> # sandbox=podman by default
|
||||
bin/jiggablend runner --api-key <key> --sandbox none # disable sandbox
|
||||
# Optional: allow network inside the jail (default off)
|
||||
# bin/jiggablend runner --api-key <key> --sandbox-network
|
||||
|
||||
# Using environment variables
|
||||
JIGGABLEND_MANAGER=http://localhost:8080 JIGGABLEND_API_KEY=<key> bin/jiggablend runner
|
||||
```
|
||||
|
||||
### Blender sandbox notes
|
||||
|
||||
- **Blender versions** are still the manager-served Linux tarballs on the host; they are **bind-mounted** into the container (no per-version Blender images).
|
||||
- **GPU**: host devices/libs are attached from detection (NVIDIA `/dev/nvidia*`, AMD `/dev/kfd`+`/dev/dri`+ROCm paths, Intel DRM). Requires the runner user to already have access to those devices on the host.
|
||||
- **podman**: rootless podman + default thin image `registry.fedoraproject.org/fedora-minimal:41` (override with `--sandbox-image`).
|
||||
|
||||
|
||||
### Render Chunk Size Note
|
||||
|
||||
For one heavy production scene/profile, chunked rendering (`frames 800-804` in one Blender process) was much slower than one-frame tasks:
|
||||
|
||||
- Chunked task (`800-804`): `27m49s` end-to-end (`Task assigned` -> last `Saved`)
|
||||
- Single-frame tasks (`800`, `801`, `802`, `803`, `804`): `15m04s` wall clock total
|
||||
|
||||
In that test, any chunk size greater than `1` caused a major slowdown after the first frame. Fresh installs should already have it set to `1`, but if you see similar performance degradation, try forcing one frame per task (hard reset Blender each frame): `jiggablend manager config set frames-per-render-task 1`. If `1` is worse on your scene/hardware, benchmark and use a higher chunk size instead.
|
||||
|
||||
### Running Both (for Testing)
|
||||
|
||||
```bash
|
||||
@@ -217,9 +243,9 @@ jiggablend/
|
||||
│ ├── executils/ # Execution utilities
|
||||
│ ├── scripts/ # Python scripts for Blender
|
||||
│ └── types/ # Shared types and models
|
||||
├── web/ # React web UI
|
||||
│ ├── src/ # Source files
|
||||
│ └── dist/ # Built files (embedded in binary)
|
||||
├── web/ # Embedded templates + static assets
|
||||
│ ├── templates/ # Go HTML templates and partials
|
||||
│ └── static/ # CSS/JS assets
|
||||
├── go.mod
|
||||
└── Makefile
|
||||
```
|
||||
@@ -266,29 +292,25 @@ jiggablend/
|
||||
- `GET /api/admin/stats` - System statistics
|
||||
|
||||
### WebSocket
|
||||
- `WS /api/ws` - WebSocket connection for real-time updates
|
||||
- Subscribe to job channels: `job:{jobId}`
|
||||
- Receive job status updates, progress, and logs
|
||||
- `WS /api/jobs/ws` - Optional API channel for advanced clients
|
||||
- The default web UI uses polling + HTMX for status updates and task views.
|
||||
|
||||
## Output Formats
|
||||
|
||||
The system supports the following output formats:
|
||||
The system supports the following output formats. Blender always renders EXR (linear); the chosen format is the deliverable (frames only or frames + video).
|
||||
|
||||
### Image Formats
|
||||
- **PNG** - Standard PNG output
|
||||
- **JPEG** - JPEG output
|
||||
- **EXR** - OpenEXR format (HDR)
|
||||
### Deliverable Formats
|
||||
- **EXR** - EXR frame sequence only (no video)
|
||||
- **EXR_264_MP4** - EXR frames + H.264 MP4 (always HDR, HLG)
|
||||
- **EXR_AV1_MP4** - EXR frames + AV1 MP4 (alpha support, always HDR)
|
||||
- **EXR_VP9_WEBM** - EXR frames + VP9 WebM (alpha and HDR)
|
||||
|
||||
### Video Formats
|
||||
- **EXR_264_MP4** - H.264 encoded MP4 from EXR sequence (SDR or HDR)
|
||||
- **EXR_AV1_MP4** - AV1 encoded MP4 from EXR sequence (with alpha channel support)
|
||||
- **EXR_VP9_WEBM** - VP9 encoded WebM from EXR sequence (with alpha channel and HDR support)
|
||||
Video encoding (EXR→video) is always HDR (HLG, 10-bit); there is no option to output SDR video. For SDR-only, download the EXR frames and encode locally.
|
||||
|
||||
Video encoding features:
|
||||
- 2-pass encoding for optimal quality
|
||||
- HDR preservation using HLG transfer function
|
||||
- EXR→video only (no PNG source); always HLG (HDR), 10-bit, full range
|
||||
- Alpha channel preservation (AV1 and VP9 only)
|
||||
- Automatic detection of source format (EXR or PNG)
|
||||
- Software encoding (libx264, libaom-av1, libvpx-vp9)
|
||||
|
||||
## Storage Structure
|
||||
@@ -320,16 +342,8 @@ go test ./... -timeout 30s
|
||||
|
||||
### Web UI Development
|
||||
|
||||
The web UI is built with React and Vite. To develop the UI:
|
||||
|
||||
```bash
|
||||
cd web
|
||||
npm install
|
||||
npm run dev # Development server
|
||||
npm run build # Build for production
|
||||
```
|
||||
|
||||
The built files are embedded in the Go binary using `embed.FS`.
|
||||
The web UI is server-rendered from embedded templates and static assets in `web/templates` and `web/static`.
|
||||
No Node/Vite build step is required.
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"jiggablend/internal/auth"
|
||||
@@ -151,7 +152,15 @@ func runManager(cmd *cobra.Command, args []string) {
|
||||
}
|
||||
|
||||
func checkBlenderAvailable() error {
|
||||
cmd := exec.Command("blender", "--version")
|
||||
blenderPath, err := exec.LookPath("blender")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to locate blender in PATH: %w", err)
|
||||
}
|
||||
blenderPath, err = filepath.Abs(blenderPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve blender path %q: %w", blenderPath, err)
|
||||
}
|
||||
cmd := exec.Command(blenderPath, "--version")
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to run 'blender --version': %w (output: %s)", err, string(output))
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"jiggablend/internal/config"
|
||||
@@ -381,6 +382,25 @@ var setGoogleOAuthCmd = &cobra.Command{
|
||||
|
||||
var setDiscordOAuthRedirectURL string
|
||||
|
||||
var setFramesPerRenderTaskCmd = &cobra.Command{
|
||||
Use: "frames-per-render-task <n>",
|
||||
Short: "Set number of frames per render task (min 1)",
|
||||
Long: `Set how many frames to batch into each render task. Job frame range is divided into chunks of this size. Default is 10.`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
n, err := strconv.Atoi(args[0])
|
||||
if err != nil || n < 1 {
|
||||
exitWithError("frames-per-render-task must be a positive integer")
|
||||
}
|
||||
withConfig(func(cfg *config.Config, db *database.DB) {
|
||||
if err := cfg.SetInt(config.KeyFramesPerRenderTask, n); err != nil {
|
||||
exitWithError("Failed to set frames_per_render_task: %v", err)
|
||||
}
|
||||
fmt.Printf("Frames per render task set to %d\n", n)
|
||||
})
|
||||
},
|
||||
}
|
||||
|
||||
var setDiscordOAuthCmd = &cobra.Command{
|
||||
Use: "discord-oauth <client-id> <client-secret>",
|
||||
Short: "Set Discord OAuth credentials",
|
||||
@@ -558,6 +578,7 @@ func init() {
|
||||
configCmd.AddCommand(setCmd)
|
||||
setCmd.AddCommand(setFixedAPIKeyCmd)
|
||||
setCmd.AddCommand(setAllowedOriginsCmd)
|
||||
setCmd.AddCommand(setFramesPerRenderTaskCmd)
|
||||
setCmd.AddCommand(setGoogleOAuthCmd)
|
||||
setCmd.AddCommand(setDiscordOAuthCmd)
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGenerateAPIKey_Format(t *testing.T) {
|
||||
key, prefix, hash, err := generateAPIKey()
|
||||
if err != nil {
|
||||
t.Fatalf("generateAPIKey failed: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(prefix, "jk_r") {
|
||||
t.Fatalf("unexpected prefix: %q", prefix)
|
||||
}
|
||||
if !strings.HasPrefix(key, prefix+"_") {
|
||||
t.Fatalf("key does not include prefix: %q", key)
|
||||
}
|
||||
if len(hash) != 64 {
|
||||
t.Fatalf("expected sha256 hex hash length, got %d", len(hash))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
package cmd
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRootCommand_HasKeySubcommands(t *testing.T) {
|
||||
names := map[string]bool{}
|
||||
for _, c := range rootCmd.Commands() {
|
||||
names[c.Name()] = true
|
||||
}
|
||||
for _, required := range []string{"manager", "runner", "version"} {
|
||||
if !names[required] {
|
||||
t.Fatalf("expected subcommand %q to be registered", required)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -37,6 +37,12 @@ func init() {
|
||||
runnerCmd.Flags().String("log-level", "info", "Log level (debug, info, warn, error)")
|
||||
runnerCmd.Flags().BoolP("verbose", "v", false, "Enable verbose logging (same as --log-level=debug)")
|
||||
runnerCmd.Flags().Duration("poll-interval", 5*time.Second, "Job polling interval")
|
||||
runnerCmd.Flags().Bool("force-cpu-rendering", false, "Force CPU rendering for all jobs (disables GPU rendering)")
|
||||
runnerCmd.Flags().Bool("disable-rt", false, "Disable GPU ray tracing acceleration (HIPRT, OptiX, etc.)")
|
||||
runnerCmd.Flags().Int("hip-gpu-sample-batch", 0, "Max samples per GPU render pass on gfx115x (0=disabled; merges batches into one EXR when >0)")
|
||||
runnerCmd.Flags().String("sandbox", "podman", "Blender sandbox backend: podman (default; bind-mounts host Blender tarball + GPU devices) or none")
|
||||
runnerCmd.Flags().Bool("sandbox-network", false, "Allow network inside the sandbox (default: isolated; manager I/O stays in the runner process)")
|
||||
runnerCmd.Flags().String("sandbox-image", "", "Thin OS image for podman backend (default: fedora-minimal; Blender is not inside the image)")
|
||||
|
||||
// Bind flags to viper with JIGGABLEND_ prefix
|
||||
runnerViper.SetEnvPrefix("JIGGABLEND")
|
||||
@@ -51,6 +57,12 @@ func init() {
|
||||
runnerViper.BindPFlag("log_level", runnerCmd.Flags().Lookup("log-level"))
|
||||
runnerViper.BindPFlag("verbose", runnerCmd.Flags().Lookup("verbose"))
|
||||
runnerViper.BindPFlag("poll_interval", runnerCmd.Flags().Lookup("poll-interval"))
|
||||
runnerViper.BindPFlag("force_cpu_rendering", runnerCmd.Flags().Lookup("force-cpu-rendering"))
|
||||
runnerViper.BindPFlag("disable_rt", runnerCmd.Flags().Lookup("disable-rt"))
|
||||
runnerViper.BindPFlag("hip_gpu_sample_batch", runnerCmd.Flags().Lookup("hip-gpu-sample-batch"))
|
||||
runnerViper.BindPFlag("sandbox", runnerCmd.Flags().Lookup("sandbox"))
|
||||
runnerViper.BindPFlag("sandbox_network", runnerCmd.Flags().Lookup("sandbox-network"))
|
||||
runnerViper.BindPFlag("sandbox_image", runnerCmd.Flags().Lookup("sandbox-image"))
|
||||
}
|
||||
|
||||
func runRunner(cmd *cobra.Command, args []string) {
|
||||
@@ -63,7 +75,12 @@ func runRunner(cmd *cobra.Command, args []string) {
|
||||
logLevel := runnerViper.GetString("log_level")
|
||||
verbose := runnerViper.GetBool("verbose")
|
||||
pollInterval := runnerViper.GetDuration("poll_interval")
|
||||
|
||||
forceCPURendering := runnerViper.GetBool("force_cpu_rendering")
|
||||
disableRT := runnerViper.GetBool("disable_rt")
|
||||
hipGPUSampleBatch := runnerViper.GetInt("hip_gpu_sample_batch")
|
||||
sandboxBackend := runnerViper.GetString("sandbox")
|
||||
sandboxNetwork := runnerViper.GetBool("sandbox_network")
|
||||
sandboxImage := runnerViper.GetString("sandbox_image")
|
||||
var r *runner.Runner
|
||||
|
||||
defer func() {
|
||||
@@ -112,13 +129,27 @@ func runRunner(cmd *cobra.Command, args []string) {
|
||||
}
|
||||
|
||||
logger.Info("Runner starting up...")
|
||||
if disableRT {
|
||||
logger.Info("GPU ray tracing acceleration disabled (--disable-rt)")
|
||||
}
|
||||
if hipGPUSampleBatch > 0 {
|
||||
logger.Infof("HIP GPU sample batching enabled: %d samples per pass", hipGPUSampleBatch)
|
||||
}
|
||||
logger.Infof("Blender sandbox backend: %s (network=%v)", sandboxBackend, sandboxNetwork)
|
||||
logger.Debugf("Generated runner ID suffix: %s", runnerIDStr)
|
||||
if logFile != "" {
|
||||
logger.Infof("Logging to file: %s", logFile)
|
||||
}
|
||||
|
||||
// Create runner
|
||||
r = runner.New(managerURL, name, hostname)
|
||||
r = runner.NewWithOptions(managerURL, name, hostname, runner.RunnerOptions{
|
||||
ForceCPURendering: forceCPURendering,
|
||||
DisableRT: disableRT,
|
||||
HipGPUSampleBatch: hipGPUSampleBatch,
|
||||
SandboxBackend: sandboxBackend,
|
||||
SandboxNetwork: sandboxNetwork,
|
||||
SandboxImage: sandboxImage,
|
||||
})
|
||||
|
||||
// Check for required tools early to fail fast
|
||||
if err := r.CheckRequiredTools(); err != nil {
|
||||
@@ -161,6 +192,9 @@ func runRunner(cmd *cobra.Command, args []string) {
|
||||
runnerID, err = r.Register(apiKey)
|
||||
if err == nil {
|
||||
logger.Infof("Registered runner with ID: %d", runnerID)
|
||||
// Detect GPU vendors/backends from host hardware so we only force CPU for Blender < 4.x when using AMD.
|
||||
logger.Info("Detecting GPU backends (AMD/NVIDIA/Intel) from host hardware for Blender < 4.x policy...")
|
||||
r.DetectAndStoreGPUBackends()
|
||||
break
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGenerateShortID_IsHex8Bytes(t *testing.T) {
|
||||
id := generateShortID()
|
||||
if len(id) != 8 {
|
||||
t.Fatalf("expected 8 hex chars, got %q", id)
|
||||
}
|
||||
if _, err := hex.DecodeString(id); err != nil {
|
||||
t.Fatalf("id should be hex: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
package cmd
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestVersionCommand_Metadata(t *testing.T) {
|
||||
if versionCmd.Use != "version" {
|
||||
t.Fatalf("unexpected command use: %q", versionCmd.Use)
|
||||
}
|
||||
if versionCmd.Run == nil {
|
||||
t.Fatal("version command run function should be set")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMainPackage_Builds(t *testing.T) {
|
||||
// Smoke test placeholder to keep package main under test compilation.
|
||||
}
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 24 MiB |
@@ -1,6 +1,6 @@
|
||||
module jiggablend
|
||||
|
||||
go 1.25.4
|
||||
go 1.27.0
|
||||
|
||||
require (
|
||||
github.com/go-chi/chi/v5 v5.2.3
|
||||
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Install the latest jiggablend binary for Linux AMD64 and create wrapper scripts.
|
||||
# Production wrappers do NOT install fixed test secrets. For local test credentials,
|
||||
# use `make init-test` from a development checkout.
|
||||
|
||||
# Dependencies: curl, jq, tar, sha256sum, sudo (for installation to /usr/local/bin)
|
||||
|
||||
REPO="s1d3sw1ped/jiggablend"
|
||||
API_URL="https://git.s1d3sw1ped.com/api/v1/repos/${REPO}/releases/latest"
|
||||
ASSET_NAME="jiggablend-linux-amd64.tar.gz"
|
||||
|
||||
echo "Fetching latest release information..."
|
||||
RELEASE_JSON=$(curl -s "$API_URL")
|
||||
|
||||
TAG=$(echo "$RELEASE_JSON" | jq -r '.tag_name')
|
||||
echo "Latest version: $TAG"
|
||||
|
||||
ASSET_URL=$(echo "$RELEASE_JSON" | jq -r ".assets[] | select(.name == \"$ASSET_NAME\") | .browser_download_url")
|
||||
if [ -z "$ASSET_URL" ]; then
|
||||
echo "Error: Asset $ASSET_NAME not found in latest release."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CHECKSUM_URL=$(echo "$RELEASE_JSON" | jq -r '.assets[] | select(.name == "checksums.txt") | .browser_download_url')
|
||||
if [ -z "$CHECKSUM_URL" ]; then
|
||||
echo "Error: checksums.txt not found in latest release."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Downloading $ASSET_NAME..."
|
||||
curl -L -o "$ASSET_NAME" "$ASSET_URL"
|
||||
|
||||
echo "Downloading checksums.txt..."
|
||||
curl -L -o "checksums.txt" "$CHECKSUM_URL"
|
||||
|
||||
echo "Verifying checksum..."
|
||||
if ! sha256sum --ignore-missing --quiet -c checksums.txt; then
|
||||
echo "Error: Checksum verification failed."
|
||||
rm -f "$ASSET_NAME" checksums.txt
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Extracting..."
|
||||
tar -xzf "$ASSET_NAME"
|
||||
|
||||
echo "Installing binary to /usr/local/bin (requires sudo)..."
|
||||
sudo install -m 0755 jiggablend /usr/local/bin/
|
||||
|
||||
echo "Creating manager wrapper script..."
|
||||
cat << 'EOF' > jiggablend-manager.sh
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Wrapper to run jiggablend manager.
|
||||
# Does NOT set fixed/test API keys or default admin passwords.
|
||||
# Bootstrap (one-time, local admin):
|
||||
# jiggablend manager config enable localauth
|
||||
# jiggablend manager config add user you@example.com 'strong-password' --admin
|
||||
# jiggablend manager config add apikey my-runner --scope manager
|
||||
# For local development only: use `make init-test` from a source checkout.
|
||||
|
||||
mkdir -p logs
|
||||
rm -f logs/manager.log
|
||||
|
||||
jiggablend manager -l logs/manager.log
|
||||
EOF
|
||||
chmod +x jiggablend-manager.sh
|
||||
sudo install -m 0755 jiggablend-manager.sh /usr/local/bin/jiggablend-manager
|
||||
rm -f jiggablend-manager.sh
|
||||
|
||||
echo "Creating runner wrapper script..."
|
||||
cat << 'EOF' > jiggablend-runner.sh
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Wrapper to run jiggablend runner.
|
||||
# Usage: jiggablend-runner [MANAGER_URL] --api-key <key> [RUNNER_FLAGS...]
|
||||
# Or set JIGGABLEND_API_KEY. Default MANAGER_URL: http://localhost:8080
|
||||
|
||||
MANAGER_URL="http://localhost:8080"
|
||||
if [[ $# -gt 0 && "$1" != -* ]]; then
|
||||
MANAGER_URL="$1"
|
||||
shift
|
||||
fi
|
||||
|
||||
EXTRA_ARGS=("$@")
|
||||
|
||||
API_KEY="${JIGGABLEND_API_KEY:-}"
|
||||
# Allow --api-key in EXTRA_ARGS; if not present and env empty, fail clearly
|
||||
HAS_KEY=0
|
||||
for arg in "${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}"; do
|
||||
case "$arg" in
|
||||
--api-key|--api-key=*|-k) HAS_KEY=1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$API_KEY" && "$HAS_KEY" -eq 0 ]]; then
|
||||
echo "Error: provide a runner API key via JIGGABLEND_API_KEY or --api-key." >&2
|
||||
echo "Create one with: jiggablend manager config add apikey <name> --scope manager" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p logs
|
||||
rm -f logs/runner.log
|
||||
|
||||
if [[ -n "$API_KEY" && "$HAS_KEY" -eq 0 ]]; then
|
||||
jiggablend runner -l logs/runner.log --api-key="$API_KEY" --manager "$MANAGER_URL" "${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}"
|
||||
else
|
||||
jiggablend runner -l logs/runner.log --manager "$MANAGER_URL" "${EXTRA_ARGS[@]+"${EXTRA_ARGS[@]}"}"
|
||||
fi
|
||||
EOF
|
||||
chmod +x jiggablend-runner.sh
|
||||
sudo install -m 0755 jiggablend-runner.sh /usr/local/bin/jiggablend-runner
|
||||
rm -f jiggablend-runner.sh
|
||||
|
||||
echo "Cleaning up..."
|
||||
rm -f "$ASSET_NAME" checksums.txt jiggablend
|
||||
|
||||
echo "Installation complete!"
|
||||
echo "Binary: jiggablend"
|
||||
echo "Wrappers: jiggablend-manager, jiggablend-runner"
|
||||
echo "Run 'jiggablend-manager' to start the manager (no fixed test secrets)."
|
||||
echo "Run 'jiggablend-runner [url] --api-key <key>' to start a runner."
|
||||
echo "Local dev only: use 'make init-test' from a source checkout for test credentials."
|
||||
echo "Note: Blender, ImageMagick, or FFmpeg may be required. See README."
|
||||
+98
-56
@@ -45,6 +45,9 @@ type Auth struct {
|
||||
sessionCache map[string]*Session // In-memory cache for performance
|
||||
cacheMu sync.RWMutex
|
||||
stopCleanup chan struct{}
|
||||
// oauthStates maps state token -> expiry for CSRF protection
|
||||
oauthStates map[string]time.Time
|
||||
oauthMu sync.Mutex
|
||||
}
|
||||
|
||||
// Session represents a user session
|
||||
@@ -63,6 +66,7 @@ func NewAuth(db *database.DB, cfg *config.Config) (*Auth, error) {
|
||||
cfg: cfg,
|
||||
sessionCache: make(map[string]*Session),
|
||||
stopCleanup: make(chan struct{}),
|
||||
oauthStates: make(map[string]time.Time),
|
||||
}
|
||||
|
||||
// Initialize Google OAuth from database config
|
||||
@@ -216,7 +220,9 @@ func (a *Auth) cleanupExpiredSessions() {
|
||||
|
||||
// initializeSettings ensures all admin settings are initialized with defaults if they don't exist
|
||||
func (a *Auth) initializeSettings() error {
|
||||
// Initialize registration_enabled setting (default: true) if it doesn't exist
|
||||
// Default registration to false (safer for internet-facing boots). Admins enable via CLI/UI.
|
||||
// In non-production, still default false — make init-test / CLI create the first admin.
|
||||
defaultReg := "false"
|
||||
var settingCount int
|
||||
err := a.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow("SELECT COUNT(*) FROM settings WHERE key = ?", "registration_enabled").Scan(&settingCount)
|
||||
@@ -227,15 +233,15 @@ func (a *Auth) initializeSettings() error {
|
||||
if settingCount == 0 {
|
||||
err = a.db.With(func(conn *sql.DB) error {
|
||||
_, err := conn.Exec(
|
||||
`INSERT INTO settings (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)`,
|
||||
"registration_enabled", "true",
|
||||
)
|
||||
`INSERT INTO settings (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)`,
|
||||
"registration_enabled", defaultReg,
|
||||
)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to initialize registration_enabled setting: %w", err)
|
||||
}
|
||||
log.Printf("Initialized admin setting: registration_enabled = true")
|
||||
log.Printf("Initialized admin setting: registration_enabled = %s", defaultReg)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -303,12 +309,44 @@ func (a *Auth) initializeTestUser() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateOAuthState mints a one-time OAuth state token and stores it until used or expired.
|
||||
func (a *Auth) CreateOAuthState() string {
|
||||
state := uuid.New().String()
|
||||
a.oauthMu.Lock()
|
||||
// Opportunistic cleanup of expired states
|
||||
now := time.Now()
|
||||
for k, exp := range a.oauthStates {
|
||||
if now.After(exp) {
|
||||
delete(a.oauthStates, k)
|
||||
}
|
||||
}
|
||||
a.oauthStates[state] = now.Add(10 * time.Minute)
|
||||
a.oauthMu.Unlock()
|
||||
return state
|
||||
}
|
||||
|
||||
// ConsumeOAuthState validates and single-use-consumes an OAuth state token.
|
||||
// Returns false if missing, unknown, or expired.
|
||||
func (a *Auth) ConsumeOAuthState(state string) bool {
|
||||
if state == "" {
|
||||
return false
|
||||
}
|
||||
a.oauthMu.Lock()
|
||||
defer a.oauthMu.Unlock()
|
||||
exp, ok := a.oauthStates[state]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
delete(a.oauthStates, state)
|
||||
return time.Now().Before(exp)
|
||||
}
|
||||
|
||||
// GoogleLoginURL returns the Google OAuth login URL
|
||||
func (a *Auth) GoogleLoginURL() (string, error) {
|
||||
if a.googleConfig == nil {
|
||||
return "", fmt.Errorf("Google OAuth not configured")
|
||||
}
|
||||
state := uuid.New().String()
|
||||
state := a.CreateOAuthState()
|
||||
return a.googleConfig.AuthCodeURL(state), nil
|
||||
}
|
||||
|
||||
@@ -317,7 +355,7 @@ func (a *Auth) DiscordLoginURL() (string, error) {
|
||||
if a.discordConfig == nil {
|
||||
return "", fmt.Errorf("Discord OAuth not configured")
|
||||
}
|
||||
state := uuid.New().String()
|
||||
state := a.CreateOAuthState()
|
||||
return a.discordConfig.AuthCodeURL(state), nil
|
||||
}
|
||||
|
||||
@@ -390,8 +428,8 @@ func (a *Auth) IsRegistrationEnabled() (bool, error) {
|
||||
return conn.QueryRow("SELECT value FROM settings WHERE key = ?", "registration_enabled").Scan(&value)
|
||||
})
|
||||
if err == sql.ErrNoRows {
|
||||
// Default to enabled if setting doesn't exist
|
||||
return true, nil
|
||||
// Default to disabled if setting doesn't exist (safer bootstrap)
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to check registration setting: %w", err)
|
||||
@@ -438,8 +476,9 @@ func (a *Auth) SetRegistrationEnabled(enabled bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// getOrCreateUser gets or creates a user in the database
|
||||
// Automatically links accounts by email across different OAuth providers and local login
|
||||
// getOrCreateUser gets or creates a user in the database.
|
||||
// Identity is (oauth_provider, oauth_id). Email collision with a different provider
|
||||
// is NOT auto-linked (prevents account takeover); the user must sign in with the original method.
|
||||
func (a *Auth) getOrCreateUser(provider, oauthID, email, name string) (*Session, error) {
|
||||
var userID int64
|
||||
var dbEmail, dbName string
|
||||
@@ -449,18 +488,18 @@ func (a *Auth) getOrCreateUser(provider, oauthID, email, name string) (*Session,
|
||||
// First, try to find by provider + oauth_id
|
||||
err := a.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow(
|
||||
"SELECT id, email, name, is_admin, oauth_provider, oauth_id FROM users WHERE oauth_provider = ? AND oauth_id = ?",
|
||||
provider, oauthID,
|
||||
).Scan(&userID, &dbEmail, &dbName, &isAdmin, &dbProvider, &dbOAuthID)
|
||||
"SELECT id, email, name, is_admin, oauth_provider, oauth_id FROM users WHERE oauth_provider = ? AND oauth_id = ?",
|
||||
provider, oauthID,
|
||||
).Scan(&userID, &dbEmail, &dbName, &isAdmin, &dbProvider, &dbOAuthID)
|
||||
})
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
// Not found by provider+oauth_id, check by email for account linking
|
||||
// Not found by provider+oauth_id — check email only to refuse silent takeover
|
||||
err = a.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow(
|
||||
"SELECT id, email, name, is_admin, oauth_provider, oauth_id FROM users WHERE email = ?",
|
||||
email,
|
||||
).Scan(&userID, &dbEmail, &dbName, &isAdmin, &dbProvider, &dbOAuthID)
|
||||
"SELECT id, email, name, is_admin, oauth_provider, oauth_id FROM users WHERE email = ?",
|
||||
email,
|
||||
).Scan(&userID, &dbEmail, &dbName, &isAdmin, &dbProvider, &dbOAuthID)
|
||||
})
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
@@ -487,7 +526,7 @@ func (a *Auth) getOrCreateUser(provider, oauthID, email, name string) (*Session,
|
||||
err = a.db.With(func(conn *sql.DB) error {
|
||||
result, err := conn.Exec(
|
||||
"INSERT INTO users (email, name, oauth_provider, oauth_id, is_admin) VALUES (?, ?, ?, ?, ?)",
|
||||
email, name, provider, oauthID, isAdmin,
|
||||
email, name, provider, oauthID, isAdmin,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -501,19 +540,8 @@ func (a *Auth) getOrCreateUser(provider, oauthID, email, name string) (*Session,
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("failed to query user by email: %w", err)
|
||||
} else {
|
||||
// User exists with same email but different provider - link accounts by updating provider info
|
||||
// This allows the user to log in with any provider that has the same email
|
||||
err = a.db.With(func(conn *sql.DB) error {
|
||||
_, err = conn.Exec(
|
||||
"UPDATE users SET oauth_provider = ?, oauth_id = ?, name = ? WHERE id = ?",
|
||||
provider, oauthID, name, userID,
|
||||
)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to link account: %w", err)
|
||||
}
|
||||
log.Printf("Linked account: user %d (email: %s) now accessible via %s provider", userID, email, provider)
|
||||
// Email already belongs to another identity — do not overwrite oauth_provider/oauth_id
|
||||
return nil, fmt.Errorf("an account with this email already exists; sign in with the original method")
|
||||
}
|
||||
} else if err != nil {
|
||||
return nil, fmt.Errorf("failed to query user: %w", err)
|
||||
@@ -522,9 +550,9 @@ func (a *Auth) getOrCreateUser(provider, oauthID, email, name string) (*Session,
|
||||
if dbEmail != email || dbName != name {
|
||||
err = a.db.With(func(conn *sql.DB) error {
|
||||
_, err = conn.Exec(
|
||||
"UPDATE users SET email = ?, name = ? WHERE id = ?",
|
||||
email, name, userID,
|
||||
)
|
||||
"UPDATE users SET email = ?, name = ? WHERE id = ?",
|
||||
email, name, userID,
|
||||
)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
@@ -652,20 +680,42 @@ func (a *Auth) DeleteSession(sessionID string) {
|
||||
}
|
||||
}
|
||||
|
||||
// IsProductionMode returns true if running in production mode
|
||||
// This is a package-level function that checks the environment variable
|
||||
// For config-based checks, use Config.IsProductionMode()
|
||||
// IsProductionMode returns true if running in production mode.
|
||||
// Prefer Config.IsProductionMode() / Auth.IsProductionModeFromConfig() for app logic.
|
||||
// This package-level helper still honors PRODUCTION=true for legacy callers.
|
||||
func IsProductionMode() bool {
|
||||
// Check environment variable first for backwards compatibility
|
||||
if os.Getenv("PRODUCTION") == "true" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
return os.Getenv("PRODUCTION") == "true"
|
||||
}
|
||||
|
||||
// IsProductionModeFromConfig returns true if production mode is enabled in config
|
||||
// or via PRODUCTION=true environment variable (OR of both sources).
|
||||
func (a *Auth) IsProductionModeFromConfig() bool {
|
||||
return a.cfg.IsProductionMode()
|
||||
if a.cfg != nil && a.cfg.IsProductionMode() {
|
||||
return true
|
||||
}
|
||||
return IsProductionMode()
|
||||
}
|
||||
|
||||
func (a *Auth) writeUnauthorized(w http.ResponseWriter, r *http.Request) {
|
||||
// Keep API behavior unchanged for programmatic clients.
|
||||
if strings.HasPrefix(r.URL.Path, "/api/") {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
||||
return
|
||||
}
|
||||
|
||||
// For HTMX UI fragment requests, trigger a full-page redirect to login.
|
||||
if strings.EqualFold(r.Header.Get("HX-Request"), "true") {
|
||||
w.Header().Set("HX-Redirect", "/login")
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
||||
return
|
||||
}
|
||||
|
||||
// For normal browser page requests, redirect to login page.
|
||||
http.Redirect(w, r, "/login", http.StatusFound)
|
||||
}
|
||||
|
||||
// Middleware creates an authentication middleware
|
||||
@@ -674,18 +724,14 @@ func (a *Auth) Middleware(next http.HandlerFunc) http.HandlerFunc {
|
||||
cookie, err := r.Cookie("session_id")
|
||||
if err != nil {
|
||||
log.Printf("Authentication failed: missing session cookie for %s %s", r.Method, r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
||||
a.writeUnauthorized(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
session, ok := a.GetSession(cookie.Value)
|
||||
if !ok {
|
||||
log.Printf("Authentication failed: invalid session cookie for %s %s", r.Method, r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
||||
a.writeUnauthorized(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -717,18 +763,14 @@ func (a *Auth) AdminMiddleware(next http.HandlerFunc) http.HandlerFunc {
|
||||
cookie, err := r.Cookie("session_id")
|
||||
if err != nil {
|
||||
log.Printf("Admin authentication failed: missing session cookie for %s %s", r.Method, r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
||||
a.writeUnauthorized(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
session, ok := a.GetSession(cookie.Value)
|
||||
if !ok {
|
||||
log.Printf("Admin authentication failed: invalid session cookie for %s %s", r.Method, r.URL.Path)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"})
|
||||
a.writeUnauthorized(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestContextHelpers(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx = context.WithValue(ctx, contextKeyUserID, int64(123))
|
||||
ctx = context.WithValue(ctx, contextKeyIsAdmin, true)
|
||||
|
||||
id, ok := GetUserID(ctx)
|
||||
if !ok || id != 123 {
|
||||
t.Fatalf("GetUserID() = (%d,%v), want (123,true)", id, ok)
|
||||
}
|
||||
if !IsAdmin(ctx) {
|
||||
t.Fatal("expected IsAdmin to be true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsProductionMode_UsesEnv(t *testing.T) {
|
||||
t.Setenv("PRODUCTION", "true")
|
||||
if !IsProductionMode() {
|
||||
t.Fatal("expected production mode true when env is set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteUnauthorized_BehaviorByRequestType(t *testing.T) {
|
||||
a := &Auth{}
|
||||
|
||||
reqAPI := httptest.NewRequest(http.MethodGet, "/api/jobs", nil)
|
||||
rrAPI := httptest.NewRecorder()
|
||||
a.writeUnauthorized(rrAPI, reqAPI)
|
||||
if rrAPI.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("api code = %d", rrAPI.Code)
|
||||
}
|
||||
|
||||
reqPage := httptest.NewRequest(http.MethodGet, "/dashboard", nil)
|
||||
rrPage := httptest.NewRecorder()
|
||||
a.writeUnauthorized(rrPage, reqPage)
|
||||
if rrPage.Code != http.StatusFound {
|
||||
t.Fatalf("page code = %d", rrPage.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsProductionMode_DefaultFalse(t *testing.T) {
|
||||
_ = os.Unsetenv("PRODUCTION")
|
||||
if IsProductionMode() {
|
||||
t.Fatal("expected false when PRODUCTION is unset")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOAuthState_CreateConsumeAndReject(t *testing.T) {
|
||||
a := &Auth{oauthStates: make(map[string]time.Time)}
|
||||
state := a.CreateOAuthState()
|
||||
if state == "" {
|
||||
t.Fatal("expected non-empty state")
|
||||
}
|
||||
if !a.ConsumeOAuthState(state) {
|
||||
t.Fatal("expected valid state to be accepted once")
|
||||
}
|
||||
if a.ConsumeOAuthState(state) {
|
||||
t.Fatal("expected state to be single-use")
|
||||
}
|
||||
if a.ConsumeOAuthState("") {
|
||||
t.Fatal("empty state must be rejected")
|
||||
}
|
||||
if a.ConsumeOAuthState("unknown-state") {
|
||||
t.Fatal("unknown state must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOAuthState_ExpiredRejected(t *testing.T) {
|
||||
a := &Auth{oauthStates: make(map[string]time.Time)}
|
||||
state := "expired-state"
|
||||
a.oauthMu.Lock()
|
||||
a.oauthStates[state] = time.Now().Add(-time.Minute)
|
||||
a.oauthMu.Unlock()
|
||||
if a.ConsumeOAuthState(state) {
|
||||
t.Fatal("expired state must be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,11 +7,54 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// JobTokenDuration is the validity period for job tokens
|
||||
const JobTokenDuration = 1 * time.Hour
|
||||
// DefaultJobTokenDuration is the minimum validity period for job tokens.
|
||||
const DefaultJobTokenDuration = 1 * time.Hour
|
||||
|
||||
// JobTokenSkew is extra lifetime added beyond configured task timeouts.
|
||||
const JobTokenSkew = 15 * time.Minute
|
||||
|
||||
// JobTokenDuration is the current validity period for newly issued job tokens.
|
||||
// Prefer JobTokenTTL() / SetJobTokenTTL; this var remains for backward-compatible reads.
|
||||
var (
|
||||
jobTokenTTL = DefaultJobTokenDuration
|
||||
jobTokenTTLMu sync.RWMutex
|
||||
)
|
||||
|
||||
// JobTokenTTL returns the current job token lifetime used when minting tokens.
|
||||
func JobTokenTTL() time.Duration {
|
||||
jobTokenTTLMu.RLock()
|
||||
defer jobTokenTTLMu.RUnlock()
|
||||
return jobTokenTTL
|
||||
}
|
||||
|
||||
// SetJobTokenTTL sets the lifetime for newly issued job tokens.
|
||||
// Values below DefaultJobTokenDuration are raised to the default.
|
||||
func SetJobTokenTTL(d time.Duration) {
|
||||
if d < DefaultJobTokenDuration {
|
||||
d = DefaultJobTokenDuration
|
||||
}
|
||||
jobTokenTTLMu.Lock()
|
||||
jobTokenTTL = d
|
||||
jobTokenTTLMu.Unlock()
|
||||
}
|
||||
|
||||
// ConfigureJobTokenTTLFromTimeouts sets token lifetime from the longest of the
|
||||
// given task timeouts (seconds) plus JobTokenSkew, floored at DefaultJobTokenDuration.
|
||||
func ConfigureJobTokenTTLFromTimeouts(timeoutSeconds ...int) time.Duration {
|
||||
maxSec := 0
|
||||
for _, s := range timeoutSeconds {
|
||||
if s > maxSec {
|
||||
maxSec = s
|
||||
}
|
||||
}
|
||||
d := time.Duration(maxSec)*time.Second + JobTokenSkew
|
||||
SetJobTokenTTL(d)
|
||||
return JobTokenTTL()
|
||||
}
|
||||
|
||||
// JobTokenClaims represents the claims in a job token
|
||||
type JobTokenClaims struct {
|
||||
@@ -41,7 +84,7 @@ func GenerateJobToken(jobID, runnerID, taskID int64) (string, error) {
|
||||
JobID: jobID,
|
||||
RunnerID: runnerID,
|
||||
TaskID: taskID,
|
||||
Exp: time.Now().Add(JobTokenDuration).Unix(),
|
||||
Exp: time.Now().Add(JobTokenTTL()).Unix(),
|
||||
}
|
||||
|
||||
// Encode claims to JSON
|
||||
@@ -112,4 +155,3 @@ func ValidateJobToken(token string) (*JobTokenClaims, error) {
|
||||
|
||||
return &claims, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGenerateAndValidateJobToken_RoundTrip(t *testing.T) {
|
||||
token, err := GenerateJobToken(10, 20, 30)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateJobToken failed: %v", err)
|
||||
}
|
||||
claims, err := ValidateJobToken(token)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateJobToken failed: %v", err)
|
||||
}
|
||||
if claims.JobID != 10 || claims.RunnerID != 20 || claims.TaskID != 30 {
|
||||
t.Fatalf("unexpected claims: %+v", claims)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateJobToken_RejectsTampering(t *testing.T) {
|
||||
token, err := GenerateJobToken(1, 2, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateJobToken failed: %v", err)
|
||||
}
|
||||
parts := strings.Split(token, ".")
|
||||
if len(parts) != 2 {
|
||||
t.Fatalf("unexpected token format: %q", token)
|
||||
}
|
||||
|
||||
rawClaims, err := base64.RawURLEncoding.DecodeString(parts[0])
|
||||
if err != nil {
|
||||
t.Fatalf("decode claims failed: %v", err)
|
||||
}
|
||||
var claims JobTokenClaims
|
||||
if err := json.Unmarshal(rawClaims, &claims); err != nil {
|
||||
t.Fatalf("unmarshal claims failed: %v", err)
|
||||
}
|
||||
claims.JobID = 999
|
||||
tamperedClaims, _ := json.Marshal(claims)
|
||||
tampered := base64.RawURLEncoding.EncodeToString(tamperedClaims) + "." + parts[1]
|
||||
|
||||
if _, err := ValidateJobToken(tampered); err == nil {
|
||||
t.Fatal("expected signature validation error for tampered token")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateJobToken_RejectsExpired(t *testing.T) {
|
||||
expiredClaims := JobTokenClaims{
|
||||
JobID: 1,
|
||||
RunnerID: 2,
|
||||
TaskID: 3,
|
||||
Exp: time.Now().Add(-time.Minute).Unix(),
|
||||
}
|
||||
claimsJSON, _ := json.Marshal(expiredClaims)
|
||||
sigToken, err := GenerateJobToken(1, 2, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateJobToken failed: %v", err)
|
||||
}
|
||||
parts := strings.Split(sigToken, ".")
|
||||
if len(parts) != 2 {
|
||||
t.Fatalf("unexpected token format: %q", sigToken)
|
||||
}
|
||||
// Re-sign expired payload with package secret.
|
||||
h := signClaimsForTest(claimsJSON)
|
||||
expiredToken := base64.RawURLEncoding.EncodeToString(claimsJSON) + "." + base64.RawURLEncoding.EncodeToString(h)
|
||||
|
||||
if _, err := ValidateJobToken(expiredToken); err == nil {
|
||||
t.Fatal("expected token expiration error")
|
||||
}
|
||||
}
|
||||
|
||||
func signClaimsForTest(claims []byte) []byte {
|
||||
h := hmac.New(sha256.New, jobTokenSecret)
|
||||
_, _ = h.Write(claims)
|
||||
return h.Sum(nil)
|
||||
}
|
||||
|
||||
func TestConfigureJobTokenTTLFromTimeouts(t *testing.T) {
|
||||
prev := JobTokenTTL()
|
||||
t.Cleanup(func() { SetJobTokenTTL(prev) })
|
||||
|
||||
// Short timeouts floor at DefaultJobTokenDuration
|
||||
got := ConfigureJobTokenTTLFromTimeouts(60, 120)
|
||||
if got < DefaultJobTokenDuration {
|
||||
t.Fatalf("TTL %v below default %v", got, DefaultJobTokenDuration)
|
||||
}
|
||||
|
||||
// Long encode timeout (24h) + skew must be reflected
|
||||
got = ConfigureJobTokenTTLFromTimeouts(3600, 86400)
|
||||
wantMin := 86400*time.Second + JobTokenSkew
|
||||
if got < wantMin {
|
||||
t.Fatalf("TTL %v < expected min %v for 24h encode", got, wantMin)
|
||||
}
|
||||
|
||||
// Newly generated tokens must use the configured TTL
|
||||
token, err := GenerateJobToken(1, 2, 3)
|
||||
if err != nil {
|
||||
t.Fatalf("GenerateJobToken: %v", err)
|
||||
}
|
||||
claims, err := ValidateJobToken(token)
|
||||
if err != nil {
|
||||
t.Fatalf("ValidateJobToken: %v", err)
|
||||
}
|
||||
// Exp should be roughly now+TTL (allow 30s clock skew in test)
|
||||
remaining := time.Until(time.Unix(claims.Exp, 0))
|
||||
if remaining < wantMin-30*time.Second {
|
||||
t.Fatalf("token remaining lifetime %v too short, want ~%v", remaining, wantMin)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package auth
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"database/sql"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
@@ -121,9 +122,13 @@ func (s *Secrets) ValidateRunnerAPIKey(apiKey string) (int64, string, error) {
|
||||
return 0, "", fmt.Errorf("API key is required")
|
||||
}
|
||||
|
||||
// Check fixed API key first (from database config)
|
||||
// Check fixed API key first (from database config). Constant-time compare.
|
||||
// Fixed keys are refused entirely when production mode is on.
|
||||
fixedKey := s.cfg.FixedAPIKey()
|
||||
if fixedKey != "" && apiKey == fixedKey {
|
||||
if fixedKey != "" && subtle.ConstantTimeCompare([]byte(apiKey), []byte(fixedKey)) == 1 {
|
||||
if s.cfg.IsProductionMode() {
|
||||
return 0, "", fmt.Errorf("fixed API key is not allowed in production mode")
|
||||
}
|
||||
// Return a special ID for fixed API key (doesn't exist in database)
|
||||
return -1, "manager", nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGenerateSecret_Length(t *testing.T) {
|
||||
secret, err := generateSecret(8)
|
||||
if err != nil {
|
||||
t.Fatalf("generateSecret failed: %v", err)
|
||||
}
|
||||
// hex encoding doubles length
|
||||
if len(secret) != 16 {
|
||||
t.Fatalf("unexpected secret length: %d", len(secret))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateAPIKey_Format(t *testing.T) {
|
||||
s := &Secrets{}
|
||||
key, err := s.generateAPIKey()
|
||||
if err != nil {
|
||||
t.Fatalf("generateAPIKey failed: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(key, "jk_r") {
|
||||
t.Fatalf("unexpected key prefix: %q", key)
|
||||
}
|
||||
if !strings.Contains(key, "_") {
|
||||
t.Fatalf("unexpected key format: %q", key)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,6 +22,15 @@ const (
|
||||
KeyRegistrationEnabled = "registration_enabled"
|
||||
KeyProductionMode = "production_mode"
|
||||
KeyAllowedOrigins = "allowed_origins"
|
||||
KeyFramesPerRenderTask = "frames_per_render_task"
|
||||
|
||||
// Operational limits (seconds / bytes / counts)
|
||||
KeyRenderTimeoutSecs = "render_timeout_seconds"
|
||||
KeyEncodeTimeoutSecs = "encode_timeout_seconds"
|
||||
KeyMaxUploadBytes = "max_upload_bytes"
|
||||
KeySessionCookieMaxAge = "session_cookie_max_age"
|
||||
KeyAPIRateLimit = "api_rate_limit"
|
||||
KeyAuthRateLimit = "auth_rate_limit"
|
||||
)
|
||||
|
||||
// Config manages application configuration stored in the database
|
||||
@@ -85,6 +94,9 @@ func (c *Config) InitializeFromEnv() error {
|
||||
|
||||
// Get retrieves a config value from the database
|
||||
func (c *Config) Get(key string) (string, error) {
|
||||
if c == nil || c.db == nil {
|
||||
return "", nil
|
||||
}
|
||||
var value string
|
||||
err := c.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow("SELECT value FROM settings WHERE key = ?", key).Scan(&value)
|
||||
@@ -291,8 +303,16 @@ func (c *Config) FixedAPIKey() string {
|
||||
return c.GetWithDefault(KeyFixedAPIKey, "")
|
||||
}
|
||||
|
||||
// IsProductionMode returns whether production mode is enabled
|
||||
// IsProductionMode returns whether production mode is enabled.
|
||||
// True if PRODUCTION=true env is set OR DB setting production_mode is true.
|
||||
// This is the single source of truth for Secure cookies, CORS, rate limits, and WS origin.
|
||||
func (c *Config) IsProductionMode() bool {
|
||||
if os.Getenv("PRODUCTION") == "true" {
|
||||
return true
|
||||
}
|
||||
if c == nil || c.db == nil {
|
||||
return false
|
||||
}
|
||||
return c.GetBoolWithDefault(KeyProductionMode, false)
|
||||
}
|
||||
|
||||
@@ -301,3 +321,43 @@ func (c *Config) AllowedOrigins() string {
|
||||
return c.GetWithDefault(KeyAllowedOrigins, "")
|
||||
}
|
||||
|
||||
// GetFramesPerRenderTask returns how many frames to include per render task (min 1, default 1).
|
||||
func (c *Config) GetFramesPerRenderTask() int {
|
||||
n := c.GetIntWithDefault(KeyFramesPerRenderTask, 1)
|
||||
if n < 1 {
|
||||
return 1
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// RenderTimeoutSeconds returns the per-frame render timeout in seconds (default 3600 = 1 hour).
|
||||
func (c *Config) RenderTimeoutSeconds() int {
|
||||
return c.GetIntWithDefault(KeyRenderTimeoutSecs, 3600)
|
||||
}
|
||||
|
||||
// EncodeTimeoutSeconds returns the video encode timeout in seconds (default 86400 = 24 hours).
|
||||
func (c *Config) EncodeTimeoutSeconds() int {
|
||||
return c.GetIntWithDefault(KeyEncodeTimeoutSecs, 86400)
|
||||
}
|
||||
|
||||
// MaxUploadBytes returns the maximum upload size in bytes (default 50 GB).
|
||||
func (c *Config) MaxUploadBytes() int64 {
|
||||
v := c.GetIntWithDefault(KeyMaxUploadBytes, 50<<30)
|
||||
return int64(v)
|
||||
}
|
||||
|
||||
// SessionCookieMaxAgeSec returns the session cookie max-age in seconds (default 86400 = 24 hours).
|
||||
func (c *Config) SessionCookieMaxAgeSec() int {
|
||||
return c.GetIntWithDefault(KeySessionCookieMaxAge, 86400)
|
||||
}
|
||||
|
||||
// APIRateLimitPerMinute returns the API rate limit (requests per minute per IP, default 100).
|
||||
func (c *Config) APIRateLimitPerMinute() int {
|
||||
return c.GetIntWithDefault(KeyAPIRateLimit, 100)
|
||||
}
|
||||
|
||||
// AuthRateLimitPerMinute returns the auth rate limit (requests per minute per IP, default 10).
|
||||
func (c *Config) AuthRateLimitPerMinute() int {
|
||||
return c.GetIntWithDefault(KeyAuthRateLimit, 10)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"jiggablend/internal/database"
|
||||
)
|
||||
|
||||
func newTestConfig(t *testing.T) *Config {
|
||||
t.Helper()
|
||||
db, err := database.NewDB(filepath.Join(t.TempDir(), "cfg.db"))
|
||||
if err != nil {
|
||||
t.Fatalf("NewDB failed: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = db.Close() })
|
||||
return NewConfig(db)
|
||||
}
|
||||
|
||||
func TestSetGetExistsDelete(t *testing.T) {
|
||||
cfg := newTestConfig(t)
|
||||
|
||||
if err := cfg.Set("alpha", "1"); err != nil {
|
||||
t.Fatalf("Set failed: %v", err)
|
||||
}
|
||||
v, err := cfg.Get("alpha")
|
||||
if err != nil {
|
||||
t.Fatalf("Get failed: %v", err)
|
||||
}
|
||||
if v != "1" {
|
||||
t.Fatalf("unexpected value: %q", v)
|
||||
}
|
||||
|
||||
exists, err := cfg.Exists("alpha")
|
||||
if err != nil {
|
||||
t.Fatalf("Exists failed: %v", err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatal("expected key to exist")
|
||||
}
|
||||
|
||||
if err := cfg.Delete("alpha"); err != nil {
|
||||
t.Fatalf("Delete failed: %v", err)
|
||||
}
|
||||
exists, err = cfg.Exists("alpha")
|
||||
if err != nil {
|
||||
t.Fatalf("Exists after delete failed: %v", err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("expected key to be deleted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetIntWithDefault_AndMinimumFrameTask(t *testing.T) {
|
||||
cfg := newTestConfig(t)
|
||||
if got := cfg.GetIntWithDefault("missing", 17); got != 17 {
|
||||
t.Fatalf("expected default value, got %d", got)
|
||||
}
|
||||
if err := cfg.SetInt(KeyFramesPerRenderTask, 0); err != nil {
|
||||
t.Fatalf("SetInt failed: %v", err)
|
||||
}
|
||||
if got := cfg.GetFramesPerRenderTask(); got != 1 {
|
||||
t.Fatalf("expected clamped value 1, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
-- SQLite does not support DROP COLUMN directly; recreate table without frame_end
|
||||
CREATE TABLE tasks_new (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
job_id INTEGER NOT NULL,
|
||||
runner_id INTEGER,
|
||||
frame INTEGER NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
output_path TEXT,
|
||||
task_type TEXT NOT NULL DEFAULT 'render',
|
||||
current_step TEXT,
|
||||
retry_count INTEGER NOT NULL DEFAULT 0,
|
||||
max_retries INTEGER NOT NULL DEFAULT 3,
|
||||
runner_failure_count INTEGER NOT NULL DEFAULT 0,
|
||||
timeout_seconds INTEGER,
|
||||
condition TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
started_at TIMESTAMP,
|
||||
completed_at TIMESTAMP,
|
||||
error_message TEXT,
|
||||
FOREIGN KEY (job_id) REFERENCES jobs(id),
|
||||
FOREIGN KEY (runner_id) REFERENCES runners(id)
|
||||
);
|
||||
INSERT INTO tasks_new (id, job_id, runner_id, frame, status, output_path, task_type, current_step, retry_count, max_retries, runner_failure_count, timeout_seconds, condition, created_at, started_at, completed_at, error_message)
|
||||
SELECT id, job_id, runner_id, frame, status, output_path, task_type, current_step, retry_count, max_retries, runner_failure_count, timeout_seconds, condition, created_at, started_at, completed_at, error_message FROM tasks;
|
||||
DROP TABLE tasks;
|
||||
ALTER TABLE tasks_new RENAME TO tasks;
|
||||
CREATE INDEX idx_tasks_job_id ON tasks(job_id);
|
||||
CREATE INDEX idx_tasks_runner_id ON tasks(runner_id);
|
||||
CREATE INDEX idx_tasks_status ON tasks(status);
|
||||
CREATE INDEX idx_tasks_job_status ON tasks(job_id, status);
|
||||
CREATE INDEX idx_tasks_started_at ON tasks(started_at);
|
||||
@@ -0,0 +1,2 @@
|
||||
-- Add frame_end to tasks for range-based render tasks (NULL = single frame, same as frame)
|
||||
ALTER TABLE tasks ADD COLUMN frame_end INTEGER;
|
||||
@@ -0,0 +1,24 @@
|
||||
-- SQLite does not support DROP COLUMN directly; recreate the table without last_used_at.
|
||||
CREATE TABLE runner_api_keys_backup (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
key_prefix TEXT NOT NULL,
|
||||
key_hash TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
scope TEXT NOT NULL DEFAULT 'user',
|
||||
is_active INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
created_by INTEGER,
|
||||
FOREIGN KEY (created_by) REFERENCES users(id),
|
||||
UNIQUE(key_prefix)
|
||||
);
|
||||
|
||||
INSERT INTO runner_api_keys_backup SELECT id, key_prefix, key_hash, name, description, scope, is_active, created_at, created_by FROM runner_api_keys;
|
||||
|
||||
DROP TABLE runner_api_keys;
|
||||
|
||||
ALTER TABLE runner_api_keys_backup RENAME TO runner_api_keys;
|
||||
|
||||
CREATE INDEX idx_runner_api_keys_prefix ON runner_api_keys(key_prefix);
|
||||
CREATE INDEX idx_runner_api_keys_active ON runner_api_keys(is_active);
|
||||
CREATE INDEX idx_runner_api_keys_created_by ON runner_api_keys(created_by);
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE runner_api_keys ADD COLUMN last_used_at TIMESTAMP;
|
||||
@@ -0,0 +1,58 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewDB_RunsMigrationsAndSupportsQueries(t *testing.T) {
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
db, err := NewDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("NewDB failed: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := db.Ping(); err != nil {
|
||||
t.Fatalf("Ping failed: %v", err)
|
||||
}
|
||||
|
||||
var exists bool
|
||||
err = db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='settings')").Scan(&exists)
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query failed: %v", err)
|
||||
}
|
||||
if !exists {
|
||||
t.Fatal("expected settings table after migrations")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithTx_RollbackOnError(t *testing.T) {
|
||||
dbPath := filepath.Join(t.TempDir(), "tx.db")
|
||||
db, err := NewDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("NewDB failed: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
_ = db.WithTx(func(tx *sql.Tx) error {
|
||||
if _, err := tx.Exec("INSERT INTO settings (key, value) VALUES (?, ?)", "rollback_key", "x"); err != nil {
|
||||
return err
|
||||
}
|
||||
return sql.ErrTxDone
|
||||
})
|
||||
|
||||
var count int
|
||||
if err := db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow("SELECT COUNT(*) FROM settings WHERE key = ?", "rollback_key").Scan(&count)
|
||||
}); err != nil {
|
||||
t.Fatalf("count query failed: %v", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("expected rollback, found %d rows", count)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package logger
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseLevel(t *testing.T) {
|
||||
if ParseLevel("debug") != LevelDebug {
|
||||
t.Fatal("debug should map to LevelDebug")
|
||||
}
|
||||
if ParseLevel("warning") != LevelWarn {
|
||||
t.Fatal("warning should map to LevelWarn")
|
||||
}
|
||||
if ParseLevel("unknown") != LevelInfo {
|
||||
t.Fatal("unknown should default to LevelInfo")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetAndGetLevel(t *testing.T) {
|
||||
SetLevel(LevelError)
|
||||
if GetLevel() != LevelError {
|
||||
t.Fatalf("GetLevel() = %v, want %v", GetLevel(), LevelError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewWithFile_CreatesFile(t *testing.T) {
|
||||
logPath := filepath.Join(t.TempDir(), "runner.log")
|
||||
l, err := NewWithFile(logPath)
|
||||
if err != nil {
|
||||
t.Fatalf("NewWithFile failed: %v", err)
|
||||
}
|
||||
defer l.Close()
|
||||
}
|
||||
|
||||
+12
-31
@@ -121,37 +121,6 @@ func (s *Manager) handleDeleteRunnerAPIKey(w http.ResponseWriter, r *http.Reques
|
||||
s.respondJSON(w, http.StatusOK, map[string]string{"message": "API key deleted"})
|
||||
}
|
||||
|
||||
// handleVerifyRunner manually verifies a runner
|
||||
func (s *Manager) handleVerifyRunner(w http.ResponseWriter, r *http.Request) {
|
||||
runnerID, err := parseID(r, "id")
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Check if runner exists
|
||||
var exists bool
|
||||
err = s.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow("SELECT EXISTS(SELECT 1 FROM runners WHERE id = ?)", runnerID).Scan(&exists)
|
||||
})
|
||||
if err != nil || !exists {
|
||||
s.respondError(w, http.StatusNotFound, "Runner not found")
|
||||
return
|
||||
}
|
||||
|
||||
// Mark runner as verified
|
||||
err = s.db.With(func(conn *sql.DB) error {
|
||||
_, err := conn.Exec("UPDATE runners SET verified = 1 WHERE id = ?", runnerID)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to verify runner: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
s.respondJSON(w, http.StatusOK, map[string]string{"message": "Runner verified"})
|
||||
}
|
||||
|
||||
// handleDeleteRunner removes a runner
|
||||
func (s *Manager) handleDeleteRunner(w http.ResponseWriter, r *http.Request) {
|
||||
runnerID, err := parseID(r, "id")
|
||||
@@ -415,6 +384,12 @@ func (s *Manager) handleSetRegistrationEnabled(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// handleSetUserAdminStatus sets a user's admin status (admin only)
|
||||
func (s *Manager) handleSetUserAdminStatus(w http.ResponseWriter, r *http.Request) {
|
||||
currentUserID, err := getUserID(r)
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusUnauthorized, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
targetUserID, err := parseID(r, "id")
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||
@@ -429,6 +404,12 @@ func (s *Manager) handleSetUserAdminStatus(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
// Prevent admins from revoking their own admin status.
|
||||
if targetUserID == currentUserID && !req.IsAdmin {
|
||||
s.respondError(w, http.StatusBadRequest, "You cannot revoke your own admin status")
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.auth.SetUserAdminStatus(targetUserID, req.IsAdmin); err != nil {
|
||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHandleGenerateRunnerAPIKey_UnauthorizedWithoutContext(t *testing.T) {
|
||||
s := &Manager{}
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/admin/runner-api-keys", bytes.NewBufferString(`{"name":"k"}`))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
s.handleGenerateRunnerAPIKey(rr, req)
|
||||
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusUnauthorized)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGenerateRunnerAPIKey_RejectsBadJSON(t *testing.T) {
|
||||
s := &Manager{}
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/admin/runner-api-keys", bytes.NewBufferString(`{`))
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
s.handleGenerateRunnerAPIKey(rr, req)
|
||||
|
||||
// No auth context means unauthorized happens first; this still validates safe
|
||||
// failure handling for malformed requests in this handler path.
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusUnauthorized)
|
||||
}
|
||||
}
|
||||
|
||||
+11
-137
@@ -3,7 +3,6 @@ package api
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/bzip2"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
@@ -16,6 +15,8 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"jiggablend/pkg/blendfile"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -331,8 +332,9 @@ func (s *Manager) GetBlenderArchivePath(version *BlenderVersion) (string, error)
|
||||
// Need to download and decompress
|
||||
log.Printf("Downloading Blender %s from %s", version.Full, version.URL)
|
||||
|
||||
// 60-minute timeout for large Blender tarballs; stream to disk via io.Copy below
|
||||
client := &http.Client{
|
||||
Timeout: 0, // No timeout for large downloads
|
||||
Timeout: 60 * time.Minute,
|
||||
}
|
||||
resp, err := client.Get(version.URL)
|
||||
if err != nil {
|
||||
@@ -438,144 +440,16 @@ func (s *Manager) cleanupExtractedBlenderFolders(blenderDir string, version *Ble
|
||||
}
|
||||
}
|
||||
|
||||
// ParseBlenderVersionFromFile parses the Blender version that a .blend file was saved with
|
||||
// This reads the file header to determine the version
|
||||
// ParseBlenderVersionFromFile parses the Blender version that a .blend file was saved with.
|
||||
// Delegates to the shared pkg/blendfile implementation.
|
||||
func ParseBlenderVersionFromFile(blendPath string) (major, minor int, err error) {
|
||||
file, err := os.Open(blendPath)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to open blend file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
return ParseBlenderVersionFromReader(file)
|
||||
return blendfile.ParseVersionFromFile(blendPath)
|
||||
}
|
||||
|
||||
// ParseBlenderVersionFromReader parses the Blender version from a reader
|
||||
// Useful for reading from uploaded files without saving to disk first
|
||||
// ParseBlenderVersionFromReader parses the Blender version from a reader.
|
||||
// Delegates to the shared pkg/blendfile implementation.
|
||||
func ParseBlenderVersionFromReader(r io.ReadSeeker) (major, minor int, err error) {
|
||||
// Read the first 12 bytes of the blend file header
|
||||
// Format: BLENDER-v<major><minor><patch> or BLENDER_v<major><minor><patch>
|
||||
// The header is: "BLENDER" (7 bytes) + pointer size (1 byte: '-' for 64-bit, '_' for 32-bit)
|
||||
// + endianness (1 byte: 'v' for little-endian, 'V' for big-endian)
|
||||
// + version (3 bytes: e.g., "402" for 4.02)
|
||||
header := make([]byte, 12)
|
||||
n, err := r.Read(header)
|
||||
if err != nil || n < 12 {
|
||||
return 0, 0, fmt.Errorf("failed to read blend file header: %w", err)
|
||||
}
|
||||
|
||||
// Check for BLENDER magic
|
||||
if string(header[:7]) != "BLENDER" {
|
||||
// Might be compressed - try to decompress
|
||||
r.Seek(0, 0)
|
||||
return parseCompressedBlendVersion(r)
|
||||
}
|
||||
|
||||
// Parse version from bytes 9-11 (3 digits)
|
||||
versionStr := string(header[9:12])
|
||||
var vMajor, vMinor int
|
||||
|
||||
// Version format changed in Blender 3.0
|
||||
// Pre-3.0: "279" = 2.79, "280" = 2.80
|
||||
// 3.0+: "300" = 3.0, "402" = 4.02, "410" = 4.10
|
||||
if len(versionStr) == 3 {
|
||||
// First digit is major version
|
||||
fmt.Sscanf(string(versionStr[0]), "%d", &vMajor)
|
||||
// Next two digits are minor version
|
||||
fmt.Sscanf(versionStr[1:3], "%d", &vMinor)
|
||||
}
|
||||
|
||||
return vMajor, vMinor, nil
|
||||
}
|
||||
|
||||
// parseCompressedBlendVersion handles gzip and zstd compressed blend files
|
||||
func parseCompressedBlendVersion(r io.ReadSeeker) (major, minor int, err error) {
|
||||
// Check for compression magic bytes
|
||||
magic := make([]byte, 4)
|
||||
if _, err := r.Read(magic); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
r.Seek(0, 0)
|
||||
|
||||
if magic[0] == 0x1f && magic[1] == 0x8b {
|
||||
// gzip compressed
|
||||
gzReader, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to create gzip reader: %w", err)
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
header := make([]byte, 12)
|
||||
n, err := gzReader.Read(header)
|
||||
if err != nil || n < 12 {
|
||||
return 0, 0, fmt.Errorf("failed to read compressed blend header: %w", err)
|
||||
}
|
||||
|
||||
if string(header[:7]) != "BLENDER" {
|
||||
return 0, 0, fmt.Errorf("invalid blend file format")
|
||||
}
|
||||
|
||||
versionStr := string(header[9:12])
|
||||
var vMajor, vMinor int
|
||||
if len(versionStr) == 3 {
|
||||
fmt.Sscanf(string(versionStr[0]), "%d", &vMajor)
|
||||
fmt.Sscanf(versionStr[1:3], "%d", &vMinor)
|
||||
}
|
||||
|
||||
return vMajor, vMinor, nil
|
||||
}
|
||||
|
||||
// Check for zstd magic (Blender 3.0+): 0x28 0xB5 0x2F 0xFD
|
||||
if magic[0] == 0x28 && magic[1] == 0xb5 && magic[2] == 0x2f && magic[3] == 0xfd {
|
||||
return parseZstdBlendVersion(r)
|
||||
}
|
||||
|
||||
return 0, 0, fmt.Errorf("unknown blend file format")
|
||||
}
|
||||
|
||||
// parseZstdBlendVersion handles zstd-compressed blend files (Blender 3.0+)
|
||||
// Uses zstd command line tool since Go doesn't have native zstd support
|
||||
func parseZstdBlendVersion(r io.ReadSeeker) (major, minor int, err error) {
|
||||
r.Seek(0, 0)
|
||||
|
||||
// We need to decompress just enough to read the header
|
||||
// Use zstd command to decompress from stdin
|
||||
cmd := exec.Command("zstd", "-d", "-c")
|
||||
cmd.Stdin = r
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to create zstd stdout pipe: %w", err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to start zstd decompression: %w", err)
|
||||
}
|
||||
|
||||
// Read just the header (12 bytes)
|
||||
header := make([]byte, 12)
|
||||
n, readErr := io.ReadFull(stdout, header)
|
||||
|
||||
// Kill the process early - we only need the header
|
||||
cmd.Process.Kill()
|
||||
cmd.Wait()
|
||||
|
||||
if readErr != nil || n < 12 {
|
||||
return 0, 0, fmt.Errorf("failed to read zstd compressed blend header: %v", readErr)
|
||||
}
|
||||
|
||||
if string(header[:7]) != "BLENDER" {
|
||||
return 0, 0, fmt.Errorf("invalid blend file format in zstd archive")
|
||||
}
|
||||
|
||||
versionStr := string(header[9:12])
|
||||
var vMajor, vMinor int
|
||||
if len(versionStr) == 3 {
|
||||
fmt.Sscanf(string(versionStr[0]), "%d", &vMajor)
|
||||
fmt.Sscanf(versionStr[1:3], "%d", &vMinor)
|
||||
}
|
||||
|
||||
return vMajor, vMinor, nil
|
||||
return blendfile.ParseVersionFromReader(r)
|
||||
}
|
||||
|
||||
// handleGetBlenderVersions returns available Blender versions
|
||||
@@ -712,7 +586,7 @@ func (s *Manager) handleDownloadBlender(w http.ResponseWriter, r *http.Request)
|
||||
tarFilename = strings.TrimSuffix(tarFilename, ".bz2")
|
||||
|
||||
w.Header().Set("Content-Type", "application/x-tar")
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", tarFilename))
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", tarFilename))
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", stat.Size()))
|
||||
w.Header().Set("X-Blender-Version", blenderVersion.Full)
|
||||
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// resolveBlenderBinaryPath resolves a Blender executable to an absolute path.
|
||||
func resolveBlenderBinaryPath(blenderBinary string) (string, error) {
|
||||
if blenderBinary == "" {
|
||||
return "", fmt.Errorf("blender binary path is empty")
|
||||
}
|
||||
|
||||
// Already contains a path component; normalize it.
|
||||
if strings.Contains(blenderBinary, string(filepath.Separator)) {
|
||||
absPath, err := filepath.Abs(blenderBinary)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to resolve blender binary path %q: %w", blenderBinary, err)
|
||||
}
|
||||
return absPath, nil
|
||||
}
|
||||
|
||||
// Bare executable name, resolve via PATH.
|
||||
resolvedPath, err := exec.LookPath(blenderBinary)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to locate blender binary %q in PATH: %w", blenderBinary, err)
|
||||
}
|
||||
absPath, err := filepath.Abs(resolvedPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to resolve blender binary path %q: %w", resolvedPath, err)
|
||||
}
|
||||
return absPath, nil
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolveBlenderBinaryPath_WithPathComponent(t *testing.T) {
|
||||
got, err := resolveBlenderBinaryPath("./blender")
|
||||
if err != nil {
|
||||
t.Fatalf("resolveBlenderBinaryPath failed: %v", err)
|
||||
}
|
||||
if !filepath.IsAbs(got) {
|
||||
t.Fatalf("expected absolute path, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBlenderBinaryPath_Empty(t *testing.T) {
|
||||
if _, err := resolveBlenderBinaryPath(""); err == nil {
|
||||
t.Fatal("expected error for empty path")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGetLatestBlenderForMajorMinor_UsesCachedVersions(t *testing.T) {
|
||||
blenderVersionCache.mu.Lock()
|
||||
blenderVersionCache.versions = []BlenderVersion{
|
||||
{Major: 4, Minor: 2, Patch: 1, Full: "4.2.1"},
|
||||
{Major: 4, Minor: 2, Patch: 3, Full: "4.2.3"},
|
||||
{Major: 4, Minor: 1, Patch: 9, Full: "4.1.9"},
|
||||
}
|
||||
blenderVersionCache.fetchedAt = time.Now()
|
||||
blenderVersionCache.mu.Unlock()
|
||||
|
||||
m := &Manager{}
|
||||
v, err := m.GetLatestBlenderForMajorMinor(4, 2)
|
||||
if err != nil {
|
||||
t.Fatalf("GetLatestBlenderForMajorMinor failed: %v", err)
|
||||
}
|
||||
if v.Full != "4.2.3" {
|
||||
t.Fatalf("expected highest patch, got %+v", *v)
|
||||
}
|
||||
}
|
||||
|
||||
+768
-504
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,305 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"jiggablend/internal/storage"
|
||||
)
|
||||
|
||||
func TestGenerateAndCheckETag(t *testing.T) {
|
||||
etag := generateETag(map[string]interface{}{"a": 1})
|
||||
if etag == "" {
|
||||
t.Fatal("expected non-empty etag")
|
||||
}
|
||||
req := httptest.NewRequest("GET", "/x", nil)
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
if !checkETag(req, etag) {
|
||||
t.Fatal("expected etag match")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSessionPhase(t *testing.T) {
|
||||
if got := uploadSessionPhase("uploading"); got != "upload" {
|
||||
t.Fatalf("unexpected phase: %q", got)
|
||||
}
|
||||
if got := uploadSessionPhase("select_blend"); got != "action_required" {
|
||||
t.Fatalf("unexpected phase: %q", got)
|
||||
}
|
||||
if got := uploadSessionPhase("something_else"); got != "processing" {
|
||||
t.Fatalf("unexpected fallback phase: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTarHeader_AndTruncateString(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "a.txt", Mode: 0644, Size: 3, Typeflag: tar.TypeReg})
|
||||
_, _ = tw.Write([]byte("abc"))
|
||||
_ = tw.Close()
|
||||
|
||||
raw := buf.Bytes()
|
||||
if len(raw) < 512 {
|
||||
t.Fatal("tar buffer unexpectedly small")
|
||||
}
|
||||
var h tar.Header
|
||||
if err := parseTarHeader(raw[:512], &h); err != nil {
|
||||
t.Fatalf("parseTarHeader failed: %v", err)
|
||||
}
|
||||
if h.Name != "a.txt" {
|
||||
t.Fatalf("unexpected parsed name: %q", h.Name)
|
||||
}
|
||||
|
||||
if got := truncateString("abcdef", 5); got != "ab..." {
|
||||
t.Fatalf("truncateString = %q, want %q", got, "ab...")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUploadSessionForJobCreation_MissingSession(t *testing.T) {
|
||||
s := &Manager{
|
||||
uploadSessions: map[string]*UploadSession{},
|
||||
}
|
||||
_, _, err := s.validateUploadSessionForJobCreation("missing", 1)
|
||||
if err == nil {
|
||||
t.Fatal("expected validation error for missing session")
|
||||
}
|
||||
sessionErr, ok := err.(*uploadSessionValidationError)
|
||||
if !ok || sessionErr.Code != uploadSessionExpiredCode {
|
||||
t.Fatalf("expected %s validation error, got %#v", uploadSessionExpiredCode, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderTaskTimeoutIsRenderNotEncode(t *testing.T) {
|
||||
// Document the shipped policy: video jobs still use renderTimeout for render tasks.
|
||||
s := &Manager{renderTimeout: 3600, videoEncodeTimeout: 86400}
|
||||
if s.renderTimeout == s.videoEncodeTimeout {
|
||||
t.Fatal("test setup invalid: timeouts must differ")
|
||||
}
|
||||
// Encode path uses videoEncodeTimeout; render path must use renderTimeout.
|
||||
// The create-job handler assigns s.renderTimeout to render tasks (see handleCreateJob).
|
||||
if s.renderTimeout != 3600 {
|
||||
t.Fatalf("renderTimeout = %d", s.renderTimeout)
|
||||
}
|
||||
if s.videoEncodeTimeout != 86400 {
|
||||
t.Fatalf("videoEncodeTimeout = %d", s.videoEncodeTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSanitize_ConsistentWriteExcludeAndBackgroundZip(t *testing.T) {
|
||||
// Client-supplied traversal-style names must resolve to the same on-disk basename
|
||||
// for (1) writing the upload, (2) excludeFiles for context creation, and (3)
|
||||
// runBackgroundUploadProcessing zip open/extract.
|
||||
st, err := storage.NewStorage(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewStorage: %v", err)
|
||||
}
|
||||
clientName := "../../evil/scene.zip"
|
||||
safeName, err := storage.SanitizeFilename(clientName)
|
||||
if err != nil {
|
||||
t.Fatalf("SanitizeFilename: %v", err)
|
||||
}
|
||||
if safeName != "scene.zip" {
|
||||
t.Fatalf("safeName = %q, want scene.zip", safeName)
|
||||
}
|
||||
|
||||
tmpDir, err := st.TempDir("upload-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write a real zip under the sanitized basename (as the upload handler does)
|
||||
zipPath := filepath.Join(tmpDir, safeName)
|
||||
if err := writeMinimalBlendZip(zipPath, "scene.blend"); err != nil {
|
||||
t.Fatalf("write zip: %v", err)
|
||||
}
|
||||
// Extract so context archive has a root .blend; ZIP remains on disk to be excluded
|
||||
if _, err := st.ExtractZip(zipPath, tmpDir); err != nil {
|
||||
t.Fatalf("ExtractZip: %v", err)
|
||||
}
|
||||
if !fileExists(filepath.Join(tmpDir, "scene.blend")) {
|
||||
t.Fatal("expected extracted scene.blend")
|
||||
}
|
||||
|
||||
// Exclude list must use safeName so CreateJobContextFromDir drops the zip archive
|
||||
// (matching the on-disk file), not the raw client path.
|
||||
contextPath, err := st.CreateJobContextFromDir(tmpDir, 1, safeName)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateJobContextFromDir with safe exclude: %v", err)
|
||||
}
|
||||
if !fileExists(contextPath) {
|
||||
t.Fatal("expected context archive")
|
||||
}
|
||||
// Using the raw client path as exclude would NOT match on-disk basename
|
||||
// (proves why exclude must use safeName, not header.Filename with traversal).
|
||||
if clientName == safeName {
|
||||
t.Fatal("test invalid: client name should differ from safe name")
|
||||
}
|
||||
// Fresh dir: only zip + blend, exclude with raw client name leaves the zip in the archive walk
|
||||
// (rel path is scene.zip; client is ../../evil/scene.zip — no match).
|
||||
tmpDir2, err := st.TempDir("upload-excl-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zip2 := filepath.Join(tmpDir2, safeName)
|
||||
if err := writeMinimalBlendZip(zip2, "scene.blend"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.ExtractZip(zip2, tmpDir2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// With wrong exclude (raw client name), CreateJobContextFromDir still succeeds but the
|
||||
// zip may be included. Safer assertion: wrong exclude string does not equal safeName.
|
||||
wrongExclude := clientName
|
||||
if filepath.Base(wrongExclude) == wrongExclude && wrongExclude == safeName {
|
||||
t.Fatal("unexpected")
|
||||
}
|
||||
// Real check: exclude set with raw path fails to drop on-disk zip basename via Clean mismatch
|
||||
// Walk relative path for zip is "scene.zip"; exclude set keys for "../../evil/scene.zip"
|
||||
// after Clean become "../evil/scene.zip" which does not match.
|
||||
_, errWrong := st.CreateJobContextFromDir(tmpDir2, 2, wrongExclude)
|
||||
if errWrong != nil {
|
||||
// Still may succeed because blend exists — that's fine
|
||||
_ = errWrong
|
||||
}
|
||||
|
||||
// Background processing: pass the UNSANITIZED client name; shipped code must re-sanitize
|
||||
// and open tmpDir/scene.zip successfully. Use a clean tmp with only the zip (as after upload).
|
||||
bgDir, err := st.TempDir("upload-bg-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bgZip := filepath.Join(bgDir, safeName)
|
||||
if err := writeMinimalBlendZip(bgZip, "scene.blend"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s := &Manager{
|
||||
storage: st,
|
||||
uploadSessions: map[string]*UploadSession{},
|
||||
}
|
||||
sessionID := "sess-traversal"
|
||||
s.uploadSessions[sessionID] = &UploadSession{
|
||||
SessionID: sessionID,
|
||||
UserID: 7,
|
||||
TempDir: bgDir,
|
||||
Status: "processing",
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
s.runBackgroundUploadProcessing(bgDir, sessionID, 7, clientName, 100, "", "")
|
||||
|
||||
s.uploadSessionsMu.RLock()
|
||||
session := s.uploadSessions[sessionID]
|
||||
s.uploadSessionsMu.RUnlock()
|
||||
if session == nil {
|
||||
t.Fatal("session missing")
|
||||
}
|
||||
if session.Status == "error" {
|
||||
t.Fatalf("background processing failed with unsanitized name: %s", session.ErrorMessage)
|
||||
}
|
||||
if session.Status != "completed" && session.Status != "select_blend" {
|
||||
t.Fatalf("status = %q, want completed or select_blend (got message %q)", session.Status, session.Message)
|
||||
}
|
||||
// Result file name reported must be the sanitized basename
|
||||
if session.ResultFileName != "" && session.ResultFileName != safeName {
|
||||
t.Fatalf("ResultFileName = %q, want %q", session.ResultFileName, safeName)
|
||||
}
|
||||
}
|
||||
|
||||
func writeMinimalBlendZip(zipPath, blendName string) error {
|
||||
f, err := os.Create(zipPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
zw := zip.NewWriter(f)
|
||||
w, err := zw.Create(blendName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Write([]byte("BLENDER-TEST")); err != nil {
|
||||
return err
|
||||
}
|
||||
return zw.Close()
|
||||
}
|
||||
|
||||
func fileExists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func TestValidateUploadSessionForJobCreation_ContextMissing(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
s := &Manager{
|
||||
uploadSessions: map[string]*UploadSession{
|
||||
"s1": {
|
||||
SessionID: "s1",
|
||||
UserID: 9,
|
||||
TempDir: tmpDir,
|
||||
Status: "completed",
|
||||
CreatedAt: time.Now(),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
if _, _, err := s.validateUploadSessionForJobCreation("s1", 9); err == nil {
|
||||
t.Fatal("expected error when context.tar is missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUploadSessionForJobCreation_NotReady(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
s := &Manager{
|
||||
uploadSessions: map[string]*UploadSession{
|
||||
"s1": {
|
||||
SessionID: "s1",
|
||||
UserID: 9,
|
||||
TempDir: tmpDir,
|
||||
Status: "processing",
|
||||
CreatedAt: time.Now(),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, _, err := s.validateUploadSessionForJobCreation("s1", 9)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for session that is not completed")
|
||||
}
|
||||
sessionErr, ok := err.(*uploadSessionValidationError)
|
||||
if !ok || sessionErr.Code != uploadSessionNotReadyCode {
|
||||
t.Fatalf("expected %s validation error, got %#v", uploadSessionNotReadyCode, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateUploadSessionForJobCreation_Success(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
contextPath := filepath.Join(tmpDir, "context.tar")
|
||||
if err := os.WriteFile(contextPath, []byte("tar-bytes"), 0644); err != nil {
|
||||
t.Fatalf("write context.tar: %v", err)
|
||||
}
|
||||
|
||||
s := &Manager{
|
||||
uploadSessions: map[string]*UploadSession{
|
||||
"s1": {
|
||||
SessionID: "s1",
|
||||
UserID: 9,
|
||||
TempDir: tmpDir,
|
||||
Status: "completed",
|
||||
CreatedAt: time.Now(),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
session, gotPath, err := s.validateUploadSessionForJobCreation("s1", 9)
|
||||
if err != nil {
|
||||
t.Fatalf("expected valid session, got error: %v", err)
|
||||
}
|
||||
if session == nil || gotPath != contextPath {
|
||||
t.Fatalf("unexpected result: session=%v path=%q", session, gotPath)
|
||||
}
|
||||
}
|
||||
|
||||
+156
-56
@@ -30,27 +30,22 @@ import (
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// Configuration constants
|
||||
// Configuration constants (non-configurable infrastructure values)
|
||||
const (
|
||||
// WebSocket timeouts
|
||||
WSReadDeadline = 90 * time.Second
|
||||
WSPingInterval = 30 * time.Second
|
||||
WSWriteDeadline = 10 * time.Second
|
||||
|
||||
// Task timeouts
|
||||
RenderTimeout = 60 * 60 // 1 hour for frame rendering
|
||||
VideoEncodeTimeout = 60 * 60 * 24 // 24 hours for encoding
|
||||
|
||||
// Limits
|
||||
MaxUploadSize = 50 << 30 // 50 GB
|
||||
// Infrastructure timers
|
||||
RunnerHeartbeatTimeout = 90 * time.Second
|
||||
TaskDistributionInterval = 10 * time.Second
|
||||
ProgressUpdateThrottle = 2 * time.Second
|
||||
|
||||
// Cookie settings
|
||||
SessionCookieMaxAge = 86400 // 24 hours
|
||||
)
|
||||
|
||||
// Operational limits are loaded from database config at Manager initialization.
|
||||
// Defaults are defined in internal/config/config.go convenience methods.
|
||||
|
||||
// Manager represents the manager server
|
||||
type Manager struct {
|
||||
db *database.DB
|
||||
@@ -59,6 +54,7 @@ type Manager struct {
|
||||
secrets *authpkg.Secrets
|
||||
storage *storage.Storage
|
||||
router *chi.Mux
|
||||
ui *uiRenderer
|
||||
|
||||
// WebSocket connections
|
||||
wsUpgrader websocket.Upgrader
|
||||
@@ -108,6 +104,12 @@ type Manager struct {
|
||||
|
||||
// Server start time for health checks
|
||||
startTime time.Time
|
||||
|
||||
// Configurable operational values loaded from config
|
||||
renderTimeout int // seconds
|
||||
videoEncodeTimeout int // seconds
|
||||
maxUploadSize int64 // bytes
|
||||
sessionCookieMaxAge int // seconds
|
||||
}
|
||||
|
||||
// ClientConnection represents a client WebSocket connection with subscriptions
|
||||
@@ -125,10 +127,24 @@ type ClientConnection struct {
|
||||
type UploadSession struct {
|
||||
SessionID string
|
||||
UserID int64
|
||||
TempDir string
|
||||
Progress float64
|
||||
Status string // "uploading", "processing", "extracting_metadata", "creating_context", "completed", "error"
|
||||
Phase string // "upload", "processing", "ready", "error", "action_required"
|
||||
Message string
|
||||
CreatedAt time.Time
|
||||
// Result fields set when Status is "completed" (for async processing)
|
||||
ResultContextArchive string
|
||||
ResultMetadata interface{} // *types.BlendMetadata when set
|
||||
ResultMainBlendFile string
|
||||
ResultFileName string
|
||||
ResultFileSize int64
|
||||
ResultZipExtracted bool
|
||||
ResultExtractedFilesCnt int
|
||||
ResultMetadataExtracted bool
|
||||
ResultMetadataError string // set when Status is "completed" but metadata extraction failed
|
||||
ErrorMessage string // set when Status is "error"
|
||||
ResultBlendFiles []string // set when Status is "select_blend" (relative paths for user to pick)
|
||||
}
|
||||
|
||||
// NewManager creates a new manager server
|
||||
@@ -137,6 +153,10 @@ func NewManager(db *database.DB, cfg *config.Config, auth *authpkg.Auth, storage
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize secrets: %w", err)
|
||||
}
|
||||
ui, err := newUIRenderer()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to initialize UI renderer: %w", err)
|
||||
}
|
||||
|
||||
s := &Manager{
|
||||
db: db,
|
||||
@@ -145,9 +165,14 @@ func NewManager(db *database.DB, cfg *config.Config, auth *authpkg.Auth, storage
|
||||
secrets: secrets,
|
||||
storage: storage,
|
||||
router: chi.NewRouter(),
|
||||
ui: ui,
|
||||
startTime: time.Now(),
|
||||
|
||||
renderTimeout: cfg.RenderTimeoutSeconds(),
|
||||
videoEncodeTimeout: cfg.EncodeTimeoutSeconds(),
|
||||
maxUploadSize: cfg.MaxUploadBytes(),
|
||||
sessionCookieMaxAge: cfg.SessionCookieMaxAgeSec(),
|
||||
wsUpgrader: websocket.Upgrader{
|
||||
CheckOrigin: checkWebSocketOrigin,
|
||||
ReadBufferSize: 1024,
|
||||
WriteBufferSize: 1024,
|
||||
},
|
||||
@@ -169,6 +194,17 @@ func NewManager(db *database.DB, cfg *config.Config, auth *authpkg.Auth, storage
|
||||
jobStatusUpdateMu: make(map[int64]*sync.Mutex),
|
||||
}
|
||||
|
||||
// Initialize rate limiters from config
|
||||
apiRateLimiter = NewRateLimiter(cfg.APIRateLimitPerMinute(), time.Minute)
|
||||
authRateLimiter = NewRateLimiter(cfg.AuthRateLimitPerMinute(), time.Minute)
|
||||
|
||||
// WebSocket origin check uses config production mode (not env-only)
|
||||
s.wsUpgrader.CheckOrigin = s.checkWebSocketOrigin
|
||||
|
||||
// Job tokens must outlive the longest task timeout so long encodes can upload results
|
||||
ttl := authpkg.ConfigureJobTokenTTLFromTimeouts(cfg.RenderTimeoutSeconds(), cfg.EncodeTimeoutSeconds())
|
||||
log.Printf("Job token TTL configured to %v (max task timeout + skew)", ttl)
|
||||
|
||||
// Check for required external tools
|
||||
if err := s.checkRequiredTools(); err != nil {
|
||||
return nil, err
|
||||
@@ -202,9 +238,9 @@ func (s *Manager) checkRequiredTools() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkWebSocketOrigin validates WebSocket connection origins
|
||||
// In production mode, only allows same-origin connections or configured allowed origins
|
||||
func checkWebSocketOrigin(r *http.Request) bool {
|
||||
// checkWebSocketOrigin validates WebSocket connection origins using the manager's
|
||||
// production mode config (single source of truth with cookies/CORS/rate limits).
|
||||
func (s *Manager) checkWebSocketOrigin(r *http.Request) bool {
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin == "" {
|
||||
// No origin header - allow (could be non-browser client like runner)
|
||||
@@ -212,14 +248,18 @@ func checkWebSocketOrigin(r *http.Request) bool {
|
||||
}
|
||||
|
||||
// In development mode, allow all origins
|
||||
// Note: This function doesn't have access to Server, so we use authpkg.IsProductionMode()
|
||||
// which checks environment variable. The server setup uses s.cfg.IsProductionMode() for consistency.
|
||||
if !authpkg.IsProductionMode() {
|
||||
if !s.cfg.IsProductionMode() {
|
||||
return true
|
||||
}
|
||||
|
||||
// In production, check against allowed origins
|
||||
allowedOrigins := os.Getenv("ALLOWED_ORIGINS")
|
||||
// In production, check against configured allowed origins (DB config, then env)
|
||||
allowedOrigins := ""
|
||||
if s.cfg != nil {
|
||||
allowedOrigins = s.cfg.AllowedOrigins()
|
||||
}
|
||||
if allowedOrigins == "" {
|
||||
allowedOrigins = os.Getenv("ALLOWED_ORIGINS")
|
||||
}
|
||||
if allowedOrigins == "" {
|
||||
// Default to same-origin only
|
||||
host := r.Host
|
||||
@@ -247,6 +287,7 @@ type RateLimiter struct {
|
||||
mu sync.RWMutex
|
||||
limit int // max requests
|
||||
window time.Duration // time window
|
||||
stopChan chan struct{}
|
||||
}
|
||||
|
||||
// NewRateLimiter creates a new rate limiter
|
||||
@@ -255,12 +296,17 @@ func NewRateLimiter(limit int, window time.Duration) *RateLimiter {
|
||||
requests: make(map[string][]time.Time),
|
||||
limit: limit,
|
||||
window: window,
|
||||
stopChan: make(chan struct{}),
|
||||
}
|
||||
// Start cleanup goroutine
|
||||
go rl.cleanup()
|
||||
return rl
|
||||
}
|
||||
|
||||
// Stop shuts down the cleanup goroutine.
|
||||
func (rl *RateLimiter) Stop() {
|
||||
close(rl.stopChan)
|
||||
}
|
||||
|
||||
// Allow checks if a request from the given IP is allowed
|
||||
func (rl *RateLimiter) Allow(ip string) bool {
|
||||
rl.mu.Lock()
|
||||
@@ -293,32 +339,37 @@ func (rl *RateLimiter) Allow(ip string) bool {
|
||||
// cleanup periodically removes old entries
|
||||
func (rl *RateLimiter) cleanup() {
|
||||
ticker := time.NewTicker(5 * time.Minute)
|
||||
for range ticker.C {
|
||||
rl.mu.Lock()
|
||||
cutoff := time.Now().Add(-rl.window)
|
||||
for ip, reqs := range rl.requests {
|
||||
validReqs := make([]time.Time, 0, len(reqs))
|
||||
for _, t := range reqs {
|
||||
if t.After(cutoff) {
|
||||
validReqs = append(validReqs, t)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
rl.mu.Lock()
|
||||
cutoff := time.Now().Add(-rl.window)
|
||||
for ip, reqs := range rl.requests {
|
||||
validReqs := make([]time.Time, 0, len(reqs))
|
||||
for _, t := range reqs {
|
||||
if t.After(cutoff) {
|
||||
validReqs = append(validReqs, t)
|
||||
}
|
||||
}
|
||||
if len(validReqs) == 0 {
|
||||
delete(rl.requests, ip)
|
||||
} else {
|
||||
rl.requests[ip] = validReqs
|
||||
}
|
||||
}
|
||||
if len(validReqs) == 0 {
|
||||
delete(rl.requests, ip)
|
||||
} else {
|
||||
rl.requests[ip] = validReqs
|
||||
}
|
||||
rl.mu.Unlock()
|
||||
case <-rl.stopChan:
|
||||
return
|
||||
}
|
||||
rl.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Global rate limiters for different endpoint types
|
||||
// Rate limiters — initialized per Manager instance in NewManager.
|
||||
var (
|
||||
// General API rate limiter: 100 requests per minute per IP
|
||||
apiRateLimiter = NewRateLimiter(100, time.Minute)
|
||||
// Auth rate limiter: 10 requests per minute per IP (stricter for login attempts)
|
||||
authRateLimiter = NewRateLimiter(10, time.Minute)
|
||||
apiRateLimiter *RateLimiter
|
||||
authRateLimiter *RateLimiter
|
||||
)
|
||||
|
||||
// rateLimitMiddleware applies rate limiting based on client IP
|
||||
@@ -353,10 +404,23 @@ func rateLimitMiddleware(limiter *RateLimiter) func(http.Handler) http.Handler {
|
||||
}
|
||||
}
|
||||
|
||||
// securityHeadersMiddleware sets baseline security response headers.
|
||||
func securityHeadersMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("X-Frame-Options", "DENY")
|
||||
w.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
|
||||
// Baseline CSP: allow same-origin scripts/styles for embedded UI
|
||||
w.Header().Set("Content-Security-Policy", "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; connect-src 'self' ws: wss:; frame-ancestors 'none'")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// setupMiddleware configures middleware
|
||||
func (s *Manager) setupMiddleware() {
|
||||
s.router.Use(middleware.Logger)
|
||||
s.router.Use(middleware.Recoverer)
|
||||
s.router.Use(securityHeadersMiddleware)
|
||||
// Note: Timeout middleware is NOT applied globally to avoid conflicts with WebSocket connections
|
||||
// WebSocket connections are long-lived and should not have HTTP timeouts
|
||||
|
||||
@@ -450,6 +514,7 @@ func (w *gzipResponseWriter) WriteHeader(statusCode int) {
|
||||
func (s *Manager) setupRoutes() {
|
||||
// Health check endpoint (unauthenticated)
|
||||
s.router.Get("/api/health", s.handleHealthCheck)
|
||||
s.setupUIRoutes()
|
||||
|
||||
// Public routes (with stricter rate limiting for auth endpoints)
|
||||
s.router.Route("/api/auth", func(r chi.Router) {
|
||||
@@ -467,7 +532,10 @@ func (s *Manager) setupRoutes() {
|
||||
r.Post("/local/login", s.handleLocalLogin)
|
||||
r.Post("/logout", s.handleLogout)
|
||||
r.Get("/me", s.handleGetMe)
|
||||
r.Post("/change-password", s.handleChangePassword)
|
||||
// Password change requires an authenticated session
|
||||
r.With(func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(s.auth.Middleware(next.ServeHTTP))
|
||||
}).Post("/change-password", s.handleChangePassword)
|
||||
})
|
||||
|
||||
// Protected routes
|
||||
@@ -477,6 +545,7 @@ func (s *Manager) setupRoutes() {
|
||||
})
|
||||
r.Post("/", s.handleCreateJob)
|
||||
r.Post("/upload", s.handleUploadFileForJobCreation) // Upload before job creation
|
||||
r.Get("/upload/status", s.handleUploadStatus) // Poll upload processing status (session_id query param)
|
||||
r.Get("/", s.handleListJobs)
|
||||
r.Get("/summary", s.handleListJobsSummary)
|
||||
r.Post("/batch", s.handleBatchGetJobs)
|
||||
@@ -487,6 +556,7 @@ func (s *Manager) setupRoutes() {
|
||||
r.Get("/{id}/files", s.handleListJobFiles)
|
||||
r.Get("/{id}/files/count", s.handleGetJobFilesCount)
|
||||
r.Get("/{id}/context", s.handleListContextArchive)
|
||||
r.Get("/{id}/files/exr-zip", s.handleDownloadEXRZip)
|
||||
r.Get("/{id}/files/{fileId}/download", s.handleDownloadJobFile)
|
||||
r.Get("/{id}/files/{fileId}/preview-exr", s.handlePreviewEXR)
|
||||
r.Get("/{id}/video", s.handleStreamVideo)
|
||||
@@ -522,7 +592,6 @@ func (s *Manager) setupRoutes() {
|
||||
r.Delete("/{id}", s.handleDeleteRunnerAPIKey)
|
||||
})
|
||||
r.Get("/", s.handleListRunnersAdmin)
|
||||
r.Post("/{id}/verify", s.handleVerifyRunner)
|
||||
r.Delete("/{id}", s.handleDeleteRunner)
|
||||
})
|
||||
r.Route("/users", func(r chi.Router) {
|
||||
@@ -555,6 +624,7 @@ func (s *Manager) setupRoutes() {
|
||||
return http.HandlerFunc(s.runnerAuthMiddleware(next.ServeHTTP))
|
||||
})
|
||||
r.Get("/blender/download", s.handleDownloadBlender)
|
||||
r.Get("/jobs/{jobId}/status", s.handleGetJobStatusForRunner)
|
||||
r.Get("/jobs/{jobId}/files", s.handleGetJobFilesForRunner)
|
||||
r.Get("/jobs/{jobId}/metadata", s.handleGetJobMetadataForRunner)
|
||||
r.Get("/files/{jobId}/{fileName}", s.handleDownloadFileForRunner)
|
||||
@@ -564,8 +634,8 @@ func (s *Manager) setupRoutes() {
|
||||
// Blender versions API (public, for job submission page)
|
||||
s.router.Get("/api/blender/versions", s.handleGetBlenderVersions)
|
||||
|
||||
// Serve static files (embedded React app with SPA fallback)
|
||||
s.router.Handle("/*", web.SPAHandler())
|
||||
// Static assets for server-rendered UI.
|
||||
s.router.Handle("/assets/*", web.StaticHandler())
|
||||
}
|
||||
|
||||
// ServeHTTP implements http.Handler
|
||||
@@ -586,19 +656,25 @@ func (s *Manager) respondError(w http.ResponseWriter, status int, message string
|
||||
s.respondJSON(w, status, map[string]string{"error": message})
|
||||
}
|
||||
|
||||
func (s *Manager) respondErrorWithCode(w http.ResponseWriter, status int, code, message string) {
|
||||
s.respondJSON(w, status, map[string]string{
|
||||
"error": message,
|
||||
"code": code,
|
||||
})
|
||||
}
|
||||
|
||||
// createSessionCookie creates a secure session cookie with appropriate flags for the environment
|
||||
func createSessionCookie(sessionID string) *http.Cookie {
|
||||
func (s *Manager) createSessionCookie(sessionID string) *http.Cookie {
|
||||
cookie := &http.Cookie{
|
||||
Name: "session_id",
|
||||
Value: sessionID,
|
||||
Path: "/",
|
||||
MaxAge: SessionCookieMaxAge,
|
||||
MaxAge: s.sessionCookieMaxAge,
|
||||
HttpOnly: true,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
}
|
||||
|
||||
// In production mode, set Secure flag to require HTTPS
|
||||
if authpkg.IsProductionMode() {
|
||||
if s.cfg.IsProductionMode() {
|
||||
cookie.Secure = true
|
||||
}
|
||||
|
||||
@@ -676,6 +752,11 @@ func (s *Manager) handleGoogleCallback(w http.ResponseWriter, r *http.Request) {
|
||||
s.respondError(w, http.StatusBadRequest, "Missing code parameter")
|
||||
return
|
||||
}
|
||||
state := r.URL.Query().Get("state")
|
||||
if !s.auth.ConsumeOAuthState(state) {
|
||||
s.respondError(w, http.StatusBadRequest, "Invalid or missing OAuth state")
|
||||
return
|
||||
}
|
||||
|
||||
session, err := s.auth.GoogleCallback(r.Context(), code)
|
||||
if err != nil {
|
||||
@@ -684,12 +765,16 @@ func (s *Manager) handleGoogleCallback(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "/?error=registration_disabled", http.StatusFound)
|
||||
return
|
||||
}
|
||||
if strings.Contains(err.Error(), "already exists") {
|
||||
http.Redirect(w, r, "/?error=account_exists", http.StatusFound)
|
||||
return
|
||||
}
|
||||
s.respondError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
sessionID := s.auth.CreateSession(session)
|
||||
http.SetCookie(w, createSessionCookie(sessionID))
|
||||
http.SetCookie(w, s.createSessionCookie(sessionID))
|
||||
|
||||
http.Redirect(w, r, "/", http.StatusFound)
|
||||
}
|
||||
@@ -709,6 +794,11 @@ func (s *Manager) handleDiscordCallback(w http.ResponseWriter, r *http.Request)
|
||||
s.respondError(w, http.StatusBadRequest, "Missing code parameter")
|
||||
return
|
||||
}
|
||||
state := r.URL.Query().Get("state")
|
||||
if !s.auth.ConsumeOAuthState(state) {
|
||||
s.respondError(w, http.StatusBadRequest, "Invalid or missing OAuth state")
|
||||
return
|
||||
}
|
||||
|
||||
session, err := s.auth.DiscordCallback(r.Context(), code)
|
||||
if err != nil {
|
||||
@@ -717,12 +807,16 @@ func (s *Manager) handleDiscordCallback(w http.ResponseWriter, r *http.Request)
|
||||
http.Redirect(w, r, "/?error=registration_disabled", http.StatusFound)
|
||||
return
|
||||
}
|
||||
if strings.Contains(err.Error(), "already exists") {
|
||||
http.Redirect(w, r, "/?error=account_exists", http.StatusFound)
|
||||
return
|
||||
}
|
||||
s.respondError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
sessionID := s.auth.CreateSession(session)
|
||||
http.SetCookie(w, createSessionCookie(sessionID))
|
||||
http.SetCookie(w, s.createSessionCookie(sessionID))
|
||||
|
||||
http.Redirect(w, r, "/", http.StatusFound)
|
||||
}
|
||||
@@ -815,7 +909,7 @@ func (s *Manager) handleLocalRegister(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
sessionID := s.auth.CreateSession(session)
|
||||
http.SetCookie(w, createSessionCookie(sessionID))
|
||||
http.SetCookie(w, s.createSessionCookie(sessionID))
|
||||
|
||||
s.respondJSON(w, http.StatusCreated, map[string]interface{}{
|
||||
"message": "Registration successful",
|
||||
@@ -852,7 +946,7 @@ func (s *Manager) handleLocalLogin(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
sessionID := s.auth.CreateSession(session)
|
||||
http.SetCookie(w, createSessionCookie(sessionID))
|
||||
http.SetCookie(w, s.createSessionCookie(sessionID))
|
||||
|
||||
s.respondJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"message": "Login successful",
|
||||
@@ -1219,11 +1313,17 @@ func (s *Manager) cleanupOldTempDirectoriesOnce() {
|
||||
now := time.Now()
|
||||
cleanedCount := 0
|
||||
|
||||
// Check upload sessions to avoid deleting active uploads
|
||||
// Check upload sessions to avoid deleting active uploads.
|
||||
// Key by TempDir path (and basename) — not session UUID.
|
||||
s.uploadSessionsMu.RLock()
|
||||
activeSessions := make(map[string]bool)
|
||||
for sessionID := range s.uploadSessions {
|
||||
activeSessions[sessionID] = true
|
||||
activeTempDirs := make(map[string]bool)
|
||||
for _, session := range s.uploadSessions {
|
||||
if session == nil || session.TempDir == "" {
|
||||
continue
|
||||
}
|
||||
clean := filepath.Clean(session.TempDir)
|
||||
activeTempDirs[clean] = true
|
||||
activeTempDirs[filepath.Base(clean)] = true
|
||||
}
|
||||
s.uploadSessionsMu.RUnlock()
|
||||
|
||||
@@ -1235,7 +1335,7 @@ func (s *Manager) cleanupOldTempDirectoriesOnce() {
|
||||
entryPath := filepath.Join(tempPath, entry.Name())
|
||||
|
||||
// Skip if this directory has an active upload session
|
||||
if activeSessions[entryPath] {
|
||||
if activeTempDirs[filepath.Clean(entryPath)] || activeTempDirs[entry.Name()] {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"jiggablend/internal/config"
|
||||
"jiggablend/internal/storage"
|
||||
)
|
||||
|
||||
func TestCheckWebSocketOrigin_DevelopmentAllowsOrigin(t *testing.T) {
|
||||
t.Setenv("PRODUCTION", "")
|
||||
s := &Manager{cfg: &config.Config{}}
|
||||
req := httptest.NewRequest("GET", "http://localhost/ws", nil)
|
||||
req.Host = "localhost:8080"
|
||||
req.Header.Set("Origin", "http://example.com")
|
||||
if !s.checkWebSocketOrigin(req) {
|
||||
t.Fatal("expected development mode to allow origin")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckWebSocketOrigin_ProductionSameHostAllowed(t *testing.T) {
|
||||
t.Setenv("PRODUCTION", "true")
|
||||
t.Setenv("ALLOWED_ORIGINS", "")
|
||||
s := &Manager{cfg: &config.Config{}}
|
||||
req := httptest.NewRequest("GET", "http://localhost/ws", nil)
|
||||
req.Host = "localhost:8080"
|
||||
req.Header.Set("Origin", "http://localhost:8080")
|
||||
if !s.checkWebSocketOrigin(req) {
|
||||
t.Fatal("expected same-host origin to be allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRespondErrorWithCode_IncludesCodeField(t *testing.T) {
|
||||
s := &Manager{}
|
||||
rr := httptest.NewRecorder()
|
||||
s.respondErrorWithCode(rr, http.StatusBadRequest, "UPLOAD_SESSION_EXPIRED", "Upload session expired.")
|
||||
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want %d", rr.Code, http.StatusBadRequest)
|
||||
}
|
||||
var payload map[string]string
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &payload); err != nil {
|
||||
t.Fatalf("failed to decode response: %v", err)
|
||||
}
|
||||
if payload["code"] != "UPLOAD_SESSION_EXPIRED" {
|
||||
t.Fatalf("unexpected code: %q", payload["code"])
|
||||
}
|
||||
if payload["error"] == "" {
|
||||
t.Fatal("expected non-empty error message")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecurityHeadersMiddleware_SetsBaselineHeaders(t *testing.T) {
|
||||
h := securityHeadersMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
h.ServeHTTP(rr, req)
|
||||
if rr.Header().Get("X-Content-Type-Options") != "nosniff" {
|
||||
t.Fatalf("missing nosniff, got %q", rr.Header().Get("X-Content-Type-Options"))
|
||||
}
|
||||
if rr.Header().Get("X-Frame-Options") != "DENY" {
|
||||
t.Fatalf("missing frame deny")
|
||||
}
|
||||
if rr.Header().Get("Content-Security-Policy") == "" {
|
||||
t.Fatal("missing CSP")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupOldTempDirectories_SkipsActiveSessionTempDir(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
st, err := storage.NewStorage(base)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStorage: %v", err)
|
||||
}
|
||||
tempPath := filepath.Join(base, "temp")
|
||||
activeDir, err := os.MkdirTemp(tempPath, "jiggablend-upload-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldTime := time.Now().Add(-2 * time.Hour)
|
||||
_ = os.Chtimes(activeDir, oldTime, oldTime)
|
||||
|
||||
staleDir := filepath.Join(tempPath, "stale-old-dir")
|
||||
if err := os.MkdirAll(staleDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = os.Chtimes(staleDir, oldTime, oldTime)
|
||||
|
||||
s := &Manager{
|
||||
storage: st,
|
||||
uploadSessions: map[string]*UploadSession{
|
||||
"active": {SessionID: "active", TempDir: activeDir},
|
||||
},
|
||||
}
|
||||
s.cleanupOldTempDirectoriesOnce()
|
||||
|
||||
if _, err := os.Stat(activeDir); err != nil {
|
||||
t.Fatalf("active session temp dir was deleted: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(staleDir); !os.IsNotExist(err) {
|
||||
t.Fatalf("stale temp dir should have been removed, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateSessionCookie_UsesConfigProductionMode(t *testing.T) {
|
||||
t.Setenv("PRODUCTION", "true")
|
||||
s := &Manager{cfg: &config.Config{}, sessionCookieMaxAge: 3600}
|
||||
cookie := s.createSessionCookie("sid")
|
||||
if !cookie.Secure {
|
||||
t.Fatal("expected Secure cookie when production mode is on via env")
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,9 @@ import (
|
||||
"jiggablend/pkg/types"
|
||||
)
|
||||
|
||||
var runMetadataCommand = executils.RunCommand
|
||||
var resolveMetadataBlenderPath = resolveBlenderBinaryPath
|
||||
|
||||
// handleGetJobMetadata retrieves metadata for a job
|
||||
func (s *Manager) handleGetJobMetadata(w http.ResponseWriter, r *http.Request) {
|
||||
userID, err := getUserID(r)
|
||||
@@ -141,16 +144,24 @@ func (s *Manager) extractMetadataFromContext(jobID int64) (*types.BlendMetadata,
|
||||
return nil, fmt.Errorf("failed to create extraction script: %w", err)
|
||||
}
|
||||
|
||||
// Make blend file path relative to tmpDir to avoid path resolution issues
|
||||
blendFileRel, err := filepath.Rel(tmpDir, blendFile)
|
||||
// Use absolute paths to avoid path normalization issues with relative traversal.
|
||||
blendFileAbs, err := filepath.Abs(blendFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get relative path for blend file: %w", err)
|
||||
return nil, fmt.Errorf("failed to get absolute path for blend file: %w", err)
|
||||
}
|
||||
scriptPathAbs, err := filepath.Abs(scriptPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get absolute path for extraction script: %w", err)
|
||||
}
|
||||
|
||||
// Execute Blender with Python script using executils
|
||||
result, err := executils.RunCommand(
|
||||
"blender",
|
||||
[]string{"-b", blendFileRel, "--python", "extract_metadata.py"},
|
||||
blenderBinary, err := resolveMetadataBlenderPath("blender")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err := runMetadataCommand(
|
||||
blenderBinary,
|
||||
[]string{"-b", blendFileAbs, "--python", scriptPathAbs},
|
||||
tmpDir,
|
||||
nil, // inherit environment
|
||||
jobID,
|
||||
@@ -225,8 +236,17 @@ func (s *Manager) extractTar(tarPath, destDir string) error {
|
||||
return fmt.Errorf("failed to read tar header: %w", err)
|
||||
}
|
||||
|
||||
// Sanitize path to prevent directory traversal
|
||||
target := filepath.Join(destDir, header.Name)
|
||||
// Sanitize path to prevent directory traversal. TAR stores "/" separators, so normalize first.
|
||||
normalizedHeaderPath := filepath.FromSlash(header.Name)
|
||||
cleanHeaderPath := filepath.Clean(normalizedHeaderPath)
|
||||
if cleanHeaderPath == "." {
|
||||
continue
|
||||
}
|
||||
if filepath.IsAbs(cleanHeaderPath) || strings.HasPrefix(cleanHeaderPath, ".."+string(os.PathSeparator)) || cleanHeaderPath == ".." {
|
||||
log.Printf("ERROR: Invalid file path in TAR - header: %s", header.Name)
|
||||
return fmt.Errorf("invalid file path in archive: %s", header.Name)
|
||||
}
|
||||
target := filepath.Join(destDir, cleanHeaderPath)
|
||||
|
||||
// Ensure target is within destDir
|
||||
cleanTarget := filepath.Clean(target)
|
||||
@@ -237,14 +257,14 @@ func (s *Manager) extractTar(tarPath, destDir string) error {
|
||||
}
|
||||
|
||||
// Create parent directories
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
||||
if err := os.MkdirAll(filepath.Dir(cleanTarget), 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Write file
|
||||
switch header.Typeflag {
|
||||
case tar.TypeReg:
|
||||
outFile, err := os.Create(target)
|
||||
outFile, err := os.Create(cleanTarget)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"jiggablend/pkg/types"
|
||||
)
|
||||
|
||||
func uploadSessionMetadata(session *UploadSession) *types.BlendMetadata {
|
||||
if session == nil || session.ResultMetadata == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch m := session.ResultMetadata.(type) {
|
||||
case *types.BlendMetadata:
|
||||
return m
|
||||
case types.BlendMetadata:
|
||||
meta := m
|
||||
return &meta
|
||||
default:
|
||||
data, err := json.Marshal(session.ResultMetadata)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var meta types.BlendMetadata
|
||||
if err := json.Unmarshal(data, &meta); err != nil {
|
||||
return nil
|
||||
}
|
||||
return &meta
|
||||
}
|
||||
}
|
||||
|
||||
// mergeBlendMetadataForJobCreate starts from upload analysis metadata when available,
|
||||
// then applies explicit job creation overrides from the request.
|
||||
func mergeBlendMetadataForJobCreate(uploadMeta *types.BlendMetadata, req *types.CreateJobRequest) (*types.BlendMetadata, error) {
|
||||
if req.FrameStart == nil || req.FrameEnd == nil {
|
||||
return nil, fmt.Errorf("frame_start and frame_end are required")
|
||||
}
|
||||
|
||||
var metadata types.BlendMetadata
|
||||
if uploadMeta != nil {
|
||||
metadata = *uploadMeta
|
||||
}
|
||||
|
||||
metadata.FrameStart = *req.FrameStart
|
||||
metadata.FrameEnd = *req.FrameEnd
|
||||
|
||||
if req.RenderSettings != nil {
|
||||
metadata.RenderSettings = *req.RenderSettings
|
||||
}
|
||||
if req.OutputFormat != nil {
|
||||
metadata.RenderSettings.OutputFormat = *req.OutputFormat
|
||||
}
|
||||
if req.BlenderVersion != nil && *req.BlenderVersion != "" {
|
||||
metadata.BlenderVersion = *req.BlenderVersion
|
||||
}
|
||||
if req.UnhideObjects != nil {
|
||||
metadata.UnhideObjects = req.UnhideObjects
|
||||
}
|
||||
if req.EnableExecution != nil {
|
||||
metadata.EnableExecution = req.EnableExecution
|
||||
}
|
||||
|
||||
if metadata.BlenderVersion == "" {
|
||||
return nil, fmt.Errorf("blender_version is required (from upload analysis or job request)")
|
||||
}
|
||||
|
||||
return &metadata, nil
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"jiggablend/pkg/types"
|
||||
)
|
||||
|
||||
func TestMergeBlendMetadataForJobCreate_PreservesUploadMetadata(t *testing.T) {
|
||||
uploadMeta := &types.BlendMetadata{
|
||||
FrameStart: -15,
|
||||
FrameEnd: 1468,
|
||||
BlenderVersion: "4.5.11",
|
||||
RenderSettings: types.RenderSettings{
|
||||
ResolutionX: 2560,
|
||||
ResolutionY: 1440,
|
||||
Engine: "cycles",
|
||||
OutputFormat: "EXR",
|
||||
},
|
||||
SceneInfo: types.SceneInfo{
|
||||
ObjectCount: 42,
|
||||
},
|
||||
}
|
||||
|
||||
frameStart := 800
|
||||
frameEnd := 800
|
||||
outputFormat := "EXR"
|
||||
req := &types.CreateJobRequest{
|
||||
FrameStart: &frameStart,
|
||||
FrameEnd: &frameEnd,
|
||||
OutputFormat: &outputFormat,
|
||||
}
|
||||
|
||||
meta, err := mergeBlendMetadataForJobCreate(uploadMeta, req)
|
||||
if err != nil {
|
||||
t.Fatalf("mergeBlendMetadataForJobCreate failed: %v", err)
|
||||
}
|
||||
|
||||
if meta.BlenderVersion != "4.5.11" {
|
||||
t.Fatalf("expected blender_version 4.5.11, got %q", meta.BlenderVersion)
|
||||
}
|
||||
if meta.FrameStart != 800 || meta.FrameEnd != 800 {
|
||||
t.Fatalf("expected frame range 800-800, got %d-%d", meta.FrameStart, meta.FrameEnd)
|
||||
}
|
||||
if meta.RenderSettings.ResolutionX != 2560 || meta.RenderSettings.ResolutionY != 1440 {
|
||||
t.Fatalf("expected resolution 2560x1440, got %dx%d", meta.RenderSettings.ResolutionX, meta.RenderSettings.ResolutionY)
|
||||
}
|
||||
if meta.RenderSettings.Engine != "cycles" {
|
||||
t.Fatalf("expected engine cycles, got %q", meta.RenderSettings.Engine)
|
||||
}
|
||||
if meta.RenderSettings.OutputFormat != "EXR" {
|
||||
t.Fatalf("expected output_format EXR, got %q", meta.RenderSettings.OutputFormat)
|
||||
}
|
||||
if meta.SceneInfo.ObjectCount != 42 {
|
||||
t.Fatalf("expected scene object_count 42, got %d", meta.SceneInfo.ObjectCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeBlendMetadataForJobCreate_RequestOverridesVersion(t *testing.T) {
|
||||
uploadMeta := &types.BlendMetadata{
|
||||
BlenderVersion: "4.5.11",
|
||||
}
|
||||
|
||||
frameStart := 1
|
||||
frameEnd := 1
|
||||
override := "4.2.3"
|
||||
req := &types.CreateJobRequest{
|
||||
FrameStart: &frameStart,
|
||||
FrameEnd: &frameEnd,
|
||||
BlenderVersion: &override,
|
||||
}
|
||||
|
||||
meta, err := mergeBlendMetadataForJobCreate(uploadMeta, req)
|
||||
if err != nil {
|
||||
t.Fatalf("mergeBlendMetadataForJobCreate failed: %v", err)
|
||||
}
|
||||
if meta.BlenderVersion != "4.2.3" {
|
||||
t.Fatalf("expected request override 4.2.3, got %q", meta.BlenderVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeBlendMetadataForJobCreate_RequiresBlenderVersion(t *testing.T) {
|
||||
frameStart := 1
|
||||
frameEnd := 1
|
||||
req := &types.CreateJobRequest{
|
||||
FrameStart: &frameStart,
|
||||
FrameEnd: &frameEnd,
|
||||
}
|
||||
|
||||
if _, err := mergeBlendMetadataForJobCreate(nil, req); err == nil {
|
||||
t.Fatal("expected error when blender_version is missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSessionMetadata(t *testing.T) {
|
||||
session := &UploadSession{
|
||||
ResultMetadata: &types.BlendMetadata{
|
||||
BlenderVersion: "4.5.11",
|
||||
},
|
||||
}
|
||||
|
||||
meta := uploadSessionMetadata(session)
|
||||
if meta == nil || meta.BlenderVersion != "4.5.11" {
|
||||
t.Fatalf("unexpected metadata: %+v", meta)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"jiggablend/internal/storage"
|
||||
"jiggablend/pkg/executils"
|
||||
)
|
||||
|
||||
func TestExtractTar_ExtractsRegularFile(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "ctx/scene.blend", Mode: 0644, Size: 4, Typeflag: tar.TypeReg})
|
||||
_, _ = tw.Write([]byte("data"))
|
||||
_ = tw.Close()
|
||||
|
||||
tarPath := filepath.Join(t.TempDir(), "ctx.tar")
|
||||
if err := os.WriteFile(tarPath, buf.Bytes(), 0644); err != nil {
|
||||
t.Fatalf("write tar: %v", err)
|
||||
}
|
||||
dest := t.TempDir()
|
||||
m := &Manager{}
|
||||
if err := m.extractTar(tarPath, dest); err != nil {
|
||||
t.Fatalf("extractTar failed: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dest, "ctx", "scene.blend")); err != nil {
|
||||
t.Fatalf("expected extracted file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTar_RejectsTraversal(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "../evil.txt", Mode: 0644, Size: 1, Typeflag: tar.TypeReg})
|
||||
_, _ = tw.Write([]byte("x"))
|
||||
_ = tw.Close()
|
||||
|
||||
tarPath := filepath.Join(t.TempDir(), "bad.tar")
|
||||
if err := os.WriteFile(tarPath, buf.Bytes(), 0644); err != nil {
|
||||
t.Fatalf("write tar: %v", err)
|
||||
}
|
||||
m := &Manager{}
|
||||
if err := m.extractTar(tarPath, t.TempDir()); err == nil {
|
||||
t.Fatal("expected path traversal error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractMetadataFromContext_UsesCommandSeam(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
st, err := storage.NewStorage(base)
|
||||
if err != nil {
|
||||
t.Fatalf("new storage: %v", err)
|
||||
}
|
||||
|
||||
jobID := int64(42)
|
||||
jobDir := st.JobPath(jobID)
|
||||
if err := os.MkdirAll(jobDir, 0755); err != nil {
|
||||
t.Fatalf("mkdir job dir: %v", err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "scene.blend", Mode: 0644, Size: 4, Typeflag: tar.TypeReg})
|
||||
_, _ = tw.Write([]byte("fake"))
|
||||
_ = tw.Close()
|
||||
if err := os.WriteFile(filepath.Join(jobDir, "context.tar"), buf.Bytes(), 0644); err != nil {
|
||||
t.Fatalf("write context tar: %v", err)
|
||||
}
|
||||
|
||||
origResolve := resolveMetadataBlenderPath
|
||||
origRun := runMetadataCommand
|
||||
resolveMetadataBlenderPath = func(_ string) (string, error) { return "/usr/bin/blender", nil }
|
||||
runMetadataCommand = func(_ string, _ []string, _ string, _ []string, _ int64, _ *executils.ProcessTracker) (*executils.CommandResult, error) {
|
||||
return &executils.CommandResult{
|
||||
Stdout: `noise
|
||||
{"frame_start":1,"frame_end":3,"has_negative_frames":false,"render_settings":{"resolution_x":1920,"resolution_y":1080,"frame_rate":24,"output_format":"PNG","engine":"CYCLES"},"scene_info":{"camera_count":1,"object_count":2,"material_count":3}}
|
||||
done`,
|
||||
}, nil
|
||||
}
|
||||
defer func() {
|
||||
resolveMetadataBlenderPath = origResolve
|
||||
runMetadataCommand = origRun
|
||||
}()
|
||||
|
||||
m := &Manager{storage: st}
|
||||
meta, err := m.extractMetadataFromContext(jobID)
|
||||
if err != nil {
|
||||
t.Fatalf("extractMetadataFromContext failed: %v", err)
|
||||
}
|
||||
if meta.FrameStart != 1 || meta.FrameEnd != 3 {
|
||||
t.Fatalf("unexpected metadata: %+v", *meta)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
authpkg "jiggablend/internal/auth"
|
||||
"jiggablend/web"
|
||||
)
|
||||
|
||||
type uiRenderer struct {
|
||||
templates *template.Template
|
||||
}
|
||||
|
||||
type pageData struct {
|
||||
Title string
|
||||
CurrentPath string
|
||||
ContentTemplate string
|
||||
PageScript string
|
||||
User *authpkg.Session
|
||||
Error string
|
||||
Notice string
|
||||
Data interface{}
|
||||
}
|
||||
|
||||
func newUIRenderer() (*uiRenderer, error) {
|
||||
tpl, err := template.New("base").Funcs(template.FuncMap{
|
||||
"formatTime": func(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return "-"
|
||||
}
|
||||
return t.Local().Format("2006-01-02 15:04:05")
|
||||
},
|
||||
"statusClass": func(status string) string {
|
||||
switch status {
|
||||
case "completed":
|
||||
return "status-completed"
|
||||
case "running":
|
||||
return "status-running"
|
||||
case "failed":
|
||||
return "status-failed"
|
||||
case "cancelled":
|
||||
return "status-cancelled"
|
||||
case "online":
|
||||
return "status-online"
|
||||
case "offline":
|
||||
return "status-offline"
|
||||
case "busy":
|
||||
return "status-busy"
|
||||
default:
|
||||
return "status-pending"
|
||||
}
|
||||
},
|
||||
"progressInt": func(v float64) int {
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
if v > 100 {
|
||||
return 100
|
||||
}
|
||||
return int(v)
|
||||
},
|
||||
"derefInt": func(v *int) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("%d", *v)
|
||||
},
|
||||
"derefString": func(v *string) string {
|
||||
if v == nil {
|
||||
return ""
|
||||
}
|
||||
return *v
|
||||
},
|
||||
"hasSuffixFold": func(value, suffix string) bool {
|
||||
return strings.HasSuffix(strings.ToLower(value), strings.ToLower(suffix))
|
||||
},
|
||||
}).ParseFS(
|
||||
web.GetTemplateFS(),
|
||||
"templates/*.html",
|
||||
"templates/partials/*.html",
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse templates: %w", err)
|
||||
}
|
||||
return &uiRenderer{templates: tpl}, nil
|
||||
}
|
||||
|
||||
func (r *uiRenderer) render(w http.ResponseWriter, data pageData) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
if err := r.templates.ExecuteTemplate(w, "base", data); err != nil {
|
||||
log.Printf("Template render error: %v", err)
|
||||
http.Error(w, "template render error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (r *uiRenderer) renderTemplate(w http.ResponseWriter, templateName string, data interface{}) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
if err := r.templates.ExecuteTemplate(w, templateName, data); err != nil {
|
||||
log.Printf("Template render error for %s: %v", templateName, err)
|
||||
http.Error(w, "template render error", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package api
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNewUIRendererParsesTemplates(t *testing.T) {
|
||||
renderer, err := newUIRenderer()
|
||||
if err != nil {
|
||||
t.Fatalf("newUIRenderer returned error: %v", err)
|
||||
}
|
||||
if renderer == nil || renderer.templates == nil {
|
||||
t.Fatalf("renderer/templates should not be nil")
|
||||
}
|
||||
}
|
||||
+513
-353
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,100 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseBlenderFrame(t *testing.T) {
|
||||
frame, ok := parseBlenderFrame("Info Fra:2470 Mem:12.00M")
|
||||
if !ok || frame != 2470 {
|
||||
t.Fatalf("parseBlenderFrame() = (%d,%v), want (2470,true)", frame, ok)
|
||||
}
|
||||
if _, ok := parseBlenderFrame("no frame here"); ok {
|
||||
t.Fatal("expected parse to fail for non-frame text")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobTaskCounts_Progress(t *testing.T) {
|
||||
c := &jobTaskCounts{total: 10, completed: 4}
|
||||
if got := c.progress(); got != 40 {
|
||||
t.Fatalf("progress() = %v, want 40", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWSTaskUpdate_LegacyErrorObject(t *testing.T) {
|
||||
// Simulates older runners that JSON-marshaled an error interface as {}
|
||||
raw := json.RawMessage(`{"task_id":99,"success":false,"error":{}}`)
|
||||
update, err := parseWSTaskUpdate(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parseWSTaskUpdate: %v", err)
|
||||
}
|
||||
if update.TaskID != 99 {
|
||||
t.Fatalf("TaskID = %d, want 99", update.TaskID)
|
||||
}
|
||||
if update.Success {
|
||||
t.Fatal("Success = true, want false")
|
||||
}
|
||||
if update.Error != "" {
|
||||
t.Fatalf("Error = %q, want empty (legacy object)", update.Error)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWSTaskUpdate_StringError(t *testing.T) {
|
||||
raw := json.RawMessage(`{"task_id":7,"success":false,"error":"blender failed: signal: segmentation fault (core dumped)","free_requeue":true}`)
|
||||
update, err := parseWSTaskUpdate(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parseWSTaskUpdate: %v", err)
|
||||
}
|
||||
if update.TaskID != 7 || update.Success {
|
||||
t.Fatalf("unexpected update: %+v", update)
|
||||
}
|
||||
if update.Error != "blender failed: signal: segmentation fault (core dumped)" {
|
||||
t.Fatalf("Error = %q", update.Error)
|
||||
}
|
||||
if !update.FreeRequeue {
|
||||
t.Fatal("expected FreeRequeue true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatInferredTaskFailure(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "task failed segfault prefix",
|
||||
in: "Task failed: blender failed: signal: segmentation fault (core dumped)",
|
||||
want: "blender failed: signal: segmentation fault (core dumped)",
|
||||
},
|
||||
{
|
||||
name: "blender render failed prefix",
|
||||
in: "Blender render failed: blender failed: signal: segmentation fault (core dumped)",
|
||||
want: "blender failed: signal: segmentation fault (core dumped)",
|
||||
},
|
||||
{
|
||||
name: "double prefix",
|
||||
in: "Task failed: Blender render failed: blender failed: signal: segmentation fault (core dumped)",
|
||||
want: "blender failed: signal: segmentation fault (core dumped)",
|
||||
},
|
||||
{
|
||||
name: "plain message",
|
||||
in: "blender failed: exit status 1",
|
||||
want: "blender failed: exit status 1",
|
||||
},
|
||||
{
|
||||
name: "empty falls back",
|
||||
in: " ",
|
||||
want: defaultUnexpectedDisconnectError,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := formatInferredTaskFailure(tt.in); got != tt.want {
|
||||
t.Fatalf("formatInferredTaskFailure(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,556 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
authpkg "jiggablend/internal/auth"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
type uiJobSummary struct {
|
||||
ID int64
|
||||
Name string
|
||||
Status string
|
||||
Progress float64
|
||||
FrameStart *int
|
||||
FrameEnd *int
|
||||
OutputFormat *string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type uiTaskSummary struct {
|
||||
ID int64
|
||||
TaskType string
|
||||
Status string
|
||||
Frame int
|
||||
FrameEnd *int
|
||||
CurrentStep string
|
||||
RetryCount int
|
||||
Error string
|
||||
StartedAt *time.Time
|
||||
CompletedAt *time.Time
|
||||
}
|
||||
|
||||
type uiFileSummary struct {
|
||||
ID int64
|
||||
FileName string
|
||||
FileType string
|
||||
FileSize int64
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
func (s *Manager) setupUIRoutes() {
|
||||
s.router.Get("/", s.handleUIRoot)
|
||||
s.router.Get("/login", s.handleUILoginPage)
|
||||
s.router.Post("/logout", s.handleUILogout)
|
||||
|
||||
s.router.Group(func(r chi.Router) {
|
||||
r.Use(func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(s.auth.Middleware(next.ServeHTTP))
|
||||
})
|
||||
r.Get("/jobs", s.handleUIJobsPage)
|
||||
r.Get("/jobs/new", s.handleUINewJobPage)
|
||||
r.Get("/jobs/{id}", s.handleUIJobDetailPage)
|
||||
|
||||
r.Get("/ui/fragments/jobs", s.handleUIJobsFragment)
|
||||
r.Get("/ui/fragments/jobs/{id}/tasks", s.handleUIJobTasksFragment)
|
||||
r.Get("/ui/fragments/jobs/{id}/files", s.handleUIJobFilesFragment)
|
||||
})
|
||||
|
||||
s.router.Group(func(r chi.Router) {
|
||||
r.Use(func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(s.auth.AdminMiddleware(next.ServeHTTP))
|
||||
})
|
||||
r.Get("/admin", s.handleUIAdminPage)
|
||||
r.Get("/ui/fragments/admin/runners", s.handleUIAdminRunnersFragment)
|
||||
r.Get("/ui/fragments/admin/users", s.handleUIAdminUsersFragment)
|
||||
r.Get("/ui/fragments/admin/apikeys", s.handleUIAdminAPIKeysFragment)
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Manager) sessionFromRequest(r *http.Request) (*authpkg.Session, bool) {
|
||||
cookie, err := r.Cookie("session_id")
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return s.auth.GetSession(cookie.Value)
|
||||
}
|
||||
|
||||
func (s *Manager) handleUIRoot(w http.ResponseWriter, r *http.Request) {
|
||||
if _, ok := s.sessionFromRequest(r); ok {
|
||||
http.Redirect(w, r, "/jobs", http.StatusFound)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/login", http.StatusFound)
|
||||
}
|
||||
|
||||
func (s *Manager) handleUILoginPage(w http.ResponseWriter, r *http.Request) {
|
||||
if _, ok := s.sessionFromRequest(r); ok {
|
||||
http.Redirect(w, r, "/jobs", http.StatusFound)
|
||||
return
|
||||
}
|
||||
|
||||
s.ui.render(w, pageData{
|
||||
Title: "Login",
|
||||
CurrentPath: "/login",
|
||||
ContentTemplate: "page_login",
|
||||
PageScript: "/assets/login.js",
|
||||
Data: map[string]interface{}{
|
||||
"google_enabled": s.auth.IsGoogleOAuthConfigured(),
|
||||
"discord_enabled": s.auth.IsDiscordOAuthConfigured(),
|
||||
"local_enabled": s.auth.IsLocalLoginEnabled(),
|
||||
"error": r.URL.Query().Get("error"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Manager) handleUILogout(w http.ResponseWriter, r *http.Request) {
|
||||
cookie, err := r.Cookie("session_id")
|
||||
if err == nil {
|
||||
s.auth.DeleteSession(cookie.Value)
|
||||
}
|
||||
expired := &http.Cookie{
|
||||
Name: "session_id",
|
||||
Value: "",
|
||||
Path: "/",
|
||||
MaxAge: -1,
|
||||
HttpOnly: true,
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
}
|
||||
if s.cfg.IsProductionMode() {
|
||||
expired.Secure = true
|
||||
}
|
||||
http.SetCookie(w, expired)
|
||||
http.Redirect(w, r, "/login", http.StatusFound)
|
||||
}
|
||||
|
||||
func (s *Manager) handleUIJobsPage(w http.ResponseWriter, r *http.Request) {
|
||||
user, _ := s.sessionFromRequest(r)
|
||||
s.ui.render(w, pageData{
|
||||
Title: "Jobs",
|
||||
CurrentPath: "/jobs",
|
||||
ContentTemplate: "page_jobs",
|
||||
PageScript: "/assets/jobs.js",
|
||||
User: user,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Manager) handleUINewJobPage(w http.ResponseWriter, r *http.Request) {
|
||||
user, _ := s.sessionFromRequest(r)
|
||||
s.ui.render(w, pageData{
|
||||
Title: "New Job",
|
||||
CurrentPath: "/jobs/new",
|
||||
ContentTemplate: "page_jobs_new",
|
||||
PageScript: "/assets/job_new.js",
|
||||
User: user,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Manager) handleUIJobDetailPage(w http.ResponseWriter, r *http.Request) {
|
||||
userID, err := getUserID(r)
|
||||
if err != nil {
|
||||
http.Redirect(w, r, "/login", http.StatusFound)
|
||||
return
|
||||
}
|
||||
isAdmin := authpkg.IsAdmin(r.Context())
|
||||
jobID, err := parseID(r, "id")
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
job, err := s.getUIJob(jobID, userID, isAdmin)
|
||||
if err != nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
user, _ := s.sessionFromRequest(r)
|
||||
s.ui.render(w, pageData{
|
||||
Title: fmt.Sprintf("Job %d", jobID),
|
||||
CurrentPath: "/jobs",
|
||||
ContentTemplate: "page_job_show",
|
||||
PageScript: "/assets/job_show.js",
|
||||
User: user,
|
||||
Data: map[string]interface{}{"job": job},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Manager) handleUIAdminPage(w http.ResponseWriter, r *http.Request) {
|
||||
user, _ := s.sessionFromRequest(r)
|
||||
regEnabled, _ := s.auth.IsRegistrationEnabled()
|
||||
s.ui.render(w, pageData{
|
||||
Title: "Admin",
|
||||
CurrentPath: "/admin",
|
||||
ContentTemplate: "page_admin",
|
||||
PageScript: "/assets/admin.js",
|
||||
User: user,
|
||||
Data: map[string]interface{}{
|
||||
"registration_enabled": regEnabled,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Manager) handleUIJobsFragment(w http.ResponseWriter, r *http.Request) {
|
||||
userID, err := getUserID(r)
|
||||
if err != nil {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
jobs, err := s.listUIJobSummaries(userID, 50, 0)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
s.ui.renderTemplate(w, "partial_jobs_table", map[string]interface{}{"jobs": jobs})
|
||||
}
|
||||
|
||||
func (s *Manager) handleUIJobTasksFragment(w http.ResponseWriter, r *http.Request) {
|
||||
userID, err := getUserID(r)
|
||||
if err != nil {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
isAdmin := authpkg.IsAdmin(r.Context())
|
||||
jobID, err := parseID(r, "id")
|
||||
if err != nil {
|
||||
http.Error(w, "invalid job id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := s.getUIJob(jobID, userID, isAdmin); err != nil {
|
||||
http.Error(w, "job not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
tasks, err := s.listUITasks(jobID)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
s.ui.renderTemplate(w, "partial_job_tasks", map[string]interface{}{
|
||||
"job_id": jobID,
|
||||
"tasks": tasks,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Manager) handleUIJobFilesFragment(w http.ResponseWriter, r *http.Request) {
|
||||
userID, err := getUserID(r)
|
||||
if err != nil {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
isAdmin := authpkg.IsAdmin(r.Context())
|
||||
jobID, err := parseID(r, "id")
|
||||
if err != nil {
|
||||
http.Error(w, "invalid job id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := s.getUIJob(jobID, userID, isAdmin); err != nil {
|
||||
http.Error(w, "job not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
files, err := s.listUIFiles(jobID, 100)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
outputFiles := make([]uiFileSummary, 0, len(files))
|
||||
adminInputFiles := make([]uiFileSummary, 0)
|
||||
for _, file := range files {
|
||||
if strings.EqualFold(file.FileType, "output") {
|
||||
outputFiles = append(outputFiles, file)
|
||||
continue
|
||||
}
|
||||
if isAdmin {
|
||||
adminInputFiles = append(adminInputFiles, file)
|
||||
}
|
||||
}
|
||||
|
||||
s.ui.renderTemplate(w, "partial_job_files", map[string]interface{}{
|
||||
"job_id": jobID,
|
||||
"files": outputFiles,
|
||||
"is_admin": isAdmin,
|
||||
"admin_input_files": adminInputFiles,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Manager) handleUIAdminRunnersFragment(w http.ResponseWriter, r *http.Request) {
|
||||
var rows *sql.Rows
|
||||
err := s.db.With(func(conn *sql.DB) error {
|
||||
var qErr error
|
||||
rows, qErr = conn.Query(`SELECT id, name, hostname, status, last_heartbeat, priority, created_at FROM runners ORDER BY created_at DESC`)
|
||||
return qErr
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type runner struct {
|
||||
ID int64
|
||||
Name string
|
||||
Hostname string
|
||||
Status string
|
||||
LastHeartbeat time.Time
|
||||
Priority int
|
||||
CreatedAt time.Time
|
||||
}
|
||||
all := make([]runner, 0)
|
||||
for rows.Next() {
|
||||
var item runner
|
||||
if scanErr := rows.Scan(&item.ID, &item.Name, &item.Hostname, &item.Status, &item.LastHeartbeat, &item.Priority, &item.CreatedAt); scanErr != nil {
|
||||
http.Error(w, scanErr.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
all = append(all, item)
|
||||
}
|
||||
s.ui.renderTemplate(w, "partial_admin_runners", map[string]interface{}{"runners": all})
|
||||
}
|
||||
|
||||
func (s *Manager) handleUIAdminUsersFragment(w http.ResponseWriter, r *http.Request) {
|
||||
currentUserID, _ := getUserID(r)
|
||||
firstUserID, _ := s.auth.GetFirstUserID()
|
||||
var rows *sql.Rows
|
||||
err := s.db.With(func(conn *sql.DB) error {
|
||||
var qErr error
|
||||
rows, qErr = conn.Query(`SELECT id, email, name, oauth_provider, is_admin, created_at FROM users ORDER BY created_at DESC`)
|
||||
return qErr
|
||||
})
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
type user struct {
|
||||
ID int64
|
||||
Email string
|
||||
Name string
|
||||
OAuthProvider string
|
||||
IsAdmin bool
|
||||
IsFirstUser bool
|
||||
CreatedAt time.Time
|
||||
}
|
||||
all := make([]user, 0)
|
||||
for rows.Next() {
|
||||
var item user
|
||||
if scanErr := rows.Scan(&item.ID, &item.Email, &item.Name, &item.OAuthProvider, &item.IsAdmin, &item.CreatedAt); scanErr != nil {
|
||||
http.Error(w, scanErr.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
item.IsFirstUser = item.ID == firstUserID
|
||||
all = append(all, item)
|
||||
}
|
||||
s.ui.renderTemplate(w, "partial_admin_users", map[string]interface{}{
|
||||
"users": all,
|
||||
"current_user_id": currentUserID,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Manager) handleUIAdminAPIKeysFragment(w http.ResponseWriter, r *http.Request) {
|
||||
keys, err := s.secrets.ListRunnerAPIKeys()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
type item struct {
|
||||
ID int64
|
||||
Name string
|
||||
Scope string
|
||||
Key string
|
||||
IsActive bool
|
||||
CreatedAt time.Time
|
||||
}
|
||||
out := make([]item, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
out = append(out, item{
|
||||
ID: key.ID,
|
||||
Name: key.Name,
|
||||
Scope: key.Scope,
|
||||
Key: key.Key,
|
||||
IsActive: key.IsActive,
|
||||
CreatedAt: key.CreatedAt,
|
||||
})
|
||||
}
|
||||
s.ui.renderTemplate(w, "partial_admin_apikeys", map[string]interface{}{"keys": out})
|
||||
}
|
||||
|
||||
func (s *Manager) listUIJobSummaries(userID int64, limit int, offset int) ([]uiJobSummary, error) {
|
||||
rows := &sql.Rows{}
|
||||
err := s.db.With(func(conn *sql.DB) error {
|
||||
var qErr error
|
||||
rows, qErr = conn.Query(
|
||||
`SELECT id, name, status, progress, frame_start, frame_end, output_format, created_at
|
||||
FROM jobs WHERE user_id = ? ORDER BY created_at DESC LIMIT ? OFFSET ?`,
|
||||
userID, limit, offset,
|
||||
)
|
||||
return qErr
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]uiJobSummary, 0)
|
||||
for rows.Next() {
|
||||
var item uiJobSummary
|
||||
var frameStart, frameEnd sql.NullInt64
|
||||
var outputFormat sql.NullString
|
||||
if scanErr := rows.Scan(&item.ID, &item.Name, &item.Status, &item.Progress, &frameStart, &frameEnd, &outputFormat, &item.CreatedAt); scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
if frameStart.Valid {
|
||||
v := int(frameStart.Int64)
|
||||
item.FrameStart = &v
|
||||
}
|
||||
if frameEnd.Valid {
|
||||
v := int(frameEnd.Int64)
|
||||
item.FrameEnd = &v
|
||||
}
|
||||
if outputFormat.Valid {
|
||||
item.OutputFormat = &outputFormat.String
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Manager) getUIJob(jobID int64, userID int64, isAdmin bool) (uiJobSummary, error) {
|
||||
var item uiJobSummary
|
||||
var frameStart, frameEnd sql.NullInt64
|
||||
var outputFormat sql.NullString
|
||||
|
||||
err := s.db.With(func(conn *sql.DB) error {
|
||||
if isAdmin {
|
||||
return conn.QueryRow(
|
||||
`SELECT id, name, status, progress, frame_start, frame_end, output_format, created_at
|
||||
FROM jobs WHERE id = ?`,
|
||||
jobID,
|
||||
).Scan(&item.ID, &item.Name, &item.Status, &item.Progress, &frameStart, &frameEnd, &outputFormat, &item.CreatedAt)
|
||||
}
|
||||
return conn.QueryRow(
|
||||
`SELECT id, name, status, progress, frame_start, frame_end, output_format, created_at
|
||||
FROM jobs WHERE id = ? AND user_id = ?`,
|
||||
jobID, userID,
|
||||
).Scan(&item.ID, &item.Name, &item.Status, &item.Progress, &frameStart, &frameEnd, &outputFormat, &item.CreatedAt)
|
||||
})
|
||||
if err != nil {
|
||||
return uiJobSummary{}, err
|
||||
}
|
||||
if frameStart.Valid {
|
||||
v := int(frameStart.Int64)
|
||||
item.FrameStart = &v
|
||||
}
|
||||
if frameEnd.Valid {
|
||||
v := int(frameEnd.Int64)
|
||||
item.FrameEnd = &v
|
||||
}
|
||||
if outputFormat.Valid {
|
||||
item.OutputFormat = &outputFormat.String
|
||||
}
|
||||
return item, nil
|
||||
}
|
||||
|
||||
func (s *Manager) listUITasks(jobID int64) ([]uiTaskSummary, error) {
|
||||
var rows *sql.Rows
|
||||
err := s.db.With(func(conn *sql.DB) error {
|
||||
var qErr error
|
||||
rows, qErr = conn.Query(
|
||||
`SELECT id, task_type, status, frame, frame_end, current_step, retry_count, error_message, started_at, completed_at
|
||||
FROM tasks WHERE job_id = ? ORDER BY id ASC`,
|
||||
jobID,
|
||||
)
|
||||
return qErr
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]uiTaskSummary, 0)
|
||||
for rows.Next() {
|
||||
var item uiTaskSummary
|
||||
var frameEnd sql.NullInt64
|
||||
var currentStep sql.NullString
|
||||
var errMsg sql.NullString
|
||||
var startedAt, completedAt sql.NullTime
|
||||
if scanErr := rows.Scan(
|
||||
&item.ID, &item.TaskType, &item.Status, &item.Frame, &frameEnd,
|
||||
¤tStep, &item.RetryCount, &errMsg, &startedAt, &completedAt,
|
||||
); scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
if frameEnd.Valid {
|
||||
v := int(frameEnd.Int64)
|
||||
item.FrameEnd = &v
|
||||
}
|
||||
if currentStep.Valid {
|
||||
item.CurrentStep = currentStep.String
|
||||
}
|
||||
if errMsg.Valid {
|
||||
item.Error = errMsg.String
|
||||
}
|
||||
if startedAt.Valid {
|
||||
item.StartedAt = &startedAt.Time
|
||||
}
|
||||
if completedAt.Valid {
|
||||
item.CompletedAt = &completedAt.Time
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *Manager) listUIFiles(jobID int64, limit int) ([]uiFileSummary, error) {
|
||||
var rows *sql.Rows
|
||||
err := s.db.With(func(conn *sql.DB) error {
|
||||
var qErr error
|
||||
rows, qErr = conn.Query(
|
||||
`SELECT id, file_name, file_type, file_size, created_at
|
||||
FROM job_files WHERE job_id = ? ORDER BY created_at DESC LIMIT ?`,
|
||||
jobID, limit,
|
||||
)
|
||||
return qErr
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]uiFileSummary, 0)
|
||||
for rows.Next() {
|
||||
var item uiFileSummary
|
||||
if scanErr := rows.Scan(&item.ID, &item.FileName, &item.FileType, &item.FileSize, &item.CreatedAt); scanErr != nil {
|
||||
return nil, scanErr
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func parseBoolForm(r *http.Request, key string) bool {
|
||||
v := strings.TrimSpace(strings.ToLower(r.FormValue(key)))
|
||||
return v == "1" || v == "true" || v == "on" || v == "yes"
|
||||
}
|
||||
|
||||
func parseIntQuery(r *http.Request, key string, fallback int) int {
|
||||
raw := strings.TrimSpace(r.URL.Query().Get(key))
|
||||
if raw == "" {
|
||||
return fallback
|
||||
}
|
||||
v, err := strconv.Atoi(raw)
|
||||
if err != nil || v < 0 {
|
||||
return fallback
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseBoolForm(t *testing.T) {
|
||||
req := httptest.NewRequest("POST", "/?flag=true", nil)
|
||||
req.ParseForm()
|
||||
req.Form.Set("enabled", "true")
|
||||
if !parseBoolForm(req, "enabled") {
|
||||
t.Fatalf("expected true for enabled=true")
|
||||
}
|
||||
|
||||
req.Form.Set("enabled", "no")
|
||||
if parseBoolForm(req, "enabled") {
|
||||
t.Fatalf("expected false for enabled=no")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseIntQuery(t *testing.T) {
|
||||
req := httptest.NewRequest("GET", "/?limit=42", nil)
|
||||
if got := parseIntQuery(req, "limit", 10); got != 42 {
|
||||
t.Fatalf("expected 42, got %d", got)
|
||||
}
|
||||
|
||||
req = httptest.NewRequest("GET", "/?limit=-1", nil)
|
||||
if got := parseIntQuery(req, "limit", 10); got != 10 {
|
||||
t.Fatalf("expected fallback 10, got %d", got)
|
||||
}
|
||||
|
||||
req = httptest.NewRequest("GET", "/?limit=abc", nil)
|
||||
if got := parseIntQuery(req, "limit", 10); got != 10 {
|
||||
t.Fatalf("expected fallback 10, got %d", got)
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ type JobConnection struct {
|
||||
writeMu sync.Mutex
|
||||
stopPing chan struct{}
|
||||
stopHeartbeat chan struct{}
|
||||
stopOnce sync.Once
|
||||
isConnected bool
|
||||
connMu sync.RWMutex
|
||||
}
|
||||
@@ -132,13 +133,12 @@ func (j *JobConnection) pingLoop() {
|
||||
|
||||
// Heartbeat sends a heartbeat message over WebSocket to keep runner online.
|
||||
func (j *JobConnection) Heartbeat() {
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
if j.conn == nil {
|
||||
return
|
||||
}
|
||||
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
|
||||
msg := map[string]interface{}{
|
||||
"type": "runner_heartbeat",
|
||||
"timestamp": time.Now().Unix(),
|
||||
@@ -178,27 +178,34 @@ func (j *JobConnection) heartbeatLoop() {
|
||||
}
|
||||
}
|
||||
|
||||
// stopLoops signals ping/heartbeat goroutines to exit.
|
||||
// Channels are closed once and left non-nil so loops can receive without racing Close.
|
||||
func (j *JobConnection) stopLoops() {
|
||||
j.stopOnce.Do(func() {
|
||||
if j.stopHeartbeat != nil {
|
||||
close(j.stopHeartbeat)
|
||||
}
|
||||
if j.stopPing != nil {
|
||||
close(j.stopPing)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Close closes the WebSocket connection.
|
||||
func (j *JobConnection) Close() {
|
||||
j.stopLoops()
|
||||
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
|
||||
j.connMu.Lock()
|
||||
j.isConnected = false
|
||||
conn := j.conn
|
||||
j.conn = nil
|
||||
j.connMu.Unlock()
|
||||
|
||||
// Stop heartbeat goroutine
|
||||
if j.stopHeartbeat != nil {
|
||||
close(j.stopHeartbeat)
|
||||
j.stopHeartbeat = nil
|
||||
}
|
||||
|
||||
// Stop ping goroutine
|
||||
if j.stopPing != nil {
|
||||
close(j.stopPing)
|
||||
j.stopPing = nil
|
||||
}
|
||||
|
||||
if j.conn != nil {
|
||||
j.conn.Close()
|
||||
j.conn = nil
|
||||
if conn != nil {
|
||||
conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -211,13 +218,12 @@ func (j *JobConnection) IsConnected() bool {
|
||||
|
||||
// Log sends a log entry to the manager.
|
||||
func (j *JobConnection) Log(taskID int64, level types.LogLevel, message string) {
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
if j.conn == nil {
|
||||
return
|
||||
}
|
||||
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
|
||||
msg := map[string]interface{}{
|
||||
"type": "log_entry",
|
||||
"data": map[string]interface{}{
|
||||
@@ -242,13 +248,12 @@ func (j *JobConnection) Log(taskID int64, level types.LogLevel, message string)
|
||||
|
||||
// Progress sends a progress update to the manager.
|
||||
func (j *JobConnection) Progress(taskID int64, progress float64) {
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
if j.conn == nil {
|
||||
return
|
||||
}
|
||||
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
|
||||
msg := map[string]interface{}{
|
||||
"type": "progress",
|
||||
"data": map[string]interface{}{
|
||||
@@ -272,13 +277,12 @@ func (j *JobConnection) Progress(taskID int64, progress float64) {
|
||||
|
||||
// OutputUploaded notifies that an output file was uploaded.
|
||||
func (j *JobConnection) OutputUploaded(taskID int64, fileName string) {
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
if j.conn == nil {
|
||||
return
|
||||
}
|
||||
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
|
||||
msg := map[string]interface{}{
|
||||
"type": "output_uploaded",
|
||||
"data": map[string]interface{}{
|
||||
@@ -301,22 +305,33 @@ func (j *JobConnection) OutputUploaded(taskID int64, fileName string) {
|
||||
}
|
||||
|
||||
// Complete sends task completion to the manager.
|
||||
func (j *JobConnection) Complete(taskID int64, success bool, errorMsg error) {
|
||||
// errorMsg must be JSON-encoded as a string: encoding an error interface
|
||||
// marshals to {} and the manager fails to unmarshal task_complete, which
|
||||
// previously surfaced as a generic "WebSocket connection lost" with no retries.
|
||||
// freeRequeue asks the manager to requeue a failure without incrementing retry_count
|
||||
// (used when this attempt newly armed GPU lockout).
|
||||
func (j *JobConnection) Complete(taskID int64, success bool, errorMsg error, freeRequeue bool) {
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
if j.conn == nil {
|
||||
log.Printf("Cannot send task complete: WebSocket connection is nil")
|
||||
return
|
||||
}
|
||||
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
data := map[string]interface{}{
|
||||
"task_id": taskID,
|
||||
"success": success,
|
||||
}
|
||||
if errorMsg != nil {
|
||||
data["error"] = errorMsg.Error()
|
||||
}
|
||||
if freeRequeue {
|
||||
data["free_requeue"] = true
|
||||
}
|
||||
|
||||
msg := map[string]interface{}{
|
||||
"type": "task_complete",
|
||||
"data": map[string]interface{}{
|
||||
"task_id": taskID,
|
||||
"success": success,
|
||||
"error": errorMsg,
|
||||
},
|
||||
"type": "task_complete",
|
||||
"data": data,
|
||||
"timestamp": time.Now().Unix(),
|
||||
}
|
||||
if err := j.conn.WriteJSON(msg); err != nil {
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// TestComplete_ErrorIsJSONString ensures task_complete encodes the failure
|
||||
// reason as a string. Marshaling a Go error interface produces {}, which the
|
||||
// manager cannot unmarshal into WSTaskUpdate.Error and used to look like a
|
||||
// silent WebSocket drop with no retries.
|
||||
func TestComplete_ErrorIsJSONString(t *testing.T) {
|
||||
upgrader := websocket.Upgrader{}
|
||||
received := make(chan map[string]interface{}, 1)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := upgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
// auth handshake
|
||||
var auth map[string]interface{}
|
||||
if err := conn.ReadJSON(&auth); err != nil {
|
||||
return
|
||||
}
|
||||
if auth["type"] == "auth" {
|
||||
_ = conn.WriteJSON(map[string]string{"type": "auth_ok"})
|
||||
}
|
||||
|
||||
var msg map[string]interface{}
|
||||
if err := conn.ReadJSON(&msg); err != nil {
|
||||
return
|
||||
}
|
||||
received <- msg
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
jc := NewJobConnection()
|
||||
if err := jc.Connect(server.URL, "/job/1", "token123"); err != nil {
|
||||
t.Fatalf("Connect failed: %v", err)
|
||||
}
|
||||
defer jc.Close()
|
||||
|
||||
jc.Complete(42, false, errors.New("blender failed: signal: segmentation fault (core dumped)"), true)
|
||||
|
||||
select {
|
||||
case msg := <-received:
|
||||
if msg["type"] != "task_complete" {
|
||||
t.Fatalf("type = %v, want task_complete", msg["type"])
|
||||
}
|
||||
data, ok := msg["data"].(map[string]interface{})
|
||||
if !ok {
|
||||
// gorilla may leave nested objects as map[string]interface{} after JSON round-trip;
|
||||
// also accept raw re-marshal path
|
||||
raw, _ := json.Marshal(msg["data"])
|
||||
data = map[string]interface{}{}
|
||||
if err := json.Unmarshal(raw, &data); err != nil {
|
||||
t.Fatalf("data type %T: %v", msg["data"], err)
|
||||
}
|
||||
}
|
||||
errVal, ok := data["error"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("error field type %T value %#v; want string (not object)", data["error"], data["error"])
|
||||
}
|
||||
if errVal != "blender failed: signal: segmentation fault (core dumped)" {
|
||||
t.Fatalf("error = %q, want segfault message", errVal)
|
||||
}
|
||||
if data["success"] != false {
|
||||
t.Fatalf("success = %v, want false", data["success"])
|
||||
}
|
||||
if data["free_requeue"] != true {
|
||||
t.Fatalf("free_requeue = %v, want true", data["free_requeue"])
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("timed out waiting for task_complete message")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobConnection_ConnectAndClose(t *testing.T) {
|
||||
upgrader := websocket.Upgrader{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := upgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
var msg map[string]interface{}
|
||||
if err := conn.ReadJSON(&msg); err != nil {
|
||||
return
|
||||
}
|
||||
if msg["type"] == "auth" {
|
||||
_ = conn.WriteJSON(map[string]string{"type": "auth_ok"})
|
||||
}
|
||||
// Keep open briefly so client can mark connected.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
jc := NewJobConnection()
|
||||
managerURL := strings.Replace(server.URL, "http://", "http://", 1)
|
||||
if err := jc.Connect(managerURL, "/job/1", "token123"); err != nil {
|
||||
t.Fatalf("Connect failed: %v", err)
|
||||
}
|
||||
if !jc.IsConnected() {
|
||||
t.Fatal("expected connection to be marked connected")
|
||||
}
|
||||
jc.Close()
|
||||
}
|
||||
|
||||
@@ -196,7 +196,8 @@ type NextJobTaskInfo struct {
|
||||
TaskID int64 `json:"task_id"`
|
||||
JobID int64 `json:"job_id"`
|
||||
JobName string `json:"job_name"`
|
||||
Frame int `json:"frame"`
|
||||
Frame int `json:"frame"` // frame start (inclusive)
|
||||
FrameEnd int `json:"frame_end"` // frame end (inclusive); same as Frame for single-frame
|
||||
TaskType string `json:"task_type"`
|
||||
Metadata *types.BlendMetadata `json:"metadata,omitempty"`
|
||||
}
|
||||
@@ -240,8 +241,8 @@ func (m *ManagerClient) DownloadContext(contextPath, jobToken string) (io.ReadCl
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("context download failed with status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
@@ -315,6 +316,28 @@ func (m *ManagerClient) GetJobMetadata(jobID int64) (*types.BlendMetadata, error
|
||||
return &metadata, nil
|
||||
}
|
||||
|
||||
// GetJobStatus retrieves the current status of a job.
|
||||
func (m *ManagerClient) GetJobStatus(jobID int64) (types.JobStatus, error) {
|
||||
path := fmt.Sprintf("/api/runner/jobs/%d/status?runner_id=%d", jobID, m.runnerID)
|
||||
resp, err := m.Request("GET", path, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return "", fmt.Errorf("failed to get job status: %s", string(body))
|
||||
}
|
||||
|
||||
var job types.Job
|
||||
if err := json.NewDecoder(resp.Body).Decode(&job); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return job.Status, nil
|
||||
}
|
||||
|
||||
// JobFile represents a file associated with a job.
|
||||
type JobFile struct {
|
||||
ID int64 `json:"id"`
|
||||
@@ -412,10 +435,39 @@ func (m *ManagerClient) DownloadBlender(version string) (io.ReadCloser, error) {
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("failed to download blender: status %d, body: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// blenderVersionsResponse is the response from GET /api/blender/versions.
|
||||
type blenderVersionsResponse struct {
|
||||
Versions []struct {
|
||||
Full string `json:"full"`
|
||||
} `json:"versions"`
|
||||
}
|
||||
|
||||
// GetLatestBlenderVersion returns the latest Blender version string (e.g. "4.2.3") from the manager.
|
||||
// Uses the flat versions list which is newest-first.
|
||||
func (m *ManagerClient) GetLatestBlenderVersion() (string, error) {
|
||||
resp, err := m.Request("GET", "/api/blender/versions", nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to fetch blender versions: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return "", fmt.Errorf("blender versions returned status %d: %s", resp.StatusCode, string(body))
|
||||
}
|
||||
var out blenderVersionsResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
return "", fmt.Errorf("failed to decode blender versions: %w", err)
|
||||
}
|
||||
if len(out.Versions) == 0 {
|
||||
return "", fmt.Errorf("no blender versions available")
|
||||
}
|
||||
return out.Versions[0].Full, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewManagerClient_TrimsTrailingSlash(t *testing.T) {
|
||||
c := NewManagerClient("http://example.com/")
|
||||
if c.GetBaseURL() != "http://example.com" {
|
||||
t.Fatalf("unexpected base url: %q", c.GetBaseURL())
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoRequest_SetsAuthorizationHeader(t *testing.T) {
|
||||
var authHeader string
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
authHeader = r.Header.Get("Authorization")
|
||||
_ = json.NewEncoder(w).Encode(map[string]bool{"ok": true})
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
c := NewManagerClient(ts.URL)
|
||||
c.SetCredentials(1, "abc123")
|
||||
|
||||
resp, err := c.Request(http.MethodGet, "/x", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Request failed: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if authHeader != "Bearer abc123" {
|
||||
t.Fatalf("unexpected Authorization header: %q", authHeader)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequest_RequiresAuth(t *testing.T) {
|
||||
c := NewManagerClient("http://example.com")
|
||||
if _, err := c.Request(http.MethodGet, "/x", nil); err == nil {
|
||||
t.Fatal("expected auth error when api key is missing")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"jiggablend/internal/runner/api"
|
||||
"jiggablend/internal/runner/workspace"
|
||||
@@ -43,8 +45,12 @@ func (m *Manager) GetBinaryPath(version string) (string, error) {
|
||||
if binaryInfo, err := os.Stat(binaryPath); err == nil {
|
||||
// Verify it's actually a file (not a directory)
|
||||
if !binaryInfo.IsDir() {
|
||||
log.Printf("Found existing Blender %s installation at %s", version, binaryPath)
|
||||
return binaryPath, nil
|
||||
absBinaryPath, err := ResolveBinaryPath(binaryPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
log.Printf("Found existing Blender %s installation at %s", version, absBinaryPath)
|
||||
return absBinaryPath, nil
|
||||
}
|
||||
}
|
||||
// Version folder exists but binary is missing - might be incomplete installation
|
||||
@@ -71,17 +77,86 @@ func (m *Manager) GetBinaryPath(version string) (string, error) {
|
||||
return "", fmt.Errorf("blender binary not found after extraction")
|
||||
}
|
||||
|
||||
log.Printf("Blender %s installed at %s", version, binaryPath)
|
||||
return binaryPath, nil
|
||||
absBinaryPath, err := ResolveBinaryPath(binaryPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
log.Printf("Blender %s installed at %s", version, absBinaryPath)
|
||||
return absBinaryPath, nil
|
||||
}
|
||||
|
||||
// GetBinaryForJob returns the Blender binary path for a job.
|
||||
// Uses the version from metadata or falls back to system blender.
|
||||
func (m *Manager) GetBinaryForJob(version string) (string, error) {
|
||||
if version == "" {
|
||||
return "blender", nil // System blender
|
||||
return ResolveBinaryPath("blender")
|
||||
}
|
||||
|
||||
return m.GetBinaryPath(version)
|
||||
}
|
||||
|
||||
// ResolveBinaryPath resolves a Blender executable to an absolute path.
|
||||
func ResolveBinaryPath(blenderBinary string) (string, error) {
|
||||
if blenderBinary == "" {
|
||||
return "", fmt.Errorf("blender binary path is empty")
|
||||
}
|
||||
|
||||
if strings.Contains(blenderBinary, string(filepath.Separator)) {
|
||||
absPath, err := filepath.Abs(blenderBinary)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to resolve blender binary path %q: %w", blenderBinary, err)
|
||||
}
|
||||
return absPath, nil
|
||||
}
|
||||
|
||||
resolvedPath, err := exec.LookPath(blenderBinary)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to locate blender binary %q in PATH: %w", blenderBinary, err)
|
||||
}
|
||||
absPath, err := filepath.Abs(resolvedPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to resolve blender binary path %q: %w", resolvedPath, err)
|
||||
}
|
||||
return absPath, nil
|
||||
}
|
||||
|
||||
// TarballEnv returns a copy of baseEnv with LD_LIBRARY_PATH set so that a
|
||||
// tarball Blender installation can find its bundled libs (e.g. lib/python3.x).
|
||||
// If blenderBinary is the system "blender" or has no path component, baseEnv is
|
||||
// returned unchanged.
|
||||
func TarballEnv(blenderBinary string, baseEnv []string) []string {
|
||||
if blenderBinary == "" || blenderBinary == "blender" {
|
||||
return baseEnv
|
||||
}
|
||||
if !strings.Contains(blenderBinary, string(os.PathSeparator)) {
|
||||
return baseEnv
|
||||
}
|
||||
blenderDir := filepath.Dir(blenderBinary)
|
||||
libDir := filepath.Join(blenderDir, "lib")
|
||||
ldLib := libDir
|
||||
for _, e := range baseEnv {
|
||||
if strings.HasPrefix(e, "LD_LIBRARY_PATH=") {
|
||||
existing := strings.TrimPrefix(e, "LD_LIBRARY_PATH=")
|
||||
if existing != "" {
|
||||
ldLib = libDir + ":" + existing
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(baseEnv)+1)
|
||||
done := false
|
||||
for _, e := range baseEnv {
|
||||
if strings.HasPrefix(e, "LD_LIBRARY_PATH=") {
|
||||
out = append(out, "LD_LIBRARY_PATH="+ldLib)
|
||||
done = true
|
||||
continue
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
if !done {
|
||||
out = append(out, "LD_LIBRARY_PATH="+ldLib)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package blender
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolveBinaryPath_AbsoluteLikePath(t *testing.T) {
|
||||
got, err := ResolveBinaryPath("./blender")
|
||||
if err != nil {
|
||||
t.Fatalf("ResolveBinaryPath failed: %v", err)
|
||||
}
|
||||
if !filepath.IsAbs(got) {
|
||||
t.Fatalf("expected absolute path, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveBinaryPath_Empty(t *testing.T) {
|
||||
if _, err := ResolveBinaryPath(""); err == nil {
|
||||
t.Fatal("expected error for empty blender binary")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTarballEnv_SetsAndExtendsLDLibraryPath(t *testing.T) {
|
||||
bin := filepath.Join(string(os.PathSeparator), "tmp", "blender", "blender")
|
||||
got := TarballEnv(bin, []string{"A=B", "LD_LIBRARY_PATH=/old"})
|
||||
joined := strings.Join(got, "\n")
|
||||
if !strings.Contains(joined, "LD_LIBRARY_PATH=/tmp/blender/lib:/old") {
|
||||
t.Fatalf("expected LD_LIBRARY_PATH to include blender lib, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
// Package blender: host GPU backend detection for AMD/NVIDIA/Intel.
|
||||
package blender
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DetectGPUBackends detects whether AMD, NVIDIA, and/or Intel GPUs are available
|
||||
// using host-level hardware probing only.
|
||||
func DetectGPUBackends() (hasAMD, hasNVIDIA, hasIntel bool, ok bool) {
|
||||
return detectGPUBackendsFromHost()
|
||||
}
|
||||
|
||||
func detectGPUBackendsFromHost() (hasAMD, hasNVIDIA, hasIntel bool, ok bool) {
|
||||
if amd, nvidia, intel, found := detectGPUBackendsFromDRM(); found {
|
||||
return amd, nvidia, intel, true
|
||||
}
|
||||
if amd, nvidia, intel, found := detectGPUBackendsFromLSPCI(); found {
|
||||
return amd, nvidia, intel, true
|
||||
}
|
||||
return false, false, false, false
|
||||
}
|
||||
|
||||
func detectGPUBackendsFromDRM() (hasAMD, hasNVIDIA, hasIntel bool, ok bool) {
|
||||
entries, err := os.ReadDir("/sys/class/drm")
|
||||
if err != nil {
|
||||
return false, false, false, false
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
name := entry.Name()
|
||||
if !isDRMCardNode(name) {
|
||||
continue
|
||||
}
|
||||
|
||||
vendorPath := filepath.Join("/sys/class/drm", name, "device", "vendor")
|
||||
vendorRaw, err := os.ReadFile(vendorPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
vendor := strings.TrimSpace(strings.ToLower(string(vendorRaw)))
|
||||
switch vendor {
|
||||
case "0x1002":
|
||||
hasAMD = true
|
||||
ok = true
|
||||
case "0x10de":
|
||||
hasNVIDIA = true
|
||||
ok = true
|
||||
case "0x8086":
|
||||
hasIntel = true
|
||||
ok = true
|
||||
}
|
||||
}
|
||||
|
||||
return hasAMD, hasNVIDIA, hasIntel, ok
|
||||
}
|
||||
|
||||
func isDRMCardNode(name string) bool {
|
||||
if !strings.HasPrefix(name, "card") {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(name, "-") {
|
||||
// Connector entries like card0-DP-1 are not GPU device nodes.
|
||||
return false
|
||||
}
|
||||
if len(name) <= len("card") {
|
||||
return false
|
||||
}
|
||||
_, err := strconv.Atoi(strings.TrimPrefix(name, "card"))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func detectGPUBackendsFromLSPCI() (hasAMD, hasNVIDIA, hasIntel bool, ok bool) {
|
||||
if _, err := exec.LookPath("lspci"); err != nil {
|
||||
return false, false, false, false
|
||||
}
|
||||
|
||||
out, err := exec.Command("lspci").CombinedOutput()
|
||||
if err != nil {
|
||||
return false, false, false, false
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(strings.NewReader(string(out)))
|
||||
for scanner.Scan() {
|
||||
line := strings.ToLower(strings.TrimSpace(scanner.Text()))
|
||||
if !isGPUControllerLine(line) {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.Contains(line, "nvidia") {
|
||||
hasNVIDIA = true
|
||||
ok = true
|
||||
}
|
||||
if strings.Contains(line, "amd") || strings.Contains(line, "ati") || strings.Contains(line, "radeon") {
|
||||
hasAMD = true
|
||||
ok = true
|
||||
}
|
||||
if strings.Contains(line, "intel") {
|
||||
hasIntel = true
|
||||
ok = true
|
||||
}
|
||||
}
|
||||
|
||||
return hasAMD, hasNVIDIA, hasIntel, ok
|
||||
}
|
||||
|
||||
func isGPUControllerLine(line string) bool {
|
||||
return strings.Contains(line, "vga compatible controller") ||
|
||||
strings.Contains(line, "3d controller") ||
|
||||
strings.Contains(line, "display controller")
|
||||
}
|
||||
|
||||
func parseRocmAgentArch(output string) (arch string, ok bool) {
|
||||
scanner := bufio.NewScanner(strings.NewReader(output))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if !strings.HasPrefix(line, "Name:") {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(strings.TrimPrefix(line, "Name:"))
|
||||
if strings.HasPrefix(name, "gfx") {
|
||||
return name, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package blender
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseRocmAgentArch(t *testing.T) {
|
||||
input := `ROCk module is loaded
|
||||
Agent 1
|
||||
Name: gfx1151
|
||||
Marketing Name: AMD Radeon Graphics
|
||||
`
|
||||
arch, ok := parseRocmAgentArch(input)
|
||||
if !ok {
|
||||
t.Fatal("expected arch to be parsed")
|
||||
}
|
||||
if arch != "gfx1151" {
|
||||
t.Fatalf("arch = %q, want gfx1151", arch)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package blender
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestIsDRMCardNode(t *testing.T) {
|
||||
tests := map[string]bool{
|
||||
"card0": true,
|
||||
"card12": true,
|
||||
"card": false,
|
||||
"card0-DP-1": false,
|
||||
"renderD128": false,
|
||||
"foo": false,
|
||||
}
|
||||
for in, want := range tests {
|
||||
if got := isDRMCardNode(in); got != want {
|
||||
t.Fatalf("isDRMCardNode(%q) = %v, want %v", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsGPUControllerLine(t *testing.T) {
|
||||
if !isGPUControllerLine("vga compatible controller: nvidia corp") {
|
||||
t.Fatal("expected VGA controller line to match")
|
||||
}
|
||||
if !isGPUControllerLine("3d controller: amd") {
|
||||
t.Fatal("expected 3d controller line to match")
|
||||
}
|
||||
if isGPUControllerLine("audio device: something") {
|
||||
t.Fatal("audio line should not match")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package blender
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"jiggablend/pkg/types"
|
||||
)
|
||||
|
||||
func TestFilterLog_FiltersNoise(t *testing.T) {
|
||||
cases := []string{
|
||||
"",
|
||||
"--------------------------------------------------------------------",
|
||||
"Failed to add relation foo",
|
||||
"BKE_modifier_set_error",
|
||||
"Depth Type Name",
|
||||
}
|
||||
for _, in := range cases {
|
||||
filtered, level := FilterLog(in)
|
||||
if !filtered {
|
||||
t.Fatalf("expected filtered for %q", in)
|
||||
}
|
||||
if level != types.LogLevelInfo {
|
||||
t.Fatalf("unexpected level for %q: %s", in, level)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterLog_KeepsNormalLine(t *testing.T) {
|
||||
filtered, _ := FilterLog("Rendering done.")
|
||||
if filtered {
|
||||
t.Fatal("normal line should not be filtered")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,143 +1,19 @@
|
||||
package blender
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
||||
"jiggablend/pkg/blendfile"
|
||||
)
|
||||
|
||||
// ParseVersionFromFile parses the Blender version that a .blend file was saved with.
|
||||
// Returns major and minor version numbers.
|
||||
// Delegates to the shared pkg/blendfile implementation.
|
||||
func ParseVersionFromFile(blendPath string) (major, minor int, err error) {
|
||||
file, err := os.Open(blendPath)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to open blend file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Read the first 12 bytes of the blend file header
|
||||
// Format: BLENDER-v<major><minor><patch> or BLENDER_v<major><minor><patch>
|
||||
// The header is: "BLENDER" (7 bytes) + pointer size (1 byte: '-' for 64-bit, '_' for 32-bit)
|
||||
// + endianness (1 byte: 'v' for little-endian, 'V' for big-endian)
|
||||
// + version (3 bytes: e.g., "402" for 4.02)
|
||||
header := make([]byte, 12)
|
||||
n, err := file.Read(header)
|
||||
if err != nil || n < 12 {
|
||||
return 0, 0, fmt.Errorf("failed to read blend file header: %w", err)
|
||||
}
|
||||
|
||||
// Check for BLENDER magic
|
||||
if string(header[:7]) != "BLENDER" {
|
||||
// Might be compressed - try to decompress
|
||||
file.Seek(0, 0)
|
||||
return parseCompressedVersion(file)
|
||||
}
|
||||
|
||||
// Parse version from bytes 9-11 (3 digits)
|
||||
versionStr := string(header[9:12])
|
||||
|
||||
// Version format changed in Blender 3.0
|
||||
// Pre-3.0: "279" = 2.79, "280" = 2.80
|
||||
// 3.0+: "300" = 3.0, "402" = 4.02, "410" = 4.10
|
||||
if len(versionStr) == 3 {
|
||||
// First digit is major version
|
||||
fmt.Sscanf(string(versionStr[0]), "%d", &major)
|
||||
// Next two digits are minor version
|
||||
fmt.Sscanf(versionStr[1:3], "%d", &minor)
|
||||
}
|
||||
|
||||
return major, minor, nil
|
||||
}
|
||||
|
||||
// parseCompressedVersion handles gzip and zstd compressed blend files.
|
||||
func parseCompressedVersion(file *os.File) (major, minor int, err error) {
|
||||
magic := make([]byte, 4)
|
||||
if _, err := file.Read(magic); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
file.Seek(0, 0)
|
||||
|
||||
if magic[0] == 0x1f && magic[1] == 0x8b {
|
||||
// gzip compressed
|
||||
gzReader, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to create gzip reader: %w", err)
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
header := make([]byte, 12)
|
||||
n, err := gzReader.Read(header)
|
||||
if err != nil || n < 12 {
|
||||
return 0, 0, fmt.Errorf("failed to read compressed blend header: %w", err)
|
||||
}
|
||||
|
||||
if string(header[:7]) != "BLENDER" {
|
||||
return 0, 0, fmt.Errorf("invalid blend file format")
|
||||
}
|
||||
|
||||
versionStr := string(header[9:12])
|
||||
if len(versionStr) == 3 {
|
||||
fmt.Sscanf(string(versionStr[0]), "%d", &major)
|
||||
fmt.Sscanf(versionStr[1:3], "%d", &minor)
|
||||
}
|
||||
|
||||
return major, minor, nil
|
||||
}
|
||||
|
||||
// Check for zstd magic (Blender 3.0+): 0x28 0xB5 0x2F 0xFD
|
||||
if magic[0] == 0x28 && magic[1] == 0xb5 && magic[2] == 0x2f && magic[3] == 0xfd {
|
||||
return parseZstdVersion(file)
|
||||
}
|
||||
|
||||
return 0, 0, fmt.Errorf("unknown blend file format")
|
||||
}
|
||||
|
||||
// parseZstdVersion handles zstd-compressed blend files (Blender 3.0+).
|
||||
// Uses zstd command line tool since Go doesn't have native zstd support.
|
||||
func parseZstdVersion(file *os.File) (major, minor int, err error) {
|
||||
file.Seek(0, 0)
|
||||
|
||||
cmd := exec.Command("zstd", "-d", "-c")
|
||||
cmd.Stdin = file
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to create zstd stdout pipe: %w", err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to start zstd decompression: %w", err)
|
||||
}
|
||||
|
||||
// Read just the header (12 bytes)
|
||||
header := make([]byte, 12)
|
||||
n, readErr := io.ReadFull(stdout, header)
|
||||
|
||||
// Kill the process early - we only need the header
|
||||
cmd.Process.Kill()
|
||||
cmd.Wait()
|
||||
|
||||
if readErr != nil || n < 12 {
|
||||
return 0, 0, fmt.Errorf("failed to read zstd compressed blend header: %v", readErr)
|
||||
}
|
||||
|
||||
if string(header[:7]) != "BLENDER" {
|
||||
return 0, 0, fmt.Errorf("invalid blend file format in zstd archive")
|
||||
}
|
||||
|
||||
versionStr := string(header[9:12])
|
||||
if len(versionStr) == 3 {
|
||||
fmt.Sscanf(string(versionStr[0]), "%d", &major)
|
||||
fmt.Sscanf(versionStr[1:3], "%d", &minor)
|
||||
}
|
||||
|
||||
return major, minor, nil
|
||||
return blendfile.ParseVersionFromFile(blendPath)
|
||||
}
|
||||
|
||||
// VersionString returns a formatted version string like "4.2".
|
||||
func VersionString(major, minor int) string {
|
||||
return fmt.Sprintf("%d.%d", major, minor)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
package blender
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestVersionString(t *testing.T) {
|
||||
if got := VersionString(4, 2); got != "4.2" {
|
||||
t.Fatalf("VersionString() = %q, want %q", got, "4.2")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,10 +20,8 @@ type EncodeConfig struct {
|
||||
StartFrame int // Starting frame number
|
||||
FrameRate float64 // Frame rate
|
||||
WorkDir string // Working directory
|
||||
UseAlpha bool // Whether to preserve alpha channel
|
||||
TwoPass bool // Whether to use 2-pass encoding
|
||||
SourceFormat string // Source format: "exr" or "png" (defaults to "exr")
|
||||
PreserveHDR bool // Whether to preserve HDR range for EXR (uses HLG with bt709 primaries)
|
||||
UseAlpha bool // Whether to preserve alpha channel
|
||||
TwoPass bool // Whether to use 2-pass encoding
|
||||
}
|
||||
|
||||
// Selector selects the software encoder.
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package encoding
|
||||
|
||||
// Pipeline: Blender outputs only EXR (linear). Encode is EXR only: linear -> sRGB -> HLG (video), 10-bit, full range.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
@@ -17,26 +19,11 @@ const (
|
||||
CRFVP9 = 30
|
||||
)
|
||||
|
||||
// tonemapFilter returns the appropriate filter for EXR input.
|
||||
// For HDR preservation: converts linear RGB (EXR) to bt2020 YUV with HLG transfer function
|
||||
// Uses zscale to properly convert colorspace from linear RGB to bt2020 YUV while preserving HDR range
|
||||
// Step 1: Ensure format is gbrpf32le (linear RGB)
|
||||
// Step 2: Convert transfer function from linear to HLG (arib-std-b67) with bt2020 primaries/matrix
|
||||
// Step 3: Convert to YUV format
|
||||
// tonemapFilter returns the production filter for EXR input (linear → sRGB → HLG, bt709).
|
||||
// This is the single source of truth used by BuildCommand and BuildPass1Command.
|
||||
// zscale numeric values: primaries 1=bt709, matrix 1=bt709, transfer 8=linear / 13=sRGB / 18=HLG.
|
||||
func tonemapFilter(useAlpha bool) string {
|
||||
// Convert from linear RGB (gbrpf32le) to HLG with bt709 primaries to match PNG appearance
|
||||
// Based on best practices: convert linear RGB directly to HLG with bt709 primaries
|
||||
// This matches PNG color appearance (bt709 primaries) while preserving HDR range (HLG transfer)
|
||||
// zscale uses numeric values:
|
||||
// primaries: 1=bt709 (matches PNG), 9=bt2020
|
||||
// matrix: 1=bt709, 9=bt2020nc, 0=gbr (RGB input)
|
||||
// transfer: 8=linear, 18=arib-std-b67 (HLG)
|
||||
// Direct conversion: linear RGB -> HLG with bt709 primaries -> bt2020 YUV (for wider gamut metadata)
|
||||
// The bt709 primaries in the conversion match PNG, but we set bt2020 in metadata for HDR displays
|
||||
// Convert linear RGB to sRGB first, then convert to HLG
|
||||
// This approach: linear -> sRGB -> HLG -> bt2020
|
||||
// Fixes red tint by using sRGB conversion, preserves HDR range with HLG
|
||||
filter := "format=gbrpf32le,zscale=transferin=8:transfer=13:primariesin=1:primaries=1:matrixin=0:matrix=1:rangein=full:range=full,zscale=transferin=13:transfer=18:primariesin=1:primaries=9:matrixin=1:matrix=9:rangein=full:range=full"
|
||||
filter := "format=gbrpf32le,zscale=transferin=8:transfer=13:primariesin=1:primaries=1:matrixin=0:matrix=1:rangein=full:range=full,zscale=transferin=13:transfer=18:primariesin=1:primaries=1:matrixin=1:matrix=1:rangein=full:range=full"
|
||||
if useAlpha {
|
||||
return filter + ",format=yuva420p10le"
|
||||
}
|
||||
@@ -55,99 +42,39 @@ func (e *SoftwareEncoder) Available() bool {
|
||||
return true // Software encoding is always available
|
||||
}
|
||||
|
||||
func (e *SoftwareEncoder) BuildCommand(config *EncodeConfig) *exec.Cmd {
|
||||
// Use HDR pixel formats for EXR, SDR for PNG
|
||||
var pixFmt string
|
||||
var colorPrimaries, colorTrc, colorspace string
|
||||
if config.SourceFormat == "png" {
|
||||
// PNG: SDR format
|
||||
pixFmt = "yuv420p"
|
||||
if config.UseAlpha {
|
||||
pixFmt = "yuva420p"
|
||||
}
|
||||
colorPrimaries = "bt709"
|
||||
colorTrc = "bt709"
|
||||
colorspace = "bt709"
|
||||
} else {
|
||||
// EXR: Use HDR encoding if PreserveHDR is true, otherwise SDR (like PNG)
|
||||
if config.PreserveHDR {
|
||||
// HDR: Use HLG transfer with bt709 primaries to preserve HDR range while matching PNG color
|
||||
pixFmt = "yuv420p10le" // 10-bit to preserve HDR range
|
||||
if config.UseAlpha {
|
||||
pixFmt = "yuva420p10le"
|
||||
}
|
||||
colorPrimaries = "bt709" // bt709 primaries to match PNG color appearance
|
||||
colorTrc = "arib-std-b67" // HLG transfer function - preserves HDR range, works on SDR displays
|
||||
colorspace = "bt709" // bt709 colorspace to match PNG
|
||||
} else {
|
||||
// SDR: Treat as SDR (like PNG) - encode as bt709
|
||||
pixFmt = "yuv420p"
|
||||
if config.UseAlpha {
|
||||
pixFmt = "yuva420p"
|
||||
}
|
||||
colorPrimaries = "bt709"
|
||||
colorTrc = "bt709"
|
||||
colorspace = "bt709"
|
||||
}
|
||||
func (e *SoftwareEncoder) buildBaseArgs(config *EncodeConfig) []string {
|
||||
pixFmt := "yuv420p10le"
|
||||
if config.UseAlpha {
|
||||
pixFmt = "yuva420p10le"
|
||||
}
|
||||
colorPrimaries, colorTrc, colorspace, colorRange := "bt709", "arib-std-b67", "bt709", "pc"
|
||||
|
||||
var codecArgs []string
|
||||
switch e.codec {
|
||||
case "libaom-av1":
|
||||
codecArgs = []string{"-crf", strconv.Itoa(CRFAV1), "-b:v", "0", "-tiles", "2x2", "-g", "240"}
|
||||
case "libvpx-vp9":
|
||||
// VP9 supports alpha and HDR, use good quality settings
|
||||
codecArgs = []string{"-crf", strconv.Itoa(CRFVP9), "-b:v", "0", "-row-mt", "1", "-g", "240"}
|
||||
default:
|
||||
// H.264: Use High 10 profile for HDR EXR (10-bit), High profile for SDR
|
||||
if config.SourceFormat != "png" && config.PreserveHDR {
|
||||
codecArgs = []string{"-preset", "veryslow", "-crf", strconv.Itoa(CRFH264), "-profile:v", "high10", "-level", "5.2", "-tune", "film", "-keyint_min", "24", "-g", "240", "-bf", "2", "-refs", "4"}
|
||||
} else {
|
||||
codecArgs = []string{"-preset", "veryslow", "-crf", strconv.Itoa(CRFH264), "-profile:v", "high", "-level", "5.2", "-tune", "film", "-keyint_min", "24", "-g", "240", "-bf", "2", "-refs", "4"}
|
||||
}
|
||||
codecArgs = []string{"-preset", "veryslow", "-crf", strconv.Itoa(CRFH264), "-profile:v", "high10", "-level", "5.2", "-tune", "film", "-keyint_min", "24", "-g", "240", "-bf", "2", "-refs", "4"}
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-y",
|
||||
"-f", "image2",
|
||||
"-start_number", fmt.Sprintf("%d", config.StartFrame),
|
||||
"-framerate", fmt.Sprintf("%.2f", config.FrameRate),
|
||||
"-i", config.InputPattern,
|
||||
"-c:v", e.codec,
|
||||
"-pix_fmt", pixFmt,
|
||||
"-r", fmt.Sprintf("%.2f", config.FrameRate),
|
||||
"-color_primaries", colorPrimaries,
|
||||
"-color_trc", colorTrc,
|
||||
"-colorspace", colorspace,
|
||||
"-color_range", "tv",
|
||||
args := []string{"-y", "-f", "image2", "-start_number", fmt.Sprintf("%d", config.StartFrame), "-framerate", fmt.Sprintf("%.2f", config.FrameRate),
|
||||
"-color_trc", "linear", "-color_primaries", "bt709"}
|
||||
args = append(args, "-i", config.InputPattern, "-c:v", e.codec, "-pix_fmt", pixFmt, "-r", fmt.Sprintf("%.2f", config.FrameRate), "-color_primaries", colorPrimaries, "-color_trc", colorTrc, "-colorspace", colorspace, "-color_range", colorRange)
|
||||
|
||||
// FFmpeg 6+ treats yuva420p10le as experimental for libvpx-vp9 / libaom-av1
|
||||
if config.UseAlpha && (e.codec == "libvpx-vp9" || e.codec == "libaom-av1") {
|
||||
args = append(args, "-strict", "experimental")
|
||||
}
|
||||
|
||||
// Add video filter for EXR: convert linear RGB based on HDR setting
|
||||
// PNG doesn't need any filter as it's already in sRGB
|
||||
if config.SourceFormat != "png" {
|
||||
var vf string
|
||||
if config.PreserveHDR {
|
||||
// HDR: Convert linear RGB -> sRGB -> HLG with bt709 primaries
|
||||
// This preserves HDR range while matching PNG color appearance
|
||||
vf = "format=gbrpf32le,zscale=transferin=8:transfer=13:primariesin=1:primaries=1:matrixin=0:matrix=1:rangein=full:range=full,zscale=transferin=13:transfer=18:primariesin=1:primaries=1:matrixin=1:matrix=1:rangein=full:range=full"
|
||||
if config.UseAlpha {
|
||||
vf += ",format=yuva420p10le"
|
||||
} else {
|
||||
vf += ",format=yuv420p10le"
|
||||
}
|
||||
} else {
|
||||
// SDR: Convert linear RGB (EXR) to sRGB (bt709) - simple conversion like Krita does
|
||||
// zscale: linear (8) -> sRGB (13) with bt709 primaries/matrix
|
||||
vf = "format=gbrpf32le,zscale=transferin=8:transfer=13:primariesin=1:primaries=1:matrixin=0:matrix=1:rangein=full:range=full"
|
||||
if config.UseAlpha {
|
||||
vf += ",format=yuva420p"
|
||||
} else {
|
||||
vf += ",format=yuv420p"
|
||||
}
|
||||
}
|
||||
args = append(args, "-vf", vf)
|
||||
}
|
||||
args = append(args, "-vf", tonemapFilter(config.UseAlpha))
|
||||
args = append(args, codecArgs...)
|
||||
return args
|
||||
}
|
||||
|
||||
func (e *SoftwareEncoder) BuildCommand(config *EncodeConfig) *exec.Cmd {
|
||||
args := e.buildBaseArgs(config)
|
||||
|
||||
if config.TwoPass {
|
||||
// For 2-pass, this builds pass 2 command
|
||||
@@ -168,99 +95,7 @@ func (e *SoftwareEncoder) BuildCommand(config *EncodeConfig) *exec.Cmd {
|
||||
|
||||
// BuildPass1Command builds the first pass command for 2-pass encoding.
|
||||
func (e *SoftwareEncoder) BuildPass1Command(config *EncodeConfig) *exec.Cmd {
|
||||
// Use HDR pixel formats for EXR, SDR for PNG
|
||||
var pixFmt string
|
||||
var colorPrimaries, colorTrc, colorspace string
|
||||
if config.SourceFormat == "png" {
|
||||
// PNG: SDR format
|
||||
pixFmt = "yuv420p"
|
||||
if config.UseAlpha {
|
||||
pixFmt = "yuva420p"
|
||||
}
|
||||
colorPrimaries = "bt709"
|
||||
colorTrc = "bt709"
|
||||
colorspace = "bt709"
|
||||
} else {
|
||||
// EXR: Use HDR encoding if PreserveHDR is true, otherwise SDR (like PNG)
|
||||
if config.PreserveHDR {
|
||||
// HDR: Use HLG transfer with bt709 primaries to preserve HDR range while matching PNG color
|
||||
pixFmt = "yuv420p10le" // 10-bit to preserve HDR range
|
||||
if config.UseAlpha {
|
||||
pixFmt = "yuva420p10le"
|
||||
}
|
||||
colorPrimaries = "bt709" // bt709 primaries to match PNG color appearance
|
||||
colorTrc = "arib-std-b67" // HLG transfer function - preserves HDR range, works on SDR displays
|
||||
colorspace = "bt709" // bt709 colorspace to match PNG
|
||||
} else {
|
||||
// SDR: Treat as SDR (like PNG) - encode as bt709
|
||||
pixFmt = "yuv420p"
|
||||
if config.UseAlpha {
|
||||
pixFmt = "yuva420p"
|
||||
}
|
||||
colorPrimaries = "bt709"
|
||||
colorTrc = "bt709"
|
||||
colorspace = "bt709"
|
||||
}
|
||||
}
|
||||
|
||||
var codecArgs []string
|
||||
switch e.codec {
|
||||
case "libaom-av1":
|
||||
codecArgs = []string{"-crf", strconv.Itoa(CRFAV1), "-b:v", "0", "-tiles", "2x2", "-g", "240"}
|
||||
case "libvpx-vp9":
|
||||
// VP9 supports alpha and HDR, use good quality settings
|
||||
codecArgs = []string{"-crf", strconv.Itoa(CRFVP9), "-b:v", "0", "-row-mt", "1", "-g", "240"}
|
||||
default:
|
||||
// H.264: Use High 10 profile for HDR EXR (10-bit), High profile for SDR
|
||||
if config.SourceFormat != "png" && config.PreserveHDR {
|
||||
codecArgs = []string{"-preset", "veryslow", "-crf", strconv.Itoa(CRFH264), "-profile:v", "high10", "-level", "5.2", "-tune", "film", "-keyint_min", "24", "-g", "240", "-bf", "2", "-refs", "4"}
|
||||
} else {
|
||||
codecArgs = []string{"-preset", "veryslow", "-crf", strconv.Itoa(CRFH264), "-profile:v", "high", "-level", "5.2", "-tune", "film", "-keyint_min", "24", "-g", "240", "-bf", "2", "-refs", "4"}
|
||||
}
|
||||
}
|
||||
|
||||
args := []string{
|
||||
"-y",
|
||||
"-f", "image2",
|
||||
"-start_number", fmt.Sprintf("%d", config.StartFrame),
|
||||
"-framerate", fmt.Sprintf("%.2f", config.FrameRate),
|
||||
"-i", config.InputPattern,
|
||||
"-c:v", e.codec,
|
||||
"-pix_fmt", pixFmt,
|
||||
"-r", fmt.Sprintf("%.2f", config.FrameRate),
|
||||
"-color_primaries", colorPrimaries,
|
||||
"-color_trc", colorTrc,
|
||||
"-colorspace", colorspace,
|
||||
"-color_range", "tv",
|
||||
}
|
||||
|
||||
// Add video filter for EXR: convert linear RGB based on HDR setting
|
||||
// PNG doesn't need any filter as it's already in sRGB
|
||||
if config.SourceFormat != "png" {
|
||||
var vf string
|
||||
if config.PreserveHDR {
|
||||
// HDR: Convert linear RGB -> sRGB -> HLG with bt709 primaries
|
||||
// This preserves HDR range while matching PNG color appearance
|
||||
vf = "format=gbrpf32le,zscale=transferin=8:transfer=13:primariesin=1:primaries=1:matrixin=0:matrix=1:rangein=full:range=full,zscale=transferin=13:transfer=18:primariesin=1:primaries=1:matrixin=1:matrix=1:rangein=full:range=full"
|
||||
if config.UseAlpha {
|
||||
vf += ",format=yuva420p10le"
|
||||
} else {
|
||||
vf += ",format=yuv420p10le"
|
||||
}
|
||||
} else {
|
||||
// SDR: Convert linear RGB (EXR) to sRGB (bt709) - simple conversion like Krita does
|
||||
// zscale: linear (8) -> sRGB (13) with bt709 primaries/matrix
|
||||
vf = "format=gbrpf32le,zscale=transferin=8:transfer=13:primariesin=1:primaries=1:matrixin=0:matrix=1:rangein=full:range=full"
|
||||
if config.UseAlpha {
|
||||
vf += ",format=yuva420p"
|
||||
} else {
|
||||
vf += ",format=yuv420p"
|
||||
}
|
||||
}
|
||||
args = append(args, "-vf", vf)
|
||||
}
|
||||
|
||||
args = append(args, codecArgs...)
|
||||
args := e.buildBaseArgs(config)
|
||||
args = append(args, "-pass", "1", "-f", "null", "/dev/null")
|
||||
|
||||
log.Printf("Build Software Pass 1 command: ffmpeg %s", strings.Join(args, " "))
|
||||
|
||||
@@ -18,7 +18,6 @@ func TestSoftwareEncoder_BuildCommand_H264_EXR(t *testing.T) {
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: false,
|
||||
TwoPass: true,
|
||||
SourceFormat: "exr",
|
||||
}
|
||||
|
||||
cmd := encoder.BuildCommand(config)
|
||||
@@ -37,7 +36,7 @@ func TestSoftwareEncoder_BuildCommand_H264_EXR(t *testing.T) {
|
||||
args := cmd.Args[1:] // Skip "ffmpeg"
|
||||
argsStr := strings.Join(args, " ")
|
||||
|
||||
// Check required arguments
|
||||
// EXR always uses HDR path: 10-bit, HLG, full range
|
||||
checks := []struct {
|
||||
name string
|
||||
expected string
|
||||
@@ -46,18 +45,19 @@ func TestSoftwareEncoder_BuildCommand_H264_EXR(t *testing.T) {
|
||||
{"image2 format", "-f image2"},
|
||||
{"start number", "-start_number 1"},
|
||||
{"framerate", "-framerate 24.00"},
|
||||
{"input color tag", "-color_trc linear"},
|
||||
{"input pattern", "-i frame_%04d.exr"},
|
||||
{"codec", "-c:v libx264"},
|
||||
{"pixel format", "-pix_fmt yuv420p"}, // EXR now treated as SDR (like PNG)
|
||||
{"pixel format", "-pix_fmt yuv420p10le"},
|
||||
{"frame rate", "-r 24.00"},
|
||||
{"color primaries", "-color_primaries bt709"}, // EXR now uses bt709 (SDR)
|
||||
{"color trc", "-color_trc bt709"}, // EXR now uses bt709 (SDR)
|
||||
{"color primaries", "-color_primaries bt709"},
|
||||
{"color trc", "-color_trc arib-std-b67"},
|
||||
{"colorspace", "-colorspace bt709"},
|
||||
{"color range", "-color_range tv"},
|
||||
{"color range", "-color_range pc"},
|
||||
{"video filter", "-vf"},
|
||||
{"preset", "-preset veryslow"},
|
||||
{"crf", "-crf 15"},
|
||||
{"profile", "-profile:v high"}, // EXR now uses high profile (SDR)
|
||||
{"profile", "-profile:v high10"},
|
||||
{"pass 2", "-pass 2"},
|
||||
{"output path", "output.mp4"},
|
||||
}
|
||||
@@ -68,40 +68,15 @@ func TestSoftwareEncoder_BuildCommand_H264_EXR(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Verify filter is present for EXR (linear RGB to sRGB conversion, like Krita does)
|
||||
// EXR: linear -> sRGB -> HLG filter
|
||||
if !strings.Contains(argsStr, "format=gbrpf32le") {
|
||||
t.Error("Expected format conversion filter for EXR source, but not found")
|
||||
}
|
||||
if !strings.Contains(argsStr, "zscale=transferin=8:transfer=13") {
|
||||
t.Error("Expected linear to sRGB conversion for EXR source, but not found")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSoftwareEncoder_BuildCommand_H264_PNG(t *testing.T) {
|
||||
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||
config := &EncodeConfig{
|
||||
InputPattern: "frame_%04d.png",
|
||||
OutputPath: "output.mp4",
|
||||
StartFrame: 1,
|
||||
FrameRate: 24.0,
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: false,
|
||||
TwoPass: true,
|
||||
SourceFormat: "png",
|
||||
}
|
||||
|
||||
cmd := encoder.BuildCommand(config)
|
||||
args := cmd.Args[1:]
|
||||
argsStr := strings.Join(args, " ")
|
||||
|
||||
// PNG should NOT have video filter
|
||||
if strings.Contains(argsStr, "-vf") {
|
||||
t.Error("PNG source should not have video filter, but -vf was found")
|
||||
}
|
||||
|
||||
// Should still have all other required args
|
||||
if !strings.Contains(argsStr, "-c:v libx264") {
|
||||
t.Error("Missing codec argument")
|
||||
if !strings.Contains(argsStr, "transfer=18") {
|
||||
t.Error("Expected sRGB to HLG conversion for EXR HDR, but not found")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -113,18 +88,17 @@ func TestSoftwareEncoder_BuildCommand_AV1_WithAlpha(t *testing.T) {
|
||||
StartFrame: 100,
|
||||
FrameRate: 30.0,
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: true,
|
||||
TwoPass: true,
|
||||
SourceFormat: "exr",
|
||||
UseAlpha: true,
|
||||
TwoPass: true,
|
||||
}
|
||||
|
||||
cmd := encoder.BuildCommand(config)
|
||||
args := cmd.Args[1:]
|
||||
argsStr := strings.Join(args, " ")
|
||||
|
||||
// Check alpha-specific settings
|
||||
if !strings.Contains(argsStr, "-pix_fmt yuva420p") {
|
||||
t.Error("Expected yuva420p pixel format for alpha, but not found")
|
||||
// EXR with alpha: 10-bit HDR path
|
||||
if !strings.Contains(argsStr, "-pix_fmt yuva420p10le") {
|
||||
t.Error("Expected yuva420p10le pixel format for EXR alpha, but not found")
|
||||
}
|
||||
|
||||
// Check AV1-specific arguments
|
||||
@@ -142,9 +116,13 @@ func TestSoftwareEncoder_BuildCommand_AV1_WithAlpha(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Check tonemap filter includes alpha format
|
||||
if !strings.Contains(argsStr, "format=yuva420p") {
|
||||
t.Error("Expected tonemap filter to output yuva420p for alpha, but not found")
|
||||
// Check tonemap filter includes alpha format (10-bit for EXR)
|
||||
if !strings.Contains(argsStr, "format=yuva420p10le") {
|
||||
t.Error("Expected tonemap filter to output yuva420p10le for EXR alpha, but not found")
|
||||
}
|
||||
// Alpha VP9/AV1 need -strict experimental on modern FFmpeg
|
||||
if !strings.Contains(argsStr, "-strict experimental") {
|
||||
t.Error("Expected -strict experimental for alpha encode")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -156,9 +134,8 @@ func TestSoftwareEncoder_BuildCommand_VP9(t *testing.T) {
|
||||
StartFrame: 1,
|
||||
FrameRate: 24.0,
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: true,
|
||||
TwoPass: true,
|
||||
SourceFormat: "exr",
|
||||
UseAlpha: true,
|
||||
TwoPass: true,
|
||||
}
|
||||
|
||||
cmd := encoder.BuildCommand(config)
|
||||
@@ -191,7 +168,6 @@ func TestSoftwareEncoder_BuildPass1Command(t *testing.T) {
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: false,
|
||||
TwoPass: true,
|
||||
SourceFormat: "exr",
|
||||
}
|
||||
|
||||
cmd := encoder.BuildPass1Command(config)
|
||||
@@ -227,7 +203,6 @@ func TestSoftwareEncoder_BuildPass1Command_AV1(t *testing.T) {
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: false,
|
||||
TwoPass: true,
|
||||
SourceFormat: "exr",
|
||||
}
|
||||
|
||||
cmd := encoder.BuildPass1Command(config)
|
||||
@@ -273,7 +248,6 @@ func TestSoftwareEncoder_BuildPass1Command_VP9(t *testing.T) {
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: false,
|
||||
TwoPass: true,
|
||||
SourceFormat: "exr",
|
||||
}
|
||||
|
||||
cmd := encoder.BuildPass1Command(config)
|
||||
@@ -319,7 +293,6 @@ func TestSoftwareEncoder_BuildCommand_NoTwoPass(t *testing.T) {
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: false,
|
||||
TwoPass: false,
|
||||
SourceFormat: "exr",
|
||||
}
|
||||
|
||||
cmd := encoder.BuildCommand(config)
|
||||
@@ -432,28 +405,6 @@ func TestSoftwareEncoder_Available(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeConfig_DefaultSourceFormat(t *testing.T) {
|
||||
config := &EncodeConfig{
|
||||
InputPattern: "frame_%04d.exr",
|
||||
OutputPath: "output.mp4",
|
||||
StartFrame: 1,
|
||||
FrameRate: 24.0,
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: false,
|
||||
TwoPass: false,
|
||||
// SourceFormat not set, should default to empty string (treated as exr)
|
||||
}
|
||||
|
||||
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||
cmd := encoder.BuildCommand(config)
|
||||
args := strings.Join(cmd.Args[1:], " ")
|
||||
|
||||
// Should still have tonemap filter when SourceFormat is empty (defaults to exr behavior)
|
||||
if !strings.Contains(args, "-vf") {
|
||||
t.Error("Empty SourceFormat should default to EXR behavior with tonemap filter")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCommandOrder(t *testing.T) {
|
||||
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||
config := &EncodeConfig{
|
||||
@@ -464,7 +415,6 @@ func TestCommandOrder(t *testing.T) {
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: false,
|
||||
TwoPass: true,
|
||||
SourceFormat: "exr",
|
||||
}
|
||||
|
||||
cmd := encoder.BuildCommand(config)
|
||||
@@ -519,20 +469,18 @@ func TestCommand_ColorspaceMetadata(t *testing.T) {
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: false,
|
||||
TwoPass: false,
|
||||
SourceFormat: "exr",
|
||||
PreserveHDR: false, // SDR encoding
|
||||
}
|
||||
|
||||
cmd := encoder.BuildCommand(config)
|
||||
args := cmd.Args[1:]
|
||||
argsStr := strings.Join(args, " ")
|
||||
|
||||
// Verify all SDR colorspace metadata is present for EXR (SDR encoding)
|
||||
// EXR always uses HDR path: bt709 primaries, HLG, full range
|
||||
colorspaceArgs := []string{
|
||||
"-color_primaries bt709", // EXR uses bt709 (SDR)
|
||||
"-color_trc bt709", // EXR uses bt709 (SDR)
|
||||
"-color_primaries bt709",
|
||||
"-color_trc arib-std-b67",
|
||||
"-colorspace bt709",
|
||||
"-color_range tv",
|
||||
"-color_range pc",
|
||||
}
|
||||
|
||||
for _, arg := range colorspaceArgs {
|
||||
@@ -541,17 +489,11 @@ func TestCommand_ColorspaceMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Verify SDR pixel format
|
||||
if !strings.Contains(argsStr, "-pix_fmt yuv420p") {
|
||||
t.Error("SDR encoding should use yuv420p pixel format")
|
||||
if !strings.Contains(argsStr, "-pix_fmt yuv420p10le") {
|
||||
t.Error("EXR encoding should use yuv420p10le pixel format")
|
||||
}
|
||||
|
||||
// Verify H.264 high profile (not high10)
|
||||
if !strings.Contains(argsStr, "-profile:v high") {
|
||||
t.Error("SDR encoding should use high profile")
|
||||
}
|
||||
if strings.Contains(argsStr, "-profile:v high10") {
|
||||
t.Error("SDR encoding should not use high10 profile")
|
||||
if !strings.Contains(argsStr, "-profile:v high10") {
|
||||
t.Error("EXR encoding should use high10 profile")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -565,20 +507,18 @@ func TestCommand_HDR_ColorspaceMetadata(t *testing.T) {
|
||||
WorkDir: "/tmp",
|
||||
UseAlpha: false,
|
||||
TwoPass: false,
|
||||
SourceFormat: "exr",
|
||||
PreserveHDR: true, // HDR encoding
|
||||
}
|
||||
|
||||
cmd := encoder.BuildCommand(config)
|
||||
args := cmd.Args[1:]
|
||||
argsStr := strings.Join(args, " ")
|
||||
|
||||
// Verify all HDR colorspace metadata is present for EXR (HDR encoding)
|
||||
// Verify all HDR colorspace metadata is present for EXR (full range to match zscale output)
|
||||
colorspaceArgs := []string{
|
||||
"-color_primaries bt709", // bt709 primaries to match PNG color appearance
|
||||
"-color_trc arib-std-b67", // HLG transfer function for HDR/SDR compatibility
|
||||
"-colorspace bt709", // bt709 colorspace to match PNG
|
||||
"-color_range tv",
|
||||
"-color_primaries bt709",
|
||||
"-color_trc arib-std-b67",
|
||||
"-colorspace bt709",
|
||||
"-color_range pc",
|
||||
}
|
||||
|
||||
for _, arg := range colorspaceArgs {
|
||||
@@ -656,7 +596,6 @@ func TestIntegration_Encode_EXR_H264(t *testing.T) {
|
||||
WorkDir: tmpDir,
|
||||
UseAlpha: false,
|
||||
TwoPass: false, // Use single pass for faster testing
|
||||
SourceFormat: "exr",
|
||||
}
|
||||
|
||||
// Build and run command
|
||||
@@ -687,77 +626,6 @@ func TestIntegration_Encode_EXR_H264(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegration_Encode_PNG_H264(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
}
|
||||
|
||||
// Check if example file exists
|
||||
exampleDir := filepath.Join("..", "..", "..", "examples")
|
||||
pngFile := filepath.Join(exampleDir, "frame_0800.png")
|
||||
if _, err := os.Stat(pngFile); os.IsNotExist(err) {
|
||||
t.Skipf("Example file not found: %s", pngFile)
|
||||
}
|
||||
|
||||
// Get absolute paths
|
||||
workspaceRoot, err := filepath.Abs(filepath.Join("..", "..", ".."))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get workspace root: %v", err)
|
||||
}
|
||||
exampleDirAbs, err := filepath.Abs(exampleDir)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to get example directory: %v", err)
|
||||
}
|
||||
tmpDir := filepath.Join(workspaceRoot, "tmp")
|
||||
if err := os.MkdirAll(tmpDir, 0755); err != nil {
|
||||
t.Fatalf("Failed to create tmp directory: %v", err)
|
||||
}
|
||||
|
||||
encoder := &SoftwareEncoder{codec: "libx264"}
|
||||
config := &EncodeConfig{
|
||||
InputPattern: filepath.Join(exampleDirAbs, "frame_%04d.png"),
|
||||
OutputPath: filepath.Join(tmpDir, "test_png_h264.mp4"),
|
||||
StartFrame: 800,
|
||||
FrameRate: 24.0,
|
||||
WorkDir: tmpDir,
|
||||
UseAlpha: false,
|
||||
TwoPass: false, // Use single pass for faster testing
|
||||
SourceFormat: "png",
|
||||
}
|
||||
|
||||
// Build and run command
|
||||
cmd := encoder.BuildCommand(config)
|
||||
if cmd == nil {
|
||||
t.Fatal("BuildCommand returned nil")
|
||||
}
|
||||
|
||||
// Verify no video filter is used for PNG
|
||||
argsStr := strings.Join(cmd.Args, " ")
|
||||
if strings.Contains(argsStr, "-vf") {
|
||||
t.Error("PNG encoding should not use video filter, but -vf was found in command")
|
||||
}
|
||||
|
||||
// Run the command
|
||||
cmdOutput, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Errorf("FFmpeg command failed: %v\nCommand output: %s", err, string(cmdOutput))
|
||||
return
|
||||
}
|
||||
|
||||
// Verify output file was created
|
||||
if _, err := os.Stat(config.OutputPath); os.IsNotExist(err) {
|
||||
t.Errorf("Output file was not created: %s\nCommand output: %s", config.OutputPath, string(cmdOutput))
|
||||
} else {
|
||||
t.Logf("Successfully created output file: %s", config.OutputPath)
|
||||
info, _ := os.Stat(config.OutputPath)
|
||||
if info.Size() == 0 {
|
||||
t.Error("Output file was created but is empty")
|
||||
} else {
|
||||
t.Logf("Output file size: %d bytes", info.Size())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntegration_Encode_EXR_VP9(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test in short mode")
|
||||
@@ -800,7 +668,6 @@ func TestIntegration_Encode_EXR_VP9(t *testing.T) {
|
||||
WorkDir: tmpDir,
|
||||
UseAlpha: false,
|
||||
TwoPass: false, // Use single pass for faster testing
|
||||
SourceFormat: "exr",
|
||||
}
|
||||
|
||||
// Build and run command
|
||||
@@ -873,7 +740,6 @@ func TestIntegration_Encode_EXR_AV1(t *testing.T) {
|
||||
WorkDir: tmpDir,
|
||||
UseAlpha: false,
|
||||
TwoPass: false,
|
||||
SourceFormat: "exr",
|
||||
}
|
||||
|
||||
// Build and run command
|
||||
@@ -940,7 +806,6 @@ func TestIntegration_Encode_EXR_VP9_WithAlpha(t *testing.T) {
|
||||
WorkDir: tmpDir,
|
||||
UseAlpha: true, // Test with alpha
|
||||
TwoPass: false, // Use single pass for faster testing
|
||||
SourceFormat: "exr",
|
||||
}
|
||||
|
||||
// Build and run command
|
||||
|
||||
+365
-93
@@ -4,11 +4,13 @@ package runner
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -16,6 +18,7 @@ import (
|
||||
"jiggablend/internal/runner/api"
|
||||
"jiggablend/internal/runner/blender"
|
||||
"jiggablend/internal/runner/encoding"
|
||||
"jiggablend/internal/runner/sandbox"
|
||||
"jiggablend/internal/runner/tasks"
|
||||
"jiggablend/internal/runner/workspace"
|
||||
"jiggablend/pkg/executils"
|
||||
@@ -39,12 +42,74 @@ type Runner struct {
|
||||
|
||||
fingerprint string
|
||||
fingerprintMu sync.RWMutex
|
||||
|
||||
// gpuLockedOut is set when logs indicate a GPU error (e.g. HIP "Illegal address");
|
||||
// when true, the runner forces CPU rendering for all subsequent jobs.
|
||||
gpuLockedOut bool
|
||||
gpuLockedOutMu sync.RWMutex
|
||||
|
||||
// hasAMD/hasNVIDIA/hasIntel are set at startup by hardware/Blender GPU backend detection.
|
||||
// Used to force CPU only for Blender < 4.x when AMD is present (no official HIP support pre-4).
|
||||
// gpuDetectionFailed is true when detection could not run; we then force CPU for all versions.
|
||||
gpuBackendMu sync.RWMutex
|
||||
hasAMD bool
|
||||
hasNVIDIA bool
|
||||
hasIntel bool
|
||||
gpuBackendProbed bool
|
||||
gpuDetectionFailed bool
|
||||
|
||||
// forceCPURendering forces CPU rendering for all jobs regardless of metadata/backend detection.
|
||||
forceCPURendering bool
|
||||
// disableRT disables GPU ray tracing acceleration (HIPRT, OptiX, etc.).
|
||||
disableRT bool
|
||||
// hipGPUSampleBatch limits samples per GPU pass on gfx115x (0 = disabled).
|
||||
hipGPUSampleBatch int
|
||||
|
||||
// sandbox wraps Blender invocations (none/podman).
|
||||
sandboxWrapper sandbox.Wrapper
|
||||
sandboxBackend string
|
||||
}
|
||||
|
||||
// RunnerOptions configures optional runner behavior.
|
||||
type RunnerOptions struct {
|
||||
ForceCPURendering bool
|
||||
DisableRT bool
|
||||
HipGPUSampleBatch int
|
||||
SandboxBackend string // none|podman
|
||||
SandboxNetwork bool
|
||||
SandboxImage string // podman thin runtime image
|
||||
}
|
||||
|
||||
// New creates a new runner.
|
||||
func New(managerURL, name, hostname string) *Runner {
|
||||
func New(managerURL, name, hostname string, forceCPURendering, disableRT bool, hipGPUSampleBatch int) *Runner {
|
||||
return NewWithOptions(managerURL, name, hostname, RunnerOptions{
|
||||
ForceCPURendering: forceCPURendering,
|
||||
DisableRT: disableRT,
|
||||
HipGPUSampleBatch: hipGPUSampleBatch,
|
||||
SandboxBackend: sandbox.BackendPodman,
|
||||
})
|
||||
}
|
||||
|
||||
// NewWithOptions creates a runner with full options including sandbox.
|
||||
func NewWithOptions(managerURL, name, hostname string, opts RunnerOptions) *Runner {
|
||||
manager := api.NewManagerClient(managerURL)
|
||||
|
||||
backend, err := sandbox.NormalizeBackend(opts.SandboxBackend)
|
||||
if err != nil {
|
||||
log.Printf("Invalid sandbox backend %q, falling back to none: %v", opts.SandboxBackend, err)
|
||||
backend = sandbox.BackendNone
|
||||
}
|
||||
sb, err := sandbox.New(sandbox.Options{
|
||||
Backend: backend,
|
||||
AllowNetwork: opts.SandboxNetwork,
|
||||
PodmanImage: opts.SandboxImage,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to init sandbox %q: %v; using none", backend, err)
|
||||
sb, _ = sandbox.New(sandbox.Options{Backend: sandbox.BackendNone})
|
||||
backend = sandbox.BackendNone
|
||||
}
|
||||
|
||||
r := &Runner{
|
||||
name: name,
|
||||
hostname: hostname,
|
||||
@@ -52,6 +117,12 @@ func New(managerURL, name, hostname string) *Runner {
|
||||
processes: executils.NewProcessTracker(),
|
||||
stopChan: make(chan struct{}),
|
||||
processors: make(map[string]tasks.Processor),
|
||||
|
||||
forceCPURendering: opts.ForceCPURendering,
|
||||
disableRT: opts.DisableRT,
|
||||
hipGPUSampleBatch: opts.HipGPUSampleBatch,
|
||||
sandboxWrapper: sb,
|
||||
sandboxBackend: backend,
|
||||
}
|
||||
|
||||
// Generate fingerprint
|
||||
@@ -67,32 +138,56 @@ func (r *Runner) CheckRequiredTools() error {
|
||||
}
|
||||
log.Printf("Found zstd for compressed blend file support")
|
||||
|
||||
if err := exec.Command("xvfb-run", "--help").Run(); err != nil {
|
||||
return fmt.Errorf("xvfb-run not found - required for headless Blender rendering. Install with: apt install xvfb")
|
||||
if r.sandboxWrapper != nil {
|
||||
if err := r.sandboxWrapper.Available(); err != nil {
|
||||
return fmt.Errorf("sandbox backend %q unavailable: %w", r.sandboxBackend, err)
|
||||
}
|
||||
log.Printf("Sandbox backend: %s", r.sandboxWrapper.Name())
|
||||
}
|
||||
log.Printf("Found xvfb-run for headless rendering without -b option")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var cachedCapabilities map[string]interface{} = nil
|
||||
// SandboxBackend returns the configured sandbox backend name.
|
||||
func (r *Runner) SandboxBackend() string {
|
||||
if r.sandboxBackend == "" {
|
||||
return sandbox.BackendNone
|
||||
}
|
||||
return r.sandboxBackend
|
||||
}
|
||||
|
||||
var (
|
||||
cachedCapabilities map[string]interface{}
|
||||
capabilitiesOnce sync.Once
|
||||
)
|
||||
|
||||
// ProbeCapabilities detects hardware capabilities.
|
||||
func (r *Runner) ProbeCapabilities() map[string]interface{} {
|
||||
if cachedCapabilities != nil {
|
||||
return cachedCapabilities
|
||||
capabilitiesOnce.Do(func() {
|
||||
caps := make(map[string]interface{})
|
||||
|
||||
if err := exec.Command("ffmpeg", "-version").Run(); err == nil {
|
||||
caps["ffmpeg"] = true
|
||||
} else {
|
||||
caps["ffmpeg"] = false
|
||||
}
|
||||
|
||||
// Filled later with actual backend when runner is constructed; probe is package-level once.
|
||||
// Real sandbox name is injected in ProbeCapabilities on the instance after New.
|
||||
caps["sandbox"] = "none"
|
||||
|
||||
cachedCapabilities = caps
|
||||
})
|
||||
// Overlay instance sandbox name (Once already ran with none default).
|
||||
if r != nil && r.sandboxWrapper != nil {
|
||||
out := make(map[string]interface{}, len(cachedCapabilities)+1)
|
||||
for k, v := range cachedCapabilities {
|
||||
out[k] = v
|
||||
}
|
||||
out["sandbox"] = r.sandboxWrapper.Name()
|
||||
return out
|
||||
}
|
||||
|
||||
caps := make(map[string]interface{})
|
||||
|
||||
// Check for ffmpeg and probe encoding capabilities
|
||||
if err := exec.Command("ffmpeg", "-version").Run(); err == nil {
|
||||
caps["ffmpeg"] = true
|
||||
} else {
|
||||
caps["ffmpeg"] = false
|
||||
}
|
||||
|
||||
cachedCapabilities = caps
|
||||
return caps
|
||||
return cachedCapabilities
|
||||
}
|
||||
|
||||
// Register registers the runner with the manager.
|
||||
@@ -122,6 +217,82 @@ func (r *Runner) Register(apiKey string) (int64, error) {
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// DetectAndStoreGPUBackends runs host-level backend detection and stores AMD/NVIDIA/Intel results.
|
||||
// Call after Register. Used so we only force CPU for Blender < 4.x when AMD is present.
|
||||
func (r *Runner) DetectAndStoreGPUBackends() {
|
||||
r.gpuBackendMu.Lock()
|
||||
defer r.gpuBackendMu.Unlock()
|
||||
if r.gpuBackendProbed {
|
||||
return
|
||||
}
|
||||
hasAMD, hasNVIDIA, hasIntel, ok := blender.DetectGPUBackends()
|
||||
if !ok {
|
||||
log.Printf("GPU backend detection failed (host probe unavailable). All jobs will use CPU because backend availability is unknown.")
|
||||
r.gpuBackendProbed = true
|
||||
r.gpuDetectionFailed = true
|
||||
return
|
||||
}
|
||||
|
||||
detectedTypes := 0
|
||||
if hasAMD {
|
||||
detectedTypes++
|
||||
}
|
||||
if hasNVIDIA {
|
||||
detectedTypes++
|
||||
}
|
||||
if hasIntel {
|
||||
detectedTypes++
|
||||
}
|
||||
if detectedTypes > 1 {
|
||||
log.Printf("mixed GPU vendors detected (AMD=%v NVIDIA=%v INTEL=%v): multi-vendor setups may not work reliably, but runner will continue with GPU enabled", hasAMD, hasNVIDIA, hasIntel)
|
||||
}
|
||||
|
||||
r.hasAMD = hasAMD
|
||||
r.hasNVIDIA = hasNVIDIA
|
||||
r.hasIntel = hasIntel
|
||||
r.gpuBackendProbed = true
|
||||
r.gpuDetectionFailed = false
|
||||
log.Printf("GPU backend detection: AMD=%v NVIDIA=%v INTEL=%v (Blender < 4.x will force CPU only when AMD is present)", hasAMD, hasNVIDIA, hasIntel)
|
||||
}
|
||||
|
||||
// HasAMD returns whether the runner detected AMD devices. Used to force CPU for Blender < 4.x only when AMD is present.
|
||||
func (r *Runner) HasAMD() bool {
|
||||
r.gpuBackendMu.RLock()
|
||||
defer r.gpuBackendMu.RUnlock()
|
||||
return r.hasAMD
|
||||
}
|
||||
|
||||
// HasNVIDIA returns whether the runner detected NVIDIA GPUs.
|
||||
func (r *Runner) HasNVIDIA() bool {
|
||||
r.gpuBackendMu.RLock()
|
||||
defer r.gpuBackendMu.RUnlock()
|
||||
return r.hasNVIDIA
|
||||
}
|
||||
|
||||
// HasIntel returns whether the runner detected Intel GPUs (e.g. Arc).
|
||||
func (r *Runner) HasIntel() bool {
|
||||
r.gpuBackendMu.RLock()
|
||||
defer r.gpuBackendMu.RUnlock()
|
||||
return r.hasIntel
|
||||
}
|
||||
|
||||
// DisableRT returns whether GPU ray tracing acceleration should be disabled.
|
||||
func (r *Runner) DisableRT() bool {
|
||||
return r.disableRT
|
||||
}
|
||||
|
||||
// HipGPUSampleBatch returns the per-pass GPU sample limit for gfx115x batching (0 = disabled).
|
||||
func (r *Runner) HipGPUSampleBatch() int {
|
||||
return r.hipGPUSampleBatch
|
||||
}
|
||||
|
||||
// GPUDetectionFailed returns true when startup GPU backend detection could not run or failed. When true, all jobs use CPU because backend availability is unknown.
|
||||
func (r *Runner) GPUDetectionFailed() bool {
|
||||
r.gpuBackendMu.RLock()
|
||||
defer r.gpuBackendMu.RUnlock()
|
||||
return r.gpuDetectionFailed
|
||||
}
|
||||
|
||||
// Start starts the job polling loop.
|
||||
func (r *Runner) Start(pollInterval time.Duration) {
|
||||
log.Printf("Starting job polling loop (interval: %v)", pollInterval)
|
||||
@@ -182,6 +353,24 @@ func (r *Runner) Cleanup() {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) withJobWorkspace(jobID int64, fn func(workDir string) error) error {
|
||||
workDir, err := r.workspace.CreateJobDir(jobID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create job workspace: %w", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if cleanupErr := r.workspace.CleanupJobDir(jobID); cleanupErr != nil {
|
||||
log.Printf("Warning: failed to cleanup job workspace for job %d: %v", jobID, cleanupErr)
|
||||
}
|
||||
if cleanupErr := r.workspace.CleanupVideoDir(jobID); cleanupErr != nil {
|
||||
log.Printf("Warning: failed to cleanup encode workspace for job %d: %v", jobID, cleanupErr)
|
||||
}
|
||||
}()
|
||||
|
||||
return fn(workDir)
|
||||
}
|
||||
|
||||
// executeJob handles a job using per-job WebSocket connection.
|
||||
func (r *Runner) executeJob(job *api.NextJobResponse) (err error) {
|
||||
// Recover from panics to prevent runner process crashes during task execution
|
||||
@@ -192,72 +381,101 @@ func (r *Runner) executeJob(job *api.NextJobResponse) (err error) {
|
||||
}
|
||||
}()
|
||||
|
||||
// Connect to job WebSocket (no runnerID needed - authentication handles it)
|
||||
jobConn := api.NewJobConnection()
|
||||
if err := jobConn.Connect(r.manager.GetBaseURL(), job.JobPath, job.JobToken); err != nil {
|
||||
return fmt.Errorf("failed to connect job WebSocket: %w", err)
|
||||
}
|
||||
defer jobConn.Close()
|
||||
|
||||
log.Printf("Job WebSocket authenticated for task %d", job.Task.TaskID)
|
||||
|
||||
// Create task context
|
||||
workDir := r.workspace.JobDir(job.Task.JobID)
|
||||
ctx := tasks.NewContext(
|
||||
job.Task.TaskID,
|
||||
job.Task.JobID,
|
||||
job.Task.JobName,
|
||||
job.Task.Frame,
|
||||
job.Task.TaskType,
|
||||
workDir,
|
||||
job.JobToken,
|
||||
job.Task.Metadata,
|
||||
r.manager,
|
||||
jobConn,
|
||||
r.workspace,
|
||||
r.blender,
|
||||
r.encoder,
|
||||
r.processes,
|
||||
)
|
||||
|
||||
ctx.Info(fmt.Sprintf("Task assignment received (job: %d, type: %s)",
|
||||
job.Task.JobID, job.Task.TaskType))
|
||||
|
||||
// Get processor for task type
|
||||
processor, ok := r.processors[job.Task.TaskType]
|
||||
if !ok {
|
||||
return fmt.Errorf("unknown task type: %s", job.Task.TaskType)
|
||||
}
|
||||
|
||||
// Process the task
|
||||
var processErr error
|
||||
switch job.Task.TaskType {
|
||||
case "render": // this task has a upload outputs step because the frames are not uploaded by the render task directly we have to do it manually here TODO: maybe we should make it work like the encode task
|
||||
// Download context
|
||||
contextPath := job.JobPath + "/context.tar"
|
||||
if err := r.downloadContext(job.Task.JobID, contextPath, job.JobToken); err != nil {
|
||||
jobConn.Log(job.Task.TaskID, types.LogLevelError, fmt.Sprintf("Failed to download context: %v", err))
|
||||
jobConn.Complete(job.Task.TaskID, false, fmt.Errorf("failed to download context: %v", err))
|
||||
return fmt.Errorf("failed to download context: %w", err)
|
||||
return r.withJobWorkspace(job.Task.JobID, func(workDir string) error {
|
||||
// Connect to job WebSocket (no runnerID needed - authentication handles it)
|
||||
jobConn := api.NewJobConnection()
|
||||
if err := jobConn.Connect(r.manager.GetBaseURL(), job.JobPath, job.JobToken); err != nil {
|
||||
return fmt.Errorf("failed to connect job WebSocket: %w", err)
|
||||
}
|
||||
processErr = processor.Process(ctx)
|
||||
if processErr == nil {
|
||||
processErr = r.uploadOutputs(ctx, job)
|
||||
defer jobConn.Close()
|
||||
|
||||
log.Printf("Job WebSocket authenticated for task %d", job.Task.TaskID)
|
||||
|
||||
// Create task context (frame range: Frame = start, FrameEnd = end; 0 or missing = single frame)
|
||||
frameEnd := job.Task.FrameEnd
|
||||
if frameEnd < job.Task.Frame {
|
||||
frameEnd = job.Task.Frame
|
||||
}
|
||||
ctx := tasks.NewContext(
|
||||
job.Task.TaskID,
|
||||
job.Task.JobID,
|
||||
job.Task.JobName,
|
||||
job.Task.Frame,
|
||||
frameEnd,
|
||||
job.Task.TaskType,
|
||||
workDir,
|
||||
job.JobToken,
|
||||
job.Task.Metadata,
|
||||
r.manager,
|
||||
jobConn,
|
||||
r.workspace,
|
||||
r.blender,
|
||||
r.encoder,
|
||||
r.processes,
|
||||
r.IsGPULockedOut(),
|
||||
r.HasAMD(),
|
||||
r.HasNVIDIA(),
|
||||
r.HasIntel(),
|
||||
r.GPUDetectionFailed(),
|
||||
r.forceCPURendering,
|
||||
r.disableRT,
|
||||
r.hipGPUSampleBatch,
|
||||
nil, // set below so the callback can mark this attempt
|
||||
r.sandboxWrapper,
|
||||
)
|
||||
// Arm GPU lockout at most once process-wide; if this attempt is the one that
|
||||
// arms it, mark the context so a failure requeues without burning retry_count.
|
||||
ctx.OnGPUError = func() {
|
||||
if r.SetGPULockedOut(true) {
|
||||
ctx.GPULockoutArmedThisAttempt = true
|
||||
ctx.GPULockedOut = true
|
||||
ctx.Warn("GPU error detected; GPU disabled for subsequent jobs (this attempt free-requeues without using a retry)")
|
||||
}
|
||||
}
|
||||
case "encode": // this task doesn't have a upload outputs step because the video is already uploaded by the encode task
|
||||
processErr = processor.Process(ctx)
|
||||
default:
|
||||
return fmt.Errorf("unknown task type: %s", job.Task.TaskType)
|
||||
}
|
||||
|
||||
if processErr != nil {
|
||||
ctx.Error(fmt.Sprintf("Task failed: %v", processErr))
|
||||
ctx.Complete(false, processErr)
|
||||
return processErr
|
||||
}
|
||||
ctx.Info(fmt.Sprintf("Task assignment received (job: %d, type: %s)",
|
||||
job.Task.JobID, job.Task.TaskType))
|
||||
|
||||
ctx.Complete(true, nil)
|
||||
return nil
|
||||
// Get processor for task type
|
||||
processor, ok := r.processors[job.Task.TaskType]
|
||||
if !ok {
|
||||
return fmt.Errorf("unknown task type: %s", job.Task.TaskType)
|
||||
}
|
||||
|
||||
// Process the task
|
||||
var processErr error
|
||||
switch job.Task.TaskType {
|
||||
case "render": // this task has a upload outputs step because the frames are not uploaded by the render task directly we have to do it manually here TODO: maybe we should make it work like the encode task
|
||||
// Download context
|
||||
contextPath := job.JobPath + "/context.tar"
|
||||
if err := r.downloadContext(job.Task.JobID, contextPath, job.JobToken); err != nil {
|
||||
jobConn.Log(job.Task.TaskID, types.LogLevelError, fmt.Sprintf("Failed to download context: %v", err))
|
||||
jobConn.Complete(job.Task.TaskID, false, fmt.Errorf("failed to download context: %v", err), false)
|
||||
return fmt.Errorf("failed to download context: %w", err)
|
||||
}
|
||||
processErr = processor.Process(ctx)
|
||||
if processErr == nil {
|
||||
processErr = r.uploadOutputs(ctx, job)
|
||||
}
|
||||
case "encode": // this task doesn't have a upload outputs step because the video is already uploaded by the encode task
|
||||
processErr = processor.Process(ctx)
|
||||
default:
|
||||
return fmt.Errorf("unknown task type: %s", job.Task.TaskType)
|
||||
}
|
||||
|
||||
if processErr != nil {
|
||||
if errors.Is(processErr, tasks.ErrJobCancelled) {
|
||||
ctx.Warn("Stopping task early because the job was cancelled")
|
||||
return nil
|
||||
}
|
||||
ctx.Error(fmt.Sprintf("Task failed: %v", processErr))
|
||||
ctx.Complete(false, processErr)
|
||||
return processErr
|
||||
}
|
||||
|
||||
ctx.Complete(true, nil)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Runner) downloadContext(jobID int64, contextPath, jobToken string) error {
|
||||
@@ -275,27 +493,56 @@ func (r *Runner) uploadOutputs(ctx *tasks.Context, job *api.NextJobResponse) err
|
||||
outputDir := ctx.WorkDir + "/output"
|
||||
uploadPath := fmt.Sprintf("/api/runner/jobs/%d/upload", job.Task.JobID)
|
||||
|
||||
return uploadOutputFiles(outputDir, func(filePath, fileName string) error {
|
||||
if err := r.manager.UploadFile(uploadPath, job.JobToken, filePath); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx.OutputUploaded(fileName)
|
||||
// Delete file after successful upload to prevent duplicate uploads
|
||||
if err := os.Remove(filePath); err != nil {
|
||||
log.Printf("Warning: Failed to delete file %s after upload: %v", filePath, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// uploadOutputFiles uploads all non-directory files from outputDir.
|
||||
// Fails if the directory cannot be read, any upload fails, or zero files were uploaded.
|
||||
func uploadOutputFiles(outputDir string, uploadFn func(filePath, fileName string) error) error {
|
||||
entries, err := os.ReadDir(outputDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read output directory: %w", err)
|
||||
}
|
||||
|
||||
var files []os.DirEntry
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
filePath := outputDir + "/" + entry.Name()
|
||||
if err := r.manager.UploadFile(uploadPath, job.JobToken, filePath); err != nil {
|
||||
log.Printf("Failed to upload %s: %v", filePath, err)
|
||||
} else {
|
||||
ctx.OutputUploaded(entry.Name())
|
||||
// Delete file after successful upload to prevent duplicate uploads
|
||||
if err := os.Remove(filePath); err != nil {
|
||||
log.Printf("Warning: Failed to delete file %s after upload: %v", filePath, err)
|
||||
}
|
||||
if !entry.IsDir() {
|
||||
files = append(files, entry)
|
||||
}
|
||||
}
|
||||
if len(files) == 0 {
|
||||
return fmt.Errorf("no output files found in %s", outputDir)
|
||||
}
|
||||
|
||||
var firstErr error
|
||||
uploaded := 0
|
||||
for _, entry := range files {
|
||||
filePath := filepath.Join(outputDir, entry.Name())
|
||||
if err := uploadFn(filePath, entry.Name()); err != nil {
|
||||
log.Printf("Failed to upload %s: %v", filePath, err)
|
||||
if firstErr == nil {
|
||||
firstErr = fmt.Errorf("failed to upload %s: %w", entry.Name(), err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
uploaded++
|
||||
}
|
||||
if firstErr != nil {
|
||||
return firstErr
|
||||
}
|
||||
if uploaded == 0 {
|
||||
return fmt.Errorf("no output files were uploaded from %s", outputDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -363,3 +610,28 @@ func (r *Runner) GetFingerprint() string {
|
||||
func (r *Runner) GetID() int64 {
|
||||
return r.id
|
||||
}
|
||||
|
||||
// SetGPULockedOut sets whether GPU use is locked out due to a detected GPU error.
|
||||
// When true, the runner will force CPU rendering for all jobs.
|
||||
// Returns true only on the false→true transition (first arm); subsequent calls are no-ops for logging.
|
||||
func (r *Runner) SetGPULockedOut(locked bool) (newlyEnabled bool) {
|
||||
r.gpuLockedOutMu.Lock()
|
||||
defer r.gpuLockedOutMu.Unlock()
|
||||
if locked {
|
||||
if r.gpuLockedOut {
|
||||
return false
|
||||
}
|
||||
r.gpuLockedOut = true
|
||||
log.Printf("GPU lockout enabled: GPU rendering disabled for subsequent jobs (CPU only)")
|
||||
return true
|
||||
}
|
||||
r.gpuLockedOut = false
|
||||
return false
|
||||
}
|
||||
|
||||
// IsGPULockedOut returns whether GPU use is currently locked out.
|
||||
func (r *Runner) IsGPULockedOut() bool {
|
||||
r.gpuLockedOutMu.RLock()
|
||||
defer r.gpuLockedOutMu.RUnlock()
|
||||
return r.gpuLockedOut
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewRunner_InitializesFields(t *testing.T) {
|
||||
r := New("http://localhost:8080", "runner-a", "host-a", false, false, 0)
|
||||
if r == nil {
|
||||
t.Fatal("New should return a runner")
|
||||
}
|
||||
if r.name != "runner-a" || r.hostname != "host-a" {
|
||||
t.Fatalf("unexpected runner identity: %q %q", r.name, r.hostname)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunner_GPUFlagsSetters(t *testing.T) {
|
||||
r := New("http://localhost:8080", "runner-a", "host-a", false, false, 0)
|
||||
if newly := r.SetGPULockedOut(true); !newly {
|
||||
t.Fatal("expected first SetGPULockedOut(true) to report newly enabled")
|
||||
}
|
||||
if !r.IsGPULockedOut() {
|
||||
t.Fatal("expected GPU lockout to be true")
|
||||
}
|
||||
if newly := r.SetGPULockedOut(true); newly {
|
||||
t.Fatal("expected second SetGPULockedOut(true) to be a no-op transition")
|
||||
}
|
||||
if newly := r.SetGPULockedOut(false); newly {
|
||||
t.Fatal("clearing lockout should not report newly enabled")
|
||||
}
|
||||
if r.IsGPULockedOut() {
|
||||
t.Fatal("expected GPU lockout cleared")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateFingerprint_PopulatesValue(t *testing.T) {
|
||||
r := New("http://localhost:8080", "runner-a", "host-a", false, false, 0)
|
||||
r.generateFingerprint()
|
||||
fp := r.GetFingerprint()
|
||||
if fp == "" {
|
||||
t.Fatal("fingerprint should not be empty")
|
||||
}
|
||||
if len(fp) != 64 {
|
||||
t.Fatalf("fingerprint should be sha256 hex, got %q", fp)
|
||||
}
|
||||
if _, err := hex.DecodeString(fp); err != nil {
|
||||
t.Fatalf("fingerprint should be valid hex: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadOutputFiles_FailsWhenUploadErrors(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "frame_0001.exr"), []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := uploadOutputFiles(dir, func(filePath, fileName string) error {
|
||||
return errors.New("upload denied")
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error when upload fails")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadOutputFiles_FailsWhenEmpty(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
err := uploadOutputFiles(dir, func(filePath, fileName string) error {
|
||||
t.Fatal("upload should not be called")
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty output dir")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadOutputFiles_Success(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "frame_0001.exr"), []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var seen string
|
||||
err := uploadOutputFiles(dir, func(filePath, fileName string) error {
|
||||
seen = fileName
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("uploadOutputFiles: %v", err)
|
||||
}
|
||||
if seen != "frame_0001.exr" {
|
||||
t.Fatalf("got %q", seen)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Bind describes a host path to expose inside the sandbox.
|
||||
type Bind struct {
|
||||
Host string
|
||||
// Dest is the path inside the sandbox (usually same as Host for absolute paths).
|
||||
Dest string
|
||||
// ReadOnly is true for library trees and Blender installs.
|
||||
ReadOnly bool
|
||||
// Dev is true for device nodes / device trees (podman --device or -v for dirs).
|
||||
Dev bool
|
||||
// Optional means skip if host path missing (no error).
|
||||
Optional bool
|
||||
}
|
||||
|
||||
// GPUMounts returns host paths to expose for Cycles GPU backends based on detection.
|
||||
// forceCPU skips GPU-specific devices/libs (still may include generic /dev via backend).
|
||||
func GPUMounts(hasAMD, hasNVIDIA, hasIntel, forceCPU bool) []Bind {
|
||||
if forceCPU {
|
||||
return nil
|
||||
}
|
||||
var binds []Bind
|
||||
|
||||
// DRM render nodes used by AMD, Intel, and some NVIDIA EGL paths.
|
||||
if hasAMD || hasNVIDIA || hasIntel {
|
||||
binds = append(binds, Bind{Host: "/dev/dri", Dest: "/dev/dri", Dev: true, Optional: true})
|
||||
}
|
||||
|
||||
if hasAMD {
|
||||
binds = append(binds, Bind{Host: "/dev/kfd", Dest: "/dev/kfd", Dev: true, Optional: true})
|
||||
// Common ROCm install layouts
|
||||
for _, p := range []string{"/opt/rocm", "/usr/share/libdrm"} {
|
||||
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||
}
|
||||
// Distro ROCm / amdgpu userspace libs often live under multiarch paths
|
||||
for _, p := range rocmLibHints() {
|
||||
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||
}
|
||||
}
|
||||
|
||||
if hasNVIDIA {
|
||||
for _, name := range nvidiaDeviceNames() {
|
||||
p := filepath.Join("/dev", name)
|
||||
binds = append(binds, Bind{Host: p, Dest: p, Dev: true, Optional: true})
|
||||
}
|
||||
for _, p := range nvidiaLibHints() {
|
||||
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||
}
|
||||
}
|
||||
|
||||
if hasIntel {
|
||||
for _, p := range intelLibHints() {
|
||||
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||
}
|
||||
}
|
||||
|
||||
return uniqueBinds(binds)
|
||||
}
|
||||
|
||||
func nvidiaDeviceNames() []string {
|
||||
// Fixed control nodes + any /dev/nvidiaN
|
||||
names := []string{"nvidiactl", "nvidia-uvm", "nvidia-uvm-tools", "nvidia-modeset"}
|
||||
entries, err := os.ReadDir("/dev")
|
||||
if err != nil {
|
||||
return names
|
||||
}
|
||||
for _, e := range entries {
|
||||
n := e.Name()
|
||||
if strings.HasPrefix(n, "nvidia") && !containsString(names, n) {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func nvidiaLibHints() []string {
|
||||
hints := []string{
|
||||
"/usr/lib/wsl/lib", // WSL NVIDIA
|
||||
"/usr/local/nvidia",
|
||||
"/usr/local/cuda",
|
||||
"/usr/lib/x86_64-linux-gnu",
|
||||
"/usr/lib64",
|
||||
}
|
||||
// Driver stores often under /usr/lib/libcuda* — parent dirs already covered.
|
||||
// Also scan common multiarch for libcuda.so
|
||||
for _, dir := range []string{"/usr/lib", "/usr/lib64", "/usr/lib/x86_64-linux-gnu", "/lib", "/lib64"} {
|
||||
if matchesAny(dir, "libcuda.so*") || matchesAny(dir, "libnvidia-*.so*") {
|
||||
hints = append(hints, dir)
|
||||
}
|
||||
}
|
||||
return hints
|
||||
}
|
||||
|
||||
func rocmLibHints() []string {
|
||||
hints := []string{
|
||||
"/usr/lib/x86_64-linux-gnu",
|
||||
"/usr/lib64",
|
||||
"/opt/amdgpu",
|
||||
}
|
||||
// ROCm versioned trees under /opt/rocm-*
|
||||
if entries, err := os.ReadDir("/opt"); err == nil {
|
||||
for _, e := range entries {
|
||||
if e.IsDir() && strings.HasPrefix(e.Name(), "rocm") {
|
||||
hints = append(hints, filepath.Join("/opt", e.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
return hints
|
||||
}
|
||||
|
||||
func intelLibHints() []string {
|
||||
return []string{
|
||||
"/usr/lib/x86_64-linux-gnu",
|
||||
"/usr/lib64",
|
||||
"/usr/lib/intel-opencl",
|
||||
"/etc/OpenCL",
|
||||
}
|
||||
}
|
||||
|
||||
func matchesAny(dir, glob string) bool {
|
||||
m, err := filepath.Glob(filepath.Join(dir, glob))
|
||||
return err == nil && len(m) > 0
|
||||
}
|
||||
|
||||
func containsString(ss []string, s string) bool {
|
||||
for _, x := range ss {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func uniqueBinds(in []Bind) []Bind {
|
||||
seen := make(map[string]bool)
|
||||
var out []Bind
|
||||
for _, b := range in {
|
||||
key := b.Host + "|" + b.Dest + "|"
|
||||
if b.Dev {
|
||||
key += "d"
|
||||
}
|
||||
if b.ReadOnly {
|
||||
key += "r"
|
||||
}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, b)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ExistingBinds filters to binds whose host path exists (or non-optional always kept for error reporting).
|
||||
func ExistingBinds(binds []Bind) []Bind {
|
||||
var out []Bind
|
||||
for _, b := range binds {
|
||||
if pathExists(b.Host) {
|
||||
out = append(out, b)
|
||||
continue
|
||||
}
|
||||
if !b.Optional {
|
||||
out = append(out, b) // caller may error
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package sandbox
|
||||
|
||||
import "os/exec"
|
||||
|
||||
type noneWrapper struct{}
|
||||
|
||||
func (w *noneWrapper) Name() string { return BackendNone }
|
||||
|
||||
func (w *noneWrapper) Available() error { return nil }
|
||||
|
||||
func (w *noneWrapper) Wrap(spec Spec) (*exec.Cmd, error) {
|
||||
cmd := exec.Command(spec.BlenderBinary, spec.Args...)
|
||||
cmd.Dir = spec.WorkDir
|
||||
if len(spec.Env) > 0 {
|
||||
cmd.Env = spec.Env
|
||||
}
|
||||
return cmd, nil
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package sandbox
|
||||
|
||||
import "path/filepath"
|
||||
|
||||
func execAbs(p string) (string, error) {
|
||||
return filepath.Abs(p)
|
||||
}
|
||||
|
||||
// blenderRoot returns the directory that contains the blender binary
|
||||
// (the version extract root, e.g. .../blender-versions/4.5.7).
|
||||
func blenderRoot(blenderBinary string) string {
|
||||
return filepath.Dir(mustAbs(blenderBinary))
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultPodmanImage = "registry.fedoraproject.org/fedora-minimal:41"
|
||||
|
||||
type podmanWrapper struct {
|
||||
allowNetwork bool
|
||||
image string
|
||||
}
|
||||
|
||||
func (w *podmanWrapper) Name() string { return BackendPodman }
|
||||
|
||||
func (w *podmanWrapper) Available() error {
|
||||
if _, err := LookPath("podman"); err != nil {
|
||||
return fmt.Errorf("podman not found in PATH: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *podmanWrapper) Wrap(spec Spec) (*exec.Cmd, error) {
|
||||
if err := w.Available(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bin := mustAbs(spec.BlenderBinary)
|
||||
work := mustAbs(spec.WorkDir)
|
||||
home := mustAbs(spec.HomeDir)
|
||||
if home == "" {
|
||||
home = filepath.Join(work, "home")
|
||||
}
|
||||
root := blenderRoot(bin)
|
||||
|
||||
args := []string{
|
||||
"run", "--rm",
|
||||
"--security-opt", "label=disable",
|
||||
// Keep user groups for /dev/dri access (render/video)
|
||||
"--group-add", "keep-groups",
|
||||
}
|
||||
if !w.allowNetwork {
|
||||
args = append(args, "--network=none")
|
||||
}
|
||||
// Map to same UID so bind mounts are writable
|
||||
uid := os.Getuid()
|
||||
gid := os.Getgid()
|
||||
args = append(args, "--user", fmt.Sprintf("%d:%d", uid, gid))
|
||||
|
||||
// Thin OS image + host Blender tree + job dir
|
||||
args = append(args,
|
||||
"-v", root+":"+root+":ro",
|
||||
"-v", work+":"+work+":rw",
|
||||
)
|
||||
if home != work && !strings.HasPrefix(home, work+string(os.PathSeparator)) {
|
||||
_ = os.MkdirAll(home, 0755)
|
||||
args = append(args, "-v", home+":"+home+":rw")
|
||||
}
|
||||
|
||||
// System libs for GPU ICDs / dynamic linker (Blender tarball is mostly self-contained)
|
||||
for _, p := range []string{"/usr", "/lib", "/lib64", "/etc"} {
|
||||
if pathExists(p) {
|
||||
args = append(args, "-v", p+":"+p+":ro")
|
||||
}
|
||||
}
|
||||
|
||||
// GPU devices and extra lib roots
|
||||
for _, b := range ExistingBinds(GPUMounts(spec.HasAMD, spec.HasNVIDIA, spec.HasIntel, spec.ForceCPU)) {
|
||||
if !pathExists(b.Host) {
|
||||
continue
|
||||
}
|
||||
if b.Dev {
|
||||
// --device works for character devices; for /dev/dri use volume
|
||||
info, err := os.Stat(b.Host)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if info.IsDir() {
|
||||
args = append(args, "-v", b.Host+":"+b.Dest+":ro")
|
||||
} else {
|
||||
args = append(args, "--device", b.Host)
|
||||
}
|
||||
continue
|
||||
}
|
||||
mode := "ro"
|
||||
if !b.ReadOnly {
|
||||
mode = "rw"
|
||||
}
|
||||
args = append(args, "-v", b.Host+":"+b.Dest+":"+mode)
|
||||
}
|
||||
|
||||
// Environment
|
||||
for _, e := range spec.Env {
|
||||
args = append(args, "-e", e)
|
||||
}
|
||||
if home != "" {
|
||||
args = append(args, "-e", "HOME="+home)
|
||||
}
|
||||
|
||||
args = append(args, "--workdir", work)
|
||||
args = append(args, "--entrypoint", bin)
|
||||
args = append(args, w.image)
|
||||
// entrypoint is blender; remaining args are blender args
|
||||
args = append(args, spec.Args...)
|
||||
|
||||
cmd := exec.Command("podman", args...)
|
||||
cmd.Dir = work
|
||||
return cmd, nil
|
||||
}
|
||||
|
||||
// PodmanImageDefault returns the default thin runtime image name.
|
||||
func PodmanImageDefault() string { return defaultPodmanImage }
|
||||
|
||||
// FormatUIDGID is a small helper for tests.
|
||||
func FormatUIDGID(uid, gid int) string {
|
||||
return strconv.Itoa(uid) + ":" + strconv.Itoa(gid)
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
// Package sandbox wraps Blender (and similar) invocations in optional isolation.
|
||||
//
|
||||
// Backends:
|
||||
// - none: run on the host (current behavior)
|
||||
// - podman: rootless podman container; Blender tarball is bind-mounted (not baked into an image)
|
||||
//
|
||||
// GPU access is provided by passing host device nodes and common userspace lib roots
|
||||
// discovered at wrap time (NVIDIA / AMD ROCm / Intel DRM), not by shipping per-version
|
||||
// Blender container images.
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Backend names accepted by New and CLI flags.
|
||||
const (
|
||||
BackendNone = "none"
|
||||
BackendPodman = "podman"
|
||||
)
|
||||
|
||||
// Options configures sandbox construction.
|
||||
type Options struct {
|
||||
// Backend is none|podman (default none).
|
||||
Backend string
|
||||
// AllowNetwork keeps network in the sandbox (default false for podman).
|
||||
AllowNetwork bool
|
||||
// PodmanImage is used only for backend=podman. The image is a thin OS root;
|
||||
// Blender comes from a host bind of the versioned tarball tree.
|
||||
// Default: registry.fedoraproject.org/fedora-minimal:41
|
||||
PodmanImage string
|
||||
}
|
||||
|
||||
// Spec describes a Blender process to run under the sandbox.
|
||||
type Spec struct {
|
||||
// BlenderBinary is an absolute path to the blender executable.
|
||||
BlenderBinary string
|
||||
// Args are arguments after the binary (not including argv0).
|
||||
Args []string
|
||||
// WorkDir is the process working directory (job workspace); bind-mounted RW.
|
||||
WorkDir string
|
||||
// HomeDir is Blender HOME (usually WorkDir/home); bind-mounted RW.
|
||||
HomeDir string
|
||||
// Env is the full environment for the process (HOME, LD_LIBRARY_PATH, etc.).
|
||||
Env []string
|
||||
|
||||
// GPU hints from host detection (used to select device/lib binds).
|
||||
HasAMD bool
|
||||
HasNVIDIA bool
|
||||
HasIntel bool
|
||||
// ForceCPU skips GPU device binds when true.
|
||||
ForceCPU bool
|
||||
// AllowNetwork overrides Options.AllowNetwork when non-nil... kept simple: use Options only.
|
||||
}
|
||||
|
||||
// Wrapper turns a Spec into an *exec.Cmd ready to Start.
|
||||
type Wrapper interface {
|
||||
// Name returns the backend name.
|
||||
Name() string
|
||||
// Available reports whether required host tools exist.
|
||||
Available() error
|
||||
// Wrap builds the command. Callers own Start/Wait/pipes.
|
||||
Wrap(spec Spec) (*exec.Cmd, error)
|
||||
}
|
||||
|
||||
// New returns a sandbox Wrapper for the given options.
|
||||
func New(opts Options) (Wrapper, error) {
|
||||
backend := strings.ToLower(strings.TrimSpace(opts.Backend))
|
||||
if backend == "" {
|
||||
backend = BackendPodman
|
||||
}
|
||||
switch backend {
|
||||
case BackendNone:
|
||||
return &noneWrapper{}, nil
|
||||
case BackendPodman:
|
||||
img := opts.PodmanImage
|
||||
if img == "" {
|
||||
img = defaultPodmanImage
|
||||
}
|
||||
w := &podmanWrapper{allowNetwork: opts.AllowNetwork, image: img}
|
||||
return w, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown sandbox backend %q (want none or podman)", opts.Backend)
|
||||
}
|
||||
}
|
||||
|
||||
// NormalizeBackend validates and returns a canonical backend name.
|
||||
func NormalizeBackend(s string) (string, error) {
|
||||
b := strings.ToLower(strings.TrimSpace(s))
|
||||
if b == "" {
|
||||
return BackendPodman, nil
|
||||
}
|
||||
switch b {
|
||||
case BackendNone, BackendPodman:
|
||||
return b, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown sandbox backend %q (want none or podman)", s)
|
||||
}
|
||||
}
|
||||
|
||||
// LookPath is os/exec.LookPath, overridable in tests.
|
||||
var LookPath = exec.LookPath
|
||||
|
||||
// pathExists reports whether path exists.
|
||||
func pathExists(p string) bool {
|
||||
_, err := os.Stat(p)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// mustAbs returns an absolute path or the original on error.
|
||||
func mustAbs(p string) string {
|
||||
if p == "" {
|
||||
return p
|
||||
}
|
||||
abs, err := absPath(p)
|
||||
if err != nil {
|
||||
return p
|
||||
}
|
||||
return abs
|
||||
}
|
||||
|
||||
// absPath is filepath.Abs, isolated for tests.
|
||||
var absPath = func(p string) (string, error) {
|
||||
return execAbs(p)
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeBackend(t *testing.T) {
|
||||
got, err := NormalizeBackend("PODMAN")
|
||||
if err != nil || got != BackendPodman {
|
||||
t.Fatalf("got %q err %v", got, err)
|
||||
}
|
||||
if _, err := NormalizeBackend("bwrap"); err == nil {
|
||||
t.Fatal("expected error for removed bwrap backend")
|
||||
}
|
||||
if _, err := NormalizeBackend("firecracker"); err == nil {
|
||||
t.Fatal("expected error for unknown backend")
|
||||
}
|
||||
got, err = NormalizeBackend("")
|
||||
if err != nil || got != BackendPodman {
|
||||
t.Fatalf("empty -> podman, got %q %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoneWrap_Passthrough(t *testing.T) {
|
||||
w, err := New(Options{Backend: BackendNone})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bin := "/opt/blender/blender"
|
||||
cmd, err := w.Wrap(Spec{
|
||||
BlenderBinary: bin,
|
||||
Args: []string{"-b", "scene.blend"},
|
||||
WorkDir: "/tmp/job",
|
||||
Env: []string{"HOME=/tmp/job/home"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cmd.Args) == 0 || cmd.Args[0] != bin {
|
||||
t.Fatalf("args[0]=%v path=%q", cmd.Args, cmd.Path)
|
||||
}
|
||||
if cmd.Dir != "/tmp/job" {
|
||||
t.Fatalf("Dir=%q", cmd.Dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUMounts_ForceCPUEmpty(t *testing.T) {
|
||||
m := GPUMounts(true, true, true, true)
|
||||
if len(m) != 0 {
|
||||
t.Fatalf("force CPU should skip GPU mounts, got %#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUMounts_AMDIncludesKFD(t *testing.T) {
|
||||
m := GPUMounts(true, false, false, false)
|
||||
var hosts []string
|
||||
for _, b := range m {
|
||||
hosts = append(hosts, b.Host)
|
||||
}
|
||||
joined := strings.Join(hosts, ",")
|
||||
if !strings.Contains(joined, "/dev/dri") {
|
||||
t.Fatalf("AMD mounts should include /dev/dri, got %v", hosts)
|
||||
}
|
||||
if !strings.Contains(joined, "/dev/kfd") {
|
||||
t.Fatalf("AMD mounts should include /dev/kfd, got %v", hosts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUMounts_NVIDIAIncludesCtl(t *testing.T) {
|
||||
m := GPUMounts(false, true, false, false)
|
||||
found := false
|
||||
for _, b := range m {
|
||||
if strings.Contains(b.Host, "nvidia") {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("NVIDIA mounts should include nvidia devices, got %#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodmanWrap_BuildsRunArgs(t *testing.T) {
|
||||
w := &podmanWrapper{allowNetwork: false, image: "example.com/thin:latest"}
|
||||
if err := w.Available(); err != nil {
|
||||
t.Skipf("podman not installed: %v", err)
|
||||
}
|
||||
tmp := t.TempDir()
|
||||
blendDir := filepath.Join(tmp, "bver")
|
||||
_ = os.MkdirAll(blendDir, 0755)
|
||||
bin := filepath.Join(blendDir, "blender")
|
||||
_ = os.WriteFile(bin, []byte("x"), 0755)
|
||||
job := filepath.Join(tmp, "job")
|
||||
_ = os.MkdirAll(job, 0755)
|
||||
|
||||
cmd, err := w.Wrap(Spec{
|
||||
BlenderBinary: bin,
|
||||
Args: []string{"-b", "x.blend"},
|
||||
WorkDir: job,
|
||||
HomeDir: filepath.Join(job, "home"),
|
||||
Env: []string{"HOME=" + filepath.Join(job, "home")},
|
||||
HasNVIDIA: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
joined := strings.Join(cmd.Args, " ")
|
||||
if !strings.Contains(joined, "run") || !strings.Contains(joined, "--network=none") {
|
||||
t.Fatalf("expected podman run --network=none: %s", joined)
|
||||
}
|
||||
if !strings.Contains(joined, "example.com/thin:latest") {
|
||||
t.Fatalf("expected image in args: %s", joined)
|
||||
}
|
||||
if !strings.Contains(joined, "--entrypoint") || !strings.Contains(joined, bin) {
|
||||
t.Fatalf("expected entrypoint blender: %s", joined)
|
||||
}
|
||||
if !strings.Contains(joined, blendDir) || !strings.Contains(joined, job) {
|
||||
t.Fatalf("expected volume binds: %s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew_UnknownBackend(t *testing.T) {
|
||||
if _, err := New(Options{Backend: "bwrap"}); err == nil {
|
||||
t.Fatal("expected error for bwrap")
|
||||
}
|
||||
if _, err := New(Options{Backend: "gvisor"}); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"jiggablend/internal/runner/encoding"
|
||||
)
|
||||
@@ -26,6 +27,10 @@ func NewEncodeProcessor() *EncodeProcessor {
|
||||
|
||||
// Process executes an encode task.
|
||||
func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
if err := ctx.CheckCancelled(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx.Info(fmt.Sprintf("Starting encode task: job %d", ctx.JobID))
|
||||
log.Printf("Processing encode task %d for job %d", ctx.TaskID, ctx.JobID)
|
||||
|
||||
@@ -64,23 +69,18 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
ctx.Info(fmt.Sprintf("File: %s (type: %s, size: %d)", file.FileName, file.FileType, file.FileSize))
|
||||
}
|
||||
|
||||
// Determine source format based on output format
|
||||
sourceFormat := "exr"
|
||||
// Encode from EXR frames only
|
||||
fileExt := ".exr"
|
||||
|
||||
// Find and deduplicate frame files (EXR or PNG)
|
||||
frameFileSet := make(map[string]bool)
|
||||
var frameFilesList []string
|
||||
for _, file := range files {
|
||||
if file.FileType == "output" && strings.HasSuffix(strings.ToLower(file.FileName), fileExt) {
|
||||
// Deduplicate by filename
|
||||
if !frameFileSet[file.FileName] {
|
||||
frameFileSet[file.FileName] = true
|
||||
frameFilesList = append(frameFilesList, file.FileName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(frameFilesList) == 0 {
|
||||
// Log why no files matched (deduplicate for error reporting)
|
||||
outputFileSet := make(map[string]bool)
|
||||
@@ -103,37 +103,61 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx.Error(fmt.Sprintf("no %s frame files found for encode: found %d total files, %d unique output files, %d unique %s files (with other types)", strings.ToUpper(fileExt[1:]), len(files), len(outputFiles), len(frameFilesOtherType), strings.ToUpper(fileExt[1:])))
|
||||
ctx.Error(fmt.Sprintf("no EXR frame files found for encode: found %d total files, %d unique output files, %d unique EXR files (with other types)", len(files), len(outputFiles), len(frameFilesOtherType)))
|
||||
if len(outputFiles) > 0 {
|
||||
ctx.Error(fmt.Sprintf("Output files found: %v", outputFiles))
|
||||
}
|
||||
if len(frameFilesOtherType) > 0 {
|
||||
ctx.Error(fmt.Sprintf("%s files with wrong type: %v", strings.ToUpper(fileExt[1:]), frameFilesOtherType))
|
||||
ctx.Error(fmt.Sprintf("EXR files with wrong type: %v", frameFilesOtherType))
|
||||
}
|
||||
err := fmt.Errorf("no %s frame files found for encode", strings.ToUpper(fileExt[1:]))
|
||||
err := fmt.Errorf("no EXR frame files found for encode")
|
||||
return err
|
||||
}
|
||||
|
||||
ctx.Info(fmt.Sprintf("Found %d %s frames for encode", len(frameFilesList), strings.ToUpper(fileExt[1:])))
|
||||
ctx.Info(fmt.Sprintf("Found %d EXR frames for encode", len(frameFilesList)))
|
||||
|
||||
// Download frames
|
||||
ctx.Info(fmt.Sprintf("Downloading %d %s frames for encode...", len(frameFilesList), strings.ToUpper(fileExt[1:])))
|
||||
// Download frames with bounded parallelism (8 concurrent downloads)
|
||||
const downloadWorkers = 8
|
||||
ctx.Info(fmt.Sprintf("Downloading %d EXR frames for encode...", len(frameFilesList)))
|
||||
|
||||
type result struct {
|
||||
path string
|
||||
err error
|
||||
}
|
||||
results := make([]result, len(frameFilesList))
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, downloadWorkers)
|
||||
for i, fileName := range frameFilesList {
|
||||
wg.Add(1)
|
||||
go func(i int, fileName string) {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
framePath := filepath.Join(workDir, fileName)
|
||||
err := ctx.Manager.DownloadFrame(ctx.JobID, fileName, framePath)
|
||||
if err != nil {
|
||||
ctx.Error(fmt.Sprintf("Failed to download EXR frame %s: %v", fileName, err))
|
||||
log.Printf("Failed to download EXR frame for encode %s: %v", fileName, err)
|
||||
results[i] = result{"", err}
|
||||
return
|
||||
}
|
||||
results[i] = result{framePath, nil}
|
||||
}(i, fileName)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var frameFiles []string
|
||||
for i, fileName := range frameFilesList {
|
||||
ctx.Info(fmt.Sprintf("Downloading frame %d/%d: %s", i+1, len(frameFilesList), fileName))
|
||||
framePath := filepath.Join(workDir, fileName)
|
||||
if err := ctx.Manager.DownloadFrame(ctx.JobID, fileName, framePath); err != nil {
|
||||
ctx.Error(fmt.Sprintf("Failed to download %s frame %s: %v", strings.ToUpper(fileExt[1:]), fileName, err))
|
||||
log.Printf("Failed to download %s frame for encode %s: %v", strings.ToUpper(fileExt[1:]), fileName, err)
|
||||
continue
|
||||
for _, r := range results {
|
||||
if r.err == nil && r.path != "" {
|
||||
frameFiles = append(frameFiles, r.path)
|
||||
}
|
||||
ctx.Info(fmt.Sprintf("Successfully downloaded frame %d/%d: %s", i+1, len(frameFilesList), fileName))
|
||||
frameFiles = append(frameFiles, framePath)
|
||||
}
|
||||
if err := ctx.CheckCancelled(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(frameFiles) == 0 {
|
||||
err := fmt.Errorf("failed to download any %s frames for encode", strings.ToUpper(fileExt[1:]))
|
||||
err := fmt.Errorf("failed to download any EXR frames for encode")
|
||||
ctx.Error(err.Error())
|
||||
return err
|
||||
}
|
||||
@@ -141,11 +165,9 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
sort.Strings(frameFiles)
|
||||
ctx.Info(fmt.Sprintf("Downloaded %d frames", len(frameFiles)))
|
||||
|
||||
// Check if EXR files have alpha channel and HDR content (only for EXR source format)
|
||||
// Check if EXR files have alpha channel (for encode decision)
|
||||
hasAlpha := false
|
||||
hasHDR := false
|
||||
if sourceFormat == "exr" {
|
||||
// Check first frame for alpha channel and HDR using ffprobe
|
||||
{
|
||||
firstFrame := frameFiles[0]
|
||||
hasAlpha = detectAlphaChannel(ctx, firstFrame)
|
||||
if hasAlpha {
|
||||
@@ -153,45 +175,28 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
} else {
|
||||
ctx.Info("No alpha channel detected in EXR files")
|
||||
}
|
||||
|
||||
hasHDR = detectHDR(ctx, firstFrame)
|
||||
if hasHDR {
|
||||
ctx.Info("Detected HDR content in EXR files")
|
||||
} else {
|
||||
ctx.Info("No HDR content detected in EXR files (SDR range)")
|
||||
}
|
||||
}
|
||||
|
||||
// Generate video
|
||||
// Use alpha if:
|
||||
// 1. User explicitly enabled it OR source has alpha channel AND
|
||||
// 2. Codec supports alpha (AV1 or VP9)
|
||||
preserveAlpha := ctx.ShouldPreserveAlpha()
|
||||
useAlpha := (preserveAlpha || hasAlpha) && (outputFormat == "EXR_AV1_MP4" || outputFormat == "EXR_VP9_WEBM")
|
||||
if (preserveAlpha || hasAlpha) && outputFormat == "EXR_264_MP4" {
|
||||
ctx.Warn("Alpha channel requested/detected but H.264 does not support alpha. Consider using EXR_AV1_MP4 or EXR_VP9_WEBM to preserve alpha.")
|
||||
}
|
||||
if preserveAlpha && !hasAlpha {
|
||||
ctx.Warn("Alpha preservation requested but no alpha channel detected in EXR files.")
|
||||
// Use alpha when source EXR has alpha and codec supports it (AV1 or VP9). H.264 does not support alpha.
|
||||
useAlpha := hasAlpha && (outputFormat == "EXR_AV1_MP4" || outputFormat == "EXR_VP9_WEBM")
|
||||
if hasAlpha && outputFormat == "EXR_264_MP4" {
|
||||
ctx.Warn("Alpha channel detected in EXR but H.264 does not support alpha. Use EXR_AV1_MP4 or EXR_VP9_WEBM to preserve alpha in video.")
|
||||
}
|
||||
if useAlpha {
|
||||
if preserveAlpha && hasAlpha {
|
||||
ctx.Info("Alpha preservation enabled: Using alpha channel encoding")
|
||||
} else if hasAlpha {
|
||||
ctx.Info("Alpha channel detected - automatically enabling alpha encoding")
|
||||
}
|
||||
ctx.Info("Alpha channel detected - encoding with alpha (AV1/VP9)")
|
||||
}
|
||||
var outputExt string
|
||||
switch outputFormat {
|
||||
case "EXR_VP9_WEBM":
|
||||
outputExt = "webm"
|
||||
ctx.Info("Encoding WebM video with VP9 codec (with alpha channel and HDR support)...")
|
||||
ctx.Info("Encoding WebM video with VP9 codec (alpha, HDR)...")
|
||||
case "EXR_AV1_MP4":
|
||||
outputExt = "mp4"
|
||||
ctx.Info("Encoding MP4 video with AV1 codec (with alpha channel)...")
|
||||
ctx.Info("Encoding MP4 video with AV1 codec (alpha, HDR)...")
|
||||
default:
|
||||
outputExt = "mp4"
|
||||
ctx.Info("Encoding MP4 video with H.264 codec...")
|
||||
ctx.Info("Encoding MP4 video with H.264 codec (HDR, HLG)...")
|
||||
}
|
||||
|
||||
outputVideo := filepath.Join(workDir, fmt.Sprintf("output_%d.%s", ctx.JobID, outputExt))
|
||||
@@ -231,11 +236,6 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
// Pass 1
|
||||
ctx.Info("Pass 1/2: Analyzing content for optimal encode...")
|
||||
softEncoder := encoder.(*encoding.SoftwareEncoder)
|
||||
// Use HDR if: user explicitly enabled it OR HDR content was detected
|
||||
preserveHDR := (ctx.ShouldPreserveHDR() || hasHDR) && sourceFormat == "exr"
|
||||
if hasHDR && !ctx.ShouldPreserveHDR() {
|
||||
ctx.Info("HDR content detected - automatically enabling HDR preservation")
|
||||
}
|
||||
pass1Cmd := softEncoder.BuildPass1Command(&encoding.EncodeConfig{
|
||||
InputPattern: patternPath,
|
||||
OutputPath: outputVideo,
|
||||
@@ -244,25 +244,16 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
WorkDir: workDir,
|
||||
UseAlpha: useAlpha,
|
||||
TwoPass: true,
|
||||
SourceFormat: sourceFormat,
|
||||
PreserveHDR: preserveHDR,
|
||||
})
|
||||
if err := pass1Cmd.Run(); err != nil {
|
||||
// Pass 1 is analysis-only (writes to /dev/null). FFmpeg often exits non-zero
|
||||
// on benign codec/option warnings while still producing passlogfile stats.
|
||||
ctx.Warn(fmt.Sprintf("Pass 1 completed (warnings expected): %v", err))
|
||||
}
|
||||
|
||||
// Pass 2
|
||||
ctx.Info("Pass 2/2: Encoding with optimal quality...")
|
||||
|
||||
preserveHDR = (ctx.ShouldPreserveHDR() || hasHDR) && sourceFormat == "exr"
|
||||
if preserveHDR {
|
||||
if hasHDR && !ctx.ShouldPreserveHDR() {
|
||||
ctx.Info("HDR preservation enabled (auto-detected): Using HLG transfer with bt709 primaries")
|
||||
} else {
|
||||
ctx.Info("HDR preservation enabled: Using HLG transfer with bt709 primaries")
|
||||
}
|
||||
}
|
||||
|
||||
config := &encoding.EncodeConfig{
|
||||
InputPattern: patternPath,
|
||||
OutputPath: outputVideo,
|
||||
@@ -271,8 +262,6 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
WorkDir: workDir,
|
||||
UseAlpha: useAlpha,
|
||||
TwoPass: true, // Software encoding always uses 2-pass for quality
|
||||
SourceFormat: sourceFormat,
|
||||
PreserveHDR: preserveHDR,
|
||||
}
|
||||
|
||||
cmd := encoder.BuildCommand(config)
|
||||
@@ -294,6 +283,8 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
if err := cmd.Start(); err != nil {
|
||||
return fmt.Errorf("failed to start encode command: %w", err)
|
||||
}
|
||||
stopMonitor := ctx.StartCancellationMonitor(cmd, "encode")
|
||||
defer stopMonitor()
|
||||
|
||||
ctx.Processes.Track(ctx.TaskID, cmd)
|
||||
defer ctx.Processes.Untrack(ctx.TaskID)
|
||||
@@ -309,6 +300,9 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
ctx.Info(line)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Printf("Error reading encode stdout: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Stream stderr
|
||||
@@ -322,6 +316,9 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
ctx.Warn(line)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Printf("Error reading encode stderr: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
err = cmd.Wait()
|
||||
@@ -329,6 +326,9 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
<-stderrDone
|
||||
|
||||
if err != nil {
|
||||
if cancelled, checkErr := ctx.IsJobCancelled(); checkErr == nil && cancelled {
|
||||
return ErrJobCancelled
|
||||
}
|
||||
var errMsg string
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
if exitErr.ExitCode() == 137 {
|
||||
@@ -387,7 +387,7 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
func detectAlphaChannel(ctx *Context, filePath string) bool {
|
||||
// Use ffprobe to check pixel format and stream properties
|
||||
// EXR files with alpha will have formats like gbrapf32le (RGBA) vs gbrpf32le (RGB)
|
||||
cmd := exec.Command("ffprobe",
|
||||
cmd := execCommand("ffprobe",
|
||||
"-v", "error",
|
||||
"-select_streams", "v:0",
|
||||
"-show_entries", "stream=pix_fmt:stream=codec_name",
|
||||
@@ -420,7 +420,7 @@ func detectAlphaChannel(ctx *Context, filePath string) bool {
|
||||
// detectHDR checks if an EXR file contains HDR content using ffprobe
|
||||
func detectHDR(ctx *Context, filePath string) bool {
|
||||
// First, check if the pixel format supports HDR (32-bit float)
|
||||
cmd := exec.Command("ffprobe",
|
||||
cmd := execCommand("ffprobe",
|
||||
"-v", "error",
|
||||
"-select_streams", "v:0",
|
||||
"-show_entries", "stream=pix_fmt",
|
||||
@@ -448,7 +448,7 @@ func detectHDR(ctx *Context, filePath string) bool {
|
||||
// For 32-bit float EXR, sample pixels to check if values exceed SDR range (> 1.0)
|
||||
// Use ffmpeg to extract pixel statistics - check max pixel values
|
||||
// This is more efficient than sampling individual pixels
|
||||
cmd = exec.Command("ffmpeg",
|
||||
cmd = execCommand("ffmpeg",
|
||||
"-v", "error",
|
||||
"-i", filePath,
|
||||
"-vf", "signalstats",
|
||||
@@ -491,7 +491,7 @@ func detectHDRBySampling(ctx *Context, filePath string) bool {
|
||||
}
|
||||
|
||||
for _, region := range sampleRegions {
|
||||
cmd := exec.Command("ffmpeg",
|
||||
cmd := execCommand("ffmpeg",
|
||||
"-v", "error",
|
||||
"-i", filePath,
|
||||
"-vf", fmt.Sprintf("%s,scale=1:1", region),
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFloat32FromBytes(t *testing.T) {
|
||||
got := float32FromBytes([]byte{0x00, 0x00, 0x80, 0x3f}) // 1.0 little-endian
|
||||
if got != 1.0 {
|
||||
t.Fatalf("float32FromBytes() = %v, want 1.0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMax(t *testing.T) {
|
||||
if got := max(1, 2); got != 2 {
|
||||
t.Fatalf("max() = %v, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFrameNumber(t *testing.T) {
|
||||
if got := extractFrameNumber("render_0042.png"); got != 42 {
|
||||
t.Fatalf("extractFrameNumber() = %d, want 42", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckFFmpegSizeError(t *testing.T) {
|
||||
err := checkFFmpegSizeError("hardware does not support encoding at size ... constraints: width 128-4096 height 128-4096")
|
||||
if err == nil {
|
||||
t.Fatal("expected a size error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectAlphaChannel_UsesExecSeam(t *testing.T) {
|
||||
orig := execCommand
|
||||
execCommand = fakeExecCommand
|
||||
defer func() { execCommand = orig }()
|
||||
|
||||
if !detectAlphaChannel(&Context{}, "/tmp/frame.exr") {
|
||||
t.Fatal("expected alpha channel detection via mocked ffprobe output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDetectHDR_UsesExecSeam(t *testing.T) {
|
||||
orig := execCommand
|
||||
execCommand = fakeExecCommand
|
||||
defer func() { execCommand = orig }()
|
||||
|
||||
if !detectHDR(&Context{}, "/tmp/frame.exr") {
|
||||
t.Fatal("expected HDR detection via mocked ffmpeg sampling output")
|
||||
}
|
||||
}
|
||||
|
||||
func fakeExecCommand(command string, args ...string) *exec.Cmd {
|
||||
cs := []string{"-test.run=TestExecHelperProcess", "--", command}
|
||||
cs = append(cs, args...)
|
||||
cmd := exec.Command(os.Args[0], cs...)
|
||||
cmd.Env = append(os.Environ(), "GO_WANT_HELPER_PROCESS=1")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func TestExecHelperProcess(t *testing.T) {
|
||||
if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" {
|
||||
return
|
||||
}
|
||||
|
||||
idx := 0
|
||||
for i, a := range os.Args {
|
||||
if a == "--" {
|
||||
idx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if idx == 0 || idx+1 >= len(os.Args) {
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
cmdName := os.Args[idx+1]
|
||||
cmdArgs := os.Args[idx+2:]
|
||||
|
||||
switch cmdName {
|
||||
case "ffprobe":
|
||||
if containsArg(cmdArgs, "stream=pix_fmt:stream=codec_name") {
|
||||
_, _ = os.Stdout.WriteString("pix_fmt=gbrapf32le\ncodec_name=exr\n")
|
||||
os.Exit(0)
|
||||
}
|
||||
_, _ = os.Stdout.WriteString("gbrpf32le\n")
|
||||
os.Exit(0)
|
||||
case "ffmpeg":
|
||||
if containsArg(cmdArgs, "signalstats") {
|
||||
_, _ = os.Stderr.WriteString("signalstats failed")
|
||||
os.Exit(1)
|
||||
}
|
||||
if containsArg(cmdArgs, "rawvideo") {
|
||||
buf := make([]byte, 12)
|
||||
binary.LittleEndian.PutUint32(buf[0:4], math.Float32bits(1.5))
|
||||
binary.LittleEndian.PutUint32(buf[4:8], math.Float32bits(0.2))
|
||||
binary.LittleEndian.PutUint32(buf[8:12], math.Float32bits(0.1))
|
||||
_, _ = os.Stdout.Write(buf)
|
||||
os.Exit(0)
|
||||
}
|
||||
os.Exit(0)
|
||||
default:
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
func containsArg(args []string, target string) bool {
|
||||
for _, a := range args {
|
||||
if strings.Contains(a, target) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
package tasks
|
||||
|
||||
import "os/exec"
|
||||
|
||||
// execCommand is a seam for process execution in tests.
|
||||
var execCommand = exec.Command
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// mergeEXRFiles averages multiple linear EXR renders into one (Monte Carlo accumulation).
|
||||
func mergeEXRFiles(output string, inputs []string) error {
|
||||
if len(inputs) == 0 {
|
||||
return fmt.Errorf("no input EXR files to merge")
|
||||
}
|
||||
if len(inputs) == 1 {
|
||||
return copyFile(inputs[0], output)
|
||||
}
|
||||
|
||||
args := append([]string{}, inputs...)
|
||||
args = append(args, "-evaluate-sequence", "mean", output)
|
||||
|
||||
if path, err := exec.LookPath("magick"); err == nil {
|
||||
cmd := exec.Command(path, args...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("magick merge failed: %w (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if path, err := exec.LookPath("convert"); err == nil {
|
||||
cmd := exec.Command(path, args...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("convert merge failed: %w (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("ImageMagick (magick or convert) required to merge batched EXR renders")
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
cmd := exec.Command("cp", "-f", src, dst)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("copy %s to %s: %w (%s)", src, dst, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -2,12 +2,18 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"jiggablend/internal/runner/api"
|
||||
"jiggablend/internal/runner/blender"
|
||||
"jiggablend/internal/runner/encoding"
|
||||
"jiggablend/internal/runner/sandbox"
|
||||
"jiggablend/internal/runner/workspace"
|
||||
"jiggablend/pkg/executils"
|
||||
"jiggablend/pkg/types"
|
||||
"os/exec"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Processor handles a specific task type.
|
||||
@@ -20,7 +26,8 @@ type Context struct {
|
||||
TaskID int64
|
||||
JobID int64
|
||||
JobName string
|
||||
Frame int
|
||||
Frame int // frame start (inclusive); kept for backward compat
|
||||
FrameEnd int // frame end (inclusive); same as Frame for single-frame
|
||||
TaskType string
|
||||
WorkDir string
|
||||
JobToken string
|
||||
@@ -32,13 +39,41 @@ type Context struct {
|
||||
Blender *blender.Manager
|
||||
Encoder *encoding.Selector
|
||||
Processes *executils.ProcessTracker
|
||||
|
||||
// GPULockedOut is set when the runner has detected a GPU error (e.g. HIP) and disables GPU for all jobs.
|
||||
GPULockedOut bool
|
||||
// GPULockoutArmedThisAttempt is true when this task attempt newly enabled runner GPU lockout.
|
||||
// Failures with this set free-requeue without incrementing retry_count (expected ROCm warmup).
|
||||
GPULockoutArmedThisAttempt bool
|
||||
// HasAMD is true when the runner detected AMD devices at startup.
|
||||
HasAMD bool
|
||||
// HasNVIDIA is true when the runner detected NVIDIA GPUs at startup.
|
||||
HasNVIDIA bool
|
||||
// HasIntel is true when the runner detected Intel GPUs (e.g. Arc) at startup.
|
||||
HasIntel bool
|
||||
// GPUDetectionFailed is true when startup GPU backend detection could not run; we force CPU for all versions (backend availability unknown).
|
||||
GPUDetectionFailed bool
|
||||
// OnGPUError is called when a GPU error line is seen in render logs; typically sets runner GPU lockout.
|
||||
OnGPUError func()
|
||||
// ForceCPURendering is a runner-level override that forces CPU rendering for all jobs.
|
||||
ForceCPURendering bool
|
||||
// DisableRT disables GPU ray tracing acceleration (runner-level flag).
|
||||
DisableRT bool
|
||||
// HipGPUSampleBatch limits samples per GPU render pass on gfx115x (0 = no batching).
|
||||
HipGPUSampleBatch int
|
||||
// Sandbox wraps Blender execution (none/podman). Nil means none.
|
||||
Sandbox sandbox.Wrapper
|
||||
}
|
||||
|
||||
// NewContext creates a new task context.
|
||||
// ErrJobCancelled indicates the manager-side job was cancelled during execution.
|
||||
var ErrJobCancelled = errors.New("job cancelled")
|
||||
|
||||
// NewContext creates a new task context. frameEnd should be >= frame; if 0 or less than frame, it is treated as single-frame (frameEnd = frame).
|
||||
// gpuLockedOut is the runner's current GPU lockout state; gpuDetectionFailed means detection failed at startup (force CPU for all versions); onGPUError is called when a GPU error is detected in logs (may be nil).
|
||||
func NewContext(
|
||||
taskID, jobID int64,
|
||||
jobName string,
|
||||
frame int,
|
||||
frameStart, frameEnd int,
|
||||
taskType string,
|
||||
workDir string,
|
||||
jobToken string,
|
||||
@@ -49,22 +84,46 @@ func NewContext(
|
||||
blenderMgr *blender.Manager,
|
||||
encoder *encoding.Selector,
|
||||
processes *executils.ProcessTracker,
|
||||
gpuLockedOut bool,
|
||||
hasAMD bool,
|
||||
hasNVIDIA bool,
|
||||
hasIntel bool,
|
||||
gpuDetectionFailed bool,
|
||||
forceCPURendering bool,
|
||||
disableRT bool,
|
||||
hipGPUSampleBatch int,
|
||||
onGPUError func(),
|
||||
sb sandbox.Wrapper,
|
||||
) *Context {
|
||||
if frameEnd < frameStart {
|
||||
frameEnd = frameStart
|
||||
}
|
||||
return &Context{
|
||||
TaskID: taskID,
|
||||
JobID: jobID,
|
||||
JobName: jobName,
|
||||
Frame: frame,
|
||||
TaskType: taskType,
|
||||
WorkDir: workDir,
|
||||
JobToken: jobToken,
|
||||
Metadata: metadata,
|
||||
Manager: manager,
|
||||
JobConn: jobConn,
|
||||
Workspace: ws,
|
||||
Blender: blenderMgr,
|
||||
Encoder: encoder,
|
||||
Processes: processes,
|
||||
TaskID: taskID,
|
||||
JobID: jobID,
|
||||
JobName: jobName,
|
||||
Frame: frameStart,
|
||||
FrameEnd: frameEnd,
|
||||
TaskType: taskType,
|
||||
WorkDir: workDir,
|
||||
JobToken: jobToken,
|
||||
Metadata: metadata,
|
||||
Manager: manager,
|
||||
JobConn: jobConn,
|
||||
Workspace: ws,
|
||||
Blender: blenderMgr,
|
||||
Encoder: encoder,
|
||||
Processes: processes,
|
||||
GPULockedOut: gpuLockedOut,
|
||||
HasAMD: hasAMD,
|
||||
HasNVIDIA: hasNVIDIA,
|
||||
HasIntel: hasIntel,
|
||||
GPUDetectionFailed: gpuDetectionFailed,
|
||||
ForceCPURendering: forceCPURendering,
|
||||
DisableRT: disableRT,
|
||||
HipGPUSampleBatch: hipGPUSampleBatch,
|
||||
OnGPUError: onGPUError,
|
||||
Sandbox: sb,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,18 +164,22 @@ func (c *Context) OutputUploaded(fileName string) {
|
||||
}
|
||||
|
||||
// Complete sends task completion.
|
||||
// When the failure newly armed GPU lockout, free_requeue is set so the manager
|
||||
// requeues without burning retry_count.
|
||||
func (c *Context) Complete(success bool, errorMsg error) {
|
||||
if c.JobConn != nil {
|
||||
c.JobConn.Complete(c.TaskID, success, errorMsg)
|
||||
freeRequeue := !success && c.GPULockoutArmedThisAttempt
|
||||
c.JobConn.Complete(c.TaskID, success, errorMsg, freeRequeue)
|
||||
}
|
||||
}
|
||||
|
||||
// GetOutputFormat returns the output format from metadata or default.
|
||||
// Product default is EXR (Blender always renders EXR; deliverable may add video).
|
||||
func (c *Context) GetOutputFormat() string {
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.OutputFormat != "" {
|
||||
return c.Metadata.RenderSettings.OutputFormat
|
||||
}
|
||||
return "PNG"
|
||||
return "EXR"
|
||||
}
|
||||
|
||||
// GetFrameRate returns the frame rate from metadata or default.
|
||||
@@ -140,17 +203,158 @@ func (c *Context) ShouldUnhideObjects() bool {
|
||||
return c.Metadata != nil && c.Metadata.UnhideObjects != nil && *c.Metadata.UnhideObjects
|
||||
}
|
||||
|
||||
// ShouldEnableExecution returns whether to enable auto-execution.
|
||||
// ShouldEnableExecution returns whether to pass Blender --enable-autoexec
|
||||
// (Python drivers/scripts inside the .blend). Job-bundled blender_addons/ install
|
||||
// independently whenever that folder is present in the context.
|
||||
func (c *Context) ShouldEnableExecution() bool {
|
||||
return c.Metadata != nil && c.Metadata.EnableExecution != nil && *c.Metadata.EnableExecution
|
||||
}
|
||||
|
||||
// ShouldPreserveHDR returns whether to preserve HDR range for EXR encoding.
|
||||
func (c *Context) ShouldPreserveHDR() bool {
|
||||
return c.Metadata != nil && c.Metadata.PreserveHDR != nil && *c.Metadata.PreserveHDR
|
||||
// ShouldForceCPU returns true if GPU should be disabled and CPU rendering forced
|
||||
// (runner GPU lockout, GPU detection failed at startup, or metadata force_cpu).
|
||||
func (c *Context) ShouldForceCPU() bool {
|
||||
if c.ForceCPURendering {
|
||||
return true
|
||||
}
|
||||
if c.GPULockedOut {
|
||||
return true
|
||||
}
|
||||
// Detection failed at startup: backend availability unknown, so force CPU for all versions.
|
||||
if c.GPUDetectionFailed {
|
||||
return true
|
||||
}
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||
if v, ok := c.Metadata.RenderSettings.EngineSettings["force_cpu"]; ok {
|
||||
if b, ok := v.(bool); ok && b {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ShouldPreserveAlpha returns whether to preserve alpha channel for EXR encoding.
|
||||
func (c *Context) ShouldPreserveAlpha() bool {
|
||||
return c.Metadata != nil && c.Metadata.PreserveAlpha != nil && *c.Metadata.PreserveAlpha
|
||||
// GetCyclesSamples returns the target Cycles sample count from metadata (default 128).
|
||||
func (c *Context) GetCyclesSamples() int {
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||
if v, ok := c.Metadata.RenderSettings.EngineSettings["samples"]; ok {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
if int(n) > 0 {
|
||||
return int(n)
|
||||
}
|
||||
case int:
|
||||
if n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 128
|
||||
}
|
||||
|
||||
// GetCyclesSeed returns the Cycles seed from metadata (default 0).
|
||||
func (c *Context) GetCyclesSeed() int {
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||
if v, ok := c.Metadata.RenderSettings.EngineSettings["seed"]; ok {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return int(n)
|
||||
case int:
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ShouldBatchGPUSamples returns true when HIP GPU renders should be split into sample batches.
|
||||
func (c *Context) ShouldBatchGPUSamples() bool {
|
||||
if c.ShouldForceCPU() || c.HipGPUSampleBatch <= 0 {
|
||||
return false
|
||||
}
|
||||
return c.GetCyclesSamples() > c.HipGPUSampleBatch
|
||||
}
|
||||
|
||||
// ShouldDisableRT returns true when GPU ray tracing acceleration should be disabled.
|
||||
func (c *Context) ShouldDisableRT() bool {
|
||||
if c.DisableRT {
|
||||
return true
|
||||
}
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||
settings := c.Metadata.RenderSettings.EngineSettings
|
||||
if v, ok := settings["disable_rt"]; ok {
|
||||
if b, ok := v.(bool); ok && b {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if v, ok := settings["disable_hiprt"]; ok {
|
||||
if b, ok := v.(bool); ok && b {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsJobCancelled checks whether the manager marked this job as cancelled.
|
||||
func (c *Context) IsJobCancelled() (bool, error) {
|
||||
if c.Manager == nil {
|
||||
return false, nil
|
||||
}
|
||||
status, err := c.Manager.GetJobStatus(c.JobID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return status == types.JobStatusCancelled, nil
|
||||
}
|
||||
|
||||
// CheckCancelled returns ErrJobCancelled if the job was cancelled.
|
||||
func (c *Context) CheckCancelled() error {
|
||||
cancelled, err := c.IsJobCancelled()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to check job status: %w", err)
|
||||
}
|
||||
if cancelled {
|
||||
return ErrJobCancelled
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StartCancellationMonitor polls manager status and kills cmd if job is cancelled.
|
||||
// Caller must invoke returned stop function when cmd exits.
|
||||
func (c *Context) StartCancellationMonitor(cmd *exec.Cmd, taskLabel string) func() {
|
||||
stop := make(chan struct{})
|
||||
var once sync.Once
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(2 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-ticker.C:
|
||||
cancelled, err := c.IsJobCancelled()
|
||||
if err != nil {
|
||||
c.Warn(fmt.Sprintf("Could not check cancellation for %s task: %v", taskLabel, err))
|
||||
continue
|
||||
}
|
||||
if !cancelled {
|
||||
continue
|
||||
}
|
||||
c.Warn(fmt.Sprintf("Job %d was cancelled, stopping %s task early", c.JobID, taskLabel))
|
||||
if cmd != nil && cmd.Process != nil {
|
||||
_ = cmd.Process.Kill()
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return func() {
|
||||
once.Do(func() {
|
||||
close(stop)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"jiggablend/pkg/types"
|
||||
)
|
||||
|
||||
func TestNewContext_NormalizesFrameEnd(t *testing.T) {
|
||||
ctx := NewContext(1, 2, "job", 10, 1, "render", "/tmp", "tok", nil, nil, nil, nil, nil, nil, nil, false, false, false, false, false, false, false, 0, nil, nil)
|
||||
if ctx.FrameEnd != 10 {
|
||||
t.Fatalf("expected FrameEnd to be normalized to Frame, got %d", ctx.FrameEnd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContext_GetOutputFormat_Default(t *testing.T) {
|
||||
ctx := &Context{}
|
||||
if got := ctx.GetOutputFormat(); got != "EXR" {
|
||||
t.Fatalf("GetOutputFormat() = %q, want EXR", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContext_ShouldForceCPU(t *testing.T) {
|
||||
ctx := &Context{ForceCPURendering: true}
|
||||
if !ctx.ShouldForceCPU() {
|
||||
t.Fatal("expected force cpu when runner-level flag is set")
|
||||
}
|
||||
|
||||
force := true
|
||||
ctx = &Context{Metadata: &types.BlendMetadata{RenderSettings: types.RenderSettings{EngineSettings: map[string]interface{}{"force_cpu": force}}}}
|
||||
if !ctx.ShouldForceCPU() {
|
||||
t.Fatal("expected force cpu when metadata requests it")
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrJobCancelled_IsSentinel(t *testing.T) {
|
||||
if !errors.Is(ErrJobCancelled, ErrJobCancelled) {
|
||||
t.Fatal("sentinel error should be self-identical")
|
||||
}
|
||||
}
|
||||
|
||||
+322
-99
@@ -12,6 +12,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"jiggablend/internal/runner/blender"
|
||||
"jiggablend/internal/runner/sandbox"
|
||||
"jiggablend/internal/runner/workspace"
|
||||
"jiggablend/pkg/scripts"
|
||||
"jiggablend/pkg/types"
|
||||
@@ -25,11 +26,53 @@ func NewRenderProcessor() *RenderProcessor {
|
||||
return &RenderProcessor{}
|
||||
}
|
||||
|
||||
// gpuErrorSubstrings are log line substrings that indicate a GPU backend error (matched case-insensitively); any match triggers full GPU lockout.
|
||||
var gpuErrorSubstrings = []string{
|
||||
"illegal address in hip", // HIP (AMD) e.g. "Illegal address in HIP" or "Illegal address in hip"
|
||||
"memory access fault", // AMDGPU page fault during HIP/HIPRT (not necessarily OOM)
|
||||
"page not present", // AMDGPU GCVM page fault detail
|
||||
"hiperror", // hipError* codes
|
||||
"hip error",
|
||||
"cuda error",
|
||||
"cuerror",
|
||||
"optix error",
|
||||
"oneapi error",
|
||||
"opencl error",
|
||||
}
|
||||
|
||||
// checkGPUErrorLine checks a log line for GPU error indicators and triggers runner GPU lockout if found.
|
||||
// Once lockout is already active for this attempt (or was active at context creation), further
|
||||
// matching lines are ignored so we do not spam lockout callbacks/logs on multi-line fault dumps.
|
||||
func (p *RenderProcessor) checkGPUErrorLine(ctx *Context, line string) {
|
||||
if ctx.GPULockedOut || ctx.GPULockoutArmedThisAttempt {
|
||||
return
|
||||
}
|
||||
lower := strings.ToLower(line)
|
||||
for _, sub := range gpuErrorSubstrings {
|
||||
if strings.Contains(lower, sub) {
|
||||
if ctx.OnGPUError != nil {
|
||||
ctx.OnGPUError()
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process executes a render task.
|
||||
func (p *RenderProcessor) Process(ctx *Context) error {
|
||||
ctx.Info(fmt.Sprintf("Starting task: job %d, frame %d, format: %s",
|
||||
ctx.JobID, ctx.Frame, ctx.GetOutputFormat()))
|
||||
log.Printf("Processing task %d: job %d, frame %d", ctx.TaskID, ctx.JobID, ctx.Frame)
|
||||
if err := ctx.CheckCancelled(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ctx.FrameEnd > ctx.Frame {
|
||||
ctx.Info(fmt.Sprintf("Starting task: job %d, frames %d-%d, format: %s",
|
||||
ctx.JobID, ctx.Frame, ctx.FrameEnd, ctx.GetOutputFormat()))
|
||||
log.Printf("Processing task %d: job %d, frames %d-%d", ctx.TaskID, ctx.JobID, ctx.Frame, ctx.FrameEnd)
|
||||
} else {
|
||||
ctx.Info(fmt.Sprintf("Starting task: job %d, frame %d, format: %s",
|
||||
ctx.JobID, ctx.Frame, ctx.GetOutputFormat()))
|
||||
log.Printf("Processing task %d: job %d, frame %d", ctx.TaskID, ctx.JobID, ctx.Frame)
|
||||
}
|
||||
|
||||
// Find .blend file
|
||||
blendFile, err := workspace.FindFirstBlendFile(ctx.WorkDir)
|
||||
@@ -37,21 +80,24 @@ func (p *RenderProcessor) Process(ctx *Context) error {
|
||||
return fmt.Errorf("failed to find blend file: %w", err)
|
||||
}
|
||||
|
||||
// Get Blender binary
|
||||
blenderBinary := "blender"
|
||||
if version := ctx.GetBlenderVersion(); version != "" {
|
||||
ctx.Info(fmt.Sprintf("Job requires Blender %s", version))
|
||||
binaryPath, err := ctx.Blender.GetBinaryPath(version)
|
||||
if err != nil {
|
||||
ctx.Warn(fmt.Sprintf("Could not get Blender %s, using system blender: %v", version, err))
|
||||
} else {
|
||||
blenderBinary = binaryPath
|
||||
ctx.Info(fmt.Sprintf("Using Blender binary: %s", blenderBinary))
|
||||
}
|
||||
} else {
|
||||
ctx.Info("No Blender version specified, using system blender")
|
||||
// Runners must use manager-provided Blender versions; never fall back to system blender.
|
||||
version := ctx.GetBlenderVersion()
|
||||
if version == "" {
|
||||
return fmt.Errorf("job metadata missing blender_version: runner cannot use system blender")
|
||||
}
|
||||
|
||||
ctx.Info(fmt.Sprintf("Job requires Blender %s", version))
|
||||
binaryPath, err := ctx.Blender.GetBinaryPath(version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get Blender %s from manager: %w", version, err)
|
||||
}
|
||||
|
||||
blenderBinary, err := blender.ResolveBinaryPath(binaryPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve Blender %s binary: %w", version, err)
|
||||
}
|
||||
ctx.Info(fmt.Sprintf("Using Blender binary: %s", blenderBinary))
|
||||
|
||||
// Create output directory
|
||||
outputDir := filepath.Join(ctx.WorkDir, "output")
|
||||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||
@@ -64,36 +110,135 @@ func (p *RenderProcessor) Process(ctx *Context) error {
|
||||
return fmt.Errorf("failed to create Blender home directory: %w", err)
|
||||
}
|
||||
|
||||
// Determine render format
|
||||
outputFormat := ctx.GetOutputFormat()
|
||||
renderFormat := outputFormat
|
||||
if outputFormat == "EXR_264_MP4" || outputFormat == "EXR_AV1_MP4" || outputFormat == "EXR_VP9_WEBM" {
|
||||
renderFormat = "EXR" // Use EXR for maximum quality
|
||||
}
|
||||
// We always render EXR (linear) for VFX accuracy; job output_format is the deliverable (EXR sequence or video).
|
||||
renderFormat := "EXR"
|
||||
|
||||
// Create render script
|
||||
if err := p.createRenderScript(ctx, renderFormat); err != nil {
|
||||
return err
|
||||
if ctx.ShouldForceCPU() {
|
||||
if ctx.ForceCPURendering {
|
||||
ctx.Info("Runner compatibility flag is enabled: forcing CPU rendering for this job")
|
||||
} else if ctx.GPUDetectionFailed {
|
||||
ctx.Info("GPU backend detection failed at startup—we could not determine available GPU backends, so rendering will use CPU to avoid compatibility issues")
|
||||
} else {
|
||||
ctx.Info("GPU lockout active: using CPU rendering only")
|
||||
}
|
||||
} else if ctx.ShouldDisableRT() {
|
||||
ctx.Info("GPU ray tracing acceleration disabled for this job (--disable-rt)")
|
||||
}
|
||||
if ctx.ShouldBatchGPUSamples() {
|
||||
total := ctx.GetCyclesSamples()
|
||||
ctx.Info(fmt.Sprintf(
|
||||
"gfx115x HIP sample batching: rendering %d samples in passes of %d (ROCm driver limit, not system RAM)",
|
||||
total, ctx.HipGPUSampleBatch,
|
||||
))
|
||||
}
|
||||
|
||||
// Render
|
||||
ctx.Info(fmt.Sprintf("Starting Blender render for frame %d...", ctx.Frame))
|
||||
if err := p.runBlender(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome); err != nil {
|
||||
if ctx.FrameEnd > ctx.Frame {
|
||||
ctx.Info(fmt.Sprintf("Starting Blender render for frames %d-%d...", ctx.Frame, ctx.FrameEnd))
|
||||
} else {
|
||||
ctx.Info(fmt.Sprintf("Starting Blender render for frame %d...", ctx.Frame))
|
||||
}
|
||||
if err := p.renderFrames(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome); err != nil {
|
||||
if errors.Is(err, ErrJobCancelled) {
|
||||
ctx.Warn("Render stopped because job was cancelled")
|
||||
return err
|
||||
}
|
||||
ctx.Error(fmt.Sprintf("Blender render failed: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
// Verify output
|
||||
if _, err := p.findOutputFile(ctx, outputDir, renderFormat); err != nil {
|
||||
// Verify output (range or single frame)
|
||||
if err := p.verifyOutputRange(ctx, outputDir, renderFormat); err != nil {
|
||||
ctx.Error(fmt.Sprintf("Output verification failed: %v", err))
|
||||
return err
|
||||
}
|
||||
ctx.Info(fmt.Sprintf("Blender render completed for frame %d", ctx.Frame))
|
||||
if ctx.FrameEnd > ctx.Frame {
|
||||
ctx.Info(fmt.Sprintf("Blender render completed for frames %d-%d", ctx.Frame, ctx.FrameEnd))
|
||||
} else {
|
||||
ctx.Info(fmt.Sprintf("Blender render completed for frame %d", ctx.Frame))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string) error {
|
||||
type renderPassOptions struct {
|
||||
samplesOverride *int
|
||||
seedOverride *int
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) renderFrames(ctx *Context, blenderBinary, blendFile, outputDir, renderFormat, blenderHome string) error {
|
||||
if !ctx.ShouldBatchGPUSamples() {
|
||||
if err := p.createRenderScript(ctx, renderFormat, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.runBlender(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome)
|
||||
}
|
||||
|
||||
totalSamples := ctx.GetCyclesSamples()
|
||||
batchSize := ctx.HipGPUSampleBatch
|
||||
baseSeed := ctx.GetCyclesSeed()
|
||||
batchDir := filepath.Join(ctx.WorkDir, "sample_batches")
|
||||
if err := os.MkdirAll(batchDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create sample batch directory: %w", err)
|
||||
}
|
||||
|
||||
var batchOutputs []string
|
||||
remaining := totalSamples
|
||||
batchNum := 0
|
||||
for remaining > 0 {
|
||||
if err := ctx.CheckCancelled(); err != nil {
|
||||
return err
|
||||
}
|
||||
passSamples := batchSize
|
||||
if remaining < passSamples {
|
||||
passSamples = remaining
|
||||
}
|
||||
seed := baseSeed + batchNum
|
||||
batchNum++
|
||||
ctx.Info(fmt.Sprintf("GPU sample batch %d: %d samples (seed %d, %d remaining)", batchNum, passSamples, seed, remaining-passSamples))
|
||||
|
||||
passOutput := filepath.Join(batchDir, fmt.Sprintf("batch_%03d", batchNum))
|
||||
if err := os.MkdirAll(passOutput, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
opts := &renderPassOptions{
|
||||
samplesOverride: &passSamples,
|
||||
seedOverride: &seed,
|
||||
}
|
||||
if err := p.createRenderScript(ctx, renderFormat, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.runBlender(ctx, blenderBinary, blendFile, passOutput, renderFormat, blenderHome); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.verifyOutputRange(ctx, passOutput, renderFormat); err != nil {
|
||||
return err
|
||||
}
|
||||
batchOutputs = append(batchOutputs, p.firstFrameOutputPath(ctx, passOutput, renderFormat))
|
||||
remaining -= passSamples
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
finalOutput := p.firstFrameOutputPath(ctx, outputDir, renderFormat)
|
||||
ctx.Info(fmt.Sprintf("Merging %d GPU sample batches into final EXR...", len(batchOutputs)))
|
||||
if err := mergeEXRFiles(finalOutput, batchOutputs); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx.Info("GPU sample batch merge completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) firstFrameOutputPath(ctx *Context, outputDir, renderFormat string) string {
|
||||
ext := strings.ToLower(renderFormat)
|
||||
if ctx.FrameEnd > ctx.Frame {
|
||||
return filepath.Join(outputDir, fmt.Sprintf("frame_%04d.%s", ctx.Frame, ext))
|
||||
}
|
||||
return filepath.Join(outputDir, fmt.Sprintf("frame_%04d.%s", ctx.Frame, ext))
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string, opts *renderPassOptions) error {
|
||||
formatFilePath := filepath.Join(ctx.WorkDir, "output_format.txt")
|
||||
renderSettingsFilePath := filepath.Join(ctx.WorkDir, "render_settings.json")
|
||||
|
||||
@@ -103,7 +248,9 @@ func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string)
|
||||
unhideCode = scripts.UnhideObjects
|
||||
}
|
||||
|
||||
// Load template and replace placeholders
|
||||
// Load template and replace placeholders.
|
||||
// Job-bundled blender_addons/ always auto-install when present (users ship addons with scenes).
|
||||
// enable_execution only controls Blender --enable-autoexec (scripts inside .blend).
|
||||
scriptContent := scripts.RenderBlenderTemplate
|
||||
scriptContent = strings.ReplaceAll(scriptContent, "{{UNHIDE_CODE}}", unhideCode)
|
||||
scriptContent = strings.ReplaceAll(scriptContent, "{{FORMAT_FILE_PATH}}", fmt.Sprintf("%q", formatFilePath))
|
||||
@@ -116,32 +263,88 @@ func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string)
|
||||
return errors.New(errMsg)
|
||||
}
|
||||
|
||||
// Write output format
|
||||
outputFormat := ctx.GetOutputFormat()
|
||||
ctx.Info(fmt.Sprintf("Writing output format '%s' to format file", outputFormat))
|
||||
if err := os.WriteFile(formatFilePath, []byte(outputFormat), 0644); err != nil {
|
||||
// Write EXR to format file so Blender script sets OPEN_EXR (job output_format is for downstream deliverable only).
|
||||
ctx.Info("Writing output format 'EXR' to format file")
|
||||
if err := os.WriteFile(formatFilePath, []byte("EXR"), 0644); err != nil {
|
||||
errMsg := fmt.Sprintf("failed to create format file: %v", err)
|
||||
ctx.Error(errMsg)
|
||||
return errors.New(errMsg)
|
||||
}
|
||||
|
||||
// Write render settings if available
|
||||
// Write render settings: merge job metadata with runner force_cpu (GPU lockout)
|
||||
var settingsMap map[string]interface{}
|
||||
if ctx.Metadata != nil && ctx.Metadata.RenderSettings.EngineSettings != nil {
|
||||
settingsJSON, err := json.Marshal(ctx.Metadata.RenderSettings)
|
||||
raw, err := json.Marshal(ctx.Metadata.RenderSettings)
|
||||
if err == nil {
|
||||
if err := os.WriteFile(renderSettingsFilePath, settingsJSON, 0644); err != nil {
|
||||
ctx.Warn(fmt.Sprintf("Failed to write render settings file: %v", err))
|
||||
}
|
||||
_ = json.Unmarshal(raw, &settingsMap)
|
||||
}
|
||||
}
|
||||
if settingsMap == nil {
|
||||
settingsMap = make(map[string]interface{})
|
||||
}
|
||||
settingsMap["force_cpu"] = ctx.ShouldForceCPU()
|
||||
settingsMap["disable_rt"] = ctx.ShouldDisableRT()
|
||||
if ctx.ShouldBatchGPUSamples() {
|
||||
// gfx115x: large tiles trigger AMDGPU page faults during HIP renders.
|
||||
settingsMap["tile_size_override"] = 256
|
||||
settingsMap["use_auto_tile_override"] = true
|
||||
}
|
||||
if opts != nil {
|
||||
if opts.samplesOverride != nil {
|
||||
settingsMap["samples_override"] = *opts.samplesOverride
|
||||
}
|
||||
if opts.seedOverride != nil {
|
||||
settingsMap["seed_override"] = *opts.seedOverride
|
||||
}
|
||||
}
|
||||
settingsJSON, err := json.Marshal(settingsMap)
|
||||
if err == nil {
|
||||
if err := os.WriteFile(renderSettingsFilePath, settingsJSON, 0644); err != nil {
|
||||
ctx.Warn(fmt.Sprintf("Failed to write render settings file: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// wrapBlenderCmd builds the Blender *exec.Cmd, optionally through the sandbox wrapper.
|
||||
func wrapBlenderCmd(ctx *Context, blenderBinary string, args, env []string, blenderHome string) (*exec.Cmd, error) {
|
||||
sb := ctx.Sandbox
|
||||
if sb == nil {
|
||||
var err error
|
||||
sb, err = sandbox.New(sandbox.Options{Backend: sandbox.BackendNone})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if sb.Name() != sandbox.BackendNone {
|
||||
ctx.Info(fmt.Sprintf("Running Blender under sandbox backend %q", sb.Name()))
|
||||
}
|
||||
return sb.Wrap(sandbox.Spec{
|
||||
BlenderBinary: blenderBinary,
|
||||
Args: args,
|
||||
WorkDir: ctx.WorkDir,
|
||||
HomeDir: blenderHome,
|
||||
Env: env,
|
||||
HasAMD: ctx.HasAMD,
|
||||
HasNVIDIA: ctx.HasNVIDIA,
|
||||
HasIntel: ctx.HasIntel,
|
||||
ForceCPU: ctx.ShouldForceCPU(),
|
||||
})
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, outputDir, renderFormat, blenderHome string) error {
|
||||
scriptPath := filepath.Join(ctx.WorkDir, "enable_gpu.py")
|
||||
blendFileAbs, err := filepath.Abs(blendFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve blend file path: %w", err)
|
||||
}
|
||||
scriptPathAbs, err := filepath.Abs(scriptPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve blender script path: %w", err)
|
||||
}
|
||||
|
||||
args := []string{"-b", blendFile, "--python", scriptPath}
|
||||
args := []string{"-b", blendFileAbs, "--python", scriptPathAbs}
|
||||
if ctx.ShouldEnableExecution() {
|
||||
args = append(args, "--enable-autoexec")
|
||||
}
|
||||
@@ -151,17 +354,16 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
outputAbsPattern, _ := filepath.Abs(outputPattern)
|
||||
args = append(args, "-o", outputAbsPattern)
|
||||
|
||||
args = append(args, "-f", fmt.Sprintf("%d", ctx.Frame))
|
||||
// Render single frame or range: -f N for one frame, -s start -e end -a for range
|
||||
if ctx.FrameEnd > ctx.Frame {
|
||||
args = append(args, "-s", fmt.Sprintf("%d", ctx.Frame), "-e", fmt.Sprintf("%d", ctx.FrameEnd), "-a")
|
||||
} else {
|
||||
args = append(args, "-f", fmt.Sprintf("%d", ctx.Frame))
|
||||
}
|
||||
|
||||
// Wrap with xvfb-run
|
||||
xvfbArgs := []string{"-a", "-s", "-screen 0 800x600x24", blenderBinary}
|
||||
xvfbArgs = append(xvfbArgs, args...)
|
||||
cmd := exec.Command("xvfb-run", xvfbArgs...)
|
||||
cmd.Dir = ctx.WorkDir
|
||||
|
||||
// Set up environment with custom HOME directory
|
||||
// Set up environment: LD_LIBRARY_PATH for tarball Blender, then custom HOME
|
||||
env := os.Environ()
|
||||
// Remove existing HOME if present and add our custom one
|
||||
env = blender.TarballEnv(blenderBinary, env)
|
||||
newEnv := make([]string, 0, len(env)+1)
|
||||
for _, e := range env {
|
||||
if !strings.HasPrefix(e, "HOME=") {
|
||||
@@ -169,7 +371,11 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
}
|
||||
}
|
||||
newEnv = append(newEnv, fmt.Sprintf("HOME=%s", blenderHome))
|
||||
cmd.Env = newEnv
|
||||
|
||||
cmd, err := wrapBlenderCmd(ctx, blenderBinary, args, newEnv, blenderHome)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to build sandboxed blender command: %w", err)
|
||||
}
|
||||
|
||||
// Set up pipes
|
||||
stdoutPipe, err := cmd.StdoutPipe()
|
||||
@@ -185,12 +391,14 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
if err := cmd.Start(); err != nil {
|
||||
return fmt.Errorf("failed to start blender: %w", err)
|
||||
}
|
||||
stopMonitor := ctx.StartCancellationMonitor(cmd, "render")
|
||||
defer stopMonitor()
|
||||
|
||||
// Track process
|
||||
ctx.Processes.Track(ctx.TaskID, cmd)
|
||||
defer ctx.Processes.Untrack(ctx.TaskID)
|
||||
|
||||
// Stream stdout
|
||||
// Stream stdout and watch for GPU error lines (lock out all GPU on any backend error)
|
||||
stdoutDone := make(chan bool)
|
||||
go func() {
|
||||
defer close(stdoutDone)
|
||||
@@ -198,15 +406,19 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if line != "" {
|
||||
p.checkGPUErrorLine(ctx, line)
|
||||
shouldFilter, logLevel := blender.FilterLog(line)
|
||||
if !shouldFilter {
|
||||
ctx.Log(logLevel, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Printf("Error reading stdout: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Stream stderr
|
||||
// Stream stderr and watch for GPU error lines
|
||||
stderrDone := make(chan bool)
|
||||
go func() {
|
||||
defer close(stderrDone)
|
||||
@@ -214,6 +426,7 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if line != "" {
|
||||
p.checkGPUErrorLine(ctx, line)
|
||||
shouldFilter, logLevel := blender.FilterLog(line)
|
||||
if !shouldFilter {
|
||||
if logLevel == types.LogLevelInfo {
|
||||
@@ -223,6 +436,9 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
log.Printf("Error reading stderr: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for completion
|
||||
@@ -231,6 +447,9 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
<-stderrDone
|
||||
|
||||
if err != nil {
|
||||
if cancelled, checkErr := ctx.IsJobCancelled(); checkErr == nil && cancelled {
|
||||
return ErrJobCancelled
|
||||
}
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
if exitErr.ExitCode() == 137 {
|
||||
return errors.New("Blender was killed due to excessive memory usage (OOM)")
|
||||
@@ -242,60 +461,64 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) findOutputFile(ctx *Context, outputDir, renderFormat string) (string, error) {
|
||||
// verifyOutputRange checks that output files exist for the task's frame range (first and last at minimum).
|
||||
func (p *RenderProcessor) verifyOutputRange(ctx *Context, outputDir, renderFormat string) error {
|
||||
entries, err := os.ReadDir(outputDir)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read output directory: %w", err)
|
||||
return fmt.Errorf("failed to read output directory: %w", err)
|
||||
}
|
||||
|
||||
ctx.Info("Checking output directory for files...")
|
||||
ext := strings.ToLower(renderFormat)
|
||||
|
||||
// Try exact match first
|
||||
expectedFile := filepath.Join(outputDir, fmt.Sprintf("frame_%04d.%s", ctx.Frame, strings.ToLower(renderFormat)))
|
||||
if _, err := os.Stat(expectedFile); err == nil {
|
||||
ctx.Info(fmt.Sprintf("Found output file: %s", filepath.Base(expectedFile)))
|
||||
return expectedFile, nil
|
||||
// Check first and last frame in range (minimum required for range; single frame = one check)
|
||||
framesToCheck := []int{ctx.Frame}
|
||||
if ctx.FrameEnd > ctx.Frame {
|
||||
framesToCheck = append(framesToCheck, ctx.FrameEnd)
|
||||
}
|
||||
|
||||
// Try without zero padding
|
||||
altFile := filepath.Join(outputDir, fmt.Sprintf("frame_%d.%s", ctx.Frame, strings.ToLower(renderFormat)))
|
||||
if _, err := os.Stat(altFile); err == nil {
|
||||
ctx.Info(fmt.Sprintf("Found output file: %s", filepath.Base(altFile)))
|
||||
return altFile, nil
|
||||
}
|
||||
|
||||
// Try just frame number
|
||||
altFile2 := filepath.Join(outputDir, fmt.Sprintf("%04d.%s", ctx.Frame, strings.ToLower(renderFormat)))
|
||||
if _, err := os.Stat(altFile2); err == nil {
|
||||
ctx.Info(fmt.Sprintf("Found output file: %s", filepath.Base(altFile2)))
|
||||
return altFile2, nil
|
||||
}
|
||||
|
||||
// Search through all files
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
fileName := entry.Name()
|
||||
if strings.Contains(fileName, "%04d") || strings.Contains(fileName, "%d") {
|
||||
ctx.Warn(fmt.Sprintf("Skipping file with literal pattern: %s", fileName))
|
||||
continue
|
||||
}
|
||||
frameStr := fmt.Sprintf("%d", ctx.Frame)
|
||||
frameStrPadded := fmt.Sprintf("%04d", ctx.Frame)
|
||||
if strings.Contains(fileName, frameStrPadded) ||
|
||||
(strings.Contains(fileName, frameStr) && strings.HasSuffix(strings.ToLower(fileName), strings.ToLower(renderFormat))) {
|
||||
outputFile := filepath.Join(outputDir, fileName)
|
||||
ctx.Info(fmt.Sprintf("Found output file: %s", fileName))
|
||||
return outputFile, nil
|
||||
for _, frame := range framesToCheck {
|
||||
found := false
|
||||
// Try frame_0001.ext, frame_1.ext, 0001.ext
|
||||
for _, name := range []string{
|
||||
fmt.Sprintf("frame_%04d.%s", frame, ext),
|
||||
fmt.Sprintf("frame_%d.%s", frame, ext),
|
||||
fmt.Sprintf("%04d.%s", frame, ext),
|
||||
} {
|
||||
if _, err := os.Stat(filepath.Join(outputDir, name)); err == nil {
|
||||
found = true
|
||||
ctx.Info(fmt.Sprintf("Found output file: %s", name))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Not found
|
||||
fileList := []string{}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
fileList = append(fileList, entry.Name())
|
||||
if !found {
|
||||
// Search entries for this frame number
|
||||
frameStr := fmt.Sprintf("%d", frame)
|
||||
frameStrPadded := fmt.Sprintf("%04d", frame)
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
fileName := entry.Name()
|
||||
if strings.Contains(fileName, "%04d") || strings.Contains(fileName, "%d") {
|
||||
continue
|
||||
}
|
||||
if (strings.Contains(fileName, frameStrPadded) ||
|
||||
strings.Contains(fileName, frameStr)) && strings.HasSuffix(strings.ToLower(fileName), ext) {
|
||||
found = true
|
||||
ctx.Info(fmt.Sprintf("Found output file: %s", fileName))
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
fileList := []string{}
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
fileList = append(fileList, e.Name())
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("output file for frame %d not found; files in output directory: %v", frame, fileList)
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("output file not found: %s\nFiles in output directory: %v", expectedFile, fileList)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package tasks
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCheckGPUErrorLine_TriggersCallback(t *testing.T) {
|
||||
p := NewRenderProcessor()
|
||||
triggered := 0
|
||||
ctx := &Context{
|
||||
OnGPUError: func() {
|
||||
triggered++
|
||||
// Simulate runner arming lockout for this attempt.
|
||||
},
|
||||
}
|
||||
p.checkGPUErrorLine(ctx, "Fatal: Illegal address in HIP kernel execution")
|
||||
if triggered != 1 {
|
||||
t.Fatalf("expected GPU error callback once, got %d", triggered)
|
||||
}
|
||||
// After arming, further fault lines must not re-fire (spam protection).
|
||||
ctx.GPULockoutArmedThisAttempt = true
|
||||
p.checkGPUErrorLine(ctx, "page not present in GPU memory")
|
||||
if triggered != 1 {
|
||||
t.Fatalf("expected no further callbacks after arm, got %d", triggered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckGPUErrorLine_SkipsWhenAlreadyLockedOut(t *testing.T) {
|
||||
p := NewRenderProcessor()
|
||||
triggered := false
|
||||
ctx := &Context{
|
||||
GPULockedOut: true,
|
||||
OnGPUError: func() { triggered = true },
|
||||
}
|
||||
p.checkGPUErrorLine(ctx, "Illegal address in HIP")
|
||||
if triggered {
|
||||
t.Fatal("did not expect callback when GPU already locked out")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBlenderVersion_EmptyWhenMissing(t *testing.T) {
|
||||
ctx := &Context{}
|
||||
if got := ctx.GetBlenderVersion(); got != "" {
|
||||
t.Fatalf("expected empty blender version, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckGPUErrorLine_IgnoresNormalLine(t *testing.T) {
|
||||
p := NewRenderProcessor()
|
||||
triggered := false
|
||||
ctx := &Context{
|
||||
OnGPUError: func() { triggered = true },
|
||||
}
|
||||
p.checkGPUErrorLine(ctx, "Render completed successfully")
|
||||
if triggered {
|
||||
t.Fatal("did not expect GPU callback for normal line")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,6 +99,11 @@ func ExtractTarStripPrefix(reader io.Reader, destDir string) error {
|
||||
|
||||
targetPath := filepath.Join(destDir, name)
|
||||
|
||||
// Sanitize path to prevent directory traversal
|
||||
if !strings.HasPrefix(filepath.Clean(targetPath), filepath.Clean(destDir)+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("invalid file path in tar: %s", header.Name)
|
||||
}
|
||||
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(targetPath, os.FileMode(header.Mode)); err != nil {
|
||||
@@ -123,6 +128,20 @@ func ExtractTarStripPrefix(reader io.Reader, destDir string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
// Reject absolute or escaping symlink targets so extraction cannot
|
||||
// plant links that point outside destDir.
|
||||
linkTarget := header.Linkname
|
||||
if linkTarget == "" {
|
||||
return fmt.Errorf("invalid empty symlink target in tar: %s", header.Name)
|
||||
}
|
||||
if filepath.IsAbs(linkTarget) {
|
||||
return fmt.Errorf("absolute symlink target not allowed in tar: %s -> %s", header.Name, linkTarget)
|
||||
}
|
||||
resolvedLink := filepath.Clean(filepath.Join(filepath.Dir(targetPath), linkTarget))
|
||||
cleanDest := filepath.Clean(destDir)
|
||||
if resolvedLink != cleanDest && !strings.HasPrefix(resolvedLink, cleanDest+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("symlink target escapes extract root: %s -> %s", header.Name, linkTarget)
|
||||
}
|
||||
os.Remove(targetPath) // Remove existing symlink if present
|
||||
if err := os.Symlink(header.Linkname, targetPath); err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func createTarBuffer(files map[string]string) *bytes.Buffer {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
for name, content := range files {
|
||||
hdr := &tar.Header{
|
||||
Name: name,
|
||||
Mode: 0644,
|
||||
Size: int64(len(content)),
|
||||
}
|
||||
tw.WriteHeader(hdr)
|
||||
tw.Write([]byte(content))
|
||||
}
|
||||
tw.Close()
|
||||
return &buf
|
||||
}
|
||||
|
||||
func TestExtractTarStripPrefix_RejectsEscapingSymlink(t *testing.T) {
|
||||
destDir := t.TempDir()
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
// Top-level prefix like Blender archives
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "blender-x/", Typeflag: tar.TypeDir, Mode: 0755})
|
||||
_ = tw.WriteHeader(&tar.Header{
|
||||
Name: "blender-x/evil",
|
||||
Typeflag: tar.TypeSymlink,
|
||||
Linkname: "../../outside",
|
||||
Mode: 0777,
|
||||
})
|
||||
_ = tw.Close()
|
||||
|
||||
if err := ExtractTarStripPrefix(&buf, destDir); err == nil {
|
||||
t.Fatal("expected escaping symlink to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTar(t *testing.T) {
|
||||
destDir := t.TempDir()
|
||||
|
||||
buf := createTarBuffer(map[string]string{
|
||||
"hello.txt": "world",
|
||||
"sub/a.txt": "nested",
|
||||
})
|
||||
|
||||
if err := ExtractTar(buf, destDir); err != nil {
|
||||
t.Fatalf("ExtractTar: %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(destDir, "hello.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("read hello.txt: %v", err)
|
||||
}
|
||||
if string(data) != "world" {
|
||||
t.Errorf("hello.txt = %q, want %q", data, "world")
|
||||
}
|
||||
|
||||
data, err = os.ReadFile(filepath.Join(destDir, "sub", "a.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("read sub/a.txt: %v", err)
|
||||
}
|
||||
if string(data) != "nested" {
|
||||
t.Errorf("sub/a.txt = %q, want %q", data, "nested")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTarStripPrefix(t *testing.T) {
|
||||
destDir := t.TempDir()
|
||||
|
||||
buf := createTarBuffer(map[string]string{
|
||||
"toplevel/": "",
|
||||
"toplevel/foo.txt": "bar",
|
||||
})
|
||||
|
||||
if err := ExtractTarStripPrefix(buf, destDir); err != nil {
|
||||
t.Fatalf("ExtractTarStripPrefix: %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(destDir, "foo.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("read foo.txt: %v", err)
|
||||
}
|
||||
if string(data) != "bar" {
|
||||
t.Errorf("foo.txt = %q, want %q", data, "bar")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTarStripPrefix_PathTraversal(t *testing.T) {
|
||||
destDir := t.TempDir()
|
||||
|
||||
buf := createTarBuffer(map[string]string{
|
||||
"prefix/../../../etc/passwd": "pwned",
|
||||
})
|
||||
|
||||
err := ExtractTarStripPrefix(buf, destDir)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for path traversal, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTar_PathTraversal(t *testing.T) {
|
||||
destDir := t.TempDir()
|
||||
|
||||
buf := createTarBuffer(map[string]string{
|
||||
"../../../etc/passwd": "pwned",
|
||||
})
|
||||
|
||||
err := ExtractTar(buf, destDir)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for path traversal, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTarFile(t *testing.T) {
|
||||
destDir := t.TempDir()
|
||||
tarPath := filepath.Join(t.TempDir(), "archive.tar")
|
||||
|
||||
buf := createTarBuffer(map[string]string{
|
||||
"hello.txt": "world",
|
||||
})
|
||||
if err := os.WriteFile(tarPath, buf.Bytes(), 0644); err != nil {
|
||||
t.Fatalf("write tar file: %v", err)
|
||||
}
|
||||
|
||||
if err := ExtractTarFile(tarPath, destDir); err != nil {
|
||||
t.Fatalf("ExtractTarFile: %v", err)
|
||||
}
|
||||
|
||||
got, err := os.ReadFile(filepath.Join(destDir, "hello.txt"))
|
||||
if err != nil {
|
||||
t.Fatalf("read extracted file: %v", err)
|
||||
}
|
||||
if string(got) != "world" {
|
||||
t.Fatalf("unexpected file content: %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSanitizeName_ReplacesUnsafeChars(t *testing.T) {
|
||||
got := sanitizeName("runner / with\\bad:chars")
|
||||
if strings.ContainsAny(got, " /\\:") {
|
||||
t.Fatalf("sanitizeName did not sanitize input: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindBlendFiles_IgnoresBlendSaveFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "scene.blend"), []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "scene.blend1"), []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files, err := FindBlendFiles(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("FindBlendFiles failed: %v", err)
|
||||
}
|
||||
if len(files) != 1 || files[0] != "scene.blend" {
|
||||
t.Fatalf("unexpected files: %#v", files)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindFirstBlendFile_ReturnsErrorWhenMissing(t *testing.T) {
|
||||
_, err := FindFirstBlendFile(t.TempDir())
|
||||
if err == nil {
|
||||
t.Fatal("expected error when no blend file exists")
|
||||
}
|
||||
}
|
||||
|
||||
+193
-6
@@ -60,7 +60,7 @@ func (s *Storage) TempDir(pattern string) (string, error) {
|
||||
if err := os.MkdirAll(s.tempPath(), 0755); err != nil {
|
||||
return "", fmt.Errorf("failed to create temp directory: %w", err)
|
||||
}
|
||||
|
||||
|
||||
// Create temp directory under storage base path
|
||||
return os.MkdirTemp(s.tempPath(), pattern)
|
||||
}
|
||||
@@ -80,8 +80,55 @@ func (s *Storage) JobPath(jobID int64) string {
|
||||
return filepath.Join(s.basePath, "jobs", fmt.Sprintf("%d", jobID))
|
||||
}
|
||||
|
||||
// SanitizeFilename returns a safe basename for client-supplied names.
|
||||
// Rejects empty, ".", "..", and root-like basenames.
|
||||
func SanitizeFilename(name string) (string, error) {
|
||||
cleaned := filepath.Clean(strings.ReplaceAll(name, "\\", "/"))
|
||||
base := filepath.Base(cleaned)
|
||||
// filepath.Base("..") is ".." ; Clean("/../..") style inputs can yield "/" or "."
|
||||
if base == "" || base == "." || base == ".." || base == "/" || base == string(os.PathSeparator) {
|
||||
return "", fmt.Errorf("invalid filename: %q", name)
|
||||
}
|
||||
// Reject if the cleaned path still has parent references after basenaming was skipped
|
||||
if strings.Contains(cleaned, "..") && base == cleaned {
|
||||
return "", fmt.Errorf("invalid filename: %q", name)
|
||||
}
|
||||
return base, nil
|
||||
}
|
||||
|
||||
// SafePathUnderRoot joins root and a relative path and ensures the result stays under root.
|
||||
func SafePathUnderRoot(root, rel string) (string, error) {
|
||||
if rel == "" {
|
||||
return "", fmt.Errorf("empty path")
|
||||
}
|
||||
// Normalize to forward slashes then clean
|
||||
rel = filepath.ToSlash(rel)
|
||||
if filepath.IsAbs(rel) || strings.HasPrefix(rel, "/") {
|
||||
return "", fmt.Errorf("absolute path not allowed: %q", rel)
|
||||
}
|
||||
cleanRel := filepath.Clean(rel)
|
||||
if cleanRel == ".." || strings.HasPrefix(cleanRel, ".."+string(os.PathSeparator)) {
|
||||
return "", fmt.Errorf("path escapes root: %q", rel)
|
||||
}
|
||||
cleanRoot := filepath.Clean(root)
|
||||
full := filepath.Join(cleanRoot, cleanRel)
|
||||
cleanFull := filepath.Clean(full)
|
||||
sep := string(os.PathSeparator)
|
||||
if cleanFull != cleanRoot && !strings.HasPrefix(cleanFull, cleanRoot+sep) {
|
||||
return "", fmt.Errorf("path escapes root: %q", rel)
|
||||
}
|
||||
return cleanFull, nil
|
||||
}
|
||||
|
||||
// SaveUpload saves an uploaded file
|
||||
func (s *Storage) SaveUpload(jobID int64, filename string, reader io.Reader) (string, error) {
|
||||
// Sanitize filename to prevent path traversal
|
||||
var err error
|
||||
filename, err = SanitizeFilename(filename)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
jobPath := s.JobPath(jobID)
|
||||
if err := os.MkdirAll(jobPath, 0755); err != nil {
|
||||
return "", fmt.Errorf("failed to create job directory: %w", err)
|
||||
@@ -103,12 +150,26 @@ func (s *Storage) SaveUpload(jobID int64, filename string, reader io.Reader) (st
|
||||
|
||||
// SaveOutput saves an output file
|
||||
func (s *Storage) SaveOutput(jobID int64, filename string, reader io.Reader) (string, error) {
|
||||
// Sanitize filename to prevent path traversal (parity with SaveUpload)
|
||||
var err error
|
||||
filename, err = SanitizeFilename(filename)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
outputPath := filepath.Join(s.outputsPath(), fmt.Sprintf("%d", jobID))
|
||||
if err := os.MkdirAll(outputPath, 0755); err != nil {
|
||||
return "", fmt.Errorf("failed to create output directory: %w", err)
|
||||
}
|
||||
|
||||
filePath := filepath.Join(outputPath, filename)
|
||||
// Defense in depth: ensure resolved path stays under output dir
|
||||
if resolved, err := SafePathUnderRoot(outputPath, filename); err != nil {
|
||||
return "", err
|
||||
} else {
|
||||
filePath = resolved
|
||||
}
|
||||
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create file: %w", err)
|
||||
@@ -166,12 +227,12 @@ func (s *Storage) GetFileSize(filePath string) (int64, error) {
|
||||
// Returns a list of all extracted file paths
|
||||
func (s *Storage) ExtractZip(zipPath, destDir string) ([]string, error) {
|
||||
log.Printf("Extracting ZIP archive: %s -> %s", zipPath, destDir)
|
||||
|
||||
|
||||
// Ensure destination directory exists
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("failed to create destination directory: %w", err)
|
||||
}
|
||||
|
||||
|
||||
r, err := zip.OpenReader(zipPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open ZIP file: %w", err)
|
||||
@@ -187,7 +248,7 @@ func (s *Storage) ExtractZip(zipPath, destDir string) ([]string, error) {
|
||||
for _, f := range r.File {
|
||||
// Sanitize file path to prevent directory traversal
|
||||
destPath := filepath.Join(destDir, f.Name)
|
||||
|
||||
|
||||
cleanDestPath := filepath.Clean(destPath)
|
||||
cleanDestDir := filepath.Clean(destDir)
|
||||
if !strings.HasPrefix(cleanDestPath, cleanDestDir+string(os.PathSeparator)) && cleanDestPath != cleanDestDir {
|
||||
@@ -520,7 +581,7 @@ func (s *Storage) CreateJobContextFromDir(sourceDir string, jobID int64, exclude
|
||||
if commonPrefix != "" && strings.HasPrefix(tarPath, commonPrefix) {
|
||||
tarPath = strings.TrimPrefix(tarPath, commonPrefix)
|
||||
}
|
||||
|
||||
|
||||
// Check if it's a .blend file at root (no path separators after prefix stripping)
|
||||
if strings.HasSuffix(strings.ToLower(tarPath), ".blend") {
|
||||
// Check if it's at root level (no directory separators)
|
||||
@@ -566,7 +627,7 @@ func (s *Storage) CreateJobContextFromDir(sourceDir string, jobID int64, exclude
|
||||
// Get relative path and strip common prefix if present
|
||||
relPath := relPaths[i]
|
||||
tarPath := filepath.ToSlash(relPath)
|
||||
|
||||
|
||||
// Strip common prefix if found
|
||||
if commonPrefix != "" && strings.HasPrefix(tarPath, commonPrefix) {
|
||||
tarPath = strings.TrimPrefix(tarPath, commonPrefix)
|
||||
@@ -608,3 +669,129 @@ func (s *Storage) CreateJobContextFromDir(sourceDir string, jobID int64, exclude
|
||||
return contextPath, nil
|
||||
}
|
||||
|
||||
// CreateContextArchiveFromDirToPath creates a context archive from files in sourceDir at destPath.
|
||||
// This is used for pre-job upload sessions where the archive is staged before a job ID exists.
|
||||
func (s *Storage) CreateContextArchiveFromDirToPath(sourceDir, destPath string, excludeFiles ...string) (string, error) {
|
||||
excludeSet := make(map[string]bool)
|
||||
for _, excludeFile := range excludeFiles {
|
||||
excludePath := filepath.Clean(excludeFile)
|
||||
excludeSet[excludePath] = true
|
||||
excludeSet[filepath.ToSlash(excludePath)] = true
|
||||
}
|
||||
|
||||
var filesToInclude []string
|
||||
err := filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
if isBlenderSaveFile(info.Name()) {
|
||||
return nil
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(sourceDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cleanRelPath := filepath.Clean(relPath)
|
||||
if strings.HasPrefix(cleanRelPath, "..") {
|
||||
return fmt.Errorf("invalid file path: %s", relPath)
|
||||
}
|
||||
if excludeSet[cleanRelPath] || excludeSet[filepath.ToSlash(cleanRelPath)] {
|
||||
return nil
|
||||
}
|
||||
|
||||
filesToInclude = append(filesToInclude, path)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to walk source directory: %w", err)
|
||||
}
|
||||
if len(filesToInclude) == 0 {
|
||||
return "", fmt.Errorf("no files found to include in context archive")
|
||||
}
|
||||
|
||||
relPaths := make([]string, 0, len(filesToInclude))
|
||||
for _, filePath := range filesToInclude {
|
||||
relPath, err := filepath.Rel(sourceDir, filePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get relative path: %w", err)
|
||||
}
|
||||
relPaths = append(relPaths, relPath)
|
||||
}
|
||||
|
||||
commonPrefix := findCommonPrefix(relPaths)
|
||||
blendFilesAtRoot := 0
|
||||
for _, relPath := range relPaths {
|
||||
tarPath := filepath.ToSlash(relPath)
|
||||
if commonPrefix != "" && strings.HasPrefix(tarPath, commonPrefix) {
|
||||
tarPath = strings.TrimPrefix(tarPath, commonPrefix)
|
||||
}
|
||||
if strings.HasSuffix(strings.ToLower(tarPath), ".blend") && !strings.Contains(tarPath, "/") {
|
||||
blendFilesAtRoot++
|
||||
}
|
||||
}
|
||||
if blendFilesAtRoot == 0 {
|
||||
return "", fmt.Errorf("no .blend file found at root level in context archive - .blend files must be at the root level of the uploaded archive, not in subdirectories")
|
||||
}
|
||||
if blendFilesAtRoot > 1 {
|
||||
return "", fmt.Errorf("multiple .blend files found at root level in context archive (found %d, expected 1)", blendFilesAtRoot)
|
||||
}
|
||||
|
||||
contextFile, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create context file: %w", err)
|
||||
}
|
||||
defer contextFile.Close()
|
||||
|
||||
tarWriter := tar.NewWriter(contextFile)
|
||||
defer tarWriter.Close()
|
||||
copyBuf := make([]byte, 32*1024)
|
||||
|
||||
for i, filePath := range filesToInclude {
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return "", fmt.Errorf("failed to stat file: %w", err)
|
||||
}
|
||||
|
||||
tarPath := filepath.ToSlash(relPaths[i])
|
||||
if commonPrefix != "" && strings.HasPrefix(tarPath, commonPrefix) {
|
||||
tarPath = strings.TrimPrefix(tarPath, commonPrefix)
|
||||
}
|
||||
|
||||
header, err := tar.FileInfoHeader(info, "")
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return "", fmt.Errorf("failed to create tar header: %w", err)
|
||||
}
|
||||
header.Name = tarPath
|
||||
|
||||
if err := tarWriter.WriteHeader(header); err != nil {
|
||||
file.Close()
|
||||
return "", fmt.Errorf("failed to write tar header: %w", err)
|
||||
}
|
||||
|
||||
if _, err := io.CopyBuffer(tarWriter, file, copyBuf); err != nil {
|
||||
file.Close()
|
||||
return "", fmt.Errorf("failed to write file to tar: %w", err)
|
||||
}
|
||||
file.Close()
|
||||
}
|
||||
|
||||
if err := tarWriter.Close(); err != nil {
|
||||
return "", fmt.Errorf("failed to close tar writer: %w", err)
|
||||
}
|
||||
if err := contextFile.Close(); err != nil {
|
||||
return "", fmt.Errorf("failed to close context file: %w", err)
|
||||
}
|
||||
|
||||
return destPath, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func setupStorage(t *testing.T) *Storage {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
s, err := NewStorage(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStorage: %v", err)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func TestSaveUpload(t *testing.T) {
|
||||
s := setupStorage(t)
|
||||
path, err := s.SaveUpload(1, "test.blend", strings.NewReader("data"))
|
||||
if err != nil {
|
||||
t.Fatalf("SaveUpload: %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read saved file: %v", err)
|
||||
}
|
||||
if string(data) != "data" {
|
||||
t.Errorf("got %q, want %q", data, "data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveUpload_PathTraversal(t *testing.T) {
|
||||
s := setupStorage(t)
|
||||
path, err := s.SaveUpload(1, "../../etc/passwd", strings.NewReader("evil"))
|
||||
if err != nil {
|
||||
t.Fatalf("SaveUpload: %v", err)
|
||||
}
|
||||
|
||||
// filepath.Base strips traversal, so the file should be inside the job dir
|
||||
if !strings.HasPrefix(path, s.JobPath(1)) {
|
||||
t.Errorf("saved file %q escaped job directory %q", path, s.JobPath(1))
|
||||
}
|
||||
|
||||
if filepath.Base(path) != "passwd" {
|
||||
t.Errorf("expected basename 'passwd', got %q", filepath.Base(path))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveOutput(t *testing.T) {
|
||||
s := setupStorage(t)
|
||||
path, err := s.SaveOutput(42, "output.png", strings.NewReader("img"))
|
||||
if err != nil {
|
||||
t.Fatalf("SaveOutput: %v", err)
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatalf("read saved output: %v", err)
|
||||
}
|
||||
if string(data) != "img" {
|
||||
t.Errorf("got %q, want %q", data, "img")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSaveOutput_PathTraversal(t *testing.T) {
|
||||
s := setupStorage(t)
|
||||
path, err := s.SaveOutput(42, "../../etc/passwd", strings.NewReader("evil"))
|
||||
if err != nil {
|
||||
t.Fatalf("SaveOutput: %v", err)
|
||||
}
|
||||
outRoot := filepath.Join(s.BasePath(), "outputs", "42")
|
||||
if !strings.HasPrefix(path, outRoot) {
|
||||
t.Errorf("saved file %q escaped output directory %q", path, outRoot)
|
||||
}
|
||||
if filepath.Base(path) != "passwd" {
|
||||
t.Errorf("expected basename passwd, got %q", filepath.Base(path))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeFilename(t *testing.T) {
|
||||
got, err := SanitizeFilename("../../etc/passwd")
|
||||
if err != nil {
|
||||
t.Fatalf("SanitizeFilename: %v", err)
|
||||
}
|
||||
if got != "passwd" {
|
||||
t.Fatalf("got %q want passwd", got)
|
||||
}
|
||||
if _, err := SanitizeFilename(".."); err == nil {
|
||||
t.Fatal("expected error for ..")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSafePathUnderRoot(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
ok, err := SafePathUnderRoot(root, "subdir/file.blend")
|
||||
if err != nil {
|
||||
t.Fatalf("SafePathUnderRoot: %v", err)
|
||||
}
|
||||
if !strings.HasPrefix(ok, root) {
|
||||
t.Fatalf("path %q not under root %q", ok, root)
|
||||
}
|
||||
if _, err := SafePathUnderRoot(root, "../escape.blend"); err == nil {
|
||||
t.Fatal("expected escape to fail")
|
||||
}
|
||||
if _, err := SafePathUnderRoot(root, "/abs.blend"); err == nil {
|
||||
t.Fatal("expected absolute path to fail")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetFile(t *testing.T) {
|
||||
s := setupStorage(t)
|
||||
savedPath, err := s.SaveUpload(1, "readme.txt", strings.NewReader("hello"))
|
||||
if err != nil {
|
||||
t.Fatalf("SaveUpload: %v", err)
|
||||
}
|
||||
|
||||
f, err := s.GetFile(savedPath)
|
||||
if err != nil {
|
||||
t.Fatalf("GetFile: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
buf := make([]byte, 64)
|
||||
n, _ := f.Read(buf)
|
||||
if string(buf[:n]) != "hello" {
|
||||
t.Errorf("got %q, want %q", string(buf[:n]), "hello")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobPath(t *testing.T) {
|
||||
s := setupStorage(t)
|
||||
path := s.JobPath(99)
|
||||
if !strings.Contains(path, "99") {
|
||||
t.Errorf("JobPath(99) = %q, expected to contain '99'", path)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package blendfile
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
)
|
||||
|
||||
// ParseVersionFromReader parses the Blender version from a reader.
|
||||
// Returns major and minor version numbers.
|
||||
//
|
||||
// Blend file header layout (12 bytes):
|
||||
//
|
||||
// "BLENDER" (7) + pointer-size (1: '-'=64, '_'=32) + endian (1: 'v'=LE, 'V'=BE)
|
||||
// + version (3 digits, e.g. "402" = 4.02)
|
||||
//
|
||||
// Supports uncompressed, gzip-compressed, and zstd-compressed blend files.
|
||||
func ParseVersionFromReader(r io.ReadSeeker) (major, minor int, err error) {
|
||||
header := make([]byte, 12)
|
||||
n, err := r.Read(header)
|
||||
if err != nil || n < 12 {
|
||||
return 0, 0, fmt.Errorf("failed to read blend file header: %w", err)
|
||||
}
|
||||
|
||||
if string(header[:7]) != "BLENDER" {
|
||||
r.Seek(0, 0)
|
||||
return parseCompressedVersion(r)
|
||||
}
|
||||
|
||||
return parseVersionDigits(header[9:12])
|
||||
}
|
||||
|
||||
// ParseVersionFromFile opens a blend file and parses the Blender version.
|
||||
func ParseVersionFromFile(blendPath string) (major, minor int, err error) {
|
||||
file, err := os.Open(blendPath)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to open blend file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
return ParseVersionFromReader(file)
|
||||
}
|
||||
|
||||
// VersionString returns a formatted version string like "4.2".
|
||||
func VersionString(major, minor int) string {
|
||||
return fmt.Sprintf("%d.%d", major, minor)
|
||||
}
|
||||
|
||||
func parseVersionDigits(versionBytes []byte) (major, minor int, err error) {
|
||||
if len(versionBytes) != 3 {
|
||||
return 0, 0, fmt.Errorf("expected 3 version digits, got %d", len(versionBytes))
|
||||
}
|
||||
fmt.Sscanf(string(versionBytes[0]), "%d", &major)
|
||||
fmt.Sscanf(string(versionBytes[1:3]), "%d", &minor)
|
||||
return major, minor, nil
|
||||
}
|
||||
|
||||
func parseCompressedVersion(r io.ReadSeeker) (major, minor int, err error) {
|
||||
magic := make([]byte, 4)
|
||||
if _, err := r.Read(magic); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
r.Seek(0, 0)
|
||||
|
||||
// gzip: 0x1f 0x8b
|
||||
if magic[0] == 0x1f && magic[1] == 0x8b {
|
||||
gzReader, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to create gzip reader: %w", err)
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
header := make([]byte, 12)
|
||||
n, err := gzReader.Read(header)
|
||||
if err != nil || n < 12 {
|
||||
return 0, 0, fmt.Errorf("failed to read compressed blend header: %w", err)
|
||||
}
|
||||
if string(header[:7]) != "BLENDER" {
|
||||
return 0, 0, fmt.Errorf("invalid blend file format")
|
||||
}
|
||||
return parseVersionDigits(header[9:12])
|
||||
}
|
||||
|
||||
// zstd: 0x28 0xB5 0x2F 0xFD
|
||||
if magic[0] == 0x28 && magic[1] == 0xb5 && magic[2] == 0x2f && magic[3] == 0xfd {
|
||||
return parseZstdVersion(r)
|
||||
}
|
||||
|
||||
return 0, 0, fmt.Errorf("unknown blend file format")
|
||||
}
|
||||
|
||||
func parseZstdVersion(r io.ReadSeeker) (major, minor int, err error) {
|
||||
r.Seek(0, 0)
|
||||
|
||||
cmd := exec.Command("zstd", "-d", "-c")
|
||||
cmd.Stdin = r
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to create zstd stdout pipe: %w", err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return 0, 0, fmt.Errorf("failed to start zstd decompression: %w", err)
|
||||
}
|
||||
|
||||
header := make([]byte, 12)
|
||||
n, readErr := io.ReadFull(stdout, header)
|
||||
|
||||
cmd.Process.Kill()
|
||||
cmd.Wait()
|
||||
|
||||
if readErr != nil || n < 12 {
|
||||
return 0, 0, fmt.Errorf("failed to read zstd compressed blend header: %v", readErr)
|
||||
}
|
||||
if string(header[:7]) != "BLENDER" {
|
||||
return 0, 0, fmt.Errorf("invalid blend file format in zstd archive")
|
||||
}
|
||||
|
||||
return parseVersionDigits(header[9:12])
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package blendfile
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func makeBlendHeader(major, minor int) []byte {
|
||||
header := make([]byte, 12)
|
||||
copy(header[:7], "BLENDER")
|
||||
header[7] = '-'
|
||||
header[8] = 'v'
|
||||
header[9] = byte('0' + major)
|
||||
header[10] = byte('0' + minor/10)
|
||||
header[11] = byte('0' + minor%10)
|
||||
return header
|
||||
}
|
||||
|
||||
func TestParseVersionFromReader_Uncompressed(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
major int
|
||||
minor int
|
||||
wantMajor int
|
||||
wantMinor int
|
||||
}{
|
||||
{"Blender 4.02", 4, 2, 4, 2},
|
||||
{"Blender 3.06", 3, 6, 3, 6},
|
||||
{"Blender 2.79", 2, 79, 2, 79},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
header := makeBlendHeader(tt.major, tt.minor)
|
||||
r := bytes.NewReader(header)
|
||||
|
||||
major, minor, err := ParseVersionFromReader(r)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseVersionFromReader: %v", err)
|
||||
}
|
||||
if major != tt.wantMajor || minor != tt.wantMinor {
|
||||
t.Errorf("got %d.%d, want %d.%d", major, minor, tt.wantMajor, tt.wantMinor)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseVersionFromReader_GzipCompressed(t *testing.T) {
|
||||
header := makeBlendHeader(4, 2)
|
||||
// Pad to ensure gzip has enough data for a full read
|
||||
data := make([]byte, 128)
|
||||
copy(data, header)
|
||||
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
gz.Write(data)
|
||||
gz.Close()
|
||||
|
||||
r := bytes.NewReader(buf.Bytes())
|
||||
major, minor, err := ParseVersionFromReader(r)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseVersionFromReader (gzip): %v", err)
|
||||
}
|
||||
if major != 4 || minor != 2 {
|
||||
t.Errorf("got %d.%d, want 4.2", major, minor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseVersionFromReader_InvalidMagic(t *testing.T) {
|
||||
data := []byte("NOT_BLEND_DATA_HERE")
|
||||
r := bytes.NewReader(data)
|
||||
|
||||
_, _, err := ParseVersionFromReader(r)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for invalid magic, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseVersionFromReader_TooShort(t *testing.T) {
|
||||
data := []byte("SHORT")
|
||||
r := bytes.NewReader(data)
|
||||
|
||||
_, _, err := ParseVersionFromReader(r)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for short data, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionString(t *testing.T) {
|
||||
got := VersionString(4, 2)
|
||||
want := "4.2"
|
||||
if got != want {
|
||||
t.Errorf("VersionString(4, 2) = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
+95
-2
@@ -2,10 +2,13 @@ package executils
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -107,6 +110,78 @@ type CommandResult struct {
|
||||
ExitCode int
|
||||
}
|
||||
|
||||
// RunCommandWithTimeout is like RunCommand but kills the process after timeout.
|
||||
// A zero timeout means no timeout.
|
||||
func RunCommandWithTimeout(
|
||||
timeout time.Duration,
|
||||
cmdPath string,
|
||||
args []string,
|
||||
dir string,
|
||||
env []string,
|
||||
taskID int64,
|
||||
tracker *ProcessTracker,
|
||||
) (*CommandResult, error) {
|
||||
if timeout <= 0 {
|
||||
return RunCommand(cmdPath, args, dir, env, taskID, tracker)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, cmdPath, args...)
|
||||
cmd.Dir = dir
|
||||
if env != nil {
|
||||
cmd.Env = env
|
||||
}
|
||||
stdoutPipe, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create stdout pipe: %w", err)
|
||||
}
|
||||
stderrPipe, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create stderr pipe: %w", err)
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("failed to start command: %w", err)
|
||||
}
|
||||
if tracker != nil {
|
||||
tracker.Track(taskID, cmd)
|
||||
defer tracker.Untrack(taskID)
|
||||
}
|
||||
var stdoutBuf, stderrBuf []byte
|
||||
var stdoutErr, stderrErr error
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
stdoutBuf, stdoutErr = readAll(stdoutPipe)
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
stderrBuf, stderrErr = readAll(stderrPipe)
|
||||
}()
|
||||
waitErr := cmd.Wait()
|
||||
wg.Wait()
|
||||
if stdoutErr != nil && !isBenignPipeReadError(stdoutErr) {
|
||||
return nil, fmt.Errorf("failed to read stdout: %w", stdoutErr)
|
||||
}
|
||||
if stderrErr != nil && !isBenignPipeReadError(stderrErr) {
|
||||
return nil, fmt.Errorf("failed to read stderr: %w", stderrErr)
|
||||
}
|
||||
result := &CommandResult{Stdout: string(stdoutBuf), Stderr: string(stderrBuf)}
|
||||
if waitErr != nil {
|
||||
if exitErr, ok := waitErr.(*exec.ExitError); ok {
|
||||
result.ExitCode = exitErr.ExitCode()
|
||||
} else {
|
||||
result.ExitCode = -1
|
||||
}
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return result, fmt.Errorf("command timed out after %v: %w", timeout, waitErr)
|
||||
}
|
||||
return result, waitErr
|
||||
}
|
||||
result.ExitCode = 0
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// RunCommand executes a command and returns the output
|
||||
// If tracker is provided, the process will be registered for tracking
|
||||
// This is useful for commands where you need to capture output (like metadata extraction)
|
||||
@@ -164,10 +239,10 @@ func RunCommand(
|
||||
wg.Wait()
|
||||
|
||||
// Check for read errors
|
||||
if stdoutErr != nil {
|
||||
if stdoutErr != nil && !isBenignPipeReadError(stdoutErr) {
|
||||
return nil, fmt.Errorf("failed to read stdout: %w", stdoutErr)
|
||||
}
|
||||
if stderrErr != nil {
|
||||
if stderrErr != nil && !isBenignPipeReadError(stderrErr) {
|
||||
return nil, fmt.Errorf("failed to read stderr: %w", stderrErr)
|
||||
}
|
||||
|
||||
@@ -208,6 +283,18 @@ func readAll(r interface{ Read([]byte) (int, error) }) ([]byte, error) {
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
// isBenignPipeReadError treats EOF-like pipe close races as non-fatal.
|
||||
func isBenignPipeReadError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, os.ErrClosed) || errors.Is(err, io.ErrClosedPipe) {
|
||||
return true
|
||||
}
|
||||
// Some platforms return wrapped messages that don't map cleanly to sentinel errors.
|
||||
return strings.Contains(strings.ToLower(err.Error()), "file already closed")
|
||||
}
|
||||
|
||||
// LogSender is a function type for sending logs
|
||||
type LogSender func(taskID int, level types.LogLevel, message string, stepName string)
|
||||
|
||||
@@ -274,6 +361,9 @@ func RunCommandWithStreaming(
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil && !isBenignPipeReadError(err) {
|
||||
logSender(taskID, types.LogLevelWarn, fmt.Sprintf("stdout read error: %v", err), stepName)
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
@@ -288,6 +378,9 @@ func RunCommandWithStreaming(
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil && !isBenignPipeReadError(err) {
|
||||
logSender(taskID, types.LogLevelWarn, fmt.Sprintf("stderr read error: %v", err), stepName)
|
||||
}
|
||||
}()
|
||||
|
||||
err = cmd.Wait()
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package executils
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestIsBenignPipeReadError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{name: "nil", err: nil, want: false},
|
||||
{name: "eof", err: io.EOF, want: true},
|
||||
{name: "closed", err: os.ErrClosed, want: true},
|
||||
{name: "closed pipe", err: io.ErrClosedPipe, want: true},
|
||||
{name: "wrapped closed", err: errors.New("read |0: file already closed"), want: true},
|
||||
{name: "other", err: errors.New("permission denied"), want: false},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := isBenignPipeReadError(tc.err)
|
||||
if got != tc.want {
|
||||
t.Fatalf("got %v, want %v (err=%v)", got, tc.want, tc.err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessTracker_TrackUntrack(t *testing.T) {
|
||||
pt := NewProcessTracker()
|
||||
cmd := exec.Command("sh", "-c", "sleep 1")
|
||||
pt.Track(1, cmd)
|
||||
if count := pt.Count(); count != 1 {
|
||||
t.Fatalf("Count() = %d, want 1", count)
|
||||
}
|
||||
pt.Untrack(1)
|
||||
if count := pt.Count(); count != 0 {
|
||||
t.Fatalf("Count() = %d, want 0", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunCommandWithTimeout_TimesOut(t *testing.T) {
|
||||
pt := NewProcessTracker()
|
||||
_, err := RunCommandWithTimeout(200*time.Millisecond, "sh", []string{"-c", "sleep 2"}, "", nil, 99, pt)
|
||||
if err == nil {
|
||||
t.Fatal("expected timeout error")
|
||||
}
|
||||
}
|
||||
@@ -12,8 +12,9 @@ try:
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not make paths relative: {e}")
|
||||
|
||||
# Auto-enable addons from blender_addons folder in context
|
||||
# Supports .zip files (installed via Blender API) and already-extracted addons
|
||||
# Auto-install addons from blender_addons/ in the job context when present.
|
||||
# Users ship scene-required addons with the upload — no admin step and no job flag.
|
||||
# (Blender --enable-autoexec is separate: controlled by enable_execution on the job.)
|
||||
blend_dir = os.path.dirname(bpy.data.filepath) if bpy.data.filepath else os.getcwd()
|
||||
addons_dir = os.path.join(blend_dir, "blender_addons")
|
||||
|
||||
@@ -95,31 +96,20 @@ if current_device:
|
||||
print(f"Blend file output format: {current_output_format}")
|
||||
|
||||
# Override output format if specified
|
||||
# The format file always takes precedence (it's written specifically for this job)
|
||||
# Render output is EXR only and must remain linear for the encode pipeline (linear -> sRGB -> HLG).
|
||||
if output_format_override:
|
||||
print(f"Overriding output format from '{current_output_format}' to '{output_format_override}'")
|
||||
# Map common format names to Blender's format constants
|
||||
# For video formats, we render as appropriate frame format first
|
||||
format_to_use = output_format_override.upper()
|
||||
if format_to_use in ['EXR_264_MP4', 'EXR_AV1_MP4', 'EXR_VP9_WEBM']:
|
||||
format_to_use = 'EXR' # Render as EXR for EXR video formats
|
||||
|
||||
format_map = {
|
||||
'PNG': 'PNG',
|
||||
'JPEG': 'JPEG',
|
||||
'JPG': 'JPEG',
|
||||
'EXR': 'OPEN_EXR',
|
||||
'OPEN_EXR': 'OPEN_EXR',
|
||||
'TARGA': 'TARGA',
|
||||
'TIFF': 'TIFF',
|
||||
'BMP': 'BMP',
|
||||
}
|
||||
blender_format = format_map.get(format_to_use, format_to_use)
|
||||
print(f"Overriding output format from '{current_output_format}' to OPEN_EXR (always EXR for pipeline)")
|
||||
try:
|
||||
scene.render.image_settings.file_format = blender_format
|
||||
print(f"Successfully set output format to: {blender_format}")
|
||||
scene.render.image_settings.file_format = 'OPEN_EXR'
|
||||
# Lock output color space to linear (defense in depth; EXR is linear for encode pipeline)
|
||||
if getattr(scene.render.image_settings, 'has_linear_colorspace', False) and hasattr(scene.render.image_settings, 'linear_colorspace_settings'):
|
||||
try:
|
||||
scene.render.image_settings.linear_colorspace_settings.name = 'Linear'
|
||||
except Exception as ex:
|
||||
print(f"Note: Could not set linear output: {ex}")
|
||||
print("Successfully set output format to: OPEN_EXR")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not set output format to {blender_format}: {e}")
|
||||
print(f"Warning: Could not set output format to OPEN_EXR: {e}")
|
||||
print(f"Using blend file's format: {current_output_format}")
|
||||
else:
|
||||
print(f"Using blend file's output format: {current_output_format}")
|
||||
@@ -182,6 +172,34 @@ if render_settings_override:
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not set resolution_y: {e}")
|
||||
|
||||
# Runner sample batching overrides (gfx115x HIP driver workaround)
|
||||
if current_engine.upper() == 'CYCLES':
|
||||
cycles = scene.cycles
|
||||
if 'samples_override' in render_settings_override:
|
||||
try:
|
||||
cycles.samples = int(render_settings_override['samples_override'])
|
||||
print(f"Set Cycles.samples override = {cycles.samples}")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not set samples override: {e}")
|
||||
if 'seed_override' in render_settings_override:
|
||||
try:
|
||||
cycles.seed = int(render_settings_override['seed_override'])
|
||||
print(f"Set Cycles.seed override = {cycles.seed}")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not set seed override: {e}")
|
||||
if 'tile_size_override' in render_settings_override:
|
||||
try:
|
||||
cycles.tile_size = int(render_settings_override['tile_size_override'])
|
||||
print(f"Set Cycles.tile_size override = {cycles.tile_size}")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not set tile_size override: {e}")
|
||||
if render_settings_override.get('use_auto_tile_override'):
|
||||
try:
|
||||
cycles.use_auto_tile = True
|
||||
print(f"Set Cycles.use_auto_tile override = True")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not set use_auto_tile override: {e}")
|
||||
|
||||
# Only override device selection if using Cycles (other engines handle GPU differently)
|
||||
if current_engine == 'CYCLES':
|
||||
# Check if CPU rendering is forced
|
||||
@@ -220,9 +238,19 @@ if current_engine == 'CYCLES':
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
# Check all devices and choose the best GPU type
|
||||
# Device type preference order (most performant first)
|
||||
device_type_preference = ['OPTIX', 'CUDA', 'HIP', 'ONEAPI', 'METAL']
|
||||
# Check all devices and choose the best GPU type.
|
||||
# Explicit fallback policy: NVIDIA -> Intel -> AMD -> CPU.
|
||||
# (OPTIX/CUDA are NVIDIA, ONEAPI is Intel, HIP/OPENCL are AMD)
|
||||
disable_rt = False
|
||||
if render_settings_override:
|
||||
disable_rt = bool(
|
||||
render_settings_override.get('disable_rt', render_settings_override.get('disable_hiprt', False))
|
||||
)
|
||||
if disable_rt:
|
||||
# Prefer non-RT backends when GPU ray tracing acceleration is disabled.
|
||||
device_type_preference = ['CUDA', 'ONEAPI', 'HIP', 'OPENCL', 'OPTIX']
|
||||
else:
|
||||
device_type_preference = ['OPTIX', 'CUDA', 'ONEAPI', 'HIP', 'OPENCL']
|
||||
gpu_available = False
|
||||
best_device_type = None
|
||||
best_gpu_devices = []
|
||||
@@ -328,50 +356,42 @@ if current_engine == 'CYCLES':
|
||||
except Exception as e:
|
||||
print(f" Warning: Could not enable device {getattr(device, 'name', 'Unknown')}: {e}")
|
||||
|
||||
# Enable ray tracing acceleration for supported device types
|
||||
# Configure GPU ray tracing acceleration for supported device types
|
||||
try:
|
||||
if best_device_type == 'HIP':
|
||||
# HIPRT (HIP Ray Tracing) for AMD GPUs
|
||||
if disable_rt:
|
||||
if hasattr(cycles_prefs, 'use_hiprt'):
|
||||
cycles_prefs.use_hiprt = False
|
||||
if hasattr(scene.cycles, 'use_hiprt'):
|
||||
scene.cycles.use_hiprt = False
|
||||
if hasattr(scene.cycles, 'use_optix_denoising'):
|
||||
scene.cycles.use_optix_denoising = False
|
||||
print(" GPU ray tracing acceleration disabled (--disable-rt)")
|
||||
elif best_device_type == 'HIP':
|
||||
if hasattr(cycles_prefs, 'use_hiprt'):
|
||||
cycles_prefs.use_hiprt = True
|
||||
print(f" Enabled HIPRT (HIP Ray Tracing) for faster rendering")
|
||||
print(" Enabled HIPRT (HIP Ray Tracing) for faster rendering")
|
||||
elif hasattr(scene.cycles, 'use_hiprt'):
|
||||
scene.cycles.use_hiprt = True
|
||||
print(f" Enabled HIPRT (HIP Ray Tracing) for faster rendering")
|
||||
print(" Enabled HIPRT (HIP Ray Tracing) for faster rendering")
|
||||
else:
|
||||
print(f" HIPRT not available (requires Blender 4.0+)")
|
||||
print(" HIPRT not available (requires Blender 4.0+)")
|
||||
elif best_device_type == 'OPTIX':
|
||||
# OptiX is already enabled when using OPTIX device type
|
||||
# But we can check if there are any OptiX-specific settings
|
||||
if hasattr(scene.cycles, 'use_optix_denoising'):
|
||||
scene.cycles.use_optix_denoising = True
|
||||
print(f" Enabled OptiX denoising")
|
||||
print(f" OptiX ray tracing is active (using OPTIX device type)")
|
||||
print(" Enabled OptiX denoising")
|
||||
print(" OptiX ray tracing is active (using OPTIX device type)")
|
||||
elif best_device_type == 'CUDA':
|
||||
# CUDA can use OptiX if available, but it's usually automatic
|
||||
# Check if we can prefer OptiX over CUDA
|
||||
if hasattr(scene.cycles, 'use_optix_denoising'):
|
||||
scene.cycles.use_optix_denoising = True
|
||||
print(f" Enabled OptiX denoising (if OptiX available)")
|
||||
print(f" CUDA ray tracing active")
|
||||
elif best_device_type == 'METAL':
|
||||
# MetalRT for Apple Silicon (if available)
|
||||
if hasattr(scene.cycles, 'use_metalrt'):
|
||||
scene.cycles.use_metalrt = True
|
||||
print(f" Enabled MetalRT (Metal Ray Tracing) for faster rendering")
|
||||
elif hasattr(cycles_prefs, 'use_metalrt'):
|
||||
cycles_prefs.use_metalrt = True
|
||||
print(f" Enabled MetalRT (Metal Ray Tracing) for faster rendering")
|
||||
else:
|
||||
print(f" MetalRT not available")
|
||||
print(" Enabled OptiX denoising (if OptiX available)")
|
||||
print(" CUDA ray tracing active")
|
||||
elif best_device_type == 'ONEAPI':
|
||||
# Intel oneAPI - Embree might be available
|
||||
if hasattr(scene.cycles, 'use_embree'):
|
||||
scene.cycles.use_embree = True
|
||||
print(f" Enabled Embree for faster CPU ray tracing")
|
||||
print(f" oneAPI ray tracing active")
|
||||
print(" Enabled Embree for faster CPU ray tracing")
|
||||
print(" oneAPI ray tracing active")
|
||||
except Exception as e:
|
||||
print(f" Could not enable ray tracing acceleration: {e}")
|
||||
print(f" Could not configure ray tracing acceleration: {e}")
|
||||
|
||||
print(f"SUCCESS: Enabled {enabled_count} GPU device(s) for {best_device_type}")
|
||||
gpu_available = True
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package scripts
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEmbeddedScripts_ArePresent(t *testing.T) {
|
||||
if strings.TrimSpace(ExtractMetadata) == "" {
|
||||
t.Fatal("ExtractMetadata script should not be empty")
|
||||
}
|
||||
if strings.TrimSpace(UnhideObjects) == "" {
|
||||
t.Fatal("UnhideObjects script should not be empty")
|
||||
}
|
||||
if strings.TrimSpace(RenderBlenderTemplate) == "" {
|
||||
t.Fatal("RenderBlenderTemplate should not be empty")
|
||||
}
|
||||
}
|
||||
|
||||
+4
-7
@@ -93,7 +93,8 @@ type Task struct {
|
||||
ID int64 `json:"id"`
|
||||
JobID int64 `json:"job_id"`
|
||||
RunnerID *int64 `json:"runner_id,omitempty"`
|
||||
Frame int `json:"frame"`
|
||||
Frame int `json:"frame"` // frame start (inclusive) for render tasks
|
||||
FrameEnd *int `json:"frame_end,omitempty"` // frame end (inclusive); nil = single frame
|
||||
TaskType TaskType `json:"task_type"`
|
||||
Status TaskStatus `json:"status"`
|
||||
CurrentStep string `json:"current_step,omitempty"`
|
||||
@@ -136,10 +137,8 @@ type CreateJobRequest struct {
|
||||
RenderSettings *RenderSettings `json:"render_settings,omitempty"` // Optional: Override blend file render settings
|
||||
UploadSessionID *string `json:"upload_session_id,omitempty"` // Optional: Session ID from file upload
|
||||
UnhideObjects *bool `json:"unhide_objects,omitempty"` // Optional: Enable unhide tweaks for objects/collections
|
||||
EnableExecution *bool `json:"enable_execution,omitempty"` // Optional: Enable auto-execution in Blender (adds --enable-autoexec flag, defaults to false)
|
||||
EnableExecution *bool `json:"enable_execution,omitempty"` // Optional: allow Python autoexec inside the .blend (--enable-autoexec). Job blender_addons/ always install when present.
|
||||
BlenderVersion *string `json:"blender_version,omitempty"` // Optional: Override Blender version (e.g., "4.2" or "4.2.3")
|
||||
PreserveHDR *bool `json:"preserve_hdr,omitempty"` // Optional: Preserve HDR range for EXR encoding (uses HLG with bt709 primaries)
|
||||
PreserveAlpha *bool `json:"preserve_alpha,omitempty"` // Optional: Preserve alpha channel for EXR encoding (requires AV1 or VP9 codec)
|
||||
}
|
||||
|
||||
// UpdateJobProgressRequest represents a request to update job progress
|
||||
@@ -232,10 +231,8 @@ type BlendMetadata struct {
|
||||
SceneInfo SceneInfo `json:"scene_info"`
|
||||
MissingFilesInfo *MissingFilesInfo `json:"missing_files_info,omitempty"`
|
||||
UnhideObjects *bool `json:"unhide_objects,omitempty"` // Enable unhide tweaks for objects/collections
|
||||
EnableExecution *bool `json:"enable_execution,omitempty"` // Enable auto-execution in Blender (adds --enable-autoexec flag, defaults to false)
|
||||
EnableExecution *bool `json:"enable_execution,omitempty"` // Allow .blend Python autoexec (--enable-autoexec); separate from blender_addons/ install
|
||||
BlenderVersion string `json:"blender_version,omitempty"` // Detected or overridden Blender version (e.g., "4.2" or "4.2.3")
|
||||
PreserveHDR *bool `json:"preserve_hdr,omitempty"` // Preserve HDR range for EXR encoding (uses HLG with bt709 primaries)
|
||||
PreserveAlpha *bool `json:"preserve_alpha,omitempty"` // Preserve alpha channel for EXR encoding (requires AV1 or VP9 codec)
|
||||
}
|
||||
|
||||
// MissingFilesInfo represents information about missing files/addons
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package types
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestJobJSON_RoundTrip(t *testing.T) {
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
frameStart, frameEnd := 1, 10
|
||||
format := "PNG"
|
||||
job := Job{
|
||||
ID: 42,
|
||||
UserID: 7,
|
||||
JobType: JobTypeRender,
|
||||
Name: "demo",
|
||||
Status: JobStatusPending,
|
||||
Progress: 12.5,
|
||||
FrameStart: &frameStart,
|
||||
FrameEnd: &frameEnd,
|
||||
OutputFormat: &format,
|
||||
BlendMetadata: &BlendMetadata{
|
||||
FrameStart: 1,
|
||||
FrameEnd: 10,
|
||||
RenderSettings: RenderSettings{
|
||||
ResolutionX: 1920,
|
||||
ResolutionY: 1080,
|
||||
FrameRate: 24.0,
|
||||
},
|
||||
},
|
||||
CreatedAt: now,
|
||||
}
|
||||
|
||||
raw, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal failed: %v", err)
|
||||
}
|
||||
|
||||
var out Job
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
t.Fatalf("unmarshal failed: %v", err)
|
||||
}
|
||||
|
||||
if out.ID != job.ID || out.JobType != JobTypeRender || out.BlendMetadata == nil {
|
||||
t.Fatalf("unexpected roundtrip result: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package version
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestInitDefaults_AreSet(t *testing.T) {
|
||||
if Version == "" {
|
||||
t.Fatal("Version should be initialized")
|
||||
}
|
||||
if Date == "" {
|
||||
t.Fatal("Date should be initialized")
|
||||
}
|
||||
if !strings.Contains(Version, ".") {
|
||||
t.Fatalf("Version should look semantic-ish, got %q", Version)
|
||||
}
|
||||
}
|
||||
|
||||
-269
@@ -1,269 +0,0 @@
|
||||
const API_BASE = '/api';
|
||||
|
||||
let currentUser = null;
|
||||
|
||||
// Check authentication on load
|
||||
async function init() {
|
||||
await checkAuth();
|
||||
setupEventListeners();
|
||||
if (currentUser) {
|
||||
showMainPage();
|
||||
loadJobs();
|
||||
loadRunners();
|
||||
} else {
|
||||
showLoginPage();
|
||||
}
|
||||
}
|
||||
|
||||
async function checkAuth() {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/auth/me`);
|
||||
if (response.ok) {
|
||||
currentUser = await response.json();
|
||||
return true;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Auth check failed:', error);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function showLoginPage() {
|
||||
document.getElementById('login-page').classList.remove('hidden');
|
||||
document.getElementById('main-page').classList.add('hidden');
|
||||
}
|
||||
|
||||
function showMainPage() {
|
||||
document.getElementById('login-page').classList.add('hidden');
|
||||
document.getElementById('main-page').classList.remove('hidden');
|
||||
if (currentUser) {
|
||||
document.getElementById('user-name').textContent = currentUser.name || currentUser.email;
|
||||
}
|
||||
}
|
||||
|
||||
function setupEventListeners() {
|
||||
// Navigation
|
||||
document.querySelectorAll('.nav-btn').forEach(btn => {
|
||||
btn.addEventListener('click', (e) => {
|
||||
const page = e.target.dataset.page;
|
||||
switchPage(page);
|
||||
});
|
||||
});
|
||||
|
||||
// Logout
|
||||
document.getElementById('logout-btn').addEventListener('click', async () => {
|
||||
await fetch(`${API_BASE}/auth/logout`, { method: 'POST' });
|
||||
currentUser = null;
|
||||
showLoginPage();
|
||||
});
|
||||
|
||||
// Job form
|
||||
document.getElementById('job-form').addEventListener('submit', async (e) => {
|
||||
e.preventDefault();
|
||||
await submitJob();
|
||||
});
|
||||
}
|
||||
|
||||
function switchPage(page) {
|
||||
document.querySelectorAll('.content-page').forEach(p => p.classList.add('hidden'));
|
||||
document.querySelectorAll('.nav-btn').forEach(b => b.classList.remove('active'));
|
||||
|
||||
document.getElementById(`${page}-page`).classList.remove('hidden');
|
||||
document.querySelector(`[data-page="${page}"]`).classList.add('active');
|
||||
|
||||
if (page === 'jobs') {
|
||||
loadJobs();
|
||||
} else if (page === 'runners') {
|
||||
loadRunners();
|
||||
}
|
||||
}
|
||||
|
||||
async function submitJob() {
|
||||
const form = document.getElementById('job-form');
|
||||
const formData = new FormData(form);
|
||||
|
||||
const jobData = {
|
||||
name: document.getElementById('job-name').value,
|
||||
frame_start: parseInt(document.getElementById('frame-start').value),
|
||||
frame_end: parseInt(document.getElementById('frame-end').value),
|
||||
output_format: document.getElementById('output-format').value,
|
||||
};
|
||||
|
||||
try {
|
||||
// Create job
|
||||
const jobResponse = await fetch(`${API_BASE}/jobs`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(jobData),
|
||||
});
|
||||
|
||||
if (!jobResponse.ok) {
|
||||
throw new Error('Failed to create job');
|
||||
}
|
||||
|
||||
const job = await jobResponse.json();
|
||||
|
||||
// Upload file
|
||||
const fileInput = document.getElementById('blend-file');
|
||||
if (fileInput.files.length > 0) {
|
||||
const fileFormData = new FormData();
|
||||
fileFormData.append('file', fileInput.files[0]);
|
||||
|
||||
const fileResponse = await fetch(`${API_BASE}/jobs/${job.id}/upload`, {
|
||||
method: 'POST',
|
||||
body: fileFormData,
|
||||
});
|
||||
|
||||
if (!fileResponse.ok) {
|
||||
throw new Error('Failed to upload file');
|
||||
}
|
||||
}
|
||||
|
||||
alert('Job submitted successfully!');
|
||||
form.reset();
|
||||
switchPage('jobs');
|
||||
loadJobs();
|
||||
} catch (error) {
|
||||
alert('Failed to submit job: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadJobs() {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/jobs`);
|
||||
if (!response.ok) throw new Error('Failed to load jobs');
|
||||
|
||||
const jobs = await response.json();
|
||||
displayJobs(jobs);
|
||||
} catch (error) {
|
||||
console.error('Failed to load jobs:', error);
|
||||
}
|
||||
}
|
||||
|
||||
function displayJobs(jobs) {
|
||||
const container = document.getElementById('jobs-list');
|
||||
if (jobs.length === 0) {
|
||||
container.innerHTML = '<p>No jobs yet. Submit a job to get started!</p>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = jobs.map(job => `
|
||||
<div class="job-card">
|
||||
<h3>${escapeHtml(job.name)}</h3>
|
||||
<div class="job-meta">
|
||||
<span>Frames: ${job.frame_start}-${job.frame_end}</span>
|
||||
<span>Format: ${job.output_format}</span>
|
||||
<span>Created: ${new Date(job.created_at).toLocaleString()}</span>
|
||||
</div>
|
||||
<div class="job-status ${job.status}">${job.status}</div>
|
||||
<div class="progress-bar">
|
||||
<div class="progress-fill" style="width: ${job.progress}%"></div>
|
||||
</div>
|
||||
<div class="job-actions">
|
||||
<button onclick="viewJob(${job.id})" class="btn btn-primary">View Details</button>
|
||||
${job.status === 'pending' || job.status === 'running' ?
|
||||
`<button onclick="cancelJob(${job.id})" class="btn btn-secondary">Cancel</button>` : ''}
|
||||
</div>
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
async function viewJob(jobId) {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/jobs/${jobId}`);
|
||||
if (!response.ok) throw new Error('Failed to load job');
|
||||
|
||||
const job = await response.json();
|
||||
|
||||
// Load files
|
||||
const filesResponse = await fetch(`${API_BASE}/jobs/${jobId}/files`);
|
||||
const files = filesResponse.ok ? await filesResponse.json() : [];
|
||||
|
||||
const outputFiles = files.filter(f => f.file_type === 'output');
|
||||
if (outputFiles.length > 0) {
|
||||
let message = 'Output files:\n';
|
||||
outputFiles.forEach(file => {
|
||||
message += `- ${file.file_name}\n`;
|
||||
});
|
||||
message += '\nWould you like to download them?';
|
||||
if (confirm(message)) {
|
||||
outputFiles.forEach(file => {
|
||||
window.open(`${API_BASE}/jobs/${jobId}/files/${file.id}/download`, '_blank');
|
||||
});
|
||||
}
|
||||
} else {
|
||||
alert(`Job: ${job.name}\nStatus: ${job.status}\nProgress: ${job.progress.toFixed(1)}%`);
|
||||
}
|
||||
} catch (error) {
|
||||
alert('Failed to load job details: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function cancelJob(jobId) {
|
||||
if (!confirm('Are you sure you want to cancel this job?')) return;
|
||||
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/jobs/${jobId}`, {
|
||||
method: 'DELETE',
|
||||
});
|
||||
if (!response.ok) throw new Error('Failed to cancel job');
|
||||
loadJobs();
|
||||
} catch (error) {
|
||||
alert('Failed to cancel job: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadRunners() {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/runners`);
|
||||
if (!response.ok) throw new Error('Failed to load runners');
|
||||
|
||||
const runners = await response.json();
|
||||
displayRunners(runners);
|
||||
} catch (error) {
|
||||
console.error('Failed to load runners:', error);
|
||||
}
|
||||
}
|
||||
|
||||
function displayRunners(runners) {
|
||||
const container = document.getElementById('runners-list');
|
||||
if (runners.length === 0) {
|
||||
container.innerHTML = '<p>No runners connected.</p>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = runners.map(runner => {
|
||||
const lastHeartbeat = new Date(runner.last_heartbeat);
|
||||
const isOnline = (Date.now() - lastHeartbeat.getTime()) < 60000; // 1 minute
|
||||
|
||||
return `
|
||||
<div class="runner-card">
|
||||
<h3>${escapeHtml(runner.name)}</h3>
|
||||
<div class="runner-info">
|
||||
<span>Hostname: ${escapeHtml(runner.hostname)}</span>
|
||||
<span>Last heartbeat: ${lastHeartbeat.toLocaleString()}</span>
|
||||
</div>
|
||||
<div class="runner-status ${isOnline ? 'online' : 'offline'}">
|
||||
${isOnline ? 'Online' : 'Offline'}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function escapeHtml(text) {
|
||||
const div = document.createElement('div');
|
||||
div.textContent = text;
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
// Auto-refresh jobs every 5 seconds
|
||||
setInterval(() => {
|
||||
if (currentUser && document.getElementById('jobs-page').classList.contains('hidden') === false) {
|
||||
loadJobs();
|
||||
}
|
||||
}, 5000);
|
||||
|
||||
// Initialize on load
|
||||
init();
|
||||
|
||||
+12
-28
@@ -4,42 +4,26 @@ import (
|
||||
"embed"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
//go:embed dist/*
|
||||
var distFS embed.FS
|
||||
//go:embed templates templates/partials static
|
||||
var uiFS embed.FS
|
||||
|
||||
// GetFileSystem returns an http.FileSystem for the embedded web UI files
|
||||
func GetFileSystem() http.FileSystem {
|
||||
subFS, err := fs.Sub(distFS, "dist")
|
||||
// GetStaticFileSystem returns an http.FileSystem for embedded UI assets.
|
||||
func GetStaticFileSystem() http.FileSystem {
|
||||
subFS, err := fs.Sub(uiFS, "static")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return http.FS(subFS)
|
||||
}
|
||||
|
||||
// SPAHandler returns an http.Handler that serves the embedded SPA
|
||||
// It serves static files if they exist, otherwise falls back to index.html
|
||||
func SPAHandler() http.Handler {
|
||||
fsys := GetFileSystem()
|
||||
fileServer := http.FileServer(fsys)
|
||||
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := r.URL.Path
|
||||
|
||||
// Try to open the file
|
||||
f, err := fsys.Open(strings.TrimPrefix(path, "/"))
|
||||
if err != nil {
|
||||
// File doesn't exist, serve index.html for SPA routing
|
||||
r.URL.Path = "/"
|
||||
fileServer.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
f.Close()
|
||||
|
||||
// File exists, serve it
|
||||
fileServer.ServeHTTP(w, r)
|
||||
})
|
||||
// StaticHandler serves /assets/* files from embedded static assets.
|
||||
func StaticHandler() http.Handler {
|
||||
return http.StripPrefix("/assets/", http.FileServer(GetStaticFileSystem()))
|
||||
}
|
||||
|
||||
// GetTemplateFS returns the embedded template filesystem.
|
||||
func GetTemplateFS() fs.FS {
|
||||
return uiFS
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGetStaticFileSystem_NonNil(t *testing.T) {
|
||||
fs := GetStaticFileSystem()
|
||||
if fs == nil {
|
||||
t.Fatal("static filesystem should not be nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaticHandler_ServesWithoutPanic(t *testing.T) {
|
||||
h := StaticHandler()
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest("GET", "/assets/does-not-exist.txt", nil)
|
||||
h.ServeHTTP(rr, req)
|
||||
if rr.Code == 0 {
|
||||
t.Fatal("handler should write a status code")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTemplateFS_NonNil(t *testing.T) {
|
||||
if GetTemplateFS() == nil {
|
||||
t.Fatal("template fs should not be nil")
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user