Compare commits
12 Commits
94490237fe
...
0.0.3
| Author | SHA1 | Date | |
|---|---|---|---|
| d3c5ee0dba | |||
| bb57ce8659 | |||
| 1a8836e6aa | |||
| b51b96a618 | |||
| 8e561922c9 | |||
| 1c4bd78f56 | |||
| 3f2982ddb3 | |||
| 0b852c5087 | |||
| 5e56c7f0e8 | |||
| 0a8f40b9cb | |||
| 7440511740 | |||
| c7c8762164 |
24
.gitea/workflows/release-tag.yaml
Normal file
24
.gitea/workflows/release-tag.yaml
Normal file
@@ -0,0 +1,24 @@
|
|||||||
|
name: Release Tag
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
tags:
|
||||||
|
- '*'
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
release:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@main
|
||||||
|
with:
|
||||||
|
fetch-depth: 0
|
||||||
|
- run: git fetch --force --tags
|
||||||
|
- uses: actions/setup-go@main
|
||||||
|
with:
|
||||||
|
go-version-file: 'go.mod'
|
||||||
|
- uses: goreleaser/goreleaser-action@master
|
||||||
|
with:
|
||||||
|
distribution: goreleaser
|
||||||
|
version: 'latest'
|
||||||
|
args: release
|
||||||
|
env:
|
||||||
|
GITEA_TOKEN: ${{secrets.RELEASE_TOKEN}}
|
||||||
17
.gitea/workflows/test-pr.yaml
Normal file
17
.gitea/workflows/test-pr.yaml
Normal file
@@ -0,0 +1,17 @@
|
|||||||
|
name: PR Check
|
||||||
|
on:
|
||||||
|
- pull_request
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
check-and-test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@main
|
||||||
|
- uses: actions/setup-go@main
|
||||||
|
with:
|
||||||
|
go-version-file: 'go.mod'
|
||||||
|
- uses: FedericoCarboni/setup-ffmpeg@v3
|
||||||
|
- run: go mod tidy
|
||||||
|
- run: cd web && npm install && npm run build
|
||||||
|
- run: go build ./...
|
||||||
|
- run: go test -race -v -shuffle=on ./...
|
||||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -65,6 +65,7 @@ lerna-debug.log*
|
|||||||
*.o
|
*.o
|
||||||
*.a
|
*.a
|
||||||
*.so
|
*.so
|
||||||
|
/dist/
|
||||||
|
|
||||||
# Temporary files
|
# Temporary files
|
||||||
*.tmp
|
*.tmp
|
||||||
|
|||||||
48
.goreleaser.yaml
Normal file
48
.goreleaser.yaml
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
version: 2
|
||||||
|
|
||||||
|
before:
|
||||||
|
hooks:
|
||||||
|
- go mod tidy -v
|
||||||
|
- sh -c "cd web && npm install && npm run build"
|
||||||
|
|
||||||
|
builds:
|
||||||
|
- id: default
|
||||||
|
main: ./cmd/jiggablend
|
||||||
|
binary: jiggablend
|
||||||
|
ldflags:
|
||||||
|
- -X jiggablend/version.Version={{.Version}}
|
||||||
|
- -X jiggablend/version.Date={{.Date}}
|
||||||
|
env:
|
||||||
|
- CGO_ENABLED=1
|
||||||
|
goos:
|
||||||
|
- linux
|
||||||
|
goarch:
|
||||||
|
- amd64
|
||||||
|
|
||||||
|
checksum:
|
||||||
|
name_template: "checksums.txt"
|
||||||
|
|
||||||
|
archives:
|
||||||
|
- id: default
|
||||||
|
name_template: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}"
|
||||||
|
formats: tar.gz
|
||||||
|
format_overrides:
|
||||||
|
- goos: windows
|
||||||
|
formats: zip
|
||||||
|
files:
|
||||||
|
- README.md
|
||||||
|
- LICENSE
|
||||||
|
|
||||||
|
changelog:
|
||||||
|
sort: asc
|
||||||
|
filters:
|
||||||
|
exclude:
|
||||||
|
- "^docs:"
|
||||||
|
- "^test:"
|
||||||
|
|
||||||
|
release:
|
||||||
|
name_template: "{{ .ProjectName }}-{{ .Version }}"
|
||||||
|
|
||||||
|
gitea_urls:
|
||||||
|
api: https://git.s1d3sw1ped.com/api/v1
|
||||||
|
download: https://git.s1d3sw1ped.com
|
||||||
21
LICENSE
Normal file
21
LICENSE
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
The MIT License (MIT)
|
||||||
|
|
||||||
|
Copyright © 2026 s1d3sw1ped
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in
|
||||||
|
all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||||
|
THE SOFTWARE.
|
||||||
50
Makefile
50
Makefile
@@ -1,12 +1,11 @@
|
|||||||
.PHONY: build build-web run run-manager run-runner cleanup cleanup-manager cleanup-runner kill-all clean-bin clean-web test help install
|
.PHONY: build build-web run run-manager run-runner cleanup cleanup-manager cleanup-runner clean-bin clean-web test help install
|
||||||
|
|
||||||
# Build the jiggablend binary (includes embedded web UI)
|
# Build the jiggablend binary (includes embedded web UI)
|
||||||
build: clean-bin build-web
|
build:
|
||||||
go build -o bin/jiggablend ./cmd/jiggablend
|
@echo "Building with GoReleaser..."
|
||||||
|
goreleaser build --clean --snapshot --single-target
|
||||||
# Build for Linux (cross-compile)
|
@mkdir -p bin
|
||||||
build-linux: clean-bin build-web
|
@find dist -name jiggablend -type f -exec cp {} bin/jiggablend \;
|
||||||
GOOS=linux GOARCH=amd64 go build -o bin/jiggablend ./cmd/jiggablend
|
|
||||||
|
|
||||||
# Build web UI
|
# Build web UI
|
||||||
build-web: clean-web
|
build-web: clean-web
|
||||||
@@ -27,10 +26,6 @@ cleanup-runner:
|
|||||||
# Cleanup both manager and runner logs
|
# Cleanup both manager and runner logs
|
||||||
cleanup: cleanup-manager cleanup-runner
|
cleanup: cleanup-manager cleanup-runner
|
||||||
|
|
||||||
# Kill all jiggablend processes
|
|
||||||
kill-all:
|
|
||||||
@echo "Not implemented"
|
|
||||||
|
|
||||||
# Run manager and runner in parallel (for testing)
|
# Run manager and runner in parallel (for testing)
|
||||||
run: cleanup build init-test
|
run: cleanup build init-test
|
||||||
@echo "Starting manager and runner in parallel..."
|
@echo "Starting manager and runner in parallel..."
|
||||||
@@ -74,37 +69,30 @@ clean-web:
|
|||||||
test:
|
test:
|
||||||
go test ./... -timeout 30s
|
go test ./... -timeout 30s
|
||||||
|
|
||||||
# Install to /usr/local/bin
|
|
||||||
install: build
|
|
||||||
sudo cp bin/jiggablend /usr/local/bin/
|
|
||||||
|
|
||||||
# Show help
|
# Show help
|
||||||
help:
|
help:
|
||||||
@echo "Jiggablend Build and Run Makefile"
|
@echo "Jiggablend Build and Run Makefile"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "Build targets:"
|
@echo "Build targets:"
|
||||||
@echo " build - Build jiggablend binary with embedded web UI"
|
@echo " build - Build jiggablend binary with embedded web UI"
|
||||||
@echo " build-linux - Cross-compile for Linux amd64"
|
@echo " build-web - Build web UI only"
|
||||||
@echo " build-web - Build web UI only"
|
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "Run targets:"
|
@echo "Run targets:"
|
||||||
@echo " run - Run manager and runner in parallel (for testing)"
|
@echo " run - Run manager and runner in parallel (for testing)"
|
||||||
@echo " run-manager - Run manager server"
|
@echo " run-manager - Run manager server"
|
||||||
@echo " run-runner - Run runner with test API key"
|
@echo " run-runner - Run runner with test API key"
|
||||||
@echo " init-test - Initialize test configuration (run once)"
|
@echo " init-test - Initialize test configuration (run once)"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "Cleanup targets:"
|
@echo "Cleanup targets:"
|
||||||
@echo " cleanup - Clean all logs"
|
@echo " cleanup - Clean all logs"
|
||||||
@echo " cleanup-manager - Clean manager logs"
|
@echo " cleanup-manager - Clean manager logs"
|
||||||
@echo " cleanup-runner - Clean runner logs"
|
@echo " cleanup-runner - Clean runner logs"
|
||||||
@echo " kill-all - Kill all running jiggablend processes"
|
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "Other targets:"
|
@echo "Other targets:"
|
||||||
@echo " clean-bin - Clean build artifacts"
|
@echo " clean-bin - Clean build artifacts"
|
||||||
@echo " clean-web - Clean web build artifacts"
|
@echo " clean-web - Clean web build artifacts"
|
||||||
@echo " test - Run Go tests"
|
@echo " test - Run Go tests"
|
||||||
@echo " install - Install to /usr/local/bin"
|
@echo " help - Show this help"
|
||||||
@echo " help - Show this help"
|
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "CLI Usage:"
|
@echo "CLI Usage:"
|
||||||
@echo " jiggablend manager serve - Start the manager server"
|
@echo " jiggablend manager serve - Start the manager server"
|
||||||
|
|||||||
266
README.md
266
README.md
@@ -4,28 +4,41 @@ A distributed Blender render farm system built with Go. The system consists of a
|
|||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
- **Manager**: Central server with REST API, web UI, DuckDB database, and local file storage
|
- **Manager**: Central server with REST API, embedded web UI, SQLite database, and local file storage
|
||||||
- **Runner**: Linux amd64 client that connects to manager, receives jobs, executes Blender renders, and reports back
|
- **Runner**: Linux amd64 client that connects to manager, receives jobs, executes Blender renders, and reports back
|
||||||
|
|
||||||
|
Both manager and runner are part of a single binary (`jiggablend`) with subcommands.
|
||||||
|
|
||||||
## Features
|
## Features
|
||||||
|
|
||||||
- OAuth authentication (Google and Discord)
|
- **Authentication**: OAuth (Google and Discord) and local authentication with user management
|
||||||
- Web-based job submission and monitoring
|
- **Web UI**: Modern React-based interface for job submission and monitoring
|
||||||
- Distributed rendering across multiple runners
|
- **Distributed Rendering**: Scale across multiple runners with automatic job distribution
|
||||||
- Real-time job progress tracking
|
- **Real-time Updates**: WebSocket-based progress tracking and job status updates
|
||||||
- File upload/download for Blender files and rendered outputs
|
- **Video Encoding**: Automatic video encoding from EXR/PNG sequences with multiple codec support:
|
||||||
- Runner health monitoring
|
- H.264 (MP4) - SDR and HDR support
|
||||||
|
- AV1 (MP4) - With alpha channel support
|
||||||
|
- VP9 (WebM) - With alpha channel and HDR support
|
||||||
|
- **Output Formats**: PNG, JPEG, EXR, and video formats (MP4, WebM)
|
||||||
|
- **Blender Version Management**: Support for multiple Blender versions with automatic detection
|
||||||
|
- **Metadata Extraction**: Automatic extraction of scene metadata from Blender files
|
||||||
|
- **Admin Panel**: User and runner management interface
|
||||||
|
- **Runner Management**: API key-based authentication for runners with health monitoring
|
||||||
|
- **HDR Support**: Preserve HDR range in video encoding with HLG transfer function
|
||||||
|
- **Alpha Channel**: Preserve alpha channel in video encoding (AV1 and VP9)
|
||||||
|
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
### Manager
|
### Manager
|
||||||
- Go 1.21 or later
|
- Go 1.25.4 or later
|
||||||
- DuckDB (via Go driver)
|
- SQLite (via Go driver)
|
||||||
|
- Blender installed and in PATH (for metadata extraction)
|
||||||
|
- ImageMagick installed (for EXR preview conversion)
|
||||||
|
|
||||||
### Runner
|
### Runner
|
||||||
- Linux amd64
|
- Linux amd64
|
||||||
- Blender installed and in PATH
|
- Blender installed (can use bundled versions from storage)
|
||||||
- FFmpeg installed (optional, for video processing)
|
- FFmpeg installed (required for video encoding)
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
@@ -44,44 +57,87 @@ go mod download
|
|||||||
|
|
||||||
### Manager
|
### Manager
|
||||||
|
|
||||||
Set the following environment variables for authentication (optional):
|
Configuration is managed through the CLI using `jiggablend manager config` commands. The configuration is stored in the SQLite database.
|
||||||
|
|
||||||
|
#### Initial Setup
|
||||||
|
|
||||||
|
For testing, use the Makefile helper:
|
||||||
|
```bash
|
||||||
|
make init-test
|
||||||
|
```
|
||||||
|
|
||||||
|
This will:
|
||||||
|
- Enable local authentication
|
||||||
|
- Set a fixed API key for testing
|
||||||
|
- Create a test admin user (test@example.com / testpassword)
|
||||||
|
|
||||||
|
#### Manual Configuration
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# OAuth Providers (optional)
|
# Enable local authentication
|
||||||
export GOOGLE_CLIENT_ID="your-google-client-id"
|
jiggablend manager config enable localauth
|
||||||
export GOOGLE_CLIENT_SECRET="your-google-client-secret"
|
|
||||||
export GOOGLE_REDIRECT_URL="http://localhost:8080/api/auth/google/callback"
|
|
||||||
|
|
||||||
export DISCORD_CLIENT_ID="your-discord-client-id"
|
# Add a user
|
||||||
export DISCORD_CLIENT_SECRET="your-discord-client-secret"
|
jiggablend manager config add user <email> <password> --admin
|
||||||
export DISCORD_REDIRECT_URL="http://localhost:8080/api/auth/discord/callback"
|
|
||||||
|
|
||||||
# Local Authentication (optional)
|
# Generate an API key for runners
|
||||||
export ENABLE_LOCAL_AUTH="true"
|
jiggablend manager config add apikey <name> --scope manager
|
||||||
|
|
||||||
# Test User (optional, for testing only)
|
# Set OAuth credentials
|
||||||
# Creates a local user on startup if it doesn't exist
|
jiggablend manager config set google-oauth <client-id> <client-secret> --redirect-url <url>
|
||||||
export LOCAL_TEST_EMAIL="test@example.com"
|
jiggablend manager config set discord-oauth <client-id> <client-secret> --redirect-url <url>
|
||||||
export LOCAL_TEST_PASSWORD="testpassword"
|
|
||||||
|
# View current configuration
|
||||||
|
jiggablend manager config show
|
||||||
|
|
||||||
|
# List users and API keys
|
||||||
|
jiggablend manager config list users
|
||||||
|
jiggablend manager config list apikeys
|
||||||
```
|
```
|
||||||
|
|
||||||
|
#### Environment Variables
|
||||||
|
|
||||||
|
You can also use environment variables with the `JIGGABLEND_` prefix:
|
||||||
|
- `JIGGABLEND_PORT` - Server port (default: 8080)
|
||||||
|
- `JIGGABLEND_DB` - Database path (default: jiggablend.db)
|
||||||
|
- `JIGGABLEND_STORAGE` - Storage path (default: ./jiggablend-storage)
|
||||||
|
- `JIGGABLEND_LOG_FILE` - Log file path
|
||||||
|
- `JIGGABLEND_LOG_LEVEL` - Log level (debug, info, warn, error)
|
||||||
|
- `JIGGABLEND_VERBOSE` - Enable verbose logging
|
||||||
|
|
||||||
### Runner
|
### Runner
|
||||||
|
|
||||||
No configuration required. Runner will auto-detect hostname and IP.
|
The runner requires an API key to connect to the manager. The runner will auto-detect hostname and IP.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
|
### Building
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Build the unified binary (includes embedded web UI)
|
||||||
|
make build
|
||||||
|
|
||||||
|
# Or build directly
|
||||||
|
go build -o bin/jiggablend ./cmd/jiggablend
|
||||||
|
|
||||||
|
# Build web UI separately
|
||||||
|
make build-web
|
||||||
|
```
|
||||||
|
|
||||||
### Running the Manager
|
### Running the Manager
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Using make
|
# Using make (includes test setup)
|
||||||
make run-manager
|
make run-manager
|
||||||
|
|
||||||
# Or directly
|
# Or directly
|
||||||
go run ./cmd/manager
|
bin/jiggablend manager
|
||||||
|
|
||||||
# With custom options
|
# With custom options
|
||||||
go run ./cmd/manager -port 8080 -db jiggablend.db -storage ./storage
|
bin/jiggablend manager --port 8080 --db jiggablend.db --storage ./jiggablend-storage --log-file manager.log
|
||||||
|
|
||||||
|
# Using environment variables
|
||||||
|
JIGGABLEND_PORT=8080 JIGGABLEND_DB=jiggablend.db bin/jiggablend manager
|
||||||
```
|
```
|
||||||
|
|
||||||
The manager will start on `http://localhost:8080` by default.
|
The manager will start on `http://localhost:8080` by default.
|
||||||
@@ -89,26 +145,28 @@ The manager will start on `http://localhost:8080` by default.
|
|||||||
### Running a Runner
|
### Running a Runner
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Using make
|
# Using make (uses test API key)
|
||||||
make run-runner
|
make run-runner
|
||||||
|
|
||||||
# Or directly
|
# Or directly (requires API key)
|
||||||
go run ./cmd/runner
|
bin/jiggablend runner --api-key <your-api-key>
|
||||||
|
|
||||||
# With custom options
|
# With custom options
|
||||||
go run ./cmd/runner -manager http://localhost:8080 -name my-runner
|
bin/jiggablend runner --manager http://localhost:8080 --name my-runner --api-key <key> --log-file runner.log
|
||||||
|
|
||||||
|
# Using environment variables
|
||||||
|
JIGGABLEND_MANAGER=http://localhost:8080 JIGGABLEND_API_KEY=<key> bin/jiggablend runner
|
||||||
```
|
```
|
||||||
|
|
||||||
### Building
|
### Running Both (for Testing)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Build manager
|
# Run manager and runner in parallel
|
||||||
make build-manager
|
make run
|
||||||
|
|
||||||
# Build runner (Linux amd64)
|
|
||||||
make build-runner
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
This will start both the manager and a test runner with a fixed API key.
|
||||||
|
|
||||||
## OAuth Setup
|
## OAuth Setup
|
||||||
|
|
||||||
### Google OAuth
|
### Google OAuth
|
||||||
@@ -118,7 +176,10 @@ make build-runner
|
|||||||
3. Enable Google+ API
|
3. Enable Google+ API
|
||||||
4. Create OAuth 2.0 credentials
|
4. Create OAuth 2.0 credentials
|
||||||
5. Add authorized redirect URI: `http://localhost:8080/api/auth/google/callback`
|
5. Add authorized redirect URI: `http://localhost:8080/api/auth/google/callback`
|
||||||
6. Set environment variables with Client ID and Secret
|
6. Configure using CLI:
|
||||||
|
```bash
|
||||||
|
jiggablend manager config set google-oauth <client-id> <client-secret> --redirect-url http://localhost:8080/api/auth/google/callback
|
||||||
|
```
|
||||||
|
|
||||||
### Discord OAuth
|
### Discord OAuth
|
||||||
|
|
||||||
@@ -126,25 +187,39 @@ make build-runner
|
|||||||
2. Create a new application
|
2. Create a new application
|
||||||
3. Go to OAuth2 section
|
3. Go to OAuth2 section
|
||||||
4. Add redirect URI: `http://localhost:8080/api/auth/discord/callback`
|
4. Add redirect URI: `http://localhost:8080/api/auth/discord/callback`
|
||||||
5. Set environment variables with Client ID and Secret
|
5. Configure using CLI:
|
||||||
|
```bash
|
||||||
|
jiggablend manager config set discord-oauth <client-id> <client-secret> --redirect-url http://localhost:8080/api/auth/discord/callback
|
||||||
|
```
|
||||||
|
|
||||||
## Project Structure
|
## Project Structure
|
||||||
|
|
||||||
```
|
```
|
||||||
jiggablend/
|
jiggablend/
|
||||||
├── cmd/
|
├── cmd/
|
||||||
│ ├── manager/ # Manager server application
|
│ └── jiggablend/ # Unified CLI application
|
||||||
│ └── runner/ # Runner client application
|
│ ├── cmd/ # Cobra command definitions
|
||||||
|
│ └── main.go # Entry point
|
||||||
├── internal/
|
├── internal/
|
||||||
│ ├── api/ # REST API handlers
|
│ ├── auth/ # Authentication (OAuth, local, sessions)
|
||||||
│ ├── auth/ # OAuth authentication
|
│ ├── config/ # Configuration management
|
||||||
│ ├── database/ # DuckDB database models and migrations
|
│ ├── database/ # SQLite database models and migrations
|
||||||
│ ├── queue/ # Job queue management
|
│ ├── logger/ # Logging utilities
|
||||||
│ ├── storage/ # File storage operations
|
│ ├── manager/ # Manager server logic
|
||||||
│ └── runner/ # Runner management logic
|
│ ├── runner/ # Runner client logic
|
||||||
|
│ │ ├── api/ # Manager API client
|
||||||
|
│ │ ├── blender/ # Blender version detection
|
||||||
|
│ │ ├── encoding/ # Video encoding (H.264, AV1, VP9)
|
||||||
|
│ │ ├── tasks/ # Task execution (render, encode, process)
|
||||||
|
│ │ └── workspace/ # Workspace management
|
||||||
|
│ └── storage/ # File storage operations
|
||||||
├── pkg/
|
├── pkg/
|
||||||
│ └── types/ # Shared types and models
|
│ ├── executils/ # Execution utilities
|
||||||
├── web/ # Static web UI files
|
│ ├── scripts/ # Python scripts for Blender
|
||||||
|
│ └── types/ # Shared types and models
|
||||||
|
├── web/ # React web UI
|
||||||
|
│ ├── src/ # Source files
|
||||||
|
│ └── dist/ # Built files (embedded in binary)
|
||||||
├── go.mod
|
├── go.mod
|
||||||
└── Makefile
|
└── Makefile
|
||||||
```
|
```
|
||||||
@@ -156,27 +231,106 @@ jiggablend/
|
|||||||
- `GET /api/auth/google/callback` - Google OAuth callback
|
- `GET /api/auth/google/callback` - Google OAuth callback
|
||||||
- `GET /api/auth/discord/login` - Initiate Discord OAuth
|
- `GET /api/auth/discord/login` - Initiate Discord OAuth
|
||||||
- `GET /api/auth/discord/callback` - Discord OAuth callback
|
- `GET /api/auth/discord/callback` - Discord OAuth callback
|
||||||
|
- `POST /api/auth/login` - Local authentication login
|
||||||
|
- `POST /api/auth/register` - User registration (if enabled)
|
||||||
- `POST /api/auth/logout` - Logout
|
- `POST /api/auth/logout` - Logout
|
||||||
- `GET /api/auth/me` - Get current user
|
- `GET /api/auth/me` - Get current user
|
||||||
|
- `POST /api/auth/password/change` - Change password
|
||||||
|
|
||||||
### Jobs
|
### Jobs
|
||||||
- `POST /api/jobs` - Create a new job
|
- `POST /api/jobs` - Create a new job
|
||||||
- `GET /api/jobs` - List user's jobs
|
- `GET /api/jobs` - List user's jobs
|
||||||
- `GET /api/jobs/{id}` - Get job details
|
- `GET /api/jobs/{id}` - Get job details
|
||||||
- `DELETE /api/jobs/{id}` - Cancel a job
|
- `DELETE /api/jobs/{id}` - Cancel a job
|
||||||
- `POST /api/jobs/{id}/upload` - Upload job file
|
- `POST /api/jobs/{id}/upload` - Upload job file (Blender file)
|
||||||
- `GET /api/jobs/{id}/files` - List job files
|
- `GET /api/jobs/{id}/files` - List job files
|
||||||
- `GET /api/jobs/{id}/files/{fileId}/download` - Download job file
|
- `GET /api/jobs/{id}/files/{fileId}/download` - Download job file
|
||||||
|
- `GET /api/jobs/{id}/metadata` - Extract metadata from uploaded file
|
||||||
|
- `GET /api/jobs/{id}/outputs` - List job output files
|
||||||
|
|
||||||
### Runners
|
### Blender
|
||||||
- `GET /api/admin/runners` - List all runners (admin only)
|
- `GET /api/blender/versions` - List available Blender versions
|
||||||
- `POST /api/runner/register` - Register a runner (uses registration token)
|
|
||||||
- `POST /api/runner/heartbeat` - Update runner heartbeat (runner authenticated)
|
### Runners (Internal API)
|
||||||
|
- `POST /api/runner/register` - Register a runner (uses API key)
|
||||||
|
- `POST /api/runner/heartbeat` - Update runner heartbeat
|
||||||
- `GET /api/runner/tasks` - Get pending tasks for runner
|
- `GET /api/runner/tasks` - Get pending tasks for runner
|
||||||
- `POST /api/runner/tasks/{id}/complete` - Mark task as complete
|
- `POST /api/runner/tasks/{id}/complete` - Mark task as complete
|
||||||
- `GET /api/runner/files/{jobId}/{fileName}` - Download file for runner
|
- `GET /api/runner/files/{jobId}/{fileName}` - Download file for runner
|
||||||
- `POST /api/runner/files/{jobId}/upload` - Upload file from runner
|
- `POST /api/runner/files/{jobId}/upload` - Upload file from runner
|
||||||
|
|
||||||
|
### Admin (Admin Only)
|
||||||
|
- `GET /api/admin/runners` - List all runners
|
||||||
|
- `GET /api/admin/jobs` - List all jobs
|
||||||
|
- `GET /api/admin/users` - List all users
|
||||||
|
- `GET /api/admin/stats` - System statistics
|
||||||
|
|
||||||
|
### WebSocket
|
||||||
|
- `WS /api/ws` - WebSocket connection for real-time updates
|
||||||
|
- Subscribe to job channels: `job:{jobId}`
|
||||||
|
- Receive job status updates, progress, and logs
|
||||||
|
|
||||||
|
## Output Formats
|
||||||
|
|
||||||
|
The system supports the following output formats:
|
||||||
|
|
||||||
|
### Image Formats
|
||||||
|
- **PNG** - Standard PNG output
|
||||||
|
- **JPEG** - JPEG output
|
||||||
|
- **EXR** - OpenEXR format (HDR)
|
||||||
|
|
||||||
|
### Video Formats
|
||||||
|
- **EXR_264_MP4** - H.264 encoded MP4 from EXR sequence (SDR or HDR)
|
||||||
|
- **EXR_AV1_MP4** - AV1 encoded MP4 from EXR sequence (with alpha channel support)
|
||||||
|
- **EXR_VP9_WEBM** - VP9 encoded WebM from EXR sequence (with alpha channel and HDR support)
|
||||||
|
|
||||||
|
Video encoding features:
|
||||||
|
- 2-pass encoding for optimal quality
|
||||||
|
- HDR preservation using HLG transfer function
|
||||||
|
- Alpha channel preservation (AV1 and VP9 only)
|
||||||
|
- Automatic detection of source format (EXR or PNG)
|
||||||
|
- Software encoding (libx264, libaom-av1, libvpx-vp9)
|
||||||
|
|
||||||
|
## Storage Structure
|
||||||
|
|
||||||
|
The manager uses a local storage directory (default: `./jiggablend-storage`) with the following structure:
|
||||||
|
|
||||||
|
```
|
||||||
|
jiggablend-storage/
|
||||||
|
├── blender-versions/ # Bundled Blender versions
|
||||||
|
│ └── <version>/
|
||||||
|
├── jobs/ # Job context files
|
||||||
|
│ └── <job-id>/
|
||||||
|
│ └── context.tar
|
||||||
|
├── outputs/ # Rendered outputs
|
||||||
|
│ └── <job-id>/
|
||||||
|
├── temp/ # Temporary files
|
||||||
|
└── uploads/ # Uploaded files
|
||||||
|
```
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
### Running Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
make test
|
||||||
|
# Or directly
|
||||||
|
go test ./... -timeout 30s
|
||||||
|
```
|
||||||
|
|
||||||
|
### Web UI Development
|
||||||
|
|
||||||
|
The web UI is built with React and Vite. To develop the UI:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd web
|
||||||
|
npm install
|
||||||
|
npm run dev # Development server
|
||||||
|
npm run build # Build for production
|
||||||
|
```
|
||||||
|
|
||||||
|
The built files are embedded in the Go binary using `embed.FS`.
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
MIT
|
MIT
|
||||||
|
|||||||
@@ -32,4 +32,3 @@ func exitWithError(msg string, args ...interface{}) {
|
|||||||
fmt.Fprintf(os.Stderr, "Error: "+msg+"\n", args...)
|
fmt.Fprintf(os.Stderr, "Error: "+msg+"\n", args...)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
25
cmd/jiggablend/cmd/version.go
Normal file
25
cmd/jiggablend/cmd/version.go
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"jiggablend/version"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var versionCmd = &cobra.Command{
|
||||||
|
Use: "version",
|
||||||
|
Short: "Print the version information",
|
||||||
|
Long: `Print the version and build date of jiggablend.`,
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
fmt.Printf("jiggablend version %s\n", version.Version)
|
||||||
|
if version.Date != "" {
|
||||||
|
fmt.Printf("Build date: %s\n", version.Date)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
rootCmd.AddCommand(versionCmd)
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
|
|
||||||
"jiggablend/cmd/jiggablend/cmd"
|
"jiggablend/cmd/jiggablend/cmd"
|
||||||
|
_ "jiggablend/version"
|
||||||
)
|
)
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
@@ -11,4 +12,3 @@ func main() {
|
|||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
4
go.mod
4
go.mod
@@ -4,6 +4,8 @@ go 1.25.4
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/go-chi/chi/v5 v5.2.3
|
github.com/go-chi/chi/v5 v5.2.3
|
||||||
|
github.com/go-chi/cors v1.2.2
|
||||||
|
github.com/golang-migrate/migrate/v4 v4.19.0
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/gorilla/websocket v1.5.3
|
github.com/gorilla/websocket v1.5.3
|
||||||
github.com/mattn/go-sqlite3 v1.14.32
|
github.com/mattn/go-sqlite3 v1.14.32
|
||||||
@@ -17,9 +19,7 @@ require (
|
|||||||
cloud.google.com/go/compute/metadata v0.5.0 // indirect
|
cloud.google.com/go/compute/metadata v0.5.0 // indirect
|
||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||||
github.com/go-chi/cors v1.2.2 // indirect
|
|
||||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||||
github.com/golang-migrate/migrate/v4 v4.19.0 // indirect
|
|
||||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
|
|||||||
6
go.sum
6
go.sum
@@ -1,5 +1,3 @@
|
|||||||
cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
|
|
||||||
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
|
|
||||||
cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
|
cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
|
||||||
cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
|
cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
|
||||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
@@ -34,8 +32,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
|||||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||||
github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
|
github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
|
||||||
github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||||
|
|||||||
@@ -944,7 +944,7 @@ func (s *Manager) handleCancelJob(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
// Cancel all pending tasks
|
// Cancel all pending tasks
|
||||||
_, err = conn.Exec(
|
_, err = conn.Exec(
|
||||||
`UPDATE tasks SET status = ? WHERE job_id = ? AND status = ?`,
|
`UPDATE tasks SET status = ?, runner_id = NULL WHERE job_id = ? AND status = ?`,
|
||||||
types.TaskStatusFailed, jobID, types.TaskStatusPending,
|
types.TaskStatusFailed, jobID, types.TaskStatusPending,
|
||||||
)
|
)
|
||||||
return err
|
return err
|
||||||
@@ -3024,9 +3024,6 @@ func (s *Manager) handleListJobTasks(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
if err != nil {
|
|
||||||
total = -1
|
|
||||||
}
|
|
||||||
|
|
||||||
tasks := []types.Task{}
|
tasks := []types.Task{}
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
|
|||||||
@@ -89,6 +89,9 @@ type Manager struct {
|
|||||||
// Throttling for task status updates (per task)
|
// Throttling for task status updates (per task)
|
||||||
taskUpdateTimes map[int64]time.Time // key: taskID
|
taskUpdateTimes map[int64]time.Time // key: taskID
|
||||||
taskUpdateTimesMu sync.RWMutex
|
taskUpdateTimesMu sync.RWMutex
|
||||||
|
// Per-job mutexes to serialize updateJobStatusFromTasks calls and prevent race conditions
|
||||||
|
jobStatusUpdateMu map[int64]*sync.Mutex // key: jobID
|
||||||
|
jobStatusUpdateMuMu sync.RWMutex
|
||||||
|
|
||||||
// Client WebSocket connections (new unified WebSocket)
|
// Client WebSocket connections (new unified WebSocket)
|
||||||
// Key is "userID:connID" to support multiple tabs per user
|
// Key is "userID:connID" to support multiple tabs per user
|
||||||
@@ -162,6 +165,8 @@ func NewManager(db *database.DB, cfg *config.Config, auth *authpkg.Auth, storage
|
|||||||
runnerJobConns: make(map[string]*websocket.Conn),
|
runnerJobConns: make(map[string]*websocket.Conn),
|
||||||
runnerJobConnsWriteMu: make(map[string]*sync.Mutex),
|
runnerJobConnsWriteMu: make(map[string]*sync.Mutex),
|
||||||
runnerJobConnsWriteMuMu: sync.RWMutex{}, // Initialize the new field
|
runnerJobConnsWriteMuMu: sync.RWMutex{}, // Initialize the new field
|
||||||
|
// Per-job mutexes for serializing status updates
|
||||||
|
jobStatusUpdateMu: make(map[int64]*sync.Mutex),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check for required external tools
|
// Check for required external tools
|
||||||
|
|||||||
@@ -390,7 +390,7 @@ func (s *Manager) handleNextJob(w http.ResponseWriter, r *http.Request) {
|
|||||||
t.condition
|
t.condition
|
||||||
FROM tasks t
|
FROM tasks t
|
||||||
JOIN jobs j ON t.job_id = j.id
|
JOIN jobs j ON t.job_id = j.id
|
||||||
WHERE t.status = ? AND j.status != ?
|
WHERE t.status = ? AND t.runner_id IS NULL AND j.status != ?
|
||||||
ORDER BY t.created_at ASC
|
ORDER BY t.created_at ASC
|
||||||
LIMIT 50`,
|
LIMIT 50`,
|
||||||
types.TaskStatusPending, types.JobStatusCancelled,
|
types.TaskStatusPending, types.JobStatusCancelled,
|
||||||
@@ -1019,6 +1019,10 @@ func (s *Manager) handleGetJobStatusForRunner(w http.ResponseWriter, r *http.Req
|
|||||||
&job.CreatedAt, &startedAt, &completedAt, &errorMessage,
|
&job.CreatedAt, &startedAt, &completedAt, &errorMessage,
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
if err == sql.ErrNoRows {
|
||||||
|
s.respondError(w, http.StatusNotFound, "Job not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query job: %v", err))
|
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query job: %v", err))
|
||||||
return
|
return
|
||||||
@@ -1037,15 +1041,6 @@ func (s *Manager) handleGetJobStatusForRunner(w http.ResponseWriter, r *http.Req
|
|||||||
job.OutputFormat = &outputFormat.String
|
job.OutputFormat = &outputFormat.String
|
||||||
}
|
}
|
||||||
|
|
||||||
if err == sql.ErrNoRows {
|
|
||||||
s.respondError(w, http.StatusNotFound, "Job not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query job: %v", err))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if startedAt.Valid {
|
if startedAt.Valid {
|
||||||
job.StartedAt = &startedAt.Time
|
job.StartedAt = &startedAt.Time
|
||||||
}
|
}
|
||||||
@@ -1368,7 +1363,7 @@ func (s *Manager) handleRunnerJobWebSocket(w http.ResponseWriter, r *http.Reques
|
|||||||
log.Printf("Job WebSocket disconnected unexpectedly for task %d, marking as failed", taskID)
|
log.Printf("Job WebSocket disconnected unexpectedly for task %d, marking as failed", taskID)
|
||||||
s.db.With(func(conn *sql.DB) error {
|
s.db.With(func(conn *sql.DB) error {
|
||||||
_, err := conn.Exec(
|
_, err := conn.Exec(
|
||||||
`UPDATE tasks SET status = ?, error_message = ?, completed_at = ? WHERE id = ?`,
|
`UPDATE tasks SET status = ?, runner_id = NULL, error_message = ?, completed_at = ? WHERE id = ?`,
|
||||||
types.TaskStatusFailed, "WebSocket connection lost", time.Now(), taskID,
|
types.TaskStatusFailed, "WebSocket connection lost", time.Now(), taskID,
|
||||||
)
|
)
|
||||||
return err
|
return err
|
||||||
@@ -1683,11 +1678,10 @@ func (s *Manager) handleWebSocketTaskComplete(runnerID int64, taskUpdate WSTaskU
|
|||||||
} else {
|
} else {
|
||||||
// No retries remaining - mark as failed
|
// No retries remaining - mark as failed
|
||||||
err = s.db.WithTx(func(tx *sql.Tx) error {
|
err = s.db.WithTx(func(tx *sql.Tx) error {
|
||||||
_, err := tx.Exec(`UPDATE tasks SET status = ? WHERE id = ?`, types.TaskStatusFailed, taskUpdate.TaskID)
|
_, err := tx.Exec(
|
||||||
if err != nil {
|
`UPDATE tasks SET status = ?, runner_id = NULL, completed_at = ? WHERE id = ?`,
|
||||||
return err
|
types.TaskStatusFailed, now, taskUpdate.TaskID,
|
||||||
}
|
)
|
||||||
_, err = tx.Exec(`UPDATE tasks SET completed_at = ? WHERE id = ?`, now, taskUpdate.TaskID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -1854,7 +1848,7 @@ func (s *Manager) cancelActiveTasksForJob(jobID int64) error {
|
|||||||
// Tasks don't have a cancelled status - mark them as failed instead
|
// Tasks don't have a cancelled status - mark them as failed instead
|
||||||
err := s.db.With(func(conn *sql.DB) error {
|
err := s.db.With(func(conn *sql.DB) error {
|
||||||
_, err := conn.Exec(
|
_, err := conn.Exec(
|
||||||
`UPDATE tasks SET status = ?, error_message = ? WHERE job_id = ? AND status IN (?, ?)`,
|
`UPDATE tasks SET status = ?, runner_id = NULL, error_message = ? WHERE job_id = ? AND status IN (?, ?)`,
|
||||||
types.TaskStatusFailed, "Job cancelled", jobID, types.TaskStatusPending, types.TaskStatusRunning,
|
types.TaskStatusFailed, "Job cancelled", jobID, types.TaskStatusPending, types.TaskStatusRunning,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1920,8 +1914,37 @@ func (s *Manager) evaluateTaskCondition(taskID int64, jobID int64, conditionJSON
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// getJobStatusUpdateMutex returns the mutex for a specific jobID, creating it if needed.
|
||||||
|
// This ensures serialized execution of updateJobStatusFromTasks per job to prevent race conditions.
|
||||||
|
func (s *Manager) getJobStatusUpdateMutex(jobID int64) *sync.Mutex {
|
||||||
|
s.jobStatusUpdateMuMu.Lock()
|
||||||
|
defer s.jobStatusUpdateMuMu.Unlock()
|
||||||
|
|
||||||
|
mu, exists := s.jobStatusUpdateMu[jobID]
|
||||||
|
if !exists {
|
||||||
|
mu = &sync.Mutex{}
|
||||||
|
s.jobStatusUpdateMu[jobID] = mu
|
||||||
|
}
|
||||||
|
return mu
|
||||||
|
}
|
||||||
|
|
||||||
|
// cleanupJobStatusUpdateMutex removes the mutex for a jobID after it's no longer needed.
|
||||||
|
// Should only be called when the job is in a final state (completed/failed) and no more updates are expected.
|
||||||
|
func (s *Manager) cleanupJobStatusUpdateMutex(jobID int64) {
|
||||||
|
s.jobStatusUpdateMuMu.Lock()
|
||||||
|
defer s.jobStatusUpdateMuMu.Unlock()
|
||||||
|
delete(s.jobStatusUpdateMu, jobID)
|
||||||
|
}
|
||||||
|
|
||||||
// updateJobStatusFromTasks updates job status and progress based on task states
|
// updateJobStatusFromTasks updates job status and progress based on task states
|
||||||
|
// This function is serialized per jobID to prevent race conditions when multiple tasks
|
||||||
|
// complete concurrently and trigger status updates simultaneously.
|
||||||
func (s *Manager) updateJobStatusFromTasks(jobID int64) {
|
func (s *Manager) updateJobStatusFromTasks(jobID int64) {
|
||||||
|
// Serialize updates per job to prevent race conditions
|
||||||
|
mu := s.getJobStatusUpdateMutex(jobID)
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
|
||||||
now := time.Now()
|
now := time.Now()
|
||||||
|
|
||||||
// All jobs now use parallel runners (one task per frame), so we always use task-based progress
|
// All jobs now use parallel runners (one task per frame), so we always use task-based progress
|
||||||
@@ -2087,6 +2110,11 @@ func (s *Manager) updateJobStatusFromTasks(jobID int64) {
|
|||||||
"progress": progress,
|
"progress": progress,
|
||||||
"completed_at": now,
|
"completed_at": now,
|
||||||
})
|
})
|
||||||
|
// Clean up mutex for jobs in final states (completed or failed)
|
||||||
|
// No more status updates will occur for these jobs
|
||||||
|
if jobStatus == string(types.JobStatusCompleted) || jobStatus == string(types.JobStatusFailed) {
|
||||||
|
s.cleanupJobStatusUpdateMutex(jobID)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -289,6 +289,10 @@ func (r *Runner) uploadOutputs(ctx *tasks.Context, job *api.NextJobResponse) err
|
|||||||
log.Printf("Failed to upload %s: %v", filePath, err)
|
log.Printf("Failed to upload %s: %v", filePath, err)
|
||||||
} else {
|
} else {
|
||||||
ctx.OutputUploaded(entry.Name())
|
ctx.OutputUploaded(entry.Name())
|
||||||
|
// Delete file after successful upload to prevent duplicate uploads
|
||||||
|
if err := os.Remove(filePath); err != nil {
|
||||||
|
log.Printf("Warning: Failed to delete file %s after upload: %v", filePath, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -373,6 +373,12 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
|||||||
|
|
||||||
ctx.Info(fmt.Sprintf("Successfully uploaded %s: %s", strings.ToUpper(outputExt), filepath.Base(outputVideo)))
|
ctx.Info(fmt.Sprintf("Successfully uploaded %s: %s", strings.ToUpper(outputExt), filepath.Base(outputVideo)))
|
||||||
|
|
||||||
|
// Delete file after successful upload to prevent duplicate uploads
|
||||||
|
if err := os.Remove(outputVideo); err != nil {
|
||||||
|
log.Printf("Warning: Failed to delete video file %s after upload: %v", outputVideo, err)
|
||||||
|
ctx.Warn(fmt.Sprintf("Warning: Failed to delete video file after upload: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
log.Printf("Successfully generated and uploaded %s for job %d: %s", strings.ToUpper(outputExt), ctx.JobID, filepath.Base(outputVideo))
|
log.Printf("Successfully generated and uploaded %s for job %d: %s", strings.ToUpper(outputExt), ctx.JobID, filepath.Base(outputVideo))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
16
version/version.go
Normal file
16
version/version.go
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
// version/version.go
|
||||||
|
package version
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
var Version string
|
||||||
|
var Date string
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
if Version == "" {
|
||||||
|
Version = "0.0.0-dev"
|
||||||
|
}
|
||||||
|
if Date == "" {
|
||||||
|
Date = time.Now().Format("2006-01-02 15:04:05")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -111,6 +111,41 @@ export default function JobDetails({ job, onClose, onUpdate }) {
|
|||||||
});
|
});
|
||||||
}, [taskData]);
|
}, [taskData]);
|
||||||
|
|
||||||
|
// Helper function to load all files with pagination
|
||||||
|
const loadAllFiles = async (jobId, signal) => {
|
||||||
|
const allFiles = [];
|
||||||
|
let offset = 0;
|
||||||
|
const limit = 100; // Load 100 files per page
|
||||||
|
let hasMore = true;
|
||||||
|
|
||||||
|
while (hasMore && !signal?.aborted) {
|
||||||
|
const fileList = await jobs.getFiles(jobId, { limit, offset, signal });
|
||||||
|
|
||||||
|
// Check for superseded sentinel
|
||||||
|
if (fileList === REQUEST_SUPERSEDED) {
|
||||||
|
return REQUEST_SUPERSEDED;
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileData = fileList?.data || fileList;
|
||||||
|
const files = Array.isArray(fileData) ? fileData : [];
|
||||||
|
allFiles.push(...files);
|
||||||
|
|
||||||
|
// Check if there are more files to load
|
||||||
|
const total = fileList?.total;
|
||||||
|
if (total !== undefined) {
|
||||||
|
hasMore = offset + files.length < total;
|
||||||
|
} else {
|
||||||
|
// If total is not provided, check if we got a full page (or more)
|
||||||
|
// Use >= to safely handle edge cases where API returns different amounts
|
||||||
|
hasMore = files.length >= limit;
|
||||||
|
}
|
||||||
|
|
||||||
|
offset += files.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
return allFiles;
|
||||||
|
};
|
||||||
|
|
||||||
const loadDetails = async () => {
|
const loadDetails = async () => {
|
||||||
// Guard against undefined job or job.id
|
// Guard against undefined job or job.id
|
||||||
if (!job || !job.id) {
|
if (!job || !job.id) {
|
||||||
@@ -122,9 +157,9 @@ export default function JobDetails({ job, onClose, onUpdate }) {
|
|||||||
setLoading(true);
|
setLoading(true);
|
||||||
// Use summary endpoint for tasks initially - much faster
|
// Use summary endpoint for tasks initially - much faster
|
||||||
const signal = abortControllerRef.current?.signal;
|
const signal = abortControllerRef.current?.signal;
|
||||||
const [details, fileList, taskListResult] = await Promise.all([
|
const [details, allFilesResult, taskListResult] = await Promise.all([
|
||||||
jobs.get(job.id, { signal }),
|
jobs.get(job.id, { signal }),
|
||||||
jobs.getFiles(job.id, { limit: 50, signal }), // Only load first page of files
|
loadAllFiles(job.id, signal), // Load all files with pagination
|
||||||
jobs.getTasksSummary(job.id, { sort: 'frame:asc', signal }), // Get all tasks
|
jobs.getTasksSummary(job.id, { sort: 'frame:asc', signal }), // Get all tasks
|
||||||
]);
|
]);
|
||||||
|
|
||||||
@@ -135,11 +170,10 @@ export default function JobDetails({ job, onClose, onUpdate }) {
|
|||||||
setJobDetails(details);
|
setJobDetails(details);
|
||||||
|
|
||||||
// Handle paginated file response - check for superseded sentinel
|
// Handle paginated file response - check for superseded sentinel
|
||||||
if (fileList === REQUEST_SUPERSEDED) {
|
if (allFilesResult === REQUEST_SUPERSEDED) {
|
||||||
return; // Request was superseded, skip this update
|
return; // Request was superseded, skip this update
|
||||||
}
|
}
|
||||||
const fileData = fileList?.data || fileList;
|
setFiles(Array.isArray(allFilesResult) ? allFilesResult : []);
|
||||||
setFiles(Array.isArray(fileData) ? fileData : []);
|
|
||||||
|
|
||||||
// Handle paginated task summary response - check for superseded sentinel
|
// Handle paginated task summary response - check for superseded sentinel
|
||||||
if (taskListResult === REQUEST_SUPERSEDED) {
|
if (taskListResult === REQUEST_SUPERSEDED) {
|
||||||
@@ -617,16 +651,22 @@ export default function JobDetails({ job, onClose, onUpdate }) {
|
|||||||
};
|
};
|
||||||
reloadTasks();
|
reloadTasks();
|
||||||
} else if (data.type === 'file_added' && data.data) {
|
} else if (data.type === 'file_added' && data.data) {
|
||||||
// New file was added - reload file list
|
// New file was added - reload all files
|
||||||
const reloadFiles = async () => {
|
const reloadFiles = async () => {
|
||||||
try {
|
try {
|
||||||
const fileList = await jobs.getFiles(job.id, { limit: 50 });
|
const signal = abortControllerRef.current?.signal;
|
||||||
|
const allFilesResult = await loadAllFiles(job.id, signal);
|
||||||
|
|
||||||
|
// Check if request was aborted
|
||||||
|
if (signal?.aborted) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Check for superseded sentinel
|
// Check for superseded sentinel
|
||||||
if (fileList === REQUEST_SUPERSEDED) {
|
if (allFilesResult === REQUEST_SUPERSEDED) {
|
||||||
return; // Request was superseded, skip this update
|
return; // Request was superseded, skip this update
|
||||||
}
|
}
|
||||||
const fileData = fileList.data || fileList;
|
setFiles(Array.isArray(allFilesResult) ? allFilesResult : []);
|
||||||
setFiles(Array.isArray(fileData) ? fileData : []);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Failed to reload files:', error);
|
console.error('Failed to reload files:', error);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user