11 Commits

Author SHA1 Message Date
bb57ce8659 Update task status handling to reset runner_id on job cancellation and failure
All checks were successful
Release Tag / release (push) Successful in 20s
- Modified SQL queries in multiple functions to set runner_id to NULL when updating task statuses for cancelled jobs and failed tasks.
- Ensured that tasks are properly marked as failed with the correct error messages and updated completion timestamps.
- Improved handling of task statuses to prevent potential issues with task assignment and execution.
2026-01-03 09:01:08 -06:00
1a8836e6aa Merge pull request 'Refactor job status handling to prevent race conditions' (#4) from fix-race into master
All checks were successful
Release Tag / release (push) Successful in 18s
Reviewed-on: #4
2026-01-02 18:25:17 -06:00
b51b96a618 Refactor job status handling to prevent race conditions
All checks were successful
PR Check / check-and-test (pull_request) Successful in 26s
- Removed redundant error handling in handleListJobTasks.
- Introduced per-job mutexes in Manager to serialize updateJobStatusFromTasks calls, ensuring thread safety during concurrent task completions.
- Added methods to manage job status update mutexes, including creation and cleanup after job completion or failure.
- Improved error handling in handleGetJobStatusForRunner by consolidating error checks.
2026-01-02 18:22:55 -06:00
8e561922c9 Merge pull request 'Implement file deletion after successful uploads in runner and encoding processes' (#3) from fix-uploads into master
Reviewed-on: #3
2026-01-02 17:51:19 -06:00
1c4bd78f56 Add FFmpeg setup step to Gitea workflow for enhanced media processing
All checks were successful
PR Check / check-and-test (pull_request) Successful in 1m6s
- Included a new step in the test-pr.yaml workflow to set up FFmpeg, improving the project's media handling capabilities.
- This addition complements the existing build steps for Go and frontend assets, ensuring a more comprehensive build environment.
2026-01-02 17:48:46 -06:00
3f2982ddb3 Update Gitea workflow to include frontend build step and adjust Go build command
Some checks failed
PR Check / check-and-test (pull_request) Failing after 42s
- Added a step to install and build the frontend using npm in the test-pr.yaml workflow.
- Modified the Go build command to compile all packages instead of specifying the output binary location.
- This change improves the build process by integrating frontend assets with the backend build.
2026-01-02 17:46:03 -06:00
0b852c5087 Update Gitea workflow to specify output binary location for jiggablend build
Some checks failed
PR Check / check-and-test (pull_request) Failing after 8s
- Changed the build command in the test-pr.yaml workflow to output the jiggablend binary to the bin directory.
- This modification enhances the organization of build artifacts and aligns with project structure.
2026-01-02 17:40:34 -06:00
5e56c7f0e8 Implement file deletion after successful uploads in runner and encoding processes
Some checks failed
PR Check / check-and-test (pull_request) Failing after 9s
- Added logic to delete files after successful uploads in both runner and encode tasks to prevent duplicate uploads.
- Included logging for any errors encountered during file deletion to ensure visibility of issues.
2026-01-02 17:34:41 -06:00
0a8f40b9cb Add MIT License file to the project
All checks were successful
Release Tag / release (push) Successful in 47s
- Introduced LICENSE file with the MIT License terms.
- Ensured compliance with open-source distribution and usage rights.
2026-01-02 14:55:53 -06:00
7440511740 Add GoReleaser configuration and update Makefile for streamlined builds
Some checks failed
Release Tag / release (push) Failing after 46s
- Introduced .goreleaser.yaml for automated release management.
- Updated Makefile to utilize GoReleaser for building the jiggablend binary.
- Added new workflows for release tagging and pull request checks in Gitea.
- Updated dependencies in go.mod and go.sum, including new packages for versioning.
- Enhanced .gitignore to exclude build artifacts in the dist directory.
2026-01-02 14:28:03 -06:00
c7c8762164 Update README.md to reflect significant changes in architecture and features. Replace DuckDB with SQLite for the database, enhance authentication options, and introduce a modern React-based web UI. Expand job management capabilities, including video encoding support and metadata extraction. Revise installation and configuration instructions, and clarify output formats and storage structure. Improve development guidelines for building and testing the application. 2026-01-02 14:03:03 -06:00
18 changed files with 447 additions and 116 deletions

View File

@@ -0,0 +1,24 @@
name: Release Tag
on:
push:
tags:
- '*'
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@main
with:
fetch-depth: 0
- run: git fetch --force --tags
- uses: actions/setup-go@main
with:
go-version-file: 'go.mod'
- uses: goreleaser/goreleaser-action@master
with:
distribution: goreleaser
version: 'latest'
args: release
env:
GITEA_TOKEN: ${{secrets.RELEASE_TOKEN}}

View File

@@ -0,0 +1,17 @@
name: PR Check
on:
- pull_request
jobs:
check-and-test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@main
- uses: actions/setup-go@main
with:
go-version-file: 'go.mod'
- uses: FedericoCarboni/setup-ffmpeg@v3
- run: go mod tidy
- run: cd web && npm install && npm run build
- run: go build ./...
- run: go test -race -v -shuffle=on ./...

1
.gitignore vendored
View File

@@ -65,6 +65,7 @@ lerna-debug.log*
*.o *.o
*.a *.a
*.so *.so
/dist/
# Temporary files # Temporary files
*.tmp *.tmp

48
.goreleaser.yaml Normal file
View File

@@ -0,0 +1,48 @@
version: 2
before:
hooks:
- go mod tidy -v
- sh -c "cd web && npm install && npm run build"
builds:
- id: default
main: ./cmd/jiggablend
binary: jiggablend
ldflags:
- -X jiggablend/version.Version={{.Version}}
- -X jiggablend/version.Date={{.Date}}
env:
- CGO_ENABLED=1
goos:
- linux
goarch:
- amd64
checksum:
name_template: "checksums.txt"
archives:
- id: default
name_template: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}"
formats: tar.gz
format_overrides:
- goos: windows
formats: zip
files:
- README.md
- LICENSE
changelog:
sort: asc
filters:
exclude:
- "^docs:"
- "^test:"
release:
name_template: "{{ .ProjectName }}-{{ .Version }}"
gitea_urls:
api: https://git.s1d3sw1ped.com/api/v1
download: https://git.s1d3sw1ped.com

21
LICENSE Normal file
View File

@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright © 2026 s1d3sw1ped
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

View File

@@ -1,12 +1,11 @@
.PHONY: build build-web run run-manager run-runner cleanup cleanup-manager cleanup-runner kill-all clean-bin clean-web test help install .PHONY: build build-web run run-manager run-runner cleanup cleanup-manager cleanup-runner clean-bin clean-web test help install
# Build the jiggablend binary (includes embedded web UI) # Build the jiggablend binary (includes embedded web UI)
build: clean-bin build-web build:
go build -o bin/jiggablend ./cmd/jiggablend @echo "Building with GoReleaser..."
goreleaser build --clean --snapshot --single-target
# Build for Linux (cross-compile) @mkdir -p bin
build-linux: clean-bin build-web @find dist -name jiggablend -type f -exec cp {} bin/jiggablend \;
GOOS=linux GOARCH=amd64 go build -o bin/jiggablend ./cmd/jiggablend
# Build web UI # Build web UI
build-web: clean-web build-web: clean-web
@@ -27,10 +26,6 @@ cleanup-runner:
# Cleanup both manager and runner logs # Cleanup both manager and runner logs
cleanup: cleanup-manager cleanup-runner cleanup: cleanup-manager cleanup-runner
# Kill all jiggablend processes
kill-all:
@echo "Not implemented"
# Run manager and runner in parallel (for testing) # Run manager and runner in parallel (for testing)
run: cleanup build init-test run: cleanup build init-test
@echo "Starting manager and runner in parallel..." @echo "Starting manager and runner in parallel..."
@@ -74,37 +69,30 @@ clean-web:
test: test:
go test ./... -timeout 30s go test ./... -timeout 30s
# Install to /usr/local/bin
install: build
sudo cp bin/jiggablend /usr/local/bin/
# Show help # Show help
help: help:
@echo "Jiggablend Build and Run Makefile" @echo "Jiggablend Build and Run Makefile"
@echo "" @echo ""
@echo "Build targets:" @echo "Build targets:"
@echo " build - Build jiggablend binary with embedded web UI" @echo " build - Build jiggablend binary with embedded web UI"
@echo " build-linux - Cross-compile for Linux amd64" @echo " build-web - Build web UI only"
@echo " build-web - Build web UI only"
@echo "" @echo ""
@echo "Run targets:" @echo "Run targets:"
@echo " run - Run manager and runner in parallel (for testing)" @echo " run - Run manager and runner in parallel (for testing)"
@echo " run-manager - Run manager server" @echo " run-manager - Run manager server"
@echo " run-runner - Run runner with test API key" @echo " run-runner - Run runner with test API key"
@echo " init-test - Initialize test configuration (run once)" @echo " init-test - Initialize test configuration (run once)"
@echo "" @echo ""
@echo "Cleanup targets:" @echo "Cleanup targets:"
@echo " cleanup - Clean all logs" @echo " cleanup - Clean all logs"
@echo " cleanup-manager - Clean manager logs" @echo " cleanup-manager - Clean manager logs"
@echo " cleanup-runner - Clean runner logs" @echo " cleanup-runner - Clean runner logs"
@echo " kill-all - Kill all running jiggablend processes"
@echo "" @echo ""
@echo "Other targets:" @echo "Other targets:"
@echo " clean-bin - Clean build artifacts" @echo " clean-bin - Clean build artifacts"
@echo " clean-web - Clean web build artifacts" @echo " clean-web - Clean web build artifacts"
@echo " test - Run Go tests" @echo " test - Run Go tests"
@echo " install - Install to /usr/local/bin" @echo " help - Show this help"
@echo " help - Show this help"
@echo "" @echo ""
@echo "CLI Usage:" @echo "CLI Usage:"
@echo " jiggablend manager serve - Start the manager server" @echo " jiggablend manager serve - Start the manager server"

266
README.md
View File

@@ -4,28 +4,41 @@ A distributed Blender render farm system built with Go. The system consists of a
## Architecture ## Architecture
- **Manager**: Central server with REST API, web UI, DuckDB database, and local file storage - **Manager**: Central server with REST API, embedded web UI, SQLite database, and local file storage
- **Runner**: Linux amd64 client that connects to manager, receives jobs, executes Blender renders, and reports back - **Runner**: Linux amd64 client that connects to manager, receives jobs, executes Blender renders, and reports back
Both manager and runner are part of a single binary (`jiggablend`) with subcommands.
## Features ## Features
- OAuth authentication (Google and Discord) - **Authentication**: OAuth (Google and Discord) and local authentication with user management
- Web-based job submission and monitoring - **Web UI**: Modern React-based interface for job submission and monitoring
- Distributed rendering across multiple runners - **Distributed Rendering**: Scale across multiple runners with automatic job distribution
- Real-time job progress tracking - **Real-time Updates**: WebSocket-based progress tracking and job status updates
- File upload/download for Blender files and rendered outputs - **Video Encoding**: Automatic video encoding from EXR/PNG sequences with multiple codec support:
- Runner health monitoring - H.264 (MP4) - SDR and HDR support
- AV1 (MP4) - With alpha channel support
- VP9 (WebM) - With alpha channel and HDR support
- **Output Formats**: PNG, JPEG, EXR, and video formats (MP4, WebM)
- **Blender Version Management**: Support for multiple Blender versions with automatic detection
- **Metadata Extraction**: Automatic extraction of scene metadata from Blender files
- **Admin Panel**: User and runner management interface
- **Runner Management**: API key-based authentication for runners with health monitoring
- **HDR Support**: Preserve HDR range in video encoding with HLG transfer function
- **Alpha Channel**: Preserve alpha channel in video encoding (AV1 and VP9)
## Prerequisites ## Prerequisites
### Manager ### Manager
- Go 1.21 or later - Go 1.25.4 or later
- DuckDB (via Go driver) - SQLite (via Go driver)
- Blender installed and in PATH (for metadata extraction)
- ImageMagick installed (for EXR preview conversion)
### Runner ### Runner
- Linux amd64 - Linux amd64
- Blender installed and in PATH - Blender installed (can use bundled versions from storage)
- FFmpeg installed (optional, for video processing) - FFmpeg installed (required for video encoding)
## Installation ## Installation
@@ -44,44 +57,87 @@ go mod download
### Manager ### Manager
Set the following environment variables for authentication (optional): Configuration is managed through the CLI using `jiggablend manager config` commands. The configuration is stored in the SQLite database.
#### Initial Setup
For testing, use the Makefile helper:
```bash
make init-test
```
This will:
- Enable local authentication
- Set a fixed API key for testing
- Create a test admin user (test@example.com / testpassword)
#### Manual Configuration
```bash ```bash
# OAuth Providers (optional) # Enable local authentication
export GOOGLE_CLIENT_ID="your-google-client-id" jiggablend manager config enable localauth
export GOOGLE_CLIENT_SECRET="your-google-client-secret"
export GOOGLE_REDIRECT_URL="http://localhost:8080/api/auth/google/callback"
export DISCORD_CLIENT_ID="your-discord-client-id" # Add a user
export DISCORD_CLIENT_SECRET="your-discord-client-secret" jiggablend manager config add user <email> <password> --admin
export DISCORD_REDIRECT_URL="http://localhost:8080/api/auth/discord/callback"
# Local Authentication (optional) # Generate an API key for runners
export ENABLE_LOCAL_AUTH="true" jiggablend manager config add apikey <name> --scope manager
# Test User (optional, for testing only) # Set OAuth credentials
# Creates a local user on startup if it doesn't exist jiggablend manager config set google-oauth <client-id> <client-secret> --redirect-url <url>
export LOCAL_TEST_EMAIL="test@example.com" jiggablend manager config set discord-oauth <client-id> <client-secret> --redirect-url <url>
export LOCAL_TEST_PASSWORD="testpassword"
# View current configuration
jiggablend manager config show
# List users and API keys
jiggablend manager config list users
jiggablend manager config list apikeys
``` ```
#### Environment Variables
You can also use environment variables with the `JIGGABLEND_` prefix:
- `JIGGABLEND_PORT` - Server port (default: 8080)
- `JIGGABLEND_DB` - Database path (default: jiggablend.db)
- `JIGGABLEND_STORAGE` - Storage path (default: ./jiggablend-storage)
- `JIGGABLEND_LOG_FILE` - Log file path
- `JIGGABLEND_LOG_LEVEL` - Log level (debug, info, warn, error)
- `JIGGABLEND_VERBOSE` - Enable verbose logging
### Runner ### Runner
No configuration required. Runner will auto-detect hostname and IP. The runner requires an API key to connect to the manager. The runner will auto-detect hostname and IP.
## Usage ## Usage
### Building
```bash
# Build the unified binary (includes embedded web UI)
make build
# Or build directly
go build -o bin/jiggablend ./cmd/jiggablend
# Build web UI separately
make build-web
```
### Running the Manager ### Running the Manager
```bash ```bash
# Using make # Using make (includes test setup)
make run-manager make run-manager
# Or directly # Or directly
go run ./cmd/manager bin/jiggablend manager
# With custom options # With custom options
go run ./cmd/manager -port 8080 -db jiggablend.db -storage ./storage bin/jiggablend manager --port 8080 --db jiggablend.db --storage ./jiggablend-storage --log-file manager.log
# Using environment variables
JIGGABLEND_PORT=8080 JIGGABLEND_DB=jiggablend.db bin/jiggablend manager
``` ```
The manager will start on `http://localhost:8080` by default. The manager will start on `http://localhost:8080` by default.
@@ -89,26 +145,28 @@ The manager will start on `http://localhost:8080` by default.
### Running a Runner ### Running a Runner
```bash ```bash
# Using make # Using make (uses test API key)
make run-runner make run-runner
# Or directly # Or directly (requires API key)
go run ./cmd/runner bin/jiggablend runner --api-key <your-api-key>
# With custom options # With custom options
go run ./cmd/runner -manager http://localhost:8080 -name my-runner bin/jiggablend runner --manager http://localhost:8080 --name my-runner --api-key <key> --log-file runner.log
# Using environment variables
JIGGABLEND_MANAGER=http://localhost:8080 JIGGABLEND_API_KEY=<key> bin/jiggablend runner
``` ```
### Building ### Running Both (for Testing)
```bash ```bash
# Build manager # Run manager and runner in parallel
make build-manager make run
# Build runner (Linux amd64)
make build-runner
``` ```
This will start both the manager and a test runner with a fixed API key.
## OAuth Setup ## OAuth Setup
### Google OAuth ### Google OAuth
@@ -118,7 +176,10 @@ make build-runner
3. Enable Google+ API 3. Enable Google+ API
4. Create OAuth 2.0 credentials 4. Create OAuth 2.0 credentials
5. Add authorized redirect URI: `http://localhost:8080/api/auth/google/callback` 5. Add authorized redirect URI: `http://localhost:8080/api/auth/google/callback`
6. Set environment variables with Client ID and Secret 6. Configure using CLI:
```bash
jiggablend manager config set google-oauth <client-id> <client-secret> --redirect-url http://localhost:8080/api/auth/google/callback
```
### Discord OAuth ### Discord OAuth
@@ -126,25 +187,39 @@ make build-runner
2. Create a new application 2. Create a new application
3. Go to OAuth2 section 3. Go to OAuth2 section
4. Add redirect URI: `http://localhost:8080/api/auth/discord/callback` 4. Add redirect URI: `http://localhost:8080/api/auth/discord/callback`
5. Set environment variables with Client ID and Secret 5. Configure using CLI:
```bash
jiggablend manager config set discord-oauth <client-id> <client-secret> --redirect-url http://localhost:8080/api/auth/discord/callback
```
## Project Structure ## Project Structure
``` ```
jiggablend/ jiggablend/
├── cmd/ ├── cmd/
── manager/ # Manager server application ── jiggablend/ # Unified CLI application
└── runner/ # Runner client application ├── cmd/ # Cobra command definitions
│ └── main.go # Entry point
├── internal/ ├── internal/
│ ├── api/ # REST API handlers │ ├── auth/ # Authentication (OAuth, local, sessions)
│ ├── auth/ # OAuth authentication │ ├── config/ # Configuration management
│ ├── database/ # DuckDB database models and migrations │ ├── database/ # SQLite database models and migrations
│ ├── queue/ # Job queue management │ ├── logger/ # Logging utilities
│ ├── storage/ # File storage operations │ ├── manager/ # Manager server logic
── runner/ # Runner management logic ── runner/ # Runner client logic
│ │ ├── api/ # Manager API client
│ │ ├── blender/ # Blender version detection
│ │ ├── encoding/ # Video encoding (H.264, AV1, VP9)
│ │ ├── tasks/ # Task execution (render, encode, process)
│ │ └── workspace/ # Workspace management
│ └── storage/ # File storage operations
├── pkg/ ├── pkg/
── types/ # Shared types and models ── executils/ # Execution utilities
├── web/ # Static web UI files │ ├── scripts/ # Python scripts for Blender
│ └── types/ # Shared types and models
├── web/ # React web UI
│ ├── src/ # Source files
│ └── dist/ # Built files (embedded in binary)
├── go.mod ├── go.mod
└── Makefile └── Makefile
``` ```
@@ -156,27 +231,106 @@ jiggablend/
- `GET /api/auth/google/callback` - Google OAuth callback - `GET /api/auth/google/callback` - Google OAuth callback
- `GET /api/auth/discord/login` - Initiate Discord OAuth - `GET /api/auth/discord/login` - Initiate Discord OAuth
- `GET /api/auth/discord/callback` - Discord OAuth callback - `GET /api/auth/discord/callback` - Discord OAuth callback
- `POST /api/auth/login` - Local authentication login
- `POST /api/auth/register` - User registration (if enabled)
- `POST /api/auth/logout` - Logout - `POST /api/auth/logout` - Logout
- `GET /api/auth/me` - Get current user - `GET /api/auth/me` - Get current user
- `POST /api/auth/password/change` - Change password
### Jobs ### Jobs
- `POST /api/jobs` - Create a new job - `POST /api/jobs` - Create a new job
- `GET /api/jobs` - List user's jobs - `GET /api/jobs` - List user's jobs
- `GET /api/jobs/{id}` - Get job details - `GET /api/jobs/{id}` - Get job details
- `DELETE /api/jobs/{id}` - Cancel a job - `DELETE /api/jobs/{id}` - Cancel a job
- `POST /api/jobs/{id}/upload` - Upload job file - `POST /api/jobs/{id}/upload` - Upload job file (Blender file)
- `GET /api/jobs/{id}/files` - List job files - `GET /api/jobs/{id}/files` - List job files
- `GET /api/jobs/{id}/files/{fileId}/download` - Download job file - `GET /api/jobs/{id}/files/{fileId}/download` - Download job file
- `GET /api/jobs/{id}/metadata` - Extract metadata from uploaded file
- `GET /api/jobs/{id}/outputs` - List job output files
### Runners ### Blender
- `GET /api/admin/runners` - List all runners (admin only) - `GET /api/blender/versions` - List available Blender versions
- `POST /api/runner/register` - Register a runner (uses registration token)
- `POST /api/runner/heartbeat` - Update runner heartbeat (runner authenticated) ### Runners (Internal API)
- `POST /api/runner/register` - Register a runner (uses API key)
- `POST /api/runner/heartbeat` - Update runner heartbeat
- `GET /api/runner/tasks` - Get pending tasks for runner - `GET /api/runner/tasks` - Get pending tasks for runner
- `POST /api/runner/tasks/{id}/complete` - Mark task as complete - `POST /api/runner/tasks/{id}/complete` - Mark task as complete
- `GET /api/runner/files/{jobId}/{fileName}` - Download file for runner - `GET /api/runner/files/{jobId}/{fileName}` - Download file for runner
- `POST /api/runner/files/{jobId}/upload` - Upload file from runner - `POST /api/runner/files/{jobId}/upload` - Upload file from runner
### Admin (Admin Only)
- `GET /api/admin/runners` - List all runners
- `GET /api/admin/jobs` - List all jobs
- `GET /api/admin/users` - List all users
- `GET /api/admin/stats` - System statistics
### WebSocket
- `WS /api/ws` - WebSocket connection for real-time updates
- Subscribe to job channels: `job:{jobId}`
- Receive job status updates, progress, and logs
## Output Formats
The system supports the following output formats:
### Image Formats
- **PNG** - Standard PNG output
- **JPEG** - JPEG output
- **EXR** - OpenEXR format (HDR)
### Video Formats
- **EXR_264_MP4** - H.264 encoded MP4 from EXR sequence (SDR or HDR)
- **EXR_AV1_MP4** - AV1 encoded MP4 from EXR sequence (with alpha channel support)
- **EXR_VP9_WEBM** - VP9 encoded WebM from EXR sequence (with alpha channel and HDR support)
Video encoding features:
- 2-pass encoding for optimal quality
- HDR preservation using HLG transfer function
- Alpha channel preservation (AV1 and VP9 only)
- Automatic detection of source format (EXR or PNG)
- Software encoding (libx264, libaom-av1, libvpx-vp9)
## Storage Structure
The manager uses a local storage directory (default: `./jiggablend-storage`) with the following structure:
```
jiggablend-storage/
├── blender-versions/ # Bundled Blender versions
│ └── <version>/
├── jobs/ # Job context files
│ └── <job-id>/
│ └── context.tar
├── outputs/ # Rendered outputs
│ └── <job-id>/
├── temp/ # Temporary files
└── uploads/ # Uploaded files
```
## Development
### Running Tests
```bash
make test
# Or directly
go test ./... -timeout 30s
```
### Web UI Development
The web UI is built with React and Vite. To develop the UI:
```bash
cd web
npm install
npm run dev # Development server
npm run build # Build for production
```
The built files are embedded in the Go binary using `embed.FS`.
## License ## License
MIT MIT

View File

@@ -32,4 +32,3 @@ func exitWithError(msg string, args ...interface{}) {
fmt.Fprintf(os.Stderr, "Error: "+msg+"\n", args...) fmt.Fprintf(os.Stderr, "Error: "+msg+"\n", args...)
os.Exit(1) os.Exit(1)
} }

View File

@@ -0,0 +1,25 @@
package cmd
import (
"fmt"
"jiggablend/version"
"github.com/spf13/cobra"
)
var versionCmd = &cobra.Command{
Use: "version",
Short: "Print the version information",
Long: `Print the version and build date of jiggablend.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Printf("jiggablend version %s\n", version.Version)
if version.Date != "" {
fmt.Printf("Build date: %s\n", version.Date)
}
},
}
func init() {
rootCmd.AddCommand(versionCmd)
}

View File

@@ -4,6 +4,7 @@ import (
"os" "os"
"jiggablend/cmd/jiggablend/cmd" "jiggablend/cmd/jiggablend/cmd"
_ "jiggablend/version"
) )
func main() { func main() {
@@ -11,4 +12,3 @@ func main() {
os.Exit(1) os.Exit(1)
} }
} }

4
go.mod
View File

@@ -4,6 +4,8 @@ go 1.25.4
require ( require (
github.com/go-chi/chi/v5 v5.2.3 github.com/go-chi/chi/v5 v5.2.3
github.com/go-chi/cors v1.2.2
github.com/golang-migrate/migrate/v4 v4.19.0
github.com/google/uuid v1.6.0 github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3 github.com/gorilla/websocket v1.5.3
github.com/mattn/go-sqlite3 v1.14.32 github.com/mattn/go-sqlite3 v1.14.32
@@ -17,9 +19,7 @@ require (
cloud.google.com/go/compute/metadata v0.5.0 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-chi/cors v1.2.2 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/golang-migrate/migrate/v4 v4.19.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect

6
go.sum
View File

@@ -1,5 +1,3 @@
cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
@@ -34,8 +32,8 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs= github.com/mattn/go-sqlite3 v1.14.32 h1:JD12Ag3oLy1zQA+BNn74xRgaBbdhbNIDYvQUEuuErjs=
github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.32/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=

View File

@@ -944,7 +944,7 @@ func (s *Manager) handleCancelJob(w http.ResponseWriter, r *http.Request) {
// Cancel all pending tasks // Cancel all pending tasks
_, err = conn.Exec( _, err = conn.Exec(
`UPDATE tasks SET status = ? WHERE job_id = ? AND status = ?`, `UPDATE tasks SET status = ?, runner_id = NULL WHERE job_id = ? AND status = ?`,
types.TaskStatusFailed, jobID, types.TaskStatusPending, types.TaskStatusFailed, jobID, types.TaskStatusPending,
) )
return err return err
@@ -3024,9 +3024,6 @@ func (s *Manager) handleListJobTasks(w http.ResponseWriter, r *http.Request) {
return return
} }
defer rows.Close() defer rows.Close()
if err != nil {
total = -1
}
tasks := []types.Task{} tasks := []types.Task{}
for rows.Next() { for rows.Next() {

View File

@@ -89,6 +89,9 @@ type Manager struct {
// Throttling for task status updates (per task) // Throttling for task status updates (per task)
taskUpdateTimes map[int64]time.Time // key: taskID taskUpdateTimes map[int64]time.Time // key: taskID
taskUpdateTimesMu sync.RWMutex taskUpdateTimesMu sync.RWMutex
// Per-job mutexes to serialize updateJobStatusFromTasks calls and prevent race conditions
jobStatusUpdateMu map[int64]*sync.Mutex // key: jobID
jobStatusUpdateMuMu sync.RWMutex
// Client WebSocket connections (new unified WebSocket) // Client WebSocket connections (new unified WebSocket)
// Key is "userID:connID" to support multiple tabs per user // Key is "userID:connID" to support multiple tabs per user
@@ -162,6 +165,8 @@ func NewManager(db *database.DB, cfg *config.Config, auth *authpkg.Auth, storage
runnerJobConns: make(map[string]*websocket.Conn), runnerJobConns: make(map[string]*websocket.Conn),
runnerJobConnsWriteMu: make(map[string]*sync.Mutex), runnerJobConnsWriteMu: make(map[string]*sync.Mutex),
runnerJobConnsWriteMuMu: sync.RWMutex{}, // Initialize the new field runnerJobConnsWriteMuMu: sync.RWMutex{}, // Initialize the new field
// Per-job mutexes for serializing status updates
jobStatusUpdateMu: make(map[int64]*sync.Mutex),
} }
// Check for required external tools // Check for required external tools

View File

@@ -390,7 +390,7 @@ func (s *Manager) handleNextJob(w http.ResponseWriter, r *http.Request) {
t.condition t.condition
FROM tasks t FROM tasks t
JOIN jobs j ON t.job_id = j.id JOIN jobs j ON t.job_id = j.id
WHERE t.status = ? AND j.status != ? WHERE t.status = ? AND t.runner_id IS NULL AND j.status != ?
ORDER BY t.created_at ASC ORDER BY t.created_at ASC
LIMIT 50`, LIMIT 50`,
types.TaskStatusPending, types.JobStatusCancelled, types.TaskStatusPending, types.JobStatusCancelled,
@@ -1019,6 +1019,10 @@ func (s *Manager) handleGetJobStatusForRunner(w http.ResponseWriter, r *http.Req
&job.CreatedAt, &startedAt, &completedAt, &errorMessage, &job.CreatedAt, &startedAt, &completedAt, &errorMessage,
) )
}) })
if err == sql.ErrNoRows {
s.respondError(w, http.StatusNotFound, "Job not found")
return
}
if err != nil { if err != nil {
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query job: %v", err)) s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query job: %v", err))
return return
@@ -1037,15 +1041,6 @@ func (s *Manager) handleGetJobStatusForRunner(w http.ResponseWriter, r *http.Req
job.OutputFormat = &outputFormat.String job.OutputFormat = &outputFormat.String
} }
if err == sql.ErrNoRows {
s.respondError(w, http.StatusNotFound, "Job not found")
return
}
if err != nil {
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query job: %v", err))
return
}
if startedAt.Valid { if startedAt.Valid {
job.StartedAt = &startedAt.Time job.StartedAt = &startedAt.Time
} }
@@ -1368,7 +1363,7 @@ func (s *Manager) handleRunnerJobWebSocket(w http.ResponseWriter, r *http.Reques
log.Printf("Job WebSocket disconnected unexpectedly for task %d, marking as failed", taskID) log.Printf("Job WebSocket disconnected unexpectedly for task %d, marking as failed", taskID)
s.db.With(func(conn *sql.DB) error { s.db.With(func(conn *sql.DB) error {
_, err := conn.Exec( _, err := conn.Exec(
`UPDATE tasks SET status = ?, error_message = ?, completed_at = ? WHERE id = ?`, `UPDATE tasks SET status = ?, runner_id = NULL, error_message = ?, completed_at = ? WHERE id = ?`,
types.TaskStatusFailed, "WebSocket connection lost", time.Now(), taskID, types.TaskStatusFailed, "WebSocket connection lost", time.Now(), taskID,
) )
return err return err
@@ -1683,11 +1678,10 @@ func (s *Manager) handleWebSocketTaskComplete(runnerID int64, taskUpdate WSTaskU
} else { } else {
// No retries remaining - mark as failed // No retries remaining - mark as failed
err = s.db.WithTx(func(tx *sql.Tx) error { err = s.db.WithTx(func(tx *sql.Tx) error {
_, err := tx.Exec(`UPDATE tasks SET status = ? WHERE id = ?`, types.TaskStatusFailed, taskUpdate.TaskID) _, err := tx.Exec(
if err != nil { `UPDATE tasks SET status = ?, runner_id = NULL, completed_at = ? WHERE id = ?`,
return err types.TaskStatusFailed, now, taskUpdate.TaskID,
} )
_, err = tx.Exec(`UPDATE tasks SET completed_at = ? WHERE id = ?`, now, taskUpdate.TaskID)
if err != nil { if err != nil {
return err return err
} }
@@ -1854,7 +1848,7 @@ func (s *Manager) cancelActiveTasksForJob(jobID int64) error {
// Tasks don't have a cancelled status - mark them as failed instead // Tasks don't have a cancelled status - mark them as failed instead
err := s.db.With(func(conn *sql.DB) error { err := s.db.With(func(conn *sql.DB) error {
_, err := conn.Exec( _, err := conn.Exec(
`UPDATE tasks SET status = ?, error_message = ? WHERE job_id = ? AND status IN (?, ?)`, `UPDATE tasks SET status = ?, runner_id = NULL, error_message = ? WHERE job_id = ? AND status IN (?, ?)`,
types.TaskStatusFailed, "Job cancelled", jobID, types.TaskStatusPending, types.TaskStatusRunning, types.TaskStatusFailed, "Job cancelled", jobID, types.TaskStatusPending, types.TaskStatusRunning,
) )
if err != nil { if err != nil {
@@ -1920,8 +1914,37 @@ func (s *Manager) evaluateTaskCondition(taskID int64, jobID int64, conditionJSON
} }
} }
// getJobStatusUpdateMutex returns the mutex for a specific jobID, creating it if needed.
// This ensures serialized execution of updateJobStatusFromTasks per job to prevent race conditions.
func (s *Manager) getJobStatusUpdateMutex(jobID int64) *sync.Mutex {
s.jobStatusUpdateMuMu.Lock()
defer s.jobStatusUpdateMuMu.Unlock()
mu, exists := s.jobStatusUpdateMu[jobID]
if !exists {
mu = &sync.Mutex{}
s.jobStatusUpdateMu[jobID] = mu
}
return mu
}
// cleanupJobStatusUpdateMutex removes the mutex for a jobID after it's no longer needed.
// Should only be called when the job is in a final state (completed/failed) and no more updates are expected.
func (s *Manager) cleanupJobStatusUpdateMutex(jobID int64) {
s.jobStatusUpdateMuMu.Lock()
defer s.jobStatusUpdateMuMu.Unlock()
delete(s.jobStatusUpdateMu, jobID)
}
// updateJobStatusFromTasks updates job status and progress based on task states // updateJobStatusFromTasks updates job status and progress based on task states
// This function is serialized per jobID to prevent race conditions when multiple tasks
// complete concurrently and trigger status updates simultaneously.
func (s *Manager) updateJobStatusFromTasks(jobID int64) { func (s *Manager) updateJobStatusFromTasks(jobID int64) {
// Serialize updates per job to prevent race conditions
mu := s.getJobStatusUpdateMutex(jobID)
mu.Lock()
defer mu.Unlock()
now := time.Now() now := time.Now()
// All jobs now use parallel runners (one task per frame), so we always use task-based progress // All jobs now use parallel runners (one task per frame), so we always use task-based progress
@@ -2087,6 +2110,11 @@ func (s *Manager) updateJobStatusFromTasks(jobID int64) {
"progress": progress, "progress": progress,
"completed_at": now, "completed_at": now,
}) })
// Clean up mutex for jobs in final states (completed or failed)
// No more status updates will occur for these jobs
if jobStatus == string(types.JobStatusCompleted) || jobStatus == string(types.JobStatusFailed) {
s.cleanupJobStatusUpdateMutex(jobID)
}
} }
} }

View File

@@ -289,6 +289,10 @@ func (r *Runner) uploadOutputs(ctx *tasks.Context, job *api.NextJobResponse) err
log.Printf("Failed to upload %s: %v", filePath, err) log.Printf("Failed to upload %s: %v", filePath, err)
} else { } else {
ctx.OutputUploaded(entry.Name()) ctx.OutputUploaded(entry.Name())
// Delete file after successful upload to prevent duplicate uploads
if err := os.Remove(filePath); err != nil {
log.Printf("Warning: Failed to delete file %s after upload: %v", filePath, err)
}
} }
} }

View File

@@ -373,6 +373,12 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
ctx.Info(fmt.Sprintf("Successfully uploaded %s: %s", strings.ToUpper(outputExt), filepath.Base(outputVideo))) ctx.Info(fmt.Sprintf("Successfully uploaded %s: %s", strings.ToUpper(outputExt), filepath.Base(outputVideo)))
// Delete file after successful upload to prevent duplicate uploads
if err := os.Remove(outputVideo); err != nil {
log.Printf("Warning: Failed to delete video file %s after upload: %v", outputVideo, err)
ctx.Warn(fmt.Sprintf("Warning: Failed to delete video file after upload: %v", err))
}
log.Printf("Successfully generated and uploaded %s for job %d: %s", strings.ToUpper(outputExt), ctx.JobID, filepath.Base(outputVideo)) log.Printf("Successfully generated and uploaded %s for job %d: %s", strings.ToUpper(outputExt), ctx.JobID, filepath.Base(outputVideo))
return nil return nil
} }

16
version/version.go Normal file
View File

@@ -0,0 +1,16 @@
// version/version.go
package version
import "time"
var Version string
var Date string
func init() {
if Version == "" {
Version = "0.0.0-dev"
}
if Date == "" {
Date = time.Now().Format("2006-01-02 15:04:05")
}
}