s1d3sw1ped/jiggablend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

Refactor installation and runner scripts for enhanced usability and security

Browse Source 
- Updated the installation script to clarify that production wrappers do not include fixed test secrets, improving user guidance for local testing.
- Modified the jiggablend-manager and jiggablend-runner scripts to remove fixed test configurations, emphasizing the use of local test credentials via `make init-test`.
- Enhanced error handling in the runner script to ensure that an API key is provided, improving security and user feedback.
- Added new flags and options for the runner, including sandboxing capabilities and GPU ray tracing control, enhancing flexibility for users.
- Improved README documentation to reflect changes in usage and configuration, ensuring users have clear instructions for setup and execution.
This commit is contained in:
Justin Harms
2026-07-12 10:01:15 -05:00
parent a3defe5cf6
commit 1a69fcfd04
47 changed files with 2718 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/runner/runner.go
+154 -18
View File
@@ -10,6 +10,7 @@ import (
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"time"
@@ -17,6 +18,7 @@ import (
"jiggablend/internal/runner/api"
"jiggablend/internal/runner/blender"
"jiggablend/internal/runner/encoding"
"jiggablend/internal/runner/sandbox"
"jiggablend/internal/runner/tasks"
"jiggablend/internal/runner/workspace"
"jiggablend/pkg/executils"
@@ -58,12 +60,56 @@ type Runner struct {
// forceCPURendering forces CPU rendering for all jobs regardless of metadata/backend detection.
forceCPURendering bool
// disableRT disables GPU ray tracing acceleration (HIPRT, OptiX, etc.).
disableRT bool
// hipGPUSampleBatch limits samples per GPU pass on gfx115x (0 = disabled).
hipGPUSampleBatch int
// sandbox wraps Blender invocations (none/podman).
sandboxWrapper sandbox.Wrapper
sandboxBackend string
}
// RunnerOptions configures optional runner behavior.
type RunnerOptions struct {
ForceCPURendering bool
DisableRT bool
HipGPUSampleBatch int
SandboxBackend string // none|podman
SandboxNetwork bool
SandboxImage string // podman thin runtime image
}
// New creates a new runner.
func New(managerURL, name, hostname string, forceCPURendering bool) *Runner {
func New(managerURL, name, hostname string, forceCPURendering, disableRT bool, hipGPUSampleBatch int) *Runner {
return NewWithOptions(managerURL, name, hostname, RunnerOptions{
ForceCPURendering: forceCPURendering,
DisableRT: disableRT,
HipGPUSampleBatch: hipGPUSampleBatch,
SandboxBackend: sandbox.BackendPodman,
})
}
// NewWithOptions creates a runner with full options including sandbox.
func NewWithOptions(managerURL, name, hostname string, opts RunnerOptions) *Runner {
manager := api.NewManagerClient(managerURL)
backend, err := sandbox.NormalizeBackend(opts.SandboxBackend)
if err != nil {
log.Printf("Invalid sandbox backend %q, falling back to none: %v", opts.SandboxBackend, err)
backend = sandbox.BackendNone
}
sb, err := sandbox.New(sandbox.Options{
Backend: backend,
AllowNetwork: opts.SandboxNetwork,
PodmanImage: opts.SandboxImage,
})
if err != nil {
log.Printf("Failed to init sandbox %q: %v; using none", backend, err)
sb, _ = sandbox.New(sandbox.Options{Backend: sandbox.BackendNone})
backend = sandbox.BackendNone
}
r := &Runner{
name: name,
hostname: hostname,
@@ -72,7 +118,11 @@ func New(managerURL, name, hostname string, forceCPURendering bool) *Runner {
stopChan: make(chan struct{}),
processors: make(map[string]tasks.Processor),
forceCPURendering: forceCPURendering,
forceCPURendering: opts.ForceCPURendering,
disableRT: opts.DisableRT,
hipGPUSampleBatch: opts.HipGPUSampleBatch,
sandboxWrapper: sb,
sandboxBackend: backend,
}
// Generate fingerprint
@@ -88,9 +138,24 @@ func (r *Runner) CheckRequiredTools() error {
}
log.Printf("Found zstd for compressed blend file support")
if r.sandboxWrapper != nil {
if err := r.sandboxWrapper.Available(); err != nil {
return fmt.Errorf("sandbox backend %q unavailable: %w", r.sandboxBackend, err)
}
log.Printf("Sandbox backend: %s", r.sandboxWrapper.Name())
}
return nil
}
// SandboxBackend returns the configured sandbox backend name.
func (r *Runner) SandboxBackend() string {
if r.sandboxBackend == "" {
return sandbox.BackendNone
}
return r.sandboxBackend
}
var (
cachedCapabilities map[string]interface{}
capabilitiesOnce sync.Once
@@ -107,8 +172,21 @@ func (r *Runner) ProbeCapabilities() map[string]interface{} {
caps["ffmpeg"] = false
}
// Filled later with actual backend when runner is constructed; probe is package-level once.
// Real sandbox name is injected in ProbeCapabilities on the instance after New.
caps["sandbox"] = "none"
cachedCapabilities = caps
})
// Overlay instance sandbox name (Once already ran with none default).
if r != nil && r.sandboxWrapper != nil {
out := make(map[string]interface{}, len(cachedCapabilities)+1)
for k, v := range cachedCapabilities {
out[k] = v
}
out["sandbox"] = r.sandboxWrapper.Name()
return out
}
return cachedCapabilities
}
@@ -198,6 +276,16 @@ func (r *Runner) HasIntel() bool {
return r.hasIntel
}
// DisableRT returns whether GPU ray tracing acceleration should be disabled.
func (r *Runner) DisableRT() bool {
return r.disableRT
}
// HipGPUSampleBatch returns the per-pass GPU sample limit for gfx115x batching (0 = disabled).
func (r *Runner) HipGPUSampleBatch() int {
return r.hipGPUSampleBatch
}
// GPUDetectionFailed returns true when startup GPU backend detection could not run or failed. When true, all jobs use CPU because backend availability is unknown.
func (r *Runner) GPUDetectionFailed() bool {
r.gpuBackendMu.RLock()
@@ -330,8 +418,20 @@ func (r *Runner) executeJob(job *api.NextJobResponse) (err error) {
r.HasIntel(),
r.GPUDetectionFailed(),
r.forceCPURendering,
func() { r.SetGPULockedOut(true) },
r.disableRT,
r.hipGPUSampleBatch,
nil, // set below so the callback can mark this attempt
r.sandboxWrapper,
)
// Arm GPU lockout at most once process-wide; if this attempt is the one that
// arms it, mark the context so a failure requeues without burning retry_count.
ctx.OnGPUError = func() {
if r.SetGPULockedOut(true) {
ctx.GPULockoutArmedThisAttempt = true
ctx.GPULockedOut = true
ctx.Warn("GPU error detected; GPU disabled for subsequent jobs (this attempt free-requeues without using a retry)")
}
}
ctx.Info(fmt.Sprintf("Task assignment received (job: %d, type: %s)",
job.Task.JobID, job.Task.TaskType))
@@ -350,7 +450,7 @@ func (r *Runner) executeJob(job *api.NextJobResponse) (err error) {
contextPath := job.JobPath + "/context.tar"
if err := r.downloadContext(job.Task.JobID, contextPath, job.JobToken); err != nil {
jobConn.Log(job.Task.TaskID, types.LogLevelError, fmt.Sprintf("Failed to download context: %v", err))
jobConn.Complete(job.Task.TaskID, false, fmt.Errorf("failed to download context: %v", err))
jobConn.Complete(job.Task.TaskID, false, fmt.Errorf("failed to download context: %v", err), false)
return fmt.Errorf("failed to download context: %w", err)
}
processErr = processor.Process(ctx)
@@ -393,27 +493,56 @@ func (r *Runner) uploadOutputs(ctx *tasks.Context, job *api.NextJobResponse) err
outputDir := ctx.WorkDir + "/output"
uploadPath := fmt.Sprintf("/api/runner/jobs/%d/upload", job.Task.JobID)
return uploadOutputFiles(outputDir, func(filePath, fileName string) error {
if err := r.manager.UploadFile(uploadPath, job.JobToken, filePath); err != nil {
return err
}
ctx.OutputUploaded(fileName)
// Delete file after successful upload to prevent duplicate uploads
if err := os.Remove(filePath); err != nil {
log.Printf("Warning: Failed to delete file %s after upload: %v", filePath, err)
}
return nil
})
}
// uploadOutputFiles uploads all non-directory files from outputDir.
// Fails if the directory cannot be read, any upload fails, or zero files were uploaded.
func uploadOutputFiles(outputDir string, uploadFn func(filePath, fileName string) error) error {
entries, err := os.ReadDir(outputDir)
if err != nil {
return fmt.Errorf("failed to read output directory: %w", err)
}
var files []os.DirEntry
for _, entry := range entries {
if entry.IsDir() {
continue
}
filePath := outputDir + "/" + entry.Name()
if err := r.manager.UploadFile(uploadPath, job.JobToken, filePath); err != nil {
log.Printf("Failed to upload %s: %v", filePath, err)
} else {
ctx.OutputUploaded(entry.Name())
// Delete file after successful upload to prevent duplicate uploads
if err := os.Remove(filePath); err != nil {
log.Printf("Warning: Failed to delete file %s after upload: %v", filePath, err)
}
if !entry.IsDir() {
files = append(files, entry)
}
}
if len(files) == 0 {
return fmt.Errorf("no output files found in %s", outputDir)
}
var firstErr error
uploaded := 0
for _, entry := range files {
filePath := filepath.Join(outputDir, entry.Name())
if err := uploadFn(filePath, entry.Name()); err != nil {
log.Printf("Failed to upload %s: %v", filePath, err)
if firstErr == nil {
firstErr = fmt.Errorf("failed to upload %s: %w", entry.Name(), err)
}
continue
}
uploaded++
}
if firstErr != nil {
return firstErr
}
if uploaded == 0 {
return fmt.Errorf("no output files were uploaded from %s", outputDir)
}
return nil
}
@@ -484,13 +613,20 @@ func (r *Runner) GetID() int64 {
// SetGPULockedOut sets whether GPU use is locked out due to a detected GPU error.
// When true, the runner will force CPU rendering for all jobs.
func (r *Runner) SetGPULockedOut(locked bool) {
// Returns true only on the false→true transition (first arm); subsequent calls are no-ops for logging.
func (r *Runner) SetGPULockedOut(locked bool) (newlyEnabled bool) {
r.gpuLockedOutMu.Lock()
defer r.gpuLockedOutMu.Unlock()
r.gpuLockedOut = locked
if locked {
if r.gpuLockedOut {
return false
}
r.gpuLockedOut = true
log.Printf("GPU lockout enabled: GPU rendering disabled for subsequent jobs (CPU only)")
return true
}
r.gpuLockedOut = false
return false
}
// IsGPULockedOut returns whether GPU use is currently locked out.