// Package runner provides the Jiggablend render runner. package runner import ( "crypto/sha256" "encoding/hex" "errors" "fmt" "log" "net" "os" "os/exec" "path/filepath" "strings" "sync" "time" "jiggablend/internal/runner/api" "jiggablend/internal/runner/blender" "jiggablend/internal/runner/encoding" "jiggablend/internal/runner/sandbox" "jiggablend/internal/runner/tasks" "jiggablend/internal/runner/workspace" "jiggablend/pkg/executils" "jiggablend/pkg/types" ) // Runner is the main render runner. type Runner struct { id int64 name string hostname string manager *api.ManagerClient workspace *workspace.Manager blender *blender.Manager encoder *encoding.Selector processes *executils.ProcessTracker processors map[string]tasks.Processor stopChan chan struct{} fingerprint string fingerprintMu sync.RWMutex // gpuLockedOut is set when logs indicate a GPU error (e.g. HIP "Illegal address"); // when true, the runner forces CPU rendering for all subsequent jobs. gpuLockedOut bool gpuLockedOutMu sync.RWMutex // hasAMD/hasNVIDIA/hasIntel are set at startup by hardware/Blender GPU backend detection. // Used to force CPU only for Blender < 4.x when AMD is present (no official HIP support pre-4). // gpuDetectionFailed is true when detection could not run; we then force CPU for all versions. gpuBackendMu sync.RWMutex hasAMD bool hasNVIDIA bool hasIntel bool gpuBackendProbed bool gpuDetectionFailed bool // forceCPURendering forces CPU rendering for all jobs regardless of metadata/backend detection. forceCPURendering bool // disableRT disables GPU ray tracing acceleration (HIPRT, OptiX, etc.). disableRT bool // hipGPUSampleBatch limits samples per GPU pass on gfx115x (0 = disabled). hipGPUSampleBatch int // sandbox wraps Blender invocations (none/podman). sandboxWrapper sandbox.Wrapper sandboxBackend string } // RunnerOptions configures optional runner behavior. type RunnerOptions struct { ForceCPURendering bool DisableRT bool HipGPUSampleBatch int SandboxBackend string // none|podman SandboxNetwork bool SandboxImage string // podman thin runtime image } // New creates a new runner. func New(managerURL, name, hostname string, forceCPURendering, disableRT bool, hipGPUSampleBatch int) *Runner { return NewWithOptions(managerURL, name, hostname, RunnerOptions{ ForceCPURendering: forceCPURendering, DisableRT: disableRT, HipGPUSampleBatch: hipGPUSampleBatch, SandboxBackend: sandbox.BackendPodman, }) } // NewWithOptions creates a runner with full options including sandbox. func NewWithOptions(managerURL, name, hostname string, opts RunnerOptions) *Runner { manager := api.NewManagerClient(managerURL) backend, err := sandbox.NormalizeBackend(opts.SandboxBackend) if err != nil { log.Printf("Invalid sandbox backend %q, falling back to none: %v", opts.SandboxBackend, err) backend = sandbox.BackendNone } sb, err := sandbox.New(sandbox.Options{ Backend: backend, AllowNetwork: opts.SandboxNetwork, PodmanImage: opts.SandboxImage, }) if err != nil { log.Printf("Failed to init sandbox %q: %v; using none", backend, err) sb, _ = sandbox.New(sandbox.Options{Backend: sandbox.BackendNone}) backend = sandbox.BackendNone } r := &Runner{ name: name, hostname: hostname, manager: manager, processes: executils.NewProcessTracker(), stopChan: make(chan struct{}), processors: make(map[string]tasks.Processor), forceCPURendering: opts.ForceCPURendering, disableRT: opts.DisableRT, hipGPUSampleBatch: opts.HipGPUSampleBatch, sandboxWrapper: sb, sandboxBackend: backend, } // Generate fingerprint r.generateFingerprint() return r } // CheckRequiredTools verifies that required external tools are available. func (r *Runner) CheckRequiredTools() error { if err := exec.Command("zstd", "--version").Run(); err != nil { return fmt.Errorf("zstd not found - required for compressed blend file support. Install with: apt install zstd") } log.Printf("Found zstd for compressed blend file support") if r.sandboxWrapper != nil { if err := r.sandboxWrapper.Available(); err != nil { return fmt.Errorf("sandbox backend %q unavailable: %w", r.sandboxBackend, err) } log.Printf("Sandbox backend: %s", r.sandboxWrapper.Name()) } return nil } // SandboxBackend returns the configured sandbox backend name. func (r *Runner) SandboxBackend() string { if r.sandboxBackend == "" { return sandbox.BackendNone } return r.sandboxBackend } var ( cachedCapabilities map[string]interface{} capabilitiesOnce sync.Once ) // ProbeCapabilities detects hardware capabilities. func (r *Runner) ProbeCapabilities() map[string]interface{} { capabilitiesOnce.Do(func() { caps := make(map[string]interface{}) if err := exec.Command("ffmpeg", "-version").Run(); err == nil { caps["ffmpeg"] = true } else { caps["ffmpeg"] = false } // Filled later with actual backend when runner is constructed; probe is package-level once. // Real sandbox name is injected in ProbeCapabilities on the instance after New. caps["sandbox"] = "none" cachedCapabilities = caps }) // Overlay instance sandbox name (Once already ran with none default). if r != nil && r.sandboxWrapper != nil { out := make(map[string]interface{}, len(cachedCapabilities)+1) for k, v := range cachedCapabilities { out[k] = v } out["sandbox"] = r.sandboxWrapper.Name() return out } return cachedCapabilities } // Register registers the runner with the manager. func (r *Runner) Register(apiKey string) (int64, error) { caps := r.ProbeCapabilities() id, err := r.manager.Register(r.name, r.hostname, caps, apiKey, r.GetFingerprint()) if err != nil { return 0, err } r.id = id // Initialize workspace after registration r.workspace = workspace.NewManager(r.name) // Initialize blender manager r.blender = blender.NewManager(r.manager, r.workspace.BaseDir()) // Initialize encoder selector r.encoder = encoding.NewSelector() // Register task processors r.processors["render"] = tasks.NewRenderProcessor() r.processors["encode"] = tasks.NewEncodeProcessor() return id, nil } // DetectAndStoreGPUBackends runs host-level backend detection and stores AMD/NVIDIA/Intel results. // Call after Register. Used so we only force CPU for Blender < 4.x when AMD is present. func (r *Runner) DetectAndStoreGPUBackends() { r.gpuBackendMu.Lock() defer r.gpuBackendMu.Unlock() if r.gpuBackendProbed { return } hasAMD, hasNVIDIA, hasIntel, ok := blender.DetectGPUBackends() if !ok { log.Printf("GPU backend detection failed (host probe unavailable). All jobs will use CPU because backend availability is unknown.") r.gpuBackendProbed = true r.gpuDetectionFailed = true return } detectedTypes := 0 if hasAMD { detectedTypes++ } if hasNVIDIA { detectedTypes++ } if hasIntel { detectedTypes++ } if detectedTypes > 1 { log.Printf("mixed GPU vendors detected (AMD=%v NVIDIA=%v INTEL=%v): multi-vendor setups may not work reliably, but runner will continue with GPU enabled", hasAMD, hasNVIDIA, hasIntel) } r.hasAMD = hasAMD r.hasNVIDIA = hasNVIDIA r.hasIntel = hasIntel r.gpuBackendProbed = true r.gpuDetectionFailed = false log.Printf("GPU backend detection: AMD=%v NVIDIA=%v INTEL=%v (Blender < 4.x will force CPU only when AMD is present)", hasAMD, hasNVIDIA, hasIntel) } // HasAMD returns whether the runner detected AMD devices. Used to force CPU for Blender < 4.x only when AMD is present. func (r *Runner) HasAMD() bool { r.gpuBackendMu.RLock() defer r.gpuBackendMu.RUnlock() return r.hasAMD } // HasNVIDIA returns whether the runner detected NVIDIA GPUs. func (r *Runner) HasNVIDIA() bool { r.gpuBackendMu.RLock() defer r.gpuBackendMu.RUnlock() return r.hasNVIDIA } // HasIntel returns whether the runner detected Intel GPUs (e.g. Arc). func (r *Runner) HasIntel() bool { r.gpuBackendMu.RLock() defer r.gpuBackendMu.RUnlock() return r.hasIntel } // DisableRT returns whether GPU ray tracing acceleration should be disabled. func (r *Runner) DisableRT() bool { return r.disableRT } // HipGPUSampleBatch returns the per-pass GPU sample limit for gfx115x batching (0 = disabled). func (r *Runner) HipGPUSampleBatch() int { return r.hipGPUSampleBatch } // GPUDetectionFailed returns true when startup GPU backend detection could not run or failed. When true, all jobs use CPU because backend availability is unknown. func (r *Runner) GPUDetectionFailed() bool { r.gpuBackendMu.RLock() defer r.gpuBackendMu.RUnlock() return r.gpuDetectionFailed } // Start starts the job polling loop. func (r *Runner) Start(pollInterval time.Duration) { log.Printf("Starting job polling loop (interval: %v)", pollInterval) for { select { case <-r.stopChan: log.Printf("Stopping job polling loop") return default: } log.Printf("Polling for next job (runner ID: %d)", r.id) job, err := r.manager.PollNextJob() if err != nil { log.Printf("Error polling for job: %v", err) time.Sleep(pollInterval) continue } if job == nil { log.Printf("No job available, sleeping for %v", pollInterval) time.Sleep(pollInterval) continue } log.Printf("Received job assignment: task=%d, job=%d, type=%s", job.Task.TaskID, job.Task.JobID, job.Task.TaskType) if err := r.executeJob(job); err != nil { log.Printf("Error processing job: %v", err) } } } // Stop stops the runner. func (r *Runner) Stop() { close(r.stopChan) } // KillAllProcesses kills all running processes. func (r *Runner) KillAllProcesses() { log.Printf("Killing all running processes...") killedCount := r.processes.KillAll() // Release all allocated devices if r.encoder != nil { // Device pool cleanup is handled internally } log.Printf("Killed %d process(es)", killedCount) } // Cleanup removes the workspace directory. func (r *Runner) Cleanup() { if r.workspace != nil { r.workspace.Cleanup() } } func (r *Runner) withJobWorkspace(jobID int64, fn func(workDir string) error) error { workDir, err := r.workspace.CreateJobDir(jobID) if err != nil { return fmt.Errorf("failed to create job workspace: %w", err) } defer func() { if cleanupErr := r.workspace.CleanupJobDir(jobID); cleanupErr != nil { log.Printf("Warning: failed to cleanup job workspace for job %d: %v", jobID, cleanupErr) } if cleanupErr := r.workspace.CleanupVideoDir(jobID); cleanupErr != nil { log.Printf("Warning: failed to cleanup encode workspace for job %d: %v", jobID, cleanupErr) } }() return fn(workDir) } // executeJob handles a job using per-job WebSocket connection. func (r *Runner) executeJob(job *api.NextJobResponse) (err error) { // Recover from panics to prevent runner process crashes during task execution defer func() { if rec := recover(); rec != nil { log.Printf("Task execution panicked: %v", rec) err = fmt.Errorf("task execution panicked: %v", rec) } }() return r.withJobWorkspace(job.Task.JobID, func(workDir string) error { // Connect to job WebSocket (no runnerID needed - authentication handles it) jobConn := api.NewJobConnection() if err := jobConn.Connect(r.manager.GetBaseURL(), job.JobPath, job.JobToken); err != nil { return fmt.Errorf("failed to connect job WebSocket: %w", err) } defer jobConn.Close() log.Printf("Job WebSocket authenticated for task %d", job.Task.TaskID) // Create task context (frame range: Frame = start, FrameEnd = end; 0 or missing = single frame) frameEnd := job.Task.FrameEnd if frameEnd < job.Task.Frame { frameEnd = job.Task.Frame } ctx := tasks.NewContext( job.Task.TaskID, job.Task.JobID, job.Task.JobName, job.Task.Frame, frameEnd, job.Task.TaskType, workDir, job.JobToken, job.Task.Metadata, r.manager, jobConn, r.workspace, r.blender, r.encoder, r.processes, r.IsGPULockedOut(), r.HasAMD(), r.HasNVIDIA(), r.HasIntel(), r.GPUDetectionFailed(), r.forceCPURendering, r.disableRT, r.hipGPUSampleBatch, nil, // set below so the callback can mark this attempt r.sandboxWrapper, ) // Arm GPU lockout at most once process-wide; if this attempt is the one that // arms it, mark the context so a failure requeues without burning retry_count. ctx.OnGPUError = func() { if r.SetGPULockedOut(true) { ctx.GPULockoutArmedThisAttempt = true ctx.GPULockedOut = true ctx.Warn("GPU error detected; GPU disabled for subsequent jobs (this attempt free-requeues without using a retry)") } } ctx.Info(fmt.Sprintf("Task assignment received (job: %d, type: %s)", job.Task.JobID, job.Task.TaskType)) // Get processor for task type processor, ok := r.processors[job.Task.TaskType] if !ok { return fmt.Errorf("unknown task type: %s", job.Task.TaskType) } // Process the task var processErr error switch job.Task.TaskType { case "render": // this task has a upload outputs step because the frames are not uploaded by the render task directly we have to do it manually here TODO: maybe we should make it work like the encode task // Download context contextPath := job.JobPath + "/context.tar" if err := r.downloadContext(job.Task.JobID, contextPath, job.JobToken); err != nil { jobConn.Log(job.Task.TaskID, types.LogLevelError, fmt.Sprintf("Failed to download context: %v", err)) jobConn.Complete(job.Task.TaskID, false, fmt.Errorf("failed to download context: %v", err), false) return fmt.Errorf("failed to download context: %w", err) } processErr = processor.Process(ctx) if processErr == nil { processErr = r.uploadOutputs(ctx, job) } case "encode": // this task doesn't have a upload outputs step because the video is already uploaded by the encode task processErr = processor.Process(ctx) default: return fmt.Errorf("unknown task type: %s", job.Task.TaskType) } if processErr != nil { if errors.Is(processErr, tasks.ErrJobCancelled) { ctx.Warn("Stopping task early because the job was cancelled") return nil } ctx.Error(fmt.Sprintf("Task failed: %v", processErr)) ctx.Complete(false, processErr) return processErr } ctx.Complete(true, nil) return nil }) } func (r *Runner) downloadContext(jobID int64, contextPath, jobToken string) error { reader, err := r.manager.DownloadContext(contextPath, jobToken) if err != nil { return err } defer reader.Close() jobDir := r.workspace.JobDir(jobID) return workspace.ExtractTar(reader, jobDir) } func (r *Runner) uploadOutputs(ctx *tasks.Context, job *api.NextJobResponse) error { outputDir := ctx.WorkDir + "/output" uploadPath := fmt.Sprintf("/api/runner/jobs/%d/upload", job.Task.JobID) return uploadOutputFiles(outputDir, func(filePath, fileName string) error { if err := r.manager.UploadFile(uploadPath, job.JobToken, filePath); err != nil { return err } ctx.OutputUploaded(fileName) // Delete file after successful upload to prevent duplicate uploads if err := os.Remove(filePath); err != nil { log.Printf("Warning: Failed to delete file %s after upload: %v", filePath, err) } return nil }) } // uploadOutputFiles uploads all non-directory files from outputDir. // Fails if the directory cannot be read, any upload fails, or zero files were uploaded. func uploadOutputFiles(outputDir string, uploadFn func(filePath, fileName string) error) error { entries, err := os.ReadDir(outputDir) if err != nil { return fmt.Errorf("failed to read output directory: %w", err) } var files []os.DirEntry for _, entry := range entries { if !entry.IsDir() { files = append(files, entry) } } if len(files) == 0 { return fmt.Errorf("no output files found in %s", outputDir) } var firstErr error uploaded := 0 for _, entry := range files { filePath := filepath.Join(outputDir, entry.Name()) if err := uploadFn(filePath, entry.Name()); err != nil { log.Printf("Failed to upload %s: %v", filePath, err) if firstErr == nil { firstErr = fmt.Errorf("failed to upload %s: %w", entry.Name(), err) } continue } uploaded++ } if firstErr != nil { return firstErr } if uploaded == 0 { return fmt.Errorf("no output files were uploaded from %s", outputDir) } return nil } // generateFingerprint creates a unique hardware fingerprint. func (r *Runner) generateFingerprint() { r.fingerprintMu.Lock() defer r.fingerprintMu.Unlock() var components []string components = append(components, r.hostname) if machineID, err := os.ReadFile("/etc/machine-id"); err == nil { components = append(components, strings.TrimSpace(string(machineID))) } if productUUID, err := os.ReadFile("/sys/class/dmi/id/product_uuid"); err == nil { components = append(components, strings.TrimSpace(string(productUUID))) } if macAddr, err := r.getMACAddress(); err == nil { components = append(components, macAddr) } if len(components) <= 1 { components = append(components, fmt.Sprintf("%d", os.Getpid())) components = append(components, fmt.Sprintf("%d", time.Now().Unix())) } h := sha256.New() for _, comp := range components { h.Write([]byte(comp)) h.Write([]byte{0}) } r.fingerprint = hex.EncodeToString(h.Sum(nil)) } func (r *Runner) getMACAddress() (string, error) { interfaces, err := net.Interfaces() if err != nil { return "", err } for _, iface := range interfaces { if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 { continue } if len(iface.HardwareAddr) == 0 { continue } return iface.HardwareAddr.String(), nil } return "", fmt.Errorf("no suitable network interface found") } // GetFingerprint returns the runner's hardware fingerprint. func (r *Runner) GetFingerprint() string { r.fingerprintMu.RLock() defer r.fingerprintMu.RUnlock() return r.fingerprint } // GetID returns the runner ID. func (r *Runner) GetID() int64 { return r.id } // SetGPULockedOut sets whether GPU use is locked out due to a detected GPU error. // When true, the runner will force CPU rendering for all jobs. // Returns true only on the false→true transition (first arm); subsequent calls are no-ops for logging. func (r *Runner) SetGPULockedOut(locked bool) (newlyEnabled bool) { r.gpuLockedOutMu.Lock() defer r.gpuLockedOutMu.Unlock() if locked { if r.gpuLockedOut { return false } r.gpuLockedOut = true log.Printf("GPU lockout enabled: GPU rendering disabled for subsequent jobs (CPU only)") return true } r.gpuLockedOut = false return false } // IsGPULockedOut returns whether GPU use is currently locked out. func (r *Runner) IsGPULockedOut() bool { r.gpuLockedOutMu.RLock() defer r.gpuLockedOutMu.RUnlock() return r.gpuLockedOut }