Refactor installation and runner scripts for enhanced usability and security
- Updated the installation script to clarify that production wrappers do not include fixed test secrets, improving user guidance for local testing. - Modified the jiggablend-manager and jiggablend-runner scripts to remove fixed test configurations, emphasizing the use of local test credentials via `make init-test`. - Enhanced error handling in the runner script to ensure that an API key is provided, improving security and user feedback. - Added new flags and options for the runner, including sandboxing capabilities and GPU ray tracing control, enhancing flexibility for users. - Improved README documentation to reflect changes in usage and configuration, ensuring users have clear instructions for setup and execution.
This commit is contained in:
@@ -301,7 +301,12 @@ func (j *JobConnection) OutputUploaded(taskID int64, fileName string) {
|
||||
}
|
||||
|
||||
// Complete sends task completion to the manager.
|
||||
func (j *JobConnection) Complete(taskID int64, success bool, errorMsg error) {
|
||||
// errorMsg must be JSON-encoded as a string: encoding an error interface
|
||||
// marshals to {} and the manager fails to unmarshal task_complete, which
|
||||
// previously surfaced as a generic "WebSocket connection lost" with no retries.
|
||||
// freeRequeue asks the manager to requeue a failure without incrementing retry_count
|
||||
// (used when this attempt newly armed GPU lockout).
|
||||
func (j *JobConnection) Complete(taskID int64, success bool, errorMsg error, freeRequeue bool) {
|
||||
if j.conn == nil {
|
||||
log.Printf("Cannot send task complete: WebSocket connection is nil")
|
||||
return
|
||||
@@ -310,13 +315,20 @@ func (j *JobConnection) Complete(taskID int64, success bool, errorMsg error) {
|
||||
j.writeMu.Lock()
|
||||
defer j.writeMu.Unlock()
|
||||
|
||||
data := map[string]interface{}{
|
||||
"task_id": taskID,
|
||||
"success": success,
|
||||
}
|
||||
if errorMsg != nil {
|
||||
data["error"] = errorMsg.Error()
|
||||
}
|
||||
if freeRequeue {
|
||||
data["free_requeue"] = true
|
||||
}
|
||||
|
||||
msg := map[string]interface{}{
|
||||
"type": "task_complete",
|
||||
"data": map[string]interface{}{
|
||||
"task_id": taskID,
|
||||
"success": success,
|
||||
"error": errorMsg,
|
||||
},
|
||||
"type": "task_complete",
|
||||
"data": data,
|
||||
"timestamp": time.Now().Unix(),
|
||||
}
|
||||
if err := j.conn.WriteJSON(msg); err != nil {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
@@ -10,6 +12,79 @@ import (
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
// TestComplete_ErrorIsJSONString ensures task_complete encodes the failure
|
||||
// reason as a string. Marshaling a Go error interface produces {}, which the
|
||||
// manager cannot unmarshal into WSTaskUpdate.Error and used to look like a
|
||||
// silent WebSocket drop with no retries.
|
||||
func TestComplete_ErrorIsJSONString(t *testing.T) {
|
||||
upgrader := websocket.Upgrader{}
|
||||
received := make(chan map[string]interface{}, 1)
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := upgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
// auth handshake
|
||||
var auth map[string]interface{}
|
||||
if err := conn.ReadJSON(&auth); err != nil {
|
||||
return
|
||||
}
|
||||
if auth["type"] == "auth" {
|
||||
_ = conn.WriteJSON(map[string]string{"type": "auth_ok"})
|
||||
}
|
||||
|
||||
var msg map[string]interface{}
|
||||
if err := conn.ReadJSON(&msg); err != nil {
|
||||
return
|
||||
}
|
||||
received <- msg
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
jc := NewJobConnection()
|
||||
if err := jc.Connect(server.URL, "/job/1", "token123"); err != nil {
|
||||
t.Fatalf("Connect failed: %v", err)
|
||||
}
|
||||
defer jc.Close()
|
||||
|
||||
jc.Complete(42, false, errors.New("blender failed: signal: segmentation fault (core dumped)"), true)
|
||||
|
||||
select {
|
||||
case msg := <-received:
|
||||
if msg["type"] != "task_complete" {
|
||||
t.Fatalf("type = %v, want task_complete", msg["type"])
|
||||
}
|
||||
data, ok := msg["data"].(map[string]interface{})
|
||||
if !ok {
|
||||
// gorilla may leave nested objects as map[string]interface{} after JSON round-trip;
|
||||
// also accept raw re-marshal path
|
||||
raw, _ := json.Marshal(msg["data"])
|
||||
data = map[string]interface{}{}
|
||||
if err := json.Unmarshal(raw, &data); err != nil {
|
||||
t.Fatalf("data type %T: %v", msg["data"], err)
|
||||
}
|
||||
}
|
||||
errVal, ok := data["error"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("error field type %T value %#v; want string (not object)", data["error"], data["error"])
|
||||
}
|
||||
if errVal != "blender failed: signal: segmentation fault (core dumped)" {
|
||||
t.Fatalf("error = %q, want segfault message", errVal)
|
||||
}
|
||||
if data["success"] != false {
|
||||
t.Fatalf("success = %v, want false", data["success"])
|
||||
}
|
||||
if data["free_requeue"] != true {
|
||||
t.Fatalf("free_requeue = %v, want true", data["free_requeue"])
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("timed out waiting for task_complete message")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobConnection_ConnectAndClose(t *testing.T) {
|
||||
upgrader := websocket.Upgrader{}
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -114,3 +114,18 @@ func isGPUControllerLine(line string) bool {
|
||||
strings.Contains(line, "3d controller") ||
|
||||
strings.Contains(line, "display controller")
|
||||
}
|
||||
|
||||
func parseRocmAgentArch(output string) (arch string, ok bool) {
|
||||
scanner := bufio.NewScanner(strings.NewReader(output))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if !strings.HasPrefix(line, "Name:") {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimSpace(strings.TrimPrefix(line, "Name:"))
|
||||
if strings.HasPrefix(name, "gfx") {
|
||||
return name, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
package blender
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestParseRocmAgentArch(t *testing.T) {
|
||||
input := `ROCk module is loaded
|
||||
Agent 1
|
||||
Name: gfx1151
|
||||
Marketing Name: AMD Radeon Graphics
|
||||
`
|
||||
arch, ok := parseRocmAgentArch(input)
|
||||
if !ok {
|
||||
t.Fatal("expected arch to be parsed")
|
||||
}
|
||||
if arch != "gfx1151" {
|
||||
t.Fatalf("arch = %q, want gfx1151", arch)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,26 +19,11 @@ const (
|
||||
CRFVP9 = 30
|
||||
)
|
||||
|
||||
// tonemapFilter returns the appropriate filter for EXR input.
|
||||
// For HDR preservation: converts linear RGB (EXR) to bt2020 YUV with HLG transfer function
|
||||
// Uses zscale to properly convert colorspace from linear RGB to bt2020 YUV while preserving HDR range
|
||||
// Step 1: Ensure format is gbrpf32le (linear RGB)
|
||||
// Step 2: Convert transfer function from linear to HLG (arib-std-b67) with bt2020 primaries/matrix
|
||||
// Step 3: Convert to YUV format
|
||||
// tonemapFilter returns the production filter for EXR input (linear → sRGB → HLG, bt709).
|
||||
// This is the single source of truth used by BuildCommand and BuildPass1Command.
|
||||
// zscale numeric values: primaries 1=bt709, matrix 1=bt709, transfer 8=linear / 13=sRGB / 18=HLG.
|
||||
func tonemapFilter(useAlpha bool) string {
|
||||
// Convert from linear RGB (gbrpf32le) to HLG with bt709 primaries to match PNG appearance
|
||||
// Based on best practices: convert linear RGB directly to HLG with bt709 primaries
|
||||
// This matches PNG color appearance (bt709 primaries) while preserving HDR range (HLG transfer)
|
||||
// zscale uses numeric values:
|
||||
// primaries: 1=bt709 (matches PNG), 9=bt2020
|
||||
// matrix: 1=bt709, 9=bt2020nc, 0=gbr (RGB input)
|
||||
// transfer: 8=linear, 18=arib-std-b67 (HLG)
|
||||
// Direct conversion: linear RGB -> HLG with bt709 primaries -> bt2020 YUV (for wider gamut metadata)
|
||||
// The bt709 primaries in the conversion match PNG, but we set bt2020 in metadata for HDR displays
|
||||
// Convert linear RGB to sRGB first, then convert to HLG
|
||||
// This approach: linear -> sRGB -> HLG -> bt2020
|
||||
// Fixes red tint by using sRGB conversion, preserves HDR range with HLG
|
||||
filter := "format=gbrpf32le,zscale=transferin=8:transfer=13:primariesin=1:primaries=1:matrixin=0:matrix=1:rangein=full:range=full,zscale=transferin=13:transfer=18:primariesin=1:primaries=9:matrixin=1:matrix=9:rangein=full:range=full"
|
||||
filter := "format=gbrpf32le,zscale=transferin=8:transfer=13:primariesin=1:primaries=1:matrixin=0:matrix=1:rangein=full:range=full,zscale=transferin=13:transfer=18:primariesin=1:primaries=1:matrixin=1:matrix=1:rangein=full:range=full"
|
||||
if useAlpha {
|
||||
return filter + ",format=yuva420p10le"
|
||||
}
|
||||
@@ -57,8 +42,7 @@ func (e *SoftwareEncoder) Available() bool {
|
||||
return true // Software encoding is always available
|
||||
}
|
||||
|
||||
func (e *SoftwareEncoder) BuildCommand(config *EncodeConfig) *exec.Cmd {
|
||||
// EXR only: HDR path (HLG, 10-bit, full range)
|
||||
func (e *SoftwareEncoder) buildBaseArgs(config *EncodeConfig) []string {
|
||||
pixFmt := "yuv420p10le"
|
||||
if config.UseAlpha {
|
||||
pixFmt = "yuva420p10le"
|
||||
@@ -79,14 +63,18 @@ func (e *SoftwareEncoder) BuildCommand(config *EncodeConfig) *exec.Cmd {
|
||||
"-color_trc", "linear", "-color_primaries", "bt709"}
|
||||
args = append(args, "-i", config.InputPattern, "-c:v", e.codec, "-pix_fmt", pixFmt, "-r", fmt.Sprintf("%.2f", config.FrameRate), "-color_primaries", colorPrimaries, "-color_trc", colorTrc, "-colorspace", colorspace, "-color_range", colorRange)
|
||||
|
||||
vf := "format=gbrpf32le,zscale=transferin=8:transfer=13:primariesin=1:primaries=1:matrixin=0:matrix=1:rangein=full:range=full,zscale=transferin=13:transfer=18:primariesin=1:primaries=1:matrixin=1:matrix=1:rangein=full:range=full"
|
||||
if config.UseAlpha {
|
||||
vf += ",format=yuva420p10le"
|
||||
} else {
|
||||
vf += ",format=yuv420p10le"
|
||||
// FFmpeg 6+ treats yuva420p10le as experimental for libvpx-vp9 / libaom-av1
|
||||
if config.UseAlpha && (e.codec == "libvpx-vp9" || e.codec == "libaom-av1") {
|
||||
args = append(args, "-strict", "experimental")
|
||||
}
|
||||
args = append(args, "-vf", vf)
|
||||
|
||||
args = append(args, "-vf", tonemapFilter(config.UseAlpha))
|
||||
args = append(args, codecArgs...)
|
||||
return args
|
||||
}
|
||||
|
||||
func (e *SoftwareEncoder) BuildCommand(config *EncodeConfig) *exec.Cmd {
|
||||
args := e.buildBaseArgs(config)
|
||||
|
||||
if config.TwoPass {
|
||||
// For 2-pass, this builds pass 2 command
|
||||
@@ -107,35 +95,7 @@ func (e *SoftwareEncoder) BuildCommand(config *EncodeConfig) *exec.Cmd {
|
||||
|
||||
// BuildPass1Command builds the first pass command for 2-pass encoding.
|
||||
func (e *SoftwareEncoder) BuildPass1Command(config *EncodeConfig) *exec.Cmd {
|
||||
pixFmt := "yuv420p10le"
|
||||
if config.UseAlpha {
|
||||
pixFmt = "yuva420p10le"
|
||||
}
|
||||
colorPrimaries, colorTrc, colorspace, colorRange := "bt709", "arib-std-b67", "bt709", "pc"
|
||||
|
||||
var codecArgs []string
|
||||
switch e.codec {
|
||||
case "libaom-av1":
|
||||
codecArgs = []string{"-crf", strconv.Itoa(CRFAV1), "-b:v", "0", "-tiles", "2x2", "-g", "240"}
|
||||
case "libvpx-vp9":
|
||||
codecArgs = []string{"-crf", strconv.Itoa(CRFVP9), "-b:v", "0", "-row-mt", "1", "-g", "240"}
|
||||
default:
|
||||
codecArgs = []string{"-preset", "veryslow", "-crf", strconv.Itoa(CRFH264), "-profile:v", "high10", "-level", "5.2", "-tune", "film", "-keyint_min", "24", "-g", "240", "-bf", "2", "-refs", "4"}
|
||||
}
|
||||
|
||||
args := []string{"-y", "-f", "image2", "-start_number", fmt.Sprintf("%d", config.StartFrame), "-framerate", fmt.Sprintf("%.2f", config.FrameRate),
|
||||
"-color_trc", "linear", "-color_primaries", "bt709"}
|
||||
args = append(args, "-i", config.InputPattern, "-c:v", e.codec, "-pix_fmt", pixFmt, "-r", fmt.Sprintf("%.2f", config.FrameRate), "-color_primaries", colorPrimaries, "-color_trc", colorTrc, "-colorspace", colorspace, "-color_range", colorRange)
|
||||
|
||||
vf := "format=gbrpf32le,zscale=transferin=8:transfer=13:primariesin=1:primaries=1:matrixin=0:matrix=1:rangein=full:range=full,zscale=transferin=13:transfer=18:primariesin=1:primaries=1:matrixin=1:matrix=1:rangein=full:range=full"
|
||||
if config.UseAlpha {
|
||||
vf += ",format=yuva420p10le"
|
||||
} else {
|
||||
vf += ",format=yuv420p10le"
|
||||
}
|
||||
args = append(args, "-vf", vf)
|
||||
|
||||
args = append(args, codecArgs...)
|
||||
args := e.buildBaseArgs(config)
|
||||
args = append(args, "-pass", "1", "-f", "null", "/dev/null")
|
||||
|
||||
log.Printf("Build Software Pass 1 command: ffmpeg %s", strings.Join(args, " "))
|
||||
|
||||
@@ -120,6 +120,10 @@ func TestSoftwareEncoder_BuildCommand_AV1_WithAlpha(t *testing.T) {
|
||||
if !strings.Contains(argsStr, "format=yuva420p10le") {
|
||||
t.Error("Expected tonemap filter to output yuva420p10le for EXR alpha, but not found")
|
||||
}
|
||||
// Alpha VP9/AV1 need -strict experimental on modern FFmpeg
|
||||
if !strings.Contains(argsStr, "-strict experimental") {
|
||||
t.Error("Expected -strict experimental for alpha encode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSoftwareEncoder_BuildCommand_VP9(t *testing.T) {
|
||||
|
||||
+154
-18
@@ -10,6 +10,7 @@ import (
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"jiggablend/internal/runner/api"
|
||||
"jiggablend/internal/runner/blender"
|
||||
"jiggablend/internal/runner/encoding"
|
||||
"jiggablend/internal/runner/sandbox"
|
||||
"jiggablend/internal/runner/tasks"
|
||||
"jiggablend/internal/runner/workspace"
|
||||
"jiggablend/pkg/executils"
|
||||
@@ -58,12 +60,56 @@ type Runner struct {
|
||||
|
||||
// forceCPURendering forces CPU rendering for all jobs regardless of metadata/backend detection.
|
||||
forceCPURendering bool
|
||||
// disableRT disables GPU ray tracing acceleration (HIPRT, OptiX, etc.).
|
||||
disableRT bool
|
||||
// hipGPUSampleBatch limits samples per GPU pass on gfx115x (0 = disabled).
|
||||
hipGPUSampleBatch int
|
||||
|
||||
// sandbox wraps Blender invocations (none/podman).
|
||||
sandboxWrapper sandbox.Wrapper
|
||||
sandboxBackend string
|
||||
}
|
||||
|
||||
// RunnerOptions configures optional runner behavior.
|
||||
type RunnerOptions struct {
|
||||
ForceCPURendering bool
|
||||
DisableRT bool
|
||||
HipGPUSampleBatch int
|
||||
SandboxBackend string // none|podman
|
||||
SandboxNetwork bool
|
||||
SandboxImage string // podman thin runtime image
|
||||
}
|
||||
|
||||
// New creates a new runner.
|
||||
func New(managerURL, name, hostname string, forceCPURendering bool) *Runner {
|
||||
func New(managerURL, name, hostname string, forceCPURendering, disableRT bool, hipGPUSampleBatch int) *Runner {
|
||||
return NewWithOptions(managerURL, name, hostname, RunnerOptions{
|
||||
ForceCPURendering: forceCPURendering,
|
||||
DisableRT: disableRT,
|
||||
HipGPUSampleBatch: hipGPUSampleBatch,
|
||||
SandboxBackend: sandbox.BackendPodman,
|
||||
})
|
||||
}
|
||||
|
||||
// NewWithOptions creates a runner with full options including sandbox.
|
||||
func NewWithOptions(managerURL, name, hostname string, opts RunnerOptions) *Runner {
|
||||
manager := api.NewManagerClient(managerURL)
|
||||
|
||||
backend, err := sandbox.NormalizeBackend(opts.SandboxBackend)
|
||||
if err != nil {
|
||||
log.Printf("Invalid sandbox backend %q, falling back to none: %v", opts.SandboxBackend, err)
|
||||
backend = sandbox.BackendNone
|
||||
}
|
||||
sb, err := sandbox.New(sandbox.Options{
|
||||
Backend: backend,
|
||||
AllowNetwork: opts.SandboxNetwork,
|
||||
PodmanImage: opts.SandboxImage,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to init sandbox %q: %v; using none", backend, err)
|
||||
sb, _ = sandbox.New(sandbox.Options{Backend: sandbox.BackendNone})
|
||||
backend = sandbox.BackendNone
|
||||
}
|
||||
|
||||
r := &Runner{
|
||||
name: name,
|
||||
hostname: hostname,
|
||||
@@ -72,7 +118,11 @@ func New(managerURL, name, hostname string, forceCPURendering bool) *Runner {
|
||||
stopChan: make(chan struct{}),
|
||||
processors: make(map[string]tasks.Processor),
|
||||
|
||||
forceCPURendering: forceCPURendering,
|
||||
forceCPURendering: opts.ForceCPURendering,
|
||||
disableRT: opts.DisableRT,
|
||||
hipGPUSampleBatch: opts.HipGPUSampleBatch,
|
||||
sandboxWrapper: sb,
|
||||
sandboxBackend: backend,
|
||||
}
|
||||
|
||||
// Generate fingerprint
|
||||
@@ -88,9 +138,24 @@ func (r *Runner) CheckRequiredTools() error {
|
||||
}
|
||||
log.Printf("Found zstd for compressed blend file support")
|
||||
|
||||
if r.sandboxWrapper != nil {
|
||||
if err := r.sandboxWrapper.Available(); err != nil {
|
||||
return fmt.Errorf("sandbox backend %q unavailable: %w", r.sandboxBackend, err)
|
||||
}
|
||||
log.Printf("Sandbox backend: %s", r.sandboxWrapper.Name())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SandboxBackend returns the configured sandbox backend name.
|
||||
func (r *Runner) SandboxBackend() string {
|
||||
if r.sandboxBackend == "" {
|
||||
return sandbox.BackendNone
|
||||
}
|
||||
return r.sandboxBackend
|
||||
}
|
||||
|
||||
var (
|
||||
cachedCapabilities map[string]interface{}
|
||||
capabilitiesOnce sync.Once
|
||||
@@ -107,8 +172,21 @@ func (r *Runner) ProbeCapabilities() map[string]interface{} {
|
||||
caps["ffmpeg"] = false
|
||||
}
|
||||
|
||||
// Filled later with actual backend when runner is constructed; probe is package-level once.
|
||||
// Real sandbox name is injected in ProbeCapabilities on the instance after New.
|
||||
caps["sandbox"] = "none"
|
||||
|
||||
cachedCapabilities = caps
|
||||
})
|
||||
// Overlay instance sandbox name (Once already ran with none default).
|
||||
if r != nil && r.sandboxWrapper != nil {
|
||||
out := make(map[string]interface{}, len(cachedCapabilities)+1)
|
||||
for k, v := range cachedCapabilities {
|
||||
out[k] = v
|
||||
}
|
||||
out["sandbox"] = r.sandboxWrapper.Name()
|
||||
return out
|
||||
}
|
||||
return cachedCapabilities
|
||||
}
|
||||
|
||||
@@ -198,6 +276,16 @@ func (r *Runner) HasIntel() bool {
|
||||
return r.hasIntel
|
||||
}
|
||||
|
||||
// DisableRT returns whether GPU ray tracing acceleration should be disabled.
|
||||
func (r *Runner) DisableRT() bool {
|
||||
return r.disableRT
|
||||
}
|
||||
|
||||
// HipGPUSampleBatch returns the per-pass GPU sample limit for gfx115x batching (0 = disabled).
|
||||
func (r *Runner) HipGPUSampleBatch() int {
|
||||
return r.hipGPUSampleBatch
|
||||
}
|
||||
|
||||
// GPUDetectionFailed returns true when startup GPU backend detection could not run or failed. When true, all jobs use CPU because backend availability is unknown.
|
||||
func (r *Runner) GPUDetectionFailed() bool {
|
||||
r.gpuBackendMu.RLock()
|
||||
@@ -330,8 +418,20 @@ func (r *Runner) executeJob(job *api.NextJobResponse) (err error) {
|
||||
r.HasIntel(),
|
||||
r.GPUDetectionFailed(),
|
||||
r.forceCPURendering,
|
||||
func() { r.SetGPULockedOut(true) },
|
||||
r.disableRT,
|
||||
r.hipGPUSampleBatch,
|
||||
nil, // set below so the callback can mark this attempt
|
||||
r.sandboxWrapper,
|
||||
)
|
||||
// Arm GPU lockout at most once process-wide; if this attempt is the one that
|
||||
// arms it, mark the context so a failure requeues without burning retry_count.
|
||||
ctx.OnGPUError = func() {
|
||||
if r.SetGPULockedOut(true) {
|
||||
ctx.GPULockoutArmedThisAttempt = true
|
||||
ctx.GPULockedOut = true
|
||||
ctx.Warn("GPU error detected; GPU disabled for subsequent jobs (this attempt free-requeues without using a retry)")
|
||||
}
|
||||
}
|
||||
|
||||
ctx.Info(fmt.Sprintf("Task assignment received (job: %d, type: %s)",
|
||||
job.Task.JobID, job.Task.TaskType))
|
||||
@@ -350,7 +450,7 @@ func (r *Runner) executeJob(job *api.NextJobResponse) (err error) {
|
||||
contextPath := job.JobPath + "/context.tar"
|
||||
if err := r.downloadContext(job.Task.JobID, contextPath, job.JobToken); err != nil {
|
||||
jobConn.Log(job.Task.TaskID, types.LogLevelError, fmt.Sprintf("Failed to download context: %v", err))
|
||||
jobConn.Complete(job.Task.TaskID, false, fmt.Errorf("failed to download context: %v", err))
|
||||
jobConn.Complete(job.Task.TaskID, false, fmt.Errorf("failed to download context: %v", err), false)
|
||||
return fmt.Errorf("failed to download context: %w", err)
|
||||
}
|
||||
processErr = processor.Process(ctx)
|
||||
@@ -393,27 +493,56 @@ func (r *Runner) uploadOutputs(ctx *tasks.Context, job *api.NextJobResponse) err
|
||||
outputDir := ctx.WorkDir + "/output"
|
||||
uploadPath := fmt.Sprintf("/api/runner/jobs/%d/upload", job.Task.JobID)
|
||||
|
||||
return uploadOutputFiles(outputDir, func(filePath, fileName string) error {
|
||||
if err := r.manager.UploadFile(uploadPath, job.JobToken, filePath); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx.OutputUploaded(fileName)
|
||||
// Delete file after successful upload to prevent duplicate uploads
|
||||
if err := os.Remove(filePath); err != nil {
|
||||
log.Printf("Warning: Failed to delete file %s after upload: %v", filePath, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// uploadOutputFiles uploads all non-directory files from outputDir.
|
||||
// Fails if the directory cannot be read, any upload fails, or zero files were uploaded.
|
||||
func uploadOutputFiles(outputDir string, uploadFn func(filePath, fileName string) error) error {
|
||||
entries, err := os.ReadDir(outputDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read output directory: %w", err)
|
||||
}
|
||||
|
||||
var files []os.DirEntry
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
filePath := outputDir + "/" + entry.Name()
|
||||
if err := r.manager.UploadFile(uploadPath, job.JobToken, filePath); err != nil {
|
||||
log.Printf("Failed to upload %s: %v", filePath, err)
|
||||
} else {
|
||||
ctx.OutputUploaded(entry.Name())
|
||||
// Delete file after successful upload to prevent duplicate uploads
|
||||
if err := os.Remove(filePath); err != nil {
|
||||
log.Printf("Warning: Failed to delete file %s after upload: %v", filePath, err)
|
||||
}
|
||||
if !entry.IsDir() {
|
||||
files = append(files, entry)
|
||||
}
|
||||
}
|
||||
if len(files) == 0 {
|
||||
return fmt.Errorf("no output files found in %s", outputDir)
|
||||
}
|
||||
|
||||
var firstErr error
|
||||
uploaded := 0
|
||||
for _, entry := range files {
|
||||
filePath := filepath.Join(outputDir, entry.Name())
|
||||
if err := uploadFn(filePath, entry.Name()); err != nil {
|
||||
log.Printf("Failed to upload %s: %v", filePath, err)
|
||||
if firstErr == nil {
|
||||
firstErr = fmt.Errorf("failed to upload %s: %w", entry.Name(), err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
uploaded++
|
||||
}
|
||||
if firstErr != nil {
|
||||
return firstErr
|
||||
}
|
||||
if uploaded == 0 {
|
||||
return fmt.Errorf("no output files were uploaded from %s", outputDir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -484,13 +613,20 @@ func (r *Runner) GetID() int64 {
|
||||
|
||||
// SetGPULockedOut sets whether GPU use is locked out due to a detected GPU error.
|
||||
// When true, the runner will force CPU rendering for all jobs.
|
||||
func (r *Runner) SetGPULockedOut(locked bool) {
|
||||
// Returns true only on the false→true transition (first arm); subsequent calls are no-ops for logging.
|
||||
func (r *Runner) SetGPULockedOut(locked bool) (newlyEnabled bool) {
|
||||
r.gpuLockedOutMu.Lock()
|
||||
defer r.gpuLockedOutMu.Unlock()
|
||||
r.gpuLockedOut = locked
|
||||
if locked {
|
||||
if r.gpuLockedOut {
|
||||
return false
|
||||
}
|
||||
r.gpuLockedOut = true
|
||||
log.Printf("GPU lockout enabled: GPU rendering disabled for subsequent jobs (CPU only)")
|
||||
return true
|
||||
}
|
||||
r.gpuLockedOut = false
|
||||
return false
|
||||
}
|
||||
|
||||
// IsGPULockedOut returns whether GPU use is currently locked out.
|
||||
|
||||
@@ -2,11 +2,14 @@ package runner
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewRunner_InitializesFields(t *testing.T) {
|
||||
r := New("http://localhost:8080", "runner-a", "host-a", false)
|
||||
r := New("http://localhost:8080", "runner-a", "host-a", false, false, 0)
|
||||
if r == nil {
|
||||
t.Fatal("New should return a runner")
|
||||
}
|
||||
@@ -16,15 +19,26 @@ func TestNewRunner_InitializesFields(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunner_GPUFlagsSetters(t *testing.T) {
|
||||
r := New("http://localhost:8080", "runner-a", "host-a", false)
|
||||
r.SetGPULockedOut(true)
|
||||
r := New("http://localhost:8080", "runner-a", "host-a", false, false, 0)
|
||||
if newly := r.SetGPULockedOut(true); !newly {
|
||||
t.Fatal("expected first SetGPULockedOut(true) to report newly enabled")
|
||||
}
|
||||
if !r.IsGPULockedOut() {
|
||||
t.Fatal("expected GPU lockout to be true")
|
||||
}
|
||||
if newly := r.SetGPULockedOut(true); newly {
|
||||
t.Fatal("expected second SetGPULockedOut(true) to be a no-op transition")
|
||||
}
|
||||
if newly := r.SetGPULockedOut(false); newly {
|
||||
t.Fatal("clearing lockout should not report newly enabled")
|
||||
}
|
||||
if r.IsGPULockedOut() {
|
||||
t.Fatal("expected GPU lockout cleared")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateFingerprint_PopulatesValue(t *testing.T) {
|
||||
r := New("http://localhost:8080", "runner-a", "host-a", false)
|
||||
r := New("http://localhost:8080", "runner-a", "host-a", false, false, 0)
|
||||
r.generateFingerprint()
|
||||
fp := r.GetFingerprint()
|
||||
if fp == "" {
|
||||
@@ -38,3 +52,45 @@ func TestGenerateFingerprint_PopulatesValue(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadOutputFiles_FailsWhenUploadErrors(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "frame_0001.exr"), []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := uploadOutputFiles(dir, func(filePath, fileName string) error {
|
||||
return errors.New("upload denied")
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error when upload fails")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadOutputFiles_FailsWhenEmpty(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
err := uploadOutputFiles(dir, func(filePath, fileName string) error {
|
||||
t.Fatal("upload should not be called")
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for empty output dir")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadOutputFiles_Success(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "frame_0001.exr"), []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var seen string
|
||||
err := uploadOutputFiles(dir, func(filePath, fileName string) error {
|
||||
seen = fileName
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("uploadOutputFiles: %v", err)
|
||||
}
|
||||
if seen != "frame_0001.exr" {
|
||||
t.Fatalf("got %q", seen)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Bind describes a host path to expose inside the sandbox.
|
||||
type Bind struct {
|
||||
Host string
|
||||
// Dest is the path inside the sandbox (usually same as Host for absolute paths).
|
||||
Dest string
|
||||
// ReadOnly is true for library trees and Blender installs.
|
||||
ReadOnly bool
|
||||
// Dev is true for device nodes / device trees (podman --device or -v for dirs).
|
||||
Dev bool
|
||||
// Optional means skip if host path missing (no error).
|
||||
Optional bool
|
||||
}
|
||||
|
||||
// GPUMounts returns host paths to expose for Cycles GPU backends based on detection.
|
||||
// forceCPU skips GPU-specific devices/libs (still may include generic /dev via backend).
|
||||
func GPUMounts(hasAMD, hasNVIDIA, hasIntel, forceCPU bool) []Bind {
|
||||
if forceCPU {
|
||||
return nil
|
||||
}
|
||||
var binds []Bind
|
||||
|
||||
// DRM render nodes used by AMD, Intel, and some NVIDIA EGL paths.
|
||||
if hasAMD || hasNVIDIA || hasIntel {
|
||||
binds = append(binds, Bind{Host: "/dev/dri", Dest: "/dev/dri", Dev: true, Optional: true})
|
||||
}
|
||||
|
||||
if hasAMD {
|
||||
binds = append(binds, Bind{Host: "/dev/kfd", Dest: "/dev/kfd", Dev: true, Optional: true})
|
||||
// Common ROCm install layouts
|
||||
for _, p := range []string{"/opt/rocm", "/usr/share/libdrm"} {
|
||||
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||
}
|
||||
// Distro ROCm / amdgpu userspace libs often live under multiarch paths
|
||||
for _, p := range rocmLibHints() {
|
||||
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||
}
|
||||
}
|
||||
|
||||
if hasNVIDIA {
|
||||
for _, name := range nvidiaDeviceNames() {
|
||||
p := filepath.Join("/dev", name)
|
||||
binds = append(binds, Bind{Host: p, Dest: p, Dev: true, Optional: true})
|
||||
}
|
||||
for _, p := range nvidiaLibHints() {
|
||||
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||
}
|
||||
}
|
||||
|
||||
if hasIntel {
|
||||
for _, p := range intelLibHints() {
|
||||
binds = append(binds, Bind{Host: p, Dest: p, ReadOnly: true, Optional: true})
|
||||
}
|
||||
}
|
||||
|
||||
return uniqueBinds(binds)
|
||||
}
|
||||
|
||||
func nvidiaDeviceNames() []string {
|
||||
// Fixed control nodes + any /dev/nvidiaN
|
||||
names := []string{"nvidiactl", "nvidia-uvm", "nvidia-uvm-tools", "nvidia-modeset"}
|
||||
entries, err := os.ReadDir("/dev")
|
||||
if err != nil {
|
||||
return names
|
||||
}
|
||||
for _, e := range entries {
|
||||
n := e.Name()
|
||||
if strings.HasPrefix(n, "nvidia") && !containsString(names, n) {
|
||||
names = append(names, n)
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func nvidiaLibHints() []string {
|
||||
hints := []string{
|
||||
"/usr/lib/wsl/lib", // WSL NVIDIA
|
||||
"/usr/local/nvidia",
|
||||
"/usr/local/cuda",
|
||||
"/usr/lib/x86_64-linux-gnu",
|
||||
"/usr/lib64",
|
||||
}
|
||||
// Driver stores often under /usr/lib/libcuda* — parent dirs already covered.
|
||||
// Also scan common multiarch for libcuda.so
|
||||
for _, dir := range []string{"/usr/lib", "/usr/lib64", "/usr/lib/x86_64-linux-gnu", "/lib", "/lib64"} {
|
||||
if matchesAny(dir, "libcuda.so*") || matchesAny(dir, "libnvidia-*.so*") {
|
||||
hints = append(hints, dir)
|
||||
}
|
||||
}
|
||||
return hints
|
||||
}
|
||||
|
||||
func rocmLibHints() []string {
|
||||
hints := []string{
|
||||
"/usr/lib/x86_64-linux-gnu",
|
||||
"/usr/lib64",
|
||||
"/opt/amdgpu",
|
||||
}
|
||||
// ROCm versioned trees under /opt/rocm-*
|
||||
if entries, err := os.ReadDir("/opt"); err == nil {
|
||||
for _, e := range entries {
|
||||
if e.IsDir() && strings.HasPrefix(e.Name(), "rocm") {
|
||||
hints = append(hints, filepath.Join("/opt", e.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
return hints
|
||||
}
|
||||
|
||||
func intelLibHints() []string {
|
||||
return []string{
|
||||
"/usr/lib/x86_64-linux-gnu",
|
||||
"/usr/lib64",
|
||||
"/usr/lib/intel-opencl",
|
||||
"/etc/OpenCL",
|
||||
}
|
||||
}
|
||||
|
||||
func matchesAny(dir, glob string) bool {
|
||||
m, err := filepath.Glob(filepath.Join(dir, glob))
|
||||
return err == nil && len(m) > 0
|
||||
}
|
||||
|
||||
func containsString(ss []string, s string) bool {
|
||||
for _, x := range ss {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func uniqueBinds(in []Bind) []Bind {
|
||||
seen := make(map[string]bool)
|
||||
var out []Bind
|
||||
for _, b := range in {
|
||||
key := b.Host + "|" + b.Dest + "|"
|
||||
if b.Dev {
|
||||
key += "d"
|
||||
}
|
||||
if b.ReadOnly {
|
||||
key += "r"
|
||||
}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, b)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ExistingBinds filters to binds whose host path exists (or non-optional always kept for error reporting).
|
||||
func ExistingBinds(binds []Bind) []Bind {
|
||||
var out []Bind
|
||||
for _, b := range binds {
|
||||
if pathExists(b.Host) {
|
||||
out = append(out, b)
|
||||
continue
|
||||
}
|
||||
if !b.Optional {
|
||||
out = append(out, b) // caller may error
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package sandbox
|
||||
|
||||
import "os/exec"
|
||||
|
||||
type noneWrapper struct{}
|
||||
|
||||
func (w *noneWrapper) Name() string { return BackendNone }
|
||||
|
||||
func (w *noneWrapper) Available() error { return nil }
|
||||
|
||||
func (w *noneWrapper) Wrap(spec Spec) (*exec.Cmd, error) {
|
||||
cmd := exec.Command(spec.BlenderBinary, spec.Args...)
|
||||
cmd.Dir = spec.WorkDir
|
||||
if len(spec.Env) > 0 {
|
||||
cmd.Env = spec.Env
|
||||
}
|
||||
return cmd, nil
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package sandbox
|
||||
|
||||
import "path/filepath"
|
||||
|
||||
func execAbs(p string) (string, error) {
|
||||
return filepath.Abs(p)
|
||||
}
|
||||
|
||||
// blenderRoot returns the directory that contains the blender binary
|
||||
// (the version extract root, e.g. .../blender-versions/4.5.7).
|
||||
func blenderRoot(blenderBinary string) string {
|
||||
return filepath.Dir(mustAbs(blenderBinary))
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultPodmanImage = "registry.fedoraproject.org/fedora-minimal:41"
|
||||
|
||||
type podmanWrapper struct {
|
||||
allowNetwork bool
|
||||
image string
|
||||
}
|
||||
|
||||
func (w *podmanWrapper) Name() string { return BackendPodman }
|
||||
|
||||
func (w *podmanWrapper) Available() error {
|
||||
if _, err := LookPath("podman"); err != nil {
|
||||
return fmt.Errorf("podman not found in PATH: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *podmanWrapper) Wrap(spec Spec) (*exec.Cmd, error) {
|
||||
if err := w.Available(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bin := mustAbs(spec.BlenderBinary)
|
||||
work := mustAbs(spec.WorkDir)
|
||||
home := mustAbs(spec.HomeDir)
|
||||
if home == "" {
|
||||
home = filepath.Join(work, "home")
|
||||
}
|
||||
root := blenderRoot(bin)
|
||||
|
||||
args := []string{
|
||||
"run", "--rm",
|
||||
"--security-opt", "label=disable",
|
||||
// Keep user groups for /dev/dri access (render/video)
|
||||
"--group-add", "keep-groups",
|
||||
}
|
||||
if !w.allowNetwork {
|
||||
args = append(args, "--network=none")
|
||||
}
|
||||
// Map to same UID so bind mounts are writable
|
||||
uid := os.Getuid()
|
||||
gid := os.Getgid()
|
||||
args = append(args, "--user", fmt.Sprintf("%d:%d", uid, gid))
|
||||
|
||||
// Thin OS image + host Blender tree + job dir
|
||||
args = append(args,
|
||||
"-v", root+":"+root+":ro",
|
||||
"-v", work+":"+work+":rw",
|
||||
)
|
||||
if home != work && !strings.HasPrefix(home, work+string(os.PathSeparator)) {
|
||||
_ = os.MkdirAll(home, 0755)
|
||||
args = append(args, "-v", home+":"+home+":rw")
|
||||
}
|
||||
|
||||
// System libs for GPU ICDs / dynamic linker (Blender tarball is mostly self-contained)
|
||||
for _, p := range []string{"/usr", "/lib", "/lib64", "/etc"} {
|
||||
if pathExists(p) {
|
||||
args = append(args, "-v", p+":"+p+":ro")
|
||||
}
|
||||
}
|
||||
|
||||
// GPU devices and extra lib roots
|
||||
for _, b := range ExistingBinds(GPUMounts(spec.HasAMD, spec.HasNVIDIA, spec.HasIntel, spec.ForceCPU)) {
|
||||
if !pathExists(b.Host) {
|
||||
continue
|
||||
}
|
||||
if b.Dev {
|
||||
// --device works for character devices; for /dev/dri use volume
|
||||
info, err := os.Stat(b.Host)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if info.IsDir() {
|
||||
args = append(args, "-v", b.Host+":"+b.Dest+":ro")
|
||||
} else {
|
||||
args = append(args, "--device", b.Host)
|
||||
}
|
||||
continue
|
||||
}
|
||||
mode := "ro"
|
||||
if !b.ReadOnly {
|
||||
mode = "rw"
|
||||
}
|
||||
args = append(args, "-v", b.Host+":"+b.Dest+":"+mode)
|
||||
}
|
||||
|
||||
// Environment
|
||||
for _, e := range spec.Env {
|
||||
args = append(args, "-e", e)
|
||||
}
|
||||
if home != "" {
|
||||
args = append(args, "-e", "HOME="+home)
|
||||
}
|
||||
|
||||
args = append(args, "--workdir", work)
|
||||
args = append(args, "--entrypoint", bin)
|
||||
args = append(args, w.image)
|
||||
// entrypoint is blender; remaining args are blender args
|
||||
args = append(args, spec.Args...)
|
||||
|
||||
cmd := exec.Command("podman", args...)
|
||||
cmd.Dir = work
|
||||
return cmd, nil
|
||||
}
|
||||
|
||||
// PodmanImageDefault returns the default thin runtime image name.
|
||||
func PodmanImageDefault() string { return defaultPodmanImage }
|
||||
|
||||
// FormatUIDGID is a small helper for tests.
|
||||
func FormatUIDGID(uid, gid int) string {
|
||||
return strconv.Itoa(uid) + ":" + strconv.Itoa(gid)
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
// Package sandbox wraps Blender (and similar) invocations in optional isolation.
|
||||
//
|
||||
// Backends:
|
||||
// - none: run on the host (current behavior)
|
||||
// - podman: rootless podman container; Blender tarball is bind-mounted (not baked into an image)
|
||||
//
|
||||
// GPU access is provided by passing host device nodes and common userspace lib roots
|
||||
// discovered at wrap time (NVIDIA / AMD ROCm / Intel DRM), not by shipping per-version
|
||||
// Blender container images.
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Backend names accepted by New and CLI flags.
|
||||
const (
|
||||
BackendNone = "none"
|
||||
BackendPodman = "podman"
|
||||
)
|
||||
|
||||
// Options configures sandbox construction.
|
||||
type Options struct {
|
||||
// Backend is none|podman (default none).
|
||||
Backend string
|
||||
// AllowNetwork keeps network in the sandbox (default false for podman).
|
||||
AllowNetwork bool
|
||||
// PodmanImage is used only for backend=podman. The image is a thin OS root;
|
||||
// Blender comes from a host bind of the versioned tarball tree.
|
||||
// Default: registry.fedoraproject.org/fedora-minimal:41
|
||||
PodmanImage string
|
||||
}
|
||||
|
||||
// Spec describes a Blender process to run under the sandbox.
|
||||
type Spec struct {
|
||||
// BlenderBinary is an absolute path to the blender executable.
|
||||
BlenderBinary string
|
||||
// Args are arguments after the binary (not including argv0).
|
||||
Args []string
|
||||
// WorkDir is the process working directory (job workspace); bind-mounted RW.
|
||||
WorkDir string
|
||||
// HomeDir is Blender HOME (usually WorkDir/home); bind-mounted RW.
|
||||
HomeDir string
|
||||
// Env is the full environment for the process (HOME, LD_LIBRARY_PATH, etc.).
|
||||
Env []string
|
||||
|
||||
// GPU hints from host detection (used to select device/lib binds).
|
||||
HasAMD bool
|
||||
HasNVIDIA bool
|
||||
HasIntel bool
|
||||
// ForceCPU skips GPU device binds when true.
|
||||
ForceCPU bool
|
||||
// AllowNetwork overrides Options.AllowNetwork when non-nil... kept simple: use Options only.
|
||||
}
|
||||
|
||||
// Wrapper turns a Spec into an *exec.Cmd ready to Start.
|
||||
type Wrapper interface {
|
||||
// Name returns the backend name.
|
||||
Name() string
|
||||
// Available reports whether required host tools exist.
|
||||
Available() error
|
||||
// Wrap builds the command. Callers own Start/Wait/pipes.
|
||||
Wrap(spec Spec) (*exec.Cmd, error)
|
||||
}
|
||||
|
||||
// New returns a sandbox Wrapper for the given options.
|
||||
func New(opts Options) (Wrapper, error) {
|
||||
backend := strings.ToLower(strings.TrimSpace(opts.Backend))
|
||||
if backend == "" {
|
||||
backend = BackendPodman
|
||||
}
|
||||
switch backend {
|
||||
case BackendNone:
|
||||
return &noneWrapper{}, nil
|
||||
case BackendPodman:
|
||||
img := opts.PodmanImage
|
||||
if img == "" {
|
||||
img = defaultPodmanImage
|
||||
}
|
||||
w := &podmanWrapper{allowNetwork: opts.AllowNetwork, image: img}
|
||||
return w, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown sandbox backend %q (want none or podman)", opts.Backend)
|
||||
}
|
||||
}
|
||||
|
||||
// NormalizeBackend validates and returns a canonical backend name.
|
||||
func NormalizeBackend(s string) (string, error) {
|
||||
b := strings.ToLower(strings.TrimSpace(s))
|
||||
if b == "" {
|
||||
return BackendPodman, nil
|
||||
}
|
||||
switch b {
|
||||
case BackendNone, BackendPodman:
|
||||
return b, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown sandbox backend %q (want none or podman)", s)
|
||||
}
|
||||
}
|
||||
|
||||
// LookPath is os/exec.LookPath, overridable in tests.
|
||||
var LookPath = exec.LookPath
|
||||
|
||||
// pathExists reports whether path exists.
|
||||
func pathExists(p string) bool {
|
||||
_, err := os.Stat(p)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// mustAbs returns an absolute path or the original on error.
|
||||
func mustAbs(p string) string {
|
||||
if p == "" {
|
||||
return p
|
||||
}
|
||||
abs, err := absPath(p)
|
||||
if err != nil {
|
||||
return p
|
||||
}
|
||||
return abs
|
||||
}
|
||||
|
||||
// absPath is filepath.Abs, isolated for tests.
|
||||
var absPath = func(p string) (string, error) {
|
||||
return execAbs(p)
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package sandbox
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeBackend(t *testing.T) {
|
||||
got, err := NormalizeBackend("PODMAN")
|
||||
if err != nil || got != BackendPodman {
|
||||
t.Fatalf("got %q err %v", got, err)
|
||||
}
|
||||
if _, err := NormalizeBackend("bwrap"); err == nil {
|
||||
t.Fatal("expected error for removed bwrap backend")
|
||||
}
|
||||
if _, err := NormalizeBackend("firecracker"); err == nil {
|
||||
t.Fatal("expected error for unknown backend")
|
||||
}
|
||||
got, err = NormalizeBackend("")
|
||||
if err != nil || got != BackendPodman {
|
||||
t.Fatalf("empty -> podman, got %q %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoneWrap_Passthrough(t *testing.T) {
|
||||
w, err := New(Options{Backend: BackendNone})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bin := "/opt/blender/blender"
|
||||
cmd, err := w.Wrap(Spec{
|
||||
BlenderBinary: bin,
|
||||
Args: []string{"-b", "scene.blend"},
|
||||
WorkDir: "/tmp/job",
|
||||
Env: []string{"HOME=/tmp/job/home"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cmd.Args) == 0 || cmd.Args[0] != bin {
|
||||
t.Fatalf("args[0]=%v path=%q", cmd.Args, cmd.Path)
|
||||
}
|
||||
if cmd.Dir != "/tmp/job" {
|
||||
t.Fatalf("Dir=%q", cmd.Dir)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUMounts_ForceCPUEmpty(t *testing.T) {
|
||||
m := GPUMounts(true, true, true, true)
|
||||
if len(m) != 0 {
|
||||
t.Fatalf("force CPU should skip GPU mounts, got %#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUMounts_AMDIncludesKFD(t *testing.T) {
|
||||
m := GPUMounts(true, false, false, false)
|
||||
var hosts []string
|
||||
for _, b := range m {
|
||||
hosts = append(hosts, b.Host)
|
||||
}
|
||||
joined := strings.Join(hosts, ",")
|
||||
if !strings.Contains(joined, "/dev/dri") {
|
||||
t.Fatalf("AMD mounts should include /dev/dri, got %v", hosts)
|
||||
}
|
||||
if !strings.Contains(joined, "/dev/kfd") {
|
||||
t.Fatalf("AMD mounts should include /dev/kfd, got %v", hosts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGPUMounts_NVIDIAIncludesCtl(t *testing.T) {
|
||||
m := GPUMounts(false, true, false, false)
|
||||
found := false
|
||||
for _, b := range m {
|
||||
if strings.Contains(b.Host, "nvidia") {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("NVIDIA mounts should include nvidia devices, got %#v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodmanWrap_BuildsRunArgs(t *testing.T) {
|
||||
w := &podmanWrapper{allowNetwork: false, image: "example.com/thin:latest"}
|
||||
if err := w.Available(); err != nil {
|
||||
t.Skipf("podman not installed: %v", err)
|
||||
}
|
||||
tmp := t.TempDir()
|
||||
blendDir := filepath.Join(tmp, "bver")
|
||||
_ = os.MkdirAll(blendDir, 0755)
|
||||
bin := filepath.Join(blendDir, "blender")
|
||||
_ = os.WriteFile(bin, []byte("x"), 0755)
|
||||
job := filepath.Join(tmp, "job")
|
||||
_ = os.MkdirAll(job, 0755)
|
||||
|
||||
cmd, err := w.Wrap(Spec{
|
||||
BlenderBinary: bin,
|
||||
Args: []string{"-b", "x.blend"},
|
||||
WorkDir: job,
|
||||
HomeDir: filepath.Join(job, "home"),
|
||||
Env: []string{"HOME=" + filepath.Join(job, "home")},
|
||||
HasNVIDIA: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
joined := strings.Join(cmd.Args, " ")
|
||||
if !strings.Contains(joined, "run") || !strings.Contains(joined, "--network=none") {
|
||||
t.Fatalf("expected podman run --network=none: %s", joined)
|
||||
}
|
||||
if !strings.Contains(joined, "example.com/thin:latest") {
|
||||
t.Fatalf("expected image in args: %s", joined)
|
||||
}
|
||||
if !strings.Contains(joined, "--entrypoint") || !strings.Contains(joined, bin) {
|
||||
t.Fatalf("expected entrypoint blender: %s", joined)
|
||||
}
|
||||
if !strings.Contains(joined, blendDir) || !strings.Contains(joined, job) {
|
||||
t.Fatalf("expected volume binds: %s", joined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNew_UnknownBackend(t *testing.T) {
|
||||
if _, err := New(Options{Backend: "bwrap"}); err == nil {
|
||||
t.Fatal("expected error for bwrap")
|
||||
}
|
||||
if _, err := New(Options{Backend: "gvisor"}); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
}
|
||||
@@ -246,6 +246,8 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
TwoPass: true,
|
||||
})
|
||||
if err := pass1Cmd.Run(); err != nil {
|
||||
// Pass 1 is analysis-only (writes to /dev/null). FFmpeg often exits non-zero
|
||||
// on benign codec/option warnings while still producing passlogfile stats.
|
||||
ctx.Warn(fmt.Sprintf("Pass 1 completed (warnings expected): %v", err))
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// mergeEXRFiles averages multiple linear EXR renders into one (Monte Carlo accumulation).
|
||||
func mergeEXRFiles(output string, inputs []string) error {
|
||||
if len(inputs) == 0 {
|
||||
return fmt.Errorf("no input EXR files to merge")
|
||||
}
|
||||
if len(inputs) == 1 {
|
||||
return copyFile(inputs[0], output)
|
||||
}
|
||||
|
||||
args := append([]string{}, inputs...)
|
||||
args = append(args, "-evaluate-sequence", "mean", output)
|
||||
|
||||
if path, err := exec.LookPath("magick"); err == nil {
|
||||
cmd := exec.Command(path, args...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("magick merge failed: %w (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if path, err := exec.LookPath("convert"); err == nil {
|
||||
cmd := exec.Command(path, args...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("convert merge failed: %w (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("ImageMagick (magick or convert) required to merge batched EXR renders")
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
cmd := exec.Command("cp", "-f", src, dst)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("copy %s to %s: %w (%s)", src, dst, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"jiggablend/internal/runner/api"
|
||||
"jiggablend/internal/runner/blender"
|
||||
"jiggablend/internal/runner/encoding"
|
||||
"jiggablend/internal/runner/sandbox"
|
||||
"jiggablend/internal/runner/workspace"
|
||||
"jiggablend/pkg/executils"
|
||||
"jiggablend/pkg/types"
|
||||
@@ -41,6 +42,9 @@ type Context struct {
|
||||
|
||||
// GPULockedOut is set when the runner has detected a GPU error (e.g. HIP) and disables GPU for all jobs.
|
||||
GPULockedOut bool
|
||||
// GPULockoutArmedThisAttempt is true when this task attempt newly enabled runner GPU lockout.
|
||||
// Failures with this set free-requeue without incrementing retry_count (expected ROCm warmup).
|
||||
GPULockoutArmedThisAttempt bool
|
||||
// HasAMD is true when the runner detected AMD devices at startup.
|
||||
HasAMD bool
|
||||
// HasNVIDIA is true when the runner detected NVIDIA GPUs at startup.
|
||||
@@ -53,6 +57,12 @@ type Context struct {
|
||||
OnGPUError func()
|
||||
// ForceCPURendering is a runner-level override that forces CPU rendering for all jobs.
|
||||
ForceCPURendering bool
|
||||
// DisableRT disables GPU ray tracing acceleration (runner-level flag).
|
||||
DisableRT bool
|
||||
// HipGPUSampleBatch limits samples per GPU render pass on gfx115x (0 = no batching).
|
||||
HipGPUSampleBatch int
|
||||
// Sandbox wraps Blender execution (none/podman). Nil means none.
|
||||
Sandbox sandbox.Wrapper
|
||||
}
|
||||
|
||||
// ErrJobCancelled indicates the manager-side job was cancelled during execution.
|
||||
@@ -80,7 +90,10 @@ func NewContext(
|
||||
hasIntel bool,
|
||||
gpuDetectionFailed bool,
|
||||
forceCPURendering bool,
|
||||
disableRT bool,
|
||||
hipGPUSampleBatch int,
|
||||
onGPUError func(),
|
||||
sb sandbox.Wrapper,
|
||||
) *Context {
|
||||
if frameEnd < frameStart {
|
||||
frameEnd = frameStart
|
||||
@@ -107,7 +120,10 @@ func NewContext(
|
||||
HasIntel: hasIntel,
|
||||
GPUDetectionFailed: gpuDetectionFailed,
|
||||
ForceCPURendering: forceCPURendering,
|
||||
OnGPUError: onGPUError,
|
||||
DisableRT: disableRT,
|
||||
HipGPUSampleBatch: hipGPUSampleBatch,
|
||||
OnGPUError: onGPUError,
|
||||
Sandbox: sb,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,18 +164,22 @@ func (c *Context) OutputUploaded(fileName string) {
|
||||
}
|
||||
|
||||
// Complete sends task completion.
|
||||
// When the failure newly armed GPU lockout, free_requeue is set so the manager
|
||||
// requeues without burning retry_count.
|
||||
func (c *Context) Complete(success bool, errorMsg error) {
|
||||
if c.JobConn != nil {
|
||||
c.JobConn.Complete(c.TaskID, success, errorMsg)
|
||||
freeRequeue := !success && c.GPULockoutArmedThisAttempt
|
||||
c.JobConn.Complete(c.TaskID, success, errorMsg, freeRequeue)
|
||||
}
|
||||
}
|
||||
|
||||
// GetOutputFormat returns the output format from metadata or default.
|
||||
// Product default is EXR (Blender always renders EXR; deliverable may add video).
|
||||
func (c *Context) GetOutputFormat() string {
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.OutputFormat != "" {
|
||||
return c.Metadata.RenderSettings.OutputFormat
|
||||
}
|
||||
return "PNG"
|
||||
return "EXR"
|
||||
}
|
||||
|
||||
// GetFrameRate returns the frame rate from metadata or default.
|
||||
@@ -183,7 +203,9 @@ func (c *Context) ShouldUnhideObjects() bool {
|
||||
return c.Metadata != nil && c.Metadata.UnhideObjects != nil && *c.Metadata.UnhideObjects
|
||||
}
|
||||
|
||||
// ShouldEnableExecution returns whether to enable auto-execution.
|
||||
// ShouldEnableExecution returns whether to pass Blender --enable-autoexec
|
||||
// (Python drivers/scripts inside the .blend). Job-bundled blender_addons/ install
|
||||
// independently whenever that folder is present in the context.
|
||||
func (c *Context) ShouldEnableExecution() bool {
|
||||
return c.Metadata != nil && c.Metadata.EnableExecution != nil && *c.Metadata.EnableExecution
|
||||
}
|
||||
@@ -211,6 +233,69 @@ func (c *Context) ShouldForceCPU() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// GetCyclesSamples returns the target Cycles sample count from metadata (default 128).
|
||||
func (c *Context) GetCyclesSamples() int {
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||
if v, ok := c.Metadata.RenderSettings.EngineSettings["samples"]; ok {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
if int(n) > 0 {
|
||||
return int(n)
|
||||
}
|
||||
case int:
|
||||
if n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 128
|
||||
}
|
||||
|
||||
// GetCyclesSeed returns the Cycles seed from metadata (default 0).
|
||||
func (c *Context) GetCyclesSeed() int {
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||
if v, ok := c.Metadata.RenderSettings.EngineSettings["seed"]; ok {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return int(n)
|
||||
case int:
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ShouldBatchGPUSamples returns true when HIP GPU renders should be split into sample batches.
|
||||
func (c *Context) ShouldBatchGPUSamples() bool {
|
||||
if c.ShouldForceCPU() || c.HipGPUSampleBatch <= 0 {
|
||||
return false
|
||||
}
|
||||
return c.GetCyclesSamples() > c.HipGPUSampleBatch
|
||||
}
|
||||
|
||||
// ShouldDisableRT returns true when GPU ray tracing acceleration should be disabled.
|
||||
func (c *Context) ShouldDisableRT() bool {
|
||||
if c.DisableRT {
|
||||
return true
|
||||
}
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||
settings := c.Metadata.RenderSettings.EngineSettings
|
||||
if v, ok := settings["disable_rt"]; ok {
|
||||
if b, ok := v.(bool); ok && b {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if v, ok := settings["disable_hiprt"]; ok {
|
||||
if b, ok := v.(bool); ok && b {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsJobCancelled checks whether the manager marked this job as cancelled.
|
||||
func (c *Context) IsJobCancelled() (bool, error) {
|
||||
if c.Manager == nil {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
)
|
||||
|
||||
func TestNewContext_NormalizesFrameEnd(t *testing.T) {
|
||||
ctx := NewContext(1, 2, "job", 10, 1, "render", "/tmp", "tok", nil, nil, nil, nil, nil, nil, nil, false, false, false, false, false, false, nil)
|
||||
ctx := NewContext(1, 2, "job", 10, 1, "render", "/tmp", "tok", nil, nil, nil, nil, nil, nil, nil, false, false, false, false, false, false, false, 0, nil, nil)
|
||||
if ctx.FrameEnd != 10 {
|
||||
t.Fatalf("expected FrameEnd to be normalized to Frame, got %d", ctx.FrameEnd)
|
||||
}
|
||||
@@ -16,8 +16,8 @@ func TestNewContext_NormalizesFrameEnd(t *testing.T) {
|
||||
|
||||
func TestContext_GetOutputFormat_Default(t *testing.T) {
|
||||
ctx := &Context{}
|
||||
if got := ctx.GetOutputFormat(); got != "PNG" {
|
||||
t.Fatalf("GetOutputFormat() = %q, want PNG", got)
|
||||
if got := ctx.GetOutputFormat(); got != "EXR" {
|
||||
t.Fatalf("GetOutputFormat() = %q, want EXR", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+156
-27
@@ -12,6 +12,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"jiggablend/internal/runner/blender"
|
||||
"jiggablend/internal/runner/sandbox"
|
||||
"jiggablend/internal/runner/workspace"
|
||||
"jiggablend/pkg/scripts"
|
||||
"jiggablend/pkg/types"
|
||||
@@ -28,6 +29,8 @@ func NewRenderProcessor() *RenderProcessor {
|
||||
// gpuErrorSubstrings are log line substrings that indicate a GPU backend error (matched case-insensitively); any match triggers full GPU lockout.
|
||||
var gpuErrorSubstrings = []string{
|
||||
"illegal address in hip", // HIP (AMD) e.g. "Illegal address in HIP" or "Illegal address in hip"
|
||||
"memory access fault", // AMDGPU page fault during HIP/HIPRT (not necessarily OOM)
|
||||
"page not present", // AMDGPU GCVM page fault detail
|
||||
"hiperror", // hipError* codes
|
||||
"hip error",
|
||||
"cuda error",
|
||||
@@ -38,14 +41,18 @@ var gpuErrorSubstrings = []string{
|
||||
}
|
||||
|
||||
// checkGPUErrorLine checks a log line for GPU error indicators and triggers runner GPU lockout if found.
|
||||
// Once lockout is already active for this attempt (or was active at context creation), further
|
||||
// matching lines are ignored so we do not spam lockout callbacks/logs on multi-line fault dumps.
|
||||
func (p *RenderProcessor) checkGPUErrorLine(ctx *Context, line string) {
|
||||
if ctx.GPULockedOut || ctx.GPULockoutArmedThisAttempt {
|
||||
return
|
||||
}
|
||||
lower := strings.ToLower(line)
|
||||
for _, sub := range gpuErrorSubstrings {
|
||||
if strings.Contains(lower, sub) {
|
||||
if ctx.OnGPUError != nil {
|
||||
ctx.OnGPUError()
|
||||
}
|
||||
ctx.Warn(fmt.Sprintf("GPU error detected in log (%q); GPU disabled for subsequent jobs", sub))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -73,26 +80,24 @@ func (p *RenderProcessor) Process(ctx *Context) error {
|
||||
return fmt.Errorf("failed to find blend file: %w", err)
|
||||
}
|
||||
|
||||
// Get Blender binary
|
||||
blenderBinary := "blender"
|
||||
if version := ctx.GetBlenderVersion(); version != "" {
|
||||
ctx.Info(fmt.Sprintf("Job requires Blender %s", version))
|
||||
binaryPath, err := ctx.Blender.GetBinaryPath(version)
|
||||
if err != nil {
|
||||
ctx.Warn(fmt.Sprintf("Could not get Blender %s, using system blender: %v", version, err))
|
||||
} else {
|
||||
blenderBinary = binaryPath
|
||||
ctx.Info(fmt.Sprintf("Using Blender binary: %s", blenderBinary))
|
||||
}
|
||||
} else {
|
||||
ctx.Info("No Blender version specified, using system blender")
|
||||
// Runners must use manager-provided Blender versions; never fall back to system blender.
|
||||
version := ctx.GetBlenderVersion()
|
||||
if version == "" {
|
||||
return fmt.Errorf("job metadata missing blender_version: runner cannot use system blender")
|
||||
}
|
||||
|
||||
blenderBinary, err = blender.ResolveBinaryPath(blenderBinary)
|
||||
ctx.Info(fmt.Sprintf("Job requires Blender %s", version))
|
||||
binaryPath, err := ctx.Blender.GetBinaryPath(version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve blender binary: %w", err)
|
||||
return fmt.Errorf("failed to get Blender %s from manager: %w", version, err)
|
||||
}
|
||||
|
||||
blenderBinary, err := blender.ResolveBinaryPath(binaryPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve Blender %s binary: %w", version, err)
|
||||
}
|
||||
ctx.Info(fmt.Sprintf("Using Blender binary: %s", blenderBinary))
|
||||
|
||||
// Create output directory
|
||||
outputDir := filepath.Join(ctx.WorkDir, "output")
|
||||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||
@@ -116,11 +121,15 @@ func (p *RenderProcessor) Process(ctx *Context) error {
|
||||
} else {
|
||||
ctx.Info("GPU lockout active: using CPU rendering only")
|
||||
}
|
||||
} else if ctx.ShouldDisableRT() {
|
||||
ctx.Info("GPU ray tracing acceleration disabled for this job (--disable-rt)")
|
||||
}
|
||||
|
||||
// Create render script
|
||||
if err := p.createRenderScript(ctx, renderFormat); err != nil {
|
||||
return err
|
||||
if ctx.ShouldBatchGPUSamples() {
|
||||
total := ctx.GetCyclesSamples()
|
||||
ctx.Info(fmt.Sprintf(
|
||||
"gfx115x HIP sample batching: rendering %d samples in passes of %d (ROCm driver limit, not system RAM)",
|
||||
total, ctx.HipGPUSampleBatch,
|
||||
))
|
||||
}
|
||||
|
||||
// Render
|
||||
@@ -129,7 +138,7 @@ func (p *RenderProcessor) Process(ctx *Context) error {
|
||||
} else {
|
||||
ctx.Info(fmt.Sprintf("Starting Blender render for frame %d...", ctx.Frame))
|
||||
}
|
||||
if err := p.runBlender(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome); err != nil {
|
||||
if err := p.renderFrames(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome); err != nil {
|
||||
if errors.Is(err, ErrJobCancelled) {
|
||||
ctx.Warn("Render stopped because job was cancelled")
|
||||
return err
|
||||
@@ -152,7 +161,84 @@ func (p *RenderProcessor) Process(ctx *Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string) error {
|
||||
type renderPassOptions struct {
|
||||
samplesOverride *int
|
||||
seedOverride *int
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) renderFrames(ctx *Context, blenderBinary, blendFile, outputDir, renderFormat, blenderHome string) error {
|
||||
if !ctx.ShouldBatchGPUSamples() {
|
||||
if err := p.createRenderScript(ctx, renderFormat, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.runBlender(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome)
|
||||
}
|
||||
|
||||
totalSamples := ctx.GetCyclesSamples()
|
||||
batchSize := ctx.HipGPUSampleBatch
|
||||
baseSeed := ctx.GetCyclesSeed()
|
||||
batchDir := filepath.Join(ctx.WorkDir, "sample_batches")
|
||||
if err := os.MkdirAll(batchDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create sample batch directory: %w", err)
|
||||
}
|
||||
|
||||
var batchOutputs []string
|
||||
remaining := totalSamples
|
||||
batchNum := 0
|
||||
for remaining > 0 {
|
||||
if err := ctx.CheckCancelled(); err != nil {
|
||||
return err
|
||||
}
|
||||
passSamples := batchSize
|
||||
if remaining < passSamples {
|
||||
passSamples = remaining
|
||||
}
|
||||
seed := baseSeed + batchNum
|
||||
batchNum++
|
||||
ctx.Info(fmt.Sprintf("GPU sample batch %d: %d samples (seed %d, %d remaining)", batchNum, passSamples, seed, remaining-passSamples))
|
||||
|
||||
passOutput := filepath.Join(batchDir, fmt.Sprintf("batch_%03d", batchNum))
|
||||
if err := os.MkdirAll(passOutput, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
opts := &renderPassOptions{
|
||||
samplesOverride: &passSamples,
|
||||
seedOverride: &seed,
|
||||
}
|
||||
if err := p.createRenderScript(ctx, renderFormat, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.runBlender(ctx, blenderBinary, blendFile, passOutput, renderFormat, blenderHome); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.verifyOutputRange(ctx, passOutput, renderFormat); err != nil {
|
||||
return err
|
||||
}
|
||||
batchOutputs = append(batchOutputs, p.firstFrameOutputPath(ctx, passOutput, renderFormat))
|
||||
remaining -= passSamples
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
finalOutput := p.firstFrameOutputPath(ctx, outputDir, renderFormat)
|
||||
ctx.Info(fmt.Sprintf("Merging %d GPU sample batches into final EXR...", len(batchOutputs)))
|
||||
if err := mergeEXRFiles(finalOutput, batchOutputs); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx.Info("GPU sample batch merge completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) firstFrameOutputPath(ctx *Context, outputDir, renderFormat string) string {
|
||||
ext := strings.ToLower(renderFormat)
|
||||
if ctx.FrameEnd > ctx.Frame {
|
||||
return filepath.Join(outputDir, fmt.Sprintf("frame_%04d.%s", ctx.Frame, ext))
|
||||
}
|
||||
return filepath.Join(outputDir, fmt.Sprintf("frame_%04d.%s", ctx.Frame, ext))
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string, opts *renderPassOptions) error {
|
||||
formatFilePath := filepath.Join(ctx.WorkDir, "output_format.txt")
|
||||
renderSettingsFilePath := filepath.Join(ctx.WorkDir, "render_settings.json")
|
||||
|
||||
@@ -162,7 +248,9 @@ func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string)
|
||||
unhideCode = scripts.UnhideObjects
|
||||
}
|
||||
|
||||
// Load template and replace placeholders
|
||||
// Load template and replace placeholders.
|
||||
// Job-bundled blender_addons/ always auto-install when present (users ship addons with scenes).
|
||||
// enable_execution only controls Blender --enable-autoexec (scripts inside .blend).
|
||||
scriptContent := scripts.RenderBlenderTemplate
|
||||
scriptContent = strings.ReplaceAll(scriptContent, "{{UNHIDE_CODE}}", unhideCode)
|
||||
scriptContent = strings.ReplaceAll(scriptContent, "{{FORMAT_FILE_PATH}}", fmt.Sprintf("%q", formatFilePath))
|
||||
@@ -195,6 +283,20 @@ func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string)
|
||||
settingsMap = make(map[string]interface{})
|
||||
}
|
||||
settingsMap["force_cpu"] = ctx.ShouldForceCPU()
|
||||
settingsMap["disable_rt"] = ctx.ShouldDisableRT()
|
||||
if ctx.ShouldBatchGPUSamples() {
|
||||
// gfx115x: large tiles trigger AMDGPU page faults during HIP renders.
|
||||
settingsMap["tile_size_override"] = 256
|
||||
settingsMap["use_auto_tile_override"] = true
|
||||
}
|
||||
if opts != nil {
|
||||
if opts.samplesOverride != nil {
|
||||
settingsMap["samples_override"] = *opts.samplesOverride
|
||||
}
|
||||
if opts.seedOverride != nil {
|
||||
settingsMap["seed_override"] = *opts.seedOverride
|
||||
}
|
||||
}
|
||||
settingsJSON, err := json.Marshal(settingsMap)
|
||||
if err == nil {
|
||||
if err := os.WriteFile(renderSettingsFilePath, settingsJSON, 0644); err != nil {
|
||||
@@ -205,6 +307,32 @@ func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string)
|
||||
return nil
|
||||
}
|
||||
|
||||
// wrapBlenderCmd builds the Blender *exec.Cmd, optionally through the sandbox wrapper.
|
||||
func wrapBlenderCmd(ctx *Context, blenderBinary string, args, env []string, blenderHome string) (*exec.Cmd, error) {
|
||||
sb := ctx.Sandbox
|
||||
if sb == nil {
|
||||
var err error
|
||||
sb, err = sandbox.New(sandbox.Options{Backend: sandbox.BackendNone})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if sb.Name() != sandbox.BackendNone {
|
||||
ctx.Info(fmt.Sprintf("Running Blender under sandbox backend %q", sb.Name()))
|
||||
}
|
||||
return sb.Wrap(sandbox.Spec{
|
||||
BlenderBinary: blenderBinary,
|
||||
Args: args,
|
||||
WorkDir: ctx.WorkDir,
|
||||
HomeDir: blenderHome,
|
||||
Env: env,
|
||||
HasAMD: ctx.HasAMD,
|
||||
HasNVIDIA: ctx.HasNVIDIA,
|
||||
HasIntel: ctx.HasIntel,
|
||||
ForceCPU: ctx.ShouldForceCPU(),
|
||||
})
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, outputDir, renderFormat, blenderHome string) error {
|
||||
scriptPath := filepath.Join(ctx.WorkDir, "enable_gpu.py")
|
||||
blendFileAbs, err := filepath.Abs(blendFile)
|
||||
@@ -233,9 +361,6 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
args = append(args, "-f", fmt.Sprintf("%d", ctx.Frame))
|
||||
}
|
||||
|
||||
cmd := execCommand(blenderBinary, args...)
|
||||
cmd.Dir = ctx.WorkDir
|
||||
|
||||
// Set up environment: LD_LIBRARY_PATH for tarball Blender, then custom HOME
|
||||
env := os.Environ()
|
||||
env = blender.TarballEnv(blenderBinary, env)
|
||||
@@ -246,7 +371,11 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
}
|
||||
}
|
||||
newEnv = append(newEnv, fmt.Sprintf("HOME=%s", blenderHome))
|
||||
cmd.Env = newEnv
|
||||
|
||||
cmd, err := wrapBlenderCmd(ctx, blenderBinary, args, newEnv, blenderHome)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to build sandboxed blender command: %w", err)
|
||||
}
|
||||
|
||||
// Set up pipes
|
||||
stdoutPipe, err := cmd.StdoutPipe()
|
||||
|
||||
@@ -4,13 +4,42 @@ import "testing"
|
||||
|
||||
func TestCheckGPUErrorLine_TriggersCallback(t *testing.T) {
|
||||
p := NewRenderProcessor()
|
||||
triggered := false
|
||||
triggered := 0
|
||||
ctx := &Context{
|
||||
OnGPUError: func() { triggered = true },
|
||||
OnGPUError: func() {
|
||||
triggered++
|
||||
// Simulate runner arming lockout for this attempt.
|
||||
},
|
||||
}
|
||||
p.checkGPUErrorLine(ctx, "Fatal: Illegal address in HIP kernel execution")
|
||||
if !triggered {
|
||||
t.Fatal("expected GPU error callback to be triggered")
|
||||
if triggered != 1 {
|
||||
t.Fatalf("expected GPU error callback once, got %d", triggered)
|
||||
}
|
||||
// After arming, further fault lines must not re-fire (spam protection).
|
||||
ctx.GPULockoutArmedThisAttempt = true
|
||||
p.checkGPUErrorLine(ctx, "page not present in GPU memory")
|
||||
if triggered != 1 {
|
||||
t.Fatalf("expected no further callbacks after arm, got %d", triggered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckGPUErrorLine_SkipsWhenAlreadyLockedOut(t *testing.T) {
|
||||
p := NewRenderProcessor()
|
||||
triggered := false
|
||||
ctx := &Context{
|
||||
GPULockedOut: true,
|
||||
OnGPUError: func() { triggered = true },
|
||||
}
|
||||
p.checkGPUErrorLine(ctx, "Illegal address in HIP")
|
||||
if triggered {
|
||||
t.Fatal("did not expect callback when GPU already locked out")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBlenderVersion_EmptyWhenMissing(t *testing.T) {
|
||||
ctx := &Context{}
|
||||
if got := ctx.GetBlenderVersion(); got != "" {
|
||||
t.Fatalf("expected empty blender version, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -128,6 +128,20 @@ func ExtractTarStripPrefix(reader io.Reader, destDir string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
// Reject absolute or escaping symlink targets so extraction cannot
|
||||
// plant links that point outside destDir.
|
||||
linkTarget := header.Linkname
|
||||
if linkTarget == "" {
|
||||
return fmt.Errorf("invalid empty symlink target in tar: %s", header.Name)
|
||||
}
|
||||
if filepath.IsAbs(linkTarget) {
|
||||
return fmt.Errorf("absolute symlink target not allowed in tar: %s -> %s", header.Name, linkTarget)
|
||||
}
|
||||
resolvedLink := filepath.Clean(filepath.Join(filepath.Dir(targetPath), linkTarget))
|
||||
cleanDest := filepath.Clean(destDir)
|
||||
if resolvedLink != cleanDest && !strings.HasPrefix(resolvedLink, cleanDest+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("symlink target escapes extract root: %s -> %s", header.Name, linkTarget)
|
||||
}
|
||||
os.Remove(targetPath) // Remove existing symlink if present
|
||||
if err := os.Symlink(header.Linkname, targetPath); err != nil {
|
||||
return err
|
||||
|
||||
@@ -24,6 +24,25 @@ func createTarBuffer(files map[string]string) *bytes.Buffer {
|
||||
return &buf
|
||||
}
|
||||
|
||||
func TestExtractTarStripPrefix_RejectsEscapingSymlink(t *testing.T) {
|
||||
destDir := t.TempDir()
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
// Top-level prefix like Blender archives
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "blender-x/", Typeflag: tar.TypeDir, Mode: 0755})
|
||||
_ = tw.WriteHeader(&tar.Header{
|
||||
Name: "blender-x/evil",
|
||||
Typeflag: tar.TypeSymlink,
|
||||
Linkname: "../../outside",
|
||||
Mode: 0777,
|
||||
})
|
||||
_ = tw.Close()
|
||||
|
||||
if err := ExtractTarStripPrefix(&buf, destDir); err == nil {
|
||||
t.Fatal("expected escaping symlink to be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractTar(t *testing.T) {
|
||||
destDir := t.TempDir()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user