Refactor installation and runner scripts for enhanced usability and security

- Updated the installation script to clarify that production wrappers do not include fixed test secrets, improving user guidance for local testing. - Modified the jiggablend-manager and jiggablend-runner scripts to remove fixed test configurations, emphasizing the use of local test credentials via `make init-test`. - Enhanced error handling in the runner script to ensure that an API key is provided, improving security and user feedback. - Added new flags and options for the runner, including sandboxing capabilities and GPU ray tracing control, enhancing flexibility for users. - Improved README documentation to reflect changes in usage and configuration, ensuring users have clear instructions for setup and execution.
2026-07-12 10:01:15 -05:00
parent a3defe5cf6
commit 1a69fcfd04
47 changed files with 2718 additions and 402 deletions
@@ -12,6 +12,7 @@ import (
 	"strings"

 	"jiggablend/internal/runner/blender"
+	"jiggablend/internal/runner/sandbox"
 	"jiggablend/internal/runner/workspace"
 	"jiggablend/pkg/scripts"
 	"jiggablend/pkg/types"
@@ -28,6 +29,8 @@ func NewRenderProcessor() *RenderProcessor {
 // gpuErrorSubstrings are log line substrings that indicate a GPU backend error (matched case-insensitively); any match triggers full GPU lockout.
 var gpuErrorSubstrings = []string{
 	"illegal address in hip", // HIP (AMD) e.g. "Illegal address in HIP" or "Illegal address in hip"
+	"memory access fault",    // AMDGPU page fault during HIP/HIPRT (not necessarily OOM)
+	"page not present",       // AMDGPU GCVM page fault detail
 	"hiperror",               // hipError* codes
 	"hip error",
 	"cuda error",
@@ -38,14 +41,18 @@ var gpuErrorSubstrings = []string{
 }

 // checkGPUErrorLine checks a log line for GPU error indicators and triggers runner GPU lockout if found.
+// Once lockout is already active for this attempt (or was active at context creation), further
+// matching lines are ignored so we do not spam lockout callbacks/logs on multi-line fault dumps.
 func (p *RenderProcessor) checkGPUErrorLine(ctx *Context, line string) {
+	if ctx.GPULockedOut || ctx.GPULockoutArmedThisAttempt {
+		return
+	}
 	lower := strings.ToLower(line)
 	for _, sub := range gpuErrorSubstrings {
 		if strings.Contains(lower, sub) {
 			if ctx.OnGPUError != nil {
 				ctx.OnGPUError()
 			}
-			ctx.Warn(fmt.Sprintf("GPU error detected in log (%q); GPU disabled for subsequent jobs", sub))
 			return
 		}
 	}
@@ -73,26 +80,24 @@ func (p *RenderProcessor) Process(ctx *Context) error {
 		return fmt.Errorf("failed to find blend file: %w", err)
 	}

-	// Get Blender binary
-	blenderBinary := "blender"
-	if version := ctx.GetBlenderVersion(); version != "" {
-		ctx.Info(fmt.Sprintf("Job requires Blender %s", version))
-		binaryPath, err := ctx.Blender.GetBinaryPath(version)
-		if err != nil {
-			ctx.Warn(fmt.Sprintf("Could not get Blender %s, using system blender: %v", version, err))
-		} else {
-			blenderBinary = binaryPath
-			ctx.Info(fmt.Sprintf("Using Blender binary: %s", blenderBinary))
-		}
-	} else {
-		ctx.Info("No Blender version specified, using system blender")
+	// Runners must use manager-provided Blender versions; never fall back to system blender.
+	version := ctx.GetBlenderVersion()
+	if version == "" {
+		return fmt.Errorf("job metadata missing blender_version: runner cannot use system blender")
 	}

-	blenderBinary, err = blender.ResolveBinaryPath(blenderBinary)
+	ctx.Info(fmt.Sprintf("Job requires Blender %s", version))
+	binaryPath, err := ctx.Blender.GetBinaryPath(version)
 	if err != nil {
-		return fmt.Errorf("failed to resolve blender binary: %w", err)
+		return fmt.Errorf("failed to get Blender %s from manager: %w", version, err)
 	}

+	blenderBinary, err := blender.ResolveBinaryPath(binaryPath)
+	if err != nil {
+		return fmt.Errorf("failed to resolve Blender %s binary: %w", version, err)
+	}
+	ctx.Info(fmt.Sprintf("Using Blender binary: %s", blenderBinary))
+
 	// Create output directory
 	outputDir := filepath.Join(ctx.WorkDir, "output")
 	if err := os.MkdirAll(outputDir, 0755); err != nil {
@@ -116,11 +121,15 @@ func (p *RenderProcessor) Process(ctx *Context) error {
 		} else {
 			ctx.Info("GPU lockout active: using CPU rendering only")
 		}
+	} else if ctx.ShouldDisableRT() {
+		ctx.Info("GPU ray tracing acceleration disabled for this job (--disable-rt)")
 	}
-
-	// Create render script
-	if err := p.createRenderScript(ctx, renderFormat); err != nil {
-		return err
+	if ctx.ShouldBatchGPUSamples() {
+		total := ctx.GetCyclesSamples()
+		ctx.Info(fmt.Sprintf(
+			"gfx115x HIP sample batching: rendering %d samples in passes of %d (ROCm driver limit, not system RAM)",
+			total, ctx.HipGPUSampleBatch,
+		))
 	}

 	// Render
@@ -129,7 +138,7 @@ func (p *RenderProcessor) Process(ctx *Context) error {
 	} else {
 		ctx.Info(fmt.Sprintf("Starting Blender render for frame %d...", ctx.Frame))
 	}
-	if err := p.runBlender(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome); err != nil {
+	if err := p.renderFrames(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome); err != nil {
 		if errors.Is(err, ErrJobCancelled) {
 			ctx.Warn("Render stopped because job was cancelled")
 			return err
@@ -152,7 +161,84 @@ func (p *RenderProcessor) Process(ctx *Context) error {
 	return nil
 }

-func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string) error {
+type renderPassOptions struct {
+	samplesOverride *int
+	seedOverride    *int
+}
+
+func (p *RenderProcessor) renderFrames(ctx *Context, blenderBinary, blendFile, outputDir, renderFormat, blenderHome string) error {
+	if !ctx.ShouldBatchGPUSamples() {
+		if err := p.createRenderScript(ctx, renderFormat, nil); err != nil {
+			return err
+		}
+		return p.runBlender(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome)
+	}
+
+	totalSamples := ctx.GetCyclesSamples()
+	batchSize := ctx.HipGPUSampleBatch
+	baseSeed := ctx.GetCyclesSeed()
+	batchDir := filepath.Join(ctx.WorkDir, "sample_batches")
+	if err := os.MkdirAll(batchDir, 0755); err != nil {
+		return fmt.Errorf("failed to create sample batch directory: %w", err)
+	}
+
+	var batchOutputs []string
+	remaining := totalSamples
+	batchNum := 0
+	for remaining > 0 {
+		if err := ctx.CheckCancelled(); err != nil {
+			return err
+		}
+		passSamples := batchSize
+		if remaining < passSamples {
+			passSamples = remaining
+		}
+		seed := baseSeed + batchNum
+		batchNum++
+		ctx.Info(fmt.Sprintf("GPU sample batch %d: %d samples (seed %d, %d remaining)", batchNum, passSamples, seed, remaining-passSamples))
+
+		passOutput := filepath.Join(batchDir, fmt.Sprintf("batch_%03d", batchNum))
+		if err := os.MkdirAll(passOutput, 0755); err != nil {
+			return err
+		}
+		opts := &renderPassOptions{
+			samplesOverride: &passSamples,
+			seedOverride:    &seed,
+		}
+		if err := p.createRenderScript(ctx, renderFormat, opts); err != nil {
+			return err
+		}
+		if err := p.runBlender(ctx, blenderBinary, blendFile, passOutput, renderFormat, blenderHome); err != nil {
+			return err
+		}
+		if err := p.verifyOutputRange(ctx, passOutput, renderFormat); err != nil {
+			return err
+		}
+		batchOutputs = append(batchOutputs, p.firstFrameOutputPath(ctx, passOutput, renderFormat))
+		remaining -= passSamples
+	}
+
+	if err := os.MkdirAll(outputDir, 0755); err != nil {
+		return err
+	}
+	finalOutput := p.firstFrameOutputPath(ctx, outputDir, renderFormat)
+	ctx.Info(fmt.Sprintf("Merging %d GPU sample batches into final EXR...", len(batchOutputs)))
+	if err := mergeEXRFiles(finalOutput, batchOutputs); err != nil {
+		return err
+	}
+	ctx.Info("GPU sample batch merge completed")
+	return nil
+}
+
+func (p *RenderProcessor) firstFrameOutputPath(ctx *Context, outputDir, renderFormat string) string {
+	ext := strings.ToLower(renderFormat)
+	if ctx.FrameEnd > ctx.Frame {
+		return filepath.Join(outputDir, fmt.Sprintf("frame_%04d.%s", ctx.Frame, ext))
+	}
+	return filepath.Join(outputDir, fmt.Sprintf("frame_%04d.%s", ctx.Frame, ext))
+}
+
+func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string, opts *renderPassOptions) error {
 	formatFilePath := filepath.Join(ctx.WorkDir, "output_format.txt")
 	renderSettingsFilePath := filepath.Join(ctx.WorkDir, "render_settings.json")

@@ -162,7 +248,9 @@ func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string)
 		unhideCode = scripts.UnhideObjects
 	}

-	// Load template and replace placeholders
+	// Load template and replace placeholders.
+	// Job-bundled blender_addons/ always auto-install when present (users ship addons with scenes).
+	// enable_execution only controls Blender --enable-autoexec (scripts inside .blend).
 	scriptContent := scripts.RenderBlenderTemplate
 	scriptContent = strings.ReplaceAll(scriptContent, "{{UNHIDE_CODE}}", unhideCode)
 	scriptContent = strings.ReplaceAll(scriptContent, "{{FORMAT_FILE_PATH}}", fmt.Sprintf("%q", formatFilePath))
@@ -195,6 +283,20 @@ func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string)
 		settingsMap = make(map[string]interface{})
 	}
 	settingsMap["force_cpu"] = ctx.ShouldForceCPU()
+	settingsMap["disable_rt"] = ctx.ShouldDisableRT()
+	if ctx.ShouldBatchGPUSamples() {
+		// gfx115x: large tiles trigger AMDGPU page faults during HIP renders.
+		settingsMap["tile_size_override"] = 256
+		settingsMap["use_auto_tile_override"] = true
+	}
+	if opts != nil {
+		if opts.samplesOverride != nil {
+			settingsMap["samples_override"] = *opts.samplesOverride
+		}
+		if opts.seedOverride != nil {
+			settingsMap["seed_override"] = *opts.seedOverride
+		}
+	}
 	settingsJSON, err := json.Marshal(settingsMap)
 	if err == nil {
 		if err := os.WriteFile(renderSettingsFilePath, settingsJSON, 0644); err != nil {
@@ -205,6 +307,32 @@ func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string)
 	return nil
 }

+// wrapBlenderCmd builds the Blender *exec.Cmd, optionally through the sandbox wrapper.
+func wrapBlenderCmd(ctx *Context, blenderBinary string, args, env []string, blenderHome string) (*exec.Cmd, error) {
+	sb := ctx.Sandbox
+	if sb == nil {
+		var err error
+		sb, err = sandbox.New(sandbox.Options{Backend: sandbox.BackendNone})
+		if err != nil {
+			return nil, err
+		}
+	}
+	if sb.Name() != sandbox.BackendNone {
+		ctx.Info(fmt.Sprintf("Running Blender under sandbox backend %q", sb.Name()))
+	}
+	return sb.Wrap(sandbox.Spec{
+		BlenderBinary: blenderBinary,
+		Args:          args,
+		WorkDir:       ctx.WorkDir,
+		HomeDir:       blenderHome,
+		Env:           env,
+		HasAMD:        ctx.HasAMD,
+		HasNVIDIA:     ctx.HasNVIDIA,
+		HasIntel:      ctx.HasIntel,
+		ForceCPU:      ctx.ShouldForceCPU(),
+	})
+}
+
 func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, outputDir, renderFormat, blenderHome string) error {
 	scriptPath := filepath.Join(ctx.WorkDir, "enable_gpu.py")
 	blendFileAbs, err := filepath.Abs(blendFile)
@@ -233,9 +361,6 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
 		args = append(args, "-f", fmt.Sprintf("%d", ctx.Frame))
 	}

-	cmd := execCommand(blenderBinary, args...)
-	cmd.Dir = ctx.WorkDir
-
 	// Set up environment: LD_LIBRARY_PATH for tarball Blender, then custom HOME
 	env := os.Environ()
 	env = blender.TarballEnv(blenderBinary, env)
@@ -246,7 +371,11 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
 		}
 	}
 	newEnv = append(newEnv, fmt.Sprintf("HOME=%s", blenderHome))
-	cmd.Env = newEnv
+
+	cmd, err := wrapBlenderCmd(ctx, blenderBinary, args, newEnv, blenderHome)
+	if err != nil {
+		return fmt.Errorf("failed to build sandboxed blender command: %w", err)
+	}

 	// Set up pipes
 	stdoutPipe, err := cmd.StdoutPipe()