Refactor installation and runner scripts for enhanced usability and security
- Updated the installation script to clarify that production wrappers do not include fixed test secrets, improving user guidance for local testing. - Modified the jiggablend-manager and jiggablend-runner scripts to remove fixed test configurations, emphasizing the use of local test credentials via `make init-test`. - Enhanced error handling in the runner script to ensure that an API key is provided, improving security and user feedback. - Added new flags and options for the runner, including sandboxing capabilities and GPU ray tracing control, enhancing flexibility for users. - Improved README documentation to reflect changes in usage and configuration, ensuring users have clear instructions for setup and execution.
This commit is contained in:
@@ -246,6 +246,8 @@ func (p *EncodeProcessor) Process(ctx *Context) error {
|
||||
TwoPass: true,
|
||||
})
|
||||
if err := pass1Cmd.Run(); err != nil {
|
||||
// Pass 1 is analysis-only (writes to /dev/null). FFmpeg often exits non-zero
|
||||
// on benign codec/option warnings while still producing passlogfile stats.
|
||||
ctx.Warn(fmt.Sprintf("Pass 1 completed (warnings expected): %v", err))
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// mergeEXRFiles averages multiple linear EXR renders into one (Monte Carlo accumulation).
|
||||
func mergeEXRFiles(output string, inputs []string) error {
|
||||
if len(inputs) == 0 {
|
||||
return fmt.Errorf("no input EXR files to merge")
|
||||
}
|
||||
if len(inputs) == 1 {
|
||||
return copyFile(inputs[0], output)
|
||||
}
|
||||
|
||||
args := append([]string{}, inputs...)
|
||||
args = append(args, "-evaluate-sequence", "mean", output)
|
||||
|
||||
if path, err := exec.LookPath("magick"); err == nil {
|
||||
cmd := exec.Command(path, args...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("magick merge failed: %w (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if path, err := exec.LookPath("convert"); err == nil {
|
||||
cmd := exec.Command(path, args...)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("convert merge failed: %w (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("ImageMagick (magick or convert) required to merge batched EXR renders")
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
cmd := exec.Command("cp", "-f", src, dst)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("copy %s to %s: %w (%s)", src, dst, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"jiggablend/internal/runner/api"
|
||||
"jiggablend/internal/runner/blender"
|
||||
"jiggablend/internal/runner/encoding"
|
||||
"jiggablend/internal/runner/sandbox"
|
||||
"jiggablend/internal/runner/workspace"
|
||||
"jiggablend/pkg/executils"
|
||||
"jiggablend/pkg/types"
|
||||
@@ -41,6 +42,9 @@ type Context struct {
|
||||
|
||||
// GPULockedOut is set when the runner has detected a GPU error (e.g. HIP) and disables GPU for all jobs.
|
||||
GPULockedOut bool
|
||||
// GPULockoutArmedThisAttempt is true when this task attempt newly enabled runner GPU lockout.
|
||||
// Failures with this set free-requeue without incrementing retry_count (expected ROCm warmup).
|
||||
GPULockoutArmedThisAttempt bool
|
||||
// HasAMD is true when the runner detected AMD devices at startup.
|
||||
HasAMD bool
|
||||
// HasNVIDIA is true when the runner detected NVIDIA GPUs at startup.
|
||||
@@ -53,6 +57,12 @@ type Context struct {
|
||||
OnGPUError func()
|
||||
// ForceCPURendering is a runner-level override that forces CPU rendering for all jobs.
|
||||
ForceCPURendering bool
|
||||
// DisableRT disables GPU ray tracing acceleration (runner-level flag).
|
||||
DisableRT bool
|
||||
// HipGPUSampleBatch limits samples per GPU render pass on gfx115x (0 = no batching).
|
||||
HipGPUSampleBatch int
|
||||
// Sandbox wraps Blender execution (none/podman). Nil means none.
|
||||
Sandbox sandbox.Wrapper
|
||||
}
|
||||
|
||||
// ErrJobCancelled indicates the manager-side job was cancelled during execution.
|
||||
@@ -80,7 +90,10 @@ func NewContext(
|
||||
hasIntel bool,
|
||||
gpuDetectionFailed bool,
|
||||
forceCPURendering bool,
|
||||
disableRT bool,
|
||||
hipGPUSampleBatch int,
|
||||
onGPUError func(),
|
||||
sb sandbox.Wrapper,
|
||||
) *Context {
|
||||
if frameEnd < frameStart {
|
||||
frameEnd = frameStart
|
||||
@@ -107,7 +120,10 @@ func NewContext(
|
||||
HasIntel: hasIntel,
|
||||
GPUDetectionFailed: gpuDetectionFailed,
|
||||
ForceCPURendering: forceCPURendering,
|
||||
OnGPUError: onGPUError,
|
||||
DisableRT: disableRT,
|
||||
HipGPUSampleBatch: hipGPUSampleBatch,
|
||||
OnGPUError: onGPUError,
|
||||
Sandbox: sb,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,18 +164,22 @@ func (c *Context) OutputUploaded(fileName string) {
|
||||
}
|
||||
|
||||
// Complete sends task completion.
|
||||
// When the failure newly armed GPU lockout, free_requeue is set so the manager
|
||||
// requeues without burning retry_count.
|
||||
func (c *Context) Complete(success bool, errorMsg error) {
|
||||
if c.JobConn != nil {
|
||||
c.JobConn.Complete(c.TaskID, success, errorMsg)
|
||||
freeRequeue := !success && c.GPULockoutArmedThisAttempt
|
||||
c.JobConn.Complete(c.TaskID, success, errorMsg, freeRequeue)
|
||||
}
|
||||
}
|
||||
|
||||
// GetOutputFormat returns the output format from metadata or default.
|
||||
// Product default is EXR (Blender always renders EXR; deliverable may add video).
|
||||
func (c *Context) GetOutputFormat() string {
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.OutputFormat != "" {
|
||||
return c.Metadata.RenderSettings.OutputFormat
|
||||
}
|
||||
return "PNG"
|
||||
return "EXR"
|
||||
}
|
||||
|
||||
// GetFrameRate returns the frame rate from metadata or default.
|
||||
@@ -183,7 +203,9 @@ func (c *Context) ShouldUnhideObjects() bool {
|
||||
return c.Metadata != nil && c.Metadata.UnhideObjects != nil && *c.Metadata.UnhideObjects
|
||||
}
|
||||
|
||||
// ShouldEnableExecution returns whether to enable auto-execution.
|
||||
// ShouldEnableExecution returns whether to pass Blender --enable-autoexec
|
||||
// (Python drivers/scripts inside the .blend). Job-bundled blender_addons/ install
|
||||
// independently whenever that folder is present in the context.
|
||||
func (c *Context) ShouldEnableExecution() bool {
|
||||
return c.Metadata != nil && c.Metadata.EnableExecution != nil && *c.Metadata.EnableExecution
|
||||
}
|
||||
@@ -211,6 +233,69 @@ func (c *Context) ShouldForceCPU() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// GetCyclesSamples returns the target Cycles sample count from metadata (default 128).
|
||||
func (c *Context) GetCyclesSamples() int {
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||
if v, ok := c.Metadata.RenderSettings.EngineSettings["samples"]; ok {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
if int(n) > 0 {
|
||||
return int(n)
|
||||
}
|
||||
case int:
|
||||
if n > 0 {
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 128
|
||||
}
|
||||
|
||||
// GetCyclesSeed returns the Cycles seed from metadata (default 0).
|
||||
func (c *Context) GetCyclesSeed() int {
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||
if v, ok := c.Metadata.RenderSettings.EngineSettings["seed"]; ok {
|
||||
switch n := v.(type) {
|
||||
case float64:
|
||||
return int(n)
|
||||
case int:
|
||||
return n
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// ShouldBatchGPUSamples returns true when HIP GPU renders should be split into sample batches.
|
||||
func (c *Context) ShouldBatchGPUSamples() bool {
|
||||
if c.ShouldForceCPU() || c.HipGPUSampleBatch <= 0 {
|
||||
return false
|
||||
}
|
||||
return c.GetCyclesSamples() > c.HipGPUSampleBatch
|
||||
}
|
||||
|
||||
// ShouldDisableRT returns true when GPU ray tracing acceleration should be disabled.
|
||||
func (c *Context) ShouldDisableRT() bool {
|
||||
if c.DisableRT {
|
||||
return true
|
||||
}
|
||||
if c.Metadata != nil && c.Metadata.RenderSettings.EngineSettings != nil {
|
||||
settings := c.Metadata.RenderSettings.EngineSettings
|
||||
if v, ok := settings["disable_rt"]; ok {
|
||||
if b, ok := v.(bool); ok && b {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if v, ok := settings["disable_hiprt"]; ok {
|
||||
if b, ok := v.(bool); ok && b {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsJobCancelled checks whether the manager marked this job as cancelled.
|
||||
func (c *Context) IsJobCancelled() (bool, error) {
|
||||
if c.Manager == nil {
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
)
|
||||
|
||||
func TestNewContext_NormalizesFrameEnd(t *testing.T) {
|
||||
ctx := NewContext(1, 2, "job", 10, 1, "render", "/tmp", "tok", nil, nil, nil, nil, nil, nil, nil, false, false, false, false, false, false, nil)
|
||||
ctx := NewContext(1, 2, "job", 10, 1, "render", "/tmp", "tok", nil, nil, nil, nil, nil, nil, nil, false, false, false, false, false, false, false, 0, nil, nil)
|
||||
if ctx.FrameEnd != 10 {
|
||||
t.Fatalf("expected FrameEnd to be normalized to Frame, got %d", ctx.FrameEnd)
|
||||
}
|
||||
@@ -16,8 +16,8 @@ func TestNewContext_NormalizesFrameEnd(t *testing.T) {
|
||||
|
||||
func TestContext_GetOutputFormat_Default(t *testing.T) {
|
||||
ctx := &Context{}
|
||||
if got := ctx.GetOutputFormat(); got != "PNG" {
|
||||
t.Fatalf("GetOutputFormat() = %q, want PNG", got)
|
||||
if got := ctx.GetOutputFormat(); got != "EXR" {
|
||||
t.Fatalf("GetOutputFormat() = %q, want EXR", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+156
-27
@@ -12,6 +12,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"jiggablend/internal/runner/blender"
|
||||
"jiggablend/internal/runner/sandbox"
|
||||
"jiggablend/internal/runner/workspace"
|
||||
"jiggablend/pkg/scripts"
|
||||
"jiggablend/pkg/types"
|
||||
@@ -28,6 +29,8 @@ func NewRenderProcessor() *RenderProcessor {
|
||||
// gpuErrorSubstrings are log line substrings that indicate a GPU backend error (matched case-insensitively); any match triggers full GPU lockout.
|
||||
var gpuErrorSubstrings = []string{
|
||||
"illegal address in hip", // HIP (AMD) e.g. "Illegal address in HIP" or "Illegal address in hip"
|
||||
"memory access fault", // AMDGPU page fault during HIP/HIPRT (not necessarily OOM)
|
||||
"page not present", // AMDGPU GCVM page fault detail
|
||||
"hiperror", // hipError* codes
|
||||
"hip error",
|
||||
"cuda error",
|
||||
@@ -38,14 +41,18 @@ var gpuErrorSubstrings = []string{
|
||||
}
|
||||
|
||||
// checkGPUErrorLine checks a log line for GPU error indicators and triggers runner GPU lockout if found.
|
||||
// Once lockout is already active for this attempt (or was active at context creation), further
|
||||
// matching lines are ignored so we do not spam lockout callbacks/logs on multi-line fault dumps.
|
||||
func (p *RenderProcessor) checkGPUErrorLine(ctx *Context, line string) {
|
||||
if ctx.GPULockedOut || ctx.GPULockoutArmedThisAttempt {
|
||||
return
|
||||
}
|
||||
lower := strings.ToLower(line)
|
||||
for _, sub := range gpuErrorSubstrings {
|
||||
if strings.Contains(lower, sub) {
|
||||
if ctx.OnGPUError != nil {
|
||||
ctx.OnGPUError()
|
||||
}
|
||||
ctx.Warn(fmt.Sprintf("GPU error detected in log (%q); GPU disabled for subsequent jobs", sub))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -73,26 +80,24 @@ func (p *RenderProcessor) Process(ctx *Context) error {
|
||||
return fmt.Errorf("failed to find blend file: %w", err)
|
||||
}
|
||||
|
||||
// Get Blender binary
|
||||
blenderBinary := "blender"
|
||||
if version := ctx.GetBlenderVersion(); version != "" {
|
||||
ctx.Info(fmt.Sprintf("Job requires Blender %s", version))
|
||||
binaryPath, err := ctx.Blender.GetBinaryPath(version)
|
||||
if err != nil {
|
||||
ctx.Warn(fmt.Sprintf("Could not get Blender %s, using system blender: %v", version, err))
|
||||
} else {
|
||||
blenderBinary = binaryPath
|
||||
ctx.Info(fmt.Sprintf("Using Blender binary: %s", blenderBinary))
|
||||
}
|
||||
} else {
|
||||
ctx.Info("No Blender version specified, using system blender")
|
||||
// Runners must use manager-provided Blender versions; never fall back to system blender.
|
||||
version := ctx.GetBlenderVersion()
|
||||
if version == "" {
|
||||
return fmt.Errorf("job metadata missing blender_version: runner cannot use system blender")
|
||||
}
|
||||
|
||||
blenderBinary, err = blender.ResolveBinaryPath(blenderBinary)
|
||||
ctx.Info(fmt.Sprintf("Job requires Blender %s", version))
|
||||
binaryPath, err := ctx.Blender.GetBinaryPath(version)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve blender binary: %w", err)
|
||||
return fmt.Errorf("failed to get Blender %s from manager: %w", version, err)
|
||||
}
|
||||
|
||||
blenderBinary, err := blender.ResolveBinaryPath(binaryPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve Blender %s binary: %w", version, err)
|
||||
}
|
||||
ctx.Info(fmt.Sprintf("Using Blender binary: %s", blenderBinary))
|
||||
|
||||
// Create output directory
|
||||
outputDir := filepath.Join(ctx.WorkDir, "output")
|
||||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||
@@ -116,11 +121,15 @@ func (p *RenderProcessor) Process(ctx *Context) error {
|
||||
} else {
|
||||
ctx.Info("GPU lockout active: using CPU rendering only")
|
||||
}
|
||||
} else if ctx.ShouldDisableRT() {
|
||||
ctx.Info("GPU ray tracing acceleration disabled for this job (--disable-rt)")
|
||||
}
|
||||
|
||||
// Create render script
|
||||
if err := p.createRenderScript(ctx, renderFormat); err != nil {
|
||||
return err
|
||||
if ctx.ShouldBatchGPUSamples() {
|
||||
total := ctx.GetCyclesSamples()
|
||||
ctx.Info(fmt.Sprintf(
|
||||
"gfx115x HIP sample batching: rendering %d samples in passes of %d (ROCm driver limit, not system RAM)",
|
||||
total, ctx.HipGPUSampleBatch,
|
||||
))
|
||||
}
|
||||
|
||||
// Render
|
||||
@@ -129,7 +138,7 @@ func (p *RenderProcessor) Process(ctx *Context) error {
|
||||
} else {
|
||||
ctx.Info(fmt.Sprintf("Starting Blender render for frame %d...", ctx.Frame))
|
||||
}
|
||||
if err := p.runBlender(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome); err != nil {
|
||||
if err := p.renderFrames(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome); err != nil {
|
||||
if errors.Is(err, ErrJobCancelled) {
|
||||
ctx.Warn("Render stopped because job was cancelled")
|
||||
return err
|
||||
@@ -152,7 +161,84 @@ func (p *RenderProcessor) Process(ctx *Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string) error {
|
||||
type renderPassOptions struct {
|
||||
samplesOverride *int
|
||||
seedOverride *int
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) renderFrames(ctx *Context, blenderBinary, blendFile, outputDir, renderFormat, blenderHome string) error {
|
||||
if !ctx.ShouldBatchGPUSamples() {
|
||||
if err := p.createRenderScript(ctx, renderFormat, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
return p.runBlender(ctx, blenderBinary, blendFile, outputDir, renderFormat, blenderHome)
|
||||
}
|
||||
|
||||
totalSamples := ctx.GetCyclesSamples()
|
||||
batchSize := ctx.HipGPUSampleBatch
|
||||
baseSeed := ctx.GetCyclesSeed()
|
||||
batchDir := filepath.Join(ctx.WorkDir, "sample_batches")
|
||||
if err := os.MkdirAll(batchDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create sample batch directory: %w", err)
|
||||
}
|
||||
|
||||
var batchOutputs []string
|
||||
remaining := totalSamples
|
||||
batchNum := 0
|
||||
for remaining > 0 {
|
||||
if err := ctx.CheckCancelled(); err != nil {
|
||||
return err
|
||||
}
|
||||
passSamples := batchSize
|
||||
if remaining < passSamples {
|
||||
passSamples = remaining
|
||||
}
|
||||
seed := baseSeed + batchNum
|
||||
batchNum++
|
||||
ctx.Info(fmt.Sprintf("GPU sample batch %d: %d samples (seed %d, %d remaining)", batchNum, passSamples, seed, remaining-passSamples))
|
||||
|
||||
passOutput := filepath.Join(batchDir, fmt.Sprintf("batch_%03d", batchNum))
|
||||
if err := os.MkdirAll(passOutput, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
opts := &renderPassOptions{
|
||||
samplesOverride: &passSamples,
|
||||
seedOverride: &seed,
|
||||
}
|
||||
if err := p.createRenderScript(ctx, renderFormat, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.runBlender(ctx, blenderBinary, blendFile, passOutput, renderFormat, blenderHome); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := p.verifyOutputRange(ctx, passOutput, renderFormat); err != nil {
|
||||
return err
|
||||
}
|
||||
batchOutputs = append(batchOutputs, p.firstFrameOutputPath(ctx, passOutput, renderFormat))
|
||||
remaining -= passSamples
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
finalOutput := p.firstFrameOutputPath(ctx, outputDir, renderFormat)
|
||||
ctx.Info(fmt.Sprintf("Merging %d GPU sample batches into final EXR...", len(batchOutputs)))
|
||||
if err := mergeEXRFiles(finalOutput, batchOutputs); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx.Info("GPU sample batch merge completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) firstFrameOutputPath(ctx *Context, outputDir, renderFormat string) string {
|
||||
ext := strings.ToLower(renderFormat)
|
||||
if ctx.FrameEnd > ctx.Frame {
|
||||
return filepath.Join(outputDir, fmt.Sprintf("frame_%04d.%s", ctx.Frame, ext))
|
||||
}
|
||||
return filepath.Join(outputDir, fmt.Sprintf("frame_%04d.%s", ctx.Frame, ext))
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string, opts *renderPassOptions) error {
|
||||
formatFilePath := filepath.Join(ctx.WorkDir, "output_format.txt")
|
||||
renderSettingsFilePath := filepath.Join(ctx.WorkDir, "render_settings.json")
|
||||
|
||||
@@ -162,7 +248,9 @@ func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string)
|
||||
unhideCode = scripts.UnhideObjects
|
||||
}
|
||||
|
||||
// Load template and replace placeholders
|
||||
// Load template and replace placeholders.
|
||||
// Job-bundled blender_addons/ always auto-install when present (users ship addons with scenes).
|
||||
// enable_execution only controls Blender --enable-autoexec (scripts inside .blend).
|
||||
scriptContent := scripts.RenderBlenderTemplate
|
||||
scriptContent = strings.ReplaceAll(scriptContent, "{{UNHIDE_CODE}}", unhideCode)
|
||||
scriptContent = strings.ReplaceAll(scriptContent, "{{FORMAT_FILE_PATH}}", fmt.Sprintf("%q", formatFilePath))
|
||||
@@ -195,6 +283,20 @@ func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string)
|
||||
settingsMap = make(map[string]interface{})
|
||||
}
|
||||
settingsMap["force_cpu"] = ctx.ShouldForceCPU()
|
||||
settingsMap["disable_rt"] = ctx.ShouldDisableRT()
|
||||
if ctx.ShouldBatchGPUSamples() {
|
||||
// gfx115x: large tiles trigger AMDGPU page faults during HIP renders.
|
||||
settingsMap["tile_size_override"] = 256
|
||||
settingsMap["use_auto_tile_override"] = true
|
||||
}
|
||||
if opts != nil {
|
||||
if opts.samplesOverride != nil {
|
||||
settingsMap["samples_override"] = *opts.samplesOverride
|
||||
}
|
||||
if opts.seedOverride != nil {
|
||||
settingsMap["seed_override"] = *opts.seedOverride
|
||||
}
|
||||
}
|
||||
settingsJSON, err := json.Marshal(settingsMap)
|
||||
if err == nil {
|
||||
if err := os.WriteFile(renderSettingsFilePath, settingsJSON, 0644); err != nil {
|
||||
@@ -205,6 +307,32 @@ func (p *RenderProcessor) createRenderScript(ctx *Context, renderFormat string)
|
||||
return nil
|
||||
}
|
||||
|
||||
// wrapBlenderCmd builds the Blender *exec.Cmd, optionally through the sandbox wrapper.
|
||||
func wrapBlenderCmd(ctx *Context, blenderBinary string, args, env []string, blenderHome string) (*exec.Cmd, error) {
|
||||
sb := ctx.Sandbox
|
||||
if sb == nil {
|
||||
var err error
|
||||
sb, err = sandbox.New(sandbox.Options{Backend: sandbox.BackendNone})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if sb.Name() != sandbox.BackendNone {
|
||||
ctx.Info(fmt.Sprintf("Running Blender under sandbox backend %q", sb.Name()))
|
||||
}
|
||||
return sb.Wrap(sandbox.Spec{
|
||||
BlenderBinary: blenderBinary,
|
||||
Args: args,
|
||||
WorkDir: ctx.WorkDir,
|
||||
HomeDir: blenderHome,
|
||||
Env: env,
|
||||
HasAMD: ctx.HasAMD,
|
||||
HasNVIDIA: ctx.HasNVIDIA,
|
||||
HasIntel: ctx.HasIntel,
|
||||
ForceCPU: ctx.ShouldForceCPU(),
|
||||
})
|
||||
}
|
||||
|
||||
func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, outputDir, renderFormat, blenderHome string) error {
|
||||
scriptPath := filepath.Join(ctx.WorkDir, "enable_gpu.py")
|
||||
blendFileAbs, err := filepath.Abs(blendFile)
|
||||
@@ -233,9 +361,6 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
args = append(args, "-f", fmt.Sprintf("%d", ctx.Frame))
|
||||
}
|
||||
|
||||
cmd := execCommand(blenderBinary, args...)
|
||||
cmd.Dir = ctx.WorkDir
|
||||
|
||||
// Set up environment: LD_LIBRARY_PATH for tarball Blender, then custom HOME
|
||||
env := os.Environ()
|
||||
env = blender.TarballEnv(blenderBinary, env)
|
||||
@@ -246,7 +371,11 @@ func (p *RenderProcessor) runBlender(ctx *Context, blenderBinary, blendFile, out
|
||||
}
|
||||
}
|
||||
newEnv = append(newEnv, fmt.Sprintf("HOME=%s", blenderHome))
|
||||
cmd.Env = newEnv
|
||||
|
||||
cmd, err := wrapBlenderCmd(ctx, blenderBinary, args, newEnv, blenderHome)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to build sandboxed blender command: %w", err)
|
||||
}
|
||||
|
||||
// Set up pipes
|
||||
stdoutPipe, err := cmd.StdoutPipe()
|
||||
|
||||
@@ -4,13 +4,42 @@ import "testing"
|
||||
|
||||
func TestCheckGPUErrorLine_TriggersCallback(t *testing.T) {
|
||||
p := NewRenderProcessor()
|
||||
triggered := false
|
||||
triggered := 0
|
||||
ctx := &Context{
|
||||
OnGPUError: func() { triggered = true },
|
||||
OnGPUError: func() {
|
||||
triggered++
|
||||
// Simulate runner arming lockout for this attempt.
|
||||
},
|
||||
}
|
||||
p.checkGPUErrorLine(ctx, "Fatal: Illegal address in HIP kernel execution")
|
||||
if !triggered {
|
||||
t.Fatal("expected GPU error callback to be triggered")
|
||||
if triggered != 1 {
|
||||
t.Fatalf("expected GPU error callback once, got %d", triggered)
|
||||
}
|
||||
// After arming, further fault lines must not re-fire (spam protection).
|
||||
ctx.GPULockoutArmedThisAttempt = true
|
||||
p.checkGPUErrorLine(ctx, "page not present in GPU memory")
|
||||
if triggered != 1 {
|
||||
t.Fatalf("expected no further callbacks after arm, got %d", triggered)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckGPUErrorLine_SkipsWhenAlreadyLockedOut(t *testing.T) {
|
||||
p := NewRenderProcessor()
|
||||
triggered := false
|
||||
ctx := &Context{
|
||||
GPULockedOut: true,
|
||||
OnGPUError: func() { triggered = true },
|
||||
}
|
||||
p.checkGPUErrorLine(ctx, "Illegal address in HIP")
|
||||
if triggered {
|
||||
t.Fatal("did not expect callback when GPU already locked out")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBlenderVersion_EmptyWhenMissing(t *testing.T) {
|
||||
ctx := &Context{}
|
||||
if got := ctx.GetBlenderVersion(); got != "" {
|
||||
t.Fatalf("expected empty blender version, got %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user