Enhance logging and context handling in job management. Introduce a logger initialization with configurable parameters in the manager and runner commands. Update job context handling to use tar files instead of tar.gz, and implement ETag generation for improved caching. Refactor API endpoints to support new context file structure and enhance error handling in job submissions. Add support for unhide objects and auto-execution options in job creation requests.
This commit is contained in:
@@ -1,10 +1,10 @@
|
||||
package runner
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"jiggablend/pkg/scripts"
|
||||
"jiggablend/pkg/types"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
@@ -53,18 +54,20 @@ type Client struct {
|
||||
vaapiDevicesMu sync.RWMutex // Protects vaapiDevices
|
||||
allocatedDevices map[int64]string // map[taskID]device - tracks which device is allocated to which task
|
||||
allocatedDevicesMu sync.RWMutex // Protects allocatedDevices
|
||||
longRunningClient *http.Client // HTTP client for long-running operations (no timeout)
|
||||
}
|
||||
|
||||
// NewClient creates a new runner client
|
||||
func NewClient(managerURL, name, hostname, ipAddress string) *Client {
|
||||
return &Client{
|
||||
managerURL: managerURL,
|
||||
name: name,
|
||||
hostname: hostname,
|
||||
ipAddress: ipAddress,
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
stopChan: make(chan struct{}),
|
||||
stepStartTimes: make(map[string]time.Time),
|
||||
managerURL: managerURL,
|
||||
name: name,
|
||||
hostname: hostname,
|
||||
ipAddress: ipAddress,
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
longRunningClient: &http.Client{Timeout: 0}, // No timeout for long-running operations (context downloads, file uploads/downloads)
|
||||
stopChan: make(chan struct{}),
|
||||
stepStartTimes: make(map[string]time.Time),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -465,6 +468,17 @@ func (c *Client) Register(registrationToken string) (int64, string, string, erro
|
||||
// doSignedRequest performs an authenticated HTTP request using shared secret
|
||||
// queryParams is optional and will be appended to the URL
|
||||
func (c *Client) doSignedRequest(method, path string, body []byte, queryParams ...string) (*http.Response, error) {
|
||||
return c.doSignedRequestWithClient(method, path, body, c.httpClient, queryParams...)
|
||||
}
|
||||
|
||||
// doSignedRequestLong performs an authenticated HTTP request using the long-running client (no timeout)
|
||||
// Use this for context downloads, file uploads/downloads, and other operations that may take a long time
|
||||
func (c *Client) doSignedRequestLong(method, path string, body []byte, queryParams ...string) (*http.Response, error) {
|
||||
return c.doSignedRequestWithClient(method, path, body, c.longRunningClient, queryParams...)
|
||||
}
|
||||
|
||||
// doSignedRequestWithClient performs an authenticated HTTP request using the specified client
|
||||
func (c *Client) doSignedRequestWithClient(method, path string, body []byte, client *http.Client, queryParams ...string) (*http.Response, error) {
|
||||
if c.runnerSecret == "" {
|
||||
return nil, fmt.Errorf("runner not authenticated")
|
||||
}
|
||||
@@ -483,7 +497,7 @@ func (c *Client) doSignedRequest(method, path string, body []byte, queryParams .
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("X-Runner-Secret", c.runnerSecret)
|
||||
|
||||
return c.httpClient.Do(req)
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
// ConnectWebSocket establishes a WebSocket connection to the manager
|
||||
@@ -969,16 +983,16 @@ func (c *Client) processTask(task map[string]interface{}, jobName string, output
|
||||
// Clean up expired cache entries periodically
|
||||
c.cleanupExpiredContextCache()
|
||||
|
||||
// Download context tar.gz
|
||||
contextPath := filepath.Join(workDir, "context.tar.gz")
|
||||
// Download context tar
|
||||
contextPath := filepath.Join(workDir, "context.tar")
|
||||
if err := c.downloadJobContext(jobID, contextPath); err != nil {
|
||||
c.sendStepUpdate(taskID, "download", types.StepStatusFailed, err.Error())
|
||||
return fmt.Errorf("failed to download context: %w", err)
|
||||
}
|
||||
|
||||
// Extract context tar.gz
|
||||
// Extract context tar
|
||||
c.sendLog(taskID, types.LogLevelInfo, "Extracting context...", "download")
|
||||
if err := c.extractTarGz(contextPath, workDir); err != nil {
|
||||
if err := c.extractTar(contextPath, workDir); err != nil {
|
||||
c.sendStepUpdate(taskID, "download", types.StepStatusFailed, err.Error())
|
||||
return fmt.Errorf("failed to extract context: %w", err)
|
||||
}
|
||||
@@ -1077,662 +1091,24 @@ func (c *Client) processTask(task map[string]interface{}, jobName string, output
|
||||
// This script will override the blend file's settings based on job metadata
|
||||
formatFilePath := filepath.Join(workDir, "output_format.txt")
|
||||
renderSettingsFilePath := filepath.Join(workDir, "render_settings.json")
|
||||
scriptContent := fmt.Sprintf(`import bpy
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
|
||||
# Make all file paths relative to the blend file location FIRST
|
||||
# This must be done immediately after file load, before any other operations
|
||||
# to prevent Blender from trying to access external files with absolute paths
|
||||
try:
|
||||
bpy.ops.file.make_paths_relative()
|
||||
print("Made all file paths relative to blend file")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not make paths relative: {e}")
|
||||
|
||||
# Check for missing addons that the blend file requires
|
||||
# Blender marks missing addons with "_missing" suffix in preferences
|
||||
missing = []
|
||||
try:
|
||||
for mod in bpy.context.preferences.addons:
|
||||
if mod.module.endswith("_missing"):
|
||||
missing.append(mod.module.rsplit("_", 1)[0])
|
||||
|
||||
if missing:
|
||||
print("Missing add-ons required by this .blend:")
|
||||
for name in missing:
|
||||
print(" -", name)
|
||||
else:
|
||||
print("No missing add-ons detected – file is headless-safe")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not check for missing addons: {e}")
|
||||
|
||||
# Fix objects and collections hidden from render
|
||||
vl = bpy.context.view_layer
|
||||
|
||||
# 1. Objects hidden in view layer
|
||||
print("Checking for objects hidden from render that need to be enabled...")
|
||||
try:
|
||||
for obj in bpy.data.objects:
|
||||
if obj.hide_get(view_layer=vl):
|
||||
if any(k in obj.name.lower() for k in ["scrotum|","cage","genital","penis","dick","collision","body.001","couch"]):
|
||||
obj.hide_set(False, view_layer=vl)
|
||||
print("Enabled object:", obj.name)
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not check/fix hidden render objects: {e}")
|
||||
|
||||
# 2. Collections disabled in renders OR set to Holdout (the final killer)
|
||||
print("Checking for collections hidden from render that need to be enabled...")
|
||||
try:
|
||||
for col in bpy.data.collections:
|
||||
if col.hide_render or (vl.layer_collection.children.get(col.name) and not vl.layer_collection.children[col.name].exclude == False):
|
||||
if any(k in col.name.lower() for k in ["genital","nsfw","dick","private","hidden","cage","scrotum","collision","dick"]):
|
||||
col.hide_render = False
|
||||
if col.name in vl.layer_collection.children:
|
||||
vl.layer_collection.children[col.name].exclude = False
|
||||
vl.layer_collection.children[col.name].holdout = False
|
||||
vl.layer_collection.children[col.name].indirect_only = False
|
||||
print("Enabled collection:", col.name)
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not check/fix hidden render collections: {e}")
|
||||
|
||||
# Read output format from file (created by Go code)
|
||||
format_file_path = %q
|
||||
output_format_override = None
|
||||
if os.path.exists(format_file_path):
|
||||
try:
|
||||
with open(format_file_path, 'r') as f:
|
||||
output_format_override = f.read().strip().upper()
|
||||
print(f"Read output format from file: '{output_format_override}'")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read output format file: {e}")
|
||||
else:
|
||||
print(f"Warning: Output format file does not exist: {format_file_path}")
|
||||
|
||||
# Read render settings from JSON file (created by Go code)
|
||||
render_settings_file = %q
|
||||
render_settings_override = None
|
||||
if os.path.exists(render_settings_file):
|
||||
try:
|
||||
with open(render_settings_file, 'r') as f:
|
||||
render_settings_override = json.load(f)
|
||||
print(f"Loaded render settings from job metadata")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not read render settings file: {e}")
|
||||
`, formatFilePath, renderSettingsFilePath) + `
|
||||
|
||||
# Get current scene settings (preserve blend file preferences)
|
||||
scene = bpy.context.scene
|
||||
current_engine = scene.render.engine
|
||||
current_device = scene.cycles.device if hasattr(scene, 'cycles') and scene.cycles else None
|
||||
current_output_format = scene.render.image_settings.file_format
|
||||
|
||||
print(f"Blend file render engine: {current_engine}")
|
||||
if current_device:
|
||||
print(f"Blend file device setting: {current_device}")
|
||||
print(f"Blend file output format: {current_output_format}")
|
||||
|
||||
# Override output format if specified
|
||||
# The format file always takes precedence (it's written specifically for this job)
|
||||
if output_format_override:
|
||||
print(f"Overriding output format from '{current_output_format}' to '{output_format_override}'")
|
||||
# Map common format names to Blender's format constants
|
||||
# For video formats (EXR_264_MP4, EXR_AV1_MP4), we render as EXR frames first
|
||||
format_to_use = output_format_override.upper()
|
||||
if format_to_use in ['EXR_264_MP4', 'EXR_AV1_MP4']:
|
||||
format_to_use = 'EXR' # Render as EXR for video formats
|
||||
|
||||
format_map = {
|
||||
'PNG': 'PNG',
|
||||
'JPEG': 'JPEG',
|
||||
'JPG': 'JPEG',
|
||||
'EXR': 'OPEN_EXR',
|
||||
'OPEN_EXR': 'OPEN_EXR',
|
||||
'TARGA': 'TARGA',
|
||||
'TIFF': 'TIFF',
|
||||
'BMP': 'BMP',
|
||||
}
|
||||
blender_format = format_map.get(format_to_use, format_to_use)
|
||||
try:
|
||||
scene.render.image_settings.file_format = blender_format
|
||||
print(f"Successfully set output format to: {blender_format}")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not set output format to {blender_format}: {e}")
|
||||
print(f"Using blend file's format: {current_output_format}")
|
||||
else:
|
||||
print(f"Using blend file's output format: {current_output_format}")
|
||||
|
||||
# Apply render settings from job metadata if provided
|
||||
# Note: output_format is NOT applied from render_settings_override - it's already set from format file above
|
||||
if render_settings_override:
|
||||
engine_override = render_settings_override.get('engine', '').upper()
|
||||
engine_settings = render_settings_override.get('engine_settings', {})
|
||||
|
||||
# Switch engine if specified
|
||||
if engine_override and engine_override != current_engine.upper():
|
||||
print(f"Switching render engine from '{current_engine}' to '{engine_override}'")
|
||||
try:
|
||||
scene.render.engine = engine_override
|
||||
current_engine = engine_override
|
||||
print(f"Successfully switched to {engine_override} engine")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not switch engine to {engine_override}: {e}")
|
||||
print(f"Using blend file's engine: {current_engine}")
|
||||
|
||||
# Apply engine-specific settings
|
||||
if engine_settings:
|
||||
if current_engine.upper() == 'CYCLES':
|
||||
cycles = scene.cycles
|
||||
print("Applying Cycles render settings from job metadata...")
|
||||
for key, value in engine_settings.items():
|
||||
try:
|
||||
if hasattr(cycles, key):
|
||||
setattr(cycles, key, value)
|
||||
print(f" Set Cycles.{key} = {value}")
|
||||
else:
|
||||
print(f" Warning: Cycles has no attribute '{key}'")
|
||||
except Exception as e:
|
||||
print(f" Warning: Could not set Cycles.{key} = {value}: {e}")
|
||||
elif current_engine.upper() in ['EEVEE', 'EEVEE_NEXT']:
|
||||
eevee = scene.eevee
|
||||
print("Applying EEVEE render settings from job metadata...")
|
||||
for key, value in engine_settings.items():
|
||||
try:
|
||||
if hasattr(eevee, key):
|
||||
setattr(eevee, key, value)
|
||||
print(f" Set EEVEE.{key} = {value}")
|
||||
else:
|
||||
print(f" Warning: EEVEE has no attribute '{key}'")
|
||||
except Exception as e:
|
||||
print(f" Warning: Could not set EEVEE.{key} = {value}: {e}")
|
||||
|
||||
# Apply resolution if specified
|
||||
if 'resolution_x' in render_settings_override:
|
||||
try:
|
||||
scene.render.resolution_x = render_settings_override['resolution_x']
|
||||
print(f"Set resolution_x = {render_settings_override['resolution_x']}")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not set resolution_x: {e}")
|
||||
if 'resolution_y' in render_settings_override:
|
||||
try:
|
||||
scene.render.resolution_y = render_settings_override['resolution_y']
|
||||
print(f"Set resolution_y = {render_settings_override['resolution_y']}")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not set resolution_y: {e}")
|
||||
|
||||
# Only override device selection if using Cycles (other engines handle GPU differently)
|
||||
if current_engine == 'CYCLES':
|
||||
# Check if CPU rendering is forced
|
||||
force_cpu = False
|
||||
if render_settings_override and render_settings_override.get('force_cpu'):
|
||||
force_cpu = render_settings_override.get('force_cpu', False)
|
||||
print("Force CPU rendering is enabled - skipping GPU detection")
|
||||
|
||||
# Ensure Cycles addon is enabled
|
||||
try:
|
||||
if 'cycles' not in bpy.context.preferences.addons:
|
||||
bpy.ops.preferences.addon_enable(module='cycles')
|
||||
print("Enabled Cycles addon")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not enable Cycles addon: {e}")
|
||||
|
||||
# If CPU is forced, skip GPU detection and set CPU directly
|
||||
if force_cpu:
|
||||
scene.cycles.device = 'CPU'
|
||||
print("Forced CPU rendering (skipping GPU detection)")
|
||||
else:
|
||||
# Access Cycles preferences
|
||||
prefs = bpy.context.preferences
|
||||
try:
|
||||
cycles_prefs = prefs.addons['cycles'].preferences
|
||||
except (KeyError, AttributeError):
|
||||
try:
|
||||
cycles_addon = prefs.addons.get('cycles')
|
||||
if cycles_addon:
|
||||
cycles_prefs = cycles_addon.preferences
|
||||
else:
|
||||
raise Exception("Cycles addon not found")
|
||||
except Exception as e:
|
||||
print(f"ERROR: Could not access Cycles preferences: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
# Check all devices and choose the best GPU type
|
||||
# Device type preference order (most performant first)
|
||||
device_type_preference = ['OPTIX', 'CUDA', 'HIP', 'ONEAPI', 'METAL']
|
||||
gpu_available = False
|
||||
best_device_type = None
|
||||
best_gpu_devices = []
|
||||
devices_by_type = {} # {device_type: [devices]}
|
||||
seen_device_ids = set() # Track device IDs to avoid duplicates
|
||||
|
||||
print("Checking for GPU availability...")
|
||||
|
||||
# Try to get all devices - try each device type to see what's available
|
||||
for device_type in device_type_preference:
|
||||
try:
|
||||
cycles_prefs.compute_device_type = device_type
|
||||
cycles_prefs.refresh_devices()
|
||||
|
||||
# Get devices for this type
|
||||
devices = None
|
||||
if hasattr(cycles_prefs, 'devices'):
|
||||
try:
|
||||
devices_prop = cycles_prefs.devices
|
||||
if devices_prop:
|
||||
devices = list(devices_prop) if hasattr(devices_prop, '__iter__') else [devices_prop]
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
if not devices or len(devices) == 0:
|
||||
try:
|
||||
devices = cycles_prefs.get_devices()
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
if devices and len(devices) > 0:
|
||||
# Categorize devices by their type attribute, avoiding duplicates
|
||||
for device in devices:
|
||||
if hasattr(device, 'type'):
|
||||
device_type_str = str(device.type).upper()
|
||||
device_id = getattr(device, 'id', None)
|
||||
|
||||
# Use device ID to avoid duplicates (same device appears when checking different compute_device_types)
|
||||
if device_id and device_id in seen_device_ids:
|
||||
continue
|
||||
|
||||
if device_id:
|
||||
seen_device_ids.add(device_id)
|
||||
|
||||
if device_type_str not in devices_by_type:
|
||||
devices_by_type[device_type_str] = []
|
||||
devices_by_type[device_type_str].append(device)
|
||||
except (ValueError, AttributeError, KeyError, TypeError):
|
||||
# Device type not supported, continue
|
||||
continue
|
||||
except Exception as e:
|
||||
# Other errors - log but continue
|
||||
print(f" Error checking {device_type}: {e}")
|
||||
continue
|
||||
|
||||
# Print what we found
|
||||
print(f"Found devices by type: {list(devices_by_type.keys())}")
|
||||
for dev_type, dev_list in devices_by_type.items():
|
||||
print(f" {dev_type}: {len(dev_list)} device(s)")
|
||||
for device in dev_list:
|
||||
device_name = getattr(device, 'name', 'Unknown')
|
||||
print(f" - {device_name}")
|
||||
|
||||
# Choose the best GPU type based on preference
|
||||
for preferred_type in device_type_preference:
|
||||
if preferred_type in devices_by_type:
|
||||
gpu_devices = [d for d in devices_by_type[preferred_type] if preferred_type in ['CUDA', 'OPENCL', 'OPTIX', 'HIP', 'METAL', 'ONEAPI']]
|
||||
if gpu_devices:
|
||||
best_device_type = preferred_type
|
||||
best_gpu_devices = [(d, preferred_type) for d in gpu_devices]
|
||||
print(f"Selected {preferred_type} as best GPU type with {len(gpu_devices)} device(s)")
|
||||
break
|
||||
|
||||
# Second pass: Enable the best GPU we found
|
||||
if best_device_type and best_gpu_devices:
|
||||
print(f"\nEnabling GPU devices for {best_device_type}...")
|
||||
try:
|
||||
# Set the device type again
|
||||
cycles_prefs.compute_device_type = best_device_type
|
||||
cycles_prefs.refresh_devices()
|
||||
|
||||
# First, disable all CPU devices to ensure only GPU is used
|
||||
print(f" Disabling CPU devices...")
|
||||
all_devices = cycles_prefs.devices if hasattr(cycles_prefs, 'devices') else cycles_prefs.get_devices()
|
||||
if all_devices:
|
||||
for device in all_devices:
|
||||
if hasattr(device, 'type') and str(device.type).upper() == 'CPU':
|
||||
try:
|
||||
device.use = False
|
||||
device_name = getattr(device, 'name', 'Unknown')
|
||||
print(f" Disabled CPU: {device_name}")
|
||||
except Exception as e:
|
||||
print(f" Warning: Could not disable CPU device {getattr(device, 'name', 'Unknown')}: {e}")
|
||||
|
||||
# Enable all GPU devices
|
||||
enabled_count = 0
|
||||
for device, device_type in best_gpu_devices:
|
||||
try:
|
||||
device.use = True
|
||||
enabled_count += 1
|
||||
device_name = getattr(device, 'name', 'Unknown')
|
||||
print(f" Enabled: {device_name}")
|
||||
except Exception as e:
|
||||
print(f" Warning: Could not enable device {getattr(device, 'name', 'Unknown')}: {e}")
|
||||
|
||||
# Enable ray tracing acceleration for supported device types
|
||||
try:
|
||||
if best_device_type == 'HIP':
|
||||
# HIPRT (HIP Ray Tracing) for AMD GPUs
|
||||
if hasattr(cycles_prefs, 'use_hiprt'):
|
||||
cycles_prefs.use_hiprt = True
|
||||
print(f" Enabled HIPRT (HIP Ray Tracing) for faster rendering")
|
||||
elif hasattr(scene.cycles, 'use_hiprt'):
|
||||
scene.cycles.use_hiprt = True
|
||||
print(f" Enabled HIPRT (HIP Ray Tracing) for faster rendering")
|
||||
else:
|
||||
print(f" HIPRT not available (requires Blender 4.0+)")
|
||||
elif best_device_type == 'OPTIX':
|
||||
# OptiX is already enabled when using OPTIX device type
|
||||
# But we can check if there are any OptiX-specific settings
|
||||
if hasattr(scene.cycles, 'use_optix_denoising'):
|
||||
scene.cycles.use_optix_denoising = True
|
||||
print(f" Enabled OptiX denoising")
|
||||
print(f" OptiX ray tracing is active (using OPTIX device type)")
|
||||
elif best_device_type == 'CUDA':
|
||||
# CUDA can use OptiX if available, but it's usually automatic
|
||||
# Check if we can prefer OptiX over CUDA
|
||||
if hasattr(scene.cycles, 'use_optix_denoising'):
|
||||
scene.cycles.use_optix_denoising = True
|
||||
print(f" Enabled OptiX denoising (if OptiX available)")
|
||||
print(f" CUDA ray tracing active")
|
||||
elif best_device_type == 'METAL':
|
||||
# MetalRT for Apple Silicon (if available)
|
||||
if hasattr(scene.cycles, 'use_metalrt'):
|
||||
scene.cycles.use_metalrt = True
|
||||
print(f" Enabled MetalRT (Metal Ray Tracing) for faster rendering")
|
||||
elif hasattr(cycles_prefs, 'use_metalrt'):
|
||||
cycles_prefs.use_metalrt = True
|
||||
print(f" Enabled MetalRT (Metal Ray Tracing) for faster rendering")
|
||||
else:
|
||||
print(f" MetalRT not available")
|
||||
elif best_device_type == 'ONEAPI':
|
||||
# Intel oneAPI - Embree might be available
|
||||
if hasattr(scene.cycles, 'use_embree'):
|
||||
scene.cycles.use_embree = True
|
||||
print(f" Enabled Embree for faster CPU ray tracing")
|
||||
print(f" oneAPI ray tracing active")
|
||||
except Exception as e:
|
||||
print(f" Could not enable ray tracing acceleration: {e}")
|
||||
|
||||
print(f"SUCCESS: Enabled {enabled_count} GPU device(s) for {best_device_type}")
|
||||
gpu_available = True
|
||||
except Exception as e:
|
||||
print(f"ERROR: Failed to enable GPU devices: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# Set device based on availability (prefer GPU, fallback to CPU)
|
||||
if gpu_available:
|
||||
scene.cycles.device = 'GPU'
|
||||
print(f"Using GPU for rendering (blend file had: {current_device})")
|
||||
else:
|
||||
scene.cycles.device = 'CPU'
|
||||
print(f"GPU not available, using CPU for rendering (blend file had: {current_device})")
|
||||
|
||||
# Verify device setting
|
||||
if current_engine == 'CYCLES':
|
||||
final_device = scene.cycles.device
|
||||
print(f"Final Cycles device: {final_device}")
|
||||
else:
|
||||
# For other engines (EEVEE, etc.), respect blend file settings
|
||||
print(f"Using {current_engine} engine - respecting blend file settings")
|
||||
|
||||
# Enable GPU acceleration for EEVEE viewport rendering (if using EEVEE)
|
||||
if current_engine == 'EEVEE' or current_engine == 'EEVEE_NEXT':
|
||||
try:
|
||||
if hasattr(bpy.context.preferences.system, 'gpu_backend'):
|
||||
bpy.context.preferences.system.gpu_backend = 'OPENGL'
|
||||
print("Enabled OpenGL GPU backend for EEVEE")
|
||||
except Exception as e:
|
||||
print(f"Could not set EEVEE GPU backend: {e}")
|
||||
|
||||
# Enable GPU acceleration for compositing (if compositing is enabled)
|
||||
try:
|
||||
if scene.use_nodes and hasattr(scene, 'node_tree') and scene.node_tree:
|
||||
if hasattr(scene.node_tree, 'use_gpu_compositing'):
|
||||
scene.node_tree.use_gpu_compositing = True
|
||||
print("Enabled GPU compositing")
|
||||
except Exception as e:
|
||||
print(f"Could not enable GPU compositing: {e}")
|
||||
|
||||
# CRITICAL: Initialize headless rendering to prevent black images
|
||||
# This ensures the render engine is properly initialized before rendering
|
||||
print("Initializing headless rendering context...")
|
||||
try:
|
||||
# Ensure world exists and has proper settings
|
||||
if not scene.world:
|
||||
# Create a default world if none exists
|
||||
world = bpy.data.worlds.new("World")
|
||||
scene.world = world
|
||||
print("Created default world")
|
||||
|
||||
# Ensure world has a background shader (not just black)
|
||||
if scene.world:
|
||||
# Enable nodes if not already enabled
|
||||
if not scene.world.use_nodes:
|
||||
scene.world.use_nodes = True
|
||||
print("Enabled world nodes")
|
||||
|
||||
world_nodes = scene.world.node_tree
|
||||
if world_nodes:
|
||||
# Find or create background shader
|
||||
bg_shader = None
|
||||
for node in world_nodes.nodes:
|
||||
if node.type == 'BACKGROUND':
|
||||
bg_shader = node
|
||||
break
|
||||
|
||||
if not bg_shader:
|
||||
bg_shader = world_nodes.nodes.new(type='ShaderNodeBackground')
|
||||
# Connect to output
|
||||
output = world_nodes.nodes.get('World Output')
|
||||
if not output:
|
||||
output = world_nodes.nodes.new(type='ShaderNodeOutputWorld')
|
||||
output.name = 'World Output'
|
||||
if output and bg_shader:
|
||||
# Connect background to surface input
|
||||
if 'Surface' in output.inputs and 'Background' in bg_shader.outputs:
|
||||
world_nodes.links.new(bg_shader.outputs['Background'], output.inputs['Surface'])
|
||||
print("Created background shader for world")
|
||||
|
||||
# Ensure background has some color (not pure black)
|
||||
if bg_shader:
|
||||
# Only set if it's pure black (0,0,0)
|
||||
if hasattr(bg_shader.inputs, 'Color'):
|
||||
color = bg_shader.inputs['Color'].default_value
|
||||
if len(color) >= 3 and color[0] == 0.0 and color[1] == 0.0 and color[2] == 0.0:
|
||||
# Set to a very dark gray instead of pure black
|
||||
bg_shader.inputs['Color'].default_value = (0.01, 0.01, 0.01, 1.0)
|
||||
print("Adjusted world background color to prevent black renders")
|
||||
else:
|
||||
# Fallback: use legacy world color if nodes aren't working
|
||||
if hasattr(scene.world, 'color'):
|
||||
color = scene.world.color
|
||||
if len(color) >= 3 and color[0] == 0.0 and color[1] == 0.0 and color[2] == 0.0:
|
||||
scene.world.color = (0.01, 0.01, 0.01)
|
||||
print("Adjusted legacy world color to prevent black renders")
|
||||
|
||||
# For EEVEE, force viewport update to initialize render engine
|
||||
if current_engine in ['EEVEE', 'EEVEE_NEXT']:
|
||||
# Force EEVEE to update its internal state
|
||||
try:
|
||||
# Update depsgraph to ensure everything is initialized
|
||||
depsgraph = bpy.context.evaluated_depsgraph_get()
|
||||
if depsgraph:
|
||||
# Force update
|
||||
depsgraph.update()
|
||||
print("Forced EEVEE depsgraph update for headless rendering")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not force EEVEE update: {e}")
|
||||
|
||||
# Ensure EEVEE settings are applied
|
||||
try:
|
||||
# Force a material update to ensure shaders are compiled
|
||||
for obj in scene.objects:
|
||||
if obj.type == 'MESH' and obj.data.materials:
|
||||
for mat in obj.data.materials:
|
||||
if mat and mat.use_nodes:
|
||||
# Touch the material to force update
|
||||
mat.use_nodes = mat.use_nodes
|
||||
print("Forced material updates for EEVEE")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not update materials: {e}")
|
||||
|
||||
# For Cycles, ensure proper initialization
|
||||
if current_engine == 'CYCLES':
|
||||
# Ensure samples are set (even if 1 for preview)
|
||||
if not hasattr(scene.cycles, 'samples') or scene.cycles.samples < 1:
|
||||
scene.cycles.samples = 1
|
||||
print("Set minimum Cycles samples")
|
||||
|
||||
# Check for lights in the scene
|
||||
lights = [obj for obj in scene.objects if obj.type == 'LIGHT']
|
||||
print(f"Found {len(lights)} light(s) in scene")
|
||||
if len(lights) == 0:
|
||||
print("WARNING: No lights found in scene - rendering may be black!")
|
||||
print(" Consider adding lights or ensuring world background emits light")
|
||||
|
||||
# Ensure world background emits light (critical for Cycles)
|
||||
if scene.world and scene.world.use_nodes:
|
||||
world_nodes = scene.world.node_tree
|
||||
if world_nodes:
|
||||
bg_shader = None
|
||||
for node in world_nodes.nodes:
|
||||
if node.type == 'BACKGROUND':
|
||||
bg_shader = node
|
||||
break
|
||||
|
||||
if bg_shader:
|
||||
# Check and set strength - Cycles needs this to emit light!
|
||||
if hasattr(bg_shader.inputs, 'Strength'):
|
||||
strength = bg_shader.inputs['Strength'].default_value
|
||||
if strength <= 0.0:
|
||||
bg_shader.inputs['Strength'].default_value = 1.0
|
||||
print("Set world background strength to 1.0 for Cycles lighting")
|
||||
else:
|
||||
print(f"World background strength: {strength}")
|
||||
# Also ensure color is not pure black
|
||||
if hasattr(bg_shader.inputs, 'Color'):
|
||||
color = bg_shader.inputs['Color'].default_value
|
||||
if len(color) >= 3 and color[0] == 0.0 and color[1] == 0.0 and color[2] == 0.0:
|
||||
bg_shader.inputs['Color'].default_value = (1.0, 1.0, 1.0, 1.0)
|
||||
print("Set world background color to white for Cycles lighting")
|
||||
|
||||
# Check film_transparent setting - if enabled, background will be transparent/black
|
||||
if hasattr(scene.cycles, 'film_transparent') and scene.cycles.film_transparent:
|
||||
print("WARNING: film_transparent is enabled - background will be transparent")
|
||||
print(" If you see black renders, try disabling film_transparent")
|
||||
|
||||
# Force Cycles to update/compile materials and shaders
|
||||
try:
|
||||
# Update depsgraph to ensure everything is initialized
|
||||
depsgraph = bpy.context.evaluated_depsgraph_get()
|
||||
if depsgraph:
|
||||
depsgraph.update()
|
||||
print("Forced Cycles depsgraph update")
|
||||
|
||||
# Force material updates to ensure shaders are compiled
|
||||
for obj in scene.objects:
|
||||
if obj.type == 'MESH' and obj.data.materials:
|
||||
for mat in obj.data.materials:
|
||||
if mat and mat.use_nodes:
|
||||
# Force material update
|
||||
mat.use_nodes = mat.use_nodes
|
||||
print("Forced Cycles material updates")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not force Cycles updates: {e}")
|
||||
|
||||
# Verify device is actually set correctly
|
||||
if hasattr(scene.cycles, 'device'):
|
||||
actual_device = scene.cycles.device
|
||||
print(f"Cycles device setting: {actual_device}")
|
||||
if actual_device == 'GPU':
|
||||
# Try to verify GPU is actually available
|
||||
try:
|
||||
prefs = bpy.context.preferences
|
||||
cycles_prefs = prefs.addons['cycles'].preferences
|
||||
devices = cycles_prefs.devices
|
||||
enabled_devices = [d for d in devices if d.use]
|
||||
if len(enabled_devices) == 0:
|
||||
print("WARNING: GPU device set but no GPU devices are enabled!")
|
||||
print(" Falling back to CPU may cause issues")
|
||||
except Exception as e:
|
||||
print(f"Could not verify GPU devices: {e}")
|
||||
|
||||
# Ensure camera exists and is active
|
||||
if scene.camera is None:
|
||||
# Find first camera in scene
|
||||
for obj in scene.objects:
|
||||
if obj.type == 'CAMERA':
|
||||
scene.camera = obj
|
||||
print(f"Set active camera: {obj.name}")
|
||||
break
|
||||
|
||||
# Fix objects and collections hidden from render
|
||||
vl = bpy.context.view_layer
|
||||
|
||||
# 1. Objects hidden in view layer
|
||||
for obj in bpy.data.objects:
|
||||
if obj.hide_get(view_layer=vl):
|
||||
if any(k in obj.name.lower() for k in ["scrotum|","cage","genital","penis","dick","collision","body.001","couch"]):
|
||||
obj.hide_set(False, view_layer=vl)
|
||||
print("Enabled object:", obj.name)
|
||||
|
||||
# 2. Collections disabled in renders OR set to Holdout (the final killer)
|
||||
for col in bpy.data.collections:
|
||||
if col.hide_render or (vl.layer_collection.children.get(col.name) and not vl.layer_collection.children[col.name].exclude == False):
|
||||
if any(k in col.name.lower() for k in ["genital","nsfw","dick","private","hidden","cage","scrotum","collision","dick"]):
|
||||
col.hide_render = False
|
||||
if col.name in vl.layer_collection.children:
|
||||
vl.layer_collection.children[col.name].exclude = False
|
||||
vl.layer_collection.children[col.name].holdout = False
|
||||
vl.layer_collection.children[col.name].indirect_only = False
|
||||
print("Enabled collection:", col.name)
|
||||
|
||||
print("Headless rendering initialization complete")
|
||||
except Exception as e:
|
||||
print(f"Warning: Headless rendering initialization had issues: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
# Final verification before rendering
|
||||
print("\n=== Pre-render verification ===")
|
||||
try:
|
||||
scene = bpy.context.scene
|
||||
print(f"Render engine: {scene.render.engine}")
|
||||
print(f"Active camera: {scene.camera.name if scene.camera else 'None'}")
|
||||
|
||||
if scene.render.engine == 'CYCLES':
|
||||
print(f"Cycles device: {scene.cycles.device}")
|
||||
print(f"Cycles samples: {scene.cycles.samples}")
|
||||
lights = [obj for obj in scene.objects if obj.type == 'LIGHT']
|
||||
print(f"Lights in scene: {len(lights)}")
|
||||
if scene.world:
|
||||
if scene.world.use_nodes:
|
||||
world_nodes = scene.world.node_tree
|
||||
if world_nodes:
|
||||
bg_shader = None
|
||||
for node in world_nodes.nodes:
|
||||
if node.type == 'BACKGROUND':
|
||||
bg_shader = node
|
||||
break
|
||||
if bg_shader:
|
||||
if hasattr(bg_shader.inputs, 'Strength'):
|
||||
strength = bg_shader.inputs['Strength'].default_value
|
||||
print(f"World background strength: {strength}")
|
||||
if hasattr(bg_shader.inputs, 'Color'):
|
||||
color = bg_shader.inputs['Color'].default_value
|
||||
print(f"World background color: ({color[0]:.2f}, {color[1]:.2f}, {color[2]:.2f})")
|
||||
else:
|
||||
print("World exists but nodes are disabled")
|
||||
else:
|
||||
print("WARNING: No world in scene!")
|
||||
|
||||
print("=== Verification complete ===\n")
|
||||
except Exception as e:
|
||||
print(f"Warning: Verification failed: {e}")
|
||||
|
||||
print("Device configuration complete - blend file settings preserved, device optimized")
|
||||
sys.stdout.flush()
|
||||
`
|
||||
|
||||
// Check if unhide_objects is enabled
|
||||
unhideObjects := false
|
||||
if jobMetadata != nil && jobMetadata.UnhideObjects != nil && *jobMetadata.UnhideObjects {
|
||||
unhideObjects = true
|
||||
}
|
||||
|
||||
// Build unhide code conditionally from embedded script
|
||||
unhideCode := ""
|
||||
if unhideObjects {
|
||||
unhideCode = scripts.UnhideObjects
|
||||
}
|
||||
|
||||
// Load template and replace placeholders
|
||||
scriptContent := scripts.RenderBlenderTemplate
|
||||
scriptContent = strings.ReplaceAll(scriptContent, "{{UNHIDE_CODE}}", unhideCode)
|
||||
scriptContent = strings.ReplaceAll(scriptContent, "{{FORMAT_FILE_PATH}}", fmt.Sprintf("%q", formatFilePath))
|
||||
scriptContent = strings.ReplaceAll(scriptContent, "{{RENDER_SETTINGS_FILE}}", fmt.Sprintf("%q", renderSettingsFilePath))
|
||||
scriptPath := filepath.Join(workDir, "enable_gpu.py")
|
||||
if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil {
|
||||
errMsg := fmt.Sprintf("failed to create GPU enable script: %v", err)
|
||||
@@ -1765,23 +1141,30 @@ sys.stdout.flush()
|
||||
}
|
||||
}
|
||||
|
||||
// Check if execution should be enabled (defaults to false/off)
|
||||
enableExecution := false
|
||||
if jobMetadata != nil && jobMetadata.EnableExecution != nil && *jobMetadata.EnableExecution {
|
||||
enableExecution = true
|
||||
}
|
||||
|
||||
// Run Blender with GPU enabled via Python script
|
||||
// Use -s (start) and -e (end) for frame ranges, or -f for single frame
|
||||
var cmd *exec.Cmd
|
||||
args := []string{"-b", blendFile, "--python", scriptPath}
|
||||
if enableExecution {
|
||||
args = append(args, "--enable-autoexec")
|
||||
}
|
||||
if frameStart == frameEnd {
|
||||
// Single frame
|
||||
cmd = exec.Command("blender", "-b", blendFile,
|
||||
"--python", scriptPath,
|
||||
"-o", absOutputPattern,
|
||||
"-f", fmt.Sprintf("%d", frameStart))
|
||||
args = append(args, "-o", absOutputPattern, "-f", fmt.Sprintf("%d", frameStart))
|
||||
cmd = exec.Command("blender", args...)
|
||||
} else {
|
||||
// Frame range
|
||||
cmd = exec.Command("blender", "-b", blendFile,
|
||||
"--python", scriptPath,
|
||||
"-o", absOutputPattern,
|
||||
args = append(args, "-o", absOutputPattern,
|
||||
"-s", fmt.Sprintf("%d", frameStart),
|
||||
"-e", fmt.Sprintf("%d", frameEnd),
|
||||
"-a") // -a renders animation (all frames in range)
|
||||
cmd = exec.Command("blender", args...)
|
||||
}
|
||||
cmd.Dir = workDir
|
||||
|
||||
@@ -3261,8 +2644,11 @@ func (c *Client) getJobMetadata(jobID int64) (*types.BlendMetadata, error) {
|
||||
|
||||
// downloadFrameFile downloads a frame file for MP4 generation
|
||||
func (c *Client) downloadFrameFile(jobID int64, fileName, destPath string) error {
|
||||
path := fmt.Sprintf("/api/runner/files/%d/%s", jobID, fileName)
|
||||
resp, err := c.doSignedRequest("GET", path, nil, fmt.Sprintf("runner_id=%d", c.runnerID))
|
||||
// URL encode the fileName to handle special characters in filenames
|
||||
encodedFileName := url.PathEscape(fileName)
|
||||
path := fmt.Sprintf("/api/runner/files/%d/%s", jobID, encodedFileName)
|
||||
// Use long-running client for file downloads (no timeout) - EXR files can be large
|
||||
resp, err := c.doSignedRequestLong("GET", path, nil, fmt.Sprintf("runner_id=%d", c.runnerID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -3330,7 +2716,8 @@ func (c *Client) downloadFileToPath(filePath, destPath string) error {
|
||||
downloadPath += "/" + filepath.Base(filePath)
|
||||
}
|
||||
|
||||
resp, err := c.doSignedRequest("GET", downloadPath, nil, fmt.Sprintf("runner_id=%d", c.runnerID))
|
||||
// Use long-running client for file downloads (no timeout)
|
||||
resp, err := c.doSignedRequestLong("GET", downloadPath, nil, fmt.Sprintf("runner_id=%d", c.runnerID))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download file: %w", err)
|
||||
}
|
||||
@@ -3392,7 +2779,8 @@ func (c *Client) uploadFile(jobID int64, filePath string) (string, error) {
|
||||
req.Header.Set("Content-Type", formWriter.FormDataContentType())
|
||||
req.Header.Set("X-Runner-Secret", c.runnerSecret)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
// Use long-running client for file uploads (no timeout)
|
||||
resp, err := c.longRunningClient.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to upload file: %w", err)
|
||||
}
|
||||
@@ -3424,7 +2812,7 @@ func (c *Client) getContextCacheKey(jobID int64) string {
|
||||
func (c *Client) getContextCachePath(cacheKey string) string {
|
||||
cacheDir := filepath.Join(c.getWorkspaceDir(), "cache", "contexts")
|
||||
os.MkdirAll(cacheDir, 0755)
|
||||
return filepath.Join(cacheDir, cacheKey+".tar.gz")
|
||||
return filepath.Join(cacheDir, cacheKey+".tar")
|
||||
}
|
||||
|
||||
// isContextCacheValid checks if a cached context file exists and is not expired (1 hour TTL)
|
||||
@@ -3437,7 +2825,7 @@ func (c *Client) isContextCacheValid(cachePath string) bool {
|
||||
return time.Since(info.ModTime()) < time.Hour
|
||||
}
|
||||
|
||||
// downloadJobContext downloads the job context tar.gz, using cache if available
|
||||
// downloadJobContext downloads the job context tar, using cache if available
|
||||
func (c *Client) downloadJobContext(jobID int64, destPath string) error {
|
||||
cacheKey := c.getContextCacheKey(jobID)
|
||||
cachePath := c.getContextCachePath(cacheKey)
|
||||
@@ -3464,9 +2852,9 @@ func (c *Client) downloadJobContext(jobID int64, destPath string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Download from manager
|
||||
path := fmt.Sprintf("/api/runner/jobs/%d/context.tar.gz", jobID)
|
||||
resp, err := c.doSignedRequest("GET", path, nil, fmt.Sprintf("runner_id=%d", c.runnerID))
|
||||
// Download from manager - use long-running client (no timeout) for large context files
|
||||
path := fmt.Sprintf("/api/runner/jobs/%d/context.tar", jobID)
|
||||
resp, err := c.doSignedRequestLong("GET", path, nil, fmt.Sprintf("runner_id=%d", c.runnerID))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to download context: %w", err)
|
||||
}
|
||||
@@ -3517,24 +2905,17 @@ func (c *Client) downloadJobContext(jobID int64, destPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractTarGz extracts a tar.gz file to the destination directory
|
||||
func (c *Client) extractTarGz(tarGzPath, destDir string) error {
|
||||
// Open the tar.gz file
|
||||
file, err := os.Open(tarGzPath)
|
||||
// extractTar extracts a tar file to the destination directory
|
||||
func (c *Client) extractTar(tarPath, destDir string) error {
|
||||
// Open the tar file
|
||||
file, err := os.Open(tarPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open tar.gz file: %w", err)
|
||||
return fmt.Errorf("failed to open tar file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Create gzip reader
|
||||
gzReader, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create gzip reader: %w", err)
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
// Create tar reader
|
||||
tarReader := tar.NewReader(gzReader)
|
||||
tarReader := tar.NewReader(file)
|
||||
|
||||
// Extract files
|
||||
for {
|
||||
@@ -3635,16 +3016,16 @@ func (c *Client) processMetadataTask(task map[string]interface{}, jobID int64, i
|
||||
c.sendStepUpdate(taskID, "download", types.StepStatusRunning, "")
|
||||
c.sendLog(taskID, types.LogLevelInfo, "Downloading job context...", "download")
|
||||
|
||||
// Download context tar.gz
|
||||
contextPath := filepath.Join(workDir, "context.tar.gz")
|
||||
// Download context tar
|
||||
contextPath := filepath.Join(workDir, "context.tar")
|
||||
if err := c.downloadJobContext(jobID, contextPath); err != nil {
|
||||
c.sendStepUpdate(taskID, "download", types.StepStatusFailed, err.Error())
|
||||
return fmt.Errorf("failed to download context: %w", err)
|
||||
}
|
||||
|
||||
// Extract context tar.gz
|
||||
// Extract context tar
|
||||
c.sendLog(taskID, types.LogLevelInfo, "Extracting context...", "download")
|
||||
if err := c.extractTarGz(contextPath, workDir); err != nil {
|
||||
if err := c.extractTar(contextPath, workDir); err != nil {
|
||||
c.sendStepUpdate(taskID, "download", types.StepStatusFailed, err.Error())
|
||||
return fmt.Errorf("failed to extract context: %w", err)
|
||||
}
|
||||
@@ -3881,6 +3262,7 @@ sys.stdout.flush()
|
||||
}
|
||||
|
||||
// Execute Blender with Python script
|
||||
// Note: disable_execution flag is not applied to metadata extraction for safety
|
||||
cmd := exec.Command("blender", "-b", blendFile, "--python", scriptPath)
|
||||
cmd.Dir = workDir
|
||||
|
||||
|
||||
Reference in New Issue
Block a user