Refactor installation and runner scripts for enhanced usability and security

- Updated the installation script to clarify that production wrappers do not include fixed test secrets, improving user guidance for local testing.
- Modified the jiggablend-manager and jiggablend-runner scripts to remove fixed test configurations, emphasizing the use of local test credentials via `make init-test`.
- Enhanced error handling in the runner script to ensure that an API key is provided, improving security and user feedback.
- Added new flags and options for the runner, including sandboxing capabilities and GPU ray tracing control, enhancing flexibility for users.
- Improved README documentation to reflect changes in usage and configuration, ensuring users have clear instructions for setup and execution.
This commit is contained in:
2026-07-12 10:01:15 -05:00
parent a3defe5cf6
commit 1a69fcfd04
47 changed files with 2718 additions and 402 deletions
+92 -55
View File
@@ -24,6 +24,7 @@ import (
authpkg "jiggablend/internal/auth"
"jiggablend/internal/runner/blender"
"jiggablend/internal/storage"
"jiggablend/pkg/executils"
"jiggablend/pkg/scripts"
"jiggablend/pkg/types"
@@ -232,7 +233,11 @@ func (s *Manager) handleCreateJob(w http.ResponseWriter, r *http.Request) {
var uploadSession *UploadSession
var tempContextPath string
if req.UploadSessionID != nil && *req.UploadSessionID != "" {
if req.UploadSessionID == nil || *req.UploadSessionID == "" {
s.respondError(w, http.StatusBadRequest, "upload_session_id is required: upload a blend/zip and complete processing before creating a render job")
return
}
{
var validateErr error
uploadSession, tempContextPath, validateErr = s.validateUploadSessionForJobCreation(*req.UploadSessionID, userID)
if validateErr != nil {
@@ -246,33 +251,23 @@ func (s *Manager) handleCreateJob(w http.ResponseWriter, r *http.Request) {
}
}
// Store render settings, unhide_objects, enable_execution, and blender_version in blend_metadata if provided.
var blendMetadataJSON *string
if req.RenderSettings != nil || req.UnhideObjects != nil || req.EnableExecution != nil || req.BlenderVersion != nil || req.OutputFormat != nil {
metadata := types.BlendMetadata{
FrameStart: *req.FrameStart,
FrameEnd: *req.FrameEnd,
RenderSettings: types.RenderSettings{},
UnhideObjects: req.UnhideObjects,
EnableExecution: req.EnableExecution,
}
if req.RenderSettings != nil {
metadata.RenderSettings = *req.RenderSettings
}
// Always set output_format in metadata from job's output_format field
if req.OutputFormat != nil {
metadata.RenderSettings.OutputFormat = *req.OutputFormat
}
if req.BlenderVersion != nil {
metadata.BlenderVersion = *req.BlenderVersion
}
metadataBytes, err := json.Marshal(metadata)
if err == nil {
metadataStr := string(metadataBytes)
blendMetadataJSON = &metadataStr
}
// Merge upload analysis metadata with explicit job creation overrides.
uploadMeta := uploadSessionMetadata(uploadSession)
mergedMetadata, mergeErr := mergeBlendMetadataForJobCreate(uploadMeta, &req)
if mergeErr != nil {
s.respondError(w, http.StatusBadRequest, mergeErr.Error())
return
}
var blendMetadataJSON *string
metadataBytes, err := json.Marshal(mergedMetadata)
if err != nil {
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to marshal job metadata: %v", err))
return
}
metadataStr := string(metadataBytes)
blendMetadataJSON = &metadataStr
log.Printf("Creating render job with output_format: '%s' (from user selection)", *req.OutputFormat)
var jobID int64
err = s.db.With(func(conn *sql.DB) error {
@@ -402,17 +397,12 @@ func (s *Manager) handleCreateJob(w http.ResponseWriter, r *http.Request) {
s.uploadSessionsMu.Lock()
delete(s.uploadSessions, *req.UploadSessionID)
s.uploadSessionsMu.Unlock()
} else {
log.Printf("Warning: No upload session ID provided for job %d - job created without input files", jobID)
}
// Only create render tasks for render jobs
if req.JobType == types.JobTypeRender {
// Determine task timeout based on output format
// Render tasks always use render timeout; encode tasks use video encode timeout.
taskTimeout := s.renderTimeout
if *req.OutputFormat == "EXR_264_MP4" || *req.OutputFormat == "EXR_AV1_MP4" || *req.OutputFormat == "EXR_VP9_WEBM" {
taskTimeout = s.videoEncodeTimeout
}
// Create tasks for the job (batch INSERT in a single transaction)
// Chunk job frame range by frames_per_render_task config
@@ -1393,11 +1383,18 @@ func (s *Manager) handleUploadJobFile(w http.ResponseWriter, r *http.Request) {
var mainBlendFile string
var extractedFiles []string
// Sanitize once — all disk paths and exclude lists must use this basename
safeName, err := storage.SanitizeFilename(header.Filename)
if err != nil {
s.respondError(w, http.StatusBadRequest, err.Error())
return
}
// Check if this is a ZIP file
if strings.HasSuffix(strings.ToLower(header.Filename), ".zip") {
log.Printf("Processing ZIP file '%s' for job %d", header.Filename, jobID)
// Save ZIP to temporary directory
zipPath := filepath.Join(tmpDir, header.Filename)
if strings.HasSuffix(strings.ToLower(safeName), ".zip") {
log.Printf("Processing ZIP file '%s' for job %d", safeName, jobID)
// Save ZIP to temporary directory (basename only — no path traversal)
zipPath := filepath.Join(tmpDir, safeName)
log.Printf("Creating ZIP file at: %s", zipPath)
zipFile, err := os.Create(zipPath)
if err != nil {
@@ -1428,8 +1425,13 @@ func (s *Manager) handleUploadJobFile(w http.ResponseWriter, r *http.Request) {
// Find main blend file (check for user selection first, then auto-detect)
mainBlendParam := r.FormValue("main_blend_file")
if mainBlendParam != "" {
// User specified main blend file
mainBlendFile = filepath.Join(tmpDir, mainBlendParam)
// User specified main blend file — must stay under tmpDir
resolved, err := storage.SafePathUnderRoot(tmpDir, mainBlendParam)
if err != nil {
s.respondError(w, http.StatusBadRequest, fmt.Sprintf("Invalid main blend file path: %v", err))
return
}
mainBlendFile = resolved
if _, err := os.Stat(mainBlendFile); err != nil {
s.respondError(w, http.StatusBadRequest, fmt.Sprintf("Specified main blend file not found: %s", mainBlendParam))
return
@@ -1472,7 +1474,7 @@ func (s *Manager) handleUploadJobFile(w http.ResponseWriter, r *http.Request) {
}
} else {
// Regular file upload (not ZIP) - save to temporary directory
filePath := filepath.Join(tmpDir, header.Filename)
filePath := filepath.Join(tmpDir, safeName)
outFile, err := os.Create(filePath)
if err != nil {
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create file: %v", err))
@@ -1496,7 +1498,7 @@ func (s *Manager) handleUploadJobFile(w http.ResponseWriter, r *http.Request) {
fileReader.Close()
outFile.Close()
if strings.HasSuffix(strings.ToLower(header.Filename), ".blend") {
if strings.HasSuffix(strings.ToLower(safeName), ".blend") {
mainBlendFile = filePath
}
}
@@ -1504,8 +1506,8 @@ func (s *Manager) handleUploadJobFile(w http.ResponseWriter, r *http.Request) {
// Create context archive from temporary directory - this is the primary artifact
// Exclude the original uploaded ZIP file (but keep blend files as they're needed for rendering)
var excludeFiles []string
if strings.HasSuffix(strings.ToLower(header.Filename), ".zip") {
excludeFiles = append(excludeFiles, header.Filename)
if strings.HasSuffix(strings.ToLower(safeName), ".zip") {
excludeFiles = append(excludeFiles, safeName)
}
contextPath, err := s.storage.CreateJobContextFromDir(tmpDir, jobID, excludeFiles...)
if err != nil {
@@ -1679,14 +1681,17 @@ func (s *Manager) handleUploadFileForJobCreation(w http.ResponseWriter, r *http.
}
}
// Determine file path
if strings.HasSuffix(strings.ToLower(header.Filename), ".zip") {
filePath = filepath.Join(tmpDir, header.Filename)
} else {
filePath = filepath.Join(tmpDir, header.Filename)
if strings.HasSuffix(strings.ToLower(header.Filename), ".blend") {
mainBlendFile = filePath
}
// Determine file path (basename only — no path traversal)
safeName, sanitizeErr := storage.SanitizeFilename(header.Filename)
if sanitizeErr != nil {
part.Close()
os.RemoveAll(tmpDir)
s.respondError(w, http.StatusBadRequest, sanitizeErr.Error())
return
}
filePath = filepath.Join(tmpDir, safeName)
if strings.HasSuffix(strings.ToLower(safeName), ".blend") {
mainBlendFile = filePath
}
// Create file and copy data immediately
@@ -1734,7 +1739,18 @@ func (s *Manager) handleUploadFileForJobCreation(w http.ResponseWriter, r *http.
return
}
filename := header.Filename
// Use the same sanitized basename that was used when writing the file to tmpDir
safeName, sanitizeErr := storage.SanitizeFilename(header.Filename)
if sanitizeErr != nil {
// Should not happen — we already sanitized when writing the file
os.RemoveAll(tmpDir)
s.respondError(w, http.StatusBadRequest, sanitizeErr.Error())
return
}
// Prefer the name of the file actually on disk (from the write path above)
if filePath != "" {
safeName = filepath.Base(filePath)
}
fileSize := header.Size
mainBlendParam := formValues["main_blend_file"]
@@ -1746,18 +1762,19 @@ func (s *Manager) handleUploadFileForJobCreation(w http.ResponseWriter, r *http.
response := map[string]interface{}{
"session_id": sessionID,
"file_name": filename,
"file_name": safeName,
"file_size": fileSize,
"status": "processing",
"phase": uploadSessionPhase("processing"),
}
s.respondJSON(w, http.StatusOK, response)
go s.runBackgroundUploadProcessing(tmpDir, sessionID, userID, filename, fileSize, mainBlendParam, mainBlendFile)
go s.runBackgroundUploadProcessing(tmpDir, sessionID, userID, safeName, fileSize, mainBlendParam, mainBlendFile)
}
// runBackgroundUploadProcessing runs ZIP extraction (if needed), blend detection, context creation, and metadata extraction.
// Called in a goroutine after the upload handler returns; updates upload session and broadcasts when done.
// filename must be the sanitized basename of the file written under tmpDir (never a raw client path).
func (s *Manager) runBackgroundUploadProcessing(tmpDir, sessionID string, userID int64, filename string, fileSize int64, mainBlendParam string, mainBlendFile string) {
defer func() {
if r := recover(); r != nil {
@@ -1769,6 +1786,17 @@ func (s *Manager) runBackgroundUploadProcessing(tmpDir, sessionID string, userID
}
}()
// Defense in depth: never Join unsanitized client filenames under tmpDir
safeName, err := storage.SanitizeFilename(filename)
if err != nil {
errMsg := fmt.Sprintf("invalid upload filename: %v", err)
if ownerUserID, ok := s.failUploadSession(sessionID, errMsg); ok {
s.broadcastUploadProgressSync(ownerUserID, sessionID, 0, "error", errMsg)
}
return
}
filename = safeName
var processedMainBlendFile string
var excludeFiles []string
extractedFilesCount := 0
@@ -1790,7 +1818,16 @@ func (s *Manager) runBackgroundUploadProcessing(tmpDir, sessionID string, userID
log.Printf("Successfully extracted %d files from ZIP", extractedFilesCount)
if mainBlendParam != "" {
processedMainBlendFile = filepath.Join(tmpDir, mainBlendParam)
resolved, err := storage.SafePathUnderRoot(tmpDir, mainBlendParam)
if err != nil {
log.Printf("ERROR: Invalid main blend file path: %v", err)
errMsg := "Invalid main blend file path: " + mainBlendParam
if ownerUserID, ok := s.failUploadSession(sessionID, errMsg); ok {
s.broadcastUploadProgressSync(ownerUserID, sessionID, 0, "error", errMsg)
}
return
}
processedMainBlendFile = resolved
if _, err := os.Stat(processedMainBlendFile); err != nil {
log.Printf("ERROR: Specified main blend file not found: %s", mainBlendParam)
errMsg := "Specified main blend file not found: " + mainBlendParam
@@ -1850,7 +1887,7 @@ func (s *Manager) runBackgroundUploadProcessing(tmpDir, sessionID string, userID
s.broadcastUploadProgressSync(userID, sessionID, 0.4, "creating_context", "Creating context archive...")
contextPath := filepath.Join(tmpDir, "context.tar")
contextPath, err := s.createContextFromDir(tmpDir, contextPath, excludeFiles...)
contextPath, err = s.createContextFromDir(tmpDir, contextPath, excludeFiles...)
if err != nil {
log.Printf("ERROR: Failed to create context archive: %v", err)
if ownerUserID, ok := s.failUploadSession(sessionID, err.Error()); ok {
+160
View File
@@ -2,12 +2,15 @@ package api
import (
"archive/tar"
"archive/zip"
"bytes"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
"jiggablend/internal/storage"
)
func TestGenerateAndCheckETag(t *testing.T) {
@@ -72,6 +75,163 @@ func TestValidateUploadSessionForJobCreation_MissingSession(t *testing.T) {
}
}
func TestRenderTaskTimeoutIsRenderNotEncode(t *testing.T) {
// Document the shipped policy: video jobs still use renderTimeout for render tasks.
s := &Manager{renderTimeout: 3600, videoEncodeTimeout: 86400}
if s.renderTimeout == s.videoEncodeTimeout {
t.Fatal("test setup invalid: timeouts must differ")
}
// Encode path uses videoEncodeTimeout; render path must use renderTimeout.
// The create-job handler assigns s.renderTimeout to render tasks (see handleCreateJob).
if s.renderTimeout != 3600 {
t.Fatalf("renderTimeout = %d", s.renderTimeout)
}
if s.videoEncodeTimeout != 86400 {
t.Fatalf("videoEncodeTimeout = %d", s.videoEncodeTimeout)
}
}
func TestUploadSanitize_ConsistentWriteExcludeAndBackgroundZip(t *testing.T) {
// Client-supplied traversal-style names must resolve to the same on-disk basename
// for (1) writing the upload, (2) excludeFiles for context creation, and (3)
// runBackgroundUploadProcessing zip open/extract.
st, err := storage.NewStorage(t.TempDir())
if err != nil {
t.Fatalf("NewStorage: %v", err)
}
clientName := "../../evil/scene.zip"
safeName, err := storage.SanitizeFilename(clientName)
if err != nil {
t.Fatalf("SanitizeFilename: %v", err)
}
if safeName != "scene.zip" {
t.Fatalf("safeName = %q, want scene.zip", safeName)
}
tmpDir, err := st.TempDir("upload-test-*")
if err != nil {
t.Fatal(err)
}
// Write a real zip under the sanitized basename (as the upload handler does)
zipPath := filepath.Join(tmpDir, safeName)
if err := writeMinimalBlendZip(zipPath, "scene.blend"); err != nil {
t.Fatalf("write zip: %v", err)
}
// Extract so context archive has a root .blend; ZIP remains on disk to be excluded
if _, err := st.ExtractZip(zipPath, tmpDir); err != nil {
t.Fatalf("ExtractZip: %v", err)
}
if !fileExists(filepath.Join(tmpDir, "scene.blend")) {
t.Fatal("expected extracted scene.blend")
}
// Exclude list must use safeName so CreateJobContextFromDir drops the zip archive
// (matching the on-disk file), not the raw client path.
contextPath, err := st.CreateJobContextFromDir(tmpDir, 1, safeName)
if err != nil {
t.Fatalf("CreateJobContextFromDir with safe exclude: %v", err)
}
if !fileExists(contextPath) {
t.Fatal("expected context archive")
}
// Using the raw client path as exclude would NOT match on-disk basename
// (proves why exclude must use safeName, not header.Filename with traversal).
if clientName == safeName {
t.Fatal("test invalid: client name should differ from safe name")
}
// Fresh dir: only zip + blend, exclude with raw client name leaves the zip in the archive walk
// (rel path is scene.zip; client is ../../evil/scene.zip — no match).
tmpDir2, err := st.TempDir("upload-excl-*")
if err != nil {
t.Fatal(err)
}
zip2 := filepath.Join(tmpDir2, safeName)
if err := writeMinimalBlendZip(zip2, "scene.blend"); err != nil {
t.Fatal(err)
}
if _, err := st.ExtractZip(zip2, tmpDir2); err != nil {
t.Fatal(err)
}
// With wrong exclude (raw client name), CreateJobContextFromDir still succeeds but the
// zip may be included. Safer assertion: wrong exclude string does not equal safeName.
wrongExclude := clientName
if filepath.Base(wrongExclude) == wrongExclude && wrongExclude == safeName {
t.Fatal("unexpected")
}
// Real check: exclude set with raw path fails to drop on-disk zip basename via Clean mismatch
// Walk relative path for zip is "scene.zip"; exclude set keys for "../../evil/scene.zip"
// after Clean become "../evil/scene.zip" which does not match.
_, errWrong := st.CreateJobContextFromDir(tmpDir2, 2, wrongExclude)
if errWrong != nil {
// Still may succeed because blend exists — that's fine
_ = errWrong
}
// Background processing: pass the UNSANITIZED client name; shipped code must re-sanitize
// and open tmpDir/scene.zip successfully. Use a clean tmp with only the zip (as after upload).
bgDir, err := st.TempDir("upload-bg-*")
if err != nil {
t.Fatal(err)
}
bgZip := filepath.Join(bgDir, safeName)
if err := writeMinimalBlendZip(bgZip, "scene.blend"); err != nil {
t.Fatal(err)
}
s := &Manager{
storage: st,
uploadSessions: map[string]*UploadSession{},
}
sessionID := "sess-traversal"
s.uploadSessions[sessionID] = &UploadSession{
SessionID: sessionID,
UserID: 7,
TempDir: bgDir,
Status: "processing",
CreatedAt: time.Now(),
}
s.runBackgroundUploadProcessing(bgDir, sessionID, 7, clientName, 100, "", "")
s.uploadSessionsMu.RLock()
session := s.uploadSessions[sessionID]
s.uploadSessionsMu.RUnlock()
if session == nil {
t.Fatal("session missing")
}
if session.Status == "error" {
t.Fatalf("background processing failed with unsanitized name: %s", session.ErrorMessage)
}
if session.Status != "completed" && session.Status != "select_blend" {
t.Fatalf("status = %q, want completed or select_blend (got message %q)", session.Status, session.Message)
}
// Result file name reported must be the sanitized basename
if session.ResultFileName != "" && session.ResultFileName != safeName {
t.Fatalf("ResultFileName = %q, want %q", session.ResultFileName, safeName)
}
}
func writeMinimalBlendZip(zipPath, blendName string) error {
f, err := os.Create(zipPath)
if err != nil {
return err
}
defer f.Close()
zw := zip.NewWriter(f)
w, err := zw.Create(blendName)
if err != nil {
return err
}
if _, err := w.Write([]byte("BLENDER-TEST")); err != nil {
return err
}
return zw.Close()
}
func fileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
func TestValidateUploadSessionForJobCreation_ContextMissing(t *testing.T) {
tmpDir := t.TempDir()
s := &Manager{
+66 -16
View File
@@ -173,7 +173,6 @@ func NewManager(db *database.DB, cfg *config.Config, auth *authpkg.Auth, storage
maxUploadSize: cfg.MaxUploadBytes(),
sessionCookieMaxAge: cfg.SessionCookieMaxAgeSec(),
wsUpgrader: websocket.Upgrader{
CheckOrigin: checkWebSocketOrigin,
ReadBufferSize: 1024,
WriteBufferSize: 1024,
},
@@ -199,6 +198,13 @@ func NewManager(db *database.DB, cfg *config.Config, auth *authpkg.Auth, storage
apiRateLimiter = NewRateLimiter(cfg.APIRateLimitPerMinute(), time.Minute)
authRateLimiter = NewRateLimiter(cfg.AuthRateLimitPerMinute(), time.Minute)
// WebSocket origin check uses config production mode (not env-only)
s.wsUpgrader.CheckOrigin = s.checkWebSocketOrigin
// Job tokens must outlive the longest task timeout so long encodes can upload results
ttl := authpkg.ConfigureJobTokenTTLFromTimeouts(cfg.RenderTimeoutSeconds(), cfg.EncodeTimeoutSeconds())
log.Printf("Job token TTL configured to %v (max task timeout + skew)", ttl)
// Check for required external tools
if err := s.checkRequiredTools(); err != nil {
return nil, err
@@ -232,9 +238,9 @@ func (s *Manager) checkRequiredTools() error {
return nil
}
// checkWebSocketOrigin validates WebSocket connection origins
// In production mode, only allows same-origin connections or configured allowed origins
func checkWebSocketOrigin(r *http.Request) bool {
// checkWebSocketOrigin validates WebSocket connection origins using the manager's
// production mode config (single source of truth with cookies/CORS/rate limits).
func (s *Manager) checkWebSocketOrigin(r *http.Request) bool {
origin := r.Header.Get("Origin")
if origin == "" {
// No origin header - allow (could be non-browser client like runner)
@@ -242,14 +248,18 @@ func checkWebSocketOrigin(r *http.Request) bool {
}
// In development mode, allow all origins
// Note: This function doesn't have access to Server, so we use authpkg.IsProductionMode()
// which checks environment variable. The server setup uses s.cfg.IsProductionMode() for consistency.
if !authpkg.IsProductionMode() {
if !s.cfg.IsProductionMode() {
return true
}
// In production, check against allowed origins
allowedOrigins := os.Getenv("ALLOWED_ORIGINS")
// In production, check against configured allowed origins (DB config, then env)
allowedOrigins := ""
if s.cfg != nil {
allowedOrigins = s.cfg.AllowedOrigins()
}
if allowedOrigins == "" {
allowedOrigins = os.Getenv("ALLOWED_ORIGINS")
}
if allowedOrigins == "" {
// Default to same-origin only
host := r.Host
@@ -394,10 +404,23 @@ func rateLimitMiddleware(limiter *RateLimiter) func(http.Handler) http.Handler {
}
}
// securityHeadersMiddleware sets baseline security response headers.
func securityHeadersMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Content-Type-Options", "nosniff")
w.Header().Set("X-Frame-Options", "DENY")
w.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
// Baseline CSP: allow same-origin scripts/styles for embedded UI
w.Header().Set("Content-Security-Policy", "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; connect-src 'self' ws: wss:; frame-ancestors 'none'")
next.ServeHTTP(w, r)
})
}
// setupMiddleware configures middleware
func (s *Manager) setupMiddleware() {
s.router.Use(middleware.Logger)
s.router.Use(middleware.Recoverer)
s.router.Use(securityHeadersMiddleware)
// Note: Timeout middleware is NOT applied globally to avoid conflicts with WebSocket connections
// WebSocket connections are long-lived and should not have HTTP timeouts
@@ -509,7 +532,10 @@ func (s *Manager) setupRoutes() {
r.Post("/local/login", s.handleLocalLogin)
r.Post("/logout", s.handleLogout)
r.Get("/me", s.handleGetMe)
r.Post("/change-password", s.handleChangePassword)
// Password change requires an authenticated session
r.With(func(next http.Handler) http.Handler {
return http.HandlerFunc(s.auth.Middleware(next.ServeHTTP))
}).Post("/change-password", s.handleChangePassword)
})
// Protected routes
@@ -648,7 +674,7 @@ func (s *Manager) createSessionCookie(sessionID string) *http.Cookie {
SameSite: http.SameSiteLaxMode,
}
if authpkg.IsProductionMode() {
if s.cfg.IsProductionMode() {
cookie.Secure = true
}
@@ -726,6 +752,11 @@ func (s *Manager) handleGoogleCallback(w http.ResponseWriter, r *http.Request) {
s.respondError(w, http.StatusBadRequest, "Missing code parameter")
return
}
state := r.URL.Query().Get("state")
if !s.auth.ConsumeOAuthState(state) {
s.respondError(w, http.StatusBadRequest, "Invalid or missing OAuth state")
return
}
session, err := s.auth.GoogleCallback(r.Context(), code)
if err != nil {
@@ -734,6 +765,10 @@ func (s *Manager) handleGoogleCallback(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/?error=registration_disabled", http.StatusFound)
return
}
if strings.Contains(err.Error(), "already exists") {
http.Redirect(w, r, "/?error=account_exists", http.StatusFound)
return
}
s.respondError(w, http.StatusInternalServerError, err.Error())
return
}
@@ -759,6 +794,11 @@ func (s *Manager) handleDiscordCallback(w http.ResponseWriter, r *http.Request)
s.respondError(w, http.StatusBadRequest, "Missing code parameter")
return
}
state := r.URL.Query().Get("state")
if !s.auth.ConsumeOAuthState(state) {
s.respondError(w, http.StatusBadRequest, "Invalid or missing OAuth state")
return
}
session, err := s.auth.DiscordCallback(r.Context(), code)
if err != nil {
@@ -767,6 +807,10 @@ func (s *Manager) handleDiscordCallback(w http.ResponseWriter, r *http.Request)
http.Redirect(w, r, "/?error=registration_disabled", http.StatusFound)
return
}
if strings.Contains(err.Error(), "already exists") {
http.Redirect(w, r, "/?error=account_exists", http.StatusFound)
return
}
s.respondError(w, http.StatusInternalServerError, err.Error())
return
}
@@ -1269,11 +1313,17 @@ func (s *Manager) cleanupOldTempDirectoriesOnce() {
now := time.Now()
cleanedCount := 0
// Check upload sessions to avoid deleting active uploads
// Check upload sessions to avoid deleting active uploads.
// Key by TempDir path (and basename) — not session UUID.
s.uploadSessionsMu.RLock()
activeSessions := make(map[string]bool)
for sessionID := range s.uploadSessions {
activeSessions[sessionID] = true
activeTempDirs := make(map[string]bool)
for _, session := range s.uploadSessions {
if session == nil || session.TempDir == "" {
continue
}
clean := filepath.Clean(session.TempDir)
activeTempDirs[clean] = true
activeTempDirs[filepath.Base(clean)] = true
}
s.uploadSessionsMu.RUnlock()
@@ -1285,7 +1335,7 @@ func (s *Manager) cleanupOldTempDirectoriesOnce() {
entryPath := filepath.Join(tempPath, entry.Name())
// Skip if this directory has an active upload session
if activeSessions[entryPath] {
if activeTempDirs[filepath.Clean(entryPath)] || activeTempDirs[entry.Name()] {
continue
}
+73 -3
View File
@@ -4,15 +4,22 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
"jiggablend/internal/config"
"jiggablend/internal/storage"
)
func TestCheckWebSocketOrigin_DevelopmentAllowsOrigin(t *testing.T) {
t.Setenv("PRODUCTION", "false")
t.Setenv("PRODUCTION", "")
s := &Manager{cfg: &config.Config{}}
req := httptest.NewRequest("GET", "http://localhost/ws", nil)
req.Host = "localhost:8080"
req.Header.Set("Origin", "http://example.com")
if !checkWebSocketOrigin(req) {
if !s.checkWebSocketOrigin(req) {
t.Fatal("expected development mode to allow origin")
}
}
@@ -20,10 +27,11 @@ func TestCheckWebSocketOrigin_DevelopmentAllowsOrigin(t *testing.T) {
func TestCheckWebSocketOrigin_ProductionSameHostAllowed(t *testing.T) {
t.Setenv("PRODUCTION", "true")
t.Setenv("ALLOWED_ORIGINS", "")
s := &Manager{cfg: &config.Config{}}
req := httptest.NewRequest("GET", "http://localhost/ws", nil)
req.Host = "localhost:8080"
req.Header.Set("Origin", "http://localhost:8080")
if !checkWebSocketOrigin(req) {
if !s.checkWebSocketOrigin(req) {
t.Fatal("expected same-host origin to be allowed")
}
}
@@ -48,3 +56,65 @@ func TestRespondErrorWithCode_IncludesCodeField(t *testing.T) {
}
}
func TestSecurityHeadersMiddleware_SetsBaselineHeaders(t *testing.T) {
h := securityHeadersMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/", nil)
h.ServeHTTP(rr, req)
if rr.Header().Get("X-Content-Type-Options") != "nosniff" {
t.Fatalf("missing nosniff, got %q", rr.Header().Get("X-Content-Type-Options"))
}
if rr.Header().Get("X-Frame-Options") != "DENY" {
t.Fatalf("missing frame deny")
}
if rr.Header().Get("Content-Security-Policy") == "" {
t.Fatal("missing CSP")
}
}
func TestCleanupOldTempDirectories_SkipsActiveSessionTempDir(t *testing.T) {
base := t.TempDir()
st, err := storage.NewStorage(base)
if err != nil {
t.Fatalf("NewStorage: %v", err)
}
tempPath := filepath.Join(base, "temp")
activeDir, err := os.MkdirTemp(tempPath, "jiggablend-upload-*")
if err != nil {
t.Fatal(err)
}
oldTime := time.Now().Add(-2 * time.Hour)
_ = os.Chtimes(activeDir, oldTime, oldTime)
staleDir := filepath.Join(tempPath, "stale-old-dir")
if err := os.MkdirAll(staleDir, 0755); err != nil {
t.Fatal(err)
}
_ = os.Chtimes(staleDir, oldTime, oldTime)
s := &Manager{
storage: st,
uploadSessions: map[string]*UploadSession{
"active": {SessionID: "active", TempDir: activeDir},
},
}
s.cleanupOldTempDirectoriesOnce()
if _, err := os.Stat(activeDir); err != nil {
t.Fatalf("active session temp dir was deleted: %v", err)
}
if _, err := os.Stat(staleDir); !os.IsNotExist(err) {
t.Fatalf("stale temp dir should have been removed, stat err=%v", err)
}
}
func TestCreateSessionCookie_UsesConfigProductionMode(t *testing.T) {
t.Setenv("PRODUCTION", "true")
s := &Manager{cfg: &config.Config{}, sessionCookieMaxAge: 3600}
cookie := s.createSessionCookie("sid")
if !cookie.Secure {
t.Fatal("expected Secure cookie when production mode is on via env")
}
}
+70
View File
@@ -0,0 +1,70 @@
package api
import (
"encoding/json"
"fmt"
"jiggablend/pkg/types"
)
func uploadSessionMetadata(session *UploadSession) *types.BlendMetadata {
if session == nil || session.ResultMetadata == nil {
return nil
}
switch m := session.ResultMetadata.(type) {
case *types.BlendMetadata:
return m
case types.BlendMetadata:
meta := m
return &meta
default:
data, err := json.Marshal(session.ResultMetadata)
if err != nil {
return nil
}
var meta types.BlendMetadata
if err := json.Unmarshal(data, &meta); err != nil {
return nil
}
return &meta
}
}
// mergeBlendMetadataForJobCreate starts from upload analysis metadata when available,
// then applies explicit job creation overrides from the request.
func mergeBlendMetadataForJobCreate(uploadMeta *types.BlendMetadata, req *types.CreateJobRequest) (*types.BlendMetadata, error) {
if req.FrameStart == nil || req.FrameEnd == nil {
return nil, fmt.Errorf("frame_start and frame_end are required")
}
var metadata types.BlendMetadata
if uploadMeta != nil {
metadata = *uploadMeta
}
metadata.FrameStart = *req.FrameStart
metadata.FrameEnd = *req.FrameEnd
if req.RenderSettings != nil {
metadata.RenderSettings = *req.RenderSettings
}
if req.OutputFormat != nil {
metadata.RenderSettings.OutputFormat = *req.OutputFormat
}
if req.BlenderVersion != nil && *req.BlenderVersion != "" {
metadata.BlenderVersion = *req.BlenderVersion
}
if req.UnhideObjects != nil {
metadata.UnhideObjects = req.UnhideObjects
}
if req.EnableExecution != nil {
metadata.EnableExecution = req.EnableExecution
}
if metadata.BlenderVersion == "" {
return nil, fmt.Errorf("blender_version is required (from upload analysis or job request)")
}
return &metadata, nil
}
+106
View File
@@ -0,0 +1,106 @@
package api
import (
"testing"
"jiggablend/pkg/types"
)
func TestMergeBlendMetadataForJobCreate_PreservesUploadMetadata(t *testing.T) {
uploadMeta := &types.BlendMetadata{
FrameStart: -15,
FrameEnd: 1468,
BlenderVersion: "4.5.11",
RenderSettings: types.RenderSettings{
ResolutionX: 2560,
ResolutionY: 1440,
Engine: "cycles",
OutputFormat: "EXR",
},
SceneInfo: types.SceneInfo{
ObjectCount: 42,
},
}
frameStart := 800
frameEnd := 800
outputFormat := "EXR"
req := &types.CreateJobRequest{
FrameStart: &frameStart,
FrameEnd: &frameEnd,
OutputFormat: &outputFormat,
}
meta, err := mergeBlendMetadataForJobCreate(uploadMeta, req)
if err != nil {
t.Fatalf("mergeBlendMetadataForJobCreate failed: %v", err)
}
if meta.BlenderVersion != "4.5.11" {
t.Fatalf("expected blender_version 4.5.11, got %q", meta.BlenderVersion)
}
if meta.FrameStart != 800 || meta.FrameEnd != 800 {
t.Fatalf("expected frame range 800-800, got %d-%d", meta.FrameStart, meta.FrameEnd)
}
if meta.RenderSettings.ResolutionX != 2560 || meta.RenderSettings.ResolutionY != 1440 {
t.Fatalf("expected resolution 2560x1440, got %dx%d", meta.RenderSettings.ResolutionX, meta.RenderSettings.ResolutionY)
}
if meta.RenderSettings.Engine != "cycles" {
t.Fatalf("expected engine cycles, got %q", meta.RenderSettings.Engine)
}
if meta.RenderSettings.OutputFormat != "EXR" {
t.Fatalf("expected output_format EXR, got %q", meta.RenderSettings.OutputFormat)
}
if meta.SceneInfo.ObjectCount != 42 {
t.Fatalf("expected scene object_count 42, got %d", meta.SceneInfo.ObjectCount)
}
}
func TestMergeBlendMetadataForJobCreate_RequestOverridesVersion(t *testing.T) {
uploadMeta := &types.BlendMetadata{
BlenderVersion: "4.5.11",
}
frameStart := 1
frameEnd := 1
override := "4.2.3"
req := &types.CreateJobRequest{
FrameStart: &frameStart,
FrameEnd: &frameEnd,
BlenderVersion: &override,
}
meta, err := mergeBlendMetadataForJobCreate(uploadMeta, req)
if err != nil {
t.Fatalf("mergeBlendMetadataForJobCreate failed: %v", err)
}
if meta.BlenderVersion != "4.2.3" {
t.Fatalf("expected request override 4.2.3, got %q", meta.BlenderVersion)
}
}
func TestMergeBlendMetadataForJobCreate_RequiresBlenderVersion(t *testing.T) {
frameStart := 1
frameEnd := 1
req := &types.CreateJobRequest{
FrameStart: &frameStart,
FrameEnd: &frameEnd,
}
if _, err := mergeBlendMetadataForJobCreate(nil, req); err == nil {
t.Fatal("expected error when blender_version is missing")
}
}
func TestUploadSessionMetadata(t *testing.T) {
session := &UploadSession{
ResultMetadata: &types.BlendMetadata{
BlenderVersion: "4.5.11",
},
}
meta := uploadSessionMetadata(session)
if meta == nil || meta.BlenderVersion != "4.5.11" {
t.Fatalf("unexpected metadata: %+v", meta)
}
}
+271 -89
View File
@@ -63,21 +63,20 @@ func (s *Manager) runnerAuthMiddleware(next http.HandlerFunc) http.HandlerFunc {
return
}
// For fixed API keys, skip database verification
// Always verify the runner exists. For non-fixed keys, also check API key ownership.
var dbAPIKeyID sql.NullInt64
err = s.db.With(func(conn *sql.DB) error {
return conn.QueryRow("SELECT api_key_id FROM runners WHERE id = ?", runnerID).Scan(&dbAPIKeyID)
})
if err == sql.ErrNoRows {
s.respondError(w, http.StatusNotFound, "runner not found")
return
}
if err != nil {
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("failed to query runner API key: %v", err))
return
}
if apiKeyID != -1 {
// Verify runner exists and uses this API key
var dbAPIKeyID sql.NullInt64
err = s.db.With(func(conn *sql.DB) error {
return conn.QueryRow("SELECT api_key_id FROM runners WHERE id = ?", runnerID).Scan(&dbAPIKeyID)
})
if err == sql.ErrNoRows {
s.respondError(w, http.StatusNotFound, "runner not found")
return
}
if err != nil {
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("failed to query runner API key: %v", err))
return
}
if !dbAPIKeyID.Valid || dbAPIKeyID.Int64 != apiKeyID {
s.respondError(w, http.StatusForbidden, "runner does not belong to this API key")
return
@@ -1000,6 +999,32 @@ func (s *Manager) handleUploadFileWithToken(w http.ResponseWriter, r *http.Reque
})
}
// runnerCanAccessJob returns true if the runner has been assigned at least one task on the job.
// Used to scope file/metadata/status APIs so a valid API key cannot read arbitrary jobs.
func (s *Manager) runnerCanAccessJob(runnerID, jobID int64) bool {
if runnerID <= 0 || jobID <= 0 {
return false
}
var n int
err := s.db.With(func(conn *sql.DB) error {
return conn.QueryRow(
`SELECT COUNT(*) FROM tasks WHERE job_id = ? AND runner_id = ?`,
jobID, runnerID,
).Scan(&n)
})
return err == nil && n > 0
}
// requireRunnerJobAccess enforces that the authenticated runner may access the job.
func (s *Manager) requireRunnerJobAccess(w http.ResponseWriter, r *http.Request, jobID int64) bool {
runnerID, _ := r.Context().Value(runnerIDContextKey).(int64)
if !s.runnerCanAccessJob(runnerID, jobID) {
s.respondError(w, http.StatusForbidden, "runner is not assigned to this job")
return false
}
return true
}
// handleGetJobStatusForRunner allows runners to check job status
func (s *Manager) handleGetJobStatusForRunner(w http.ResponseWriter, r *http.Request) {
jobID, err := parseID(r, "jobId")
@@ -1007,6 +1032,9 @@ func (s *Manager) handleGetJobStatusForRunner(w http.ResponseWriter, r *http.Req
s.respondError(w, http.StatusBadRequest, err.Error())
return
}
if !s.requireRunnerJobAccess(w, r, jobID) {
return
}
var job types.Job
var startedAt, completedAt sql.NullTime
@@ -1069,6 +1097,9 @@ func (s *Manager) handleGetJobFilesForRunner(w http.ResponseWriter, r *http.Requ
s.respondError(w, http.StatusBadRequest, err.Error())
return
}
if !s.requireRunnerJobAccess(w, r, jobID) {
return
}
runnerID := r.URL.Query().Get("runner_id")
log.Printf("GetJobFiles request for job %d from runner %s", jobID, runnerID)
@@ -1127,6 +1158,9 @@ func (s *Manager) handleGetJobMetadataForRunner(w http.ResponseWriter, r *http.R
s.respondError(w, http.StatusBadRequest, err.Error())
return
}
if !s.requireRunnerJobAccess(w, r, jobID) {
return
}
var blendMetadataJSON sql.NullString
err = s.db.With(func(conn *sql.DB) error {
@@ -1166,6 +1200,9 @@ func (s *Manager) handleDownloadFileForRunner(w http.ResponseWriter, r *http.Req
s.respondError(w, http.StatusBadRequest, err.Error())
return
}
if !s.requireRunnerJobAccess(w, r, jobID) {
return
}
// Get fileName from URL path (may need URL decoding)
fileName := chi.URLParam(r, "fileName")
@@ -1259,11 +1296,14 @@ type WSLogEntry struct {
}
type WSTaskUpdate struct {
TaskID int64 `json:"task_id"`
Status string `json:"status"`
OutputPath string `json:"output_path,omitempty"`
Success bool `json:"success"`
Error string `json:"error,omitempty"`
TaskID int64 `json:"task_id"`
Status string `json:"status"`
OutputPath string `json:"output_path,omitempty"`
Success bool `json:"success"`
Error string `json:"error,omitempty"`
// FreeRequeue asks the manager to requeue without incrementing retry_count
// (e.g. this attempt newly armed GPU lockout after a ROCm fault).
FreeRequeue bool `json:"free_requeue,omitempty"`
}
// handleRunnerJobWebSocket handles per-job WebSocket connections from runners
@@ -1362,25 +1402,18 @@ func (s *Manager) handleRunnerJobWebSocket(w http.ResponseWriter, r *http.Reques
delete(s.runnerJobConnsWriteMu, connKey)
s.runnerJobConnsMu.Unlock()
// Check if task is still running - if so, mark as failed
// If the runner never delivered task_complete (crash, bad payload, network drop),
// treat this like a task failure with retries. Prefer the last ERROR log so
// segfaults surface as such instead of a generic WebSocket message.
var currentStatus string
s.db.With(func(conn *sql.DB) error {
return conn.QueryRow("SELECT status FROM tasks WHERE id = ?", taskID).Scan(&currentStatus)
})
if currentStatus == string(types.TaskStatusRunning) {
log.Printf("Job WebSocket disconnected unexpectedly for task %d, marking as failed", taskID)
s.db.With(func(conn *sql.DB) error {
_, err := conn.Exec(
`UPDATE tasks SET status = ?, runner_id = NULL, error_message = ?, completed_at = ? WHERE id = ?`,
types.TaskStatusFailed, "WebSocket connection lost", time.Now(), taskID,
)
return err
})
s.broadcastTaskUpdate(jobID, taskID, "task_update", map[string]interface{}{
"status": types.TaskStatusFailed,
"error_message": "WebSocket connection lost",
})
s.updateJobStatusFromTasks(jobID)
errorMsg := s.inferTaskFailureFromLogs(taskID)
freeRequeue := s.taskLogsIndicateGPULockoutArm(taskID)
log.Printf("Job WebSocket disconnected unexpectedly for task %d: %s (free_requeue=%v)", taskID, errorMsg, freeRequeue)
s.handleTaskFailureWithRetry(runnerID, taskID, jobID, errorMsg, freeRequeue)
}
log.Printf("Job WebSocket closed: job=%d, runner=%d, task=%d", jobID, runnerID, taskID)
@@ -1467,13 +1500,19 @@ func (s *Manager) handleRunnerJobWebSocket(w http.ResponseWriter, r *http.Reques
}
case "task_complete":
var taskUpdate WSTaskUpdate
if err := json.Unmarshal(msg.Data, &taskUpdate); err == nil {
if taskUpdate.TaskID == taskID {
s.handleWebSocketTaskComplete(runnerID, taskUpdate)
// Task is done, close connection
return
}
taskUpdate, err := parseWSTaskUpdate(msg.Data)
if err != nil {
log.Printf("Job WebSocket task_complete for task %d: %v", taskID, err)
// Still close; disconnect handler will infer failure from ERROR logs.
return
}
if taskUpdate.TaskID == 0 {
taskUpdate.TaskID = taskID
}
if taskUpdate.TaskID == taskID {
s.handleWebSocketTaskComplete(runnerID, taskUpdate)
// Task is done, close connection
return
}
case "runner_heartbeat":
s.handleWSRunnerHeartbeat(conn, jobID)
@@ -1576,12 +1615,11 @@ func (s *Manager) handleWebSocketTaskComplete(runnerID int64, taskUpdate WSTaskU
// Verify task belongs to runner and get task info
var taskRunnerID sql.NullInt64
var jobID int64
var retryCount, maxRetries int
err := s.db.With(func(conn *sql.DB) error {
return conn.QueryRow(
"SELECT runner_id, job_id, retry_count, max_retries FROM tasks WHERE id = ?",
"SELECT runner_id, job_id FROM tasks WHERE id = ?",
taskUpdate.TaskID,
).Scan(&taskRunnerID, &jobID, &retryCount, &maxRetries)
).Scan(&taskRunnerID, &jobID)
})
if err != nil {
log.Printf("Failed to get task %d info: %v", taskUpdate.TaskID, err)
@@ -1597,7 +1635,10 @@ func (s *Manager) handleWebSocketTaskComplete(runnerID int64, taskUpdate WSTaskU
// Handle successful completion
if taskUpdate.Success {
err = s.db.WithTx(func(tx *sql.Tx) error {
_, err := tx.Exec(`UPDATE tasks SET status = ? WHERE id = ?`, types.TaskStatusCompleted, taskUpdate.TaskID)
_, err := tx.Exec(
`UPDATE tasks SET status = ?, error_message = NULL, completed_at = ? WHERE id = ?`,
types.TaskStatusCompleted, now, taskUpdate.TaskID,
)
if err != nil {
return err
}
@@ -1607,8 +1648,7 @@ func (s *Manager) handleWebSocketTaskComplete(runnerID int64, taskUpdate WSTaskU
return err
}
}
_, err = tx.Exec(`UPDATE tasks SET completed_at = ? WHERE id = ?`, now, taskUpdate.TaskID)
return err
return nil
})
if err != nil {
log.Printf("Failed to update task %d: %v", taskUpdate.TaskID, err)
@@ -1617,89 +1657,231 @@ func (s *Manager) handleWebSocketTaskComplete(runnerID int64, taskUpdate WSTaskU
// Broadcast task update
s.broadcastTaskUpdate(jobID, taskUpdate.TaskID, "task_update", map[string]interface{}{
"status": types.TaskStatusCompleted,
"output_path": taskUpdate.OutputPath,
"completed_at": now,
"status": types.TaskStatusCompleted,
"output_path": taskUpdate.OutputPath,
"completed_at": now,
"error_message": nil,
})
s.updateJobStatusFromTasks(jobID)
return
}
// Handle task failure - this is an actual task failure (e.g., Blender crash)
// Check if we have retries remaining
if retryCount < maxRetries {
// Reset to pending for retry - increment retry_count
// Actual task failure (e.g. Blender segfault) — retry when retries remain.
// free_requeue: failure newly armed GPU lockout; requeue without burning a retry.
errorMsg := taskUpdate.Error
if errorMsg == "" {
errorMsg = s.inferTaskFailureFromLogs(taskUpdate.TaskID)
}
freeRequeue := taskUpdate.FreeRequeue || s.taskLogsIndicateGPULockoutArm(taskUpdate.TaskID)
s.handleTaskFailureWithRetry(runnerID, taskUpdate.TaskID, jobID, errorMsg, freeRequeue)
}
// defaultUnexpectedDisconnectError is used when the runner drops the job WebSocket
// without a task_complete message and without any ERROR logs.
const defaultUnexpectedDisconnectError = "WebSocket connection lost"
// parseWSTaskUpdate decodes a task_complete payload. Older runners marshaled
// the error field as a Go error interface (JSON object {}), which fails strict
// unmarshaling into a string — fall back to extracting fields loosely so
// success/failure is still recognized.
func parseWSTaskUpdate(data json.RawMessage) (WSTaskUpdate, error) {
var update WSTaskUpdate
if err := json.Unmarshal(data, &update); err == nil {
return update, nil
}
var loose map[string]interface{}
if err := json.Unmarshal(data, &loose); err != nil {
return WSTaskUpdate{}, err
}
if v, ok := loose["task_id"].(float64); ok {
update.TaskID = int64(v)
}
if v, ok := loose["success"].(bool); ok {
update.Success = v
}
if v, ok := loose["output_path"].(string); ok {
update.OutputPath = v
}
if v, ok := loose["status"].(string); ok {
update.Status = v
}
if v, ok := loose["free_requeue"].(bool); ok {
update.FreeRequeue = v
}
switch v := loose["error"].(type) {
case string:
update.Error = v
case map[string]interface{}:
// Legacy runner bug: error interface marshaled as {}
update.Error = ""
}
return update, nil
}
// taskLogsIndicateGPULockoutArm is true when this attempt's logs show the runner
// newly armed GPU lockout (free-requeue path when task_complete lacked free_requeue).
func (s *Manager) taskLogsIndicateGPULockoutArm(taskID int64) bool {
var n int
err := s.db.With(func(conn *sql.DB) error {
return conn.QueryRow(
`SELECT COUNT(*) FROM task_logs
WHERE task_id = ?
AND (
message LIKE '%GPU error detected%'
OR message LIKE '%free-requeue%'
OR message LIKE '%GPU disabled for subsequent jobs%'
)`,
taskID,
).Scan(&n)
})
return err == nil && n > 0
}
// inferTaskFailureFromLogs returns the best failure reason for a task that died
// without a usable task_complete payload. Prefers the latest ERROR log so Blender
// segfaults (and similar crashes) are stored instead of a generic disconnect message.
func (s *Manager) inferTaskFailureFromLogs(taskID int64) string {
var lastError string
err := s.db.With(func(conn *sql.DB) error {
return conn.QueryRow(
`SELECT message FROM task_logs
WHERE task_id = ? AND UPPER(log_level) = ?
ORDER BY id DESC LIMIT 1`,
taskID, string(types.LogLevelError),
).Scan(&lastError)
})
if err != nil || strings.TrimSpace(lastError) == "" {
return defaultUnexpectedDisconnectError
}
return formatInferredTaskFailure(lastError)
}
// formatInferredTaskFailure normalizes streamed runner ERROR lines into a concise
// task error_message. Blender crashes often arrive as "Task failed: blender failed: signal: ..."
// or "Blender render failed: ..."; keep the underlying failure text.
func formatInferredTaskFailure(logMessage string) string {
msg := strings.TrimSpace(logMessage)
for _, prefix := range []string{
"Task failed: ",
"Blender render failed: ",
} {
if strings.HasPrefix(msg, prefix) {
msg = strings.TrimSpace(strings.TrimPrefix(msg, prefix))
}
}
if msg == "" {
return defaultUnexpectedDisconnectError
}
return msg
}
// handleTaskFailureWithRetry applies the task failure policy used for Blender
// crashes and unexpected job-WebSocket drops: requeue as pending while
// retry_count < max_retries, otherwise mark failed permanently with error_message.
// When freeRequeue is true (GPU lockout newly armed this attempt), the task is
// always requeued as pending without incrementing retry_count.
func (s *Manager) handleTaskFailureWithRetry(runnerID, taskID, jobID int64, errorMsg string, freeRequeue bool) {
var retryCount, maxRetries int
err := s.db.With(func(conn *sql.DB) error {
return conn.QueryRow(
"SELECT retry_count, max_retries FROM tasks WHERE id = ?",
taskID,
).Scan(&retryCount, &maxRetries)
})
if err != nil {
log.Printf("Failed to get retry info for task %d: %v", taskID, err)
return
}
if errorMsg == "" {
errorMsg = defaultUnexpectedDisconnectError
}
if freeRequeue {
// Make the reason obvious in the UI without looking like a permanent fail.
if !strings.Contains(strings.ToLower(errorMsg), "gpu lockout") {
errorMsg = "GPU lockout armed (free requeue): " + errorMsg
}
}
now := time.Now()
// Free requeue always goes back to pending (expected ROCm GPU fault that armed lockout).
// Otherwise only requeue while retries remain.
canRequeue := freeRequeue || retryCount < maxRetries
if canRequeue {
newRetryCount := retryCount
retrySQL := `UPDATE tasks SET status = ?, runner_id = NULL, current_step = NULL,
error_message = ?, started_at = NULL, completed_at = NULL
WHERE id = ?`
args := []interface{}{types.TaskStatusPending, errorMsg, taskID}
if !freeRequeue {
retrySQL = `UPDATE tasks SET status = ?, runner_id = NULL, current_step = NULL,
error_message = ?, retry_count = retry_count + 1, started_at = NULL, completed_at = NULL
WHERE id = ?`
newRetryCount = retryCount + 1
}
err = s.db.WithTx(func(tx *sql.Tx) error {
_, err := tx.Exec(
`UPDATE tasks SET status = ?, runner_id = NULL, current_step = NULL,
retry_count = retry_count + 1, started_at = NULL, completed_at = NULL
WHERE id = ?`,
types.TaskStatusPending, taskUpdate.TaskID,
)
_, err := tx.Exec(retrySQL, args...)
if err != nil {
return err
}
// Clear steps and logs for fresh retry
_, err = tx.Exec(`DELETE FROM task_steps WHERE task_id = ?`, taskUpdate.TaskID)
_, err = tx.Exec(`DELETE FROM task_steps WHERE task_id = ?`, taskID)
if err != nil {
return err
}
_, err = tx.Exec(`DELETE FROM task_logs WHERE task_id = ?`, taskUpdate.TaskID)
_, err = tx.Exec(`DELETE FROM task_logs WHERE task_id = ?`, taskID)
return err
})
if err != nil {
log.Printf("Failed to reset task %d for retry: %v", taskUpdate.TaskID, err)
log.Printf("Failed to reset task %d for retry: %v", taskID, err)
return
}
// Broadcast task reset to clients (includes steps_cleared and logs_cleared flags)
s.broadcastTaskUpdate(jobID, taskUpdate.TaskID, "task_reset", map[string]interface{}{
s.broadcastTaskUpdate(jobID, taskID, "task_reset", map[string]interface{}{
"status": types.TaskStatusPending,
"retry_count": retryCount + 1,
"error_message": taskUpdate.Error,
"retry_count": newRetryCount,
"error_message": errorMsg,
"steps_cleared": true,
"logs_cleared": true,
"free_requeue": freeRequeue,
})
log.Printf("Task %d failed but has retries remaining (%d/%d), reset to pending", taskUpdate.TaskID, retryCount+1, maxRetries)
if freeRequeue {
log.Printf("Task %d failed (%s); free-requeued without burning retry (still %d/%d used)",
taskID, errorMsg, retryCount, maxRetries)
} else {
log.Printf("Task %d failed (%s) but has retries remaining (%d/%d), reset to pending",
taskID, errorMsg, newRetryCount, maxRetries)
}
} else {
// No retries remaining - mark as failed
err = s.db.WithTx(func(tx *sql.Tx) error {
_, err := tx.Exec(
`UPDATE tasks SET status = ?, runner_id = NULL, completed_at = ? WHERE id = ?`,
types.TaskStatusFailed, now, taskUpdate.TaskID,
`UPDATE tasks SET status = ?, runner_id = NULL, completed_at = ?, error_message = ? WHERE id = ?`,
types.TaskStatusFailed, now, errorMsg, taskID,
)
if err != nil {
return err
}
if taskUpdate.Error != "" {
_, err = tx.Exec(`UPDATE tasks SET error_message = ? WHERE id = ?`, taskUpdate.Error, taskUpdate.TaskID)
if err != nil {
return err
}
}
return nil
return err
})
if err != nil {
log.Printf("Failed to mark task %d as failed: %v", taskUpdate.TaskID, err)
log.Printf("Failed to mark task %d as failed: %v", taskID, err)
return
}
// Log the final failure
s.logTaskEvent(taskUpdate.TaskID, &runnerID, types.LogLevelError,
fmt.Sprintf("Task failed permanently after %d retries: %s", maxRetries, taskUpdate.Error), "")
s.logTaskEvent(taskID, &runnerID, types.LogLevelError,
fmt.Sprintf("Task failed permanently after %d retries: %s", maxRetries, errorMsg), "")
// Broadcast task update
s.broadcastTaskUpdate(jobID, taskUpdate.TaskID, "task_update", map[string]interface{}{
s.broadcastTaskUpdate(jobID, taskID, "task_update", map[string]interface{}{
"status": types.TaskStatusFailed,
"completed_at": now,
"error_message": taskUpdate.Error,
"error_message": errorMsg,
})
log.Printf("Task %d failed permanently after %d retries", taskUpdate.TaskID, maxRetries)
log.Printf("Task %d failed permanently after %d retries: %s", taskID, maxRetries, errorMsg)
}
// Update job status and progress
s.updateJobStatusFromTasks(jobID)
}
+80 -1
View File
@@ -1,6 +1,9 @@
package api
import "testing"
import (
"encoding/json"
"testing"
)
func TestParseBlenderFrame(t *testing.T) {
frame, ok := parseBlenderFrame("Info Fra:2470 Mem:12.00M")
@@ -19,3 +22,79 @@ func TestJobTaskCounts_Progress(t *testing.T) {
}
}
func TestParseWSTaskUpdate_LegacyErrorObject(t *testing.T) {
// Simulates older runners that JSON-marshaled an error interface as {}
raw := json.RawMessage(`{"task_id":99,"success":false,"error":{}}`)
update, err := parseWSTaskUpdate(raw)
if err != nil {
t.Fatalf("parseWSTaskUpdate: %v", err)
}
if update.TaskID != 99 {
t.Fatalf("TaskID = %d, want 99", update.TaskID)
}
if update.Success {
t.Fatal("Success = true, want false")
}
if update.Error != "" {
t.Fatalf("Error = %q, want empty (legacy object)", update.Error)
}
}
func TestParseWSTaskUpdate_StringError(t *testing.T) {
raw := json.RawMessage(`{"task_id":7,"success":false,"error":"blender failed: signal: segmentation fault (core dumped)","free_requeue":true}`)
update, err := parseWSTaskUpdate(raw)
if err != nil {
t.Fatalf("parseWSTaskUpdate: %v", err)
}
if update.TaskID != 7 || update.Success {
t.Fatalf("unexpected update: %+v", update)
}
if update.Error != "blender failed: signal: segmentation fault (core dumped)" {
t.Fatalf("Error = %q", update.Error)
}
if !update.FreeRequeue {
t.Fatal("expected FreeRequeue true")
}
}
func TestFormatInferredTaskFailure(t *testing.T) {
tests := []struct {
name string
in string
want string
}{
{
name: "task failed segfault prefix",
in: "Task failed: blender failed: signal: segmentation fault (core dumped)",
want: "blender failed: signal: segmentation fault (core dumped)",
},
{
name: "blender render failed prefix",
in: "Blender render failed: blender failed: signal: segmentation fault (core dumped)",
want: "blender failed: signal: segmentation fault (core dumped)",
},
{
name: "double prefix",
in: "Task failed: Blender render failed: blender failed: signal: segmentation fault (core dumped)",
want: "blender failed: signal: segmentation fault (core dumped)",
},
{
name: "plain message",
in: "blender failed: exit status 1",
want: "blender failed: exit status 1",
},
{
name: "empty falls back",
in: " ",
want: defaultUnexpectedDisconnectError,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := formatInferredTaskFailure(tt.in); got != tt.want {
t.Fatalf("formatInferredTaskFailure(%q) = %q, want %q", tt.in, got, tt.want)
}
})
}
}