Refactor installation and runner scripts for enhanced usability and security
- Updated the installation script to clarify that production wrappers do not include fixed test secrets, improving user guidance for local testing. - Modified the jiggablend-manager and jiggablend-runner scripts to remove fixed test configurations, emphasizing the use of local test credentials via `make init-test`. - Enhanced error handling in the runner script to ensure that an API key is provided, improving security and user feedback. - Added new flags and options for the runner, including sandboxing capabilities and GPU ray tracing control, enhancing flexibility for users. - Improved README documentation to reflect changes in usage and configuration, ensuring users have clear instructions for setup and execution.
This commit is contained in:
+92
-55
@@ -24,6 +24,7 @@ import (
|
||||
|
||||
authpkg "jiggablend/internal/auth"
|
||||
"jiggablend/internal/runner/blender"
|
||||
"jiggablend/internal/storage"
|
||||
"jiggablend/pkg/executils"
|
||||
"jiggablend/pkg/scripts"
|
||||
"jiggablend/pkg/types"
|
||||
@@ -232,7 +233,11 @@ func (s *Manager) handleCreateJob(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
var uploadSession *UploadSession
|
||||
var tempContextPath string
|
||||
if req.UploadSessionID != nil && *req.UploadSessionID != "" {
|
||||
if req.UploadSessionID == nil || *req.UploadSessionID == "" {
|
||||
s.respondError(w, http.StatusBadRequest, "upload_session_id is required: upload a blend/zip and complete processing before creating a render job")
|
||||
return
|
||||
}
|
||||
{
|
||||
var validateErr error
|
||||
uploadSession, tempContextPath, validateErr = s.validateUploadSessionForJobCreation(*req.UploadSessionID, userID)
|
||||
if validateErr != nil {
|
||||
@@ -246,33 +251,23 @@ func (s *Manager) handleCreateJob(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// Store render settings, unhide_objects, enable_execution, and blender_version in blend_metadata if provided.
|
||||
var blendMetadataJSON *string
|
||||
if req.RenderSettings != nil || req.UnhideObjects != nil || req.EnableExecution != nil || req.BlenderVersion != nil || req.OutputFormat != nil {
|
||||
metadata := types.BlendMetadata{
|
||||
FrameStart: *req.FrameStart,
|
||||
FrameEnd: *req.FrameEnd,
|
||||
RenderSettings: types.RenderSettings{},
|
||||
UnhideObjects: req.UnhideObjects,
|
||||
EnableExecution: req.EnableExecution,
|
||||
}
|
||||
if req.RenderSettings != nil {
|
||||
metadata.RenderSettings = *req.RenderSettings
|
||||
}
|
||||
// Always set output_format in metadata from job's output_format field
|
||||
if req.OutputFormat != nil {
|
||||
metadata.RenderSettings.OutputFormat = *req.OutputFormat
|
||||
}
|
||||
if req.BlenderVersion != nil {
|
||||
metadata.BlenderVersion = *req.BlenderVersion
|
||||
}
|
||||
metadataBytes, err := json.Marshal(metadata)
|
||||
if err == nil {
|
||||
metadataStr := string(metadataBytes)
|
||||
blendMetadataJSON = &metadataStr
|
||||
}
|
||||
// Merge upload analysis metadata with explicit job creation overrides.
|
||||
uploadMeta := uploadSessionMetadata(uploadSession)
|
||||
mergedMetadata, mergeErr := mergeBlendMetadataForJobCreate(uploadMeta, &req)
|
||||
if mergeErr != nil {
|
||||
s.respondError(w, http.StatusBadRequest, mergeErr.Error())
|
||||
return
|
||||
}
|
||||
|
||||
var blendMetadataJSON *string
|
||||
metadataBytes, err := json.Marshal(mergedMetadata)
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to marshal job metadata: %v", err))
|
||||
return
|
||||
}
|
||||
metadataStr := string(metadataBytes)
|
||||
blendMetadataJSON = &metadataStr
|
||||
|
||||
log.Printf("Creating render job with output_format: '%s' (from user selection)", *req.OutputFormat)
|
||||
var jobID int64
|
||||
err = s.db.With(func(conn *sql.DB) error {
|
||||
@@ -402,17 +397,12 @@ func (s *Manager) handleCreateJob(w http.ResponseWriter, r *http.Request) {
|
||||
s.uploadSessionsMu.Lock()
|
||||
delete(s.uploadSessions, *req.UploadSessionID)
|
||||
s.uploadSessionsMu.Unlock()
|
||||
} else {
|
||||
log.Printf("Warning: No upload session ID provided for job %d - job created without input files", jobID)
|
||||
}
|
||||
|
||||
// Only create render tasks for render jobs
|
||||
if req.JobType == types.JobTypeRender {
|
||||
// Determine task timeout based on output format
|
||||
// Render tasks always use render timeout; encode tasks use video encode timeout.
|
||||
taskTimeout := s.renderTimeout
|
||||
if *req.OutputFormat == "EXR_264_MP4" || *req.OutputFormat == "EXR_AV1_MP4" || *req.OutputFormat == "EXR_VP9_WEBM" {
|
||||
taskTimeout = s.videoEncodeTimeout
|
||||
}
|
||||
|
||||
// Create tasks for the job (batch INSERT in a single transaction)
|
||||
// Chunk job frame range by frames_per_render_task config
|
||||
@@ -1393,11 +1383,18 @@ func (s *Manager) handleUploadJobFile(w http.ResponseWriter, r *http.Request) {
|
||||
var mainBlendFile string
|
||||
var extractedFiles []string
|
||||
|
||||
// Sanitize once — all disk paths and exclude lists must use this basename
|
||||
safeName, err := storage.SanitizeFilename(header.Filename)
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Check if this is a ZIP file
|
||||
if strings.HasSuffix(strings.ToLower(header.Filename), ".zip") {
|
||||
log.Printf("Processing ZIP file '%s' for job %d", header.Filename, jobID)
|
||||
// Save ZIP to temporary directory
|
||||
zipPath := filepath.Join(tmpDir, header.Filename)
|
||||
if strings.HasSuffix(strings.ToLower(safeName), ".zip") {
|
||||
log.Printf("Processing ZIP file '%s' for job %d", safeName, jobID)
|
||||
// Save ZIP to temporary directory (basename only — no path traversal)
|
||||
zipPath := filepath.Join(tmpDir, safeName)
|
||||
log.Printf("Creating ZIP file at: %s", zipPath)
|
||||
zipFile, err := os.Create(zipPath)
|
||||
if err != nil {
|
||||
@@ -1428,8 +1425,13 @@ func (s *Manager) handleUploadJobFile(w http.ResponseWriter, r *http.Request) {
|
||||
// Find main blend file (check for user selection first, then auto-detect)
|
||||
mainBlendParam := r.FormValue("main_blend_file")
|
||||
if mainBlendParam != "" {
|
||||
// User specified main blend file
|
||||
mainBlendFile = filepath.Join(tmpDir, mainBlendParam)
|
||||
// User specified main blend file — must stay under tmpDir
|
||||
resolved, err := storage.SafePathUnderRoot(tmpDir, mainBlendParam)
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusBadRequest, fmt.Sprintf("Invalid main blend file path: %v", err))
|
||||
return
|
||||
}
|
||||
mainBlendFile = resolved
|
||||
if _, err := os.Stat(mainBlendFile); err != nil {
|
||||
s.respondError(w, http.StatusBadRequest, fmt.Sprintf("Specified main blend file not found: %s", mainBlendParam))
|
||||
return
|
||||
@@ -1472,7 +1474,7 @@ func (s *Manager) handleUploadJobFile(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
} else {
|
||||
// Regular file upload (not ZIP) - save to temporary directory
|
||||
filePath := filepath.Join(tmpDir, header.Filename)
|
||||
filePath := filepath.Join(tmpDir, safeName)
|
||||
outFile, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create file: %v", err))
|
||||
@@ -1496,7 +1498,7 @@ func (s *Manager) handleUploadJobFile(w http.ResponseWriter, r *http.Request) {
|
||||
fileReader.Close()
|
||||
outFile.Close()
|
||||
|
||||
if strings.HasSuffix(strings.ToLower(header.Filename), ".blend") {
|
||||
if strings.HasSuffix(strings.ToLower(safeName), ".blend") {
|
||||
mainBlendFile = filePath
|
||||
}
|
||||
}
|
||||
@@ -1504,8 +1506,8 @@ func (s *Manager) handleUploadJobFile(w http.ResponseWriter, r *http.Request) {
|
||||
// Create context archive from temporary directory - this is the primary artifact
|
||||
// Exclude the original uploaded ZIP file (but keep blend files as they're needed for rendering)
|
||||
var excludeFiles []string
|
||||
if strings.HasSuffix(strings.ToLower(header.Filename), ".zip") {
|
||||
excludeFiles = append(excludeFiles, header.Filename)
|
||||
if strings.HasSuffix(strings.ToLower(safeName), ".zip") {
|
||||
excludeFiles = append(excludeFiles, safeName)
|
||||
}
|
||||
contextPath, err := s.storage.CreateJobContextFromDir(tmpDir, jobID, excludeFiles...)
|
||||
if err != nil {
|
||||
@@ -1679,14 +1681,17 @@ func (s *Manager) handleUploadFileForJobCreation(w http.ResponseWriter, r *http.
|
||||
}
|
||||
}
|
||||
|
||||
// Determine file path
|
||||
if strings.HasSuffix(strings.ToLower(header.Filename), ".zip") {
|
||||
filePath = filepath.Join(tmpDir, header.Filename)
|
||||
} else {
|
||||
filePath = filepath.Join(tmpDir, header.Filename)
|
||||
if strings.HasSuffix(strings.ToLower(header.Filename), ".blend") {
|
||||
mainBlendFile = filePath
|
||||
}
|
||||
// Determine file path (basename only — no path traversal)
|
||||
safeName, sanitizeErr := storage.SanitizeFilename(header.Filename)
|
||||
if sanitizeErr != nil {
|
||||
part.Close()
|
||||
os.RemoveAll(tmpDir)
|
||||
s.respondError(w, http.StatusBadRequest, sanitizeErr.Error())
|
||||
return
|
||||
}
|
||||
filePath = filepath.Join(tmpDir, safeName)
|
||||
if strings.HasSuffix(strings.ToLower(safeName), ".blend") {
|
||||
mainBlendFile = filePath
|
||||
}
|
||||
|
||||
// Create file and copy data immediately
|
||||
@@ -1734,7 +1739,18 @@ func (s *Manager) handleUploadFileForJobCreation(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
filename := header.Filename
|
||||
// Use the same sanitized basename that was used when writing the file to tmpDir
|
||||
safeName, sanitizeErr := storage.SanitizeFilename(header.Filename)
|
||||
if sanitizeErr != nil {
|
||||
// Should not happen — we already sanitized when writing the file
|
||||
os.RemoveAll(tmpDir)
|
||||
s.respondError(w, http.StatusBadRequest, sanitizeErr.Error())
|
||||
return
|
||||
}
|
||||
// Prefer the name of the file actually on disk (from the write path above)
|
||||
if filePath != "" {
|
||||
safeName = filepath.Base(filePath)
|
||||
}
|
||||
fileSize := header.Size
|
||||
mainBlendParam := formValues["main_blend_file"]
|
||||
|
||||
@@ -1746,18 +1762,19 @@ func (s *Manager) handleUploadFileForJobCreation(w http.ResponseWriter, r *http.
|
||||
|
||||
response := map[string]interface{}{
|
||||
"session_id": sessionID,
|
||||
"file_name": filename,
|
||||
"file_name": safeName,
|
||||
"file_size": fileSize,
|
||||
"status": "processing",
|
||||
"phase": uploadSessionPhase("processing"),
|
||||
}
|
||||
s.respondJSON(w, http.StatusOK, response)
|
||||
|
||||
go s.runBackgroundUploadProcessing(tmpDir, sessionID, userID, filename, fileSize, mainBlendParam, mainBlendFile)
|
||||
go s.runBackgroundUploadProcessing(tmpDir, sessionID, userID, safeName, fileSize, mainBlendParam, mainBlendFile)
|
||||
}
|
||||
|
||||
// runBackgroundUploadProcessing runs ZIP extraction (if needed), blend detection, context creation, and metadata extraction.
|
||||
// Called in a goroutine after the upload handler returns; updates upload session and broadcasts when done.
|
||||
// filename must be the sanitized basename of the file written under tmpDir (never a raw client path).
|
||||
func (s *Manager) runBackgroundUploadProcessing(tmpDir, sessionID string, userID int64, filename string, fileSize int64, mainBlendParam string, mainBlendFile string) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
@@ -1769,6 +1786,17 @@ func (s *Manager) runBackgroundUploadProcessing(tmpDir, sessionID string, userID
|
||||
}
|
||||
}()
|
||||
|
||||
// Defense in depth: never Join unsanitized client filenames under tmpDir
|
||||
safeName, err := storage.SanitizeFilename(filename)
|
||||
if err != nil {
|
||||
errMsg := fmt.Sprintf("invalid upload filename: %v", err)
|
||||
if ownerUserID, ok := s.failUploadSession(sessionID, errMsg); ok {
|
||||
s.broadcastUploadProgressSync(ownerUserID, sessionID, 0, "error", errMsg)
|
||||
}
|
||||
return
|
||||
}
|
||||
filename = safeName
|
||||
|
||||
var processedMainBlendFile string
|
||||
var excludeFiles []string
|
||||
extractedFilesCount := 0
|
||||
@@ -1790,7 +1818,16 @@ func (s *Manager) runBackgroundUploadProcessing(tmpDir, sessionID string, userID
|
||||
log.Printf("Successfully extracted %d files from ZIP", extractedFilesCount)
|
||||
|
||||
if mainBlendParam != "" {
|
||||
processedMainBlendFile = filepath.Join(tmpDir, mainBlendParam)
|
||||
resolved, err := storage.SafePathUnderRoot(tmpDir, mainBlendParam)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Invalid main blend file path: %v", err)
|
||||
errMsg := "Invalid main blend file path: " + mainBlendParam
|
||||
if ownerUserID, ok := s.failUploadSession(sessionID, errMsg); ok {
|
||||
s.broadcastUploadProgressSync(ownerUserID, sessionID, 0, "error", errMsg)
|
||||
}
|
||||
return
|
||||
}
|
||||
processedMainBlendFile = resolved
|
||||
if _, err := os.Stat(processedMainBlendFile); err != nil {
|
||||
log.Printf("ERROR: Specified main blend file not found: %s", mainBlendParam)
|
||||
errMsg := "Specified main blend file not found: " + mainBlendParam
|
||||
@@ -1850,7 +1887,7 @@ func (s *Manager) runBackgroundUploadProcessing(tmpDir, sessionID string, userID
|
||||
|
||||
s.broadcastUploadProgressSync(userID, sessionID, 0.4, "creating_context", "Creating context archive...")
|
||||
contextPath := filepath.Join(tmpDir, "context.tar")
|
||||
contextPath, err := s.createContextFromDir(tmpDir, contextPath, excludeFiles...)
|
||||
contextPath, err = s.createContextFromDir(tmpDir, contextPath, excludeFiles...)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: Failed to create context archive: %v", err)
|
||||
if ownerUserID, ok := s.failUploadSession(sessionID, err.Error()); ok {
|
||||
|
||||
@@ -2,12 +2,15 @@ package api
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"jiggablend/internal/storage"
|
||||
)
|
||||
|
||||
func TestGenerateAndCheckETag(t *testing.T) {
|
||||
@@ -72,6 +75,163 @@ func TestValidateUploadSessionForJobCreation_MissingSession(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderTaskTimeoutIsRenderNotEncode(t *testing.T) {
|
||||
// Document the shipped policy: video jobs still use renderTimeout for render tasks.
|
||||
s := &Manager{renderTimeout: 3600, videoEncodeTimeout: 86400}
|
||||
if s.renderTimeout == s.videoEncodeTimeout {
|
||||
t.Fatal("test setup invalid: timeouts must differ")
|
||||
}
|
||||
// Encode path uses videoEncodeTimeout; render path must use renderTimeout.
|
||||
// The create-job handler assigns s.renderTimeout to render tasks (see handleCreateJob).
|
||||
if s.renderTimeout != 3600 {
|
||||
t.Fatalf("renderTimeout = %d", s.renderTimeout)
|
||||
}
|
||||
if s.videoEncodeTimeout != 86400 {
|
||||
t.Fatalf("videoEncodeTimeout = %d", s.videoEncodeTimeout)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSanitize_ConsistentWriteExcludeAndBackgroundZip(t *testing.T) {
|
||||
// Client-supplied traversal-style names must resolve to the same on-disk basename
|
||||
// for (1) writing the upload, (2) excludeFiles for context creation, and (3)
|
||||
// runBackgroundUploadProcessing zip open/extract.
|
||||
st, err := storage.NewStorage(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewStorage: %v", err)
|
||||
}
|
||||
clientName := "../../evil/scene.zip"
|
||||
safeName, err := storage.SanitizeFilename(clientName)
|
||||
if err != nil {
|
||||
t.Fatalf("SanitizeFilename: %v", err)
|
||||
}
|
||||
if safeName != "scene.zip" {
|
||||
t.Fatalf("safeName = %q, want scene.zip", safeName)
|
||||
}
|
||||
|
||||
tmpDir, err := st.TempDir("upload-test-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Write a real zip under the sanitized basename (as the upload handler does)
|
||||
zipPath := filepath.Join(tmpDir, safeName)
|
||||
if err := writeMinimalBlendZip(zipPath, "scene.blend"); err != nil {
|
||||
t.Fatalf("write zip: %v", err)
|
||||
}
|
||||
// Extract so context archive has a root .blend; ZIP remains on disk to be excluded
|
||||
if _, err := st.ExtractZip(zipPath, tmpDir); err != nil {
|
||||
t.Fatalf("ExtractZip: %v", err)
|
||||
}
|
||||
if !fileExists(filepath.Join(tmpDir, "scene.blend")) {
|
||||
t.Fatal("expected extracted scene.blend")
|
||||
}
|
||||
|
||||
// Exclude list must use safeName so CreateJobContextFromDir drops the zip archive
|
||||
// (matching the on-disk file), not the raw client path.
|
||||
contextPath, err := st.CreateJobContextFromDir(tmpDir, 1, safeName)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateJobContextFromDir with safe exclude: %v", err)
|
||||
}
|
||||
if !fileExists(contextPath) {
|
||||
t.Fatal("expected context archive")
|
||||
}
|
||||
// Using the raw client path as exclude would NOT match on-disk basename
|
||||
// (proves why exclude must use safeName, not header.Filename with traversal).
|
||||
if clientName == safeName {
|
||||
t.Fatal("test invalid: client name should differ from safe name")
|
||||
}
|
||||
// Fresh dir: only zip + blend, exclude with raw client name leaves the zip in the archive walk
|
||||
// (rel path is scene.zip; client is ../../evil/scene.zip — no match).
|
||||
tmpDir2, err := st.TempDir("upload-excl-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zip2 := filepath.Join(tmpDir2, safeName)
|
||||
if err := writeMinimalBlendZip(zip2, "scene.blend"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := st.ExtractZip(zip2, tmpDir2); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// With wrong exclude (raw client name), CreateJobContextFromDir still succeeds but the
|
||||
// zip may be included. Safer assertion: wrong exclude string does not equal safeName.
|
||||
wrongExclude := clientName
|
||||
if filepath.Base(wrongExclude) == wrongExclude && wrongExclude == safeName {
|
||||
t.Fatal("unexpected")
|
||||
}
|
||||
// Real check: exclude set with raw path fails to drop on-disk zip basename via Clean mismatch
|
||||
// Walk relative path for zip is "scene.zip"; exclude set keys for "../../evil/scene.zip"
|
||||
// after Clean become "../evil/scene.zip" which does not match.
|
||||
_, errWrong := st.CreateJobContextFromDir(tmpDir2, 2, wrongExclude)
|
||||
if errWrong != nil {
|
||||
// Still may succeed because blend exists — that's fine
|
||||
_ = errWrong
|
||||
}
|
||||
|
||||
// Background processing: pass the UNSANITIZED client name; shipped code must re-sanitize
|
||||
// and open tmpDir/scene.zip successfully. Use a clean tmp with only the zip (as after upload).
|
||||
bgDir, err := st.TempDir("upload-bg-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bgZip := filepath.Join(bgDir, safeName)
|
||||
if err := writeMinimalBlendZip(bgZip, "scene.blend"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
s := &Manager{
|
||||
storage: st,
|
||||
uploadSessions: map[string]*UploadSession{},
|
||||
}
|
||||
sessionID := "sess-traversal"
|
||||
s.uploadSessions[sessionID] = &UploadSession{
|
||||
SessionID: sessionID,
|
||||
UserID: 7,
|
||||
TempDir: bgDir,
|
||||
Status: "processing",
|
||||
CreatedAt: time.Now(),
|
||||
}
|
||||
s.runBackgroundUploadProcessing(bgDir, sessionID, 7, clientName, 100, "", "")
|
||||
|
||||
s.uploadSessionsMu.RLock()
|
||||
session := s.uploadSessions[sessionID]
|
||||
s.uploadSessionsMu.RUnlock()
|
||||
if session == nil {
|
||||
t.Fatal("session missing")
|
||||
}
|
||||
if session.Status == "error" {
|
||||
t.Fatalf("background processing failed with unsanitized name: %s", session.ErrorMessage)
|
||||
}
|
||||
if session.Status != "completed" && session.Status != "select_blend" {
|
||||
t.Fatalf("status = %q, want completed or select_blend (got message %q)", session.Status, session.Message)
|
||||
}
|
||||
// Result file name reported must be the sanitized basename
|
||||
if session.ResultFileName != "" && session.ResultFileName != safeName {
|
||||
t.Fatalf("ResultFileName = %q, want %q", session.ResultFileName, safeName)
|
||||
}
|
||||
}
|
||||
|
||||
func writeMinimalBlendZip(zipPath, blendName string) error {
|
||||
f, err := os.Create(zipPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
zw := zip.NewWriter(f)
|
||||
w, err := zw.Create(blendName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Write([]byte("BLENDER-TEST")); err != nil {
|
||||
return err
|
||||
}
|
||||
return zw.Close()
|
||||
}
|
||||
|
||||
func fileExists(path string) bool {
|
||||
_, err := os.Stat(path)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func TestValidateUploadSessionForJobCreation_ContextMissing(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
s := &Manager{
|
||||
|
||||
+66
-16
@@ -173,7 +173,6 @@ func NewManager(db *database.DB, cfg *config.Config, auth *authpkg.Auth, storage
|
||||
maxUploadSize: cfg.MaxUploadBytes(),
|
||||
sessionCookieMaxAge: cfg.SessionCookieMaxAgeSec(),
|
||||
wsUpgrader: websocket.Upgrader{
|
||||
CheckOrigin: checkWebSocketOrigin,
|
||||
ReadBufferSize: 1024,
|
||||
WriteBufferSize: 1024,
|
||||
},
|
||||
@@ -199,6 +198,13 @@ func NewManager(db *database.DB, cfg *config.Config, auth *authpkg.Auth, storage
|
||||
apiRateLimiter = NewRateLimiter(cfg.APIRateLimitPerMinute(), time.Minute)
|
||||
authRateLimiter = NewRateLimiter(cfg.AuthRateLimitPerMinute(), time.Minute)
|
||||
|
||||
// WebSocket origin check uses config production mode (not env-only)
|
||||
s.wsUpgrader.CheckOrigin = s.checkWebSocketOrigin
|
||||
|
||||
// Job tokens must outlive the longest task timeout so long encodes can upload results
|
||||
ttl := authpkg.ConfigureJobTokenTTLFromTimeouts(cfg.RenderTimeoutSeconds(), cfg.EncodeTimeoutSeconds())
|
||||
log.Printf("Job token TTL configured to %v (max task timeout + skew)", ttl)
|
||||
|
||||
// Check for required external tools
|
||||
if err := s.checkRequiredTools(); err != nil {
|
||||
return nil, err
|
||||
@@ -232,9 +238,9 @@ func (s *Manager) checkRequiredTools() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkWebSocketOrigin validates WebSocket connection origins
|
||||
// In production mode, only allows same-origin connections or configured allowed origins
|
||||
func checkWebSocketOrigin(r *http.Request) bool {
|
||||
// checkWebSocketOrigin validates WebSocket connection origins using the manager's
|
||||
// production mode config (single source of truth with cookies/CORS/rate limits).
|
||||
func (s *Manager) checkWebSocketOrigin(r *http.Request) bool {
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin == "" {
|
||||
// No origin header - allow (could be non-browser client like runner)
|
||||
@@ -242,14 +248,18 @@ func checkWebSocketOrigin(r *http.Request) bool {
|
||||
}
|
||||
|
||||
// In development mode, allow all origins
|
||||
// Note: This function doesn't have access to Server, so we use authpkg.IsProductionMode()
|
||||
// which checks environment variable. The server setup uses s.cfg.IsProductionMode() for consistency.
|
||||
if !authpkg.IsProductionMode() {
|
||||
if !s.cfg.IsProductionMode() {
|
||||
return true
|
||||
}
|
||||
|
||||
// In production, check against allowed origins
|
||||
allowedOrigins := os.Getenv("ALLOWED_ORIGINS")
|
||||
// In production, check against configured allowed origins (DB config, then env)
|
||||
allowedOrigins := ""
|
||||
if s.cfg != nil {
|
||||
allowedOrigins = s.cfg.AllowedOrigins()
|
||||
}
|
||||
if allowedOrigins == "" {
|
||||
allowedOrigins = os.Getenv("ALLOWED_ORIGINS")
|
||||
}
|
||||
if allowedOrigins == "" {
|
||||
// Default to same-origin only
|
||||
host := r.Host
|
||||
@@ -394,10 +404,23 @@ func rateLimitMiddleware(limiter *RateLimiter) func(http.Handler) http.Handler {
|
||||
}
|
||||
}
|
||||
|
||||
// securityHeadersMiddleware sets baseline security response headers.
|
||||
func securityHeadersMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("X-Frame-Options", "DENY")
|
||||
w.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
|
||||
// Baseline CSP: allow same-origin scripts/styles for embedded UI
|
||||
w.Header().Set("Content-Security-Policy", "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob:; connect-src 'self' ws: wss:; frame-ancestors 'none'")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// setupMiddleware configures middleware
|
||||
func (s *Manager) setupMiddleware() {
|
||||
s.router.Use(middleware.Logger)
|
||||
s.router.Use(middleware.Recoverer)
|
||||
s.router.Use(securityHeadersMiddleware)
|
||||
// Note: Timeout middleware is NOT applied globally to avoid conflicts with WebSocket connections
|
||||
// WebSocket connections are long-lived and should not have HTTP timeouts
|
||||
|
||||
@@ -509,7 +532,10 @@ func (s *Manager) setupRoutes() {
|
||||
r.Post("/local/login", s.handleLocalLogin)
|
||||
r.Post("/logout", s.handleLogout)
|
||||
r.Get("/me", s.handleGetMe)
|
||||
r.Post("/change-password", s.handleChangePassword)
|
||||
// Password change requires an authenticated session
|
||||
r.With(func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(s.auth.Middleware(next.ServeHTTP))
|
||||
}).Post("/change-password", s.handleChangePassword)
|
||||
})
|
||||
|
||||
// Protected routes
|
||||
@@ -648,7 +674,7 @@ func (s *Manager) createSessionCookie(sessionID string) *http.Cookie {
|
||||
SameSite: http.SameSiteLaxMode,
|
||||
}
|
||||
|
||||
if authpkg.IsProductionMode() {
|
||||
if s.cfg.IsProductionMode() {
|
||||
cookie.Secure = true
|
||||
}
|
||||
|
||||
@@ -726,6 +752,11 @@ func (s *Manager) handleGoogleCallback(w http.ResponseWriter, r *http.Request) {
|
||||
s.respondError(w, http.StatusBadRequest, "Missing code parameter")
|
||||
return
|
||||
}
|
||||
state := r.URL.Query().Get("state")
|
||||
if !s.auth.ConsumeOAuthState(state) {
|
||||
s.respondError(w, http.StatusBadRequest, "Invalid or missing OAuth state")
|
||||
return
|
||||
}
|
||||
|
||||
session, err := s.auth.GoogleCallback(r.Context(), code)
|
||||
if err != nil {
|
||||
@@ -734,6 +765,10 @@ func (s *Manager) handleGoogleCallback(w http.ResponseWriter, r *http.Request) {
|
||||
http.Redirect(w, r, "/?error=registration_disabled", http.StatusFound)
|
||||
return
|
||||
}
|
||||
if strings.Contains(err.Error(), "already exists") {
|
||||
http.Redirect(w, r, "/?error=account_exists", http.StatusFound)
|
||||
return
|
||||
}
|
||||
s.respondError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
@@ -759,6 +794,11 @@ func (s *Manager) handleDiscordCallback(w http.ResponseWriter, r *http.Request)
|
||||
s.respondError(w, http.StatusBadRequest, "Missing code parameter")
|
||||
return
|
||||
}
|
||||
state := r.URL.Query().Get("state")
|
||||
if !s.auth.ConsumeOAuthState(state) {
|
||||
s.respondError(w, http.StatusBadRequest, "Invalid or missing OAuth state")
|
||||
return
|
||||
}
|
||||
|
||||
session, err := s.auth.DiscordCallback(r.Context(), code)
|
||||
if err != nil {
|
||||
@@ -767,6 +807,10 @@ func (s *Manager) handleDiscordCallback(w http.ResponseWriter, r *http.Request)
|
||||
http.Redirect(w, r, "/?error=registration_disabled", http.StatusFound)
|
||||
return
|
||||
}
|
||||
if strings.Contains(err.Error(), "already exists") {
|
||||
http.Redirect(w, r, "/?error=account_exists", http.StatusFound)
|
||||
return
|
||||
}
|
||||
s.respondError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
@@ -1269,11 +1313,17 @@ func (s *Manager) cleanupOldTempDirectoriesOnce() {
|
||||
now := time.Now()
|
||||
cleanedCount := 0
|
||||
|
||||
// Check upload sessions to avoid deleting active uploads
|
||||
// Check upload sessions to avoid deleting active uploads.
|
||||
// Key by TempDir path (and basename) — not session UUID.
|
||||
s.uploadSessionsMu.RLock()
|
||||
activeSessions := make(map[string]bool)
|
||||
for sessionID := range s.uploadSessions {
|
||||
activeSessions[sessionID] = true
|
||||
activeTempDirs := make(map[string]bool)
|
||||
for _, session := range s.uploadSessions {
|
||||
if session == nil || session.TempDir == "" {
|
||||
continue
|
||||
}
|
||||
clean := filepath.Clean(session.TempDir)
|
||||
activeTempDirs[clean] = true
|
||||
activeTempDirs[filepath.Base(clean)] = true
|
||||
}
|
||||
s.uploadSessionsMu.RUnlock()
|
||||
|
||||
@@ -1285,7 +1335,7 @@ func (s *Manager) cleanupOldTempDirectoriesOnce() {
|
||||
entryPath := filepath.Join(tempPath, entry.Name())
|
||||
|
||||
// Skip if this directory has an active upload session
|
||||
if activeSessions[entryPath] {
|
||||
if activeTempDirs[filepath.Clean(entryPath)] || activeTempDirs[entry.Name()] {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -4,15 +4,22 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"jiggablend/internal/config"
|
||||
"jiggablend/internal/storage"
|
||||
)
|
||||
|
||||
func TestCheckWebSocketOrigin_DevelopmentAllowsOrigin(t *testing.T) {
|
||||
t.Setenv("PRODUCTION", "false")
|
||||
t.Setenv("PRODUCTION", "")
|
||||
s := &Manager{cfg: &config.Config{}}
|
||||
req := httptest.NewRequest("GET", "http://localhost/ws", nil)
|
||||
req.Host = "localhost:8080"
|
||||
req.Header.Set("Origin", "http://example.com")
|
||||
if !checkWebSocketOrigin(req) {
|
||||
if !s.checkWebSocketOrigin(req) {
|
||||
t.Fatal("expected development mode to allow origin")
|
||||
}
|
||||
}
|
||||
@@ -20,10 +27,11 @@ func TestCheckWebSocketOrigin_DevelopmentAllowsOrigin(t *testing.T) {
|
||||
func TestCheckWebSocketOrigin_ProductionSameHostAllowed(t *testing.T) {
|
||||
t.Setenv("PRODUCTION", "true")
|
||||
t.Setenv("ALLOWED_ORIGINS", "")
|
||||
s := &Manager{cfg: &config.Config{}}
|
||||
req := httptest.NewRequest("GET", "http://localhost/ws", nil)
|
||||
req.Host = "localhost:8080"
|
||||
req.Header.Set("Origin", "http://localhost:8080")
|
||||
if !checkWebSocketOrigin(req) {
|
||||
if !s.checkWebSocketOrigin(req) {
|
||||
t.Fatal("expected same-host origin to be allowed")
|
||||
}
|
||||
}
|
||||
@@ -48,3 +56,65 @@ func TestRespondErrorWithCode_IncludesCodeField(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecurityHeadersMiddleware_SetsBaselineHeaders(t *testing.T) {
|
||||
h := securityHeadersMiddleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}))
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
h.ServeHTTP(rr, req)
|
||||
if rr.Header().Get("X-Content-Type-Options") != "nosniff" {
|
||||
t.Fatalf("missing nosniff, got %q", rr.Header().Get("X-Content-Type-Options"))
|
||||
}
|
||||
if rr.Header().Get("X-Frame-Options") != "DENY" {
|
||||
t.Fatalf("missing frame deny")
|
||||
}
|
||||
if rr.Header().Get("Content-Security-Policy") == "" {
|
||||
t.Fatal("missing CSP")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupOldTempDirectories_SkipsActiveSessionTempDir(t *testing.T) {
|
||||
base := t.TempDir()
|
||||
st, err := storage.NewStorage(base)
|
||||
if err != nil {
|
||||
t.Fatalf("NewStorage: %v", err)
|
||||
}
|
||||
tempPath := filepath.Join(base, "temp")
|
||||
activeDir, err := os.MkdirTemp(tempPath, "jiggablend-upload-*")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldTime := time.Now().Add(-2 * time.Hour)
|
||||
_ = os.Chtimes(activeDir, oldTime, oldTime)
|
||||
|
||||
staleDir := filepath.Join(tempPath, "stale-old-dir")
|
||||
if err := os.MkdirAll(staleDir, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = os.Chtimes(staleDir, oldTime, oldTime)
|
||||
|
||||
s := &Manager{
|
||||
storage: st,
|
||||
uploadSessions: map[string]*UploadSession{
|
||||
"active": {SessionID: "active", TempDir: activeDir},
|
||||
},
|
||||
}
|
||||
s.cleanupOldTempDirectoriesOnce()
|
||||
|
||||
if _, err := os.Stat(activeDir); err != nil {
|
||||
t.Fatalf("active session temp dir was deleted: %v", err)
|
||||
}
|
||||
if _, err := os.Stat(staleDir); !os.IsNotExist(err) {
|
||||
t.Fatalf("stale temp dir should have been removed, stat err=%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateSessionCookie_UsesConfigProductionMode(t *testing.T) {
|
||||
t.Setenv("PRODUCTION", "true")
|
||||
s := &Manager{cfg: &config.Config{}, sessionCookieMaxAge: 3600}
|
||||
cookie := s.createSessionCookie("sid")
|
||||
if !cookie.Secure {
|
||||
t.Fatal("expected Secure cookie when production mode is on via env")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"jiggablend/pkg/types"
|
||||
)
|
||||
|
||||
func uploadSessionMetadata(session *UploadSession) *types.BlendMetadata {
|
||||
if session == nil || session.ResultMetadata == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch m := session.ResultMetadata.(type) {
|
||||
case *types.BlendMetadata:
|
||||
return m
|
||||
case types.BlendMetadata:
|
||||
meta := m
|
||||
return &meta
|
||||
default:
|
||||
data, err := json.Marshal(session.ResultMetadata)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var meta types.BlendMetadata
|
||||
if err := json.Unmarshal(data, &meta); err != nil {
|
||||
return nil
|
||||
}
|
||||
return &meta
|
||||
}
|
||||
}
|
||||
|
||||
// mergeBlendMetadataForJobCreate starts from upload analysis metadata when available,
|
||||
// then applies explicit job creation overrides from the request.
|
||||
func mergeBlendMetadataForJobCreate(uploadMeta *types.BlendMetadata, req *types.CreateJobRequest) (*types.BlendMetadata, error) {
|
||||
if req.FrameStart == nil || req.FrameEnd == nil {
|
||||
return nil, fmt.Errorf("frame_start and frame_end are required")
|
||||
}
|
||||
|
||||
var metadata types.BlendMetadata
|
||||
if uploadMeta != nil {
|
||||
metadata = *uploadMeta
|
||||
}
|
||||
|
||||
metadata.FrameStart = *req.FrameStart
|
||||
metadata.FrameEnd = *req.FrameEnd
|
||||
|
||||
if req.RenderSettings != nil {
|
||||
metadata.RenderSettings = *req.RenderSettings
|
||||
}
|
||||
if req.OutputFormat != nil {
|
||||
metadata.RenderSettings.OutputFormat = *req.OutputFormat
|
||||
}
|
||||
if req.BlenderVersion != nil && *req.BlenderVersion != "" {
|
||||
metadata.BlenderVersion = *req.BlenderVersion
|
||||
}
|
||||
if req.UnhideObjects != nil {
|
||||
metadata.UnhideObjects = req.UnhideObjects
|
||||
}
|
||||
if req.EnableExecution != nil {
|
||||
metadata.EnableExecution = req.EnableExecution
|
||||
}
|
||||
|
||||
if metadata.BlenderVersion == "" {
|
||||
return nil, fmt.Errorf("blender_version is required (from upload analysis or job request)")
|
||||
}
|
||||
|
||||
return &metadata, nil
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"jiggablend/pkg/types"
|
||||
)
|
||||
|
||||
func TestMergeBlendMetadataForJobCreate_PreservesUploadMetadata(t *testing.T) {
|
||||
uploadMeta := &types.BlendMetadata{
|
||||
FrameStart: -15,
|
||||
FrameEnd: 1468,
|
||||
BlenderVersion: "4.5.11",
|
||||
RenderSettings: types.RenderSettings{
|
||||
ResolutionX: 2560,
|
||||
ResolutionY: 1440,
|
||||
Engine: "cycles",
|
||||
OutputFormat: "EXR",
|
||||
},
|
||||
SceneInfo: types.SceneInfo{
|
||||
ObjectCount: 42,
|
||||
},
|
||||
}
|
||||
|
||||
frameStart := 800
|
||||
frameEnd := 800
|
||||
outputFormat := "EXR"
|
||||
req := &types.CreateJobRequest{
|
||||
FrameStart: &frameStart,
|
||||
FrameEnd: &frameEnd,
|
||||
OutputFormat: &outputFormat,
|
||||
}
|
||||
|
||||
meta, err := mergeBlendMetadataForJobCreate(uploadMeta, req)
|
||||
if err != nil {
|
||||
t.Fatalf("mergeBlendMetadataForJobCreate failed: %v", err)
|
||||
}
|
||||
|
||||
if meta.BlenderVersion != "4.5.11" {
|
||||
t.Fatalf("expected blender_version 4.5.11, got %q", meta.BlenderVersion)
|
||||
}
|
||||
if meta.FrameStart != 800 || meta.FrameEnd != 800 {
|
||||
t.Fatalf("expected frame range 800-800, got %d-%d", meta.FrameStart, meta.FrameEnd)
|
||||
}
|
||||
if meta.RenderSettings.ResolutionX != 2560 || meta.RenderSettings.ResolutionY != 1440 {
|
||||
t.Fatalf("expected resolution 2560x1440, got %dx%d", meta.RenderSettings.ResolutionX, meta.RenderSettings.ResolutionY)
|
||||
}
|
||||
if meta.RenderSettings.Engine != "cycles" {
|
||||
t.Fatalf("expected engine cycles, got %q", meta.RenderSettings.Engine)
|
||||
}
|
||||
if meta.RenderSettings.OutputFormat != "EXR" {
|
||||
t.Fatalf("expected output_format EXR, got %q", meta.RenderSettings.OutputFormat)
|
||||
}
|
||||
if meta.SceneInfo.ObjectCount != 42 {
|
||||
t.Fatalf("expected scene object_count 42, got %d", meta.SceneInfo.ObjectCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeBlendMetadataForJobCreate_RequestOverridesVersion(t *testing.T) {
|
||||
uploadMeta := &types.BlendMetadata{
|
||||
BlenderVersion: "4.5.11",
|
||||
}
|
||||
|
||||
frameStart := 1
|
||||
frameEnd := 1
|
||||
override := "4.2.3"
|
||||
req := &types.CreateJobRequest{
|
||||
FrameStart: &frameStart,
|
||||
FrameEnd: &frameEnd,
|
||||
BlenderVersion: &override,
|
||||
}
|
||||
|
||||
meta, err := mergeBlendMetadataForJobCreate(uploadMeta, req)
|
||||
if err != nil {
|
||||
t.Fatalf("mergeBlendMetadataForJobCreate failed: %v", err)
|
||||
}
|
||||
if meta.BlenderVersion != "4.2.3" {
|
||||
t.Fatalf("expected request override 4.2.3, got %q", meta.BlenderVersion)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeBlendMetadataForJobCreate_RequiresBlenderVersion(t *testing.T) {
|
||||
frameStart := 1
|
||||
frameEnd := 1
|
||||
req := &types.CreateJobRequest{
|
||||
FrameStart: &frameStart,
|
||||
FrameEnd: &frameEnd,
|
||||
}
|
||||
|
||||
if _, err := mergeBlendMetadataForJobCreate(nil, req); err == nil {
|
||||
t.Fatal("expected error when blender_version is missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadSessionMetadata(t *testing.T) {
|
||||
session := &UploadSession{
|
||||
ResultMetadata: &types.BlendMetadata{
|
||||
BlenderVersion: "4.5.11",
|
||||
},
|
||||
}
|
||||
|
||||
meta := uploadSessionMetadata(session)
|
||||
if meta == nil || meta.BlenderVersion != "4.5.11" {
|
||||
t.Fatalf("unexpected metadata: %+v", meta)
|
||||
}
|
||||
}
|
||||
+271
-89
@@ -63,21 +63,20 @@ func (s *Manager) runnerAuthMiddleware(next http.HandlerFunc) http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// For fixed API keys, skip database verification
|
||||
// Always verify the runner exists. For non-fixed keys, also check API key ownership.
|
||||
var dbAPIKeyID sql.NullInt64
|
||||
err = s.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow("SELECT api_key_id FROM runners WHERE id = ?", runnerID).Scan(&dbAPIKeyID)
|
||||
})
|
||||
if err == sql.ErrNoRows {
|
||||
s.respondError(w, http.StatusNotFound, "runner not found")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("failed to query runner API key: %v", err))
|
||||
return
|
||||
}
|
||||
if apiKeyID != -1 {
|
||||
// Verify runner exists and uses this API key
|
||||
var dbAPIKeyID sql.NullInt64
|
||||
err = s.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow("SELECT api_key_id FROM runners WHERE id = ?", runnerID).Scan(&dbAPIKeyID)
|
||||
})
|
||||
if err == sql.ErrNoRows {
|
||||
s.respondError(w, http.StatusNotFound, "runner not found")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("failed to query runner API key: %v", err))
|
||||
return
|
||||
}
|
||||
if !dbAPIKeyID.Valid || dbAPIKeyID.Int64 != apiKeyID {
|
||||
s.respondError(w, http.StatusForbidden, "runner does not belong to this API key")
|
||||
return
|
||||
@@ -1000,6 +999,32 @@ func (s *Manager) handleUploadFileWithToken(w http.ResponseWriter, r *http.Reque
|
||||
})
|
||||
}
|
||||
|
||||
// runnerCanAccessJob returns true if the runner has been assigned at least one task on the job.
|
||||
// Used to scope file/metadata/status APIs so a valid API key cannot read arbitrary jobs.
|
||||
func (s *Manager) runnerCanAccessJob(runnerID, jobID int64) bool {
|
||||
if runnerID <= 0 || jobID <= 0 {
|
||||
return false
|
||||
}
|
||||
var n int
|
||||
err := s.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow(
|
||||
`SELECT COUNT(*) FROM tasks WHERE job_id = ? AND runner_id = ?`,
|
||||
jobID, runnerID,
|
||||
).Scan(&n)
|
||||
})
|
||||
return err == nil && n > 0
|
||||
}
|
||||
|
||||
// requireRunnerJobAccess enforces that the authenticated runner may access the job.
|
||||
func (s *Manager) requireRunnerJobAccess(w http.ResponseWriter, r *http.Request, jobID int64) bool {
|
||||
runnerID, _ := r.Context().Value(runnerIDContextKey).(int64)
|
||||
if !s.runnerCanAccessJob(runnerID, jobID) {
|
||||
s.respondError(w, http.StatusForbidden, "runner is not assigned to this job")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// handleGetJobStatusForRunner allows runners to check job status
|
||||
func (s *Manager) handleGetJobStatusForRunner(w http.ResponseWriter, r *http.Request) {
|
||||
jobID, err := parseID(r, "jobId")
|
||||
@@ -1007,6 +1032,9 @@ func (s *Manager) handleGetJobStatusForRunner(w http.ResponseWriter, r *http.Req
|
||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if !s.requireRunnerJobAccess(w, r, jobID) {
|
||||
return
|
||||
}
|
||||
|
||||
var job types.Job
|
||||
var startedAt, completedAt sql.NullTime
|
||||
@@ -1069,6 +1097,9 @@ func (s *Manager) handleGetJobFilesForRunner(w http.ResponseWriter, r *http.Requ
|
||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if !s.requireRunnerJobAccess(w, r, jobID) {
|
||||
return
|
||||
}
|
||||
|
||||
runnerID := r.URL.Query().Get("runner_id")
|
||||
log.Printf("GetJobFiles request for job %d from runner %s", jobID, runnerID)
|
||||
@@ -1127,6 +1158,9 @@ func (s *Manager) handleGetJobMetadataForRunner(w http.ResponseWriter, r *http.R
|
||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if !s.requireRunnerJobAccess(w, r, jobID) {
|
||||
return
|
||||
}
|
||||
|
||||
var blendMetadataJSON sql.NullString
|
||||
err = s.db.With(func(conn *sql.DB) error {
|
||||
@@ -1166,6 +1200,9 @@ func (s *Manager) handleDownloadFileForRunner(w http.ResponseWriter, r *http.Req
|
||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
if !s.requireRunnerJobAccess(w, r, jobID) {
|
||||
return
|
||||
}
|
||||
|
||||
// Get fileName from URL path (may need URL decoding)
|
||||
fileName := chi.URLParam(r, "fileName")
|
||||
@@ -1259,11 +1296,14 @@ type WSLogEntry struct {
|
||||
}
|
||||
|
||||
type WSTaskUpdate struct {
|
||||
TaskID int64 `json:"task_id"`
|
||||
Status string `json:"status"`
|
||||
OutputPath string `json:"output_path,omitempty"`
|
||||
Success bool `json:"success"`
|
||||
Error string `json:"error,omitempty"`
|
||||
TaskID int64 `json:"task_id"`
|
||||
Status string `json:"status"`
|
||||
OutputPath string `json:"output_path,omitempty"`
|
||||
Success bool `json:"success"`
|
||||
Error string `json:"error,omitempty"`
|
||||
// FreeRequeue asks the manager to requeue without incrementing retry_count
|
||||
// (e.g. this attempt newly armed GPU lockout after a ROCm fault).
|
||||
FreeRequeue bool `json:"free_requeue,omitempty"`
|
||||
}
|
||||
|
||||
// handleRunnerJobWebSocket handles per-job WebSocket connections from runners
|
||||
@@ -1362,25 +1402,18 @@ func (s *Manager) handleRunnerJobWebSocket(w http.ResponseWriter, r *http.Reques
|
||||
delete(s.runnerJobConnsWriteMu, connKey)
|
||||
s.runnerJobConnsMu.Unlock()
|
||||
|
||||
// Check if task is still running - if so, mark as failed
|
||||
// If the runner never delivered task_complete (crash, bad payload, network drop),
|
||||
// treat this like a task failure with retries. Prefer the last ERROR log so
|
||||
// segfaults surface as such instead of a generic WebSocket message.
|
||||
var currentStatus string
|
||||
s.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow("SELECT status FROM tasks WHERE id = ?", taskID).Scan(¤tStatus)
|
||||
})
|
||||
if currentStatus == string(types.TaskStatusRunning) {
|
||||
log.Printf("Job WebSocket disconnected unexpectedly for task %d, marking as failed", taskID)
|
||||
s.db.With(func(conn *sql.DB) error {
|
||||
_, err := conn.Exec(
|
||||
`UPDATE tasks SET status = ?, runner_id = NULL, error_message = ?, completed_at = ? WHERE id = ?`,
|
||||
types.TaskStatusFailed, "WebSocket connection lost", time.Now(), taskID,
|
||||
)
|
||||
return err
|
||||
})
|
||||
s.broadcastTaskUpdate(jobID, taskID, "task_update", map[string]interface{}{
|
||||
"status": types.TaskStatusFailed,
|
||||
"error_message": "WebSocket connection lost",
|
||||
})
|
||||
s.updateJobStatusFromTasks(jobID)
|
||||
errorMsg := s.inferTaskFailureFromLogs(taskID)
|
||||
freeRequeue := s.taskLogsIndicateGPULockoutArm(taskID)
|
||||
log.Printf("Job WebSocket disconnected unexpectedly for task %d: %s (free_requeue=%v)", taskID, errorMsg, freeRequeue)
|
||||
s.handleTaskFailureWithRetry(runnerID, taskID, jobID, errorMsg, freeRequeue)
|
||||
}
|
||||
|
||||
log.Printf("Job WebSocket closed: job=%d, runner=%d, task=%d", jobID, runnerID, taskID)
|
||||
@@ -1467,13 +1500,19 @@ func (s *Manager) handleRunnerJobWebSocket(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
|
||||
case "task_complete":
|
||||
var taskUpdate WSTaskUpdate
|
||||
if err := json.Unmarshal(msg.Data, &taskUpdate); err == nil {
|
||||
if taskUpdate.TaskID == taskID {
|
||||
s.handleWebSocketTaskComplete(runnerID, taskUpdate)
|
||||
// Task is done, close connection
|
||||
return
|
||||
}
|
||||
taskUpdate, err := parseWSTaskUpdate(msg.Data)
|
||||
if err != nil {
|
||||
log.Printf("Job WebSocket task_complete for task %d: %v", taskID, err)
|
||||
// Still close; disconnect handler will infer failure from ERROR logs.
|
||||
return
|
||||
}
|
||||
if taskUpdate.TaskID == 0 {
|
||||
taskUpdate.TaskID = taskID
|
||||
}
|
||||
if taskUpdate.TaskID == taskID {
|
||||
s.handleWebSocketTaskComplete(runnerID, taskUpdate)
|
||||
// Task is done, close connection
|
||||
return
|
||||
}
|
||||
case "runner_heartbeat":
|
||||
s.handleWSRunnerHeartbeat(conn, jobID)
|
||||
@@ -1576,12 +1615,11 @@ func (s *Manager) handleWebSocketTaskComplete(runnerID int64, taskUpdate WSTaskU
|
||||
// Verify task belongs to runner and get task info
|
||||
var taskRunnerID sql.NullInt64
|
||||
var jobID int64
|
||||
var retryCount, maxRetries int
|
||||
err := s.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow(
|
||||
"SELECT runner_id, job_id, retry_count, max_retries FROM tasks WHERE id = ?",
|
||||
"SELECT runner_id, job_id FROM tasks WHERE id = ?",
|
||||
taskUpdate.TaskID,
|
||||
).Scan(&taskRunnerID, &jobID, &retryCount, &maxRetries)
|
||||
).Scan(&taskRunnerID, &jobID)
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to get task %d info: %v", taskUpdate.TaskID, err)
|
||||
@@ -1597,7 +1635,10 @@ func (s *Manager) handleWebSocketTaskComplete(runnerID int64, taskUpdate WSTaskU
|
||||
// Handle successful completion
|
||||
if taskUpdate.Success {
|
||||
err = s.db.WithTx(func(tx *sql.Tx) error {
|
||||
_, err := tx.Exec(`UPDATE tasks SET status = ? WHERE id = ?`, types.TaskStatusCompleted, taskUpdate.TaskID)
|
||||
_, err := tx.Exec(
|
||||
`UPDATE tasks SET status = ?, error_message = NULL, completed_at = ? WHERE id = ?`,
|
||||
types.TaskStatusCompleted, now, taskUpdate.TaskID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1607,8 +1648,7 @@ func (s *Manager) handleWebSocketTaskComplete(runnerID int64, taskUpdate WSTaskU
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = tx.Exec(`UPDATE tasks SET completed_at = ? WHERE id = ?`, now, taskUpdate.TaskID)
|
||||
return err
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to update task %d: %v", taskUpdate.TaskID, err)
|
||||
@@ -1617,89 +1657,231 @@ func (s *Manager) handleWebSocketTaskComplete(runnerID int64, taskUpdate WSTaskU
|
||||
|
||||
// Broadcast task update
|
||||
s.broadcastTaskUpdate(jobID, taskUpdate.TaskID, "task_update", map[string]interface{}{
|
||||
"status": types.TaskStatusCompleted,
|
||||
"output_path": taskUpdate.OutputPath,
|
||||
"completed_at": now,
|
||||
"status": types.TaskStatusCompleted,
|
||||
"output_path": taskUpdate.OutputPath,
|
||||
"completed_at": now,
|
||||
"error_message": nil,
|
||||
})
|
||||
s.updateJobStatusFromTasks(jobID)
|
||||
return
|
||||
}
|
||||
|
||||
// Handle task failure - this is an actual task failure (e.g., Blender crash)
|
||||
// Check if we have retries remaining
|
||||
if retryCount < maxRetries {
|
||||
// Reset to pending for retry - increment retry_count
|
||||
// Actual task failure (e.g. Blender segfault) — retry when retries remain.
|
||||
// free_requeue: failure newly armed GPU lockout; requeue without burning a retry.
|
||||
errorMsg := taskUpdate.Error
|
||||
if errorMsg == "" {
|
||||
errorMsg = s.inferTaskFailureFromLogs(taskUpdate.TaskID)
|
||||
}
|
||||
freeRequeue := taskUpdate.FreeRequeue || s.taskLogsIndicateGPULockoutArm(taskUpdate.TaskID)
|
||||
s.handleTaskFailureWithRetry(runnerID, taskUpdate.TaskID, jobID, errorMsg, freeRequeue)
|
||||
}
|
||||
|
||||
// defaultUnexpectedDisconnectError is used when the runner drops the job WebSocket
|
||||
// without a task_complete message and without any ERROR logs.
|
||||
const defaultUnexpectedDisconnectError = "WebSocket connection lost"
|
||||
|
||||
// parseWSTaskUpdate decodes a task_complete payload. Older runners marshaled
|
||||
// the error field as a Go error interface (JSON object {}), which fails strict
|
||||
// unmarshaling into a string — fall back to extracting fields loosely so
|
||||
// success/failure is still recognized.
|
||||
func parseWSTaskUpdate(data json.RawMessage) (WSTaskUpdate, error) {
|
||||
var update WSTaskUpdate
|
||||
if err := json.Unmarshal(data, &update); err == nil {
|
||||
return update, nil
|
||||
}
|
||||
|
||||
var loose map[string]interface{}
|
||||
if err := json.Unmarshal(data, &loose); err != nil {
|
||||
return WSTaskUpdate{}, err
|
||||
}
|
||||
|
||||
if v, ok := loose["task_id"].(float64); ok {
|
||||
update.TaskID = int64(v)
|
||||
}
|
||||
if v, ok := loose["success"].(bool); ok {
|
||||
update.Success = v
|
||||
}
|
||||
if v, ok := loose["output_path"].(string); ok {
|
||||
update.OutputPath = v
|
||||
}
|
||||
if v, ok := loose["status"].(string); ok {
|
||||
update.Status = v
|
||||
}
|
||||
if v, ok := loose["free_requeue"].(bool); ok {
|
||||
update.FreeRequeue = v
|
||||
}
|
||||
switch v := loose["error"].(type) {
|
||||
case string:
|
||||
update.Error = v
|
||||
case map[string]interface{}:
|
||||
// Legacy runner bug: error interface marshaled as {}
|
||||
update.Error = ""
|
||||
}
|
||||
return update, nil
|
||||
}
|
||||
|
||||
// taskLogsIndicateGPULockoutArm is true when this attempt's logs show the runner
|
||||
// newly armed GPU lockout (free-requeue path when task_complete lacked free_requeue).
|
||||
func (s *Manager) taskLogsIndicateGPULockoutArm(taskID int64) bool {
|
||||
var n int
|
||||
err := s.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow(
|
||||
`SELECT COUNT(*) FROM task_logs
|
||||
WHERE task_id = ?
|
||||
AND (
|
||||
message LIKE '%GPU error detected%'
|
||||
OR message LIKE '%free-requeue%'
|
||||
OR message LIKE '%GPU disabled for subsequent jobs%'
|
||||
)`,
|
||||
taskID,
|
||||
).Scan(&n)
|
||||
})
|
||||
return err == nil && n > 0
|
||||
}
|
||||
|
||||
// inferTaskFailureFromLogs returns the best failure reason for a task that died
|
||||
// without a usable task_complete payload. Prefers the latest ERROR log so Blender
|
||||
// segfaults (and similar crashes) are stored instead of a generic disconnect message.
|
||||
func (s *Manager) inferTaskFailureFromLogs(taskID int64) string {
|
||||
var lastError string
|
||||
err := s.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow(
|
||||
`SELECT message FROM task_logs
|
||||
WHERE task_id = ? AND UPPER(log_level) = ?
|
||||
ORDER BY id DESC LIMIT 1`,
|
||||
taskID, string(types.LogLevelError),
|
||||
).Scan(&lastError)
|
||||
})
|
||||
if err != nil || strings.TrimSpace(lastError) == "" {
|
||||
return defaultUnexpectedDisconnectError
|
||||
}
|
||||
return formatInferredTaskFailure(lastError)
|
||||
}
|
||||
|
||||
// formatInferredTaskFailure normalizes streamed runner ERROR lines into a concise
|
||||
// task error_message. Blender crashes often arrive as "Task failed: blender failed: signal: ..."
|
||||
// or "Blender render failed: ..."; keep the underlying failure text.
|
||||
func formatInferredTaskFailure(logMessage string) string {
|
||||
msg := strings.TrimSpace(logMessage)
|
||||
for _, prefix := range []string{
|
||||
"Task failed: ",
|
||||
"Blender render failed: ",
|
||||
} {
|
||||
if strings.HasPrefix(msg, prefix) {
|
||||
msg = strings.TrimSpace(strings.TrimPrefix(msg, prefix))
|
||||
}
|
||||
}
|
||||
if msg == "" {
|
||||
return defaultUnexpectedDisconnectError
|
||||
}
|
||||
return msg
|
||||
}
|
||||
|
||||
// handleTaskFailureWithRetry applies the task failure policy used for Blender
|
||||
// crashes and unexpected job-WebSocket drops: requeue as pending while
|
||||
// retry_count < max_retries, otherwise mark failed permanently with error_message.
|
||||
// When freeRequeue is true (GPU lockout newly armed this attempt), the task is
|
||||
// always requeued as pending without incrementing retry_count.
|
||||
func (s *Manager) handleTaskFailureWithRetry(runnerID, taskID, jobID int64, errorMsg string, freeRequeue bool) {
|
||||
var retryCount, maxRetries int
|
||||
err := s.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow(
|
||||
"SELECT retry_count, max_retries FROM tasks WHERE id = ?",
|
||||
taskID,
|
||||
).Scan(&retryCount, &maxRetries)
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to get retry info for task %d: %v", taskID, err)
|
||||
return
|
||||
}
|
||||
|
||||
if errorMsg == "" {
|
||||
errorMsg = defaultUnexpectedDisconnectError
|
||||
}
|
||||
if freeRequeue {
|
||||
// Make the reason obvious in the UI without looking like a permanent fail.
|
||||
if !strings.Contains(strings.ToLower(errorMsg), "gpu lockout") {
|
||||
errorMsg = "GPU lockout armed (free requeue): " + errorMsg
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
// Free requeue always goes back to pending (expected ROCm GPU fault that armed lockout).
|
||||
// Otherwise only requeue while retries remain.
|
||||
canRequeue := freeRequeue || retryCount < maxRetries
|
||||
if canRequeue {
|
||||
newRetryCount := retryCount
|
||||
retrySQL := `UPDATE tasks SET status = ?, runner_id = NULL, current_step = NULL,
|
||||
error_message = ?, started_at = NULL, completed_at = NULL
|
||||
WHERE id = ?`
|
||||
args := []interface{}{types.TaskStatusPending, errorMsg, taskID}
|
||||
if !freeRequeue {
|
||||
retrySQL = `UPDATE tasks SET status = ?, runner_id = NULL, current_step = NULL,
|
||||
error_message = ?, retry_count = retry_count + 1, started_at = NULL, completed_at = NULL
|
||||
WHERE id = ?`
|
||||
newRetryCount = retryCount + 1
|
||||
}
|
||||
|
||||
err = s.db.WithTx(func(tx *sql.Tx) error {
|
||||
_, err := tx.Exec(
|
||||
`UPDATE tasks SET status = ?, runner_id = NULL, current_step = NULL,
|
||||
retry_count = retry_count + 1, started_at = NULL, completed_at = NULL
|
||||
WHERE id = ?`,
|
||||
types.TaskStatusPending, taskUpdate.TaskID,
|
||||
)
|
||||
_, err := tx.Exec(retrySQL, args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Clear steps and logs for fresh retry
|
||||
_, err = tx.Exec(`DELETE FROM task_steps WHERE task_id = ?`, taskUpdate.TaskID)
|
||||
_, err = tx.Exec(`DELETE FROM task_steps WHERE task_id = ?`, taskID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(`DELETE FROM task_logs WHERE task_id = ?`, taskUpdate.TaskID)
|
||||
_, err = tx.Exec(`DELETE FROM task_logs WHERE task_id = ?`, taskID)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to reset task %d for retry: %v", taskUpdate.TaskID, err)
|
||||
log.Printf("Failed to reset task %d for retry: %v", taskID, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Broadcast task reset to clients (includes steps_cleared and logs_cleared flags)
|
||||
s.broadcastTaskUpdate(jobID, taskUpdate.TaskID, "task_reset", map[string]interface{}{
|
||||
s.broadcastTaskUpdate(jobID, taskID, "task_reset", map[string]interface{}{
|
||||
"status": types.TaskStatusPending,
|
||||
"retry_count": retryCount + 1,
|
||||
"error_message": taskUpdate.Error,
|
||||
"retry_count": newRetryCount,
|
||||
"error_message": errorMsg,
|
||||
"steps_cleared": true,
|
||||
"logs_cleared": true,
|
||||
"free_requeue": freeRequeue,
|
||||
})
|
||||
|
||||
log.Printf("Task %d failed but has retries remaining (%d/%d), reset to pending", taskUpdate.TaskID, retryCount+1, maxRetries)
|
||||
if freeRequeue {
|
||||
log.Printf("Task %d failed (%s); free-requeued without burning retry (still %d/%d used)",
|
||||
taskID, errorMsg, retryCount, maxRetries)
|
||||
} else {
|
||||
log.Printf("Task %d failed (%s) but has retries remaining (%d/%d), reset to pending",
|
||||
taskID, errorMsg, newRetryCount, maxRetries)
|
||||
}
|
||||
} else {
|
||||
// No retries remaining - mark as failed
|
||||
err = s.db.WithTx(func(tx *sql.Tx) error {
|
||||
_, err := tx.Exec(
|
||||
`UPDATE tasks SET status = ?, runner_id = NULL, completed_at = ? WHERE id = ?`,
|
||||
types.TaskStatusFailed, now, taskUpdate.TaskID,
|
||||
`UPDATE tasks SET status = ?, runner_id = NULL, completed_at = ?, error_message = ? WHERE id = ?`,
|
||||
types.TaskStatusFailed, now, errorMsg, taskID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if taskUpdate.Error != "" {
|
||||
_, err = tx.Exec(`UPDATE tasks SET error_message = ? WHERE id = ?`, taskUpdate.Error, taskUpdate.TaskID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to mark task %d as failed: %v", taskUpdate.TaskID, err)
|
||||
log.Printf("Failed to mark task %d as failed: %v", taskID, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Log the final failure
|
||||
s.logTaskEvent(taskUpdate.TaskID, &runnerID, types.LogLevelError,
|
||||
fmt.Sprintf("Task failed permanently after %d retries: %s", maxRetries, taskUpdate.Error), "")
|
||||
s.logTaskEvent(taskID, &runnerID, types.LogLevelError,
|
||||
fmt.Sprintf("Task failed permanently after %d retries: %s", maxRetries, errorMsg), "")
|
||||
|
||||
// Broadcast task update
|
||||
s.broadcastTaskUpdate(jobID, taskUpdate.TaskID, "task_update", map[string]interface{}{
|
||||
s.broadcastTaskUpdate(jobID, taskID, "task_update", map[string]interface{}{
|
||||
"status": types.TaskStatusFailed,
|
||||
"completed_at": now,
|
||||
"error_message": taskUpdate.Error,
|
||||
"error_message": errorMsg,
|
||||
})
|
||||
|
||||
log.Printf("Task %d failed permanently after %d retries", taskUpdate.TaskID, maxRetries)
|
||||
log.Printf("Task %d failed permanently after %d retries: %s", taskID, maxRetries, errorMsg)
|
||||
}
|
||||
|
||||
// Update job status and progress
|
||||
s.updateJobStatusFromTasks(jobID)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package api
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseBlenderFrame(t *testing.T) {
|
||||
frame, ok := parseBlenderFrame("Info Fra:2470 Mem:12.00M")
|
||||
@@ -19,3 +22,79 @@ func TestJobTaskCounts_Progress(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWSTaskUpdate_LegacyErrorObject(t *testing.T) {
|
||||
// Simulates older runners that JSON-marshaled an error interface as {}
|
||||
raw := json.RawMessage(`{"task_id":99,"success":false,"error":{}}`)
|
||||
update, err := parseWSTaskUpdate(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parseWSTaskUpdate: %v", err)
|
||||
}
|
||||
if update.TaskID != 99 {
|
||||
t.Fatalf("TaskID = %d, want 99", update.TaskID)
|
||||
}
|
||||
if update.Success {
|
||||
t.Fatal("Success = true, want false")
|
||||
}
|
||||
if update.Error != "" {
|
||||
t.Fatalf("Error = %q, want empty (legacy object)", update.Error)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseWSTaskUpdate_StringError(t *testing.T) {
|
||||
raw := json.RawMessage(`{"task_id":7,"success":false,"error":"blender failed: signal: segmentation fault (core dumped)","free_requeue":true}`)
|
||||
update, err := parseWSTaskUpdate(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("parseWSTaskUpdate: %v", err)
|
||||
}
|
||||
if update.TaskID != 7 || update.Success {
|
||||
t.Fatalf("unexpected update: %+v", update)
|
||||
}
|
||||
if update.Error != "blender failed: signal: segmentation fault (core dumped)" {
|
||||
t.Fatalf("Error = %q", update.Error)
|
||||
}
|
||||
if !update.FreeRequeue {
|
||||
t.Fatal("expected FreeRequeue true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatInferredTaskFailure(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "task failed segfault prefix",
|
||||
in: "Task failed: blender failed: signal: segmentation fault (core dumped)",
|
||||
want: "blender failed: signal: segmentation fault (core dumped)",
|
||||
},
|
||||
{
|
||||
name: "blender render failed prefix",
|
||||
in: "Blender render failed: blender failed: signal: segmentation fault (core dumped)",
|
||||
want: "blender failed: signal: segmentation fault (core dumped)",
|
||||
},
|
||||
{
|
||||
name: "double prefix",
|
||||
in: "Task failed: Blender render failed: blender failed: signal: segmentation fault (core dumped)",
|
||||
want: "blender failed: signal: segmentation fault (core dumped)",
|
||||
},
|
||||
{
|
||||
name: "plain message",
|
||||
in: "blender failed: exit status 1",
|
||||
want: "blender failed: exit status 1",
|
||||
},
|
||||
{
|
||||
name: "empty falls back",
|
||||
in: " ",
|
||||
want: defaultUnexpectedDisconnectError,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := formatInferredTaskFailure(tt.in); got != tt.want {
|
||||
t.Fatalf("formatInferredTaskFailure(%q) = %q, want %q", tt.in, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user