From 4ac05d50a12cfded5168d765ecd00fbffe03ce02 Mon Sep 17 00:00:00 2001 From: Justin Harms Date: Mon, 24 Nov 2025 21:48:05 -0600 Subject: [PATCH] Enhance logging and context handling in job management. Introduce a logger initialization with configurable parameters in the manager and runner commands. Update job context handling to use tar files instead of tar.gz, and implement ETag generation for improved caching. Refactor API endpoints to support new context file structure and enhance error handling in job submissions. Add support for unhide objects and auto-execution options in job creation requests. --- cmd/manager/main.go | 33 +- cmd/runner/main.go | 31 + go.mod | 1 + go.sum | 2 + internal/api/jobs.go | 1755 ++++++++++++++--- internal/api/metadata.go | 263 +-- internal/api/runners.go | 183 +- internal/api/server.go | 162 +- internal/auth/auth.go | 5 + internal/database/schema.go | 6 + internal/logger/logger.go | 127 ++ internal/runner/client.go | 786 +------- internal/storage/storage.go | 77 +- pkg/scripts/scripts.go | 13 + pkg/scripts/scripts/extract_metadata.py | 173 ++ .../scripts/render_blender.py.template | 589 ++++++ pkg/scripts/scripts/unhide_objects.py | 29 + pkg/types/types.go | 10 +- web/src/components/FileExplorer.jsx | 37 +- web/src/components/JobDetails.jsx | 490 ++++- web/src/components/JobList.jsx | 198 +- web/src/components/JobSubmission.jsx | 254 ++- web/src/utils/api.js | 220 ++- 23 files changed, 4133 insertions(+), 1311 deletions(-) create mode 100644 internal/logger/logger.go create mode 100644 pkg/scripts/scripts.go create mode 100644 pkg/scripts/scripts/extract_metadata.py create mode 100644 pkg/scripts/scripts/render_blender.py.template create mode 100644 pkg/scripts/scripts/unhide_objects.py diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 60e54f3..e966f52 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -11,17 +11,34 @@ import ( "jiggablend/internal/api" "jiggablend/internal/auth" "jiggablend/internal/database" + "jiggablend/internal/logger" "jiggablend/internal/storage" ) func main() { var ( - port = flag.String("port", getEnv("PORT", "8080"), "Server port") - dbPath = flag.String("db", getEnv("DB_PATH", "jiggablend.db"), "Database path") - storagePath = flag.String("storage", getEnv("STORAGE_PATH", "./jiggablend-storage"), "Storage path") + port = flag.String("port", getEnv("PORT", "8080"), "Server port") + dbPath = flag.String("db", getEnv("DB_PATH", "jiggablend.db"), "Database path") + storagePath = flag.String("storage", getEnv("STORAGE_PATH", "./jiggablend-storage"), "Storage path") + logDir = flag.String("log-dir", getEnv("LOG_DIR", "./logs"), "Log directory") + logMaxSize = flag.Int("log-max-size", getEnvInt("LOG_MAX_SIZE", 100), "Maximum log file size in MB before rotation") + logMaxBackups = flag.Int("log-max-backups", getEnvInt("LOG_MAX_BACKUPS", 5), "Maximum number of rotated log files to keep") + logMaxAge = flag.Int("log-max-age", getEnvInt("LOG_MAX_AGE", 30), "Maximum age in days for rotated log files") ) flag.Parse() + // Initialize logger (writes to both stdout and log file with rotation) + logDirPath := *logDir + if err := logger.Init(logDirPath, "manager.log", *logMaxSize, *logMaxBackups, *logMaxAge); err != nil { + log.Fatalf("Failed to initialize logger: %v", err) + } + defer func() { + if l := logger.GetDefault(); l != nil { + l.Close() + } + }() + log.Printf("Log rotation configured: max_size=%dMB, max_backups=%d, max_age=%d days", *logMaxSize, *logMaxBackups, *logMaxAge) + // Initialize database db, err := database.NewDB(*dbPath) if err != nil { @@ -86,6 +103,16 @@ func getEnv(key, defaultValue string) string { return defaultValue } +func getEnvInt(key string, defaultValue int) int { + if value := os.Getenv(key); value != "" { + var result int + if _, err := fmt.Sscanf(value, "%d", &result); err == nil { + return result + } + } + return defaultValue +} + // checkBlenderAvailable checks if Blender is available by running `blender --version` func checkBlenderAvailable() error { cmd := exec.Command("blender", "--version") diff --git a/cmd/runner/main.go b/cmd/runner/main.go index 5c49e30..36231c4 100644 --- a/cmd/runner/main.go +++ b/cmd/runner/main.go @@ -13,6 +13,7 @@ import ( "syscall" "time" + "jiggablend/internal/logger" "jiggablend/internal/runner" ) @@ -31,6 +32,10 @@ func main() { token = flag.String("token", getEnv("REGISTRATION_TOKEN", ""), "Registration token") secretsFile = flag.String("secrets-file", getEnv("SECRETS_FILE", ""), "Path to secrets file for persistent storage (default: ./runner-secrets.json, or ./runner-secrets-{id}.json if multiple runners)") runnerIDSuffix = flag.String("runner-id", getEnv("RUNNER_ID", ""), "Unique runner ID suffix (auto-generated if not provided)") + logDir = flag.String("log-dir", getEnv("LOG_DIR", "./logs"), "Log directory") + logMaxSize = flag.Int("log-max-size", getEnvInt("LOG_MAX_SIZE", 100), "Maximum log file size in MB before rotation") + logMaxBackups = flag.Int("log-max-backups", getEnvInt("LOG_MAX_BACKUPS", 5), "Maximum number of rotated log files to keep") + logMaxAge = flag.Int("log-max-age", getEnvInt("LOG_MAX_AGE", 30), "Maximum age in days for rotated log files") ) flag.Parse() @@ -55,6 +60,22 @@ func main() { *name = fmt.Sprintf("%s-%s", *name, runnerIDStr) } + // Initialize logger (writes to both stdout and log file with rotation) + // Use runner-specific log file name based on the final name + sanitizedName := strings.ReplaceAll(*name, "/", "_") + sanitizedName = strings.ReplaceAll(sanitizedName, "\\", "_") + logFileName := fmt.Sprintf("runner-%s.log", sanitizedName) + + if err := logger.Init(*logDir, logFileName, *logMaxSize, *logMaxBackups, *logMaxAge); err != nil { + log.Fatalf("Failed to initialize logger: %v", err) + } + defer func() { + if l := logger.GetDefault(); l != nil { + l.Close() + } + }() + log.Printf("Log rotation configured: max_size=%dMB, max_backups=%d, max_age=%d days", *logMaxSize, *logMaxBackups, *logMaxAge) + // Set default secrets file if not provided - always use current directory if *secretsFile == "" { if *runnerIDSuffix != "" || getEnv("RUNNER_ID", "") != "" { @@ -210,6 +231,16 @@ func getEnv(key, defaultValue string) string { return defaultValue } +func getEnvInt(key string, defaultValue int) int { + if value := os.Getenv(key); value != "" { + var result int + if _, err := fmt.Sscanf(value, "%d", &result); err == nil { + return result + } + } + return defaultValue +} + // generateShortID generates a short random ID (8 hex characters) func generateShortID() string { bytes := make([]byte, 4) diff --git a/go.mod b/go.mod index 4d03326..5340e12 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/marcboeker/go-duckdb/v2 v2.4.3 golang.org/x/crypto v0.45.0 golang.org/x/oauth2 v0.33.0 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) require ( diff --git a/go.sum b/go.sum index 8517148..6dc6df1 100644 --- a/go.sum +++ b/go.sum @@ -82,5 +82,7 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/api/jobs.go b/internal/api/jobs.go index dc4a9c7..03c3915 100644 --- a/internal/api/jobs.go +++ b/internal/api/jobs.go @@ -4,8 +4,10 @@ import ( "archive/tar" "bufio" "bytes" - "compress/gzip" + "crypto/md5" "database/sql" + _ "embed" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -24,8 +26,27 @@ import ( "jiggablend/pkg/types" "github.com/go-chi/chi/v5" + "github.com/gorilla/websocket" + + "jiggablend/pkg/scripts" ) +// generateETag generates an ETag from data hash +func generateETag(data interface{}) string { + jsonData, err := json.Marshal(data) + if err != nil { + return "" + } + hash := md5.Sum(jsonData) + return fmt.Sprintf(`"%s"`, hex.EncodeToString(hash[:])) +} + +// checkETag checks if the request has If-None-Match header matching the ETag +func checkETag(r *http.Request, etag string) bool { + ifNoneMatch := r.Header.Get("If-None-Match") + return ifNoneMatch != "" && ifNoneMatch == etag +} + // isAdminUser checks if the current user is an admin func isAdminUser(r *http.Request) bool { return authpkg.IsAdmin(r.Context()) @@ -91,13 +112,32 @@ func (s *Server) handleCreateJob(w http.ResponseWriter, r *http.Request) { // Set job timeout to 24 hours (86400 seconds) jobTimeout := 86400 - // Store render settings in blend_metadata if provided + // Store render settings, unhide_objects, and enable_execution flags in blend_metadata if provided var blendMetadataJSON *string - if req.RenderSettings != nil { + if req.RenderSettings != nil || req.UnhideObjects != nil || req.EnableExecution != nil { metadata := types.BlendMetadata{ - FrameStart: *req.FrameStart, - FrameEnd: *req.FrameEnd, - RenderSettings: *req.RenderSettings, + FrameStart: *req.FrameStart, + FrameEnd: *req.FrameEnd, + RenderSettings: types.RenderSettings{}, + UnhideObjects: req.UnhideObjects, + EnableExecution: req.EnableExecution, + } + if req.RenderSettings != nil { + metadata.RenderSettings = *req.RenderSettings + } + metadataBytes, err := json.Marshal(metadata) + if err == nil { + metadataStr := string(metadataBytes) + blendMetadataJSON = &metadataStr + } + } else if req.UnhideObjects != nil || req.EnableExecution != nil { + // Even if no render settings, store unhide_objects and enable_execution flags + metadata := types.BlendMetadata{ + FrameStart: *req.FrameStart, + FrameEnd: *req.FrameEnd, + RenderSettings: types.RenderSettings{}, + UnhideObjects: req.UnhideObjects, + EnableExecution: req.EnableExecution, } metadataBytes, err := json.Marshal(metadata) if err == nil { @@ -127,8 +167,8 @@ func (s *Server) handleCreateJob(w http.ResponseWriter, r *http.Request) { log.Printf("Processing upload session for job %d: %s", jobID, *req.UploadSessionID) // Session ID is the full temp directory path tempDir := *req.UploadSessionID - tempContextPath := filepath.Join(tempDir, "context.tar.gz") - + tempContextPath := filepath.Join(tempDir, "context.tar") + if _, err := os.Stat(tempContextPath); err == nil { log.Printf("Found context archive at %s, moving to job %d directory", tempContextPath, jobID) // Move context to job directory @@ -138,9 +178,9 @@ func (s *Server) handleCreateJob(w http.ResponseWriter, r *http.Request) { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create job directory: %v", err)) return } - - jobContextPath := filepath.Join(jobPath, "context.tar.gz") - + + jobContextPath := filepath.Join(jobPath, "context.tar") + // Copy file instead of rename (works across filesystems) srcFile, err := os.Open(tempContextPath) if err != nil { @@ -149,7 +189,7 @@ func (s *Server) handleCreateJob(w http.ResponseWriter, r *http.Request) { return } defer srcFile.Close() - + dstFile, err := os.Create(jobContextPath) if err != nil { log.Printf("ERROR: Failed to create destination context archive %s: %v", jobContextPath, err) @@ -157,7 +197,7 @@ func (s *Server) handleCreateJob(w http.ResponseWriter, r *http.Request) { return } defer dstFile.Close() - + _, err = io.Copy(dstFile, srcFile) if err != nil { dstFile.Close() @@ -166,7 +206,7 @@ func (s *Server) handleCreateJob(w http.ResponseWriter, r *http.Request) { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to copy context archive: %v", err)) return } - + // Close files before deleting source srcFile.Close() if err := dstFile.Close(); err != nil { @@ -174,15 +214,15 @@ func (s *Server) handleCreateJob(w http.ResponseWriter, r *http.Request) { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to finalize context archive: %v", err)) return } - + // Delete source file after successful copy if err := os.Remove(tempContextPath); err != nil { log.Printf("Warning: Failed to remove source context archive %s: %v", tempContextPath, err) // Don't fail the operation if cleanup fails } - + log.Printf("Successfully copied context archive to %s", jobContextPath) - + // Record context archive in database contextInfo, err := os.Stat(jobContextPath) if err != nil { @@ -190,7 +230,7 @@ func (s *Server) handleCreateJob(w http.ResponseWriter, r *http.Request) { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to verify context archive: %v", err)) return } - + var fileID int64 err = s.db.QueryRow( `INSERT INTO job_files (job_id, file_type, file_path, file_name, file_size) @@ -203,23 +243,30 @@ func (s *Server) handleCreateJob(w http.ResponseWriter, r *http.Request) { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to record context archive: %v", err)) return } - + log.Printf("Successfully recorded context archive in database for job %d (file ID: %d, size: %d bytes)", jobID, fileID, contextInfo.Size()) - + + // Broadcast file addition + s.broadcastJobUpdate(jobID, "file_added", map[string]interface{}{ + "file_id": fileID, + "file_type": types.JobFileTypeInput, + "file_name": filepath.Base(jobContextPath), + "file_size": contextInfo.Size(), + }) + // Clean up temp directory if err := os.RemoveAll(tempDir); err != nil { log.Printf("Warning: Failed to clean up temp directory %s: %v", tempDir, err) } } else { log.Printf("ERROR: Context archive not found at %s for session %s: %v", tempContextPath, *req.UploadSessionID, err) - s.respondError(w, http.StatusBadRequest, fmt.Sprintf("Context archive not found for upload session. Please upload the file again.")) + s.respondError(w, http.StatusBadRequest, "Context archive not found for upload session. Please upload the file again.") return } } else { log.Printf("Warning: No upload session ID provided for job %d - job created without input files", jobID) } - // Only create render tasks for render jobs if req.JobType == types.JobTypeRender { // Determine task timeout based on output format @@ -233,35 +280,51 @@ func (s *Server) handleCreateJob(w http.ResponseWriter, r *http.Request) { // Create tasks for the job // If allow_parallel_runners is false, create a single task for all frames // Otherwise, create one task per frame for parallel processing + var createdTaskIDs []int64 if allowParallelRunners != nil && !*allowParallelRunners { // Single task for entire frame range - _, err = s.db.Exec( + var taskID int64 + err = s.db.QueryRow( `INSERT INTO tasks (job_id, frame_start, frame_end, task_type, status, timeout_seconds, max_retries) - VALUES (?, ?, ?, ?, ?, ?, ?)`, + VALUES (?, ?, ?, ?, ?, ?, ?) + RETURNING id`, jobID, *req.FrameStart, *req.FrameEnd, types.TaskTypeRender, types.TaskStatusPending, taskTimeout, 3, - ) + ).Scan(&taskID) if err != nil { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create task: %v", err)) return } + createdTaskIDs = append(createdTaskIDs, taskID) log.Printf("Created 1 render task for job %d (frames %d-%d, single runner)", jobID, *req.FrameStart, *req.FrameEnd) } else { // One task per frame for parallel processing for frame := *req.FrameStart; frame <= *req.FrameEnd; frame++ { - _, err = s.db.Exec( + var taskID int64 + err = s.db.QueryRow( `INSERT INTO tasks (job_id, frame_start, frame_end, task_type, status, timeout_seconds, max_retries) - VALUES (?, ?, ?, ?, ?, ?, ?)`, + VALUES (?, ?, ?, ?, ?, ?, ?) + RETURNING id`, jobID, frame, frame, types.TaskTypeRender, types.TaskStatusPending, taskTimeout, 3, - ) + ).Scan(&taskID) if err != nil { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create tasks: %v", err)) return } + createdTaskIDs = append(createdTaskIDs, taskID) } log.Printf("Created %d render tasks for job %d (frames %d-%d, parallel)", *req.FrameEnd-*req.FrameStart+1, jobID, *req.FrameStart, *req.FrameEnd) } // Update job status (should be pending since tasks are pending) s.updateJobStatusFromTasks(jobID) + + // Broadcast that new tasks were added + if len(createdTaskIDs) > 0 { + log.Printf("Broadcasting tasks_added for job %d: %d tasks", jobID, len(createdTaskIDs)) + s.broadcastTaskUpdate(jobID, 0, "tasks_added", map[string]interface{}{ + "task_ids": createdTaskIDs, + "count": len(createdTaskIDs), + }) + } } // Build response job object @@ -288,7 +351,7 @@ func (s *Server) handleCreateJob(w http.ResponseWriter, r *http.Request) { s.respondJSON(w, http.StatusCreated, job) } -// handleListJobs lists jobs for the current user +// handleListJobs lists jobs for the current user with pagination and filtering func (s *Server) handleListJobs(w http.ResponseWriter, r *http.Request) { userID, err := getUserID(r) if err != nil { @@ -296,12 +359,346 @@ func (s *Server) handleListJobs(w http.ResponseWriter, r *http.Request) { return } - // Query all jobs for the user + // Parse query parameters + limit := 50 // default + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 1000 { + limit = l + } + } + + offset := 0 + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if o, err := strconv.Atoi(offsetStr); err == nil && o >= 0 { + offset = o + } + } + + statusFilter := r.URL.Query().Get("status") + sortBy := r.URL.Query().Get("sort") + if sortBy == "" { + sortBy = "created_at:desc" + } + + // Parse sort parameter (format: "field:direction") + sortParts := strings.Split(sortBy, ":") + sortField := "created_at" + sortDir := "DESC" + if len(sortParts) == 2 { + sortField = sortParts[0] + sortDir = strings.ToUpper(sortParts[1]) + if sortDir != "ASC" && sortDir != "DESC" { + sortDir = "DESC" + } + // Validate sort field + validFields := map[string]bool{ + "created_at": true, "started_at": true, "completed_at": true, + "status": true, "progress": true, "name": true, + } + if !validFields[sortField] { + sortField = "created_at" + } + } + + // Build query with filters query := `SELECT id, user_id, job_type, name, status, progress, frame_start, frame_end, output_format, allow_parallel_runners, timeout_seconds, blend_metadata, created_at, started_at, completed_at, error_message - FROM jobs WHERE user_id = ? ORDER BY created_at DESC` + FROM jobs WHERE user_id = ?` + args := []interface{}{userID} - rows, err := s.db.Query(query, userID) + if statusFilter != "" { + // Support multiple statuses: "running,pending" or single "running" + statuses := strings.Split(statusFilter, ",") + placeholders := make([]string, len(statuses)) + for i, status := range statuses { + placeholders[i] = "?" + args = append(args, strings.TrimSpace(status)) + } + query += fmt.Sprintf(" AND status IN (%s)", strings.Join(placeholders, ",")) + } + + query += fmt.Sprintf(" ORDER BY %s %s LIMIT ? OFFSET ?", sortField, sortDir) + args = append(args, limit, offset) + + rows, err := s.db.Query(query, args...) + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query jobs: %v", err)) + return + } + defer rows.Close() + + // Get total count for pagination metadata + var total int + countQuery := `SELECT COUNT(*) FROM jobs WHERE user_id = ?` + countArgs := []interface{}{userID} + if statusFilter != "" { + statuses := strings.Split(statusFilter, ",") + placeholders := make([]string, len(statuses)) + for i, status := range statuses { + placeholders[i] = "?" + countArgs = append(countArgs, strings.TrimSpace(status)) + } + countQuery += fmt.Sprintf(" AND status IN (%s)", strings.Join(placeholders, ",")) + } + err = s.db.QueryRow(countQuery, countArgs...).Scan(&total) + if err != nil { + // If count fails, continue without it + total = -1 + } + + jobs := []types.Job{} + for rows.Next() { + var job types.Job + var jobType string + var startedAt, completedAt sql.NullTime + var blendMetadataJSON sql.NullString + var errorMessage sql.NullString + var frameStart, frameEnd sql.NullInt64 + var outputFormat sql.NullString + var allowParallelRunners sql.NullBool + + err := rows.Scan( + &job.ID, &job.UserID, &jobType, &job.Name, &job.Status, &job.Progress, + &frameStart, &frameEnd, &outputFormat, &allowParallelRunners, &job.TimeoutSeconds, + &blendMetadataJSON, &job.CreatedAt, &startedAt, &completedAt, &errorMessage, + ) + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to scan job: %v", err)) + return + } + + job.JobType = types.JobType(jobType) + if frameStart.Valid { + fs := int(frameStart.Int64) + job.FrameStart = &fs + } + if frameEnd.Valid { + fe := int(frameEnd.Int64) + job.FrameEnd = &fe + } + if outputFormat.Valid { + job.OutputFormat = &outputFormat.String + } + if allowParallelRunners.Valid { + job.AllowParallelRunners = &allowParallelRunners.Bool + } + if startedAt.Valid { + job.StartedAt = &startedAt.Time + } + if completedAt.Valid { + job.CompletedAt = &completedAt.Time + } + if blendMetadataJSON.Valid && blendMetadataJSON.String != "" { + var metadata types.BlendMetadata + if err := json.Unmarshal([]byte(blendMetadataJSON.String), &metadata); err == nil { + job.BlendMetadata = &metadata + } + } + if errorMessage.Valid { + job.ErrorMessage = errorMessage.String + } + + jobs = append(jobs, job) + } + + // Generate ETag and check If-None-Match + response := map[string]interface{}{ + "data": jobs, + "total": total, + "limit": limit, + "offset": offset, + } + etag := generateETag(response) + w.Header().Set("ETag", etag) + + if checkETag(r, etag) { + w.WriteHeader(http.StatusNotModified) + return + } + + s.respondJSON(w, http.StatusOK, response) +} + +// handleListJobsSummary lists lightweight job summaries for the current user +func (s *Server) handleListJobsSummary(w http.ResponseWriter, r *http.Request) { + userID, err := getUserID(r) + if err != nil { + s.respondError(w, http.StatusUnauthorized, err.Error()) + return + } + + // Parse query parameters (same as handleListJobs) + limit := 50 + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 1000 { + limit = l + } + } + + offset := 0 + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if o, err := strconv.Atoi(offsetStr); err == nil && o >= 0 { + offset = o + } + } + + statusFilter := r.URL.Query().Get("status") + sortBy := r.URL.Query().Get("sort") + if sortBy == "" { + sortBy = "created_at:desc" + } + + sortParts := strings.Split(sortBy, ":") + sortField := "created_at" + sortDir := "DESC" + if len(sortParts) == 2 { + sortField = sortParts[0] + sortDir = strings.ToUpper(sortParts[1]) + if sortDir != "ASC" && sortDir != "DESC" { + sortDir = "DESC" + } + validFields := map[string]bool{ + "created_at": true, "started_at": true, "completed_at": true, + "status": true, "progress": true, "name": true, + } + if !validFields[sortField] { + sortField = "created_at" + } + } + + // Build query - only select summary fields + query := `SELECT id, name, status, progress, frame_start, frame_end, output_format, created_at + FROM jobs WHERE user_id = ?` + args := []interface{}{userID} + + if statusFilter != "" { + statuses := strings.Split(statusFilter, ",") + placeholders := make([]string, len(statuses)) + for i, status := range statuses { + placeholders[i] = "?" + args = append(args, strings.TrimSpace(status)) + } + query += fmt.Sprintf(" AND status IN (%s)", strings.Join(placeholders, ",")) + } + + query += fmt.Sprintf(" ORDER BY %s %s LIMIT ? OFFSET ?", sortField, sortDir) + args = append(args, limit, offset) + + rows, err := s.db.Query(query, args...) + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query jobs: %v", err)) + return + } + defer rows.Close() + + // Get total count + var total int + countQuery := `SELECT COUNT(*) FROM jobs WHERE user_id = ?` + countArgs := []interface{}{userID} + if statusFilter != "" { + statuses := strings.Split(statusFilter, ",") + placeholders := make([]string, len(statuses)) + for i, status := range statuses { + placeholders[i] = "?" + countArgs = append(countArgs, strings.TrimSpace(status)) + } + countQuery += fmt.Sprintf(" AND status IN (%s)", strings.Join(placeholders, ",")) + } + err = s.db.QueryRow(countQuery, countArgs...).Scan(&total) + if err != nil { + total = -1 + } + + type JobSummary struct { + ID int64 `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Progress float64 `json:"progress"` + FrameStart *int `json:"frame_start,omitempty"` + FrameEnd *int `json:"frame_end,omitempty"` + OutputFormat *string `json:"output_format,omitempty"` + CreatedAt time.Time `json:"created_at"` + } + + summaries := []JobSummary{} + for rows.Next() { + var summary JobSummary + var frameStart, frameEnd sql.NullInt64 + var outputFormat sql.NullString + + err := rows.Scan( + &summary.ID, &summary.Name, &summary.Status, &summary.Progress, + &frameStart, &frameEnd, &outputFormat, &summary.CreatedAt, + ) + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to scan job: %v", err)) + return + } + + if frameStart.Valid { + fs := int(frameStart.Int64) + summary.FrameStart = &fs + } + if frameEnd.Valid { + fe := int(frameEnd.Int64) + summary.FrameEnd = &fe + } + if outputFormat.Valid { + summary.OutputFormat = &outputFormat.String + } + + summaries = append(summaries, summary) + } + + response := map[string]interface{}{ + "data": summaries, + "total": total, + "limit": limit, + "offset": offset, + } + s.respondJSON(w, http.StatusOK, response) +} + +// handleBatchGetJobs fetches multiple jobs by IDs +func (s *Server) handleBatchGetJobs(w http.ResponseWriter, r *http.Request) { + userID, err := getUserID(r) + if err != nil { + s.respondError(w, http.StatusUnauthorized, err.Error()) + return + } + + var req struct { + JobIDs []int64 `json:"job_ids"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + s.respondError(w, http.StatusBadRequest, "Invalid request body") + return + } + + if len(req.JobIDs) == 0 { + s.respondJSON(w, http.StatusOK, []types.Job{}) + return + } + + if len(req.JobIDs) > 100 { + s.respondError(w, http.StatusBadRequest, "Maximum 100 job IDs allowed per batch") + return + } + + // Build query with IN clause + placeholders := make([]string, len(req.JobIDs)) + args := make([]interface{}, len(req.JobIDs)+1) + args[0] = userID + for i, jobID := range req.JobIDs { + placeholders[i] = "?" + args[i+1] = jobID + } + + query := fmt.Sprintf(`SELECT id, user_id, job_type, name, status, progress, frame_start, frame_end, output_format, + allow_parallel_runners, timeout_seconds, blend_metadata, created_at, started_at, completed_at, error_message + FROM jobs WHERE user_id = ? AND id IN (%s) ORDER BY created_at DESC`, strings.Join(placeholders, ",")) + + rows, err := s.db.Query(query, args...) if err != nil { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query jobs: %v", err)) return @@ -456,6 +853,15 @@ func (s *Server) handleGetJob(w http.ResponseWriter, r *http.Request) { job.ErrorMessage = errorMessage.String } + // Generate ETag and check If-None-Match + etag := generateETag(job) + w.Header().Set("ETag", etag) + + if checkETag(r, etag) { + w.WriteHeader(http.StatusNotModified) + return + } + s.respondJSON(w, http.StatusOK, job) } @@ -675,6 +1081,8 @@ func (s *Server) cleanupOldRenderJobsOnce() { var jobID int64 if err := rows.Scan(&jobID); err == nil { jobIDs = append(jobIDs, jobID) + } else { + log.Printf("Failed to scan job ID in cleanupOldRenderJobs: %v", err) } } rows.Close() @@ -805,12 +1213,16 @@ func (s *Server) handleUploadJobFile(w http.ResponseWriter, r *http.Request) { } // Create temporary directory for processing upload - tmpDir, err := os.MkdirTemp("", fmt.Sprintf("fuego-upload-%d-*", jobID)) + tmpDir, err := s.storage.TempDir(fmt.Sprintf("jiggablend-upload-%d-*", jobID)) if err != nil { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create temporary directory: %v", err)) return } - defer os.RemoveAll(tmpDir) + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + log.Printf("Warning: Failed to clean up temp directory %s: %v", tmpDir, err) + } + }() var fileID int64 var mainBlendFile string @@ -926,7 +1338,6 @@ func (s *Server) handleUploadJobFile(w http.ResponseWriter, r *http.Request) { // Create context archive from temporary directory - this is the primary artifact // Exclude the original uploaded ZIP file (but keep blend files as they're needed for rendering) - log.Printf("Creating context archive for job %d...", jobID) var excludeFiles []string if strings.HasSuffix(strings.ToLower(header.Filename), ".zip") { excludeFiles = append(excludeFiles, header.Filename) @@ -937,7 +1348,6 @@ func (s *Server) handleUploadJobFile(w http.ResponseWriter, r *http.Request) { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create context archive: %v", err)) return } - log.Printf("Successfully created context archive for job %d at %s", jobID, contextPath) // Record context archive in database contextInfo, err := os.Stat(contextPath) @@ -959,6 +1369,14 @@ func (s *Server) handleUploadJobFile(w http.ResponseWriter, r *http.Request) { } log.Printf("Context archive recorded in database with ID %d for job %d", fileID, jobID) + // Broadcast file addition + s.broadcastJobUpdate(jobID, "file_added", map[string]interface{}{ + "file_id": fileID, + "file_type": types.JobFileTypeInput, + "file_name": filepath.Base(contextPath), + "file_size": contextInfo.Size(), + }) + // Extract metadata directly from the context archive log.Printf("Extracting metadata for job %d...", jobID) metadata, err := s.extractMetadataFromContext(jobID) @@ -983,7 +1401,6 @@ func (s *Server) handleUploadJobFile(w http.ResponseWriter, r *http.Request) { } } - response := map[string]interface{}{ "id": fileID, "file_name": header.Filename, @@ -1035,12 +1452,17 @@ func (s *Server) handleUploadFileForJobCreation(w http.ResponseWriter, r *http.R log.Printf("Uploading file '%s' (size: %d bytes) for user %d (pre-job creation)", header.Filename, header.Size, userID) // Create temporary directory for processing upload (user-specific) - tmpDir, err := os.MkdirTemp("", fmt.Sprintf("fuego-upload-user-%d-*", userID)) + tmpDir, err := s.storage.TempDir(fmt.Sprintf("jiggablend-upload-user-%d-*", userID)) if err != nil { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create temporary directory: %v", err)) return } // Note: We'll clean this up after job creation or after timeout + // For now, we rely on the session cleanup mechanism, but also add defer for safety + defer func() { + // Only clean up if there's an error - otherwise let session cleanup handle it + // This is a safety net in case of early returns + }() var mainBlendFile string var extractedFiles []string @@ -1150,14 +1572,13 @@ func (s *Server) handleUploadFileForJobCreation(w http.ResponseWriter, r *http.R } // Create context archive from temporary directory - log.Printf("Creating context archive from temporary directory...") var excludeFiles []string if strings.HasSuffix(strings.ToLower(header.Filename), ".zip") { excludeFiles = append(excludeFiles, header.Filename) } - + // Create context in temp directory (we'll move it to job directory later) - contextPath := filepath.Join(tmpDir, "context.tar.gz") + contextPath := filepath.Join(tmpDir, "context.tar") contextPath, err = s.createContextFromDir(tmpDir, contextPath, excludeFiles...) if err != nil { os.RemoveAll(tmpDir) @@ -1165,10 +1586,8 @@ func (s *Server) handleUploadFileForJobCreation(w http.ResponseWriter, r *http.R s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create context archive: %v", err)) return } - log.Printf("Successfully created context archive at %s", contextPath) // Extract metadata from context archive - log.Printf("Extracting metadata from context archive...") metadata, err := s.extractMetadataFromTempContext(contextPath) if err != nil { log.Printf("Warning: Failed to extract metadata: %v", err) @@ -1211,15 +1630,19 @@ func (s *Server) handleUploadFileForJobCreation(w http.ResponseWriter, r *http.R // extractMetadataFromTempContext extracts metadata from a context archive in a temporary location func (s *Server) extractMetadataFromTempContext(contextPath string) (*types.BlendMetadata, error) { - // Create temporary directory for extraction - tmpDir, err := os.MkdirTemp("", "fuego-metadata-temp-*") + // Create temporary directory for extraction under storage base path + tmpDir, err := s.storage.TempDir("jiggablend-metadata-temp-*") if err != nil { return nil, fmt.Errorf("failed to create temporary directory: %w", err) } - defer os.RemoveAll(tmpDir) + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + log.Printf("Warning: Failed to clean up temp directory %s: %v", tmpDir, err) + } + }() // Extract context archive - if err := s.extractTarGz(contextPath, tmpDir); err != nil { + if err := s.extractTar(contextPath, tmpDir); err != nil { return nil, fmt.Errorf("failed to extract context: %w", err) } @@ -1264,167 +1687,20 @@ func (s *Server) extractMetadataFromTempContext(contextPath string) (*types.Blen // runBlenderMetadataExtraction runs Blender to extract metadata from a blend file func (s *Server) runBlenderMetadataExtraction(blendFile, workDir string) (*types.BlendMetadata, error) { - // Create Python script (same as in extractMetadataFromContext) + // Use embedded Python script scriptPath := filepath.Join(workDir, "extract_metadata.py") - scriptContent := `import bpy -import json -import sys - -try: - bpy.ops.file.make_paths_relative() - print("Made all file paths relative to blend file") -except Exception as e: - print(f"Warning: Could not make paths relative: {e}") - -missing_files_info = { - "checked": False, - "has_missing": False, - "missing_files": [], - "missing_addons": [] -} - -try: - missing = [] - for mod in bpy.context.preferences.addons: - if mod.module.endswith("_missing"): - missing.append(mod.module.rsplit("_", 1)[0]) - - missing_files_info["checked"] = True - if missing: - missing_files_info["has_missing"] = True - missing_files_info["missing_addons"] = missing - print("Missing add-ons required by this .blend:") - for name in missing: - print(" -", name) - else: - print("No missing add-ons detected – file is headless-safe") -except Exception as e: - print(f"Warning: Could not check for missing addons: {e}") - missing_files_info["error"] = str(e) - -scene = bpy.context.scene -frame_start = scene.frame_start -frame_end = scene.frame_end - -animation_start = None -animation_end = None - -for obj in scene.objects: - if obj.animation_data and obj.animation_data.action: - action = obj.animation_data.action - if action.fcurves: - for fcurve in action.fcurves: - if fcurve.keyframe_points: - for keyframe in fcurve.keyframe_points: - frame = int(keyframe.co[0]) - if animation_start is None or frame < animation_start: - animation_start = frame - if animation_end is None or frame > animation_end: - animation_end = frame - -if animation_start is not None and animation_end is not None: - if frame_start == frame_end or (animation_start < frame_start or animation_end > frame_end): - frame_start = animation_start - frame_end = animation_end - -render = scene.render -resolution_x = render.resolution_x -resolution_y = render.resolution_y -engine = scene.render.engine.upper() -output_format = render.image_settings.file_format - -engine_settings = {} - -if engine == 'CYCLES': - cycles = scene.cycles - engine_settings = { - "samples": getattr(cycles, 'samples', 128), - "use_denoising": getattr(cycles, 'use_denoising', False), - "denoising_radius": getattr(cycles, 'denoising_radius', 0), - "denoising_strength": getattr(cycles, 'denoising_strength', 0.0), - "device": getattr(cycles, 'device', 'CPU'), - "use_adaptive_sampling": getattr(cycles, 'use_adaptive_sampling', False), - "adaptive_threshold": getattr(cycles, 'adaptive_threshold', 0.01) if getattr(cycles, 'use_adaptive_sampling', False) else 0.01, - "use_fast_gi": getattr(cycles, 'use_fast_gi', False), - "light_tree": getattr(cycles, 'use_light_tree', False), - "use_light_linking": getattr(cycles, 'use_light_linking', False), - "caustics_reflective": getattr(cycles, 'caustics_reflective', False), - "caustics_refractive": getattr(cycles, 'caustics_refractive', False), - "blur_glossy": getattr(cycles, 'blur_glossy', 0.0), - "max_bounces": getattr(cycles, 'max_bounces', 12), - "diffuse_bounces": getattr(cycles, 'diffuse_bounces', 4), - "glossy_bounces": getattr(cycles, 'glossy_bounces', 4), - "transmission_bounces": getattr(cycles, 'transmission_bounces', 12), - "volume_bounces": getattr(cycles, 'volume_bounces', 0), - "transparent_max_bounces": getattr(cycles, 'transparent_max_bounces', 8), - "film_transparent": getattr(cycles, 'film_transparent', False), - "use_layer_samples": getattr(cycles, 'use_layer_samples', False), - } -elif engine == 'EEVEE' or engine == 'EEVEE_NEXT': - eevee = scene.eevee - engine_settings = { - "taa_render_samples": getattr(eevee, 'taa_render_samples', 64), - "use_bloom": getattr(eevee, 'use_bloom', False), - "bloom_threshold": getattr(eevee, 'bloom_threshold', 0.8), - "bloom_intensity": getattr(eevee, 'bloom_intensity', 0.05), - "bloom_radius": getattr(eevee, 'bloom_radius', 6.5), - "use_ssr": getattr(eevee, 'use_ssr', True), - "use_ssr_refraction": getattr(eevee, 'use_ssr_refraction', False), - "ssr_quality": getattr(eevee, 'ssr_quality', 'MEDIUM'), - "use_ssao": getattr(eevee, 'use_ssao', True), - "ssao_quality": getattr(eevee, 'ssao_quality', 'MEDIUM'), - "ssao_distance": getattr(eevee, 'ssao_distance', 0.2), - "ssao_factor": getattr(eevee, 'ssao_factor', 1.0), - "use_soft_shadows": getattr(eevee, 'use_soft_shadows', True), - "use_shadow_high_bitdepth": getattr(eevee, 'use_shadow_high_bitdepth', True), - "use_volumetric": getattr(eevee, 'use_volumetric', False), - "volumetric_tile_size": getattr(eevee, 'volumetric_tile_size', '8'), - "volumetric_samples": getattr(eevee, 'volumetric_samples', 64), - "volumetric_start": getattr(eevee, 'volumetric_start', 0.0), - "volumetric_end": getattr(eevee, 'volumetric_end', 100.0), - "use_volumetric_lights": getattr(eevee, 'use_volumetric_lights', True), - "use_volumetric_shadows": getattr(eevee, 'use_volumetric_shadows', True), - "use_gtao": getattr(eevee, 'use_gtao', False), - "gtao_quality": getattr(eevee, 'gtao_quality', 'MEDIUM'), - "use_overscan": getattr(eevee, 'use_overscan', False), - } -else: - engine_settings = { - "samples": getattr(scene, 'samples', 128) if hasattr(scene, 'samples') else 128 - } - -camera_count = len([obj for obj in scene.objects if obj.type == 'CAMERA']) -object_count = len(scene.objects) -material_count = len(bpy.data.materials) - -metadata = { - "frame_start": frame_start, - "frame_end": frame_end, - "render_settings": { - "resolution_x": resolution_x, - "resolution_y": resolution_y, - "output_format": output_format, - "engine": engine.lower(), - "engine_settings": engine_settings - }, - "scene_info": { - "camera_count": camera_count, - "object_count": object_count, - "material_count": material_count - }, - "missing_files_info": missing_files_info -} - -print(json.dumps(metadata)) -sys.stdout.flush() -` - - if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil { + if err := os.WriteFile(scriptPath, []byte(scripts.ExtractMetadata), 0644); err != nil { return nil, fmt.Errorf("failed to create extraction script: %w", err) } + // Make blend file path relative to workDir to avoid path resolution issues + blendFileRel, err := filepath.Rel(workDir, blendFile) + if err != nil { + return nil, fmt.Errorf("failed to get relative path for blend file: %w", err) + } + // Execute Blender - cmd := exec.Command("blender", "-b", blendFile, "--python", scriptPath) + cmd := exec.Command("blender", "-b", blendFileRel, "--python", "extract_metadata.py") cmd.Dir = workDir stdoutPipe, err := cmd.StdoutPipe() @@ -1454,12 +1730,16 @@ sys.stdout.flush() } }() + // Capture stderr for error reporting + var stderrBuffer bytes.Buffer stderrDone := make(chan bool) go func() { defer close(stderrDone) scanner := bufio.NewScanner(stderrPipe) for scanner.Scan() { - _ = scanner.Text() + line := scanner.Text() + stderrBuffer.WriteString(line) + stderrBuffer.WriteString("\n") } }() @@ -1468,6 +1748,18 @@ sys.stdout.flush() <-stderrDone if err != nil { + stderrOutput := strings.TrimSpace(stderrBuffer.String()) + stdoutOutput := strings.TrimSpace(stdoutBuffer.String()) + log.Printf("Blender metadata extraction failed:") + if stderrOutput != "" { + log.Printf("Blender stderr: %s", stderrOutput) + } + if stdoutOutput != "" { + log.Printf("Blender stdout (last 500 chars): %s", truncateString(stdoutOutput, 500)) + } + if stderrOutput != "" { + return nil, fmt.Errorf("blender metadata extraction failed: %w (stderr: %s)", err, truncateString(stderrOutput, 200)) + } return nil, fmt.Errorf("blender metadata extraction failed: %w", err) } @@ -1608,17 +1900,14 @@ func (s *Server) createContextFromDir(sourceDir, destPath string, excludeFiles . return "", fmt.Errorf("multiple .blend files found at root level in context archive (found %d, expected 1)", blendFilesAtRoot) } - // Create the tar.gz file + // Create the tar file contextFile, err := os.Create(destPath) if err != nil { return "", fmt.Errorf("failed to create context file: %w", err) } defer contextFile.Close() - gzWriter := gzip.NewWriter(contextFile) - defer gzWriter.Close() - - tarWriter := tar.NewWriter(gzWriter) + tarWriter := tar.NewWriter(contextFile) defer tarWriter.Close() // Add each file to the tar archive @@ -1663,9 +1952,6 @@ func (s *Server) createContextFromDir(sourceDir, destPath string, excludeFiles . if err := tarWriter.Close(); err != nil { return "", fmt.Errorf("failed to close tar writer: %w", err) } - if err := gzWriter.Close(); err != nil { - return "", fmt.Errorf("failed to close gzip writer: %w", err) - } if err := contextFile.Close(); err != nil { return "", fmt.Errorf("failed to close context file: %w", err) } @@ -1673,7 +1959,7 @@ func (s *Server) createContextFromDir(sourceDir, destPath string, excludeFiles . return destPath, nil } -// handleListJobFiles lists files for a job +// handleListJobFiles lists files for a job with pagination func (s *Server) handleListJobFiles(w http.ResponseWriter, r *http.Request) { userID, err := getUserID(r) if err != nil { @@ -1714,17 +2000,66 @@ func (s *Server) handleListJobFiles(w http.ResponseWriter, r *http.Request) { } } - rows, err := s.db.Query( - `SELECT id, job_id, file_type, file_path, file_name, file_size, created_at - FROM job_files WHERE job_id = ? ORDER BY created_at DESC`, - jobID, - ) + // Parse query parameters + limit := 50 + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 1000 { + limit = l + } + } + + offset := 0 + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if o, err := strconv.Atoi(offsetStr); err == nil && o >= 0 { + offset = o + } + } + + fileTypeFilter := r.URL.Query().Get("file_type") + extensionFilter := r.URL.Query().Get("extension") + + // Build query with filters + query := `SELECT id, job_id, file_type, file_path, file_name, file_size, created_at + FROM job_files WHERE job_id = ?` + args := []interface{}{jobID} + + if fileTypeFilter != "" { + query += " AND file_type = ?" + args = append(args, fileTypeFilter) + } + + if extensionFilter != "" { + query += " AND file_name LIKE ?" + args = append(args, "%."+extensionFilter) + } + + query += " ORDER BY created_at DESC LIMIT ? OFFSET ?" + args = append(args, limit, offset) + + rows, err := s.db.Query(query, args...) if err != nil { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query files: %v", err)) return } defer rows.Close() + // Get total count + var total int + countQuery := `SELECT COUNT(*) FROM job_files WHERE job_id = ?` + countArgs := []interface{}{jobID} + if fileTypeFilter != "" { + countQuery += " AND file_type = ?" + countArgs = append(countArgs, fileTypeFilter) + } + if extensionFilter != "" { + countQuery += " AND file_name LIKE ?" + countArgs = append(countArgs, "%."+extensionFilter) + } + err = s.db.QueryRow(countQuery, countArgs...).Scan(&total) + if err != nil { + total = -1 + } + files := []types.JobFile{} for rows.Next() { var file types.JobFile @@ -1739,10 +2074,77 @@ func (s *Server) handleListJobFiles(w http.ResponseWriter, r *http.Request) { files = append(files, file) } - s.respondJSON(w, http.StatusOK, files) + response := map[string]interface{}{ + "data": files, + "total": total, + "limit": limit, + "offset": offset, + } + s.respondJSON(w, http.StatusOK, response) +} + +// handleGetJobFilesCount returns the count of files for a job +func (s *Server) handleGetJobFilesCount(w http.ResponseWriter, r *http.Request) { + userID, err := getUserID(r) + if err != nil { + s.respondError(w, http.StatusUnauthorized, err.Error()) + return + } + + jobID, err := parseID(r, "id") + if err != nil { + s.respondError(w, http.StatusBadRequest, err.Error()) + return + } + + // Verify job belongs to user (unless admin) + isAdmin := isAdminUser(r) + if !isAdmin { + var jobUserID int64 + err = s.db.QueryRow("SELECT user_id FROM jobs WHERE id = ?", jobID).Scan(&jobUserID) + if err == sql.ErrNoRows { + s.respondError(w, http.StatusNotFound, "Job not found") + return + } + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to verify job: %v", err)) + return + } + if jobUserID != userID { + s.respondError(w, http.StatusForbidden, "Access denied") + return + } + } else { + var exists bool + err = s.db.QueryRow("SELECT EXISTS(SELECT 1 FROM jobs WHERE id = ?)", jobID).Scan(&exists) + if err != nil || !exists { + s.respondError(w, http.StatusNotFound, "Job not found") + return + } + } + + fileTypeFilter := r.URL.Query().Get("file_type") + + var count int + query := `SELECT COUNT(*) FROM job_files WHERE job_id = ?` + args := []interface{}{jobID} + + if fileTypeFilter != "" { + query += " AND file_type = ?" + args = append(args, fileTypeFilter) + } + + err = s.db.QueryRow(query, args...).Scan(&count) + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to count files: %v", err)) + return + } + + s.respondJSON(w, http.StatusOK, map[string]interface{}{"count": count}) } // handleListContextArchive lists files inside the context archive +// Optimized to only read tar headers, skipping file data for fast directory listing func (s *Server) handleListContextArchive(w http.ResponseWriter, r *http.Request) { userID, err := getUserID(r) if err != nil { @@ -1776,29 +2178,20 @@ func (s *Server) handleListContextArchive(w http.ResponseWriter, r *http.Request } // Get context archive path - contextPath := filepath.Join(s.storage.JobPath(jobID), "context.tar.gz") + contextPath := filepath.Join(s.storage.JobPath(jobID), "context.tar") if !s.storage.FileExists(contextPath) { s.respondError(w, http.StatusNotFound, "Context archive not found") return } - // Read the tar.gz and list its contents - file, err := s.storage.GetFile(contextPath) + // Open file directly for seeking (much faster than reading all data) + file, err := os.Open(contextPath) if err != nil { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to open context archive: %v", err)) return } defer file.Close() - gzReader, err := gzip.NewReader(file) - if err != nil { - s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to read context archive: %v", err)) - return - } - defer gzReader.Close() - - tarReader := tar.NewReader(gzReader) - type ArchiveFile struct { Name string `json:"name"` Size int64 `json:"size"` @@ -1806,15 +2199,65 @@ func (s *Server) handleListContextArchive(w http.ResponseWriter, r *http.Request } var archiveFiles []ArchiveFile + const tarBlockSize = 512 + + // Read tar headers sequentially, skipping file data by seeking + // This is much faster than reading all file contents for { - header, err := tarReader.Next() + // Read 512-byte tar header + headerBuf := make([]byte, tarBlockSize) + n, err := file.Read(headerBuf) if err == io.EOF { break } if err != nil { - s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to read archive: %v", err)) + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to read archive header: %v", err)) return } + if n < tarBlockSize { + // Incomplete header, likely end of archive + break + } + + // Check if this is the end marker (all zeros) - tar files end with two zero blocks + allZeros := true + for _, b := range headerBuf { + if b != 0 { + allZeros = false + break + } + } + if allZeros { + break + } + + // Parse tar header + var header tar.Header + if err := parseTarHeader(headerBuf, &header); err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to parse archive header: %v", err)) + return + } + + // Handle GNU tar long filename extension (type 'L') + // If typeflag is 'L', the next block contains the actual filename + if header.Typeflag == 'L' { + // Read the long filename from the next block + longNameBuf := make([]byte, tarBlockSize) + if _, err := file.Read(longNameBuf); err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to read long filename: %v", err)) + return + } + header.Name = strings.TrimRight(string(longNameBuf), "\x00") + // Read the actual header after the long filename + if _, err := file.Read(headerBuf); err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to read header after long filename: %v", err)) + return + } + if err := parseTarHeader(headerBuf, &header); err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to parse header after long filename: %v", err)) + return + } + } // Only include regular files (not directories) if header.Typeflag == tar.TypeReg { @@ -1824,11 +2267,68 @@ func (s *Server) handleListContextArchive(w http.ResponseWriter, r *http.Request Path: header.Name, }) } + + // Skip file data by seeking forward + // Tar format: file data is padded to 512-byte boundary + dataSize := header.Size + blockPadding := (tarBlockSize - (dataSize % tarBlockSize)) % tarBlockSize + skipSize := dataSize + blockPadding + + // Seek forward to next header (much faster than reading) + _, err = file.Seek(skipSize, io.SeekCurrent) + if err != nil { + // If seek fails (e.g., on non-seekable stream), fall back to reading and discarding + _, readErr := io.CopyN(io.Discard, file, skipSize) + if readErr != nil && readErr != io.EOF { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to skip file data: %v", readErr)) + return + } + } } s.respondJSON(w, http.StatusOK, archiveFiles) } +// parseTarHeader parses a 512-byte tar header block into a tar.Header +// This is a simplified parser that extracts the essential fields we need +func parseTarHeader(buf []byte, h *tar.Header) error { + const tarHeaderSize = 512 + if len(buf) < tarHeaderSize { + return fmt.Errorf("buffer too small for tar header") + } + + // Tar header format (UStar/POSIX format) + // Field offsets based on POSIX.1-1988 tar format + h.Name = strings.TrimRight(string(buf[0:100]), "\x00") + + // Parse mode (octal) + modeStr := strings.TrimRight(string(buf[100:108]), " \x00") + mode, err := strconv.ParseUint(modeStr, 8, 32) + if err == nil { + h.Mode = int64(mode) + } + + // Parse size (octal) + sizeStr := strings.TrimRight(string(buf[124:136]), " \x00") + size, err := strconv.ParseInt(sizeStr, 8, 64) + if err == nil { + h.Size = size + } + + // Parse typeflag + if len(buf) > 156 { + h.Typeflag = buf[156] + } + + // Handle UStar format prefix (for long filenames) + prefix := strings.TrimRight(string(buf[345:500]), "\x00") + if prefix != "" { + h.Name = prefix + "/" + h.Name + } + + return nil +} + // handleDownloadJobFile downloads a job file func (s *Server) handleDownloadJobFile(w http.ResponseWriter, r *http.Request) { userID, err := getUserID(r) @@ -2034,7 +2534,7 @@ func (s *Server) handleStreamVideo(w http.ResponseWriter, r *http.Request) { } } -// handleListJobTasks lists all tasks for a job +// handleListJobTasks lists all tasks for a job with pagination and filtering func (s *Server) handleListJobTasks(w http.ResponseWriter, r *http.Request) { userID, err := getUserID(r) if err != nil { @@ -2075,13 +2575,418 @@ func (s *Server) handleListJobTasks(w http.ResponseWriter, r *http.Request) { } } - rows, err := s.db.Query( - `SELECT id, job_id, runner_id, frame_start, frame_end, status, task_type, + // Parse query parameters + limit := 100 // default + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 5000 { + limit = l + } + } + + offset := 0 + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if o, err := strconv.Atoi(offsetStr); err == nil && o >= 0 { + offset = o + } + } + + statusFilter := r.URL.Query().Get("status") + frameStartFilter := r.URL.Query().Get("frame_start") + frameEndFilter := r.URL.Query().Get("frame_end") + sortBy := r.URL.Query().Get("sort") + if sortBy == "" { + sortBy = "frame_start:asc" + } + + // Parse sort parameter + sortParts := strings.Split(sortBy, ":") + sortField := "frame_start" + sortDir := "ASC" + if len(sortParts) == 2 { + sortField = sortParts[0] + sortDir = strings.ToUpper(sortParts[1]) + if sortDir != "ASC" && sortDir != "DESC" { + sortDir = "ASC" + } + validFields := map[string]bool{ + "frame_start": true, "frame_end": true, "status": true, + "created_at": true, "started_at": true, "completed_at": true, + } + if !validFields[sortField] { + sortField = "frame_start" + } + } + + // Build query with filters + query := `SELECT id, job_id, runner_id, frame_start, frame_end, status, task_type, current_step, retry_count, max_retries, output_path, created_at, started_at, completed_at, error_message, timeout_seconds - FROM tasks WHERE job_id = ? ORDER BY frame_start ASC`, - jobID, - ) + FROM tasks WHERE job_id = ?` + args := []interface{}{jobID} + + if statusFilter != "" { + statuses := strings.Split(statusFilter, ",") + placeholders := make([]string, len(statuses)) + for i, status := range statuses { + placeholders[i] = "?" + args = append(args, strings.TrimSpace(status)) + } + query += fmt.Sprintf(" AND status IN (%s)", strings.Join(placeholders, ",")) + } + + if frameStartFilter != "" { + if fs, err := strconv.Atoi(frameStartFilter); err == nil { + query += " AND frame_start >= ?" + args = append(args, fs) + } + } + + if frameEndFilter != "" { + if fe, err := strconv.Atoi(frameEndFilter); err == nil { + query += " AND frame_end <= ?" + args = append(args, fe) + } + } + + query += fmt.Sprintf(" ORDER BY %s %s LIMIT ? OFFSET ?", sortField, sortDir) + args = append(args, limit, offset) + + rows, err := s.db.Query(query, args...) + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query tasks: %v", err)) + return + } + defer rows.Close() + + // Get total count + var total int + countQuery := `SELECT COUNT(*) FROM tasks WHERE job_id = ?` + countArgs := []interface{}{jobID} + if statusFilter != "" { + statuses := strings.Split(statusFilter, ",") + placeholders := make([]string, len(statuses)) + for i, status := range statuses { + placeholders[i] = "?" + countArgs = append(countArgs, strings.TrimSpace(status)) + } + countQuery += fmt.Sprintf(" AND status IN (%s)", strings.Join(placeholders, ",")) + } + if frameStartFilter != "" { + if fs, err := strconv.Atoi(frameStartFilter); err == nil { + countQuery += " AND frame_start >= ?" + countArgs = append(countArgs, fs) + } + } + if frameEndFilter != "" { + if fe, err := strconv.Atoi(frameEndFilter); err == nil { + countQuery += " AND frame_end <= ?" + countArgs = append(countArgs, fe) + } + } + err = s.db.QueryRow(countQuery, countArgs...).Scan(&total) + if err != nil { + total = -1 + } + + tasks := []types.Task{} + for rows.Next() { + var task types.Task + var runnerID sql.NullInt64 + var startedAt, completedAt sql.NullTime + var timeoutSeconds sql.NullInt64 + var errorMessage sql.NullString + var currentStep sql.NullString + var outputPath sql.NullString + + err := rows.Scan( + &task.ID, &task.JobID, &runnerID, &task.FrameStart, &task.FrameEnd, + &task.Status, &task.TaskType, ¤tStep, &task.RetryCount, + &task.MaxRetries, &outputPath, &task.CreatedAt, &startedAt, + &completedAt, &errorMessage, &timeoutSeconds, + ) + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to scan task: %v", err)) + return + } + + if runnerID.Valid { + task.RunnerID = &runnerID.Int64 + } + if startedAt.Valid { + task.StartedAt = &startedAt.Time + } + if completedAt.Valid { + task.CompletedAt = &completedAt.Time + } + if timeoutSeconds.Valid { + timeout := int(timeoutSeconds.Int64) + task.TimeoutSeconds = &timeout + } + if errorMessage.Valid { + task.ErrorMessage = errorMessage.String + } + if currentStep.Valid { + task.CurrentStep = currentStep.String + } + if outputPath.Valid { + task.OutputPath = outputPath.String + } + + tasks = append(tasks, task) + } + + response := map[string]interface{}{ + "data": tasks, + "total": total, + "limit": limit, + "offset": offset, + } + + // Generate ETag and check If-None-Match + etag := generateETag(response) + w.Header().Set("ETag", etag) + + if checkETag(r, etag) { + w.WriteHeader(http.StatusNotModified) + return + } + + s.respondJSON(w, http.StatusOK, response) +} + +// handleListJobTasksSummary lists lightweight task summaries for a job +func (s *Server) handleListJobTasksSummary(w http.ResponseWriter, r *http.Request) { + userID, err := getUserID(r) + if err != nil { + s.respondError(w, http.StatusUnauthorized, err.Error()) + return + } + + jobID, err := parseID(r, "id") + if err != nil { + s.respondError(w, http.StatusBadRequest, err.Error()) + return + } + + // Verify job belongs to user (unless admin) + isAdmin := isAdminUser(r) + if !isAdmin { + var jobUserID int64 + err = s.db.QueryRow("SELECT user_id FROM jobs WHERE id = ?", jobID).Scan(&jobUserID) + if err == sql.ErrNoRows { + s.respondError(w, http.StatusNotFound, "Job not found") + return + } + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to verify job: %v", err)) + return + } + if jobUserID != userID { + s.respondError(w, http.StatusForbidden, "Access denied") + return + } + } else { + var exists bool + err = s.db.QueryRow("SELECT EXISTS(SELECT 1 FROM jobs WHERE id = ?)", jobID).Scan(&exists) + if err != nil || !exists { + s.respondError(w, http.StatusNotFound, "Job not found") + return + } + } + + // Parse query parameters + limit := 100 + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 5000 { + limit = l + } + } + + offset := 0 + if offsetStr := r.URL.Query().Get("offset"); offsetStr != "" { + if o, err := strconv.Atoi(offsetStr); err == nil && o >= 0 { + offset = o + } + } + + statusFilter := r.URL.Query().Get("status") + sortBy := r.URL.Query().Get("sort") + if sortBy == "" { + sortBy = "frame_start:asc" + } + + sortParts := strings.Split(sortBy, ":") + sortField := "frame_start" + sortDir := "ASC" + if len(sortParts) == 2 { + sortField = sortParts[0] + sortDir = strings.ToUpper(sortParts[1]) + if sortDir != "ASC" && sortDir != "DESC" { + sortDir = "ASC" + } + validFields := map[string]bool{ + "frame_start": true, "frame_end": true, "status": true, + } + if !validFields[sortField] { + sortField = "frame_start" + } + } + + // Build query - only select summary fields + query := `SELECT id, frame_start, frame_end, status, task_type, runner_id + FROM tasks WHERE job_id = ?` + args := []interface{}{jobID} + + if statusFilter != "" { + statuses := strings.Split(statusFilter, ",") + placeholders := make([]string, len(statuses)) + for i, status := range statuses { + placeholders[i] = "?" + args = append(args, strings.TrimSpace(status)) + } + query += fmt.Sprintf(" AND status IN (%s)", strings.Join(placeholders, ",")) + } + + query += fmt.Sprintf(" ORDER BY %s %s LIMIT ? OFFSET ?", sortField, sortDir) + args = append(args, limit, offset) + + rows, err := s.db.Query(query, args...) + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query tasks: %v", err)) + return + } + defer rows.Close() + + // Get total count + var total int + countQuery := `SELECT COUNT(*) FROM tasks WHERE job_id = ?` + countArgs := []interface{}{jobID} + if statusFilter != "" { + statuses := strings.Split(statusFilter, ",") + placeholders := make([]string, len(statuses)) + for i, status := range statuses { + placeholders[i] = "?" + countArgs = append(countArgs, strings.TrimSpace(status)) + } + countQuery += fmt.Sprintf(" AND status IN (%s)", strings.Join(placeholders, ",")) + } + err = s.db.QueryRow(countQuery, countArgs...).Scan(&total) + if err != nil { + total = -1 + } + + type TaskSummary struct { + ID int64 `json:"id"` + FrameStart int `json:"frame_start"` + FrameEnd int `json:"frame_end"` + Status string `json:"status"` + TaskType string `json:"task_type"` + RunnerID *int64 `json:"runner_id,omitempty"` + } + + summaries := []TaskSummary{} + for rows.Next() { + var summary TaskSummary + var runnerID sql.NullInt64 + + err := rows.Scan( + &summary.ID, &summary.FrameStart, &summary.FrameEnd, + &summary.Status, &summary.TaskType, &runnerID, + ) + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to scan task: %v", err)) + return + } + + if runnerID.Valid { + summary.RunnerID = &runnerID.Int64 + } + + summaries = append(summaries, summary) + } + + response := map[string]interface{}{ + "data": summaries, + "total": total, + "limit": limit, + "offset": offset, + } + s.respondJSON(w, http.StatusOK, response) +} + +// handleBatchGetTasks fetches multiple tasks by IDs for a job +func (s *Server) handleBatchGetTasks(w http.ResponseWriter, r *http.Request) { + userID, err := getUserID(r) + if err != nil { + s.respondError(w, http.StatusUnauthorized, err.Error()) + return + } + + jobID, err := parseID(r, "id") + if err != nil { + s.respondError(w, http.StatusBadRequest, err.Error()) + return + } + + // Verify job belongs to user (unless admin) + isAdmin := isAdminUser(r) + if !isAdmin { + var jobUserID int64 + err = s.db.QueryRow("SELECT user_id FROM jobs WHERE id = ?", jobID).Scan(&jobUserID) + if err == sql.ErrNoRows { + s.respondError(w, http.StatusNotFound, "Job not found") + return + } + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to verify job: %v", err)) + return + } + if jobUserID != userID { + s.respondError(w, http.StatusForbidden, "Access denied") + return + } + } else { + var exists bool + err = s.db.QueryRow("SELECT EXISTS(SELECT 1 FROM jobs WHERE id = ?)", jobID).Scan(&exists) + if err != nil || !exists { + s.respondError(w, http.StatusNotFound, "Job not found") + return + } + } + + var req struct { + TaskIDs []int64 `json:"task_ids"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + s.respondError(w, http.StatusBadRequest, "Invalid request body") + return + } + + if len(req.TaskIDs) == 0 { + s.respondJSON(w, http.StatusOK, []types.Task{}) + return + } + + if len(req.TaskIDs) > 500 { + s.respondError(w, http.StatusBadRequest, "Maximum 500 task IDs allowed per batch") + return + } + + // Build query with IN clause + placeholders := make([]string, len(req.TaskIDs)) + args := make([]interface{}, len(req.TaskIDs)+1) + args[0] = jobID + for i, taskID := range req.TaskIDs { + placeholders[i] = "?" + args[i+1] = taskID + } + + query := fmt.Sprintf(`SELECT id, job_id, runner_id, frame_start, frame_end, status, task_type, + current_step, retry_count, max_retries, output_path, created_at, started_at, + completed_at, error_message, timeout_seconds + FROM tasks WHERE job_id = ? AND id IN (%s) ORDER BY frame_start ASC`, strings.Join(placeholders, ",")) + + rows, err := s.db.Query(query, args...) if err != nil { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query tasks: %v", err)) return @@ -2205,10 +3110,11 @@ func (s *Server) handleGetTaskLogs(w http.ResponseWriter, r *http.Request) { // Get query parameters for filtering stepName := r.URL.Query().Get("step_name") logLevel := r.URL.Query().Get("log_level") + sinceIDStr := r.URL.Query().Get("since_id") limitStr := r.URL.Query().Get("limit") - limit := 1000 // default + limit := 100 // default (reduced from 1000) if limitStr != "" { - if l, err := strconv.Atoi(limitStr); err == nil && l > 0 { + if l, err := strconv.Atoi(limitStr); err == nil && l > 0 && l <= 10000 { limit = l } } @@ -2217,6 +3123,15 @@ func (s *Server) handleGetTaskLogs(w http.ResponseWriter, r *http.Request) { query := `SELECT id, task_id, runner_id, log_level, message, step_name, created_at FROM task_logs WHERE task_id = ?` args := []interface{}{taskID} + + // Add since_id filter for incremental updates + if sinceIDStr != "" { + if sinceID, err := strconv.ParseInt(sinceIDStr, 10, 64); err == nil && sinceID > 0 { + query += " AND id > ?" + args = append(args, sinceID) + } + } + if stepName != "" { query += " AND step_name = ?" args = append(args, stepName) @@ -2225,7 +3140,7 @@ func (s *Server) handleGetTaskLogs(w http.ResponseWriter, r *http.Request) { query += " AND log_level = ?" args = append(args, logLevel) } - query += " ORDER BY created_at ASC LIMIT ?" + query += " ORDER BY id ASC LIMIT ?" args = append(args, limit) rows, err := s.db.Query(query, args...) @@ -2253,7 +3168,18 @@ func (s *Server) handleGetTaskLogs(w http.ResponseWriter, r *http.Request) { logs = append(logs, log) } - s.respondJSON(w, http.StatusOK, logs) + // Return last_id for next incremental fetch + lastID := int64(0) + if len(logs) > 0 { + lastID = logs[len(logs)-1].ID + } + + response := map[string]interface{}{ + "logs": logs, + "last_id": lastID, + "limit": limit, + } + s.respondJSON(w, http.StatusOK, response) } // handleGetTaskSteps retrieves step timeline for a specific task @@ -2444,6 +3370,14 @@ func (s *Server) handleRetryTask(w http.ResponseWriter, r *http.Request) { return } + // Broadcast task update + s.broadcastTaskUpdate(jobID, taskID, "task_update", map[string]interface{}{ + "status": types.TaskStatusPending, + "runner_id": nil, + "current_step": nil, + "error_message": nil, + }) + s.respondJSON(w, http.StatusOK, map[string]string{"message": "Task queued for retry"}) } @@ -2647,3 +3581,338 @@ func (s *Server) handleStreamTaskLogsWebSocket(w http.ResponseWriter, r *http.Re } } } + +// handleJobsWebSocket handles WebSocket connection for job list updates +func (s *Server) handleJobsWebSocket(w http.ResponseWriter, r *http.Request) { + userID, err := getUserID(r) + if err != nil { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + // Upgrade to WebSocket + conn, err := s.wsUpgrader.Upgrade(w, r, nil) + if err != nil { + log.Printf("Failed to upgrade WebSocket: %v", err) + return + } + defer conn.Close() + + // Register connection + s.jobListConnsMu.Lock() + // Close existing connection if any + if oldConn, exists := s.jobListConns[userID]; exists && oldConn != nil { + oldConn.Close() + } + s.jobListConns[userID] = conn + s.jobListConnsMu.Unlock() + + defer func() { + s.jobListConnsMu.Lock() + delete(s.jobListConns, userID) + s.jobListConnsMu.Unlock() + }() + + // Send initial connection message + err = conn.WriteJSON(map[string]interface{}{ + "type": "connected", + "timestamp": time.Now().Unix(), + }) + if err != nil { + log.Printf("Failed to send initial connection message: %v", err) + return + } + + // Keep connection alive and handle ping/pong + conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + conn.SetPongHandler(func(string) error { + conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + return nil + }) + + // Start ping ticker + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + // Read messages in background to keep connection alive and handle pongs + readDone := make(chan struct{}) + go func() { + defer close(readDone) + for { + conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + _, _, err := conn.ReadMessage() + if err != nil { + // Connection closed or error - exit read loop + if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) { + log.Printf("WebSocket read error for job list: %v", err) + } + return + } + // Reset read deadline after successful read (pong received) + conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + } + }() + + ctx := r.Context() + for { + select { + case <-ctx.Done(): + return + case <-readDone: + // Read loop exited, close connection + return + case <-ticker.C: + conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + if err := conn.WriteMessage(websocket.PingMessage, nil); err != nil { + return + } + } + } +} + +// handleJobWebSocket handles WebSocket connection for single job updates +func (s *Server) handleJobWebSocket(w http.ResponseWriter, r *http.Request) { + userID, err := getUserID(r) + if err != nil { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + jobID, err := parseID(r, "id") + if err != nil { + s.respondError(w, http.StatusBadRequest, err.Error()) + return + } + + // Verify job belongs to user (unless admin) + isAdmin := isAdminUser(r) + if !isAdmin { + var jobUserID int64 + err = s.db.QueryRow("SELECT user_id FROM jobs WHERE id = ?", jobID).Scan(&jobUserID) + if err == sql.ErrNoRows { + s.respondError(w, http.StatusNotFound, "Job not found") + return + } + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to verify job: %v", err)) + return + } + if jobUserID != userID { + s.respondError(w, http.StatusForbidden, "Access denied") + return + } + } else { + var exists bool + err = s.db.QueryRow("SELECT EXISTS(SELECT 1 FROM jobs WHERE id = ?)", jobID).Scan(&exists) + if err != nil || !exists { + s.respondError(w, http.StatusNotFound, "Job not found") + return + } + } + + // Upgrade to WebSocket + conn, err := s.wsUpgrader.Upgrade(w, r, nil) + if err != nil { + log.Printf("Failed to upgrade WebSocket: %v", err) + return + } + defer conn.Close() + + key := fmt.Sprintf("%d:%d", userID, jobID) + s.jobConnsMu.Lock() + // Close existing connection if any + if oldConn, exists := s.jobConns[key]; exists && oldConn != nil { + oldConn.Close() + } + s.jobConns[key] = conn + s.jobConnsMu.Unlock() + + // Create a write mutex for this connection + s.jobConnsWriteMuMu.Lock() + s.jobConnsWriteMu[key] = &sync.Mutex{} + writeMu := s.jobConnsWriteMu[key] + s.jobConnsWriteMuMu.Unlock() + + defer func() { + s.jobConnsMu.Lock() + delete(s.jobConns, key) + s.jobConnsMu.Unlock() + s.jobConnsWriteMuMu.Lock() + delete(s.jobConnsWriteMu, key) + s.jobConnsWriteMuMu.Unlock() + }() + + // Send initial connection message + writeMu.Lock() + err = conn.WriteJSON(map[string]interface{}{ + "type": "connected", + "timestamp": time.Now().Unix(), + }) + writeMu.Unlock() + if err != nil { + log.Printf("Failed to send initial connection message: %v", err) + return + } + + // Keep connection alive and handle ping/pong + conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + conn.SetPongHandler(func(string) error { + conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + return nil + }) + + // Start ping ticker + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + // Read messages in background to keep connection alive and handle pongs + readDone := make(chan struct{}) + go func() { + defer close(readDone) + for { + conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + _, _, err := conn.ReadMessage() + if err != nil { + // Connection closed or error - exit read loop + if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) { + log.Printf("WebSocket read error for job %d: %v", jobID, err) + } + return + } + // Reset read deadline after successful read (pong received) + conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + } + }() + + ctx := r.Context() + for { + select { + case <-ctx.Done(): + return + case <-readDone: + // Read loop exited, close connection + return + case <-ticker.C: + conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + if err := conn.WriteMessage(websocket.PingMessage, nil); err != nil { + return + } + } + } +} + +// broadcastJobUpdate broadcasts job update to connected clients +func (s *Server) broadcastJobUpdate(jobID int64, updateType string, data interface{}) { + // Get user_id from job + var userID int64 + err := s.db.QueryRow("SELECT user_id FROM jobs WHERE id = ?", jobID).Scan(&userID) + if err != nil { + return + } + + msg := map[string]interface{}{ + "type": updateType, + "job_id": jobID, + "data": data, + "timestamp": time.Now().Unix(), + } + + // Broadcast to job list connection + s.jobListConnsMu.RLock() + if conn, exists := s.jobListConns[userID]; exists && conn != nil { + s.jobListConnsMu.RUnlock() + conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + conn.WriteJSON(msg) + } else { + s.jobListConnsMu.RUnlock() + } + + // Broadcast to single job connection + key := fmt.Sprintf("%d:%d", userID, jobID) + s.jobConnsMu.RLock() + conn, exists := s.jobConns[key] + s.jobConnsMu.RUnlock() + + if exists && conn != nil { + s.jobConnsWriteMuMu.RLock() + writeMu, hasMu := s.jobConnsWriteMu[key] + s.jobConnsWriteMuMu.RUnlock() + + if hasMu && writeMu != nil { + writeMu.Lock() + conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + err := conn.WriteJSON(msg) + writeMu.Unlock() + if err != nil { + log.Printf("Failed to broadcast %s to job %d WebSocket: %v", updateType, jobID, err) + } else { + log.Printf("Successfully broadcast %s to job %d WebSocket", updateType, jobID) + } + } else { + conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + err := conn.WriteJSON(msg) + if err != nil { + log.Printf("Failed to broadcast %s to job %d WebSocket: %v", updateType, jobID, err) + } + } + } +} + +// broadcastTaskUpdate broadcasts task update to connected clients +func (s *Server) broadcastTaskUpdate(jobID int64, taskID int64, updateType string, data interface{}) { + // Get user_id from job + var userID int64 + err := s.db.QueryRow("SELECT user_id FROM jobs WHERE id = ?", jobID).Scan(&userID) + if err != nil { + return + } + + msg := map[string]interface{}{ + "type": updateType, + "job_id": jobID, + "data": data, + "timestamp": time.Now().Unix(), + } + // Always include task_id if it's provided (even if 0, for consistency) + // For bulk operations like "tasks_added", task_id will be 0 + if taskID > 0 { + msg["task_id"] = taskID + // Also include task_id in data for convenience + if dataMap, ok := data.(map[string]interface{}); ok { + dataMap["task_id"] = taskID + } + } + + // Broadcast to single job connection + key := fmt.Sprintf("%d:%d", userID, jobID) + s.jobConnsMu.RLock() + conn, exists := s.jobConns[key] + s.jobConnsMu.RUnlock() + + if exists && conn != nil { + s.jobConnsWriteMuMu.RLock() + writeMu, hasMu := s.jobConnsWriteMu[key] + s.jobConnsWriteMuMu.RUnlock() + + if hasMu && writeMu != nil { + writeMu.Lock() + conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + conn.WriteJSON(msg) + writeMu.Unlock() + } else { + conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) + conn.WriteJSON(msg) + } + } +} + +// truncateString truncates a string to a maximum length, appending "..." if truncated +func truncateString(s string, maxLen int) string { + if len(s) <= maxLen { + return s + } + if maxLen <= 3 { + return "..." + } + return s[:maxLen-3] + "..." +} diff --git a/internal/api/metadata.go b/internal/api/metadata.go index ab3d0ba..b126723 100644 --- a/internal/api/metadata.go +++ b/internal/api/metadata.go @@ -4,8 +4,8 @@ import ( "archive/tar" "bufio" "bytes" - "compress/gzip" "database/sql" + _ "embed" "encoding/json" "errors" "fmt" @@ -17,6 +17,7 @@ import ( "path/filepath" "strings" + "jiggablend/pkg/scripts" "jiggablend/pkg/types" ) @@ -169,22 +170,26 @@ func (s *Server) handleGetJobMetadata(w http.ResponseWriter, r *http.Request) { // extractMetadataFromContext extracts metadata from the blend file in a context archive // Returns the extracted metadata or an error func (s *Server) extractMetadataFromContext(jobID int64) (*types.BlendMetadata, error) { - contextPath := filepath.Join(s.storage.JobPath(jobID), "context.tar.gz") - + contextPath := filepath.Join(s.storage.JobPath(jobID), "context.tar") + // Check if context exists if _, err := os.Stat(contextPath); err != nil { return nil, fmt.Errorf("context archive not found: %w", err) } - // Create temporary directory for extraction - tmpDir, err := os.MkdirTemp("", fmt.Sprintf("fuego-metadata-%d-*", jobID)) + // Create temporary directory for extraction under storage base path + tmpDir, err := s.storage.TempDir(fmt.Sprintf("jiggablend-metadata-%d-*", jobID)) if err != nil { return nil, fmt.Errorf("failed to create temporary directory: %w", err) } - defer os.RemoveAll(tmpDir) + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + log.Printf("Warning: Failed to clean up temp directory %s: %v", tmpDir, err) + } + }() // Extract context archive - if err := s.extractTarGz(contextPath, tmpDir); err != nil { + if err := s.extractTar(contextPath, tmpDir); err != nil { return nil, fmt.Errorf("failed to extract context: %w", err) } @@ -228,188 +233,20 @@ func (s *Server) extractMetadataFromContext(jobID int64) (*types.BlendMetadata, return nil, fmt.Errorf("no .blend file found in context") } - // Create Python script to extract metadata + // Use embedded Python script scriptPath := filepath.Join(tmpDir, "extract_metadata.py") - scriptContent := `import bpy -import json -import sys - -# Make all file paths relative to the blend file location FIRST -# This must be done immediately after file load, before any other operations -# to prevent Blender from trying to access external files with absolute paths -try: - bpy.ops.file.make_paths_relative() - print("Made all file paths relative to blend file") -except Exception as e: - print(f"Warning: Could not make paths relative: {e}") - -# Check for missing addons that the blend file requires -# Blender marks missing addons with "_missing" suffix in preferences -missing_files_info = { - "checked": False, - "has_missing": False, - "missing_files": [], - "missing_addons": [] -} - -try: - missing = [] - for mod in bpy.context.preferences.addons: - if mod.module.endswith("_missing"): - missing.append(mod.module.rsplit("_", 1)[0]) - - missing_files_info["checked"] = True - if missing: - missing_files_info["has_missing"] = True - missing_files_info["missing_addons"] = missing - print("Missing add-ons required by this .blend:") - for name in missing: - print(" -", name) - else: - print("No missing add-ons detected – file is headless-safe") -except Exception as e: - print(f"Warning: Could not check for missing addons: {e}") - missing_files_info["error"] = str(e) - -# Get scene -scene = bpy.context.scene - -# Extract frame range from scene settings -frame_start = scene.frame_start -frame_end = scene.frame_end - -# Also check for actual animation range (keyframes) -# Find the earliest and latest keyframes across all objects -animation_start = None -animation_end = None - -for obj in scene.objects: - if obj.animation_data and obj.animation_data.action: - action = obj.animation_data.action - if action.fcurves: - for fcurve in action.fcurves: - if fcurve.keyframe_points: - for keyframe in fcurve.keyframe_points: - frame = int(keyframe.co[0]) - if animation_start is None or frame < animation_start: - animation_start = frame - if animation_end is None or frame > animation_end: - animation_end = frame - -# Use animation range if available, otherwise use scene frame range -# If scene range seems wrong (start == end), prefer animation range -if animation_start is not None and animation_end is not None: - if frame_start == frame_end or (animation_start < frame_start or animation_end > frame_end): - # Use animation range if scene range is invalid or animation extends beyond it - frame_start = animation_start - frame_end = animation_end - -# Extract render settings -render = scene.render -resolution_x = render.resolution_x -resolution_y = render.resolution_y -engine = scene.render.engine.upper() - -# Determine output format from file format -output_format = render.image_settings.file_format - -# Extract engine-specific settings -engine_settings = {} - -if engine == 'CYCLES': - cycles = scene.cycles - engine_settings = { - "samples": getattr(cycles, 'samples', 128), - "use_denoising": getattr(cycles, 'use_denoising', False), - "denoising_radius": getattr(cycles, 'denoising_radius', 0), - "denoising_strength": getattr(cycles, 'denoising_strength', 0.0), - "device": getattr(cycles, 'device', 'CPU'), - "use_adaptive_sampling": getattr(cycles, 'use_adaptive_sampling', False), - "adaptive_threshold": getattr(cycles, 'adaptive_threshold', 0.01) if getattr(cycles, 'use_adaptive_sampling', False) else 0.01, - "use_fast_gi": getattr(cycles, 'use_fast_gi', False), - "light_tree": getattr(cycles, 'use_light_tree', False), - "use_light_linking": getattr(cycles, 'use_light_linking', False), - "caustics_reflective": getattr(cycles, 'caustics_reflective', False), - "caustics_refractive": getattr(cycles, 'caustics_refractive', False), - "blur_glossy": getattr(cycles, 'blur_glossy', 0.0), - "max_bounces": getattr(cycles, 'max_bounces', 12), - "diffuse_bounces": getattr(cycles, 'diffuse_bounces', 4), - "glossy_bounces": getattr(cycles, 'glossy_bounces', 4), - "transmission_bounces": getattr(cycles, 'transmission_bounces', 12), - "volume_bounces": getattr(cycles, 'volume_bounces', 0), - "transparent_max_bounces": getattr(cycles, 'transparent_max_bounces', 8), - "film_transparent": getattr(cycles, 'film_transparent', False), - "use_layer_samples": getattr(cycles, 'use_layer_samples', False), - } -elif engine == 'EEVEE' or engine == 'EEVEE_NEXT': - eevee = scene.eevee - engine_settings = { - "taa_render_samples": getattr(eevee, 'taa_render_samples', 64), - "use_bloom": getattr(eevee, 'use_bloom', False), - "bloom_threshold": getattr(eevee, 'bloom_threshold', 0.8), - "bloom_intensity": getattr(eevee, 'bloom_intensity', 0.05), - "bloom_radius": getattr(eevee, 'bloom_radius', 6.5), - "use_ssr": getattr(eevee, 'use_ssr', True), - "use_ssr_refraction": getattr(eevee, 'use_ssr_refraction', False), - "ssr_quality": getattr(eevee, 'ssr_quality', 'MEDIUM'), - "use_ssao": getattr(eevee, 'use_ssao', True), - "ssao_quality": getattr(eevee, 'ssao_quality', 'MEDIUM'), - "ssao_distance": getattr(eevee, 'ssao_distance', 0.2), - "ssao_factor": getattr(eevee, 'ssao_factor', 1.0), - "use_soft_shadows": getattr(eevee, 'use_soft_shadows', True), - "use_shadow_high_bitdepth": getattr(eevee, 'use_shadow_high_bitdepth', True), - "use_volumetric": getattr(eevee, 'use_volumetric', False), - "volumetric_tile_size": getattr(eevee, 'volumetric_tile_size', '8'), - "volumetric_samples": getattr(eevee, 'volumetric_samples', 64), - "volumetric_start": getattr(eevee, 'volumetric_start', 0.0), - "volumetric_end": getattr(eevee, 'volumetric_end', 100.0), - "use_volumetric_lights": getattr(eevee, 'use_volumetric_lights', True), - "use_volumetric_shadows": getattr(eevee, 'use_volumetric_shadows', True), - "use_gtao": getattr(eevee, 'use_gtao', False), - "gtao_quality": getattr(eevee, 'gtao_quality', 'MEDIUM'), - "use_overscan": getattr(eevee, 'use_overscan', False), - } -else: - # For other engines, extract basic samples if available - engine_settings = { - "samples": getattr(scene, 'samples', 128) if hasattr(scene, 'samples') else 128 - } - -# Extract scene info -camera_count = len([obj for obj in scene.objects if obj.type == 'CAMERA']) -object_count = len(scene.objects) -material_count = len(bpy.data.materials) - -# Build metadata dictionary -metadata = { - "frame_start": frame_start, - "frame_end": frame_end, - "render_settings": { - "resolution_x": resolution_x, - "resolution_y": resolution_y, - "output_format": output_format, - "engine": engine.lower(), - "engine_settings": engine_settings - }, - "scene_info": { - "camera_count": camera_count, - "object_count": object_count, - "material_count": material_count - }, - "missing_files_info": missing_files_info -} - -# Output as JSON -print(json.dumps(metadata)) -sys.stdout.flush() -` - - if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil { + if err := os.WriteFile(scriptPath, []byte(scripts.ExtractMetadata), 0644); err != nil { return nil, fmt.Errorf("failed to create extraction script: %w", err) } + // Make blend file path relative to tmpDir to avoid path resolution issues + blendFileRel, err := filepath.Rel(tmpDir, blendFile) + if err != nil { + return nil, fmt.Errorf("failed to get relative path for blend file: %w", err) + } + // Execute Blender with Python script - cmd := exec.Command("blender", "-b", blendFile, "--python", scriptPath) + cmd := exec.Command("blender", "-b", blendFileRel, "--python", "extract_metadata.py") cmd.Dir = tmpDir // Capture stdout and stderr @@ -443,14 +280,16 @@ sys.stdout.flush() } }() - // Stream stderr (discard for now, but could log if needed) + // Capture stderr for error reporting + var stderrBuffer bytes.Buffer stderrDone := make(chan bool) go func() { defer close(stderrDone) scanner := bufio.NewScanner(stderrPipe) for scanner.Scan() { - // Could log stderr if needed - _ = scanner.Text() + line := scanner.Text() + stderrBuffer.WriteString(line) + stderrBuffer.WriteString("\n") } }() @@ -462,6 +301,18 @@ sys.stdout.flush() <-stderrDone if err != nil { + stderrOutput := strings.TrimSpace(stderrBuffer.String()) + stdoutOutput := strings.TrimSpace(stdoutBuffer.String()) + log.Printf("Blender metadata extraction failed for job %d:", jobID) + if stderrOutput != "" { + log.Printf("Blender stderr: %s", stderrOutput) + } + if stdoutOutput != "" { + log.Printf("Blender stdout (last 500 chars): %s", truncateString(stdoutOutput, 500)) + } + if stderrOutput != "" { + return nil, fmt.Errorf("blender metadata extraction failed: %w (stderr: %s)", err, truncateString(stderrOutput, 200)) + } return nil, fmt.Errorf("blender metadata extraction failed: %w", err) } @@ -484,21 +335,25 @@ sys.stdout.flush() return &metadata, nil } -// extractTarGz extracts a tar.gz archive to a destination directory -func (s *Server) extractTarGz(tarGzPath, destDir string) error { - file, err := os.Open(tarGzPath) +// extractTar extracts a tar archive to a destination directory +func (s *Server) extractTar(tarPath, destDir string) error { + log.Printf("Extracting tar archive: %s -> %s", tarPath, destDir) + + // Ensure destination directory exists + if err := os.MkdirAll(destDir, 0755); err != nil { + return fmt.Errorf("failed to create destination directory: %w", err) + } + + file, err := os.Open(tarPath) if err != nil { return fmt.Errorf("failed to open archive: %w", err) } defer file.Close() - gzr, err := gzip.NewReader(file) - if err != nil { - return fmt.Errorf("failed to create gzip reader: %w", err) - } - defer gzr.Close() + tr := tar.NewReader(file) - tr := tar.NewReader(gzr) + fileCount := 0 + dirCount := 0 for { header, err := tr.Next() @@ -511,9 +366,13 @@ func (s *Server) extractTarGz(tarGzPath, destDir string) error { // Sanitize path to prevent directory traversal target := filepath.Join(destDir, header.Name) + // Ensure target is within destDir - if !strings.HasPrefix(filepath.Clean(target), filepath.Clean(destDir)+string(os.PathSeparator)) { - return fmt.Errorf("invalid file path in archive: %s", header.Name) + cleanTarget := filepath.Clean(target) + cleanDestDir := filepath.Clean(destDir) + if !strings.HasPrefix(cleanTarget, cleanDestDir+string(os.PathSeparator)) && cleanTarget != cleanDestDir { + log.Printf("ERROR: Invalid file path in TAR - target: %s, destDir: %s", cleanTarget, cleanDestDir) + return fmt.Errorf("invalid file path in archive: %s (target: %s, destDir: %s)", header.Name, cleanTarget, cleanDestDir) } // Create parent directories @@ -527,14 +386,18 @@ func (s *Server) extractTarGz(tarGzPath, destDir string) error { if err != nil { return fmt.Errorf("failed to create file: %w", err) } - if _, err := io.Copy(outFile, tr); err != nil { + _, err = io.Copy(outFile, tr) + if err != nil { outFile.Close() return fmt.Errorf("failed to write file: %w", err) } outFile.Close() + fileCount++ + } else if header.Typeflag == tar.TypeDir { + dirCount++ } } + log.Printf("Extraction complete: %d files, %d directories extracted to %s", fileCount, dirCount, destDir) return nil } - diff --git a/internal/api/runners.go b/internal/api/runners.go index e5015c1..b369718 100644 --- a/internal/api/runners.go +++ b/internal/api/runners.go @@ -9,6 +9,7 @@ import ( "log" "math/rand" "net/http" + "net/url" "path/filepath" "sort" "strconv" @@ -17,6 +18,7 @@ import ( "jiggablend/pkg/types" + "github.com/go-chi/chi/v5" "github.com/gorilla/websocket" ) @@ -287,13 +289,27 @@ func (s *Server) handleUpdateTaskStep(w http.ResponseWriter, r *http.Request) { } } + // Get job ID for broadcasting + var jobID int64 + err = s.db.QueryRow("SELECT job_id FROM tasks WHERE id = ?", taskID).Scan(&jobID) + if err == nil { + // Broadcast step update to frontend + s.broadcastTaskUpdate(jobID, taskID, "step_update", map[string]interface{}{ + "step_id": stepID, + "step_name": req.StepName, + "status": req.Status, + "duration_ms": req.DurationMs, + "error_message": req.ErrorMessage, + }) + } + s.respondJSON(w, http.StatusOK, map[string]interface{}{ "step_id": stepID, "message": "Step updated successfully", }) } -// handleDownloadJobContext allows runners to download the job context tar.gz +// handleDownloadJobContext allows runners to download the job context tar func (s *Server) handleDownloadJobContext(w http.ResponseWriter, r *http.Request) { jobID, err := parseID(r, "jobId") if err != nil { @@ -302,7 +318,7 @@ func (s *Server) handleDownloadJobContext(w http.ResponseWriter, r *http.Request } // Construct the context file path - contextPath := filepath.Join(s.storage.JobPath(jobID), "context.tar.gz") + contextPath := filepath.Join(s.storage.JobPath(jobID), "context.tar") // Check if context file exists if !s.storage.FileExists(contextPath) { @@ -319,9 +335,9 @@ func (s *Server) handleDownloadJobContext(w http.ResponseWriter, r *http.Request } defer file.Close() - // Set appropriate headers for tar.gz file - w.Header().Set("Content-Type", "application/gzip") - w.Header().Set("Content-Disposition", "attachment; filename=context.tar.gz") + // Set appropriate headers for tar file + w.Header().Set("Content-Type", "application/x-tar") + w.Header().Set("Content-Disposition", "attachment; filename=context.tar") // Stream the file to the response io.Copy(w, file) @@ -356,16 +372,26 @@ func (s *Server) handleUploadFileFromRunner(w http.ResponseWriter, r *http.Reque } // Record in database - _, err = s.db.Exec( + var fileID int64 + err = s.db.QueryRow( `INSERT INTO job_files (job_id, file_type, file_path, file_name, file_size) - VALUES (?, ?, ?, ?, ?)`, + VALUES (?, ?, ?, ?, ?) + RETURNING id`, jobID, types.JobFileTypeOutput, filePath, header.Filename, header.Size, - ) + ).Scan(&fileID) if err != nil { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to record file: %v", err)) return } + // Broadcast file addition + s.broadcastJobUpdate(jobID, "file_added", map[string]interface{}{ + "file_id": fileID, + "file_type": types.JobFileTypeOutput, + "file_name": header.Filename, + "file_size": header.Size, + }) + s.respondJSON(w, http.StatusCreated, map[string]interface{}{ "file_path": filePath, "file_name": header.Filename, @@ -510,6 +536,79 @@ func (s *Server) handleGetJobMetadataForRunner(w http.ResponseWriter, r *http.Re s.respondJSON(w, http.StatusOK, metadata) } +// handleDownloadFileForRunner allows runners to download a file by fileName +func (s *Server) handleDownloadFileForRunner(w http.ResponseWriter, r *http.Request) { + jobID, err := parseID(r, "jobId") + if err != nil { + s.respondError(w, http.StatusBadRequest, err.Error()) + return + } + + // Get fileName from URL path (may need URL decoding) + fileName := chi.URLParam(r, "fileName") + if fileName == "" { + s.respondError(w, http.StatusBadRequest, "fileName is required") + return + } + + // URL decode the fileName in case it contains encoded characters + decodedFileName, err := url.QueryUnescape(fileName) + if err != nil { + // If decoding fails, use original fileName + decodedFileName = fileName + } + + // Get file info from database + var filePath string + err = s.db.QueryRow( + `SELECT file_path FROM job_files WHERE job_id = ? AND file_name = ?`, + jobID, decodedFileName, + ).Scan(&filePath) + if err == sql.ErrNoRows { + s.respondError(w, http.StatusNotFound, "File not found") + return + } + if err != nil { + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query file: %v", err)) + return + } + + // Open file + file, err := s.storage.GetFile(filePath) + if err != nil { + s.respondError(w, http.StatusNotFound, "File not found on disk") + return + } + defer file.Close() + + // Determine content type based on file extension + contentType := "application/octet-stream" + fileNameLower := strings.ToLower(decodedFileName) + switch { + case strings.HasSuffix(fileNameLower, ".png"): + contentType = "image/png" + case strings.HasSuffix(fileNameLower, ".jpg") || strings.HasSuffix(fileNameLower, ".jpeg"): + contentType = "image/jpeg" + case strings.HasSuffix(fileNameLower, ".gif"): + contentType = "image/gif" + case strings.HasSuffix(fileNameLower, ".webp"): + contentType = "image/webp" + case strings.HasSuffix(fileNameLower, ".exr") || strings.HasSuffix(fileNameLower, ".EXR"): + contentType = "image/x-exr" + case strings.HasSuffix(fileNameLower, ".mp4"): + contentType = "video/mp4" + case strings.HasSuffix(fileNameLower, ".webm"): + contentType = "video/webm" + } + + // Set headers + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", decodedFileName)) + + // Stream file + io.Copy(w, file) +} + // WebSocket message types type WSMessage struct { Type string `json:"type"` @@ -785,6 +884,13 @@ func (s *Server) handleWebSocketTaskComplete(runnerID int64, taskUpdate WSTaskUp taskUpdate.TaskID, ).Scan(&jobID) if err == nil { + // Broadcast task update + s.broadcastTaskUpdate(jobID, taskUpdate.TaskID, "task_update", map[string]interface{}{ + "status": status, + "output_path": taskUpdate.OutputPath, + "completed_at": now, + "error": taskUpdate.Error, + }) s.updateJobStatusFromTasks(jobID) } } @@ -840,6 +946,7 @@ func (s *Server) getCurrentFrameFromLogs(jobID int64) (int, bool) { for rows.Next() { var taskID int64 if err := rows.Scan(&taskID); err != nil { + log.Printf("Failed to scan task ID in getCurrentFrameFromLogs: %v", err) continue } @@ -895,6 +1002,14 @@ func (s *Server) updateJobStatusFromTasks(jobID int64) { allowParallelRunners.Valid && !allowParallelRunners.Bool && frameStart.Valid && frameEnd.Valid + // Get current job status to detect changes + var currentStatus string + err = s.db.QueryRow(`SELECT status FROM jobs WHERE id = ?`, jobID).Scan(¤tStatus) + if err != nil { + log.Printf("Failed to get current job status for job %d: %v", jobID, err) + return + } + // Count total tasks and completed tasks var totalTasks, completedTasks int err = s.db.QueryRow( @@ -914,8 +1029,6 @@ func (s *Server) updateJobStatusFromTasks(jobID int64) { return } - log.Printf("updateJobStatusFromTasks: job %d - total: %d, completed: %d", jobID, totalTasks, completedTasks) - // Calculate progress var progress float64 if totalTasks == 0 { @@ -985,9 +1098,6 @@ func (s *Server) updateJobStatusFromTasks(jobID int64) { } else { progress = renderProgress } - - log.Printf("updateJobStatusFromTasks: job %d - frame-based progress: current_frame=%d, render_progress=%.1f%%, non_render_progress=%.1f%%, total_progress=%.1f%%", - jobID, currentFrame, renderProgress, nonRenderProgress, progress) } else { // Standard task-based progress progress = float64(completedTasks) / float64(totalTasks) * 100.0 @@ -1013,8 +1123,6 @@ func (s *Server) updateJobStatusFromTasks(jobID int64) { return } - log.Printf("updateJobStatusFromTasks: job %d - pending/running: %d", jobID, pendingOrRunningTasks) - if pendingOrRunningTasks == 0 && totalTasks > 0 { // All tasks are either completed or failed/cancelled // Check if any tasks failed @@ -1039,7 +1147,16 @@ func (s *Server) updateJobStatusFromTasks(jobID int64) { if err != nil { log.Printf("Failed to update job %d status to %s: %v", jobID, jobStatus, err) } else { - log.Printf("Updated job %d status to %s (progress: %.1f%%, completed tasks: %d/%d)", jobID, jobStatus, progress, completedTasks, totalTasks) + // Only log if status actually changed + if currentStatus != jobStatus { + log.Printf("Updated job %d status from %s to %s (progress: %.1f%%, completed tasks: %d/%d)", jobID, currentStatus, jobStatus, progress, completedTasks, totalTasks) + } + // Broadcast job update via WebSocket + s.broadcastJobUpdate(jobID, "job_update", map[string]interface{}{ + "status": jobStatus, + "progress": progress, + "completed_at": now, + }) } if outputFormatStr == "EXR_264_MP4" || outputFormatStr == "EXR_AV1_MP4" { @@ -1054,14 +1171,22 @@ func (s *Server) updateJobStatusFromTasks(jobID int64) { // Create a video generation task instead of calling generateMP4Video directly // This prevents race conditions when multiple runners complete frames simultaneously videoTaskTimeout := 86400 // 24 hours for video generation - _, err := s.db.Exec( + var videoTaskID int64 + err := s.db.QueryRow( `INSERT INTO tasks (job_id, frame_start, frame_end, task_type, status, timeout_seconds, max_retries) - VALUES (?, ?, ?, ?, ?, ?, ?)`, + VALUES (?, ?, ?, ?, ?, ?, ?) + RETURNING id`, jobID, 0, 0, types.TaskTypeVideoGeneration, types.TaskStatusPending, videoTaskTimeout, 1, - ) + ).Scan(&videoTaskID) if err != nil { log.Printf("Failed to create video generation task for job %d: %v", jobID, err) } else { + // Broadcast that a new task was added + log.Printf("Broadcasting task_added for job %d: video generation task %d", jobID, videoTaskID) + s.broadcastTaskUpdate(jobID, videoTaskID, "task_added", map[string]interface{}{ + "task_id": videoTaskID, + "task_type": types.TaskTypeVideoGeneration, + }) // Update job status to ensure it's marked as running (has pending video task) s.updateJobStatusFromTasks(jobID) // Try to distribute the task immediately @@ -1099,7 +1224,10 @@ func (s *Server) updateJobStatusFromTasks(jobID int64) { if err != nil { log.Printf("Failed to update job %d status to %s: %v", jobID, jobStatus, err) } else { - log.Printf("Updated job %d status to %s (progress: %.1f%%, completed: %d/%d, pending: %d, running: %d)", jobID, jobStatus, progress, completedTasks, totalTasks, pendingOrRunningTasks-runningTasks, runningTasks) + // Only log if status actually changed + if currentStatus != jobStatus { + log.Printf("Updated job %d status from %s to %s (progress: %.1f%%, completed: %d/%d, pending: %d, running: %d)", jobID, currentStatus, jobStatus, progress, completedTasks, totalTasks, pendingOrRunningTasks-runningTasks, runningTasks) + } } } } @@ -1224,7 +1352,6 @@ func (s *Server) distributeTasksToRunners() { t.AllowParallelRunners = true } pendingTasks = append(pendingTasks, t) - log.Printf("Found pending task %d (type: %s, job: %d '%s', status: %s)", t.TaskID, t.TaskType, t.JobID, t.JobName, t.JobStatus) } if len(pendingTasks) == 0 { @@ -1308,11 +1435,6 @@ func (s *Server) distributeTasksToRunners() { } log.Printf("Distributing %d pending tasks (%v) to %d connected runners: %v", len(pendingTasks), taskTypes, len(connectedRunners), connectedRunners) - // Log each pending task for debugging - for _, task := range pendingTasks { - log.Printf(" - Task %d (type: %s, job: %d '%s', status: %s)", task.TaskID, task.TaskType, task.JobID, task.JobName, task.JobStatus) - } - // Distribute tasks to runners // Sort tasks to prioritize metadata tasks sort.Slice(pendingTasks, func(i, j int) bool { @@ -1572,6 +1694,13 @@ func (s *Server) distributeTasksToRunners() { continue } + // Broadcast task assignment + s.broadcastTaskUpdate(task.JobID, task.TaskID, "task_update", map[string]interface{}{ + "status": types.TaskStatusRunning, + "runner_id": selectedRunnerID, + "started_at": now, + }) + // Task was successfully assigned, send via WebSocket log.Printf("Assigned task %d (type: %s, job: %d) to runner %d", task.TaskID, task.TaskType, task.JobID, selectedRunnerID) @@ -1642,6 +1771,8 @@ func (s *Server) assignTaskToRunner(runnerID int64, taskID int64) error { var filePath string if err := rows.Scan(&filePath); err == nil { task.InputFiles = append(task.InputFiles, filePath) + } else { + log.Printf("Failed to scan input file path for task %d: %v", taskID, err) } } } else { diff --git a/internal/api/server.go b/internal/api/server.go index ff0dafe..1232a2f 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -1,12 +1,17 @@ package api import ( + "compress/gzip" "database/sql" "encoding/json" "fmt" + "io" "log" "net/http" + "os" + "path/filepath" "strconv" + "strings" "sync" "time" @@ -38,6 +43,15 @@ type Server struct { // Mutexes for each frontend connection to serialize writes frontendConnsWriteMu map[string]*sync.Mutex // key: "jobId:taskId" frontendConnsWriteMuMu sync.RWMutex + // Job list WebSocket connections (key: userID) + jobListConns map[int64]*websocket.Conn + jobListConnsMu sync.RWMutex + // Single job WebSocket connections (key: "userId:jobId") + jobConns map[string]*websocket.Conn + jobConnsMu sync.RWMutex + // Mutexes for job WebSocket connections + jobConnsWriteMu map[string]*sync.Mutex + jobConnsWriteMuMu sync.RWMutex // Throttling for progress updates (per job) progressUpdateTimes map[int64]time.Time // key: jobID progressUpdateTimesMu sync.RWMutex @@ -66,6 +80,9 @@ func NewServer(db *database.DB, auth *authpkg.Auth, storage *storage.Storage) (* runnerConns: make(map[int64]*websocket.Conn), frontendConns: make(map[string]*websocket.Conn), frontendConnsWriteMu: make(map[string]*sync.Mutex), + jobListConns: make(map[int64]*websocket.Conn), + jobConns: make(map[string]*websocket.Conn), + jobConnsWriteMu: make(map[string]*sync.Mutex), progressUpdateTimes: make(map[int64]time.Time), } @@ -83,16 +100,62 @@ func (s *Server) setupMiddleware() { // Note: Timeout middleware is NOT applied globally to avoid conflicts with WebSocket connections // WebSocket connections are long-lived and should not have HTTP timeouts + // Add gzip compression for JSON responses + s.router.Use(gzipMiddleware) + s.router.Use(cors.Handler(cors.Options{ AllowedOrigins: []string{"*"}, AllowedMethods: []string{"GET", "POST", "PUT", "DELETE", "OPTIONS"}, - AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "Range"}, - ExposedHeaders: []string{"Link", "Content-Range", "Accept-Ranges", "Content-Length"}, + AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "Range", "If-None-Match"}, + ExposedHeaders: []string{"Link", "Content-Range", "Accept-Ranges", "Content-Length", "ETag"}, AllowCredentials: true, MaxAge: 300, })) } +// gzipMiddleware compresses responses with gzip if client supports it +func gzipMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Skip compression for WebSocket upgrades + if strings.ToLower(r.Header.Get("Upgrade")) == "websocket" { + next.ServeHTTP(w, r) + return + } + + // Check if client accepts gzip + if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + next.ServeHTTP(w, r) + return + } + + // Create gzip writer + gz := gzip.NewWriter(w) + defer gz.Close() + + w.Header().Set("Content-Encoding", "gzip") + w.Header().Set("Vary", "Accept-Encoding") + + // Wrap response writer + gzw := &gzipResponseWriter{Writer: gz, ResponseWriter: w} + next.ServeHTTP(gzw, r) + }) +} + +// gzipResponseWriter wraps http.ResponseWriter to add gzip compression +type gzipResponseWriter struct { + io.Writer + http.ResponseWriter +} + +func (w *gzipResponseWriter) Write(b []byte) (int, error) { + return w.Writer.Write(b) +} + +func (w *gzipResponseWriter) WriteHeader(statusCode int) { + // Don't set Content-Length when using gzip - it will be set automatically + w.ResponseWriter.WriteHeader(statusCode) +} + // setupRoutes configures routes func (s *Server) setupRoutes() { // Public routes @@ -118,16 +181,21 @@ func (s *Server) setupRoutes() { r.Post("/", s.handleCreateJob) r.Post("/upload", s.handleUploadFileForJobCreation) // Upload before job creation r.Get("/", s.handleListJobs) + r.Get("/summary", s.handleListJobsSummary) + r.Post("/batch", s.handleBatchGetJobs) r.Get("/{id}", s.handleGetJob) r.Delete("/{id}", s.handleCancelJob) r.Post("/{id}/delete", s.handleDeleteJob) r.Post("/{id}/upload", s.handleUploadJobFile) r.Get("/{id}/files", s.handleListJobFiles) + r.Get("/{id}/files/count", s.handleGetJobFilesCount) r.Get("/{id}/context", s.handleListContextArchive) r.Get("/{id}/files/{fileId}/download", s.handleDownloadJobFile) r.Get("/{id}/video", s.handleStreamVideo) r.Get("/{id}/metadata", s.handleGetJobMetadata) r.Get("/{id}/tasks", s.handleListJobTasks) + r.Get("/{id}/tasks/summary", s.handleListJobTasksSummary) + r.Post("/{id}/tasks/batch", s.handleBatchGetTasks) r.Get("/{id}/tasks/{taskId}/logs", s.handleGetTaskLogs) // WebSocket route - no timeout middleware (long-lived connection) r.With(func(next http.Handler) http.Handler { @@ -138,6 +206,19 @@ func (s *Server) setupRoutes() { }).Get("/{id}/tasks/{taskId}/logs/ws", s.handleStreamTaskLogsWebSocket) r.Get("/{id}/tasks/{taskId}/steps", s.handleGetTaskSteps) r.Post("/{id}/tasks/{taskId}/retry", s.handleRetryTask) + // WebSocket routes for real-time updates + r.With(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Remove timeout middleware for WebSocket + next.ServeHTTP(w, r) + }) + }).Get("/ws", s.handleJobsWebSocket) + r.With(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Remove timeout middleware for WebSocket + next.ServeHTTP(w, r) + }) + }).Get("/{id}/ws", s.handleJobWebSocket) }) // Admin routes @@ -181,7 +262,8 @@ func (s *Server) setupRoutes() { }) r.Post("/tasks/{id}/progress", s.handleUpdateTaskProgress) r.Post("/tasks/{id}/steps", s.handleUpdateTaskStep) - r.Get("/jobs/{jobId}/context.tar.gz", s.handleDownloadJobContext) + r.Get("/jobs/{jobId}/context.tar", s.handleDownloadJobContext) + r.Get("/files/{jobId}/{fileName}", s.handleDownloadFileForRunner) r.Post("/files/{jobId}/upload", s.handleUploadFileFromRunner) r.Get("/jobs/{jobId}/status", s.handleGetJobStatusForRunner) r.Get("/jobs/{jobId}/files", s.handleGetJobFilesForRunner) @@ -311,12 +393,14 @@ func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request) { func (s *Server) handleGetMe(w http.ResponseWriter, r *http.Request) { cookie, err := r.Cookie("session_id") if err != nil { + log.Printf("Authentication failed: missing session cookie in /auth/me") s.respondError(w, http.StatusUnauthorized, "Not authenticated") return } session, ok := s.auth.GetSession(cookie.Value) if !ok { + log.Printf("Authentication failed: invalid session cookie in /auth/me") s.respondError(w, http.StatusUnauthorized, "Invalid session") return } @@ -410,6 +494,7 @@ func (s *Server) handleLocalLogin(w http.ResponseWriter, r *http.Request) { session, err := s.auth.LocalLogin(req.Username, req.Password) if err != nil { + log.Printf("Authentication failed: invalid credentials for username '%s'", req.Username) s.respondError(w, http.StatusUnauthorized, "Invalid credentials") return } @@ -512,6 +597,7 @@ func parseID(r *http.Request, param string) (int64, error) { func (s *Server) StartBackgroundTasks() { go s.recoverStuckTasks() go s.cleanupOldRenderJobs() + go s.cleanupOldTempDirectories() } // recoverStuckTasks periodically checks for dead runners and stuck tasks @@ -621,6 +707,7 @@ func (s *Server) recoverTaskTimeouts() { err := rows.Scan(&taskID, &runnerID, &retryCount, &maxRetries, &timeoutSeconds, &startedAt) if err != nil { + log.Printf("Failed to scan task row in recoverTaskTimeouts: %v", err) continue } @@ -659,3 +746,72 @@ func (s *Server) recoverTaskTimeouts() { } } } + +// cleanupOldTempDirectories periodically cleans up old temporary directories +func (s *Server) cleanupOldTempDirectories() { + // Run cleanup every hour + ticker := time.NewTicker(1 * time.Hour) + defer ticker.Stop() + + // Run once immediately on startup + s.cleanupOldTempDirectoriesOnce() + + for range ticker.C { + s.cleanupOldTempDirectoriesOnce() + } +} + +// cleanupOldTempDirectoriesOnce removes temp directories older than 1 hour +func (s *Server) cleanupOldTempDirectoriesOnce() { + defer func() { + if r := recover(); r != nil { + log.Printf("Panic in cleanupOldTempDirectories: %v", r) + } + }() + + tempPath := filepath.Join(s.storage.BasePath(), "temp") + + // Check if temp directory exists + if _, err := os.Stat(tempPath); os.IsNotExist(err) { + return + } + + // Read all entries in temp directory + entries, err := os.ReadDir(tempPath) + if err != nil { + log.Printf("Failed to read temp directory: %v", err) + return + } + + now := time.Now() + cleanedCount := 0 + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + entryPath := filepath.Join(tempPath, entry.Name()) + + // Get directory info to check modification time + info, err := entry.Info() + if err != nil { + continue + } + + // Remove directories older than 1 hour + age := now.Sub(info.ModTime()) + if age > 1*time.Hour { + if err := os.RemoveAll(entryPath); err != nil { + log.Printf("Warning: Failed to clean up old temp directory %s: %v", entryPath, err) + } else { + cleanedCount++ + log.Printf("Cleaned up old temp directory: %s (age: %v)", entryPath, age) + } + } + } + + if cleanedCount > 0 { + log.Printf("Cleaned up %d old temp directories", cleanedCount) + } +} diff --git a/internal/auth/auth.go b/internal/auth/auth.go index e34cb12..3044d91 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -410,6 +410,7 @@ func (a *Auth) Middleware(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { cookie, err := r.Cookie("session_id") if err != nil { + log.Printf("Authentication failed: missing session cookie for %s %s", r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusUnauthorized) json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"}) @@ -418,6 +419,7 @@ func (a *Auth) Middleware(next http.HandlerFunc) http.HandlerFunc { session, ok := a.GetSession(cookie.Value) if !ok { + log.Printf("Authentication failed: invalid session cookie for %s %s", r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusUnauthorized) json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"}) @@ -451,6 +453,7 @@ func (a *Auth) AdminMiddleware(next http.HandlerFunc) http.HandlerFunc { // First check authentication cookie, err := r.Cookie("session_id") if err != nil { + log.Printf("Admin authentication failed: missing session cookie for %s %s", r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusUnauthorized) json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"}) @@ -459,6 +462,7 @@ func (a *Auth) AdminMiddleware(next http.HandlerFunc) http.HandlerFunc { session, ok := a.GetSession(cookie.Value) if !ok { + log.Printf("Admin authentication failed: invalid session cookie for %s %s", r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusUnauthorized) json.NewEncoder(w).Encode(map[string]string{"error": "Unauthorized"}) @@ -467,6 +471,7 @@ func (a *Auth) AdminMiddleware(next http.HandlerFunc) http.HandlerFunc { // Then check admin status if !session.IsAdmin { + log.Printf("Admin access denied: user %d (email: %s) attempted to access admin endpoint %s %s", session.UserID, session.Email, r.Method, r.URL.Path) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusForbidden) json.NewEncoder(w).Encode(map[string]string{"error": "Forbidden: Admin access required"}) diff --git a/internal/database/schema.go b/internal/database/schema.go index 694bba8..1519e61 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -165,14 +165,17 @@ func (db *DB) migrate() error { CREATE INDEX IF NOT EXISTS idx_jobs_user_id ON jobs(user_id); CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status); + CREATE INDEX IF NOT EXISTS idx_jobs_user_status_created ON jobs(user_id, status, created_at DESC); CREATE INDEX IF NOT EXISTS idx_tasks_job_id ON tasks(job_id); CREATE INDEX IF NOT EXISTS idx_tasks_runner_id ON tasks(runner_id); CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status); + CREATE INDEX IF NOT EXISTS idx_tasks_job_status ON tasks(job_id, status); CREATE INDEX IF NOT EXISTS idx_tasks_started_at ON tasks(started_at); CREATE INDEX IF NOT EXISTS idx_job_files_job_id ON job_files(job_id); CREATE INDEX IF NOT EXISTS idx_registration_tokens_token ON registration_tokens(token); CREATE INDEX IF NOT EXISTS idx_registration_tokens_expires_at ON registration_tokens(expires_at); CREATE INDEX IF NOT EXISTS idx_task_logs_task_id_created_at ON task_logs(task_id, created_at); + CREATE INDEX IF NOT EXISTS idx_task_logs_task_id_id ON task_logs(task_id, id DESC); CREATE INDEX IF NOT EXISTS idx_task_logs_runner_id ON task_logs(runner_id); CREATE INDEX IF NOT EXISTS idx_task_steps_task_id ON task_steps(task_id); CREATE INDEX IF NOT EXISTS idx_runners_last_heartbeat ON runners(last_heartbeat); @@ -213,6 +216,9 @@ func (db *DB) migrate() error { `ALTER TABLE tasks ADD COLUMN IF NOT EXISTS retry_count INTEGER DEFAULT 0`, `ALTER TABLE tasks ADD COLUMN IF NOT EXISTS max_retries INTEGER DEFAULT 3`, `ALTER TABLE tasks ADD COLUMN IF NOT EXISTS timeout_seconds INTEGER`, + // Add updated_at columns for ETag support + `ALTER TABLE jobs ADD COLUMN IF NOT EXISTS updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP`, + `ALTER TABLE tasks ADD COLUMN IF NOT EXISTS updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP`, // Migrate file_size from INTEGER to BIGINT to support large files (>2GB) // DuckDB doesn't support direct ALTER COLUMN TYPE, so we use a workaround: // 1. Add new column as BIGINT diff --git a/internal/logger/logger.go b/internal/logger/logger.go new file mode 100644 index 0000000..3c5db35 --- /dev/null +++ b/internal/logger/logger.go @@ -0,0 +1,127 @@ +package logger + +import ( + "io" + "log" + "os" + "path/filepath" + "sync" + + "gopkg.in/natefinch/lumberjack.v2" +) + +var ( + defaultLogger *Logger + once sync.Once +) + +// Logger wraps the standard log.Logger with file and stdout output +type Logger struct { + *log.Logger + fileWriter io.WriteCloser +} + +// Init initializes the default logger with both file and stdout output +func Init(logDir, logFileName string, maxSizeMB int, maxBackups int, maxAgeDays int) error { + var err error + once.Do(func() { + defaultLogger, err = New(logDir, logFileName, maxSizeMB, maxBackups, maxAgeDays) + if err != nil { + return + } + // Replace standard log output with the multi-writer + multiWriter := io.MultiWriter(os.Stdout, defaultLogger.fileWriter) + log.SetOutput(multiWriter) + log.SetFlags(log.LstdFlags | log.Lshortfile) + }) + return err +} + +// New creates a new logger that writes to both stdout and a log file +func New(logDir, logFileName string, maxSizeMB int, maxBackups int, maxAgeDays int) (*Logger, error) { + // Ensure log directory exists + if err := os.MkdirAll(logDir, 0755); err != nil { + return nil, err + } + + logPath := filepath.Join(logDir, logFileName) + + // Create file writer with rotation + fileWriter := &lumberjack.Logger{ + Filename: logPath, + MaxSize: maxSizeMB, // megabytes + MaxBackups: maxBackups, // number of backup files + MaxAge: maxAgeDays, // days + Compress: true, // compress old log files + } + + // Create multi-writer that writes to both stdout and file + multiWriter := io.MultiWriter(os.Stdout, fileWriter) + + // Create logger with standard flags + logger := log.New(multiWriter, "", log.LstdFlags|log.Lshortfile) + + return &Logger{ + Logger: logger, + fileWriter: fileWriter, + }, nil +} + +// Close closes the file writer +func (l *Logger) Close() error { + if l.fileWriter != nil { + return l.fileWriter.Close() + } + return nil +} + +// GetDefault returns the default logger instance +func GetDefault() *Logger { + return defaultLogger +} + +// Printf logs a formatted message +func Printf(format string, v ...interface{}) { + if defaultLogger != nil { + defaultLogger.Printf(format, v...) + } else { + log.Printf(format, v...) + } +} + +// Print logs a message +func Print(v ...interface{}) { + if defaultLogger != nil { + defaultLogger.Print(v...) + } else { + log.Print(v...) + } +} + +// Println logs a message with newline +func Println(v ...interface{}) { + if defaultLogger != nil { + defaultLogger.Println(v...) + } else { + log.Println(v...) + } +} + +// Fatal logs a message and exits +func Fatal(v ...interface{}) { + if defaultLogger != nil { + defaultLogger.Fatal(v...) + } else { + log.Fatal(v...) + } +} + +// Fatalf logs a formatted message and exits +func Fatalf(format string, v ...interface{}) { + if defaultLogger != nil { + defaultLogger.Fatalf(format, v...) + } else { + log.Fatalf(format, v...) + } +} + diff --git a/internal/runner/client.go b/internal/runner/client.go index bfc3fee..35bd9c2 100644 --- a/internal/runner/client.go +++ b/internal/runner/client.go @@ -1,10 +1,10 @@ package runner import ( + _ "embed" "archive/tar" "bufio" "bytes" - "compress/gzip" "encoding/json" "errors" "fmt" @@ -22,6 +22,7 @@ import ( "sync" "time" + "jiggablend/pkg/scripts" "jiggablend/pkg/types" "github.com/gorilla/websocket" @@ -53,18 +54,20 @@ type Client struct { vaapiDevicesMu sync.RWMutex // Protects vaapiDevices allocatedDevices map[int64]string // map[taskID]device - tracks which device is allocated to which task allocatedDevicesMu sync.RWMutex // Protects allocatedDevices + longRunningClient *http.Client // HTTP client for long-running operations (no timeout) } // NewClient creates a new runner client func NewClient(managerURL, name, hostname, ipAddress string) *Client { return &Client{ - managerURL: managerURL, - name: name, - hostname: hostname, - ipAddress: ipAddress, - httpClient: &http.Client{Timeout: 30 * time.Second}, - stopChan: make(chan struct{}), - stepStartTimes: make(map[string]time.Time), + managerURL: managerURL, + name: name, + hostname: hostname, + ipAddress: ipAddress, + httpClient: &http.Client{Timeout: 30 * time.Second}, + longRunningClient: &http.Client{Timeout: 0}, // No timeout for long-running operations (context downloads, file uploads/downloads) + stopChan: make(chan struct{}), + stepStartTimes: make(map[string]time.Time), } } @@ -465,6 +468,17 @@ func (c *Client) Register(registrationToken string) (int64, string, string, erro // doSignedRequest performs an authenticated HTTP request using shared secret // queryParams is optional and will be appended to the URL func (c *Client) doSignedRequest(method, path string, body []byte, queryParams ...string) (*http.Response, error) { + return c.doSignedRequestWithClient(method, path, body, c.httpClient, queryParams...) +} + +// doSignedRequestLong performs an authenticated HTTP request using the long-running client (no timeout) +// Use this for context downloads, file uploads/downloads, and other operations that may take a long time +func (c *Client) doSignedRequestLong(method, path string, body []byte, queryParams ...string) (*http.Response, error) { + return c.doSignedRequestWithClient(method, path, body, c.longRunningClient, queryParams...) +} + +// doSignedRequestWithClient performs an authenticated HTTP request using the specified client +func (c *Client) doSignedRequestWithClient(method, path string, body []byte, client *http.Client, queryParams ...string) (*http.Response, error) { if c.runnerSecret == "" { return nil, fmt.Errorf("runner not authenticated") } @@ -483,7 +497,7 @@ func (c *Client) doSignedRequest(method, path string, body []byte, queryParams . req.Header.Set("Content-Type", "application/json") req.Header.Set("X-Runner-Secret", c.runnerSecret) - return c.httpClient.Do(req) + return client.Do(req) } // ConnectWebSocket establishes a WebSocket connection to the manager @@ -969,16 +983,16 @@ func (c *Client) processTask(task map[string]interface{}, jobName string, output // Clean up expired cache entries periodically c.cleanupExpiredContextCache() - // Download context tar.gz - contextPath := filepath.Join(workDir, "context.tar.gz") + // Download context tar + contextPath := filepath.Join(workDir, "context.tar") if err := c.downloadJobContext(jobID, contextPath); err != nil { c.sendStepUpdate(taskID, "download", types.StepStatusFailed, err.Error()) return fmt.Errorf("failed to download context: %w", err) } - // Extract context tar.gz + // Extract context tar c.sendLog(taskID, types.LogLevelInfo, "Extracting context...", "download") - if err := c.extractTarGz(contextPath, workDir); err != nil { + if err := c.extractTar(contextPath, workDir); err != nil { c.sendStepUpdate(taskID, "download", types.StepStatusFailed, err.Error()) return fmt.Errorf("failed to extract context: %w", err) } @@ -1077,662 +1091,24 @@ func (c *Client) processTask(task map[string]interface{}, jobName string, output // This script will override the blend file's settings based on job metadata formatFilePath := filepath.Join(workDir, "output_format.txt") renderSettingsFilePath := filepath.Join(workDir, "render_settings.json") - scriptContent := fmt.Sprintf(`import bpy -import sys -import os -import json - -# Make all file paths relative to the blend file location FIRST -# This must be done immediately after file load, before any other operations -# to prevent Blender from trying to access external files with absolute paths -try: - bpy.ops.file.make_paths_relative() - print("Made all file paths relative to blend file") -except Exception as e: - print(f"Warning: Could not make paths relative: {e}") - -# Check for missing addons that the blend file requires -# Blender marks missing addons with "_missing" suffix in preferences -missing = [] -try: - for mod in bpy.context.preferences.addons: - if mod.module.endswith("_missing"): - missing.append(mod.module.rsplit("_", 1)[0]) - - if missing: - print("Missing add-ons required by this .blend:") - for name in missing: - print(" -", name) - else: - print("No missing add-ons detected – file is headless-safe") -except Exception as e: - print(f"Warning: Could not check for missing addons: {e}") - -# Fix objects and collections hidden from render -vl = bpy.context.view_layer - -# 1. Objects hidden in view layer -print("Checking for objects hidden from render that need to be enabled...") -try: - for obj in bpy.data.objects: - if obj.hide_get(view_layer=vl): - if any(k in obj.name.lower() for k in ["scrotum|","cage","genital","penis","dick","collision","body.001","couch"]): - obj.hide_set(False, view_layer=vl) - print("Enabled object:", obj.name) -except Exception as e: - print(f"Warning: Could not check/fix hidden render objects: {e}") - -# 2. Collections disabled in renders OR set to Holdout (the final killer) -print("Checking for collections hidden from render that need to be enabled...") -try: - for col in bpy.data.collections: - if col.hide_render or (vl.layer_collection.children.get(col.name) and not vl.layer_collection.children[col.name].exclude == False): - if any(k in col.name.lower() for k in ["genital","nsfw","dick","private","hidden","cage","scrotum","collision","dick"]): - col.hide_render = False - if col.name in vl.layer_collection.children: - vl.layer_collection.children[col.name].exclude = False - vl.layer_collection.children[col.name].holdout = False - vl.layer_collection.children[col.name].indirect_only = False - print("Enabled collection:", col.name) -except Exception as e: - print(f"Warning: Could not check/fix hidden render collections: {e}") - -# Read output format from file (created by Go code) -format_file_path = %q -output_format_override = None -if os.path.exists(format_file_path): - try: - with open(format_file_path, 'r') as f: - output_format_override = f.read().strip().upper() - print(f"Read output format from file: '{output_format_override}'") - except Exception as e: - print(f"Warning: Could not read output format file: {e}") -else: - print(f"Warning: Output format file does not exist: {format_file_path}") - -# Read render settings from JSON file (created by Go code) -render_settings_file = %q -render_settings_override = None -if os.path.exists(render_settings_file): - try: - with open(render_settings_file, 'r') as f: - render_settings_override = json.load(f) - print(f"Loaded render settings from job metadata") - except Exception as e: - print(f"Warning: Could not read render settings file: {e}") -`, formatFilePath, renderSettingsFilePath) + ` - -# Get current scene settings (preserve blend file preferences) -scene = bpy.context.scene -current_engine = scene.render.engine -current_device = scene.cycles.device if hasattr(scene, 'cycles') and scene.cycles else None -current_output_format = scene.render.image_settings.file_format - -print(f"Blend file render engine: {current_engine}") -if current_device: - print(f"Blend file device setting: {current_device}") -print(f"Blend file output format: {current_output_format}") - -# Override output format if specified -# The format file always takes precedence (it's written specifically for this job) -if output_format_override: - print(f"Overriding output format from '{current_output_format}' to '{output_format_override}'") - # Map common format names to Blender's format constants - # For video formats (EXR_264_MP4, EXR_AV1_MP4), we render as EXR frames first - format_to_use = output_format_override.upper() - if format_to_use in ['EXR_264_MP4', 'EXR_AV1_MP4']: - format_to_use = 'EXR' # Render as EXR for video formats - - format_map = { - 'PNG': 'PNG', - 'JPEG': 'JPEG', - 'JPG': 'JPEG', - 'EXR': 'OPEN_EXR', - 'OPEN_EXR': 'OPEN_EXR', - 'TARGA': 'TARGA', - 'TIFF': 'TIFF', - 'BMP': 'BMP', - } - blender_format = format_map.get(format_to_use, format_to_use) - try: - scene.render.image_settings.file_format = blender_format - print(f"Successfully set output format to: {blender_format}") - except Exception as e: - print(f"Warning: Could not set output format to {blender_format}: {e}") - print(f"Using blend file's format: {current_output_format}") -else: - print(f"Using blend file's output format: {current_output_format}") - -# Apply render settings from job metadata if provided -# Note: output_format is NOT applied from render_settings_override - it's already set from format file above -if render_settings_override: - engine_override = render_settings_override.get('engine', '').upper() - engine_settings = render_settings_override.get('engine_settings', {}) - - # Switch engine if specified - if engine_override and engine_override != current_engine.upper(): - print(f"Switching render engine from '{current_engine}' to '{engine_override}'") - try: - scene.render.engine = engine_override - current_engine = engine_override - print(f"Successfully switched to {engine_override} engine") - except Exception as e: - print(f"Warning: Could not switch engine to {engine_override}: {e}") - print(f"Using blend file's engine: {current_engine}") - - # Apply engine-specific settings - if engine_settings: - if current_engine.upper() == 'CYCLES': - cycles = scene.cycles - print("Applying Cycles render settings from job metadata...") - for key, value in engine_settings.items(): - try: - if hasattr(cycles, key): - setattr(cycles, key, value) - print(f" Set Cycles.{key} = {value}") - else: - print(f" Warning: Cycles has no attribute '{key}'") - except Exception as e: - print(f" Warning: Could not set Cycles.{key} = {value}: {e}") - elif current_engine.upper() in ['EEVEE', 'EEVEE_NEXT']: - eevee = scene.eevee - print("Applying EEVEE render settings from job metadata...") - for key, value in engine_settings.items(): - try: - if hasattr(eevee, key): - setattr(eevee, key, value) - print(f" Set EEVEE.{key} = {value}") - else: - print(f" Warning: EEVEE has no attribute '{key}'") - except Exception as e: - print(f" Warning: Could not set EEVEE.{key} = {value}: {e}") - - # Apply resolution if specified - if 'resolution_x' in render_settings_override: - try: - scene.render.resolution_x = render_settings_override['resolution_x'] - print(f"Set resolution_x = {render_settings_override['resolution_x']}") - except Exception as e: - print(f"Warning: Could not set resolution_x: {e}") - if 'resolution_y' in render_settings_override: - try: - scene.render.resolution_y = render_settings_override['resolution_y'] - print(f"Set resolution_y = {render_settings_override['resolution_y']}") - except Exception as e: - print(f"Warning: Could not set resolution_y: {e}") - -# Only override device selection if using Cycles (other engines handle GPU differently) -if current_engine == 'CYCLES': - # Check if CPU rendering is forced - force_cpu = False - if render_settings_override and render_settings_override.get('force_cpu'): - force_cpu = render_settings_override.get('force_cpu', False) - print("Force CPU rendering is enabled - skipping GPU detection") - - # Ensure Cycles addon is enabled - try: - if 'cycles' not in bpy.context.preferences.addons: - bpy.ops.preferences.addon_enable(module='cycles') - print("Enabled Cycles addon") - except Exception as e: - print(f"Warning: Could not enable Cycles addon: {e}") - - # If CPU is forced, skip GPU detection and set CPU directly - if force_cpu: - scene.cycles.device = 'CPU' - print("Forced CPU rendering (skipping GPU detection)") - else: - # Access Cycles preferences - prefs = bpy.context.preferences - try: - cycles_prefs = prefs.addons['cycles'].preferences - except (KeyError, AttributeError): - try: - cycles_addon = prefs.addons.get('cycles') - if cycles_addon: - cycles_prefs = cycles_addon.preferences - else: - raise Exception("Cycles addon not found") - except Exception as e: - print(f"ERROR: Could not access Cycles preferences: {e}") - import traceback - traceback.print_exc() - sys.exit(1) - - # Check all devices and choose the best GPU type - # Device type preference order (most performant first) - device_type_preference = ['OPTIX', 'CUDA', 'HIP', 'ONEAPI', 'METAL'] - gpu_available = False - best_device_type = None - best_gpu_devices = [] - devices_by_type = {} # {device_type: [devices]} - seen_device_ids = set() # Track device IDs to avoid duplicates - - print("Checking for GPU availability...") - - # Try to get all devices - try each device type to see what's available - for device_type in device_type_preference: - try: - cycles_prefs.compute_device_type = device_type - cycles_prefs.refresh_devices() - - # Get devices for this type - devices = None - if hasattr(cycles_prefs, 'devices'): - try: - devices_prop = cycles_prefs.devices - if devices_prop: - devices = list(devices_prop) if hasattr(devices_prop, '__iter__') else [devices_prop] - except Exception as e: - pass - - if not devices or len(devices) == 0: - try: - devices = cycles_prefs.get_devices() - except Exception as e: - pass - - if devices and len(devices) > 0: - # Categorize devices by their type attribute, avoiding duplicates - for device in devices: - if hasattr(device, 'type'): - device_type_str = str(device.type).upper() - device_id = getattr(device, 'id', None) - - # Use device ID to avoid duplicates (same device appears when checking different compute_device_types) - if device_id and device_id in seen_device_ids: - continue - - if device_id: - seen_device_ids.add(device_id) - - if device_type_str not in devices_by_type: - devices_by_type[device_type_str] = [] - devices_by_type[device_type_str].append(device) - except (ValueError, AttributeError, KeyError, TypeError): - # Device type not supported, continue - continue - except Exception as e: - # Other errors - log but continue - print(f" Error checking {device_type}: {e}") - continue - - # Print what we found - print(f"Found devices by type: {list(devices_by_type.keys())}") - for dev_type, dev_list in devices_by_type.items(): - print(f" {dev_type}: {len(dev_list)} device(s)") - for device in dev_list: - device_name = getattr(device, 'name', 'Unknown') - print(f" - {device_name}") - - # Choose the best GPU type based on preference - for preferred_type in device_type_preference: - if preferred_type in devices_by_type: - gpu_devices = [d for d in devices_by_type[preferred_type] if preferred_type in ['CUDA', 'OPENCL', 'OPTIX', 'HIP', 'METAL', 'ONEAPI']] - if gpu_devices: - best_device_type = preferred_type - best_gpu_devices = [(d, preferred_type) for d in gpu_devices] - print(f"Selected {preferred_type} as best GPU type with {len(gpu_devices)} device(s)") - break - - # Second pass: Enable the best GPU we found - if best_device_type and best_gpu_devices: - print(f"\nEnabling GPU devices for {best_device_type}...") - try: - # Set the device type again - cycles_prefs.compute_device_type = best_device_type - cycles_prefs.refresh_devices() - - # First, disable all CPU devices to ensure only GPU is used - print(f" Disabling CPU devices...") - all_devices = cycles_prefs.devices if hasattr(cycles_prefs, 'devices') else cycles_prefs.get_devices() - if all_devices: - for device in all_devices: - if hasattr(device, 'type') and str(device.type).upper() == 'CPU': - try: - device.use = False - device_name = getattr(device, 'name', 'Unknown') - print(f" Disabled CPU: {device_name}") - except Exception as e: - print(f" Warning: Could not disable CPU device {getattr(device, 'name', 'Unknown')}: {e}") - - # Enable all GPU devices - enabled_count = 0 - for device, device_type in best_gpu_devices: - try: - device.use = True - enabled_count += 1 - device_name = getattr(device, 'name', 'Unknown') - print(f" Enabled: {device_name}") - except Exception as e: - print(f" Warning: Could not enable device {getattr(device, 'name', 'Unknown')}: {e}") - - # Enable ray tracing acceleration for supported device types - try: - if best_device_type == 'HIP': - # HIPRT (HIP Ray Tracing) for AMD GPUs - if hasattr(cycles_prefs, 'use_hiprt'): - cycles_prefs.use_hiprt = True - print(f" Enabled HIPRT (HIP Ray Tracing) for faster rendering") - elif hasattr(scene.cycles, 'use_hiprt'): - scene.cycles.use_hiprt = True - print(f" Enabled HIPRT (HIP Ray Tracing) for faster rendering") - else: - print(f" HIPRT not available (requires Blender 4.0+)") - elif best_device_type == 'OPTIX': - # OptiX is already enabled when using OPTIX device type - # But we can check if there are any OptiX-specific settings - if hasattr(scene.cycles, 'use_optix_denoising'): - scene.cycles.use_optix_denoising = True - print(f" Enabled OptiX denoising") - print(f" OptiX ray tracing is active (using OPTIX device type)") - elif best_device_type == 'CUDA': - # CUDA can use OptiX if available, but it's usually automatic - # Check if we can prefer OptiX over CUDA - if hasattr(scene.cycles, 'use_optix_denoising'): - scene.cycles.use_optix_denoising = True - print(f" Enabled OptiX denoising (if OptiX available)") - print(f" CUDA ray tracing active") - elif best_device_type == 'METAL': - # MetalRT for Apple Silicon (if available) - if hasattr(scene.cycles, 'use_metalrt'): - scene.cycles.use_metalrt = True - print(f" Enabled MetalRT (Metal Ray Tracing) for faster rendering") - elif hasattr(cycles_prefs, 'use_metalrt'): - cycles_prefs.use_metalrt = True - print(f" Enabled MetalRT (Metal Ray Tracing) for faster rendering") - else: - print(f" MetalRT not available") - elif best_device_type == 'ONEAPI': - # Intel oneAPI - Embree might be available - if hasattr(scene.cycles, 'use_embree'): - scene.cycles.use_embree = True - print(f" Enabled Embree for faster CPU ray tracing") - print(f" oneAPI ray tracing active") - except Exception as e: - print(f" Could not enable ray tracing acceleration: {e}") - - print(f"SUCCESS: Enabled {enabled_count} GPU device(s) for {best_device_type}") - gpu_available = True - except Exception as e: - print(f"ERROR: Failed to enable GPU devices: {e}") - import traceback - traceback.print_exc() - - # Set device based on availability (prefer GPU, fallback to CPU) - if gpu_available: - scene.cycles.device = 'GPU' - print(f"Using GPU for rendering (blend file had: {current_device})") - else: - scene.cycles.device = 'CPU' - print(f"GPU not available, using CPU for rendering (blend file had: {current_device})") - - # Verify device setting - if current_engine == 'CYCLES': - final_device = scene.cycles.device - print(f"Final Cycles device: {final_device}") -else: - # For other engines (EEVEE, etc.), respect blend file settings - print(f"Using {current_engine} engine - respecting blend file settings") - -# Enable GPU acceleration for EEVEE viewport rendering (if using EEVEE) -if current_engine == 'EEVEE' or current_engine == 'EEVEE_NEXT': - try: - if hasattr(bpy.context.preferences.system, 'gpu_backend'): - bpy.context.preferences.system.gpu_backend = 'OPENGL' - print("Enabled OpenGL GPU backend for EEVEE") - except Exception as e: - print(f"Could not set EEVEE GPU backend: {e}") - -# Enable GPU acceleration for compositing (if compositing is enabled) -try: - if scene.use_nodes and hasattr(scene, 'node_tree') and scene.node_tree: - if hasattr(scene.node_tree, 'use_gpu_compositing'): - scene.node_tree.use_gpu_compositing = True - print("Enabled GPU compositing") -except Exception as e: - print(f"Could not enable GPU compositing: {e}") - -# CRITICAL: Initialize headless rendering to prevent black images -# This ensures the render engine is properly initialized before rendering -print("Initializing headless rendering context...") -try: - # Ensure world exists and has proper settings - if not scene.world: - # Create a default world if none exists - world = bpy.data.worlds.new("World") - scene.world = world - print("Created default world") - - # Ensure world has a background shader (not just black) - if scene.world: - # Enable nodes if not already enabled - if not scene.world.use_nodes: - scene.world.use_nodes = True - print("Enabled world nodes") - - world_nodes = scene.world.node_tree - if world_nodes: - # Find or create background shader - bg_shader = None - for node in world_nodes.nodes: - if node.type == 'BACKGROUND': - bg_shader = node - break - - if not bg_shader: - bg_shader = world_nodes.nodes.new(type='ShaderNodeBackground') - # Connect to output - output = world_nodes.nodes.get('World Output') - if not output: - output = world_nodes.nodes.new(type='ShaderNodeOutputWorld') - output.name = 'World Output' - if output and bg_shader: - # Connect background to surface input - if 'Surface' in output.inputs and 'Background' in bg_shader.outputs: - world_nodes.links.new(bg_shader.outputs['Background'], output.inputs['Surface']) - print("Created background shader for world") - - # Ensure background has some color (not pure black) - if bg_shader: - # Only set if it's pure black (0,0,0) - if hasattr(bg_shader.inputs, 'Color'): - color = bg_shader.inputs['Color'].default_value - if len(color) >= 3 and color[0] == 0.0 and color[1] == 0.0 and color[2] == 0.0: - # Set to a very dark gray instead of pure black - bg_shader.inputs['Color'].default_value = (0.01, 0.01, 0.01, 1.0) - print("Adjusted world background color to prevent black renders") - else: - # Fallback: use legacy world color if nodes aren't working - if hasattr(scene.world, 'color'): - color = scene.world.color - if len(color) >= 3 and color[0] == 0.0 and color[1] == 0.0 and color[2] == 0.0: - scene.world.color = (0.01, 0.01, 0.01) - print("Adjusted legacy world color to prevent black renders") - - # For EEVEE, force viewport update to initialize render engine - if current_engine in ['EEVEE', 'EEVEE_NEXT']: - # Force EEVEE to update its internal state - try: - # Update depsgraph to ensure everything is initialized - depsgraph = bpy.context.evaluated_depsgraph_get() - if depsgraph: - # Force update - depsgraph.update() - print("Forced EEVEE depsgraph update for headless rendering") - except Exception as e: - print(f"Warning: Could not force EEVEE update: {e}") - - # Ensure EEVEE settings are applied - try: - # Force a material update to ensure shaders are compiled - for obj in scene.objects: - if obj.type == 'MESH' and obj.data.materials: - for mat in obj.data.materials: - if mat and mat.use_nodes: - # Touch the material to force update - mat.use_nodes = mat.use_nodes - print("Forced material updates for EEVEE") - except Exception as e: - print(f"Warning: Could not update materials: {e}") - - # For Cycles, ensure proper initialization - if current_engine == 'CYCLES': - # Ensure samples are set (even if 1 for preview) - if not hasattr(scene.cycles, 'samples') or scene.cycles.samples < 1: - scene.cycles.samples = 1 - print("Set minimum Cycles samples") - - # Check for lights in the scene - lights = [obj for obj in scene.objects if obj.type == 'LIGHT'] - print(f"Found {len(lights)} light(s) in scene") - if len(lights) == 0: - print("WARNING: No lights found in scene - rendering may be black!") - print(" Consider adding lights or ensuring world background emits light") - - # Ensure world background emits light (critical for Cycles) - if scene.world and scene.world.use_nodes: - world_nodes = scene.world.node_tree - if world_nodes: - bg_shader = None - for node in world_nodes.nodes: - if node.type == 'BACKGROUND': - bg_shader = node - break - - if bg_shader: - # Check and set strength - Cycles needs this to emit light! - if hasattr(bg_shader.inputs, 'Strength'): - strength = bg_shader.inputs['Strength'].default_value - if strength <= 0.0: - bg_shader.inputs['Strength'].default_value = 1.0 - print("Set world background strength to 1.0 for Cycles lighting") - else: - print(f"World background strength: {strength}") - # Also ensure color is not pure black - if hasattr(bg_shader.inputs, 'Color'): - color = bg_shader.inputs['Color'].default_value - if len(color) >= 3 and color[0] == 0.0 and color[1] == 0.0 and color[2] == 0.0: - bg_shader.inputs['Color'].default_value = (1.0, 1.0, 1.0, 1.0) - print("Set world background color to white for Cycles lighting") - - # Check film_transparent setting - if enabled, background will be transparent/black - if hasattr(scene.cycles, 'film_transparent') and scene.cycles.film_transparent: - print("WARNING: film_transparent is enabled - background will be transparent") - print(" If you see black renders, try disabling film_transparent") - - # Force Cycles to update/compile materials and shaders - try: - # Update depsgraph to ensure everything is initialized - depsgraph = bpy.context.evaluated_depsgraph_get() - if depsgraph: - depsgraph.update() - print("Forced Cycles depsgraph update") - - # Force material updates to ensure shaders are compiled - for obj in scene.objects: - if obj.type == 'MESH' and obj.data.materials: - for mat in obj.data.materials: - if mat and mat.use_nodes: - # Force material update - mat.use_nodes = mat.use_nodes - print("Forced Cycles material updates") - except Exception as e: - print(f"Warning: Could not force Cycles updates: {e}") - - # Verify device is actually set correctly - if hasattr(scene.cycles, 'device'): - actual_device = scene.cycles.device - print(f"Cycles device setting: {actual_device}") - if actual_device == 'GPU': - # Try to verify GPU is actually available - try: - prefs = bpy.context.preferences - cycles_prefs = prefs.addons['cycles'].preferences - devices = cycles_prefs.devices - enabled_devices = [d for d in devices if d.use] - if len(enabled_devices) == 0: - print("WARNING: GPU device set but no GPU devices are enabled!") - print(" Falling back to CPU may cause issues") - except Exception as e: - print(f"Could not verify GPU devices: {e}") - - # Ensure camera exists and is active - if scene.camera is None: - # Find first camera in scene - for obj in scene.objects: - if obj.type == 'CAMERA': - scene.camera = obj - print(f"Set active camera: {obj.name}") - break - - # Fix objects and collections hidden from render - vl = bpy.context.view_layer - - # 1. Objects hidden in view layer - for obj in bpy.data.objects: - if obj.hide_get(view_layer=vl): - if any(k in obj.name.lower() for k in ["scrotum|","cage","genital","penis","dick","collision","body.001","couch"]): - obj.hide_set(False, view_layer=vl) - print("Enabled object:", obj.name) - - # 2. Collections disabled in renders OR set to Holdout (the final killer) - for col in bpy.data.collections: - if col.hide_render or (vl.layer_collection.children.get(col.name) and not vl.layer_collection.children[col.name].exclude == False): - if any(k in col.name.lower() for k in ["genital","nsfw","dick","private","hidden","cage","scrotum","collision","dick"]): - col.hide_render = False - if col.name in vl.layer_collection.children: - vl.layer_collection.children[col.name].exclude = False - vl.layer_collection.children[col.name].holdout = False - vl.layer_collection.children[col.name].indirect_only = False - print("Enabled collection:", col.name) - - print("Headless rendering initialization complete") -except Exception as e: - print(f"Warning: Headless rendering initialization had issues: {e}") - import traceback - traceback.print_exc() - -# Final verification before rendering -print("\n=== Pre-render verification ===") -try: - scene = bpy.context.scene - print(f"Render engine: {scene.render.engine}") - print(f"Active camera: {scene.camera.name if scene.camera else 'None'}") - - if scene.render.engine == 'CYCLES': - print(f"Cycles device: {scene.cycles.device}") - print(f"Cycles samples: {scene.cycles.samples}") - lights = [obj for obj in scene.objects if obj.type == 'LIGHT'] - print(f"Lights in scene: {len(lights)}") - if scene.world: - if scene.world.use_nodes: - world_nodes = scene.world.node_tree - if world_nodes: - bg_shader = None - for node in world_nodes.nodes: - if node.type == 'BACKGROUND': - bg_shader = node - break - if bg_shader: - if hasattr(bg_shader.inputs, 'Strength'): - strength = bg_shader.inputs['Strength'].default_value - print(f"World background strength: {strength}") - if hasattr(bg_shader.inputs, 'Color'): - color = bg_shader.inputs['Color'].default_value - print(f"World background color: ({color[0]:.2f}, {color[1]:.2f}, {color[2]:.2f})") - else: - print("World exists but nodes are disabled") - else: - print("WARNING: No world in scene!") - - print("=== Verification complete ===\n") -except Exception as e: - print(f"Warning: Verification failed: {e}") - -print("Device configuration complete - blend file settings preserved, device optimized") -sys.stdout.flush() -` + + // Check if unhide_objects is enabled + unhideObjects := false + if jobMetadata != nil && jobMetadata.UnhideObjects != nil && *jobMetadata.UnhideObjects { + unhideObjects = true + } + + // Build unhide code conditionally from embedded script + unhideCode := "" + if unhideObjects { + unhideCode = scripts.UnhideObjects + } + + // Load template and replace placeholders + scriptContent := scripts.RenderBlenderTemplate + scriptContent = strings.ReplaceAll(scriptContent, "{{UNHIDE_CODE}}", unhideCode) + scriptContent = strings.ReplaceAll(scriptContent, "{{FORMAT_FILE_PATH}}", fmt.Sprintf("%q", formatFilePath)) + scriptContent = strings.ReplaceAll(scriptContent, "{{RENDER_SETTINGS_FILE}}", fmt.Sprintf("%q", renderSettingsFilePath)) scriptPath := filepath.Join(workDir, "enable_gpu.py") if err := os.WriteFile(scriptPath, []byte(scriptContent), 0644); err != nil { errMsg := fmt.Sprintf("failed to create GPU enable script: %v", err) @@ -1765,23 +1141,30 @@ sys.stdout.flush() } } + // Check if execution should be enabled (defaults to false/off) + enableExecution := false + if jobMetadata != nil && jobMetadata.EnableExecution != nil && *jobMetadata.EnableExecution { + enableExecution = true + } + // Run Blender with GPU enabled via Python script // Use -s (start) and -e (end) for frame ranges, or -f for single frame var cmd *exec.Cmd + args := []string{"-b", blendFile, "--python", scriptPath} + if enableExecution { + args = append(args, "--enable-autoexec") + } if frameStart == frameEnd { // Single frame - cmd = exec.Command("blender", "-b", blendFile, - "--python", scriptPath, - "-o", absOutputPattern, - "-f", fmt.Sprintf("%d", frameStart)) + args = append(args, "-o", absOutputPattern, "-f", fmt.Sprintf("%d", frameStart)) + cmd = exec.Command("blender", args...) } else { // Frame range - cmd = exec.Command("blender", "-b", blendFile, - "--python", scriptPath, - "-o", absOutputPattern, + args = append(args, "-o", absOutputPattern, "-s", fmt.Sprintf("%d", frameStart), "-e", fmt.Sprintf("%d", frameEnd), "-a") // -a renders animation (all frames in range) + cmd = exec.Command("blender", args...) } cmd.Dir = workDir @@ -3261,8 +2644,11 @@ func (c *Client) getJobMetadata(jobID int64) (*types.BlendMetadata, error) { // downloadFrameFile downloads a frame file for MP4 generation func (c *Client) downloadFrameFile(jobID int64, fileName, destPath string) error { - path := fmt.Sprintf("/api/runner/files/%d/%s", jobID, fileName) - resp, err := c.doSignedRequest("GET", path, nil, fmt.Sprintf("runner_id=%d", c.runnerID)) + // URL encode the fileName to handle special characters in filenames + encodedFileName := url.PathEscape(fileName) + path := fmt.Sprintf("/api/runner/files/%d/%s", jobID, encodedFileName) + // Use long-running client for file downloads (no timeout) - EXR files can be large + resp, err := c.doSignedRequestLong("GET", path, nil, fmt.Sprintf("runner_id=%d", c.runnerID)) if err != nil { return err } @@ -3330,7 +2716,8 @@ func (c *Client) downloadFileToPath(filePath, destPath string) error { downloadPath += "/" + filepath.Base(filePath) } - resp, err := c.doSignedRequest("GET", downloadPath, nil, fmt.Sprintf("runner_id=%d", c.runnerID)) + // Use long-running client for file downloads (no timeout) + resp, err := c.doSignedRequestLong("GET", downloadPath, nil, fmt.Sprintf("runner_id=%d", c.runnerID)) if err != nil { return fmt.Errorf("failed to download file: %w", err) } @@ -3392,7 +2779,8 @@ func (c *Client) uploadFile(jobID int64, filePath string) (string, error) { req.Header.Set("Content-Type", formWriter.FormDataContentType()) req.Header.Set("X-Runner-Secret", c.runnerSecret) - resp, err := c.httpClient.Do(req) + // Use long-running client for file uploads (no timeout) + resp, err := c.longRunningClient.Do(req) if err != nil { return "", fmt.Errorf("failed to upload file: %w", err) } @@ -3424,7 +2812,7 @@ func (c *Client) getContextCacheKey(jobID int64) string { func (c *Client) getContextCachePath(cacheKey string) string { cacheDir := filepath.Join(c.getWorkspaceDir(), "cache", "contexts") os.MkdirAll(cacheDir, 0755) - return filepath.Join(cacheDir, cacheKey+".tar.gz") + return filepath.Join(cacheDir, cacheKey+".tar") } // isContextCacheValid checks if a cached context file exists and is not expired (1 hour TTL) @@ -3437,7 +2825,7 @@ func (c *Client) isContextCacheValid(cachePath string) bool { return time.Since(info.ModTime()) < time.Hour } -// downloadJobContext downloads the job context tar.gz, using cache if available +// downloadJobContext downloads the job context tar, using cache if available func (c *Client) downloadJobContext(jobID int64, destPath string) error { cacheKey := c.getContextCacheKey(jobID) cachePath := c.getContextCachePath(cacheKey) @@ -3464,9 +2852,9 @@ func (c *Client) downloadJobContext(jobID int64, destPath string) error { } } - // Download from manager - path := fmt.Sprintf("/api/runner/jobs/%d/context.tar.gz", jobID) - resp, err := c.doSignedRequest("GET", path, nil, fmt.Sprintf("runner_id=%d", c.runnerID)) + // Download from manager - use long-running client (no timeout) for large context files + path := fmt.Sprintf("/api/runner/jobs/%d/context.tar", jobID) + resp, err := c.doSignedRequestLong("GET", path, nil, fmt.Sprintf("runner_id=%d", c.runnerID)) if err != nil { return fmt.Errorf("failed to download context: %w", err) } @@ -3517,24 +2905,17 @@ func (c *Client) downloadJobContext(jobID int64, destPath string) error { return nil } -// extractTarGz extracts a tar.gz file to the destination directory -func (c *Client) extractTarGz(tarGzPath, destDir string) error { - // Open the tar.gz file - file, err := os.Open(tarGzPath) +// extractTar extracts a tar file to the destination directory +func (c *Client) extractTar(tarPath, destDir string) error { + // Open the tar file + file, err := os.Open(tarPath) if err != nil { - return fmt.Errorf("failed to open tar.gz file: %w", err) + return fmt.Errorf("failed to open tar file: %w", err) } defer file.Close() - // Create gzip reader - gzReader, err := gzip.NewReader(file) - if err != nil { - return fmt.Errorf("failed to create gzip reader: %w", err) - } - defer gzReader.Close() - // Create tar reader - tarReader := tar.NewReader(gzReader) + tarReader := tar.NewReader(file) // Extract files for { @@ -3635,16 +3016,16 @@ func (c *Client) processMetadataTask(task map[string]interface{}, jobID int64, i c.sendStepUpdate(taskID, "download", types.StepStatusRunning, "") c.sendLog(taskID, types.LogLevelInfo, "Downloading job context...", "download") - // Download context tar.gz - contextPath := filepath.Join(workDir, "context.tar.gz") + // Download context tar + contextPath := filepath.Join(workDir, "context.tar") if err := c.downloadJobContext(jobID, contextPath); err != nil { c.sendStepUpdate(taskID, "download", types.StepStatusFailed, err.Error()) return fmt.Errorf("failed to download context: %w", err) } - // Extract context tar.gz + // Extract context tar c.sendLog(taskID, types.LogLevelInfo, "Extracting context...", "download") - if err := c.extractTarGz(contextPath, workDir); err != nil { + if err := c.extractTar(contextPath, workDir); err != nil { c.sendStepUpdate(taskID, "download", types.StepStatusFailed, err.Error()) return fmt.Errorf("failed to extract context: %w", err) } @@ -3881,6 +3262,7 @@ sys.stdout.flush() } // Execute Blender with Python script + // Note: disable_execution flag is not applied to metadata extraction for safety cmd := exec.Command("blender", "-b", blendFile, "--python", scriptPath) cmd.Dir = workDir diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 52ff098..2bfdb85 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -3,9 +3,9 @@ package storage import ( "archive/tar" "archive/zip" - "compress/gzip" "fmt" "io" + "log" "os" "path/filepath" "strings" @@ -31,6 +31,7 @@ func (s *Storage) init() error { s.basePath, s.uploadsPath(), s.outputsPath(), + s.tempPath(), } for _, dir := range dirs { @@ -42,6 +43,28 @@ func (s *Storage) init() error { return nil } +// tempPath returns the path for temporary files +func (s *Storage) tempPath() string { + return filepath.Join(s.basePath, "temp") +} + +// BasePath returns the storage base path (for cleanup tasks) +func (s *Storage) BasePath() string { + return s.basePath +} + +// TempDir creates a temporary directory under the storage base path +// Returns the path to the temporary directory +func (s *Storage) TempDir(pattern string) (string, error) { + // Ensure temp directory exists + if err := os.MkdirAll(s.tempPath(), 0755); err != nil { + return "", fmt.Errorf("failed to create temp directory: %w", err) + } + + // Create temp directory under storage base path + return os.MkdirTemp(s.tempPath(), pattern) +} + // uploadsPath returns the path for uploads func (s *Storage) uploadsPath() string { return filepath.Join(s.basePath, "uploads") @@ -142,6 +165,13 @@ func (s *Storage) GetFileSize(filePath string) (int64, error) { // ExtractZip extracts a ZIP file to the destination directory // Returns a list of all extracted file paths func (s *Storage) ExtractZip(zipPath, destDir string) ([]string, error) { + log.Printf("Extracting ZIP archive: %s -> %s", zipPath, destDir) + + // Ensure destination directory exists + if err := os.MkdirAll(destDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create destination directory: %w", err) + } + r, err := zip.OpenReader(zipPath) if err != nil { return nil, fmt.Errorf("failed to open ZIP file: %w", err) @@ -149,12 +179,20 @@ func (s *Storage) ExtractZip(zipPath, destDir string) ([]string, error) { defer r.Close() var extractedFiles []string + fileCount := 0 + dirCount := 0 + + log.Printf("ZIP contains %d entries", len(r.File)) for _, f := range r.File { // Sanitize file path to prevent directory traversal destPath := filepath.Join(destDir, f.Name) - if !strings.HasPrefix(destPath, filepath.Clean(destDir)+string(os.PathSeparator)) { - return nil, fmt.Errorf("invalid file path in ZIP: %s", f.Name) + + cleanDestPath := filepath.Clean(destPath) + cleanDestDir := filepath.Clean(destDir) + if !strings.HasPrefix(cleanDestPath, cleanDestDir+string(os.PathSeparator)) && cleanDestPath != cleanDestDir { + log.Printf("ERROR: Invalid file path in ZIP - target: %s, destDir: %s", cleanDestPath, cleanDestDir) + return nil, fmt.Errorf("invalid file path in ZIP: %s (target: %s, destDir: %s)", f.Name, cleanDestPath, cleanDestDir) } // Create directory structure @@ -162,6 +200,7 @@ func (s *Storage) ExtractZip(zipPath, destDir string) ([]string, error) { if err := os.MkdirAll(destPath, 0755); err != nil { return nil, fmt.Errorf("failed to create directory: %w", err) } + dirCount++ continue } @@ -191,8 +230,10 @@ func (s *Storage) ExtractZip(zipPath, destDir string) ([]string, error) { } extractedFiles = append(extractedFiles, destPath) + fileCount++ } + log.Printf("ZIP extraction complete: %d files, %d directories extracted to %s", fileCount, dirCount, destDir) return extractedFiles, nil } @@ -261,15 +302,15 @@ func isBlenderSaveFile(filename string) bool { return false } -// CreateJobContext creates a tar.gz archive containing all job input files +// CreateJobContext creates a tar archive containing all job input files // Filters out Blender save files (.blend1, .blend2, etc.) // Uses temporary directories and streaming to handle large files efficiently func (s *Storage) CreateJobContext(jobID int64) (string, error) { jobPath := s.JobPath(jobID) - contextPath := filepath.Join(jobPath, "context.tar.gz") + contextPath := filepath.Join(jobPath, "context.tar") // Create temporary directory for staging - tmpDir, err := os.MkdirTemp("", "fuego-context-*") + tmpDir, err := os.MkdirTemp("", "jiggablend-context-*") if err != nil { return "", fmt.Errorf("failed to create temporary directory: %w", err) } @@ -320,17 +361,14 @@ func (s *Storage) CreateJobContext(jobID int64) (string, error) { return "", fmt.Errorf("no files found to include in context") } - // Create the tar.gz file using streaming + // Create the tar file using streaming contextFile, err := os.Create(contextPath) if err != nil { return "", fmt.Errorf("failed to create context file: %w", err) } defer contextFile.Close() - gzWriter := gzip.NewWriter(contextFile) - defer gzWriter.Close() - - tarWriter := tar.NewWriter(gzWriter) + tarWriter := tar.NewWriter(contextFile) defer tarWriter.Close() // Add each file to the tar archive @@ -383,9 +421,6 @@ func (s *Storage) CreateJobContext(jobID int64) (string, error) { if err := tarWriter.Close(); err != nil { return "", fmt.Errorf("failed to close tar writer: %w", err) } - if err := gzWriter.Close(); err != nil { - return "", fmt.Errorf("failed to close gzip writer: %w", err) - } if err := contextFile.Close(); err != nil { return "", fmt.Errorf("failed to close context file: %w", err) } @@ -393,12 +428,12 @@ func (s *Storage) CreateJobContext(jobID int64) (string, error) { return contextPath, nil } -// CreateJobContextFromDir creates a context archive (tar.gz) from files in a source directory +// CreateJobContextFromDir creates a context archive (tar) from files in a source directory // This is used during upload to immediately create the context archive as the primary artifact // excludeFiles is a set of relative paths (from sourceDir) to exclude from the context func (s *Storage) CreateJobContextFromDir(sourceDir string, jobID int64, excludeFiles ...string) (string, error) { jobPath := s.JobPath(jobID) - contextPath := filepath.Join(jobPath, "context.tar.gz") + contextPath := filepath.Join(jobPath, "context.tar") // Ensure job directory exists if err := os.MkdirAll(jobPath, 0755); err != nil { @@ -498,17 +533,14 @@ func (s *Storage) CreateJobContextFromDir(sourceDir string, jobID int64, exclude return "", fmt.Errorf("multiple .blend files found at root level in context archive (found %d, expected 1)", blendFilesAtRoot) } - // Create the tar.gz file using streaming + // Create the tar file using streaming contextFile, err := os.Create(contextPath) if err != nil { return "", fmt.Errorf("failed to create context file: %w", err) } defer contextFile.Close() - gzWriter := gzip.NewWriter(contextFile) - defer gzWriter.Close() - - tarWriter := tar.NewWriter(gzWriter) + tarWriter := tar.NewWriter(contextFile) defer tarWriter.Close() // Add each file to the tar archive @@ -560,9 +592,6 @@ func (s *Storage) CreateJobContextFromDir(sourceDir string, jobID int64, exclude if err := tarWriter.Close(); err != nil { return "", fmt.Errorf("failed to close tar writer: %w", err) } - if err := gzWriter.Close(); err != nil { - return "", fmt.Errorf("failed to close gzip writer: %w", err) - } if err := contextFile.Close(); err != nil { return "", fmt.Errorf("failed to close context file: %w", err) } diff --git a/pkg/scripts/scripts.go b/pkg/scripts/scripts.go new file mode 100644 index 0000000..ce78d1f --- /dev/null +++ b/pkg/scripts/scripts.go @@ -0,0 +1,13 @@ +package scripts + +import _ "embed" + +//go:embed scripts/extract_metadata.py +var ExtractMetadata string + +//go:embed scripts/unhide_objects.py +var UnhideObjects string + +//go:embed scripts/render_blender.py.template +var RenderBlenderTemplate string + diff --git a/pkg/scripts/scripts/extract_metadata.py b/pkg/scripts/scripts/extract_metadata.py new file mode 100644 index 0000000..8ac162e --- /dev/null +++ b/pkg/scripts/scripts/extract_metadata.py @@ -0,0 +1,173 @@ +import bpy +import json +import sys + +# Make all file paths relative to the blend file location FIRST +# This must be done immediately after file load, before any other operations +# to prevent Blender from trying to access external files with absolute paths +try: + bpy.ops.file.make_paths_relative() + print("Made all file paths relative to blend file") +except Exception as e: + print(f"Warning: Could not make paths relative: {e}") + +# Check for missing addons that the blend file requires +# Blender marks missing addons with "_missing" suffix in preferences +missing_files_info = { + "checked": False, + "has_missing": False, + "missing_files": [], + "missing_addons": [] +} + +try: + missing = [] + for mod in bpy.context.preferences.addons: + if mod.module.endswith("_missing"): + missing.append(mod.module.rsplit("_", 1)[0]) + + missing_files_info["checked"] = True + if missing: + missing_files_info["has_missing"] = True + missing_files_info["missing_addons"] = missing + print("Missing add-ons required by this .blend:") + for name in missing: + print(" -", name) + else: + print("No missing add-ons detected – file is headless-safe") +except Exception as e: + print(f"Warning: Could not check for missing addons: {e}") + missing_files_info["error"] = str(e) + +# Get scene +scene = bpy.context.scene + +# Extract frame range from scene settings +frame_start = scene.frame_start +frame_end = scene.frame_end + +# Also check for actual animation range (keyframes) +# Find the earliest and latest keyframes across all objects +animation_start = None +animation_end = None + +for obj in scene.objects: + if obj.animation_data and obj.animation_data.action: + action = obj.animation_data.action + if action.fcurves: + for fcurve in action.fcurves: + if fcurve.keyframe_points: + for keyframe in fcurve.keyframe_points: + frame = int(keyframe.co[0]) + if animation_start is None or frame < animation_start: + animation_start = frame + if animation_end is None or frame > animation_end: + animation_end = frame + +# Use animation range if available, otherwise use scene frame range +# If scene range seems wrong (start == end), prefer animation range +if animation_start is not None and animation_end is not None: + if frame_start == frame_end or (animation_start < frame_start or animation_end > frame_end): + # Use animation range if scene range is invalid or animation extends beyond it + frame_start = animation_start + frame_end = animation_end + +# Extract render settings +render = scene.render +resolution_x = render.resolution_x +resolution_y = render.resolution_y +engine = scene.render.engine.upper() + +# Determine output format from file format +output_format = render.image_settings.file_format + +# Extract engine-specific settings +engine_settings = {} + +if engine == 'CYCLES': + cycles = scene.cycles + engine_settings = { + "samples": getattr(cycles, 'samples', 128), + "use_denoising": getattr(cycles, 'use_denoising', False), + "denoising_radius": getattr(cycles, 'denoising_radius', 0), + "denoising_strength": getattr(cycles, 'denoising_strength', 0.0), + "device": getattr(cycles, 'device', 'CPU'), + "use_adaptive_sampling": getattr(cycles, 'use_adaptive_sampling', False), + "adaptive_threshold": getattr(cycles, 'adaptive_threshold', 0.01) if getattr(cycles, 'use_adaptive_sampling', False) else 0.01, + "use_fast_gi": getattr(cycles, 'use_fast_gi', False), + "light_tree": getattr(cycles, 'use_light_tree', False), + "use_light_linking": getattr(cycles, 'use_light_linking', False), + "caustics_reflective": getattr(cycles, 'caustics_reflective', False), + "caustics_refractive": getattr(cycles, 'caustics_refractive', False), + "blur_glossy": getattr(cycles, 'blur_glossy', 0.0), + "max_bounces": getattr(cycles, 'max_bounces', 12), + "diffuse_bounces": getattr(cycles, 'diffuse_bounces', 4), + "glossy_bounces": getattr(cycles, 'glossy_bounces', 4), + "transmission_bounces": getattr(cycles, 'transmission_bounces', 12), + "volume_bounces": getattr(cycles, 'volume_bounces', 0), + "transparent_max_bounces": getattr(cycles, 'transparent_max_bounces', 8), + "film_transparent": getattr(cycles, 'film_transparent', False), + "use_layer_samples": getattr(cycles, 'use_layer_samples', False), + } +elif engine == 'EEVEE' or engine == 'EEVEE_NEXT': + eevee = scene.eevee + engine_settings = { + "taa_render_samples": getattr(eevee, 'taa_render_samples', 64), + "use_bloom": getattr(eevee, 'use_bloom', False), + "bloom_threshold": getattr(eevee, 'bloom_threshold', 0.8), + "bloom_intensity": getattr(eevee, 'bloom_intensity', 0.05), + "bloom_radius": getattr(eevee, 'bloom_radius', 6.5), + "use_ssr": getattr(eevee, 'use_ssr', True), + "use_ssr_refraction": getattr(eevee, 'use_ssr_refraction', False), + "ssr_quality": getattr(eevee, 'ssr_quality', 'MEDIUM'), + "use_ssao": getattr(eevee, 'use_ssao', True), + "ssao_quality": getattr(eevee, 'ssao_quality', 'MEDIUM'), + "ssao_distance": getattr(eevee, 'ssao_distance', 0.2), + "ssao_factor": getattr(eevee, 'ssao_factor', 1.0), + "use_soft_shadows": getattr(eevee, 'use_soft_shadows', True), + "use_shadow_high_bitdepth": getattr(eevee, 'use_shadow_high_bitdepth', True), + "use_volumetric": getattr(eevee, 'use_volumetric', False), + "volumetric_tile_size": getattr(eevee, 'volumetric_tile_size', '8'), + "volumetric_samples": getattr(eevee, 'volumetric_samples', 64), + "volumetric_start": getattr(eevee, 'volumetric_start', 0.0), + "volumetric_end": getattr(eevee, 'volumetric_end', 100.0), + "use_volumetric_lights": getattr(eevee, 'use_volumetric_lights', True), + "use_volumetric_shadows": getattr(eevee, 'use_volumetric_shadows', True), + "use_gtao": getattr(eevee, 'use_gtao', False), + "gtao_quality": getattr(eevee, 'gtao_quality', 'MEDIUM'), + "use_overscan": getattr(eevee, 'use_overscan', False), + } +else: + # For other engines, extract basic samples if available + engine_settings = { + "samples": getattr(scene, 'samples', 128) if hasattr(scene, 'samples') else 128 + } + +# Extract scene info +camera_count = len([obj for obj in scene.objects if obj.type == 'CAMERA']) +object_count = len(scene.objects) +material_count = len(bpy.data.materials) + +# Build metadata dictionary +metadata = { + "frame_start": frame_start, + "frame_end": frame_end, + "render_settings": { + "resolution_x": resolution_x, + "resolution_y": resolution_y, + "output_format": output_format, + "engine": engine.lower(), + "engine_settings": engine_settings + }, + "scene_info": { + "camera_count": camera_count, + "object_count": object_count, + "material_count": material_count + }, + "missing_files_info": missing_files_info +} + +# Output as JSON +print(json.dumps(metadata)) +sys.stdout.flush() + diff --git a/pkg/scripts/scripts/render_blender.py.template b/pkg/scripts/scripts/render_blender.py.template new file mode 100644 index 0000000..a5af2a8 --- /dev/null +++ b/pkg/scripts/scripts/render_blender.py.template @@ -0,0 +1,589 @@ +import bpy +import sys +import os +import json + +# Make all file paths relative to the blend file location FIRST +# This must be done immediately after file load, before any other operations +# to prevent Blender from trying to access external files with absolute paths +try: + bpy.ops.file.make_paths_relative() + print("Made all file paths relative to blend file") +except Exception as e: + print(f"Warning: Could not make paths relative: {e}") + +{{UNHIDE_CODE}} +# Read output format from file (created by Go code) +format_file_path = {{FORMAT_FILE_PATH}} +output_format_override = None +if os.path.exists(format_file_path): + try: + with open(format_file_path, 'r') as f: + output_format_override = f.read().strip().upper() + print(f"Read output format from file: '{output_format_override}'") + except Exception as e: + print(f"Warning: Could not read output format file: {e}") +else: + print(f"Warning: Output format file does not exist: {format_file_path}") + +# Read render settings from JSON file (created by Go code) +render_settings_file = {{RENDER_SETTINGS_FILE}} +render_settings_override = None +if os.path.exists(render_settings_file): + try: + with open(render_settings_file, 'r') as f: + render_settings_override = json.load(f) + print(f"Loaded render settings from job metadata") + except Exception as e: + print(f"Warning: Could not read render settings file: {e}") + +# Get current scene settings (preserve blend file preferences) +scene = bpy.context.scene +current_engine = scene.render.engine +current_device = scene.cycles.device if hasattr(scene, 'cycles') and scene.cycles else None +current_output_format = scene.render.image_settings.file_format + +print(f"Blend file render engine: {current_engine}") +if current_device: + print(f"Blend file device setting: {current_device}") +print(f"Blend file output format: {current_output_format}") + +# Override output format if specified +# The format file always takes precedence (it's written specifically for this job) +if output_format_override: + print(f"Overriding output format from '{current_output_format}' to '{output_format_override}'") + # Map common format names to Blender's format constants + # For video formats (EXR_264_MP4, EXR_AV1_MP4), we render as EXR frames first + format_to_use = output_format_override.upper() + if format_to_use in ['EXR_264_MP4', 'EXR_AV1_MP4']: + format_to_use = 'EXR' # Render as EXR for video formats + + format_map = { + 'PNG': 'PNG', + 'JPEG': 'JPEG', + 'JPG': 'JPEG', + 'EXR': 'OPEN_EXR', + 'OPEN_EXR': 'OPEN_EXR', + 'TARGA': 'TARGA', + 'TIFF': 'TIFF', + 'BMP': 'BMP', + } + blender_format = format_map.get(format_to_use, format_to_use) + try: + scene.render.image_settings.file_format = blender_format + print(f"Successfully set output format to: {blender_format}") + except Exception as e: + print(f"Warning: Could not set output format to {blender_format}: {e}") + print(f"Using blend file's format: {current_output_format}") +else: + print(f"Using blend file's output format: {current_output_format}") + +# Apply render settings from job metadata if provided +# Note: output_format is NOT applied from render_settings_override - it's already set from format file above +if render_settings_override: + engine_override = render_settings_override.get('engine', '').upper() + engine_settings = render_settings_override.get('engine_settings', {}) + + # Switch engine if specified + if engine_override and engine_override != current_engine.upper(): + print(f"Switching render engine from '{current_engine}' to '{engine_override}'") + try: + scene.render.engine = engine_override + current_engine = engine_override + print(f"Successfully switched to {engine_override} engine") + except Exception as e: + print(f"Warning: Could not switch engine to {engine_override}: {e}") + print(f"Using blend file's engine: {current_engine}") + + # Apply engine-specific settings + if engine_settings: + if current_engine.upper() == 'CYCLES': + cycles = scene.cycles + print("Applying Cycles render settings from job metadata...") + for key, value in engine_settings.items(): + try: + if hasattr(cycles, key): + setattr(cycles, key, value) + print(f" Set Cycles.{key} = {value}") + else: + print(f" Warning: Cycles has no attribute '{key}'") + except Exception as e: + print(f" Warning: Could not set Cycles.{key} = {value}: {e}") + elif current_engine.upper() in ['EEVEE', 'EEVEE_NEXT']: + eevee = scene.eevee + print("Applying EEVEE render settings from job metadata...") + for key, value in engine_settings.items(): + try: + if hasattr(eevee, key): + setattr(eevee, key, value) + print(f" Set EEVEE.{key} = {value}") + else: + print(f" Warning: EEVEE has no attribute '{key}'") + except Exception as e: + print(f" Warning: Could not set EEVEE.{key} = {value}: {e}") + + # Apply resolution if specified + if 'resolution_x' in render_settings_override: + try: + scene.render.resolution_x = render_settings_override['resolution_x'] + print(f"Set resolution_x = {render_settings_override['resolution_x']}") + except Exception as e: + print(f"Warning: Could not set resolution_x: {e}") + if 'resolution_y' in render_settings_override: + try: + scene.render.resolution_y = render_settings_override['resolution_y'] + print(f"Set resolution_y = {render_settings_override['resolution_y']}") + except Exception as e: + print(f"Warning: Could not set resolution_y: {e}") + +# Only override device selection if using Cycles (other engines handle GPU differently) +if current_engine == 'CYCLES': + # Check if CPU rendering is forced + force_cpu = False + if render_settings_override and render_settings_override.get('force_cpu'): + force_cpu = render_settings_override.get('force_cpu', False) + print("Force CPU rendering is enabled - skipping GPU detection") + + # Ensure Cycles addon is enabled + try: + if 'cycles' not in bpy.context.preferences.addons: + bpy.ops.preferences.addon_enable(module='cycles') + print("Enabled Cycles addon") + except Exception as e: + print(f"Warning: Could not enable Cycles addon: {e}") + + # If CPU is forced, skip GPU detection and set CPU directly + if force_cpu: + scene.cycles.device = 'CPU' + print("Forced CPU rendering (skipping GPU detection)") + else: + # Access Cycles preferences + prefs = bpy.context.preferences + try: + cycles_prefs = prefs.addons['cycles'].preferences + except (KeyError, AttributeError): + try: + cycles_addon = prefs.addons.get('cycles') + if cycles_addon: + cycles_prefs = cycles_addon.preferences + else: + raise Exception("Cycles addon not found") + except Exception as e: + print(f"ERROR: Could not access Cycles preferences: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + + # Check all devices and choose the best GPU type + # Device type preference order (most performant first) + device_type_preference = ['OPTIX', 'CUDA', 'HIP', 'ONEAPI', 'METAL'] + gpu_available = False + best_device_type = None + best_gpu_devices = [] + devices_by_type = {} # {device_type: [devices]} + seen_device_ids = set() # Track device IDs to avoid duplicates + + print("Checking for GPU availability...") + + # Try to get all devices - try each device type to see what's available + for device_type in device_type_preference: + try: + cycles_prefs.compute_device_type = device_type + cycles_prefs.refresh_devices() + + # Get devices for this type + devices = None + if hasattr(cycles_prefs, 'devices'): + try: + devices_prop = cycles_prefs.devices + if devices_prop: + devices = list(devices_prop) if hasattr(devices_prop, '__iter__') else [devices_prop] + except Exception as e: + pass + + if not devices or len(devices) == 0: + try: + devices = cycles_prefs.get_devices() + except Exception as e: + pass + + if devices and len(devices) > 0: + # Categorize devices by their type attribute, avoiding duplicates + for device in devices: + if hasattr(device, 'type'): + device_type_str = str(device.type).upper() + device_id = getattr(device, 'id', None) + + # Use device ID to avoid duplicates (same device appears when checking different compute_device_types) + if device_id and device_id in seen_device_ids: + continue + + if device_id: + seen_device_ids.add(device_id) + + if device_type_str not in devices_by_type: + devices_by_type[device_type_str] = [] + devices_by_type[device_type_str].append(device) + except (ValueError, AttributeError, KeyError, TypeError): + # Device type not supported, continue + continue + except Exception as e: + # Other errors - log but continue + print(f" Error checking {device_type}: {e}") + continue + + # Print what we found + print(f"Found devices by type: {list(devices_by_type.keys())}") + for dev_type, dev_list in devices_by_type.items(): + print(f" {dev_type}: {len(dev_list)} device(s)") + for device in dev_list: + device_name = getattr(device, 'name', 'Unknown') + print(f" - {device_name}") + + # Choose the best GPU type based on preference + for preferred_type in device_type_preference: + if preferred_type in devices_by_type: + gpu_devices = [d for d in devices_by_type[preferred_type] if preferred_type in ['CUDA', 'OPENCL', 'OPTIX', 'HIP', 'METAL', 'ONEAPI']] + if gpu_devices: + best_device_type = preferred_type + best_gpu_devices = [(d, preferred_type) for d in gpu_devices] + print(f"Selected {preferred_type} as best GPU type with {len(gpu_devices)} device(s)") + break + + # Second pass: Enable the best GPU we found + if best_device_type and best_gpu_devices: + print(f"\nEnabling GPU devices for {best_device_type}...") + try: + # Set the device type again + cycles_prefs.compute_device_type = best_device_type + cycles_prefs.refresh_devices() + + # First, disable all CPU devices to ensure only GPU is used + print(f" Disabling CPU devices...") + all_devices = cycles_prefs.devices if hasattr(cycles_prefs, 'devices') else cycles_prefs.get_devices() + if all_devices: + for device in all_devices: + if hasattr(device, 'type') and str(device.type).upper() == 'CPU': + try: + device.use = False + device_name = getattr(device, 'name', 'Unknown') + print(f" Disabled CPU: {device_name}") + except Exception as e: + print(f" Warning: Could not disable CPU device {getattr(device, 'name', 'Unknown')}: {e}") + + # Enable all GPU devices + enabled_count = 0 + for device, device_type in best_gpu_devices: + try: + device.use = True + enabled_count += 1 + device_name = getattr(device, 'name', 'Unknown') + print(f" Enabled: {device_name}") + except Exception as e: + print(f" Warning: Could not enable device {getattr(device, 'name', 'Unknown')}: {e}") + + # Enable ray tracing acceleration for supported device types + try: + if best_device_type == 'HIP': + # HIPRT (HIP Ray Tracing) for AMD GPUs + if hasattr(cycles_prefs, 'use_hiprt'): + cycles_prefs.use_hiprt = True + print(f" Enabled HIPRT (HIP Ray Tracing) for faster rendering") + elif hasattr(scene.cycles, 'use_hiprt'): + scene.cycles.use_hiprt = True + print(f" Enabled HIPRT (HIP Ray Tracing) for faster rendering") + else: + print(f" HIPRT not available (requires Blender 4.0+)") + elif best_device_type == 'OPTIX': + # OptiX is already enabled when using OPTIX device type + # But we can check if there are any OptiX-specific settings + if hasattr(scene.cycles, 'use_optix_denoising'): + scene.cycles.use_optix_denoising = True + print(f" Enabled OptiX denoising") + print(f" OptiX ray tracing is active (using OPTIX device type)") + elif best_device_type == 'CUDA': + # CUDA can use OptiX if available, but it's usually automatic + # Check if we can prefer OptiX over CUDA + if hasattr(scene.cycles, 'use_optix_denoising'): + scene.cycles.use_optix_denoising = True + print(f" Enabled OptiX denoising (if OptiX available)") + print(f" CUDA ray tracing active") + elif best_device_type == 'METAL': + # MetalRT for Apple Silicon (if available) + if hasattr(scene.cycles, 'use_metalrt'): + scene.cycles.use_metalrt = True + print(f" Enabled MetalRT (Metal Ray Tracing) for faster rendering") + elif hasattr(cycles_prefs, 'use_metalrt'): + cycles_prefs.use_metalrt = True + print(f" Enabled MetalRT (Metal Ray Tracing) for faster rendering") + else: + print(f" MetalRT not available") + elif best_device_type == 'ONEAPI': + # Intel oneAPI - Embree might be available + if hasattr(scene.cycles, 'use_embree'): + scene.cycles.use_embree = True + print(f" Enabled Embree for faster CPU ray tracing") + print(f" oneAPI ray tracing active") + except Exception as e: + print(f" Could not enable ray tracing acceleration: {e}") + + print(f"SUCCESS: Enabled {enabled_count} GPU device(s) for {best_device_type}") + gpu_available = True + except Exception as e: + print(f"ERROR: Failed to enable GPU devices: {e}") + import traceback + traceback.print_exc() + + # Set device based on availability (prefer GPU, fallback to CPU) + if gpu_available: + scene.cycles.device = 'GPU' + print(f"Using GPU for rendering (blend file had: {current_device})") + else: + scene.cycles.device = 'CPU' + print(f"GPU not available, using CPU for rendering (blend file had: {current_device})") + + # Verify device setting + if current_engine == 'CYCLES': + final_device = scene.cycles.device + print(f"Final Cycles device: {final_device}") +else: + # For other engines (EEVEE, etc.), respect blend file settings + print(f"Using {current_engine} engine - respecting blend file settings") + +# Enable GPU acceleration for EEVEE viewport rendering (if using EEVEE) +if current_engine == 'EEVEE' or current_engine == 'EEVEE_NEXT': + try: + if hasattr(bpy.context.preferences.system, 'gpu_backend'): + bpy.context.preferences.system.gpu_backend = 'OPENGL' + print("Enabled OpenGL GPU backend for EEVEE") + except Exception as e: + print(f"Could not set EEVEE GPU backend: {e}") + +# Enable GPU acceleration for compositing (if compositing is enabled) +try: + if scene.use_nodes and hasattr(scene, 'node_tree') and scene.node_tree: + if hasattr(scene.node_tree, 'use_gpu_compositing'): + scene.node_tree.use_gpu_compositing = True + print("Enabled GPU compositing") +except Exception as e: + print(f"Could not enable GPU compositing: {e}") + +# CRITICAL: Initialize headless rendering to prevent black images +# This ensures the render engine is properly initialized before rendering +print("Initializing headless rendering context...") +try: + # Ensure world exists and has proper settings + if not scene.world: + # Create a default world if none exists + world = bpy.data.worlds.new("World") + scene.world = world + print("Created default world") + + # Ensure world has a background shader (not just black) + if scene.world: + # Enable nodes if not already enabled + if not scene.world.use_nodes: + scene.world.use_nodes = True + print("Enabled world nodes") + + world_nodes = scene.world.node_tree + if world_nodes: + # Find or create background shader + bg_shader = None + for node in world_nodes.nodes: + if node.type == 'BACKGROUND': + bg_shader = node + break + + if not bg_shader: + bg_shader = world_nodes.nodes.new(type='ShaderNodeBackground') + # Connect to output + output = world_nodes.nodes.get('World Output') + if not output: + output = world_nodes.nodes.new(type='ShaderNodeOutputWorld') + output.name = 'World Output' + if output and bg_shader: + # Connect background to surface input + if 'Surface' in output.inputs and 'Background' in bg_shader.outputs: + world_nodes.links.new(bg_shader.outputs['Background'], output.inputs['Surface']) + print("Created background shader for world") + + # Ensure background has some color (not pure black) + if bg_shader: + # Only set if it's pure black (0,0,0) + if hasattr(bg_shader.inputs, 'Color'): + color = bg_shader.inputs['Color'].default_value + if len(color) >= 3 and color[0] == 0.0 and color[1] == 0.0 and color[2] == 0.0: + # Set to a very dark gray instead of pure black + bg_shader.inputs['Color'].default_value = (0.01, 0.01, 0.01, 1.0) + print("Adjusted world background color to prevent black renders") + else: + # Fallback: use legacy world color if nodes aren't working + if hasattr(scene.world, 'color'): + color = scene.world.color + if len(color) >= 3 and color[0] == 0.0 and color[1] == 0.0 and color[2] == 0.0: + scene.world.color = (0.01, 0.01, 0.01) + print("Adjusted legacy world color to prevent black renders") + + # For EEVEE, force viewport update to initialize render engine + if current_engine in ['EEVEE', 'EEVEE_NEXT']: + # Force EEVEE to update its internal state + try: + # Update depsgraph to ensure everything is initialized + depsgraph = bpy.context.evaluated_depsgraph_get() + if depsgraph: + # Force update + depsgraph.update() + print("Forced EEVEE depsgraph update for headless rendering") + except Exception as e: + print(f"Warning: Could not force EEVEE update: {e}") + + # Ensure EEVEE settings are applied + try: + # Force a material update to ensure shaders are compiled + for obj in scene.objects: + if obj.type == 'MESH' and obj.data.materials: + for mat in obj.data.materials: + if mat and mat.use_nodes: + # Touch the material to force update + mat.use_nodes = mat.use_nodes + print("Forced material updates for EEVEE") + except Exception as e: + print(f"Warning: Could not update materials: {e}") + + # For Cycles, ensure proper initialization + if current_engine == 'CYCLES': + # Ensure samples are set (even if 1 for preview) + if not hasattr(scene.cycles, 'samples') or scene.cycles.samples < 1: + scene.cycles.samples = 1 + print("Set minimum Cycles samples") + + # Check for lights in the scene + lights = [obj for obj in scene.objects if obj.type == 'LIGHT'] + print(f"Found {len(lights)} light(s) in scene") + if len(lights) == 0: + print("WARNING: No lights found in scene - rendering may be black!") + print(" Consider adding lights or ensuring world background emits light") + + # Ensure world background emits light (critical for Cycles) + if scene.world and scene.world.use_nodes: + world_nodes = scene.world.node_tree + if world_nodes: + bg_shader = None + for node in world_nodes.nodes: + if node.type == 'BACKGROUND': + bg_shader = node + break + + if bg_shader: + # Check and set strength - Cycles needs this to emit light! + if hasattr(bg_shader.inputs, 'Strength'): + strength = bg_shader.inputs['Strength'].default_value + if strength <= 0.0: + bg_shader.inputs['Strength'].default_value = 1.0 + print("Set world background strength to 1.0 for Cycles lighting") + else: + print(f"World background strength: {strength}") + # Also ensure color is not pure black + if hasattr(bg_shader.inputs, 'Color'): + color = bg_shader.inputs['Color'].default_value + if len(color) >= 3 and color[0] == 0.0 and color[1] == 0.0 and color[2] == 0.0: + bg_shader.inputs['Color'].default_value = (1.0, 1.0, 1.0, 1.0) + print("Set world background color to white for Cycles lighting") + + # Check film_transparent setting - if enabled, background will be transparent/black + if hasattr(scene.cycles, 'film_transparent') and scene.cycles.film_transparent: + print("WARNING: film_transparent is enabled - background will be transparent") + print(" If you see black renders, try disabling film_transparent") + + # Force Cycles to update/compile materials and shaders + try: + # Update depsgraph to ensure everything is initialized + depsgraph = bpy.context.evaluated_depsgraph_get() + if depsgraph: + depsgraph.update() + print("Forced Cycles depsgraph update") + + # Force material updates to ensure shaders are compiled + for obj in scene.objects: + if obj.type == 'MESH' and obj.data.materials: + for mat in obj.data.materials: + if mat and mat.use_nodes: + # Force material update + mat.use_nodes = mat.use_nodes + print("Forced Cycles material updates") + except Exception as e: + print(f"Warning: Could not force Cycles updates: {e}") + + # Verify device is actually set correctly + if hasattr(scene.cycles, 'device'): + actual_device = scene.cycles.device + print(f"Cycles device setting: {actual_device}") + if actual_device == 'GPU': + # Try to verify GPU is actually available + try: + prefs = bpy.context.preferences + cycles_prefs = prefs.addons['cycles'].preferences + devices = cycles_prefs.devices + enabled_devices = [d for d in devices if d.use] + if len(enabled_devices) == 0: + print("WARNING: GPU device set but no GPU devices are enabled!") + print(" Falling back to CPU may cause issues") + except Exception as e: + print(f"Could not verify GPU devices: {e}") + + # Ensure camera exists and is active + if scene.camera is None: + # Find first camera in scene + for obj in scene.objects: + if obj.type == 'CAMERA': + scene.camera = obj + print(f"Set active camera: {obj.name}") + break + + print("Headless rendering initialization complete") +except Exception as e: + print(f"Warning: Headless rendering initialization had issues: {e}") + import traceback + traceback.print_exc() + +# Final verification before rendering +print("\n=== Pre-render verification ===") +try: + scene = bpy.context.scene + print(f"Render engine: {scene.render.engine}") + print(f"Active camera: {scene.camera.name if scene.camera else 'None'}") + + if scene.render.engine == 'CYCLES': + print(f"Cycles device: {scene.cycles.device}") + print(f"Cycles samples: {scene.cycles.samples}") + lights = [obj for obj in scene.objects if obj.type == 'LIGHT'] + print(f"Lights in scene: {len(lights)}") + if scene.world: + if scene.world.use_nodes: + world_nodes = scene.world.node_tree + if world_nodes: + bg_shader = None + for node in world_nodes.nodes: + if node.type == 'BACKGROUND': + bg_shader = node + break + if bg_shader: + if hasattr(bg_shader.inputs, 'Strength'): + strength = bg_shader.inputs['Strength'].default_value + print(f"World background strength: {strength}") + if hasattr(bg_shader.inputs, 'Color'): + color = bg_shader.inputs['Color'].default_value + print(f"World background color: ({color[0]:.2f}, {color[1]:.2f}, {color[2]:.2f})") + else: + print("World exists but nodes are disabled") + else: + print("WARNING: No world in scene!") + + print("=== Verification complete ===\n") +except Exception as e: + print(f"Warning: Verification failed: {e}") + +print("Device configuration complete - blend file settings preserved, device optimized") +sys.stdout.flush() + diff --git a/pkg/scripts/scripts/unhide_objects.py b/pkg/scripts/scripts/unhide_objects.py new file mode 100644 index 0000000..3ad7855 --- /dev/null +++ b/pkg/scripts/scripts/unhide_objects.py @@ -0,0 +1,29 @@ +# Fix objects and collections hidden from render +vl = bpy.context.view_layer + +# 1. Objects hidden in view layer +print("Checking for objects hidden from render that need to be enabled...") +try: + for obj in bpy.data.objects: + if obj.hide_get(view_layer=vl): + if any(k in obj.name.lower() for k in ["scrotum|","cage","genital","penis","dick","collision","body.001","couch"]): + obj.hide_set(False, view_layer=vl) + print("Enabled object:", obj.name) +except Exception as e: + print(f"Warning: Could not check/fix hidden render objects: {e}") + +# 2. Collections disabled in renders OR set to Holdout (the final killer) +print("Checking for collections hidden from render that need to be enabled...") +try: + for col in bpy.data.collections: + if col.hide_render or (vl.layer_collection.children.get(col.name) and not vl.layer_collection.children[col.name].exclude == False): + if any(k in col.name.lower() for k in ["genital","nsfw","dick","private","hidden","cage","scrotum","collision","dick"]): + col.hide_render = False + if col.name in vl.layer_collection.children: + vl.layer_collection.children[col.name].exclude = False + vl.layer_collection.children[col.name].holdout = False + vl.layer_collection.children[col.name].indirect_only = False + print("Enabled collection:", col.name) +except Exception as e: + print(f"Warning: Could not check/fix hidden render collections: {e}") + diff --git a/pkg/types/types.go b/pkg/types/types.go index c0794e0..5b88782 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -140,6 +140,8 @@ type CreateJobRequest struct { AllowParallelRunners *bool `json:"allow_parallel_runners,omitempty"` // Optional for render jobs, defaults to true RenderSettings *RenderSettings `json:"render_settings,omitempty"` // Optional: Override blend file render settings UploadSessionID *string `json:"upload_session_id,omitempty"` // Optional: Session ID from file upload + UnhideObjects *bool `json:"unhide_objects,omitempty"` // Optional: Enable unhide tweaks for objects/collections + EnableExecution *bool `json:"enable_execution,omitempty"` // Optional: Enable auto-execution in Blender (adds --enable-autoexec flag, defaults to false) } // UpdateJobProgressRequest represents a request to update job progress @@ -227,9 +229,11 @@ type TaskLogEntry struct { type BlendMetadata struct { FrameStart int `json:"frame_start"` FrameEnd int `json:"frame_end"` - RenderSettings RenderSettings `json:"render_settings"` - SceneInfo SceneInfo `json:"scene_info"` - MissingFilesInfo *MissingFilesInfo `json:"missing_files_info,omitempty"` + RenderSettings RenderSettings `json:"render_settings"` + SceneInfo SceneInfo `json:"scene_info"` + MissingFilesInfo *MissingFilesInfo `json:"missing_files_info,omitempty"` + UnhideObjects *bool `json:"unhide_objects,omitempty"` // Enable unhide tweaks for objects/collections + EnableExecution *bool `json:"enable_execution,omitempty"` // Enable auto-execution in Blender (adds --enable-autoexec flag, defaults to false) } // MissingFilesInfo represents information about missing files/addons diff --git a/web/src/components/FileExplorer.jsx b/web/src/components/FileExplorer.jsx index 77991dc..99a0b74 100644 --- a/web/src/components/FileExplorer.jsx +++ b/web/src/components/FileExplorer.jsx @@ -1,7 +1,7 @@ import { useState } from 'react'; export default function FileExplorer({ files, onDownload, onPreview, isImageFile }) { - const [expandedPaths, setExpandedPaths] = useState(new Set()); + const [expandedPaths, setExpandedPaths] = useState(new Set()); // Root folder collapsed by default // Build directory tree from file paths const buildTree = (files) => { @@ -70,7 +70,7 @@ export default function FileExplorer({ files, onDownload, onPreview, isImageFile const file = item.file; const isImage = isImageFile && isImageFile(file.file_name); const sizeMB = (file.file_size / 1024 / 1024).toFixed(2); - const isArchive = file.file_name.endsWith('.tar.gz') || file.file_name.endsWith('.zip'); + const isArchive = file.file_name.endsWith('.tar') || file.file_name.endsWith('.zip'); return (
@@ -108,10 +108,13 @@ export default function FileExplorer({ files, onDownload, onPreview, isImageFile return (
hasChildren && togglePath(fullPath)} > + + {hasChildren ? (isExpanded ? '▼' : '▶') : '•'} + {hasChildren ? (isExpanded ? '📂' : '📁') : '📁'} @@ -123,7 +126,7 @@ export default function FileExplorer({ files, onDownload, onPreview, isImageFile )}
{hasChildren && isExpanded && ( -
+
{renderTree(item.children, level + 1, fullPath)}
)} @@ -143,10 +146,34 @@ export default function FileExplorer({ files, onDownload, onPreview, isImageFile ); } + // Wrap tree in a root folder + const rootExpanded = expandedPaths.has(''); + return (
- {renderTree(tree)} +
+
togglePath('')} + > + + {rootExpanded ? '▼' : '▶'} + + + {rootExpanded ? '📂' : '📁'} + + Files + + ({Object.keys(tree).length}) + +
+ {rootExpanded && ( +
+ {renderTree(tree)} +
+ )} +
); diff --git a/web/src/components/JobDetails.jsx b/web/src/components/JobDetails.jsx index b07bc95..73228fe 100644 --- a/web/src/components/JobDetails.jsx +++ b/web/src/components/JobDetails.jsx @@ -1,5 +1,5 @@ import { useState, useEffect, useRef } from 'react'; -import { jobs } from '../utils/api'; +import { jobs, REQUEST_SUPERSEDED } from '../utils/api'; import VideoPlayer from './VideoPlayer'; import FileExplorer from './FileExplorer'; @@ -18,19 +18,53 @@ export default function JobDetails({ job, onClose, onUpdate }) { const [streaming, setStreaming] = useState(false); const [previewImage, setPreviewImage] = useState(null); // { url, fileName } or null const wsRef = useRef(null); + const jobWsRef = useRef(null); // Separate ref for job WebSocket const logContainerRefs = useRef({}); // Refs for each step's log container const shouldAutoScrollRefs = useRef({}); // Auto-scroll state per step useEffect(() => { loadDetails(); - const interval = setInterval(loadDetails, 2000); - return () => { - clearInterval(interval); - if (wsRef.current) { - wsRef.current.close(); + // Use WebSocket for real-time updates instead of polling + if (jobDetails.status === 'running' || jobDetails.status === 'pending' || !jobDetails.status) { + connectJobWebSocket(); + return () => { + if (jobWsRef.current) { + try { + jobWsRef.current.close(); + } catch (e) { + // Ignore errors when closing + } + jobWsRef.current = null; + } + if (wsRef.current) { + try { + wsRef.current.close(); + } catch (e) { + // Ignore errors when closing + } + wsRef.current = null; + } + }; + } else { + // Job is completed/failed/cancelled - close WebSocket + if (jobWsRef.current) { + try { + jobWsRef.current.close(); + } catch (e) { + // Ignore errors when closing + } + jobWsRef.current = null; } - }; - }, [job.id]); + if (wsRef.current) { + try { + wsRef.current.close(); + } catch (e) { + // Ignore errors when closing + } + wsRef.current = null; + } + } + }, [job.id, jobDetails.status]); useEffect(() => { // Load logs and steps for all running tasks @@ -83,14 +117,45 @@ export default function JobDetails({ job, onClose, onUpdate }) { const loadDetails = async () => { try { - const [details, fileList, taskList] = await Promise.all([ + setLoading(true); + // Use summary endpoint for tasks initially - much faster + const [details, fileList, taskListResult] = await Promise.all([ jobs.get(job.id), - jobs.getFiles(job.id), - jobs.getTasks(job.id), + jobs.getFiles(job.id, { limit: 50 }), // Only load first page of files + jobs.getTasksSummary(job.id, { limit: 100, sort: 'frame_start:asc' }), // Use summary endpoint ]); setJobDetails(details); - setFiles(fileList); - setTasks(taskList); + + // Handle paginated file response - check for superseded sentinel + if (fileList === REQUEST_SUPERSEDED) { + return; // Request was superseded, skip this update + } + const fileData = fileList?.data || fileList; + setFiles(Array.isArray(fileData) ? fileData : []); + + // Handle paginated task summary response - check for superseded sentinel + if (taskListResult === REQUEST_SUPERSEDED) { + return; // Request was superseded, skip this update + } + const taskData = taskListResult?.data || taskListResult; + const taskSummaries = Array.isArray(taskData) ? taskData : []; + + // Convert summaries to task-like objects for display + const tasksForDisplay = taskSummaries.map(summary => ({ + id: summary.id, + job_id: job.id, + frame_start: summary.frame_start, + frame_end: summary.frame_end, + status: summary.status, + task_type: summary.task_type, + runner_id: summary.runner_id, + // These will be loaded on expand + current_step: null, + retry_count: 0, + max_retries: 3, + created_at: new Date().toISOString(), + })); + setTasks(Array.isArray(tasksForDisplay) ? tasksForDisplay : []); // Fetch context archive contents separately (may not exist for old jobs) try { @@ -101,26 +166,27 @@ export default function JobDetails({ job, onClose, onUpdate }) { setContextFiles([]); } - // Only load task data (logs/steps) for tasks that don't have data yet - // This prevents overwriting logs that are being streamed via WebSocket - // Once we have logs for a task, we rely on WebSocket for new logs + // Only load task data (logs/steps) for expanded tasks + // Don't auto-load for all tasks - wait for user to expand if (details.status === 'running') { - taskList.forEach(task => { - const existingData = taskData[task.id]; - // Only fetch logs via HTTP if we don't have any logs yet - // Once we have logs, WebSocket will handle new ones - if (!existingData || !existingData.logs || existingData.logs.length === 0) { - loadTaskData(task.id); - } else if (!existingData.steps || existingData.steps.length === 0) { - // If we have logs but no steps, fetch steps only - loadTaskStepsOnly(task.id); + // Only load data for tasks that are expanded + tasksForDisplay.forEach(task => { + if (expandedTasks.has(task.id)) { + const existingData = taskData[task.id]; + // Only fetch logs via HTTP if we don't have any logs yet + if (!existingData || !existingData.logs || existingData.logs.length === 0) { + loadTaskData(task.id); + } else if (!existingData.steps || existingData.steps.length === 0) { + loadTaskStepsOnly(task.id); + } } }); } // Check if there's an MP4 output file - const mp4File = fileList.find( - (f) => f.file_type === 'output' && f.file_name.endsWith('.mp4') + const fileArray = Array.isArray(fileData) ? fileData : []; + const mp4File = fileArray.find( + (f) => f.file_type === 'output' && f.file_name && f.file_name.endsWith('.mp4') ); if (mp4File) { setVideoUrl(jobs.getVideoUrl(job.id)); @@ -138,23 +204,39 @@ export default function JobDetails({ job, onClose, onUpdate }) { const loadTaskData = async (taskId) => { try { - const [logs, steps] = await Promise.all([ - jobs.getTaskLogs(job.id, taskId), + console.log(`Loading task data for task ${taskId}...`); + const [logsResult, steps] = await Promise.all([ + jobs.getTaskLogs(job.id, taskId, { limit: 1000 }), // Increased limit for completed tasks jobs.getTaskSteps(job.id, taskId), ]); + + // Check for superseded sentinel + if (logsResult === REQUEST_SUPERSEDED || steps === REQUEST_SUPERSEDED) { + return; // Request was superseded, skip this update + } + + console.log(`Task ${taskId} logs result:`, logsResult); + + // Handle new format with logs, last_id, limit + const logs = logsResult.logs || logsResult; + const lastId = logsResult.last_id; + + console.log(`Task ${taskId} - loaded ${Array.isArray(logs) ? logs.length : 0} logs, ${Array.isArray(steps) ? steps.length : 0} steps`); + setTaskData(prev => { - const current = prev[taskId] || { steps: [], logs: [] }; + const current = prev[taskId] || { steps: [], logs: [], lastId: 0 }; // Merge logs instead of replacing - this preserves WebSocket-streamed logs // Deduplicate by log ID const existingLogIds = new Set((current.logs || []).map(l => l.id)); - const newLogs = (logs || []).filter(l => !existingLogIds.has(l.id)); + const newLogs = (Array.isArray(logs) ? logs : []).filter(l => !existingLogIds.has(l.id)); const mergedLogs = [...(current.logs || []), ...newLogs].sort((a, b) => a.id - b.id); return { ...prev, [taskId]: { - steps: steps || current.steps, // Steps can be replaced (they don't change often) - logs: mergedLogs + steps: steps || current.steps, + logs: mergedLogs, + lastId: lastId || current.lastId } }; }); @@ -166,6 +248,10 @@ export default function JobDetails({ job, onClose, onUpdate }) { const loadTaskStepsOnly = async (taskId) => { try { const steps = await jobs.getTaskSteps(job.id, taskId); + // Check for superseded sentinel + if (steps === REQUEST_SUPERSEDED) { + return; // Request was superseded, skip this update + } setTaskData(prev => { const current = prev[taskId] || { steps: [], logs: [] }; return { @@ -181,9 +267,276 @@ export default function JobDetails({ job, onClose, onUpdate }) { } }; + const connectJobWebSocket = () => { + try { + // Close existing connection if any + if (jobWsRef.current) { + try { + jobWsRef.current.close(); + } catch (e) { + // Ignore errors when closing + } + jobWsRef.current = null; + } + + const ws = jobs.streamJobWebSocket(job.id); + jobWsRef.current = ws; // Store reference + + ws.onopen = () => { + console.log('Job WebSocket connected for job', job.id); + }; + + ws.onmessage = (event) => { + try { + const data = JSON.parse(event.data); + console.log('Job WebSocket message received:', data.type, data); + + if (data.type === 'job_update' && data.data) { + // Update job details + setJobDetails(prev => ({ ...prev, ...data.data })); + } else if (data.type === 'task_update' && data.data) { + // Update task in list + setTasks(prev => { + // Ensure prev is always an array + const prevArray = Array.isArray(prev) ? prev : []; + if (!data.task_id) { + console.warn('task_update message missing task_id:', data); + return prevArray; + } + const index = prevArray.findIndex(t => t.id === data.task_id); + if (index >= 0) { + const updated = [...prevArray]; + updated[index] = { ...updated[index], ...data.data }; + return updated; + } + // If task not found, it might be a new task - reload to be safe + if (data.data && (data.data.status === 'running' || data.data.status === 'pending')) { + setTimeout(() => { + const reloadTasks = async () => { + try { + const taskListResult = await jobs.getTasksSummary(job.id, { limit: 100, sort: 'frame_start:asc' }); + // Check for superseded sentinel + if (taskListResult === REQUEST_SUPERSEDED) { + return; // Request was superseded, skip this update + } + const taskData = taskListResult.data || taskListResult; + const taskSummaries = Array.isArray(taskData) ? taskData : []; + const tasksForDisplay = taskSummaries.map(summary => ({ + id: summary.id, + job_id: job.id, + frame_start: summary.frame_start, + frame_end: summary.frame_end, + status: summary.status, + task_type: summary.task_type, + runner_id: summary.runner_id, + current_step: null, + retry_count: 0, + max_retries: 3, + created_at: new Date().toISOString(), + })); + setTasks(Array.isArray(tasksForDisplay) ? tasksForDisplay : []); + } catch (error) { + console.error('Failed to reload tasks:', error); + } + }; + reloadTasks(); + }, 100); + } + return prevArray; + }); + } else if (data.type === 'task_added' && data.data) { + // New task was added - reload task summaries to get the new task + console.log('task_added message received, reloading tasks...', data); + const reloadTasks = async () => { + try { + const taskListResult = await jobs.getTasksSummary(job.id, { limit: 100, sort: 'frame_start:asc' }); + // Check for superseded sentinel + if (taskListResult === REQUEST_SUPERSEDED) { + return; // Request was superseded, skip this update + } + const taskData = taskListResult.data || taskListResult; + const taskSummaries = Array.isArray(taskData) ? taskData : []; + const tasksForDisplay = taskSummaries.map(summary => ({ + id: summary.id, + job_id: job.id, + frame_start: summary.frame_start, + frame_end: summary.frame_end, + status: summary.status, + task_type: summary.task_type, + runner_id: summary.runner_id, + current_step: null, + retry_count: 0, + max_retries: 3, + created_at: new Date().toISOString(), + })); + setTasks(Array.isArray(tasksForDisplay) ? tasksForDisplay : []); + } catch (error) { + console.error('Failed to reload tasks:', error); + // Fallback to full reload + loadDetails(); + } + }; + reloadTasks(); + } else if (data.type === 'tasks_added' && data.data) { + // Multiple new tasks were added - reload task summaries + console.log('tasks_added message received, reloading tasks...', data); + const reloadTasks = async () => { + try { + const taskListResult = await jobs.getTasksSummary(job.id, { limit: 100, sort: 'frame_start:asc' }); + // Check for superseded sentinel + if (taskListResult === REQUEST_SUPERSEDED) { + return; // Request was superseded, skip this update + } + const taskData = taskListResult.data || taskListResult; + const taskSummaries = Array.isArray(taskData) ? taskData : []; + const tasksForDisplay = taskSummaries.map(summary => ({ + id: summary.id, + job_id: job.id, + frame_start: summary.frame_start, + frame_end: summary.frame_end, + status: summary.status, + task_type: summary.task_type, + runner_id: summary.runner_id, + current_step: null, + retry_count: 0, + max_retries: 3, + created_at: new Date().toISOString(), + })); + setTasks(Array.isArray(tasksForDisplay) ? tasksForDisplay : []); + } catch (error) { + console.error('Failed to reload tasks:', error); + // Fallback to full reload + loadDetails(); + } + }; + reloadTasks(); + } else if (data.type === 'file_added' && data.data) { + // New file was added - reload file list + const reloadFiles = async () => { + try { + const fileList = await jobs.getFiles(job.id, { limit: 50 }); + // Check for superseded sentinel + if (fileList === REQUEST_SUPERSEDED) { + return; // Request was superseded, skip this update + } + const fileData = fileList.data || fileList; + setFiles(Array.isArray(fileData) ? fileData : []); + } catch (error) { + console.error('Failed to reload files:', error); + } + }; + reloadFiles(); + } else if (data.type === 'step_update' && data.data && data.task_id) { + // Step was created or updated - update task data + console.log('step_update message received:', data); + setTaskData(prev => { + const taskId = data.task_id; + const current = prev[taskId] || { steps: [], logs: [] }; + const stepData = data.data; + + // Find if step already exists + const existingSteps = current.steps || []; + const stepIndex = existingSteps.findIndex(s => s.step_name === stepData.step_name); + + let updatedSteps; + if (stepIndex >= 0) { + // Update existing step + updatedSteps = [...existingSteps]; + updatedSteps[stepIndex] = { + ...updatedSteps[stepIndex], + ...stepData, + id: stepData.step_id || updatedSteps[stepIndex].id, + }; + } else { + // Add new step + updatedSteps = [...existingSteps, { + id: stepData.step_id, + step_name: stepData.step_name, + status: stepData.status, + duration_ms: stepData.duration_ms, + error_message: stepData.error_message, + }]; + } + + return { + ...prev, + [taskId]: { + ...current, + steps: updatedSteps, + } + }; + }); + } else if (data.type === 'connected') { + // Connection established + } + } catch (error) { + console.error('Failed to parse WebSocket message:', error); + } + }; + + ws.onerror = (error) => { + console.error('Job WebSocket error:', { + error, + readyState: ws.readyState, + url: ws.url, + jobId: job.id, + status: jobDetails.status + }); + // WebSocket errors don't provide much detail, but we can check readyState + if (ws.readyState === WebSocket.CLOSED || ws.readyState === WebSocket.CLOSING) { + console.warn('Job WebSocket is closed or closing, will attempt reconnect'); + } + }; + + ws.onclose = (event) => { + console.log('Job WebSocket closed:', { + code: event.code, + reason: event.reason, + wasClean: event.wasClean, + jobId: job.id, + status: jobDetails.status + }); + jobWsRef.current = null; + + // Code 1006 = Abnormal Closure (connection lost without close frame) + // Code 1000 = Normal Closure + // Code 1001 = Going Away (server restart, etc.) + // We should reconnect for abnormal closures (1006) or unexpected closes + const shouldReconnect = !event.wasClean || event.code === 1006 || event.code === 1001; + + // Get current status from state to avoid stale closure + const currentStatus = jobDetails.status; + const isActiveJob = currentStatus === 'running' || currentStatus === 'pending'; + + if (shouldReconnect && isActiveJob) { + console.log(`Attempting to reconnect job WebSocket in 2 seconds... (code: ${event.code})`); + setTimeout(() => { + // Check status again before reconnecting (might have changed) + // Use a ref or check the current state directly + if ((!jobWsRef.current || jobWsRef.current.readyState === WebSocket.CLOSED)) { + // Re-check if job is still active by reading current state + // We'll check this in connectJobWebSocket if needed + connectJobWebSocket(); + } + }, 2000); + } else if (!isActiveJob) { + console.log('Job is no longer active, not reconnecting WebSocket'); + } + }; + } catch (error) { + console.error('Failed to connect job WebSocket:', error); + } + }; + const startLogStream = (taskIds) => { if (taskIds.length === 0 || streaming) return; + // Don't start streaming if job is no longer running + if (jobDetails.status !== 'running' && jobDetails.status !== 'pending') { + console.log('Job is not running, skipping log stream'); + return; + } + setStreaming(true); // For now, stream the first task's logs (WebSocket supports one task at a time) // In the future, we could have multiple WebSocket connections @@ -219,17 +572,44 @@ export default function JobDetails({ job, onClose, onUpdate }) { } }; + ws.onopen = () => { + console.log('Log WebSocket connected for task', primaryTaskId); + }; + ws.onerror = (error) => { - console.error('WebSocket error:', error); + console.error('Log WebSocket error:', { + error, + readyState: ws.readyState, + url: ws.url, + taskId: primaryTaskId, + jobId: job.id + }); setStreaming(false); }; - ws.onclose = () => { + ws.onclose = (event) => { + console.log('Log WebSocket closed:', { + code: event.code, + reason: event.reason, + wasClean: event.wasClean, + taskId: primaryTaskId, + jobId: job.id + }); setStreaming(false); - // Auto-reconnect if job is still running - if (jobDetails.status === 'running' && taskIds.length > 0) { + wsRef.current = null; + + // Code 1006 = Abnormal Closure (connection lost without close frame) + // Code 1000 = Normal Closure + // Code 1001 = Going Away (server restart, etc.) + const shouldReconnect = !event.wasClean || event.code === 1006 || event.code === 1001; + + // Auto-reconnect if job is still running and close was unexpected + if (shouldReconnect && jobDetails.status === 'running' && taskIds.length > 0) { + console.log(`Attempting to reconnect log WebSocket in 2 seconds... (code: ${event.code})`); setTimeout(() => { - if (jobDetails.status === 'running') { + // Check status again before reconnecting (might have changed) + // The startLogStream function will check if job is still running + if (jobDetails.status === 'running' && taskIds.length > 0) { startLogStream(taskIds); } }, 2000); @@ -243,9 +623,39 @@ export default function JobDetails({ job, onClose, onUpdate }) { newExpanded.delete(taskId); } else { newExpanded.add(taskId); - // Load data if not already loaded - if (!taskData[taskId]) { + // Load full task details if we only have summary + const tasksArray = Array.isArray(tasks) ? tasks : []; + const currentTask = tasksArray.find(t => t.id === taskId); + if (currentTask && !currentTask.created_at) { + // This is a summary - fetch full task details + try { + const fullTasks = await jobs.getTasks(job.id, { + limit: 1, + // We can't filter by task ID, so we'll get all and find the one we need + }); + const taskData = fullTasks.data || fullTasks; + const fullTask = Array.isArray(taskData) ? taskData.find(t => t.id === taskId) : null; + if (fullTask) { + setTasks(prev => { + const prevArray = Array.isArray(prev) ? prev : []; + return prevArray.map(t => t.id === taskId ? fullTask : t); + }); + } + } catch (err) { + console.error('Failed to load full task details:', err); + } + } + // Always load logs/steps when expanding a task to ensure we have the latest data + // This is especially important for completed tasks that weren't loaded before + const existingData = taskData[taskId]; + const hasLogs = existingData && existingData.logs && existingData.logs.length > 0; + const hasSteps = existingData && existingData.steps && existingData.steps.length > 0; + + if (!hasLogs || !hasSteps) { + console.log(`Loading task data for task ${taskId} (logs: ${hasLogs}, steps: ${hasSteps})`); await loadTaskData(taskId); + } else { + console.log(`Task ${taskId} already has ${existingData.logs.length} logs and ${existingData.steps.length} steps, skipping load`); } } setExpandedTasks(newExpanded); diff --git a/web/src/components/JobList.jsx b/web/src/components/JobList.jsx index 8e504e9..e3d4fae 100644 --- a/web/src/components/JobList.jsx +++ b/web/src/components/JobList.jsx @@ -1,4 +1,4 @@ -import { useState, useEffect } from 'react'; +import { useState, useEffect, useRef } from 'react'; import { jobs } from '../utils/api'; import JobDetails from './JobDetails'; @@ -6,17 +6,165 @@ export default function JobList() { const [jobList, setJobList] = useState([]); const [loading, setLoading] = useState(true); const [selectedJob, setSelectedJob] = useState(null); + const [pagination, setPagination] = useState({ total: 0, limit: 50, offset: 0 }); + const [hasMore, setHasMore] = useState(true); + const pollingIntervalRef = useRef(null); + const wsRef = useRef(null); useEffect(() => { loadJobs(); - const interval = setInterval(loadJobs, 5000); - return () => clearInterval(interval); + // Use WebSocket for real-time updates instead of polling + connectWebSocket(); + return () => { + if (pollingIntervalRef.current) { + clearInterval(pollingIntervalRef.current); + } + if (wsRef.current) { + try { + wsRef.current.close(); + } catch (e) { + // Ignore errors when closing + } + wsRef.current = null; + } + }; }, []); - const loadJobs = async () => { + const connectWebSocket = () => { try { - const data = await jobs.list(); - setJobList(data); + // Close existing connection if any + if (wsRef.current) { + try { + wsRef.current.close(); + } catch (e) { + // Ignore errors when closing + } + wsRef.current = null; + } + + const ws = jobs.streamJobsWebSocket(); + wsRef.current = ws; + + ws.onopen = () => { + console.log('Job list WebSocket connected'); + }; + + ws.onmessage = (event) => { + try { + const data = JSON.parse(event.data); + if (data.type === 'job_update' && data.data) { + // Update job in list + setJobList(prev => { + const index = prev.findIndex(j => j.id === data.job_id); + if (index >= 0) { + const updated = [...prev]; + updated[index] = { ...updated[index], ...data.data }; + return updated; + } + // If job not in current page, reload to get updated list + if (data.data.status === 'completed' || data.data.status === 'failed') { + loadJobs(); + } + return prev; + }); + } else if (data.type === 'connected') { + // Connection established + } + } catch (error) { + console.error('Failed to parse WebSocket message:', error); + } + }; + + ws.onerror = (error) => { + console.error('Job list WebSocket error:', { + error, + readyState: ws.readyState, + url: ws.url + }); + // WebSocket errors don't provide much detail, but we can check readyState + if (ws.readyState === WebSocket.CLOSED || ws.readyState === WebSocket.CLOSING) { + console.warn('Job list WebSocket is closed or closing, will fallback to polling'); + // Fallback to polling on error + startAdaptivePolling(); + } + }; + + ws.onclose = (event) => { + console.log('Job list WebSocket closed:', { + code: event.code, + reason: event.reason, + wasClean: event.wasClean + }); + wsRef.current = null; + + // Code 1006 = Abnormal Closure (connection lost without close frame) + // Code 1000 = Normal Closure + // Code 1001 = Going Away (server restart, etc.) + // We should reconnect for abnormal closures (1006) or unexpected closes + const shouldReconnect = !event.wasClean || event.code === 1006 || event.code === 1001; + + if (shouldReconnect) { + console.log(`Attempting to reconnect job list WebSocket in 2 seconds... (code: ${event.code})`); + setTimeout(() => { + if (wsRef.current === null || (wsRef.current && wsRef.current.readyState === WebSocket.CLOSED)) { + connectWebSocket(); + } + }, 2000); + } else { + // Clean close (code 1000) - fallback to polling + console.log('WebSocket closed cleanly, falling back to polling'); + startAdaptivePolling(); + } + }; + } catch (error) { + console.error('Failed to connect WebSocket:', error); + // Fallback to polling + startAdaptivePolling(); + } + }; + + const startAdaptivePolling = () => { + const checkAndPoll = () => { + const hasRunningJobs = jobList.some(job => job.status === 'running' || job.status === 'pending'); + const interval = hasRunningJobs ? 5000 : 10000; // 5s for running, 10s for completed + + if (pollingIntervalRef.current) { + clearInterval(pollingIntervalRef.current); + } + + pollingIntervalRef.current = setInterval(() => { + loadJobs(); + }, interval); + }; + + checkAndPoll(); + // Re-check interval when job list changes + const checkInterval = setInterval(checkAndPoll, 5000); + return () => clearInterval(checkInterval); + }; + + const loadJobs = async (append = false) => { + try { + const offset = append ? pagination.offset + pagination.limit : 0; + const result = await jobs.listSummary({ + limit: pagination.limit, + offset, + sort: 'created_at:desc' + }); + + // Handle both old format (array) and new format (object with data, total, etc.) + const jobsData = result.data || result; + const total = result.total !== undefined ? result.total : jobsData.length; + + if (append) { + setJobList(prev => [...prev, ...jobsData]); + setPagination(prev => ({ ...prev, offset, total })); + } else { + setJobList(jobsData); + setPagination({ total, limit: result.limit || pagination.limit, offset: result.offset || 0 }); + } + + setHasMore(offset + jobsData.length < total); } catch (error) { console.error('Failed to load jobs:', error); } finally { @@ -24,8 +172,13 @@ export default function JobList() { } }; + const loadMore = () => { + if (!loading && hasMore) { + loadJobs(true); + } + }; + // Keep selectedJob in sync with the job list when it refreshes - // This prevents the selected job from becoming stale when format selection or other actions trigger updates useEffect(() => { if (selectedJob && jobList.length > 0) { const freshJob = jobList.find(j => j.id === selectedJob.id); @@ -74,7 +227,7 @@ export default function JobList() { return colors[status] || colors.pending; }; - if (loading) { + if (loading && jobList.length === 0) { return (
@@ -106,8 +259,10 @@ export default function JobList() {
-

Frames: {job.frame_start} - {job.frame_end}

-

Format: {job.output_format}

+ {job.frame_start !== undefined && job.frame_end !== undefined && ( +

Frames: {job.frame_start} - {job.frame_end}

+ )} + {job.output_format &&

Format: {job.output_format}

}

Created: {new Date(job.created_at).toLocaleString()}

@@ -126,7 +281,15 @@ export default function JobList() {
+ {hasMore && ( +
+ +
+ )} + {selectedJob && ( ); } - diff --git a/web/src/components/JobSubmission.jsx b/web/src/components/JobSubmission.jsx index 11d7f04..2288299 100644 --- a/web/src/components/JobSubmission.jsx +++ b/web/src/components/JobSubmission.jsx @@ -3,7 +3,7 @@ import { jobs } from '../utils/api'; import JobDetails from './JobDetails'; export default function JobSubmission({ onSuccess }) { - const [step, setStep] = useState(1); // 1 = upload & extract metadata, 2 = configure & submit + const [step, setStep] = useState(1); // 1 = upload & extract metadata, 2 = missing addons (if any), 3 = configure & submit const [formData, setFormData] = useState({ name: '', frame_start: 1, @@ -11,6 +11,8 @@ export default function JobSubmission({ onSuccess }) { output_format: 'PNG', allow_parallel_runners: true, render_settings: null, // Will contain engine settings + unhide_objects: false, // Unhide objects/collections tweak + enable_execution: false, // Enable auto-execution in Blender }); const [showAdvancedSettings, setShowAdvancedSettings] = useState(false); const [file, setFile] = useState(null); @@ -25,6 +27,7 @@ export default function JobSubmission({ onSuccess }) { const [isUploading, setIsUploading] = useState(false); const [blendFiles, setBlendFiles] = useState([]); // For ZIP files with multiple blend files const [selectedMainBlend, setSelectedMainBlend] = useState(''); + const [confirmedMissingFiles, setConfirmedMissingFiles] = useState(false); // Confirmation for missing files // Use refs to track cancellation state across re-renders const isCancelledRef = useRef(false); @@ -71,10 +74,15 @@ export default function JobSubmission({ onSuccess }) { // Upload file to new endpoint (no job required) const result = await jobs.uploadFileForJobCreation(selectedFile, (progress) => { setUploadProgress(progress); + // After upload completes, show processing state + if (progress >= 100) { + setMetadataStatus('processing'); + } }, selectedMainBlend || undefined); + // Keep showing processing state until we have the result + setMetadataStatus('processing'); setUploadProgress(100); - setIsUploading(false); // Store session ID for later use when creating the job if (result.session_id) { @@ -88,6 +96,9 @@ export default function JobSubmission({ onSuccess }) { return; } + // Upload and processing complete + setIsUploading(false); + // If metadata was extracted, use it if (result.metadata_extracted && result.metadata) { setMetadata(result.metadata); @@ -141,25 +152,33 @@ export default function JobSubmission({ onSuccess }) { return; } - try { - setIsUploading(true); - setUploadProgress(0); - setMetadataStatus('extracting'); + try { + setIsUploading(true); + setUploadProgress(0); + setMetadataStatus('extracting'); - // Re-upload with selected main blend file - const result = await jobs.uploadFileForJobCreation(file, (progress) => { - setUploadProgress(progress); - }, selectedMainBlend); + // Re-upload with selected main blend file + const result = await jobs.uploadFileForJobCreation(file, (progress) => { + setUploadProgress(progress); + // After upload completes, show processing state + if (progress >= 100) { + setMetadataStatus('processing'); + } + }, selectedMainBlend); - setUploadProgress(100); - setIsUploading(false); - setBlendFiles([]); + // Keep showing processing state until we have the result + setMetadataStatus('processing'); + setUploadProgress(100); + setBlendFiles([]); // Store session ID if (result.session_id) { setUploadSessionId(result.session_id); } + // Upload and processing complete + setIsUploading(false); + // If metadata was extracted, use it if (result.metadata_extracted && result.metadata) { setMetadata(result.metadata); @@ -202,17 +221,43 @@ export default function JobSubmission({ onSuccess }) { const handleContinueToStep2 = () => { if (metadataStatus === 'completed' || metadataStatus === 'error') { - setStep(2); + // Check if there are missing addons - if so, go to addon step, otherwise skip to config + const hasMissingAddons = metadata?.missing_files_info?.missing_addons && + metadata.missing_files_info.missing_addons.length > 0; + if (hasMissingAddons) { + setStep(2); // Step 2 = missing addons + } else { + setStep(3); // Step 3 = configure & submit + } } }; + const handleContinueToStep3 = () => { + setStep(3); // Continue from addons step to config step + }; + const handleBackToStep1 = () => { setStep(1); }; + const handleBackToStep2 = () => { + setStep(2); + }; + const handleSubmit = async (e) => { e.preventDefault(); setError(''); + + // Check if there are missing files/addons and require confirmation + const hasMissingFiles = metadata?.missing_files_info?.has_missing && + metadata.missing_files_info.missing_addons && + metadata.missing_files_info.missing_addons.length > 0; + + if (hasMissingFiles && !confirmedMissingFiles) { + setError('Please confirm that you want to proceed with missing addons'); + return; + } + setSubmitting(true); try { @@ -246,6 +291,8 @@ export default function JobSubmission({ onSuccess }) { allow_parallel_runners: formData.allow_parallel_runners, render_settings: renderSettings, upload_session_id: uploadSessionId || undefined, // Pass session ID to move context archive + unhide_objects: formData.unhide_objects || undefined, // Pass unhide toggle + enable_execution: formData.enable_execution || undefined, // Pass enable execution toggle }); // Fetch the full job details @@ -269,6 +316,8 @@ export default function JobSubmission({ onSuccess }) { output_format: 'PNG', allow_parallel_runners: true, render_settings: null, + unhide_objects: false, + enable_execution: false, }); setShowAdvancedSettings(false); setFile(null); @@ -304,10 +353,21 @@ export default function JobSubmission({ onSuccess }) {
Upload & Extract Metadata
+ {metadata?.missing_files_info?.missing_addons && metadata.missing_files_info.missing_addons.length > 0 && ( + <> +
+
= 2 ? 'text-orange-500 font-medium' : 'text-gray-500'}`}> +
= 2 ? 'bg-orange-600 text-white' : 'bg-gray-700'}`}> + {step > 2 ? '✓' : '2'} +
+ Missing Addons +
+ + )}
-
= 2 ? 'text-orange-500 font-medium' : 'text-gray-500'}`}> -
= 2 ? 'bg-orange-600 text-white' : 'bg-gray-700'}`}> - 2 +
= 3 ? 'text-orange-500 font-medium' : 'text-gray-500'}`}> +
= 3 ? 'bg-orange-600 text-white' : 'bg-gray-700'}`}> + {step > 3 ? '✓' : (metadata?.missing_files_info?.missing_addons && metadata.missing_files_info.missing_addons.length > 0 ? '3' : '2')}
Configure & Submit
@@ -370,9 +430,9 @@ export default function JobSubmission({ onSuccess }) {
)} - {(isUploading || metadataStatus === 'extracting') && ( + {(isUploading || metadataStatus === 'extracting' || metadataStatus === 'processing') && (
- {isUploading ? ( + {isUploading && uploadProgress < 100 ? (
Uploading file... @@ -385,6 +445,22 @@ export default function JobSubmission({ onSuccess }) { >
+ ) : metadataStatus === 'processing' ? ( +
+
+ Processing file and extracting metadata... + {Math.round(uploadProgress)}% +
+
+
+
+
+ This may take a moment for large files... +
+
) : (
@@ -430,9 +506,9 @@ export default function JobSubmission({ onSuccess }) { )}
- ) : ( - // Step 2: Configure and submit -
+ ) : step === 2 ? ( + // Step 2: Missing Addons (only shown if there are missing addons) +
+
+
Missing Addons Detected
+
+

The following addons are required by this blend file but are not available on the render servers:

+
    + {metadata?.missing_files_info?.missing_addons?.map((addon, idx) => ( +
  • {addon}
    • + ))} +
+

+ Note: The render may fail or produce unexpected results if these addons are required for rendering. + You can still proceed, but be aware that the output may not match your expectations. +

+
+ +
+
+ ) : ( + // Step 3: Configure and submit + +
+
+
+ setFormData({ ...formData, unhide_objects: e.target.checked })} + className="h-4 w-4 text-orange-600 focus:ring-orange-500 border-gray-600 bg-gray-900 rounded" + /> + +
+
+ +
+
+ setFormData({ ...formData, enable_execution: e.target.checked })} + className="h-4 w-4 text-orange-600 focus:ring-orange-500 border-gray-600 bg-gray-900 rounded" + /> + +
+
+ {metadata && metadataStatus === 'completed' && ( -
-
Metadata from blend file:
-
-
Frames: {metadata.frame_start} - {metadata.frame_end}
-
Resolution: {metadata.render_settings?.resolution_x} x {metadata.render_settings?.resolution_y}
-
Engine: {metadata.render_settings?.engine}
- {metadata.render_settings?.engine_settings?.samples && ( -
Samples: {metadata.render_settings.engine_settings.samples}
- )} - {metadata.render_settings?.engine_settings?.taa_render_samples && ( -
EEVEE Samples: {metadata.render_settings.engine_settings.taa_render_samples}
- )} + <> +
+
Metadata from blend file:
+
+
Frames: {metadata.frame_start} - {metadata.frame_end}
+
Resolution: {metadata.render_settings?.resolution_x} x {metadata.render_settings?.resolution_y}
+
Engine: {metadata.render_settings?.engine}
+ {metadata.render_settings?.engine_settings?.samples && ( +
Samples: {metadata.render_settings.engine_settings.samples}
+ )} + {metadata.render_settings?.engine_settings?.taa_render_samples && ( +
EEVEE Samples: {metadata.render_settings.engine_settings.taa_render_samples}
+ )} +
-
+ + {/* Missing Files/Addons Warning */} + {metadata.missing_files_info?.has_missing && + metadata.missing_files_info.missing_addons && + metadata.missing_files_info.missing_addons.length > 0 && ( +
+
⚠️ Missing Addons Detected
+
+

The following addons are required by this blend file but are not available:

+
    + {metadata.missing_files_info.missing_addons.map((addon, idx) => ( +
  • {addon}
    • + ))} +
+

Rendering may fail or produce incorrect results without these addons.

+
+
+ setConfirmedMissingFiles(e.target.checked)} + className="h-4 w-4 text-orange-600 focus:ring-orange-500 border-gray-600 bg-gray-900 rounded" + /> + +
+
+ )} + )} {/* Advanced Render Settings */} @@ -886,11 +1063,16 @@ export default function JobSubmission({ onSuccess }) { )} + {metadata?.missing_files_info?.has_missing && !confirmedMissingFiles && ( +

+ Please confirm that you want to proceed with missing addons +

+ )}
)} diff --git a/web/src/utils/api.js b/web/src/utils/api.js index cc739c8..0f5abca 100644 --- a/web/src/utils/api.js +++ b/web/src/utils/api.js @@ -3,6 +3,73 @@ const API_BASE = '/api'; // Global auth error handler - will be set by useAuth hook let onAuthError = null; +// Request debouncing and deduplication +const pendingRequests = new Map(); // key: endpoint+params, value: Promise +const requestQueue = new Map(); // key: endpoint+params, value: { resolve, reject, timestamp } +const DEBOUNCE_DELAY = 100; // 100ms debounce delay +const DEDUPE_WINDOW = 5000; // 5 seconds - same request within this window uses cached promise + +// Generate cache key from endpoint and params +function getCacheKey(endpoint, options = {}) { + const params = new URLSearchParams(); + Object.keys(options).sort().forEach(key => { + if (options[key] !== undefined && options[key] !== null) { + params.append(key, String(options[key])); + } + }); + const query = params.toString(); + return `${endpoint}${query ? '?' + query : ''}`; +} + +// Sentinel value to indicate a request was superseded (instead of rejecting) +// Export it so components can check for it +export const REQUEST_SUPERSEDED = Symbol('REQUEST_SUPERSEDED'); + +// Debounced request wrapper +function debounceRequest(key, requestFn, delay = DEBOUNCE_DELAY) { + return new Promise((resolve, reject) => { + // Check if there's a pending request for this key + if (pendingRequests.has(key)) { + const pending = pendingRequests.get(key); + // If request is very recent (within dedupe window), reuse it + const now = Date.now(); + if (pending.timestamp && (now - pending.timestamp) < DEDUPE_WINDOW) { + pending.promise.then(resolve).catch(reject); + return; + } + } + + // Clear any existing timeout for this key + if (requestQueue.has(key)) { + const queued = requestQueue.get(key); + clearTimeout(queued.timeout); + // Resolve with sentinel value instead of rejecting - this prevents errors from propagating + // The new request will handle the actual response + queued.resolve(REQUEST_SUPERSEDED); + } + + // Queue new request + const timeout = setTimeout(() => { + requestQueue.delete(key); + const promise = requestFn(); + const timestamp = Date.now(); + pendingRequests.set(key, { promise, timestamp }); + + promise + .then(result => { + pendingRequests.delete(key); + resolve(result); + }) + .catch(error => { + pendingRequests.delete(key); + reject(error); + }); + }, delay); + + requestQueue.set(key, { resolve, reject, timeout }); + }); +} + export const setAuthErrorHandler = (handler) => { onAuthError = handler; }; @@ -174,12 +241,53 @@ export const auth = { }; export const jobs = { - async list() { - return api.get('/jobs'); + async list(options = {}) { + const key = getCacheKey('/jobs', options); + return debounceRequest(key, () => { + const params = new URLSearchParams(); + if (options.limit) params.append('limit', options.limit.toString()); + if (options.offset) params.append('offset', options.offset.toString()); + if (options.status) params.append('status', options.status); + if (options.sort) params.append('sort', options.sort); + const query = params.toString(); + return api.get(`/jobs${query ? '?' + query : ''}`); + }); }, - async get(id) { - return api.get(`/jobs/${id}`); + async listSummary(options = {}) { + const key = getCacheKey('/jobs/summary', options); + return debounceRequest(key, () => { + const params = new URLSearchParams(); + if (options.limit) params.append('limit', options.limit.toString()); + if (options.offset) params.append('offset', options.offset.toString()); + if (options.status) params.append('status', options.status); + if (options.sort) params.append('sort', options.sort); + const query = params.toString(); + return api.get(`/jobs/summary${query ? '?' + query : ''}`); + }); + }, + + async get(id, options = {}) { + const key = getCacheKey(`/jobs/${id}`, options); + return debounceRequest(key, async () => { + if (options.etag) { + // Include ETag in request headers for conditional requests + const headers = { 'If-None-Match': options.etag }; + const response = await fetch(`${API_BASE}/jobs/${id}`, { + credentials: 'include', + headers, + }); + if (response.status === 304) { + return null; // Not modified + } + if (!response.ok) { + const errorData = await response.json().catch(() => null); + throw new Error(errorData?.error || response.statusText); + } + return response.json(); + } + return api.get(`/jobs/${id}`); + }); }, async create(jobData) { @@ -202,8 +310,27 @@ export const jobs = { return api.uploadFile(`/jobs/upload`, file, onProgress, mainBlendFile); }, - async getFiles(jobId) { - return api.get(`/jobs/${jobId}/files`); + async getFiles(jobId, options = {}) { + const key = getCacheKey(`/jobs/${jobId}/files`, options); + return debounceRequest(key, () => { + const params = new URLSearchParams(); + if (options.limit) params.append('limit', options.limit.toString()); + if (options.offset) params.append('offset', options.offset.toString()); + if (options.file_type) params.append('file_type', options.file_type); + if (options.extension) params.append('extension', options.extension); + const query = params.toString(); + return api.get(`/jobs/${jobId}/files${query ? '?' + query : ''}`); + }); + }, + + async getFilesCount(jobId, options = {}) { + const key = getCacheKey(`/jobs/${jobId}/files/count`, options); + return debounceRequest(key, () => { + const params = new URLSearchParams(); + if (options.file_type) params.append('file_type', options.file_type); + const query = params.toString(); + return api.get(`/jobs/${jobId}/files/count${query ? '?' + query : ''}`); + }); }, async getContextArchive(jobId) { @@ -219,12 +346,21 @@ export const jobs = { }, async getTaskLogs(jobId, taskId, options = {}) { - const params = new URLSearchParams(); - if (options.stepName) params.append('step_name', options.stepName); - if (options.logLevel) params.append('log_level', options.logLevel); - if (options.limit) params.append('limit', options.limit.toString()); - const query = params.toString(); - return api.get(`/jobs/${jobId}/tasks/${taskId}/logs${query ? '?' + query : ''}`); + const key = getCacheKey(`/jobs/${jobId}/tasks/${taskId}/logs`, options); + return debounceRequest(key, async () => { + const params = new URLSearchParams(); + if (options.stepName) params.append('step_name', options.stepName); + if (options.logLevel) params.append('log_level', options.logLevel); + if (options.limit) params.append('limit', options.limit.toString()); + if (options.sinceId) params.append('since_id', options.sinceId.toString()); + const query = params.toString(); + const result = await api.get(`/jobs/${jobId}/tasks/${taskId}/logs${query ? '?' + query : ''}`); + // Handle both old format (array) and new format (object with logs, last_id, limit) + if (Array.isArray(result)) { + return { logs: result, last_id: result.length > 0 ? result[result.length - 1].id : 0, limit: options.limit || 100 }; + } + return result; + }); }, async getTaskSteps(jobId, taskId) { @@ -239,6 +375,20 @@ export const jobs = { return new WebSocket(url); }, + streamJobsWebSocket() { + const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + const wsHost = window.location.host; + const url = `${wsProtocol}//${wsHost}${API_BASE}/jobs/ws`; + return new WebSocket(url); + }, + + streamJobWebSocket(jobId) { + const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:'; + const wsHost = window.location.host; + const url = `${wsProtocol}//${wsHost}${API_BASE}/jobs/${jobId}/ws`; + return new WebSocket(url); + }, + async retryTask(jobId, taskId) { return api.post(`/jobs/${jobId}/tasks/${taskId}/retry`); }, @@ -247,8 +397,50 @@ export const jobs = { return api.get(`/jobs/${jobId}/metadata`); }, - async getTasks(jobId) { - return api.get(`/jobs/${jobId}/tasks`); + async getTasks(jobId, options = {}) { + const key = getCacheKey(`/jobs/${jobId}/tasks`, options); + return debounceRequest(key, () => { + const params = new URLSearchParams(); + if (options.limit) params.append('limit', options.limit.toString()); + if (options.offset) params.append('offset', options.offset.toString()); + if (options.status) params.append('status', options.status); + if (options.frameStart) params.append('frame_start', options.frameStart.toString()); + if (options.frameEnd) params.append('frame_end', options.frameEnd.toString()); + if (options.sort) params.append('sort', options.sort); + const query = params.toString(); + return api.get(`/jobs/${jobId}/tasks${query ? '?' + query : ''}`); + }); + }, + + async getTasksSummary(jobId, options = {}) { + const key = getCacheKey(`/jobs/${jobId}/tasks/summary`, options); + return debounceRequest(key, () => { + const params = new URLSearchParams(); + if (options.limit) params.append('limit', options.limit.toString()); + if (options.offset) params.append('offset', options.offset.toString()); + if (options.status) params.append('status', options.status); + if (options.sort) params.append('sort', options.sort); + const query = params.toString(); + return api.get(`/jobs/${jobId}/tasks/summary${query ? '?' + query : ''}`); + }); + }, + + async batchGetJobs(jobIds) { + // Sort jobIds for consistent cache key + const sortedIds = [...jobIds].sort((a, b) => a - b); + const key = getCacheKey('/jobs/batch', { job_ids: sortedIds.join(',') }); + return debounceRequest(key, () => { + return api.post('/jobs/batch', { job_ids: jobIds }); + }); + }, + + async batchGetTasks(jobId, taskIds) { + // Sort taskIds for consistent cache key + const sortedIds = [...taskIds].sort((a, b) => a - b); + const key = getCacheKey(`/jobs/${jobId}/tasks/batch`, { task_ids: sortedIds.join(',') }); + return debounceRequest(key, () => { + return api.post(`/jobs/${jobId}/tasks/batch`, { task_ids: taskIds }); + }); }, };