diff --git a/cmd/manager/main.go b/cmd/manager/main.go index aec72ce..27f3cb2 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -46,13 +46,26 @@ func main() { log.Fatalf("Failed to create server: %v", err) } - // Start server + // Start server with increased request body size limit for large file uploads addr := fmt.Sprintf(":%s", *port) log.Printf("Starting manager server on %s", addr) log.Printf("Database: %s", *dbPath) log.Printf("Storage: %s", *storagePath) - if err := http.ListenAndServe(addr, server); err != nil { + httpServer := &http.Server{ + Addr: addr, + Handler: server, + MaxHeaderBytes: 1 << 20, // 1 MB for headers + ReadTimeout: 0, // No read timeout (for large uploads) + WriteTimeout: 0, // No write timeout (for large uploads) + } + + // Note: MaxRequestBodySize is not directly configurable in http.Server + // It's handled by ParseMultipartForm in handlers, which we've already configured + // But we need to ensure the server can handle large requests + // The default limit is 10MB, but we bypass it by using ParseMultipartForm with larger limit + + if err := httpServer.ListenAndServe(); err != nil { log.Fatalf("Server failed: %v", err) } } diff --git a/internal/api/admin.go b/internal/api/admin.go index fade020..fd97728 100644 --- a/internal/api/admin.go +++ b/internal/api/admin.go @@ -25,8 +25,13 @@ func (s *Server) handleGenerateRegistrationToken(w http.ResponseWriter, r *http. ExpiresInHours int `json:"expires_in_hours,omitempty"` } if r.Body != nil && r.ContentLength > 0 { - if err := json.NewDecoder(r.Body).Decode(&req); err == nil && req.ExpiresInHours > 0 { - expiresIn = time.Duration(req.ExpiresInHours) * time.Hour + if err := json.NewDecoder(r.Body).Decode(&req); err == nil { + if req.ExpiresInHours == 0 { + // 0 hours means infinite expiration + expiresIn = 0 + } else if req.ExpiresInHours > 0 { + expiresIn = time.Duration(req.ExpiresInHours) * time.Hour + } } } @@ -36,11 +41,17 @@ func (s *Server) handleGenerateRegistrationToken(w http.ResponseWriter, r *http. return } - s.respondJSON(w, http.StatusCreated, map[string]interface{}{ - "token": token, - "expires_in": expiresIn.String(), - "expires_at": time.Now().Add(expiresIn), - }) + response := map[string]interface{}{ + "token": token, + } + if expiresIn == 0 { + response["expires_in"] = "infinite" + response["expires_at"] = nil + } else { + response["expires_in"] = expiresIn.String() + response["expires_at"] = time.Now().Add(expiresIn) + } + s.respondJSON(w, http.StatusCreated, response) } // handleListRegistrationTokens lists all registration tokens diff --git a/internal/api/jobs.go b/internal/api/jobs.go index a93dcab..3f460da 100644 --- a/internal/api/jobs.go +++ b/internal/api/jobs.go @@ -841,20 +841,25 @@ func (s *Server) handleUploadJobFile(w http.ResponseWriter, r *http.Request) { return } - // Parse multipart form - err = r.ParseMultipartForm(500 << 20) // 500 MB (larger for ZIP files) + // Parse multipart form with large limit for big files + // Note: For very large files, this will use temporary files on disk + err = r.ParseMultipartForm(20 << 30) // 20 GB (for large ZIP files and blend files) if err != nil { - s.respondError(w, http.StatusBadRequest, "Failed to parse form") + log.Printf("Error parsing multipart form for job %d: %v", jobID, err) + s.respondError(w, http.StatusBadRequest, fmt.Sprintf("Failed to parse form: %v", err)) return } file, header, err := r.FormFile("file") if err != nil { - s.respondError(w, http.StatusBadRequest, "No file provided") + log.Printf("Error getting file from form for job %d: %v", jobID, err) + s.respondError(w, http.StatusBadRequest, fmt.Sprintf("No file provided: %v", err)) return } defer file.Close() + log.Printf("Uploading file '%s' (size: %d bytes) for job %d", header.Filename, header.Size, jobID) + jobPath := s.storage.JobPath(jobID) if err := os.MkdirAll(jobPath, 0755); err != nil { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create job directory: %v", err)) @@ -867,22 +872,34 @@ func (s *Server) handleUploadJobFile(w http.ResponseWriter, r *http.Request) { // Check if this is a ZIP file if strings.HasSuffix(strings.ToLower(header.Filename), ".zip") { + log.Printf("Processing ZIP file '%s' for job %d", header.Filename, jobID) // Extract ZIP file zipPath := filepath.Join(jobPath, header.Filename) + log.Printf("Creating ZIP file at: %s", zipPath) zipFile, err := os.Create(zipPath) if err != nil { + log.Printf("ERROR: Failed to create ZIP file for job %d: %v", jobID, err) s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to create ZIP file: %v", err)) return } - _, err = io.Copy(zipFile, file) + log.Printf("Copying %d bytes to ZIP file for job %d...", header.Size, jobID) + copied, err := io.Copy(zipFile, file) zipFile.Close() if err != nil { + log.Printf("ERROR: Failed to save ZIP file for job %d (copied %d bytes): %v", jobID, copied, err) s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to save ZIP file: %v", err)) return } + log.Printf("Successfully copied %d bytes to ZIP file for job %d", copied, jobID) // Record ZIP file in database - zipInfo, _ := os.Stat(zipPath) + zipInfo, err := os.Stat(zipPath) + if err != nil { + log.Printf("ERROR: Failed to stat ZIP file for job %d: %v", jobID, err) + s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to stat ZIP file: %v", err)) + return + } + log.Printf("Recording ZIP file in database for job %d (size: %d bytes)", jobID, zipInfo.Size()) err = s.db.QueryRow( `INSERT INTO job_files (job_id, file_type, file_path, file_name, file_size) VALUES (?, ?, ?, ?, ?) @@ -890,16 +907,21 @@ func (s *Server) handleUploadJobFile(w http.ResponseWriter, r *http.Request) { jobID, types.JobFileTypeInput, zipPath, header.Filename, zipInfo.Size(), ).Scan(&fileID) if err != nil { + log.Printf("ERROR: Failed to record ZIP file in database for job %d: %v", jobID, err) s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to record ZIP file: %v", err)) return } + log.Printf("ZIP file recorded in database with ID %d for job %d", fileID, jobID) // Extract ZIP file + log.Printf("Extracting ZIP file for job %d...", jobID) extractedFiles, err = s.storage.ExtractZip(zipPath, jobPath) if err != nil { + log.Printf("ERROR: Failed to extract ZIP file for job %d: %v", jobID, err) s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to extract ZIP file: %v", err)) return } + log.Printf("Successfully extracted %d files from ZIP for job %d", len(extractedFiles), jobID) // Find main blend file (check for user selection first, then auto-detect) mainBlendParam := r.FormValue("main_blend_file") diff --git a/internal/api/runners.go b/internal/api/runners.go index 7bec667..68caab3 100644 --- a/internal/api/runners.go +++ b/internal/api/runners.go @@ -70,6 +70,9 @@ func (s *Server) runnerAuthMiddleware(next http.HandlerFunc) http.HandlerFunc { } // handleRegisterRunner registers a new runner +// Note: Token expiration only affects whether the token can be used for registration. +// Once a runner is registered, it receives its own runner_secret and manager_secret +// and operates independently. The token expiration does not affect registered runners. func (s *Server) handleRegisterRunner(w http.ResponseWriter, r *http.Request) { var req struct { types.RegisterRunnerRequest @@ -90,7 +93,7 @@ func (s *Server) handleRegisterRunner(w http.ResponseWriter, r *http.Request) { return } - // Validate registration token + // Validate registration token (expiration only affects token usability, not registered runners) result, err := s.secrets.ValidateRegistrationTokenDetailed(req.RegistrationToken) if err != nil { s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to validate token: %v", err)) @@ -119,7 +122,7 @@ func (s *Server) handleRegisterRunner(w http.ResponseWriter, r *http.Request) { return } - // Generate runner secret + // Generate runner secret (runner will use this for all future authentication, independent of token) runnerSecret, err := s.secrets.GenerateRunnerSecret() if err != nil { s.respondError(w, http.StatusInternalServerError, "Failed to generate runner secret") @@ -347,7 +350,7 @@ func (s *Server) handleUploadFileFromRunner(w http.ResponseWriter, r *http.Reque return } - err = r.ParseMultipartForm(100 << 20) // 100 MB + err = r.ParseMultipartForm(50 << 30) // 50 GB (for large output files) if err != nil { s.respondError(w, http.StatusBadRequest, "Failed to parse form") return diff --git a/internal/auth/secrets.go b/internal/auth/secrets.go index fa4a699..9135d41 100644 --- a/internal/auth/secrets.go +++ b/internal/auth/secrets.go @@ -76,13 +76,22 @@ func (s *Secrets) GetManagerSecret() (string, error) { } // GenerateRegistrationToken generates a new registration token +// If expiresIn is 0, the token will never expire (uses far future date) +// Note: Token expiration only affects whether the token can be used for registration. +// Once a runner registers, it operates independently using its own secrets. func (s *Secrets) GenerateRegistrationToken(createdBy int64, expiresIn time.Duration) (string, error) { token, err := generateSecret(32) if err != nil { return "", fmt.Errorf("failed to generate token: %w", err) } - expiresAt := time.Now().Add(expiresIn) + var expiresAt time.Time + if expiresIn == 0 { + // Use far future date (year 9999) to represent infinite expiration + expiresAt = time.Date(9999, 12, 31, 23, 59, 59, 0, time.UTC) + } else { + expiresAt = time.Now().Add(expiresIn) + } _, err = s.db.Exec( "INSERT INTO registration_tokens (token, expires_at, created_by) VALUES (?, ?, ?)", @@ -141,9 +150,16 @@ func (s *Secrets) ValidateRegistrationTokenDetailed(token string) (*TokenValidat return &TokenValidationResult{Valid: false, Reason: "already_used"}, nil } - if time.Now().After(expiresAt) { - return &TokenValidationResult{Valid: false, Reason: "expired"}, nil + // Check if token has infinite expiration (year 9999 or later) + // Tokens with infinite expiration never expire + infiniteExpirationThreshold := time.Date(3000, 1, 1, 0, 0, 0, 0, time.UTC) + if expiresAt.Before(infiniteExpirationThreshold) { + // Normal expiration check for tokens with finite expiration + if time.Now().After(expiresAt) { + return &TokenValidationResult{Valid: false, Reason: "expired"}, nil + } } + // If expiresAt is after the threshold, treat it as infinite (never expires) // Mark token as used _, err = s.db.Exec("UPDATE registration_tokens SET used = 1 WHERE id = ?", id) diff --git a/internal/database/schema.go b/internal/database/schema.go index 200828a..694bba8 100644 --- a/internal/database/schema.go +++ b/internal/database/schema.go @@ -122,7 +122,7 @@ func (db *DB) migrate() error { file_type TEXT NOT NULL, file_path TEXT NOT NULL, file_name TEXT NOT NULL, - file_size INTEGER NOT NULL, + file_size BIGINT NOT NULL, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); @@ -213,6 +213,17 @@ func (db *DB) migrate() error { `ALTER TABLE tasks ADD COLUMN IF NOT EXISTS retry_count INTEGER DEFAULT 0`, `ALTER TABLE tasks ADD COLUMN IF NOT EXISTS max_retries INTEGER DEFAULT 3`, `ALTER TABLE tasks ADD COLUMN IF NOT EXISTS timeout_seconds INTEGER`, + // Migrate file_size from INTEGER to BIGINT to support large files (>2GB) + // DuckDB doesn't support direct ALTER COLUMN TYPE, so we use a workaround: + // 1. Add new column as BIGINT + // 2. Copy data from old column + // 3. Drop old column + // 4. Rename new column + // Note: This will only run if the column exists and is INTEGER + `ALTER TABLE job_files ADD COLUMN IF NOT EXISTS file_size_new BIGINT`, + `UPDATE job_files SET file_size_new = CAST(file_size AS BIGINT) WHERE file_size_new IS NULL`, + `ALTER TABLE job_files DROP COLUMN IF EXISTS file_size`, + `ALTER TABLE job_files RENAME COLUMN file_size_new TO file_size`, } for _, migration := range migrations { @@ -220,6 +231,7 @@ func (db *DB) migrate() error { if _, err := db.Exec(migration); err != nil { // Log but don't fail - column might already exist or table might not exist yet // This is fine for migrations that run after schema creation + // For the file_size migration, if it fails (e.g., already BIGINT), that's fine } } @@ -235,16 +247,6 @@ func (db *DB) migrate() error { } return nil - - for _, migration := range migrations { - // DuckDB supports IF NOT EXISTS for ALTER TABLE, so we can safely execute - if _, err := db.Exec(migration); err != nil { - // Log but don't fail - column might already exist or table might not exist yet - // This is fine for migrations that run after schema creation - } - } - - return nil } // Close closes the database connection