s1d3sw1ped/jiggablend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Enhance logging and context handling in job management. Introduce a logger initialization with configurable parameters in the manager and runner commands. Update job context handling to use tar files instead of tar.gz, and implement ETag generation for improved caching. Refactor API endpoints to support new context file structure and enhance error handling in job submissions. Add support for unhide objects and auto-execution options in job creation requests.

Browse Source
This commit is contained in:
Justin Harms
2025-11-24 21:48:05 -06:00
parent a029714e08
commit 4ac05d50a1
23 changed files with 4133 additions and 1311 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
internal/storage/storage.go
View File

@@ -3,9 +3,9 @@ package storage
import (
"archive/tar"
"archive/zip"
"compress/gzip"
"fmt"
"io"
"log"
"os"
"path/filepath"
"strings"
@@ -31,6 +31,7 @@ func (s *Storage) init() error {
s.basePath,
s.uploadsPath(),
s.outputsPath(),
s.tempPath(),
}
for _, dir := range dirs {
@@ -42,6 +43,28 @@ func (s *Storage) init() error {
return nil
}
// tempPath returns the path for temporary files
func (s *Storage) tempPath() string {
return filepath.Join(s.basePath, "temp")
}
// BasePath returns the storage base path (for cleanup tasks)
func (s *Storage) BasePath() string {
return s.basePath
}
// TempDir creates a temporary directory under the storage base path
// Returns the path to the temporary directory
func (s *Storage) TempDir(pattern string) (string, error) {
// Ensure temp directory exists
if err := os.MkdirAll(s.tempPath(), 0755); err != nil {
return "", fmt.Errorf("failed to create temp directory: %w", err)
}
// Create temp directory under storage base path
return os.MkdirTemp(s.tempPath(), pattern)
}
// uploadsPath returns the path for uploads
func (s *Storage) uploadsPath() string {
return filepath.Join(s.basePath, "uploads")
@@ -142,6 +165,13 @@ func (s *Storage) GetFileSize(filePath string) (int64, error) {
// ExtractZip extracts a ZIP file to the destination directory
// Returns a list of all extracted file paths
func (s *Storage) ExtractZip(zipPath, destDir string) ([]string, error) {
log.Printf("Extracting ZIP archive: %s -> %s", zipPath, destDir)
// Ensure destination directory exists
if err := os.MkdirAll(destDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create destination directory: %w", err)
}
r, err := zip.OpenReader(zipPath)
if err != nil {
return nil, fmt.Errorf("failed to open ZIP file: %w", err)
@@ -149,12 +179,20 @@ func (s *Storage) ExtractZip(zipPath, destDir string) ([]string, error) {
defer r.Close()
var extractedFiles []string
fileCount := 0
dirCount := 0
log.Printf("ZIP contains %d entries", len(r.File))
for _, f := range r.File {
// Sanitize file path to prevent directory traversal
destPath := filepath.Join(destDir, f.Name)
if !strings.HasPrefix(destPath, filepath.Clean(destDir)+string(os.PathSeparator)) {
return nil, fmt.Errorf("invalid file path in ZIP: %s", f.Name)
cleanDestPath := filepath.Clean(destPath)
cleanDestDir := filepath.Clean(destDir)
if !strings.HasPrefix(cleanDestPath, cleanDestDir+string(os.PathSeparator)) && cleanDestPath != cleanDestDir {
log.Printf("ERROR: Invalid file path in ZIP - target: %s, destDir: %s", cleanDestPath, cleanDestDir)
return nil, fmt.Errorf("invalid file path in ZIP: %s (target: %s, destDir: %s)", f.Name, cleanDestPath, cleanDestDir)
}
// Create directory structure
@@ -162,6 +200,7 @@ func (s *Storage) ExtractZip(zipPath, destDir string) ([]string, error) {
if err := os.MkdirAll(destPath, 0755); err != nil {
return nil, fmt.Errorf("failed to create directory: %w", err)
}
dirCount++
continue
}
@@ -191,8 +230,10 @@ func (s *Storage) ExtractZip(zipPath, destDir string) ([]string, error) {
}
extractedFiles = append(extractedFiles, destPath)
fileCount++
}
log.Printf("ZIP extraction complete: %d files, %d directories extracted to %s", fileCount, dirCount, destDir)
return extractedFiles, nil
}
@@ -261,15 +302,15 @@ func isBlenderSaveFile(filename string) bool {
return false
}
// CreateJobContext creates a tar.gz archive containing all job input files
// CreateJobContext creates a tar archive containing all job input files
// Filters out Blender save files (.blend1, .blend2, etc.)
// Uses temporary directories and streaming to handle large files efficiently
func (s *Storage) CreateJobContext(jobID int64) (string, error) {
jobPath := s.JobPath(jobID)
contextPath := filepath.Join(jobPath, "context.tar.gz")
contextPath := filepath.Join(jobPath, "context.tar")
// Create temporary directory for staging
tmpDir, err := os.MkdirTemp("", "fuego-context-*")
tmpDir, err := os.MkdirTemp("", "jiggablend-context-*")
if err != nil {
return "", fmt.Errorf("failed to create temporary directory: %w", err)
}
@@ -320,17 +361,14 @@ func (s *Storage) CreateJobContext(jobID int64) (string, error) {
return "", fmt.Errorf("no files found to include in context")
}
// Create the tar.gz file using streaming
// Create the tar file using streaming
contextFile, err := os.Create(contextPath)
if err != nil {
return "", fmt.Errorf("failed to create context file: %w", err)
}
defer contextFile.Close()
gzWriter := gzip.NewWriter(contextFile)
defer gzWriter.Close()
tarWriter := tar.NewWriter(gzWriter)
tarWriter := tar.NewWriter(contextFile)
defer tarWriter.Close()
// Add each file to the tar archive
@@ -383,9 +421,6 @@ func (s *Storage) CreateJobContext(jobID int64) (string, error) {
if err := tarWriter.Close(); err != nil {
return "", fmt.Errorf("failed to close tar writer: %w", err)
}
if err := gzWriter.Close(); err != nil {
return "", fmt.Errorf("failed to close gzip writer: %w", err)
}
if err := contextFile.Close(); err != nil {
return "", fmt.Errorf("failed to close context file: %w", err)
}
@@ -393,12 +428,12 @@ func (s *Storage) CreateJobContext(jobID int64) (string, error) {
return contextPath, nil
}
// CreateJobContextFromDir creates a context archive (tar.gz) from files in a source directory
// CreateJobContextFromDir creates a context archive (tar) from files in a source directory
// This is used during upload to immediately create the context archive as the primary artifact
// excludeFiles is a set of relative paths (from sourceDir) to exclude from the context
func (s *Storage) CreateJobContextFromDir(sourceDir string, jobID int64, excludeFiles ...string) (string, error) {
jobPath := s.JobPath(jobID)
contextPath := filepath.Join(jobPath, "context.tar.gz")
contextPath := filepath.Join(jobPath, "context.tar")
// Ensure job directory exists
if err := os.MkdirAll(jobPath, 0755); err != nil {
@@ -498,17 +533,14 @@ func (s *Storage) CreateJobContextFromDir(sourceDir string, jobID int64, exclude
return "", fmt.Errorf("multiple .blend files found at root level in context archive (found %d, expected 1)", blendFilesAtRoot)
}
// Create the tar.gz file using streaming
// Create the tar file using streaming
contextFile, err := os.Create(contextPath)
if err != nil {
return "", fmt.Errorf("failed to create context file: %w", err)
}
defer contextFile.Close()
gzWriter := gzip.NewWriter(contextFile)
defer gzWriter.Close()
tarWriter := tar.NewWriter(gzWriter)
tarWriter := tar.NewWriter(contextFile)
defer tarWriter.Close()
// Add each file to the tar archive
@@ -560,9 +592,6 @@ func (s *Storage) CreateJobContextFromDir(sourceDir string, jobID int64, exclude
if err := tarWriter.Close(); err != nil {
return "", fmt.Errorf("failed to close tar writer: %w", err)
}
if err := gzWriter.Close(); err != nil {
return "", fmt.Errorf("failed to close gzip writer: %w", err)
}
if err := contextFile.Close(); err != nil {
return "", fmt.Errorf("failed to close context file: %w", err)
}