Update .gitignore to include log files and database journal files. Modify go.mod to update dependencies for go-sqlite3 and cloud.google.com/go/compute/metadata. Enhance Makefile to include logging options for manager and runner commands. Introduce new job token handling in auth package and implement database migration scripts. Refactor manager and runner components to improve job processing and metadata extraction. Add support for video preview in frontend components and enhance WebSocket management for channel subscriptions.
This commit is contained in:
265
internal/manager/metadata.go
Normal file
265
internal/manager/metadata.go
Normal file
@@ -0,0 +1,265 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"database/sql"
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"jiggablend/pkg/executils"
|
||||
"jiggablend/pkg/scripts"
|
||||
"jiggablend/pkg/types"
|
||||
)
|
||||
|
||||
// handleGetJobMetadata retrieves metadata for a job
|
||||
func (s *Manager) handleGetJobMetadata(w http.ResponseWriter, r *http.Request) {
|
||||
userID, err := getUserID(r)
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusUnauthorized, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
jobID, err := parseID(r, "id")
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Verify job belongs to user
|
||||
var jobUserID int64
|
||||
var blendMetadataJSON sql.NullString
|
||||
err = s.db.With(func(conn *sql.DB) error {
|
||||
return conn.QueryRow(
|
||||
`SELECT user_id, blend_metadata FROM jobs WHERE id = ?`,
|
||||
jobID,
|
||||
).Scan(&jobUserID, &blendMetadataJSON)
|
||||
})
|
||||
if err == sql.ErrNoRows {
|
||||
s.respondError(w, http.StatusNotFound, "Job not found")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.respondError(w, http.StatusInternalServerError, fmt.Sprintf("Failed to query job: %v", err))
|
||||
return
|
||||
}
|
||||
if jobUserID != userID {
|
||||
s.respondError(w, http.StatusForbidden, "Access denied")
|
||||
return
|
||||
}
|
||||
|
||||
if !blendMetadataJSON.Valid || blendMetadataJSON.String == "" {
|
||||
s.respondJSON(w, http.StatusOK, nil)
|
||||
return
|
||||
}
|
||||
|
||||
var metadata types.BlendMetadata
|
||||
if err := json.Unmarshal([]byte(blendMetadataJSON.String), &metadata); err != nil {
|
||||
s.respondError(w, http.StatusInternalServerError, "Failed to parse metadata")
|
||||
return
|
||||
}
|
||||
|
||||
s.respondJSON(w, http.StatusOK, metadata)
|
||||
}
|
||||
|
||||
// extractMetadataFromContext extracts metadata from the blend file in a context archive
|
||||
// Returns the extracted metadata or an error
|
||||
func (s *Manager) extractMetadataFromContext(jobID int64) (*types.BlendMetadata, error) {
|
||||
contextPath := filepath.Join(s.storage.JobPath(jobID), "context.tar")
|
||||
|
||||
// Check if context exists
|
||||
if _, err := os.Stat(contextPath); err != nil {
|
||||
return nil, fmt.Errorf("context archive not found: %w", err)
|
||||
}
|
||||
|
||||
// Create temporary directory for extraction under storage base path
|
||||
tmpDir, err := s.storage.TempDir(fmt.Sprintf("jiggablend-metadata-%d-*", jobID))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create temporary directory: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(tmpDir); err != nil {
|
||||
log.Printf("Warning: Failed to clean up temp directory %s: %v", tmpDir, err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Extract context archive
|
||||
if err := s.extractTar(contextPath, tmpDir); err != nil {
|
||||
return nil, fmt.Errorf("failed to extract context: %w", err)
|
||||
}
|
||||
|
||||
// Find .blend file in extracted contents
|
||||
blendFile := ""
|
||||
err = filepath.Walk(tmpDir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() && strings.HasSuffix(strings.ToLower(info.Name()), ".blend") {
|
||||
// Check it's not a Blender save file (.blend1, .blend2, etc.)
|
||||
lower := strings.ToLower(info.Name())
|
||||
idx := strings.LastIndex(lower, ".blend")
|
||||
if idx != -1 {
|
||||
suffix := lower[idx+len(".blend"):]
|
||||
// If there are digits after .blend, it's a save file
|
||||
isSaveFile := false
|
||||
if len(suffix) > 0 {
|
||||
isSaveFile = true
|
||||
for _, r := range suffix {
|
||||
if r < '0' || r > '9' {
|
||||
isSaveFile = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !isSaveFile {
|
||||
blendFile = path
|
||||
return filepath.SkipAll // Stop walking once we find a blend file
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find blend file: %w", err)
|
||||
}
|
||||
|
||||
if blendFile == "" {
|
||||
return nil, fmt.Errorf("no .blend file found in context - the uploaded context archive must contain at least one .blend file for metadata extraction")
|
||||
}
|
||||
|
||||
// Use embedded Python script
|
||||
scriptPath := filepath.Join(tmpDir, "extract_metadata.py")
|
||||
if err := os.WriteFile(scriptPath, []byte(scripts.ExtractMetadata), 0644); err != nil {
|
||||
return nil, fmt.Errorf("failed to create extraction script: %w", err)
|
||||
}
|
||||
|
||||
// Make blend file path relative to tmpDir to avoid path resolution issues
|
||||
blendFileRel, err := filepath.Rel(tmpDir, blendFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get relative path for blend file: %w", err)
|
||||
}
|
||||
|
||||
// Execute Blender with Python script using executils
|
||||
result, err := executils.RunCommand(
|
||||
"blender",
|
||||
[]string{"-b", blendFileRel, "--python", "extract_metadata.py"},
|
||||
tmpDir,
|
||||
nil, // inherit environment
|
||||
jobID,
|
||||
nil, // no process tracker needed for metadata extraction
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
stderrOutput := ""
|
||||
stdoutOutput := ""
|
||||
if result != nil {
|
||||
stderrOutput = strings.TrimSpace(result.Stderr)
|
||||
stdoutOutput = strings.TrimSpace(result.Stdout)
|
||||
}
|
||||
log.Printf("Blender metadata extraction failed for job %d:", jobID)
|
||||
if stderrOutput != "" {
|
||||
log.Printf("Blender stderr: %s", stderrOutput)
|
||||
}
|
||||
if stdoutOutput != "" {
|
||||
log.Printf("Blender stdout (last 500 chars): %s", truncateString(stdoutOutput, 500))
|
||||
}
|
||||
if stderrOutput != "" {
|
||||
return nil, fmt.Errorf("blender metadata extraction failed: %w (stderr: %s)", err, truncateString(stderrOutput, 200))
|
||||
}
|
||||
return nil, fmt.Errorf("blender metadata extraction failed: %w", err)
|
||||
}
|
||||
|
||||
// Parse output (metadata is printed to stdout)
|
||||
metadataJSON := strings.TrimSpace(result.Stdout)
|
||||
// Extract JSON from output (Blender may print other stuff)
|
||||
jsonStart := strings.Index(metadataJSON, "{")
|
||||
jsonEnd := strings.LastIndex(metadataJSON, "}")
|
||||
if jsonStart == -1 || jsonEnd == -1 || jsonEnd <= jsonStart {
|
||||
return nil, errors.New("failed to extract JSON from Blender output")
|
||||
}
|
||||
metadataJSON = metadataJSON[jsonStart : jsonEnd+1]
|
||||
|
||||
var metadata types.BlendMetadata
|
||||
if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse metadata JSON: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("Metadata extracted for job %d: frame_start=%d, frame_end=%d", jobID, metadata.FrameStart, metadata.FrameEnd)
|
||||
return &metadata, nil
|
||||
}
|
||||
|
||||
// extractTar extracts a tar archive to a destination directory
|
||||
func (s *Manager) extractTar(tarPath, destDir string) error {
|
||||
log.Printf("Extracting tar archive: %s -> %s", tarPath, destDir)
|
||||
|
||||
// Ensure destination directory exists
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create destination directory: %w", err)
|
||||
}
|
||||
|
||||
file, err := os.Open(tarPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open archive: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
tr := tar.NewReader(file)
|
||||
|
||||
fileCount := 0
|
||||
dirCount := 0
|
||||
|
||||
for {
|
||||
header, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read tar header: %w", err)
|
||||
}
|
||||
|
||||
// Sanitize path to prevent directory traversal
|
||||
target := filepath.Join(destDir, header.Name)
|
||||
|
||||
// Ensure target is within destDir
|
||||
cleanTarget := filepath.Clean(target)
|
||||
cleanDestDir := filepath.Clean(destDir)
|
||||
if !strings.HasPrefix(cleanTarget, cleanDestDir+string(os.PathSeparator)) && cleanTarget != cleanDestDir {
|
||||
log.Printf("ERROR: Invalid file path in TAR - target: %s, destDir: %s", cleanTarget, cleanDestDir)
|
||||
return fmt.Errorf("invalid file path in archive: %s (target: %s, destDir: %s)", header.Name, cleanTarget, cleanDestDir)
|
||||
}
|
||||
|
||||
// Create parent directories
|
||||
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
// Write file
|
||||
switch header.Typeflag {
|
||||
case tar.TypeReg:
|
||||
outFile, err := os.Create(target)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
_, err = io.Copy(outFile, tr)
|
||||
if err != nil {
|
||||
outFile.Close()
|
||||
return fmt.Errorf("failed to write file: %w", err)
|
||||
}
|
||||
outFile.Close()
|
||||
fileCount++
|
||||
case tar.TypeDir:
|
||||
dirCount++
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("Extraction complete: %d files, %d directories extracted to %s", fileCount, dirCount, destDir)
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user