Update .gitignore to include log files and database journal files. Modify go.mod to update dependencies for go-sqlite3 and cloud.google.com/go/compute/metadata. Enhance Makefile to include logging options for manager and runner commands. Introduce new job token handling in auth package and implement database migration scripts. Refactor manager and runner components to improve job processing and metadata extraction. Add support for video preview in frontend components and enhance WebSocket management for channel subscriptions.
This commit is contained in:
146
internal/runner/workspace/archive.go
Normal file
146
internal/runner/workspace/archive.go
Normal file
@@ -0,0 +1,146 @@
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ExtractTar extracts a tar archive from a reader to a directory.
|
||||
func ExtractTar(reader io.Reader, destDir string) error {
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create destination directory: %w", err)
|
||||
}
|
||||
|
||||
tarReader := tar.NewReader(reader)
|
||||
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read tar header: %w", err)
|
||||
}
|
||||
|
||||
// Sanitize path to prevent directory traversal
|
||||
targetPath := filepath.Join(destDir, header.Name)
|
||||
if !strings.HasPrefix(filepath.Clean(targetPath), filepath.Clean(destDir)+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("invalid file path in tar: %s", header.Name)
|
||||
}
|
||||
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(targetPath, os.FileMode(header.Mode)); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
case tar.TypeReg:
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||
return fmt.Errorf("failed to create parent directory: %w", err)
|
||||
}
|
||||
|
||||
outFile, err := os.Create(targetPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create file: %w", err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(outFile, tarReader); err != nil {
|
||||
outFile.Close()
|
||||
return fmt.Errorf("failed to write file: %w", err)
|
||||
}
|
||||
outFile.Close()
|
||||
|
||||
if err := os.Chmod(targetPath, os.FileMode(header.Mode)); err != nil {
|
||||
log.Printf("Warning: failed to set file permissions: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExtractTarStripPrefix extracts a tar archive, stripping the top-level directory.
|
||||
// Useful for Blender archives like "blender-4.2.3-linux-x64/".
|
||||
func ExtractTarStripPrefix(reader io.Reader, destDir string) error {
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tarReader := tar.NewReader(reader)
|
||||
stripPrefix := ""
|
||||
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Determine strip prefix from first entry (e.g., "blender-4.2.3-linux-x64/")
|
||||
if stripPrefix == "" {
|
||||
parts := strings.SplitN(header.Name, "/", 2)
|
||||
if len(parts) > 0 {
|
||||
stripPrefix = parts[0] + "/"
|
||||
}
|
||||
}
|
||||
|
||||
// Strip the top-level directory
|
||||
name := strings.TrimPrefix(header.Name, stripPrefix)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
targetPath := filepath.Join(destDir, name)
|
||||
|
||||
switch header.Typeflag {
|
||||
case tar.TypeDir:
|
||||
if err := os.MkdirAll(targetPath, os.FileMode(header.Mode)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
case tar.TypeReg:
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
outFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(header.Mode))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(outFile, tarReader); err != nil {
|
||||
outFile.Close()
|
||||
return err
|
||||
}
|
||||
outFile.Close()
|
||||
|
||||
case tar.TypeSymlink:
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
os.Remove(targetPath) // Remove existing symlink if present
|
||||
if err := os.Symlink(header.Linkname, targetPath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExtractTarFile extracts a tar file to a directory.
|
||||
func ExtractTarFile(tarPath, destDir string) error {
|
||||
file, err := os.Open(tarPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open tar file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
return ExtractTar(file, destDir)
|
||||
}
|
||||
|
||||
217
internal/runner/workspace/workspace.go
Normal file
217
internal/runner/workspace/workspace.go
Normal file
@@ -0,0 +1,217 @@
|
||||
// Package workspace manages runner workspace directories.
|
||||
package workspace
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Manager handles workspace directory operations.
|
||||
type Manager struct {
|
||||
baseDir string
|
||||
runnerName string
|
||||
}
|
||||
|
||||
// NewManager creates a new workspace manager.
|
||||
func NewManager(runnerName string) *Manager {
|
||||
m := &Manager{
|
||||
runnerName: sanitizeName(runnerName),
|
||||
}
|
||||
m.init()
|
||||
return m
|
||||
}
|
||||
|
||||
func sanitizeName(name string) string {
|
||||
name = strings.ReplaceAll(name, " ", "_")
|
||||
name = strings.ReplaceAll(name, "/", "_")
|
||||
name = strings.ReplaceAll(name, "\\", "_")
|
||||
name = strings.ReplaceAll(name, ":", "_")
|
||||
return name
|
||||
}
|
||||
|
||||
func (m *Manager) init() {
|
||||
// Prefer current directory if writable, otherwise use temp
|
||||
baseDir := os.TempDir()
|
||||
if cwd, err := os.Getwd(); err == nil {
|
||||
baseDir = cwd
|
||||
}
|
||||
|
||||
m.baseDir = filepath.Join(baseDir, "jiggablend-workspaces", m.runnerName)
|
||||
if err := os.MkdirAll(m.baseDir, 0755); err != nil {
|
||||
log.Printf("Warning: Failed to create workspace directory %s: %v", m.baseDir, err)
|
||||
// Fallback to temp directory
|
||||
m.baseDir = filepath.Join(os.TempDir(), "jiggablend-workspaces", m.runnerName)
|
||||
if err := os.MkdirAll(m.baseDir, 0755); err != nil {
|
||||
log.Printf("Error: Failed to create fallback workspace directory: %v", err)
|
||||
// Last resort
|
||||
m.baseDir = filepath.Join(os.TempDir(), "jiggablend-runner")
|
||||
os.MkdirAll(m.baseDir, 0755)
|
||||
}
|
||||
}
|
||||
log.Printf("Runner workspace initialized at: %s", m.baseDir)
|
||||
}
|
||||
|
||||
// BaseDir returns the base workspace directory.
|
||||
func (m *Manager) BaseDir() string {
|
||||
return m.baseDir
|
||||
}
|
||||
|
||||
// JobDir returns the directory for a specific job.
|
||||
func (m *Manager) JobDir(jobID int64) string {
|
||||
return filepath.Join(m.baseDir, fmt.Sprintf("job-%d", jobID))
|
||||
}
|
||||
|
||||
// VideoDir returns the directory for encoding.
|
||||
func (m *Manager) VideoDir(jobID int64) string {
|
||||
return filepath.Join(m.baseDir, fmt.Sprintf("job-%d-video", jobID))
|
||||
}
|
||||
|
||||
// BlenderDir returns the directory for Blender installations.
|
||||
func (m *Manager) BlenderDir() string {
|
||||
return filepath.Join(m.baseDir, "blender-versions")
|
||||
}
|
||||
|
||||
// CreateJobDir creates and returns the job directory.
|
||||
func (m *Manager) CreateJobDir(jobID int64) (string, error) {
|
||||
dir := m.JobDir(jobID)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return "", fmt.Errorf("failed to create job directory: %w", err)
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// CreateVideoDir creates and returns the encode directory.
|
||||
func (m *Manager) CreateVideoDir(jobID int64) (string, error) {
|
||||
dir := m.VideoDir(jobID)
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return "", fmt.Errorf("failed to create video directory: %w", err)
|
||||
}
|
||||
return dir, nil
|
||||
}
|
||||
|
||||
// CleanupJobDir removes a job directory.
|
||||
func (m *Manager) CleanupJobDir(jobID int64) error {
|
||||
dir := m.JobDir(jobID)
|
||||
return os.RemoveAll(dir)
|
||||
}
|
||||
|
||||
// CleanupVideoDir removes an encode directory.
|
||||
func (m *Manager) CleanupVideoDir(jobID int64) error {
|
||||
dir := m.VideoDir(jobID)
|
||||
return os.RemoveAll(dir)
|
||||
}
|
||||
|
||||
// Cleanup removes the entire workspace directory.
|
||||
func (m *Manager) Cleanup() {
|
||||
if m.baseDir != "" {
|
||||
log.Printf("Cleaning up workspace directory: %s", m.baseDir)
|
||||
if err := os.RemoveAll(m.baseDir); err != nil {
|
||||
log.Printf("Warning: Failed to remove workspace directory %s: %v", m.baseDir, err)
|
||||
} else {
|
||||
log.Printf("Successfully removed workspace directory: %s", m.baseDir)
|
||||
}
|
||||
}
|
||||
|
||||
// Also clean up any orphaned jiggablend directories
|
||||
cleanupOrphanedWorkspaces()
|
||||
}
|
||||
|
||||
// cleanupOrphanedWorkspaces removes any jiggablend workspace directories
|
||||
// that might be left behind from previous runs or crashes.
|
||||
func cleanupOrphanedWorkspaces() {
|
||||
log.Printf("Cleaning up orphaned jiggablend workspace directories...")
|
||||
|
||||
dirsToCheck := []string{".", os.TempDir()}
|
||||
for _, baseDir := range dirsToCheck {
|
||||
workspaceDir := filepath.Join(baseDir, "jiggablend-workspaces")
|
||||
if _, err := os.Stat(workspaceDir); err == nil {
|
||||
log.Printf("Removing orphaned workspace directory: %s", workspaceDir)
|
||||
if err := os.RemoveAll(workspaceDir); err != nil {
|
||||
log.Printf("Warning: Failed to remove workspace directory %s: %v", workspaceDir, err)
|
||||
} else {
|
||||
log.Printf("Successfully removed workspace directory: %s", workspaceDir)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FindBlendFiles finds all .blend files in a directory.
|
||||
func FindBlendFiles(dir string) ([]string, error) {
|
||||
var blendFiles []string
|
||||
|
||||
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() && strings.HasSuffix(strings.ToLower(info.Name()), ".blend") {
|
||||
// Check it's not a Blender save file (.blend1, .blend2, etc.)
|
||||
lower := strings.ToLower(info.Name())
|
||||
idx := strings.LastIndex(lower, ".blend")
|
||||
if idx != -1 {
|
||||
suffix := lower[idx+len(".blend"):]
|
||||
isSaveFile := false
|
||||
if len(suffix) > 0 {
|
||||
isSaveFile = true
|
||||
for _, r := range suffix {
|
||||
if r < '0' || r > '9' {
|
||||
isSaveFile = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !isSaveFile {
|
||||
relPath, _ := filepath.Rel(dir, path)
|
||||
blendFiles = append(blendFiles, relPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
return blendFiles, err
|
||||
}
|
||||
|
||||
// FindFirstBlendFile finds the first .blend file in a directory.
|
||||
func FindFirstBlendFile(dir string) (string, error) {
|
||||
var blendFile string
|
||||
|
||||
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() && strings.HasSuffix(strings.ToLower(info.Name()), ".blend") {
|
||||
lower := strings.ToLower(info.Name())
|
||||
idx := strings.LastIndex(lower, ".blend")
|
||||
if idx != -1 {
|
||||
suffix := lower[idx+len(".blend"):]
|
||||
isSaveFile := false
|
||||
if len(suffix) > 0 {
|
||||
isSaveFile = true
|
||||
for _, r := range suffix {
|
||||
if r < '0' || r > '9' {
|
||||
isSaveFile = false
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !isSaveFile {
|
||||
blendFile = path
|
||||
return filepath.SkipAll
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if blendFile == "" {
|
||||
return "", fmt.Errorf("no .blend file found in %s", dir)
|
||||
}
|
||||
return blendFile, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user