Enhance logging and context handling in job management. Introduce a logger initialization with configurable parameters in the manager and runner commands. Update job context handling to use tar files instead of tar.gz, and implement ETag generation for improved caching. Refactor API endpoints to support new context file structure and enhance error handling in job submissions. Add support for unhide objects and auto-execution options in job creation requests.

internal/database/schema.go

@@ -165,14 +165,17 @@ func (db *DB) migrate() error {
 
 	CREATE INDEX IF NOT EXISTS idx_jobs_user_id ON jobs(user_id);
 	CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status);
+	CREATE INDEX IF NOT EXISTS idx_jobs_user_status_created ON jobs(user_id, status, created_at DESC);
 	CREATE INDEX IF NOT EXISTS idx_tasks_job_id ON tasks(job_id);
 	CREATE INDEX IF NOT EXISTS idx_tasks_runner_id ON tasks(runner_id);
 	CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status);
+	CREATE INDEX IF NOT EXISTS idx_tasks_job_status ON tasks(job_id, status);
 	CREATE INDEX IF NOT EXISTS idx_tasks_started_at ON tasks(started_at);
 	CREATE INDEX IF NOT EXISTS idx_job_files_job_id ON job_files(job_id);
 	CREATE INDEX IF NOT EXISTS idx_registration_tokens_token ON registration_tokens(token);
 	CREATE INDEX IF NOT EXISTS idx_registration_tokens_expires_at ON registration_tokens(expires_at);
 	CREATE INDEX IF NOT EXISTS idx_task_logs_task_id_created_at ON task_logs(task_id, created_at);
+	CREATE INDEX IF NOT EXISTS idx_task_logs_task_id_id ON task_logs(task_id, id DESC);
 	CREATE INDEX IF NOT EXISTS idx_task_logs_runner_id ON task_logs(runner_id);
 	CREATE INDEX IF NOT EXISTS idx_task_steps_task_id ON task_steps(task_id);
 	CREATE INDEX IF NOT EXISTS idx_runners_last_heartbeat ON runners(last_heartbeat);
@@ -213,6 +216,9 @@ func (db *DB) migrate() error {
 		`ALTER TABLE tasks ADD COLUMN IF NOT EXISTS retry_count INTEGER DEFAULT 0`,
 		`ALTER TABLE tasks ADD COLUMN IF NOT EXISTS max_retries INTEGER DEFAULT 3`,
 		`ALTER TABLE tasks ADD COLUMN IF NOT EXISTS timeout_seconds INTEGER`,
+		// Add updated_at columns for ETag support
+		`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP`,
+		`ALTER TABLE tasks ADD COLUMN IF NOT EXISTS updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP`,
 		// Migrate file_size from INTEGER to BIGINT to support large files (>2GB)
 		// DuckDB doesn't support direct ALTER COLUMN TYPE, so we use a workaround:
 		// 1. Add new column as BIGINT