feat(service): add TaskService, FileStagingService, and refactor ApplicationService for task submission

2026-04-15 21:31:02 +08:00
parent acf8c1d62b
commit ec64300ff2
9 changed files with 2394 additions and 136 deletions
--- a/internal/service/task_service_test.go
+++ b/internal/service/task_service_test.go
@@ -0,0 +1,538 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"gcy_hpc_server/internal/model"
+	"gcy_hpc_server/internal/slurm"
+	"gcy_hpc_server/internal/store"
+
+	"go.uber.org/zap"
+	"gorm.io/driver/sqlite"
+	gormlogger "gorm.io/gorm/logger"
+
+	"gorm.io/gorm"
+)
+
+func setupTaskTestDB(t *testing.T) *gorm.DB {
+	t.Helper()
+	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{
+		Logger: gormlogger.Default.LogMode(gormlogger.Silent),
+	})
+	if err != nil {
+		t.Fatalf("open sqlite: %v", err)
+	}
+	if err := db.AutoMigrate(&model.Task{}, &model.Application{}, &model.File{}, &model.FileBlob{}); err != nil {
+		t.Fatalf("auto migrate: %v", err)
+	}
+	return db
+}
+
+type taskTestEnv struct {
+	taskStore   *store.TaskStore
+	appStore    *store.ApplicationStore
+	fileStore   *store.FileStore
+	blobStore   *store.BlobStore
+	svc         *TaskService
+	srv         *httptest.Server
+	db          *gorm.DB
+	workDirBase string
+}
+
+func newTaskTestEnv(t *testing.T, slurmHandler http.HandlerFunc) *taskTestEnv {
+	t.Helper()
+	db := setupTaskTestDB(t)
+
+	ts := store.NewTaskStore(db)
+	as := store.NewApplicationStore(db)
+	fs := store.NewFileStore(db)
+	bs := store.NewBlobStore(db)
+
+	srv := httptest.NewServer(slurmHandler)
+	client, _ := slurm.NewClient(srv.URL, srv.Client())
+	jobSvc := NewJobService(client, zap.NewNop())
+
+	workDirBase := filepath.Join(t.TempDir(), "workdir")
+	os.MkdirAll(workDirBase, 0777)
+
+	svc := NewTaskService(ts, as, fs, bs, nil, jobSvc, workDirBase, zap.NewNop())
+
+	return &taskTestEnv{
+		taskStore:   ts,
+		appStore:    as,
+		fileStore:   fs,
+		blobStore:   bs,
+		svc:         svc,
+		srv:         srv,
+		db:          db,
+		workDirBase: workDirBase,
+	}
+}
+
+func (e *taskTestEnv) close() {
+	e.srv.Close()
+}
+
+func (e *taskTestEnv) createApp(t *testing.T, name, script string, params json.RawMessage) int64 {
+	t.Helper()
+	id, err := e.appStore.Create(context.Background(), &model.CreateApplicationRequest{
+		Name:           name,
+		ScriptTemplate: script,
+		Parameters:     params,
+	})
+	if err != nil {
+		t.Fatalf("create app: %v", err)
+	}
+	return id
+}
+
+func TestTaskService_CreateTask_Success(t *testing.T) {
+	env := newTaskTestEnv(t, nil)
+	defer env.close()
+
+	appID := env.createApp(t, "my-app", "#!/bin/bash\necho hello", json.RawMessage(`[]`))
+
+	task, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+		AppID:    appID,
+		TaskName: "test-task",
+		Values:   map[string]string{"KEY": "val"},
+	})
+	if err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+	if task.ID == 0 {
+		t.Error("expected non-zero task ID")
+	}
+	if task.AppID != appID {
+		t.Errorf("AppID = %d, want %d", task.AppID, appID)
+	}
+	if task.AppName != "my-app" {
+		t.Errorf("AppName = %q, want %q", task.AppName, "my-app")
+	}
+	if task.Status != model.TaskStatusSubmitted {
+		t.Errorf("Status = %q, want %q", task.Status, model.TaskStatusSubmitted)
+	}
+	if task.TaskName != "test-task" {
+		t.Errorf("TaskName = %q, want %q", task.TaskName, "test-task")
+	}
+
+	var values map[string]string
+	if err := json.Unmarshal(task.Values, &values); err != nil {
+		t.Fatalf("unmarshal values: %v", err)
+	}
+	if values["KEY"] != "val" {
+		t.Errorf("values[KEY] = %q, want %q", values["KEY"], "val")
+	}
+}
+
+func TestTaskService_CreateTask_InvalidAppID(t *testing.T) {
+	env := newTaskTestEnv(t, nil)
+	defer env.close()
+
+	_, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+		AppID: 999,
+	})
+	if err == nil {
+		t.Fatal("expected error for invalid app_id")
+	}
+	if !strings.Contains(err.Error(), "not found") {
+		t.Errorf("error should mention 'not found', got: %v", err)
+	}
+}
+
+func TestTaskService_CreateTask_ExceedsFileLimit(t *testing.T) {
+	env := newTaskTestEnv(t, nil)
+	defer env.close()
+
+	appID := env.createApp(t, "app", "#!/bin/bash\necho hi", nil)
+
+	fileIDs := make([]int64, 101)
+	for i := range fileIDs {
+		fileIDs[i] = int64(i + 1)
+	}
+
+	_, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+		AppID:        appID,
+		InputFileIDs: fileIDs,
+	})
+	if err == nil {
+		t.Fatal("expected error for exceeding file limit")
+	}
+	if !strings.Contains(err.Error(), "exceeds limit") {
+		t.Errorf("error should mention limit, got: %v", err)
+	}
+}
+
+func TestTaskService_CreateTask_DuplicateFileIDs(t *testing.T) {
+	env := newTaskTestEnv(t, nil)
+	defer env.close()
+
+	appID := env.createApp(t, "app", "#!/bin/bash\necho hi", nil)
+
+	ctx := context.Background()
+	for _, id := range []int64{1, 2} {
+		f := &model.File{
+			Name:       "file.txt",
+			BlobSHA256: "abc123",
+		}
+		if err := env.fileStore.Create(ctx, f); err != nil {
+			t.Fatalf("create file: %v", err)
+		}
+		if f.ID != id {
+			t.Fatalf("expected file ID %d, got %d", id, f.ID)
+		}
+	}
+
+	task, err := env.svc.CreateTask(ctx, &model.CreateTaskRequest{
+		AppID:        appID,
+		InputFileIDs: []int64{1, 1, 2, 2},
+	})
+	if err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+
+	var fileIDs []int64
+	if err := json.Unmarshal(task.InputFileIDs, &fileIDs); err != nil {
+		t.Fatalf("unmarshal file ids: %v", err)
+	}
+	if len(fileIDs) != 2 {
+		t.Fatalf("expected 2 deduplicated file IDs, got %d: %v", len(fileIDs), fileIDs)
+	}
+	if fileIDs[0] != 1 || fileIDs[1] != 2 {
+		t.Errorf("expected [1,2], got %v", fileIDs)
+	}
+}
+
+func TestTaskService_CreateTask_AutoName(t *testing.T) {
+	env := newTaskTestEnv(t, nil)
+	defer env.close()
+
+	appID := env.createApp(t, "My Cool App", "#!/bin/bash\necho hi", nil)
+
+	task, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+		AppID: appID,
+	})
+	if err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+	if !strings.HasPrefix(task.TaskName, "My_Cool_App_") {
+		t.Errorf("auto-generated name should start with 'My_Cool_App_', got %q", task.TaskName)
+	}
+}
+
+func TestTaskService_CreateTask_NilValues(t *testing.T) {
+	env := newTaskTestEnv(t, nil)
+	defer env.close()
+
+	appID := env.createApp(t, "app", "#!/bin/bash\necho hi", nil)
+
+	task, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+		AppID:  appID,
+		Values: nil,
+	})
+	if err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+	if string(task.Values) != `{}` {
+		t.Errorf("Values = %q, want {}", string(task.Values))
+	}
+}
+
+func TestTaskService_ProcessTask_Success(t *testing.T) {
+	jobID := int32(42)
+	env := newTaskTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(slurm.OpenapiJobSubmitResponse{
+			Result: &slurm.JobSubmitResponseMsg{JobID: &jobID},
+		})
+	}))
+	defer env.close()
+
+	appID := env.createApp(t, "test-app", "#!/bin/bash\necho $INPUT", json.RawMessage(`[{"name":"INPUT","type":"string","required":true}]`))
+
+	task, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+		AppID:  appID,
+		Values: map[string]string{"INPUT": "hello"},
+	})
+	if err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+
+	err = env.svc.ProcessTask(context.Background(), task.ID)
+	if err != nil {
+		t.Fatalf("ProcessTask: %v", err)
+	}
+
+	updated, _ := env.taskStore.GetByID(context.Background(), task.ID)
+	if updated.Status != model.TaskStatusQueued {
+		t.Errorf("Status = %q, want %q", updated.Status, model.TaskStatusQueued)
+	}
+	if updated.SlurmJobID == nil || *updated.SlurmJobID != 42 {
+		t.Errorf("SlurmJobID = %v, want 42", updated.SlurmJobID)
+	}
+	if updated.WorkDir == "" {
+		t.Error("WorkDir should not be empty")
+	}
+	if !strings.HasPrefix(updated.WorkDir, env.workDirBase) {
+		t.Errorf("WorkDir = %q, should start with %q", updated.WorkDir, env.workDirBase)
+	}
+
+	info, err := os.Stat(updated.WorkDir)
+	if err != nil {
+		t.Fatalf("stat workdir: %v", err)
+	}
+	if !info.IsDir() {
+		t.Error("WorkDir should be a directory")
+	}
+}
+
+func TestTaskService_ProcessTask_TaskNotFound(t *testing.T) {
+	env := newTaskTestEnv(t, nil)
+	defer env.close()
+
+	err := env.svc.ProcessTask(context.Background(), 999)
+	if err == nil {
+		t.Fatal("expected error for non-existent task")
+	}
+	if !strings.Contains(err.Error(), "not found") {
+		t.Errorf("error should mention 'not found', got: %v", err)
+	}
+}
+
+func TestTaskService_ProcessTask_SlurmError(t *testing.T) {
+	env := newTaskTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(`{"error":"slurm down"}`))
+	}))
+	defer env.close()
+
+	appID := env.createApp(t, "test-app", "#!/bin/bash\necho hello", nil)
+
+	task, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+		AppID: appID,
+	})
+	if err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+
+	err = env.svc.ProcessTask(context.Background(), task.ID)
+	if err == nil {
+		t.Fatal("expected error from Slurm")
+	}
+
+	updated, _ := env.taskStore.GetByID(context.Background(), task.ID)
+	if updated.Status != model.TaskStatusFailed {
+		t.Errorf("Status = %q, want %q", updated.Status, model.TaskStatusFailed)
+	}
+	if updated.CurrentStep != model.TaskStepSubmitting {
+		t.Errorf("CurrentStep = %q, want %q", updated.CurrentStep, model.TaskStepSubmitting)
+	}
+	if !strings.Contains(updated.ErrorMessage, "submit job") {
+		t.Errorf("ErrorMessage should mention 'submit job', got: %q", updated.ErrorMessage)
+	}
+}
+
+func TestTaskService_ProcessTaskSync(t *testing.T) {
+	jobID := int32(42)
+	env := newTaskTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(slurm.OpenapiJobSubmitResponse{
+			Result: &slurm.JobSubmitResponseMsg{JobID: &jobID},
+		})
+	}))
+	defer env.close()
+
+	appID := env.createApp(t, "sync-app", "#!/bin/bash\necho hello", nil)
+
+	resp, err := env.svc.ProcessTaskSync(context.Background(), &model.CreateTaskRequest{
+		AppID: appID,
+	})
+	if err != nil {
+		t.Fatalf("ProcessTaskSync: %v", err)
+	}
+	if resp.JobID != 42 {
+		t.Errorf("JobID = %d, want 42", resp.JobID)
+	}
+}
+
+func TestTaskService_ProcessTaskSync_NoMinIO(t *testing.T) {
+	jobID := int32(42)
+	env := newTaskTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(slurm.OpenapiJobSubmitResponse{
+			Result: &slurm.JobSubmitResponseMsg{JobID: &jobID},
+		})
+	}))
+	defer env.close()
+
+	appID := env.createApp(t, "no-minio-app", "#!/bin/bash\necho hello", nil)
+
+	resp, err := env.svc.ProcessTaskSync(context.Background(), &model.CreateTaskRequest{
+		AppID:        appID,
+		InputFileIDs: nil,
+	})
+	if err != nil {
+		t.Fatalf("ProcessTaskSync: %v", err)
+	}
+	if resp.JobID != 42 {
+		t.Errorf("JobID = %d, want 42", resp.JobID)
+	}
+}
+
+func TestTaskService_ProcessTask_NilValues(t *testing.T) {
+	jobID := int32(55)
+	env := newTaskTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(slurm.OpenapiJobSubmitResponse{
+			Result: &slurm.JobSubmitResponseMsg{JobID: &jobID},
+		})
+	}))
+	defer env.close()
+
+	appID := env.createApp(t, "nil-val-app", "#!/bin/bash\necho hello", nil)
+
+	task, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+		AppID:  appID,
+		Values: nil,
+	})
+	if err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+
+	err = env.svc.ProcessTask(context.Background(), task.ID)
+	if err != nil {
+		t.Fatalf("ProcessTask: %v", err)
+	}
+
+	updated, _ := env.taskStore.GetByID(context.Background(), task.ID)
+	if updated.Status != model.TaskStatusQueued {
+		t.Errorf("Status = %q, want %q", updated.Status, model.TaskStatusQueued)
+	}
+}
+
+func TestTaskService_ListTasks(t *testing.T) {
+	env := newTaskTestEnv(t, nil)
+	defer env.close()
+
+	appID := env.createApp(t, "list-app", "#!/bin/bash\necho hi", nil)
+
+	for i := 0; i < 3; i++ {
+		_, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+			AppID:    appID,
+			TaskName: "task-" + string(rune('A'+i)),
+		})
+		if err != nil {
+			t.Fatalf("CreateTask %d: %v", i, err)
+		}
+	}
+
+	tasks, total, err := env.svc.ListTasks(context.Background(), &model.TaskListQuery{
+		Page:     1,
+		PageSize: 10,
+	})
+	if err != nil {
+		t.Fatalf("ListTasks: %v", err)
+	}
+	if total != 3 {
+		t.Errorf("total = %d, want 3", total)
+	}
+	if len(tasks) != 3 {
+		t.Errorf("len(tasks) = %d, want 3", len(tasks))
+	}
+}
+
+func TestTaskService_ProcessTask_ValidateParams_MissingRequired(t *testing.T) {
+	jobID := int32(42)
+	env := newTaskTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(slurm.OpenapiJobSubmitResponse{
+			Result: &slurm.JobSubmitResponseMsg{JobID: &jobID},
+		})
+	}))
+	defer env.close()
+
+	// App requires INPUT param, but we submit without it
+	appID := env.createApp(t, "validation-app", "#!/bin/bash\necho $INPUT", json.RawMessage(`[{"name":"INPUT","type":"string","required":true}]`))
+
+	task, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+		AppID:  appID,
+		Values: map[string]string{}, // missing required INPUT
+	})
+	if err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+
+	err = env.svc.ProcessTask(context.Background(), task.ID)
+	if err == nil {
+		t.Fatal("expected error for missing required parameter, got nil — ValidateParams is not being called in ProcessTask pipeline")
+	}
+	errStr := err.Error()
+	if !strings.Contains(errStr, "validation") && !strings.Contains(errStr, "missing") && !strings.Contains(errStr, "INPUT") {
+		t.Errorf("error should mention 'validation', 'missing', or 'INPUT', got: %v", err)
+	}
+}
+
+func TestTaskService_ProcessTask_ValidateParams_InvalidInteger(t *testing.T) {
+	jobID := int32(42)
+	env := newTaskTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(slurm.OpenapiJobSubmitResponse{
+			Result: &slurm.JobSubmitResponseMsg{JobID: &jobID},
+		})
+	}))
+	defer env.close()
+
+	// App expects integer param NUM, but we submit "abc"
+	appID := env.createApp(t, "int-validation-app", "#!/bin/bash\necho $NUM", json.RawMessage(`[{"name":"NUM","type":"integer","required":true}]`))
+
+	task, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+		AppID:  appID,
+		Values: map[string]string{"NUM": "abc"}, // invalid integer
+	})
+	if err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+
+	err = env.svc.ProcessTask(context.Background(), task.ID)
+	if err == nil {
+		t.Fatal("expected error for invalid integer parameter, got nil — ValidateParams is not being called in ProcessTask pipeline")
+	}
+	errStr := err.Error()
+	if !strings.Contains(errStr, "integer") && !strings.Contains(errStr, "validation") && !strings.Contains(errStr, "NUM") {
+		t.Errorf("error should mention 'integer', 'validation', or 'NUM', got: %v", err)
+	}
+}
+
+func TestTaskService_ProcessTask_ValidateParams_ValidParamsSucceed(t *testing.T) {
+	jobID := int32(99)
+	env := newTaskTestEnv(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(slurm.OpenapiJobSubmitResponse{
+			Result: &slurm.JobSubmitResponseMsg{JobID: &jobID},
+		})
+	}))
+	defer env.close()
+
+	appID := env.createApp(t, "valid-params-app", "#!/bin/bash\necho $INPUT", json.RawMessage(`[{"name":"INPUT","type":"string","required":true}]`))
+
+	task, err := env.svc.CreateTask(context.Background(), &model.CreateTaskRequest{
+		AppID:  appID,
+		Values: map[string]string{"INPUT": "hello"},
+	})
+	if err != nil {
+		t.Fatalf("CreateTask: %v", err)
+	}
+
+	err = env.svc.ProcessTask(context.Background(), task.ID)
+	if err != nil {
+		t.Fatalf("ProcessTask with valid params: %v", err)
+	}
+
+	updated, _ := env.taskStore.GetByID(context.Background(), task.ID)
+	if updated.Status != model.TaskStatusQueued {
+		t.Errorf("Status = %q, want %q", updated.Status, model.TaskStatusQueued)
+	}
+	if updated.SlurmJobID == nil || *updated.SlurmJobID != 99 {
+		t.Errorf("SlurmJobID = %v, want 99", updated.SlurmJobID)
+	}
+}