Files
hpc/internal/testutil/mockslurm/server_test.go
dailz b9b2f0d9b4 feat(testutil): add MockSlurm, MockMinIO, TestEnv and 37 integration tests
- mockminio: in-memory ObjectStorage with all 11 methods, thread-safe, SHA256 ETag, Range support
- mockslurm: httptest server with 11 Slurm REST API endpoints, job eviction from active to history queue
- testenv: one-line test environment factory (SQLite + MockSlurm + MockMinIO + all stores/services/handlers + httptest server)
- integration tests: 37 tests covering Jobs(5), Cluster(5), App(6), Upload(5), File(4), Folder(4), Task(4), E2E(1)
- no external dependencies, no existing files modified
2026-04-16 13:23:27 +08:00

680 lines
20 KiB
Go

package mockslurm
import (
"context"
"encoding/json"
"strconv"
"strings"
"testing"
"gcy_hpc_server/internal/slurm"
)
func setupTestClient(t *testing.T) (*slurm.Client, *MockSlurm) {
t.Helper()
srv, mock := NewMockSlurmServer()
t.Cleanup(srv.Close)
client, err := slurm.NewClientWithOpts(srv.URL, slurm.WithHTTPClient(srv.Client()))
if err != nil {
t.Fatalf("failed to create client: %v", err)
}
return client, mock
}
func submitTestJob(t *testing.T, client *slurm.Client, name, partition, workDir, script string) int32 {
t.Helper()
ctx := context.Background()
resp, _, err := client.Jobs.SubmitJob(ctx, &slurm.JobSubmitReq{
Script: slurm.Ptr(script),
Job: &slurm.JobDescMsg{
Name: slurm.Ptr(name),
Partition: slurm.Ptr(partition),
CurrentWorkingDirectory: slurm.Ptr(workDir),
},
})
if err != nil {
t.Fatalf("SubmitJob failed: %v", err)
}
if resp.JobID == nil {
t.Fatal("SubmitJob returned nil JobID")
}
return *resp.JobID
}
// ---------------------------------------------------------------------------
// P0: Submit Job
// ---------------------------------------------------------------------------
func TestSubmitJob(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
resp, _, err := client.Jobs.SubmitJob(ctx, &slurm.JobSubmitReq{
Script: slurm.Ptr("#!/bin/bash\necho hello"),
Job: &slurm.JobDescMsg{
Name: slurm.Ptr("test-job"),
Partition: slurm.Ptr("normal"),
CurrentWorkingDirectory: slurm.Ptr("/tmp/work"),
},
})
if err != nil {
t.Fatalf("SubmitJob failed: %v", err)
}
if resp.JobID == nil || *resp.JobID != 1 {
t.Errorf("JobID = %v, want 1", resp.JobID)
}
if resp.StepID == nil || *resp.StepID != "Scalar" {
t.Errorf("StepID = %v, want Scalar", resp.StepID)
}
if resp.Result == nil || resp.Result.JobID == nil || *resp.Result.JobID != 1 {
t.Errorf("Result.JobID = %v, want 1", resp.Result)
}
}
func TestSubmitJobAutoIncrement(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
for i := 1; i <= 3; i++ {
resp, _, err := client.Jobs.SubmitJob(ctx, &slurm.JobSubmitReq{
Script: slurm.Ptr("#!/bin/bash\necho " + strconv.Itoa(i)),
})
if err != nil {
t.Fatalf("SubmitJob %d failed: %v", i, err)
}
if resp.JobID == nil || *resp.JobID != int32(i) {
t.Errorf("job %d: JobID = %v, want %d", i, resp.JobID, i)
}
}
}
// ---------------------------------------------------------------------------
// P0: Get All Jobs
// ---------------------------------------------------------------------------
func TestGetJobsEmpty(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
resp, _, err := client.Jobs.GetJobs(ctx, nil)
if err != nil {
t.Fatalf("GetJobs failed: %v", err)
}
if len(resp.Jobs) != 0 {
t.Errorf("len(Jobs) = %d, want 0", len(resp.Jobs))
}
}
func TestGetJobsWithSubmitted(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
submitTestJob(t, client, "job-a", "normal", "/tmp/a", "#!/bin/bash\ntrue")
submitTestJob(t, client, "job-b", "gpu", "/tmp/b", "#!/bin/bash\nfalse")
resp, _, err := client.Jobs.GetJobs(ctx, nil)
if err != nil {
t.Fatalf("GetJobs failed: %v", err)
}
if len(resp.Jobs) != 2 {
t.Fatalf("len(Jobs) = %d, want 2", len(resp.Jobs))
}
names := map[string]bool{}
for _, j := range resp.Jobs {
if j.Name != nil {
names[*j.Name] = true
}
if len(j.JobState) == 0 || j.JobState[0] != "PENDING" {
t.Errorf("JobState = %v, want [PENDING]", j.JobState)
}
}
if !names["job-a"] || !names["job-b"] {
t.Errorf("expected job-a and job-b, got %v", names)
}
}
// ---------------------------------------------------------------------------
// P0: Get Job By ID
// ---------------------------------------------------------------------------
func TestGetJobByID(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
jobID := submitTestJob(t, client, "single-job", "normal", "/tmp/work", "#!/bin/bash\necho hi")
resp, _, err := client.Jobs.GetJob(ctx, strconv.Itoa(int(jobID)), nil)
if err != nil {
t.Fatalf("GetJob failed: %v", err)
}
if len(resp.Jobs) != 1 {
t.Fatalf("len(Jobs) = %d, want 1", len(resp.Jobs))
}
job := resp.Jobs[0]
if job.JobID == nil || *job.JobID != jobID {
t.Errorf("JobID = %v, want %d", job.JobID, jobID)
}
if job.Name == nil || *job.Name != "single-job" {
t.Errorf("Name = %v, want single-job", job.Name)
}
if job.Partition == nil || *job.Partition != "normal" {
t.Errorf("Partition = %v, want normal", job.Partition)
}
if job.CurrentWorkingDirectory == nil || *job.CurrentWorkingDirectory != "/tmp/work" {
t.Errorf("CurrentWorkingDirectory = %v, want /tmp/work", job.CurrentWorkingDirectory)
}
if job.SubmitTime == nil || job.SubmitTime.Number == nil {
t.Error("SubmitTime should be non-nil")
}
}
func TestGetJobByIDNotFound(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
_, _, err := client.Jobs.GetJob(ctx, "999", nil)
if err == nil {
t.Fatal("expected error for unknown job ID, got nil")
}
if !slurm.IsNotFound(err) {
t.Errorf("error type = %T, want SlurmAPIError with 404", err)
}
}
// ---------------------------------------------------------------------------
// P0: Delete Job (triggers eviction)
// ---------------------------------------------------------------------------
func TestDeleteJob(t *testing.T) {
client, mock := setupTestClient(t)
ctx := context.Background()
jobID := submitTestJob(t, client, "cancel-me", "normal", "/tmp", "#!/bin/bash\nsleep 99")
resp, _, err := client.Jobs.DeleteJob(ctx, strconv.Itoa(int(jobID)), nil)
if err != nil {
t.Fatalf("DeleteJob failed: %v", err)
}
if resp == nil {
t.Fatal("DeleteJob returned nil response")
}
if len(mock.GetAllActiveJobs()) != 0 {
t.Error("active jobs should be empty after delete")
}
if len(mock.GetAllHistoryJobs()) != 1 {
t.Error("history should contain 1 job after delete")
}
if mock.GetJobState(jobID) != "CANCELLED" {
t.Errorf("job state = %q, want CANCELLED", mock.GetJobState(jobID))
}
}
func TestDeleteJobEvictsFromActive(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
jobID := submitTestJob(t, client, "to-delete", "normal", "/tmp", "#!/bin/bash\ntrue")
_, _, err := client.Jobs.DeleteJob(ctx, strconv.Itoa(int(jobID)), nil)
if err != nil {
t.Fatalf("DeleteJob failed: %v", err)
}
_, _, err = client.Jobs.GetJob(ctx, strconv.Itoa(int(jobID)), nil)
if err == nil {
t.Fatal("expected 404 after delete, got nil error")
}
if !slurm.IsNotFound(err) {
t.Errorf("error = %v, want not-found", err)
}
}
// ---------------------------------------------------------------------------
// P0: Job State format ([]string, not bare string)
// ---------------------------------------------------------------------------
func TestJobStateIsStringArray(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
submitTestJob(t, client, "state-test", "normal", "/tmp", "#!/bin/bash\necho")
resp, _, err := client.Jobs.GetJobs(ctx, nil)
if err != nil {
t.Fatalf("GetJobs failed: %v", err)
}
if len(resp.Jobs) == 0 {
t.Fatal("expected at least one job")
}
job := resp.Jobs[0]
if len(job.JobState) == 0 {
t.Fatal("JobState is empty — must be non-empty []string to avoid mapSlurmStateToTaskStatus silent failure")
}
if job.JobState[0] != "PENDING" {
t.Errorf("JobState[0] = %q, want %q", job.JobState[0], "PENDING")
}
raw, err := json.Marshal(job)
if err != nil {
t.Fatalf("Marshal job: %v", err)
}
if !strings.Contains(string(raw), `"job_state":["PENDING"]`) {
t.Errorf("JobState JSON = %s, want array format [\"PENDING\"]", string(raw))
}
}
// ---------------------------------------------------------------------------
// P0: Full Job Lifecycle
// ---------------------------------------------------------------------------
func TestJobLifecycle(t *testing.T) {
client, mock := setupTestClient(t)
ctx := context.Background()
jobID := submitTestJob(t, client, "lifecycle", "normal", "/tmp/lc", "#!/bin/bash\necho lifecycle")
// Verify PENDING in active
resp, _, err := client.Jobs.GetJob(ctx, strconv.Itoa(int(jobID)), nil)
if err != nil {
t.Fatalf("GetJob PENDING: %v", err)
}
if resp.Jobs[0].JobState[0] != "PENDING" {
t.Errorf("initial state = %v, want PENDING", resp.Jobs[0].JobState)
}
if len(mock.GetAllActiveJobs()) != 1 {
t.Error("should have 1 active job")
}
// Transition to RUNNING
mock.SetJobState(jobID, "RUNNING")
resp, _, err = client.Jobs.GetJob(ctx, strconv.Itoa(int(jobID)), nil)
if err != nil {
t.Fatalf("GetJob RUNNING: %v", err)
}
if resp.Jobs[0].JobState[0] != "RUNNING" {
t.Errorf("running state = %v, want RUNNING", resp.Jobs[0].JobState)
}
if resp.Jobs[0].StartTime == nil || resp.Jobs[0].StartTime.Number == nil {
t.Error("StartTime should be set for RUNNING job")
}
if len(mock.GetAllActiveJobs()) != 1 {
t.Error("should still have 1 active job after RUNNING")
}
// Transition to COMPLETED — triggers eviction
mock.SetJobState(jobID, "COMPLETED")
_, _, err = client.Jobs.GetJob(ctx, strconv.Itoa(int(jobID)), nil)
if err == nil {
t.Fatal("expected 404 after COMPLETED (evicted from active)")
}
if !slurm.IsNotFound(err) {
t.Errorf("error = %v, want not-found", err)
}
if len(mock.GetAllActiveJobs()) != 0 {
t.Error("active jobs should be empty after COMPLETED")
}
if len(mock.GetAllHistoryJobs()) != 1 {
t.Error("history should contain 1 job after COMPLETED")
}
if mock.GetJobState(jobID) != "COMPLETED" {
t.Errorf("state = %q, want COMPLETED", mock.GetJobState(jobID))
}
// Verify history endpoint returns the job
histResp, _, err := client.SlurmdbJobs.GetJob(ctx, strconv.Itoa(int(jobID)))
if err != nil {
t.Fatalf("SlurmdbJobs.GetJob: %v", err)
}
if len(histResp.Jobs) != 1 {
t.Fatalf("history jobs = %d, want 1", len(histResp.Jobs))
}
histJob := histResp.Jobs[0]
if histJob.State == nil || len(histJob.State.Current) == 0 || histJob.State.Current[0] != "COMPLETED" {
t.Errorf("history state = %v, want current=[COMPLETED]", histJob.State)
}
if histJob.ExitCode == nil || histJob.ExitCode.ReturnCode == nil || histJob.ExitCode.ReturnCode.Number == nil {
t.Error("history ExitCode should be set")
} else if *histJob.ExitCode.ReturnCode.Number != 0 {
t.Errorf("exit code = %d, want 0 for COMPLETED", *histJob.ExitCode.ReturnCode.Number)
}
}
// ---------------------------------------------------------------------------
// P1: Nodes
// ---------------------------------------------------------------------------
func TestGetNodes(t *testing.T) {
client, mock := setupTestClient(t)
_ = mock
ctx := context.Background()
resp, _, err := client.Nodes.GetNodes(ctx, nil)
if err != nil {
t.Fatalf("GetNodes failed: %v", err)
}
if resp.Nodes == nil {
t.Fatal("Nodes is nil")
}
if len(*resp.Nodes) != 3 {
t.Errorf("len(Nodes) = %d, want 3", len(*resp.Nodes))
}
names := make([]string, len(*resp.Nodes))
for i, n := range *resp.Nodes {
if n.Name == nil {
t.Errorf("node %d: Name is nil", i)
} else {
names[i] = *n.Name
}
}
for _, expected := range []string{"node01", "node02", "node03"} {
found := false
for _, n := range names {
if n == expected {
found = true
break
}
}
if !found {
t.Errorf("missing node %q in %v", expected, names)
}
}
}
func TestGetNode(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
resp, _, err := client.Nodes.GetNode(ctx, "node02", nil)
if err != nil {
t.Fatalf("GetNode failed: %v", err)
}
if resp.Nodes == nil || len(*resp.Nodes) != 1 {
t.Fatalf("expected 1 node, got %v", resp.Nodes)
}
if (*resp.Nodes)[0].Name == nil || *(*resp.Nodes)[0].Name != "node02" {
t.Errorf("Name = %v, want node02", (*resp.Nodes)[0].Name)
}
}
func TestGetNodeNotFound(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
_, _, err := client.Nodes.GetNode(ctx, "nonexistent", nil)
if err == nil {
t.Fatal("expected error for unknown node, got nil")
}
if !slurm.IsNotFound(err) {
t.Errorf("error = %v, want not-found", err)
}
}
// ---------------------------------------------------------------------------
// P1: Partitions
// ---------------------------------------------------------------------------
func TestGetPartitions(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
resp, _, err := client.Partitions.GetPartitions(ctx, nil)
if err != nil {
t.Fatalf("GetPartitions failed: %v", err)
}
if resp.Partitions == nil {
t.Fatal("Partitions is nil")
}
if len(*resp.Partitions) != 2 {
t.Errorf("len(Partitions) = %d, want 2", len(*resp.Partitions))
}
names := map[string]bool{}
for _, p := range *resp.Partitions {
if p.Name != nil {
names[*p.Name] = true
}
}
if !names["normal"] || !names["gpu"] {
t.Errorf("expected normal and gpu partitions, got %v", names)
}
}
func TestGetPartition(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
resp, _, err := client.Partitions.GetPartition(ctx, "gpu", nil)
if err != nil {
t.Fatalf("GetPartition failed: %v", err)
}
if resp.Partitions == nil || len(*resp.Partitions) != 1 {
t.Fatalf("expected 1 partition, got %v", resp.Partitions)
}
if (*resp.Partitions)[0].Name == nil || *(*resp.Partitions)[0].Name != "gpu" {
t.Errorf("Name = %v, want gpu", (*resp.Partitions)[0].Name)
}
}
func TestGetPartitionNotFound(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
_, _, err := client.Partitions.GetPartition(ctx, "nonexistent", nil)
if err == nil {
t.Fatal("expected error for unknown partition, got nil")
}
if !slurm.IsNotFound(err) {
t.Errorf("error = %v, want not-found", err)
}
}
// ---------------------------------------------------------------------------
// P1: Diag
// ---------------------------------------------------------------------------
func TestGetDiag(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
resp, _, err := client.Diag.GetDiag(ctx)
if err != nil {
t.Fatalf("GetDiag failed: %v", err)
}
if resp.Statistics == nil {
t.Fatal("Statistics is nil")
}
if resp.Statistics.ServerThreadCount == nil || *resp.Statistics.ServerThreadCount != 3 {
t.Errorf("ServerThreadCount = %v, want 3", resp.Statistics.ServerThreadCount)
}
if resp.Statistics.AgentQueueSize == nil || *resp.Statistics.AgentQueueSize != 0 {
t.Errorf("AgentQueueSize = %v, want 0", resp.Statistics.AgentQueueSize)
}
}
// ---------------------------------------------------------------------------
// P1: SlurmDB Jobs
// ---------------------------------------------------------------------------
func TestSlurmdbGetJobsEmpty(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
resp, _, err := client.SlurmdbJobs.GetJobs(ctx, nil)
if err != nil {
t.Fatalf("GetJobs failed: %v", err)
}
if len(resp.Jobs) != 0 {
t.Errorf("len(Jobs) = %d, want 0 (no history)", len(resp.Jobs))
}
}
func TestSlurmdbGetJobsAfterEviction(t *testing.T) {
client, mock := setupTestClient(t)
ctx := context.Background()
jobID := submitTestJob(t, client, "hist-job", "normal", "/tmp/h", "#!/bin/bash\necho hist")
mock.SetJobState(jobID, "RUNNING")
mock.SetJobState(jobID, "COMPLETED")
resp, _, err := client.SlurmdbJobs.GetJobs(ctx, nil)
if err != nil {
t.Fatalf("GetJobs failed: %v", err)
}
if len(resp.Jobs) != 1 {
t.Fatalf("len(Jobs) = %d, want 1", len(resp.Jobs))
}
job := resp.Jobs[0]
if job.Name == nil || *job.Name != "hist-job" {
t.Errorf("Name = %v, want hist-job", job.Name)
}
if job.State == nil || len(job.State.Current) == 0 || job.State.Current[0] != "COMPLETED" {
t.Errorf("State = %v, want current=[COMPLETED]", job.State)
}
if job.Time == nil || job.Time.Submission == nil {
t.Error("Time.Submission should be set")
}
}
func TestSlurmdbGetJobByID(t *testing.T) {
client, mock := setupTestClient(t)
ctx := context.Background()
jobID := submitTestJob(t, client, "single-hist", "normal", "/tmp/sh", "#!/bin/bash\nexit 1")
mock.SetJobState(jobID, "FAILED")
resp, _, err := client.SlurmdbJobs.GetJob(ctx, strconv.Itoa(int(jobID)))
if err != nil {
t.Fatalf("GetJob failed: %v", err)
}
if len(resp.Jobs) != 1 {
t.Fatalf("len(Jobs) = %d, want 1", len(resp.Jobs))
}
job := resp.Jobs[0]
if job.JobID == nil || *job.JobID != jobID {
t.Errorf("JobID = %v, want %d", job.JobID, jobID)
}
if job.State == nil || len(job.State.Current) == 0 || job.State.Current[0] != "FAILED" {
t.Errorf("State = %v, want current=[FAILED]", job.State)
}
if job.ExitCode == nil || job.ExitCode.ReturnCode == nil || job.ExitCode.ReturnCode.Number == nil {
t.Error("ExitCode should be set")
} else if *job.ExitCode.ReturnCode.Number != 1 {
t.Errorf("exit code = %d, want 1 for FAILED", *job.ExitCode.ReturnCode.Number)
}
}
func TestSlurmdbGetJobNotFound(t *testing.T) {
client, _ := setupTestClient(t)
ctx := context.Background()
_, _, err := client.SlurmdbJobs.GetJob(ctx, "999")
if err == nil {
t.Fatal("expected error for unknown history job, got nil")
}
if !slurm.IsNotFound(err) {
t.Errorf("error = %v, want not-found", err)
}
}
func TestSlurmdbJobStateIsNested(t *testing.T) {
client, mock := setupTestClient(t)
ctx := context.Background()
jobID := submitTestJob(t, client, "nested-state", "gpu", "/tmp/ns", "#!/bin/bash\ntrue")
mock.SetJobState(jobID, "COMPLETED")
resp, _, err := client.SlurmdbJobs.GetJob(ctx, strconv.Itoa(int(jobID)))
if err != nil {
t.Fatalf("GetJob failed: %v", err)
}
job := resp.Jobs[0]
if job.State == nil {
t.Fatal("State is nil — must be nested {current: [...], reason: \"\"}")
}
if len(job.State.Current) == 0 {
t.Fatal("State.Current is empty")
}
if job.State.Current[0] != "COMPLETED" {
t.Errorf("State.Current[0] = %q, want COMPLETED", job.State.Current[0])
}
if job.State.Reason == nil || *job.State.Reason != "" {
t.Errorf("State.Reason = %v, want empty string", job.State.Reason)
}
raw, err := json.Marshal(job)
if err != nil {
t.Fatalf("Marshal: %v", err)
}
rawStr := string(raw)
if !strings.Contains(rawStr, `"state":{"current":["COMPLETED"]`) {
t.Errorf("state JSON should use nested format, got: %s", rawStr)
}
}
func TestSlurmdbJobsFilterByName(t *testing.T) {
client, mock := setupTestClient(t)
ctx := context.Background()
id1 := submitTestJob(t, client, "match-me", "normal", "/tmp", "#!/bin/bash\ntrue")
id2 := submitTestJob(t, client, "other-job", "normal", "/tmp", "#!/bin/bash\ntrue")
mock.SetJobState(id1, "COMPLETED")
mock.SetJobState(id2, "COMPLETED")
resp, _, err := client.SlurmdbJobs.GetJobs(ctx, &slurm.GetSlurmdbJobsOptions{
JobName: slurm.Ptr("match-me"),
})
if err != nil {
t.Fatalf("GetJobs with filter: %v", err)
}
if len(resp.Jobs) != 1 {
t.Fatalf("len(Jobs) = %d, want 1 (filtered by name)", len(resp.Jobs))
}
if resp.Jobs[0].Name == nil || *resp.Jobs[0].Name != "match-me" {
t.Errorf("Name = %v, want match-me", resp.Jobs[0].Name)
}
}
// ---------------------------------------------------------------------------
// SetJobState terminal state exit codes
// ---------------------------------------------------------------------------
func TestSetJobStateExitCodes(t *testing.T) {
client, mock := setupTestClient(t)
ctx := context.Background()
cases := []struct {
state string
wantExit int64
}{
{"COMPLETED", 0},
{"FAILED", 1},
{"CANCELLED", 1},
{"TIMEOUT", 1},
}
for i, tc := range cases {
jobID := submitTestJob(t, client, "exit-"+strconv.Itoa(i), "normal", "/tmp", "#!/bin/bash\ntrue")
mock.SetJobState(jobID, tc.state)
resp, _, err := client.SlurmdbJobs.GetJob(ctx, strconv.Itoa(int(jobID)))
if err != nil {
t.Fatalf("GetJob(%d) %s: %v", jobID, tc.state, err)
}
job := resp.Jobs[0]
if job.ExitCode == nil || job.ExitCode.ReturnCode == nil || job.ExitCode.ReturnCode.Number == nil {
t.Errorf("%s: ExitCode not set", tc.state)
continue
}
if *job.ExitCode.ReturnCode.Number != tc.wantExit {
t.Errorf("%s: exit code = %d, want %d", tc.state, *job.ExitCode.ReturnCode.Number, tc.wantExit)
}
}
}