package mockslurm import ( "context" "encoding/json" "strconv" "strings" "testing" "gcy_hpc_server/internal/slurm" ) func setupTestClient(t *testing.T) (*slurm.Client, *MockSlurm) { t.Helper() srv, mock := NewMockSlurmServer() t.Cleanup(srv.Close) client, err := slurm.NewClientWithOpts(srv.URL, slurm.WithHTTPClient(srv.Client())) if err != nil { t.Fatalf("failed to create client: %v", err) } return client, mock } func submitTestJob(t *testing.T, client *slurm.Client, name, partition, workDir, script string) int32 { t.Helper() ctx := context.Background() resp, _, err := client.Jobs.SubmitJob(ctx, &slurm.JobSubmitReq{ Script: slurm.Ptr(script), Job: &slurm.JobDescMsg{ Name: slurm.Ptr(name), Partition: slurm.Ptr(partition), CurrentWorkingDirectory: slurm.Ptr(workDir), }, }) if err != nil { t.Fatalf("SubmitJob failed: %v", err) } if resp.JobID == nil { t.Fatal("SubmitJob returned nil JobID") } return *resp.JobID } // --------------------------------------------------------------------------- // P0: Submit Job // --------------------------------------------------------------------------- func TestSubmitJob(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() resp, _, err := client.Jobs.SubmitJob(ctx, &slurm.JobSubmitReq{ Script: slurm.Ptr("#!/bin/bash\necho hello"), Job: &slurm.JobDescMsg{ Name: slurm.Ptr("test-job"), Partition: slurm.Ptr("normal"), CurrentWorkingDirectory: slurm.Ptr("/tmp/work"), }, }) if err != nil { t.Fatalf("SubmitJob failed: %v", err) } if resp.JobID == nil || *resp.JobID != 1 { t.Errorf("JobID = %v, want 1", resp.JobID) } if resp.StepID == nil || *resp.StepID != "Scalar" { t.Errorf("StepID = %v, want Scalar", resp.StepID) } if resp.Result == nil || resp.Result.JobID == nil || *resp.Result.JobID != 1 { t.Errorf("Result.JobID = %v, want 1", resp.Result) } } func TestSubmitJobAutoIncrement(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() for i := 1; i <= 3; i++ { resp, _, err := client.Jobs.SubmitJob(ctx, &slurm.JobSubmitReq{ Script: slurm.Ptr("#!/bin/bash\necho " + strconv.Itoa(i)), }) if err != nil { t.Fatalf("SubmitJob %d failed: %v", i, err) } if resp.JobID == nil || *resp.JobID != int32(i) { t.Errorf("job %d: JobID = %v, want %d", i, resp.JobID, i) } } } // --------------------------------------------------------------------------- // P0: Get All Jobs // --------------------------------------------------------------------------- func TestGetJobsEmpty(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() resp, _, err := client.Jobs.GetJobs(ctx, nil) if err != nil { t.Fatalf("GetJobs failed: %v", err) } if len(resp.Jobs) != 0 { t.Errorf("len(Jobs) = %d, want 0", len(resp.Jobs)) } } func TestGetJobsWithSubmitted(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() submitTestJob(t, client, "job-a", "normal", "/tmp/a", "#!/bin/bash\ntrue") submitTestJob(t, client, "job-b", "gpu", "/tmp/b", "#!/bin/bash\nfalse") resp, _, err := client.Jobs.GetJobs(ctx, nil) if err != nil { t.Fatalf("GetJobs failed: %v", err) } if len(resp.Jobs) != 2 { t.Fatalf("len(Jobs) = %d, want 2", len(resp.Jobs)) } names := map[string]bool{} for _, j := range resp.Jobs { if j.Name != nil { names[*j.Name] = true } if len(j.JobState) == 0 || j.JobState[0] != "PENDING" { t.Errorf("JobState = %v, want [PENDING]", j.JobState) } } if !names["job-a"] || !names["job-b"] { t.Errorf("expected job-a and job-b, got %v", names) } } // --------------------------------------------------------------------------- // P0: Get Job By ID // --------------------------------------------------------------------------- func TestGetJobByID(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() jobID := submitTestJob(t, client, "single-job", "normal", "/tmp/work", "#!/bin/bash\necho hi") resp, _, err := client.Jobs.GetJob(ctx, strconv.Itoa(int(jobID)), nil) if err != nil { t.Fatalf("GetJob failed: %v", err) } if len(resp.Jobs) != 1 { t.Fatalf("len(Jobs) = %d, want 1", len(resp.Jobs)) } job := resp.Jobs[0] if job.JobID == nil || *job.JobID != jobID { t.Errorf("JobID = %v, want %d", job.JobID, jobID) } if job.Name == nil || *job.Name != "single-job" { t.Errorf("Name = %v, want single-job", job.Name) } if job.Partition == nil || *job.Partition != "normal" { t.Errorf("Partition = %v, want normal", job.Partition) } if job.CurrentWorkingDirectory == nil || *job.CurrentWorkingDirectory != "/tmp/work" { t.Errorf("CurrentWorkingDirectory = %v, want /tmp/work", job.CurrentWorkingDirectory) } if job.SubmitTime == nil || job.SubmitTime.Number == nil { t.Error("SubmitTime should be non-nil") } } func TestGetJobByIDNotFound(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() _, _, err := client.Jobs.GetJob(ctx, "999", nil) if err == nil { t.Fatal("expected error for unknown job ID, got nil") } if !slurm.IsNotFound(err) { t.Errorf("error type = %T, want SlurmAPIError with 404", err) } } // --------------------------------------------------------------------------- // P0: Delete Job (triggers eviction) // --------------------------------------------------------------------------- func TestDeleteJob(t *testing.T) { client, mock := setupTestClient(t) ctx := context.Background() jobID := submitTestJob(t, client, "cancel-me", "normal", "/tmp", "#!/bin/bash\nsleep 99") resp, _, err := client.Jobs.DeleteJob(ctx, strconv.Itoa(int(jobID)), nil) if err != nil { t.Fatalf("DeleteJob failed: %v", err) } if resp == nil { t.Fatal("DeleteJob returned nil response") } if len(mock.GetAllActiveJobs()) != 0 { t.Error("active jobs should be empty after delete") } if len(mock.GetAllHistoryJobs()) != 1 { t.Error("history should contain 1 job after delete") } if mock.GetJobState(jobID) != "CANCELLED" { t.Errorf("job state = %q, want CANCELLED", mock.GetJobState(jobID)) } } func TestDeleteJobEvictsFromActive(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() jobID := submitTestJob(t, client, "to-delete", "normal", "/tmp", "#!/bin/bash\ntrue") _, _, err := client.Jobs.DeleteJob(ctx, strconv.Itoa(int(jobID)), nil) if err != nil { t.Fatalf("DeleteJob failed: %v", err) } _, _, err = client.Jobs.GetJob(ctx, strconv.Itoa(int(jobID)), nil) if err == nil { t.Fatal("expected 404 after delete, got nil error") } if !slurm.IsNotFound(err) { t.Errorf("error = %v, want not-found", err) } } // --------------------------------------------------------------------------- // P0: Job State format ([]string, not bare string) // --------------------------------------------------------------------------- func TestJobStateIsStringArray(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() submitTestJob(t, client, "state-test", "normal", "/tmp", "#!/bin/bash\necho") resp, _, err := client.Jobs.GetJobs(ctx, nil) if err != nil { t.Fatalf("GetJobs failed: %v", err) } if len(resp.Jobs) == 0 { t.Fatal("expected at least one job") } job := resp.Jobs[0] if len(job.JobState) == 0 { t.Fatal("JobState is empty — must be non-empty []string to avoid mapSlurmStateToTaskStatus silent failure") } if job.JobState[0] != "PENDING" { t.Errorf("JobState[0] = %q, want %q", job.JobState[0], "PENDING") } raw, err := json.Marshal(job) if err != nil { t.Fatalf("Marshal job: %v", err) } if !strings.Contains(string(raw), `"job_state":["PENDING"]`) { t.Errorf("JobState JSON = %s, want array format [\"PENDING\"]", string(raw)) } } // --------------------------------------------------------------------------- // P0: Full Job Lifecycle // --------------------------------------------------------------------------- func TestJobLifecycle(t *testing.T) { client, mock := setupTestClient(t) ctx := context.Background() jobID := submitTestJob(t, client, "lifecycle", "normal", "/tmp/lc", "#!/bin/bash\necho lifecycle") // Verify PENDING in active resp, _, err := client.Jobs.GetJob(ctx, strconv.Itoa(int(jobID)), nil) if err != nil { t.Fatalf("GetJob PENDING: %v", err) } if resp.Jobs[0].JobState[0] != "PENDING" { t.Errorf("initial state = %v, want PENDING", resp.Jobs[0].JobState) } if len(mock.GetAllActiveJobs()) != 1 { t.Error("should have 1 active job") } // Transition to RUNNING mock.SetJobState(jobID, "RUNNING") resp, _, err = client.Jobs.GetJob(ctx, strconv.Itoa(int(jobID)), nil) if err != nil { t.Fatalf("GetJob RUNNING: %v", err) } if resp.Jobs[0].JobState[0] != "RUNNING" { t.Errorf("running state = %v, want RUNNING", resp.Jobs[0].JobState) } if resp.Jobs[0].StartTime == nil || resp.Jobs[0].StartTime.Number == nil { t.Error("StartTime should be set for RUNNING job") } if len(mock.GetAllActiveJobs()) != 1 { t.Error("should still have 1 active job after RUNNING") } // Transition to COMPLETED — triggers eviction mock.SetJobState(jobID, "COMPLETED") _, _, err = client.Jobs.GetJob(ctx, strconv.Itoa(int(jobID)), nil) if err == nil { t.Fatal("expected 404 after COMPLETED (evicted from active)") } if !slurm.IsNotFound(err) { t.Errorf("error = %v, want not-found", err) } if len(mock.GetAllActiveJobs()) != 0 { t.Error("active jobs should be empty after COMPLETED") } if len(mock.GetAllHistoryJobs()) != 1 { t.Error("history should contain 1 job after COMPLETED") } if mock.GetJobState(jobID) != "COMPLETED" { t.Errorf("state = %q, want COMPLETED", mock.GetJobState(jobID)) } // Verify history endpoint returns the job histResp, _, err := client.SlurmdbJobs.GetJob(ctx, strconv.Itoa(int(jobID))) if err != nil { t.Fatalf("SlurmdbJobs.GetJob: %v", err) } if len(histResp.Jobs) != 1 { t.Fatalf("history jobs = %d, want 1", len(histResp.Jobs)) } histJob := histResp.Jobs[0] if histJob.State == nil || len(histJob.State.Current) == 0 || histJob.State.Current[0] != "COMPLETED" { t.Errorf("history state = %v, want current=[COMPLETED]", histJob.State) } if histJob.ExitCode == nil || histJob.ExitCode.ReturnCode == nil || histJob.ExitCode.ReturnCode.Number == nil { t.Error("history ExitCode should be set") } else if *histJob.ExitCode.ReturnCode.Number != 0 { t.Errorf("exit code = %d, want 0 for COMPLETED", *histJob.ExitCode.ReturnCode.Number) } } // --------------------------------------------------------------------------- // P1: Nodes // --------------------------------------------------------------------------- func TestGetNodes(t *testing.T) { client, mock := setupTestClient(t) _ = mock ctx := context.Background() resp, _, err := client.Nodes.GetNodes(ctx, nil) if err != nil { t.Fatalf("GetNodes failed: %v", err) } if resp.Nodes == nil { t.Fatal("Nodes is nil") } if len(*resp.Nodes) != 3 { t.Errorf("len(Nodes) = %d, want 3", len(*resp.Nodes)) } names := make([]string, len(*resp.Nodes)) for i, n := range *resp.Nodes { if n.Name == nil { t.Errorf("node %d: Name is nil", i) } else { names[i] = *n.Name } } for _, expected := range []string{"node01", "node02", "node03"} { found := false for _, n := range names { if n == expected { found = true break } } if !found { t.Errorf("missing node %q in %v", expected, names) } } } func TestGetNode(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() resp, _, err := client.Nodes.GetNode(ctx, "node02", nil) if err != nil { t.Fatalf("GetNode failed: %v", err) } if resp.Nodes == nil || len(*resp.Nodes) != 1 { t.Fatalf("expected 1 node, got %v", resp.Nodes) } if (*resp.Nodes)[0].Name == nil || *(*resp.Nodes)[0].Name != "node02" { t.Errorf("Name = %v, want node02", (*resp.Nodes)[0].Name) } } func TestGetNodeNotFound(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() _, _, err := client.Nodes.GetNode(ctx, "nonexistent", nil) if err == nil { t.Fatal("expected error for unknown node, got nil") } if !slurm.IsNotFound(err) { t.Errorf("error = %v, want not-found", err) } } // --------------------------------------------------------------------------- // P1: Partitions // --------------------------------------------------------------------------- func TestGetPartitions(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() resp, _, err := client.Partitions.GetPartitions(ctx, nil) if err != nil { t.Fatalf("GetPartitions failed: %v", err) } if resp.Partitions == nil { t.Fatal("Partitions is nil") } if len(*resp.Partitions) != 2 { t.Errorf("len(Partitions) = %d, want 2", len(*resp.Partitions)) } names := map[string]bool{} for _, p := range *resp.Partitions { if p.Name != nil { names[*p.Name] = true } } if !names["normal"] || !names["gpu"] { t.Errorf("expected normal and gpu partitions, got %v", names) } } func TestGetPartition(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() resp, _, err := client.Partitions.GetPartition(ctx, "gpu", nil) if err != nil { t.Fatalf("GetPartition failed: %v", err) } if resp.Partitions == nil || len(*resp.Partitions) != 1 { t.Fatalf("expected 1 partition, got %v", resp.Partitions) } if (*resp.Partitions)[0].Name == nil || *(*resp.Partitions)[0].Name != "gpu" { t.Errorf("Name = %v, want gpu", (*resp.Partitions)[0].Name) } } func TestGetPartitionNotFound(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() _, _, err := client.Partitions.GetPartition(ctx, "nonexistent", nil) if err == nil { t.Fatal("expected error for unknown partition, got nil") } if !slurm.IsNotFound(err) { t.Errorf("error = %v, want not-found", err) } } // --------------------------------------------------------------------------- // P1: Diag // --------------------------------------------------------------------------- func TestGetDiag(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() resp, _, err := client.Diag.GetDiag(ctx) if err != nil { t.Fatalf("GetDiag failed: %v", err) } if resp.Statistics == nil { t.Fatal("Statistics is nil") } if resp.Statistics.ServerThreadCount == nil || *resp.Statistics.ServerThreadCount != 3 { t.Errorf("ServerThreadCount = %v, want 3", resp.Statistics.ServerThreadCount) } if resp.Statistics.AgentQueueSize == nil || *resp.Statistics.AgentQueueSize != 0 { t.Errorf("AgentQueueSize = %v, want 0", resp.Statistics.AgentQueueSize) } } // --------------------------------------------------------------------------- // P1: SlurmDB Jobs // --------------------------------------------------------------------------- func TestSlurmdbGetJobsEmpty(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() resp, _, err := client.SlurmdbJobs.GetJobs(ctx, nil) if err != nil { t.Fatalf("GetJobs failed: %v", err) } if len(resp.Jobs) != 0 { t.Errorf("len(Jobs) = %d, want 0 (no history)", len(resp.Jobs)) } } func TestSlurmdbGetJobsAfterEviction(t *testing.T) { client, mock := setupTestClient(t) ctx := context.Background() jobID := submitTestJob(t, client, "hist-job", "normal", "/tmp/h", "#!/bin/bash\necho hist") mock.SetJobState(jobID, "RUNNING") mock.SetJobState(jobID, "COMPLETED") resp, _, err := client.SlurmdbJobs.GetJobs(ctx, nil) if err != nil { t.Fatalf("GetJobs failed: %v", err) } if len(resp.Jobs) != 1 { t.Fatalf("len(Jobs) = %d, want 1", len(resp.Jobs)) } job := resp.Jobs[0] if job.Name == nil || *job.Name != "hist-job" { t.Errorf("Name = %v, want hist-job", job.Name) } if job.State == nil || len(job.State.Current) == 0 || job.State.Current[0] != "COMPLETED" { t.Errorf("State = %v, want current=[COMPLETED]", job.State) } if job.Time == nil || job.Time.Submission == nil { t.Error("Time.Submission should be set") } } func TestSlurmdbGetJobByID(t *testing.T) { client, mock := setupTestClient(t) ctx := context.Background() jobID := submitTestJob(t, client, "single-hist", "normal", "/tmp/sh", "#!/bin/bash\nexit 1") mock.SetJobState(jobID, "FAILED") resp, _, err := client.SlurmdbJobs.GetJob(ctx, strconv.Itoa(int(jobID))) if err != nil { t.Fatalf("GetJob failed: %v", err) } if len(resp.Jobs) != 1 { t.Fatalf("len(Jobs) = %d, want 1", len(resp.Jobs)) } job := resp.Jobs[0] if job.JobID == nil || *job.JobID != jobID { t.Errorf("JobID = %v, want %d", job.JobID, jobID) } if job.State == nil || len(job.State.Current) == 0 || job.State.Current[0] != "FAILED" { t.Errorf("State = %v, want current=[FAILED]", job.State) } if job.ExitCode == nil || job.ExitCode.ReturnCode == nil || job.ExitCode.ReturnCode.Number == nil { t.Error("ExitCode should be set") } else if *job.ExitCode.ReturnCode.Number != 1 { t.Errorf("exit code = %d, want 1 for FAILED", *job.ExitCode.ReturnCode.Number) } } func TestSlurmdbGetJobNotFound(t *testing.T) { client, _ := setupTestClient(t) ctx := context.Background() _, _, err := client.SlurmdbJobs.GetJob(ctx, "999") if err == nil { t.Fatal("expected error for unknown history job, got nil") } if !slurm.IsNotFound(err) { t.Errorf("error = %v, want not-found", err) } } func TestSlurmdbJobStateIsNested(t *testing.T) { client, mock := setupTestClient(t) ctx := context.Background() jobID := submitTestJob(t, client, "nested-state", "gpu", "/tmp/ns", "#!/bin/bash\ntrue") mock.SetJobState(jobID, "COMPLETED") resp, _, err := client.SlurmdbJobs.GetJob(ctx, strconv.Itoa(int(jobID))) if err != nil { t.Fatalf("GetJob failed: %v", err) } job := resp.Jobs[0] if job.State == nil { t.Fatal("State is nil — must be nested {current: [...], reason: \"\"}") } if len(job.State.Current) == 0 { t.Fatal("State.Current is empty") } if job.State.Current[0] != "COMPLETED" { t.Errorf("State.Current[0] = %q, want COMPLETED", job.State.Current[0]) } if job.State.Reason == nil || *job.State.Reason != "" { t.Errorf("State.Reason = %v, want empty string", job.State.Reason) } raw, err := json.Marshal(job) if err != nil { t.Fatalf("Marshal: %v", err) } rawStr := string(raw) if !strings.Contains(rawStr, `"state":{"current":["COMPLETED"]`) { t.Errorf("state JSON should use nested format, got: %s", rawStr) } } func TestSlurmdbJobsFilterByName(t *testing.T) { client, mock := setupTestClient(t) ctx := context.Background() id1 := submitTestJob(t, client, "match-me", "normal", "/tmp", "#!/bin/bash\ntrue") id2 := submitTestJob(t, client, "other-job", "normal", "/tmp", "#!/bin/bash\ntrue") mock.SetJobState(id1, "COMPLETED") mock.SetJobState(id2, "COMPLETED") resp, _, err := client.SlurmdbJobs.GetJobs(ctx, &slurm.GetSlurmdbJobsOptions{ JobName: slurm.Ptr("match-me"), }) if err != nil { t.Fatalf("GetJobs with filter: %v", err) } if len(resp.Jobs) != 1 { t.Fatalf("len(Jobs) = %d, want 1 (filtered by name)", len(resp.Jobs)) } if resp.Jobs[0].Name == nil || *resp.Jobs[0].Name != "match-me" { t.Errorf("Name = %v, want match-me", resp.Jobs[0].Name) } } // --------------------------------------------------------------------------- // SetJobState terminal state exit codes // --------------------------------------------------------------------------- func TestSetJobStateExitCodes(t *testing.T) { client, mock := setupTestClient(t) ctx := context.Background() cases := []struct { state string wantExit int64 }{ {"COMPLETED", 0}, {"FAILED", 1}, {"CANCELLED", 1}, {"TIMEOUT", 1}, } for i, tc := range cases { jobID := submitTestJob(t, client, "exit-"+strconv.Itoa(i), "normal", "/tmp", "#!/bin/bash\ntrue") mock.SetJobState(jobID, tc.state) resp, _, err := client.SlurmdbJobs.GetJob(ctx, strconv.Itoa(int(jobID))) if err != nil { t.Fatalf("GetJob(%d) %s: %v", jobID, tc.state, err) } job := resp.Jobs[0] if job.ExitCode == nil || job.ExitCode.ReturnCode == nil || job.ExitCode.ReturnCode.Number == nil { t.Errorf("%s: ExitCode not set", tc.state) continue } if *job.ExitCode.ReturnCode.Number != tc.wantExit { t.Errorf("%s: exit code = %d, want %d", tc.state, *job.ExitCode.ReturnCode.Number, tc.wantExit) } } }