feat(service): add GetJob fallback to SlurmDBD history and expand query params

GetJob now falls back to SlurmDBD history when active queue returns 404 or empty jobs. Expand JobHistoryQuery from 7 to 16 filter params (add SubmitTime, Cluster, Qos, Constraints, ExitCode, Node, Reservation, Groups, Wckey). Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
2026-04-10 13:43:31 +08:00
parent b3d787c97b
commit f4177dd287
3 changed files with 245 additions and 11 deletions
--- a/internal/service/job_service.go
+++ b/internal/service/job_service.go
@@ -116,7 +116,8 @@ func (s *JobService) GetJobs(ctx context.Context) ([]model.JobResponse, error) {
 	return jobs, nil
 }

-// GetJob retrieves a single job by ID.
+// GetJob retrieves a single job by ID. If the job is not found in the active
+// queue (404 or empty result), it falls back to querying SlurmDBD history.
 func (s *JobService) GetJob(ctx context.Context, jobID string) (*model.JobResponse, error) {
 	s.logger.Debug("slurm API request",
 		zap.String("operation", "GetJob"),
@@ -128,6 +129,12 @@ func (s *JobService) GetJob(ctx context.Context, jobID string) (*model.JobRespon
 	took := time.Since(start)

 	if err != nil {
+		if slurm.IsNotFound(err) {
+			s.logger.Debug("job not in active queue, querying history",
+				zap.String("job_id", jobID),
+			)
+			return s.getJobFromHistory(ctx, jobID)
+		}
 		s.logger.Debug("slurm API error response",
 			zap.String("operation", "GetJob"),
 			zap.String("job_id", jobID),
@@ -146,13 +153,49 @@ func (s *JobService) GetJob(ctx context.Context, jobID string) (*model.JobRespon
 	)

 	if len(result.Jobs) == 0 {
-		return nil, nil
+		s.logger.Debug("empty jobs response, querying history",
+			zap.String("job_id", jobID),
+		)
+		return s.getJobFromHistory(ctx, jobID)
 	}

 	resp := mapJobInfo(&result.Jobs[0])
 	return &resp, nil
 }

+func (s *JobService) getJobFromHistory(ctx context.Context, jobID string) (*model.JobResponse, error) {
+	start := time.Now()
+	result, _, err := s.client.SlurmdbJobs.GetJob(ctx, jobID)
+	took := time.Since(start)
+
+	if err != nil {
+		s.logger.Debug("slurmdb API error response",
+			zap.String("operation", "getJobFromHistory"),
+			zap.String("job_id", jobID),
+			zap.Duration("took", took),
+			zap.Error(err),
+		)
+		if slurm.IsNotFound(err) {
+			return nil, nil
+		}
+		return nil, fmt.Errorf("get job history %s: %w", jobID, err)
+	}
+
+	s.logger.Debug("slurmdb API response",
+		zap.String("operation", "getJobFromHistory"),
+		zap.String("job_id", jobID),
+		zap.Duration("took", took),
+		zap.Any("body", result),
+	)
+
+	if len(result.Jobs) == 0 {
+		return nil, nil
+	}
+
+	resp := mapSlurmdbJob(&result.Jobs[0])
+	return &resp, nil
+}
+
 // CancelJob cancels a job by ID.
 func (s *JobService) CancelJob(ctx context.Context, jobID string) error {
 	s.logger.Debug("slurm API request",
@@ -209,6 +252,33 @@ func (s *JobService) GetJobHistory(ctx context.Context, query *model.JobHistoryQ
 	if query.EndTime != "" {
 		opts.EndTime = strToPtr(query.EndTime)
 	}
+	if query.SubmitTime != "" {
+		opts.SubmitTime = strToPtr(query.SubmitTime)
+	}
+	if query.Cluster != "" {
+		opts.Cluster = strToPtr(query.Cluster)
+	}
+	if query.Qos != "" {
+		opts.Qos = strToPtr(query.Qos)
+	}
+	if query.Constraints != "" {
+		opts.Constraints = strToPtr(query.Constraints)
+	}
+	if query.ExitCode != "" {
+		opts.ExitCode = strToPtr(query.ExitCode)
+	}
+	if query.Node != "" {
+		opts.Node = strToPtr(query.Node)
+	}
+	if query.Reservation != "" {
+		opts.Reservation = strToPtr(query.Reservation)
+	}
+	if query.Groups != "" {
+		opts.Groups = strToPtr(query.Groups)
+	}
+	if query.Wckey != "" {
+		opts.Wckey = strToPtr(query.Wckey)
+	}

 	s.logger.Debug("slurm API request",
 		zap.String("operation", "GetJobHistory"),
--- a/internal/service/job_service_test.go
+++ b/internal/service/job_service_test.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"

 	"gcy_hpc_server/internal/model"
@@ -701,3 +702,157 @@ func TestJobService_GetJobHistory_ErrorLog(t *testing.T) {
 		t.Error("expected error field in log entry")
 	}
 }
+
+// ---------------------------------------------------------------------------
+// Fallback to SlurmDBD history tests
+// ---------------------------------------------------------------------------
+
+func TestGetJob_FallbackToHistory_Found(t *testing.T) {
+	jobID := int32(198)
+	name := "hist-job"
+	ts := int64(1700000000)
+
+	client, cleanup := mockJobServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+
+		switch r.URL.Path {
+		case "/slurm/v0.0.40/job/198":
+			w.WriteHeader(http.StatusNotFound)
+			json.NewEncoder(w).Encode(map[string]interface{}{
+				"errors": []map[string]interface{}{
+					{
+						"description":  "Unable to query JobId=198",
+						"error_number": float64(2017),
+						"error":        "Invalid job id specified",
+						"source":       "_handle_job_get",
+					},
+				},
+				"jobs": []interface{}{},
+			})
+		case "/slurmdb/v0.0.40/job/198":
+			resp := slurm.OpenapiSlurmdbdJobsResp{
+				Jobs: slurm.JobList{
+					{
+						JobID: &jobID,
+						Name:  &name,
+						State: &slurm.JobState{Current: []string{"COMPLETED"}},
+						Time:  &slurm.JobTime{Submission: &ts, Start: &ts, End: &ts},
+					},
+				},
+			}
+			json.NewEncoder(w).Encode(resp)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer cleanup()
+
+	svc := NewJobService(client, zap.NewNop())
+	job, err := svc.GetJob(context.Background(), "198")
+	if err != nil {
+		t.Fatalf("GetJob: %v", err)
+	}
+	if job == nil {
+		t.Fatal("expected job, got nil")
+	}
+	if job.JobID != 198 {
+		t.Errorf("expected JobID 198, got %d", job.JobID)
+	}
+	if job.Name != "hist-job" {
+		t.Errorf("expected Name hist-job, got %s", job.Name)
+	}
+}
+
+func TestGetJob_FallbackToHistory_NotFound(t *testing.T) {
+	client, cleanup := mockJobServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusNotFound)
+	}))
+	defer cleanup()
+
+	svc := NewJobService(client, zap.NewNop())
+	job, err := svc.GetJob(context.Background(), "999")
+	if err != nil {
+		t.Fatalf("GetJob: %v", err)
+	}
+	if job != nil {
+		t.Errorf("expected nil, got %+v", job)
+	}
+}
+
+func TestGetJob_FallbackToHistory_HistoryError(t *testing.T) {
+	client, cleanup := mockJobServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+
+		switch r.URL.Path {
+		case "/slurm/v0.0.40/job/500":
+			w.WriteHeader(http.StatusNotFound)
+			json.NewEncoder(w).Encode(map[string]interface{}{
+				"errors": []map[string]interface{}{
+					{
+						"description":  "Unable to query JobId=500",
+						"error_number": float64(2017),
+						"error":        "Invalid job id specified",
+						"source":       "_handle_job_get",
+					},
+				},
+				"jobs": []interface{}{},
+			})
+		case "/slurmdb/v0.0.40/job/500":
+			w.WriteHeader(http.StatusInternalServerError)
+			w.Write([]byte(`{"errors":[{"error":"db error"}]}`))
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer cleanup()
+
+	svc := NewJobService(client, zap.NewNop())
+	job, err := svc.GetJob(context.Background(), "500")
+	if err == nil {
+		t.Fatal("expected error, got nil")
+	}
+	if job != nil {
+		t.Errorf("expected nil job, got %+v", job)
+	}
+	if !strings.Contains(err.Error(), "get job history") {
+		t.Errorf("expected error to contain 'get job history', got %s", err.Error())
+	}
+}
+
+func TestGetJob_FallbackToHistory_EmptyHistory(t *testing.T) {
+	client, cleanup := mockJobServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+
+		switch r.URL.Path {
+		case "/slurm/v0.0.40/job/777":
+			w.WriteHeader(http.StatusNotFound)
+			json.NewEncoder(w).Encode(map[string]interface{}{
+				"errors": []map[string]interface{}{
+					{
+						"description":  "Unable to query JobId=777",
+						"error_number": float64(2017),
+						"error":        "Invalid job id specified",
+						"source":       "_handle_job_get",
+					},
+				},
+				"jobs": []interface{}{},
+			})
+		case "/slurmdb/v0.0.40/job/777":
+			resp := slurm.OpenapiSlurmdbdJobsResp{Jobs: slurm.JobList{}}
+			json.NewEncoder(w).Encode(resp)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer cleanup()
+
+	svc := NewJobService(client, zap.NewNop())
+	job, err := svc.GetJob(context.Background(), "777")
+	if err != nil {
+		t.Fatalf("GetJob: %v", err)
+	}
+	if job != nil {
+		t.Errorf("expected nil, got %+v", job)
+	}
+}