feat(testutil): add MockSlurm, MockMinIO, TestEnv and 37 integration tests

- mockminio: in-memory ObjectStorage with all 11 methods, thread-safe, SHA256 ETag, Range support
- mockslurm: httptest server with 11 Slurm REST API endpoints, job eviction from active to history queue
- testenv: one-line test environment factory (SQLite + MockSlurm + MockMinIO + all stores/services/handlers + httptest server)
- integration tests: 37 tests covering Jobs(5), Cluster(5), App(6), Upload(5), File(4), Folder(4), Task(4), E2E(1)
- no external dependencies, no existing files modified
This commit is contained in:
dailz
2026-04-16 13:23:27 +08:00
parent 73504f9fdb
commit b9b2f0d9b4
16 changed files with 4685 additions and 0 deletions

View File

@@ -0,0 +1,239 @@
// Package mockminio provides an in-memory implementation of storage.ObjectStorage
// for use in tests. It is thread-safe and supports Range reads.
package mockminio
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"sort"
"strings"
"sync"
"time"
"gcy_hpc_server/internal/storage"
)
// Compile-time interface check.
var _ storage.ObjectStorage = (*InMemoryStorage)(nil)
// objectMeta holds metadata for a stored object.
type objectMeta struct {
size int64
etag string
lastModified time.Time
contentType string
}
// InMemoryStorage is a thread-safe, in-memory implementation of
// storage.ObjectStorage. All data is kept in memory; no network or disk I/O
// is performed.
type InMemoryStorage struct {
mu sync.RWMutex
objects map[string][]byte
meta map[string]objectMeta
buckets map[string]bool
}
// NewInMemoryStorage returns a ready-to-use InMemoryStorage.
func NewInMemoryStorage() *InMemoryStorage {
return &InMemoryStorage{
objects: make(map[string][]byte),
meta: make(map[string]objectMeta),
buckets: make(map[string]bool),
}
}
// PutObject reads all bytes from reader and stores them under key.
// The ETag is the SHA-256 hash of the data, formatted as hex.
func (s *InMemoryStorage) PutObject(_ context.Context, _, key string, reader io.Reader, _ int64, opts storage.PutObjectOptions) (storage.UploadInfo, error) {
data, err := io.ReadAll(reader)
if err != nil {
return storage.UploadInfo{}, fmt.Errorf("read all: %w", err)
}
h := sha256.Sum256(data)
etag := hex.EncodeToString(h[:])
s.mu.Lock()
s.objects[key] = data
s.meta[key] = objectMeta{
size: int64(len(data)),
etag: etag,
lastModified: time.Now(),
contentType: opts.ContentType,
}
s.mu.Unlock()
return storage.UploadInfo{ETag: etag, Size: int64(len(data))}, nil
}
// GetObject retrieves an object. opts.Start and opts.End control byte-range
// reads. Four cases are supported:
// 1. No range (both nil) → return entire object
// 2. Start only (End nil) → from start to end of object
// 3. End only (Start nil) → from byte 0 to end
// 4. Start + End → standard byte range
func (s *InMemoryStorage) GetObject(_ context.Context, _, key string, opts storage.GetOptions) (io.ReadCloser, storage.ObjectInfo, error) {
s.mu.RLock()
data, ok := s.objects[key]
meta := s.meta[key]
s.mu.RUnlock()
if !ok {
return nil, storage.ObjectInfo{}, fmt.Errorf("object %s not found", key)
}
size := int64(len(data))
// Full object info (Size is always the total object size).
info := storage.ObjectInfo{
Key: key,
Size: size,
ETag: meta.etag,
LastModified: meta.lastModified,
ContentType: meta.contentType,
}
// No range requested → return everything.
if opts.Start == nil && opts.End == nil {
return io.NopCloser(bytes.NewReader(data)), info, nil
}
// Build range. Check each pointer individually to avoid nil dereference.
start := int64(0)
if opts.Start != nil {
start = *opts.Start
}
end := size - 1
if opts.End != nil {
end = *opts.End
}
// Clamp end to last byte.
if end >= size {
end = size - 1
}
if start > end || start < 0 {
return nil, storage.ObjectInfo{}, fmt.Errorf("invalid range: start=%d, end=%d, size=%d", start, end, size)
}
section := io.NewSectionReader(bytes.NewReader(data), start, end-start+1)
return io.NopCloser(section), info, nil
}
// ComposeObject concatenates source objects (in order) into dst.
func (s *InMemoryStorage) ComposeObject(_ context.Context, _, dst string, sources []string) (storage.UploadInfo, error) {
s.mu.Lock()
defer s.mu.Unlock()
var buf bytes.Buffer
for _, src := range sources {
data, ok := s.objects[src]
if !ok {
return storage.UploadInfo{}, fmt.Errorf("source object %s not found", src)
}
buf.Write(data)
}
combined := buf.Bytes()
h := sha256.Sum256(combined)
etag := hex.EncodeToString(h[:])
s.objects[dst] = combined
s.meta[dst] = objectMeta{
size: int64(len(combined)),
etag: etag,
lastModified: time.Now(),
}
return storage.UploadInfo{ETag: etag, Size: int64(len(combined))}, nil
}
// RemoveObject deletes a single object.
func (s *InMemoryStorage) RemoveObject(_ context.Context, _, key string, _ storage.RemoveObjectOptions) error {
s.mu.Lock()
delete(s.objects, key)
delete(s.meta, key)
s.mu.Unlock()
return nil
}
// RemoveObjects deletes multiple objects by key.
func (s *InMemoryStorage) RemoveObjects(_ context.Context, _ string, keys []string, _ storage.RemoveObjectsOptions) error {
s.mu.Lock()
for _, k := range keys {
delete(s.objects, k)
delete(s.meta, k)
}
s.mu.Unlock()
return nil
}
// ListObjects returns object info for all objects matching prefix, sorted by key.
func (s *InMemoryStorage) ListObjects(_ context.Context, _, prefix string, _ bool) ([]storage.ObjectInfo, error) {
s.mu.RLock()
defer s.mu.RUnlock()
var result []storage.ObjectInfo
for k, m := range s.meta {
if strings.HasPrefix(k, prefix) {
result = append(result, storage.ObjectInfo{
Key: k,
Size: m.size,
ETag: m.etag,
LastModified: m.lastModified,
ContentType: m.contentType,
})
}
}
sort.Slice(result, func(i, j int) bool { return result[i].Key < result[j].Key })
return result, nil
}
// BucketExists reports whether the named bucket exists.
func (s *InMemoryStorage) BucketExists(_ context.Context, bucket string) (bool, error) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.buckets[bucket], nil
}
// MakeBucket creates a bucket.
func (s *InMemoryStorage) MakeBucket(_ context.Context, bucket string, _ storage.MakeBucketOptions) error {
s.mu.Lock()
s.buckets[bucket] = true
s.mu.Unlock()
return nil
}
// StatObject returns metadata about an object without downloading it.
func (s *InMemoryStorage) StatObject(_ context.Context, _, key string, _ storage.StatObjectOptions) (storage.ObjectInfo, error) {
s.mu.RLock()
m, ok := s.meta[key]
s.mu.RUnlock()
if !ok {
return storage.ObjectInfo{}, fmt.Errorf("object %s not found", key)
}
return storage.ObjectInfo{
Key: key,
Size: m.size,
ETag: m.etag,
LastModified: m.lastModified,
ContentType: m.contentType,
}, nil
}
// AbortMultipartUpload is a no-op for the in-memory implementation.
func (s *InMemoryStorage) AbortMultipartUpload(_ context.Context, _, _, _ string) error {
return nil
}
// RemoveIncompleteUpload is a no-op for the in-memory implementation.
func (s *InMemoryStorage) RemoveIncompleteUpload(_ context.Context, _, _ string) error {
return nil
}

View File

@@ -0,0 +1,378 @@
package mockminio
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"sync"
"testing"
"gcy_hpc_server/internal/storage"
)
func sha256Hex(data []byte) string {
h := sha256.Sum256(data)
return hex.EncodeToString(h[:])
}
func TestNewInMemoryStorage_ReturnsInitialized(t *testing.T) {
s := NewInMemoryStorage()
if s == nil {
t.Fatal("expected non-nil storage")
}
}
func TestPutObject_StoresData(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
data := []byte("hello world")
info, err := s.PutObject(ctx, "bucket", "key1", bytes.NewReader(data), int64(len(data)), storage.PutObjectOptions{ContentType: "text/plain"})
if err != nil {
t.Fatalf("PutObject: %v", err)
}
wantETag := sha256Hex(data)
if info.ETag != wantETag {
t.Errorf("ETag = %q, want %q", info.ETag, wantETag)
}
if info.Size != int64(len(data)) {
t.Errorf("Size = %d, want %d", info.Size, len(data))
}
}
func TestGetObject_FullObject(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
data := []byte("hello world")
s.PutObject(ctx, "bucket", "key1", bytes.NewReader(data), int64(len(data)), storage.PutObjectOptions{})
rc, info, err := s.GetObject(ctx, "bucket", "key1", storage.GetOptions{})
if err != nil {
t.Fatalf("GetObject: %v", err)
}
defer rc.Close()
got, err := io.ReadAll(rc)
if err != nil {
t.Fatalf("ReadAll: %v", err)
}
if !bytes.Equal(got, data) {
t.Errorf("got %q, want %q", got, data)
}
if info.Size != int64(len(data)) {
t.Errorf("info.Size = %d, want %d", info.Size, len(data))
}
}
func TestGetObject_RangeStartOnly(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
data := []byte("0123456789")
s.PutObject(ctx, "bucket", "key1", bytes.NewReader(data), int64(len(data)), storage.PutObjectOptions{})
start := int64(5)
rc, _, err := s.GetObject(ctx, "bucket", "key1", storage.GetOptions{Start: &start})
if err != nil {
t.Fatalf("GetObject: %v", err)
}
defer rc.Close()
got, err := io.ReadAll(rc)
if err != nil {
t.Fatalf("ReadAll: %v", err)
}
want := data[5:]
if !bytes.Equal(got, want) {
t.Errorf("got %q, want %q", got, want)
}
}
func TestGetObject_RangeEndOnly(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
data := []byte("0123456789")
s.PutObject(ctx, "bucket", "key1", bytes.NewReader(data), int64(len(data)), storage.PutObjectOptions{})
end := int64(4)
rc, _, err := s.GetObject(ctx, "bucket", "key1", storage.GetOptions{End: &end})
if err != nil {
t.Fatalf("GetObject: %v", err)
}
defer rc.Close()
got, err := io.ReadAll(rc)
if err != nil {
t.Fatalf("ReadAll: %v", err)
}
want := data[:5]
if !bytes.Equal(got, want) {
t.Errorf("got %q, want %q", got, want)
}
}
func TestGetObject_RangeStartAndEnd(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
data := []byte("0123456789")
s.PutObject(ctx, "bucket", "key1", bytes.NewReader(data), int64(len(data)), storage.PutObjectOptions{})
start := int64(2)
end := int64(5)
rc, _, err := s.GetObject(ctx, "bucket", "key1", storage.GetOptions{Start: &start, End: &end})
if err != nil {
t.Fatalf("GetObject: %v", err)
}
defer rc.Close()
got, err := io.ReadAll(rc)
if err != nil {
t.Fatalf("ReadAll: %v", err)
}
want := data[2:6]
if !bytes.Equal(got, want) {
t.Errorf("got %q, want %q", got, want)
}
}
func TestGetObject_NotFound(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
_, _, err := s.GetObject(ctx, "bucket", "nonexistent", storage.GetOptions{})
if err == nil {
t.Fatal("expected error for missing object")
}
}
func TestComposeObject_ConcatenatesSources(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
s.PutObject(ctx, "bucket", "part1", bytes.NewReader([]byte("hello ")), 6, storage.PutObjectOptions{})
s.PutObject(ctx, "bucket", "part2", bytes.NewReader([]byte("world")), 5, storage.PutObjectOptions{})
info, err := s.ComposeObject(ctx, "bucket", "combined", []string{"part1", "part2"})
if err != nil {
t.Fatalf("ComposeObject: %v", err)
}
want := []byte("hello world")
if info.Size != int64(len(want)) {
t.Errorf("Size = %d, want %d", info.Size, len(want))
}
wantETag := sha256Hex(want)
if info.ETag != wantETag {
t.Errorf("ETag = %q, want %q", info.ETag, wantETag)
}
rc, _, err := s.GetObject(ctx, "bucket", "combined", storage.GetOptions{})
if err != nil {
t.Fatalf("GetObject combined: %v", err)
}
defer rc.Close()
got, _ := io.ReadAll(rc)
if !bytes.Equal(got, want) {
t.Errorf("got %q, want %q", got, want)
}
}
func TestComposeObject_MissingSource(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
_, err := s.ComposeObject(ctx, "bucket", "dst", []string{"missing"})
if err == nil {
t.Fatal("expected error for missing source")
}
}
func TestRemoveObject(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
s.PutObject(ctx, "bucket", "key1", bytes.NewReader([]byte("data")), 4, storage.PutObjectOptions{})
err := s.RemoveObject(ctx, "bucket", "key1", storage.RemoveObjectOptions{})
if err != nil {
t.Fatalf("RemoveObject: %v", err)
}
_, _, err = s.GetObject(ctx, "bucket", "key1", storage.GetOptions{})
if err == nil {
t.Fatal("expected error after removal")
}
}
func TestRemoveObjects(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
for i := 0; i < 5; i++ {
key := fmt.Sprintf("key%d", i)
s.PutObject(ctx, "bucket", key, bytes.NewReader([]byte(key)), int64(len(key)), storage.PutObjectOptions{})
}
err := s.RemoveObjects(ctx, "bucket", []string{"key1", "key3"}, storage.RemoveObjectsOptions{})
if err != nil {
t.Fatalf("RemoveObjects: %v", err)
}
objects, _ := s.ListObjects(ctx, "bucket", "", true)
if len(objects) != 3 {
t.Errorf("got %d objects, want 3", len(objects))
}
}
func TestListObjects(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
s.PutObject(ctx, "bucket", "dir/a", bytes.NewReader([]byte("a")), 1, storage.PutObjectOptions{})
s.PutObject(ctx, "bucket", "dir/b", bytes.NewReader([]byte("bb")), 2, storage.PutObjectOptions{})
s.PutObject(ctx, "bucket", "other/c", bytes.NewReader([]byte("ccc")), 3, storage.PutObjectOptions{})
objects, err := s.ListObjects(ctx, "bucket", "dir/", true)
if err != nil {
t.Fatalf("ListObjects: %v", err)
}
if len(objects) != 2 {
t.Fatalf("got %d objects, want 2", len(objects))
}
if objects[0].Key != "dir/a" || objects[1].Key != "dir/b" {
t.Errorf("keys = %v, want [dir/a dir/b]", []string{objects[0].Key, objects[1].Key})
}
if objects[0].Size != 1 || objects[1].Size != 2 {
t.Errorf("sizes = %v, want [1 2]", []int64{objects[0].Size, objects[1].Size})
}
}
func TestBucketExists(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
ok, _ := s.BucketExists(ctx, "mybucket")
if ok {
t.Error("bucket should not exist yet")
}
s.MakeBucket(ctx, "mybucket", storage.MakeBucketOptions{})
ok, _ = s.BucketExists(ctx, "mybucket")
if !ok {
t.Error("bucket should exist after MakeBucket")
}
}
func TestMakeBucket(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
err := s.MakeBucket(ctx, "test-bucket", storage.MakeBucketOptions{Region: "us-east-1"})
if err != nil {
t.Fatalf("MakeBucket: %v", err)
}
ok, _ := s.BucketExists(ctx, "test-bucket")
if !ok {
t.Error("bucket should exist")
}
}
func TestStatObject(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
data := []byte("test data")
s.PutObject(ctx, "bucket", "key1", bytes.NewReader(data), int64(len(data)), storage.PutObjectOptions{ContentType: "text/plain"})
info, err := s.StatObject(ctx, "bucket", "key1", storage.StatObjectOptions{})
if err != nil {
t.Fatalf("StatObject: %v", err)
}
wantETag := sha256Hex(data)
if info.Key != "key1" {
t.Errorf("Key = %q, want %q", info.Key, "key1")
}
if info.Size != int64(len(data)) {
t.Errorf("Size = %d, want %d", info.Size, len(data))
}
if info.ETag != wantETag {
t.Errorf("ETag = %q, want %q", info.ETag, wantETag)
}
if info.ContentType != "text/plain" {
t.Errorf("ContentType = %q, want %q", info.ContentType, "text/plain")
}
if info.LastModified.IsZero() {
t.Error("LastModified should not be zero")
}
}
func TestStatObject_NotFound(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
_, err := s.StatObject(ctx, "bucket", "nonexistent", storage.StatObjectOptions{})
if err == nil {
t.Fatal("expected error for missing object")
}
}
func TestAbortMultipartUpload(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
err := s.AbortMultipartUpload(ctx, "bucket", "key", "upload-id")
if err != nil {
t.Fatalf("AbortMultipartUpload: %v", err)
}
}
func TestRemoveIncompleteUpload(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
err := s.RemoveIncompleteUpload(ctx, "bucket", "key")
if err != nil {
t.Fatalf("RemoveIncompleteUpload: %v", err)
}
}
func TestConcurrentAccess(t *testing.T) {
s := NewInMemoryStorage()
ctx := context.Background()
const goroutines = 50
var wg sync.WaitGroup
wg.Add(goroutines * 2)
for i := 0; i < goroutines; i++ {
go func(i int) {
defer wg.Done()
key := fmt.Sprintf("key%d", i%10)
data := []byte(fmt.Sprintf("data-%d", i))
s.PutObject(ctx, "bucket", key, bytes.NewReader(data), int64(len(data)), storage.PutObjectOptions{})
}(i)
go func(i int) {
defer wg.Done()
key := fmt.Sprintf("key%d", i%10)
rc, _, _ := s.GetObject(ctx, "bucket", key, storage.GetOptions{})
if rc != nil {
rc.Close()
}
}(i)
}
wg.Wait()
}