- mockminio: in-memory ObjectStorage with all 11 methods, thread-safe, SHA256 ETag, Range support - mockslurm: httptest server with 11 Slurm REST API endpoints, job eviction from active to history queue - testenv: one-line test environment factory (SQLite + MockSlurm + MockMinIO + all stores/services/handlers + httptest server) - integration tests: 37 tests covering Jobs(5), Cluster(5), App(6), Upload(5), File(4), Folder(4), Task(4), E2E(1) - no external dependencies, no existing files modified
240 lines
6.5 KiB
Go
240 lines
6.5 KiB
Go
// Package mockminio provides an in-memory implementation of storage.ObjectStorage
|
|
// for use in tests. It is thread-safe and supports Range reads.
|
|
package mockminio
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"fmt"
|
|
"io"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"gcy_hpc_server/internal/storage"
|
|
)
|
|
|
|
// Compile-time interface check.
|
|
var _ storage.ObjectStorage = (*InMemoryStorage)(nil)
|
|
|
|
// objectMeta holds metadata for a stored object.
|
|
type objectMeta struct {
|
|
size int64
|
|
etag string
|
|
lastModified time.Time
|
|
contentType string
|
|
}
|
|
|
|
// InMemoryStorage is a thread-safe, in-memory implementation of
|
|
// storage.ObjectStorage. All data is kept in memory; no network or disk I/O
|
|
// is performed.
|
|
type InMemoryStorage struct {
|
|
mu sync.RWMutex
|
|
objects map[string][]byte
|
|
meta map[string]objectMeta
|
|
buckets map[string]bool
|
|
}
|
|
|
|
// NewInMemoryStorage returns a ready-to-use InMemoryStorage.
|
|
func NewInMemoryStorage() *InMemoryStorage {
|
|
return &InMemoryStorage{
|
|
objects: make(map[string][]byte),
|
|
meta: make(map[string]objectMeta),
|
|
buckets: make(map[string]bool),
|
|
}
|
|
}
|
|
|
|
// PutObject reads all bytes from reader and stores them under key.
|
|
// The ETag is the SHA-256 hash of the data, formatted as hex.
|
|
func (s *InMemoryStorage) PutObject(_ context.Context, _, key string, reader io.Reader, _ int64, opts storage.PutObjectOptions) (storage.UploadInfo, error) {
|
|
data, err := io.ReadAll(reader)
|
|
if err != nil {
|
|
return storage.UploadInfo{}, fmt.Errorf("read all: %w", err)
|
|
}
|
|
|
|
h := sha256.Sum256(data)
|
|
etag := hex.EncodeToString(h[:])
|
|
|
|
s.mu.Lock()
|
|
s.objects[key] = data
|
|
s.meta[key] = objectMeta{
|
|
size: int64(len(data)),
|
|
etag: etag,
|
|
lastModified: time.Now(),
|
|
contentType: opts.ContentType,
|
|
}
|
|
s.mu.Unlock()
|
|
|
|
return storage.UploadInfo{ETag: etag, Size: int64(len(data))}, nil
|
|
}
|
|
|
|
// GetObject retrieves an object. opts.Start and opts.End control byte-range
|
|
// reads. Four cases are supported:
|
|
// 1. No range (both nil) → return entire object
|
|
// 2. Start only (End nil) → from start to end of object
|
|
// 3. End only (Start nil) → from byte 0 to end
|
|
// 4. Start + End → standard byte range
|
|
func (s *InMemoryStorage) GetObject(_ context.Context, _, key string, opts storage.GetOptions) (io.ReadCloser, storage.ObjectInfo, error) {
|
|
s.mu.RLock()
|
|
data, ok := s.objects[key]
|
|
meta := s.meta[key]
|
|
s.mu.RUnlock()
|
|
|
|
if !ok {
|
|
return nil, storage.ObjectInfo{}, fmt.Errorf("object %s not found", key)
|
|
}
|
|
|
|
size := int64(len(data))
|
|
|
|
// Full object info (Size is always the total object size).
|
|
info := storage.ObjectInfo{
|
|
Key: key,
|
|
Size: size,
|
|
ETag: meta.etag,
|
|
LastModified: meta.lastModified,
|
|
ContentType: meta.contentType,
|
|
}
|
|
|
|
// No range requested → return everything.
|
|
if opts.Start == nil && opts.End == nil {
|
|
return io.NopCloser(bytes.NewReader(data)), info, nil
|
|
}
|
|
|
|
// Build range. Check each pointer individually to avoid nil dereference.
|
|
start := int64(0)
|
|
if opts.Start != nil {
|
|
start = *opts.Start
|
|
}
|
|
|
|
end := size - 1
|
|
if opts.End != nil {
|
|
end = *opts.End
|
|
}
|
|
|
|
// Clamp end to last byte.
|
|
if end >= size {
|
|
end = size - 1
|
|
}
|
|
|
|
if start > end || start < 0 {
|
|
return nil, storage.ObjectInfo{}, fmt.Errorf("invalid range: start=%d, end=%d, size=%d", start, end, size)
|
|
}
|
|
|
|
section := io.NewSectionReader(bytes.NewReader(data), start, end-start+1)
|
|
return io.NopCloser(section), info, nil
|
|
}
|
|
|
|
// ComposeObject concatenates source objects (in order) into dst.
|
|
func (s *InMemoryStorage) ComposeObject(_ context.Context, _, dst string, sources []string) (storage.UploadInfo, error) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
|
|
var buf bytes.Buffer
|
|
for _, src := range sources {
|
|
data, ok := s.objects[src]
|
|
if !ok {
|
|
return storage.UploadInfo{}, fmt.Errorf("source object %s not found", src)
|
|
}
|
|
buf.Write(data)
|
|
}
|
|
|
|
combined := buf.Bytes()
|
|
h := sha256.Sum256(combined)
|
|
etag := hex.EncodeToString(h[:])
|
|
|
|
s.objects[dst] = combined
|
|
s.meta[dst] = objectMeta{
|
|
size: int64(len(combined)),
|
|
etag: etag,
|
|
lastModified: time.Now(),
|
|
}
|
|
|
|
return storage.UploadInfo{ETag: etag, Size: int64(len(combined))}, nil
|
|
}
|
|
|
|
// RemoveObject deletes a single object.
|
|
func (s *InMemoryStorage) RemoveObject(_ context.Context, _, key string, _ storage.RemoveObjectOptions) error {
|
|
s.mu.Lock()
|
|
delete(s.objects, key)
|
|
delete(s.meta, key)
|
|
s.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
// RemoveObjects deletes multiple objects by key.
|
|
func (s *InMemoryStorage) RemoveObjects(_ context.Context, _ string, keys []string, _ storage.RemoveObjectsOptions) error {
|
|
s.mu.Lock()
|
|
for _, k := range keys {
|
|
delete(s.objects, k)
|
|
delete(s.meta, k)
|
|
}
|
|
s.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
// ListObjects returns object info for all objects matching prefix, sorted by key.
|
|
func (s *InMemoryStorage) ListObjects(_ context.Context, _, prefix string, _ bool) ([]storage.ObjectInfo, error) {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
|
|
var result []storage.ObjectInfo
|
|
for k, m := range s.meta {
|
|
if strings.HasPrefix(k, prefix) {
|
|
result = append(result, storage.ObjectInfo{
|
|
Key: k,
|
|
Size: m.size,
|
|
ETag: m.etag,
|
|
LastModified: m.lastModified,
|
|
ContentType: m.contentType,
|
|
})
|
|
}
|
|
}
|
|
sort.Slice(result, func(i, j int) bool { return result[i].Key < result[j].Key })
|
|
return result, nil
|
|
}
|
|
|
|
// BucketExists reports whether the named bucket exists.
|
|
func (s *InMemoryStorage) BucketExists(_ context.Context, bucket string) (bool, error) {
|
|
s.mu.RLock()
|
|
defer s.mu.RUnlock()
|
|
return s.buckets[bucket], nil
|
|
}
|
|
|
|
// MakeBucket creates a bucket.
|
|
func (s *InMemoryStorage) MakeBucket(_ context.Context, bucket string, _ storage.MakeBucketOptions) error {
|
|
s.mu.Lock()
|
|
s.buckets[bucket] = true
|
|
s.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
// StatObject returns metadata about an object without downloading it.
|
|
func (s *InMemoryStorage) StatObject(_ context.Context, _, key string, _ storage.StatObjectOptions) (storage.ObjectInfo, error) {
|
|
s.mu.RLock()
|
|
m, ok := s.meta[key]
|
|
s.mu.RUnlock()
|
|
if !ok {
|
|
return storage.ObjectInfo{}, fmt.Errorf("object %s not found", key)
|
|
}
|
|
return storage.ObjectInfo{
|
|
Key: key,
|
|
Size: m.size,
|
|
ETag: m.etag,
|
|
LastModified: m.lastModified,
|
|
ContentType: m.contentType,
|
|
}, nil
|
|
}
|
|
|
|
// AbortMultipartUpload is a no-op for the in-memory implementation.
|
|
func (s *InMemoryStorage) AbortMultipartUpload(_ context.Context, _, _, _ string) error {
|
|
return nil
|
|
}
|
|
|
|
// RemoveIncompleteUpload is a no-op for the in-memory implementation.
|
|
func (s *InMemoryStorage) RemoveIncompleteUpload(_ context.Context, _, _ string) error {
|
|
return nil
|
|
}
|