feat(app): wire file storage DI, cleanup worker, and integration tests
Add DI wiring with graceful MinIO fallback, background cleanup worker for expired sessions and leaked multipart uploads, and end-to-end integration tests. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
dailz
266
internal/app/cleanup_test.go
Normal file
266
internal/app/cleanup_test.go
Normal file
@@ -0,0 +1,266 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gcy_hpc_server/internal/model"
|
||||
"gcy_hpc_server/internal/storage"
|
||||
"gcy_hpc_server/internal/store"
|
||||
|
||||
"go.uber.org/zap"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// mockCleanupStorage implements ObjectStorage for cleanup tests.
|
||||
type mockCleanupStorage struct {
|
||||
listObjectsFn func(ctx context.Context, bucket, prefix string, recursive bool) ([]storage.ObjectInfo, error)
|
||||
removeObjectsFn func(ctx context.Context, bucket string, keys []string, opts storage.RemoveObjectsOptions) error
|
||||
removeIncompleteFn func(ctx context.Context, bucket, object string) error
|
||||
}
|
||||
|
||||
func (m *mockCleanupStorage) PutObject(_ context.Context, _ string, _ string, _ io.Reader, _ int64, _ storage.PutObjectOptions) (storage.UploadInfo, error) {
|
||||
return storage.UploadInfo{}, nil
|
||||
}
|
||||
|
||||
func (m *mockCleanupStorage) GetObject(_ context.Context, _ string, _ string, _ storage.GetOptions) (io.ReadCloser, storage.ObjectInfo, error) {
|
||||
return nil, storage.ObjectInfo{}, nil
|
||||
}
|
||||
|
||||
func (m *mockCleanupStorage) ComposeObject(_ context.Context, _ string, _ string, _ []string) (storage.UploadInfo, error) {
|
||||
return storage.UploadInfo{}, nil
|
||||
}
|
||||
|
||||
func (m *mockCleanupStorage) AbortMultipartUpload(_ context.Context, _ string, _ string, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockCleanupStorage) RemoveIncompleteUpload(_ context.Context, _ string, _ string) error {
|
||||
if m.removeIncompleteFn != nil {
|
||||
return m.removeIncompleteFn(context.Background(), "", "")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockCleanupStorage) RemoveObject(_ context.Context, _ string, _ string, _ storage.RemoveObjectOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockCleanupStorage) ListObjects(ctx context.Context, bucket string, prefix string, recursive bool) ([]storage.ObjectInfo, error) {
|
||||
if m.listObjectsFn != nil {
|
||||
return m.listObjectsFn(ctx, bucket, prefix, recursive)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (m *mockCleanupStorage) RemoveObjects(ctx context.Context, bucket string, keys []string, opts storage.RemoveObjectsOptions) error {
|
||||
if m.removeObjectsFn != nil {
|
||||
return m.removeObjectsFn(ctx, bucket, keys, opts)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockCleanupStorage) BucketExists(_ context.Context, _ string) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (m *mockCleanupStorage) MakeBucket(_ context.Context, _ string, _ storage.MakeBucketOptions) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *mockCleanupStorage) StatObject(_ context.Context, _ string, _ string, _ storage.StatObjectOptions) (storage.ObjectInfo, error) {
|
||||
return storage.ObjectInfo{}, nil
|
||||
}
|
||||
|
||||
func setupCleanupTestDB(t *testing.T) (*gorm.DB, *store.UploadStore) {
|
||||
t.Helper()
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&model.UploadSession{}, &model.UploadChunk{}); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
return db, store.NewUploadStore(db)
|
||||
}
|
||||
|
||||
func TestCleanupExpiredSessions(t *testing.T) {
|
||||
_, uploadStore := setupCleanupTestDB(t)
|
||||
ctx := context.Background()
|
||||
|
||||
past := time.Now().Add(-1 * time.Hour)
|
||||
err := uploadStore.CreateSession(ctx, &model.UploadSession{
|
||||
FileName: "expired.bin",
|
||||
FileSize: 1024,
|
||||
ChunkSize: 16 << 20,
|
||||
TotalChunks: 1,
|
||||
SHA256: "expired_hash",
|
||||
Status: "pending",
|
||||
MinioPrefix: "uploads/99/",
|
||||
ExpiresAt: past,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create session: %v", err)
|
||||
}
|
||||
|
||||
var listedPrefix string
|
||||
var removedKeys []string
|
||||
|
||||
mockStore := &mockCleanupStorage{
|
||||
listObjectsFn: func(_ context.Context, _, prefix string, _ bool) ([]storage.ObjectInfo, error) {
|
||||
listedPrefix = prefix
|
||||
return []storage.ObjectInfo{{Key: "uploads/99/chunk_00000", Size: 100}}, nil
|
||||
},
|
||||
removeObjectsFn: func(_ context.Context, _ string, keys []string, _ storage.RemoveObjectsOptions) error {
|
||||
removedKeys = keys
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
cleanupExpiredSessions(ctx, uploadStore, mockStore, "test-bucket", zap.NewNop())
|
||||
|
||||
if listedPrefix != "uploads/99/" {
|
||||
t.Errorf("listed prefix = %q, want %q", listedPrefix, "uploads/99/")
|
||||
}
|
||||
if len(removedKeys) != 1 || removedKeys[0] != "uploads/99/chunk_00000" {
|
||||
t.Errorf("removed keys = %v, want [uploads/99/chunk_00000]", removedKeys)
|
||||
}
|
||||
|
||||
session, err := uploadStore.GetSession(ctx, 1)
|
||||
if err != nil {
|
||||
t.Fatalf("get session: %v", err)
|
||||
}
|
||||
if session != nil {
|
||||
t.Error("session should be deleted after cleanup")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupExpiredSessions_Empty(t *testing.T) {
|
||||
_, uploadStore := setupCleanupTestDB(t)
|
||||
ctx := context.Background()
|
||||
|
||||
called := false
|
||||
mockStore := &mockCleanupStorage{
|
||||
listObjectsFn: func(_ context.Context, _, _ string, _ bool) ([]storage.ObjectInfo, error) {
|
||||
called = true
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
|
||||
cleanupExpiredSessions(ctx, uploadStore, mockStore, "test-bucket", zap.NewNop())
|
||||
|
||||
if called {
|
||||
t.Error("ListObjects should not be called when no expired sessions exist")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupExpiredSessions_CompletedNotCleaned(t *testing.T) {
|
||||
_, uploadStore := setupCleanupTestDB(t)
|
||||
ctx := context.Background()
|
||||
|
||||
past := time.Now().Add(-1 * time.Hour)
|
||||
err := uploadStore.CreateSession(ctx, &model.UploadSession{
|
||||
FileName: "completed.bin",
|
||||
FileSize: 1024,
|
||||
ChunkSize: 16 << 20,
|
||||
TotalChunks: 1,
|
||||
SHA256: "completed_hash",
|
||||
Status: "completed",
|
||||
MinioPrefix: "uploads/100/",
|
||||
ExpiresAt: past,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create session: %v", err)
|
||||
}
|
||||
|
||||
called := false
|
||||
mockStore := &mockCleanupStorage{
|
||||
listObjectsFn: func(_ context.Context, _, _ string, _ bool) ([]storage.ObjectInfo, error) {
|
||||
called = true
|
||||
return nil, nil
|
||||
},
|
||||
}
|
||||
|
||||
cleanupExpiredSessions(ctx, uploadStore, mockStore, "test-bucket", zap.NewNop())
|
||||
|
||||
if called {
|
||||
t.Error("ListObjects should not be called for completed sessions")
|
||||
}
|
||||
|
||||
session, _ := uploadStore.GetSession(ctx, 1)
|
||||
if session == nil {
|
||||
t.Error("completed session should not be deleted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupWorker_StopsOnContextCancel(t *testing.T) {
|
||||
_, uploadStore := setupCleanupTestDB(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
mockStore := &mockCleanupStorage{}
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
startCleanupWorker(ctx, uploadStore, mockStore, "test-bucket", zap.NewNop())
|
||||
close(done)
|
||||
}()
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Error("worker did not stop after context cancel")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCleanupExpiredSessions_NoObjects(t *testing.T) {
|
||||
_, uploadStore := setupCleanupTestDB(t)
|
||||
ctx := context.Background()
|
||||
|
||||
past := time.Now().Add(-1 * time.Hour)
|
||||
err := uploadStore.CreateSession(ctx, &model.UploadSession{
|
||||
FileName: "empty.bin",
|
||||
FileSize: 1024,
|
||||
ChunkSize: 16 << 20,
|
||||
TotalChunks: 1,
|
||||
SHA256: "empty_hash",
|
||||
Status: "pending",
|
||||
MinioPrefix: "uploads/200/",
|
||||
ExpiresAt: past,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create session: %v", err)
|
||||
}
|
||||
|
||||
listCalled := false
|
||||
removeCalled := false
|
||||
mockStore := &mockCleanupStorage{
|
||||
listObjectsFn: func(_ context.Context, _, _ string, _ bool) ([]storage.ObjectInfo, error) {
|
||||
listCalled = true
|
||||
return nil, nil
|
||||
},
|
||||
removeObjectsFn: func(_ context.Context, _ string, _ []string, _ storage.RemoveObjectsOptions) error {
|
||||
removeCalled = true
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
cleanupExpiredSessions(ctx, uploadStore, mockStore, "test-bucket", zap.NewNop())
|
||||
|
||||
if !listCalled {
|
||||
t.Error("ListObjects should be called")
|
||||
}
|
||||
if removeCalled {
|
||||
t.Error("RemoveObjects should not be called when no objects found")
|
||||
}
|
||||
|
||||
session, _ := uploadStore.GetSession(ctx, 1)
|
||||
if session != nil {
|
||||
t.Error("session should be deleted even when no objects found")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user