f0847d3978
Add UploadService (dedup, chunk lifecycle, ComposeObject), DownloadService (Range support), FileService (ref counting), FolderService (path validation). Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
485 lines
14 KiB
Go
485 lines
14 KiB
Go
package service
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gcy_hpc_server/internal/model"
|
|
"gcy_hpc_server/internal/storage"
|
|
"gcy_hpc_server/internal/store"
|
|
|
|
"go.uber.org/zap"
|
|
"gorm.io/driver/sqlite"
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
type mockFileStorage struct {
|
|
getObjectFn func(ctx context.Context, bucket, key string, opts storage.GetOptions) (io.ReadCloser, storage.ObjectInfo, error)
|
|
removeObjectFn func(ctx context.Context, bucket, key string, opts storage.RemoveObjectOptions) error
|
|
}
|
|
|
|
func (m *mockFileStorage) PutObject(_ context.Context, _, _ string, _ io.Reader, _ int64, _ storage.PutObjectOptions) (storage.UploadInfo, error) {
|
|
return storage.UploadInfo{}, nil
|
|
}
|
|
|
|
func (m *mockFileStorage) GetObject(ctx context.Context, bucket, key string, opts storage.GetOptions) (io.ReadCloser, storage.ObjectInfo, error) {
|
|
if m.getObjectFn != nil {
|
|
return m.getObjectFn(ctx, bucket, key, opts)
|
|
}
|
|
return io.NopCloser(strings.NewReader("data")), storage.ObjectInfo{}, nil
|
|
}
|
|
|
|
func (m *mockFileStorage) ComposeObject(_ context.Context, _ string, _ string, _ []string) (storage.UploadInfo, error) {
|
|
return storage.UploadInfo{}, nil
|
|
}
|
|
|
|
func (m *mockFileStorage) AbortMultipartUpload(_ context.Context, _, _, _ string) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockFileStorage) RemoveIncompleteUpload(_ context.Context, _, _ string) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockFileStorage) RemoveObject(ctx context.Context, bucket, key string, opts storage.RemoveObjectOptions) error {
|
|
if m.removeObjectFn != nil {
|
|
return m.removeObjectFn(ctx, bucket, key, opts)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *mockFileStorage) ListObjects(_ context.Context, _ string, _ string, _ bool) ([]storage.ObjectInfo, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (m *mockFileStorage) RemoveObjects(_ context.Context, _ string, _ []string, _ storage.RemoveObjectsOptions) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockFileStorage) BucketExists(_ context.Context, _ string) (bool, error) {
|
|
return true, nil
|
|
}
|
|
|
|
func (m *mockFileStorage) MakeBucket(_ context.Context, _ string, _ storage.MakeBucketOptions) error {
|
|
return nil
|
|
}
|
|
|
|
func (m *mockFileStorage) StatObject(_ context.Context, _, _ string, _ storage.StatObjectOptions) (storage.ObjectInfo, error) {
|
|
return storage.ObjectInfo{}, nil
|
|
}
|
|
|
|
func setupFileTestDB(t *testing.T) *gorm.DB {
|
|
t.Helper()
|
|
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
|
if err != nil {
|
|
t.Fatalf("open sqlite: %v", err)
|
|
}
|
|
if err := db.AutoMigrate(&model.File{}, &model.FileBlob{}); err != nil {
|
|
t.Fatalf("migrate: %v", err)
|
|
}
|
|
return db
|
|
}
|
|
|
|
func setupFileService(t *testing.T) (*FileService, *mockFileStorage, *gorm.DB) {
|
|
t.Helper()
|
|
db := setupFileTestDB(t)
|
|
ms := &mockFileStorage{}
|
|
svc := NewFileService(ms, store.NewBlobStore(db), store.NewFileStore(db), "test-bucket", db, zap.NewNop())
|
|
return svc, ms, db
|
|
}
|
|
|
|
func createTestBlob(t *testing.T, db *gorm.DB, sha256, minioKey, mimeType string, fileSize int64, refCount int) *model.FileBlob {
|
|
t.Helper()
|
|
blob := &model.FileBlob{
|
|
SHA256: sha256,
|
|
MinioKey: minioKey,
|
|
FileSize: fileSize,
|
|
MimeType: mimeType,
|
|
RefCount: refCount,
|
|
}
|
|
if err := db.Create(blob).Error; err != nil {
|
|
t.Fatalf("create blob: %v", err)
|
|
}
|
|
return blob
|
|
}
|
|
|
|
func createTestFile(t *testing.T, db *gorm.DB, name, blobSHA256 string, folderID *int64) *model.File {
|
|
t.Helper()
|
|
file := &model.File{
|
|
Name: name,
|
|
FolderID: folderID,
|
|
BlobSHA256: blobSHA256,
|
|
}
|
|
if err := db.Create(file).Error; err != nil {
|
|
t.Fatalf("create file: %v", err)
|
|
}
|
|
return file
|
|
}
|
|
|
|
func TestListFiles_Empty(t *testing.T) {
|
|
svc, _, _ := setupFileService(t)
|
|
|
|
files, total, err := svc.ListFiles(context.Background(), nil, 1, 10, "")
|
|
if err != nil {
|
|
t.Fatalf("ListFiles: %v", err)
|
|
}
|
|
if total != 0 {
|
|
t.Errorf("expected total 0, got %d", total)
|
|
}
|
|
if len(files) != 0 {
|
|
t.Errorf("expected empty files, got %d", len(files))
|
|
}
|
|
}
|
|
|
|
func TestListFiles_WithFiles(t *testing.T) {
|
|
svc, _, db := setupFileService(t)
|
|
|
|
blob := createTestBlob(t, db, "sha256abc", "blobs/abc", "text/plain", 1024, 2)
|
|
createTestFile(t, db, "file1.txt", blob.SHA256, nil)
|
|
createTestFile(t, db, "file2.txt", blob.SHA256, nil)
|
|
|
|
files, total, err := svc.ListFiles(context.Background(), nil, 1, 10, "")
|
|
if err != nil {
|
|
t.Fatalf("ListFiles: %v", err)
|
|
}
|
|
if total != 2 {
|
|
t.Errorf("expected total 2, got %d", total)
|
|
}
|
|
if len(files) != 2 {
|
|
t.Fatalf("expected 2 files, got %d", len(files))
|
|
}
|
|
for _, f := range files {
|
|
if f.Size != 1024 {
|
|
t.Errorf("expected size 1024, got %d", f.Size)
|
|
}
|
|
if f.MimeType != "text/plain" {
|
|
t.Errorf("expected mime text/plain, got %s", f.MimeType)
|
|
}
|
|
if f.SHA256 != "sha256abc" {
|
|
t.Errorf("expected sha256 sha256abc, got %s", f.SHA256)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestListFiles_Search(t *testing.T) {
|
|
svc, _, db := setupFileService(t)
|
|
|
|
blob := createTestBlob(t, db, "sha256search", "blobs/search", "image/png", 2048, 1)
|
|
createTestFile(t, db, "photo.png", blob.SHA256, nil)
|
|
createTestFile(t, db, "document.pdf", "sha256other", nil)
|
|
createTestBlob(t, db, "sha256other", "blobs/other", "application/pdf", 512, 1)
|
|
|
|
files, total, err := svc.ListFiles(context.Background(), nil, 1, 10, "photo")
|
|
if err != nil {
|
|
t.Fatalf("ListFiles: %v", err)
|
|
}
|
|
if total != 1 {
|
|
t.Errorf("expected total 1, got %d", total)
|
|
}
|
|
if len(files) != 1 {
|
|
t.Fatalf("expected 1 file, got %d", len(files))
|
|
}
|
|
if files[0].Name != "photo.png" {
|
|
t.Errorf("expected photo.png, got %s", files[0].Name)
|
|
}
|
|
}
|
|
|
|
func TestGetFileMetadata_Found(t *testing.T) {
|
|
svc, _, db := setupFileService(t)
|
|
|
|
blob := createTestBlob(t, db, "sha256meta", "blobs/meta", "application/json", 42, 1)
|
|
file := createTestFile(t, db, "data.json", blob.SHA256, nil)
|
|
|
|
gotFile, gotBlob, err := svc.GetFileMetadata(context.Background(), file.ID)
|
|
if err != nil {
|
|
t.Fatalf("GetFileMetadata: %v", err)
|
|
}
|
|
if gotFile.ID != file.ID {
|
|
t.Errorf("expected file id %d, got %d", file.ID, gotFile.ID)
|
|
}
|
|
if gotBlob.SHA256 != blob.SHA256 {
|
|
t.Errorf("expected blob sha256 %s, got %s", blob.SHA256, gotBlob.SHA256)
|
|
}
|
|
}
|
|
|
|
func TestGetFileMetadata_NotFound(t *testing.T) {
|
|
svc, _, _ := setupFileService(t)
|
|
|
|
_, _, err := svc.GetFileMetadata(context.Background(), 9999)
|
|
if err == nil {
|
|
t.Fatal("expected error for missing file")
|
|
}
|
|
}
|
|
|
|
func TestDownloadFile_Full(t *testing.T) {
|
|
svc, ms, db := setupFileService(t)
|
|
|
|
blob := createTestBlob(t, db, "sha256dl", "blobs/dl", "text/plain", 100, 1)
|
|
file := createTestFile(t, db, "download.txt", blob.SHA256, nil)
|
|
|
|
content := []byte("hello world")
|
|
ms.getObjectFn = func(_ context.Context, _ string, _ string, opts storage.GetOptions) (io.ReadCloser, storage.ObjectInfo, error) {
|
|
if opts.Start == nil || opts.End == nil {
|
|
t.Error("expected start and end to be set")
|
|
} else if *opts.Start != 0 || *opts.End != 99 {
|
|
t.Errorf("expected range 0-99, got %d-%d", *opts.Start, *opts.End)
|
|
}
|
|
return io.NopCloser(bytes.NewReader(content)), storage.ObjectInfo{}, nil
|
|
}
|
|
|
|
reader, gotFile, gotBlob, start, end, err := svc.DownloadFile(context.Background(), file.ID, "")
|
|
if err != nil {
|
|
t.Fatalf("DownloadFile: %v", err)
|
|
}
|
|
defer reader.Close()
|
|
|
|
if gotFile.ID != file.ID {
|
|
t.Errorf("expected file id %d, got %d", file.ID, gotFile.ID)
|
|
}
|
|
if gotBlob.SHA256 != blob.SHA256 {
|
|
t.Errorf("expected blob sha256 %s, got %s", blob.SHA256, gotBlob.SHA256)
|
|
}
|
|
if start != 0 {
|
|
t.Errorf("expected start 0, got %d", start)
|
|
}
|
|
if end != 99 {
|
|
t.Errorf("expected end 99, got %d", end)
|
|
}
|
|
|
|
data, _ := io.ReadAll(reader)
|
|
if string(data) != "hello world" {
|
|
t.Errorf("expected 'hello world', got %q", string(data))
|
|
}
|
|
}
|
|
|
|
func TestDownloadFile_WithRange(t *testing.T) {
|
|
svc, ms, db := setupFileService(t)
|
|
|
|
blob := createTestBlob(t, db, "sha256range", "blobs/range", "text/plain", 1000, 1)
|
|
file := createTestFile(t, db, "range.txt", blob.SHA256, nil)
|
|
|
|
ms.getObjectFn = func(_ context.Context, _ string, _ string, opts storage.GetOptions) (io.ReadCloser, storage.ObjectInfo, error) {
|
|
if opts.Start != nil && *opts.Start != 100 {
|
|
t.Errorf("expected start 100, got %d", *opts.Start)
|
|
}
|
|
if opts.End != nil && *opts.End != 199 {
|
|
t.Errorf("expected end 199, got %d", *opts.End)
|
|
}
|
|
return io.NopCloser(strings.NewReader("partial")), storage.ObjectInfo{}, nil
|
|
}
|
|
|
|
reader, _, _, start, end, err := svc.DownloadFile(context.Background(), file.ID, "bytes=100-199")
|
|
if err != nil {
|
|
t.Fatalf("DownloadFile: %v", err)
|
|
}
|
|
defer reader.Close()
|
|
|
|
if start != 100 {
|
|
t.Errorf("expected start 100, got %d", start)
|
|
}
|
|
if end != 199 {
|
|
t.Errorf("expected end 199, got %d", end)
|
|
}
|
|
}
|
|
|
|
func TestDeleteFile_LastRef(t *testing.T) {
|
|
svc, ms, db := setupFileService(t)
|
|
|
|
blob := createTestBlob(t, db, "sha256del", "blobs/del", "text/plain", 50, 1)
|
|
file := createTestFile(t, db, "delete-me.txt", blob.SHA256, nil)
|
|
|
|
removed := false
|
|
ms.removeObjectFn = func(_ context.Context, bucket, key string, _ storage.RemoveObjectOptions) error {
|
|
if bucket != "test-bucket" {
|
|
t.Errorf("expected bucket 'test-bucket', got %q", bucket)
|
|
}
|
|
if key != "blobs/del" {
|
|
t.Errorf("expected key 'blobs/del', got %q", key)
|
|
}
|
|
removed = true
|
|
return nil
|
|
}
|
|
|
|
if err := svc.DeleteFile(context.Background(), file.ID); err != nil {
|
|
t.Fatalf("DeleteFile: %v", err)
|
|
}
|
|
|
|
if !removed {
|
|
t.Error("expected RemoveObject to be called")
|
|
}
|
|
|
|
var count int64
|
|
db.Model(&model.FileBlob{}).Where("sha256 = ?", "sha256del").Count(&count)
|
|
if count != 0 {
|
|
t.Errorf("expected blob to be hard deleted, found %d records", count)
|
|
}
|
|
|
|
var fileCount int64
|
|
db.Unscoped().Model(&model.File{}).Where("id = ?", file.ID).Count(&fileCount)
|
|
if fileCount != 1 {
|
|
t.Errorf("expected file to still exist (soft deleted), found %d", fileCount)
|
|
}
|
|
|
|
var deletedFile model.File
|
|
db.Unscoped().First(&deletedFile, file.ID)
|
|
if deletedFile.DeletedAt.Time.IsZero() {
|
|
t.Error("expected deleted_at to be set")
|
|
}
|
|
}
|
|
|
|
func TestDeleteFile_OtherRefsExist(t *testing.T) {
|
|
svc, ms, db := setupFileService(t)
|
|
|
|
blob := createTestBlob(t, db, "sha256multi", "blobs/multi", "text/plain", 50, 3)
|
|
file1 := createTestFile(t, db, "ref1.txt", blob.SHA256, nil)
|
|
createTestFile(t, db, "ref2.txt", blob.SHA256, nil)
|
|
createTestFile(t, db, "ref3.txt", blob.SHA256, nil)
|
|
|
|
removed := false
|
|
ms.removeObjectFn = func(_ context.Context, _, _ string, _ storage.RemoveObjectOptions) error {
|
|
removed = true
|
|
return nil
|
|
}
|
|
|
|
if err := svc.DeleteFile(context.Background(), file1.ID); err != nil {
|
|
t.Fatalf("DeleteFile: %v", err)
|
|
}
|
|
|
|
if removed {
|
|
t.Error("expected RemoveObject NOT to be called since other refs exist")
|
|
}
|
|
|
|
var updatedBlob model.FileBlob
|
|
db.Where("sha256 = ?", "sha256multi").First(&updatedBlob)
|
|
if updatedBlob.RefCount != 3 {
|
|
t.Errorf("expected ref_count to remain 3, got %d", updatedBlob.RefCount)
|
|
}
|
|
}
|
|
|
|
func TestDeleteFile_SoftDeleteNotAffectRefcount(t *testing.T) {
|
|
svc, _, db := setupFileService(t)
|
|
|
|
blob := createTestBlob(t, db, "sha256soft", "blobs/soft", "text/plain", 50, 2)
|
|
file1 := createTestFile(t, db, "soft1.txt", blob.SHA256, nil)
|
|
file2 := createTestFile(t, db, "soft2.txt", blob.SHA256, nil)
|
|
|
|
if err := svc.DeleteFile(context.Background(), file1.ID); err != nil {
|
|
t.Fatalf("DeleteFile: %v", err)
|
|
}
|
|
|
|
var updatedBlob model.FileBlob
|
|
db.Where("sha256 = ?", "sha256soft").First(&updatedBlob)
|
|
if updatedBlob.RefCount != 2 {
|
|
t.Errorf("expected ref_count to remain 2 (soft delete should not decrement), got %d", updatedBlob.RefCount)
|
|
}
|
|
|
|
activeCount, err := store.NewFileStore(db).CountByBlobSHA256(context.Background(), "sha256soft")
|
|
if err != nil {
|
|
t.Fatalf("CountByBlobSHA256: %v", err)
|
|
}
|
|
if activeCount != 1 {
|
|
t.Errorf("expected 1 active ref after soft delete, got %d", activeCount)
|
|
}
|
|
|
|
var allFiles []model.File
|
|
db.Unscoped().Where("blob_sha256 = ?", "sha256soft").Find(&allFiles)
|
|
if len(allFiles) != 2 {
|
|
t.Errorf("expected 2 total files (one soft deleted), got %d", len(allFiles))
|
|
}
|
|
|
|
if err := svc.DeleteFile(context.Background(), file2.ID); err != nil {
|
|
t.Fatalf("DeleteFile second: %v", err)
|
|
}
|
|
|
|
activeCount2, err := store.NewFileStore(db).CountByBlobSHA256(context.Background(), "sha256soft")
|
|
if err != nil {
|
|
t.Fatalf("CountByBlobSHA256 after second delete: %v", err)
|
|
}
|
|
if activeCount2 != 0 {
|
|
t.Errorf("expected 0 active refs after both deleted, got %d", activeCount2)
|
|
}
|
|
|
|
var finalBlob model.FileBlob
|
|
db.Where("sha256 = ?", "sha256soft").First(&finalBlob)
|
|
if finalBlob.RefCount != 1 {
|
|
t.Errorf("expected ref_count=1 (decremented once from 2), got %d", finalBlob.RefCount)
|
|
}
|
|
}
|
|
|
|
func TestDeleteFile_NotFound(t *testing.T) {
|
|
svc, _, _ := setupFileService(t)
|
|
|
|
err := svc.DeleteFile(context.Background(), 9999)
|
|
if err == nil {
|
|
t.Fatal("expected error for missing file")
|
|
}
|
|
}
|
|
|
|
func TestDownloadFile_StorageError(t *testing.T) {
|
|
svc, ms, db := setupFileService(t)
|
|
|
|
blob := createTestBlob(t, db, "sha256err", "blobs/err", "text/plain", 100, 1)
|
|
file := createTestFile(t, db, "error.txt", blob.SHA256, nil)
|
|
|
|
ms.getObjectFn = func(_ context.Context, _ string, _ string, _ storage.GetOptions) (io.ReadCloser, storage.ObjectInfo, error) {
|
|
return nil, storage.ObjectInfo{}, errors.New("storage unavailable")
|
|
}
|
|
|
|
_, _, _, _, _, err := svc.DownloadFile(context.Background(), file.ID, "")
|
|
if err == nil {
|
|
t.Fatal("expected error from storage")
|
|
}
|
|
if !strings.Contains(err.Error(), "storage unavailable") {
|
|
t.Errorf("expected storage error, got: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestListFiles_WithFolderFilter(t *testing.T) {
|
|
svc, _, db := setupFileService(t)
|
|
|
|
blob := createTestBlob(t, db, "sha256folder", "blobs/folder", "text/plain", 100, 2)
|
|
folderID := int64(1)
|
|
createTestFile(t, db, "in_folder.txt", blob.SHA256, &folderID)
|
|
createTestFile(t, db, "root.txt", blob.SHA256, nil)
|
|
|
|
files, total, err := svc.ListFiles(context.Background(), &folderID, 1, 10, "")
|
|
if err != nil {
|
|
t.Fatalf("ListFiles: %v", err)
|
|
}
|
|
if total != 1 {
|
|
t.Errorf("expected total 1, got %d", total)
|
|
}
|
|
if len(files) != 1 {
|
|
t.Fatalf("expected 1 file, got %d", len(files))
|
|
}
|
|
if files[0].Name != "in_folder.txt" {
|
|
t.Errorf("expected in_folder.txt, got %s", files[0].Name)
|
|
}
|
|
|
|
rootFiles, rootTotal, err := svc.ListFiles(context.Background(), nil, 1, 10, "")
|
|
if err != nil {
|
|
t.Fatalf("ListFiles root: %v", err)
|
|
}
|
|
if rootTotal != 1 {
|
|
t.Errorf("expected root total 1, got %d", rootTotal)
|
|
}
|
|
if len(rootFiles) != 1 || rootFiles[0].Name != "root.txt" {
|
|
t.Errorf("expected root.txt in root listing")
|
|
}
|
|
}
|
|
|
|
func TestGetFileMetadata_BlobMissing(t *testing.T) {
|
|
svc, _, db := setupFileService(t)
|
|
|
|
file := createTestFile(t, db, "orphan.txt", "nonexistent_sha256", nil)
|
|
|
|
_, _, err := svc.GetFileMetadata(context.Background(), file.ID)
|
|
if err == nil {
|
|
t.Fatal("expected error when blob is missing")
|
|
}
|
|
}
|