From 44895214d4bc49894caf306d740c4f8d6668f422 Mon Sep 17 00:00:00 2001 From: dailz Date: Wed, 15 Apr 2026 09:22:18 +0800 Subject: [PATCH] feat(config): add MinIO object storage configuration Add MinioConfig struct with connection, bucket, chunk size, and session TTL settings. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus --- internal/config/config.example.yaml | 11 +++ internal/config/config.go | 41 +++++++-- internal/config/config_test.go | 132 ++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 7 deletions(-) diff --git a/internal/config/config.example.yaml b/internal/config/config.example.yaml index 3d86927..0cc557b 100644 --- a/internal/config/config.example.yaml +++ b/internal/config/config.example.yaml @@ -15,3 +15,14 @@ log: max_age: 30 # days to retain old log files compress: true # gzip rotated log files gorm_level: "warn" # GORM SQL log level: silent, error, warn, info + +minio: + endpoint: "http://fnos.dailz.cn:15001" # MinIO server address + access_key: "3dgDu9ncwflLoRQW2OeP" # access key + secret_key: "g2GLBNTPxJ9sdFwh37jtfilRSacEO5yQepMkDrnV" # secret key + bucket: "test" # bucket name + use_ssl: false # use TLS connection + chunk_size: 16777216 # upload chunk size in bytes (default: 16MB) + max_file_size: 53687091200 # max file size in bytes (default: 50GB) + min_chunk_size: 5242880 # minimum chunk size in bytes (default: 5MB) + session_ttl: 48 # session TTL in hours (default: 48) diff --git a/internal/config/config.go b/internal/config/config.go index 996326a..4b70583 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -20,15 +20,29 @@ type LogConfig struct { GormLevel string `yaml:"gorm_level"` // GORM SQL log level (default: warn) } +// MinioConfig holds MinIO object storage configuration values. +type MinioConfig struct { + Endpoint string `yaml:"endpoint"` // MinIO server address + AccessKey string `yaml:"access_key"` // access key + SecretKey string `yaml:"secret_key"` // secret key + Bucket string `yaml:"bucket"` // bucket name + UseSSL bool `yaml:"use_ssl"` // use TLS connection + ChunkSize int64 `yaml:"chunk_size"` // upload chunk size in bytes (default: 16MB) + MaxFileSize int64 `yaml:"max_file_size"` // max file size in bytes (default: 50GB) + MinChunkSize int64 `yaml:"min_chunk_size"` // minimum chunk size in bytes (default: 5MB) + SessionTTL int `yaml:"session_ttl"` // session TTL in hours (default: 48) +} + // Config holds all application configuration values. type Config struct { - ServerPort string `yaml:"server_port"` - SlurmAPIURL string `yaml:"slurm_api_url"` - SlurmUserName string `yaml:"slurm_user_name"` - SlurmJWTKeyPath string `yaml:"slurm_jwt_key_path"` - MySQLDSN string `yaml:"mysql_dsn"` - WorkDirBase string `yaml:"work_dir_base"` // base directory for job work dirs - Log LogConfig `yaml:"log"` + ServerPort string `yaml:"server_port"` + SlurmAPIURL string `yaml:"slurm_api_url"` + SlurmUserName string `yaml:"slurm_user_name"` + SlurmJWTKeyPath string `yaml:"slurm_jwt_key_path"` + MySQLDSN string `yaml:"mysql_dsn"` + WorkDirBase string `yaml:"work_dir_base"` // base directory for job work dirs + Log LogConfig `yaml:"log"` + Minio MinioConfig `yaml:"minio"` } // Load reads a YAML configuration file and returns a parsed Config. @@ -48,5 +62,18 @@ func Load(path string) (*Config, error) { return nil, fmt.Errorf("parse config file %s: %w", path, err) } + if cfg.Minio.ChunkSize == 0 { + cfg.Minio.ChunkSize = 16 << 20 // 16MB + } + if cfg.Minio.MaxFileSize == 0 { + cfg.Minio.MaxFileSize = 50 << 30 // 50GB + } + if cfg.Minio.MinChunkSize == 0 { + cfg.Minio.MinChunkSize = 5 << 20 // 5MB + } + if cfg.Minio.SessionTTL == 0 { + cfg.Minio.SessionTTL = 48 + } + return &cfg, nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 38af357..0be2989 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -254,3 +254,135 @@ log: t.Errorf("Log.FilePath = %q, want %q", cfg.Log.FilePath, "/var/log/app.log") } } + +func TestLoadWithMinioConfig(t *testing.T) { + content := []byte(`server_port: "9090" +slurm_api_url: "http://slurm.example.com:6820" +slurm_user_name: "admin" +slurm_jwt_key_path: "/etc/slurm/jwt.key" +mysql_dsn: "user:pass@tcp(10.0.0.1:3306)/testdb?parseTime=true" +minio: + endpoint: "minio.example.com:9000" + access_key: "myaccesskey" + secret_key: "mysecretkey" + bucket: "test-bucket" + use_ssl: true + chunk_size: 33554432 + max_file_size: 107374182400 + min_chunk_size: 10485760 + session_ttl: 24 +`) + + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, content, 0644); err != nil { + t.Fatalf("write temp config: %v", err) + } + + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + + if cfg.Minio.Endpoint != "minio.example.com:9000" { + t.Errorf("Minio.Endpoint = %q, want %q", cfg.Minio.Endpoint, "minio.example.com:9000") + } + if cfg.Minio.AccessKey != "myaccesskey" { + t.Errorf("Minio.AccessKey = %q, want %q", cfg.Minio.AccessKey, "myaccesskey") + } + if cfg.Minio.SecretKey != "mysecretkey" { + t.Errorf("Minio.SecretKey = %q, want %q", cfg.Minio.SecretKey, "mysecretkey") + } + if cfg.Minio.Bucket != "test-bucket" { + t.Errorf("Minio.Bucket = %q, want %q", cfg.Minio.Bucket, "test-bucket") + } + if cfg.Minio.UseSSL != true { + t.Errorf("Minio.UseSSL = %v, want %v", cfg.Minio.UseSSL, true) + } + if cfg.Minio.ChunkSize != 33554432 { + t.Errorf("Minio.ChunkSize = %d, want %d", cfg.Minio.ChunkSize, 33554432) + } + if cfg.Minio.MaxFileSize != 107374182400 { + t.Errorf("Minio.MaxFileSize = %d, want %d", cfg.Minio.MaxFileSize, 107374182400) + } + if cfg.Minio.MinChunkSize != 10485760 { + t.Errorf("Minio.MinChunkSize = %d, want %d", cfg.Minio.MinChunkSize, 10485760) + } + if cfg.Minio.SessionTTL != 24 { + t.Errorf("Minio.SessionTTL = %d, want %d", cfg.Minio.SessionTTL, 24) + } +} + +func TestLoadWithoutMinioConfig(t *testing.T) { + content := []byte(`server_port: "8080" +slurm_api_url: "http://localhost:6820" +slurm_user_name: "root" +slurm_jwt_key_path: "/etc/slurm/jwt_hs256.key" +mysql_dsn: "root:@tcp(127.0.0.1:3306)/hpc_platform?parseTime=true" +`) + + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, content, 0644); err != nil { + t.Fatalf("write temp config: %v", err) + } + + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + + if cfg.Minio.Endpoint != "" { + t.Errorf("Minio.Endpoint = %q, want empty string", cfg.Minio.Endpoint) + } + if cfg.Minio.AccessKey != "" { + t.Errorf("Minio.AccessKey = %q, want empty string", cfg.Minio.AccessKey) + } + if cfg.Minio.SecretKey != "" { + t.Errorf("Minio.SecretKey = %q, want empty string", cfg.Minio.SecretKey) + } + if cfg.Minio.Bucket != "" { + t.Errorf("Minio.Bucket = %q, want empty string", cfg.Minio.Bucket) + } + if cfg.Minio.UseSSL != false { + t.Errorf("Minio.UseSSL = %v, want false", cfg.Minio.UseSSL) + } +} + +func TestLoadMinioDefaults(t *testing.T) { + content := []byte(`server_port: "8080" +slurm_api_url: "http://localhost:6820" +slurm_user_name: "root" +slurm_jwt_key_path: "/etc/slurm/jwt_hs256.key" +mysql_dsn: "root:@tcp(127.0.0.1:3306)/hpc_platform?parseTime=true" +minio: + endpoint: "localhost:9000" + access_key: "minioadmin" + secret_key: "minioadmin" + bucket: "uploads" +`) + + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, content, 0644); err != nil { + t.Fatalf("write temp config: %v", err) + } + + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + + if cfg.Minio.ChunkSize != 16<<20 { + t.Errorf("Minio.ChunkSize = %d, want %d", cfg.Minio.ChunkSize, 16<<20) + } + if cfg.Minio.MaxFileSize != 50<<30 { + t.Errorf("Minio.MaxFileSize = %d, want %d", cfg.Minio.MaxFileSize, 50<<30) + } + if cfg.Minio.MinChunkSize != 5<<20 { + t.Errorf("Minio.MinChunkSize = %d, want %d", cfg.Minio.MinChunkSize, 5<<20) + } + if cfg.Minio.SessionTTL != 48 { + t.Errorf("Minio.SessionTTL = %d, want %d", cfg.Minio.SessionTTL, 48) + } +}