dailz/hpc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(config): add MinIO object storage configuration

Browse Source 
Add MinioConfig struct with connection, bucket, chunk size, and session TTL settings.

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
dailz
2026-04-15 09:22:18 +08:00
parent a65c8762af
commit 44895214d4
3 changed files with 177 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
internal/config/config.example.yaml
View File

@@ -15,3 +15,14 @@ log:
max_age: 30 # days to retain old log files
compress: true # gzip rotated log files
gorm_level: "warn" # GORM SQL log level: silent, error, warn, info
minio:
endpoint: "http://fnos.dailz.cn:15001" # MinIO server address
access_key: "3dgDu9ncwflLoRQW2OeP" # access key
secret_key: "g2GLBNTPxJ9sdFwh37jtfilRSacEO5yQepMkDrnV" # secret key
bucket: "test" # bucket name
use_ssl: false # use TLS connection
chunk_size: 16777216 # upload chunk size in bytes (default: 16MB)
max_file_size: 53687091200 # max file size in bytes (default: 50GB)
min_chunk_size: 5242880 # minimum chunk size in bytes (default: 5MB)
session_ttl: 48 # session TTL in hours (default: 48)

41
internal/config/config.go
View File

@@ -20,15 +20,29 @@ type LogConfig struct {
GormLevel string `yaml:"gorm_level"` // GORM SQL log level (default: warn)
}
// MinioConfig holds MinIO object storage configuration values.
type MinioConfig struct {
Endpoint string `yaml:"endpoint"` // MinIO server address
AccessKey string `yaml:"access_key"` // access key
SecretKey string `yaml:"secret_key"` // secret key
Bucket string `yaml:"bucket"` // bucket name
UseSSL bool `yaml:"use_ssl"` // use TLS connection
ChunkSize int64 `yaml:"chunk_size"` // upload chunk size in bytes (default: 16MB)
MaxFileSize int64 `yaml:"max_file_size"` // max file size in bytes (default: 50GB)
MinChunkSize int64 `yaml:"min_chunk_size"` // minimum chunk size in bytes (default: 5MB)
SessionTTL int `yaml:"session_ttl"` // session TTL in hours (default: 48)
}
// Config holds all application configuration values.
type Config struct {
ServerPort string `yaml:"server_port"`
SlurmAPIURL string `yaml:"slurm_api_url"`
SlurmUserName string `yaml:"slurm_user_name"`
SlurmJWTKeyPath string `yaml:"slurm_jwt_key_path"`
MySQLDSN string `yaml:"mysql_dsn"`
WorkDirBase string `yaml:"work_dir_base"` // base directory for job work dirs
Log LogConfig `yaml:"log"`
ServerPort string `yaml:"server_port"`
SlurmAPIURL string `yaml:"slurm_api_url"`
SlurmUserName string `yaml:"slurm_user_name"`
SlurmJWTKeyPath string `yaml:"slurm_jwt_key_path"`
MySQLDSN string `yaml:"mysql_dsn"`
WorkDirBase string `yaml:"work_dir_base"` // base directory for job work dirs
Log LogConfig `yaml:"log"`
Minio MinioConfig `yaml:"minio"`
}
// Load reads a YAML configuration file and returns a parsed Config.
@@ -48,5 +62,18 @@ func Load(path string) (*Config, error) {
return nil, fmt.Errorf("parse config file %s: %w", path, err)
}
if cfg.Minio.ChunkSize == 0 {
cfg.Minio.ChunkSize = 16 << 20 // 16MB
}
if cfg.Minio.MaxFileSize == 0 {
cfg.Minio.MaxFileSize = 50 << 30 // 50GB
}
if cfg.Minio.MinChunkSize == 0 {
cfg.Minio.MinChunkSize = 5 << 20 // 5MB
}
if cfg.Minio.SessionTTL == 0 {
cfg.Minio.SessionTTL = 48
}
return &cfg, nil
}

132
internal/config/config_test.go
View File

@@ -254,3 +254,135 @@ log:
t.Errorf("Log.FilePath = %q, want %q", cfg.Log.FilePath, "/var/log/app.log")
}
}
func TestLoadWithMinioConfig(t *testing.T) {
content := []byte(`server_port: "9090"
slurm_api_url: "http://slurm.example.com:6820"
slurm_user_name: "admin"
slurm_jwt_key_path: "/etc/slurm/jwt.key"
mysql_dsn: "user:pass@tcp(10.0.0.1:3306)/testdb?parseTime=true"
minio:
endpoint: "minio.example.com:9000"
access_key: "myaccesskey"
secret_key: "mysecretkey"
bucket: "test-bucket"
use_ssl: true
chunk_size: 33554432
max_file_size: 107374182400
min_chunk_size: 10485760
session_ttl: 24
`)
dir := t.TempDir()
path := filepath.Join(dir, "config.yaml")
if err := os.WriteFile(path, content, 0644); err != nil {
t.Fatalf("write temp config: %v", err)
}
cfg, err := Load(path)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Minio.Endpoint != "minio.example.com:9000" {
t.Errorf("Minio.Endpoint = %q, want %q", cfg.Minio.Endpoint, "minio.example.com:9000")
}
if cfg.Minio.AccessKey != "myaccesskey" {
t.Errorf("Minio.AccessKey = %q, want %q", cfg.Minio.AccessKey, "myaccesskey")
}
if cfg.Minio.SecretKey != "mysecretkey" {
t.Errorf("Minio.SecretKey = %q, want %q", cfg.Minio.SecretKey, "mysecretkey")
}
if cfg.Minio.Bucket != "test-bucket" {
t.Errorf("Minio.Bucket = %q, want %q", cfg.Minio.Bucket, "test-bucket")
}
if cfg.Minio.UseSSL != true {
t.Errorf("Minio.UseSSL = %v, want %v", cfg.Minio.UseSSL, true)
}
if cfg.Minio.ChunkSize != 33554432 {
t.Errorf("Minio.ChunkSize = %d, want %d", cfg.Minio.ChunkSize, 33554432)
}
if cfg.Minio.MaxFileSize != 107374182400 {
t.Errorf("Minio.MaxFileSize = %d, want %d", cfg.Minio.MaxFileSize, 107374182400)
}
if cfg.Minio.MinChunkSize != 10485760 {
t.Errorf("Minio.MinChunkSize = %d, want %d", cfg.Minio.MinChunkSize, 10485760)
}
if cfg.Minio.SessionTTL != 24 {
t.Errorf("Minio.SessionTTL = %d, want %d", cfg.Minio.SessionTTL, 24)
}
}
func TestLoadWithoutMinioConfig(t *testing.T) {
content := []byte(`server_port: "8080"
slurm_api_url: "http://localhost:6820"
slurm_user_name: "root"
slurm_jwt_key_path: "/etc/slurm/jwt_hs256.key"
mysql_dsn: "root:@tcp(127.0.0.1:3306)/hpc_platform?parseTime=true"
`)
dir := t.TempDir()
path := filepath.Join(dir, "config.yaml")
if err := os.WriteFile(path, content, 0644); err != nil {
t.Fatalf("write temp config: %v", err)
}
cfg, err := Load(path)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Minio.Endpoint != "" {
t.Errorf("Minio.Endpoint = %q, want empty string", cfg.Minio.Endpoint)
}
if cfg.Minio.AccessKey != "" {
t.Errorf("Minio.AccessKey = %q, want empty string", cfg.Minio.AccessKey)
}
if cfg.Minio.SecretKey != "" {
t.Errorf("Minio.SecretKey = %q, want empty string", cfg.Minio.SecretKey)
}
if cfg.Minio.Bucket != "" {
t.Errorf("Minio.Bucket = %q, want empty string", cfg.Minio.Bucket)
}
if cfg.Minio.UseSSL != false {
t.Errorf("Minio.UseSSL = %v, want false", cfg.Minio.UseSSL)
}
}
func TestLoadMinioDefaults(t *testing.T) {
content := []byte(`server_port: "8080"
slurm_api_url: "http://localhost:6820"
slurm_user_name: "root"
slurm_jwt_key_path: "/etc/slurm/jwt_hs256.key"
mysql_dsn: "root:@tcp(127.0.0.1:3306)/hpc_platform?parseTime=true"
minio:
endpoint: "localhost:9000"
access_key: "minioadmin"
secret_key: "minioadmin"
bucket: "uploads"
`)
dir := t.TempDir()
path := filepath.Join(dir, "config.yaml")
if err := os.WriteFile(path, content, 0644); err != nil {
t.Fatalf("write temp config: %v", err)
}
cfg, err := Load(path)
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if cfg.Minio.ChunkSize != 16<<20 {
t.Errorf("Minio.ChunkSize = %d, want %d", cfg.Minio.ChunkSize, 16<<20)
}
if cfg.Minio.MaxFileSize != 50<<30 {
t.Errorf("Minio.MaxFileSize = %d, want %d", cfg.Minio.MaxFileSize, 50<<30)
}
if cfg.Minio.MinChunkSize != 5<<20 {
t.Errorf("Minio.MinChunkSize = %d, want %d", cfg.Minio.MinChunkSize, 5<<20)
}
if cfg.Minio.SessionTTL != 48 {
t.Errorf("Minio.SessionTTL = %d, want %d", cfg.Minio.SessionTTL, 48)
}
}