huanghaosheng/VLoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Stabilize docker service startup

Browse Source
This commit is contained in:
leonincs
2026-04-26 15:49:16 +08:00
parent 624da23a1c
commit a45a64c1ff
3 changed files with 210 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
backend/internal/config/loadconfig.go
View File

@@ -1,17 +1,19 @@
package config package config
import ( import (
"errors"
"fmt" "fmt"
"os" "os"
"errors" "strconv"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
type Config struct { type Config struct {
Server ServerConfig `yaml:"server"` Server ServerConfig `yaml:"server"`
Database DatabaseConfig `yaml:"database"` Database DatabaseConfig `yaml:"database"`
Redis RedisConfig `yaml:"redis"` Redis RedisConfig `yaml:"redis"`
RabbitMQ RabbitMQConfig `yaml:"rabbitmq"` RabbitMQ RabbitMQConfig `yaml:"rabbitmq"`
ObservabilityConfig ObservabilityConfig `yaml:"observability"` ObservabilityConfig ObservabilityConfig `yaml:"observability"`
} }
@@ -45,10 +47,11 @@ type ObservabilityConfig struct {
Pprof PprofConfig `yaml:"pprof"` Pprof PprofConfig `yaml:"pprof"`
} }
type PprofConfig struct { type PprofConfig struct {
Enabled bool `yaml:"enabled"` Enabled bool `yaml:"enabled"`
ApiAddr string `yaml:"api_addr"` ApiAddr string `yaml:"api_addr"`
WorkerAddr string `yaml:"worker_addr"` WorkerAddr string `yaml:"worker_addr"`
} }
func Load(filename string) (Config, error) { func Load(filename string) (Config, error) {
data, err := os.ReadFile(filename) data, err := os.ReadFile(filename)
if err != nil { if err != nil {
@@ -60,9 +63,71 @@ func Load(filename string) (Config, error) {
return Config{}, fmt.Errorf("parse config %s: %w", filename, err) return Config{}, fmt.Errorf("parse config %s: %w", filename, err)
} }
ApplyEnvOverrides(&cfg)
return cfg, nil return cfg, nil
} }
func ApplyEnvOverrides(cfg *Config) {
if cfg == nil {
return
}
if v := os.Getenv("SERVER_PORT"); v != "" {
if port, err := strconv.Atoi(v); err == nil {
cfg.Server.Port = port
}
}
if v := os.Getenv("MYSQL_HOST"); v != "" {
cfg.Database.Host = v
}
if v := os.Getenv("MYSQL_PORT"); v != "" {
if port, err := strconv.Atoi(v); err == nil {
cfg.Database.Port = port
}
}
if v := os.Getenv("MYSQL_USER"); v != "" {
cfg.Database.User = v
}
if v := os.Getenv("MYSQL_ROOT_PASSWORD"); v != "" {
cfg.Database.Password = v
}
if v := os.Getenv("MYSQL_PASSWORD"); v != "" {
cfg.Database.Password = v
}
if v := os.Getenv("MYSQL_DATABASE"); v != "" {
cfg.Database.DBName = v
}
if v := os.Getenv("REDIS_HOST"); v != "" {
cfg.Redis.Host = v
}
if v := os.Getenv("REDIS_PORT"); v != "" {
if port, err := strconv.Atoi(v); err == nil {
cfg.Redis.Port = port
}
}
if v := os.Getenv("REDIS_PASSWORD"); v != "" {
cfg.Redis.Password = v
}
if v := os.Getenv("REDIS_DB"); v != "" {
if db, err := strconv.Atoi(v); err == nil {
cfg.Redis.DB = db
}
}
if v := os.Getenv("RABBITMQ_HOST"); v != "" {
cfg.RabbitMQ.Host = v
}
if v := os.Getenv("RABBITMQ_PORT"); v != "" {
if port, err := strconv.Atoi(v); err == nil {
cfg.RabbitMQ.Port = port
}
}
if v := os.Getenv("RABBITMQ_USER"); v != "" {
cfg.RabbitMQ.Username = v
}
if v := os.Getenv("RABBITMQ_PASS"); v != "" {
cfg.RabbitMQ.Password = v
}
}
// bool用来表示是否使用了默认配置，true表示使用了默认配置 // bool用来表示是否使用了默认配置，true表示使用了默认配置
func LoadLocalDev(filename string) (Config, bool, error) { func LoadLocalDev(filename string) (Config, bool, error) {
cfg, err := Load(filename) cfg, err := Load(filename)
@@ -76,14 +141,14 @@ func LoadLocalDev(filename string) (Config, bool, error) {
} }
func DefaultLocalConfig() Config { func DefaultLocalConfig() Config {
return Config{ cfg := Config{
Server: ServerConfig{ Server: ServerConfig{
Port: 8080, Port: 8080,
}, },
Database: DatabaseConfig{ Database: DatabaseConfig{
Host: "localhost", Host: "localhost",
Port: 3306, Port: 3306,
User: "root", User: "root",
Password: "123456", Password: "123456",
DBName: "feedsystem", DBName: "feedsystem",
}, },
@@ -101,10 +166,12 @@ func DefaultLocalConfig() Config {
}, },
ObservabilityConfig: ObservabilityConfig{ ObservabilityConfig: ObservabilityConfig{
Pprof: PprofConfig{ Pprof: PprofConfig{
Enabled: true, Enabled: true,
ApiAddr: "localhost:6060", ApiAddr: "localhost:6060",
WorkerAddr: "localhost:6061", WorkerAddr: "localhost:6061",
}, },
}, },
} }
} ApplyEnvOverrides(&cfg)
return cfg
}

200
backend/internal/worker/outboxworker.go
View File

@@ -1,93 +1,107 @@
package worker package worker
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"feedsystem_video_go/internal/middleware/rabbitmq" "feedsystem_video_go/internal/middleware/rabbitmq"
"feedsystem_video_go/internal/middleware/redis" "feedsystem_video_go/internal/middleware/redis"
"feedsystem_video_go/internal/video" "feedsystem_video_go/internal/video"
"fmt" "fmt"
"log" "log"
"time" "time"
oredis "github.com/redis/go-redis/v9" oredis "github.com/redis/go-redis/v9"
"gorm.io/gorm" "gorm.io/gorm"
) )
func StartOutboxPoller(db *gorm.DB, tmq *rabbitmq.TimelineMQ) { func StartOutboxPoller(db *gorm.DB, tmq *rabbitmq.TimelineMQ) {
go func() { if db == nil || tmq == nil || tmq.RabbitMQ == nil || tmq.Ch == nil {
for { log.Printf("Outbox poller disabled: timeline mq is not initialized")
var messages []video.OutboxMsg return
}
err := db.Where("status = ?", "pending").Order("create_time ASC").Limit(100).Find(&messages).Error
go func() {
if err != nil || len(messages) == 0 { for {
time.Sleep(1 * time.Second) var messages []video.OutboxMsg
continue
} err := db.Where("status = ?", "pending").Order("create_time ASC").Limit(100).Find(&messages).Error
for _, msg := range messages { if err != nil || len(messages) == 0 {
err := tmq.PublishVideo(context.Background(), msg.VideoID, msg.CreateTime) time.Sleep(1 * time.Second)
continue
if err == nil { }
db.Delete(&msg)
} else { for _, msg := range messages {
log.Printf("投递MQ失败: VideoID: %d, err: %v", msg.VideoID, err) err := tmq.PublishVideo(context.Background(), msg.VideoID, msg.CreateTime)
}
} if err == nil {
} db.Delete(&msg)
}() } else {
} log.Printf("投递MQ失败: VideoID: %d, err: %v", msg.VideoID, err)
}
func StartConsumer(tmq *rabbitmq.TimelineMQ, queueName string, redisClient *redis.Client) { }
msgs, err := tmq.Ch.Consume( }
queueName, }()
"", }
false,
false, func StartConsumer(tmq *rabbitmq.TimelineMQ, queueName string, redisClient *redis.Client) {
false, if tmq == nil || tmq.RabbitMQ == nil || tmq.Ch == nil {
false, log.Printf("Timeline consumer disabled: timeline mq is not initialized")
nil, return
) }
if redisClient == nil {
if err != nil { log.Printf("Timeline consumer disabled: redis is not initialized")
log.Printf("注册消费失败") return
return }
}
msgs, err := tmq.Ch.Consume(
go func() { queueName,
for msg := range msgs { "",
var event rabbitmq.TimelineEvent false,
err := json.Unmarshal(msg.Body, &event) false,
false,
if err != nil { false,
log.Printf("反序列化失败") nil,
msg.Ack(false) )
continue
} if err != nil {
log.Printf("注册消费失败")
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) return
timelineKey := redisClient.Key("feed:global_timeline") }
err = redisClient.ZAdd(ctx, timelineKey, oredis.Z{
Score: float64(event.CreateTime), go func() {
Member: fmt.Sprintf("%d", event.VideoID), for msg := range msgs {
}) var event rabbitmq.TimelineEvent
err := json.Unmarshal(msg.Body, &event)
if err != nil {
log.Printf("写入Zset失败") if err != nil {
msg.Nack(false, true) log.Printf("反序列化失败")
cancel() msg.Ack(false)
continue continue
} }
err = redisClient.ZRemRangeByRank(ctx, timelineKey, 0, -1001) ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
timelineKey := redisClient.Key("feed:global_timeline")
if err != nil { err = redisClient.ZAdd(ctx, timelineKey, oredis.Z{
log.Printf("ZRem失败") Score: float64(event.CreateTime),
} Member: fmt.Sprintf("%d", event.VideoID),
})
msg.Ack(false)
cancel() if err != nil {
} log.Printf("写入Zset失败")
}() msg.Nack(false, true)
} cancel()
continue
}
err = redisClient.ZRemRangeByRank(ctx, timelineKey, 0, -1001)
if err != nil {
log.Printf("ZRem失败")
}
msg.Ack(false)
cancel()
}
}()
}

36
docker-compose.yml
View File

@@ -16,22 +16,24 @@ services:
- --default-authentication-plugin=mysql_native_password - --default-authentication-plugin=mysql_native_password
- --character-set-server=utf8mb4 - --character-set-server=utf8mb4
- --collation-server=utf8mb4_0900_ai_ci - --collation-server=utf8mb4_0900_ai_ci
healthcheck: healthcheck:
test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -p123456 --silent"] test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -p$${MYSQL_ROOT_PASSWORD} --silent"]
interval: 5s interval: 5s
timeout: 5s timeout: 5s
retries: 20 retries: 20
redis: redis:
image: redis:7-alpine image: redis:7-alpine
restart: always restart: always
command: ["redis-server", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD:-123456}"] environment:
REDIS_PASSWORD: ${REDIS_PASSWORD:-123456}
command: ["redis-server", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD:-123456}"]
ports: ports:
- "6379:6379" - "6379:6379"
volumes: volumes:
- redis_data:/data - redis_data:/data
healthcheck: healthcheck:
test: ["CMD", "redis-cli", "-a", "123456", "ping"] test: ["CMD-SHELL", "redis-cli -a \"$${REDIS_PASSWORD}\" ping"]
interval: 5s interval: 5s
timeout: 3s timeout: 3s
retries: 20 retries: 20
@@ -59,8 +61,13 @@ services:
dockerfile: backend/Dockerfile dockerfile: backend/Dockerfile
target: api target: api
restart: always restart: always
environment: environment:
JWT_SECRET: ${JWT_SECRET:-feedsystem-dev-secret-key} JWT_SECRET: ${JWT_SECRET:-feedsystem-dev-secret-key}
MYSQL_DATABASE: ${MYSQL_DATABASE:-feedsystem}
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-123456}
REDIS_PASSWORD: ${REDIS_PASSWORD:-123456}
RABBITMQ_USER: ${RABBITMQ_USER:-admin}
RABBITMQ_PASS: ${RABBITMQ_PASS:-password123}
ports: ports:
- "8080:8080" - "8080:8080"
volumes: volumes:
@@ -85,8 +92,13 @@ services:
dockerfile: backend/Dockerfile dockerfile: backend/Dockerfile
target: worker target: worker
restart: always restart: always
environment: environment:
JWT_SECRET: ${JWT_SECRET:-feedsystem-dev-secret-key} JWT_SECRET: ${JWT_SECRET:-feedsystem-dev-secret-key}
MYSQL_DATABASE: ${MYSQL_DATABASE:-feedsystem}
MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-123456}
REDIS_PASSWORD: ${REDIS_PASSWORD:-123456}
RABBITMQ_USER: ${RABBITMQ_USER:-admin}
RABBITMQ_PASS: ${RABBITMQ_PASS:-password123}
volumes: volumes:
- ./backend/configs/config.docker.yaml:/app/configs/config.yaml:ro - ./backend/configs/config.docker.yaml:/app/configs/config.yaml:ro
depends_on: depends_on: