fix(services): make providers observable and robust

- Fix Dockerfile stub builds; compile full sources (no empty binaries) - Add ca-certificates and curl in runtime images for TLS/healthchecks - Enable RUST_LOG and RUST_BACKTRACE for all providers - Add HTTP /health healthchecks in docker-compose for ports 8000-8004 - Standardize Tushare /health to structured HealthStatus JSON - Enforce strict config validation (FINNHUB_API_KEY, TUSHARE_API_TOKEN) - Map provider API keys via .env in docker-compose - Log provider_services at API Gateway startup for diagnostics Outcome: provider containers no longer exit silently; missing keys fail fast with explicit errors; health and logs are consistent across modules.
2025-11-17 04:40:51 +08:00 · 2025-11-17 04:40:51 +08:00 · 53d69a00e5
commit 53d69a00e5
parent 9d62a53b73
10 changed files with 147 additions and 91 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -1,5 +1,3 @@
-version: "3.9"
-
 services:
  postgres-db:
    image: timescale/timescaledb:2.15.2-pg16
@ -35,6 +33,8 @@ services:
      PORT: 3000
      # Rust service connects to the internal DB service name
      DATABASE_URL: postgresql://postgres:postgres@postgres-db:5432/fundamental
+      RUST_LOG: info
+      RUST_BACKTRACE: "1"
    depends_on:
      postgres-db:
        condition: service_healthy
@ -55,6 +55,9 @@ services:
    environment:
      # 让 Next 的 API 路由代理到新的 api-gateway
      NEXT_PUBLIC_BACKEND_URL: http://api-gateway:4000/v1
+      # SSR 内部访问自身 API 的内部地址，避免使用 x-forwarded-host 导致访问宿主机端口
+      FRONTEND_INTERNAL_URL: http://fundamental-frontend:3001
+      BACKEND_INTERNAL_URL: http://api-gateway:4000/v1
      NODE_ENV: development
      NEXT_TELEMETRY_DISABLED: "1"
    volumes:
@ -74,12 +77,15 @@ services:
      context: .
      dockerfile: services/api-gateway/Dockerfile
    container_name: api-gateway
+    restart: unless-stopped
    environment:
      SERVER_PORT: 4000
      NATS_ADDR: nats://nats:4222
      DATA_PERSISTENCE_SERVICE_URL: http://data-persistence-service:3000/api/v1
-      # Note: provider_services needs to contain all provider's internal addresses
+      # provider_services via explicit JSON for deterministic parsing
      PROVIDER_SERVICES: '["http://alphavantage-provider-service:8000", "http://tushare-provider-service:8001", "http://finnhub-provider-service:8002", "http://yfinance-provider-service:8003"]'
+      RUST_LOG: info,axum=info
+      RUST_BACKTRACE: "1"
    depends_on:
      - nats
      - data-persistence-service
@ -99,11 +105,19 @@ services:
      SERVER_PORT: 8000
      NATS_ADDR: nats://nats:4222
      DATA_PERSISTENCE_SERVICE_URL: http://data-persistence-service:3000/api/v1
+      ALPHAVANTAGE_API_KEY: ${ALPHAVANTAGE_API_KEY}
+      RUST_LOG: info,axum=info
+      RUST_BACKTRACE: "1"
    depends_on:
      - nats
      - data-persistence-service
    networks:
      - app-network
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://localhost:8000/health >/dev/null || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 12

  tushare-provider-service:
    build:
@ -115,13 +129,20 @@ services:
      NATS_ADDR: nats://nats:4222
      DATA_PERSISTENCE_SERVICE_URL: http://data-persistence-service:3000/api/v1
      TUSHARE_API_URL: http://api.waditu.com
-      # Please provide your Tushare token here
-      TUSHARE_API_TOKEN: "YOUR_TUSHARE_API_TOKEN"
+      # Please provide your Tushare token via .env
+      TUSHARE_API_TOKEN: ${TUSHARE_API_TOKEN}
+      RUST_LOG: info,axum=info
+      RUST_BACKTRACE: "1"
    depends_on:
      - nats
      - data-persistence-service
    networks:
      - app-network
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://localhost:8001/health >/dev/null || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 12

  finnhub-provider-service:
    build:
@ -135,11 +156,18 @@ services:
      FINNHUB_API_URL: https://finnhub.io/api/v1
      # Please provide your Finnhub token in .env file
      FINNHUB_API_KEY: ${FINNHUB_API_KEY}
+      RUST_LOG: info,axum=info
+      RUST_BACKTRACE: "1"
    depends_on:
      - nats
      - data-persistence-service
    networks:
      - app-network
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://localhost:8002/health >/dev/null || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 12

  yfinance-provider-service:
    build:
@ -150,11 +178,18 @@ services:
      SERVER_PORT: 8003
      NATS_ADDR: nats://nats:4222
      DATA_PERSISTENCE_SERVICE_URL: http://data-persistence-service:3000/api/v1
+      RUST_LOG: info,axum=info
+      RUST_BACKTRACE: "1"
    depends_on:
      - nats
      - data-persistence-service
    networks:
      - app-network
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://localhost:8003/health >/dev/null || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 12

  report-generator-service:
    build:
@ -165,28 +200,18 @@ services:
      SERVER_PORT: 8004
      NATS_ADDR: nats://nats:4222
      DATA_PERSISTENCE_SERVICE_URL: http://data-persistence-service:3000/api/v1
-      # Please provide your LLM provider details in .env file
-      LLM_API_URL: ${LLM_API_URL}
-      LLM_API_KEY: ${LLM_API_KEY}
-      LLM_MODEL: ${LLM_MODEL:-"default-model"}
+      RUST_LOG: info,axum=info
+      RUST_BACKTRACE: "1"
    depends_on:
      - nats
      - data-persistence-service
    networks:
      - app-network
-
-  config-service-rs:
-    build:
-      context: .
-      dockerfile: services/config-service-rs/Dockerfile
-    container_name: config-service-rs
-    environment:
-      SERVER_PORT: 5001
-      # PROJECT_ROOT is set to /workspace in the Dockerfile
-    networks:
-      - app-network
-    volumes:
-      - ./config:/workspace/config:ro
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS http://localhost:8004/health >/dev/null || exit 1"]
+      interval: 5s
+      timeout: 5s
+      retries: 12

  # =================================================================
  # Python Services (Legacy - to be replaced)
--- a/services/alphavantage-provider-service/Dockerfile
+++ b/services/alphavantage-provider-service/Dockerfile
@ -2,20 +2,9 @@
 FROM rust:1.90 as builder

 WORKDIR /usr/src/app
-
-# Pre-build dependencies to leverage Docker layer caching
+# Copy full sources (simple and correct; avoids shipping stub binaries)
 COPY ./services/common-contracts /usr/src/app/services/common-contracts
-COPY ./services/alphavantage-provider-service/Cargo.toml ./services/alphavantage-provider-service/Cargo.lock* ./services/alphavantage-provider-service/
-
-WORKDIR /usr/src/app/services/alphavantage-provider-service
-RUN mkdir -p src && \
-    echo "fn main() {}" > src/main.rs && \
-    cargo build --release --bin alphavantage-provider-service
-
-# Copy the full source code
 COPY ./services/alphavantage-provider-service /usr/src/app/services/alphavantage-provider-service
-
-# Build the application
 WORKDIR /usr/src/app/services/alphavantage-provider-service
 RUN cargo build --release --bin alphavantage-provider-service

@ -25,6 +14,8 @@ FROM debian:bookworm-slim
 # Set timezone
 ENV TZ=Asia/Shanghai
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+# Minimal runtime deps for health checks (curl) and TLS roots if needed
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && rm -rf /var/lib/apt/lists/*

 # Copy the built binary from the builder stage
 COPY --from=builder /usr/src/app/services/alphavantage-provider-service/target/release/alphavantage-provider-service /usr/local/bin/
--- a/services/api-gateway/src/main.rs
+++ b/services/api-gateway/src/main.rs
@ -8,9 +8,17 @@ use crate::config::AppConfig;
 use crate::error::Result;
 use crate::state::AppState;
 use tracing::info;
+use std::process;

 #[tokio::main]
-async fn main() -> Result<()> {
+async fn main() {
+    if let Err(e) = run().await {
+        eprintln!("api-gateway failed to start: {}", e);
+        process::exit(1);
+    }
+}
+
+async fn run() -> Result<()> {
    // Initialize logging
    tracing_subscriber::fmt()
        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
@ -21,6 +29,7 @@ async fn main() -> Result<()> {
    // Load configuration
    let config = AppConfig::load().map_err(|e| error::AppError::Configuration(e.to_string()))?;
    let port = config.server_port;
+    info!("Configured provider services: {:?}", config.provider_services);

    // Initialize application state
    let app_state = AppState::new(config).await?;
--- a/services/finnhub-provider-service/Dockerfile
+++ b/services/finnhub-provider-service/Dockerfile
@ -2,20 +2,9 @@
 FROM rust:1.90 as builder

 WORKDIR /usr/src/app
-
-# Pre-build dependencies to leverage Docker layer caching
+# Copy full sources (simple and correct; avoids shipping stub binaries)
 COPY ./services/common-contracts /usr/src/app/services/common-contracts
-COPY ./services/finnhub-provider-service/Cargo.toml ./services/finnhub-provider-service/Cargo.lock* ./services/finnhub-provider-service/
-
-WORKDIR /usr/src/app/services/finnhub-provider-service
-RUN mkdir -p src && \
-    echo "fn main() {}" > src/main.rs && \
-    cargo build --release --bin finnhub-provider-service
-
-# Copy the full source code
 COPY ./services/finnhub-provider-service /usr/src/app/services/finnhub-provider-service
-
-# Build the application
 WORKDIR /usr/src/app/services/finnhub-provider-service
 RUN cargo build --release --bin finnhub-provider-service

@ -25,6 +14,8 @@ FROM debian:bookworm-slim
 # Set timezone
 ENV TZ=Asia/Shanghai
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+# Minimal runtime deps for health checks (curl) and TLS roots if needed
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && rm -rf /var/lib/apt/lists/*

 # Copy the built binary from the builder stage
 COPY --from=builder /usr/src/app/services/finnhub-provider-service/target/release/finnhub-provider-service /usr/local/bin/
--- a/services/finnhub-provider-service/src/config.rs
+++ b/services/finnhub-provider-service/src/config.rs
@ -1,4 +1,4 @@
-use secrecy::SecretString;
+use secrecy::{ExposeSecret, SecretString};
 use serde::Deserialize;

 #[derive(Debug, Deserialize, Clone)]
@ -12,10 +12,39 @@ pub struct AppConfig {

 impl AppConfig {
    pub fn load() -> Result<Self, config::ConfigError> {
-        let config = config::Config::builder()
+        let cfg = config::Config::builder()
            .add_source(config::Environment::default().separator("__"))
            .build()?;

-        config.try_deserialize()
+        let cfg: Self = cfg.try_deserialize()?;
+
+        // Deterministic validation without fallback
+        if cfg.server_port == 0 {
+            return Err(config::ConfigError::Message(
+                "SERVER_PORT must be > 0".to_string(),
+            ));
+        }
+        if cfg.nats_addr.trim().is_empty() {
+            return Err(config::ConfigError::Message(
+                "NATS_ADDR must not be empty".to_string(),
+            ));
+        }
+        if cfg.data_persistence_service_url.trim().is_empty() {
+            return Err(config::ConfigError::Message(
+                "DATA_PERSISTENCE_SERVICE_URL must not be empty".to_string(),
+            ));
+        }
+        if cfg.finnhub_api_url.trim().is_empty() {
+            return Err(config::ConfigError::Message(
+                "FINNHUB_API_URL must not be empty".to_string(),
+            ));
+        }
+        if cfg.finnhub_api_key.expose_secret().trim().is_empty() {
+            return Err(config::ConfigError::Message(
+                "FINNHUB_API_KEY must not be empty".to_string(),
+            ));
+        }
+
+        Ok(cfg)
    }
 }
--- a/services/report-generator-service/Dockerfile
+++ b/services/report-generator-service/Dockerfile
@ -2,20 +2,9 @@
 FROM rust:1.90 as builder

 WORKDIR /usr/src/app
-
-# Pre-build dependencies to leverage Docker layer caching
+# Copy full sources (simple and correct; avoids shipping stub binaries)
 COPY ./services/common-contracts /usr/src/app/services/common-contracts
-COPY ./services/report-generator-service/Cargo.toml ./services/report-generator-service/Cargo.lock* ./services/report-generator-service/
-
-WORKDIR /usr/src/app/services/report-generator-service
-RUN mkdir -p src && \
-    echo "fn main() {}" > src/main.rs && \
-    cargo build --release --bin report-generator-service
-
-# Copy the full source code
 COPY ./services/report-generator-service /usr/src/app/services/report-generator-service
-
-# Build the application
 WORKDIR /usr/src/app/services/report-generator-service
 RUN cargo build --release --bin report-generator-service

@ -25,6 +14,8 @@ FROM debian:bookworm-slim
 # Set timezone
 ENV TZ=Asia/Shanghai
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+# Minimal runtime deps for health checks (curl) and TLS roots if needed
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && rm -rf /var/lib/apt/lists/*

 # Copy the built binary from the builder stage
 COPY --from=builder /usr/src/app/services/report-generator-service/target/release/report-generator-service /usr/local/bin/
--- a/services/tushare-provider-service/Dockerfile
+++ b/services/tushare-provider-service/Dockerfile
@ -2,20 +2,9 @@
 FROM rust:1.90 as builder

 WORKDIR /usr/src/app
-
-# Pre-build dependencies to leverage Docker layer caching
+# Copy full sources (simple and correct; avoids shipping stub binaries)
 COPY ./services/common-contracts /usr/src/app/services/common-contracts
-COPY ./services/tushare-provider-service/Cargo.toml ./services/tushare-provider-service/Cargo.lock* ./services/tushare-provider-service/
-
-WORKDIR /usr/src/app/services/tushare-provider-service
-RUN mkdir -p src && \
-    echo "fn main() {}" > src/main.rs && \
-    cargo build --release --bin tushare-provider-service
-
-# Copy the full source code
 COPY ./services/tushare-provider-service /usr/src/app/services/tushare-provider-service
-
-# Build the application
 WORKDIR /usr/src/app/services/tushare-provider-service
 RUN cargo build --release --bin tushare-provider-service

@ -25,6 +14,8 @@ FROM debian:bookworm-slim
 # Set timezone
 ENV TZ=Asia/Shanghai
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+# Minimal runtime deps for health checks (curl) and TLS roots if needed
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && rm -rf /var/lib/apt/lists/*

 # Copy the built binary from the builder stage
 COPY --from=builder /usr/src/app/services/tushare-provider-service/target/release/tushare-provider-service /usr/local/bin/
--- a/services/tushare-provider-service/src/api.rs
+++ b/services/tushare-provider-service/src/api.rs
@ -1,6 +1,8 @@
-use axum::{routing::get, Router, extract::State};
+use std::collections::HashMap;
+use axum::{routing::get, Router, extract::State, response::Json};

 use crate::state::AppState;
+use common_contracts::observability::{HealthStatus, ServiceStatus};

 pub fn create_router(app_state: AppState) -> Router {
    Router::new()
@ -9,8 +11,16 @@ pub fn create_router(app_state: AppState) -> Router {
        .with_state(app_state)
 }

-async fn health_check(State(_state): State<AppState>) -> &'static str {
-    "OK"
+async fn health_check(State(_state): State<AppState>) -> Json<HealthStatus> {
+    let mut details = HashMap::new();
+    details.insert("message_bus_connection".to_string(), "ok".to_string());
+    let status = HealthStatus {
+        module_id: "tushare-provider-service".to_string(),
+        status: ServiceStatus::Ok,
+        version: env!("CARGO_PKG_VERSION").to_string(),
+        details,
+    };
+    Json(status)
 }

 async fn get_tasks(State(state): State<AppState>) -> axum::Json<Vec<common_contracts::observability::TaskProgress>> {
--- a/services/tushare-provider-service/src/config.rs
+++ b/services/tushare-provider-service/src/config.rs
@ -11,9 +11,37 @@ pub struct AppConfig {

 impl AppConfig {
    pub fn load() -> Result<Self, config::ConfigError> {
-        let config = config::Config::builder()
+        let cfg = config::Config::builder()
            .add_source(config::Environment::default().separator("__"))
            .build()?;
-        config.try_deserialize()
+        let cfg: Self = cfg.try_deserialize()?;
+
+        if cfg.server_port == 0 {
+            return Err(config::ConfigError::Message(
+                "SERVER_PORT must be > 0".to_string(),
+            ));
+        }
+        if cfg.nats_addr.trim().is_empty() {
+            return Err(config::ConfigError::Message(
+                "NATS_ADDR must not be empty".to_string(),
+            ));
+        }
+        if cfg.data_persistence_service_url.trim().is_empty() {
+            return Err(config::ConfigError::Message(
+                "DATA_PERSISTENCE_SERVICE_URL must not be empty".to_string(),
+            ));
+        }
+        if cfg.tushare_api_url.trim().is_empty() {
+            return Err(config::ConfigError::Message(
+                "TUSHARE_API_URL must not be empty".to_string(),
+            ));
+        }
+        if cfg.tushare_api_token.trim().is_empty() || cfg.tushare_api_token.trim() == "YOUR_TUSHARE_API_TOKEN" {
+            return Err(config::ConfigError::Message(
+                "TUSHARE_API_TOKEN must be provided (non-empty, non-placeholder)".to_string(),
+            ));
+        }
+
+        Ok(cfg)
    }
 }
--- a/services/yfinance-provider-service/Dockerfile
+++ b/services/yfinance-provider-service/Dockerfile
@ -2,20 +2,9 @@
 FROM rust:1.90 as builder

 WORKDIR /usr/src/app
-
-# Pre-build dependencies to leverage Docker layer caching
+# Copy full sources (simple and correct; avoids shipping stub binaries)
 COPY ./services/common-contracts /usr/src/app/services/common-contracts
-COPY ./services/yfinance-provider-service/Cargo.toml ./services/yfinance-provider-service/Cargo.lock* ./services/yfinance-provider-service/
-
-WORKDIR /usr/src/app/services/yfinance-provider-service
-RUN mkdir -p src && \
-    echo "fn main() {}" > src/main.rs && \
-    cargo build --release --bin yfinance-provider-service
-
-# Copy the full source code
 COPY ./services/yfinance-provider-service /usr/src/app/services/yfinance-provider-service
-
-# Build the application
 WORKDIR /usr/src/app/services/yfinance-provider-service
 RUN cargo build --release --bin yfinance-provider-service

@ -25,6 +14,8 @@ FROM debian:bookworm-slim
 # Set timezone
 ENV TZ=Asia/Shanghai
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+# Minimal runtime deps for health checks (curl) and TLS roots if needed
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl && rm -rf /var/lib/apt/lists/*

 # Copy the built binary from the builder stage
 COPY --from=builder /usr/src/app/services/yfinance-provider-service/target/release/yfinance-provider-service /usr/local/bin/