diff --git a/.dockerignore b/.dockerignore index 49cc4f4..ab48493 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,7 +21,10 @@ __pycache__ *.pyc # Large reference/resources not needed in images -ref/ +# ref/ is usually ignored, but we need service_kit_mirror for build context +# We use exclusion pattern (!) to allow specific subdirectories +ref/* +!ref/service_kit_mirror archive/ docs/ diff --git a/config/analysis-config.json b/config/analysis-config.json index 402b68e..ad7fd4b 100644 --- a/config/analysis-config.json +++ b/config/analysis-config.json @@ -8,19 +8,19 @@ "fundamental_analysis": { "name": "基本面分析", "model": "qwen-flash-2025-07-28", - "prompt_template": "# 角色\n你是一位专注于长期价值投资的顶级证券分析师，擅长从基本面出发，对公司进行深入、全面的分析。你的分析报告以客观、严谨、逻辑清晰、数据详实著称。\n# 任务\n为公司 {company_name} (股票代码: {ts_code}) 生成一份全面、专业、结构化的投资分析报告。\n# 输出要求\n直接开始：不要进行任何自我介绍或客套话，直接输出报告正文。\nMarkdown格式：使用清晰的多级Markdown标题（如 ## 和 ###）来组织报告结构。\n专业口吻：保持客观、中立、分析性的专业语调。\n信息缺失处理：如果某些信息在公开渠道无法获取，请明确指出“相关信息未公开披露”或类似说明。\n\n# 报告核心结构与分析要点\n一、 公司基本面分析 (Fundamental Analysis)\n1.1 护城河与核心竞争力\n公司通过何种独有优势（如品牌、技术、成本、网络效应、牌照等）获取超额利润？\n该护城河是在增强、维持还是在削弱？请提供论据。\n1.2 管理层与公司治理\n管理能力：管理层过往的战略决策和执行能力如何？是否有卓越的业界声誉？\n股东回报：管理层及大股东是否珍惜股权价值？（分析历史上的增持/减持行为、分红派息政策、是否存在损害小股东利益的体外资产等）\n激励与目标：公司的经营目标是长期主义还是短期化？管理层的激励机制（如股权激励、考核指标）是否与长期战略目标一致？\n1.3 企业文化与财务政策\n公司是否有独特且可观察到的企业文化？（例如：创新文化、成本控制文化等）\n公司的财务政策（如资本结构、现金流管理、投资策略）与同行业公司相比有何显著特点？是激进还是保守？\n1.4 发展历程与战略规划\n梳理公司发展史上的关键事件、重大业务转型或里程碑。\n公司是否有清晰的长期战略目标（未来5-10年）？计划成为一家什么样的企业？\n二、 业务与市场分析 (Business & Market Analysis)\n2.1 产品与客户价值\n公司为客户提供什么核心产品/服务？其核心价值点是什么？客户为何选择公司的产品而非竞争对手的？\n产品的更新迭代是颠覆性的还是渐进积累型的？分析产品历年的产量、价格及销量变化，并探讨其背后的驱动因素。\n2.2 市场需求与景气度\n客户所处行业的需求是趋势性的高增长，还是周期性波动？或是两者结合？当前处于何种阶段？\n目标客户群体的经营状况和现金流是否健康？\n2.3 议价能力与客户关系\n公司对下游客户的议价能力强弱如何？（结合应收账款周转天数、账龄结构、毛利率等数据进行佐证）\n公司与核心客户的关系是否稳定？客户对公司的评价如何（例如：客户忠诚度、满意度）？\n三、 竞争格局分析 (Competitive Landscape Analysis)\n3.1 竞争对手画像\n列出公司的主要竞争对手，并分析各自的优势与劣势。\n公司的竞争对手是在增多还是减少？行业进入壁垒是在增高还是降低？\n是否存在潜在的跨界竞争者？\n四、 供应链与外部关系 (Supply Chain & External Relations)\n4.1 供应链议价能力\n公司对上游供应商的议价能力如何？（结合应付账款周转天数、采购成本控制等数据进行佐证）\n核心供应商的经营是否稳定？供应链是否存在集中度过高的风险？\n4.2 金融机构关系与融资需求\n公司与金融机构的关系如何？融资渠道是否通畅？\n公司未来的发展是否依赖于大规模的债务或股权融资？\n五、 监管环境与政策风险 (Regulatory Environment & Policy Risks)\n公司所处行业是否存在重要的监管部门？主要的监管政策有哪些？\n监管政策是否稳定？未来可能发生哪些重大变化？对公司有何潜在影响？\n公司是否具备影响或适应监管政策变化的能力？" + "prompt_template": "# 角色\n你是一位专注于长期价值投资的顶级证券分析师，擅长从基本面出发，对公司进行深入、全面的分析。你的分析报告以客观、严谨、逻辑清晰、数据详实著称。\n# 任务\n为公司 {company_name} (股票代码: {ts_code}) 生成一份全面、专业、结构化的投资分析报告。\n# 输出要求\n直接开始：不要进行任何自我介绍或客套话，直接输出报告正文。\nMarkdown格式：使用清晰的多级Markdown标题（如 ## 和 ###）来组织报告结构。\n专业口吻：保持客观、中立、分析性的专业语调。\n信息缺失处理：如果某些信息在公开渠道无法获取，请明确指出“相关信息未公开披露”或类似说明。\n\n# 报告核心结构与分析要点\n一、 公司基本面分析 (Fundamental Analysis)\n1.1 护城河与核心竞争力\n公司通过何种独有优势（如品牌、技术、成本、网络效应、牌照等）获取超额利润？\n该护城河是在增强、维持还是在削弱？请提供论据。\n1.2 管理层与公司治理\n管理能力：管理层过往的战略决策和执行能力如何？是否有卓越的业界声誉？\n股东回报：管理层及大股东是否珍惜股权价值？（分析历史上的增持/减持行为、分红派息政策、是否存在损害小股东利益的体外资产等）\n激励与目标：公司的经营目标是长期主义还是短期化？管理层的激励机制（如股权激励、考核指标）是否与长期战略目标一致？\n1.3 企业文化与财务政策\n公司是否有独特且可观察到的企业文化？（例如：创新文化、成本控制文化等）\n公司的财务政策（如资本结构、现金流管理、投资策略）与同行业公司相比有何显著特点？是激进还是保守？\n1.4 发展历程与战略规划\n梳理公司发展史上的关键事件、重大业务转型或里程碑。\n公司是否有清晰的长期战略目标（未来5-10年）？计划成为一家什么样的企业？\n二、 业务与市场分析 (Business & Market Analysis)\n2.1 产品与客户价值\n公司为客户提供什么核心产品/服务？其核心价值点是什么？客户为何选择公司的产品而非竞争对手的？\n产品的更新迭代是颠覆性的还是渐进积累型的？分析产品历年的产量、价格及销量变化，并探讨其背后的驱动因素。\n2.2 市场需求与景气度\n客户所处行业的需求是趋势性的高增长，还是周期性波动？或是两者结合？当前处于何种阶段？\n目标客户群体的经营状况和现金流是否健康？\n2.3 议价能力与客户关系\n公司对下游客户的议价能力强弱如何？（结合应收账款周转天数、账龄结构、毛利率等数据进行佐证）\n公司与核心客户的关系是否稳定？客户对公司的评价如何（例如：客户忠诚度、满意度）？\n三、 竞争格局分析 (Competitive Landscape Analysis)\n3.1 竞争对手画像\n列出公司的主要竞争对手，并分析各自的优势与劣势。\n公司的竞争对手是在增多还是减少？行业进入壁垒是在增高还是降低？\n是否存在潜在的跨界竞争者？\n四、 供应链与外部关系 (Supply Chain & External Relations)\n4.1 供应链议价能力\n公司对上游供应商的议价能力如何？（结合应付账款周转天数、采购成本控制等数据进行佐证）\n核心供应商的经营是否稳定？供应链是否存在集中度过高的风险？\n4.2 金融机构关系与融资需求\n公司与金融机构的关系如何？融资渠道是否通畅？\n公司未来的发展是否依赖于大规模的债务或股权融资？\n五、 监管环境与政策风险 (Regulatory Environment & Policy Risks)\n公司所处行业是否存在重要的监管部门？主要的监管政策有哪些？\n监管政策是否稳定？未来可能发生哪些重大变化？对公司有何潜在影响？\n公司是否具备影响或适应监管政策变化的能力？\n\n# 附录：财务数据\n以下是该公司的财务数据摘要，请在分析中充分引用这些数据：\n{{ financial_data }}" }, "bull_case": { "name": "看涨分析", "model": "qwen-flash-2025-07-28", "dependencies": [], - "prompt_template": "#### # 角色\n你是一位顶级的成长股投资分析师，拥有敏锐的洞察力，尤其擅长**挖掘市场尚未充分认识到的潜在价值**和**判断长期行业趋势**。你的任务是为目标公司构建一个令人信服的、由证据支持的看涨论述（Bull Case）。\n\n#### # 任务\n为公司 **{company_name}** (股票代码: **{ts_code}**) 生成一份深入的看涨分析报告。报告的核心是论证该公司拥有被市场低估的隐藏资产、持续加深的护城河，并且其所处行业将迎来至少3年以上的景气周期。\n\n#### # 输出要求\n1. **直奔主题**：直接开始分析，无需引言。\n2. **Markdown格式**：使用清晰的标题结构来组织你的论点。\n3. **数据与来源**：所有关键论点都必须有数据、事实或合理的逻辑推演作为支撑。请用*斜体*注明信息来源（如：*来源：公司2023年投资者交流纪要* 或 *来源：中信证券行业研报*）。\n4. **聚焦看涨逻辑**：报告内容应完全围绕支撑看涨观点的论据展开，暂时忽略风险和负面因素。\n5. **前瞻性视角**：分析应侧重于未来3-5年的发展潜力，而不仅仅是回顾历史。\n6. **信息缺失处理**：如果某些推论需要的数据无法公开获取，可以基于现有信息进行合理的逻辑推测，并明确标注“（此为基于...的推测）”。\n\n---\n\n### # 看涨核心论证框架\n\n## 一、 深度挖掘：公司的隐藏资产与未被市场充分定价的价值\n\n### 1.1 资产负债表之外的价值 (Off-Balance Sheet Value)\n- **无形资产**：公司是否拥有未被充分计价的核心技术专利、软件著作权、特许经营权或强大的品牌价值？请量化或举例说明其潜在商业价值。\n- **数据资产**：公司是否积累了具有巨大潜在价值的用户或行业数据？这些数据未来可能的变现途径是什么？\n\n### 1.2 低估的实体或股权资产 (Undervalued Physical or Equity Assets)\n- **土地/物业重估**：公司持有的土地、房产等固定资产，其当前市场公允价值是否远超账面价值？\n- **子公司/投资价值**：公司旗下是否有快速增长但未被市场充分关注的子公司或有价值的长期股权投资？分析其独立估值的潜力。\n\n### 1.3 运营中的“隐形冠军” (Operational \"Hidden Champions\")\n- 公司是否存在独特的、难以复制的生产工艺、供应链管理能力或运营效率优势，而这些优势尚未完全体现在当前的利润率中？\n\n## 二、 护城河的加深：竞争优势的动态强化分析\n\n### 2.1 护城河的动态演变：是静态还是在拓宽？\n- 论证公司的核心护城河（例如：网络效应、转换成本、成本优势、技术壁垒）在未来几年将如何被**强化**而非削弱。请提供具体证据（如：研发投入的持续增长、客户续约率的提升、市场份额的扩大等）。\n\n### 2.2 技术与创新壁垒的领先优势\n- 公司的研发投入和创新产出，如何确保其在未来3-5年内保持对竞争对手的技术代差或领先地位？\n- 是否有即将商业化的“杀手级”新产品或新技术？\n\n### 2.3 品牌与客户粘性的正反馈循环\n- 公司的品牌价值或客户关系如何形成一个正反馈循环（即：强品牌带来高议价能力 -> 高利润投入研发/营销 -> 品牌更强）？\n- 客户为何难以转向竞争对手？分析其高昂的转换成本。\n\n## 三、 长期景气度：行业未来3年以上的持续增长动力\n\n### 3.1 长期需求驱动力（Demand-Side Drivers）\n- 驱动行业增长的核心动力是短期的周期性复苏，还是长期的结构性变迁（如：技术革命、消费升级、国产替代、政策驱动）？请深入论证。\n- 行业的市场渗透率是否仍有巨大提升空间？分析未来市场规模（TAM）的扩张潜力。\n\n### 3.2 供给侧格局优化（Supply-Side Dynamics）\n- 行业供给侧是否出现集中度提升、落后产能出清的趋势？这是否意味着龙头企业的定价权和盈利能力将持续增强？\n- 行业的进入壁垒是否在显著提高（如：技术、资金、资质壁垒），从而限制新竞争者的涌入？\n\n### 3.3 关键催化剂（Key Catalysts）\n- 未来1-2年内，是否存在可以显著提升公司估值或盈利的潜在催化剂事件（如：新产品发布、重要政策落地、海外市场突破等）？" + "prompt_template": "#### # 角色\n你是一位顶级的成长股投资分析师，拥有敏锐的洞察力，尤其擅长**挖掘市场尚未充分认识到的潜在价值**和**判断长期行业趋势**。你的任务是为目标公司构建一个令人信服的、由证据支持的看涨论述（Bull Case）。\n\n#### # 任务\n为公司 **{company_name}** (股票代码: **{ts_code}**) 生成一份深入的看涨分析报告。报告的核心是论证该公司拥有被市场低估的隐藏资产、持续加深的护城河，并且其所处行业将迎来至少3年以上的景气周期。\n\n#### # 输出要求\n1. **直奔主题**：直接开始分析，无需引言。\n2. **Markdown格式**：使用清晰的标题结构来组织你的论点。\n3. **数据与来源**：所有关键论点都必须有数据、事实或合理的逻辑推演作为支撑。请用*斜体*注明信息来源（如：*来源：公司2023年投资者交流纪要* 或 *来源：中信证券行业研报*）。\n4. **聚焦看涨逻辑**：报告内容应完全围绕支撑看涨观点的论据展开，暂时忽略风险和负面因素。\n5. **前瞻性视角**：分析应侧重于未来3-5年的发展潜力，而不仅仅是回顾历史。\n6. **信息缺失处理**：如果某些推论需要的数据无法公开获取，可以基于现有信息进行合理的逻辑推测，并明确标注“（此为基于...的推测）”。\n\n---\n\n### # 看涨核心论证框架\n\n## 一、 深度挖掘：公司的隐藏资产与未被市场充分定价的价值\n\n### 1.1 资产负债表之外的价值 (Off-Balance Sheet Value)\n- **无形资产**：公司是否拥有未被充分计价的核心技术专利、软件著作权、特许经营权或强大的品牌价值？请量化或举例说明其潜在商业价值。\n- **数据资产**：公司是否积累了具有巨大潜在价值的用户或行业数据？这些数据未来可能的变现途径是什么？\n\n### 1.2 低估的实体或股权资产 (Undervalued Physical or Equity Assets)\n- **土地/物业重估**：公司持有的土地、房产等固定资产，其当前市场公允价值是否远超账面价值？\n- **子公司/投资价值**：公司旗下是否有快速增长但未被市场充分关注的子公司或有价值的长期股权投资？分析其独立估值的潜力。\n\n### 1.3 运营中的“隐形冠军” (Operational \"Hidden Champions\")\n- 公司是否存在独特的、难以复制的生产工艺、供应链管理能力或运营效率优势，而这些优势尚未完全体现在当前的利润率中？\n\n## 二、 护城河的加深：竞争优势的动态强化分析\n\n### 2.1 护城河的动态演变：是静态还是在拓宽？\n- 论证公司的核心护城河（例如：网络效应、转换成本、成本优势、技术壁垒）在未来几年将如何被**强化**而非削弱。请提供具体证据（如：研发投入的持续增长、客户续约率的提升、市场份额的扩大等）。\n\n### 2.2 技术与创新壁垒的领先优势\n- 公司的研发投入和创新产出，如何确保其在未来3-5年内保持对竞争对手的技术代差或领先地位？\n- 是否有即将商业化的“杀手级”新产品或新技术？\n\n### 2.3 品牌与客户粘性的正反馈循环\n- 公司的品牌价值或客户关系如何形成一个正反馈循环（即：强品牌带来高议价能力 -> 高利润投入研发/营销 -> 品牌更强）？\n- 客户为何难以转向竞争对手？分析其高昂的转换成本。\n\n## 三、 长期景气度：行业未来3年以上的持续增长动力\n\n### 3.1 长期需求驱动力（Demand-Side Drivers）\n- 驱动行业增长的核心动力是短期的周期性复苏，还是长期的结构性变迁（如：技术革命、消费升级、国产替代、政策驱动）？请深入论证。\n- 行业的市场渗透率是否仍有巨大提升空间？分析未来市场规模（TAM）的扩张潜力。\n\n### 3.2 供给侧格局优化（Supply-Side Dynamics）\n- 行业供给侧是否出现集中度提升、落后产能出清的趋势？这是否意味着龙头企业的定价权和盈利能力将持续增强？\n- 行业的进入壁垒是否在显著提高（如：技术、资金、资质壁垒），从而限制新竞争者的涌入？\n\n### 3.3 关键催化剂（Key Catalysts）\n- 未来1-2年内，是否存在可以显著提升公司估值或盈利的潜在催化剂事件（如：新产品发布、重要政策落地、海外市场突破等）？\n\n# 附录：财务数据\n以下是该公司的财务数据摘要，请在分析中充分引用这些数据：\n{{ financial_data }}" }, "bear_case": { "name": "看跌分析", "model": "qwen-flash-2025-07-28", "dependencies": [], - "prompt_template": "#### # 角色\n你是一位经验丰富的风险控制分析师和审慎的价值投资者，以“能看到别人看不到的风险”而闻名。你的核心任务是**进行压力测试**，识别出公司潜在的、可能导致价值毁灭的重大风险点，并评估其在最坏情况下的价值底线。\n\n#### # 任务\n为公司 **{company_name}** (股票代码: **{ts_code}**) 生成一份审慎的看跌分析报告（Bear Case）。报告需要深入探讨可能侵蚀公司护城河的因素、被市场忽视的潜在风险、行业可能面临的逆风，并对公司的价值底线进行评估。\n\n#### # 输出要求\n1. **直奔主题**：直接开始风险分析，无需引言。\n2. **Markdown格式**：使用清晰的标题结构组织风险论点。\n3. **证据驱动**：所有风险点都必须基于事实、数据或严谨的逻辑推演。请用*斜体*注明信息来源（如：*来源：竞争对手2023年财报* 或 *来源：行业监管政策草案*）。\n4. **聚焦看跌逻辑**：报告应完全围绕看跌观点展开，旨在识别和放大潜在的负面因素。\n5. **底线思维**：分析的核心是评估“事情最坏能到什么程度”，并判断公司的安全边际。\n6. **信息缺失处理**：对于难以量化的风险（如管理层风险），进行定性分析和逻辑阐述。\n\n---\n\n### # 看跌核心论证框架\n\n## 一、 护城河的侵蚀：竞争优势的脆弱性分析 (Moat Erosion: Vulnerability of Competitive Advantages)\n\n### 1.1 现有护城河的潜在威胁\n- 公司的核心护城河（技术、品牌、成本等）是否面临被颠覆的风险？（例如：新技术的出现、竞争对手的模仿或价格战）\n- 客户的转换成本是否真的足够高？是否存在某些因素（如行业标准化）可能降低客户的转换壁垒？\n\n### 1.2 竞争格局的恶化\n- 是否有新的、强大的“跨界”竞争者进入市场？\n- 行业是否从“蓝海”变为“红海”？分析导致竞争加剧的因素（如：产能过剩、产品同质化）。\n- 竞争对手的哪些战略举动可能对公司构成致命打击？\n\n## 二、 隐藏的负债与风险：资产负债表之外的“地雷” (Hidden Liabilities & Risks: Off-Balance Sheet \"Mines\")\n\n### 2.1 潜在的财务风险\n- 公司是否存在大量的或有负债、对外担保或未入表的债务？\n- 公司的现金流健康状况是否脆弱？分析其经营现金流能否覆盖资本开支和债务利息，尤其是在收入下滑的情况下。\n- 应收账款或存货是否存在潜在的暴雷风险？（分析其账龄、周转率和减值计提的充分性）\n\n### 2.2 运营与管理风险\n- 公司是否对单一供应商、单一客户或单一市场存在过度依赖？\n- 公司是否存在“关键人物风险”？创始团队或核心技术人员的离开会对公司造成多大影响？\n- 公司的企业文化或治理结构是否存在可能导致重大决策失误的缺陷？\n\n## 三、 行业逆风与最坏情况分析 (Industry Headwinds & Worst-Case Scenario)\n\n### 3.1 行业天花板与需求逆转\n- 行业渗透率是否已接近饱和？未来的增长空间是否被高估？\n- 驱动行业增长的核心因素是否可持续？是否存在可能导致需求突然逆转的黑天鹅事件（如：政策突变、技术路线改变、消费者偏好转移）？\n\n### 3.2 价值链上的压力传导\n- 上游供应商的议价能力是否在增强，从而挤压公司利润空间？\n- 下游客户的需求是否在萎缩，或者客户的财务状况是否在恶化？\n\n### 3.3 最坏情况压力测试 (Worst-Case Stress Test)\n- **情景假设**：假设行业需求下滑30%，或主要竞争对手发起价格战，公司的收入、利润和现金流会受到多大冲击？\n- **破产风险评估**：在这种极端情况下，公司是否有足够的现金储备和融资能力来度过危机？公司的生存底线在哪里？\n\n### 3.4 价值底线评估：清算价值分析 (Bottom-Line Valuation: Liquidation Value Analysis)\n- **核心假设**：在公司被迫停止经营并清算的极端情况下，其资产的真实变现价值是多少？\n- **资产逐项折价**：请对资产负债表中的主要科目进行折价估算。例如：\n - *现金及等价物*：按100%计算。\n - *应收账款*：根据账龄和客户质量，估计一个合理的回收率（如50%-80%）。\n - *存货*：根据存货类型（原材料、产成品）和市场状况，估计一个变现折扣（如30%-70%）。\n - *固定资产（厂房、设备）*：估计其二手市场的变现价值，通常远低于账面净值。\n - *无形资产/商誉*：大部分在清算时价值归零。\n- **负债计算**：公司的总负债（包括所有表内及表外负债）需要被优先偿还。\n- **清算价值估算**：计算**（折价后的总资产 - 总负债）/ 总股本**，得出每股清算价值。这是公司价值的绝对底线。\n\n## 四、 估值陷阱分析 (Valuation Trap Analysis)\n\n### 4.1 增长预期的证伪\n- 当前的高估值是否隐含了过于乐观的增长预期？论证这些预期为何可能无法实现。\n- 市场是否忽略了公司盈利能力的周期性，而将其误判为长期成长性？\n\n### 4.2 资产质量重估\n- 公司的资产（尤其是商誉、无形资产）是否存在大幅减值的风险？\n- 公司的真实盈利能力（扣除非经常性损益后）是否低于报表利润？\n" + "prompt_template": "#### # 角色\n你是一位经验丰富的风险控制分析师和审慎的价值投资者，以“能看到别人看不到的风险”而闻名。你的核心任务是**进行压力测试**，识别出公司潜在的、可能导致价值毁灭的重大风险点，并评估其在最坏情况下的价值底线。\n\n#### # 任务\n为公司 **{company_name}** (股票代码: **{ts_code}**) 生成一份审慎的看跌分析报告（Bear Case）。报告需要深入探讨可能侵蚀公司护城河的因素、被市场忽视的潜在风险、行业可能面临的逆风，并对公司的价值底线进行评估。\n\n#### # 输出要求\n1. **直奔主题**：直接开始风险分析，无需引言。\n2. **Markdown格式**：使用清晰的标题结构组织风险论点。\n3. **证据驱动**：所有风险点都必须基于事实、数据或严谨的逻辑推演。请用*斜体*注明信息来源（如：*来源：竞争对手2023年财报* 或 *来源：行业监管政策草案*）。\n4. **聚焦看跌逻辑**：报告应完全围绕看跌观点展开，旨在识别和放大潜在的负面因素。\n5. **底线思维**：分析的核心是评估“事情最坏能到什么程度”，并判断公司的安全边际。\n6. **信息缺失处理**：对于难以量化的风险（如管理层风险），进行定性分析和逻辑阐述。\n\n---\n\n### # 看跌核心论证框架\n\n## 一、 护城河的侵蚀：竞争优势的脆弱性分析 (Moat Erosion: Vulnerability of Competitive Advantages)\n\n### 1.1 现有护城河的潜在威胁\n- 公司的核心护城河（技术、品牌、成本等）是否面临被颠覆的风险？（例如：新技术的出现、竞争对手的模仿或价格战）\n- 客户的转换成本是否真的足够高？是否存在某些因素（如行业标准化）可能降低客户的转换壁垒？\n\n### 1.2 竞争格局的恶化\n- 是否有新的、强大的“跨界”竞争者进入市场？\n- 行业是否从“蓝海”变为“红海”？分析导致竞争加剧的因素（如：产能过剩、产品同质化）。\n- 竞争对手的哪些战略举动可能对公司构成致命打击？\n\n## 二、 隐藏的负债与风险：资产负债表之外的“地雷” (Hidden Liabilities & Risks: Off-Balance Sheet \"Mines\")\n\n### 2.1 潜在的财务风险\n- 公司是否存在大量的或有负债、对外担保或未入表的债务？\n- 公司的现金流健康状况是否脆弱？分析其经营现金流能否覆盖资本开支和债务利息，尤其是在收入下滑的情况下。\n- 应收账款或存货是否存在潜在的暴雷风险？（分析其账龄、周转率和减值计提的充分性）\n\n### 2.2 运营与管理风险\n- 公司是否对单一供应商、单一客户或单一市场存在过度依赖？\n- 公司是否存在“关键人物风险”？创始团队或核心技术人员的离开会对公司造成多大影响？\n- 公司的企业文化或治理结构是否存在可能导致重大决策失误的缺陷？\n\n## 三、 行业逆风与最坏情况分析 (Industry Headwinds & Worst-Case Scenario)\n\n### 3.1 行业天花板与需求逆转\n- 行业渗透率是否已接近饱和？未来的增长空间是否被高估？\n- 驱动行业增长的核心因素是否可持续？是否存在可能导致需求突然逆转的黑天鹅事件（如：政策突变、技术路线改变、消费者偏好转移）？\n\n### 3.2 价值链上的压力传导\n- 上游供应商的议价能力是否在增强，从而挤压公司利润空间？\n- 下游客户的需求是否在萎缩，或者客户的财务状况是否在恶化？\n\n### 3.3 最坏情况压力测试 (Worst-Case Stress Test)\n- **情景假设**：假设行业需求下滑30%，或主要竞争对手发起价格战，公司的收入、利润和现金流会受到多大冲击？\n- **破产风险评估**：在这种极端情况下，公司是否有足够的现金储备和融资能力来度过危机？公司的生存底线在哪里？\n\n### 3.4 价值底线评估：清算价值分析 (Bottom-Line Valuation: Liquidation Value Analysis)\n- **核心假设**：在公司被迫停止经营并清算的极端情况下，其资产的真实变现价值是多少？\n- **资产逐项折价**：请对资产负债表中的主要科目进行折价估算。例如：\n - *现金及等价物*：按100%计算。\n - *应收账款*：根据账龄和客户质量，估计一个合理的回收率（如50%-80%）。\n - *存货*：根据存货类型（原材料、产成品）和市场状况，估计一个变现折扣（如30%-70%）。\n - *固定资产（厂房、设备）*：估计其二手市场的变现价值，通常远低于账面净值。\n - *无形资产/商誉*：大部分在清算时价值归零。\n- **负债计算**：公司的总负债（包括所有表内及表外负债）需要被优先偿还。\n- **清算价值估算**：计算**（折价后的总资产 - 总负债）/ 总股本**，得出每股清算价值。这是公司价值的绝对底线。\n\n## 四、 估值陷阱分析 (Valuation Trap Analysis)\n\n### 4.1 增长预期的证伪\n- 当前的高估值是否隐含了过于乐观的增长预期？论证这些预期为何可能无法实现。\n- 市场是否忽略了公司盈利能力的周期性，而将其误判为长期成长性？\n\n### 4.2 资产质量重估\n- 公司的资产（尤其是商誉、无形资产）是否存在大幅减值的风险？\n- 公司的真实盈利能力（扣除非经常性损益后）是否低于报表利润？\n\n\n# 附录：财务数据\n以下是该公司的财务数据摘要，请在分析中充分引用这些数据：\n{{ financial_data }}" }, "market_analysis": { "name": "市场分析", diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml new file mode 100644 index 0000000..0953e4d --- /dev/null +++ b/docker-compose.e2e.yml @@ -0,0 +1,10 @@ +services: + api-gateway: + ports: + - "4000:4000" + + workflow-orchestrator-service: + ports: + - "8005:8005" # Expose for debugging if needed + + diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..1ee472a --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,51 @@ +services: + postgres-test: + image: timescale/timescaledb:2.15.2-pg16 + container_name: fundamental-postgres-test + command: -c shared_preload_libraries=timescaledb + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: fundamental_test + ports: + - "5433:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d fundamental_test"] + interval: 5s + timeout: 5s + retries: 10 + networks: + - test-network + + nats-test: + image: nats:2.9 + container_name: fundamental-nats-test + ports: + - "4223:4222" + networks: + - test-network + + data-persistence-test: + build: + context: . + dockerfile: services/data-persistence-service/Dockerfile + container_name: data-persistence-service-test + environment: + HOST: 0.0.0.0 + PORT: 3000 + # Connect to postgres-test using internal docker network alias + DATABASE_URL: postgresql://postgres:postgres@postgres-test:5432/fundamental_test + RUST_LOG: info + RUST_BACKTRACE: "1" + ports: + - "3001:3000" + depends_on: + postgres-test: + condition: service_healthy + networks: + - test-network + +networks: + test-network: + + diff --git a/docker-compose.yml b/docker-compose.yml index 3c84409..2225282 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: build: context: . dockerfile: services/data-persistence-service/Dockerfile + # Override build context to ensure ignored files are included if needed, or rely on .dockerignore container_name: data-persistence-service environment: HOST: 0.0.0.0 @@ -84,8 +85,6 @@ services: NATS_ADDR: nats://nats:4222 DATA_PERSISTENCE_SERVICE_URL: http://data-persistence-service:3000/api/v1 REPORT_GENERATOR_SERVICE_URL: http://report-generator-service:8004 - # provider_services via explicit JSON for deterministic parsing - PROVIDER_SERVICES: '["http://alphavantage-provider-service:8000", "http://tushare-provider-service:8001", "http://finnhub-provider-service:8002", "http://yfinance-provider-service:8003"]' RUST_LOG: info,axum=info RUST_BACKTRACE: "1" depends_on: @@ -113,6 +112,8 @@ services: SERVER_PORT: 8000 NATS_ADDR: nats://nats:4222 DATA_PERSISTENCE_SERVICE_URL: http://data-persistence-service:3000/api/v1 + API_GATEWAY_URL: http://api-gateway:4000 + SERVICE_HOST: alphavantage-provider-service RUST_LOG: info,axum=info RUST_BACKTRACE: "1" depends_on: @@ -136,6 +137,8 @@ services: NATS_ADDR: nats://nats:4222 DATA_PERSISTENCE_SERVICE_URL: http://data-persistence-service:3000/api/v1 TUSHARE_API_URL: http://api.waditu.com + API_GATEWAY_URL: http://api-gateway:4000 + SERVICE_HOST: tushare-provider-service RUST_LOG: info,axum=info RUST_BACKTRACE: "1" depends_on: @@ -159,6 +162,8 @@ services: NATS_ADDR: nats://nats:4222 DATA_PERSISTENCE_SERVICE_URL: http://data-persistence-service:3000/api/v1 FINNHUB_API_URL: https://finnhub.io/api/v1 + API_GATEWAY_URL: http://api-gateway:4000 + SERVICE_HOST: finnhub-provider-service RUST_LOG: info,axum=info RUST_BACKTRACE: "1" depends_on: @@ -181,6 +186,8 @@ services: SERVER_PORT: 8003 NATS_ADDR: nats://nats:4222 DATA_PERSISTENCE_SERVICE_URL: http://data-persistence-service:3000/api/v1 + API_GATEWAY_URL: http://api-gateway:4000 + SERVICE_HOST: yfinance-provider-service RUST_LOG: info,axum=info RUST_BACKTRACE: "1" depends_on: @@ -219,6 +226,28 @@ services: timeout: 5s retries: 12 + workflow-orchestrator-service: + build: + context: . + dockerfile: services/workflow-orchestrator-service/Dockerfile + container_name: workflow-orchestrator-service + environment: + SERVER_PORT: 8005 + NATS_ADDR: nats://nats:4222 + DATA_PERSISTENCE_SERVICE_URL: http://data-persistence-service:3000/api/v1 + RUST_LOG: info + RUST_BACKTRACE: "1" + depends_on: + - nats + - data-persistence-service + networks: + - app-network + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:8005/health >/dev/null || exit 1"] + interval: 5s + timeout: 5s + retries: 12 + # ================================================================= # Python Services (Legacy - to be replaced) # ================================================================= diff --git a/docs/3_project_management/tasks/pending/20251118_analysis_template_integration.md b/docs/3_project_management/tasks/completed/20251118_analysis_template_integration.md similarity index 100% rename from docs/3_project_management/tasks/pending/20251118_analysis_template_integration.md rename to docs/3_project_management/tasks/completed/20251118_analysis_template_integration.md diff --git a/docs/3_project_management/tasks/pending/20251118_analysis_workflow_optimization.md b/docs/3_project_management/tasks/completed/20251118_analysis_workflow_optimization.md similarity index 100% rename from docs/3_project_management/tasks/pending/20251118_analysis_workflow_optimization.md rename to docs/3_project_management/tasks/completed/20251118_analysis_workflow_optimization.md diff --git a/docs/3_project_management/tasks/completed/20251119_frontend_refactoring.md b/docs/3_project_management/tasks/completed/20251119_frontend_refactoring.md new file mode 100644 index 0000000..156e46d --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251119_frontend_refactoring.md @@ -0,0 +1,193 @@ +# 前端报告页面重构设计文档 (Frontend Refactoring Design Doc) + +**日期**: 2025-11-19 +**状态**: 待评审 (Draft) +**目标**: 重构 `app/report/[symbol]` 页面，消除历史技术债务，严格对齐 V2 后端微服务架构。 + +## 1. 核心原则 + +1. **单一数据源 (SSOT)**: 前端不再维护任务进度、依赖关系或倒计时。所有状态严格来自后端 API (`/api/tasks/{id}`, `/api/analysis-results`). +2. **无隐式逻辑 (No Implicit Logic)**: 严格按照用户选择的 Template ID 渲染，后端未返回的数据即视为不存在，不进行客户端推断或 Fallback。 +3. **真·流式传输 (True Streaming)**: 废弃数据库轮询方案。采用 **Server-Sent Events (SSE)** 技术。 + * 后端在内存中维护 `tokio::sync::broadcast` 通道。 + * LLM 生成的 Token 实时推送到通道，直达前端。 + * 数据库只负责存储**最终完成**的分析结果 (Persistence)，不参与流式传输过程。 + +## 2. 页面布局设计 + +页面采用“固定框架 + 动态内容”的布局模式。 + +```text ++-----------------------------------------------------------------------+ +| [Header Area] | +| Symbol: AAPL | Market: US | Price: $230.5 (Snapshot) | [Status Badge]| +| Control: [ Template Select Dropdown [v] ] [ Trigger Analysis Button ]| ++-----------------------------------------------------------------------+ +| | +| [ Tab Navigation Bar ] | +| +-----------+ +--------------+ +------------+ +------------+ +-----+ | +| | 股价图表 | | 基本面数据 | | 分析模块A | | 分析模块B | | ... | | +| +-----------+ +--------------+ +------------+ +------------+ +-----+ | +| | | ++-----------------------------------------------------------------------+ +| [ Main Content Area ] | +| | +| (Content changes based on selected Tab) | +| | +| SCENARIO 1: Stock Chart Tab | +| +-------------------------------------------------+ | +| | [ PLACEHOLDER: TradingView / K-Line Chart ] | | +| | (Future: Connect to Time-Series DB) | | +| +-------------------------------------------------+ | +| | +| SCENARIO 2: Fundamental Data Tab | +| +-------------------------------------------------+ | +| | Status: Waiting for Providers (2/3)... | | +| | --------------------------------------------- | | +| | [Tushare]: OK (JSON/Table Dump) | | +| | [Finnhub]: OK (JSON/Table Dump) | | +| | [AlphaV ]: Pending... | | +| +-------------------------------------------------+ | +| | +| SCENARIO 3: Analysis Module Tab (e.g., Valuation) | +| +-------------------------------------------------+ | +| | [Markdown Renderer] | | +| | ## Valuation Analysis | | +| | Based on the PE ratio of 30... | | +| | (Streaming Cursor) _ | | +| +-------------------------------------------------+ | +| | ++-----------------------------------------------------------------------+ +| [ Execution Details Footer / Tab ] | +| Total Time: 12s | Tokens: 4050 | Cost: $0.02 | ++-----------------------------------------------------------------------+ +``` + +## 3. 数据流与状态机 + +### 3.1 固定 Tab 定义 +无论选择何种模板，以下 Tab 始终存在（Fixed Tabs）： + +1. **股价图表 (Stock Chart)** + * **数据源**: 独立的实时行情 API / 时间序列数据库。 + * **当前实现**: 占位符 (Placeholder)。 +2. **基本面数据 (Fundamental Data)** + * **定义**: 所有已启用的 Data Providers 返回的原始数据聚合。 + * **状态逻辑**: + * 此 Tab 代表“数据准备阶段”。 + * 必须等待后端 `FetchCompanyDataCommand` 对应的 Task 状态为 Completed/Partial/Failed。 + * UI 展示所有 Provider 的回执。只有当所有 Provider 都有定论（成功或失败），此阶段才算结束。 + * **作为后续分析的“门控”**: 此阶段未完成前，后续分析 Tab 处于“等待中”状态。 +3. **执行详情 (Execution Details)** + * **定义**: 工作流的元数据汇总。 + * **内容**: 耗时统计、Token 消耗、API 调用清单。 + +### 3.2 动态 Tab 定义 (Analysis Modules) +* **来源**: 根据当前选中的 `Template ID` 从后端获取 `AnalysisTemplateConfig`。 +* **生成逻辑**: + * Template 中定义了 Modules: `[Module A, Module B, Module C]`. + * 前端直接映射为 Tab A, Tab B, Tab C。 +* **渲染**: + * **Loading**: 后端 `AnalysisResult` 状态为 `processing`。 + * **Streaming**: 通过 SSE (`/api/analysis-results/stream`) 接收增量内容。 + * **Done**: 后端流结束，或直接从 DB 读取完整内容。 + +### 3.3 状态机 (useReportEngine Hook) + +我们将废弃旧的 Hook，实现一个纯粹的 `useReportEngine`。 + +```typescript +interface ReportState { + // 1. 配置上下文 + symbol: string; + templateId: string; + templateConfig: AnalysisTemplateSet | null; // 用于生成动态 Tab + + // 2. 阶段状态 + fetchStatus: 'idle' | 'fetching' | 'complete' | 'error'; // 基本面数据阶段 + analysisStatus: 'idle' | 'running' | 'complete'; // 分析阶段 + + // 3. 数据持有 + fundamentalData: any[]; // 来自各个 Provider 的原始数据 + analysisResults: Record; // Key: ModuleID + + // 4. 进度 + executionMeta: { + startTime: number; + elapsed: number; + tokens: number; + } +} +``` + +## 4. 交互流程 + +1. **初始化**: + * 用户进入页面 -> 加载 `api/configs/analysis_template_sets` -> 填充下拉框。 + * 如果 URL 或历史数据中有 `template_id`，自动选中。 + +2. **触发 (Trigger)**: + * 用户点击“开始分析”。 + * 前端 POST `/api/data-requests` (payload: `{ symbol, template_id }`)。 + * **前端重置所有动态 Tab 内容为空**。 + * 进入 `fetchStatus: fetching`。 + +3. **阶段一：基本面数据获取**: + * 前端轮询 `/api/tasks/{request_id}`。 + * **基本面 Tab** 高亮/显示 Spinner。 + * 展示各个 Provider 的子任务进度。 + * 当 Task 状态 = Completed -> 进入阶段二。 + +4. **阶段二：流式分析 (SSE)**: + * 前端建立 EventSource 连接 `/api/analysis-results/stream?request_id={id}`。 + * **智能切换 Tab**: (可选) 当某个 Module 开始生成 (收到 SSE 事件 `module_start`) 时，UI 可以自动切换到该 Tab。 + * **渲染**: 收到 `content` 事件，追加到对应 Module 的内容中。 + * **持久化**: 只有当 SSE 收到 `DONE` 事件时，后端才保证数据已落库。 + +5. **完成**: + * SSE 连接关闭。 + * 状态转为 `complete`。 + +## 5. 架构设计 (Architecture Design) + +为了实现真流式传输，后端架构调整如下： + +1. **内存状态管理 (In-Memory State)**: + * `AppState` 中增加 `stream_manager: StreamManager`。 + * `StreamManager` 维护 `HashMap>`。 + * 这消除了对数据库的中间状态写入压力。 +2. **Worker 职责**: + * Worker 执行 LLM 请求。 + * 收到 Token -> 写入 `BroadcastSender` (Fire and forget)。 + * 同时将 Token 累积在内存 Buffer 中。 + * 生成结束 -> 将完整 Buffer 写入数据库 (PostgreSQL) -> 广播 `ModuleDone` 事件。 +3. **API 职责**: + * `GET /stream`: + * 检查内存中是否有对应的 `BroadcastSender`? + * **有**: 建立 SSE 连接，订阅并转发事件。 + * **无**: 检查数据库是否已完成? + * **已完成**: 一次性返回完整内容 (模拟 SSE 或直接返回 JSON)。 + * **未开始/不存在**: 返回 404 或等待。 + +## 6. 迁移计划 (Action Items) + +### 6.1 清理与归档 (Cleanup) +- [x] 创建 `frontend/archive/v1_report` 目录。 +- [x] 移动 `app/report/[symbol]/components` 下的旧组件（`ExecutionDetails.tsx`, `TaskStatus.tsx`, `ReportHeader.tsx`, `AnalysisContent.tsx`）到 archive。 +- [x] 移动 `app/report/[symbol]/hooks` 下的 `useAnalysisRunner.ts` 和 `useReportData.ts` 到 archive。 + +### 6.2 核心构建 (Core Scaffolding) +- [x] 创建 `hooks/useReportEngine.ts`: 实现上述状态机，严格对接后端 API。 +- [x] 创建 `components/ReportLayout.tsx`: 实现新的布局框架（Header + Tabs + Content）。 +- [x] 创建 `components/RawDataViewer.tsx`: 用于展示基本面原始数据（JSON View）。 +- [x] 创建 `components/AnalysisViewer.tsx`: 用于展示分析结果（Markdown Streaming）。 + +### 6.3 页面集成 (Integration) +- [x] 重写 `app/report/[symbol]/page.tsx`: 引入 `useReportEngine` 和新组件。 +- [ ] 验证全流程：Trigger -> Task Fetching -> Analysis Streaming -> Finish。 + +### 6.4 后端重构 (Backend Refactoring) - NEW +- [x] **State Upgrade**: 更新 `AppState` 引入 `tokio::sync::broadcast` 用于流式广播。 +- [x] **Worker Update**: 修改 `run_report_generation_workflow`，不再生成完才写库，也不中间写库，而是**中间发广播，最后写库**。 +- [x] **API Update**: 新增 `GET /api/analysis-results/stream` (SSE Endpoint)，对接广播通道。 +- [x] **Frontend Update**: 修改 `useReportEngine.ts`，将轮询 `analysis-results` 改为 `EventSource` 连接。 diff --git a/docs/3_project_management/tasks/completed/20251119_provider_cache_design.md b/docs/3_project_management/tasks/completed/20251119_provider_cache_design.md new file mode 100644 index 0000000..0cd24df --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251119_provider_cache_design.md @@ -0,0 +1,148 @@ +# 供应商隔离的数据新鲜度与缓存设计方案 + +## 1. 背景 (Background) + +当前系统使用 `company_profiles` 表中的全局 `updated_at` 时间戳来判断某个股票的数据是否“新鲜”（例如：过去 24 小时内更新过）。 + +**现有问题：** +这种方法在多供应商（Multi-Provider）环境中会导致严重的竞态条件（Race Condition）： +1. **Tushare**（A股数据源）通常响应较快，获取数据并更新了 `company_profiles` 表的 `updated_at`。 +2. `updated_at` 时间戳被更新为 `NOW()`。 +3. **YFinance** 或 **AlphaVantage**（全球数据源）稍后启动任务。 +4. 它们检查 `company_profiles` 表，发现 `updated_at` 非常新，因此错误地认为**自己的**数据也是最新的。 +5. 结果：YFinance/AlphaVantage 跳过执行，导致这些特定字段的数据为空或陈旧。 + +## 2. 目标 (Objective) + +实现一个**供应商隔离的缓存机制**，允许每个数据供应商（Tushare, YFinance, AlphaVantage, Finnhub）能够： +1. 独立追踪其最后一次成功更新数据的时间。 +2. 仅根据**自己的**数据新鲜度来决定是否执行任务。 +3. 避免干扰其他供应商的执行逻辑。 + +## 3. 设计原则 (Design Principles) + +1. **不新增数据表**：利用数据库现有的文档-关系混合特性（Document-Relational）。具体来说，使用 `company_profiles` 表中的 `additional_info` (JSONB) 字段。 +2. **服务层抽象**：解析和管理这些元数据的复杂性应封装在 `Data Persistence Service` 内部，向各 Provider Service 暴露简洁的 API。 +3. **并发安全**：确保不同供应商的并发更新不会覆盖彼此的元数据状态。 + +## 4. 数据结构设计 (Data Structure Design) + +我们将利用现有的 `company_profiles.additional_info` 字段（类型：`JSONB`）来存储一个供应商状态字典。 + +### `additional_info` JSON Schema 设计 + +```json +{ + "provider_status": { + "tushare": { + "last_updated": "2025-11-19T10:00:00Z", + "data_version": "v1", + "status": "success" + }, + "yfinance": { + "last_updated": "2025-11-18T09:30:00Z", + "status": "success" + }, + "alphavantage": { + "last_updated": "2025-11-15T14:00:00Z", + "status": "partial_success" // 例如：触发了速率限制 + } + }, + "other_metadata": "..." // 保留其他现有元数据 +} +``` + +## 5. 实施计划 (Implementation Plan) + +### 5.1. 数据持久化服务更新 (Data Persistence Service) + +我们需要扩展 `PersistenceClient` 及其底层 API，以支持细粒度的元数据更新。 + +**新增/更新 API 端点：** + +1. **`PUT /companies/{symbol}/providers/{provider_id}/status`** (新增) + * **目的**：原子更新特定供应商的状态，无需读取/写入完整的 profile。 + * **实现**：使用 Postgres 的 `jsonb_set` 函数，直接更新 JSON 路径 `['provider_status', provider_id]`。 + * **Payload**: + ```json + { + "last_updated": "2025-11-19T12:00:00Z", + "status": "success" + } + ``` + +2. **`GET /companies/{symbol}/providers/{provider_id}/status`** (新增) + * **目的**：辅助接口，用于获取特定供应商的当前缓存状态。 + +### 5.2. 供应商服务工作流更新 (Provider Service) + +每个 Provider Service（例如 `yfinance-provider-service`）将修改其 `worker.rs` 中的逻辑： + +**现有逻辑（有缺陷）：** +```rust +let profile = client.get_company_profile(symbol).await?; +if profile.updated_at > 24h_ago { return; } // 全局检查 +``` + +**新逻辑：** +```rust +// 1. 检查 Provider 专属缓存 +let status = client.get_provider_status(symbol, "yfinance").await?; +if let Some(s) = status { + if s.last_updated > 24h_ago { + info!("YFinance 数据较新，跳过执行。"); + return; + } +} + +// 2. 获取并持久化数据 +// ... fetch ... +client.upsert_company_profile(profile).await?; // 更新基本信息 +client.batch_insert_financials(financials).await?; + +// 3. 更新 Provider 状态 +client.update_provider_status(symbol, "yfinance", ProviderStatus { + last_updated: Utc::now(), + status: "success" +}).await?; +``` + +## 6. 风险管理与迁移 (Risk Management & Migration) + +* **竞态条件 (Race Conditions)**：通过在数据库层使用 `jsonb_set` 进行部分更新，我们避免了“读-改-写”的竞态条件，确保 Provider A 的更新不会覆盖 Provider B 同时写入的状态。 +* **数据迁移 (Migration)**： + * **策略**：**Lazy Migration (懒迁移)**。 + * 现有数据中没有 `provider_status` 字段。代码将优雅地处理 `null` 或缺失键的情况（将其视为“陈旧/从未运行”，触发重新获取）。 + * **无需**编写专门的 SQL 迁移脚本去清洗历史数据。旧数据会随着新的抓取任务运行而自动补充上状态信息。 + * 如果必须清理，可以直接执行 `UPDATE company_profiles SET additional_info = additional_info - 'provider_status';` 来重置所有缓存状态。 + +## 7. 实施清单 (Implementation Checklist) + +- [x] **Phase 1: Common Contracts & DTOs** + - [x] 在 `services/common-contracts/src/dtos.rs` 中定义 `ProviderStatusDto`. + +- [x] **Phase 2: Data Persistence Service API** + - [x] 实现 DB 层逻辑: `get_provider_status` (读取 JSONB). + - [x] 实现 DB 层逻辑: `update_provider_status` (使用 `jsonb_set`). + - [x] 添加 API Handler: `GET /companies/{symbol}/providers/{provider_id}/status`. + - [x] 添加 API Handler: `PUT /companies/{symbol}/providers/{provider_id}/status`. + - [x] 注册路由并测试接口. + +- [x] **Phase 3: Client Logic Update** + - [x] 更新各服务中的 `PersistenceClient` (如 `services/yfinance-provider-service/src/persistence.rs` 等)，增加 `get_provider_status` 和 `update_provider_status` 方法. + +- [x] **Phase 4: Provider Services Integration** + - [x] **Tushare Service**: 更新 `worker.rs`，集成新的缓存检查逻辑. + - [x] **YFinance Service**: 更新 `worker.rs`，集成新的缓存检查逻辑. + - [x] **AlphaVantage Service**: 更新 `worker.rs`，集成新的缓存检查逻辑. + - [x] **Finnhub Service**: 更新 `worker.rs`，集成新的缓存检查逻辑. + +- [ ] **Phase 5: Verification (验证)** + - [ ] 运行 `scripts/test_data_fetch.py` 验证全流程. + - [ ] 验证不同 Provider 的状态互不干扰. + +- [ ] **Phase 6: Caching Logic Abstraction (缓存逻辑抽象 - 智能客户端)** + - [ ] 将 `PersistenceClient` 迁移至 `services/common-contracts/src/persistence_client.rs`（或新建 `service-sdk` 库），消除重复代码。 + - [ ] 在共享客户端中实现高层方法 `should_fetch_data(symbol, provider, ttl)`。 + - [ ] 重构所有 Provider Service 以使用共享的 `PersistenceClient`。 + - [ ] 验证所有 Provider 的缓存逻辑是否一致且无需手动实现。 diff --git a/docs/3_project_management/tasks/completed/20251119_report_generation_optimization.md b/docs/3_project_management/tasks/completed/20251119_report_generation_optimization.md new file mode 100644 index 0000000..020fe6d --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251119_report_generation_optimization.md @@ -0,0 +1,128 @@ +# 报告生成优化与 UI 状态反馈改进设计文档 + +**状态**: Draft +**日期**: 2025-11-19 +**涉及模块**: Report Generator Service (Backend), Frontend (UI) + +## 1. 背景与问题分析 + +当前系统的报告生成流程存在两个主要痛点，导致用户体验不佳且生成内容质量低下： + +1. **数据注入缺失 (Data Injection Gap)**: + * 后端在执行 Prompt 渲染时，`financial_data` 被硬编码为 `"..."`。 + * 大模型（LLM）缺乏上下文输入，导致输出“幻觉”内容（如自我介绍、复读指令）或通用废话。 + * 依赖链条虽然在拓扑排序上是正确的，但由于上游（如“基本面分析”）输出无效内容，下游（如“最终结论”）的输入也随之失效。 + +2. **UI 状态反馈缺失 (UI/UX Gap)**: + * 前端仅有简单的“有数据/无数据”判断。 + * 点击“重新生成”时，UI 往往显示旧的缓存数据，缺乏“生成中”或“进度更新”的实时反馈。 + * 用户无法区分“旧报告”和“正在生成的新报告”。 + +## 2. 后端优化设计 (Report Generator Service) + +### 2.1 数据注入逻辑修复 (Fixing Financial Data Injection) + +我们将把当前的“基本面数据获取”视为一个**内置的基础工具（Native Tool）**。 + +* **当前逻辑**: 直接透传数据库 Raw Data。 +* **改进逻辑**: 在 `worker.rs` 中实现一个数据格式化器，将 `Vec` 转换为 LLM 易读的 Markdown 表格或结构化文本。 + +**实现细节**: +1. **格式化函数**: 实现 `format_financials_to_markdown(financials: &[TimeSeriesFinancialDto]) -> String`。 + * 按年份/季度降序排列。 + * 提取关键指标（营收、净利润、ROE、毛利率等）。 + * 生成 Markdown Table。 +2. **注入 Context**: + * 在 `Tera` 模板渲染前，调用上述函数。 + * 替换占位符: `context.insert("financial_data", &formatted_data);`。 +3. **上游依赖注入 (保持不变)**: + * 继续保留现有的 `generated_results` 注入逻辑，确保上游模块（如 `market_analysis`）的输出能正确传递给下游（如 `final_conclusion`）。 + +### 2.2 执行状态管理 (Execution Status Management) + +为了支持前端的“实时状态”，后端需要能够区分“排队中”、“生成中”和“已完成”。 + +* **现状**: 只有生成完成后才写入 `analysis_results` 表。 +* **改进**: 引入任务状态流转。 + +**方案 A (基于数据库 - 推荐 MVP)**: +利用现有的 `analysis_results` 表或新建 `analysis_tasks` 表。 +1. **任务开始时**: + * Worker 开始处理某个 `module_id` 时，立即写入/更新一条记录。 + * `status`: `PROCESSING` + * `content`: 空或 "Analysis in progress..." +2. **任务完成时**: + * 更新记录。 + * `status`: `COMPLETED` + * `content`: 实际生成的 Markdown。 +3. **任务失败时**: + * `status`: `FAILED` + * `content`: 错误信息。 + +### 2.3 未来扩展性：工具模块 (Future Tool Module) + +* 当前设计中，`financial_data` 是硬编码注入的。 +* **未来规划**: 在 Prompt 模板配置中，增加 `tools` 字段。 + ```json + "tools": ["financial_aggregator", "news_search", "calculator"] + ``` +* Worker 在渲染 Prompt 前，先解析 `tools` 配置，并行执行对应的工具函数（如 Python 数据清洗脚本），获取输出后注入 Context。当前修复的 `financial_data` 本质上就是 `financial_aggregator` 工具的默认实现。 + +## 3. 前端优化设计 (Frontend) + +### 3.1 状态感知与交互 + +**目标**: 让用户清晰感知到“正在生成”。 + +1. **重新生成按钮行为**: + * 点击“重新生成”后，**立即**将当前模块的 UI 状态置为 `GENERATING`。 + * **视觉反馈**: + * 方案一（简单）：清空旧内容，显示 Skeleton（骨架屏）+ 进度条/Spinner。 + * 方案二（平滑）：保留旧内容，但在上方覆盖一层半透明遮罩，并显示“正在更新分析...”。(推荐方案二，避免内容跳动)。 + +2. **状态轮询 (Polling)**: + * 由于后端暂未实现 SSE (Server-Sent Events)，前端需采用轮询机制。 + * 当状态为 `GENERATING` 时，每隔 2-3 秒调用一次 API 检查该 `module_id` 的状态。 + * 当后端返回状态变更为 `COMPLETED` 时，停止轮询，刷新显示内容。 + +### 3.2 组件结构调整 + +修改 `AnalysisContent.tsx` 组件： + +```typescript +interface AnalysisState { + status: 'idle' | 'loading' | 'success' | 'error'; + data: string | null; // Markdown content + isStale: boolean; // 标记当前显示的是否为旧缓存 +} +``` + +* **Idle**: 初始状态。 +* **Loading**: 点击生成后，显示加载动画。 +* **Success**: 获取到新数据。 +* **IsStale**: 点击重新生成瞬间，将 `isStale` 设为 true。UI 上可以给旧文本加灰色滤镜，直到新数据到来。 + +## 4. 实施计划 (Action Plan) + +### Phase 1: 后端数据修正 (Backend Core) +- [ ] 修改 `services/report-generator-service/src/worker.rs`。 +- [ ] 实现 `format_financial_data` 辅助函数。 +- [ ] 将格式化后的数据注入 Tera Context。 +- [ ] 验证大模型输出不再包含“幻觉”文本。 + +### Phase 2: 后端状态透出 (Backend API) +- [ ] 确认 `NewAnalysisResult` 或相关 DTO 是否支持状态字段。 +- [ ] 在 Worker 开始处理模块时，写入 `PROCESSING` 状态到数据库。 +- [ ] 确保 API 查询接口能返回 `status` 字段。 + +### Phase 3: 前端体验升级 (Frontend UI) +- [ ] 修改 `AnalysisContent.tsx`，增加对 `status` 字段的处理。 +- [ ] 实现“重新生成”时的 UI 遮罩或 Loading 状态，不再单纯依赖 `useQuery` 的缓存。 +- [ ] 优化 Markdown 渲染区的用户体验。 + +## 5. 验收标准 (Acceptance Criteria) + +1. **内容质量**: 市场分析、基本面分析报告中包含具体的财务数字（如营收、利润），且引用正确，不再出现“请提供数据”的字样。 +2. **流程闭环**: 点击“重新生成”，UI 显示加载状态 -> 后端处理 -> UI 自动刷新为新内容。 +3. **无闪烁**: 页面不会因为轮询而频繁闪烁，状态切换平滑。 + diff --git a/docs/3_project_management/tasks/completed/20251120_architecture_refactor_orchestrator.md b/docs/3_project_management/tasks/completed/20251120_architecture_refactor_orchestrator.md new file mode 100644 index 0000000..17f224c --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251120_architecture_refactor_orchestrator.md @@ -0,0 +1,225 @@ +# 架构重构设计文档：引入 Workflow Orchestrator + +## 1. 背景与目标 +当前系统存在 `api-gateway` 职责过载、业务逻辑分散、状态机隐式且脆弱、前后端状态不同步等核心问题。为了彻底解决这些架构痛点，本设计提出引入 **Workflow Orchestrator Service**，作为系统的“大脑”，负责集中管理业务流程、状态流转与事件协调。 + +### 核心目标 +1. **解耦 (Decoupling)**: 将业务协调逻辑从 `api-gateway` 剥离，Gateway 回归纯粹的流量入口和连接管理职责。 +2. **状态一致性 (Consistency)**: 建立单一事实来源 (Single Source of Truth)，所有业务状态由 Orchestrator 统一维护并广播。 +3. **细粒度任务编排 (Fine-Grained Orchestration)**: 废除粗粒度的“阶段”概念，转向基于 DAG (有向无环图) 的任务编排。后端只负责执行任务和广播每个任务的状态，前端根据任务状态自由决定呈现逻辑。 + +## 2. 架构全景图 (Architecture Overview) + +### 2.1 服务角色重定义 + +| 服务 | 现有职责 | **新职责** | +| :--- | :--- | :--- | +| **API Gateway** | 路由, 鉴权, 注册发现, 业务聚合, 流程触发 | 路由, 鉴权, 注册发现, **SSE/WS 代理 (Frontend Proxy)** | +| **Workflow Orchestrator** | *(新服务)* | **DAG 调度**, **任务依赖管理**, **事件广播**, **状态快照** | +| **Data Providers** | 数据抓取, 存库, 发 NATS 消息 | (保持不变) 接收指令 -> 干活 -> 发结果事件 | +| **Report Generator** | 报告生成, 发 NATS 消息 | (保持不变) 接收指令 -> 干活 -> 发进度/结果事件 | +| **Data Processors** | *(新服务类型)* | **数据清洗/转换** (接收上下文 -> 转换 -> 更新上下文) | + +### 2.2 数据流向 (Data Flow) + +1. **启动**: 前端 -> Gateway (`POST /start`) -> **Orchestrator** (NATS: `StartWorkflow`) +2. **调度**: **Orchestrator** 解析模板构建 DAG -> NATS: 触发无依赖的 Tasks (如 Data Fetching) +3. **反馈**: Executors (Providers/ReportGen/Processors) -> NATS: `TaskCompleted` -> **Orchestrator** +4. **流转**: **Orchestrator** 检查依赖 -> NATS: 触发下一层 Tasks +5. **广播**: **Orchestrator** -> NATS: `WorkflowEvent` (Task Status Updates) -> Gateway -> 前端 (SSE) + +## 3. 接口与协议定义 (Contracts & Schemas) + +需在 `services/common-contracts` 中进行以下调整： + +### 3.1 新增 Commands (NATS Subject: `workflow.commands.*`) + +```rust +// Topic: workflow.commands.start +#[derive(Serialize, Deserialize, Debug)] +pub struct StartWorkflowCommand { + pub request_id: Uuid, + pub symbol: CanonicalSymbol, + pub market: String, + pub template_id: String, +} + +// 新增：用于手动请求状态对齐 (Reconnect Scenario) +// Topic: workflow.commands.sync_state +#[derive(Serialize, Deserialize, Debug)] +pub struct SyncStateCommand { + pub request_id: Uuid, +} +``` + +### 3.2 新增 Events (NATS Subject: `events.workflow.{request_id}`) + +这是前端唯一需要订阅的流。 + +```rust +// Topic: events.workflow.{request_id} +#[derive(Serialize, Deserialize, Debug)] +#[serde(tag = "type", content = "payload")] +pub enum WorkflowEvent { + // 1. 流程初始化 (携带完整的任务依赖图) + WorkflowStarted { + timestamp: i64, + // 定义所有任务及其依赖关系，前端可据此绘制流程图或进度条 + task_graph: WorkflowDag + }, + + // 2. 任务状态变更 (核心事件) + TaskStateChanged { + task_id: String, // e.g., "fetch:tushare", "process:clean_financials", "module:swot_analysis" + task_type: TaskType, // DataFetch | DataProcessing | Analysis + status: TaskStatus, // Pending, Scheduled, Running, Completed, Failed, Skipped + message: Option, + timestamp: i64 + }, + + // 3. 任务流式输出 (用于 LLM 打字机效果) + TaskStreamUpdate { + task_id: String, + content_delta: String, + index: u32 + }, + + // 4. 流程整体结束 + WorkflowCompleted { + result_summary: serde_json::Value, + end_timestamp: i64 + }, + + WorkflowFailed { + reason: String, + is_fatal: bool, + end_timestamp: i64 + }, + + // 5. 状态快照 (用于重连/丢包恢复) + // 当前端重连或显式发送 SyncStateCommand 时，Orchestrator 发送此事件 + WorkflowStateSnapshot { + timestamp: i64, + task_graph: WorkflowDag, + tasks_status: HashMap, // 当前所有任务的最新状态 + tasks_output: HashMap> // (可选) 已完成任务的关键输出摘要 + } +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct WorkflowDag { + pub nodes: Vec, + pub edges: Vec // from -> to +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct TaskNode { + pub id: String, + pub name: String, + pub type: TaskType, + pub initial_status: TaskStatus +} + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +pub enum TaskType { + DataFetch, // 创造原始上下文 + DataProcessing, // 消耗并转换上下文 (New) + Analysis // 读取上下文生成新内容 +} + +#[derive(Serialize, Deserialize, Debug, PartialEq)] +pub enum TaskStatus { + Pending, // 等待依赖 + Scheduled, // 依赖满足，已下发给 Worker + Running, // Worker 正在执行 + Completed, // 执行成功 + Failed, // 执行失败 + Skipped // 因上游失败或策略原因被跳过 +} +``` + +### 3.3 调整现有 Messages + +* **`FetchCompanyDataCommand`**: Publisher 变更为 `Workflow Orchestrator`。 +* **`GenerateReportCommand`**: Publisher 变更为 `Workflow Orchestrator`。 + +## 4. Workflow Orchestrator 内部设计 + +### 4.1 DAG 调度器 (DAG Scheduler) +每个 `request_id` 对应一个 DAG 实例。 + +1. **初始化**: 根据 `TemplateID` 读取配置。 + * 创建 Data Fetch Tasks (作为 DAG 的 Root Nodes)。 + * 创建 Analysis Module Tasks (根据 `dependencies` 配置连接边)。 +2. **依赖检查**: + * 监听 Task 状态变更。 + * 当 Task A 变成 `Completed` -> 检查依赖 A 的 Task B。 + * 如果 Task B 的所有依赖都 `Completed` -> 触发 Task B。 + * 如果 Task A `Failed` -> 将依赖 A 的 Task B 标记为 `Skipped` (除非有容错策略)。 + +### 4.2 状态对齐机制 (State Alignment / Snapshot) +为了解决前端刷新或网络丢包导致的状态不一致： + +1. **主动推送快照 (On Connect)**: + * Gateway 在前端建立 SSE 连接时，向 Orchestrator 发送 `SyncStateCommand`。 + * Orchestrator 收到命令后，将当前内存中的完整 DAG 状态打包成 `WorkflowStateSnapshot` 事件发送。 +2. **前端合并逻辑**: + * 前端收到 Snapshot 后，全量替换本地的任务状态树。 + * 如果 Snapshot 显示某任务 `Running`，前端恢复 Loading 动画。 + * 如果 Snapshot 显示某任务 `Completed`，前端渲染结果。 + +### 4.3 容错策略 (Policy) +Orchestrator 需要内置策略来处理非二元结果。 +* **Data Fetch Policy**: 并非所有 Data Fetch 必须成功。可以配置 "At least one data source" 策略。如果满足策略，Orchestrator 将下游的 Analysis Task 依赖视为满足。 + +## 5. 实施步骤 (Implementation Checklist) + +### Phase 1: Contract & Interface +- [x] **Update common-contracts**: + - [x] Add `StartWorkflowCommand` and `SyncStateCommand`. + - [x] Add `WorkflowEvent` enum (incl. Started, StateChanged, StreamUpdate, Completed, Failed, Snapshot). + - [x] Add `WorkflowDag`, `TaskNode`, `TaskType`, `TaskStatus` structs. + - [x] Update publishers for `FetchCompanyDataCommand` and `GenerateReportCommand`. + - [x] Bump version and publish crate. + +### Phase 2: Workflow Orchestrator Service (New) +- [x] **Scaffold Service**: + - [x] Create new Rust service `services/workflow-orchestrator-service`. + - [x] Setup `Dockerfile`, `Cargo.toml`, and `main.rs`. + - [x] Implement NATS connection and multi-topic subscription. +- [x] **Core Logic - State Machine**: + - [x] Implement `WorkflowState` struct (InMemory + Redis/DB persistence optional for MVP). + - [x] Implement `DagScheduler`: Logic to parse template and build dependency graph. +- [x] **Core Logic - Handlers**: + - [x] Handle `StartWorkflowCommand`: Init DAG, fire initial tasks. + - [x] Handle `TaskCompleted` events (from Providers/ReportGen): Update DAG, trigger next tasks. + - [x] Handle `SyncStateCommand`: Serialize current state and emit `WorkflowStateSnapshot`. +- [x] **Policy Engine**: + - [x] Implement "At least one provider" policy for data fetching. + +### Phase 3: API Gateway Refactoring +- [x] **Remove Legacy Logic**: + - [x] Delete `aggregator.rs` completely. + - [x] Remove `trigger_data_fetch` aggregation logic. + - [x] Remove `/api/tasks` polling endpoint. +- [x] **Implement Proxy Logic**: + - [x] Add `POST /api/v2/workflow/start` -> Publishes `StartWorkflowCommand`. + - [x] Add `GET /api/v2/workflow/events/{id}` -> Subscribes to NATS, sends `SyncStateCommand` on open, proxies events to SSE. + +### Phase 4: Integration & Frontend +- [x] **Docker Compose**: Add `workflow-orchestrator-service` to stack. +- [x] **Frontend Adapter**: + - [x] **Type Definitions**: Define `WorkflowEvent`, `WorkflowDag`, `TaskStatus` in `src/types/workflow.ts`. + - [x] **API Proxy**: Implement Next.js Route Handlers for `POST /workflow/start` and `GET /workflow/events/{id}` (SSE). + - [x] **Core Logic (`useWorkflow`)**: + - [x] Implement SSE connection management with auto-reconnect. + - [x] Handle `WorkflowStarted`, `TaskStreamUpdate`, `WorkflowCompleted`. + - [x] Implement state restoration via `WorkflowStateSnapshot`. + - [x] **UI Components**: + - [x] `WorkflowVisualizer`: Task list and status tracking. + - [x] `TaskOutputViewer`: Markdown-rendered stream output. + - [x] `WorkflowReportLayout`: Integrated analysis page layout. + - [x] **Page Integration**: Refactor `app/report/[symbol]/page.tsx` to use the new workflow engine. + +--- +*Updated: 2025-11-20 - Added Implementation Checklist* diff --git a/docs/3_project_management/tasks/completed/20251120_architecture_session_isolation.md b/docs/3_project_management/tasks/completed/20251120_architecture_session_isolation.md new file mode 100644 index 0000000..e6d1371 --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251120_architecture_session_isolation.md @@ -0,0 +1,175 @@ +# 架构修订：基于会话的数据快照与分层存储 (Session-Based Data Snapshotting) + +## 1. 核心理念修订 (Core Philosophy Refinement) + +基于您的反馈，我们修正了架构的核心逻辑，将数据明确划分为两类，并采取不同的存储策略。 + +### 1.1 数据分类 (Data Classification) + +1. **客观历史数据 (Objective History / Time-Series)** + * **定义**: 股价、成交量、K线图等交易数据。 + * **特性**: "出现即历史"，不可篡改，全球唯一。 + * **存储策略**: **全局共享存储**。不需要按 Session 隔离，不需要存多份。 + * **表**: 现有的 `daily_market_data` (TimescaleDB) 保持不变。 + +2. **观测型数据 (Observational Data / Fundamentals)** + * **定义**: 财务报表、公司简介、以及 Provider 返回的原始非结构化或半结构化信息。 + * **特性**: 不同来源（Providers）说法不一；可能随时间修正（Restatement）；分析依赖于“当时”获取的版本。 + * **存储策略**: **基于 Session 的快照存储**。每一次 Session 都必须保存一份当时获取的原始数据的完整副本。 + * **表**: 新增 `session_raw_data` 表。 + +### 1.2 解决的问题 +* **会话隔离**: 新的 Session 拥有自己独立的一套基础面数据，不受历史 Session 干扰，也不污染未来 Session。 +* **历史回溯**: 即使 Provider 变了，查看历史 Report 时，依然能看到当时是基于什么数据得出的结论。 +* **数据清洗解耦**: 我们现在只负责“收集并快照”，不负责“清洗和聚合”。复杂的清洗逻辑（WASM/AI）留待后续模块处理。 + +--- + +## 2. 数据库架构设计 (Schema Design) + +### 2.1 新增：会话原始数据表 (`session_raw_data`) + +这是本次架构调整的核心。我们不再试图把财务数据强行塞进一个全局唯一的标准表，而是忠实记录每个 Provider 在该 Session 中返回的内容。 + +```sql +CREATE TABLE session_raw_data ( + id BIGSERIAL PRIMARY KEY, + request_id UUID NOT NULL, -- 关联的 Session ID + symbol VARCHAR(32) NOT NULL, + provider VARCHAR(64) NOT NULL, -- e.g., 'tushare', 'alphavantage' + data_type VARCHAR(32) NOT NULL, -- e.g., 'financial_statements', 'company_profile' + + -- 核心：直接存储 Provider 返回的（或稍微标准化的）完整 JSON + data_payload JSONB NOT NULL, + + created_at TIMESTAMPTZ DEFAULT NOW(), + + -- 索引：为了快速查询某次 Session 的数据 + CONSTRAINT fk_request_id FOREIGN KEY (request_id) REFERENCES requests(id) ON DELETE CASCADE +); + +CREATE INDEX idx_session_data_req ON session_raw_data(request_id); +``` + +### 2.2 新增：供应商缓存表 (`provider_response_cache`) + +为了优化性能和节省 API 调用次数，我们在全局层引入缓存。但请注意：**缓存仅作为读取源，不作为 Session 的存储地。** + +```sql +CREATE TABLE provider_response_cache ( + cache_key VARCHAR(255) PRIMARY KEY, -- e.g., "tushare:AAPL:financials" + data_payload JSONB NOT NULL, + updated_at TIMESTAMPTZ DEFAULT NOW(), + expires_at TIMESTAMPTZ NOT NULL +); +``` + +### 2.3 保持不变：市场数据表 (`daily_market_data`) +* 继续使用 TimescaleDB 存储 `open`, `high`, `low`, `close`, `volume`。 +* 所有 Session 共享读取此表。 + +--- + +## 3. 数据流转逻辑 (Data Lifecycle) + +### Phase 1: Session 启动与数据获取 (Acquisition) + +1. **Start**: API Gateway 生成 `request_id`。 +2. **Fetch & Cache Logic (在 Provider Service 中执行)**: + * Provider 收到任务 (Symbol: AAPL)。 + * **Check Cache**: 查询 `provider_response_cache`。 + * *Hit*: 拿出现成的 JSON。 + * *Miss*: 调用外部 API，获得 JSON，写入 Cache (设置过期时间如 24h)。 +3. **Snapshot (关键步骤)**: + * Provider 将拿到的 JSON (无论来自 Cache 还是 API)，作为一条**新记录**写入 `session_raw_data`。 + * 字段: `request_id=UUID`, `provider=tushare`, `data=JSON`。 + +### Phase 2: 展示与分析 (Consumption) + +1. **Frontend Raw View (UI)**: + * 前端调用 `GET /api/v1/session/{request_id}/raw-data`。 + * 后端 `SELECT * FROM session_raw_data WHERE request_id = ...`。 + * UI 依然可以使用之前的 Accordion 结构，展示 "Tushare: Financials", "AlphaVantage: Profile"。这就是用户看到的“本次调查的原始底稿”。 + +2. **Analysis (LLM)**: + * Report Generator 获取 `request_id` 对应的所有 raw data。 + * 将这些 Raw Data 作为 Context 喂给 LLM。 + * (未来扩展): 在这一步之前，插入一个 "Data Cleaning Agent/Wasm"，读取 raw data，输出 clean data，再喂给 LLM。 + +### Phase 3: 归档与清理 (Cleanup) + +* **Session Deletion**: 当我们需要清理某个历史 Session 时，只需 `DELETE FROM session_raw_data WHERE request_id = ...`。 +* **副作用**: 零。因为 `daily_market_data` 是共享的（留着也没事），而 Session 独享的 `raw_data` 被彻底删除了。 + +--- + +## 4. 实施路线图 (Implementation Roadmap) + +1. **Database Migration**: + * 创建 `session_raw_data` 表。 + * 创建 `provider_response_cache` 表。 + * (清理旧表): 废弃 `time_series_financials` 表（原计划用于存标准化的财务指标，现在确认不需要。我们只存 `session_raw_data` 中的原始基本面数据，财务报表由原始数据动态推导）。 + * **保留** `daily_market_data` 表（存储股价、K线等客观时间序列数据，保持全局共享）。 + +2. **Provider Services**: + * 引入 Cache 检查逻辑。 + * 修改输出逻辑：不再尝试 Upsert 全局表，而是 Insert `session_raw_data`。 + +3. **Frontend Refactor**: + * 修改 `RawDataViewer` 的数据源，从读取“最后一次更新”改为读取“当前 Session 的 Raw Data”。 + * 这完美解决了“刷新页面看到旧数据”的问题——如果是一个新 Session ID，它的 `session_raw_data` 一开始是空的，UI 就会显示为空/Loading，直到新的 Snapshot 写入。 + +4. **Future Extensibility (Aggregation)**: + * 当前架构下，Frontend 直接展示 Raw Data。 + * 未来：新增 `DataProcessorService`。它监听 "Data Fetched" 事件，读取 `session_raw_data`，执行聚合逻辑，将结果写入 `session_clean_data` (假想表)，供 UI 显示“完美报表”。 + +--- + +## 5. Step-by-Step Task List + +### Phase 1: Data Persistence Service & Database (Foundation) +- [x] **Task 1.1**: Create new SQL migration file. + - Define `session_raw_data` table (Columns: `id`, `request_id`, `symbol`, `provider`, `data_type`, `data_payload`, `created_at`). + - Define `provider_response_cache` table (Columns: `cache_key`, `data_payload`, `updated_at`, `expires_at`). + - (Optional) Rename `time_series_financials` to `_deprecated_time_series_financials` to prevent accidental usage. +- [x] **Task 1.2**: Run SQL migration (`sqlx migrate run`). +- [x] **Task 1.3**: Implement `db/session_data.rs` in Data Persistence Service. + - Function: `insert_session_data(pool, request_id, provider, data_type, payload)`. + - Function: `get_session_data(pool, request_id)`. +- [x] **Task 1.4**: Implement `db/provider_cache.rs` in Data Persistence Service. + - Function: `get_cache(pool, key) -> Option`. + - Function: `set_cache(pool, key, payload, ttl)`. +- [x] **Task 1.5**: Expose new API endpoints in `api/`. + - `POST /api/v1/session-data` (Internal use by Providers). + - `GET /api/v1/session-data/:request_id` (Used by ReportGen & Frontend). + - `GET/POST /api/v1/provider-cache` (Internal use by Providers). + +### Phase 2: Common Contracts & SDK (Glue Code) +- [x] **Task 2.1**: Update `common-contracts`. + - Add DTOs for `SessionData` and `CacheEntry`. + - Update `PersistenceClient` struct to include methods for calling new endpoints (`save_session_data`, `get_cache`, `set_cache`). + +### Phase 3: Provider Services (Logic Update) +- [x] **Task 3.1**: Refactor `tushare-provider-service`. + - Update Worker to check Cache first. + - On Cache Miss: Call Tushare API -> Save to Cache. + - **Final Step**: Post data to `POST /api/v1/session-data` (instead of old batch insert). + - Ensure `request_id` is propagated correctly. +- [x] **Task 3.2**: Refactor `alphavantage-provider-service` (same logic). +- [x] **Task 3.3**: Refactor `yfinance-provider-service` (same logic). +- [x] **Task 3.4**: Verify `FinancialsPersistedEvent` is still emitted (or similar event) to trigger Gateway aggregation. + +### Phase 4: API Gateway & Report Generator (Consumption) +- [x] **Task 4.1**: Update `api-gateway` routing. + - Proxy `GET /api/v1/session-data/:request_id` for Frontend. +- [x] **Task 4.2**: Update `report-generator-service`. + - In `worker.rs`, change data fetching logic. + - Instead of `get_financials_by_symbol`, call `get_session_data(request_id)`. + - Pass the raw JSON list to the LLM Context Builder. + +### Phase 5: Frontend (UI Update) +- [x] **Task 5.1**: Update `useReportEngine.ts`. + - Change polling/fetching logic to request `GET /api/v1/session-data/${requestId}`. +- [x] **Task 5.2**: Update `RawDataViewer.tsx`. + - Adapt to new data structure (List of `{ provider, data_type, payload }`). + - Ensure the UI correctly groups these raw snapshots by Provider. diff --git a/docs/3_project_management/tasks/completed/20251120_dynamic_service_registration_proposal.md b/docs/3_project_management/tasks/completed/20251120_dynamic_service_registration_proposal.md new file mode 100644 index 0000000..66757bd --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251120_dynamic_service_registration_proposal.md @@ -0,0 +1,110 @@ +# 动态服务注册与发现机制设计方案 (Dynamic Service Registration & Discovery Proposal) + +## 1. 问题陈述 (Problem Statement) +目前的 **API Gateway** 依赖于静态配置（环境变量中的 `provider_services` 映射表）来获知可用的数据提供商服务 (Data Provider Services)。 +* **脆弱性 (Brittleness)**: 增加或迁移 Provider 需要修改 Gateway 配置并重启。 +* **缺乏健康感知 (Lack of Health Awareness)**: Gateway 会盲目地尝试连接配置的 URL。如果某个服务挂了（但配置还在），请求会遭遇超时或连接错误。 +* **运维复杂 (Operational Complexity)**: 手动管理 URL 既机械又容易出错。 + +## 2. 解决方案：动态注册系统 (Dynamic Registration System) +我们将实施**服务注册 (Service Registry)** 模式，由 API Gateway 充当注册中心。 + +### 2.1. "注册" 生命周期 +1. **启动 (Startup)**: 当一个 Provider Service (例如 Tushare) 启动时，它向 API Gateway 发送 `POST /v1/registry/register` 请求。 + * 载荷包括：服务 ID、基础 URL、能力标识（如 "tushare"）。 +2. **存活心跳 (Liveness/Heartbeat)**: Provider Service 运行一个后台任务，每隔 **N 秒** (建议 **10秒**) 发送一次 `POST /v1/registry/heartbeat`。 + * **注意**: 由于我们主要在本地容器网络运行，网络开销极低，我们可以使用较短的心跳周期（如 10秒）来实现快速的故障检测。 +3. **发现 (Discovery)**: API Gateway 在内存中维护活跃服务列表。 + * 如果超过 **2 * N 秒** (如 20秒) 未收到心跳，该服务将被标记为“不健康”或被移除。 +4. **关闭 (Shutdown)**: 在优雅退出 (Graceful Shutdown, SIGTERM/SIGINT) 时，Provider 发送 `POST /v1/registry/deregister`。 + +### 2.2. 架构变更 + +#### A. 共享契约 (`common-contracts`) +定义注册所需的数据结构。 + +```rust +// services/common-contracts/src/registry.rs + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ServiceRegistration { + pub service_id: String, // 唯一ID, 例如 "tushare-provider-1" + pub service_name: String, // 类型, 例如 "tushare" + pub base_url: String, // 例如 "http://10.0.1.5:8000" + pub health_check_url: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct Heartbeat { + pub service_id: String, + pub status: ServiceStatus, // Active, Degraded +} +``` + +#### B. API Gateway (`api-gateway`) +* **新组件**: `ServiceRegistry` (带 TTL 的线程安全 Map)。 +* **新接口**: + * `POST /v1/registry/register`: 添加/更新条目。 + * `POST /v1/registry/heartbeat`: 刷新 TTL。 + * `POST /v1/registry/deregister`: 移除条目。 +* **逻辑变更**: `get_task_progress` 和 `trigger_data_fetch` 将不再读取静态配置，而是查询动态的 `ServiceRegistry`。 + +#### C. Provider Services (`*-provider-service`) +我们需要一个统一的机制来处理这个生命周期。 +建议在 `common-contracts` 中引入一个标准的生命周期处理模块。 + +**建议的 Trait / 辅助结构体:** + +```rust +// services/common-contracts/src/lifecycle.rs (New) + +pub struct ServiceRegistrar { + gateway_url: String, + registration: ServiceRegistration, + // ... +} + +impl ServiceRegistrar { + /// 注册服务 (重试直到成功) + pub async fn register(&self) -> Result<()>; + /// 启动后台心跳循环 (10s 间隔) + pub async fn start_heartbeat_loop(&self); + /// 注销服务 + pub async fn deregister(&self) -> Result<()>; +} +``` + +## 3. 实施计划 (TODO List) + +### Phase 1: 基础建设 (Infrastructure) +* [ ] **Task 1.1 (Contracts)**: 在 `services/common-contracts` 中创建 `registry.rs`，定义 `ServiceRegistration` 和 `Heartbeat` 结构体。 +* [ ] **Task 1.2 (Library)**: 在 `services/common-contracts` 中实现 `ServiceRegistrar` 逻辑。 + * 包含重试机制的 `register`。 + * 包含 `tokio::time::interval` (10s) 的 `start_heartbeat_loop`。 + * 确保能从环境变量 (如 `API_GATEWAY_URL`) 获取 Gateway 地址。 +* [ ] **Task 1.3 (Gateway Core)**: 在 `api-gateway` 中实现 `ServiceRegistry` 状态管理（使用 `Arc>>`）。 +* [ ] **Task 1.4 (Gateway API)**: 在 `api-gateway` 中添加 `/v1/registry/*` 路由并挂载 Handler。 + +### Phase 2: Provider 改造 (Provider Migration) +*由于所有 Provider 架构一致，以下步骤需在 `tushare`, `finnhub`, `alphavantage`, `yfinance` 四个服务中重复执行：* + +* [ ] **Task 2.1 (Config)**: 更新 `AppConfig`，增加 `gateway_url` 配置项。 +* [ ] **Task 2.2 (Main Loop)**: 修改 `main.rs`。 + * 初始化 `ServiceRegistrar`。 + * 在 HTTP Server 启动前（或同时）调用 `registrar.register().await`。 + * 使用 `tokio::spawn` 启动 `registrar.start_heartbeat_loop()`。 +* [ ] **Task 2.3 (Shutdown)**: 添加 Graceful Shutdown 钩子，确保在收到 Ctrl+C 时调用 `registrar.deregister()`。 + +### Phase 3: 消费端适配 (Gateway Consumption) +* [ ] **Task 3.1**: 修改 `api-gateway` 的 `test_data_source_config`，不再查 Config，改为查 Registry。 +* [ ] **Task 3.2**: 修改 `api-gateway` 的 `trigger_data_fetch`，根据 `service_name` (如 "tushare") 从 Registry 查找可用的 `base_url`。 + * 如果找到多个同名服务，可以做简单的 Load Balance（轮询）。 +* [ ] **Task 3.3**: 修改 `api-gateway` 的 `get_task_progress`，遍历 Registry 中的所有服务来聚合状态。 + +### Phase 4: 清理 (Cleanup) +* [ ] **Task 4.1**: 移除 `api-gateway` 中关于 `provider_services` 的静态配置代码和环境变量。 + +## 4. 预期收益 +* **即插即用 (Plug-and-Play)**: 启动一个新的 Provider 实例，它会自动出现在系统中。 +* **自愈 (Self-Healing)**: 如果 Provider 崩溃，它会从注册表中消失（TTL 过期），Gateway 不会再向其发送请求，避免了无意义的等待和超时。 +* **零配置 (Zero-Config)**: 扩容或迁移 Provider 时无需修改 Gateway 环境变量。 diff --git a/docs/3_project_management/tasks/completed/20251120_frontend_refactor_plan.md b/docs/3_project_management/tasks/completed/20251120_frontend_refactor_plan.md new file mode 100644 index 0000000..ba7ce7d --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251120_frontend_refactor_plan.md @@ -0,0 +1,45 @@ +# 前端架构重构计划：状态管理与工作流控制权移交 + +## 1. 背景与现状 +当前的 `fundamental-analysis` 前端项目源自一个 POC (Proof of Concept) 原型。在快速迭代过程中，遗留了大量“为了跑通流程而写”的临时逻辑。核心问题在于**前端承担了过多的业务控制逻辑**，导致前后端状态不一致、错误处理困难、用户体验割裂。 + +### 核心痛点 +1. **“自嗨式”状态流转**：前端自行判断何时从“数据获取”切换到“分析报告”阶段（基于轮询结果推断），而非响应后端的明确指令。 +2. **脆弱的 Polling + SSE 混合模式**：前端先轮询 HTTP 接口查询进度，再断开连接 SSE 流。这两者之间存在状态断层，且严重依赖 HTTP 接口的实时性（而这个接口又是后端实时聚合下游得来的，极易超时）。 +3. **缺乏统一的状态源 (Source of Truth)**：前端维护了一套复杂的 `ReportState`，后端也有一套状态，两者通过不稳定的网络请求同步，经常出现“前端显示完成，后端还在跑”或“后端报错，前端还在转圈”的情况。 + +## 2. 重构目标 +**原则：前端归前端（UI展示），后端归后端（业务逻辑与流转控制）。** + +1. **控制权移交**：所有涉及业务流程流转（Phase Transition）的逻辑，必须由后端通过事件或状态字段明确驱动。前端只负责渲染当前状态。 +2. **单一数据流 (Single Stream)**：废除“HTTP Polling -> SSE”的混合模式，建立统一的 WebSocket 或 SSE 通道。从发请求那一刻起，所有状态变更（包括数据获取进度、分析进度、报错）全由服务端推送。 +3. **简化状态机**：前端 `useReportEngine` 钩子应简化为单纯的“状态订阅者”，不再包含复杂的判断逻辑（如 `if (tasks.every(t => t.success)) switchPhase()`）。 + +## 3. 实施方案 (Tasks) + +### Phase 1: 后端基础设施准备 (Backend Readiness) +- [ ] **统一事件流接口**：在 `api-gateway` 实现一个统一的 SSE/WebSocket 端点（如 `/v2/workflow/events`）。 + - 该端点应聚合：`DataFetchProgress` (NATS), `WorkflowStart` (NATS), `ModuleProgress` (ReportGenerator), `WorkflowComplete`。 +- [ ] **Gateway 状态缓存**：`api-gateway` 需要维护一个轻量级的 Request 状态缓存（Redis 或 内存），不再实时透传查询请求给下游 Provider，而是直接返回缓存的最新状态。 +- [ ] **定义统一状态协议**：制定前后端通用的状态枚举（`PENDING`, `DATA_FETCHING`, `ANALYZING`, `COMPLETED`, `FAILED`）。 + +### Phase 2: 前端逻辑剥离 (Frontend Refactoring) +- [ ] **废除 useReportEngine 里的推断逻辑**：删除所有 `useEffect` 里关于状态切换的 `if/else` 判断代码。 +- [ ] **实现 Event-Driven Hook**：重写 `useReportEngine`，使其核心逻辑变为：连接流 -> 收到事件 -> 更新 State。 + - 收到 `STATUS_CHANGED: DATA_FETCHING` -> 显示数据加载 UI。 + - 收到 `STATUS_CHANGED: ANALYZING` -> 自动切换到分析 UI（无需前端判断数据是否齐备）。 + - 收到 `ERROR` -> 显示错误 UI。 +- [ ] **清理旧代码**：移除对 `/api/tasks` 轮询的依赖代码。 + +### Phase 3: 验证与兜底 +- [ ] **断线重连机制**：实现 SSE/WS 的自动重连，并能从后端获取“当前快照”来恢复状态，防止刷新页面丢失进度。 +- [ ] **超时兜底**：仅保留最基本的网络超时提示（如“服务器连接中断”），不再处理业务逻辑超时。 + +## 4. 复杂度评估与建议 +- **复杂度**：中等偏高 (Medium-High)。涉及前后端协议变更和核心 Hook 重写。 +- **风险**：高。这是系统的心脏部位，重构期间可能会导致整个分析流程暂时不可用。 +- **建议**：**单独开一个线程（Branch/Session）进行**。不要在当前修复 Bug 的线程中混合进行。这需要系统性的设计和逐步替换，无法通过简单的 Patch 完成。 + +--- +*Created: 2025-11-20* + diff --git a/docs/3_project_management/tasks/completed/20251120_log_analysis_and_debugging.md b/docs/3_project_management/tasks/completed/20251120_log_analysis_and_debugging.md new file mode 100644 index 0000000..ec7375f --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251120_log_analysis_and_debugging.md @@ -0,0 +1,59 @@ +# 系统日志分析与调试报告 (2025-11-20) + +## 1. 系统现状快照 + +基于 `scripts/inspect_logs.sh` 的执行结果，当前系统各服务状态如下： + +| 服务名称 | 状态 | 关键日志/行为 | +| :--- | :--- | :--- | +| **API Gateway** | 🟢 Running | 成功接收数据获取请求 (`FetchCompanyDataCommand`)；成功注册服务；**未观测到**发送 `GenerateReportCommand`。 | +| **Data Persistence** | 🟢 Running | 数据库连接正常；成功写入 `session_data` (Source: `yfinance`, `tushare`)。 | +| **Report Generator** | 🟢 Running | 已启动并连接 NATS；**无**收到任务的日志；服务似乎在 13:43 重启过。 | +| **Alphavantage** | 🟢 Running | 任务执行成功 (Task Completed)。 | +| **YFinance** | 🟢 Running | 任务执行成功 (Cache HIT)。 | +| **Tushare** | 🟢 Running | 配置轮询正常；有数据写入记录。 | +| **Finnhub** | 🟡 Degraded | **配置错误**：`No enabled Finnhub configuration found`，导致服务降级，无法执行任务。 | +| **NATS** | 🟢 Running | 正常运行。 | + +## 2. 现象分析 + +### 2.1 核心问题：报告生成流程中断 +用户反馈 "点击后无反应/报错"，日志显示： +1. **数据获取阶段 (Data Fetch)**： + * API Gateway 接收到了数据获取请求 (Req ID: `935e6999...`)。 + * Alphavantage, YFinance, Tushare 成功响应并写入数据。 + * **Finnhub 失败/超时**：由于配置缺失，Finnhub Provider 处于降级状态，无法处理请求。 + * API Gateway 的 Aggregator 显示 `Received 2/4 responses`。它可能在等待所有 Provider 返回，导致整体任务状态卡在 "InProgress"。 + +2. **报告生成阶段 (Report Generation)**： + * **完全未触发**。`api-gateway` 日志中没有 `Publishing analysis generation command`。 + * `report-generator-service` 日志中没有 `Received NATS command`。 + +### 2.2 根因推断 +前端 (Frontend) 或 API Gateway 的聚合逻辑可能存在**"全有或全无" (All-or-Nothing)** 的依赖： +* 前端通常轮询 `/tasks/{id}`。 +* 如果 Finnhub 任务从未完成（挂起或失败未上报），聚合状态可能永远不是 "Completed"。 +* 前端因此卡在进度条，从未发送 `POST /analysis-requests/{symbol}` 来触发下一步的报告生成。 + +## 3. 潜在风险与待办 + +1. **Finnhub 配置缺失**：导致服务不可用，拖累整体流程。 +2. **容错性不足**：单个 Provider (Finnhub) 的失败似乎阻塞了整个 Pipeline。我们需要确保 "部分成功" 也能继续后续流程。 +3. **Report Generator 重启**：日志显示该服务在 13:43 重启。如果此前有请求，可能因 Crash 丢失。需要关注其稳定性。 + +## 4. 下一步调试与修复计划 + +### Phase 1: 修复阻塞点 +- [ ] **修复 Finnhub 配置**：检查数据库中的 `data_sources_config`，确保 Finnhub 有效启用且 API Key 正确。 +- [ ] **验证容错逻辑**：检查 API Gateway 的 `Aggregator` 和 Frontend 的 `useReportEngine`，确保设置超时机制。如果 3/4 成功，1/4 超时，应允许用户继续生成报告。 + +### Phase 2: 验证报告生成器 +- [ ] **手动触发**：使用 Postman 或 `curl` 直接调用 `POST http://localhost:4000/v1/analysis-requests/{symbol}`，绕过前端等待逻辑，验证 Report Generator 是否能正常工作。 +- [ ] **观察日志**：确认 Report Generator 收到指令并开始流式输出。 + +### Phase 3: 增强可观测性 +- [ ] **完善日志**：Report Generator 的日志偏少，建议增加 "Start processing module X" 等详细步骤日志。 + +--- +*Report generated by AI Assistant.* + diff --git a/docs/3_project_management/tasks/completed/20251120_parallel_provider_status_ui.md b/docs/3_project_management/tasks/completed/20251120_parallel_provider_status_ui.md new file mode 100644 index 0000000..4c51aba --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251120_parallel_provider_status_ui.md @@ -0,0 +1,90 @@ +# UI Improvement: Parallel Data Provider Status & Error Reporting + +## 1. Problem Statement +Currently, the Fundamental Analysis page shows a generic "Fetching Data..." loading state. The detailed status and errors from individual data providers (Tushare, YFinance, AlphaVantage) are aggregated into a single status in the API Gateway. + +This causes two issues: +1. **Ambiguity**: Users cannot see which provider is working, finished, or failed. +2. **Hidden Errors**: If one provider fails (e.g., database error) but the overall task is still "in progress" (or generic failed), the specific error details are lost or not displayed prominently. + +## 2. Goal +Update the API and UI to reflect the parallel nature of data fetching. The UI should display a "control panel" style view where each Data Provider has its own status card, showing: +- Provider Name (e.g., "Tushare") +- Current Status (Queued, In Progress, Completed, Failed) +- Progress Details (e.g., "Fetching data...", "Persisting...", "Error: 500 Internal Server Error") + +## 3. Proposed Changes + +### 3.1 Backend (API Gateway) +**Endpoint**: `GET /v1/tasks/{request_id}` + +**Current Behavior**: Returns a single `TaskProgress` object (the first one found). + +**New Behavior**: Returns a list of all tasks associated with the `request_id`. + +**Response Schema Change**: +```json +// BEFORE +{ + "request_id": "uuid", + "task_name": "tushare:600519.SS", + "status": "in_progress", + ... +} + +// AFTER +[ + { + "request_id": "uuid", + "task_name": "tushare:600519.SS", + "status": "failed", + "details": "Error: 500 ...", + ... + }, + { + "request_id": "uuid", + "task_name": "yfinance:600519.SS", + "status": "completed", + ... + } +] +``` + +### 3.2 Frontend + +#### Types +Update `TaskProgress` handling to support array responses. + +#### Logic (`useReportEngine` & `useTaskProgress`) +- **Aggregation Logic**: + - The overall "Phase Status" (Fetching vs Complete) depends on *all* provider tasks. + - **Fetching**: If *any* task is `queued` or `in_progress`. + - **Complete**: When *all* tasks are `completed` or `failed`. +- **Error Handling**: Do not fail the whole report if one provider fails. Allow partial success. + +#### UI (`RawDataViewer` & `FinancialTable`) +Replace the single loader with a grid layout: + +```tsx +// Conceptual Layout +
+ + + +
+``` + +**Card States**: +- **Waiting**: Gray / Spinner +- **Success**: Green Checkmark + "Data retrieved" +- **Error**: Red X + Error Message (expanded or tooltip) + +## 4. Implementation Steps +1. **Backend**: Modify `services/api-gateway/src/api.rs` to return `Vec`. +2. **Frontend**: + - Update `TaskProgress` type definition. + - Update `useTaskProgress` fetcher. + - Update `useReportEngine` polling logic to handle array. + - Create `ProviderStatusCard` component. + - Update `RawDataViewer` to render the grid. + diff --git a/docs/3_project_management/tasks/completed/20251120_system_lifecycle_analysis.md b/docs/3_project_management/tasks/completed/20251120_system_lifecycle_analysis.md new file mode 100644 index 0000000..8e22b33 --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251120_system_lifecycle_analysis.md @@ -0,0 +1,99 @@ +# 系统生命周期与异常处理分析 (System Lifecycle Analysis) + +## 1. 核心问题 (Core Issue) +目前系统的业务逻辑缺乏**确定性 (Determinism)** 和 **闭环 (Closed-loop Lifecycle)**。 +虽然各个微服务独立运行，但缺乏统一的状态协调机制。当“快乐路径” (Happy Path) 被打断（如DB报错）时，下游服务无法感知上游的失败，导致系统处于“僵尸状态” (Zombie State)。 + +> **用户反馈**：“有始必有终...你接了这个任务你就要负责把它结束掉...我们既然是微服务,那这个有始有终,可以说是跟生命性一样重要的一个基本原则。” + +## 2. 现状分析 (Current State Analysis) + +### 2.1 当前的数据流与控制流 +```mermaid +sequenceDiagram + User->>API Gateway: 1. POST /data-requests + API Gateway->>NATS: 2. Pub "data_fetch_commands" + + par Provider Execution + NATS->>Provider: 3. Receive Command + Provider->>Provider: 4. Fetch External Data + Provider-->>DB: 5. Persist Data (Upsert) + end + + rect rgb(20, 0, 0) + Note right of DB: [CRITICAL FAILURE POINT] + DB-->>Provider: 500 Error (Panic) + end + + alt Happy Path + Provider->>NATS: 6. Pub "events.financials_persisted" + NATS->>Report Gen: 7. Trigger Analysis + else Failure Path (Current) + Provider->>Log: Log Error + Provider->>TaskStore: Update Task = Failed + Note right of Provider: 链条在此断裂 (Chain Breaks Here) + end + + User->>API Gateway: 8. Poll Task Status + API Gateway-->>User: All Failed + + User->>User: 9. Frontend Logic: "All Done" -> Switch to Analysis UI + User->>API Gateway: 10. Connect SSE (Analysis Stream) + Note right of User: Hangs forever (Waiting for Report Gen that never started) +``` + +### 2.2 存在的具体缺陷 + +1. **隐式依赖链 (Implicit Dependency Chain)**: + * Report Generator 被动等待 `FinancialsPersistedEvent`。如果 Provider 挂了，事件永远不发，Report Generator 就像一个不知道此时该上班的工人，一直在睡觉。 + +2. **缺乏全局协调者 (Lack of Orchestration)**: + * API Gateway 把命令发出去就不管了（除了被动提供查询）。 + * 没有人负责说：“嘿，数据获取全部失败了，取消本次分析任务。” + +3. **前端的状态误判**: + * 前端认为 `Failed` 也是一种 `Completed`（终止态），这是对的。但前端错误地假设“只要终止了就可以进行下一步”。 + * **修正原则**：只有 `Success` 才能驱动下一步。`Failed` 应该导致整个工作流的**熔断 (Circuit Break)**。 + +## 3. 改进方案 (Improvement Plan) + +我们需要引入**Rustic**的确定性原则：**如果不能保证成功，就明确地失败。** + +### 3.1 方案一：引入显式的工作流状态 (Explicit Workflow State) - 推荐 +我们不需要引入沉重的 Workflow Engine (如 Temporal)，但在逻辑上必须闭环。 + +**后端改进：** +1. **修复数据库错误**：这是首要任务。`unexpected null` 必须被修复。 +2. **事件驱动的失败传播 (Failure Propagation)**： + * 如果 Provider 失败，发送 `events.data_fetch_failed`。 + * Report Generator 或者 API Gateway 监听这个失败事件？ + * **更好方案**：Report Generator 不需要监听失败。API Gateway 需要聚合状态。 + +**前端/交互改进：** +1. **熔断机制**： + * 在 `useReportEngine` 中，如果所有 Task 都是 `Failed`，**绝对不要**进入 Analysis 阶段。 + * 直接在界面显示：“数据获取失败，无法生成最新报告。是否查看历史数据？” + +### 3.2 具体的实施步骤 (Action Items) + +#### Phase 1: 修复根本错误 (Fix the Root Cause) +* **Task**: 调试并修复 `data-persistence-service` 中的 `500 Internal Server Error`。 + * 原因推测：数据库 schema 中某列允许 NULL，但 Rust 代码中定义为非 Option 类型；或者反之。 + * 错误日志：`unexpected null; try decoding as an Option`。 + +#### Phase 2: 完善生命周期逻辑 (Lifecycle Logic) +* **Task (Frontend)**: 修改 `useReportEngine`。 + * 逻辑变更：`if (allTasksFailed) { stop(); show_error(); }` + * 逻辑变更：`if (partialSuccess) { proceed_with_warning(); }` +* **Task (Backend - ReportGen)**: 增加超时机制。 + * 如果用户连接了 SSE 但长时间没有数据（因为没收到事件），应该发送一个 Timeout 消息给前端，结束连接，而不是无限挂起。 + +## 4. 结论 +目前的“卡在 Analyzing”是因为**上游失败导致下游触发器丢失**，叠加**前端盲目推进流程**导致的。 +我们必须： +1. 修好 DB 错误（让快乐路径通畅）。 +2. 在前端增加“失败熔断”，不要在没有新数据的情况下假装去分析。 + +--- +*Created: 2025-11-20* + diff --git a/docs/3_project_management/tasks/completed/20251120_system_status_and_debugging_guide.md b/docs/3_project_management/tasks/completed/20251120_system_status_and_debugging_guide.md new file mode 100644 index 0000000..ba70e6f --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251120_system_status_and_debugging_guide.md @@ -0,0 +1,110 @@ +# 系统日志分析与调试操作指南 (System Debugging Guide) + +本文档旨在记录当前系统的运行状况、已知问题以及标准化的调试流程。它将指导开发人员如何利用现有工具（如 Docker、Tilt、自定义脚本）快速定位问题。 + +## 1. 系统现状 (System Status Snapshot) + +截至 2025-11-20，Fundamental Analysis 系统由多个微服务组成，采用 Docker Compose 编排，并通过 Tilt 进行开发环境的热重载管理。 + +### 1.1 服务概览 + +| 服务名称 | 职责 | 当前状态 | 关键依赖 | +| :--- | :--- | :--- | :--- | +| **API Gateway** | 流量入口，任务分发，服务发现 | 🟢 正常 | NATS, Providers | +| **Report Generator** | 接收指令，调用 LLM 生成报告 | 🟢 正常 (但在等待任务) | NATS, Data Persistence, LLM API | +| **Data Persistence** | 数据库读写，配置管理，Session 数据隔离 | 🟢 正常 (已恢复 Seeding) | Postgres | +| **Alphavantage** | 美股数据 Provider | 🟢 正常 | NATS, External API | +| **YFinance** | 雅虎财经 Provider | 🟢 正常 | NATS, External API | +| **Tushare** | A股数据 Provider | 🟢 正常 | NATS, External API | +| **Finnhub** | 市场数据 Provider | 🟡 **降级 (Degraded)** | 缺少 API Key 配置 | + +### 1.2 核心问题：报告生成流程阻塞 +目前用户在前端点击 "生成报告" 后无反应。 +* **现象**：API Gateway 未收到生成报告的请求，Report Generator 未收到 NATS 消息。 +* **原因推断**：Finnhub Provider 因配置缺失处于 "Degraded" 状态，导致前端轮询的任务列表 (`GET /tasks/{id}`) 中始终包含未完成/失败的任务。前端逻辑可能因等待所有 Provider 完成而阻塞了后续 "Generate Report" 请求的发送。 + +--- + +## 2. 运维与开发流程 (DevOps & Workflow) + +我们使用 **Tilt** 管理 Docker Compose 环境。这意味着你不需要手动 `docker-compose up/down` 来应用代码变更。 + +### 2.1 启动与更新 +1. **启动环境**： + 在项目根目录运行： + ```bash + tilt up + ``` + 这会启动所有服务，并打开 Tilt UI (通常在 `http://localhost:10350`)。 + +2. **代码更新**： + * 直接在 IDE 中修改代码并保存。 + * **Tilt 会自动检测变更**： + * 如果是前端代码，Tilt 会触发前端热更新。 + * 如果是 Rust 服务代码，Tilt 会在容器内或宿主机触发增量编译并重启服务。 + * **操作建议**：修改代码后，只需**等待一会儿**，观察 Tilt UI 变绿即可。无需手动重启容器。 + +3. **配置变更**： + * 如果修改了 `docker-compose.yml` 或 `.env`，Tilt 通常也会检测到并重建相关资源。 + +### 2.2 快速重置数据库 (如有必要) +如果遇到严重的数据不一致或认证问题，可使用以下命令重置数据库（**警告：数据将丢失，但会自动 Seed 默认模板**）： +```bash +docker-compose down postgres-db +docker volume rm fundamental_analysis_pgdata +docker-compose up -d postgres-db +# 等待几秒后 +# Tilt 会自动重启依赖 DB 的服务，触发 Seeding +``` + +--- + +## 3. 调试与分析工具 (Debugging Tools) + +为了快速诊断跨服务的问题，我们提供了一个能够聚合查看所有容器最新日志的脚本。 + +### 3.1 `inspect_logs.sh` 使用指南 + +该脚本位于 `scripts/inspect_logs.sh`。它能一次性输出所有关键服务的最后 N 行日志，避免手动切换容器查看。 + +* **基本用法** (默认显示最后 10 行)： + ```bash + ./scripts/inspect_logs.sh + ``` + +* **指定行数** (例如查看最后 50 行)： + ```bash + ./scripts/inspect_logs.sh 50 + ``` + +### 3.2 分析策略 + +当遇到 "点击无反应" 或 "流程卡住" 时，请按以下步骤操作： + +1. **运行脚本**：`./scripts/inspect_logs.sh 20` +2. **检查 API Gateway**： + * 是否有 `Received data fetch request`？ -> 如果无，说明前端没发请求。 + * 是否有 `Publishing analysis generation command`？ -> 如果无，说明 Gateway 没收到生成指令，或者内部逻辑（如等待 Provider）卡住了。 +3. **检查 Provider**： + * 是否有 `Degraded` 或 `Error` 日志？(如当前的 Finnhub 问题) +4. **检查 Report Generator**： + * 是否有 `Received NATS command`？ -> 如果无，说明消息没发过来。 + +--- + +## 4. 当前待办与修复建议 (Action Items) + +为了打通流程，我们需要解决 Finnhub 导致的阻塞问题。 + +1. **修复配置**： + * 在 `config/data_sources.yaml` (或数据库 `configs` 表) 中配置有效的 Finnhub API Key。 + * 或者，暂时在配置中**禁用** Finnhub (`enabled: false`)，让前端忽略该 Provider。 + +2. **前端容错**： + * 检查前端 `useReportEngine.ts`。 + * 确保即使某个 Provider 失败/超时，用户依然可以强制触发 "Generate Report"。 + +3. **验证**： + * 使用 `inspect_logs.sh` 确认 Finnhub 不再报错，或已被跳过。 + * 确认 API Gateway 日志中出现 `Publishing analysis generation command`。 + diff --git a/docs/3_project_management/tasks/completed/20251120_testing_strategy.md b/docs/3_project_management/tasks/completed/20251120_testing_strategy.md new file mode 100644 index 0000000..3955b48 --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251120_testing_strategy.md @@ -0,0 +1,144 @@ +# 测试策略设计文档：基于 Docker 环境的组件测试与 Orchestrator 逻辑验证 + +> **文档使用说明**: +> 本文档不仅作为测试设计方案，也是测试实施过程中的**Living Document (活文档)**。 +> 请参阅第 4 节 "执行状态追踪 (Execution Status Tracking)" 了解当前进度、Milestones 和 Pending Tasks。 +> 在每次完成重要步骤后，请更新此文档的状态部分。 + +## 1. 策略概述 (Strategy Overview) + +响应“无 Mock、全真实环境”的要求，结合“Rustic 强类型”设计原则，我们将采用 **混合测试策略 (Hybrid Strategy)**： + +1. **I/O 密集型服务 (Providers & ReportGen)**: 采用 **基于 Docker Compose 的组件集成测试**。 + * 直接连接真实的 Postgres, NATS 和第三方 API (Alphavantage/LLM)。 + * 验证“端到端”的功能可用性（Key 是否有效、数据格式是否兼容）。 +2. **逻辑密集型服务 (Orchestrator)**: 采用 **基于 Trait 的内存测试 (In-Memory Testing)**。 + * 通过 Trait 抽象外部依赖，使用简单的内存实现 (Fake) 替代真实服务。 + * 实现毫秒级反馈，覆盖复杂的状态机跳转和边界条件。 + +--- + +## 2. 实施阶段 (Implementation Phases) + +### Phase 1: 测试基础设施 (Infrastructure) + +* **Docker Environment**: `docker-compose.test.yml` + * `postgres-test`: 端口 `5433:5432` + * `nats-test`: 端口 `4223:4222` + * `persistence-test`: 端口 `3001:3000` (Data Persistence Service 本身也视作基础设施的一部分) +* **Abstraction (Refactoring)**: + * 在 `workflow-orchestrator-service` 中定义 `WorkflowRepository` 和 `CommandPublisher` traits，用于解耦逻辑测试。 + +### Phase 2: 微服务组件测试 (IO-Heavy Services) +**执行方式**: 宿主机运行 `cargo test`，环境变量指向 Phase 1 启动的 Docker 端口。 + +#### 1. Data Providers (数据源) +验证从 API 获取数据并存入系统的能力。 +* **Alphavantage Provider**: (Key: `alphaventage_key`) + * Input: `FetchCompanyDataCommand` + * Assert: DB 中存入 SessionData (Profile/Financials)，NATS 发出 `FinancialsPersistedEvent`。 +* **Tushare Provider**: (Key: `tushare_key`) + * Input: `FetchCompanyDataCommand` (CN Market) + * Assert: 同上。 +* **Finnhub Provider**: (Key: `finnhub_key`) + * Input: `FetchCompanyDataCommand` + * Assert: 同上。 +* **YFinance Provider**: (No Key) + * Input: `FetchCompanyDataCommand` + * Assert: 同上。 + +#### 2. Report Generator (报告生成器) +验证从 Persistence 读取数据并调用 LLM 生成报告的能力。 +* **Key**: `openrouter_key` (Model: `google/gemini-flash-1.5` 或其他低成本模型) +* **Pre-condition**: 需要先往 Persistence (localhost:3001) 插入一些伪造的 SessionData (Financials/Price)，否则 LLM 上下文为空。 +* **Input**: `GenerateReportCommand` +* **Logic**: + 1. Service 从 Persistence 读取数据。 + 2. Service 组装 Prompt 调用 OpenRouter API。 + 3. Service 将生成的 Markdown 存回 Persistence。 +* **Assert**: + * NATS 收到 `ReportGeneratedEvent`。 + * Persistence 中能查到 `analysis_report` 类型的 SessionData，且内容非空。 + +### Phase 3: Orchestrator 逻辑测试 (Logic-Heavy) +**执行方式**: 纯内存单元测试，无需 Docker。 + +* **Refactoring**: 将 Orchestrator 的核心逻辑 `WorkflowEngine` 修改为接受 `Box` 和 `Box`。 +* **Test Suite**: + * **DAG Construction**: 给定不同 Template ID，验证生成的 DAG 结构（依赖关系）是否正确。 + * **State Transition**: + * Scenario 1: Happy Path (所有 Task 成功 -> Workflow 完成)。 + * Scenario 2: Dependency Failure (上游失败 -> 下游 Skipped)。 + * Scenario 3: Resume (模拟服务重启，从 Repository 加载状态并继续)。 + * **Policy Check**: 验证 "At least one provider" 策略是否生效。 + +### Phase 4: 全链路验收测试 (E2E) +**执行方式**: `scripts/run_e2e.sh` (Docker + Rust Test Runner) + +* **配置策略**: + * 动态注入测试配置 (`setup_test_environment`): + * 注册 `simple_test_analysis` 模板。 + * 配置 LLM Provider (`openrouter`/`new_api`) 使用 `google/gemini-2.5-flash-lite`。 +* **超时控制**: + * SSE 连接监听设置 60秒硬性超时，防止长连接假死。 +* **Scenarios**: + * **Scenario A (Happy Path)**: 使用 `simple_test_analysis` 模板完整运行。 + * **Scenario B (Recovery)**: 模拟 Orchestrator 重启，验证状态恢复。 (SKIPPED: Requires DB Persistence) + * **Scenario C (Partial Failure)**: 模拟非关键 Provider (Tushare) 故障，验证工作流不受影响。 + * **Scenario D (Invalid Input)**: 使用无效 Symbol，验证错误传播和快速失败。 + * **Scenario E (Module Failure)**: 模拟 Analysis 模块内部错误（如配置错误），验证工作流终止。 + * **Status**: ✅ Completed (2025-11-21) + +--- + +## 3. 执行计划 (Action Plan) + +1. **Environment**: 创建 `docker-compose.test.yml` 和控制脚本。 ✅ +2. **Providers Test**: 编写 4 个 Data Provider 的集成测试。 ✅ +3. **ReportGen Test**: 编写 Report Generator 的集成测试（含数据预埋逻辑）。 ✅ +4. **Orchestrator Refactor**: 引入 Traits 并编写内存测试。 ✅ +5. **Final Verification**: 运行全套测试。 ✅ + +--- + +## 4. 执行状态追踪 (Execution Status Tracking) + +### 当前状态 (Current Status) +* **日期**: 2025-11-21 +* **阶段**: Phase 4 - E2E Testing Completed +* **最近活动**: + * 修复了测试模板配置错误导致 Scenario A 超时的问题。 + * 修复了 Orchestrator 错误广播 Analysis 失败导致 Scenario C 误判的问题。 + * 完整验证了 Scenario A, C, D, E。 + * 暂时跳过 Scenario B (待持久化层就绪后启用)。 + +### 历史记录 (Milestones) +| 日期 | 阶段 | 事件/变更 | 状态 | +| :--- | :--- | :--- | :--- | +| 2025-11-20 | Planning | 完成测试策略文档编写，确定混合测试方案。 | ✅ Completed | +| 2025-11-20 | Phase 1 | 创建 `docker-compose.test.yml` 和基础设施。 | ✅ Completed | +| 2025-11-20 | Phase 2 | 完成 Data Providers 集成测试代码。 | ✅ Completed | +| 2025-11-20 | Phase 2 | 完成 Report Generator 集成测试代码。 | ✅ Completed | +| 2025-11-20 | Phase 3 | 完成 Orchestrator 重构与内存测试。 | ✅ Completed | +| 2025-11-21 | Phase 4 | 修复 SSE 超时问题，增加动态配置注入。 | ✅ Completed | +| 2025-11-21 | Phase 4 | 实现并验证异常场景 (Partial Failure, Invalid Input, Module Error)。 | ✅ Completed | + +### 待处理项 (Next Steps) +- [ ] **Persistence**: 为 Orchestrator 引入 Postgres 存储，启用 Scenario B。 +- [ ] **CI Integration**: 将 `run_e2e.sh` 集成到 CI 流水线。 + +## 5. 未来展望 (Future Outlook) + +随着系统演进，建议增加以下测试场景： + +1. **Network Resilience (网络分区)**: + * 使用 `toxiproxy` 或 Docker Network 操作模拟网络中断。 + * 验证服务的重试机制 (Retry Policy) 和幂等性。 +2. **Concurrency & Load (并发与负载)**: + * 同时启动 10+ 个工作流，验证 Orchestrator 调度和 Provider 吞吐量。 + * 验证 Rate Limiting 是否生效（避免被上游 API 封禁）。 +3. **Long-Running Workflows (长流程)**: + * 测试包含数十个步骤、运行时间超过 5 分钟的复杂模板。 + * 验证 SSE 连接保活和超时处理。 +4. **Data Integrity (数据一致性)**: + * 验证 Fetch -> Persistence -> Report Gen 链路中的数据精度（小数位、时区）。 diff --git a/docs/3_project_management/tasks/completed/20251121_phase4_e2e_plan.md b/docs/3_project_management/tasks/completed/20251121_phase4_e2e_plan.md new file mode 100644 index 0000000..b587a85 --- /dev/null +++ b/docs/3_project_management/tasks/completed/20251121_phase4_e2e_plan.md @@ -0,0 +1,96 @@ +# Phase 4: End-to-End (E2E) 测试计划与执行方案 + +## 1. 测试目标 +本次 E2E 测试旨在验证系统在“全链路真实环境”下的行为，涵盖**正常流程**、**异常恢复**及**组件动态插拔**场景。不涉及前端 UI，而是通过模拟 HTTP/SSE 客户端直接与后端交互。 + +核心验证点： +1. **业务闭环**: 从 `POST /start` 到 SSE 接收 `WorkflowCompleted` 再到最终报告生成。 +2. **状态一致性**: Orchestrator 重启后，能否通过 `SyncStateCommand` 恢复上下文并继续执行。 +3. **容错机制**: 当部分 Data Provider 下线时，策略引擎是否按预期工作（如 "At least one provider"）。 +4. **并发稳定性**: 多个 Workflow 同时运行时互不干扰。 + +## 2. 测试环境架构 +测试运行器 (`end-to-end` Rust Crate) 将作为外部观察者和控制器。 + +```mermaid +graph TD + TestRunner[Rust E2E Runner] -->|HTTP/SSE| Gateway[API Gateway] + TestRunner -->|Docker API| Docker[Docker Engine] + + subgraph "Docker Compose Stack" + Gateway --> Orchestrator + Orchestrator --> NATS + NATS --> Providers + NATS --> ReportGen + Providers --> Postgres + end + + Docker -.->|Stop/Start| Orchestrator + Docker -.->|Stop/Start| Providers +``` + +## 3. 详细测试场景 (Scenarios) + +### Scenario A: The Happy Path (基准测试) +* **目标**: 验证标准流程无误。 +* **步骤**: + 1. 发送 `POST /api/v2/workflow/start` (Symbol: AAPL/000001.SZ)。 + 2. 建立 SSE 连接监听 `events.workflow.{id}`。 + 3. 验证接收到的事件序列： + * `WorkflowStarted` (含完整 DAG) + * `TaskStateChanged` (Pending -> Running -> Completed) + * `TaskStreamUpdate` (Report 内容流式传输) + * `WorkflowCompleted` + 4. **断言**: 最终报告内容非空，数据库中存在 Analysis 记录。 + +### Scenario B: Brain Transplant (Orchestrator 宕机恢复) +* **目标**: 验证 Orchestrator 的状态持久化与快照恢复能力。 +* **步骤**: + 1. 启动 Workflow。 + 2. 等待至少一个 Data Fetch Task 完成 (Receiving `TaskCompleted`)。 + 3. **Action**: `docker stop workflow-orchestrator-service`。 + 4. 等待 5 秒，**Action**: `docker start workflow-orchestrator-service`。 + 5. Test Runner 重新建立 SSE 连接 (自动触发 `SyncStateCommand`)。 + 6. **断言**: + * 收到 `WorkflowStateSnapshot` 事件。 + * 快照中已完成的任务状态保持 `Completed`。 + * 流程继续向下执行，直到最终完成。 + +### Scenario C: Partial Failure (组件拔插) +* **目标**: 验证 "At least one provider" 容错策略。 +* **步骤**: + 1. **Action**: `docker stop tushare-provider-service` (模拟 Tushare 挂掉)。 + 2. 启动 Workflow (Symbol: 000001.SZ，需涉及 Tushare)。 + 3. **断言**: + * Tushare 对应的 Task 状态变为 `Failed` 或 `Skipped`。 + * 由于还有其他 Provider (或模拟数据)，Orchestrator 判定满足 "At least one" 策略。 + * 下游 Analysis Task **正常启动** (而不是被 Block)。 + * 流程最终显示 `WorkflowCompleted` (可能带有 Warning)。 + 4. **Cleanup**: `docker start tushare-provider-service`。 + +### Scenario D: Network Jitter (网络中断模拟) +* **目标**: 验证 Gateway 到 Orchestrator 通讯中断后的恢复。 +* **步骤**: + 1. 启动 Workflow。 + 2. Test Runner 主动断开 SSE 连接。 + 3. 等待 10 秒。 + 4. Test Runner 重连 SSE。 + 5. **断言**: 立即收到 `WorkflowStateSnapshot`，且补齐了断连期间产生的状态变更。 + +## 4. 工程实现 (Rustic Implementation) +新建独立 Rust Crate `tests/end-to-end`，不依赖现有 workspace 的构建配置，独立编译运行。 + +**依赖栈**: +* `reqwest`: HTTP Client +* `eventsource-stream` + `futures`: SSE Handling +* `bollard`: Docker Control API +* `tokio`: Async Runtime +* `anyhow`: Error Handling +* `serde`: JSON Parsing + +**执行方式**: +```bash +# 在 tests/end-to-end 目录下 +cargo run -- --target-env test +``` + diff --git a/docs/backend_requirements_for_frontend_refactor.md b/docs/backend_requirements_for_frontend_refactor.md new file mode 100644 index 0000000..bd66b59 --- /dev/null +++ b/docs/backend_requirements_for_frontend_refactor.md @@ -0,0 +1,62 @@ +# Backend Requirements for Frontend Refactor + +由于前端正在进行“破坏式”重构，删除了所有包含业务逻辑控制、状态推断、流程编排的代码（如 `useReportEngine`, `ExecutionStepManager`），后端必须接管以下职责，以支持纯被动式（Puppet Mode）的前端。 + +## 1. 核心原则 +前端不再拥有“大脑”，只拥有“眼睛”和“耳朵”。所有状态变更、流程流转、错误判断全由后端指令驱动。 + +## 2. 接口需求 + +### 2.1 统一事件流 (Unified Event Stream) +前端将只连接**一个**长连接通道（SSE 或 WebSocket），用于接收整个分析周期的所有信息。 + +* **Endpoint**: `/api/v2/workflow/events?request_id={id}` (建议) +* **职责**: 聚合 NATS (Data Fetching), Internal State (Report Generator), Database (Persistence) 的所有事件。 + +### 2.2 事件类型定义 (Protocol) +后端需要推送以下类型的事件，且 Payload 必须包含前端渲染所需的所有上下文，前端不再发起二次请求查询详情。 + +1. **`WORKFLOW_START`** + * 标志流程开始。 + * Payload: `{ symbol, market, template_id, timestamp }` + +2. **`PHASE_CHANGED`** + * **关键**: 前端不再判断何时切换界面，完全依赖此事件。 + * Payload: `{ phase: 'DATA_FETCHING' | 'ANALYZING' | 'COMPLETED' | 'FAILED', previous_phase: '...' }` + +3. **`TASK_PROGRESS` (Data Fetching Phase)** + * 替代前端轮询 `/api/tasks`。 + * Payload: `{ task_id, provider, status, progress, message }` + * **注意**: 后端需负责聚合多个 Provider 的进度，前端只管展示列表。 + +4. **`MODULE_PROGRESS` (Analysis Phase)** + * 替代旧的 SSE 流。 + * Payload: `{ module_id, content_delta, status }` + +5. **`WORKFLOW_ERROR`** + * **关键**: 包含错误级别（Fatal/Warning）。前端只展示，不判断是否重试。 + * Payload: `{ code, message, is_fatal, suggestion }` + +## 3. 逻辑接管需求 + +### 3.1 状态机迁移 (State Transitions) +* **旧逻辑 (已删)**: 前端轮询任务 -> `if (all_tasks_done) start_analysis()`. +* **新逻辑**: 后端 `Workflow Orchestrator` 监听任务完成事件 -> 自动触发分析 -> 推送 `PHASE_CHANGED: ANALYZING` 给前端。 + +### 3.2 容错与部分成功 (Partial Success) +* **旧逻辑 (已删)**: 前端判断 `if (failed_tasks < total) continue`. +* **新逻辑**: 后端决定数据缺失量是否允许继续分析。如果允许，直接进入分析阶段；如果不允许，推送 `WORKFLOW_ERROR`。 + +### 3.3 超时控制 (Timeout) +* **旧逻辑 (已删)**: 前端 `setTimeout(10min)`. +* **新逻辑**: 后端设置执行超时。如果超时，主动推送 Error 事件关闭连接。前端仅处理网络层面的断开重连。 + +### 3.4 断点恢复 (Resume) +* **需求**: 当用户刷新页面重连 SSE 时，后端必须立即推送一条 `SNAPSHOT` 事件，包含当前所有已完成的任务、已生成的报告片段、当前所处的阶段。 +* **目的**: 防止前端因为丢失历史事件而无法渲染完整界面。 + +## 4. 废弃接口 +以下接口的前端调用代码已被删除，后端可酌情保留用于调试，但业务不再依赖： +* `GET /api/tasks/{id}` (轮询接口) +* `GET /api/analysis-results/stream` (旧的纯分析流，需升级为统一流) + diff --git a/docs/tasks/completed/20251121_refactor_nats_subjects_enum.md b/docs/tasks/completed/20251121_refactor_nats_subjects_enum.md new file mode 100644 index 0000000..95d4707 --- /dev/null +++ b/docs/tasks/completed/20251121_refactor_nats_subjects_enum.md @@ -0,0 +1,132 @@ +# NATS Subject 强类型重构设计文档 + +## 1. 背景与现状 (Background & Status Quo) + +目前，项目中微服务之间的 NATS 消息通信主要依赖于硬编码的字符串（String Literals）来指定 Subject（主题）。例如： +- `services/report-generator-service` 使用 `"events.analysis.report_generated"` 发布消息。 +- `services/workflow-orchestrator-service` 使用 `"events.analysis.>"` 订阅消息，并使用字符串匹配 `if subject == "events.analysis.report_generated"` 来区分消息类型。 + +这种方式存在以下问题： +1. **弱类型约束**：字符串拼接容易出现拼写错误（Typos），且无法在编译期捕获，只能在运行时发现，违反了 "Fail Early" 原则。 +2. **维护困难**：Subject 散落在各个服务的代码中，缺乏统一视图（Single Source of Truth），修改一个 Subject 需要全局搜索并小心替换。 +3. **缺乏契约**：Subject 与 Payload（消息体）之间的对应关系仅通过注释或隐式约定存在，缺乏代码层面的强制约束。 + +## 2. 目的 (Objectives) + +本设计旨在贯彻 Rustic 的工程原则（强类型约束、单一来源、早失败、无回退），通过以下方式重构 NATS Subject 的管理： + +1. **强类型枚举 (Enum-driven Subjects)**：在 `common-contracts` 中定义全局唯一的枚举类型，涵盖系统中所有合法的 NATS Subject。 +2. **消除魔法字符串**：禁止在业务逻辑中直接使用字符串字面量进行 publish 或 subscribe 操作。 +3. **编译期安全**：利用 Rust 的类型系统，确保 Subject 的构造和匹配是合法的。 + +## 3. 设计方案 (Design Proposal) + +### 3.1 核心数据结构 (`common-contracts`) + +在 `services/common-contracts/src/subjects.rs` 中定义 `NatsSubject` 枚举。该枚举涵盖系统中所有合法的 NATS Subject。 + +```rust +use uuid::Uuid; +use std::fmt; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NatsSubject { + // --- Commands --- + WorkflowCommandStart, // "workflow.commands.start" + WorkflowCommandSyncState, // "workflow.commands.sync_state" + DataFetchCommands, // "data_fetch_commands" + AnalysisCommandGenerateReport, // "analysis.commands.generate_report" + + // --- Events --- + // Analysis Events + AnalysisReportGenerated, // "events.analysis.report_generated" + AnalysisReportFailed, // "events.analysis.report_failed" + + // Data Events + DataFinancialsPersisted, // "events.data.financials_persisted" + DataFetchFailed, // "events.data.fetch_failed" + + // Workflow Events (Dynamic) + WorkflowProgress(Uuid), // "events.workflow.{uuid}" + + // --- Wildcards (For Subscription) --- + AnalysisEventsWildcard, // "events.analysis.>" + WorkflowCommandsWildcard, // "workflow.commands.>" + DataEventsWildcard, // "events.data.>" +} + +// ... impl Display and FromStr ... +``` + +### 3.2 使用方式 + +#### 发布消息 (Publish) + +```rust +// Old +state.nats.publish("events.analysis.report_generated", payload).await?; + +// New +use common_contracts::subjects::NatsSubject; + +state.nats.publish(NatsSubject::AnalysisReportGenerated.to_string(), payload).await?; +``` + +#### 订阅与匹配 (Subscribe & Match) + +```rust +// Old +let sub = nats.subscribe("events.analysis.>").await?; +while let Some(msg) = sub.next().await { + if msg.subject == "events.analysis.report_generated" { ... } +} + +// New +let sub = nats.subscribe(NatsSubject::AnalysisEventsWildcard.to_string()).await?; +while let Some(msg) = sub.next().await { + // 将接收到的 subject 字符串尝试转换为枚举 + match NatsSubject::try_from(msg.subject.as_str()) { + Ok(NatsSubject::AnalysisReportGenerated) => { + // Handle report generated + }, + Ok(NatsSubject::AnalysisReportFailed) => { + // Handle report failed + }, + _ => { + // Log warning or ignore + } + } +} +``` + +## 4. 实施状态 (Implementation Status) + +### 4.1 `common-contracts` +- [x] 定义 `NatsSubject` 枚举及相关 Trait (`Display`, `FromStr`) 在 `src/subjects.rs`。 +- [x] 添加单元测试确保 Round-trip 正确性。 + +### 4.2 `report-generator-service` +- [x] `src/worker.rs`: 替换 Publish Subject。 + +### 4.3 `workflow-orchestrator-service` +- [x] `src/message_consumer.rs`: 替换 Subscribe Subject 和 Match 逻辑。 + +### 4.4 `api-gateway` +- [x] `src/api.rs`: 替换 Publish Subject。 + +### 4.5 Provider Services +- [x] `finnhub-provider-service`: 替换 Subscribe Subject，移除魔法字符串常量。 +- [x] `alphavantage-provider-service`: 替换 Subscribe Subject，移除魔法字符串常量。 +- [x] `tushare-provider-service`: 替换 Subscribe Subject，移除魔法字符串常量。 +- [x] `yfinance-provider-service`: 替换 Subscribe Subject，移除魔法字符串常量。 + +## 5. 进阶优化 (Future Work) + +- [x] **关联 Payload 类型**: 利用 Rust 的 trait 系统，将 Subject 枚举与对应的 Payload 结构体关联起来，使得 `publish` 函数能够根据 Subject 自动推断 Payload 类型，从而防止 Subject 与 Payload 不匹配的问题。 + ```rust + trait SubjectMessage { + // type Payload: Serialize + DeserializeOwned; // Simplified: trait is implemented on Payload struct itself + fn subject(&self) -> NatsSubject; + } + ``` + 已在 `services/common-contracts/src/subjects.rs` 中实现 `SubjectMessage` trait，并在 `messages.rs` 中为各个 Command/Event 实现了该 trait。各服务已更新为使用 `msg.subject().to_string()` 进行发布。 diff --git a/frontend/src/app/report/[symbol]/components/AnalysisContent.tsx b/frontend/archive/v1_report/AnalysisContent.tsx similarity index 56% rename from frontend/src/app/report/[symbol]/components/AnalysisContent.tsx rename to frontend/archive/v1_report/AnalysisContent.tsx index 5cbae12..bb9a30f 100644 --- a/frontend/src/app/report/[symbol]/components/AnalysisContent.tsx +++ b/frontend/archive/v1_report/AnalysisContent.tsx @@ -35,8 +35,10 @@ export function AnalysisContent({ const contentWithoutTitle = removeTitleFromContent(state.content, analysisName); const normalizedContent = normalizeMarkdown(contentWithoutTitle); + const isGenerating = state.loading; + return ( -
+

{analysisName}（来自 {modelName || 'AI'}）

{!financials && ( @@ -64,16 +66,16 @@ export function AnalysisContent({ : '待开始'}
- {/* 始终可见的"重新生成分析"按钮 */} + {/* 重新生成按钮 */} {!state.loading && ( )} @@ -82,31 +84,43 @@ export function AnalysisContent({

加载失败: {state.error}

)} - {(state.loading || state.content) && ( -
-
-
- - {normalizedContent} - - {state.loading && ( - - - 正在生成中... - - )} -
-
-
- )} + {/* Content Area with Overlay */} +
+ {/* Overlay when generating */} + {isGenerating && ( +
+ +

+ 正在深入分析财务数据，请稍候... +

+
+ )} + + {/* Existing Content or Placeholder */} + {state.content ? ( +
+
+
+ + {normalizedContent} + +
+
+
+ ) : !isGenerating && ( +
+ 暂无分析内容，请点击生成。 +
+ )} +
)} diff --git a/frontend/src/app/report/[symbol]/components/ExecutionDetails.tsx b/frontend/archive/v1_report/ExecutionDetails.tsx similarity index 100% rename from frontend/src/app/report/[symbol]/components/ExecutionDetails.tsx rename to frontend/archive/v1_report/ExecutionDetails.tsx diff --git a/frontend/src/app/report/[symbol]/components/ReportHeader.tsx b/frontend/archive/v1_report/ReportHeader.tsx similarity index 100% rename from frontend/src/app/report/[symbol]/components/ReportHeader.tsx rename to frontend/archive/v1_report/ReportHeader.tsx diff --git a/frontend/src/app/report/[symbol]/components/TaskStatus.tsx b/frontend/archive/v1_report/TaskStatus.tsx similarity index 100% rename from frontend/src/app/report/[symbol]/components/TaskStatus.tsx rename to frontend/archive/v1_report/TaskStatus.tsx diff --git a/frontend/archive/v1_report/useAnalysisRunner.ts b/frontend/archive/v1_report/useAnalysisRunner.ts new file mode 100644 index 0000000..4607dd4 --- /dev/null +++ b/frontend/archive/v1_report/useAnalysisRunner.ts @@ -0,0 +1,303 @@ +import { useState, useRef, useEffect, useMemo } from 'react'; +import { useDataRequest, useTaskProgress, useAnalysisResults } from '@/hooks/useApi'; + +interface AnalysisState { + content: string; + loading: boolean; + error: string | null; + elapsed_ms?: number; +} + +interface AnalysisRecord { + type: string; + name: string; + status: 'pending' | 'running' | 'done' | 'error'; + start_ts?: string; + end_ts?: string; + duration_ms?: number; + tokens?: { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + }; + error?: string; +} + +export function useAnalysisRunner( + financials: any, + financialConfig: any, + normalizedMarket: string, + unifiedSymbol: string, + isLoading: boolean, + error: any, + templateSets: any // Added templateSets +) { + // --- Template Logic --- + const [selectedTemplateId, setSelectedTemplateId] = useState(''); + const reportTemplateId = financials?.meta?.template_id; + + // Sync selected template with report template when report loads + useEffect(() => { + if (reportTemplateId) { + setSelectedTemplateId(reportTemplateId); + } + }, [reportTemplateId]); + + // Set default template if nothing selected and no report template + useEffect(() => { + if (!selectedTemplateId && !reportTemplateId && templateSets && Object.keys(templateSets).length > 0) { + const defaultId = Object.keys(templateSets).find(k => k.includes('standard') || k === 'default') || Object.keys(templateSets)[0]; + setSelectedTemplateId(defaultId); + } + }, [templateSets, selectedTemplateId, reportTemplateId]); + + // Determine active template set + const activeTemplateId = selectedTemplateId; + + const activeTemplateSet = useMemo(() => { + if (!activeTemplateId || !templateSets) return null; + return templateSets[activeTemplateId] || null; + }, [activeTemplateId, templateSets]); + + // Derive effective analysis config from template set, falling back to global config if needed + const activeAnalysisConfig = useMemo(() => { + if (activeTemplateSet) { + return { + ...financialConfig, + analysis_modules: activeTemplateSet.modules, + }; + } + return financialConfig; // Fallback to global config (legacy behavior) + }, [activeTemplateSet, financialConfig]); + + // 分析类型列表 + const analysisTypes = useMemo(() => { + if (!activeAnalysisConfig?.analysis_modules) return []; + return Object.keys(activeAnalysisConfig.analysis_modules); + }, [activeAnalysisConfig]); + + // 分析状态管理 + const [analysisStates, setAnalysisStates] = useState>({}); + + const fullAnalysisTriggeredRef = useRef(false); + const isAnalysisRunningRef = useRef(false); + const analysisFetchedRefs = useRef>({}); + const stopRequestedRef = useRef(false); + const abortControllerRef = useRef(null); + const currentAnalysisTypeRef = useRef(null); + const [manualRunKey, setManualRunKey] = useState(0); + + // 当前正在执行的分析任务 + const [currentAnalysisTask, setCurrentAnalysisTask] = useState(null); + + // 计时器状态 + const [startTime, setStartTime] = useState(null); + const [elapsedSeconds, setElapsedSeconds] = useState(0); + + // 分析执行记录 + const [analysisRecords, setAnalysisRecords] = useState([]); + + // 新架构：触发分析与查看任务进度 + const { trigger: triggerAnalysisRequest, isMutating: triggering } = useDataRequest(); + const [requestId, setRequestId] = useState(null); + const { progress: taskProgress } = useTaskProgress(requestId); + + // 引入 Analysis Results 轮询 + const { data: newAnalysisResults } = useAnalysisResults(unifiedSymbol); + + // 1. Determine the Active Request ID (The one we want to display) + const activeRequestId = useMemo(() => { + // If the user manually triggered a task in this session, prioritize that + if (requestId) return requestId; + + // Otherwise, default to the most recent result's request_id from the backend + // Assuming newAnalysisResults is sorted by created_at DESC + if (newAnalysisResults && newAnalysisResults.length > 0) { + return newAnalysisResults[0].request_id; + } + return null; + }, [requestId, newAnalysisResults]); + + // 2. Filter results for the current batch + const currentBatchResults = useMemo(() => { + if (!newAnalysisResults || !activeRequestId) return []; + return newAnalysisResults.filter(r => r.request_id === activeRequestId); + }, [newAnalysisResults, activeRequestId]); + + // 3. Sync analysisStates (Content) from current batch + // We only update if we have a result for that module in the current batch. + // If not, we leave it as is (or could clear it if we wanted strict mode). + // For now, we'll update based on what we find. + useEffect(() => { + if (!currentBatchResults) return; + + setAnalysisStates(prev => { + const next = { ...prev }; + let hasChanges = false; + + currentBatchResults.forEach(result => { + const type = result.module_id; + const status = result.meta_data?.status || 'success'; + const content = result.content; + + const currentState = next[type]; + + // Only update if content changed or status changed + if ( + !currentState || + currentState.content !== content || + (status === 'processing' && !currentState.loading) || + (status === 'success' && currentState.loading) || + (status === 'error' && !currentState.error) + ) { + next[type] = { + content: content, + loading: status === 'processing', + error: status === 'error' ? result.meta_data?.error || 'Unknown error' : null, + }; + hasChanges = true; + } + }); + return hasChanges ? next : prev; + }); + }, [currentBatchResults]); + + // 4. Sync analysisRecords (Execution Details) from current batch + // This ensures Execution Details only shows the relevant modules for the current run. + useEffect(() => { + if (!currentBatchResults) return; + + // If we are starting a new run (triggered), we might want to reset records initially? + // But currentBatchResults will eventually populate. + + const records: AnalysisRecord[] = currentBatchResults.map(r => { + const statusStr = r.meta_data?.status; + let status: 'pending' | 'running' | 'done' | 'error' = 'done'; + if (statusStr === 'processing') status = 'running'; + else if (statusStr === 'error') status = 'error'; + + return { + type: r.module_id, + name: activeAnalysisConfig?.analysis_modules?.[r.module_id]?.name || r.module_id, + status: status, + duration_ms: r.meta_data?.elapsed_ms, // Backend needs to provide this in meta_data + error: r.meta_data?.error, + tokens: r.meta_data?.tokens // Backend needs to provide this + }; + }); + + // Sort records to match the defined order in activeAnalysisConfig if possible + const definedOrder = Object.keys(activeAnalysisConfig?.analysis_modules || {}); + records.sort((a, b) => { + const idxA = definedOrder.indexOf(a.type); + const idxB = definedOrder.indexOf(b.type); + if (idxA === -1) return 1; + if (idxB === -1) return -1; + return idxA - idxB; + }); + + setAnalysisRecords(records); + }, [currentBatchResults, activeAnalysisConfig]); + + + // 计算完成比例 + const completionProgress = useMemo(() => { + const totalTasks = analysisRecords.length; + if (totalTasks === 0) return 0; + const completedTasks = analysisRecords.filter(r => r.status === 'done' || r.status === 'error').length; + return (completedTasks / totalTasks) * 100; + }, [analysisRecords]); + + // 总耗时（ms） + const totalElapsedMs = useMemo(() => { + const finMs = financials?.meta?.elapsed_ms || 0; + const analysesMs = analysisRecords.reduce((sum, r) => sum + (r.duration_ms || 0), 0); + return finMs + analysesMs; + }, [financials?.meta?.elapsed_ms, analysisRecords]); + + const hasRunningTask = useMemo(() => { + if (currentAnalysisTask !== null) return true; + // Also check analysisRecords derived from backend + if (analysisRecords.some(r => r.status === 'running')) return true; + return false; + }, [currentAnalysisTask, analysisRecords]); + + // 全部任务是否完成 + const allTasksCompleted = useMemo(() => { + if (analysisRecords.length === 0) return false; + const allDoneOrErrored = analysisRecords.every(r => r.status === 'done' || r.status === 'error'); + return allDoneOrErrored && !hasRunningTask && currentAnalysisTask === null; + }, [analysisRecords, hasRunningTask, currentAnalysisTask]); + + // 所有任务完成时，停止计时器 + useEffect(() => { + if (allTasksCompleted) { + setStartTime(null); + } + }, [allTasksCompleted]); + + useEffect(() => { + if (!startTime) return; + const interval = setInterval(() => { + const now = Date.now(); + const elapsed = Math.floor((now - startTime) / 1000); + setElapsedSeconds(elapsed); + }, 1000); + return () => clearInterval(interval); + }, [startTime]); + + const retryAnalysis = async (analysisType: string) => { + // Retry logic is complicated with the new backend-driven approach. + // Ideally, we should send a backend command to retry a specific module. + // For now, we can just re-trigger the whole template or alert the user. + // Or implementation TODO: Single module retry endpoint. + alert("单个模块重试功能在新架构中尚未就绪，请重新触发完整分析。"); + }; + + const stopAll = () => { + // Clean up client-side state + stopRequestedRef.current = true; + isAnalysisRunningRef.current = false; + setStartTime(null); + // Ideally call backend to cancel job + }; + + const continuePending = () => { + // No-op in new architecture basically + }; + + const triggerAnalysis = async () => { + const reqId = await triggerAnalysisRequest(unifiedSymbol, normalizedMarket || '', selectedTemplateId); + if (reqId) { + setRequestId(reqId); + setStartTime(Date.now()); // Start timer + // Reset records to empty or wait for poll? + // Waiting for poll is safer to avoid flashing old data + setAnalysisRecords([]); + } + }; + + return { + activeAnalysisConfig, // Exported + analysisTypes, + analysisStates, + analysisRecords, + currentAnalysisTask, + triggerAnalysis, + triggering, + requestId, + setRequestId, + taskProgress, + startTime, + elapsedSeconds, + completionProgress, + totalElapsedMs, + stopAll, + continuePending, + retryAnalysis, + hasRunningTask, + isAnalysisRunning: hasRunningTask, // Simplified + selectedTemplateId, // Exported + setSelectedTemplateId, // Exported + }; +} diff --git a/frontend/src/app/report/[symbol]/hooks/useReportData.ts b/frontend/archive/v1_report/useReportData.ts similarity index 100% rename from frontend/src/app/report/[symbol]/hooks/useReportData.ts rename to frontend/archive/v1_report/useReportData.ts diff --git a/frontend/next.config.mjs b/frontend/next.config.mjs index 1f1640a..436d7f1 100644 --- a/frontend/next.config.mjs +++ b/frontend/next.config.mjs @@ -14,8 +14,16 @@ const nextConfig = { proxyTimeout: 300000, // 300 seconds (5 minutes) }, // Optimize for Docker deployment only in production - // 当 NODE_ENV 为 production 时开启 standalone 模式 output: process.env.NODE_ENV === 'production' ? 'standalone' : undefined, + + async rewrites() { + return [ + { + source: '/api/:path*', + destination: 'http://api-gateway:4000/v1/:path*', + }, + ]; + }, }; export default nextConfig; diff --git a/frontend/src/app/api/analysis-results/route.ts b/frontend/src/app/api/analysis-results/route.ts deleted file mode 100644 index 5fb38a9..0000000 --- a/frontend/src/app/api/analysis-results/route.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { NextRequest } from 'next/server'; - -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function GET(req: NextRequest) { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const { searchParams } = new URL(req.url); - const symbol = searchParams.get('symbol'); - - if (!symbol) { - return new Response('Missing symbol parameter', { status: 400 }); - } - - const resp = await fetch(`${BACKEND_BASE}/analysis-results?symbol=${encodeURIComponent(symbol)}`, { cache: 'no-store' }); - - if (!resp.ok) { - if (resp.status === 404) { - // Return empty list if not found, to avoid UI errors - return Response.json([]); - } - return new Response(resp.statusText, { status: resp.status }); - } - - const data = await resp.json(); - return Response.json(data); -} - diff --git a/frontend/src/app/api/companies/[symbol]/profile/route.ts b/frontend/src/app/api/companies/[symbol]/profile/route.ts deleted file mode 100644 index 54e74a5..0000000 --- a/frontend/src/app/api/companies/[symbol]/profile/route.ts +++ /dev/null @@ -1,23 +0,0 @@ -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function GET( - _req: Request, - context: { params: Promise<{ symbol: string }> } -) { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const { symbol } = await context.params; - const target = `${BACKEND_BASE}/companies/${encodeURIComponent(symbol)}/profile`; - const resp = await fetch(target, { headers: { 'Content-Type': 'application/json' } }); - const headers = new Headers(); - const contentType = resp.headers.get('content-type') || 'application/json; charset=utf-8'; - headers.set('content-type', contentType); - const cacheControl = resp.headers.get('cache-control'); - if (cacheControl) headers.set('cache-control', cacheControl); - const xAccelBuffering = resp.headers.get('x-accel-buffering'); - if (xAccelBuffering) headers.set('x-accel-buffering', xAccelBuffering); - return new Response(resp.body, { status: resp.status, headers }); -} - - diff --git a/frontend/src/app/api/config/route.ts b/frontend/src/app/api/config/route.ts deleted file mode 100644 index b5437e5..0000000 --- a/frontend/src/app/api/config/route.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { NextRequest } from 'next/server'; - -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -// 聚合新后端的配置，提供给旧前端调用点一个稳定入口 -export async function GET() { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - try { - const [providersResp, modulesResp] = await Promise.all([ - fetch(`${BACKEND_BASE}/configs/llm_providers`, { cache: 'no-store' }), - fetch(`${BACKEND_BASE}/configs/analysis_modules`, { cache: 'no-store' }), - ]); - const providersText = await providersResp.text(); - const modulesText = await modulesResp.text(); - let providers: unknown = {}; - let modules: unknown = {}; - try { providers = providersText ? JSON.parse(providersText) : {}; } catch { providers = {}; } - try { modules = modulesText ? JSON.parse(modulesText) : {}; } catch { modules = {}; } - return Response.json({ - llm_providers: providers, - analysis_modules: modules, - }); - } catch (e: any) { - return new Response(e?.message || 'Failed to load config', { status: 502 }); - } -} - -// 允许前端一次性提交部分配置；根据键路由到新后端 -export async function PUT(req: NextRequest) { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - try { - const incoming = await req.json().catch(() => ({})); - const tasks: Promise[] = []; - if (incoming.llm_providers) { - tasks.push(fetch(`${BACKEND_BASE}/configs/llm_providers`, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(incoming.llm_providers), - })); - } - if (incoming.analysis_modules) { - tasks.push(fetch(`${BACKEND_BASE}/configs/analysis_modules`, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(incoming.analysis_modules), - })); - } - const results = await Promise.all(tasks); - const ok = results.every(r => r.ok); - if (!ok) { - const texts = await Promise.all(results.map(r => r.text().catch(() => ''))); - return new Response(JSON.stringify({ error: 'Partial update failed', details: texts }), { - status: 502, - headers: { 'Content-Type': 'application/json' }, - }); - } - // 返回最新聚合 - const [providersResp, modulesResp] = await Promise.all([ - fetch(`${BACKEND_BASE}/configs/llm_providers`, { cache: 'no-store' }), - fetch(`${BACKEND_BASE}/configs/analysis_modules`, { cache: 'no-store' }), - ]); - const providers = await providersResp.json().catch(() => ({})); - const modules = await modulesResp.json().catch(() => ({})); - return Response.json({ - llm_providers: providers, - analysis_modules: modules, - }); - } catch (e: any) { - return new Response(e?.message || 'Failed to update config', { status: 502 }); - } -} diff --git a/frontend/src/app/api/config/test/route.ts b/frontend/src/app/api/config/test/route.ts deleted file mode 100644 index 42d7c71..0000000 --- a/frontend/src/app/api/config/test/route.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { NextRequest } from 'next/server'; - -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function POST(req: NextRequest) { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - - try { - const body = await req.json(); - const { type, data } = body; - - if (!type || !data) { - return new Response('请求体必须包含 type 和 data', { status: 400 }); - } - - // 将请求转发到 API Gateway - const targetUrl = `${BACKEND_BASE.replace(/\/$/, '')}/configs/test`; - - const backendRes = await fetch(targetUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ type, ...data }), // 转发时将 data 字段展开 - }); - - const backendResBody = await backendRes.text(); - - return new Response(backendResBody, { - status: backendRes.status, - headers: { - 'Content-Type': 'application/json', - }, - }); - - } catch (error: any) { - console.error('配置测试代理失败:', error); - return new Response(JSON.stringify({ success: false, message: error.message || '代理请求时发生未知错误' }), { - status: 500, - headers: { 'Content-Type': 'application/json' }, - }); - } -} diff --git a/frontend/src/app/api/configs/analysis_modules/route.ts b/frontend/src/app/api/configs/analysis_modules/route.ts deleted file mode 100644 index 50d107b..0000000 --- a/frontend/src/app/api/configs/analysis_modules/route.ts +++ /dev/null @@ -1,34 +0,0 @@ -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function GET() { - if (!BACKEND_BASE) { - return new Response('NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const resp = await fetch(`${BACKEND_BASE}/configs/analysis_modules`, { - headers: { 'Content-Type': 'application/json' }, - cache: 'no-store', - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); -} - -export async function PUT(req: Request) { - if (!BACKEND_BASE) { - return new Response('NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const body = await req.text(); - const resp = await fetch(`${BACKEND_BASE}/configs/analysis_modules`, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body, - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); -} - diff --git a/frontend/src/app/api/configs/analysis_template_sets/route.ts b/frontend/src/app/api/configs/analysis_template_sets/route.ts deleted file mode 100644 index 35826ae..0000000 --- a/frontend/src/app/api/configs/analysis_template_sets/route.ts +++ /dev/null @@ -1,45 +0,0 @@ -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function GET() { - if (!BACKEND_BASE) { - return new Response('NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - try { - const resp = await fetch(`${BACKEND_BASE}/configs/analysis_template_sets`, { - headers: { 'Content-Type': 'application/json' }, - cache: 'no-store', - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); - } catch (e: any) { - const errorBody = JSON.stringify({ message: e?.message || '连接后端失败' }); - return new Response(errorBody, { status: 502, headers: { 'Content-Type': 'application/json' } }); - } -} - -export async function PUT(req: Request) { - if (!BACKEND_BASE) { - return new Response('NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const body = await req.text(); - try { - const resp = await fetch(`${BACKEND_BASE}/configs/analysis_template_sets`, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body, - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); - } catch (e: any) { - const errorBody = JSON.stringify({ message: e?.message || '连接后端失败' }); - return new Response(errorBody, { status: 502, headers: { 'Content-Type': 'application/json' } }); - } -} - - diff --git a/frontend/src/app/api/configs/data_sources/route.ts b/frontend/src/app/api/configs/data_sources/route.ts deleted file mode 100644 index f66ee5b..0000000 --- a/frontend/src/app/api/configs/data_sources/route.ts +++ /dev/null @@ -1,45 +0,0 @@ -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function GET() { - if (!BACKEND_BASE) { - return new Response('NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - try { - const resp = await fetch(`${BACKEND_BASE}/configs/data_sources`, { - headers: { 'Content-Type': 'application/json' }, - cache: 'no-store', - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); - } catch (e: any) { - const errorBody = JSON.stringify({ message: e?.message || '连接后端失败' }); - return new Response(errorBody, { status: 502, headers: { 'Content-Type': 'application/json' } }); - } -} - -export async function PUT(req: Request) { - if (!BACKEND_BASE) { - return new Response('NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const body = await req.text(); - try { - const resp = await fetch(`${BACKEND_BASE}/configs/data_sources`, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body, - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); - } catch (e: any) { - const errorBody = JSON.stringify({ message: e?.message || '连接后端失败' }); - return new Response(errorBody, { status: 502, headers: { 'Content-Type': 'application/json' } }); - } -} - - diff --git a/frontend/src/app/api/configs/llm/test/route.ts b/frontend/src/app/api/configs/llm/test/route.ts deleted file mode 100644 index 892cfd1..0000000 --- a/frontend/src/app/api/configs/llm/test/route.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { NextRequest } from 'next/server'; - -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function POST(req: NextRequest) { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - - try { - const body = await req.json(); - - // 将请求转发到 API Gateway - const targetUrl = `${BACKEND_BASE.replace(/\/$/, '')}/configs/llm/test`; - - const backendRes = await fetch(targetUrl, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(body), - }); - - const backendResBody = await backendRes.text(); - - return new Response(backendResBody, { - status: backendRes.status, - headers: { - 'Content-Type': 'application/json', - }, - }); - - } catch (error: any) { - console.error('LLM测试代理失败:', error); - return new Response(JSON.stringify({ success: false, message: error.message || '代理请求时发生未知错误' }), { - status: 500, - headers: { 'Content-Type': 'application/json' }, - }); - } -} - diff --git a/frontend/src/app/api/configs/llm_providers/route.ts b/frontend/src/app/api/configs/llm_providers/route.ts deleted file mode 100644 index 5ac6c46..0000000 --- a/frontend/src/app/api/configs/llm_providers/route.ts +++ /dev/null @@ -1,44 +0,0 @@ -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function GET() { - if (!BACKEND_BASE) { - return new Response('NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - try { - const resp = await fetch(`${BACKEND_BASE}/configs/llm_providers`, { - headers: { 'Content-Type': 'application/json' }, - cache: 'no-store', - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); - } catch (e: any) { - const errorBody = JSON.stringify({ message: e?.message || '连接后端失败' }); - return new Response(errorBody, { status: 502, headers: { 'Content-Type': 'application/json' } }); - } -} - -export async function PUT(req: Request) { - if (!BACKEND_BASE) { - return new Response('NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const body = await req.text(); - try { - const resp = await fetch(`${BACKEND_BASE}/configs/llm_providers`, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body, - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); - } catch (e: any) { - const errorBody = JSON.stringify({ message: e?.message || '连接后端失败' }); - return new Response(errorBody, { status: 502, headers: { 'Content-Type': 'application/json' } }); - } -} - diff --git a/frontend/src/app/api/data-requests/route.ts b/frontend/src/app/api/data-requests/route.ts deleted file mode 100644 index 35a4bfb..0000000 --- a/frontend/src/app/api/data-requests/route.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { NextRequest } from 'next/server'; - -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function POST(req: NextRequest) { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const body = await req.text(); - const resp = await fetch(`${BACKEND_BASE}/data-requests`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body, - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); -} - - diff --git a/frontend/src/app/api/discover-models/[provider_id]/route.ts b/frontend/src/app/api/discover-models/[provider_id]/route.ts deleted file mode 100644 index be6435b..0000000 --- a/frontend/src/app/api/discover-models/[provider_id]/route.ts +++ /dev/null @@ -1,27 +0,0 @@ -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function GET( - _req: Request, - context: any -) { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const raw = context?.params; - const params = raw && typeof raw.then === 'function' ? await raw : raw; - const provider_id = params?.provider_id as string | undefined; - if (!provider_id) { - return new Response('provider_id 缺失', { status: 400 }); - } - const target = `${BACKEND_BASE}/discover-models/${encodeURIComponent(provider_id)}`; - const resp = await fetch(target, { - headers: { 'Content-Type': 'application/json' }, - cache: 'no-store', - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); -} - diff --git a/frontend/src/app/api/discover-models/route.ts b/frontend/src/app/api/discover-models/route.ts deleted file mode 100644 index cb63264..0000000 --- a/frontend/src/app/api/discover-models/route.ts +++ /dev/null @@ -1,26 +0,0 @@ -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function POST(req: Request) { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const body = await req.text(); - try { - const resp = await fetch(`${BACKEND_BASE}/discover-models`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body, - cache: 'no-store', - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); - } catch (e: any) { - const errorBody = JSON.stringify({ message: e?.message || '连接后端失败' }); - return new Response(errorBody, { status: 502, headers: { 'Content-Type': 'application/json' } }); - } -} - - diff --git a/frontend/src/app/api/financials/[...slug]/route.ts b/frontend/src/app/api/financials/[...slug]/route.ts deleted file mode 100644 index f08b330..0000000 --- a/frontend/src/app/api/financials/[...slug]/route.ts +++ /dev/null @@ -1,195 +0,0 @@ -import { NextRequest } from 'next/server'; - -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; -const FRONTEND_BASE = process.env.FRONTEND_INTERNAL_URL || 'http://localhost:3001'; - -export async function GET( - req: NextRequest, - context: { params: Promise<{ slug: string[] }> } -) { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const url = new URL(req.url); - const { slug } = await context.params; - const first = slug?.[0]; - - // 1. Match /api/financials/{market}/{symbol} - // slug[0] = market (e.g., "cn" or "us") - // slug[1] = symbol (e.g., "600519" or "AAPL") - if (slug.length === 2 && first !== 'analysis-config' && first !== 'config') { - const market = slug[0]; - const symbol = slug[1]; - const years = url.searchParams.get('years') || '10'; - - // Fetch financials from backend - // Corrected path to match new API Gateway route - const metricsParam = url.searchParams.get('metrics') || ''; - const fetchUrl = `${BACKEND_BASE}/market-data/financial-statements/${encodeURIComponent(symbol)}` + - (metricsParam ? `?metrics=${encodeURIComponent(metricsParam)}` : ''); - - const finResp = await fetch(fetchUrl, { cache: 'no-store' }); - - if (!finResp.ok) { - if (finResp.status === 404) { - return Response.json({}, { status: 200 }); // Return empty for now to not break UI - } - return new Response(finResp.statusText, { status: finResp.status }); - } - - const series = await finResp.json(); - - // Transform to frontend expected format (BatchFinancialDataResponse) - // We group by metric_name - const groupedSeries: Record = {}; - series.forEach((item: any) => { - if (!groupedSeries[item.metric_name]) { - groupedSeries[item.metric_name] = []; - } - groupedSeries[item.metric_name].push({ - period: item.period_date ? item.period_date.replace(/-/g, '') : null, // YYYY-MM-DD -> YYYYMMDD - value: item.value, - source: item.source - }); - }); - - // Fetch Company Profile to populate name/industry - // Corrected path to match new API Gateway route - const profileResp = await fetch(`${BACKEND_BASE}/companies/${encodeURIComponent(symbol)}/profile`, { cache: 'no-store' }); - let profileData: any = {}; - if (profileResp.ok) { - profileData = await profileResp.json(); - } - - // Fetch Latest Analysis Result Metadata (to get template_id) - // We search for the most recent analysis result for this symbol - const analysisResp = await fetch(`${BACKEND_BASE}/analysis-results?symbol=${encodeURIComponent(symbol)}`, { cache: 'no-store' }); - let meta: any = { - symbol: symbol, - generated_at: new Date().toISOString(), // Fallback - template_id: null // Explicitly null if not found - }; - - if (analysisResp.ok) { - const analysisList = await analysisResp.json(); - if (Array.isArray(analysisList) && analysisList.length > 0) { - // Sort by created_at desc (backend should already do this, but to be safe) - // Backend returns sorted by created_at DESC - const latest = analysisList[0]; - meta.template_id = latest.template_id || null; - meta.generated_at = latest.created_at; - } - } - - const responsePayload = { - name: profileData.name || symbol, - symbol: symbol, - market: market, - series: groupedSeries, - meta: meta - }; - - return Response.json(responsePayload); - } - - // 适配旧接口：analysis-config → 新分析模块配置 - if (first === 'analysis-config') { - const resp = await fetch(`${BACKEND_BASE}/configs/analysis_modules`, { cache: 'no-store' }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); - } - // 适配旧接口：config → 聚合配置 - if (first === 'config') { - const resp = await fetch(`${FRONTEND_BASE}/api/config`, { cache: 'no-store' }); - const text = await resp.text(); - return new Response(text, { status: resp.status, headers: { 'Content-Type': 'application/json' } }); - } - - // 2. Match /api/financials/{market}/{symbol}/analysis/{type}/stream - // slug length = 5 - // slug[0] = market - // slug[1] = symbol - // slug[2] = 'analysis' - // slug[3] = analysisType (module_id) - // slug[4] = 'stream' - if (slug.length === 5 && slug[2] === 'analysis' && slug[4] === 'stream') { - const symbol = slug[1]; - const analysisType = slug[3]; - - const encoder = new TextEncoder(); - - const stream = new ReadableStream({ - async start(controller) { - // Polling logic - // We try for up to 60 seconds - const maxRetries = 30; - let found = false; - - for (let i = 0; i < maxRetries; i++) { - try { - const resp = await fetch(`${BACKEND_BASE}/analysis-results?symbol=${encodeURIComponent(symbol)}&module_id=${encodeURIComponent(analysisType)}`, { cache: 'no-store' }); - - if (resp.ok) { - const results = await resp.json(); - // Assuming results are sorted by created_at DESC (backend behavior) - if (Array.isArray(results) && results.length > 0) { - const latest = results[0]; - // If result is found, send it and exit - if (latest && latest.content) { - controller.enqueue(encoder.encode(latest.content)); - found = true; - break; - } - } - } - } catch (e) { - console.error("Error polling analysis results", e); - } - - // Wait 2 seconds before next poll - await new Promise(resolve => setTimeout(resolve, 2000)); - } - controller.close(); - } - }); - - return new Response(stream, { - headers: { - 'Content-Type': 'text/plain; charset=utf-8', - 'Cache-Control': 'no-cache', - 'Connection': 'keep-alive' - } - }); - } - - // 其他旧 financials 端点在新架构中未实现：返回空对象以避免前端 JSON 解析错误 - return Response.json({}, { status: 200 }); -} - -export async function PUT( - req: NextRequest, - context: { params: Promise<{ slug: string[] }> } -) { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const { slug } = await context.params; - const first = slug?.[0]; - if (first === 'analysis-config') { - const body = await req.text(); - const resp = await fetch(`${BACKEND_BASE}/configs/analysis_modules`, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body, - }); - const text = await resp.text(); - return new Response(text, { - status: resp.status, - headers: { 'Content-Type': resp.headers.get('Content-Type') || 'application/json' }, - }); - } - return new Response('Not Found', { status: 404 }); -} diff --git a/frontend/src/app/api/reports/[id]/route.ts b/frontend/src/app/api/reports/[id]/route.ts deleted file mode 100644 index 6a6b295..0000000 --- a/frontend/src/app/api/reports/[id]/route.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { NextRequest } from 'next/server' - -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function GET( - req: NextRequest, - context: { params: Promise<{ id: string }> } -) { - // 优先从动态路由 params（Promise）获取，其次从 URL 最后一段兜底 - let id: string | undefined - try { - const { id: idFromParams } = await context.params - id = idFromParams - } catch { - // ignore - } - if (!id) { - id = new URL(req.url).pathname.split('/').pop() || undefined - } - - if (!id) { - return Response.json({ error: 'missing id' }, { status: 400 }) - } - - if (!BACKEND_BASE) { - return new Response('NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const resp = await fetch(`${BACKEND_BASE}/analysis-results/${encodeURIComponent(id)}`); - const text = await resp.text(); - if (!resp.ok) { - return new Response(text || 'not found', { status: resp.status }); - } - // 将后端 DTO（generated_at 等）适配为前端旧结构字段（createdAt） - try { - const dto = JSON.parse(text); - const adapted = { - id: dto.id, - symbol: dto.symbol, - createdAt: dto.generated_at || dto.generatedAt || null, - content: dto.content, - module_id: dto.module_id, - model_name: dto.model_name, - meta_data: dto.meta_data, - }; - return Response.json(adapted); - } catch { - return Response.json({ error: 'invalid response from backend' }, { status: 502 }); - } -} diff --git a/frontend/src/app/api/reports/route.ts b/frontend/src/app/api/reports/route.ts deleted file mode 100644 index b2fa7d7..0000000 --- a/frontend/src/app/api/reports/route.ts +++ /dev/null @@ -1,13 +0,0 @@ -export const runtime = 'nodejs' -import { NextRequest } from 'next/server' - -export async function GET(req: NextRequest) { - // 历史报告列表功能在新架构中由后端持久化服务统一提供。 - // 当前网关未提供“全量列表”接口（需要 symbol 条件），因此此路由返回空集合。 - return Response.json({ items: [], total: 0 }, { status: 200 }); -} - -export async function POST(req: NextRequest) { - // 新架构下，报告持久化由后端流水线/服务完成，此处不再直接创建。 - return Response.json({ error: 'Not implemented: creation is handled by backend pipeline' }, { status: 501 }); -} diff --git a/frontend/src/app/api/tasks/[request_id]/route.ts b/frontend/src/app/api/tasks/[request_id]/route.ts deleted file mode 100644 index e151167..0000000 --- a/frontend/src/app/api/tasks/[request_id]/route.ts +++ /dev/null @@ -1,23 +0,0 @@ -const BACKEND_BASE = process.env.BACKEND_INTERNAL_URL || process.env.NEXT_PUBLIC_BACKEND_URL; - -export async function GET( - _req: Request, - context: { params: Promise<{ request_id: string }> } -) { - if (!BACKEND_BASE) { - return new Response('BACKEND_INTERNAL_URL/NEXT_PUBLIC_BACKEND_URL 未配置', { status: 500 }); - } - const { request_id } = await context.params; - const target = `${BACKEND_BASE}/tasks/${encodeURIComponent(request_id)}`; - const resp = await fetch(target, { headers: { 'Content-Type': 'application/json' } }); - const headers = new Headers(); - const contentType = resp.headers.get('content-type') || 'application/json; charset=utf-8'; - headers.set('content-type', contentType); - const cacheControl = resp.headers.get('cache-control'); - if (cacheControl) headers.set('cache-control', cacheControl); - const xAccelBuffering = resp.headers.get('x-accel-buffering'); - if (xAccelBuffering) headers.set('x-accel-buffering', xAccelBuffering); - return new Response(resp.body, { status: resp.status, headers }); -} - - diff --git a/frontend/src/app/page.tsx b/frontend/src/app/page.tsx index e84a2a5..7525926 100644 --- a/frontend/src/app/page.tsx +++ b/frontend/src/app/page.tsx @@ -6,15 +6,53 @@ import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Loader2 } from "lucide-react"; export default function StockInputForm() { const [symbol, setSymbol] = useState(''); const [market, setMarket] = useState('china'); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(''); const router = useRouter(); - const handleSearch = () => { - if (symbol.trim()) { - router.push(`/report/${symbol.trim()}?market=${market}`); + const handleSearch = async (e?: React.FormEvent) => { + if (e) { + e.preventDefault(); + } + if (!symbol.trim()) return; + + setIsLoading(true); + setError(''); + + try { + // 1. 调用后端进行 Symbol 归一化，但不启动工作流 + const response = await fetch('/api/tools/resolve-symbol', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + symbol: symbol.trim(), + market: market, + }), + }); + + if (!response.ok) { + const errText = await response.text(); + throw new Error(errText || '解析股票代码失败'); + } + + const data = await response.json(); + // data 结构: { symbol: string, market: string } + + // 2. 跳转到报告页面，仅携带归一化后的 Symbol + // 此时并没有 request_id，所以详情页不会自动开始，而是等待用户点击 + router.push(`/report/${encodeURIComponent(data.symbol)}?market=${data.market}`); + + } catch (err) { + console.error(err); + setError(err instanceof Error ? err.message : '操作失败，请重试'); + setIsLoading(false); } }; @@ -25,30 +63,49 @@ export default function StockInputForm() { 基本面分析报告 输入股票代码和市场，生成综合分析报告。 - -
- - setSymbol(e.target.value)} - /> -
-
- - -
- + +
+
+ + setSymbol(e.target.value)} + disabled={isLoading} + /> +
+
+ + +
+ + {error && ( +
+ {error} +
+ )} + + +
diff --git a/frontend/src/app/report/[symbol]/components/AnalysisModulesView.tsx b/frontend/src/app/report/[symbol]/components/AnalysisModulesView.tsx new file mode 100644 index 0000000..1f55022 --- /dev/null +++ b/frontend/src/app/report/[symbol]/components/AnalysisModulesView.tsx @@ -0,0 +1,142 @@ +import React, { useState, useEffect } from 'react'; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { Badge } from "@/components/ui/badge"; +import ReactMarkdown from 'react-markdown'; +import remarkGfm from 'remark-gfm'; +import { TaskStatus } from '@/types/workflow'; +import { AnalysisModuleConfig } from '@/types/index'; +import { BrainCircuit, Terminal } from 'lucide-react'; + +interface AnalysisModulesViewProps { + taskStates: Record; + taskOutputs: Record; + modulesConfig: Record; +} + +export function AnalysisModulesView({ + taskStates, + taskOutputs, + modulesConfig +}: AnalysisModulesViewProps) { + // Identify analysis tasks based on the template config + // We assume task IDs in the DAG correspond to module IDs or follow a pattern + // For now, let's try to match tasks that are NOT fetch tasks + + // If we have config, use it to drive tabs + const moduleIds = Object.keys(modulesConfig); + + const [activeModuleId, setActiveModuleId] = useState(moduleIds[0] || ''); + + useEffect(() => { + // If no active module and we have modules, select first + if (!activeModuleId && moduleIds.length > 0) { + setActiveModuleId(moduleIds[0]); + } + }, [moduleIds, activeModuleId]); + + if (moduleIds.length === 0) { + return ( +
+ +

No analysis modules defined in this template.

+
+ ); + } + + return ( +
+ +
+ + {moduleIds.map(moduleId => { + const config = modulesConfig[moduleId]; + // Task ID might match module ID directly or be prefixed + // Heuristic: check exact match first + const taskId = moduleId; + const status = taskStates[taskId] || 'pending'; + + return ( + +
+ {config.name} + +
+ + ); + })} + +
+ + {moduleIds.map(moduleId => { + const taskId = moduleId; + const output = taskOutputs[taskId] || ''; + const status = taskStates[taskId] || 'pending'; + const config = modulesConfig[moduleId]; + + return ( + + + +
+
+ {config.name} + + {config.model_id} + +
+ +
+ + + + {output ? ( +
+ + {output} + +
+ ) : ( +
+ +

{status === 'running' ? 'Generating analysis...' : 'Waiting for input...'}

+
+ )} + + + + + ); + })} + +
+ ); +} + +function StatusDot({ status }: { status: TaskStatus }) { + let colorClass = "bg-muted"; + if (status === 'completed') colorClass = "bg-green-500"; + if (status === 'failed') colorClass = "bg-red-500"; + if (status === 'running') colorClass = "bg-blue-500 animate-pulse"; + + return
; +} + +function StatusBadge({ status }: { status: TaskStatus }) { + switch (status) { + case 'completed': + return Completed; + case 'failed': + return Failed; + case 'running': + return Generating...; + default: + return Pending; + } + } + diff --git a/frontend/src/app/report/[symbol]/components/AnalysisViewer.tsx b/frontend/src/app/report/[symbol]/components/AnalysisViewer.tsx new file mode 100644 index 0000000..673a239 --- /dev/null +++ b/frontend/src/app/report/[symbol]/components/AnalysisViewer.tsx @@ -0,0 +1,50 @@ +import React from 'react'; +import ReactMarkdown from 'react-markdown'; +import remarkGfm from 'remark-gfm'; +import { AnalysisResultDto, AnalysisModuleConfig } from '@/types'; +import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; + +interface AnalysisViewerProps { + result?: AnalysisResultDto; + config: AnalysisModuleConfig; + isActive: boolean; +} + +export function AnalysisViewer({ result, config, isActive: _isActive }: AnalysisViewerProps) { + + if (!result) { + return ( +
+

Waiting for analysis...

+ {config.name} +
+ ); + } + + return ( + + +
+ {config.name} + + {config.model_id} + +
+ + Generated by {config.provider_id} + + + +
+ + {result.content} + + {/* Simple cursor effect if we think it's still streaming (we don't have explicit stream status per module here easily without more props, but this is fine for now) */} + {/* You could add a blinking cursor here if needed, but maybe overkill if we don't know for sure if it's done */} +
+ + + ); +} + diff --git a/frontend/src/app/report/[symbol]/components/FinancialTable.tsx b/frontend/src/app/report/[symbol]/components/FinancialTable.tsx index ebf595e..c3a4156 100644 --- a/frontend/src/app/report/[symbol]/components/FinancialTable.tsx +++ b/frontend/src/app/report/[symbol]/components/FinancialTable.tsx @@ -1,569 +1,99 @@ -import { useMemo } from 'react'; -import { Spinner } from '@/components/ui/spinner'; -import { CheckCircle, XCircle } from 'lucide-react'; -import { Table, TableHeader, TableBody, TableHead, TableRow, TableCell } from '@/components/ui/table'; -import { formatReportPeriod } from '@/lib/financial-utils'; -import { numberFormatter, integerFormatter } from '../utils'; +import React, { useMemo } from 'react'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { transformFinancialData, TimeSeriesFinancialDto } from '@/lib/financial-data-transformer'; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Loader2, AlertCircle } from "lucide-react"; interface FinancialTableProps { - financials: any; - isLoading: boolean; - error: any; - financialConfig: any; + data: unknown[]; + status: 'idle' | 'fetching' | 'complete' | 'error'; } -export function FinancialTable({ financials, isLoading, error, financialConfig }: FinancialTableProps) { - // 创建 tushareParam 到 displayText 的映射 - const metricDisplayMap = useMemo(() => { - if (!financialConfig?.api_groups) return {}; - - const map: Record = {}; - const groups = Object.values((financialConfig as any).api_groups || {}) as any[][]; - groups.forEach((metrics) => { - (metrics || []).forEach((metric: any) => { - if (metric.tushareParam && metric.displayText) { - map[metric.tushareParam] = metric.displayText; - } - }); - }); - return map; - }, [financialConfig]); +export function FinancialTable({ data, status }: FinancialTableProps) { + const tableData = useMemo(() => { + if (!Array.isArray(data)) return { headers: [], rows: [] }; + return transformFinancialData(data as TimeSeriesFinancialDto[]); + }, [data]); - const metricGroupMap = useMemo(() => { - if (!financialConfig?.api_groups) return {} as Record; - const map: Record = {}; - const entries = Object.entries((financialConfig as any).api_groups || {}) as [string, any[]][]; - entries.forEach(([groupName, metrics]) => { - (metrics || []).forEach((metric: any) => { - if (metric.tushareParam) { - map[metric.tushareParam] = groupName; - } - }); - }); - return map; - }, [financialConfig]); + if (status === 'idle') { + return ( +
+ Waiting to start analysis... +
+ ); + } + + if (status === 'fetching') { + return ( +
+ +

Fetching financial data from Tushare, YFinance, and AlphaVantage...

+
+ ); + } + + if (status === 'error') { + return ( +
+ + Error fetching data. Please try again. +
+ ); + } + + if (tableData.rows.length === 0) { + return ( +
+ No financial data available. +
+ ); + } return ( -
-

财务数据

-
- {isLoading ? ( - - ) : error ? ( - - ) : ( - - )} -
- {isLoading ? '正在读取数据…' : error ? '读取失败' : '读取完成'} + + + Financial Statements + + +
+ + + + Metric + {tableData.headers.map(year => ( + {year} + ))} + + + + {tableData.rows.map((row, idx) => ( + + {row.metric} + {tableData.headers.map(year => ( + + {row[year] !== undefined + ? typeof row[year] === 'number' + ? (row[year] as number).toLocaleString(undefined, { maximumFractionDigits: 2 }) + : row[year] + : '-'} + + ))} + + ))} + +
-
- {error &&

加载失败

} - - {isLoading && ( -
- 加载中 - +
+ * Data aggregated from multiple sources. Duplicate metrics from different sources are currently overwritten by the latest received.
- )} - - {financials && ( -
- {(() => { - const series = financials?.series ?? {}; - // 统一 period：优先 p.period；若仅有 year 则映射到 `${year}1231` - const toPeriod = (p: any): string | null => { - if (!p) return null; - if (p.period) return String(p.period); - if (p.year) return `${p.year}1231`; - return null; - }; - - const displayedKeys = [ - 'roe', 'roa', 'roic', 'grossprofit_margin', 'netprofit_margin', 'revenue', 'tr_yoy', 'n_income', - 'dt_netprofit_yoy', 'n_cashflow_act', 'c_pay_acq_const_fiolta', '__free_cash_flow', - 'dividend_amount', 'repurchase_amount', 'total_assets', 'total_hldr_eqy_exc_min_int', 'goodwill', - '__sell_rate', '__admin_rate', '__rd_rate', '__other_fee_rate', '__tax_rate', '__depr_ratio', - '__money_cap_ratio', '__inventories_ratio', '__ar_ratio', '__prepay_ratio', '__fix_assets_ratio', - '__lt_invest_ratio', '__goodwill_ratio', '__other_assets_ratio', '__ap_ratio', '__adv_ratio', - '__st_borr_ratio', '__lt_borr_ratio', '__operating_assets_ratio', '__interest_bearing_debt_ratio', - 'invturn_days', 'arturn_days', 'payturn_days', 'fa_turn', 'assets_turn', - 'employees', '__rev_per_emp', '__profit_per_emp', '__salary_per_emp', - 'close', 'total_mv', 'pe', 'pb', 'holder_num' - ]; - - const displayedSeries = Object.entries(series) - .filter(([key]) => displayedKeys.includes(key)) - .map(([, value]) => value); - - const allPeriods = Array.from( - new Set( - (displayedSeries.flat() as any[]) - .map((p) => toPeriod(p)) - .filter((v): v is string => Boolean(v)) - ) - ).sort((a, b) => b.localeCompare(a)); // 最新在左（按 YYYYMMDD 排序） - - if (allPeriods.length === 0) { - return

暂无可展示的数据

; - } - const periods = allPeriods.slice(0, 10); - - const getValueByPeriod = (points: any[] | undefined, period: string): number | null => { - if (!points) return null; - const hit = points.find((pp) => toPeriod(pp) === period); - const v = hit?.value; - if (v == null) return null; - const num = typeof v === 'number' ? v : Number(v); - return Number.isFinite(num) ? num : null; - }; - return ( - - - - 指标 - {periods.map((p) => ( - {formatReportPeriod(p)} - ))} - - - - {(() => { - // 指定显示顺序（tushareParam） - const ORDER: Array<{ key: string; label?: string }> = [ - { key: 'roe' }, - { key: 'roa' }, - { key: 'roic' }, - { key: 'grossprofit_margin' }, - { key: 'netprofit_margin' }, - { key: 'revenue' }, - { key: 'tr_yoy' }, - { key: 'n_income' }, - { key: 'dt_netprofit_yoy' }, - { key: 'n_cashflow_act' }, - { key: 'c_pay_acq_const_fiolta' }, - { key: '__free_cash_flow', label: '自由现金流' }, - { key: 'dividend_amount', label: '分红' }, - { key: 'repurchase_amount', label: '回购' }, - { key: 'total_assets' }, - { key: 'total_hldr_eqy_exc_min_int' }, - { key: 'goodwill' }, - ]; - - // 在表格顶部插入"主要指标"行 - const summaryRow = ( - - 主要指标 - {periods.map((p) => ( - - ))} - - ); - - const PERCENT_KEYS = new Set([ - 'roe','roa','roic','grossprofit_margin','netprofit_margin','tr_yoy','dt_netprofit_yoy', - // Add all calculated percentage rows - '__sell_rate', '__admin_rate', '__rd_rate', '__other_fee_rate', '__tax_rate', '__depr_ratio', - '__money_cap_ratio', '__inventories_ratio', '__ar_ratio', '__prepay_ratio', - '__fix_assets_ratio', '__lt_invest_ratio', '__goodwill_ratio', '__other_assets_ratio', - '__ap_ratio', '__adv_ratio', '__st_borr_ratio', '__lt_borr_ratio', - '__operating_assets_ratio', '__interest_bearing_debt_ratio' - ]); - const rows = ORDER.map(({ key, label }) => { - const points = series[key] as any[] | undefined; - - return ( - - - {label || metricDisplayMap[key] || key} - - {periods.map((p) => { - const v = getValueByPeriod(points, p); - - const groupName = metricGroupMap[key]; - const rawNum = typeof v === 'number' ? v : (v == null ? null : Number(v)); - if (rawNum == null || Number.isNaN(rawNum)) { - return -; - } - if (PERCENT_KEYS.has(key)) { - const perc = Math.abs(rawNum) <= 1 && key !== 'tax_to_ebt' && key !== '__tax_rate' ? rawNum * 100 : rawNum; - const text = Number.isFinite(perc) ? numberFormatter.format(perc) : '-'; - const isGrowthRow = key === 'tr_yoy' || key === 'dt_netprofit_yoy'; - if (isGrowthRow) { - const isNeg = typeof perc === 'number' && perc < 0; - const isHighGrowth = typeof perc === 'number' && perc > 30; - - let content = `${text}%`; - if (key === 'dt_netprofit_yoy' && typeof perc === 'number' && perc > 1000) { - content = `${(perc / 100).toFixed(1)}x`; - } - - let tableCellClassName = 'text-right p-2'; - let spanClassName = 'italic'; - - if (isNeg) { - tableCellClassName += ' bg-red-100'; - spanClassName += ' text-red-600'; - } else if (isHighGrowth) { - tableCellClassName += ' bg-green-100'; - spanClassName += ' text-green-800 font-bold'; - } else { - spanClassName += ' text-blue-600'; - } - - return ( - - {content} - - ); - } - const isHighlighted = (key === 'roe' && typeof perc === 'number' && perc > 12.5) || - (key === 'grossprofit_margin' && typeof perc === 'number' && perc > 35) || - (key === 'netprofit_margin' && typeof perc === 'number' && perc > 15); - - if (isHighlighted) { - return ( - - {`${text}%`} - - ); - } - return ( - {`${text}%`} - ); - } else { - const isFinGroup = groupName === 'income' || groupName === 'balancesheet' || groupName === 'cashflow'; - const scaled = key === 'total_mv' - ? rawNum / 10000 - : (isFinGroup || key === '__free_cash_flow' || key === 'repurchase_amount' ? rawNum / 1e8 : rawNum); - const formatter = key === 'total_mv' ? integerFormatter : numberFormatter; - const text = Number.isFinite(scaled) ? formatter.format(scaled) : '-'; - if (key === '__free_cash_flow') { - const isNeg = typeof scaled === 'number' && scaled < 0; - return ( - - {isNeg ? {text} : text} - - ); - } - return ( - {text} - ); - } - })} - - ); - }); - - // ========================= - // 费用指标分组 - // ========================= - const feeHeaderRow = ( - - 费用指标 - {periods.map((p) => ( - - ))} - - ); - - const feeRows = [ - { key: '__sell_rate', label: '销售费用率' }, - { key: '__admin_rate', label: '管理费用率' }, - { key: '__rd_rate', label: '研发费用率' }, - { key: '__other_fee_rate', label: '其他费用率' }, - { key: '__tax_rate', label: '所得税率' }, - { key: '__depr_ratio', label: '折旧费用占比' }, - ].map(({ key, label }) => ( - - {label} - {periods.map((p) => { - const points = series[key] as any[] | undefined; - const v = getValueByPeriod(points, p); - - if (v == null || !Number.isFinite(v)) { - return -; - } - const rateText = numberFormatter.format(v); - const isNegative = v < 0; - return ( - - {isNegative ? {rateText}% : `${rateText}%`} - - ); - })} - - )); - - // ========================= - // 资产占比分组 - // ========================= - const assetHeaderRow = ( - - 资产占比 - {periods.map((p) => ( - - ))} - - ); - - const ratioCell = (value: number | null, keyStr: string) => { - if (value == null || !Number.isFinite(value)) { - return -; - } - const text = numberFormatter.format(value); - const isNegative = value < 0; - const isHighRatio = value > 30; - - let cellClassName = "text-right p-2"; - if (isHighRatio) { - cellClassName += " bg-red-100"; - } else if (isNegative) { - cellClassName += " bg-red-100"; - } - - return ( - - {isNegative ? {text}% : `${text}%`} - - ); - }; - - const assetRows = [ - { key: '__money_cap_ratio', label: '现金占比' }, - { key: '__inventories_ratio', label: '库存占比' }, - { key: '__ar_ratio', label: '应收款占比' }, - { key: '__prepay_ratio', label: '预付款占比' }, - { key: '__fix_assets_ratio', label: '固定资产占比' }, - { key: '__lt_invest_ratio', label: '长期投资占比' }, - { key: '__goodwill_ratio', label: '商誉占比' }, - { key: '__other_assets_ratio', label: '其他资产占比' }, - { key: '__ap_ratio', label: '应付款占比' }, - { key: '__adv_ratio', label: '预收款占比' }, - { key: '__st_borr_ratio', label: '短期借款占比' }, - { key: '__lt_borr_ratio', label: '长期借款占比' }, - { key: '__operating_assets_ratio', label: '运营资产占比' }, - { key: '__interest_bearing_debt_ratio', label: '有息负债率' }, - ].map(({ key, label }) => ( - - {label} - {periods.map((p) => { - const points = series[key] as any[] | undefined; - const v = getValueByPeriod(points, p); - return ratioCell(v, p); - })} - - )); - - // ========================= - // 周转能力分组 - // ========================= - const turnoverHeaderRow = ( - - 周转能力 - {periods.map((p) => ( - - ))} - - ); - - const turnoverItems: Array<{ key: string; label: string }> = [ - { key: 'invturn_days', label: '存货周转天数' }, - { key: 'arturn_days', label: '应收款周转天数' }, - { key: 'payturn_days', label: '应付款周转天数' }, - { key: 'fa_turn', label: '固定资产周转率' }, - { key: 'assets_turn', label: '总资产周转率' }, - ]; - - const turnoverRows = turnoverItems.map(({ key, label }) => ( - - {label} - {periods.map((p) => { - const points = series[key] as any[] | undefined; - const v = getValueByPeriod(points, p); - const value = typeof v === 'number' ? v : (v == null ? null : Number(v)); - - if (value == null || !Number.isFinite(value)) { - return -; - } - const text = numberFormatter.format(value); - if (key === 'arturn_days' && value > 90) { - return ( - {text} - ); - } - return {text}; - })} - - )); - - return [ - summaryRow, - ...rows, - feeHeaderRow, - ...feeRows, - assetHeaderRow, - ...assetRows, - turnoverHeaderRow, - ...turnoverRows, - // ========================= - // 人均效率分组 - // ========================= - ( - - 人均效率 - {periods.map((p) => ( - - ))} - - ), - // 员工人数（整数千分位） - ( - - 员工人数 - {periods.map((p) => { - const points = series['employees'] as any[] | undefined; - const v = getValueByPeriod(points, p); - if (v == null) { - return -; - } - return {integerFormatter.format(Math.round(v))}; - })} - - ), - // 人均创收 = 收入 / 员工人数（万元） - ( - - 人均创收（万元） - {periods.map((p) => { - const points = series['__rev_per_emp'] as any[] | undefined; - const val = getValueByPeriod(points, p); - if (val == null) { - return -; - } - return {numberFormatter.format(val)}; - })} - - ), - // 人均创利 = 净利润 / 员工人数（万元） - ( - - 人均创利（万元） - {periods.map((p) => { - const points = series['__profit_per_emp'] as any[] | undefined; - const val = getValueByPeriod(points, p); - if (val == null) { - return -; - } - return {numberFormatter.format(val)}; - })} - - ), - // 人均工资 = 支付给职工以及为职工支付的现金 / 员工人数（万元） - ( - - 人均工资（万元） - {periods.map((p) => { - const points = series['__salary_per_emp'] as any[] | undefined; - const val = getValueByPeriod(points, p); - if (val == null) { - return -; - } - return {numberFormatter.format(val)}; - })} - - ), - // ========================= - // 市场表现分组 - // ========================= - ( - - 市场表现 - {periods.map((p) => ( - - ))} - - ), - // 股价（收盘价） - ( - - 股价 - {periods.map((p) => { - const points = series['close'] as any[] | undefined; - const v = getValueByPeriod(points, p); - if (v == null) { - return -; - } - return {numberFormatter.format(v)}; - })} - - ), - // 市值（按亿为单位显示：乘以10000并整数千分位） - ( - - 市值（亿元） - {periods.map((p) => { - const points = series['total_mv'] as any[] | undefined; - const v = getValueByPeriod(points, p); - if (v == null) { - return -; - } - const scaled = v / 10000; // 转为亿元 - return {integerFormatter.format(Math.round(scaled))}; - })} - - ), - // PE - ( - - PE - {periods.map((p) => { - const points = series['pe'] as any[] | undefined; - const v = getValueByPeriod(points, p); - if (v == null) { - return -; - } - return {numberFormatter.format(v)}; - })} - - ), - // PB - ( - - PB - {periods.map((p) => { - const points = series['pb'] as any[] | undefined; - const v = getValueByPeriod(points, p); - if (v == null) { - return -; - } - return {numberFormatter.format(v)}; - })} - - ), - // 股东户数 - ( - - 股东户数 - {periods.map((p) => { - const points = series['holder_num'] as any[] | undefined; - const v = getValueByPeriod(points, p); - if (v == null) { - return -; - } - return {integerFormatter.format(Math.round(v))}; - })} - - ), - ]; - })()} - -
- ); - })()} -
- )} -
+ + ); } - diff --git a/frontend/src/app/report/[symbol]/components/FundamentalDataView.tsx b/frontend/src/app/report/[symbol]/components/FundamentalDataView.tsx new file mode 100644 index 0000000..c5cc33c --- /dev/null +++ b/frontend/src/app/report/[symbol]/components/FundamentalDataView.tsx @@ -0,0 +1,90 @@ +import React from 'react'; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { Badge } from "@/components/ui/badge"; +import { TaskStatus } from '@/types/workflow'; + +interface FundamentalDataViewProps { + taskStates: Record; + taskOutputs: Record; +} + +export function FundamentalDataView({ taskStates, taskOutputs }: FundamentalDataViewProps) { + // Filter tasks that look like data fetching tasks + const dataTasks = Object.keys(taskStates).filter(taskId => + taskId.startsWith('fetch:') // Standardized task ID format: "fetch:provider_id" + ); + + if (dataTasks.length === 0) { + return ( +
+ No data providers detected in this workflow. +
+ ); + } + + return ( +
+ {dataTasks.map(taskId => { + const status = taskStates[taskId]; + const output = taskOutputs[taskId]; + // Dynamic name resolution: extract provider ID from task ID (e.g., "fetch:tushare" -> "Tushare") + const providerId = taskId.replace('fetch:', ''); + const providerName = providerId.charAt(0).toUpperCase() + providerId.slice(1); + + return ( + + + + {providerName} + + + + + + {output ? ( + 
+                    {tryFormatJson(output)}
+
+ ) : ( +
+ {status === 'pending' || status === 'running' ? 'Waiting for data...' : 'No data returned'} +
+ )} + + + + ); + })} +
+ ); +} + +function StatusBadge({ status }: { status: TaskStatus }) { + switch (status) { + case 'completed': + return Success; + case 'failed': + return Failed; + case 'running': + return Fetching; + default: + return Pending; + } +} + +function tryFormatJson(str: string): string { + try { + // Only try to format if it looks like JSON object or array + const trimmed = str.trim(); + if ((trimmed.startsWith('{') && trimmed.endsWith('}')) || + (trimmed.startsWith('[') && trimmed.endsWith(']'))) { + const obj = JSON.parse(str); + return JSON.stringify(obj, null, 2); + } + return str; + } catch (e) { + return str; + } +} + diff --git a/frontend/src/app/report/[symbol]/components/RawDataViewer.tsx b/frontend/src/app/report/[symbol]/components/RawDataViewer.tsx new file mode 100644 index 0000000..ff2cd67 --- /dev/null +++ b/frontend/src/app/report/[symbol]/components/RawDataViewer.tsx @@ -0,0 +1,203 @@ +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { ScrollArea } from "@/components/ui/scroll-area"; +import { TaskProgress } from "@/types"; +import { useState, useMemo } from "react"; +import { ChevronDown, ChevronRight, CheckCircle2, XCircle, Loader2, AlertCircle } from "lucide-react"; +import { cn } from "@/lib/utils"; + +interface RawDataViewerProps { + data: unknown[]; + tasks: TaskProgress[]; + status: 'idle' | 'fetching' | 'complete' | 'error'; +} + +export function RawDataViewer({ data, tasks, status }: RawDataViewerProps) { + const groupedProviders = useMemo(() => { + // 1. Group Data by Source + const dataGroups: Record = {}; + if (Array.isArray(data)) { + data.forEach((item: any) => { + // Normalize source: lowercase + // In SessionDataDto, provider is the key. item.source is legacy/fallback. + const source = (item.provider || item.source || 'unknown').toLowerCase(); + if (!dataGroups[source]) dataGroups[source] = []; + // We push the whole item which contains { provider, data_type, data_payload } + dataGroups[source].push(item); + }); + } + + // 2. Map Tasks to Providers + const providerMap = new Map(); + + // Populate from Tasks first (as they represent the current execution plan) + tasks.forEach(task => { + // Task name format: "provider:symbol" or just "provider" + const providerName = task.task_name.split(':')[0].toLowerCase(); + providerMap.set(providerName, { + name: providerName, + status: task.status, + details: task.details, + data: dataGroups[providerName] || [] + }); + }); + + // Populate remaining data sources (historical or pre-fetched) that might not have a task + Object.keys(dataGroups).forEach(source => { + if (!providerMap.has(source)) { + providerMap.set(source, { + name: source, + status: 'completed', // If we have data, assume it's done + details: 'Data loaded from cache/db', + data: dataGroups[source] + }); + } + }); + + return Array.from(providerMap.values()); + }, [data, tasks]); + + if (groupedProviders.length === 0 && status === 'error') { + return ( +
+ + Error fetching data. +
+ ); + } + + if (groupedProviders.length === 0 && status === 'fetching') { + return ( +
+ + Waiting for providers... +
+ ) + } + + return ( +
+ {groupedProviders.map(provider => ( + + ))} + + {groupedProviders.length === 0 && status === 'complete' && ( +
+ No data providers found. +
+ )} +
+ ); +} + +function ProviderSection({ + name, + status, + details, + data +}: { + name: string; + status: string; + details?: string; + data: unknown[] +}) { + const [isOpen, setIsOpen] = useState(false); + + // Auto-open if there is an error to show details, otherwise keep closed to save space + // or keep open if it's the only one? Let's default to closed but maybe open if it has interesting data? + // User said "generally collapsed", so default false. + + return ( + + setIsOpen(!isOpen)} + > +
+
+ {isOpen ? : } + +
+
+ + {name} + + +
+
+
+ +
+ + {data.length} records + +
+
+ + {/* Progress/Details Text */} + {(details || status === 'in_progress') && ( +
+ {details} +
+ )} + + + {isOpen && ( + +
+ {data.length > 0 ? ( + + {/* Show structured data if available */} +
+ {data.map((item: any, idx) => ( +
+
+ {item.data_type || 'unknown'} + {item.created_at || ''} +
+ 
+                                       {/* If item.data_payload exists, show it. Otherwise show item (legacy) */}
+                                       {JSON.stringify(item.data_payload !== undefined ? item.data_payload : item, null, 2)}
+
+
+ ))} +
+ + ) : ( +
+ {status === 'failed' + ? "No data generated due to failure." + : "No data records available yet."} +
+ )} +
+ + )} + + ); +} + +function StatusBadge({ status }: { status: string }) { + const s = status.toLowerCase(); + if (s === 'completed' || s === 'success') { + return Success; + } + if (s === 'failed' || s === 'error') { + return Failed; + } + if (s === 'in_progress' || s === 'running') { + return Running; + } + return {status}; +} diff --git a/frontend/src/app/report/[symbol]/components/ReportLayout.tsx b/frontend/src/app/report/[symbol]/components/ReportLayout.tsx new file mode 100644 index 0000000..5332fc0 --- /dev/null +++ b/frontend/src/app/report/[symbol]/components/ReportLayout.tsx @@ -0,0 +1,168 @@ +import React, { useEffect, useState } from 'react'; +import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs"; +import { Card, CardHeader, CardTitle } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Badge } from "@/components/ui/badge"; +import { Loader2 } from "lucide-react"; +import { AnalysisTemplateSets } from '@/types'; +import { ReportState } from '@/types/report'; +import { RawDataViewer } from './RawDataViewer'; +import { FinancialTable } from './FinancialTable'; +import { AnalysisViewer } from './AnalysisViewer'; +import { StockChart } from './StockChart'; + +interface ReportLayoutProps { + state: ReportState; + onTemplateChange: (id: string) => void; + onTrigger: () => void; +} + +export function ReportLayout({ state, onTemplateChange, onTrigger }: ReportLayoutProps) { + const [templates, setTemplates] = useState({}); + const [activeTab, setActiveTab] = useState("chart"); + + // Load templates for dropdown + useEffect(() => { + fetch('/api/configs/analysis_template_sets') + .then(res => res.json()) + .then(data => setTemplates(data)) + .catch(err => console.error(err)); + }, []); + + // Auto-switch tabs based on state + useEffect(() => { + if (state.fetchStatus === 'fetching') { + setActiveTab("fundamental"); + } else if (state.analysisStatus === 'running' && activeTab === 'fundamental') { + // Try to switch to the first analysis tab if available + const firstModule = state.templateConfig ? Object.keys(state.templateConfig.modules)[0] : null; + if (firstModule) { + setActiveTab(firstModule); + } + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [state.fetchStatus, state.analysisStatus, state.templateConfig]); + + return ( +
+ {/* Header Area */} + + +
+ {state.symbol} +
+ Market: {state.market || 'Unknown'} + • + + {state.analysisStatus === 'running' ? 'Analyzing...' : + state.analysisStatus === 'error' ? 'Failed' : + state.fetchStatus === 'fetching' ? 'Fetching Data...' : 'Ready'} + +
+
+
+
+ +
+ +
+ + + + {/* Main Content Area */} + {/* Error Banner */} + {(state.analysisStatus === 'error' || state.fetchStatus === 'error') && state.error && ( +
+ Error: {state.error} +
+ )} + + + + Stock Chart + Fundamental Data + + {/* Dynamic Tabs from Template */} + {state.templateConfig && Object.entries(state.templateConfig.modules).map(([moduleId, moduleConfig]) => ( + + {moduleConfig.name} + {state.analysisStatus === 'running' && !state.analysisResults[moduleId] && ( + + )} + + ))} + + +
+ {/* SCENARIO 1: Stock Chart */} + + + + + {/* SCENARIO 2: Fundamental Data */} + + {/* Always show the Provider/Task Grid (RawDataViewer) as it contains the per-provider status */} + + + {/* Show Aggregated Table only when complete */} + {/* User request: Hide consolidated table temporarily to focus on raw provider data */} + {/* {state.fetchStatus === 'complete' && ( +
+
+

Aggregated Financial Statements

+
+ +
+ )} */} + + + {/* SCENARIO 3: Analysis Modules */} + {state.templateConfig && Object.entries(state.templateConfig.modules).map(([moduleId, moduleConfig]) => ( + + + + ))} +
+ + + {/* Execution Details Footer */} +
+ Request ID: {state.requestId || '-'} + + Time: {(state.executionMeta.elapsed / 1000).toFixed(1)}s + | Tokens: {state.executionMeta.tokens} + +
+
+ ); +} + diff --git a/frontend/src/app/report/[symbol]/components/StockChart.tsx b/frontend/src/app/report/[symbol]/components/StockChart.tsx index e805e2f..173defe 100644 --- a/frontend/src/app/report/[symbol]/components/StockChart.tsx +++ b/frontend/src/app/report/[symbol]/components/StockChart.tsx @@ -1,57 +1,20 @@ -import { CheckCircle } from 'lucide-react'; -import { Spinner } from '@/components/ui/spinner'; +import React from 'react'; import { TradingViewWidget } from '@/components/TradingViewWidget'; interface StockChartProps { - unifiedSymbol: string; - marketParam: string; - realtime: any; - realtimeLoading: boolean; - realtimeError: any; + symbol: string; } -export function StockChart({ - unifiedSymbol, - marketParam, - realtime, - realtimeLoading, - realtimeError, -}: StockChartProps) { - return ( -
-

股价图表（来自 TradingView）

-
-
- -
- 实时股价图表 - {unifiedSymbol} -
-
-
- {realtimeLoading ? ( - 正在获取实时报价… - ) : realtimeError ? ( - 实时报价不可用 - ) : (() => { - const priceRaw = realtime?.price; - const priceNum = typeof priceRaw === 'number' ? priceRaw : Number(priceRaw); - const tsRaw = realtime?.ts; - const tsDate = tsRaw == null ? null : new Date(typeof tsRaw === 'number' ? tsRaw : String(tsRaw)); - const tsText = tsDate && !isNaN(tsDate.getTime()) ? `（${tsDate.toLocaleString()}）` : ''; - if (Number.isFinite(priceNum)) { - return 价格 {priceNum.toLocaleString()} {tsText}; - } - return 暂无最新报价; - })()} -
-
+export function StockChart({ symbol }: StockChartProps) { + // Simple heuristic to detect market. + // If 6 digits at start or ends with .SH/.SZ, it's likely China. + // Otherwise default to US (or let TradingView handle it). + const isChina = /^\d{6}/.test(symbol) || symbol.endsWith('.SH') || symbol.endsWith('.SZ'); + const market = isChina ? 'china' : 'us'; - + return ( +
+
); } - diff --git a/frontend/src/app/report/[symbol]/components/WorkflowReportLayout.tsx b/frontend/src/app/report/[symbol]/components/WorkflowReportLayout.tsx new file mode 100644 index 0000000..3d2014d --- /dev/null +++ b/frontend/src/app/report/[symbol]/components/WorkflowReportLayout.tsx @@ -0,0 +1,275 @@ +import React, { useEffect, useState } from 'react'; +import { useRouter } from 'next/navigation'; // Import router +import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs"; +import { Card, CardHeader, CardTitle, CardContent } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Badge } from "@/components/ui/badge"; +import { Loader2, Play, RefreshCw, BrainCircuit, Activity, Database, LayoutDashboard, LineChart, FileText } from "lucide-react"; +import { WorkflowVisualizer } from '@/components/workflow/WorkflowVisualizer'; +import { StockChart } from './StockChart'; +import { useWorkflow } from '@/hooks/useWorkflow'; +import { AnalysisTemplateSets } from '@/types/index'; +import ReactMarkdown from 'react-markdown'; +import remarkGfm from 'remark-gfm'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { FundamentalDataView } from './FundamentalDataView'; +import { AnalysisModulesView } from './AnalysisModulesView'; + +interface WorkflowReportLayoutProps { + symbol: string; + initialMarket?: string; + initialTemplateId?: string; + initialRequestId?: string; +} + +export function WorkflowReportLayout({ + symbol, + initialMarket, + initialTemplateId, + initialRequestId +}: WorkflowReportLayoutProps) { + const router = useRouter(); + const workflow = useWorkflow(); + const [templates, setTemplates] = useState({}); + const [selectedTemplateId, setSelectedTemplateId] = useState(initialTemplateId || ''); + const [activeTab, setActiveTab] = useState("chart"); // Default to chart for quick overview + + // Auto-connect if initialRequestId is provided + useEffect(() => { + if (initialRequestId && workflow.status === 'idle' && !workflow.requestId) { + workflow.connectToWorkflow(initialRequestId); + } + }, [initialRequestId, workflow]); + + // Load templates + useEffect(() => { + fetch('/api/configs/analysis_template_sets') + .then(res => res.json()) + .then(data => { + setTemplates(data); + if (!selectedTemplateId && Object.keys(data).length > 0) { + // Default to 'standard_analysis' if exists, else first key + setSelectedTemplateId(data['standard_analysis'] ? 'standard_analysis' : Object.keys(data)[0]); + } + }) + .catch(err => console.error('Failed to load templates:', err)); + }, [selectedTemplateId]); + + // Auto switch to analysis tab when workflow starts + useEffect(() => { + if (workflow.status === 'connecting' || workflow.status === 'connected') { + setActiveTab("analysis"); + } + }, [workflow.status]); + + const handleStart = async () => { + if (!selectedTemplateId) return; + + const response = await workflow.startWorkflow({ + symbol, + market: initialMarket, + template_id: selectedTemplateId + }); + + // Handle Symbol Normalization Redirection + if (response && response.symbol && response.symbol !== symbol) { + console.log(`Redirecting normalized symbol: ${symbol} -> ${response.symbol}`); + const newUrl = `/report/${encodeURIComponent(response.symbol)}?template_id=${selectedTemplateId}&market=${response.market}`; + router.replace(newUrl); + } + }; + + const isRunning = workflow.status === 'connecting' || workflow.status === 'connected'; + + // Get current template config for dynamic tabs + const currentTemplate = templates[selectedTemplateId]; + const dynamicModules = currentTemplate?.modules || {}; + + return ( +
+ {/* Header Card */} + + +
+ + {symbol} + + {initialMarket || 'Unknown Market'} + + +
+ + {workflow.requestId && ( + + ID: {workflow.requestId} + + )} +
+
+ +
+
+ +
+ +
+ + + + {/* Main Content Tabs */} + + + + + Market Chart + + + + Fundamental Data + + + + Analysis Modules + + + + Workflow Monitor + + + + {/* Tab A: Market Chart */} + + + + + {/* Tab B: Fundamental Data */} + + + + + {/* Tab C: Analysis Modules */} + + {workflow.requestId ? ( + + ) : ( + + )} + + + {/* Tab D: Workflow Monitor */} + + {workflow.dag ? ( +
+
+ +
+
+ + + Execution Stats + + +
+
+ Status + {workflow.status} +
+
+ Tasks Total + {workflow.dag.nodes.length} +
+
+ Tasks Completed + + {Object.values(workflow.taskStates).filter(s => s === 'completed').length} + +
+ {workflow.finalResult && ( +
+ +
+ )} +
+ + +
+
+ ) : ( + + )} + + +
+ ); +} + +function StatusBadge({ status, error }: { status: string, error: string | null }) { + if (error) { + return Error: {error}; + } + switch (status) { + case 'connecting': + case 'connected': + return Processing; + case 'disconnected': // Usually means finished + return Finished; + case 'idle': + return Ready; + default: + return {status}; + } +} + +function EmptyState({ onStart, message }: { onStart: () => void, message: string }) { + return ( +
+ +

{message}

+ +
+ ); +} + +// Removed FinalReportView as it is now superseded by AnalysisModulesView + diff --git a/frontend/src/app/report/[symbol]/hooks/useAnalysisRunner.ts b/frontend/src/app/report/[symbol]/hooks/useAnalysisRunner.ts deleted file mode 100644 index 34282e5..0000000 --- a/frontend/src/app/report/[symbol]/hooks/useAnalysisRunner.ts +++ /dev/null @@ -1,446 +0,0 @@ -import { useState, useRef, useEffect, useMemo } from 'react'; -import { useDataRequest, useTaskProgress } from '@/hooks/useApi'; - -interface AnalysisState { - content: string; - loading: boolean; - error: string | null; - elapsed_ms?: number; -} - -interface AnalysisRecord { - type: string; - name: string; - status: 'pending' | 'running' | 'done' | 'error'; - start_ts?: string; - end_ts?: string; - duration_ms?: number; - tokens?: { - prompt_tokens: number; - completion_tokens: number; - total_tokens: number; - }; - error?: string; -} - -export function useAnalysisRunner( - financials: any, - financialConfig: any, - normalizedMarket: string, - unifiedSymbol: string, - isLoading: boolean, - error: any, - templateSets: any // Added templateSets -) { - // --- Template Logic --- - const [selectedTemplateId, setSelectedTemplateId] = useState(''); - - // Set default template - useEffect(() => { - if (!selectedTemplateId && templateSets && Object.keys(templateSets).length > 0) { - const defaultId = Object.keys(templateSets).find(k => k.includes('standard') || k === 'default') || Object.keys(templateSets)[0]; - setSelectedTemplateId(defaultId); - } - }, [templateSets, selectedTemplateId]); - - const reportTemplateId = financials?.meta?.template_id; - - // Determine active template set - const activeTemplateId = (financials && reportTemplateId) ? reportTemplateId : selectedTemplateId; - - const activeTemplateSet = useMemo(() => { - if (!activeTemplateId || !templateSets) return null; - return templateSets[activeTemplateId] || null; - }, [activeTemplateId, templateSets]); - - // Derive effective analysis config from template set, falling back to global config if needed - const activeAnalysisConfig = useMemo(() => { - if (activeTemplateSet) { - return { - ...financialConfig, - analysis_modules: activeTemplateSet.modules, - }; - } - return financialConfig; // Fallback to global config (legacy behavior) - }, [activeTemplateSet, financialConfig]); - - // 分析类型列表 - const analysisTypes = useMemo(() => { - if (!activeAnalysisConfig?.analysis_modules) return []; - return Object.keys(activeAnalysisConfig.analysis_modules); - }, [activeAnalysisConfig]); - - // 分析状态管理 - const [analysisStates, setAnalysisStates] = useState>({}); - - const fullAnalysisTriggeredRef = useRef(false); - const isAnalysisRunningRef = useRef(false); - const analysisFetchedRefs = useRef>({}); - const stopRequestedRef = useRef(false); - const abortControllerRef = useRef(null); - const currentAnalysisTypeRef = useRef(null); - const [manualRunKey, setManualRunKey] = useState(0); - - // 当前正在执行的分析任务 - const [currentAnalysisTask, setCurrentAnalysisTask] = useState(null); - - // 计时器状态 - const [startTime, setStartTime] = useState(null); - const [elapsedSeconds, setElapsedSeconds] = useState(0); - - // 分析执行记录 - const [analysisRecords, setAnalysisRecords] = useState([]); - - // 新架构：触发分析与查看任务进度 - const { trigger: triggerAnalysisRequest, isMutating: triggering } = useDataRequest(); - const [requestId, setRequestId] = useState(null); - const { progress: taskProgress } = useTaskProgress(requestId); - - // 计算完成比例 - const completionProgress = useMemo(() => { - const totalTasks = analysisRecords.length; - if (totalTasks === 0) return 0; - const completedTasks = analysisRecords.filter(r => r.status === 'done' || r.status === 'error').length; - return (completedTasks / totalTasks) * 100; - }, [analysisRecords]); - - // 总耗时（ms） - const totalElapsedMs = useMemo(() => { - const finMs = financials?.meta?.elapsed_ms || 0; - const analysesMs = analysisRecords.reduce((sum, r) => sum + (r.duration_ms || 0), 0); - return finMs + analysesMs; - }, [financials?.meta?.elapsed_ms, analysisRecords]); - - const hasRunningTask = useMemo(() => { - if (currentAnalysisTask !== null) return true; - if (analysisRecords.some(r => r.status === 'running')) return true; - return false; - }, [currentAnalysisTask, analysisRecords]); - - // 全部任务是否完成 - const allTasksCompleted = useMemo(() => { - if (analysisRecords.length === 0) return false; - const allDoneOrErrored = analysisRecords.every(r => r.status === 'done' || r.status === 'error'); - return allDoneOrErrored && !hasRunningTask && currentAnalysisTask === null; - }, [analysisRecords, hasRunningTask, currentAnalysisTask]); - - // 所有任务完成时，停止计时器 - useEffect(() => { - if (allTasksCompleted) { - setStartTime(null); - } - }, [allTasksCompleted]); - - useEffect(() => { - if (!startTime) return; - const interval = setInterval(() => { - const now = Date.now(); - const elapsed = Math.floor((now - startTime) / 1000); - setElapsedSeconds(elapsed); - }, 1000); - return () => clearInterval(interval); - }, [startTime]); - - const retryAnalysis = async (analysisType: string) => { - if (!financials || !activeAnalysisConfig?.analysis_modules) { - return; - } - analysisFetchedRefs.current[analysisType] = false; - setAnalysisStates(prev => ({ - ...prev, - [analysisType]: { content: '', loading: true, error: null } - })); - setAnalysisRecords(prev => prev.filter(record => record.type !== analysisType)); - const analysisName = - activeAnalysisConfig.analysis_modules[analysisType]?.name || analysisType; - const startTimeISO = new Date().toISOString(); - setCurrentAnalysisTask(analysisType); - setAnalysisRecords(prev => [...prev, { - type: analysisType, - name: analysisName, - status: 'running', - start_ts: startTimeISO - }]); - - try { - const startedMsLocal = Date.now(); - const response = await fetch( - `/api/financials/${normalizedMarket}/${unifiedSymbol}/analysis/${analysisType}/stream?company_name=${encodeURIComponent(financials?.name || unifiedSymbol)}` - ); - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - const reader = response.body?.getReader(); - const decoder = new TextDecoder(); - let aggregate = ''; - if (reader) { - while (true) { - const { value, done } = await reader.read(); - if (done) break; - const chunk = decoder.decode(value, { stream: true }); - aggregate += chunk; - const snapshot = aggregate; - setAnalysisStates(prev => ({ - ...prev, - [analysisType]: { - ...prev[analysisType], - content: snapshot, - loading: true, - error: null, - } - })); - } - } - const endTime = new Date().toISOString(); - const elapsedMs = Date.now() - startedMsLocal; - setAnalysisStates(prev => ({ - ...prev, - [analysisType]: { - ...prev[analysisType], - content: aggregate, - loading: false, - error: null, - elapsed_ms: elapsedMs, - } - })); - setAnalysisRecords(prev => prev.map(record => - record.type === analysisType - ? { - ...record, - status: 'done', - end_ts: endTime, - duration_ms: elapsedMs, - } - : record - )); - } catch (err) { - const errorMessage = err instanceof Error ? err.message : '加载失败'; - const endTime = new Date().toISOString(); - setAnalysisStates(prev => ({ - ...prev, - [analysisType]: { - ...prev[analysisType], - content: '', - loading: false, - error: errorMessage - } - })); - setAnalysisRecords(prev => prev.map(record => - record.type === analysisType - ? { - ...record, - status: 'error', - end_ts: endTime, - error: errorMessage - } - : record - )); - } finally { - setCurrentAnalysisTask(null); - analysisFetchedRefs.current[analysisType] = true; - } - }; - - useEffect(() => { - if (isLoading || error || !financials || !activeAnalysisConfig?.analysis_modules || analysisTypes.length === 0) { - return; - } - if (isAnalysisRunningRef.current) { - return; - } - const runAnalysesSequentially = async () => { - if (isAnalysisRunningRef.current) { - return; - } - isAnalysisRunningRef.current = true; - try { - if (!stopRequestedRef.current && !startTime) { - setStartTime(Date.now()); - } - for (let i = 0; i < analysisTypes.length; i++) { - const analysisType = analysisTypes[i]; - if (stopRequestedRef.current) { - break; - } - if (analysisFetchedRefs.current[analysisType]) { - continue; - } - if (!analysisFetchedRefs.current || !activeAnalysisConfig?.analysis_modules) { - console.error("分析配置或refs未初始化，无法进行分析。"); - continue; - } - currentAnalysisTypeRef.current = analysisType; - const analysisName = - activeAnalysisConfig.analysis_modules[analysisType]?.name || analysisType; - const startTimeISO = new Date().toISOString(); - setCurrentAnalysisTask(analysisType); - setAnalysisRecords(prev => { - const next = [...prev]; - const idx = next.findIndex(r => r.type === analysisType); - const updated: AnalysisRecord = { - type: analysisType, - name: analysisName, - status: 'running' as const, - start_ts: startTimeISO - }; - if (idx >= 0) { - next[idx] = { ...next[idx], ...updated }; - } else { - next.push(updated); - } - return next; - }); - setAnalysisStates(prev => ({ - ...prev, - [analysisType]: { content: '', loading: true, error: null } - })); - try { - abortControllerRef.current?.abort(); - abortControllerRef.current = new AbortController(); - const startedMsLocal = Date.now(); - const response = await fetch( - `/api/financials/${normalizedMarket}/${unifiedSymbol}/analysis/${analysisType}/stream?company_name=${encodeURIComponent(financials?.name || unifiedSymbol)}`, - { signal: abortControllerRef.current.signal } - ); - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - const reader = response.body?.getReader(); - const decoder = new TextDecoder(); - let aggregate = ''; - if (reader) { - // 持续读取并追加到内容 - while (true) { - const { value, done } = await reader.read(); - if (done) break; - const chunk = decoder.decode(value, { stream: true }); - aggregate += chunk; - const snapshot = aggregate; - setAnalysisStates(prev => ({ - ...prev, - [analysisType]: { - ...prev[analysisType], - content: snapshot, - loading: true, - error: null, - } - })); - } - } - const endTime = new Date().toISOString(); - const elapsedMs = Date.now() - startedMsLocal; - setAnalysisStates(prev => ({ - ...prev, - [analysisType]: { - ...prev[analysisType], - content: aggregate, - loading: false, - error: null, - elapsed_ms: elapsedMs, - } - })); - setAnalysisRecords(prev => prev.map(record => - record.type === analysisType - ? { - ...record, - status: 'done', - end_ts: endTime, - duration_ms: elapsedMs, - } - : record - )); - } catch (err) { - if (err && typeof err === 'object' && (err as any).name === 'AbortError') { - setAnalysisStates(prev => ({ - ...prev, - [analysisType]: { content: '', loading: false, error: null } - })); - setAnalysisRecords(prev => prev.map(record => - record.type === analysisType - ? { ...record, status: 'pending', start_ts: undefined } - : record - )); - analysisFetchedRefs.current[analysisType] = false; - break; - } - const errorMessage = err instanceof Error ? err.message : '加载失败'; - const endTime = new Date().toISOString(); - setAnalysisStates(prev => ({ - ...prev, - [analysisType]: { - content: '', - loading: false, - error: errorMessage - } - })); - setAnalysisRecords(prev => prev.map(record => - record.type === analysisType - ? { - ...record, - status: 'error', - end_ts: endTime, - error: errorMessage - } - : record - )); - } finally { - setCurrentAnalysisTask(null); - currentAnalysisTypeRef.current = null; - analysisFetchedRefs.current[analysisType] = true; - } - } - } finally { - isAnalysisRunningRef.current = false; - } - }; - runAnalysesSequentially(); - }, [isLoading, error, financials, activeAnalysisConfig, analysisTypes, normalizedMarket, unifiedSymbol, startTime, manualRunKey]); - - const stopAll = () => { - stopRequestedRef.current = true; - abortControllerRef.current?.abort(); - abortControllerRef.current = null; - isAnalysisRunningRef.current = false; - if (currentAnalysisTypeRef.current) { - analysisFetchedRefs.current[currentAnalysisTypeRef.current] = false; - } - setCurrentAnalysisTask(null); - setStartTime(null); - }; - - const continuePending = () => { - if (isAnalysisRunningRef.current) return; - stopRequestedRef.current = false; - setStartTime((prev) => (prev == null ? Date.now() - elapsedSeconds * 1000 : prev)); - setManualRunKey((k) => k + 1); - }; - - const triggerAnalysis = async () => { - const reqId = await triggerAnalysisRequest(unifiedSymbol, normalizedMarket || '', selectedTemplateId); - if (reqId) setRequestId(reqId); - }; - - return { - activeAnalysisConfig, // Exported - analysisTypes, - analysisStates, - analysisRecords, - currentAnalysisTask, - triggerAnalysis, - triggering, - requestId, - setRequestId, - taskProgress, - startTime, - elapsedSeconds, - completionProgress, - totalElapsedMs, - stopAll, - continuePending, - retryAnalysis, - hasRunningTask, - isAnalysisRunning: isAnalysisRunningRef.current, - selectedTemplateId, // Exported - setSelectedTemplateId, // Exported - }; -} diff --git a/frontend/src/app/report/[symbol]/page.tsx b/frontend/src/app/report/[symbol]/page.tsx index deed765..59c282a 100644 --- a/frontend/src/app/report/[symbol]/page.tsx +++ b/frontend/src/app/report/[symbol]/page.tsx @@ -1,143 +1,30 @@ 'use client'; -import { Tabs, TabsList, TabsTrigger, TabsContent } from '@/components/ui/tabs'; -import { useReportData } from './hooks/useReportData'; -import { useAnalysisRunner } from './hooks/useAnalysisRunner'; -import { ReportHeader } from './components/ReportHeader'; -import { TaskStatus } from './components/TaskStatus'; -import { StockChart } from './components/StockChart'; -import { FinancialTable } from './components/FinancialTable'; -import { AnalysisContent } from './components/AnalysisContent'; -import { ExecutionDetails } from './components/ExecutionDetails'; +import React from 'react'; +import { useParams, useSearchParams } from 'next/navigation'; +import { WorkflowReportLayout } from './components/WorkflowReportLayout'; export default function ReportPage() { - const { - unifiedSymbol, - displayMarket, - normalizedMarket, - marketParam, - financials, - isLoading, - error, - snapshot, - snapshotLoading, - realtime, - realtimeLoading, - realtimeError, - financialConfig, - templateSets, - } = useReportData(); + // Next.js 15 params handling + // Note: In client components, hooks like useParams() handle the async nature internally or return current values + const params = useParams(); + const searchParams = useSearchParams(); + + const symbol = typeof params.symbol === 'string' ? decodeURIComponent(params.symbol) : ''; + const initialTemplateId = searchParams.get('template_id') || undefined; + const initialMarket = searchParams.get('market') || undefined; + const initialRequestId = searchParams.get('request_id') || undefined; - const { - activeAnalysisConfig, - analysisTypes, - analysisStates, - analysisRecords, - currentAnalysisTask, - triggerAnalysis, - triggering, - requestId, - taskProgress, - startTime, - elapsedSeconds, - completionProgress, - totalElapsedMs, - stopAll, - continuePending, - retryAnalysis, - hasRunningTask, - isAnalysisRunning, - selectedTemplateId, - setSelectedTemplateId, - } = useAnalysisRunner(financials, financialConfig, normalizedMarket, unifiedSymbol, isLoading, error, templateSets); + if (!symbol) { + return
Invalid Symbol
; + } return ( -
-
- - -
- - - - 股价图表 - 财务数据 - {analysisTypes.map(type => ( - - {type === 'company_profile' ? '公司简介' : (activeAnalysisConfig?.analysis_modules?.[type]?.name || type)} - - ))} - 执行详情 - - - - - - - - - - - {analysisTypes.map(analysisType => ( - - - - ))} - - - - - -
+ ); } diff --git a/frontend/src/components/TradingViewWidget.tsx b/frontend/src/components/TradingViewWidget.tsx index aa09511..faa5cb4 100644 --- a/frontend/src/components/TradingViewWidget.tsx +++ b/frontend/src/components/TradingViewWidget.tsx @@ -115,7 +115,11 @@ export function TradingViewWidget({ // 延迟到下一帧，确保容器已插入并可获取 iframe.contentWindow requestAnimationFrame(() => { try { - if (container.isConnected) { + // 再次检查容器是否仍然连接在DOM上，避免组件卸载后执行 + if (container && container.isConnected) { + // TradingView 的 embed 脚本会在内部创建 iframe + // 如果容器正在被卸载，或者 iframe 尚未完全准备好，可能会触发该错误 + // 我们只是 append script，实际的 iframe 是由 TradingView 脚本注入的 container.appendChild(script); } } catch (e) { @@ -126,11 +130,9 @@ export function TradingViewWidget({ } return () => { - const c = containerRef.current; - if (c) { - try { - c.innerHTML = ''; - } catch {} + // 清理函数 + if (containerRef.current) { + containerRef.current.innerHTML = ''; } }; }, [symbol, market]); diff --git a/frontend/src/components/ui/status-bar.tsx b/frontend/src/components/ui/status-bar.tsx index c2a6148..d443b35 100644 --- a/frontend/src/components/ui/status-bar.tsx +++ b/frontend/src/components/ui/status-bar.tsx @@ -77,7 +77,7 @@ export interface StatusBarState { retryable?: boolean; } -// 预定义的执行步骤已移至 ExecutionStepManager +// 预定义的执行步骤已移至 ExecutionStepManager (Deleted) // ============================================================================ // 主组件 diff --git a/frontend/src/components/workflow/TaskOutputViewer.tsx b/frontend/src/components/workflow/TaskOutputViewer.tsx new file mode 100644 index 0000000..b4bbb14 --- /dev/null +++ b/frontend/src/components/workflow/TaskOutputViewer.tsx @@ -0,0 +1,69 @@ +import React from 'react'; +import ReactMarkdown from 'react-markdown'; +import remarkGfm from 'remark-gfm'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; +import { Badge } from '@/components/ui/badge'; +import { TaskStatus, TaskType } from '@/types/workflow'; +import { Loader2 } from 'lucide-react'; + +interface TaskOutputViewerProps { + taskId: string; + taskName: string; + taskType: TaskType; + status: TaskStatus; + content: string; +} + +export function TaskOutputViewer({ + taskId, + taskName, + taskType, + status, + content +}: TaskOutputViewerProps) { + + return ( + + +
+ {taskName} +
+ {taskId} + • + {taskType} +
+
+
+ + {status === 'Running' && } + {status} + +
+ + + + {content ? ( +
+ + {content} + +
+ ) : ( +
+ {status === 'Pending' ? 'Waiting to start...' : + status === 'Running' ? 'Processing...' : + 'No output available'} +
+ )} + + + + ); +} + diff --git a/frontend/src/components/workflow/WorkflowVisualizer.tsx b/frontend/src/components/workflow/WorkflowVisualizer.tsx new file mode 100644 index 0000000..7c09184 --- /dev/null +++ b/frontend/src/components/workflow/WorkflowVisualizer.tsx @@ -0,0 +1,159 @@ +import React, { useState, useEffect } from 'react'; +import { WorkflowDag, TaskNode, TaskStatus, TaskType } from '@/types/workflow'; +import { TaskOutputViewer } from './TaskOutputViewer'; +import { Card } from '@/components/ui/card'; +import { Badge } from '@/components/ui/badge'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { cn } from '@/lib/utils'; +import { + CheckCircle2, + Circle, + Clock, + AlertCircle, + Loader2, + SkipForward, + Database, + FileText, + BrainCircuit +} from 'lucide-react'; + +interface WorkflowVisualizerProps { + dag: WorkflowDag; + taskStates: Record; + taskOutputs: Record; + className?: string; +} + +const TYPE_ORDER: Record = { + 'DataFetch': 1, + 'DataProcessing': 2, + 'Analysis': 3 +}; + +const TYPE_ICONS: Record = { + 'DataFetch': , + 'DataProcessing': , + 'Analysis': +}; + +export function WorkflowVisualizer({ + dag, + taskStates, + taskOutputs, + className +}: WorkflowVisualizerProps) { + const [selectedTaskId, setSelectedTaskId] = useState(null); + + // Sort nodes by type then name + const sortedNodes = [...dag.nodes].sort((a, b) => { + const typeScoreA = TYPE_ORDER[a.type] || 99; + const typeScoreB = TYPE_ORDER[b.type] || 99; + if (typeScoreA !== typeScoreB) return typeScoreA - typeScoreB; + return a.name.localeCompare(b.name); + }); + + // Auto-select first node or running node if none selected + useEffect(() => { + if (!selectedTaskId && sortedNodes.length > 0) { + // Try to find a running node, or the first one + const runningNode = sortedNodes.find(n => taskStates[n.id] === 'Running'); + setSelectedTaskId(runningNode ? runningNode.id : sortedNodes[0].id); + } + }, [dag, taskStates, selectedTaskId, sortedNodes]); + + const selectedNode = dag.nodes.find(n => n.id === selectedTaskId); + + return ( +
+ {/* Left Sidebar: Task List */} +
+
+

Workflow Tasks

+

+ {dag.nodes.length} steps in pipeline +

+
+ +
+ {sortedNodes.map(node => ( + setSelectedTaskId(node.id)} + /> + ))} +
+ +
+ + {/* Right Content: Output Viewer */} +
+ {selectedNode ? ( + + ) : ( +
+ Select a task to view details +
+ )} +
+
+ ); +} + +function TaskListItem({ + node, + status, + isSelected, + onClick +}: { + node: TaskNode; + status: TaskStatus; + isSelected: boolean; + onClick: () => void; +}) { + return ( + + ); +} + +function StatusIcon({ status }: { status: TaskStatus }) { + switch (status) { + case 'Completed': + return ; + case 'Failed': + return ; + case 'Running': + return ; + case 'Scheduled': + return ; + case 'Skipped': + return ; + default: // Pending + return ; + } +} + diff --git a/frontend/src/hooks/useApi.ts b/frontend/src/hooks/useApi.ts index 553c345..62a73ed 100644 --- a/frontend/src/hooks/useApi.ts +++ b/frontend/src/hooks/useApi.ts @@ -55,28 +55,26 @@ export function useDataRequest() { } -// 用于轮询任务进度 + +// [DEPRECATED] Used for polling, logic removed. +// Backend now pushes progress via SSE. export function useTaskProgress(requestId: string | null, options?: SWRConfiguration) { - const { data, error, isLoading } = useSWR( - requestId ? `/api/tasks/${requestId}` : null, + const { data, error, isLoading } = useSWR( + null, // Disable polling fetcher, - { - refreshInterval: 2000, // 每2秒轮询一次 - ...options, - errorRetryCount: 2, - } + options ); - const isFinished = !isLoading && (data?.status?.includes('completed') || data?.status?.includes('failed') || !data); - return { - progress: data, - isLoading, - isError: error, - isFinished, + tasks: [], + progress: null, + isLoading: false, + isError: null, + isFinished: false, }; } + // --- Analysis Results Hooks (NEW) --- export function useAnalysisResults(symbol?: string) { @@ -257,11 +255,18 @@ export async function updateConfig(payload: Partial) { return updated; } -export async function testConfig(type: string, data: unknown) { - const res = await fetch('/api/config/test', { +export async function testConfig(type: string, data: any) { + // Flat the data object to match backend expectation (#[serde(flatten)]) + const payload = { + type, + ...(typeof data === 'object' ? data : {}) + }; + + // Use /api/configs/test to match backend /v1/configs/test via Rewrite + const res = await fetch('/api/configs/test', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ type, data }), + body: JSON.stringify(payload), }); const text = await res.text(); if (!res.ok) { diff --git a/frontend/src/hooks/useWorkflow.ts b/frontend/src/hooks/useWorkflow.ts new file mode 100644 index 0000000..996a112 --- /dev/null +++ b/frontend/src/hooks/useWorkflow.ts @@ -0,0 +1,206 @@ +import { useState, useRef, useCallback, useEffect } from 'react'; +import { + WorkflowEvent, + WorkflowDag, + TaskStatus, + StartWorkflowRequest, + StartWorkflowResponse +} from '@/types/workflow'; + +export type WorkflowConnectionStatus = 'idle' | 'connecting' | 'connected' | 'disconnected' | 'error'; + +interface UseWorkflowReturn { + // State + status: WorkflowConnectionStatus; + requestId: string | null; + dag: WorkflowDag | null; + taskStates: Record; + taskOutputs: Record; // Accumulates streaming content + error: string | null; + finalResult: any | null; + + // Actions + // Returns StartWorkflowResponse to allow caller to handle redirects (e.g. symbol normalization) + startWorkflow: (params: StartWorkflowRequest) => Promise; + connectToWorkflow: (requestId: string) => void; + disconnect: () => void; +} + +export function useWorkflow(): UseWorkflowReturn { + const [status, setStatus] = useState('idle'); + const [requestId, setRequestId] = useState(null); + const [dag, setDag] = useState(null); + const [taskStates, setTaskStates] = useState>({}); + const [taskOutputs, setTaskOutputs] = useState>({}); + const [error, setError] = useState(null); + const [finalResult, setFinalResult] = useState(null); + + // Ref for EventSource to handle cleanup + const eventSourceRef = useRef(null); + + // Refs for state that updates frequently to avoid closure staleness in event handlers if needed + // (Though in this React pattern, simple state updates usually suffice unless high freq) + + const disconnect = useCallback(() => { + if (eventSourceRef.current) { + eventSourceRef.current.close(); + eventSourceRef.current = null; + } + setStatus('disconnected'); + }, []); + + const handleEvent = useCallback((eventData: WorkflowEvent) => { + switch (eventData.type) { + case 'WorkflowStarted': + setDag(eventData.payload.task_graph); + // Initialize states based on graph + const initialStates: Record = {}; + eventData.payload.task_graph.nodes.forEach(node => { + initialStates[node.id] = node.initial_status; + }); + setTaskStates(initialStates); + break; + + case 'TaskStateChanged': + setTaskStates(prev => ({ + ...prev, + [eventData.payload.task_id]: eventData.payload.status + })); + break; + + case 'TaskStreamUpdate': + setTaskOutputs(prev => ({ + ...prev, + [eventData.payload.task_id]: (prev[eventData.payload.task_id] || '') + eventData.payload.content_delta + })); + break; + + case 'WorkflowStateSnapshot': + // Restore full state + setDag(eventData.payload.task_graph); + setTaskStates(eventData.payload.tasks_status); + // Restore outputs if present + const outputs: Record = {}; + Object.entries(eventData.payload.tasks_output).forEach(([k, v]) => { + if (v) outputs[k] = v; + }); + setTaskOutputs(prev => ({ ...prev, ...outputs })); + break; + + case 'WorkflowCompleted': + setFinalResult(eventData.payload.result_summary); + disconnect(); // Close connection on completion + break; + + case 'WorkflowFailed': + setError(eventData.payload.reason); + // We might want to keep connected or disconnect depending on if retry is possible + // For now, treat fatal error as disconnect reason + if (eventData.payload.is_fatal) { + disconnect(); + setStatus('error'); + } + break; + } + }, [disconnect]); + + const connectToWorkflow = useCallback((id: string) => { + if (eventSourceRef.current) { + eventSourceRef.current.close(); + } + + setRequestId(id); + setStatus('connecting'); + setError(null); + + try { + const es = new EventSource(`/api/workflow/events/${id}`); + eventSourceRef.current = es; + + es.onopen = () => { + setStatus('connected'); + }; + + es.onmessage = (event) => { + try { + const data = JSON.parse(event.data) as WorkflowEvent; + handleEvent(data); + } catch (e) { + console.error('Failed to parse workflow event:', e); + } + }; + + es.onerror = (e) => { + console.error('Workflow SSE error:', e); + // EventSource automatically retries, but we might want to handle it explicitly + // For now, let's assume if readyState is CLOSED, it's a fatal error + if (es.readyState === EventSource.CLOSED) { + setStatus('error'); + setError('Connection lost'); + es.close(); + } + }; + + } catch (e) { + console.error('Failed to create EventSource:', e); + setStatus('error'); + setError(e instanceof Error ? e.message : 'Connection initialization failed'); + } + }, [handleEvent]); + + const startWorkflow = useCallback(async (params: StartWorkflowRequest) => { + setStatus('connecting'); + setError(null); + setDag(null); + setTaskStates({}); + setTaskOutputs({}); + setFinalResult(null); + + try { + const res = await fetch('/api/workflow/start', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(params), + }); + + if (!res.ok) { + const errorBody = await res.json().catch(() => ({})); + throw new Error(errorBody.error || `HTTP ${res.status}`); + } + + const data: StartWorkflowResponse = await res.json(); + + // Start listening + connectToWorkflow(data.request_id); + + return data; // Return response so UI can handle symbol normalization redirection + + } catch (e) { + setStatus('error'); + setError(e instanceof Error ? e.message : 'Failed to start workflow'); + return undefined; + } + }, [connectToWorkflow]); + + // Cleanup on unmount + useEffect(() => { + return () => { + if (eventSourceRef.current) { + eventSourceRef.current.close(); + } + }; + }, []); + + return { + status, + requestId, + dag, + taskStates, + taskOutputs, + error, + finalResult, + startWorkflow, + connectToWorkflow, + disconnect + }; +} diff --git a/frontend/src/lib/execution-step-manager.ts b/frontend/src/lib/execution-step-manager.ts deleted file mode 100644 index 5bb8a4a..0000000 --- a/frontend/src/lib/execution-step-manager.ts +++ /dev/null @@ -1,434 +0,0 @@ -/** - * ExecutionStepManager - 可扩展的步骤执行框架 - * - * 提供步骤的动态添加、管理和执行，支持错误处理和状态回调 - * - * 主要功能： - * - 步骤的动态添加和管理 - * - 支持重试机制和错误处理 - * - 提供执行状态回调 - * - 支持并行和串行执行 - * - 可扩展的步骤定义 - * - * @author Financial Analysis Platform Team - * @version 1.0.0 - */ - -// ============================================================================ -// 类型定义 -// ============================================================================ - -/** - * 执行步骤接口 - */ -export interface ExecutionStep { - /** 步骤唯一标识符 */ - id: string; - /** 步骤显示名称 */ - name: string; - /** 步骤详细描述 */ - description: string; - /** 执行函数（可选） */ - execute?: () => Promise; -} - -/** - * 执行选项接口 - */ -export interface ExecutionOptions { - /** 步骤开始回调 */ - onStepStart?: (step: ExecutionStep, index: number, total: number) => void; - /** 步骤完成回调 */ - onStepComplete?: (step: ExecutionStep, index: number, total: number) => void; - /** 步骤错误回调 */ - onStepError?: (step: ExecutionStep, index: number, total: number, error: Error) => void; - /** 全部完成回调 */ - onComplete?: () => void; - /** 执行错误回调 */ - onError?: (error: Error) => void; - /** 最大重试次数 */ - maxRetries?: number; - /** 重试延迟（毫秒） */ - retryDelay?: number; - /** 出错时是否继续执行 */ - continueOnError?: boolean; -} - -/** - * 执行上下文接口 - */ -export interface ExecutionContext { - /** 当前执行步骤 */ - currentStep: ExecutionStep | null; - /** 当前步骤索引 */ - stepIndex: number; - /** 总步骤数 */ - totalSteps: number; - /** 是否正在运行 */ - isRunning: boolean; - /** 是否有错误 */ - hasError: boolean; - /** 错误信息 */ - errorMessage?: string; - /** 重试次数 */ - retryCount: number; - /** 最大重试次数 */ - maxRetries: number; - /** 是否可重试 */ - canRetry: boolean; -} - -export class ExecutionStepManager { - private steps: ExecutionStep[] = []; - private context: ExecutionContext = { - currentStep: null, - stepIndex: 0, - totalSteps: 0, - isRunning: false, - hasError: false, - errorMessage: undefined, - retryCount: 0, - maxRetries: 0, - canRetry: false - }; - private options: ExecutionOptions = {}; - - constructor(steps: ExecutionStep[] = [], options: ExecutionOptions = {}) { - this.steps = [...steps]; - this.options = { - maxRetries: 2, - retryDelay: 1000, - continueOnError: false, - ...options - }; - this.updateContext(); - } - - /** - * 添加执行步骤 - */ - addStep(step: ExecutionStep): void { - this.steps.push(step); - this.updateContext(); - } - - /** - * 批量添加执行步骤 - */ - addSteps(steps: ExecutionStep[]): void { - this.steps.push(...steps); - this.updateContext(); - } - - /** - * 插入步骤到指定位置 - */ - insertStep(index: number, step: ExecutionStep): void { - this.steps.splice(index, 0, step); - this.updateContext(); - } - - /** - * 移除步骤 - */ - removeStep(stepId: string): boolean { - const index = this.steps.findIndex(step => step.id === stepId); - if (index !== -1) { - this.steps.splice(index, 1); - this.updateContext(); - return true; - } - return false; - } - - /** - * 清空所有步骤 - */ - clearSteps(): void { - this.steps = []; - this.updateContext(); - } - - /** - * 获取所有步骤 - */ - getSteps(): ExecutionStep[] { - return [...this.steps]; - } - - /** - * 获取当前执行上下文 - */ - getContext(): ExecutionContext { - return { ...this.context }; - } - - /** - * 更新执行选项 - */ - setOptions(options: ExecutionOptions): void { - this.options = { ...this.options, ...options }; - } - - /** - * 执行所有步骤 - */ - async execute(): Promise { - if (this.context.isRunning) { - throw new Error('Execution is already in progress'); - } - - if (this.steps.length === 0) { - throw new Error('No steps to execute'); - } - - this.context.isRunning = true; - this.context.hasError = false; - this.context.errorMessage = undefined; - this.context.stepIndex = 0; - this.context.retryCount = 0; - this.context.maxRetries = this.options.maxRetries || 2; - - try { - for (let i = 0; i < this.steps.length; i++) { - const step = this.steps[i]; - this.context.currentStep = step; - this.context.stepIndex = i; - - // 通知步骤开始 - this.options.onStepStart?.(step, i, this.steps.length); - - let stepSuccess = false; - let lastError: Error | null = null; - - // 重试逻辑 - for (let retryAttempt = 0; retryAttempt <= this.context.maxRetries; retryAttempt++) { - try { - this.context.retryCount = retryAttempt; - - // 如果是重试，等待一段时间 - if (retryAttempt > 0 && this.options.retryDelay) { - await new Promise(resolve => setTimeout(resolve, this.options.retryDelay)); - } - - // 执行步骤（如果有执行函数） - if (step.execute) { - await step.execute(); - } - - stepSuccess = true; - break; // 成功执行，跳出重试循环 - } catch (stepError) { - lastError = stepError instanceof Error ? stepError : new Error(String(stepError)); - - // 如果还有重试机会，继续重试 - if (retryAttempt < this.context.maxRetries) { - console.warn(`Step "${step.name}" failed, retrying (${retryAttempt + 1}/${this.context.maxRetries + 1}):`, lastError.message); - continue; - } - } - } - - if (stepSuccess) { - // 通知步骤完成 - this.options.onStepComplete?.(step, i, this.steps.length); - } else { - // 所有重试都失败了 - const error = lastError || new Error('Step execution failed'); - - // 更新错误状态 - this.context.hasError = true; - this.context.errorMessage = error.message; - this.context.canRetry = true; - - // 通知步骤错误 - this.options.onStepError?.(step, i, this.steps.length, error); - - // 如果不继续执行，抛出错误 - if (!this.options.continueOnError) { - throw error; - } - } - } - - // 所有步骤执行完成 - this.options.onComplete?.(); - } catch (error) { - const execError = error instanceof Error ? error : new Error(String(error)); - - // 通知执行错误 - this.options.onError?.(execError); - - // 重新抛出错误 - throw execError; - } finally { - this.context.isRunning = false; - } - } - - /** - * 执行单个步骤 - */ - async executeStep(stepId: string): Promise { - const stepIndex = this.steps.findIndex(step => step.id === stepId); - if (stepIndex === -1) { - throw new Error(`Step with id '${stepId}' not found`); - } - - const step = this.steps[stepIndex]; - this.context.currentStep = step; - this.context.stepIndex = stepIndex; - this.context.isRunning = true; - this.context.hasError = false; - this.context.errorMessage = undefined; - - try { - // 通知步骤开始 - this.options.onStepStart?.(step, stepIndex, this.steps.length); - - // 执行步骤 - if (step.execute) { - await step.execute(); - } - - // 通知步骤完成 - this.options.onStepComplete?.(step, stepIndex, this.steps.length); - } catch (stepError) { - const error = stepError instanceof Error ? stepError : new Error(String(stepError)); - - // 更新错误状态 - this.context.hasError = true; - this.context.errorMessage = error.message; - - // 通知步骤错误 - this.options.onStepError?.(step, stepIndex, this.steps.length, error); - - throw error; - } finally { - this.context.isRunning = false; - } - } - - /** - * 停止执行（如果正在运行） - */ - stop(): void { - this.context.isRunning = false; - } - - /** - * 重试当前失败的步骤 - */ - async retry(): Promise { - if (!this.context.hasError || !this.context.canRetry) { - throw new Error('No failed step to retry'); - } - - if (this.context.isRunning) { - throw new Error('Execution is already in progress'); - } - - // 重置错误状态 - this.context.hasError = false; - this.context.errorMessage = undefined; - this.context.canRetry = false; - - // 重新执行从当前步骤开始 - try { - await this.execute(); - } catch (error) { - // 错误已经在execute方法中处理 - throw error; - } - } - - /** - * 重置执行状态 - */ - reset(): void { - this.context = { - currentStep: null, - stepIndex: 0, - totalSteps: this.steps.length, - isRunning: false, - hasError: false, - errorMessage: undefined, - retryCount: 0, - maxRetries: this.options.maxRetries || 2, - canRetry: false - }; - } - - /** - * 检查是否正在执行 - */ - isRunning(): boolean { - return this.context.isRunning; - } - - /** - * 检查是否有错误 - */ - hasError(): boolean { - return this.context.hasError; - } - - /** - * 获取错误信息 - */ - getErrorMessage(): string | undefined { - return this.context.errorMessage; - } - - /** - * 检查是否可以重试 - */ - canRetry(): boolean { - return this.context.canRetry; - } - - /** - * 更新执行上下文 - */ - private updateContext(): void { - this.context.totalSteps = this.steps.length; - this.context.maxRetries = this.options.maxRetries || 2; - if (!this.context.isRunning) { - this.context.stepIndex = 0; - this.context.currentStep = null; - this.context.retryCount = 0; - } - } - - /** - * 创建一个带有预定义步骤的管理器实例 - */ - static createWithSteps(steps: ExecutionStep[], options: ExecutionOptions = {}): ExecutionStepManager { - return new ExecutionStepManager(steps, options); - } - - /** - * 创建一个空的管理器实例 - */ - static create(options: ExecutionOptions = {}): ExecutionStepManager { - return new ExecutionStepManager([], options); - } -} - -/** - * 预定义的执行步骤 - */ -export const DEFAULT_EXECUTION_STEPS: ExecutionStep[] = [ - { - id: 'fetch_financial_data', - name: '正在读取财务数据', - description: '从Tushare API获取公司财务指标数据' - } -]; - -/** - * 创建默认的执行步骤管理器 - */ -export function createDefaultStepManager(options: ExecutionOptions = {}): ExecutionStepManager { - return ExecutionStepManager.createWithSteps(DEFAULT_EXECUTION_STEPS, options); -} \ No newline at end of file diff --git a/frontend/src/lib/financial-data-transformer.ts b/frontend/src/lib/financial-data-transformer.ts new file mode 100644 index 0000000..587c92d --- /dev/null +++ b/frontend/src/lib/financial-data-transformer.ts @@ -0,0 +1,79 @@ + +export interface TimeSeriesFinancialDto { + symbol: string; + metric_name: string; + period_date: string; + value: number; + source?: string; +} + +export interface FinancialTableRow { + metric: string; + [year: string]: string | number | undefined; +} + +export interface FinancialTableData { + headers: string[]; // Sorted years + rows: FinancialTableRow[]; +} + +/** + * Transforms a flat list of TimeSeriesFinancialDto into a pivoted table structure. + * + * Input: + * [ + * { metric_name: "Revenue", period_date: "2023-12-31", value: 100, source: "tushare" }, + * { metric_name: "Revenue", period_date: "2022-12-31", value: 90, source: "tushare" }, + * ] + * + * Output: + * { + * headers: ["2023", "2022"], + * rows: [ + * { metric: "Revenue", "2023": 100, "2022": 90 } + * ] + * } + */ +export function transformFinancialData(data: TimeSeriesFinancialDto[]): FinancialTableData { + if (!data || data.length === 0) { + return { headers: [], rows: [] }; + } + + // 1. Collect all unique years (from period_date) + const yearsSet = new Set(); + // 2. Group by metric name + const metricMap = new Map>(); + + data.forEach(item => { + if (!item.period_date) return; + // Extract year from "YYYY-MM-DD" + const year = item.period_date.substring(0, 4); + yearsSet.add(year); + + if (!metricMap.has(item.metric_name)) { + metricMap.set(item.metric_name, { metric: item.metric_name }); + } + + const row = metricMap.get(item.metric_name)!; + + // Handle potential conflicts: + // If multiple sources provide data, we currently just overwrite or keep the last one. + // A better approach might be to append source info or average, but for a table view, single value is cleaner. + // Maybe prioritize sources? Tushare > YFinance > AlphaVantage? + // For now, we just use the value. + // We could format it with source: "100 (Tushare)" but that breaks numeric sorting/formatting. + row[year] = item.value; + }); + + // Sort years descending (newest first) + const headers = Array.from(yearsSet).sort((a, b) => Number(b) - Number(a)); + + // Create rows + const rows = Array.from(metricMap.values()).map(row => { + // Ensure all header keys exist (fill with undefined/null if needed) + return row as FinancialTableRow; + }); + + return { headers, rows }; +} + diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index 8fd25ed..e1f1fd5 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -342,6 +342,23 @@ export interface ConfigSaveState { timestamp?: number; } +/** + * Task Status Enum (Matches Backend TaskStatus) + */ +export type TaskStatus = 'queued' | 'in_progress' | 'completed' | 'failed'; + +/** + * Task Progress DTO (Matches Backend TaskProgress) + */ +export interface TaskProgress { + request_id: string; + task_name: string; + status: TaskStatus; + progress_percent: number; + details: string; + started_at: string; // ISO8601 +} + // ============================================================================ // API 相关类型 // ============================================================================ diff --git a/frontend/src/types/report.ts b/frontend/src/types/report.ts new file mode 100644 index 0000000..c63f964 --- /dev/null +++ b/frontend/src/types/report.ts @@ -0,0 +1,32 @@ +import { + AnalysisTemplateSet, + AnalysisResultDto, + TaskProgress +} from '@/types'; + +export interface ReportState { + // 1. Context + symbol: string; + market: string; + templateId: string; + templateConfig: AnalysisTemplateSet | null; + + // 2. Phase Status + fetchStatus: 'idle' | 'fetching' | 'complete' | 'error'; + analysisStatus: 'idle' | 'running' | 'complete' | 'error'; + + // 3. Data + fundamentalData: unknown[]; + analysisResults: Record; // Key: ModuleID + tasks: TaskProgress[]; + + // 4. Progress + requestId: string | null; + executionMeta: { + startTime: number; + elapsed: number; + tokens: number; + }; + error: string | null; +} + diff --git a/frontend/src/types/workflow.ts b/frontend/src/types/workflow.ts new file mode 100644 index 0000000..63c6ff3 --- /dev/null +++ b/frontend/src/types/workflow.ts @@ -0,0 +1,119 @@ +/** + * Workflow Types Definition + * Corresponds to backend Rust types in `common-contracts/src/messages.rs` + * + * 遵循原则： + * 1. 强类型约束：枚举和接口严格对应 + * 2. 单一来源：通过后端定义推导前端类型 + */ + +// ============================================================================ +// Enums +// ============================================================================ + +export type TaskType = 'DataFetch' | 'DataProcessing' | 'Analysis'; + +export type TaskStatus = + | 'Pending' // 等待依赖 + | 'Scheduled' // 依赖满足，已下发给 Worker + | 'Running' // Worker 正在执行 + | 'Completed' // 执行成功 + | 'Failed' // 执行失败 + | 'Skipped'; // 因上游失败或策略原因被跳过 + +// ============================================================================ +// Graph Structure (DAG) +// ============================================================================ + +export interface TaskNode { + id: string; + name: string; + type: TaskType; + initial_status: TaskStatus; +} + +export interface TaskDependency { + from: string; + to: string; +} + +export interface WorkflowDag { + nodes: TaskNode[]; + edges: TaskDependency[]; +} + +// ============================================================================ +// Events (Server-Sent Events Payloads) +// ============================================================================ + +/** + * Base interface for all workflow events + * Discriminated union based on 'type' field + */ +export type WorkflowEvent = + | { + type: 'WorkflowStarted'; + payload: { + timestamp: number; + task_graph: WorkflowDag; + }; + } + | { + type: 'TaskStateChanged'; + payload: { + task_id: string; + task_type: TaskType; + status: TaskStatus; + message: string | null; + timestamp: number; + }; + } + | { + type: 'TaskStreamUpdate'; + payload: { + task_id: string; + content_delta: string; + index: number; + }; + } + | { + type: 'WorkflowCompleted'; + payload: { + result_summary: any; // JSON Value + end_timestamp: number; + }; + } + | { + type: 'WorkflowFailed'; + payload: { + reason: string; + is_fatal: boolean; + end_timestamp: number; + }; + } + | { + type: 'WorkflowStateSnapshot'; + payload: { + timestamp: number; + task_graph: WorkflowDag; + tasks_status: Record; + tasks_output: Record; + }; + }; + +// ============================================================================ +// API Request/Response Types +// ============================================================================ + +export interface StartWorkflowRequest { + symbol: string; + market?: string; + template_id: string; +} + +export interface StartWorkflowResponse { + request_id: string; + symbol: string; + market: string; +} + diff --git a/keys.md b/keys.md new file mode 100644 index 0000000..bb5aef6 --- /dev/null +++ b/keys.md @@ -0,0 +1,5 @@ +alphaventage_key=PUOO7UPTNXN325NN +openrouter_url=https://openrouter.ai/api/v1 +openrouter_key=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 +tushare_key="f62b415de0a5a947fcb693b66cd299dd6242868bf04ad687800c7f3f" +finnhub_key="d3fjs5pr01qolkndil0gd3fjs5pr01qolkndil10" diff --git a/scripts/check_services.sh b/scripts/check_services.sh new file mode 100755 index 0000000..c52a648 --- /dev/null +++ b/scripts/check_services.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Define the services to check (order matters for dependencies) +SERVICES=( + "services/common-contracts" + "services/data-persistence-service" + "services/workflow-orchestrator-service" + "services/api-gateway" + "services/report-generator-service" + "services/alphavantage-provider-service" + "services/tushare-provider-service" + "services/finnhub-provider-service" + "services/yfinance-provider-service" +) + +echo "========================================================" +echo " RUST SERVICES COMPILATION CHECK SEQUENCE " +echo "========================================================" +echo "" + +FAIL_COUNT=0 +FAILED_SERVICES=() + +for service_path in "${SERVICES[@]}"; do + echo "--------------------------------------------------------" + echo ">>> CHECKING: $service_path" + echo "--------------------------------------------------------" + + if [ -d "$service_path" ]; then + pushd "$service_path" > /dev/null + + # Run cargo check with SQLX_OFFLINE=true + if SQLX_OFFLINE=true cargo check --tests --all-features; then + echo "✅ SUCCESS: $service_path compiled successfully." + else + echo "❌ FAILURE: $service_path failed to compile." + FAIL_COUNT=$((FAIL_COUNT+1)) + FAILED_SERVICES+=("$service_path") + fi + + popd > /dev/null + else + echo "⚠️ WARNING: Directory $service_path not found!" + fi + echo "" +done + +echo "========================================================" +echo " CHECK COMPLETE " +echo "========================================================" + +if [ $FAIL_COUNT -eq 0 ]; then + echo "🎉 All services passed cargo check!" + exit 0 +else + echo "💥 $FAIL_COUNT services failed to compile:" + for failed in "${FAILED_SERVICES[@]}"; do + echo " - $failed" + done + echo "" + echo "Please review errors above." + exit 1 +fi diff --git a/scripts/inspect_logs.sh b/scripts/inspect_logs.sh new file mode 100755 index 0000000..2195d3a --- /dev/null +++ b/scripts/inspect_logs.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Define the services we are interested in +SERVICES=( + "api-gateway" + "report-generator-service" + "data-persistence-service" + "alphavantage-provider-service" + "tushare-provider-service" + "finnhub-provider-service" + "yfinance-provider-service" + "workflow-orchestrator-service" + "fundamental_analysis-nats-1" +) + +# Get line count from first argument, default to 10 +LINES_INPUT=${1:-10} + +echo "========================================================" +echo " FUNDAMENTAL ANALYSIS SYSTEM STATUS REPORT " +echo "========================================================" +echo "Showing last $LINES_INPUT lines of logs per service" +echo "" + +for service in "${SERVICES[@]}"; do + echo "--------------------------------------------------------" + echo ">>> SERVICE: $service" + echo "--------------------------------------------------------" + + if docker ps -a --format '{{.Names}}' | grep -q "^${service}$"; then + STATUS=$(docker inspect --format='{{.State.Status}}' "$service") + echo "Status: $STATUS" + echo "Logs (Last $LINES_INPUT lines):" + echo "" + # Execute docker logs directly without extra piping that might buffer output weirdly in some shells, though unlikely. + # The issue might be how variables are expanded or env. + # Using simple variable expansion. + docker logs "$service" --tail $LINES_INPUT 2>&1 + else + echo "Status: NOT FOUND / NOT RUNNING" + fi + echo "" + echo "" +done + +echo "========================================================" +echo " END OF LOG REPORT " +echo "========================================================" diff --git a/scripts/run_component_tests.sh b/scripts/run_component_tests.sh new file mode 100755 index 0000000..e23cdb1 --- /dev/null +++ b/scripts/run_component_tests.sh @@ -0,0 +1,141 @@ +#!/bin/bash +set -e + +# Configuration +COMPOSE_FILE="docker-compose.test.yml" +export NATS_ADDR="nats://localhost:4223" +export DATA_PERSISTENCE_SERVICE_URL="http://localhost:3001/api/v1" +# For services that might need direct DB access (e.g. persistence tests) +export DATABASE_URL="postgresql://postgres:postgres@localhost:5433/fundamental_test" + +# Fake Service Host config for providers +export SERVICE_HOST="localhost" +export API_GATEWAY_URL="http://localhost:4000" # Mock + +# Keys (Injected for testing) +export ALPHAVANTAGE_API_KEY="PUOO7UPTNXN325NN" +export TUSHARE_API_KEY="f62b415de0a5a947fcb693b66cd299dd6242868bf04ad687800c7f3f" +export FINNHUB_API_KEY="d3fjs5pr01qolkndil0gd3fjs5pr01qolkndil10" +export OPENROUTER_API_KEY="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" +export OPENROUTER_API_URL="https://openrouter.ai/api/v1" +# Common config for services +export SERVER_PORT=0 # Use random/no port for tests to avoid config errors + +# Default URLs (From Frontend Defaults) +export ALPHAVANTAGE_MCP_URL="https://mcp.alphavantage.co/mcp" +export TUSHARE_API_URL="http://api.tushare.pro" +export FINNHUB_API_URL="https://finnhub.io/api/v1" +export YFINANCE_API_URL="https://query1.finance.yahoo.com" # Generic default + +# Check for MCP URL (now set by default above, but good to keep check if user wants override) +if [ -z "$ALPHAVANTAGE_MCP_URL" ]; then + echo -e "\033[1;33m[WARNING]\033[0m ALPHAVANTAGE_MCP_URL is not set. Integration tests using it will fail." + echo "Please set it via: export ALPHAVANTAGE_MCP_URL='...'" + # Set a dummy for now to prevent crash, but test will fail connection + export ALPHAVANTAGE_MCP_URL="http://localhost:9999/sse" +fi + +function log() { + echo -e "\033[1;34m[TEST-RUNNER]\033[0m $1" +} + +function start_env() { + log "Starting test infrastructure..." + docker-compose -f $COMPOSE_FILE up -d --build + + log "Waiting for services to be healthy..." + # Simple wait loop for persistence service + local max_retries=30 + local count=0 + while ! curl -s http://localhost:3001/health > /dev/null; do + sleep 2 + count=$((count+1)) + if [ $count -ge $max_retries ]; then + log "Error: Timeout waiting for persistence service." + exit 1 + fi + echo -n "." + done + echo "" + log "Infrastructure is ready!" +} + +function stop_env() { + log "Stopping test infrastructure..." + docker-compose -f $COMPOSE_FILE down -v + log "Environment destroyed." +} + +function run_tests_in_dir() { + local dir=$1 + log "Running tests in $dir..." + if ! (cd "$dir" && cargo test -- --nocapture); then + log "Tests failed in $dir" + return 1 + fi +} + +function run_tests() { + local package=$1 + local services_dir="services" + + if [ -n "$package" ]; then + if [ -d "$services_dir/$package" ]; then + if ! run_tests_in_dir "$services_dir/$package"; then + exit 1 + fi + else + log "Error: Package directory '$services_dir/$package' not found." + exit 1 + fi + else + log "Running ALL tests in services/ directory..." + for dir in "$services_dir"/*; do + if [ -d "$dir" ] && [ -f "$dir/Cargo.toml" ]; then + if ! run_tests_in_dir "$dir"; then + log "Aborting due to test failure in $dir." + exit 1 + fi + fi + done + fi +} + +function check_env_ready() { + if curl -s http://localhost:3001/health > /dev/null; then + return 0 + else + return 1 + fi +} + +# CLI Argument Parsing +case "$1" in + "prepare"|"start") + if check_env_ready; then + log "Environment is already running." + else + start_env + fi + ;; + "destroy"|"stop") + stop_env + ;; + "test") + # Verify environment is ready + if ! check_env_ready; then + log "Error: Test environment is NOT ready." + log "Please run '$0 prepare' first to start the infrastructure." + exit 1 + fi + + run_tests $2 + ;; + *) + echo "Usage: $0 {prepare|destroy|test [package_name]}" + echo " prepare (start): Start test infrastructure (Docker)" + echo " destroy (stop): Stop and cleanup test infrastructure" + echo " test: Run cargo test (requires environment to be ready). Optional: provide package name." + exit 1 + ;; +esac diff --git a/scripts/run_e2e.sh b/scripts/run_e2e.sh new file mode 100755 index 0000000..9534685 --- /dev/null +++ b/scripts/run_e2e.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e + +ROOT_DIR=$(pwd) + +# Function to cleanup on exit +cleanup() { + echo "[E2E] Dumping logs for report-generator-service..." + docker logs report-generator-service || true + + echo "[E2E] Tearing down environment..." + cd "$ROOT_DIR" + docker-compose -f docker-compose.yml -f docker-compose.e2e.yml down +} +# Trap exit to ensure cleanup +trap cleanup EXIT + +echo "[E2E] Building and Starting Environment..." +# Build specifically the services we need to ensure latest code +docker-compose -f docker-compose.yml -f docker-compose.e2e.yml up -d --build --remove-orphans + +echo "[E2E] Waiting for API Gateway (localhost:4000)..." +MAX_RETRIES=30 +count=0 +until curl -s http://localhost:4000/health > /dev/null; do + count=$((count+1)) + if [ $count -ge $MAX_RETRIES ]; then + echo "Timeout waiting for Gateway" + exit 1 + fi + echo "Waiting for Gateway... ($count/$MAX_RETRIES)" + sleep 2 +done +echo "Gateway is ready!" + +echo "[E2E] Running Rust Test Runner..." +cd tests/end-to-end +RUST_LOG=info cargo run + +echo "[E2E] Tests Completed Successfully!" diff --git a/services/alphavantage-provider-service/Cargo.lock b/services/alphavantage-provider-service/Cargo.lock index 4640b07..3d526fe 100644 --- a/services/alphavantage-provider-service/Cargo.lock +++ b/services/alphavantage-provider-service/Cargo.lock @@ -352,12 +352,17 @@ dependencies = [ name = "common-contracts" version = "0.1.0" dependencies = [ + "anyhow", "chrono", + "log", + "reqwest", "rust_decimal", "serde", "serde_json", "service_kit", "sqlx", + "tokio", + "tracing", "utoipa", "uuid", ] @@ -2168,10 +2173,11 @@ dependencies = [ [[package]] name = "rmcp" -version = "0.8.5" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5947688160b56fb6c827e3c20a72c90392a1d7e9dec74749197aa1780ac42ca" +checksum = "acc36ea743d4bbc97e9f3c33bf0b97765a5cf338de3d9c3d2f321a6e38095615" dependencies = [ + "async-trait", "base64", "chrono", "futures", @@ -2193,9 +2199,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "0.8.5" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01263441d3f8635c628e33856c468b96ebbce1af2d3699ea712ca71432d4ee7a" +checksum = "263caba1c96f2941efca0fdcd97b03f42bcde52d2347d05e5d77c93ab18c5b58" dependencies = [ "darling", "proc-macro2", diff --git a/services/alphavantage-provider-service/Cargo.toml b/services/alphavantage-provider-service/Cargo.toml index fd421d3..954b2b9 100644 --- a/services/alphavantage-provider-service/Cargo.toml +++ b/services/alphavantage-provider-service/Cargo.toml @@ -13,7 +13,7 @@ tower-http = { version = "0.6.6", features = ["cors"] } common-contracts = { path = "../common-contracts" } # Generic MCP Client -rmcp = { version = "0.8.5", features = ["client", "transport-streamable-http-client-reqwest"] } +rmcp = { version = "0.9.0", features = ["client", "transport-streamable-http-client-reqwest"] } # Message Queue (NATS) async-nats = "0.45.0" diff --git a/services/alphavantage-provider-service/src/config.rs b/services/alphavantage-provider-service/src/config.rs index c4e4d8e..ab78e82 100644 --- a/services/alphavantage-provider-service/src/config.rs +++ b/services/alphavantage-provider-service/src/config.rs @@ -7,6 +7,10 @@ pub struct AppConfig { pub nats_addr: String, pub data_persistence_service_url: String, pub alphavantage_api_key: Option, + + // New fields + pub api_gateway_url: String, + pub service_host: String, } impl AppConfig { @@ -22,6 +26,16 @@ impl AppConfig { "DATA_PERSISTENCE_SERVICE_URL must not be empty".to_string(), )); } + if cfg.api_gateway_url.trim().is_empty() { + return Err(config::ConfigError::Message( + "API_GATEWAY_URL must not be empty".to_string(), + )); + } + if cfg.service_host.trim().is_empty() { + return Err(config::ConfigError::Message( + "SERVICE_HOST must not be empty".to_string(), + )); + } Ok(cfg) } diff --git a/services/alphavantage-provider-service/src/main.rs b/services/alphavantage-provider-service/src/main.rs index daa3df6..5b1d546 100644 --- a/services/alphavantage-provider-service/src/main.rs +++ b/services/alphavantage-provider-service/src/main.rs @@ -4,7 +4,7 @@ mod config; mod error; mod mapping; mod message_consumer; -mod persistence; +// mod persistence; // Removed mod state; mod worker; mod av_client; @@ -14,7 +14,10 @@ mod transport; use crate::config::AppConfig; use crate::error::Result; use crate::state::AppState; -use tracing::info; +use tracing::{info, warn}; +use common_contracts::lifecycle::ServiceRegistrar; +use common_contracts::registry::ServiceRegistration; +use std::sync::Arc; #[tokio::main] async fn main() -> Result<()> { @@ -30,7 +33,7 @@ async fn main() -> Result<()> { let port = config.server_port; // Initialize application state - let app_state = AppState::new(config)?; + let app_state = AppState::new(config.clone())?; // --- Start the config poller --- tokio::spawn(config_poller::run_config_poller(app_state.clone())); @@ -41,12 +44,60 @@ async fn main() -> Result<()> { // --- Start the message consumer --- tokio::spawn(message_consumer::run(app_state)); + // --- Service Registration --- + let registrar = ServiceRegistrar::new( + config.api_gateway_url.clone(), + ServiceRegistration { + service_id: format!("{}-{}", "alphavantage-provider", uuid::Uuid::new_v4()), + service_name: "alphavantage".to_string(), + role: common_contracts::registry::ServiceRole::DataProvider, + base_url: format!("http://{}:{}", config.service_host, port), + health_check_url: format!("http://{}:{}/health", config.service_host, port), + } + ); + + let _ = registrar.register().await; + + let registrar = Arc::new(registrar); + tokio::spawn(registrar.clone().start_heartbeat_loop()); + // Start the HTTP server let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{}", port)) .await .unwrap(); info!("HTTP server listening on port {}", port); - axum::serve(listener, app).await.unwrap(); + + axum::serve(listener, app) + .with_graceful_shutdown(shutdown_signal(registrar)) + .await + .unwrap(); Ok(()) } + +async fn shutdown_signal(registrar: Arc) { + let ctrl_c = async { + tokio::signal::ctrl_c() + .await + .expect("failed to install Ctrl+C handler"); + }; + + #[cfg(unix)] + let terminate = async { + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => {}, + _ = terminate => {}, + } + + info!("Shutdown signal received, deregistering service..."); + let _ = registrar.deregister().await; +} diff --git a/services/alphavantage-provider-service/src/message_consumer.rs b/services/alphavantage-provider-service/src/message_consumer.rs index 0232250..d40c921 100644 --- a/services/alphavantage-provider-service/src/message_consumer.rs +++ b/services/alphavantage-provider-service/src/message_consumer.rs @@ -1,12 +1,11 @@ use crate::error::Result; use crate::state::{AppState, ServiceOperationalStatus}; use common_contracts::messages::FetchCompanyDataCommand; +use common_contracts::subjects::NatsSubject; use futures_util::StreamExt; use std::time::Duration; use tracing::{error, info, warn}; -const SUBJECT_NAME: &str = "data_fetch_commands"; - pub async fn run(state: AppState) -> Result<()> { info!("Starting NATS message consumer..."); @@ -38,10 +37,11 @@ pub async fn run(state: AppState) -> Result<()> { } async fn subscribe_and_process(state: AppState, client: async_nats::Client) -> Result<()> { - let mut subscriber = client.subscribe(SUBJECT_NAME.to_string()).await?; + let subject = NatsSubject::DataFetchCommands.to_string(); + let mut subscriber = client.subscribe(subject.clone()).await?; info!( "Consumer started, waiting for messages on subject '{}'", - SUBJECT_NAME + subject ); while let Some(message) = subscriber.next().await { @@ -59,12 +59,17 @@ async fn subscribe_and_process(state: AppState, client: async_nats::Client) -> R tokio::spawn(async move { match serde_json::from_slice::(&message.payload) { Ok(command) => { + let request_id = command.request_id; info!("Deserialized command for symbol: {}", command.symbol); if let Err(e) = - crate::worker::handle_fetch_command(state_clone, command, publisher_clone) + crate::worker::handle_fetch_command(state_clone.clone(), command, publisher_clone) .await { error!("Error handling fetch command: {:?}", e); + if let Some(mut task) = state_clone.tasks.get_mut(&request_id) { + task.status = common_contracts::observability::TaskStatus::Failed; + task.details = format!("Worker failed: {}", e); + } } } Err(e) => { diff --git a/services/alphavantage-provider-service/src/persistence.rs b/services/alphavantage-provider-service/src/persistence.rs deleted file mode 100644 index 974a13c..0000000 --- a/services/alphavantage-provider-service/src/persistence.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! -//! 数据持久化客户端 -//! -//! 提供一个类型化的接口，用于与 `data-persistence-service` 进行通信。 -//! - -use crate::error::Result; -use common_contracts::{ - dtos::{CompanyProfileDto, RealtimeQuoteDto, TimeSeriesFinancialBatchDto, TimeSeriesFinancialDto}, -}; -use tracing::info; - -#[derive(Clone)] -pub struct PersistenceClient { - client: reqwest::Client, - base_url: String, -} - -impl PersistenceClient { - pub fn new(base_url: String) -> Self { - Self { - client: reqwest::Client::new(), - base_url, - } - } - - pub async fn get_company_profile(&self, symbol: &str) -> Result> { - let url = format!("{}/companies/{}", self.base_url, symbol); - let resp = self.client.get(&url).send().await?; - if resp.status() == reqwest::StatusCode::NOT_FOUND { - return Ok(None); - } - let profile = resp.error_for_status()?.json().await?; - Ok(Some(profile)) - } - - pub async fn upsert_company_profile(&self, profile: CompanyProfileDto) -> Result<()> { - let url = format!("{}/companies", self.base_url); - info!("Upserting company profile for {} to {}", profile.symbol, url); - self.client - .put(&url) - .json(&profile) - .send() - .await? - .error_for_status()?; - Ok(()) - } - - pub async fn upsert_realtime_quote(&self, quote: RealtimeQuoteDto) -> Result<()> { - let url = format!("{}/market-data/quotes", self.base_url); - info!("Upserting realtime quote for {} to {}", quote.symbol, url); - self.client - .post(&url) - .json("e) - .send() - .await? - .error_for_status()?; - Ok(()) - } - - pub async fn batch_insert_financials(&self, dtos: Vec) -> Result<()> { - if dtos.is_empty() { - return Ok(()); - } - let url = format!("{}/market-data/financials/batch", self.base_url); - let symbol = dtos[0].symbol.clone(); - info!("Batch inserting {} financial statements for {} to {}", dtos.len(), symbol, url); - - let batch = TimeSeriesFinancialBatchDto { records: dtos }; - - self.client - .post(&url) - .json(&batch) - .send() - .await? - .error_for_status()?; - - Ok(()) - } -} diff --git a/services/alphavantage-provider-service/src/state.rs b/services/alphavantage-provider-service/src/state.rs index 849a276..a1711a0 100644 --- a/services/alphavantage-provider-service/src/state.rs +++ b/services/alphavantage-provider-service/src/state.rs @@ -46,7 +46,12 @@ impl AppState { let mut provider_guard = self.av_provider.write().await; let mut status_guard = self.status.write().await; - match (api_key, api_url) { + // Fallback to default URL if not provided + let final_url = api_url + .filter(|s| !s.trim().is_empty()) + .or_else(|| Some("https://mcp.alphavantage.co/mcp".to_string())); + + match (api_key, final_url) { (Some(key), Some(base_url)) => { if base_url.contains('?') { *provider_guard = None; diff --git a/services/alphavantage-provider-service/src/transport.rs b/services/alphavantage-provider-service/src/transport.rs index 91feb4b..77147ec 100644 --- a/services/alphavantage-provider-service/src/transport.rs +++ b/services/alphavantage-provider-service/src/transport.rs @@ -27,7 +27,10 @@ pub struct CustomHttpClient { impl Default for CustomHttpClient { fn default() -> Self { Self { - client: reqwest::Client::new(), + client: reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()), } } } diff --git a/services/alphavantage-provider-service/src/worker.rs b/services/alphavantage-provider-service/src/worker.rs index ba65c22..e45211d 100644 --- a/services/alphavantage-provider-service/src/worker.rs +++ b/services/alphavantage-provider-service/src/worker.rs @@ -1,11 +1,12 @@ use crate::error::{Result, AppError}; use crate::mapping::{CombinedFinancials, parse_company_profile, parse_financials, parse_realtime_quote}; -use crate::persistence::PersistenceClient; +use common_contracts::persistence_client::PersistenceClient; +use common_contracts::dtos::{SessionDataDto, ProviderCacheDto, TimeSeriesFinancialDto, CompanyProfileDto}; use crate::state::{AppState, TaskStore}; use anyhow::Context; -use chrono::{Utc, Datelike}; -use common_contracts::messages::{FetchCompanyDataCommand, FinancialsPersistedEvent}; -use common_contracts::observability::TaskProgress; +use chrono::{Utc, Datelike, Duration}; +use common_contracts::messages::{FetchCompanyDataCommand, FinancialsPersistedEvent, DataFetchFailedEvent}; +use common_contracts::observability::{TaskProgress, TaskStatus}; use tracing::{error, info, instrument, warn}; use uuid::Uuid; use serde_json::Value; @@ -15,13 +16,59 @@ pub async fn handle_fetch_command( state: AppState, command: FetchCompanyDataCommand, publisher: async_nats::Client, +) -> Result<()> { + match handle_fetch_command_inner(state.clone(), &command, &publisher).await { + Ok(_) => Ok(()), + Err(e) => { + error!("AlphaVantage workflow failed: {}", e); + + // Publish failure event + let event = DataFetchFailedEvent { + request_id: command.request_id, + symbol: command.symbol.clone(), + error: e.to_string(), + provider_id: Some("alphavantage".to_string()), + }; + let _ = publisher + .publish( + "events.data.fetch_failed".to_string(), + serde_json::to_vec(&event).unwrap().into(), + ) + .await; + + // Update task status + if let Some(mut task) = state.tasks.get_mut(&command.request_id) { + task.status = TaskStatus::Failed; + task.details = format!("Failed: {}", e); + } else { + // If task doesn't exist (e.g. failed at insert), create a failed task + let task = TaskProgress { + request_id: command.request_id, + task_name: format!("alphavantage:{}", command.symbol), + status: TaskStatus::Failed, + progress_percent: 0, + details: format!("Failed: {}", e), + started_at: Utc::now(), + }; + state.tasks.insert(command.request_id, task); + } + + Err(e) + } + } +} + +async fn handle_fetch_command_inner( + state: AppState, + command: &FetchCompanyDataCommand, + publisher: &async_nats::Client, ) -> Result<()> { info!("Handling fetch data command."); let task = TaskProgress { request_id: command.request_id, - task_name: format!("fetch_data_for_{}", command.symbol), - status: "in_progress".to_string(), + task_name: format!("alphavantage:{}", command.symbol), + status: TaskStatus::InProgress, progress_percent: 0, details: "Initializing...".to_string(), started_at: Utc::now(), @@ -32,13 +79,6 @@ pub async fn handle_fetch_command( Some(p) => p, None => { let reason = "Execution failed: Alphavantage provider is not available (misconfigured).".to_string(); - error!("{}", reason); - update_task_progress( - &state.tasks, - command.request_id, - 100, - &reason, - ).await; return Err(AppError::ProviderNotAvailable(reason)); } }; @@ -47,121 +87,126 @@ pub async fn handle_fetch_command( PersistenceClient::new(state.config.data_persistence_service_url.clone()); let symbol = command.symbol.clone(); - // Check freshness - let mut is_fresh = false; - match persistence_client.get_company_profile(&command.symbol).await { - Ok(Some(p)) => { - if let Some(updated_at) = p.updated_at { - let age = chrono::Utc::now() - updated_at; - if age < chrono::Duration::hours(24) { - info!("Data for {} is fresh (age: {}h). Skipping fetch.", command.symbol, age.num_hours()); - is_fresh = true; - } - } - } - Ok(None) => {} - Err(e) => tracing::warn!("Failed to check profile freshness: {}", e), - } - - if is_fresh { - let event = FinancialsPersistedEvent { - request_id: command.request_id, - symbol: command.symbol, - years_updated: vec![], - template_id: command.template_id, - }; - let subject = "events.data.financials_persisted".to_string(); - publisher - .publish(subject, serde_json::to_vec(&event).unwrap().into()) - .await?; - - update_task_progress( - &state.tasks, - command.request_id, - 100, - "Data retrieved from cache", - ) - .await; - return Ok(()); - } - - // Symbol conversion for Chinese stocks - let av_symbol = if symbol.ends_with(".SH") { - symbol.replace(".SH", ".SS") - } else { - symbol.clone() - }; + // Symbol conversion using shared logic + let av_symbol = symbol.to_alphavantage(); info!("Using symbol for AlphaVantage: {}", av_symbol); update_task_progress( &state.tasks, command.request_id, 10, - "Fetching from AlphaVantage...", + "Checking cache...", + None, ) .await; - // --- 1. Fetch all data in parallel --- - let (overview_json, income_json, balance_json, cashflow_json, quote_json) = { - let params_overview = vec![("symbol", av_symbol.as_str())]; - let params_income = vec![("symbol", av_symbol.as_str())]; - let params_balance = vec![("symbol", av_symbol.as_str())]; - let params_cashflow = vec![("symbol", av_symbol.as_str())]; - // Add datatype=json to force JSON response if supported (or at least Python-dict like) - let params_quote = vec![("symbol", av_symbol.as_str()), ("datatype", "json")]; - - let overview_task = client.query("COMPANY_OVERVIEW", ¶ms_overview); - let income_task = client.query("INCOME_STATEMENT", ¶ms_income); - let balance_task = client.query("BALANCE_SHEET", ¶ms_balance); - let cashflow_task = client.query("CASH_FLOW", ¶ms_cashflow); - let quote_task = client.query("GLOBAL_QUOTE", ¶ms_quote); + // --- 1. Check Cache --- + let cache_key = format!("alphavantage:{}:all", av_symbol); + + let (overview_json, income_json, balance_json, cashflow_json, quote_json) = match persistence_client.get_cache(&cache_key).await.map_err(|e| AppError::Internal(e.to_string()))? { + Some(cache_entry) => { + info!("Cache HIT for {}", cache_key); + // Deserialize tuple of JSONs + let data: (Value, Value, Value, Value, Value) = serde_json::from_value(cache_entry.data_payload) + .map_err(|e| AppError::Internal(format!("Failed to deserialize cache: {}", e)))?; + + update_task_progress( + &state.tasks, + command.request_id, + 50, + "Data retrieved from cache", + None, + ).await; + data + }, + None => { + info!("Cache MISS for {}", cache_key); + update_task_progress( + &state.tasks, + command.request_id, + 20, + "Fetching from AlphaVantage API...", + None, + ).await; - match tokio::try_join!( - overview_task, - income_task, - balance_task, - cashflow_task, - quote_task - ) { - Ok(data) => data, - Err(e) => { - let error_msg = format!("Failed to fetch data from AlphaVantage: {}", e); - error!(error_msg); - update_task_progress(&state.tasks, command.request_id, 100, &error_msg).await; - return Err(e); - } + let params_overview = vec![("symbol", av_symbol.as_str())]; + let params_income = vec![("symbol", av_symbol.as_str())]; + let params_balance = vec![("symbol", av_symbol.as_str())]; + let params_cashflow = vec![("symbol", av_symbol.as_str())]; + // Add datatype=json to force JSON response if supported (or at least Python-dict like) + let params_quote = vec![("symbol", av_symbol.as_str()), ("datatype", "json")]; + + let overview_task = client.query("COMPANY_OVERVIEW", ¶ms_overview); + let income_task = client.query("INCOME_STATEMENT", ¶ms_income); + let balance_task = client.query("BALANCE_SHEET", ¶ms_balance); + let cashflow_task = client.query("CASH_FLOW", ¶ms_cashflow); + let quote_task = client.query("GLOBAL_QUOTE", ¶ms_quote); + + let data = match tokio::try_join!( + overview_task, + income_task, + balance_task, + cashflow_task, + quote_task + ) { + Ok(data) => data, + Err(e) => { + let error_msg = format!("Failed to fetch data from AlphaVantage: {}", e); + error!(error_msg); + return Err(e); + } + }; + + // Write to Cache + let payload = serde_json::json!(data); + persistence_client.set_cache(&ProviderCacheDto { + cache_key, + data_payload: payload, + expires_at: Utc::now() + Duration::hours(24), + updated_at: None, + }).await.map_err(|e| AppError::Internal(e.to_string()))?; + + data } }; update_task_progress( &state.tasks, command.request_id, - 50, - "Data fetched, transforming and persisting...", + 70, + "Data fetched, processing...", + None, ) .await; - // --- 2. Transform and persist data --- - // Profile - // Check if overview_json is empty (Symbol field check) + // --- 2. Transform and Snapshot Data --- + + // 2.1 Profile if let Some(_symbol_val) = overview_json.get("Symbol") { match parse_company_profile(overview_json) { Ok(profile_to_persist) => { - persistence_client - .upsert_company_profile(profile_to_persist) - .await?; + // Update Global Profile + // REMOVED: upsert_company_profile is deprecated. + // let _ = persistence_client.upsert_company_profile(profile_to_persist.clone()).await; + + // Snapshot Profile + persistence_client.insert_session_data(&SessionDataDto { + request_id: command.request_id, + symbol: command.symbol.to_string(), + provider: "alphavantage".to_string(), + data_type: "company_profile".to_string(), + data_payload: serde_json::to_value(&profile_to_persist).unwrap(), + created_at: None, + }).await.map_err(|e| AppError::Internal(e.to_string()))?; }, Err(e) => { warn!("Failed to parse CompanyProfile: {}", e); } } - } else { - warn!("CompanyProfile data is empty or missing 'Symbol' for {}, skipping persistence.", av_symbol); } - // Financials + // 2.2 Financials let mut years_updated: Vec = Vec::new(); - // Only attempt to parse financials if we have data (simple check if income statement has annualReports) if income_json.get("annualReports").is_some() { let combined_financials = CombinedFinancials { income: income_json, @@ -175,25 +220,28 @@ pub async fn handle_fetch_command( .iter() .map(|f| f.period_date.year() as u16) .collect(); - persistence_client - .batch_insert_financials(financials_to_persist) - .await?; + + // Snapshot Financials + persistence_client.insert_session_data(&SessionDataDto { + request_id: command.request_id, + symbol: command.symbol.to_string(), + provider: "alphavantage".to_string(), + data_type: "financial_statements".to_string(), + data_payload: serde_json::to_value(&financials_to_persist).unwrap(), + created_at: None, + }).await.map_err(|e| AppError::Internal(e.to_string()))?; } }, Err(e) => { warn!("Failed to parse Financials: {}", e); } } - } else { - warn!("Financial data missing for {}, skipping.", av_symbol); } - // Quote + // 2.3 Quote // Fix Python-dict string if necessary let fixed_quote_json = if let Some(s) = quote_json.as_str() { if s.trim().starts_with("{'Global Quote'") { - // Attempt to replace single quotes with double quotes - // Note: This is a naive fix but works for the expected format let fixed = s.replace("'", "\""); match serde_json::from_str::(&fixed) { Ok(v) => v, @@ -209,13 +257,24 @@ pub async fn handle_fetch_command( quote_json }; + // Realtime quote is global/time-series, so we still use upsert_realtime_quote + let mut summary = format!("Fetched {} years of financial data", years_updated.len()); + match parse_realtime_quote(fixed_quote_json, &command.market) { Ok(mut quote_to_persist) => { - // Restore original symbol if we converted it - quote_to_persist.symbol = command.symbol.clone(); - persistence_client - .upsert_realtime_quote(quote_to_persist) - .await?; + quote_to_persist.symbol = command.symbol.to_string(); + // Snapshot Realtime Quote + let _ = persistence_client.insert_session_data(&SessionDataDto { + request_id: command.request_id, + symbol: command.symbol.to_string(), + provider: "alphavantage".to_string(), + data_type: "realtime_quote".to_string(), + data_payload: serde_json::to_value("e_to_persist).unwrap(), + created_at: None, + }).await; + + summary = format!("Parsed Realtime Quote for {}: Price={}, Volume={:?}", + quote_to_persist.symbol, quote_to_persist.price, quote_to_persist.volume); }, Err(e) => { warn!("Failed to parse RealtimeQuote: {}", e); @@ -226,36 +285,130 @@ pub async fn handle_fetch_command( &state.tasks, command.request_id, 90, - "Data persisted, publishing events...", + "Snapshot created, publishing events...", + None, ) .await; // --- 3. Publish events --- - // Only publish if we actually updated something - // Actually, we should publish event even if partial, to signal completion? - // The command is "FetchCompanyData", implies success if we fetched *available* data. let event = FinancialsPersistedEvent { request_id: command.request_id, - symbol: command.symbol, + symbol: command.symbol.clone(), years_updated, - template_id: command.template_id, + template_id: command.template_id.clone(), + provider_id: Some("alphavantage".to_string()), + data_summary: Some(summary), }; - let subject = "events.data.financials_persisted".to_string(); // NATS subject + let subject = "events.data.financials_persisted".to_string(); publisher .publish(subject, serde_json::to_vec(&event).unwrap().into()) .await?; - state.tasks.remove(&command.request_id); - info!("Task completed successfully (Partial data may be missing if provider lacks coverage)."); + // Update Provider Status + // REMOVED: update_provider_status is deprecated or missing in client. + /* + persistence_client.update_provider_status(command.symbol.as_str(), "alphavantage", common_contracts::dtos::ProviderStatusDto { + last_updated: chrono::Utc::now(), + status: TaskStatus::Completed, + data_version: None, + }).await?; + */ + + update_task_progress( + &state.tasks, + command.request_id, + 100, + "Task completed successfully", + Some(TaskStatus::Completed), + ).await; + + info!("AlphaVantage task completed successfully."); Ok(()) } -async fn update_task_progress(tasks: &TaskStore, request_id: Uuid, percent: u8, details: &str) { +async fn update_task_progress(tasks: &TaskStore, request_id: Uuid, percent: u8, details: &str, status: Option) { if let Some(mut task) = tasks.get_mut(&request_id) { task.progress_percent = percent; task.details = details.to_string(); - info!("Task update: {}% - {}", percent, details); + if let Some(s) = status { + task.status = s; + } + info!("Task update: {}% - {} (Status: {:?})", percent, details, task.status); + } +} + +#[cfg(test)] +mod integration_tests { + use super::*; + use crate::config::AppConfig; + use crate::state::AppState; + use secrecy::SecretString; + use std::time::Duration; + use common_contracts::symbol_utils::{CanonicalSymbol, Market}; + + #[tokio::test] + async fn test_alphavantage_fetch_flow() { + // Check if running in test environment + if std::env::var("NATS_ADDR").is_err() { + // Skip if env vars not set (e.g. running cargo test without script) + // But better to panic to alert developer + // panic!("Must run integration tests with run_component_tests.sh or set env vars"); + println!("Skipping integration test (no environment)"); + return; + } + + // 1. Environment Variables + // Assumed set by external script, but we double check specific overrides for component test + // NATS_ADDR, DATA_PERSISTENCE_SERVICE_URL, ALPHAVANTAGE_API_KEY, ALPHAVANTAGE_MCP_URL + + let api_key = std::env::var("ALPHAVANTAGE_API_KEY") + .unwrap_or_else(|_| "PUOO7UPTNXN325NN".to_string()); + + let mcp_url = std::env::var("ALPHAVANTAGE_MCP_URL") + .expect("ALPHAVANTAGE_MCP_URL must be set"); + + let config = AppConfig::load().expect("Failed to load config"); + let state = AppState::new(config.clone()).expect("Failed to create state"); + + // 2. Manual Init Provider (Skip Config Poller) + state.update_provider( + Some(SecretString::new(api_key.into())), + Some(mcp_url) + ).await; + + // Wait for connection + let mut connected = false; + for _ in 0..10 { + if state.get_provider().await.is_some() { + connected = true; + break; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert!(connected, "Failed to connect to AlphaVantage MCP Provider"); + + // 3. Construct Command + let request_id = Uuid::new_v4(); + let cmd = FetchCompanyDataCommand { + request_id, + symbol: CanonicalSymbol::new("IBM", &Market::US), + market: "US".to_string(), + template_id: Some("default".to_string()), + }; + + // 4. NATS + let nats_client = async_nats::connect(&config.nats_addr).await + .expect("Failed to connect to NATS"); + + // 5. Run + let result = handle_fetch_command_inner(state.clone(), &cmd, &nats_client).await; + + // 6. Assert + assert!(result.is_ok(), "Worker execution failed: {:?}", result.err()); + + let task = state.tasks.get(&request_id).expect("Task should exist"); + assert_eq!(task.status, TaskStatus::Completed); } } diff --git a/services/api-gateway/Cargo.lock b/services/api-gateway/Cargo.lock index 42fa9f6..960ee1b 100644 --- a/services/api-gateway/Cargo.lock +++ b/services/api-gateway/Cargo.lock @@ -49,7 +49,9 @@ version = "0.1.0" dependencies = [ "anyhow", "async-nats", + "async-stream", "axum", + "chrono", "common-contracts", "config", "futures-util", @@ -113,6 +115,28 @@ dependencies = [ "url", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -345,12 +369,17 @@ dependencies = [ name = "common-contracts" version = "0.1.0" dependencies = [ + "anyhow", "chrono", + "log", + "reqwest", "rust_decimal", "serde", "serde_json", "service_kit", "sqlx", + "tokio", + "tracing", "utoipa", "uuid", ] @@ -644,6 +673,16 @@ dependencies = [ "typeid", ] +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -666,6 +705,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -701,6 +746,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -837,6 +897,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -973,6 +1052,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -1002,6 +1082,22 @@ dependencies = [ "webpki-roots 1.0.4", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.18" @@ -1021,9 +1117,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -1258,6 +1356,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "litemap" version = "0.8.1" @@ -1333,6 +1437,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nkeys" version = "0.4.5" @@ -1424,12 +1545,50 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "openssl-probe" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "ordered-multimap" version = "0.7.3" @@ -1712,7 +1871,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -1870,15 +2029,21 @@ checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ "base64", "bytes", + "encoding_rs", "futures-core", + "futures-util", + "h2", "http", "http-body", "http-body-util", "hyper", "hyper-rustls", + "hyper-tls", "hyper-util", "js-sys", "log", + "mime", + "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -1889,13 +2054,16 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-native-tls", "tokio-rustls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots 1.0.4", ] @@ -2018,6 +2186,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.35" @@ -2711,12 +2892,46 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tap" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -2859,6 +3074,16 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -3308,6 +3533,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.82" @@ -3397,6 +3635,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" diff --git a/services/api-gateway/Cargo.toml b/services/api-gateway/Cargo.toml index 2d1e626..8f3b80e 100644 --- a/services/api-gateway/Cargo.toml +++ b/services/api-gateway/Cargo.toml @@ -15,9 +15,10 @@ common-contracts = { path = "../common-contracts" } # Message Queue (NATS) async-nats = "0.45.0" futures-util = "0.3" +async-stream = "0.3" # HTTP Client -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] } # Concurrency & Async uuid = { version = "1.8", features = ["v4"] } @@ -36,3 +37,4 @@ config = "0.15.19" # Error Handling thiserror = "2.0.17" anyhow = "1.0" +chrono = "0.4.42" diff --git a/services/api-gateway/src/api.rs b/services/api-gateway/src/api.rs index 14fc9ff..2922e37 100644 --- a/services/api-gateway/src/api.rs +++ b/services/api-gateway/src/api.rs @@ -7,28 +7,31 @@ use axum::{ routing::{get, post}, Router, }; -use common_contracts::messages::{FetchCompanyDataCommand, GenerateReportCommand}; -use common_contracts::observability::{HealthStatus, ServiceStatus, TaskProgress}; +use common_contracts::messages::GenerateReportCommand; +use common_contracts::observability::{TaskProgress, TaskStatus}; +use common_contracts::subjects::{NatsSubject, SubjectMessage}; +use common_contracts::symbol_utils::{CanonicalSymbol, Market}; use futures_util::future::join_all; +use futures_util::stream::StreamExt; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use tracing::{info, warn}; +use tracing::{info, warn, error}; use uuid::Uuid; -const DATA_FETCH_QUEUE: &str = "data_fetch_commands"; -const ANALYSIS_COMMANDS_QUEUE: &str = "analysis.commands.generate_report"; +mod registry; // --- Request/Response Structs --- #[derive(Deserialize)] pub struct DataRequest { pub symbol: String, - pub market: String, - pub template_id: Option, + pub market: Option, + pub template_id: String, // Changed to required as it's mandatory for workflow } #[derive(Serialize)] pub struct RequestAcceptedResponse { pub request_id: Uuid, + pub symbol: String, + pub market: String, } #[derive(Deserialize)] @@ -41,18 +44,40 @@ pub struct AnalysisResultQuery { pub symbol: String, } +#[derive(Deserialize)] +pub struct SymbolResolveRequest { + pub symbol: String, + pub market: Option, +} + +#[derive(Serialize)] +pub struct SymbolResolveResponse { + pub symbol: String, + pub market: String, +} + // --- Router Definition --- pub fn create_router(app_state: AppState) -> Router { Router::new() .route("/health", get(health_check)) - .route("/tasks", get(get_current_tasks)) // This is the old, stateless one + .route("/tasks/{request_id}", get(get_task_progress)) .nest("/v1", create_v1_router()) .with_state(app_state) } +use common_contracts::messages::{StartWorkflowCommand, SyncStateCommand, WorkflowEvent}; + fn create_v1_router() -> Router { Router::new() - .route("/data-requests", post(trigger_data_fetch)) + // New Workflow API + .route("/workflow/start", post(start_workflow)) + .route("/workflow/events/{request_id}", get(workflow_events_stream)) + // Tools + .route("/tools/resolve-symbol", post(resolve_symbol)) + // Legacy routes (marked for removal or compatibility) + .route("/data-requests", post(trigger_data_fetch_legacy)) + .route("/session-data/{request_id}", get(proxy_get_session_data)) + .route("/analysis-results/stream", get(proxy_analysis_stream)) .route( "/analysis-requests/{symbol}", post(trigger_analysis_generation), @@ -60,8 +85,7 @@ fn create_v1_router() -> Router { .route("/analysis-results", get(get_analysis_results_by_symbol)) .route("/companies/{symbol}/profile", get(get_company_profile)) .route("/market-data/financial-statements/{symbol}", get(get_financials_by_symbol)) - .route("/tasks/{request_id}", get(get_task_progress)) - // --- New Config Routes --- + // ... Config routes remain same ... .route( "/configs/llm_providers", get(get_llm_providers_config).put(update_llm_providers_config), @@ -76,62 +100,162 @@ fn create_v1_router() -> Router { ) .route("/configs/test", post(test_data_source_config)) .route("/configs/llm/test", post(test_llm_config)) - // --- New Discover Routes --- .route("/discover-models/{provider_id}", get(discover_models)) .route("/discover-models", post(discover_models_preview)) + .route("/registry/register", post(registry::register_service)) + .route("/registry/heartbeat", post(registry::heartbeat)) + .route("/registry/deregister", post(registry::deregister_service)) } -// --- Health & Stateless Tasks --- -async fn health_check(State(state): State) -> Json { - let mut details = HashMap::new(); - // 提供确定性且无副作用的健康详情，避免访问不存在的状态字段 - details.insert("message_bus".to_string(), "nats".to_string()); - details.insert("nats_addr".to_string(), state.config.nats_addr.clone()); - - let status = HealthStatus { - module_id: "api-gateway".to_string(), - status: ServiceStatus::Ok, - version: env!("CARGO_PKG_VERSION").to_string(), - details, +// --- Helper Functions --- + +fn infer_market(symbol: &str) -> String { + if symbol.ends_with(".SS") || symbol.ends_with(".SH") { + "CN".to_string() + } else if symbol.ends_with(".HK") { + "HK".to_string() + } else { + "US".to_string() + } +} + +// --- New Workflow Handlers --- + +/// [POST /v1/tools/resolve-symbol] +/// Resolves and normalizes a symbol without starting a workflow. +async fn resolve_symbol( + Json(payload): Json, +) -> Result { + let market = if let Some(m) = payload.market { + if m.is_empty() { + infer_market(&payload.symbol) + } else { + m + } + } else { + infer_market(&payload.symbol) }; - Json(status) -} -async fn get_current_tasks() -> Json> { - Json(vec![]) + + let market_enum = Market::from(market.as_str()); + let normalized_symbol = CanonicalSymbol::new(&payload.symbol, &market_enum); + + Ok(Json(SymbolResolveResponse { + symbol: normalized_symbol.into(), + market, + })) } -// --- V1 API Handlers --- - -/// [POST /v1/data-requests] -/// Triggers the data fetching process by publishing a command to the message bus. -async fn trigger_data_fetch( +/// [POST /v1/workflow/start] +/// Initiates a new analysis workflow via the Orchestrator. +async fn start_workflow( State(state): State, Json(payload): Json, ) -> Result { let request_id = Uuid::new_v4(); - let command = FetchCompanyDataCommand { + + let market = if let Some(m) = payload.market { + if m.is_empty() { + infer_market(&payload.symbol) + } else { + m + } + } else { + infer_market(&payload.symbol) + }; + let market_enum = Market::from(market.as_str()); + let normalized_symbol = CanonicalSymbol::new(&payload.symbol, &market_enum); + + let command = StartWorkflowCommand { request_id, - symbol: payload.symbol.clone(), - market: payload.market, - template_id: payload.template_id.clone(), + symbol: normalized_symbol.clone(), + market: market.clone(), + template_id: payload.template_id, }; - info!(request_id = %request_id, "Publishing data fetch command"); + info!(request_id = %request_id, "Publishing StartWorkflowCommand to Orchestrator"); - state - .nats_client + state.nats_client .publish( - DATA_FETCH_QUEUE.to_string(), + command.subject().to_string(), serde_json::to_vec(&command).unwrap().into(), ) .await?; Ok(( StatusCode::ACCEPTED, - Json(RequestAcceptedResponse { request_id }), + Json(RequestAcceptedResponse { + request_id, + symbol: normalized_symbol.into(), + market + }), )) } +/// [GET /v1/workflow/events/:request_id] +/// SSE endpoint that proxies events from NATS to the frontend. +async fn workflow_events_stream( + State(state): State, + Path(request_id): Path, +) -> Result { + info!("Client connected to event stream for {}", request_id); + + // 1. Send SyncStateCommand to ask Orchestrator for a snapshot + // This ensures if the client reconnects, they get the latest state immediately. + let sync_cmd = SyncStateCommand { request_id }; + if let Err(e) = state.nats_client + .publish(sync_cmd.subject().to_string(), serde_json::to_vec(&sync_cmd).unwrap().into()) + .await + { + error!("Failed to send SyncStateCommand: {}", e); + } + + // 2. Subscribe to NATS topic + let topic = NatsSubject::WorkflowProgress(request_id).to_string(); + let mut subscriber = state.nats_client.subscribe(topic).await?; + + // 3. Convert NATS stream to SSE stream + let stream = async_stream::stream! { + while let Some(msg) = subscriber.next().await { + if let Ok(event) = serde_json::from_slice::(&msg.payload) { + match axum::response::sse::Event::default().json_data(event) { + Ok(sse_event) => yield Ok::<_, anyhow::Error>(sse_event), + Err(e) => error!("Failed to serialize SSE event: {}", e), + } + } + } + }; + + Ok(axum::response::Sse::new(stream) + .keep_alive(axum::response::sse::KeepAlive::default())) +} + +// --- Legacy Handler (Renamed) --- +async fn trigger_data_fetch_legacy( + State(state): State, + Json(payload): Json, +) -> Result { + // Redirect to new workflow start for compatibility if possible, or keep as is for now? + // Let's just call start_workflow to gradually migrate behavior. + start_workflow(State(state), Json(payload)).await +} + +async fn health_check() -> impl IntoResponse { + (StatusCode::OK, "OK") +} + +async fn proxy_get_session_data( + State(_state): State, + Path(_request_id): Path, +) -> Result { + Ok((StatusCode::NOT_IMPLEMENTED, Json(serde_json::json!({"error": "Not implemented"})))) +} + +async fn proxy_analysis_stream( + State(_state): State, +) -> Result { + Ok((StatusCode::NOT_IMPLEMENTED, Json(serde_json::json!({"error": "Not implemented"})))) +} + /// [POST /v1/analysis-requests/:symbol] /// Triggers the analysis report generation workflow by publishing a command. async fn trigger_analysis_generation( @@ -140,9 +264,19 @@ async fn trigger_analysis_generation( Json(payload): Json, ) -> Result { let request_id = Uuid::new_v4(); + + // Try to infer market to help normalization, defaulting to US if unclear but keeping original behavior safe + let market_str = infer_market(&symbol); + let market_enum = Market::from(market_str.as_str()); + let normalized_symbol = CanonicalSymbol::new(&symbol, &market_enum); + + if normalized_symbol.as_str() != symbol { + info!("Normalized analysis request symbol '{}' to '{}'", symbol, normalized_symbol); + } + let command = GenerateReportCommand { request_id, - symbol, + symbol: normalized_symbol.clone(), template_id: payload.template_id, }; @@ -151,14 +285,21 @@ async fn trigger_analysis_generation( state .nats_client .publish( - ANALYSIS_COMMANDS_QUEUE.to_string(), + command.subject().to_string(), serde_json::to_vec(&command).unwrap().into(), ) .await?; + + // Infer market for response consistency + let market = infer_market(normalized_symbol.as_str()); Ok(( StatusCode::ACCEPTED, - Json(RequestAcceptedResponse { request_id }), + Json(RequestAcceptedResponse { + request_id, + symbol: normalized_symbol.into(), + market + }), )) } @@ -197,25 +338,42 @@ async fn get_task_progress( Path(request_id): Path, ) -> Result { let client = reqwest::Client::new(); - let fetches = state - .config - .provider_services + let services = state.get_all_services(); + + let fetches = services .iter() - .map(|service_url| { + .map(|(service_id, service_url)| { let client = client.clone(); let url = format!("{}/tasks", service_url); + let service_id_clone = service_id.clone(); async move { match client.get(&url).send().await { Ok(resp) => match resp.json::>().await { Ok(tasks) => Some(tasks), Err(e) => { warn!("Failed to decode tasks from {}: {}", url, e); - None + // Return a synthetic error task for this provider + Some(vec![TaskProgress { + request_id, + task_name: format!("{}:unreachable", service_id_clone), + status: TaskStatus::Failed, + progress_percent: 0, + details: "Invalid response format".to_string(), + started_at: chrono::Utc::now(), + }]) } }, Err(e) => { warn!("Failed to fetch tasks from {}: {}", url, e); - None + // Return a synthetic error task for this provider + Some(vec![TaskProgress { + request_id, + task_name: format!("{}:unreachable", service_id_clone), + status: TaskStatus::Failed, + progress_percent: 0, + details: format!("Connection Error: {}", e), + started_at: chrono::Utc::now(), + }]) } } } @@ -228,12 +386,22 @@ async fn get_task_progress( merged.extend(tasks); } } - - if let Some(task) = merged.into_iter().find(|t| t.request_id == request_id) { - Ok((StatusCode::OK, Json(task)).into_response()) - } else { - Ok((StatusCode::NOT_FOUND, Json(serde_json::json!({"error": "Task not found"}))).into_response()) + + let tasks_for_req: Vec = merged.into_iter() + .filter(|t| t.request_id == request_id) + .collect(); + + if tasks_for_req.is_empty() { + // Instead of returning 404, we should probably return an empty list if we have checked everyone + // But if we really found nothing (even synthetic errors), then 404 is fine. + // With synthetic errors, this should rarely happen unless no providers are registered. + if services.is_empty() { + warn!("No providers registered to query for tasks."); + } + return Ok((StatusCode::NOT_FOUND, Json(serde_json::json!({"error": "Task not found"}))).into_response()); } + + Ok(Json(tasks_for_req).into_response()) } @@ -254,17 +422,8 @@ async fn test_data_source_config( ) -> Result { info!("test_data_source_config: type={}", payload.r#type); - let target_service_url = match payload.r#type.as_str() { - "tushare" => state.config.provider_services.iter().find(|s| s.contains("tushare")), - "finnhub" => state.config.provider_services.iter().find(|s| s.contains("finnhub")), - "alphavantage" => state.config.provider_services.iter().find(|s| s.contains("alphavantage")), - _ => { - return Ok(( - StatusCode::BAD_REQUEST, - Json(serde_json::json!({ "error": "Unsupported config type" })), - ).into_response()); - } - }; + // Dynamic discovery + let target_service_url = state.get_service_url(&payload.r#type); if let Some(base_url) = target_service_url { let client = reqwest::Client::new(); @@ -293,10 +452,10 @@ async fn test_data_source_config( let response_json: serde_json::Value = response.json().await?; Ok((StatusCode::OK, Json(response_json)).into_response()) } else { - warn!("No downstream service found for config type: {}", payload.r#type); + warn!("No downstream service registered for config type: {}", payload.r#type); Ok(( StatusCode::NOT_IMPLEMENTED, - Json(serde_json::json!({ "error": "No downstream service configured for this type" })), + Json(serde_json::json!({ "error": "No downstream service registered for this type" })), ).into_response()) } } diff --git a/services/api-gateway/src/api/registry.rs b/services/api-gateway/src/api/registry.rs new file mode 100644 index 0000000..1a66bf2 --- /dev/null +++ b/services/api-gateway/src/api/registry.rs @@ -0,0 +1,64 @@ +use axum::{ + extract::State, + http::StatusCode, + response::IntoResponse, + Json, +}; +use common_contracts::registry::{Heartbeat, ServiceRegistration}; +use std::time::Instant; +use tracing::{info, warn}; + +use crate::{error::Result, state::{AppState, RegistryEntry}}; + +/// [POST /v1/registry/register] +pub async fn register_service( + State(state): State, + Json(payload): Json, +) -> Result { + info!("Registering service: {} ({}) at {}", payload.service_id, payload.service_name, payload.base_url); + + let entry = RegistryEntry { + registration: payload.clone(), + last_heartbeat: Instant::now(), + }; + + let mut registry = state.registry.write().unwrap(); + registry.insert(payload.service_id.clone(), entry); + + Ok(StatusCode::OK) +} + +/// [POST /v1/registry/heartbeat] +pub async fn heartbeat( + State(state): State, + Json(payload): Json, +) -> Result { + let mut registry = state.registry.write().unwrap(); + + if let Some(entry) = registry.get_mut(&payload.service_id) { + entry.last_heartbeat = Instant::now(); + Ok(StatusCode::OK) + } else { + // This is the key part for self-healing: tell the provider we don't know them + warn!("Received heartbeat from unknown service: {}", payload.service_id); + Ok(StatusCode::NOT_FOUND) + } +} + +/// [POST /v1/registry/deregister] +pub async fn deregister_service( + State(state): State, + Json(payload): Json, +) -> Result { + let service_id = payload.get("service_id") + .and_then(|v| v.as_str()) + .ok_or_else(|| crate::error::AppError::BadRequest("Missing service_id".into()))?; + + info!("Deregistering service: {}", service_id); + + let mut registry = state.registry.write().unwrap(); + registry.remove(service_id); + + Ok(StatusCode::OK) +} + diff --git a/services/api-gateway/src/config.rs b/services/api-gateway/src/config.rs index cd7f2ca..3dbce28 100644 --- a/services/api-gateway/src/config.rs +++ b/services/api-gateway/src/config.rs @@ -7,7 +7,6 @@ pub struct AppConfig { pub nats_addr: String, pub data_persistence_service_url: String, pub report_generator_service_url: String, - pub provider_services: Vec, } impl AppConfig { @@ -22,34 +21,11 @@ impl AppConfig { let report_generator_service_url: String = cfg.get::("report_generator_service_url") .unwrap_or_else(|_| "http://report-generator-service:8004".to_string()); - // Parse provider_services deterministically: - // 1) prefer array from env (e.g., PROVIDER_SERVICES__0, PROVIDER_SERVICES__1, ...) - // 2) fallback to explicit JSON in PROVIDER_SERVICES - let provider_services: Vec = if let Ok(arr) = cfg.get_array("provider_services") { - let mut out: Vec = Vec::with_capacity(arr.len()); - for v in arr { - let s = v.into_string().map_err(|e| { - config::ConfigError::Message(format!("provider_services must be strings: {}", e)) - })?; - out.push(s); - } - out - } else { - let json = cfg.get_string("provider_services")?; - serde_json::from_str::>(&json).map_err(|e| { - config::ConfigError::Message(format!( - "Invalid JSON for provider_services: {}", - e - )) - })? - }; - Ok(Self { server_port, nats_addr, data_persistence_service_url, report_generator_service_url, - provider_services, }) } } diff --git a/services/api-gateway/src/error.rs b/services/api-gateway/src/error.rs index d14b025..8926a1b 100644 --- a/services/api-gateway/src/error.rs +++ b/services/api-gateway/src/error.rs @@ -23,9 +23,12 @@ pub enum AppError { #[error("HTTP request to another service failed: {0}")] ServiceRequest(#[from] reqwest::Error), + + #[error("Bad request: {0}")] + BadRequest(String), - #[error("An unexpected error occurred.")] - Anyhow(#[from] anyhow::Error), + #[error("Internal error: {0}")] + Internal(#[from] anyhow::Error), } impl IntoResponse for AppError { @@ -37,7 +40,8 @@ impl IntoResponse for AppError { AppError::MessageBusSubscribe(msg) => (StatusCode::SERVICE_UNAVAILABLE, msg.clone()), AppError::MessageBusConnect(msg) => (StatusCode::SERVICE_UNAVAILABLE, msg.clone()), AppError::ServiceRequest(err) => (StatusCode::BAD_GATEWAY, err.to_string()), - AppError::Anyhow(err) => (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), + AppError::BadRequest(msg) => (StatusCode::BAD_REQUEST, msg.clone()), + AppError::Internal(err) => (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()), }; let body = Json(json!({ "error": message })); (status, body).into_response() diff --git a/services/api-gateway/src/main.rs b/services/api-gateway/src/main.rs index 6894220..e5df622 100644 --- a/services/api-gateway/src/main.rs +++ b/services/api-gateway/src/main.rs @@ -29,11 +29,10 @@ async fn main() { eprintln!("api-gateway launching: pid={}, ts_unix={}", process::id(), ts); // Print critical environment variables relevant to configuration (no secrets) eprintln!( - "env: SERVER_PORT={:?}, NATS_ADDR={:?}, DATA_PERSISTENCE_SERVICE_URL={:?}, PROVIDER_SERVICES.len={}", + "env: SERVER_PORT={:?}, NATS_ADDR={:?}, DATA_PERSISTENCE_SERVICE_URL={:?}", std::env::var("SERVER_PORT").ok(), std::env::var("NATS_ADDR").ok(), std::env::var("DATA_PERSISTENCE_SERVICE_URL").ok(), - std::env::var("PROVIDER_SERVICES").ok().map(|s| s.len()).unwrap_or(0), ); let _ = io::stderr().flush(); @@ -67,7 +66,7 @@ async fn run() -> Result<()> { persistence_url = %config.data_persistence_service_url, "Loaded configuration" ); - info!("Configured provider services: {:?}", config.provider_services); + // info!("Configured provider services: {:?}", config.provider_services); // Removed in favor of dynamic registry // Initialize application state let app_state = AppState::new(config).await?; @@ -83,13 +82,13 @@ async fn run() -> Result<()> { Ok(l) => l, Err(e) => { error!(%addr, err = %e, "Failed to bind TCP listener"); - return Err(error::AppError::Anyhow(anyhow::anyhow!(e))); + return Err(error::AppError::Internal(anyhow::anyhow!(e))); } }; info!("HTTP server listening on port {}", port); if let Err(e) = axum::serve(listener, app).await { error!(err = %e, "HTTP server terminated with error"); - return Err(error::AppError::Anyhow(anyhow::anyhow!(e))); + return Err(error::AppError::Internal(anyhow::anyhow!(e))); } Ok(()) diff --git a/services/api-gateway/src/persistence.rs b/services/api-gateway/src/persistence.rs index 26bf202..6911b0c 100644 --- a/services/api-gateway/src/persistence.rs +++ b/services/api-gateway/src/persistence.rs @@ -46,6 +46,19 @@ impl PersistenceClient { Ok(financials) } + pub async fn get_session_data(&self, request_id: uuid::Uuid) -> Result> { + let url = format!("{}/session-data/{}", self.base_url, request_id); + let data = self + .client + .get(&url) + .send() + .await? + .error_for_status()? + .json::>() + .await?; + Ok(data) + } + pub async fn get_analysis_results(&self, symbol: &str) -> Result> { let url = format!("{}/analysis-results?symbol={}", self.base_url, symbol); let results = self diff --git a/services/api-gateway/src/state.rs b/services/api-gateway/src/state.rs index 7790edb..0a3f445 100644 --- a/services/api-gateway/src/state.rs +++ b/services/api-gateway/src/state.rs @@ -1,16 +1,26 @@ use crate::config::AppConfig; use crate::error::Result; use crate::persistence::PersistenceClient; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use async_nats::Client as NatsClient; use tokio::time::{sleep, Duration}; use tracing::{info, warn}; +use std::collections::HashMap; +use std::time::Instant; +use common_contracts::registry::{ServiceRegistration, ServiceRole}; + +#[derive(Clone, Debug)] +pub struct RegistryEntry { + pub registration: ServiceRegistration, + pub last_heartbeat: Instant, +} #[derive(Clone)] pub struct AppState { pub config: Arc, pub nats_client: NatsClient, pub persistence_client: PersistenceClient, + pub registry: Arc>>, } impl AppState { @@ -19,13 +29,43 @@ impl AppState { let persistence_client = PersistenceClient::new(config.data_persistence_service_url.clone()); - + Ok(Self { config: Arc::new(config), nats_client, persistence_client, + registry: Arc::new(RwLock::new(HashMap::new())), }) } + + /// Finds a healthy service instance by name (e.g., "tushare"). + /// Returns the base_url. + pub fn get_service_url(&self, service_name: &str) -> Option { + let registry = self.registry.read().unwrap(); + // TODO: Implement Round-Robin or check Last Heartbeat for health? + // For now, return the first match. + registry.values() + .find(|entry| entry.registration.service_name == service_name) + .map(|entry| entry.registration.base_url.clone()) + } + + /// Returns all registered services as (service_id, base_url) tuples. + pub fn get_all_services(&self) -> Vec<(String, String)> { + let registry = self.registry.read().unwrap(); + registry.values() + .map(|entry| (entry.registration.service_id.clone(), entry.registration.base_url.clone())) + .collect() + } + + pub fn get_provider_count(&self) -> usize { + let registry = self.registry.read().unwrap(); + registry.values() + .filter(|entry| { + // Strict type checking using ServiceRole + entry.registration.role == ServiceRole::DataProvider + }) + .count() + } } const MAX_NATS_CONNECT_ATTEMPTS: usize = 30; diff --git a/services/common-contracts/Cargo.lock b/services/common-contracts/Cargo.lock index b2fa315..96fbeec 100644 --- a/services/common-contracts/Cargo.lock +++ b/services/common-contracts/Cargo.lock @@ -37,6 +37,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anyhow" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + [[package]] name = "arrayvec" version = "0.7.6" @@ -52,6 +58,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" @@ -248,12 +260,17 @@ dependencies = [ name = "common-contracts" version = "0.1.0" dependencies = [ + "anyhow", "chrono", + "log", + "reqwest", "rust_decimal", "serde", "serde_json", "service_kit", "sqlx", + "tokio", + "tracing", "utoipa", "uuid", ] @@ -273,6 +290,16 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -383,12 +410,31 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -411,6 +457,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "find-msvc-tools" version = "0.1.5" @@ -440,6 +492,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -560,6 +627,25 @@ dependencies = [ "wasip2", ] +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -668,6 +754,92 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + [[package]] name = "iana-time-zone" version = "0.1.64" @@ -815,6 +987,22 @@ dependencies = [ "rustversion", ] +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "itoa" version = "1.0.15" @@ -873,6 +1061,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "litemap" version = "0.8.1" @@ -933,6 +1127,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "num-bigint-dig" version = "0.8.6" @@ -985,6 +1196,50 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking" version = "2.2.1" @@ -1242,6 +1497,46 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "reqwest" +version = "0.12.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -1321,6 +1616,19 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.35" @@ -1367,6 +1675,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "1.1.0" @@ -1405,6 +1722,29 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.228" @@ -1866,6 +2206,9 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -1878,12 +2221,46 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tap" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "2.0.17" @@ -1940,9 +2317,41 @@ dependencies = [ "mio", "pin-project-lite", "socket2", + "tokio-macros", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.17" @@ -1954,6 +2363,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-util" +version = "0.7.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.9.8" @@ -2015,6 +2437,25 @@ dependencies = [ "futures-util", "pin-project-lite", "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", "tower-layer", "tower-service", ] @@ -2063,6 +2504,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.19.0" @@ -2169,6 +2616,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2203,6 +2659,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.105" @@ -2235,6 +2704,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -2304,6 +2783,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" diff --git a/services/common-contracts/Cargo.toml b/services/common-contracts/Cargo.toml index 860639a..173d894 100644 --- a/services/common-contracts/Cargo.toml +++ b/services/common-contracts/Cargo.toml @@ -18,5 +18,8 @@ rust_decimal = { version = "1.36", features = ["serde"] } utoipa = { version = "5.4", features = ["chrono", "uuid"] } sqlx = { version = "0.8.6", features = [ "runtime-tokio-rustls", "postgres", "chrono", "uuid", "json", "rust_decimal" ] } service_kit = { version = "0.1.2" } - - +reqwest = { version = "0.12", features = ["json"] } +tokio = { version = "1", features = ["time", "sync", "macros"] } +log = "0.4" +tracing = "0.1" +anyhow = "1.0" diff --git a/services/common-contracts/src/dtos.rs b/services/common-contracts/src/dtos.rs index 9bc81eb..03e47de 100644 --- a/services/common-contracts/src/dtos.rs +++ b/services/common-contracts/src/dtos.rs @@ -91,4 +91,38 @@ pub struct RealtimeQuoteDto { pub source: Option, } +use crate::observability::TaskStatus; +#[api_dto] +pub struct ProviderStatusDto { + pub last_updated: chrono::DateTime, + pub status: TaskStatus, + pub data_version: Option, +} + +// Provider Path Params +#[api_dto] +#[derive(utoipa::IntoParams)] +pub struct ProviderPathParams { + pub symbol: String, + pub provider_id: String, +} + +// Session Data & Cache DTOs +#[api_dto] +pub struct SessionDataDto { + pub request_id: Uuid, + pub symbol: String, + pub provider: String, + pub data_type: String, + pub data_payload: JsonValue, + pub created_at: Option>, +} + +#[api_dto] +pub struct ProviderCacheDto { + pub cache_key: String, + pub data_payload: JsonValue, + pub expires_at: chrono::DateTime, + pub updated_at: Option>, +} diff --git a/services/common-contracts/src/lib.rs b/services/common-contracts/src/lib.rs index e9050a4..457335b 100644 --- a/services/common-contracts/src/lib.rs +++ b/services/common-contracts/src/lib.rs @@ -3,6 +3,9 @@ pub mod models; pub mod observability; pub mod messages; pub mod config_models; +pub mod subjects; pub mod provider; - - +pub mod registry; +pub mod lifecycle; +pub mod symbol_utils; +pub mod persistence_client; diff --git a/services/common-contracts/src/lifecycle.rs b/services/common-contracts/src/lifecycle.rs new file mode 100644 index 0000000..c675059 --- /dev/null +++ b/services/common-contracts/src/lifecycle.rs @@ -0,0 +1,138 @@ +use crate::registry::{ServiceRegistration, Heartbeat, ServiceStatus}; +use reqwest::Client; +use std::time::Duration; +use tokio::time; +use log::{info, warn, error}; +use std::sync::Arc; + +pub struct ServiceRegistrar { + gateway_url: String, + registration: ServiceRegistration, + client: Client, +} + +impl ServiceRegistrar { + pub fn new(gateway_url: String, registration: ServiceRegistration) -> Self { + Self { + gateway_url, + registration, + client: Client::new(), + } + } + + /// Registers the service with the gateway. + /// It will attempt to register multiple times before giving up or returning. + /// In a real production scenario, you might want this to block until success + /// or allow the application to start and register in the background. + pub async fn register(&self) -> Result<(), reqwest::Error> { + let url = format!("{}/v1/registry/register", self.gateway_url); + let mut attempt = 0; + let max_retries = 5; + let mut delay = Duration::from_secs(2); + + loop { + attempt += 1; + info!("Registering service (attempt {}/{})...", attempt, max_retries); + + match self.client.post(&url) + .json(&self.registration) + .send() + .await { + Ok(resp) => { + if resp.status().is_success() { + info!("Successfully registered service: {}", self.registration.service_id); + return Ok(()); + } else { + warn!("Registration failed with status: {}. Attempt: {}", resp.status(), attempt); + } + }, + Err(e) => { + warn!("Registration request failed: {}. Attempt: {}", e, attempt); + } + } + + if attempt >= max_retries { + // We stop blocking here. The heartbeat loop will try to recover registration if needed. + warn!("Max registration retries reached. Continuing, but service might not be discoverable yet."); + // We return Ok to allow the service to start up. + // Returning an error might cause the whole pod to crash loop which might be desired or not. + // Given the self-healing design, we can proceed. + return Ok(()); + } + + time::sleep(delay).await; + delay = std::cmp::min(delay * 2, Duration::from_secs(30)); + } + } + + /// Helper to register a single time without retries (used by recovery mechanism) + async fn register_once(&self) -> Result<(), reqwest::Error> { + let url = format!("{}/v1/registry/register", self.gateway_url); + let resp = self.client.post(&url) + .json(&self.registration) + .send() + .await?; + + if !resp.status().is_success() { + return Err(resp.error_for_status().unwrap_err()); + } + Ok(()) + } + + /// Starts the background heartbeat loop. + /// Requires `Arc` because it will be spawned into a static task. + pub async fn start_heartbeat_loop(self: Arc) { + let mut interval = time::interval(Duration::from_secs(10)); + let heartbeat_url = format!("{}/v1/registry/heartbeat", self.gateway_url); + + info!("Starting heartbeat loop for service: {}", self.registration.service_id); + + loop { + interval.tick().await; + + let heartbeat = Heartbeat { + service_id: self.registration.service_id.clone(), + status: ServiceStatus::Active, + }; + + match self.client.post(&heartbeat_url) + .json(&heartbeat) + .send() + .await { + Ok(resp) => { + // If the Gateway says "I don't know you" (404) or Unauthorized, we re-register. + if resp.status() == reqwest::StatusCode::NOT_FOUND || resp.status() == reqwest::StatusCode::UNAUTHORIZED { + warn!("Gateway returned {}, indicating registration loss. Re-registering...", resp.status()); + if let Err(e) = self.register_once().await { + error!("Re-registration failed: {}", e); + } else { + info!("Re-registration successful."); + } + } else if !resp.status().is_success() { + warn!("Heartbeat failed with status: {}", resp.status()); + } + }, + Err(e) => { + error!("Heartbeat request failed: {}", e); + } + } + } + } + + pub async fn deregister(&self) -> Result<(), reqwest::Error> { + let url = format!("{}/v1/registry/deregister", self.gateway_url); + info!("Deregistering service: {}", self.registration.service_id); + + let payload = serde_json::json!({ + "service_id": self.registration.service_id + }); + + let _ = self.client.post(&url) + .json(&payload) + .send() + .await?; + + Ok(()) + } +} + diff --git a/services/common-contracts/src/messages.rs b/services/common-contracts/src/messages.rs index ccc5b1f..97e1f94 100644 --- a/services/common-contracts/src/messages.rs +++ b/services/common-contracts/src/messages.rs @@ -1,42 +1,239 @@ use serde::{Serialize, Deserialize}; use uuid::Uuid; +use crate::symbol_utils::CanonicalSymbol; +use crate::subjects::{NatsSubject, SubjectMessage}; +use std::collections::HashMap; // --- Commands --- -/// + +// Topic: workflow.commands.start +/// Command to initiate a new workflow. /// Published by: `api-gateway` +/// Consumed by: `workflow-orchestrator` +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct StartWorkflowCommand { + pub request_id: Uuid, + pub symbol: CanonicalSymbol, + pub market: String, + pub template_id: String, +} + +impl SubjectMessage for StartWorkflowCommand { + fn subject(&self) -> NatsSubject { + NatsSubject::WorkflowCommandStart + } +} + +// Topic: workflow.commands.sync_state +/// Command to request a state snapshot for re-alignment. +/// Published by: `api-gateway` (on client connect/reconnect) +/// Consumed by: `workflow-orchestrator` +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SyncStateCommand { + pub request_id: Uuid, +} + +impl SubjectMessage for SyncStateCommand { + fn subject(&self) -> NatsSubject { + NatsSubject::WorkflowCommandSyncState + } +} + +/// Command to trigger data fetching. +/// Published by: `workflow-orchestrator` (previously api-gateway) /// Consumed by: `*-provider-services` #[derive(Clone, Debug, Serialize, Deserialize)] pub struct FetchCompanyDataCommand { pub request_id: Uuid, - pub symbol: String, + pub symbol: CanonicalSymbol, pub market: String, pub template_id: Option, // Optional trigger for analysis } +impl SubjectMessage for FetchCompanyDataCommand { + fn subject(&self) -> NatsSubject { + NatsSubject::DataFetchCommands + } +} + /// Command to start a full report generation workflow. -/// -/// Published by: `api-gateway` +/// Published by: `workflow-orchestrator` (previously api-gateway) /// Consumed by: `report-generator-service` #[derive(Clone, Debug, Serialize, Deserialize)] pub struct GenerateReportCommand { pub request_id: Uuid, - pub symbol: String, + pub symbol: CanonicalSymbol, pub template_id: String, } +impl SubjectMessage for GenerateReportCommand { + fn subject(&self) -> NatsSubject { + NatsSubject::AnalysisCommandGenerateReport + } +} + // --- Events --- + +// Topic: events.workflow.{request_id} +/// Unified event stream for frontend consumption. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type", content = "payload")] +pub enum WorkflowEvent { + // 1. 流程初始化 (携带完整的任务依赖图) + WorkflowStarted { + timestamp: i64, + // 定义所有任务及其依赖关系，前端可据此绘制流程图或进度条 + task_graph: WorkflowDag + }, + + // 2. 任务状态变更 (核心事件) + TaskStateChanged { + task_id: String, // e.g., "fetch:tushare", "process:clean_financials", "module:swot_analysis" + task_type: TaskType, // DataFetch | DataProcessing | Analysis + status: TaskStatus, // Pending, Scheduled, Running, Completed, Failed, Skipped + message: Option, + timestamp: i64 + }, + + // 3. 任务流式输出 (用于 LLM 打字机效果) + TaskStreamUpdate { + task_id: String, + content_delta: String, + index: u32 + }, + + // 4. 流程整体结束 + WorkflowCompleted { + result_summary: serde_json::Value, + end_timestamp: i64 + }, + + WorkflowFailed { + reason: String, + is_fatal: bool, + end_timestamp: i64 + }, + + // 5. 状态快照 (用于重连/丢包恢复) + // 当前端重连或显式发送 SyncStateCommand 时，Orchestrator 发送此事件 + WorkflowStateSnapshot { + timestamp: i64, + task_graph: WorkflowDag, + tasks_status: HashMap, // 当前所有任务的最新状态 + tasks_output: HashMap> // (可选) 已完成任务的关键输出摘要 + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct WorkflowDag { + pub nodes: Vec, + pub edges: Vec // from -> to +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TaskDependency { + pub from: String, + pub to: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct TaskNode { + pub id: String, + pub name: String, + pub r#type: TaskType, + pub initial_status: TaskStatus +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] +pub enum TaskType { + DataFetch, // 创造原始上下文 + DataProcessing, // 消耗并转换上下文 (New) + Analysis // 读取上下文生成新内容 +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] +pub enum TaskStatus { + Pending, // 等待依赖 + Scheduled, // 依赖满足，已下发给 Worker + Running, // Worker 正在执行 + Completed, // 执行成功 + Failed, // 执行失败 + Skipped // 因上游失败或策略原因被跳过 +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct CompanyProfilePersistedEvent { pub request_id: Uuid, - pub symbol: String, + pub symbol: CanonicalSymbol, } #[derive(Serialize, Deserialize, Debug, Clone)] pub struct FinancialsPersistedEvent { pub request_id: Uuid, - pub symbol: String, + pub symbol: CanonicalSymbol, pub years_updated: Vec, pub template_id: Option, // Pass-through for analysis trigger + // Identity fix: Mandatory provider ID + #[serde(default)] + pub provider_id: Option, + // Output pass-through: Optional data preview/summary + #[serde(default)] + pub data_summary: Option, +} + +impl SubjectMessage for FinancialsPersistedEvent { + fn subject(&self) -> NatsSubject { + NatsSubject::DataFinancialsPersisted + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct DataFetchFailedEvent { + pub request_id: Uuid, + pub symbol: CanonicalSymbol, + pub error: String, + // Identity fix: Mandatory provider ID + #[serde(default)] + pub provider_id: Option, +} + +impl SubjectMessage for DataFetchFailedEvent { + fn subject(&self) -> NatsSubject { + NatsSubject::DataFetchFailed + } +} + +// Topic: events.analysis.report_generated +/// Event emitted when a report generation task (or sub-module) is completed. +/// Consumed by: `workflow-orchestrator` +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ReportGeneratedEvent { + pub request_id: Uuid, + pub symbol: CanonicalSymbol, + pub module_id: String, // Which part of the analysis finished + pub content_snapshot: Option, // Optional short preview + pub model_id: Option, +} + +impl SubjectMessage for ReportGeneratedEvent { + fn subject(&self) -> NatsSubject { + NatsSubject::AnalysisReportGenerated + } +} + +// Topic: events.analysis.report_failed +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ReportFailedEvent { + pub request_id: Uuid, + pub symbol: CanonicalSymbol, + pub module_id: String, + pub error: String, +} + +impl SubjectMessage for ReportFailedEvent { + fn subject(&self) -> NatsSubject { + NatsSubject::AnalysisReportFailed + } } diff --git a/services/common-contracts/src/observability.rs b/services/common-contracts/src/observability.rs index a0deea5..5a4a31f 100644 --- a/services/common-contracts/src/observability.rs +++ b/services/common-contracts/src/observability.rs @@ -18,11 +18,20 @@ pub struct HealthStatus { pub details: HashMap, } +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum TaskStatus { + Queued, + InProgress, + Completed, + Failed, +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct TaskProgress { pub request_id: Uuid, pub task_name: String, - pub status: String, + pub status: TaskStatus, pub progress_percent: u8, pub details: String, pub started_at: DateTime, diff --git a/services/common-contracts/src/persistence_client.rs b/services/common-contracts/src/persistence_client.rs new file mode 100644 index 0000000..99600e9 --- /dev/null +++ b/services/common-contracts/src/persistence_client.rs @@ -0,0 +1,156 @@ +use crate::dtos::{ + SessionDataDto, ProviderCacheDto, CompanyProfileDto, + TimeSeriesFinancialBatchDto, TimeSeriesFinancialDto, ProviderStatusDto +}; +use crate::config_models::{ + DataSourcesConfig, LlmProvidersConfig, AnalysisTemplateSets +}; +use reqwest::{Client, StatusCode}; +use uuid::Uuid; +use anyhow::Result; + +#[derive(Clone)] +pub struct PersistenceClient { + client: Client, + base_url: String, +} + +impl PersistenceClient { + pub fn new(base_url: String) -> Self { + Self { + client: Client::new(), + base_url: base_url.trim_end_matches('/').to_string(), + } + } + + // --- Session Data --- + + pub async fn insert_session_data(&self, dto: &SessionDataDto) -> Result<()> { + let url = format!("{}/session-data", self.base_url); + self.client + .post(&url) + .json(dto) + .send() + .await? + .error_for_status()?; + Ok(()) + } + + pub async fn get_session_data(&self, request_id: Uuid) -> Result> { + let url = format!("{}/session-data/{}", self.base_url, request_id); + let resp = self.client.get(&url).send().await?.error_for_status()?; + let data = resp.json().await?; + Ok(data) + } + + // --- Provider Cache --- + + pub async fn get_cache(&self, key: &str) -> Result> { + let url = format!("{}/provider-cache", self.base_url); + let resp = self.client + .get(&url) + .query(&[("key", key)]) + .send() + .await? + .error_for_status()?; + + let data = resp.json().await?; + Ok(data) + } + + pub async fn set_cache(&self, dto: &ProviderCacheDto) -> Result<()> { + let url = format!("{}/provider-cache", self.base_url); + self.client + .post(&url) + .json(dto) + .send() + .await? + .error_for_status()?; + Ok(()) + } + + // --- Existing Methods (Ported for completeness) --- + + pub async fn get_company_profile(&self, symbol: &str) -> Result> { + let url = format!("{}/companies/{}", self.base_url, symbol); + let resp = self.client.get(&url).send().await?; + if resp.status() == StatusCode::NOT_FOUND { + return Ok(None); + } + let profile = resp.error_for_status()?.json().await?; + Ok(Some(profile)) + } + + pub async fn batch_insert_financials(&self, dtos: Vec) -> Result<()> { + if dtos.is_empty() { + return Ok(()); + } + let url = format!("{}/market-data/financials/batch", self.base_url); + let batch = TimeSeriesFinancialBatchDto { records: dtos }; + + self.client + .post(&url) + .json(&batch) + .send() + .await? + .error_for_status()?; + Ok(()) + } + + // --- Configs --- + + pub async fn get_data_sources_config(&self) -> Result { + let url = format!("{}/configs/data_sources", self.base_url); + let resp = self.client.get(&url).send().await?.error_for_status()?; + let config = resp.json().await?; + Ok(config) + } + + pub async fn update_data_sources_config(&self, config: &DataSourcesConfig) -> Result { + let url = format!("{}/configs/data_sources", self.base_url); + let resp = self.client.put(&url).json(config).send().await?.error_for_status()?; + let updated = resp.json().await?; + Ok(updated) + } + + pub async fn get_llm_providers_config(&self) -> Result { + let url = format!("{}/configs/llm_providers", self.base_url); + let resp = self.client.get(&url).send().await?.error_for_status()?; + let config = resp.json().await?; + Ok(config) + } + + pub async fn update_llm_providers_config(&self, config: &LlmProvidersConfig) -> Result { + let url = format!("{}/configs/llm_providers", self.base_url); + let resp = self.client.put(&url).json(config).send().await?.error_for_status()?; + let updated = resp.json().await?; + Ok(updated) + } + + pub async fn get_analysis_template_sets(&self) -> Result { + let url = format!("{}/configs/analysis_template_sets", self.base_url); + let resp = self.client.get(&url).send().await?.error_for_status()?; + let config = resp.json().await?; + Ok(config) + } + + pub async fn update_analysis_template_sets(&self, config: &AnalysisTemplateSets) -> Result { + let url = format!("{}/configs/analysis_template_sets", self.base_url); + let resp = self.client.put(&url).json(config).send().await?.error_for_status()?; + let updated = resp.json().await?; + Ok(updated) + } + + // --- Deprecated/Legacy Support --- + + pub async fn update_provider_status(&self, symbol: &str, provider_id: &str, status: ProviderStatusDto) -> Result<()> { + let url = format!("{}/companies/{}/providers/{}/status", self.base_url, symbol, provider_id); + self.client.put(&url).json(&status).send().await?.error_for_status()?; + Ok(()) + } + + pub async fn upsert_company_profile(&self, _profile: CompanyProfileDto) -> Result<()> { + // Deprecated: No-op + Ok(()) + } +} diff --git a/services/common-contracts/src/registry.rs b/services/common-contracts/src/registry.rs new file mode 100644 index 0000000..79fdf8d --- /dev/null +++ b/services/common-contracts/src/registry.rs @@ -0,0 +1,40 @@ +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema, PartialEq)] +pub enum ServiceStatus { + Active, + Degraded, + Maintenance, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ServiceRole { + DataProvider, + ReportGenerator, + Persistence, + Gateway, + Other, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct ServiceRegistration { + /// Unique ID for this service instance (e.g., "tushare-provider-uuid") + pub service_id: String, + /// Service type/name (e.g., "tushare") + pub service_name: String, + /// The role/category of the service + pub role: ServiceRole, + /// Base URL for the service (e.g., "http://10.0.1.5:8000") + pub base_url: String, + /// Health check endpoint + pub health_check_url: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct Heartbeat { + pub service_id: String, + pub status: ServiceStatus, +} + diff --git a/services/common-contracts/src/subjects.rs b/services/common-contracts/src/subjects.rs new file mode 100644 index 0000000..a74ae48 --- /dev/null +++ b/services/common-contracts/src/subjects.rs @@ -0,0 +1,142 @@ +use std::fmt; +use std::str::FromStr; +use uuid::Uuid; +use serde::{Serialize, de::DeserializeOwned}; + +/// Trait for messages that know their own NATS subject. +pub trait SubjectMessage: Serialize + DeserializeOwned + Send + Sync { + fn subject(&self) -> NatsSubject; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NatsSubject { + // --- Commands --- + WorkflowCommandStart, + WorkflowCommandSyncState, + DataFetchCommands, + AnalysisCommandGenerateReport, + + // --- Events --- + // Data Events + DataFinancialsPersisted, + DataFetchFailed, + + // Analysis Events + AnalysisReportGenerated, + AnalysisReportFailed, + + // Workflow Events (Dynamic) + WorkflowProgress(Uuid), + + // --- Wildcards (For Subscription) --- + AnalysisEventsWildcard, + WorkflowCommandsWildcard, + DataEventsWildcard, +} + +impl fmt::Display for NatsSubject { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WorkflowCommandStart => write!(f, "workflow.commands.start"), + Self::WorkflowCommandSyncState => write!(f, "workflow.commands.sync_state"), + Self::DataFetchCommands => write!(f, "data_fetch_commands"), + Self::AnalysisCommandGenerateReport => write!(f, "analysis.commands.generate_report"), + Self::DataFinancialsPersisted => write!(f, "events.data.financials_persisted"), + Self::DataFetchFailed => write!(f, "events.data.fetch_failed"), + Self::AnalysisReportGenerated => write!(f, "events.analysis.report_generated"), + Self::AnalysisReportFailed => write!(f, "events.analysis.report_failed"), + Self::WorkflowProgress(id) => write!(f, "events.workflow.{}", id), + Self::AnalysisEventsWildcard => write!(f, "events.analysis.>"), + Self::WorkflowCommandsWildcard => write!(f, "workflow.commands.>"), + Self::DataEventsWildcard => write!(f, "events.data.>"), + } + } +} + +impl FromStr for NatsSubject { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "workflow.commands.start" => Ok(Self::WorkflowCommandStart), + "workflow.commands.sync_state" => Ok(Self::WorkflowCommandSyncState), + "data_fetch_commands" => Ok(Self::DataFetchCommands), + "analysis.commands.generate_report" => Ok(Self::AnalysisCommandGenerateReport), + "events.data.financials_persisted" => Ok(Self::DataFinancialsPersisted), + "events.data.fetch_failed" => Ok(Self::DataFetchFailed), + "events.analysis.report_generated" => Ok(Self::AnalysisReportGenerated), + "events.analysis.report_failed" => Ok(Self::AnalysisReportFailed), + "events.analysis.>" => Ok(Self::AnalysisEventsWildcard), + "workflow.commands.>" => Ok(Self::WorkflowCommandsWildcard), + "events.data.>" => Ok(Self::DataEventsWildcard), + _ => { + if s.starts_with("events.workflow.") { + let uuid_str = s.trim_start_matches("events.workflow."); + if let Ok(uuid) = Uuid::parse_str(uuid_str) { + return Ok(Self::WorkflowProgress(uuid)); + } + } + Err(format!("Unknown or invalid subject: {}", s)) + } + } + } +} + +// Implement TryFrom for convenience +impl TryFrom<&str> for NatsSubject { + type Error = String; + + fn try_from(value: &str) -> Result { + Self::from_str(value) + } +} + +impl TryFrom for NatsSubject { + type Error = String; + + fn try_from(value: String) -> Result { + Self::from_str(&value) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_subject_round_trip() { + let subjects = vec![ + (NatsSubject::WorkflowCommandStart, "workflow.commands.start"), + (NatsSubject::WorkflowCommandSyncState, "workflow.commands.sync_state"), + (NatsSubject::DataFetchCommands, "data_fetch_commands"), + (NatsSubject::AnalysisCommandGenerateReport, "analysis.commands.generate_report"), + (NatsSubject::DataFinancialsPersisted, "events.data.financials_persisted"), + (NatsSubject::DataFetchFailed, "events.data.fetch_failed"), + (NatsSubject::AnalysisReportGenerated, "events.analysis.report_generated"), + (NatsSubject::AnalysisReportFailed, "events.analysis.report_failed"), + (NatsSubject::AnalysisEventsWildcard, "events.analysis.>"), + ]; + + for (subject, string_val) in subjects { + assert_eq!(subject.to_string(), string_val); + assert_eq!(NatsSubject::from_str(string_val).unwrap(), subject); + } + } + + #[test] + fn test_dynamic_subject() { + let id = Uuid::new_v4(); + let subject = NatsSubject::WorkflowProgress(id); + let expected = format!("events.workflow.{}", id); + + assert_eq!(subject.to_string(), expected); + assert_eq!(NatsSubject::from_str(&expected).unwrap(), subject); + } + + #[test] + fn test_invalid_subject() { + assert!(NatsSubject::from_str("invalid.subject").is_err()); + assert!(NatsSubject::from_str("events.workflow.invalid-uuid").is_err()); + } +} + diff --git a/services/common-contracts/src/symbol_utils.rs b/services/common-contracts/src/symbol_utils.rs new file mode 100644 index 0000000..a5c51b9 --- /dev/null +++ b/services/common-contracts/src/symbol_utils.rs @@ -0,0 +1,180 @@ +use serde::{Deserialize, Serialize}; +use std::fmt; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum Market { + US, + CN, + HK, + Crypto, + Unknown, +} + +impl From<&str> for Market { + fn from(s: &str) -> Self { + match s.to_uppercase().as_str() { + "US" | "USA" => Market::US, + "CN" | "CHINA" => Market::CN, + "HK" | "HKG" => Market::HK, + "CRYPTO" => Market::Crypto, + _ => Market::Unknown, + } + } +} + +/// CanonicalSymbol 是系统内部唯一的股票代码标识符类型 +/// 它封装了一个标准化的字符串（遵循 Yahoo Finance 格式） +/// 使用 newtype 模式防止与普通 String 混淆 +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(transparent)] +pub struct CanonicalSymbol(String); + +impl CanonicalSymbol { + /// 创建一个新的 CanonicalSymbol，同时执行归一化逻辑 + /// 这是从外部输入进入系统内部类型的唯一（或主要）入口 + pub fn new(input: &str, market: &Market) -> Self { + let normalized = normalize_string(input, market); + Self(normalized) + } + + /// 仅用于测试或从受信任的数据库源恢复 + pub fn from_valid(symbol: String) -> Self { + Self(symbol) + } + + /// 获取内部字符串引用 + pub fn as_str(&self) -> &str { + &self.0 + } + + // --- Provider Specific Conversions --- + + /// 转换为 Tushare 专用格式 + pub fn to_tushare(&self) -> String { + // 600519.SS -> 600519.SH + if self.0.ends_with(".SS") { + self.0.replace(".SS", ".SH") + } else { + self.0.to_string() + } + } + + /// 转换为 Yahoo 专用格式 (即自身) + pub fn to_yahoo(&self) -> String { + self.0.to_string() + } + + /// 转换为 AlphaVantage 专用格式 + /// AV 对于 A 股通常使用 .SH/.SZ，与 Tushare 类似 + pub fn to_alphavantage(&self) -> String { + if self.0.ends_with(".SS") { + self.0.replace(".SS", ".SH") + } else { + self.0.to_string() + } + } +} + +impl fmt::Display for CanonicalSymbol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +// 方便转回 String +impl From for String { + fn from(val: CanonicalSymbol) -> Self { + val.0 + } +} + +/// 内部归一化逻辑 +fn normalize_string(input: &str, market: &Market) -> String { + let input = input.trim().to_uppercase(); + + // 如果已经包含后缀，暂时假设用户知道自己在做什么，或者进行简单的修正 + if input.contains('.') { + // 处理 Tushare 特有的 .SH -> .SS 转换 (为了统一到 Yahoo 标准) + if input.ends_with(".SH") { + return input.replace(".SH", ".SS"); + } + return input; + } + + match market { + Market::CN => { + // A股规则推断 + if input.starts_with('6') || input.starts_with('9') { + format!("{}.SS", input) + } else if input.starts_with('0') || input.starts_with('3') { + format!("{}.SZ", input) + } else if input.starts_with('4') || input.starts_with('8') { + format!("{}.BJ", input) + } else { + format!("{}.SS", input) + } + }, + Market::HK => { + // 港股补零 + let num_part = input.parse::().unwrap_or(0); + format!("{:04}.HK", num_part) + }, + Market::US => { + input + }, + Market::Crypto => { + if !input.ends_with("-USD") { + format!("{}-USD", input) + } else { + input + } + }, + Market::Unknown => input, + } +} + +// 保留原有函数名作为便捷入口，或者标记为 deprecated +// 为了兼容性，这里直接让他返回 CanonicalSymbol，迫使调用方修改类型 +pub fn normalize_symbol(input: &str, market: &Market) -> CanonicalSymbol { + CanonicalSymbol::new(input, market) +} + +// 下面这些函数现在只是 helper，如果调用方还在用，需要保留并适配 +// 推荐调用方直接使用 CanonicalSymbol 的方法 +pub fn to_tushare_symbol(canonical_symbol: &str) -> String { + CanonicalSymbol(canonical_symbol.to_string()).to_tushare() +} + +pub fn to_yahoo_symbol(canonical_symbol: &str) -> String { + canonical_symbol.to_string() +} + +pub fn to_alphavantage_symbol(canonical_symbol: &str) -> String { + CanonicalSymbol(canonical_symbol.to_string()).to_alphavantage() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_cn_normalization() { + assert_eq!(CanonicalSymbol::new("600519", &Market::CN).as_str(), "600519.SS"); + assert_eq!(CanonicalSymbol::new("000001", &Market::CN).as_str(), "000001.SZ"); + assert_eq!(CanonicalSymbol::new("600519.SH", &Market::CN).as_str(), "600519.SS"); + } + + #[test] + fn test_hk_normalization() { + assert_eq!(CanonicalSymbol::new("700", &Market::HK).as_str(), "0700.HK"); + assert_eq!(CanonicalSymbol::new("0700", &Market::HK).as_str(), "0700.HK"); + } + + #[test] + fn test_conversions() { + let s = CanonicalSymbol::new("600519", &Market::CN); // 600519.SS + assert_eq!(s.to_tushare(), "600519.SH"); + assert_eq!(s.to_alphavantage(), "600519.SH"); + assert_eq!(s.to_yahoo(), "600519.SS"); + } +} diff --git a/services/data-persistence-service/.sqlx/query-21a6b3602a199978f87186634866e7bd72a083ebd55985acae1d712434e2ebb6.json b/services/data-persistence-service/.sqlx/query-21a6b3602a199978f87186634866e7bd72a083ebd55985acae1d712434e2ebb6.json deleted file mode 100644 index 817686e..0000000 --- a/services/data-persistence-service/.sqlx/query-21a6b3602a199978f87186634866e7bd72a083ebd55985acae1d712434e2ebb6.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO company_profiles (symbol, name, industry, list_date, additional_info, updated_at)\n VALUES ($1, $2, $3, $4, $5, NOW())\n ON CONFLICT (symbol) DO UPDATE SET\n name = EXCLUDED.name,\n industry = EXCLUDED.industry,\n list_date = EXCLUDED.list_date,\n additional_info = EXCLUDED.additional_info,\n updated_at = NOW()\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Date", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "21a6b3602a199978f87186634866e7bd72a083ebd55985acae1d712434e2ebb6" -} diff --git a/services/data-persistence-service/.sqlx/query-242e6f3319cfa0c19b53c4da80993a1da3cb77f58a3c0dac0260bf3adb4e501f.json b/services/data-persistence-service/.sqlx/query-242e6f3319cfa0c19b53c4da80993a1da3cb77f58a3c0dac0260bf3adb4e501f.json deleted file mode 100644 index f9a97be..0000000 --- a/services/data-persistence-service/.sqlx/query-242e6f3319cfa0c19b53c4da80993a1da3cb77f58a3c0dac0260bf3adb4e501f.json +++ /dev/null @@ -1,95 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT symbol, market, ts, price, open_price, high_price, low_price, prev_close, change, change_percent, volume, source, updated_at\n FROM realtime_quotes\n WHERE symbol = $1 AND market = $2\n ORDER BY ts DESC\n LIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "market", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "ts", - "type_info": "Timestamptz" - }, - { - "ordinal": 3, - "name": "price", - "type_info": "Numeric" - }, - { - "ordinal": 4, - "name": "open_price", - "type_info": "Numeric" - }, - { - "ordinal": 5, - "name": "high_price", - "type_info": "Numeric" - }, - { - "ordinal": 6, - "name": "low_price", - "type_info": "Numeric" - }, - { - "ordinal": 7, - "name": "prev_close", - "type_info": "Numeric" - }, - { - "ordinal": 8, - "name": "change", - "type_info": "Numeric" - }, - { - "ordinal": 9, - "name": "change_percent", - "type_info": "Numeric" - }, - { - "ordinal": 10, - "name": "volume", - "type_info": "Int8" - }, - { - "ordinal": 11, - "name": "source", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "updated_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - true, - true, - true, - true, - true, - true, - true, - true, - false - ] - }, - "hash": "242e6f3319cfa0c19b53c4da80993a1da3cb77f58a3c0dac0260bf3adb4e501f" -} diff --git a/services/data-persistence-service/.sqlx/query-c3d06b1b669d66f82fd532a7bc782621101780f7f549852fc3b4405b477870af.json b/services/data-persistence-service/.sqlx/query-40a2909c95715b8a9141254de3c9c3aa3a5c6427731bc07510e7b5ba759ccc7d.json similarity index 56% rename from services/data-persistence-service/.sqlx/query-c3d06b1b669d66f82fd532a7bc782621101780f7f549852fc3b4405b477870af.json rename to services/data-persistence-service/.sqlx/query-40a2909c95715b8a9141254de3c9c3aa3a5c6427731bc07510e7b5ba759ccc7d.json index 917d2e7..131bc70 100644 --- a/services/data-persistence-service/.sqlx/query-c3d06b1b669d66f82fd532a7bc782621101780f7f549852fc3b4405b477870af.json +++ b/services/data-persistence-service/.sqlx/query-40a2909c95715b8a9141254de3c9c3aa3a5c6427731bc07510e7b5ba759ccc7d.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, symbol, module_id, generated_at, model_name, content, meta_data\n FROM analysis_results\n WHERE id = $1\n ", + "query": "\n SELECT \n request_id, \n symbol, \n provider, \n data_type, \n data_payload, \n created_at\n FROM session_raw_data\n WHERE request_id = $1\n ORDER BY created_at ASC\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "id", + "name": "request_id", "type_info": "Uuid" }, { @@ -15,28 +15,23 @@ }, { "ordinal": 2, - "name": "module_id", + "name": "provider", "type_info": "Varchar" }, { "ordinal": 3, - "name": "generated_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 4, - "name": "model_name", + "name": "data_type", "type_info": "Varchar" }, { - "ordinal": 5, - "name": "content", - "type_info": "Text" + "ordinal": 4, + "name": "data_payload", + "type_info": "Jsonb" }, { - "ordinal": 6, - "name": "meta_data", - "type_info": "Jsonb" + "ordinal": 5, + "name": "created_at", + "type_info": "Timestamptz" } ], "parameters": { @@ -49,10 +44,9 @@ false, false, false, - true, false, - true + false ] }, - "hash": "c3d06b1b669d66f82fd532a7bc782621101780f7f549852fc3b4405b477870af" + "hash": "40a2909c95715b8a9141254de3c9c3aa3a5c6427731bc07510e7b5ba759ccc7d" } diff --git a/services/data-persistence-service/.sqlx/query-4536af5904df2b38a10e801f488cf2bd4176dccf06b0b791284d729f53ab262d.json b/services/data-persistence-service/.sqlx/query-4536af5904df2b38a10e801f488cf2bd4176dccf06b0b791284d729f53ab262d.json deleted file mode 100644 index 390aef1..0000000 --- a/services/data-persistence-service/.sqlx/query-4536af5904df2b38a10e801f488cf2bd4176dccf06b0b791284d729f53ab262d.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT symbol, metric_name, period_date, value, source\n FROM time_series_financials\n WHERE symbol = $1\n ORDER BY period_date DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "metric_name", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "period_date", - "type_info": "Date" - }, - { - "ordinal": 3, - "name": "value", - "type_info": "Numeric" - }, - { - "ordinal": 4, - "name": "source", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - true - ] - }, - "hash": "4536af5904df2b38a10e801f488cf2bd4176dccf06b0b791284d729f53ab262d" -} diff --git a/services/data-persistence-service/.sqlx/query-47dd5646e6a94d84da1db7e7aa5961ce012cf8467e5b98fc88f073f84ddd7b87.json b/services/data-persistence-service/.sqlx/query-47dd5646e6a94d84da1db7e7aa5961ce012cf8467e5b98fc88f073f84ddd7b87.json deleted file mode 100644 index 7fe5a7d..0000000 --- a/services/data-persistence-service/.sqlx/query-47dd5646e6a94d84da1db7e7aa5961ce012cf8467e5b98fc88f073f84ddd7b87.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO analysis_results (symbol, module_id, model_name, content, meta_data)\n VALUES ($1, $2, $3, $4, $5)\n RETURNING id, symbol, module_id, generated_at, model_name, content, meta_data\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "symbol", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "module_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "generated_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 4, - "name": "model_name", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "content", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "meta_data", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Text", - "Jsonb" - ] - }, - "nullable": [ - false, - false, - false, - false, - true, - false, - true - ] - }, - "hash": "47dd5646e6a94d84da1db7e7aa5961ce012cf8467e5b98fc88f073f84ddd7b87" -} diff --git a/services/data-persistence-service/.sqlx/query-5ddfe5e70c62b906ca23de28cd0056fa116a90f932567cefff259e110b6e9b1b.json b/services/data-persistence-service/.sqlx/query-5ddfe5e70c62b906ca23de28cd0056fa116a90f932567cefff259e110b6e9b1b.json deleted file mode 100644 index 6af9dec..0000000 --- a/services/data-persistence-service/.sqlx/query-5ddfe5e70c62b906ca23de28cd0056fa116a90f932567cefff259e110b6e9b1b.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT id, symbol, module_id, generated_at, model_name, content, meta_data\n FROM analysis_results\n WHERE symbol = $1\n ORDER BY generated_at DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "symbol", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "module_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "generated_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 4, - "name": "model_name", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "content", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "meta_data", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - true, - false, - true - ] - }, - "hash": "5ddfe5e70c62b906ca23de28cd0056fa116a90f932567cefff259e110b6e9b1b" -} diff --git a/services/data-persistence-service/.sqlx/query-79ac63ac22399f0ba64783b87fbca6f7637c0f331c1346211ac5275e51221654.json b/services/data-persistence-service/.sqlx/query-79ac63ac22399f0ba64783b87fbca6f7637c0f331c1346211ac5275e51221654.json deleted file mode 100644 index 43a5f79..0000000 --- a/services/data-persistence-service/.sqlx/query-79ac63ac22399f0ba64783b87fbca6f7637c0f331c1346211ac5275e51221654.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO realtime_quotes (\n symbol, market, ts, price, open_price, high_price, low_price, prev_close, change, change_percent, volume, source, updated_at\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, NOW()\n )\n ON CONFLICT (symbol, market, ts) DO UPDATE SET\n price = EXCLUDED.price,\n open_price = EXCLUDED.open_price,\n high_price = EXCLUDED.high_price,\n low_price = EXCLUDED.low_price,\n prev_close = EXCLUDED.prev_close,\n change = EXCLUDED.change,\n change_percent = EXCLUDED.change_percent,\n volume = EXCLUDED.volume,\n source = EXCLUDED.source,\n updated_at = NOW()\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Timestamptz", - "Numeric", - "Numeric", - "Numeric", - "Numeric", - "Numeric", - "Numeric", - "Numeric", - "Int8", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "79ac63ac22399f0ba64783b87fbca6f7637c0f331c1346211ac5275e51221654" -} diff --git a/services/data-persistence-service/.sqlx/query-7bc18e5f68bfc1455b7e6e74feacabb79121b6a8008c999852a9fae3a8396789.json b/services/data-persistence-service/.sqlx/query-7bc18e5f68bfc1455b7e6e74feacabb79121b6a8008c999852a9fae3a8396789.json deleted file mode 100644 index 29a3992..0000000 --- a/services/data-persistence-service/.sqlx/query-7bc18e5f68bfc1455b7e6e74feacabb79121b6a8008c999852a9fae3a8396789.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO daily_market_data (symbol, trade_date, open_price, high_price, low_price, close_price, volume, pe, pb, total_mv)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)\n ON CONFLICT (symbol, trade_date) DO UPDATE SET\n open_price = EXCLUDED.open_price,\n high_price = EXCLUDED.high_price,\n low_price = EXCLUDED.low_price,\n close_price = EXCLUDED.close_price,\n volume = EXCLUDED.volume,\n pe = EXCLUDED.pe,\n pb = EXCLUDED.pb,\n total_mv = EXCLUDED.total_mv\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Date", - "Numeric", - "Numeric", - "Numeric", - "Numeric", - "Int8", - "Numeric", - "Numeric", - "Numeric" - ] - }, - "nullable": [] - }, - "hash": "7bc18e5f68bfc1455b7e6e74feacabb79121b6a8008c999852a9fae3a8396789" -} diff --git a/services/data-persistence-service/.sqlx/query-8868e58490b2f11be13c74ae3b1ce71a3f589b61d046815b6e9a7fe67ce94886.json b/services/data-persistence-service/.sqlx/query-8868e58490b2f11be13c74ae3b1ce71a3f589b61d046815b6e9a7fe67ce94886.json deleted file mode 100644 index 77e1165..0000000 --- a/services/data-persistence-service/.sqlx/query-8868e58490b2f11be13c74ae3b1ce71a3f589b61d046815b6e9a7fe67ce94886.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT symbol, metric_name, period_date, value, source\n FROM time_series_financials\n WHERE symbol = $1 AND metric_name = ANY($2)\n ORDER BY period_date DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "metric_name", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "period_date", - "type_info": "Date" - }, - { - "ordinal": 3, - "name": "value", - "type_info": "Numeric" - }, - { - "ordinal": 4, - "name": "source", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "TextArray" - ] - }, - "nullable": [ - false, - false, - false, - false, - true - ] - }, - "hash": "8868e58490b2f11be13c74ae3b1ce71a3f589b61d046815b6e9a7fe67ce94886" -} diff --git a/services/data-persistence-service/.sqlx/query-926e80040622e569d7698396e0126fecc648346e67ecae96cb191077737f5ab5.json b/services/data-persistence-service/.sqlx/query-926e80040622e569d7698396e0126fecc648346e67ecae96cb191077737f5ab5.json deleted file mode 100644 index 2787cea..0000000 --- a/services/data-persistence-service/.sqlx/query-926e80040622e569d7698396e0126fecc648346e67ecae96cb191077737f5ab5.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT id, symbol, module_id, generated_at, model_name, content, meta_data\n FROM analysis_results\n WHERE symbol = $1 AND module_id = $2\n ORDER BY generated_at DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "symbol", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "module_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "generated_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 4, - "name": "model_name", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "content", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "meta_data", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - true, - false, - true - ] - }, - "hash": "926e80040622e569d7698396e0126fecc648346e67ecae96cb191077737f5ab5" -} diff --git a/services/data-persistence-service/.sqlx/query-a01fb0ba486147652fb7ad4e555c5205a2adc87ba01c405a73c71e5bd15ca8b6.json b/services/data-persistence-service/.sqlx/query-a01fb0ba486147652fb7ad4e555c5205a2adc87ba01c405a73c71e5bd15ca8b6.json new file mode 100644 index 0000000..a91db77 --- /dev/null +++ b/services/data-persistence-service/.sqlx/query-a01fb0ba486147652fb7ad4e555c5205a2adc87ba01c405a73c71e5bd15ca8b6.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n cache_key, \n data_payload, \n updated_at, \n expires_at\n FROM provider_response_cache\n WHERE cache_key = $1 AND expires_at > $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cache_key", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "data_payload", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "updated_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "expires_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Timestamptz" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "a01fb0ba486147652fb7ad4e555c5205a2adc87ba01c405a73c71e5bd15ca8b6" +} diff --git a/services/data-persistence-service/.sqlx/query-a487a815febf42b5c58fce44382f2d849f81b5831e733fc1d8faa62196f67dc9.json b/services/data-persistence-service/.sqlx/query-a487a815febf42b5c58fce44382f2d849f81b5831e733fc1d8faa62196f67dc9.json deleted file mode 100644 index c34bcf0..0000000 --- a/services/data-persistence-service/.sqlx/query-a487a815febf42b5c58fce44382f2d849f81b5831e733fc1d8faa62196f67dc9.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT symbol, trade_date, open_price, high_price, low_price, close_price, volume, pe, pb, total_mv\n FROM daily_market_data\n WHERE symbol = $1\n AND ($2::DATE IS NULL OR trade_date >= $2)\n AND ($3::DATE IS NULL OR trade_date <= $3)\n ORDER BY trade_date DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "trade_date", - "type_info": "Date" - }, - { - "ordinal": 2, - "name": "open_price", - "type_info": "Numeric" - }, - { - "ordinal": 3, - "name": "high_price", - "type_info": "Numeric" - }, - { - "ordinal": 4, - "name": "low_price", - "type_info": "Numeric" - }, - { - "ordinal": 5, - "name": "close_price", - "type_info": "Numeric" - }, - { - "ordinal": 6, - "name": "volume", - "type_info": "Int8" - }, - { - "ordinal": 7, - "name": "pe", - "type_info": "Numeric" - }, - { - "ordinal": 8, - "name": "pb", - "type_info": "Numeric" - }, - { - "ordinal": 9, - "name": "total_mv", - "type_info": "Numeric" - } - ], - "parameters": { - "Left": [ - "Text", - "Date", - "Date" - ] - }, - "nullable": [ - false, - false, - true, - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "a487a815febf42b5c58fce44382f2d849f81b5831e733fc1d8faa62196f67dc9" -} diff --git a/services/data-persistence-service/.sqlx/query-a857a2bbeb2b7defebc976b472df1fd3b88ab154afe1d0d6ca044e616a75e60f.json b/services/data-persistence-service/.sqlx/query-a857a2bbeb2b7defebc976b472df1fd3b88ab154afe1d0d6ca044e616a75e60f.json deleted file mode 100644 index c255d49..0000000 --- a/services/data-persistence-service/.sqlx/query-a857a2bbeb2b7defebc976b472df1fd3b88ab154afe1d0d6ca044e616a75e60f.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT symbol, name, industry, list_date, additional_info, updated_at\n FROM company_profiles\n WHERE symbol = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "symbol", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "industry", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "list_date", - "type_info": "Date" - }, - { - "ordinal": 4, - "name": "additional_info", - "type_info": "Jsonb" - }, - { - "ordinal": 5, - "name": "updated_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - true, - true, - true, - false - ] - }, - "hash": "a857a2bbeb2b7defebc976b472df1fd3b88ab154afe1d0d6ca044e616a75e60f" -} diff --git a/services/data-persistence-service/.sqlx/query-c08e82dfa0c325fe81baef633be7369ff6e4eb4534d00a41da94adfebbd44cc2.json b/services/data-persistence-service/.sqlx/query-c08e82dfa0c325fe81baef633be7369ff6e4eb4534d00a41da94adfebbd44cc2.json deleted file mode 100644 index aa9cc8c..0000000 --- a/services/data-persistence-service/.sqlx/query-c08e82dfa0c325fe81baef633be7369ff6e4eb4534d00a41da94adfebbd44cc2.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO time_series_financials (symbol, metric_name, period_date, value, source)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (symbol, metric_name, period_date) DO UPDATE SET\n value = EXCLUDED.value,\n source = EXCLUDED.source\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Date", - "Numeric", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "c08e82dfa0c325fe81baef633be7369ff6e4eb4534d00a41da94adfebbd44cc2" -} diff --git a/services/data-persistence-service/Cargo.lock b/services/data-persistence-service/Cargo.lock index 607913e..d41874d 100644 --- a/services/data-persistence-service/Cargo.lock +++ b/services/data-persistence-service/Cargo.lock @@ -327,12 +327,17 @@ dependencies = [ name = "common-contracts" version = "0.1.0" dependencies = [ + "anyhow", "chrono", + "log", + "reqwest", "rust_decimal", "serde", "serde_json", "service_kit", "sqlx", + "tokio", + "tracing", "utoipa", "uuid", ] @@ -352,6 +357,16 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -462,7 +477,7 @@ dependencies = [ "common-contracts", "dotenvy", "http-body-util", - "rmcp 0.8.5", + "rmcp", "rust-embed", "rust_decimal", "serde", @@ -546,12 +561,31 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -574,6 +608,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "find-msvc-tools" version = "0.1.5" @@ -614,6 +654,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -762,6 +817,25 @@ dependencies = [ "wasip2", ] +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -892,6 +966,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -901,6 +976,39 @@ dependencies = [ "pin-utils", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", ] [[package]] @@ -909,14 +1017,24 @@ version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" dependencies = [ + "base64", "bytes", + "futures-channel", "futures-core", + "futures-util", "http", "http-body", "hyper", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", + "socket2", + "system-configuration", "tokio", "tower-service", + "tracing", + "windows-registry", ] [[package]] @@ -1072,6 +1190,22 @@ dependencies = [ "rustversion", ] +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "itoa" version = "1.0.15" @@ -1139,6 +1273,12 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + [[package]] name = "litemap" version = "0.8.1" @@ -1228,6 +1368,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1289,6 +1446,50 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "parking" version = "2.2.1" @@ -1581,6 +1782,46 @@ dependencies = [ "bytecheck", ] +[[package]] +name = "reqwest" +version = "0.12.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "ring" version = "0.17.14" @@ -1626,31 +1867,11 @@ dependencies = [ [[package]] name = "rmcp" -version = "0.6.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ab0892f4938752b34ae47cb53910b1b0921e55e77ddb6e44df666cab17939f" -dependencies = [ - "base64", - "chrono", - "futures", - "paste", - "pin-project-lite", - "rmcp-macros 0.6.4", - "schemars", - "serde", - "serde_json", - "thiserror", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "rmcp" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5947688160b56fb6c827e3c20a72c90392a1d7e9dec74749197aa1780ac42ca" +checksum = "acc36ea743d4bbc97e9f3c33bf0b97765a5cf338de3d9c3d2f321a6e38095615" dependencies = [ + "async-trait", "base64", "bytes", "chrono", @@ -1661,7 +1882,7 @@ dependencies = [ "paste", "pin-project-lite", "rand 0.9.2", - "rmcp-macros 0.8.5", + "rmcp-macros", "schemars", "serde", "serde_json", @@ -1677,22 +1898,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "0.6.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1827cd98dab34cade0513243c6fe0351f0f0b2c9d6825460bcf45b42804bdda0" -dependencies = [ - "darling", - "proc-macro2", - "quote", - "serde_json", - "syn 2.0.110", -] - -[[package]] -name = "rmcp-macros" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01263441d3f8635c628e33856c468b96ebbce1af2d3699ea712ca71432d4ee7a" +checksum = "263caba1c96f2941efca0fdcd97b03f42bcde52d2347d05e5d77c93ab18c5b58" dependencies = [ "darling", "proc-macro2", @@ -1771,6 +1979,19 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustix" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.35" @@ -1826,6 +2047,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "1.1.0" @@ -1864,6 +2094,29 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.228" @@ -1953,8 +2206,6 @@ dependencies = [ [[package]] name = "service-kit-macros" version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "950f2a12dee6b2680baad55557688cb9f46c0006f3636117a76e842f8bf6517d" dependencies = [ "heck", "inventory", @@ -1967,15 +2218,13 @@ dependencies = [ [[package]] name = "service_kit" version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534d2ec2a54edd36e582df2d72e3d8c2aad8cea4011cb971525e4199431cea1c" dependencies = [ "axum", "inventory", "once_cell", "proc-macro2", "quote", - "rmcp 0.6.4", + "rmcp", "schemars", "serde", "serde_json", @@ -2369,6 +2618,9 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -2381,12 +2633,46 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tap" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "2.0.17" @@ -2469,6 +2755,26 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.17" @@ -2568,9 +2874,12 @@ checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "bitflags", "bytes", + "futures-util", "http", "http-body", + "iri-string", "pin-project-lite", + "tower", "tower-layer", "tower-service", "tracing", @@ -2650,6 +2959,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.19.0" @@ -2796,6 +3111,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2830,6 +3154,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.105" @@ -2862,6 +3199,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -2940,6 +3287,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" diff --git a/services/data-persistence-service/Cargo.toml b/services/data-persistence-service/Cargo.toml index b533d08..daff302 100644 --- a/services/data-persistence-service/Cargo.toml +++ b/services/data-persistence-service/Cargo.toml @@ -22,7 +22,7 @@ path = "src/bin/api-cli.rs" [dependencies] service_kit = { version = "0.1.2", default-features = true } anyhow = "1.0" -rmcp = { version = "0.8.5", features = [ +rmcp = { version = "0.9.0", features = [ "transport-streamable-http-server", "transport-worker" ] } @@ -76,10 +76,7 @@ mcp = ["service_kit/mcp"] # api-cli = ["service_kit/api-cli"] # full-data = [] -# --- For Local Development --- -# If you are developing `service_kit` locally, uncomment the following lines -# in your project's `.cargo/config.toml` file (create it if it doesn't exist) -# to make Cargo use your local version instead of the one from git. -# -# [patch.'https://github.com/lvsoft/service_kit'] -# service_kit = { path = "../service_kit" } # Note: Adjust the path if your directory structure is different. +# --- For Local Development & Docker Build --- +[patch.crates-io] +service_kit = { path = "../../ref/service_kit_mirror/service_kit/service_kit" } +service-kit-macros = { path = "../../ref/service_kit_mirror/service_kit/service_kit/service-kit-macros" } diff --git a/services/data-persistence-service/Dockerfile b/services/data-persistence-service/Dockerfile index 52a8c5e..e2e8b0f 100644 --- a/services/data-persistence-service/Dockerfile +++ b/services/data-persistence-service/Dockerfile @@ -7,6 +7,9 @@ WORKDIR /app/services/data-persistence-service # 仅复制必要的 Cargo 清单，避免大体积上下文 COPY services/common-contracts/Cargo.toml /app/services/common-contracts/Cargo.toml COPY services/data-persistence-service/Cargo.toml /app/services/data-persistence-service/Cargo.toml +# Copy service_kit mirror for dependency resolution +COPY ref/service_kit_mirror /app/ref/service_kit_mirror + RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder @@ -15,6 +18,9 @@ WORKDIR /app/services/data-persistence-service COPY --from=planner /app/services/data-persistence-service/recipe.json /app/services/data-persistence-service/recipe.json # 为了支持 path 依赖，先拷贝依赖源码再 cook COPY services/common-contracts /app/services/common-contracts +# Copy service_kit mirror again for build +COPY ref/service_kit_mirror /app/ref/service_kit_mirror + RUN cargo chef cook --release --recipe-path /app/services/data-persistence-service/recipe.json # 复制服务源码用于实际构建 COPY services/common-contracts /app/services/common-contracts diff --git a/services/data-persistence-service/src/api/analysis.rs b/services/data-persistence-service/src/api/analysis.rs index a2666e6..b40f6d6 100644 --- a/services/data-persistence-service/src/api/analysis.rs +++ b/services/data-persistence-service/src/api/analysis.rs @@ -11,6 +11,7 @@ use serde::Deserialize; use service_kit::api; use tracing::{instrument, error}; use anyhow::Error as AnyhowError; +use uuid::Uuid; #[derive(Debug, Deserialize, utoipa::IntoParams, utoipa::ToSchema)] pub struct AnalysisQuery { @@ -25,27 +26,12 @@ pub async fn create_analysis_result( State(state): State, Json(payload): Json, ) -> Result { - // Use explicit column names to avoid issues if DB schema and struct are slightly out of sync - // Also ensure we are returning all fields needed by AnalysisResult - let result = sqlx::query_as::<_, AnalysisResult>( - r#" - INSERT INTO analysis_results (request_id, symbol, template_id, module_id, content, meta_data) - VALUES ($1, $2, $3, $4, $5, $6) - RETURNING id, request_id, symbol, template_id, module_id, content, meta_data, created_at - "# - ) - .bind(&payload.request_id) - .bind(&payload.symbol) - .bind(&payload.template_id) - .bind(&payload.module_id) - .bind(&payload.content) - .bind(&payload.meta_data) - .fetch_one(state.pool()) - .await - .map_err(|e| { - error!("Database error inserting analysis result: {}", e); - AnyhowError::from(e) - })?; + let result = crate::db::create_analysis_result(state.pool(), &payload) + .await + .map_err(|e| { + error!("Database error inserting analysis result: {}", e); + AnyhowError::from(e) + })?; let dto = AnalysisResultDto { id: result.id, @@ -68,41 +54,12 @@ pub async fn get_analysis_results( State(state): State, Query(query): Query, ) -> Result>, ServerError> { - // Use string replacement for module_id to avoid lifetime issues with query_builder - // This is safe because we're not interpolating user input directly into the SQL structure, just deciding whether to add a clause. - // However, binding parameters is better. The issue with previous code was lifetime of temporary values. - - let results = if let Some(mid) = &query.module_id { - sqlx::query_as::<_, AnalysisResult>( - r#" - SELECT id, request_id, symbol, template_id, module_id, content, meta_data, created_at - FROM analysis_results - WHERE symbol = $1 AND module_id = $2 - ORDER BY created_at DESC - "# - ) - .bind(&query.symbol) - .bind(mid) - .fetch_all(state.pool()) + let results = crate::db::get_analysis_results(state.pool(), &query.symbol, query.module_id.as_deref()) .await - } else { - sqlx::query_as::<_, AnalysisResult>( - r#" - SELECT id, request_id, symbol, template_id, module_id, content, meta_data, created_at - FROM analysis_results - WHERE symbol = $1 - ORDER BY created_at DESC - "# - ) - .bind(&query.symbol) - .fetch_all(state.pool()) - .await - }; - - let results = results.map_err(|e| { - error!("Database error fetching analysis results: {}", e); - AnyhowError::from(e) - })?; + .map_err(|e| { + error!("Database error fetching analysis results: {}", e); + AnyhowError::from(e) + })?; let dtos = results .into_iter() @@ -121,31 +78,22 @@ pub async fn get_analysis_results( Ok(Json(dtos)) } -use uuid::Uuid; - /// Retrieves a single analysis result by its primary ID. #[instrument(skip(state))] #[api(GET, "/api/v1/analysis-results/{id}", output(detail = "AnalysisResultDto"))] pub async fn get_analysis_result_by_id( State(state): State, - Path(id_str): Path, + Path(id): Path, ) -> Result, ServerError> { - let id = Uuid::parse_str(&id_str).map_err(|_| ServerError::NotFound(format!("Invalid UUID: {}", id_str)))?; + let id = Uuid::parse_str(&id).map_err(|_| ServerError::NotFound(format!("Invalid UUID: {}", id)))?; - let result = sqlx::query_as::<_, AnalysisResult>( - r#" - SELECT id, request_id, symbol, template_id, module_id, content, meta_data, created_at - FROM analysis_results - WHERE id = $1 - "# - ) - .bind(&id) - .fetch_one(state.pool()) - .await - .map_err(|e| { - error!("Database error fetching analysis result by id: {}", e); - AnyhowError::from(e) - })?; + let result = crate::db::get_analysis_result_by_id(state.pool(), id) + .await + .map_err(|e| { + error!("Database error fetching analysis result by id: {}", e); + AnyhowError::from(e) + })? + .ok_or_else(|| ServerError::NotFound(format!("Analysis result not found: {}", id)))?; let dto = AnalysisResultDto { id: result.id, diff --git a/services/data-persistence-service/src/api/companies.rs b/services/data-persistence-service/src/api/companies.rs index f3176c6..0ffee6a 100644 --- a/services/data-persistence-service/src/api/companies.rs +++ b/services/data-persistence-service/src/api/companies.rs @@ -1,6 +1,6 @@ use crate::{ db, - dtos::CompanyProfileDto, + dtos::{CompanyProfileDto, ProviderStatusDto}, AppState, ServerError, }; use axum::{ @@ -17,7 +17,7 @@ pub async fn upsert_company( Json(payload): Json, ) -> Result<(), ServerError> { info!(target: "api", symbol = %payload.symbol, "PUT /companies → upsert_company called"); - db::upsert_company(&state.pool, &payload).await.map_err(AnyhowError::from)?; + db::companies::upsert_company(&state.pool, &payload).await.map_err(AnyhowError::from)?; info!(target: "api", symbol = %payload.symbol, "upsert_company completed"); Ok(()) } @@ -28,7 +28,7 @@ pub async fn get_company_by_symbol( Path(symbol): Path, ) -> Result, ServerError> { info!(target: "api", symbol = %symbol, "GET /companies/{{symbol}} → get_company_by_symbol called"); - let company = db::get_company_by_symbol(&state.pool, &symbol) + let company = db::companies::get_company_by_symbol(&state.pool, &symbol) .await .map_err(AnyhowError::from)? .ok_or_else(|| ServerError::NotFound(format!("Company with symbol '{}' not found", symbol)))?; @@ -46,3 +46,35 @@ pub async fn get_company_by_symbol( info!(target: "api", symbol = %dto.symbol, "get_company_by_symbol completed"); Ok(Json(dto)) } + + +#[api( + PUT, + "/api/v1/companies/{symbol}/providers/{provider_id}/status" +)] +pub async fn update_provider_status( + State(state): State, + Path((symbol, provider_id)): Path<(String, String)>, + Json(payload): Json, +) -> Result<(), ServerError> { + info!(target: "api", symbol = %symbol, provider = %provider_id, "PUT provider status"); + db::companies::update_provider_status(&state.pool, &symbol, &provider_id, &payload) + .await + .map_err(AnyhowError::from)?; + Ok(()) +} + +#[api( + GET, + "/api/v1/companies/{symbol}/providers/{provider_id}/status" +)] +pub async fn get_provider_status( + State(state): State, + Path((symbol, provider_id)): Path<(String, String)>, +) -> Result>, ServerError> { + info!(target: "api", symbol = %symbol, provider = %provider_id, "GET provider status"); + let status = db::companies::get_provider_status(&state.pool, &symbol, &provider_id) + .await + .map_err(AnyhowError::from)?; + Ok(Json(status)) +} diff --git a/services/data-persistence-service/src/api/market_data.rs b/services/data-persistence-service/src/api/market_data.rs index 105a3af..9bb651f 100644 --- a/services/data-persistence-service/src/api/market_data.rs +++ b/services/data-persistence-service/src/api/market_data.rs @@ -29,7 +29,7 @@ pub async fn batch_insert_financials( Ok(axum::http::StatusCode::CREATED) } -#[api(GET, "/api/v1/market-data/financial-statements/{symbol}", output(list = "TimeSeriesFinancialDto"))] +#[api(GET, "/api/v1/market-data/financials/{symbol}", output(list = "TimeSeriesFinancialDto"))] pub async fn get_financials_by_symbol( State(state): State, Path(symbol): Path, diff --git a/services/data-persistence-service/src/api/mod.rs b/services/data-persistence-service/src/api/mod.rs index a037074..73e5118 100644 --- a/services/data-persistence-service/src/api/mod.rs +++ b/services/data-persistence-service/src/api/mod.rs @@ -3,6 +3,9 @@ mod companies; mod configs; mod market_data; mod system; +mod session_data; +mod provider_cache; + use crate::AppState; use axum::{ routing::{get, post}, @@ -28,6 +31,10 @@ pub fn create_router(_state: AppState) -> Router { ) // Companies .route("/companies/{symbol}", get(companies::get_company_by_symbol)) + .route( + "/companies/{symbol}/providers/{provider_id}/status", + get(companies::get_provider_status).put(companies::update_provider_status), + ) // Market Data .route( "/market-data/financial-statements/{symbol}", @@ -39,8 +46,22 @@ pub fn create_router(_state: AppState) -> Router { post(analysis::create_analysis_result).get(analysis::get_analysis_results), ) .route( - "/analysis-results/:id", + "/analysis-results/{id}", get(analysis::get_analysis_result_by_id), + ) + // Session Data + .route( + "/session-data", + post(session_data::insert_session_data), + ) + .route( + "/session-data/{request_id}", + get(session_data::get_session_data).delete(session_data::delete_session_data), + ) + // Provider Cache + .route( + "/provider-cache", + get(provider_cache::get_cache).post(provider_cache::set_cache), ); router diff --git a/services/data-persistence-service/src/api/provider_cache.rs b/services/data-persistence-service/src/api/provider_cache.rs new file mode 100644 index 0000000..6c17b12 --- /dev/null +++ b/services/data-persistence-service/src/api/provider_cache.rs @@ -0,0 +1,39 @@ +use crate::{ + db, + AppState, ServerError, +}; +use axum::{ + extract::{Query, State}, + Json, + http::StatusCode, +}; +use common_contracts::dtos::ProviderCacheDto; +use service_kit::api; +use tracing::info; +use anyhow::Error as AnyhowError; +use serde::Deserialize; + +#[derive(Deserialize, utoipa::ToSchema)] +pub struct CacheQuery { + pub key: String, +} + +#[api(GET, "/api/v1/provider-cache")] +pub async fn get_cache( + State(state): State, + Query(query): Query, +) -> Result>, ServerError> { + // Detailed logging might be too noisy for cache hits, so keep it simple + let data = db::get_cache(&state.pool, &query.key).await.map_err(AnyhowError::from)?; + Ok(Json(data)) +} + +#[api(POST, "/api/v1/provider-cache")] +pub async fn set_cache( + State(state): State, + Json(payload): Json, +) -> Result { + db::set_cache(&state.pool, &payload).await.map_err(AnyhowError::from)?; + Ok(StatusCode::OK) +} + diff --git a/services/data-persistence-service/src/api/session_data.rs b/services/data-persistence-service/src/api/session_data.rs new file mode 100644 index 0000000..2803707 --- /dev/null +++ b/services/data-persistence-service/src/api/session_data.rs @@ -0,0 +1,47 @@ +use crate::{ + db, + AppState, ServerError, +}; +use axum::{ + extract::{Path, State}, + Json, + http::StatusCode, +}; +use common_contracts::dtos::SessionDataDto; +use service_kit::api; +use uuid::Uuid; +use tracing::info; +use anyhow::Error as AnyhowError; + +#[api(POST, "/api/v1/session-data")] +pub async fn insert_session_data( + State(state): State, + Json(payload): Json, +) -> Result { + info!(target: "api", request_id = %payload.request_id, provider = %payload.provider, "POST /session-data called"); + db::insert_session_data(&state.pool, &payload).await.map_err(AnyhowError::from)?; + Ok(StatusCode::CREATED) +} + +#[api(GET, "/api/v1/session-data/{request_id}")] +pub async fn get_session_data( + State(state): State, + Path(request_id): Path, +) -> Result>, ServerError> { + let request_id = Uuid::parse_str(&request_id).map_err(|e| AnyhowError::from(e))?; + info!(target: "api", request_id = %request_id, "GET /session-data/{request_id} called"); + let data = db::get_session_data(&state.pool, request_id).await.map_err(AnyhowError::from)?; + Ok(Json(data)) +} + +#[api(DELETE, "/api/v1/session-data/{request_id}")] +pub async fn delete_session_data( + State(state): State, + Path(request_id): Path, +) -> Result { + let request_id = Uuid::parse_str(&request_id).map_err(|e| AnyhowError::from(e))?; + info!(target: "api", request_id = %request_id, "DELETE /session-data/{request_id} called"); + db::delete_session_data(&state.pool, request_id).await.map_err(AnyhowError::from)?; + Ok(StatusCode::NO_CONTENT) +} + diff --git a/services/data-persistence-service/src/db/analysis_results.rs b/services/data-persistence-service/src/db/analysis_results.rs new file mode 100644 index 0000000..50a944e --- /dev/null +++ b/services/data-persistence-service/src/db/analysis_results.rs @@ -0,0 +1,75 @@ +use crate::models::AnalysisResult; +use common_contracts::dtos::NewAnalysisResult; +use sqlx::PgPool; +use uuid::Uuid; + +pub async fn create_analysis_result( + pool: &PgPool, + payload: &NewAnalysisResult, +) -> Result { + sqlx::query_as::<_, AnalysisResult>( + r#" + INSERT INTO analysis_results (request_id, symbol, template_id, module_id, content, meta_data) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id, request_id, symbol, template_id, module_id, content, meta_data, created_at + "# + ) + .bind(&payload.request_id) + .bind(&payload.symbol) + .bind(&payload.template_id) + .bind(&payload.module_id) + .bind(&payload.content) + .bind(&payload.meta_data) + .fetch_one(pool) + .await +} + +pub async fn get_analysis_results( + pool: &PgPool, + symbol: &str, + module_id: Option<&str>, +) -> Result, sqlx::Error> { + if let Some(mid) = module_id { + sqlx::query_as::<_, AnalysisResult>( + r#" + SELECT id, request_id, symbol, template_id, module_id, content, meta_data, created_at + FROM analysis_results + WHERE symbol = $1 AND module_id = $2 + ORDER BY created_at DESC + "# + ) + .bind(symbol) + .bind(mid) + .fetch_all(pool) + .await + } else { + sqlx::query_as::<_, AnalysisResult>( + r#" + SELECT id, request_id, symbol, template_id, module_id, content, meta_data, created_at + FROM analysis_results + WHERE symbol = $1 + ORDER BY created_at DESC + "# + ) + .bind(symbol) + .fetch_all(pool) + .await + } +} + +pub async fn get_analysis_result_by_id( + pool: &PgPool, + id: Uuid, +) -> Result, sqlx::Error> { + sqlx::query_as::<_, AnalysisResult>( + r#" + SELECT id, request_id, symbol, template_id, module_id, content, meta_data, created_at + FROM analysis_results + WHERE id = $1 + "# + ) + .bind(&id) + .fetch_optional(pool) + .await +} + diff --git a/services/data-persistence-service/src/db/companies.rs b/services/data-persistence-service/src/db/companies.rs index 7e776fd..5cefb1c 100644 --- a/services/data-persistence-service/src/db/companies.rs +++ b/services/data-persistence-service/src/db/companies.rs @@ -1,4 +1,4 @@ -use common_contracts::dtos::CompanyProfileDto; +use common_contracts::dtos::{CompanyProfileDto, ProviderStatusDto}; use common_contracts::models::CompanyProfile; use sqlx::PgPool; @@ -11,7 +11,12 @@ pub async fn upsert_company(pool: &PgPool, payload: &CompanyProfileDto) -> Resul name = EXCLUDED.name, industry = EXCLUDED.industry, list_date = EXCLUDED.list_date, - additional_info = EXCLUDED.additional_info, + additional_info = CASE + -- Preserve existing provider_status in additional_info if it exists + WHEN company_profiles.additional_info ? 'provider_status' AND EXCLUDED.additional_info IS NOT NULL THEN + jsonb_set(EXCLUDED.additional_info, '{provider_status}', company_profiles.additional_info->'provider_status') + ELSE EXCLUDED.additional_info + END, updated_at = NOW() "#, ) @@ -31,7 +36,7 @@ pub async fn get_company_by_symbol(pool: &PgPool, symbol: &str) -> Result