ycg
569c1c8813
- 重构数据访问层：引入DAO模式，支持MySQL/SQLite双数据库 - 新增数据库架构：完整的股票数据、AI分析、自选股管理表结构 - 升级AI分析服务：集成豆包大模型，支持多维度分析 - 优化API路由：分离市场数据API，提供更清晰的接口设计 - 完善项目文档：添加数据库迁移指南、新功能指南等 - 清理冗余文件：删除旧的缓存文件和无用配置 - 新增调度器：支持定时任务和数据自动更新 - 改进前端模板：简化的股票展示页面 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
403 lines
15 KiB
Python
403 lines
15 KiB
Python
"""
|
||
定时任务调度器
|
||
负责自动更新股票数据、K线数据等定时任务
|
||
"""
|
||
import asyncio
|
||
import logging
|
||
from datetime import datetime, time, timedelta
|
||
from typing import Dict, List, Optional
|
||
import threading
|
||
from app.services.market_data_service import MarketDataService
|
||
from app.services.kline_service import KlineService
|
||
from app.services.stock_service_db import StockServiceDB
|
||
from app.database import DatabaseManager
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class TaskScheduler:
|
||
def __init__(self):
|
||
self.market_service = MarketDataService()
|
||
self.kline_service = KlineService()
|
||
self.stock_service = StockServiceDB()
|
||
self.db_manager = DatabaseManager()
|
||
self.logger = logging.getLogger(__name__)
|
||
self.running = False
|
||
self.scheduler_thread = None
|
||
|
||
def start(self):
|
||
"""启动定时任务调度器"""
|
||
if self.running:
|
||
self.logger.warning("任务调度器已在运行")
|
||
return
|
||
|
||
self.running = True
|
||
self.scheduler_thread = threading.Thread(target=self._run_scheduler, daemon=True)
|
||
self.scheduler_thread.start()
|
||
self.logger.info("任务调度器已启动")
|
||
|
||
def stop(self):
|
||
"""停止定时任务调度器"""
|
||
self.running = False
|
||
if self.scheduler_thread:
|
||
self.scheduler_thread.join(timeout=10)
|
||
self.logger.info("任务调度器已停止")
|
||
|
||
def _run_scheduler(self):
|
||
"""运行调度器主循环"""
|
||
self.logger.info("任务调度器开始运行")
|
||
|
||
while self.running:
|
||
try:
|
||
current_time = datetime.now()
|
||
|
||
# 检查是否到了执行时间
|
||
self._check_and_run_tasks(current_time)
|
||
|
||
# 每5分钟检查一次
|
||
for _ in range(60): # 5分钟 = 300秒，每5秒检查一次
|
||
if not self.running:
|
||
break
|
||
asyncio.sleep(5)
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"任务调度器运行错误: {e}")
|
||
asyncio.sleep(30) # 出错后等待30秒再继续
|
||
|
||
def _check_and_run_tasks(self, current_time: datetime):
|
||
"""检查并执行定时任务"""
|
||
try:
|
||
# 每日上午9:00更新股票列表（每周一）
|
||
if current_time.weekday() == 0 and current_time.time() >= time(9, 0):
|
||
if self._should_run_task('update_stock_list', current_time):
|
||
self._run_task_async('update_stock_list', self._update_stock_list)
|
||
|
||
# 每日上午9:30更新K线数据
|
||
if current_time.time() >= time(9, 30):
|
||
if self._should_run_task('update_daily_kline', current_time):
|
||
self._run_task_async('update_daily_kline', self._update_daily_kline)
|
||
|
||
# 每日收盘后(16:00)更新市场统计
|
||
if current_time.time() >= time(16, 0):
|
||
if self._should_run_task('update_market_stats', current_time):
|
||
self._run_task_async('update_market_stats', self._update_market_statistics)
|
||
|
||
# 每日晚上20:00更新监控列表数据
|
||
if current_time.time() >= time(20, 0):
|
||
if self._should_run_task('update_watchlist', current_time):
|
||
self._run_task_async('update_watchlist', self._update_watchlist_data)
|
||
|
||
# 每周日凌晨2:00清理旧数据
|
||
if current_time.weekday() == 6 and current_time.time() >= time(2, 0):
|
||
if self._should_run_task('clean_old_data', current_time):
|
||
self._run_task_async('clean_old_data', self._clean_old_data)
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"检查和执行任务失败: {e}")
|
||
|
||
def _should_run_task(self, task_name: str, current_time: datetime) -> bool:
|
||
"""检查任务是否应该执行（避免重复执行）"""
|
||
try:
|
||
with self.db_manager.get_connection() as conn:
|
||
cursor = conn.cursor(dictionary=True)
|
||
|
||
# 检查今天是否已经执行过该任务
|
||
today = current_time.date()
|
||
query = """
|
||
SELECT COUNT(*) as count
|
||
FROM data_update_tasks
|
||
WHERE task_type = %s AND DATE(created_at) = %s AND status = 'completed'
|
||
"""
|
||
cursor.execute(query, (task_name, today))
|
||
result = cursor.fetchone()
|
||
|
||
cursor.close()
|
||
return result['count'] == 0
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"检查任务执行状态失败: {task_name}, 错误: {e}")
|
||
return False
|
||
|
||
def _run_task_async(self, task_name: str, task_func):
|
||
"""异步执行任务"""
|
||
def run_task():
|
||
try:
|
||
self._create_task_record(task_name, 'running')
|
||
|
||
start_time = datetime.now()
|
||
result = task_func()
|
||
end_time = datetime.now()
|
||
|
||
duration = (end_time - start_time).total_seconds()
|
||
|
||
if isinstance(result, dict) and 'error' in result:
|
||
self._update_task_record(task_name, 'failed',
|
||
error_message=result['error'],
|
||
duration=duration)
|
||
else:
|
||
self._update_task_record(task_name, 'completed',
|
||
processed_count=result.get('processed_count', 0),
|
||
total_count=result.get('total_count', 0),
|
||
duration=duration)
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"执行任务失败: {task_name}, 错误: {e}")
|
||
self._update_task_record(task_name, 'failed', error_message=str(e))
|
||
|
||
# 在新线程中执行任务
|
||
task_thread = threading.Thread(target=run_task, daemon=True)
|
||
task_thread.start()
|
||
|
||
def _create_task_record(self, task_name: str, task_type: str):
|
||
"""创建任务记录"""
|
||
try:
|
||
with self.db_manager.get_connection() as conn:
|
||
cursor = conn.cursor()
|
||
query = """
|
||
INSERT INTO data_update_tasks (task_name, task_type, status, start_time)
|
||
VALUES (%s, %s, %s, NOW())
|
||
"""
|
||
cursor.execute(query, (task_name, task_type, 'running'))
|
||
conn.commit()
|
||
cursor.close()
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"创建任务记录失败: {task_name}, 错误: {e}")
|
||
|
||
def _update_task_record(self, task_name: str, status: str,
|
||
processed_count: int = 0, total_count: int = 0,
|
||
error_message: str = None, duration: float = None):
|
||
"""更新任务记录"""
|
||
try:
|
||
with self.db_manager.get_connection() as conn:
|
||
cursor = conn.cursor()
|
||
query = """
|
||
UPDATE data_update_tasks
|
||
SET status = %s, end_time = NOW(),
|
||
processed_count = %s, total_count = %s,
|
||
error_message = %s
|
||
WHERE task_name = %s AND status = 'running'
|
||
ORDER BY created_at DESC
|
||
LIMIT 1
|
||
"""
|
||
cursor.execute(query, (status, processed_count, total_count, error_message, task_name))
|
||
conn.commit()
|
||
cursor.close()
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"更新任务记录失败: {task_name}, 错误: {e}")
|
||
|
||
def _update_stock_list(self) -> Dict:
|
||
"""更新股票列表"""
|
||
try:
|
||
self.logger.info("开始更新股票列表")
|
||
result = self.market_service.get_all_stock_list(force_refresh=True)
|
||
|
||
# 更新概念分类
|
||
self.market_service.update_stock_sectors()
|
||
|
||
return {
|
||
'total_count': len(result),
|
||
'processed_count': len(result)
|
||
}
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"更新股票列表失败: {e}")
|
||
return {'error': str(e)}
|
||
|
||
def _update_daily_kline(self) -> Dict:
|
||
"""更新日K线数据"""
|
||
try:
|
||
self.logger.info("开始更新日K线数据")
|
||
result = self.kline_service.batch_update_kline_data(days_back=1)
|
||
return result
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"更新日K线数据失败: {e}")
|
||
return {'error': str(e)}
|
||
|
||
def _update_watchlist_data(self) -> Dict:
|
||
"""更新监控列表数据"""
|
||
try:
|
||
self.logger.info("开始更新监控列表数据")
|
||
result = self.stock_service.batch_update_watchlist_data()
|
||
return result
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"更新监控列表数据失败: {e}")
|
||
return {'error': str(e)}
|
||
|
||
def _update_market_statistics(self) -> Dict:
|
||
"""更新市场统计数据"""
|
||
try:
|
||
self.logger.info("开始更新市场统计数据")
|
||
return self._calculate_market_stats()
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"更新市场统计数据失败: {e}")
|
||
return {'error': str(e)}
|
||
|
||
def _calculate_market_stats(self) -> Dict:
|
||
"""计算市场统计数据"""
|
||
try:
|
||
today = datetime.now().date()
|
||
|
||
with self.db_manager.get_connection() as conn:
|
||
cursor = conn.cursor(dictionary=True)
|
||
|
||
# 计算市场统计
|
||
query = """
|
||
INSERT INTO market_statistics (
|
||
stat_date, market_code, total_stocks, up_stocks, down_stocks,
|
||
flat_stocks, total_volume, total_amount, created_at
|
||
)
|
||
SELECT
|
||
%s as stat_date,
|
||
market,
|
||
COUNT(*) as total_stocks,
|
||
SUM(CASE WHEN change_percent > 0 THEN 1 ELSE 0 END) as up_stocks,
|
||
SUM(CASE WHEN change_percent < 0 THEN 1 ELSE 0 END) as down_stocks,
|
||
SUM(CASE WHEN change_percent = 0 THEN 1 ELSE 0 END) as flat_stocks,
|
||
COALESCE(SUM(volume), 0) as total_volume,
|
||
COALESCE(SUM(amount), 0) as total_amount,
|
||
NOW()
|
||
FROM (
|
||
SELECT
|
||
CASE WHEN stock_code LIKE '6%' THEN 'SH'
|
||
WHEN stock_code LIKE '0%' OR stock_code LIKE '3%' THEN 'SZ'
|
||
ELSE 'OTHER' END as market,
|
||
change_percent,
|
||
volume,
|
||
amount
|
||
FROM kline_data
|
||
WHERE kline_type = 'daily' AND trade_date = %s
|
||
) as daily_data
|
||
GROUP BY market
|
||
ON DUPLICATE KEY UPDATE
|
||
total_stocks = VALUES(total_stocks),
|
||
up_stocks = VALUES(up_stocks),
|
||
down_stocks = VALUES(down_stocks),
|
||
flat_stocks = VALUES(flat_stocks),
|
||
total_volume = VALUES(total_volume),
|
||
total_amount = VALUES(total_amount),
|
||
updated_at = NOW()
|
||
"""
|
||
|
||
cursor.execute(query, (today, today))
|
||
affected_rows = cursor.rowcount
|
||
conn.commit()
|
||
cursor.close()
|
||
|
||
return {
|
||
'processed_count': affected_rows,
|
||
'total_count': affected_rows
|
||
}
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"计算市场统计数据失败: {e}")
|
||
return {'error': str(e)}
|
||
|
||
def _clean_old_data(self) -> Dict:
|
||
"""清理旧数据"""
|
||
try:
|
||
self.logger.info("开始清理旧数据")
|
||
|
||
# 清理6个月前的K线数据
|
||
deleted_count = self.kline_service.clean_old_kline_data(days_to_keep=180)
|
||
|
||
# 清理3个月前的任务记录
|
||
cutoff_date = datetime.now() - timedelta(days=90)
|
||
with self.db_manager.get_connection() as conn:
|
||
cursor = conn.cursor()
|
||
cursor.execute("DELETE FROM data_update_tasks WHERE created_at < %s", (cutoff_date,))
|
||
task_deleted = cursor.rowcount
|
||
conn.commit()
|
||
cursor.close()
|
||
|
||
return {
|
||
'processed_count': deleted_count + task_deleted,
|
||
'deleted_kline_count': deleted_count,
|
||
'deleted_task_count': task_deleted
|
||
}
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"清理旧数据失败: {e}")
|
||
return {'error': str(e)}
|
||
|
||
def get_task_status(self, task_type: str = None, days: int = 7) -> List[Dict]:
|
||
"""获取任务执行状态"""
|
||
try:
|
||
with self.db_manager.get_connection() as conn:
|
||
cursor = conn.cursor(dictionary=True)
|
||
|
||
query = """
|
||
SELECT task_name, task_type, status, start_time, end_time,
|
||
processed_count, total_count, error_message,
|
||
TIMESTAMPDIFF(SECOND, start_time, end_time) as duration_seconds
|
||
FROM data_update_tasks
|
||
WHERE created_at >= DATE_SUB(NOW(), INTERVAL %s DAY)
|
||
"""
|
||
params = [days]
|
||
|
||
if task_type:
|
||
query += " AND task_type = %s"
|
||
params.append(task_type)
|
||
|
||
query += " ORDER BY created_at DESC"
|
||
|
||
cursor.execute(query, params)
|
||
tasks = cursor.fetchall()
|
||
cursor.close()
|
||
|
||
return tasks
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"获取任务状态失败: {e}")
|
||
return []
|
||
|
||
def run_manual_task(self, task_name: str, **kwargs) -> Dict:
|
||
"""手动执行任务"""
|
||
try:
|
||
self.logger.info(f"手动执行任务: {task_name}")
|
||
|
||
task_map = {
|
||
'update_stock_list': self._update_stock_list,
|
||
'update_daily_kline': lambda: self._update_daily_kline(),
|
||
'update_watchlist': self._update_watchlist_data,
|
||
'update_market_stats': self._update_market_statistics,
|
||
'clean_old_data': self._clean_old_data
|
||
}
|
||
|
||
if task_name not in task_map:
|
||
return {'error': f'未知任务: {task_name}'}
|
||
|
||
task_func = task_map[task_name]
|
||
return task_func()
|
||
|
||
except Exception as e:
|
||
self.logger.error(f"手动执行任务失败: {task_name}, 错误: {e}")
|
||
return {'error': str(e)}
|
||
|
||
|
||
# 全局调度器实例
|
||
task_scheduler = TaskScheduler()
|
||
|
||
|
||
def start_scheduler():
|
||
"""启动任务调度器"""
|
||
task_scheduler.start()
|
||
|
||
|
||
def stop_scheduler():
|
||
"""停止任务调度器"""
|
||
task_scheduler.stop()
|
||
|
||
|
||
def get_scheduler_status(task_type: str = None, days: int = 7) -> List[Dict]:
|
||
"""获取调度器状态"""
|
||
return task_scheduler.get_task_status(task_type, days)
|
||
|
||
|
||
def run_manual_task(task_name: str, **kwargs) -> Dict:
|
||
"""手动执行任务"""
|
||
return task_scheduler.run_manual_task(task_name, **kwargs) |