#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ 商品看板路由模組 包含：首頁儀表板、商品列表、統計數據 """ import os import json import math import time import hashlib from datetime import datetime, timezone, timedelta from flask import Blueprint, request, render_template from sqlalchemy import func, and_ from sqlalchemy.orm import joinedload from auth import login_required from config import BASE_DIR, SYSTEM_VERSION, public_url from database.manager import DatabaseManager from database.models import Product, PriceRecord from services.logger_manager import SystemLogger # 時區設定 TAIPEI_TZ = timezone(timedelta(hours=8)) # Logger sys_log = SystemLogger("DashboardRoutes").get_logger() # Blueprint 定義 dashboard_bp = Blueprint('dashboard', __name__) # ========================================== # 快取與監控變數 # ========================================== import fcntl _DASHBOARD_DATA_CACHE = { 'consolidated_data': None, # get_consolidated_data() 結果 'consolidated_timestamp': None, 'today_start': None, 'full_data': None, # 包含統計數據的完整結果 'full_timestamp': None } _DASHBOARD_CACHE_TTL = 1800 # 快取有效期 30 分鐘 _DASHBOARD_LOCK_FILE = os.path.join(BASE_DIR, 'data', '.dashboard_cache.lock') # V-Opt: 檔案鎖（跨進程） class FileLock: """簡單的檔案鎖，用於 gunicorn 多進程環境""" def __init__(self, lock_file): self.lock_file = lock_file self.fd = None def acquire(self, blocking=True): """取得鎖""" try: self.fd = open(self.lock_file, 'w') if blocking: fcntl.flock(self.fd, fcntl.LOCK_EX) else: fcntl.flock(self.fd, fcntl.LOCK_EX | fcntl.LOCK_NB) return True except (IOError, OSError): if self.fd: self.fd.close() self.fd = None return False def release(self): """釋放鎖""" if self.fd: fcntl.flock(self.fd, fcntl.LOCK_UN) self.fd.close() self.fd = None _DASHBOARD_FILE_LOCK = FileLock(_DASHBOARD_LOCK_FILE) # 慢查詢監控 _SLOW_QUERY_STATS = { 'total_queries': 0, 'slow_queries': 0, 'very_slow_queries': 0, 'total_query_time_ms': 0, 'last_slow_query': None, 'last_slow_query_time': None, } _SLOW_QUERY_THRESHOLD_MS = 1000 _VERY_SLOW_QUERY_THRESHOLD_MS = 5000 def track_query_time(query_name, duration_ms): """追蹤查詢時間，更新慢查詢統計""" global _SLOW_QUERY_STATS _SLOW_QUERY_STATS['total_queries'] += 1 _SLOW_QUERY_STATS['total_query_time_ms'] += duration_ms if duration_ms >= _VERY_SLOW_QUERY_THRESHOLD_MS: _SLOW_QUERY_STATS['very_slow_queries'] += 1 _SLOW_QUERY_STATS['slow_queries'] += 1 _SLOW_QUERY_STATS['last_slow_query'] = query_name _SLOW_QUERY_STATS['last_slow_query_time'] = datetime.now(TAIPEI_TZ).isoformat() elif duration_ms >= _SLOW_QUERY_THRESHOLD_MS: _SLOW_QUERY_STATS['slow_queries'] += 1 _SLOW_QUERY_STATS['last_slow_query'] = query_name _SLOW_QUERY_STATS['last_slow_query_time'] = datetime.now(TAIPEI_TZ).isoformat() # ========================================== # 輔助函數 # ========================================== def get_color_for_string(s): """為字串生成一個穩定且美觀的 HSL 顏色""" if not s: return "hsl(0, 0%, 85%)" hash_val = int(hashlib.md5(s.encode('utf-8'), usedforsecurity=False).hexdigest(), 16) hue = hash_val % 360 return f"hsl({hue}, 60%, 88%)" def load_scheduler_stats(): """讀取排程統計資料""" stats_path = os.path.join(BASE_DIR, 'data', 'scheduler_stats.json') if os.path.exists(stats_path): try: with open(stats_path, 'r', encoding='utf-8') as f: return json.load(f) except (IOError, json.JSONDecodeError): return {} return {} # ========================================== # 核心數據函數 # ========================================== def get_consolidated_data(): """統一封裝：獲取全分類去重後的當前數據、昨日對比及差值 (帶快取)""" global _DASHBOARD_DATA_CACHE now = datetime.now(TAIPEI_TZ) # V-Opt: 先檢查快取（無需鎖） if (_DASHBOARD_DATA_CACHE['consolidated_data'] is not None and _DASHBOARD_DATA_CACHE['consolidated_timestamp'] is not None): cache_age = (now.timestamp() - _DASHBOARD_DATA_CACHE['consolidated_timestamp']) if cache_age < _DASHBOARD_CACHE_TTL: sys_log.debug(f"[Dashboard] [Cache] ✅ 使用快取資料 | 快取年齡: {cache_age:.1f}秒") return _DASHBOARD_DATA_CACHE['consolidated_data'], _DASHBOARD_DATA_CACHE['today_start'] # V-Opt: 使用檔案鎖避免多 gunicorn worker 同時重建快取 # 注意: get_consolidated_data 通常由 get_full_dashboard_data 調用， # 後者已持有 _DASHBOARD_FILE_LOCK，因此這裡可以不重複鎖定 # 但為避免直接調用時的競爭問題，仍保留快取檢查邏輯 # 再次檢查快取（可能其他 worker 已經更新） if (_DASHBOARD_DATA_CACHE['consolidated_data'] is not None and _DASHBOARD_DATA_CACHE['consolidated_timestamp'] is not None): cache_age = (now.timestamp() - _DASHBOARD_DATA_CACHE['consolidated_timestamp']) if cache_age < _DASHBOARD_CACHE_TTL: sys_log.debug(f"[Dashboard] [Cache] ✅ 使用快取資料 (其他 worker 已更新) | 快取年齡: {cache_age:.1f}秒") return _DASHBOARD_DATA_CACHE['consolidated_data'], _DASHBOARD_DATA_CACHE['today_start'] sys_log.debug("[Dashboard] [Cache] 🔄 快取過期或不存在，重新查詢資料庫") query_start_time = time.time() db = DatabaseManager() session = db.get_session() today_start = now.replace(hour=0, minute=0, second=0, microsecond=0) # 保持台北時區 seven_days_ago = today_start - timedelta(days=7) thirty_days_ago = today_start - timedelta(days=30) try: # Query 1: Get the latest price record for every product latest_price_subq = session.query( func.max(PriceRecord.id).label('max_id') ).group_by(PriceRecord.product_id).subquery() latest_records = session.query(PriceRecord).options( joinedload(PriceRecord.product) ).join(latest_price_subq, PriceRecord.id == latest_price_subq.c.max_id).all() product_ids = [r.product_id for r in latest_records] if not product_ids: session.close() return [], today_start # Query 2: Get yesterday's closing prices for all products yesterday_prices_subq = session.query( PriceRecord.product_id, func.max(PriceRecord.id).label('max_id') ).filter( PriceRecord.product_id.in_(product_ids), PriceRecord.timestamp < today_start ).group_by(PriceRecord.product_id).subquery() yesterday_prices_q = session.query( PriceRecord.product_id, PriceRecord.price ).join( yesterday_prices_subq, PriceRecord.id == yesterday_prices_subq.c.max_id ) yesterday_prices_map = {pid: price for pid, price in yesterday_prices_q} # Query 3: Get specific historical price points (7 days ago and 30 days ago) def get_price_map_before(target_date): subq = session.query( PriceRecord.product_id, func.max(PriceRecord.timestamp).label('max_ts') ).filter( PriceRecord.product_id.in_(product_ids), PriceRecord.timestamp < target_date ).group_by(PriceRecord.product_id).subquery() q = session.query(PriceRecord.product_id, PriceRecord.price).join( subq, and_(PriceRecord.product_id == subq.c.product_id, PriceRecord.timestamp == subq.c.max_ts) ) return {pid: price for pid, price in q} prices_7d_ago_map = get_price_map_before(seven_days_ago + timedelta(days=1)) prices_30d_ago_map = get_price_map_before(thirty_days_ago + timedelta(days=1)) # Query 4: Get TODAY's records only (for sparkline/intraday change) today_records_q = session.query(PriceRecord).filter( PriceRecord.product_id.in_(product_ids), PriceRecord.timestamp >= today_start ).order_by(PriceRecord.product_id, PriceRecord.timestamp).all() today_map = {} for r in today_records_q: if r.product_id not in today_map: today_map[r.product_id] = [] today_map[r.product_id].append(r) # Final Assembly unique_items = [] for r in latest_records: pid = r.product_id price_7d = prices_7d_ago_map.get(pid) price_30d = prices_30d_ago_map.get(pid) stats_7d_diff = r.price - price_7d if price_7d is not None else 0 stats_30d_diff = r.price - price_30d if price_30d is not None else 0 today_records = today_map.get(pid, []) today_diff = 0 today_changes = [] if len(today_records) > 1: today_diff = today_records[-1].price - today_records[0].price y_price = yesterday_prices_map.get(pid) yesterday_diff = r.price - y_price if y_price is not None else 0 status = "NONE" if yesterday_diff > 0: status = "PRICE_UP" elif yesterday_diff < 0: status = "PRICE_DOWN" last_p = y_price if y_price is not None else (today_records[0].price if today_records else r.price) for tr in today_records: if tr.price != last_p: diff = tr.price - last_p today_changes.append({ 'time': tr.timestamp.strftime('%H:%M'), 'price': tr.price, 'diff': diff }) last_p = tr.price unique_items.append({ 'record': r, 'stats': {'7d_diff': stats_7d_diff, '30d_diff': stats_30d_diff, '1d_diff': today_diff}, 'yesterday_diff': yesterday_diff, 'today_changes': today_changes, 'status': status }) # 更新快取 _DASHBOARD_DATA_CACHE['consolidated_data'] = unique_items _DASHBOARD_DATA_CACHE['consolidated_timestamp'] = now.timestamp() _DASHBOARD_DATA_CACHE['today_start'] = today_start query_duration_ms = (time.time() - query_start_time) * 1000 track_query_time('get_consolidated_data', query_duration_ms) sys_log.debug(f"[Dashboard] [Cache] 快取已更新 | 商品數: {len(unique_items)} | 耗時: {query_duration_ms:.0f}ms") return unique_items, today_start finally: session.close() def get_full_dashboard_data(): """獲取完整的看板資料，包含快取清單與全部 KPIs (深度快取)""" global _DASHBOARD_DATA_CACHE now = datetime.now(TAIPEI_TZ) # V-Opt: 先檢查快取（無需鎖） if _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'): age = now.timestamp() - _DASHBOARD_DATA_CACHE['full_timestamp'] if age < _DASHBOARD_CACHE_TTL: sys_log.debug(f"[Dashboard] [Cache] ✅ 使用完整看板快取 | 快取年齡: {age:.0f}秒") return _DASHBOARD_DATA_CACHE['full_data'] # V-Opt: 使用檔案鎖避免多 gunicorn worker 同時計算 if not _DASHBOARD_FILE_LOCK.acquire(blocking=False): # 如果無法取得鎖，表示其他 worker 正在重建，等待並使用更新後的快取 sys_log.debug("[Dashboard] [Cache] ⏳ 等待其他 worker 重建快取...") _DASHBOARD_FILE_LOCK.acquire() # 等待取得鎖 _DASHBOARD_FILE_LOCK.release() # 立即釋放 # 返回更新後的快取 if _DASHBOARD_DATA_CACHE.get('full_data'): return _DASHBOARD_DATA_CACHE['full_data'] try: # 再次檢查快取（可能其他 worker 已經更新） if _DASHBOARD_DATA_CACHE.get('full_data') and _DASHBOARD_DATA_CACHE.get('full_timestamp'): age = now.timestamp() - _DASHBOARD_DATA_CACHE['full_timestamp'] if age < _DASHBOARD_CACHE_TTL: sys_log.debug(f"[Dashboard] [Cache] ✅ 使用完整看板快取 (其他 worker 已更新) | 快取年齡: {age:.0f}秒") return _DASHBOARD_DATA_CACHE['full_data'] sys_log.info("[Dashboard] [Cache] 🔄 完整快取過期，重新計算所有 KPIs 與統計數據...") query_start_time = time.time() unique_items, today_start = get_consolidated_data() today_start_db = today_start # 保持台北時區 db = DatabaseManager() session = db.get_session() try: # A. 基礎清單統計 increase_items = [item for item in unique_items if item['yesterday_diff'] > 0] decrease_items = [item for item in unique_items if item['yesterday_diff'] < 0] # B. 分類筆數統計 cat_counts = {} for item in unique_items: c = item['record'].product.category if c: cat_counts[c] = cat_counts.get(c, 0) + 1 all_categories = [f"{cat} ({count}筆)" for cat, count in sorted(cat_counts.items())] # C. 核心 KPI 統計 total_products_history = session.query(Product).count() total_price_records = session.query(PriceRecord).count() today_updates = session.query(PriceRecord).filter(PriceRecord.timestamp >= today_start_db).count() # 今日新增商品 new_pids_query = session.query(PriceRecord.product_id).group_by( PriceRecord.product_id ).having(func.min(PriceRecord.timestamp) >= today_start_db) new_product_ids = {r[0] for r in new_pids_query.all()} today_new_products = len(new_product_ids) # D. 今日下架商品處理 raw_delisted_items = session.query(Product).filter( Product.status == 'INACTIVE', Product.updated_at >= today_start_db ).all() today_delisted_items = [] if raw_delisted_items: delisted_ids = [p.id for p in raw_delisted_items] last_prices_subq = session.query( PriceRecord.product_id, func.max(PriceRecord.id).label('max_id') ).filter(PriceRecord.product_id.in_(delisted_ids)).group_by(PriceRecord.product_id).subquery() last_prices_q = session.query(PriceRecord.product_id, PriceRecord.price).join( last_prices_subq, PriceRecord.id == last_prices_subq.c.max_id).all() price_map = {pid: price for pid, price in last_prices_q} for p in raw_delisted_items: today_delisted_items.append({'product': p, 'last_price': price_map.get(p.id, 0)}) # E. 週增長 week_ago_db = now.replace(hour=0, minute=0, second=0, microsecond=0) - timedelta(days=7) week_new_products = session.query(func.count(Product.id)).filter( Product.id.in_( session.query(PriceRecord.product_id) .group_by(PriceRecord.product_id) .having(func.min(PriceRecord.timestamp) >= week_ago_db) ) ).scalar() or 0 # F. 價格穩定商品數 try: stable_count = session.query(PriceRecord.product_id).filter( PriceRecord.timestamp >= week_ago_db ).group_by(PriceRecord.product_id).having( func.count(func.distinct(PriceRecord.price)) == 1 ).count() except Exception: stable_count = 0 # G. 最大變動計算 max_change_item = None max_change_value = 0 for item in unique_items: if abs(item['yesterday_diff']) > abs(max_change_value): max_change_value = item['yesterday_diff'] max_change_item = item # H. 最活躍分類 category_activity = {} for item in increase_items + decrease_items: cat = item['record'].product.category if cat: category_activity[cat] = category_activity.get(cat, 0) + 1 most_active_category_item = max(category_activity.items(), key=lambda x: x[1]) if category_activity else (None, 0) # I. 組合結果 full_data = { 'unique_items': unique_items, 'today_start': today_start, 'today_start_db': today_start_db, 'increase_items_all': increase_items, 'decrease_items_all': decrease_items, 'all_categories': all_categories, 'new_product_ids': new_product_ids, 'total_products_history': total_products_history, 'total_price_records': total_price_records, 'today_updates': today_updates, 'today_new_products': today_new_products, 'today_delisted_count': len(raw_delisted_items), 'today_delisted_items': today_delisted_items, 'max_change_item': max_change_item, 'max_change_value': max_change_value, 'avg_increase': sum(item['yesterday_diff'] for item in increase_items) / len(increase_items) if increase_items else 0, 'avg_decrease': sum(item['yesterday_diff'] for item in decrease_items) / len(decrease_items) if decrease_items else 0, 'activity_rate': (len(increase_items) + len(decrease_items)) / total_products_history * 100 if total_products_history > 0 else 0, 'active_count': len(increase_items) + len(decrease_items), 'week_new_products': week_new_products, 'stable_count': stable_count, 'most_active_category': most_active_category_item[0], 'most_active_count': most_active_category_item[1] } # 更新快取 _DASHBOARD_DATA_CACHE['full_data'] = full_data _DASHBOARD_DATA_CACHE['full_timestamp'] = now.timestamp() query_duration_ms = (time.time() - query_start_time) * 1000 track_query_time('get_full_dashboard_data', query_duration_ms) sys_log.info(f"[Dashboard] [Cache] ✅ 完整看板快取已更新 | 耗時: {query_duration_ms:.0f}ms") return full_data except Exception as e: sys_log.error(f"[Dashboard] KPI 計算失敗: {e}") import traceback traceback.print_exc() return None finally: session.close() finally: # V-Opt: 確保釋放檔案鎖 _DASHBOARD_FILE_LOCK.release() def get_dashboard_stats(): """計算看板統計數據 (供通知使用)""" data = get_full_dashboard_data() if data: return { 'new': data['today_new_products'], 'up': len(data['increase_items_all']), 'down': len(data['decrease_items_all']), 'delisted': data['today_delisted_count'] } return {'new': 0, 'up': 0, 'down': 0, 'delisted': 0} # ========================================== # 頁面路由 # ========================================== @dashboard_bp.route('/') @login_required def index(): """商品看板首頁""" db = DatabaseManager() session = db.get_session() page = request.args.get('page', 1, type=int) category_filter = request.args.get('category', 'all') sort_by = request.args.get('sort_by', 'timestamp') filter_type = request.args.get('filter', 'all') order = request.args.get('order', 'desc') search_query = request.args.get('q', '').strip() per_page = 50 now_taipei = datetime.now(TAIPEI_TZ) today_start_db = now_taipei.replace(hour=0, minute=0, second=0, microsecond=0) # 保持台北時區 try: # 使用深度快取獲取所有數據 data = get_full_dashboard_data() if not data: return render_template('index.html', error="無法載入數據，請檢查資料庫。") unique_items = data['unique_items'] today_start = data['today_start'] today_start_db = data['today_start_db'] increase_items = data['increase_items_all'] decrease_items = data['decrease_items_all'] all_categories = data['all_categories'] new_product_ids = data['new_product_ids'] total_products_history = data['total_products_history'] today_new_products = data['today_new_products'] total_price_records = data['total_price_records'] today_updates = data['today_updates'] today_delisted_count = data['today_delisted_count'] today_delisted_items = data['today_delisted_items'] max_change_item = data['max_change_item'] max_change_value = data['max_change_value'] avg_increase = data['avg_increase'] avg_decrease = data['avg_decrease'] activity_rate = data['activity_rate'] week_new_products = data['week_new_products'] stable_count = data['stable_count'] most_active_category = data['most_active_category'] most_active_count = data['most_active_count'] active_count = data.get('active_count', 0) # 讀取系統狀態 system_status = {"status": "UNKNOWN", "message": "尚無執行紀錄", "timestamp": "-"} status_path = os.path.join(BASE_DIR, 'data/system_status.json') if os.path.exists(status_path): try: with open(status_path, 'r', encoding='utf-8') as f: system_status = json.load(f) except: pass # 後端篩選 scheduler_stats = load_scheduler_stats() # Handle old scheduler stats format if scheduler_stats.get('momo_task') and isinstance(scheduler_stats.get('momo_task'), dict): scheduler_stats['momo_task'] = [scheduler_stats['momo_task']] if scheduler_stats.get('edm_task') and isinstance(scheduler_stats.get('edm_task'), dict): scheduler_stats['edm_task'] = [scheduler_stats['edm_task']] filtered_items = [] # 先處理搜尋 if search_query: search_lower = search_query.lower() base_items = [ item for item in unique_items if (item['record'].product.name and search_lower in item['record'].product.name.lower()) or (item['record'].product.i_code and search_lower in str(item['record'].product.i_code)) ] else: base_items = unique_items # 處理狀態篩選 if filter_type == 'increase': filtered_items = [i for i in base_items if i in increase_items] elif filter_type == 'decrease': filtered_items = [i for i in base_items if i in decrease_items] elif filter_type == 'new': filtered_items = [i for i in base_items if i['record'].product_id in new_product_ids] elif filter_type == 'delisted': for item in today_delisted_items: class MockRecord: def __init__(self, p, price): self.product = p self.price = price self.timestamp = p.updated_at if not search_query or search_query.lower() in item['product'].name.lower(): filtered_items.append({ 'record': MockRecord(item['product'], item['last_price']), 'stats': {'1d_diff': 0, '7d_diff': 0, '30d_diff': 0}, 'yesterday_diff': 0, 'today_changes': [], 'status': 'DELISTED' }) else: if category_filter != 'all': real_category = category_filter if "(" in category_filter and "筆)" in category_filter: real_category = category_filter.rsplit(" (", 1)[0] filtered_items = [item for item in base_items if item['record'].product.category == real_category] else: filtered_items = base_items # 後端排序 reverse = (order == 'desc') def get_sort_key(item): def safe_get(value, default=0): return default if value is None else value if sort_by == 'i_code': return int(safe_get(item['record'].product.i_code, 0)) if sort_by == 'category': return safe_get(item['record'].product.category, '') if sort_by == 'name': return safe_get(item['record'].product.name, '') if sort_by == 'price': return safe_get(item['record'].price, 0) if sort_by == 'today_change': return safe_get(item['stats']['1d_diff'], 0) if sort_by == 'yesterday_change': return safe_get(item['yesterday_diff'], 0) if sort_by == 'week_change': return safe_get(item['stats']['7d_diff'], 0) return item['record'].timestamp sorted_items = sorted(filtered_items, key=get_sort_key, reverse=reverse) # 分頁 total_items = len(sorted_items) total_pages = math.ceil(total_items / per_page) start_idx = (page - 1) * per_page paged_items = sorted_items[start_idx: start_idx + per_page] # 為前端準備安全的 created_at 屬性 for item in paged_items: item['safe_created_at'] = getattr(item['record'].product, 'created_at', None) # 為當前頁面項目添加顏色 for item in paged_items: category_name = item['record'].product.category item['category_color'] = get_color_for_string(category_name) return render_template('dashboard.html', total_products=total_products_history, today_new_products=today_new_products, total_price_records=total_price_records, cnt_increase=len(increase_items), cnt_decrease=len(decrease_items), today_delisted_count=today_delisted_count, today_delisted_items=today_delisted_items, system_status=system_status, items=paged_items, categories=all_categories, current_page=page, total_pages=total_pages, total_items=total_items, datetime_now=now_taipei.strftime('%Y-%m-%d %H:%M:%S'), today_date=now_taipei.strftime('%Y-%m-%d'), public_url=public_url, current_category=category_filter, current_filter=filter_type, search_query=search_query, current_sort=sort_by, current_order=order, scheduler_stats=scheduler_stats, avg_increase=avg_increase, avg_decrease=avg_decrease, activity_rate=activity_rate, active_count=active_count, max_change_item=max_change_item, max_change_value=max_change_value, week_new_products=week_new_products, stable_count=stable_count, most_active_category=most_active_category, most_active_count=most_active_count, active_page='dashboard') except Exception as e: sys_log.error(f"[Web] [Dashboard] 渲染錯誤 | Error: {e}") return f"系統維護中，錯誤詳情：{e}" finally: session.close() @dashboard_bp.route('/brand_assets') @login_required def brand_assets(): """顯示品牌資產庫""" return render_template('brand_assets.html')