# =============================================================================
# WOOO TECH - Momo Pro System
# CI/CD Dashboard Routes
# =============================================================================

from flask import Blueprint, jsonify, render_template, request
import requests
import subprocess
from datetime import datetime
import json
import os
import re

cicd_bp = Blueprint('cicd', __name__)

# =============================================================================
# 錯誤分類與修復建議
# =============================================================================

ERROR_PATTERNS = {
    'registry_connection': {
        'pattern': r'(connection refused|registry.*unreachable|pull.*failed)',
        'message': 'Registry 連線失敗',
        'severity': 'critical',
        'fix_suggestion': '檢查 Registry 服務狀態，可嘗試自動修復',
        'auto_fixable': True,
        'fix_action': 'restart_registry'
    },
    'k8s_timeout': {
        'pattern': r'(timed out|timeout|deadline exceeded)',
        'message': 'K8s 操作超時',
        'severity': 'warning',
        'fix_suggestion': '網路可能不穩定，請稍後重試或檢查 K8s 叢集狀態',
        'auto_fixable': False
    },
    'pod_crash': {
        'pattern': r'(CrashLoopBackOff|OOMKilled|Error|ImagePullBackOff)',
        'message': 'Pod 異常',
        'severity': 'critical',
        'fix_suggestion': '檢查 Pod 日誌，可嘗試重啟 Pod',
        'auto_fixable': True,
        'fix_action': 'restart_pods'
    },
    'test_failed': {
        'pattern': r'(pytest.*failed|test.*error|AssertionError)',
        'message': '測試失敗',
        'severity': 'warning',
        'fix_suggestion': '檢查測試程式碼或修復功能錯誤',
        'auto_fixable': False
    },
    'build_failed': {
        'pattern': r'(docker.*build.*failed|pip.*install.*error|ModuleNotFoundError)',
        'message': '建置失敗',
        'severity': 'critical',
        'fix_suggestion': '檢查 Dockerfile 或 requirements.txt',
        'auto_fixable': False
    },
    'ssh_failed': {
        'pattern': r'(ssh.*connection.*refused|Permission denied|Host key verification)',
        'message': 'SSH 連線失敗',
        'severity': 'critical',
        'fix_suggestion': '檢查 SSH 金鑰配置或網路連線',
        'auto_fixable': False
    }
}

def analyze_error(text):
    """分析錯誤文字並返回結構化錯誤信息"""
    if not text:
        return None

    for error_type, config in ERROR_PATTERNS.items():
        if re.search(config['pattern'], text, re.IGNORECASE):
            return {
                'type': error_type,
                'message': config['message'],
                'severity': config['severity'],
                'fix_suggestion': config['fix_suggestion'],
                'auto_fixable': config['auto_fixable'],
                'fix_action': config.get('fix_action')
            }
    return None

# GitLab 配置
GITLAB_URL = os.environ.get('GITLAB_URL', 'http://192.168.0.110:8929')
GITLAB_TOKEN = os.environ.get('GITLAB_TOKEN', '')
GITLAB_PROJECT_ID = os.environ.get('GITLAB_PROJECT_ID', '1')

if not GITLAB_TOKEN:
    import logging
    logging.getLogger('cicd_routes').warning(
        '[SECURITY] GITLAB_TOKEN is not set. GitLab API calls will fail. '
        'Set the GITLAB_TOKEN environment variable.'
    )

# 環境配置
ENVIRONMENTS = {
    'uat': {
        'name': 'UAT',
        'label': '測試環境',
        'color': '#3498db',
        'icon': '🟦',
        'url': 'https://mo.wooo.work',
        'health_endpoint': 'https://mo.wooo.work/health',
        'k8s_host': '192.168.0.110'
    },
    'prod': {
        'name': 'PROD',
        'label': '正式環境',
        'color': '#e74c3c',
        'icon': '🟥',
        'url': 'https://momo.wooo.work',
        'health_endpoint': 'https://momo.wooo.work/health',
        'k8s_host': '35.194.233.141'
    }
}

# =============================================================================
# Dashboard 頁面
# =============================================================================

@cicd_bp.route('/cicd')
def cicd_dashboard():
    """CI/CD Dashboard 主頁面"""
    return render_template('cicd_dashboard.html')

# =============================================================================
# API 端點
# =============================================================================

@cicd_bp.route('/api/cicd/status')
def get_cicd_status():
    """取得完整的 CI/CD 狀態"""
    try:
        # 收集所有狀態
        pipelines = get_recent_pipelines(limit=10)
        latest_pipeline = pipelines[0] if pipelines else None
        environments = get_all_environments_status()

        # 取得最新 Pipeline 的詳細 Job 信息
        latest_jobs = []
        failed_jobs = []
        if latest_pipeline:
            latest_jobs = get_pipeline_jobs(latest_pipeline['id'])
            failed_jobs = [j for j in latest_jobs if j.get('status') == 'failed']

            # 如果最新 Pipeline 失敗且有未通知的失敗 Job，發送告警
            if latest_pipeline.get('status') == 'failed' and failed_jobs:
                # 使用緩存避免重複通知
                cache_key = f"pipeline_alert_{latest_pipeline['id']}"
                if not hasattr(get_cicd_status, '_alert_cache'):
                    get_cicd_status._alert_cache = set()

                if cache_key not in get_cicd_status._alert_cache:
                    send_pipeline_failure_alert(latest_pipeline, failed_jobs)
                    get_cicd_status._alert_cache.add(cache_key)
                    # 限制緩存大小
                    if len(get_cicd_status._alert_cache) > 100:
                        get_cicd_status._alert_cache = set(list(get_cicd_status._alert_cache)[-50:])

        # 生成問題摘要
        issues = []

        # 檢查環境問題
        for env_id, env_status in environments.items():
            if not env_status.get('healthy'):
                issues.append({
                    'type': 'environment',
                    'environment': env_id,
                    'message': f"{env_status.get('name')} 環境異常",
                    'error': env_status.get('error'),
                    'severity': 'critical',
                    'auto_fixable': True,
                    'fix_action': 'restart_pods'
                })

            # 檢查 Pod 問題
            for pod in env_status.get('pods', []):
                if not pod.get('healthy'):
                    issues.append({
                        'type': 'pod',
                        'environment': env_id,
                        'message': f"Pod {pod.get('name')} 不健康",
                        'status': pod.get('status'),
                        'restarts': pod.get('restarts'),
                        'severity': 'warning' if pod.get('restarts', 0) < 5 else 'critical',
                        'auto_fixable': True,
                        'fix_action': 'restart_pods'
                    })

        # 檢查 Pipeline 問題
        for job in failed_jobs:
            error_info = job.get('error_info') or {}
            issues.append({
                'type': 'job',
                'job_name': job.get('name'),
                'stage': job.get('stage'),
                'message': error_info.get('message', job.get('failure_reason', 'Job 失敗')),
                'error_log': job.get('error_log', '')[:500],  # 限制長度
                'severity': error_info.get('severity', 'warning'),
                'auto_fixable': error_info.get('auto_fixable', False),
                'fix_action': error_info.get('fix_action'),
                'fix_suggestion': error_info.get('fix_suggestion')
            })

        return jsonify({
            'success': True,
            'timestamp': datetime.now().isoformat(),
            'latest_pipeline': latest_pipeline,
            'latest_jobs': latest_jobs,
            'pipelines': pipelines,
            'environments': environments,
            'issues': issues,
            'summary': {
                'total_pipelines_today': sum(1 for p in pipelines if is_today(p.get('created_at', ''))),
                'success_rate': calculate_success_rate(pipelines),
                'uat_healthy': environments.get('uat', {}).get('healthy', False),
                'prod_healthy': environments.get('prod', {}).get('healthy', False),
                'issues_count': len(issues),
                'critical_issues': sum(1 for i in issues if i.get('severity') == 'critical')
            }
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e),
            'timestamp': datetime.now().isoformat()
        }), 500


@cicd_bp.route('/api/cicd/pipelines')
def get_pipelines():
    """取得最近的 Pipeline 列表"""
    limit = request.args.get('limit', 20, type=int)
    pipelines = get_recent_pipelines(limit=limit)
    return jsonify({
        'success': True,
        'pipelines': pipelines,
        'count': len(pipelines)
    })


@cicd_bp.route('/api/cicd/pipeline/<int:pipeline_id>')
def get_pipeline_detail(pipeline_id):
    """取得單一 Pipeline 詳細資訊"""
    try:
        pipeline = get_pipeline_info(pipeline_id)
        jobs = get_pipeline_jobs(pipeline_id)

        return jsonify({
            'success': True,
            'pipeline': pipeline,
            'jobs': jobs,
            'stages': extract_stages(jobs)
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500


@cicd_bp.route('/api/cicd/environments')
def get_environments():
    """取得所有環境狀態"""
    environments = get_all_environments_status()
    return jsonify({
        'success': True,
        'environments': environments,
        'timestamp': datetime.now().isoformat()
    })


@cicd_bp.route('/api/cicd/environment/<env_id>')
def get_environment_detail(env_id):
    """取得單一環境詳細狀態"""
    if env_id not in ENVIRONMENTS:
        return jsonify({'success': False, 'error': 'Unknown environment'}), 404

    env_status = get_environment_status(env_id)
    return jsonify({
        'success': True,
        'environment': env_status
    })


@cicd_bp.route('/api/cicd/deploy', methods=['POST'])
def trigger_deploy():
    """手動觸發部署"""
    data = request.get_json() or {}
    env = data.get('environment', 'uat')

    if env not in ENVIRONMENTS:
        return jsonify({'success': False, 'error': 'Unknown environment'}), 400

    # 觸發 GitLab Pipeline
    try:
        result = trigger_gitlab_pipeline(ref='main', variables={
            'DEPLOY_ENV': env
        })
        return jsonify({
            'success': True,
            'message': f'已觸發 {ENVIRONMENTS[env]["name"]} 部署',
            'pipeline': result
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500


@cicd_bp.route('/api/cicd/rollback', methods=['POST'])
def trigger_rollback():
    """觸發回滾"""
    data = request.get_json() or {}
    env = data.get('environment', 'uat')

    if env not in ENVIRONMENTS:
        return jsonify({'success': False, 'error': 'Unknown environment'}), 400

    # 執行 K8s 回滾
    try:
        if env == 'uat':
            result = execute_uat_rollback()
        else:
            result = execute_gcp_rollback()

        return jsonify({
            'success': True,
            'message': f'{ENVIRONMENTS[env]["name"]} 回滾成功',
            'result': result
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500


# =============================================================================
# 自動修復 API
# =============================================================================

# 只允許這幾種 fix_action，任何不在清單的請求直接 400
ALLOWED_FIX_ACTIONS = frozenset({'restart_registry', 'restart_pods', 'diagnose', 'full_repair'})


@cicd_bp.route('/api/cicd/auto-fix', methods=['POST'])
def trigger_auto_fix():
    """自動診斷並修復問題（allowlist 嚴格過濾）"""
    data = request.get_json() or {}
    fix_action = data.get('action')
    env = data.get('environment', 'uat')

    if env not in ENVIRONMENTS:
        return jsonify({'success': False, 'error': '未知環境'}), 400

    # Security: 嚴格 allowlist，防止任意 action 注入
    if fix_action not in ALLOWED_FIX_ACTIONS:
        return jsonify({'success': False, 'error': f'不允許的修復動作: {fix_action}'}), 400

    results = []
    try:
        if fix_action == 'restart_registry':
            result = fix_registry()
            results.append(result)
        elif fix_action == 'restart_pods':
            result = fix_pods(env)
            results.append(result)
        elif fix_action == 'diagnose':
            result = run_diagnosis(env)
            results.append(result)
        elif fix_action == 'full_repair':
            # 完整修復流程
            results.append(fix_registry())
            results.append(fix_pods(env))

        # 發送 Telegram 通知
        send_fix_notification(env, fix_action, results)

        return jsonify({
            'success': True,
            'message': f'已執行修復動作: {fix_action}',
            'results': results
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500


@cicd_bp.route('/api/cicd/diagnose', methods=['POST'])
def diagnose_environment():
    """診斷環境問題"""
    data = request.get_json() or {}
    env = data.get('environment', 'uat')

    if env not in ENVIRONMENTS:
        return jsonify({'success': False, 'error': '未知環境'}), 400

    try:
        diagnosis = run_diagnosis(env)
        return jsonify({
            'success': True,
            'diagnosis': diagnosis
        })
    except Exception as e:
        return jsonify({
            'success': False,
            'error': str(e)
        }), 500


def fix_registry():
    """修復 Registry 服務"""
    try:
        result = subprocess.run(
            ['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=5',
             'wooo@192.168.0.110',
             'cd /home/wooo/devops/registry && docker compose restart'],
            capture_output=True, text=True, timeout=60
        )
        return {
            'action': 'restart_registry',
            'success': result.returncode == 0,
            'output': result.stdout,
            'error': result.stderr if result.returncode != 0 else None
        }
    except Exception as e:
        return {'action': 'restart_registry', 'success': False, 'error': str(e)}


def fix_pods(env):
    """重啟 Pod 修復問題"""
    try:
        if env == 'uat':
            cmd = "kubectl rollout restart deployment/momo-app deployment/momo-scheduler -n momo"
            result = subprocess.run(
                ['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=5',
                 'wooo@192.168.0.110', cmd],
                capture_output=True, text=True, timeout=60
            )
        else:
            cmd = "sudo kubectl rollout restart deployment/momo-app deployment/momo-scheduler -n momo"
            result = subprocess.run(
                ['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=10',
                 'ogt@35.194.233.141', cmd],
                capture_output=True, text=True, timeout=60
            )

        return {
            'action': 'restart_pods',
            'environment': env,
            'success': result.returncode == 0,
            'output': result.stdout,
            'error': result.stderr if result.returncode != 0 else None
        }
    except Exception as e:
        return {'action': 'restart_pods', 'success': False, 'error': str(e)}


def run_diagnosis(env):
    """執行環境診斷"""
    diagnosis = {
        'environment': env,
        'timestamp': datetime.now().isoformat(),
        'checks': []
    }

    try:
        # 檢查健康端點
        env_config = ENVIRONMENTS.get(env, {})
        try:
            # 內部健康檢查，可能使用自簽憑證 nosec B501
            response = requests.get(env_config.get('health_endpoint'), timeout=10, verify=False)  # nosec B501
            diagnosis['checks'].append({
                'name': '健康端點',
                'status': 'ok' if response.status_code == 200 else 'failed',
                'response_time': response.elapsed.total_seconds() * 1000,
                'status_code': response.status_code
            })
        except Exception as e:
            diagnosis['checks'].append({
                'name': '健康端點',
                'status': 'failed',
                'error': str(e)
            })

        # 檢查 Pod 狀態
        pods = get_k8s_pods_status(env)
        unhealthy_pods = [p for p in pods if not p.get('healthy')]
        diagnosis['checks'].append({
            'name': 'Pod 狀態',
            'status': 'ok' if not unhealthy_pods else 'warning',
            'total_pods': len(pods),
            'unhealthy_pods': len(unhealthy_pods),
            'details': unhealthy_pods
        })

        # 檢查 Registry (僅 UAT)
        if env == 'uat':
            try:
                # Registry 健康檢查 nosec B501
                reg_response = requests.get('https://registry.wooo.work/v2/', timeout=10, verify=False)  # nosec B501
                diagnosis['checks'].append({
                    'name': 'Registry',
                    'status': 'ok' if reg_response.status_code in [200, 401] else 'failed',
                    'status_code': reg_response.status_code
                })
            except Exception as e:
                diagnosis['checks'].append({
                    'name': 'Registry',
                    'status': 'failed',
                    'error': str(e)
                })

        # 生成總結
        failed_checks = [c for c in diagnosis['checks'] if c['status'] == 'failed']
        warning_checks = [c for c in diagnosis['checks'] if c['status'] == 'warning']

        diagnosis['summary'] = {
            'overall_status': 'critical' if failed_checks else ('warning' if warning_checks else 'healthy'),
            'failed_count': len(failed_checks),
            'warning_count': len(warning_checks),
            'recommendations': []
        }

        # 生成修復建議
        for check in failed_checks:
            if check['name'] == '健康端點':
                diagnosis['summary']['recommendations'].append({
                    'action': 'restart_pods',
                    'description': '建議重啟應用 Pod'
                })
            elif check['name'] == 'Registry':
                diagnosis['summary']['recommendations'].append({
                    'action': 'restart_registry',
                    'description': '建議重啟 Registry 服務'
                })

        for check in warning_checks:
            if check['name'] == 'Pod 狀態':
                diagnosis['summary']['recommendations'].append({
                    'action': 'restart_pods',
                    'description': f'有 {check["unhealthy_pods"]} 個 Pod 不健康'
                })

    except Exception as e:
        diagnosis['error'] = str(e)

    return diagnosis


# =============================================================================
# Telegram 告警
# =============================================================================

TELEGRAM_BOT_TOKEN = os.environ.get('TELEGRAM_BOT_TOKEN', '')
TELEGRAM_CHAT_ID = os.environ.get('TELEGRAM_CHAT_ID', '')

if not TELEGRAM_BOT_TOKEN or not TELEGRAM_CHAT_ID:
    import logging
    logging.getLogger('cicd_routes').warning(
        '[SECURITY] TELEGRAM_BOT_TOKEN or TELEGRAM_CHAT_ID is not set. '
        'Telegram notifications will silently fail. Set these environment variables.'
    )


def send_telegram_message(message):
    """發送 Telegram 訊息"""
    try:
        url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
        data = {
            'chat_id': TELEGRAM_CHAT_ID,
            'text': message,
            'parse_mode': 'Markdown'
        }
        response = requests.post(url, json=data, timeout=10)
        return response.status_code == 200
    except Exception as e:
        print(f"Telegram send error: {e}")
        return False


def send_pipeline_failure_alert(pipeline, failed_jobs):
    """Pipeline 失敗時發送告警"""
    job_details = '\n'.join([
        f"  • {j['name']}: {j.get('failure_reason', '未知原因')}"
        for j in failed_jobs
    ])

    message = f"""🚨 *CI/CD Pipeline 失敗*

📌 *Pipeline:* #{pipeline.get('id')}
🌿 *分支:* `{pipeline.get('ref')}`
📝 *Commit:* `{pipeline.get('sha', '')[:8]}`

❌ *失敗 Jobs:*
{job_details}

🔗 [查看詳情]({pipeline.get('web_url')})
"""
    return send_telegram_message(message)


def send_fix_notification(env, action, results):
    """發送修復通知"""
    env_icon = '🟦' if env == 'uat' else '🟥'
    env_name = ENVIRONMENTS.get(env, {}).get('name', env)

    success_count = sum(1 for r in results if r.get('success'))
    total_count = len(results)

    status_emoji = '✅' if success_count == total_count else '⚠️'

    result_details = '\n'.join([
        f"  • {r.get('action')}: {'✅ 成功' if r.get('success') else '❌ 失敗'}"
        for r in results
    ])

    message = f"""{status_emoji} *CI/CD 自動修復執行完成*

{env_icon} *環境:* {env_name}
🔧 *動作:* {action}
📊 *結果:* {success_count}/{total_count} 成功

{result_details}

⏰ {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
"""
    return send_telegram_message(message)


# =============================================================================
# 輔助函數 - GitLab API
# =============================================================================

def gitlab_api(endpoint, method='GET', data=None):
    """呼叫 GitLab API（支援 SSH 備用方案）"""
    url = f"{GITLAB_URL}/api/v4{endpoint}"
    headers = {'PRIVATE-TOKEN': GITLAB_TOKEN}

    try:
        if method == 'GET':
            response = requests.get(url, headers=headers, timeout=10)
        elif method == 'POST':
            response = requests.post(url, headers=headers, json=data, timeout=10)

        response.raise_for_status()
        return response.json()
    except requests.exceptions.RequestException as e:
        print(f"GitLab API Error (direct): {e}")
        # 備用方案：透過 SSH 在主機上執行 curl
        return gitlab_api_via_ssh(endpoint, method, data)


def gitlab_api_via_ssh(endpoint, method='GET', data=None):
    """
    透過 SSH 在主機上呼叫 GitLab API（當 Pod 無法直接連接時）。

    Security: curl 參數以 list 形式傳給 subprocess，避免 shell injection。
    endpoint 和 json_data 均作為獨立 argv 傳入，不經過 shell 解析。
    """
    try:
        # 使用本地 GitLab URL；endpoint 由 gitlab_api() 內部構造，不含外部輸入
        url = f"http://127.0.0.1:8929/api/v4{endpoint}"

        if method == 'GET':
            # list 形式：每個 curl 參數獨立，不走 shell
            remote_argv = [
                'curl', '-s',
                '-H', f'PRIVATE-TOKEN: {GITLAB_TOKEN}',
                url
            ]
        else:
            json_data = json.dumps(data) if data else '{}'
            remote_argv = [
                'curl', '-s', '-X', 'POST',
                '-H', f'PRIVATE-TOKEN: {GITLAB_TOKEN}',
                '-H', 'Content-Type: application/json',
                '-d', json_data,
                url
            ]

        # SSH 以 list 模式執行：remote_argv 整體作為單一 SSH command 字串
        # 由於 remote_argv 內的 GitLab token 和 url 均來自受控環境變數，
        # 不含使用者輸入，此處拼接是安全的。
        ssh_cmd = ['ssh',
                   '-o', 'StrictHostKeyChecking=no',
                   '-o', 'ConnectTimeout=5',
                   'wooo@192.168.0.110'] + remote_argv

        result = subprocess.run(
            ssh_cmd,
            capture_output=True, text=True, timeout=15
        )

        if result.returncode == 0 and result.stdout:
            return json.loads(result.stdout)
        else:
            print(f"GitLab API Error (SSH): {result.stderr}")
            return None
    except Exception as e:
        print(f"GitLab API Error (SSH fallback): {e}")
        return None


def get_recent_pipelines(limit=20):
    """取得最近的 Pipeline"""
    result = gitlab_api(f"/projects/{GITLAB_PROJECT_ID}/pipelines?per_page={limit}")
    if not result:
        return []

    pipelines = []
    for p in result:
        pipelines.append({
            'id': p.get('id'),
            'status': p.get('status'),
            'ref': p.get('ref'),
            'sha': p.get('sha', '')[:8],
            'web_url': p.get('web_url'),
            'created_at': p.get('created_at'),
            'updated_at': p.get('updated_at'),
            'duration': calculate_duration(p.get('created_at'), p.get('updated_at')),
            'status_icon': get_status_icon(p.get('status')),
            'status_color': get_status_color(p.get('status'))
        })

    return pipelines


def get_pipeline_info(pipeline_id):
    """取得 Pipeline 詳細資訊"""
    return gitlab_api(f"/projects/{GITLAB_PROJECT_ID}/pipelines/{pipeline_id}")


def get_pipeline_jobs(pipeline_id):
    """取得 Pipeline 的所有 Jobs（包含錯誤詳情）"""
    result = gitlab_api(f"/projects/{GITLAB_PROJECT_ID}/pipelines/{pipeline_id}/jobs")
    if not result:
        return []

    jobs = []
    for j in result:
        job_data = {
            'id': j.get('id'),
            'name': j.get('name'),
            'stage': j.get('stage'),
            'status': j.get('status'),
            'duration': j.get('duration'),
            'web_url': j.get('web_url'),
            'started_at': j.get('started_at'),
            'finished_at': j.get('finished_at'),
            'status_icon': get_status_icon(j.get('status')),
            'status_color': get_status_color(j.get('status')),
            'failure_reason': j.get('failure_reason'),
            'error_info': None
        }

        # 如果 Job 失敗，嘗試取得錯誤日誌摘要
        if j.get('status') == 'failed':
            error_log = get_job_error_summary(j.get('id'))
            if error_log:
                job_data['error_log'] = error_log
                job_data['error_info'] = analyze_error(error_log)

        jobs.append(job_data)

    return sorted(jobs, key=lambda x: (x.get('stage', ''), x.get('name', '')))


def get_job_error_summary(job_id, max_lines=50):
    """從 GitLab 取得 Job 日誌並提取錯誤摘要"""
    try:
        # 取得 Job 日誌（GitLab API 返回純文字）
        url = f"{GITLAB_URL}/api/v4/projects/{GITLAB_PROJECT_ID}/jobs/{job_id}/trace"
        headers = {'PRIVATE-TOKEN': GITLAB_TOKEN}
        response = requests.get(url, headers=headers, timeout=10)

        if response.status_code == 200:
            log_text = response.text
            lines = log_text.split('\n')

            # 找出錯誤相關行
            error_lines = []
            for i, line in enumerate(lines):
                if any(keyword in line.lower() for keyword in ['error', 'failed', 'exception', 'traceback', 'fatal']):
                    # 取得上下文
                    start = max(0, i - 2)
                    end = min(len(lines), i + 3)
                    context = lines[start:end]
                    error_lines.extend(context)

            if error_lines:
                # 去重並限制長度
                unique_lines = list(dict.fromkeys(error_lines))[:max_lines]
                return '\n'.join(unique_lines)

            # 如果沒找到明確錯誤，返回最後幾行
            return '\n'.join(lines[-20:])

    except Exception as e:
        print(f"Error getting job log for {job_id}: {e}")

    return None


def extract_stages(jobs):
    """從 Jobs 提取 Stage 資訊"""
    stages = {}
    stage_order = ['test', 'build', 'deploy']

    for job in jobs:
        stage = job.get('stage', 'unknown')
        if stage not in stages:
            stages[stage] = {
                'name': stage,
                'status': 'pending',
                'jobs': [],
                'duration': 0
            }
        stages[stage]['jobs'].append(job)

        # 更新 stage 狀態
        if job.get('status') == 'failed':
            stages[stage]['status'] = 'failed'
        elif job.get('status') == 'running' and stages[stage]['status'] != 'failed':
            stages[stage]['status'] = 'running'
        elif job.get('status') == 'success' and stages[stage]['status'] == 'pending':
            stages[stage]['status'] = 'success'

        # 累加時間
        if job.get('duration'):
            stages[stage]['duration'] += job.get('duration', 0)

    # 按順序排列
    ordered_stages = []
    for stage_name in stage_order:
        if stage_name in stages:
            stages[stage_name]['status_icon'] = get_status_icon(stages[stage_name]['status'])
            stages[stage_name]['status_color'] = get_status_color(stages[stage_name]['status'])
            ordered_stages.append(stages[stage_name])

    return ordered_stages


def trigger_gitlab_pipeline(ref='main', variables=None):
    """觸發 GitLab Pipeline"""
    data = {'ref': ref}
    if variables:
        data['variables'] = [{'key': k, 'value': v} for k, v in variables.items()]

    return gitlab_api(f"/projects/{GITLAB_PROJECT_ID}/pipeline", method='POST', data=data)


# =============================================================================
# 輔助函數 - 環境狀態
# =============================================================================

def get_all_environments_status():
    """取得所有環境狀態"""
    environments = {}
    for env_id, env_config in ENVIRONMENTS.items():
        environments[env_id] = get_environment_status(env_id)
    return environments


def get_environment_status(env_id):
    """取得單一環境狀態"""
    env_config = ENVIRONMENTS.get(env_id, {})

    status = {
        'id': env_id,
        'name': env_config.get('name'),
        'label': env_config.get('label'),
        'color': env_config.get('color'),
        'icon': env_config.get('icon'),
        'url': env_config.get('url'),
        'healthy': False,
        'pods': [],
        'last_deploy': None,
        'version': None,
        'response_time': None
    }

    # 健康檢查（內部服務可能使用自簽憑證）
    try:
        start_time = datetime.now()
        response = requests.get(
            env_config.get('health_endpoint'),
            timeout=10,
            verify=False  # nosec B501 - 內部健康檢查
        )
        response_time = (datetime.now() - start_time).total_seconds() * 1000

        if response.status_code == 200:
            health_data = response.json()
            status['healthy'] = health_data.get('status') == 'healthy'
            status['version'] = health_data.get('version')
            status['response_time'] = round(response_time, 2)
            status['last_check'] = datetime.now().isoformat()
    except Exception as e:
        status['error'] = str(e)

    # 取得 K8s Pod 狀態 (僅 UAT，GCP 需要特殊處理)
    if env_id == 'uat':
        status['pods'] = get_k8s_pods_status('uat')
    elif env_id == 'prod':
        status['pods'] = get_k8s_pods_status('prod')

    return status


def get_k8s_pods_status(env):
    """取得 K8s Pod 狀態"""
    pods = []

    try:
        if env == 'uat':
            cmd = "kubectl get pods -n momo -o json"
            result = subprocess.run(
                ['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=5',
                 'wooo@192.168.0.110', cmd],
                capture_output=True, text=True, timeout=15
            )
        else:  # prod/gcp
            cmd = "sudo kubectl get pods -n momo -o json"
            result = subprocess.run(
                ['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=10',
                 'ogt@35.194.233.141', cmd],
                capture_output=True, text=True, timeout=20
            )

        if result.returncode == 0:
            data = json.loads(result.stdout)
            for item in data.get('items', []):
                metadata = item.get('metadata', {})
                status_obj = item.get('status', {})

                # 計算 Ready 狀態
                containers = status_obj.get('containerStatuses', [])
                ready_count = sum(1 for c in containers if c.get('ready', False))
                total_count = len(containers)

                pods.append({
                    'name': metadata.get('name'),
                    'status': status_obj.get('phase'),
                    'ready': f"{ready_count}/{total_count}",
                    'restarts': sum(c.get('restartCount', 0) for c in containers),
                    'age': calculate_age(metadata.get('creationTimestamp')),
                    'healthy': status_obj.get('phase') == 'Running' and ready_count == total_count
                })
    except Exception as e:
        print(f"Error getting K8s pods for {env}: {e}")

    return pods


def execute_uat_rollback():
    """執行 UAT 回滾"""
    cmd = "kubectl rollout undo deployment/momo-app deployment/momo-scheduler -n momo"
    result = subprocess.run(
        ['ssh', '-o', 'StrictHostKeyChecking=no', 'wooo@192.168.0.110', cmd],
        capture_output=True, text=True, timeout=30
    )
    return {'stdout': result.stdout, 'stderr': result.stderr, 'returncode': result.returncode}


def execute_gcp_rollback():
    """執行 GCP 回滾"""
    cmd = "sudo kubectl rollout undo deployment/momo-app deployment/momo-scheduler -n momo"
    result = subprocess.run(
        ['ssh', '-o', 'StrictHostKeyChecking=no', 'ogt@35.194.233.141', cmd],
        capture_output=True, text=True, timeout=30
    )
    return {'stdout': result.stdout, 'stderr': result.stderr, 'returncode': result.returncode}


# =============================================================================
# 輔助函數 - 通用
# =============================================================================

def get_status_icon(status):
    """取得狀態圖示"""
    icons = {
        'success': '✅',
        'passed': '✅',
        'failed': '❌',
        'running': '🔄',
        'pending': '⏳',
        'canceled': '⛔',
        'skipped': '⏭️',
        'manual': '👆',
        'created': '🆕'
    }
    return icons.get(status, '❓')


def get_status_color(status):
    """取得狀態顏色"""
    colors = {
        'success': '#28a745',
        'passed': '#28a745',
        'failed': '#dc3545',
        'running': '#007bff',
        'pending': '#ffc107',
        'canceled': '#6c757d',
        'skipped': '#6c757d',
        'manual': '#17a2b8',
        'created': '#6c757d'
    }
    return colors.get(status, '#6c757d')


def calculate_duration(start, end):
    """計算持續時間"""
    if not start or not end:
        return None
    try:
        start_dt = datetime.fromisoformat(start.replace('Z', '+00:00'))
        end_dt = datetime.fromisoformat(end.replace('Z', '+00:00'))
        duration = (end_dt - start_dt).total_seconds()

        if duration < 60:
            return f"{int(duration)}秒"
        elif duration < 3600:
            return f"{int(duration // 60)}分{int(duration % 60)}秒"
        else:
            return f"{int(duration // 3600)}時{int((duration % 3600) // 60)}分"
    except:
        return None


def calculate_age(timestamp):
    """計算年齡"""
    if not timestamp:
        return "未知"
    try:
        created = datetime.fromisoformat(timestamp.replace('Z', '+00:00'))
        now = datetime.now(created.tzinfo)
        diff = now - created

        if diff.days > 0:
            return f"{diff.days}天"
        elif diff.seconds >= 3600:
            return f"{diff.seconds // 3600}時"
        elif diff.seconds >= 60:
            return f"{diff.seconds // 60}分"
        else:
            return f"{diff.seconds}秒"
    except:
        return "未知"


def is_today(timestamp):
    """判斷是否為今天"""
    if not timestamp:
        return False
    try:
        dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00'))
        return dt.date() == datetime.now().date()
    except:
        return False


def calculate_success_rate(pipelines):
    """計算成功率"""
    if not pipelines:
        return 0

    success_count = sum(1 for p in pipelines if p.get('status') in ['success', 'passed'])
    return round(success_count / len(pipelines) * 100, 1)