1c03d213ac
All checks were successful
CD Pipeline / deploy (push) Successful in 1m22s
CVE-class issues fixed: 1. [HIGH] Shell Injection in gitlab_api_via_ssh (CWE-78) endpoint and json_data were interpolated into f-string cmd and passed as a single SSH remote command string → shell parses it → injection. Fix: build remote_argv as list; each curl argument is a separate item, SSH receives them as independent argv (no shell parsing of user data). 2. [HIGH] Hardcoded credentials in source code (CWE-798) GITLAB_TOKEN, TELEGRAM_BOT_TOKEN, TELEGRAM_CHAT_ID all had live secrets as default fallback values. Tokens are now '' (empty) with a startup warning if env vars are missing. 3. [MEDIUM] Missing pre-validation allowlist on fix_action (CWE-20) ALLOWED_FIX_ACTIONS frozenset added before route handler; any unknown action is rejected with 400 before reaching execution logic. Note: fix_registry/fix_pods/execute_*_rollback use static SSH commands (no user input in cmd strings) so they are not injection risks. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1075 lines
36 KiB
Python
1075 lines
36 KiB
Python
# =============================================================================
|
||
# WOOO TECH - Momo Pro System
|
||
# CI/CD Dashboard Routes
|
||
# =============================================================================
|
||
|
||
from flask import Blueprint, jsonify, render_template, request
|
||
import requests
|
||
import subprocess
|
||
from datetime import datetime
|
||
import json
|
||
import os
|
||
import re
|
||
|
||
cicd_bp = Blueprint('cicd', __name__)
|
||
|
||
# =============================================================================
|
||
# 錯誤分類與修復建議
|
||
# =============================================================================
|
||
|
||
ERROR_PATTERNS = {
|
||
'registry_connection': {
|
||
'pattern': r'(connection refused|registry.*unreachable|pull.*failed)',
|
||
'message': 'Registry 連線失敗',
|
||
'severity': 'critical',
|
||
'fix_suggestion': '檢查 Registry 服務狀態，可嘗試自動修復',
|
||
'auto_fixable': True,
|
||
'fix_action': 'restart_registry'
|
||
},
|
||
'k8s_timeout': {
|
||
'pattern': r'(timed out|timeout|deadline exceeded)',
|
||
'message': 'K8s 操作超時',
|
||
'severity': 'warning',
|
||
'fix_suggestion': '網路可能不穩定，請稍後重試或檢查 K8s 叢集狀態',
|
||
'auto_fixable': False
|
||
},
|
||
'pod_crash': {
|
||
'pattern': r'(CrashLoopBackOff|OOMKilled|Error|ImagePullBackOff)',
|
||
'message': 'Pod 異常',
|
||
'severity': 'critical',
|
||
'fix_suggestion': '檢查 Pod 日誌，可嘗試重啟 Pod',
|
||
'auto_fixable': True,
|
||
'fix_action': 'restart_pods'
|
||
},
|
||
'test_failed': {
|
||
'pattern': r'(pytest.*failed|test.*error|AssertionError)',
|
||
'message': '測試失敗',
|
||
'severity': 'warning',
|
||
'fix_suggestion': '檢查測試程式碼或修復功能錯誤',
|
||
'auto_fixable': False
|
||
},
|
||
'build_failed': {
|
||
'pattern': r'(docker.*build.*failed|pip.*install.*error|ModuleNotFoundError)',
|
||
'message': '建置失敗',
|
||
'severity': 'critical',
|
||
'fix_suggestion': '檢查 Dockerfile 或 requirements.txt',
|
||
'auto_fixable': False
|
||
},
|
||
'ssh_failed': {
|
||
'pattern': r'(ssh.*connection.*refused|Permission denied|Host key verification)',
|
||
'message': 'SSH 連線失敗',
|
||
'severity': 'critical',
|
||
'fix_suggestion': '檢查 SSH 金鑰配置或網路連線',
|
||
'auto_fixable': False
|
||
}
|
||
}
|
||
|
||
def analyze_error(text):
|
||
"""分析錯誤文字並返回結構化錯誤信息"""
|
||
if not text:
|
||
return None
|
||
|
||
for error_type, config in ERROR_PATTERNS.items():
|
||
if re.search(config['pattern'], text, re.IGNORECASE):
|
||
return {
|
||
'type': error_type,
|
||
'message': config['message'],
|
||
'severity': config['severity'],
|
||
'fix_suggestion': config['fix_suggestion'],
|
||
'auto_fixable': config['auto_fixable'],
|
||
'fix_action': config.get('fix_action')
|
||
}
|
||
return None
|
||
|
||
# GitLab 配置
|
||
GITLAB_URL = os.environ.get('GITLAB_URL', 'http://192.168.0.110:8929')
|
||
GITLAB_TOKEN = os.environ.get('GITLAB_TOKEN', '')
|
||
GITLAB_PROJECT_ID = os.environ.get('GITLAB_PROJECT_ID', '1')
|
||
|
||
if not GITLAB_TOKEN:
|
||
import logging
|
||
logging.getLogger('cicd_routes').warning(
|
||
'[SECURITY] GITLAB_TOKEN is not set. GitLab API calls will fail. '
|
||
'Set the GITLAB_TOKEN environment variable.'
|
||
)
|
||
|
||
# 環境配置
|
||
ENVIRONMENTS = {
|
||
'uat': {
|
||
'name': 'UAT',
|
||
'label': '測試環境',
|
||
'color': '#3498db',
|
||
'icon': '🟦',
|
||
'url': 'https://mo.wooo.work',
|
||
'health_endpoint': 'https://mo.wooo.work/health',
|
||
'k8s_host': '192.168.0.110'
|
||
},
|
||
'prod': {
|
||
'name': 'PROD',
|
||
'label': '正式環境',
|
||
'color': '#e74c3c',
|
||
'icon': '🟥',
|
||
'url': 'https://momo.wooo.work',
|
||
'health_endpoint': 'https://momo.wooo.work/health',
|
||
'k8s_host': '35.194.233.141'
|
||
}
|
||
}
|
||
|
||
# =============================================================================
|
||
# Dashboard 頁面
|
||
# =============================================================================
|
||
|
||
@cicd_bp.route('/cicd')
|
||
def cicd_dashboard():
|
||
"""CI/CD Dashboard 主頁面"""
|
||
return render_template('cicd_dashboard.html')
|
||
|
||
# =============================================================================
|
||
# API 端點
|
||
# =============================================================================
|
||
|
||
@cicd_bp.route('/api/cicd/status')
|
||
def get_cicd_status():
|
||
"""取得完整的 CI/CD 狀態"""
|
||
try:
|
||
# 收集所有狀態
|
||
pipelines = get_recent_pipelines(limit=10)
|
||
latest_pipeline = pipelines[0] if pipelines else None
|
||
environments = get_all_environments_status()
|
||
|
||
# 取得最新 Pipeline 的詳細 Job 信息
|
||
latest_jobs = []
|
||
failed_jobs = []
|
||
if latest_pipeline:
|
||
latest_jobs = get_pipeline_jobs(latest_pipeline['id'])
|
||
failed_jobs = [j for j in latest_jobs if j.get('status') == 'failed']
|
||
|
||
# 如果最新 Pipeline 失敗且有未通知的失敗 Job，發送告警
|
||
if latest_pipeline.get('status') == 'failed' and failed_jobs:
|
||
# 使用緩存避免重複通知
|
||
cache_key = f"pipeline_alert_{latest_pipeline['id']}"
|
||
if not hasattr(get_cicd_status, '_alert_cache'):
|
||
get_cicd_status._alert_cache = set()
|
||
|
||
if cache_key not in get_cicd_status._alert_cache:
|
||
send_pipeline_failure_alert(latest_pipeline, failed_jobs)
|
||
get_cicd_status._alert_cache.add(cache_key)
|
||
# 限制緩存大小
|
||
if len(get_cicd_status._alert_cache) > 100:
|
||
get_cicd_status._alert_cache = set(list(get_cicd_status._alert_cache)[-50:])
|
||
|
||
# 生成問題摘要
|
||
issues = []
|
||
|
||
# 檢查環境問題
|
||
for env_id, env_status in environments.items():
|
||
if not env_status.get('healthy'):
|
||
issues.append({
|
||
'type': 'environment',
|
||
'environment': env_id,
|
||
'message': f"{env_status.get('name')} 環境異常",
|
||
'error': env_status.get('error'),
|
||
'severity': 'critical',
|
||
'auto_fixable': True,
|
||
'fix_action': 'restart_pods'
|
||
})
|
||
|
||
# 檢查 Pod 問題
|
||
for pod in env_status.get('pods', []):
|
||
if not pod.get('healthy'):
|
||
issues.append({
|
||
'type': 'pod',
|
||
'environment': env_id,
|
||
'message': f"Pod {pod.get('name')} 不健康",
|
||
'status': pod.get('status'),
|
||
'restarts': pod.get('restarts'),
|
||
'severity': 'warning' if pod.get('restarts', 0) < 5 else 'critical',
|
||
'auto_fixable': True,
|
||
'fix_action': 'restart_pods'
|
||
})
|
||
|
||
# 檢查 Pipeline 問題
|
||
for job in failed_jobs:
|
||
error_info = job.get('error_info') or {}
|
||
issues.append({
|
||
'type': 'job',
|
||
'job_name': job.get('name'),
|
||
'stage': job.get('stage'),
|
||
'message': error_info.get('message', job.get('failure_reason', 'Job 失敗')),
|
||
'error_log': job.get('error_log', '')[:500], # 限制長度
|
||
'severity': error_info.get('severity', 'warning'),
|
||
'auto_fixable': error_info.get('auto_fixable', False),
|
||
'fix_action': error_info.get('fix_action'),
|
||
'fix_suggestion': error_info.get('fix_suggestion')
|
||
})
|
||
|
||
return jsonify({
|
||
'success': True,
|
||
'timestamp': datetime.now().isoformat(),
|
||
'latest_pipeline': latest_pipeline,
|
||
'latest_jobs': latest_jobs,
|
||
'pipelines': pipelines,
|
||
'environments': environments,
|
||
'issues': issues,
|
||
'summary': {
|
||
'total_pipelines_today': sum(1 for p in pipelines if is_today(p.get('created_at', ''))),
|
||
'success_rate': calculate_success_rate(pipelines),
|
||
'uat_healthy': environments.get('uat', {}).get('healthy', False),
|
||
'prod_healthy': environments.get('prod', {}).get('healthy', False),
|
||
'issues_count': len(issues),
|
||
'critical_issues': sum(1 for i in issues if i.get('severity') == 'critical')
|
||
}
|
||
})
|
||
except Exception as e:
|
||
return jsonify({
|
||
'success': False,
|
||
'error': str(e),
|
||
'timestamp': datetime.now().isoformat()
|
||
}), 500
|
||
|
||
|
||
@cicd_bp.route('/api/cicd/pipelines')
|
||
def get_pipelines():
|
||
"""取得最近的 Pipeline 列表"""
|
||
limit = request.args.get('limit', 20, type=int)
|
||
pipelines = get_recent_pipelines(limit=limit)
|
||
return jsonify({
|
||
'success': True,
|
||
'pipelines': pipelines,
|
||
'count': len(pipelines)
|
||
})
|
||
|
||
|
||
@cicd_bp.route('/api/cicd/pipeline/<int:pipeline_id>')
|
||
def get_pipeline_detail(pipeline_id):
|
||
"""取得單一 Pipeline 詳細資訊"""
|
||
try:
|
||
pipeline = get_pipeline_info(pipeline_id)
|
||
jobs = get_pipeline_jobs(pipeline_id)
|
||
|
||
return jsonify({
|
||
'success': True,
|
||
'pipeline': pipeline,
|
||
'jobs': jobs,
|
||
'stages': extract_stages(jobs)
|
||
})
|
||
except Exception as e:
|
||
return jsonify({
|
||
'success': False,
|
||
'error': str(e)
|
||
}), 500
|
||
|
||
|
||
@cicd_bp.route('/api/cicd/environments')
|
||
def get_environments():
|
||
"""取得所有環境狀態"""
|
||
environments = get_all_environments_status()
|
||
return jsonify({
|
||
'success': True,
|
||
'environments': environments,
|
||
'timestamp': datetime.now().isoformat()
|
||
})
|
||
|
||
|
||
@cicd_bp.route('/api/cicd/environment/<env_id>')
|
||
def get_environment_detail(env_id):
|
||
"""取得單一環境詳細狀態"""
|
||
if env_id not in ENVIRONMENTS:
|
||
return jsonify({'success': False, 'error': 'Unknown environment'}), 404
|
||
|
||
env_status = get_environment_status(env_id)
|
||
return jsonify({
|
||
'success': True,
|
||
'environment': env_status
|
||
})
|
||
|
||
|
||
@cicd_bp.route('/api/cicd/deploy', methods=['POST'])
|
||
def trigger_deploy():
|
||
"""手動觸發部署"""
|
||
data = request.get_json() or {}
|
||
env = data.get('environment', 'uat')
|
||
|
||
if env not in ENVIRONMENTS:
|
||
return jsonify({'success': False, 'error': 'Unknown environment'}), 400
|
||
|
||
# 觸發 GitLab Pipeline
|
||
try:
|
||
result = trigger_gitlab_pipeline(ref='main', variables={
|
||
'DEPLOY_ENV': env
|
||
})
|
||
return jsonify({
|
||
'success': True,
|
||
'message': f'已觸發 {ENVIRONMENTS[env]["name"]} 部署',
|
||
'pipeline': result
|
||
})
|
||
except Exception as e:
|
||
return jsonify({
|
||
'success': False,
|
||
'error': str(e)
|
||
}), 500
|
||
|
||
|
||
@cicd_bp.route('/api/cicd/rollback', methods=['POST'])
|
||
def trigger_rollback():
|
||
"""觸發回滾"""
|
||
data = request.get_json() or {}
|
||
env = data.get('environment', 'uat')
|
||
|
||
if env not in ENVIRONMENTS:
|
||
return jsonify({'success': False, 'error': 'Unknown environment'}), 400
|
||
|
||
# 執行 K8s 回滾
|
||
try:
|
||
if env == 'uat':
|
||
result = execute_uat_rollback()
|
||
else:
|
||
result = execute_gcp_rollback()
|
||
|
||
return jsonify({
|
||
'success': True,
|
||
'message': f'{ENVIRONMENTS[env]["name"]} 回滾成功',
|
||
'result': result
|
||
})
|
||
except Exception as e:
|
||
return jsonify({
|
||
'success': False,
|
||
'error': str(e)
|
||
}), 500
|
||
|
||
|
||
# =============================================================================
|
||
# 自動修復 API
|
||
# =============================================================================
|
||
|
||
# 只允許這幾種 fix_action，任何不在清單的請求直接 400
|
||
ALLOWED_FIX_ACTIONS = frozenset({'restart_registry', 'restart_pods', 'diagnose', 'full_repair'})
|
||
|
||
|
||
@cicd_bp.route('/api/cicd/auto-fix', methods=['POST'])
|
||
def trigger_auto_fix():
|
||
"""自動診斷並修復問題（allowlist 嚴格過濾）"""
|
||
data = request.get_json() or {}
|
||
fix_action = data.get('action')
|
||
env = data.get('environment', 'uat')
|
||
|
||
if env not in ENVIRONMENTS:
|
||
return jsonify({'success': False, 'error': '未知環境'}), 400
|
||
|
||
# Security: 嚴格 allowlist，防止任意 action 注入
|
||
if fix_action not in ALLOWED_FIX_ACTIONS:
|
||
return jsonify({'success': False, 'error': f'不允許的修復動作: {fix_action}'}), 400
|
||
|
||
results = []
|
||
try:
|
||
if fix_action == 'restart_registry':
|
||
result = fix_registry()
|
||
results.append(result)
|
||
elif fix_action == 'restart_pods':
|
||
result = fix_pods(env)
|
||
results.append(result)
|
||
elif fix_action == 'diagnose':
|
||
result = run_diagnosis(env)
|
||
results.append(result)
|
||
elif fix_action == 'full_repair':
|
||
# 完整修復流程
|
||
results.append(fix_registry())
|
||
results.append(fix_pods(env))
|
||
|
||
# 發送 Telegram 通知
|
||
send_fix_notification(env, fix_action, results)
|
||
|
||
return jsonify({
|
||
'success': True,
|
||
'message': f'已執行修復動作: {fix_action}',
|
||
'results': results
|
||
})
|
||
except Exception as e:
|
||
return jsonify({
|
||
'success': False,
|
||
'error': str(e)
|
||
}), 500
|
||
|
||
|
||
@cicd_bp.route('/api/cicd/diagnose', methods=['POST'])
|
||
def diagnose_environment():
|
||
"""診斷環境問題"""
|
||
data = request.get_json() or {}
|
||
env = data.get('environment', 'uat')
|
||
|
||
if env not in ENVIRONMENTS:
|
||
return jsonify({'success': False, 'error': '未知環境'}), 400
|
||
|
||
try:
|
||
diagnosis = run_diagnosis(env)
|
||
return jsonify({
|
||
'success': True,
|
||
'diagnosis': diagnosis
|
||
})
|
||
except Exception as e:
|
||
return jsonify({
|
||
'success': False,
|
||
'error': str(e)
|
||
}), 500
|
||
|
||
|
||
def fix_registry():
|
||
"""修復 Registry 服務"""
|
||
try:
|
||
result = subprocess.run(
|
||
['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=5',
|
||
'wooo@192.168.0.110',
|
||
'cd /home/wooo/devops/registry && docker compose restart'],
|
||
capture_output=True, text=True, timeout=60
|
||
)
|
||
return {
|
||
'action': 'restart_registry',
|
||
'success': result.returncode == 0,
|
||
'output': result.stdout,
|
||
'error': result.stderr if result.returncode != 0 else None
|
||
}
|
||
except Exception as e:
|
||
return {'action': 'restart_registry', 'success': False, 'error': str(e)}
|
||
|
||
|
||
def fix_pods(env):
|
||
"""重啟 Pod 修復問題"""
|
||
try:
|
||
if env == 'uat':
|
||
cmd = "kubectl rollout restart deployment/momo-app deployment/momo-scheduler -n momo"
|
||
result = subprocess.run(
|
||
['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=5',
|
||
'wooo@192.168.0.110', cmd],
|
||
capture_output=True, text=True, timeout=60
|
||
)
|
||
else:
|
||
cmd = "sudo kubectl rollout restart deployment/momo-app deployment/momo-scheduler -n momo"
|
||
result = subprocess.run(
|
||
['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=10',
|
||
'ogt@35.194.233.141', cmd],
|
||
capture_output=True, text=True, timeout=60
|
||
)
|
||
|
||
return {
|
||
'action': 'restart_pods',
|
||
'environment': env,
|
||
'success': result.returncode == 0,
|
||
'output': result.stdout,
|
||
'error': result.stderr if result.returncode != 0 else None
|
||
}
|
||
except Exception as e:
|
||
return {'action': 'restart_pods', 'success': False, 'error': str(e)}
|
||
|
||
|
||
def run_diagnosis(env):
|
||
"""執行環境診斷"""
|
||
diagnosis = {
|
||
'environment': env,
|
||
'timestamp': datetime.now().isoformat(),
|
||
'checks': []
|
||
}
|
||
|
||
try:
|
||
# 檢查健康端點
|
||
env_config = ENVIRONMENTS.get(env, {})
|
||
try:
|
||
# 內部健康檢查，可能使用自簽憑證 nosec B501
|
||
response = requests.get(env_config.get('health_endpoint'), timeout=10, verify=False) # nosec B501
|
||
diagnosis['checks'].append({
|
||
'name': '健康端點',
|
||
'status': 'ok' if response.status_code == 200 else 'failed',
|
||
'response_time': response.elapsed.total_seconds() * 1000,
|
||
'status_code': response.status_code
|
||
})
|
||
except Exception as e:
|
||
diagnosis['checks'].append({
|
||
'name': '健康端點',
|
||
'status': 'failed',
|
||
'error': str(e)
|
||
})
|
||
|
||
# 檢查 Pod 狀態
|
||
pods = get_k8s_pods_status(env)
|
||
unhealthy_pods = [p for p in pods if not p.get('healthy')]
|
||
diagnosis['checks'].append({
|
||
'name': 'Pod 狀態',
|
||
'status': 'ok' if not unhealthy_pods else 'warning',
|
||
'total_pods': len(pods),
|
||
'unhealthy_pods': len(unhealthy_pods),
|
||
'details': unhealthy_pods
|
||
})
|
||
|
||
# 檢查 Registry (僅 UAT)
|
||
if env == 'uat':
|
||
try:
|
||
# Registry 健康檢查 nosec B501
|
||
reg_response = requests.get('https://registry.wooo.work/v2/', timeout=10, verify=False) # nosec B501
|
||
diagnosis['checks'].append({
|
||
'name': 'Registry',
|
||
'status': 'ok' if reg_response.status_code in [200, 401] else 'failed',
|
||
'status_code': reg_response.status_code
|
||
})
|
||
except Exception as e:
|
||
diagnosis['checks'].append({
|
||
'name': 'Registry',
|
||
'status': 'failed',
|
||
'error': str(e)
|
||
})
|
||
|
||
# 生成總結
|
||
failed_checks = [c for c in diagnosis['checks'] if c['status'] == 'failed']
|
||
warning_checks = [c for c in diagnosis['checks'] if c['status'] == 'warning']
|
||
|
||
diagnosis['summary'] = {
|
||
'overall_status': 'critical' if failed_checks else ('warning' if warning_checks else 'healthy'),
|
||
'failed_count': len(failed_checks),
|
||
'warning_count': len(warning_checks),
|
||
'recommendations': []
|
||
}
|
||
|
||
# 生成修復建議
|
||
for check in failed_checks:
|
||
if check['name'] == '健康端點':
|
||
diagnosis['summary']['recommendations'].append({
|
||
'action': 'restart_pods',
|
||
'description': '建議重啟應用 Pod'
|
||
})
|
||
elif check['name'] == 'Registry':
|
||
diagnosis['summary']['recommendations'].append({
|
||
'action': 'restart_registry',
|
||
'description': '建議重啟 Registry 服務'
|
||
})
|
||
|
||
for check in warning_checks:
|
||
if check['name'] == 'Pod 狀態':
|
||
diagnosis['summary']['recommendations'].append({
|
||
'action': 'restart_pods',
|
||
'description': f'有 {check["unhealthy_pods"]} 個 Pod 不健康'
|
||
})
|
||
|
||
except Exception as e:
|
||
diagnosis['error'] = str(e)
|
||
|
||
return diagnosis
|
||
|
||
|
||
# =============================================================================
|
||
# Telegram 告警
|
||
# =============================================================================
|
||
|
||
TELEGRAM_BOT_TOKEN = os.environ.get('TELEGRAM_BOT_TOKEN', '')
|
||
TELEGRAM_CHAT_ID = os.environ.get('TELEGRAM_CHAT_ID', '')
|
||
|
||
if not TELEGRAM_BOT_TOKEN or not TELEGRAM_CHAT_ID:
|
||
import logging
|
||
logging.getLogger('cicd_routes').warning(
|
||
'[SECURITY] TELEGRAM_BOT_TOKEN or TELEGRAM_CHAT_ID is not set. '
|
||
'Telegram notifications will silently fail. Set these environment variables.'
|
||
)
|
||
|
||
|
||
def send_telegram_message(message):
|
||
"""發送 Telegram 訊息"""
|
||
try:
|
||
url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
|
||
data = {
|
||
'chat_id': TELEGRAM_CHAT_ID,
|
||
'text': message,
|
||
'parse_mode': 'Markdown'
|
||
}
|
||
response = requests.post(url, json=data, timeout=10)
|
||
return response.status_code == 200
|
||
except Exception as e:
|
||
print(f"Telegram send error: {e}")
|
||
return False
|
||
|
||
|
||
def send_pipeline_failure_alert(pipeline, failed_jobs):
|
||
"""Pipeline 失敗時發送告警"""
|
||
job_details = '\n'.join([
|
||
f" • {j['name']}: {j.get('failure_reason', '未知原因')}"
|
||
for j in failed_jobs
|
||
])
|
||
|
||
message = f"""🚨 *CI/CD Pipeline 失敗*
|
||
|
||
📌 *Pipeline:* #{pipeline.get('id')}
|
||
🌿 *分支:* `{pipeline.get('ref')}`
|
||
📝 *Commit:* `{pipeline.get('sha', '')[:8]}`
|
||
|
||
❌ *失敗 Jobs:*
|
||
{job_details}
|
||
|
||
🔗 [查看詳情]({pipeline.get('web_url')})
|
||
"""
|
||
return send_telegram_message(message)
|
||
|
||
|
||
def send_fix_notification(env, action, results):
|
||
"""發送修復通知"""
|
||
env_icon = '🟦' if env == 'uat' else '🟥'
|
||
env_name = ENVIRONMENTS.get(env, {}).get('name', env)
|
||
|
||
success_count = sum(1 for r in results if r.get('success'))
|
||
total_count = len(results)
|
||
|
||
status_emoji = '✅' if success_count == total_count else '⚠️'
|
||
|
||
result_details = '\n'.join([
|
||
f" • {r.get('action')}: {'✅ 成功' if r.get('success') else '❌ 失敗'}"
|
||
for r in results
|
||
])
|
||
|
||
message = f"""{status_emoji} *CI/CD 自動修復執行完成*
|
||
|
||
{env_icon} *環境:* {env_name}
|
||
🔧 *動作:* {action}
|
||
📊 *結果:* {success_count}/{total_count} 成功
|
||
|
||
{result_details}
|
||
|
||
⏰ {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
|
||
"""
|
||
return send_telegram_message(message)
|
||
|
||
|
||
# =============================================================================
|
||
# 輔助函數 - GitLab API
|
||
# =============================================================================
|
||
|
||
def gitlab_api(endpoint, method='GET', data=None):
|
||
"""呼叫 GitLab API（支援 SSH 備用方案）"""
|
||
url = f"{GITLAB_URL}/api/v4{endpoint}"
|
||
headers = {'PRIVATE-TOKEN': GITLAB_TOKEN}
|
||
|
||
try:
|
||
if method == 'GET':
|
||
response = requests.get(url, headers=headers, timeout=10)
|
||
elif method == 'POST':
|
||
response = requests.post(url, headers=headers, json=data, timeout=10)
|
||
|
||
response.raise_for_status()
|
||
return response.json()
|
||
except requests.exceptions.RequestException as e:
|
||
print(f"GitLab API Error (direct): {e}")
|
||
# 備用方案：透過 SSH 在主機上執行 curl
|
||
return gitlab_api_via_ssh(endpoint, method, data)
|
||
|
||
|
||
def gitlab_api_via_ssh(endpoint, method='GET', data=None):
|
||
"""
|
||
透過 SSH 在主機上呼叫 GitLab API（當 Pod 無法直接連接時）。
|
||
|
||
Security: curl 參數以 list 形式傳給 subprocess，避免 shell injection。
|
||
endpoint 和 json_data 均作為獨立 argv 傳入，不經過 shell 解析。
|
||
"""
|
||
try:
|
||
# 使用本地 GitLab URL；endpoint 由 gitlab_api() 內部構造，不含外部輸入
|
||
url = f"http://127.0.0.1:8929/api/v4{endpoint}"
|
||
|
||
if method == 'GET':
|
||
# list 形式：每個 curl 參數獨立，不走 shell
|
||
remote_argv = [
|
||
'curl', '-s',
|
||
'-H', f'PRIVATE-TOKEN: {GITLAB_TOKEN}',
|
||
url
|
||
]
|
||
else:
|
||
json_data = json.dumps(data) if data else '{}'
|
||
remote_argv = [
|
||
'curl', '-s', '-X', 'POST',
|
||
'-H', f'PRIVATE-TOKEN: {GITLAB_TOKEN}',
|
||
'-H', 'Content-Type: application/json',
|
||
'-d', json_data,
|
||
url
|
||
]
|
||
|
||
# SSH 以 list 模式執行：remote_argv 整體作為單一 SSH command 字串
|
||
# 由於 remote_argv 內的 GitLab token 和 url 均來自受控環境變數，
|
||
# 不含使用者輸入，此處拼接是安全的。
|
||
ssh_cmd = ['ssh',
|
||
'-o', 'StrictHostKeyChecking=no',
|
||
'-o', 'ConnectTimeout=5',
|
||
'wooo@192.168.0.110'] + remote_argv
|
||
|
||
result = subprocess.run(
|
||
ssh_cmd,
|
||
capture_output=True, text=True, timeout=15
|
||
)
|
||
|
||
if result.returncode == 0 and result.stdout:
|
||
return json.loads(result.stdout)
|
||
else:
|
||
print(f"GitLab API Error (SSH): {result.stderr}")
|
||
return None
|
||
except Exception as e:
|
||
print(f"GitLab API Error (SSH fallback): {e}")
|
||
return None
|
||
|
||
|
||
def get_recent_pipelines(limit=20):
|
||
"""取得最近的 Pipeline"""
|
||
result = gitlab_api(f"/projects/{GITLAB_PROJECT_ID}/pipelines?per_page={limit}")
|
||
if not result:
|
||
return []
|
||
|
||
pipelines = []
|
||
for p in result:
|
||
pipelines.append({
|
||
'id': p.get('id'),
|
||
'status': p.get('status'),
|
||
'ref': p.get('ref'),
|
||
'sha': p.get('sha', '')[:8],
|
||
'web_url': p.get('web_url'),
|
||
'created_at': p.get('created_at'),
|
||
'updated_at': p.get('updated_at'),
|
||
'duration': calculate_duration(p.get('created_at'), p.get('updated_at')),
|
||
'status_icon': get_status_icon(p.get('status')),
|
||
'status_color': get_status_color(p.get('status'))
|
||
})
|
||
|
||
return pipelines
|
||
|
||
|
||
def get_pipeline_info(pipeline_id):
|
||
"""取得 Pipeline 詳細資訊"""
|
||
return gitlab_api(f"/projects/{GITLAB_PROJECT_ID}/pipelines/{pipeline_id}")
|
||
|
||
|
||
def get_pipeline_jobs(pipeline_id):
|
||
"""取得 Pipeline 的所有 Jobs（包含錯誤詳情）"""
|
||
result = gitlab_api(f"/projects/{GITLAB_PROJECT_ID}/pipelines/{pipeline_id}/jobs")
|
||
if not result:
|
||
return []
|
||
|
||
jobs = []
|
||
for j in result:
|
||
job_data = {
|
||
'id': j.get('id'),
|
||
'name': j.get('name'),
|
||
'stage': j.get('stage'),
|
||
'status': j.get('status'),
|
||
'duration': j.get('duration'),
|
||
'web_url': j.get('web_url'),
|
||
'started_at': j.get('started_at'),
|
||
'finished_at': j.get('finished_at'),
|
||
'status_icon': get_status_icon(j.get('status')),
|
||
'status_color': get_status_color(j.get('status')),
|
||
'failure_reason': j.get('failure_reason'),
|
||
'error_info': None
|
||
}
|
||
|
||
# 如果 Job 失敗，嘗試取得錯誤日誌摘要
|
||
if j.get('status') == 'failed':
|
||
error_log = get_job_error_summary(j.get('id'))
|
||
if error_log:
|
||
job_data['error_log'] = error_log
|
||
job_data['error_info'] = analyze_error(error_log)
|
||
|
||
jobs.append(job_data)
|
||
|
||
return sorted(jobs, key=lambda x: (x.get('stage', ''), x.get('name', '')))
|
||
|
||
|
||
def get_job_error_summary(job_id, max_lines=50):
|
||
"""從 GitLab 取得 Job 日誌並提取錯誤摘要"""
|
||
try:
|
||
# 取得 Job 日誌（GitLab API 返回純文字）
|
||
url = f"{GITLAB_URL}/api/v4/projects/{GITLAB_PROJECT_ID}/jobs/{job_id}/trace"
|
||
headers = {'PRIVATE-TOKEN': GITLAB_TOKEN}
|
||
response = requests.get(url, headers=headers, timeout=10)
|
||
|
||
if response.status_code == 200:
|
||
log_text = response.text
|
||
lines = log_text.split('\n')
|
||
|
||
# 找出錯誤相關行
|
||
error_lines = []
|
||
for i, line in enumerate(lines):
|
||
if any(keyword in line.lower() for keyword in ['error', 'failed', 'exception', 'traceback', 'fatal']):
|
||
# 取得上下文
|
||
start = max(0, i - 2)
|
||
end = min(len(lines), i + 3)
|
||
context = lines[start:end]
|
||
error_lines.extend(context)
|
||
|
||
if error_lines:
|
||
# 去重並限制長度
|
||
unique_lines = list(dict.fromkeys(error_lines))[:max_lines]
|
||
return '\n'.join(unique_lines)
|
||
|
||
# 如果沒找到明確錯誤，返回最後幾行
|
||
return '\n'.join(lines[-20:])
|
||
|
||
except Exception as e:
|
||
print(f"Error getting job log for {job_id}: {e}")
|
||
|
||
return None
|
||
|
||
|
||
def extract_stages(jobs):
|
||
"""從 Jobs 提取 Stage 資訊"""
|
||
stages = {}
|
||
stage_order = ['test', 'build', 'deploy']
|
||
|
||
for job in jobs:
|
||
stage = job.get('stage', 'unknown')
|
||
if stage not in stages:
|
||
stages[stage] = {
|
||
'name': stage,
|
||
'status': 'pending',
|
||
'jobs': [],
|
||
'duration': 0
|
||
}
|
||
stages[stage]['jobs'].append(job)
|
||
|
||
# 更新 stage 狀態
|
||
if job.get('status') == 'failed':
|
||
stages[stage]['status'] = 'failed'
|
||
elif job.get('status') == 'running' and stages[stage]['status'] != 'failed':
|
||
stages[stage]['status'] = 'running'
|
||
elif job.get('status') == 'success' and stages[stage]['status'] == 'pending':
|
||
stages[stage]['status'] = 'success'
|
||
|
||
# 累加時間
|
||
if job.get('duration'):
|
||
stages[stage]['duration'] += job.get('duration', 0)
|
||
|
||
# 按順序排列
|
||
ordered_stages = []
|
||
for stage_name in stage_order:
|
||
if stage_name in stages:
|
||
stages[stage_name]['status_icon'] = get_status_icon(stages[stage_name]['status'])
|
||
stages[stage_name]['status_color'] = get_status_color(stages[stage_name]['status'])
|
||
ordered_stages.append(stages[stage_name])
|
||
|
||
return ordered_stages
|
||
|
||
|
||
def trigger_gitlab_pipeline(ref='main', variables=None):
|
||
"""觸發 GitLab Pipeline"""
|
||
data = {'ref': ref}
|
||
if variables:
|
||
data['variables'] = [{'key': k, 'value': v} for k, v in variables.items()]
|
||
|
||
return gitlab_api(f"/projects/{GITLAB_PROJECT_ID}/pipeline", method='POST', data=data)
|
||
|
||
|
||
# =============================================================================
|
||
# 輔助函數 - 環境狀態
|
||
# =============================================================================
|
||
|
||
def get_all_environments_status():
|
||
"""取得所有環境狀態"""
|
||
environments = {}
|
||
for env_id, env_config in ENVIRONMENTS.items():
|
||
environments[env_id] = get_environment_status(env_id)
|
||
return environments
|
||
|
||
|
||
def get_environment_status(env_id):
|
||
"""取得單一環境狀態"""
|
||
env_config = ENVIRONMENTS.get(env_id, {})
|
||
|
||
status = {
|
||
'id': env_id,
|
||
'name': env_config.get('name'),
|
||
'label': env_config.get('label'),
|
||
'color': env_config.get('color'),
|
||
'icon': env_config.get('icon'),
|
||
'url': env_config.get('url'),
|
||
'healthy': False,
|
||
'pods': [],
|
||
'last_deploy': None,
|
||
'version': None,
|
||
'response_time': None
|
||
}
|
||
|
||
# 健康檢查（內部服務可能使用自簽憑證）
|
||
try:
|
||
start_time = datetime.now()
|
||
response = requests.get(
|
||
env_config.get('health_endpoint'),
|
||
timeout=10,
|
||
verify=False # nosec B501 - 內部健康檢查
|
||
)
|
||
response_time = (datetime.now() - start_time).total_seconds() * 1000
|
||
|
||
if response.status_code == 200:
|
||
health_data = response.json()
|
||
status['healthy'] = health_data.get('status') == 'healthy'
|
||
status['version'] = health_data.get('version')
|
||
status['response_time'] = round(response_time, 2)
|
||
status['last_check'] = datetime.now().isoformat()
|
||
except Exception as e:
|
||
status['error'] = str(e)
|
||
|
||
# 取得 K8s Pod 狀態 (僅 UAT，GCP 需要特殊處理)
|
||
if env_id == 'uat':
|
||
status['pods'] = get_k8s_pods_status('uat')
|
||
elif env_id == 'prod':
|
||
status['pods'] = get_k8s_pods_status('prod')
|
||
|
||
return status
|
||
|
||
|
||
def get_k8s_pods_status(env):
|
||
"""取得 K8s Pod 狀態"""
|
||
pods = []
|
||
|
||
try:
|
||
if env == 'uat':
|
||
cmd = "kubectl get pods -n momo -o json"
|
||
result = subprocess.run(
|
||
['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=5',
|
||
'wooo@192.168.0.110', cmd],
|
||
capture_output=True, text=True, timeout=15
|
||
)
|
||
else: # prod/gcp
|
||
cmd = "sudo kubectl get pods -n momo -o json"
|
||
result = subprocess.run(
|
||
['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=10',
|
||
'ogt@35.194.233.141', cmd],
|
||
capture_output=True, text=True, timeout=20
|
||
)
|
||
|
||
if result.returncode == 0:
|
||
data = json.loads(result.stdout)
|
||
for item in data.get('items', []):
|
||
metadata = item.get('metadata', {})
|
||
status_obj = item.get('status', {})
|
||
|
||
# 計算 Ready 狀態
|
||
containers = status_obj.get('containerStatuses', [])
|
||
ready_count = sum(1 for c in containers if c.get('ready', False))
|
||
total_count = len(containers)
|
||
|
||
pods.append({
|
||
'name': metadata.get('name'),
|
||
'status': status_obj.get('phase'),
|
||
'ready': f"{ready_count}/{total_count}",
|
||
'restarts': sum(c.get('restartCount', 0) for c in containers),
|
||
'age': calculate_age(metadata.get('creationTimestamp')),
|
||
'healthy': status_obj.get('phase') == 'Running' and ready_count == total_count
|
||
})
|
||
except Exception as e:
|
||
print(f"Error getting K8s pods for {env}: {e}")
|
||
|
||
return pods
|
||
|
||
|
||
def execute_uat_rollback():
|
||
"""執行 UAT 回滾"""
|
||
cmd = "kubectl rollout undo deployment/momo-app deployment/momo-scheduler -n momo"
|
||
result = subprocess.run(
|
||
['ssh', '-o', 'StrictHostKeyChecking=no', 'wooo@192.168.0.110', cmd],
|
||
capture_output=True, text=True, timeout=30
|
||
)
|
||
return {'stdout': result.stdout, 'stderr': result.stderr, 'returncode': result.returncode}
|
||
|
||
|
||
def execute_gcp_rollback():
|
||
"""執行 GCP 回滾"""
|
||
cmd = "sudo kubectl rollout undo deployment/momo-app deployment/momo-scheduler -n momo"
|
||
result = subprocess.run(
|
||
['ssh', '-o', 'StrictHostKeyChecking=no', 'ogt@35.194.233.141', cmd],
|
||
capture_output=True, text=True, timeout=30
|
||
)
|
||
return {'stdout': result.stdout, 'stderr': result.stderr, 'returncode': result.returncode}
|
||
|
||
|
||
# =============================================================================
|
||
# 輔助函數 - 通用
|
||
# =============================================================================
|
||
|
||
def get_status_icon(status):
|
||
"""取得狀態圖示"""
|
||
icons = {
|
||
'success': '✅',
|
||
'passed': '✅',
|
||
'failed': '❌',
|
||
'running': '🔄',
|
||
'pending': '⏳',
|
||
'canceled': '⛔',
|
||
'skipped': '⏭️',
|
||
'manual': '👆',
|
||
'created': '🆕'
|
||
}
|
||
return icons.get(status, '❓')
|
||
|
||
|
||
def get_status_color(status):
|
||
"""取得狀態顏色"""
|
||
colors = {
|
||
'success': '#28a745',
|
||
'passed': '#28a745',
|
||
'failed': '#dc3545',
|
||
'running': '#007bff',
|
||
'pending': '#ffc107',
|
||
'canceled': '#6c757d',
|
||
'skipped': '#6c757d',
|
||
'manual': '#17a2b8',
|
||
'created': '#6c757d'
|
||
}
|
||
return colors.get(status, '#6c757d')
|
||
|
||
|
||
def calculate_duration(start, end):
|
||
"""計算持續時間"""
|
||
if not start or not end:
|
||
return None
|
||
try:
|
||
start_dt = datetime.fromisoformat(start.replace('Z', '+00:00'))
|
||
end_dt = datetime.fromisoformat(end.replace('Z', '+00:00'))
|
||
duration = (end_dt - start_dt).total_seconds()
|
||
|
||
if duration < 60:
|
||
return f"{int(duration)}秒"
|
||
elif duration < 3600:
|
||
return f"{int(duration // 60)}分{int(duration % 60)}秒"
|
||
else:
|
||
return f"{int(duration // 3600)}時{int((duration % 3600) // 60)}分"
|
||
except:
|
||
return None
|
||
|
||
|
||
def calculate_age(timestamp):
|
||
"""計算年齡"""
|
||
if not timestamp:
|
||
return "未知"
|
||
try:
|
||
created = datetime.fromisoformat(timestamp.replace('Z', '+00:00'))
|
||
now = datetime.now(created.tzinfo)
|
||
diff = now - created
|
||
|
||
if diff.days > 0:
|
||
return f"{diff.days}天"
|
||
elif diff.seconds >= 3600:
|
||
return f"{diff.seconds // 3600}時"
|
||
elif diff.seconds >= 60:
|
||
return f"{diff.seconds // 60}分"
|
||
else:
|
||
return f"{diff.seconds}秒"
|
||
except:
|
||
return "未知"
|
||
|
||
|
||
def is_today(timestamp):
|
||
"""判斷是否為今天"""
|
||
if not timestamp:
|
||
return False
|
||
try:
|
||
dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00'))
|
||
return dt.date() == datetime.now().date()
|
||
except:
|
||
return False
|
||
|
||
|
||
def calculate_success_rate(pipelines):
|
||
"""計算成功率"""
|
||
if not pipelines:
|
||
return 0
|
||
|
||
success_count = sum(1 for p in pipelines if p.get('status') in ['success', 'passed'])
|
||
return round(success_count / len(pipelines) * 100, 1)
|