wooo/ewoooc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(pchome): automate authorized sales acquisition
Some checks failed
CD Pipeline / deploy (push) Has been cancelled
Details

Browse Source
This commit is contained in:
ogt
2026-07-11 01:30:42 +08:00
parent f68223f0ed
commit 6e65ef00bb
21 changed files with 2428 additions and 617 deletions
Download Patch File Download Diff File Expand all files Collapse all files

523
services/import_service.py
View File

@@ -10,6 +10,7 @@ import json
import os
import re
from datetime import date, datetime, timedelta
from pathlib import Path
from typing import Any, Dict, Optional
import pandas as pd
@@ -299,7 +300,8 @@ def _read_daily_sales_excel(file_path: str) -> tuple[pd.DataFrame, dict]:
"""
candidates: list[dict] = []
with pd.ExcelFile(file_path, engine="openpyxl") as excel:
excel_engine = "xlrd" if Path(file_path).suffix.lower() == ".xls" else "openpyxl"
with pd.ExcelFile(file_path, engine=excel_engine) as excel:
for sheet_name in excel.sheet_names:
try:
preview = pd.read_excel(
@@ -399,6 +401,116 @@ def _should_quarantine_failed_import(error_message: str) -> bool:
]
return any(marker in error_message for marker in permanent_error_markers)
def _prepare_monthly_dataframe(df: pd.DataFrame, conn, job_id: int) -> tuple[pd.DataFrame, list]:
"""Normalize the sales dataframe to the existing monthly-table contract."""
monthly_table = "realtime_sales_monthly"
df_monthly = df.drop(columns=["snapshot_date"], errors="ignore").copy()
df_monthly = df_monthly.rename(
columns={
column: column.replace("%", "_pct").replace("(", "_").replace(")", "_")
for column in df_monthly.columns
}
)
target_columns = _table_columns(conn, monthly_table)
date_column = _find_daily_sales_date_column(df_monthly.columns)
if date_column and date_column != "日期" and "日期" in target_columns and "日期" not in df_monthly.columns:
df_monthly = df_monthly.rename(columns={date_column: "日期"})
extra_columns = set(df_monthly.columns) - target_columns
if extra_columns:
logger.warning("任務 %s 移除月報目標表未定義欄位: %s", job_id, sorted(extra_columns))
df_monthly = df_monthly.drop(columns=sorted(extra_columns), errors="ignore")
if "日期" not in df_monthly.columns:
raise ValueError("業績分析同步缺少日期欄位，已中止整批匯入")
if df_monthly.empty or not len(df_monthly.columns):
raise ValueError("業績分析同步沒有可寫入欄位，已中止整批匯入")
parsed_monthly_dates = pd.to_datetime(df_monthly["日期"], errors="coerce")
if parsed_monthly_dates.isna().any():
invalid_rows = int(parsed_monthly_dates.isna().sum())
raise ValueError(f"業績分析同步有 {invalid_rows} 筆無效日期，已中止整批匯入")
df_monthly["日期"] = parsed_monthly_dates.dt.strftime("%Y-%m-%d")
return df_monthly, sorted(df_monthly["日期"].unique().tolist())
def _replace_sales_tables_atomic(df: pd.DataFrame, job_id: int) -> Dict[str, Any]:
"""Replace snapshot and monthly rows in one transaction and verify both writes."""
snapshot_dates = _normalise_date_values_for_sql(df["snapshot_date"].dropna().unique())
if not snapshot_dates:
raise ValueError("當日業績沒有可驗證的資料日期，已中止整批匯入")
with engine.begin() as conn:
snapshot_placeholders, snapshot_params = _build_in_clause("snapshot_date", snapshot_dates)
snapshot_expr = _date_filter_expr("snapshot_date")
deleted_snapshot = conn.execute(
text(
"DELETE FROM daily_sales_snapshot "
f"WHERE {snapshot_expr} IN ({snapshot_placeholders})"
),
snapshot_params,
).rowcount
df.to_sql(
"daily_sales_snapshot",
conn,
if_exists="append",
index=False,
method="multi",
chunksize=1000,
)
snapshot_count = conn.execute(
text(
"SELECT COUNT(*) FROM daily_sales_snapshot "
f"WHERE {snapshot_expr} IN ({snapshot_placeholders})"
),
snapshot_params,
).scalar()
if snapshot_count != len(df):
raise RuntimeError(
f"當日業績寫入驗證失敗: 預期 {len(df)} 筆, 實際 {snapshot_count} 筆"
)
df_monthly, monthly_dates = _prepare_monthly_dataframe(df, conn, job_id)
monthly_placeholders, monthly_params = _build_in_clause("monthly_date", monthly_dates)
deleted_monthly = conn.execute(
text(
'DELETE FROM realtime_sales_monthly WHERE "日期" '
f"IN ({monthly_placeholders})"
),
monthly_params,
).rowcount
df_monthly.to_sql(
"realtime_sales_monthly",
conn,
if_exists="append",
index=False,
method="multi",
chunksize=1000,
)
monthly_count = conn.execute(
text(
'SELECT COUNT(*) FROM realtime_sales_monthly WHERE "日期" '
f"IN ({monthly_placeholders})"
),
monthly_params,
).scalar()
if monthly_count != len(df_monthly):
raise RuntimeError(
f"業績分析寫入驗證失敗: 預期 {len(df_monthly)} 筆, 實際 {monthly_count} 筆"
)
return {
"snapshot_rows": snapshot_count,
"monthly_rows": monthly_count,
"deleted_snapshot_rows": deleted_snapshot,
"deleted_monthly_rows": deleted_monthly,
"verified": True,
"atomic": True,
}
# 資料庫設定 - 使用 config.py 中的設定，支援 PostgreSQL 和 SQLite
def _create_engine_with_pool(db_path):
"""建立帶有連線池配置的資料庫引擎"""
@@ -512,8 +624,8 @@ class ImportService:
Args:
job_type: 任務類型（daily_sales 或 vendor_stockout）
drive_file_id: Google Drive 檔案 ID
drive_file_name: 檔案名稱
drive_file_id: Google Drive ID or governed source fingerprint
drive_file_name: Source file name
drive_file_size: 檔案大小
Returns:
@@ -720,6 +832,92 @@ class ImportService:
finally:
session.close()
def import_local_daily_sales(
self,
file_path: str,
*,
source_type: str,
source_fingerprint: str,
source_file_name: Optional[str] = None,
) -> Dict[str, Any]:
"""Import a provider-fetched sales file with fingerprint idempotency."""
if not os.path.isfile(file_path) or os.path.islink(file_path):
return {
'success': False,
'status': 'rejected',
'error_kind': 'invalid_local_file',
'message': '來源檔案不存在或不符合安全檔案規則。',
}
safe_source = re.sub(r'[^a-z0-9_-]+', '-', str(source_type).lower()).strip('-')[:40]
if not safe_source or not re.fullmatch(r'[0-9a-f]{64}', str(source_fingerprint or '')):
return {
'success': False,
'status': 'rejected',
'error_kind': 'invalid_source_identity',
'message': '來源識別或內容指紋無效，未執行寫入。',
}
source_identity = f'{safe_source}:sha256:{source_fingerprint}'
session = Session()
try:
existing = session.query(ImportJob).filter_by(
job_type='daily_sales',
drive_file_id=source_identity,
status='completed',
).order_by(ImportJob.id.desc()).first()
if existing:
return {
'success': True,
'status': 'duplicate_no_write',
'duplicate': True,
'job_id': existing.id,
'job': _public_import_job_payload(existing),
'message': '相同內容已完成匯入，本次以冪等方式略過寫入。',
}
finally:
session.close()
file_name = os.path.basename(source_file_name or file_path)[:500]
job_id = self.create_import_job(
'daily_sales',
source_identity,
file_name,
os.path.getsize(file_path),
)
if not job_id:
return {
'success': False,
'status': 'failed',
'error_kind': 'job_create_failed',
'message': '無法建立匯入任務，正式資料未變更。',
}
session = Session()
try:
job = session.query(ImportJob).filter_by(id=job_id).first()
if job:
job.local_file_path = file_path
session.commit()
finally:
session.close()
self.update_job_status(job_id, 'downloading', 40, '授權來源檔案已取得')
success = self.process_daily_sales_import(job_id, file_path)
public_job = self.get_job_status(job_id)
return {
'success': success,
'status': 'completed' if success else 'failed_rolled_back',
'duplicate': False,
'job_id': job_id,
'job': public_job,
'message': (
'業績檔已完成交易式匯入與雙表驗證。'
if success
else '業績檔未通過交易式匯入驗證，正式資料未變更。'
),
}
def process_daily_sales_import(self, job_id: int, file_path: str) -> bool:
"""
處理當日業績匯入
@@ -769,6 +967,15 @@ class ImportService:
logger.error(error_msg)
self.update_job_status(job_id, 'failed', 50, '日期驗證失敗', error_msg)
return False
if parsed_dates.isna().any():
invalid_rows = int(parsed_dates.isna().sum())
error_msg = (
f"Excel 日期防禦失敗：日期欄位「{date_col}」有 {invalid_rows} 筆無效日期，"
"為避免部分資料寫入已中止整批匯入"
)
logger.error(error_msg)
self.update_job_status(job_id, 'failed', 50, '日期驗證失敗', error_msg)
return False
df['snapshot_date'] = parsed_dates.dt.date
logger.info(f"使用日期欄位: {date_col}")
@@ -789,225 +996,88 @@ class ImportService:
return False
else:
# 使用當前日期
df['snapshot_date'] = datetime.now(TAIPEI_TZ).date()
fallback_date = datetime.now(TAIPEI_TZ).date()
df['snapshot_date'] = fallback_date
if '日期' not in df.columns:
df['日期'] = fallback_date.isoformat()
logger.info("未找到日期欄位，使用當前日期（台北時區）")
# 寫入資料庫 - 使用全域的 engine（支援 PostgreSQL 和 SQLite）
# 使用模組頂部定義的 engine，確保連接到正確的資料庫
# 更新進度
total_rows = len(df)
self.update_job_progress(job_id, total_rows=total_rows, processed_rows=0)
# 取得此次匯入的日期範圍
import_dates = df['snapshot_date'].unique()
logger.info(f"本次匯入包含 {len(import_dates)} 個日期的資料")
# 刪除資料庫中相同日期的舊資料（覆蓋邏輯）
if len(import_dates) > 0:
# 過濾掉 None 值
valid_dates = _normalise_date_values_for_sql(import_dates)
if valid_dates:
date_placeholders, date_params = _build_in_clause("snapshot_date", valid_dates)
snapshot_date_expr = _date_filter_expr("snapshot_date")
with engine.connect() as conn:
# 刪除相同日期的舊資料
delete_query = text(
f"DELETE FROM {table_name} WHERE {snapshot_date_expr} IN ({date_placeholders})"
)
result = conn.execute(delete_query, date_params)
deleted_count = result.rowcount
conn.commit()
if deleted_count > 0:
logger.info(f"已刪除 {deleted_count} 筆舊資料（覆蓋模式）")
# 寫入資料庫（帶驗證和重試機制）
self.update_job_status(job_id, 'importing', 60, '交易式寫入與雙表驗證...')
max_retries = 2
retry_count = 0
write_success = False
while retry_count <= max_retries and not write_success:
try:
if retry_count > 0:
logger.warning(f"任務 {job_id} 第 {retry_count} 次重試寫入...")
self.update_job_status(job_id, 'importing', 60, f'重試寫入中 ({retry_count}/{max_retries})...')
df.to_sql(
table_name,
engine,
if_exists='append',
index=False,
method='multi',
chunksize=1000
)
# V-Fix: 匯入後驗證 - 確認資料已正確寫入資料庫
self.update_job_status(job_id, 'importing', 85, '驗證資料寫入...')
# 取得本次匯入的日期
import_dates = df['snapshot_date'].dropna().unique()
if len(import_dates) > 0:
# 查詢資料庫中這些日期的資料筆數
raw_valid_dates = [d for d in import_dates if d is not None]
valid_dates = _normalise_date_values_for_sql(raw_valid_dates)
date_placeholders, date_params = _build_in_clause("verify_date", valid_dates)
snapshot_date_expr = _date_filter_expr("snapshot_date")
with engine.connect() as conn:
verify_query = text(
f"SELECT COUNT(*) FROM {table_name} WHERE {snapshot_date_expr} IN ({date_placeholders})"
)
result = conn.execute(verify_query, date_params)
db_count = result.scalar()
# 驗證：資料庫筆數應該 >= 本次匯入筆數（可能有其他日期的舊資料）
expected_count = len(df[df['snapshot_date'].isin(raw_valid_dates)])
if db_count >= expected_count:
logger.info(f"任務 {job_id} 驗證成功: 預期 {expected_count} 筆, 資料庫有 {db_count} 筆")
write_success = True
else:
logger.warning(f"任務 {job_id} 驗證失敗: 預期 {expected_count} 筆, 資料庫只有 {db_count} 筆")
retry_count += 1
else:
# 沒有有效日期，跳過驗證
logger.warning(f"任務 {job_id} 無法驗證: 沒有有效的 snapshot_date")
write_success = True
except Exception as write_error:
logger.error(f"任務 {job_id} 寫入失敗 (嘗試 {retry_count + 1}): {str(write_error)}")
retry_count += 1
if retry_count > max_retries:
raise write_error
if not write_success:
error_msg = f"資料寫入驗證失敗，已重試 {max_retries} 次"
self.update_job_status(job_id, 'failed', 85, '驗證失敗', error_msg)
logger.error(f"任務 {job_id} {error_msg}")
return False
# === V-New 2026-01-15: 同步寫入 realtime_sales_monthly ===
# 目的：讓當日業績 raw data 同時呈現在「業績分析儀表板」
# 2026-01-30 修復：加強欄位驗證、同步狀態追蹤、失敗告警
self.update_job_status(job_id, 'importing', 90, '同步至業績分析儀表板...')
sync_success = False
write_receipt = None
sync_error_msg = None
monthly_table = 'realtime_sales_monthly'
try:
# 準備資料：移除 snapshot_date 欄位（realtime_sales_monthly 不需要此欄位）
df_monthly = df.drop(columns=['snapshot_date'], errors='ignore')
# 2026-01-30 修正：強化欄位名稱轉換
# 將特殊字符轉換為 PostgreSQL 安全格式
column_mapping = {}
for col in df_monthly.columns:
new_col = col.replace('%', '_pct').replace('(', '_').replace(')', '_')
column_mapping[col] = new_col
df_monthly = df_monthly.rename(columns=column_mapping)
# 記錄轉換的欄位
converted_cols = [f"'{k}' -> '{v}'" for k, v in column_mapping.items() if k != v]
if converted_cols:
logger.info(f"任務 {job_id} 欄位名稱轉換: {', '.join(converted_cols)}")
logger.info(f"任務 {job_id} 欄位轉換完成，共 {len(df_monthly.columns)} 個欄位")
# 2026-01-30 新增：驗證 DataFrame 欄位和目標表欄位是否一致
with engine.connect() as conn:
target_columns = _table_columns(conn, monthly_table)
df_columns = set(df_monthly.columns)
missing_in_table = df_columns - target_columns
missing_in_df = target_columns - df_columns
if missing_in_table:
logger.warning(f"任務 {job_id} 欄位警告: DataFrame 有但表中沒有: {missing_in_table}")
# 移除表中沒有的欄位，避免 INSERT 失敗
df_monthly = df_monthly.drop(columns=list(missing_in_table), errors='ignore')
logger.info(f"任務 {job_id} 已移除多餘欄位，剩餘 {len(df_monthly.columns)} 個欄位")
if missing_in_df:
logger.warning(f"任務 {job_id} 欄位警告: 表中有但 DataFrame 沒有: {missing_in_df}")
# 取得本次匯入的日期列表（使用原始「日期」欄位）
unique_dates = []
if '日期' in df.columns:
unique_dates = df['日期'].dropna().unique().tolist()
logger.info(f"任務 {job_id} 準備同步 {len(unique_dates)} 個日期的資料")
if len(unique_dates) > 0:
# 刪除 realtime_sales_monthly 中相同日期的舊資料（去重）
date_placeholders, date_params = _build_in_clause("monthly_date", unique_dates)
with engine.connect() as conn:
delete_monthly_query = text(
f'DELETE FROM {monthly_table} WHERE "日期" IN ({date_placeholders})'
)
result = conn.execute(delete_monthly_query, date_params)
deleted_monthly = result.rowcount
conn.commit()
if deleted_monthly > 0:
logger.info(f"任務 {job_id} 已從 {monthly_table} 刪除 {deleted_monthly} 筆同日期舊資料")
# 寫入 realtime_sales_monthly
df_monthly.to_sql(
monthly_table,
engine,
if_exists='append',
index=False,
method='multi',
chunksize=1000
)
logger.info(f"任務 {job_id} 已同步 {len(df_monthly)} 筆資料至 {monthly_table}")
# 驗證同步結果
if len(unique_dates) > 0:
with engine.connect() as conn:
date_placeholders, date_params = _build_in_clause("monthly_verify_date", unique_dates)
verify_query = text(
f'SELECT COUNT(*) FROM {monthly_table} WHERE "日期" IN ({date_placeholders})'
for attempt in range(max_retries + 1):
try:
if attempt:
self.update_job_status(
job_id,
'importing',
60,
f'整批交易重試中 ({attempt}/{max_retries})...'
)
verify_count = conn.execute(verify_query, date_params).scalar()
write_receipt = _replace_sales_tables_atomic(df, job_id)
break
except Exception as write_error:
sync_error_msg = str(write_error)
logger.error(
"任務 %s 交易式寫入失敗 (嘗試 %s/%s): %s",
job_id,
attempt + 1,
max_retries + 1,
sync_error_msg,
exc_info=True,
)
if verify_count >= len(df_monthly):
logger.info(f"任務 {job_id} 同步驗證成功: {monthly_table} 現有 {verify_count} 筆資料")
sync_success = True
else:
sync_error_msg = f"同步驗證失敗: 預期 {len(df_monthly)} 筆, 實際 {verify_count} 筆"
logger.error(f"任務 {job_id} {sync_error_msg}")
else:
sync_success = True # 沒有日期資料時視為成功
sync_success = bool(write_receipt and write_receipt.get('verified'))
monthly_table = 'realtime_sales_monthly'
if not sync_success:
summary = {
'imported_count': 0,
'table_name': table_name,
'synced_to': None,
'sync_success': False,
'sync_error': sync_error_msg,
'verified': False,
'atomic': True,
'rolled_back': True,
'source_sheet': excel_metadata.get("sheet_name"),
'source_header_row': excel_metadata.get("header_row"),
'message': '業績資料驗證失敗，整批交易已回滾，正式資料未變更。',
}
session = Session()
try:
job = session.query(ImportJob).filter_by(id=job_id).first()
if job:
job.import_summary = json.dumps(summary, ensure_ascii=False)
session.commit()
finally:
session.close()
except Exception as sync_error:
# 同步失敗，記錄完整錯誤
import traceback
sync_error_msg = str(sync_error)
logger.error(f"任務 {job_id} 同步至 {monthly_table} 失敗: {sync_error_msg}")
logger.error(f"任務 {job_id} 同步錯誤堆疊:\n{traceback.format_exc()}")
# 2026-01-30 新增：發送同步失敗告警
self.update_job_progress(
job_id,
processed_rows=total_rows,
success_rows=0,
error_rows=total_rows,
)
self.update_job_status(
job_id,
'failed',
95,
'交易驗證失敗，已回滾',
sync_error_msg or '交易式寫入驗證失敗',
)
try:
from services.notification_manager import NotificationManager
notifier = NotificationManager()
alert_msg = (
f"⚠️ 業績資料同步失敗告警\n"
f"{'='*30}\n"
f"任務 ID: {job_id}\n"
f"目標表: {monthly_table}\n"
f"錯誤: {sync_error_msg[:200]}\n"
f"{'='*30}\n"
f"daily_sales_snapshot 已匯入成功，但業績分析儀表板需要手動同步"
)
notifier._send_telegram_messages([alert_msg])
logger.info(f"任務 {job_id} 已發送同步失敗告警")
NotificationManager()._send_telegram_messages([
f"業績匯入失敗（任務 {job_id}）：雙表交易已回滾，正式資料未變更。"
])
except Exception as notify_error:
logger.error(f"任務 {job_id} 發送告警失敗: {notify_error}")
logger.error("任務 %s 發送回滾告警失敗: %s", job_id, notify_error)
return False
self.update_job_status(job_id, 'importing', 90, '雙表交易驗證完成')
# 計算日期範圍
@@ -1170,7 +1240,7 @@ class ImportService:
if isinstance(last_date, date)
else date.fromisoformat(str(last_date)[:10])
)
days_since = (date.today() - normalized_last_date).days
days_since = (datetime.now(TAIPEI_TZ).date() - normalized_last_date).days
data_lag_days = days_since
latest_sales_date = str(normalized_last_date)
if days_since >= 3:
@@ -1212,6 +1282,7 @@ class ImportService:
total_rows = 0
all_dates = [] # 收集所有匯入的日期
failed_files = []
archive_failed_files = []
for file in files:
file_id = file['id']
@@ -1260,10 +1331,20 @@ class ImportService:
if drive_service.move_file(file_id, archive_folder):
logger.info(f"已移動 Google Drive 檔案到「{archive_folder}」: {file_name}")
self.update_job_status(job_id, 'completed', 100, '完成')
else:
logger.warning(f"無法移動 Google Drive 檔案: {file_name}")
self.update_job_status(job_id, 'completed', 100, '完成')
archive_failed_files.append({
'file': file_name,
'job_id': job_id,
'error': '業績已匯入，但 Google Drive 來源封存失敗',
})
self.update_job_status(
job_id,
'completed',
100,
'匯入完成，來源封存待自動重試',
)
imported_count += 1
# 讀取 job summary 取得匯入筆數和日期範圍
@@ -1347,15 +1428,29 @@ class ImportService:
f'找到 {len(files)} 個檔案，成功匯入 {imported_count} 個，'
f'失敗 {len(failed_files)} 個。{public_error}'
)
elif archive_failed_files:
message = (
f'成功匯入 {imported_count} 個檔案，但有 {len(archive_failed_files)} 個來源封存待重試'
)
else:
message = f'成功匯入 {imported_count} 個檔案'
return {
'success': len(failed_files) == 0,
'success': len(failed_files) == 0 and len(archive_failed_files) == 0,
'status': (
'partial_source_finalize_failed'
if archive_failed_files
else 'completed'
if not failed_files
else 'failed'
),
'message': message,
'file_count': len(files),
'imported_count': imported_count,
'failed_count': len(failed_files),
'source_finalize_ok': len(archive_failed_files) == 0,
'archive_failed_count': len(archive_failed_files),
'archive_errors': archive_failed_files,
'errors': failed_files,
'total_rows': total_rows,
'date_range': date_range