381 lines
17 KiB
Python
381 lines
17 KiB
Python
"""Authorized PChome sales report providers and file-boundary guards."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import http.client
|
|
import imaplib
|
|
import ipaddress
|
|
import os
|
|
import re
|
|
import shutil
|
|
import socket
|
|
import ssl
|
|
import tempfile
|
|
from dataclasses import dataclass
|
|
from email import policy
|
|
from email.header import decode_header, make_header
|
|
from email.parser import BytesParser
|
|
from pathlib import Path
|
|
from typing import Callable, List, Optional
|
|
from urllib.parse import unquote, urlparse
|
|
|
|
|
|
VALID_EXTENSIONS = {".xlsx", ".xls"}
|
|
XLSX_MAGIC = b"PK\x03\x04"
|
|
XLS_MAGIC = b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
|
|
|
|
|
|
class ProviderError(RuntimeError):
|
|
def __init__(self, kind: str, public_message: str):
|
|
super().__init__(public_message)
|
|
self.kind = kind
|
|
self.public_message = public_message
|
|
|
|
|
|
class _PinnedHTTPSConnection(http.client.HTTPSConnection):
|
|
"""Connect to the validated IP while preserving hostname TLS verification."""
|
|
|
|
def __init__(self, hostname: str, port: int, pinned_ip: str, timeout: int):
|
|
super().__init__(hostname, port=port, timeout=timeout, context=ssl.create_default_context())
|
|
self._pinned_ip = pinned_ip
|
|
|
|
def connect(self):
|
|
self.sock = socket.create_connection(
|
|
(self._pinned_ip, self.port),
|
|
self.timeout,
|
|
self.source_address,
|
|
)
|
|
self.sock = self._context.wrap_socket(self.sock, server_hostname=self.host)
|
|
|
|
|
|
@dataclass
|
|
class AcquisitionCandidate:
|
|
source_type: str
|
|
file_path: str
|
|
file_name: str
|
|
fingerprint: str
|
|
source_ref_hash: str
|
|
finalize_success: Callable[[], None]
|
|
finalize_rejected: Callable[[], None]
|
|
cleanup: Callable[[], None]
|
|
|
|
|
|
def env_bool(name: str, default: bool = False) -> bool:
|
|
fallback = "true" if default else "false"
|
|
return os.getenv(name, fallback).strip().lower() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
def env_int(name: str, default: int, minimum: int = 1, maximum: int = 1000) -> int:
|
|
try:
|
|
value = int(os.getenv(name, str(default)))
|
|
except (TypeError, ValueError):
|
|
value = default
|
|
return max(minimum, min(value, maximum))
|
|
|
|
|
|
def _hash_text(value: str) -> str:
|
|
return hashlib.sha256(value.encode("utf-8", errors="ignore")).hexdigest()
|
|
|
|
|
|
def _sha256_file(file_path: str) -> str:
|
|
digest = hashlib.sha256()
|
|
with open(file_path, "rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _safe_file_name(value: str, default: str = "pchome-sales.xlsx") -> str:
|
|
candidate = os.path.basename(str(value or "").replace("\x00", "")).strip()
|
|
candidate = re.sub(r"[^0-9A-Za-z._()\-\u4e00-\u9fff]+", "_", candidate)
|
|
return (candidate or default)[:180]
|
|
|
|
|
|
def _decode_mail_header(value: Optional[str]) -> str:
|
|
if not value:
|
|
return ""
|
|
try:
|
|
return str(make_header(decode_header(value)))
|
|
except Exception:
|
|
return str(value)
|
|
|
|
|
|
def _make_temp_path(suffix: str) -> str:
|
|
target = Path(os.getenv("PCHOME_SALES_TEMP_DIR", "data/temp/pchome-sales"))
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
fd, file_path = tempfile.mkstemp(prefix="sales-", suffix=suffix, dir=str(target))
|
|
os.close(fd)
|
|
return file_path
|
|
|
|
|
|
def _validate_excel_file(file_path: str, file_name: str) -> None:
|
|
path = Path(file_path)
|
|
max_bytes = env_int("PCHOME_SALES_MAX_FILE_BYTES", 25 * 1024 * 1024, 1024, 200 * 1024 * 1024)
|
|
if not path.is_file() or path.is_symlink():
|
|
raise ProviderError("unsafe_file", "來源檔案不存在或不符合安全檔案規則。")
|
|
if path.stat().st_size <= 0 or path.stat().st_size > max_bytes:
|
|
raise ProviderError("invalid_file_size", "來源檔案大小不符合匯入政策，未執行寫入。")
|
|
suffix = Path(file_name).suffix.lower()
|
|
if suffix not in VALID_EXTENSIONS:
|
|
raise ProviderError("unsupported_file_type", "來源不是允許的 Excel 業績檔，未執行寫入。")
|
|
with path.open("rb") as handle:
|
|
magic = handle.read(8)
|
|
signature_matches = (
|
|
suffix == ".xlsx" and magic.startswith(XLSX_MAGIC)
|
|
) or (
|
|
suffix == ".xls" and magic == XLS_MAGIC
|
|
)
|
|
if not signature_matches:
|
|
raise ProviderError("invalid_excel_signature", "來源檔案內容不是有效的 Excel 格式，未執行寫入。")
|
|
|
|
|
|
def _archive_file(source: Path, folder_name: str, fingerprint: str) -> None:
|
|
if not source.exists() or source.is_symlink():
|
|
return
|
|
target_dir = source.parent / folder_name
|
|
target_dir.mkdir(parents=True, exist_ok=True)
|
|
target = target_dir / source.name
|
|
if target.exists():
|
|
target = target_dir / f"{source.stem}-{fingerprint[:10]}{source.suffix}"
|
|
os.replace(str(source), str(target))
|
|
|
|
|
|
class AuthorizedSalesProviders:
|
|
"""Side-effect-bounded report providers in governed priority order."""
|
|
|
|
def validate_http_url(self, url: str) -> tuple[str, int, str]:
|
|
parsed = urlparse(url)
|
|
if parsed.scheme.lower() != "https" or not parsed.hostname or parsed.username or parsed.password:
|
|
raise ProviderError("http_url_rejected", "授權報表網址必須是無內嵌帳密的 HTTPS 網址。")
|
|
hostname = parsed.hostname.lower().rstrip(".")
|
|
try:
|
|
port = parsed.port or 443
|
|
except ValueError as exc:
|
|
raise ProviderError("http_port_rejected", "授權報表網址的連接埠格式無效。") from exc
|
|
allowed_ports = {
|
|
int(item.strip())
|
|
for item in os.getenv("PCHOME_SALES_HTTP_ALLOWED_PORTS", "443").split(",")
|
|
if item.strip().isdigit()
|
|
}
|
|
if port not in allowed_ports:
|
|
raise ProviderError("http_port_rejected", "授權報表連接埠未列入允許清單，未發出連線。")
|
|
allowed_hosts = {
|
|
item.strip().lower().rstrip(".")
|
|
for item in os.getenv("PCHOME_SALES_HTTP_ALLOWED_HOSTS", "").split(",")
|
|
if item.strip()
|
|
}
|
|
if hostname not in allowed_hosts:
|
|
raise ProviderError("http_host_not_allowed", "授權報表主機未列入允許清單，未發出連線。")
|
|
try:
|
|
addresses = {
|
|
item[4][0]
|
|
for item in socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM)
|
|
}
|
|
except OSError as exc:
|
|
raise ProviderError("http_dns_failed", "授權報表主機目前無法解析。") from exc
|
|
if not addresses:
|
|
raise ProviderError("http_dns_failed", "授權報表主機目前沒有可用位址。")
|
|
if not env_bool("PCHOME_SALES_HTTP_ALLOW_PRIVATE"):
|
|
for address in addresses:
|
|
ip = ipaddress.ip_address(address)
|
|
if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved:
|
|
raise ProviderError("http_private_address_rejected", "授權報表主機解析到受限制網段，未發出連線。")
|
|
return hostname, port, sorted(addresses)[0]
|
|
|
|
def http_candidates(self) -> List[AcquisitionCandidate]:
|
|
if not env_bool("PCHOME_SALES_HTTP_ENABLED"):
|
|
return []
|
|
url = os.getenv("PCHOME_SALES_HTTP_URL", "").strip()
|
|
if not url:
|
|
raise ProviderError("http_not_configured", "授權 HTTPS 來源已啟用但網址尚未設定。")
|
|
hostname, port, pinned_ip = self.validate_http_url(url)
|
|
parsed = urlparse(url)
|
|
headers = {"Accept": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet, application/vnd.ms-excel"}
|
|
token = os.getenv("PCHOME_SALES_HTTP_BEARER_TOKEN", "").strip()
|
|
if token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
timeout = env_int("PCHOME_SALES_HTTP_TIMEOUT_SECONDS", 30, 3, 120)
|
|
max_bytes = env_int("PCHOME_SALES_MAX_FILE_BYTES", 25 * 1024 * 1024, 1024, 200 * 1024 * 1024)
|
|
connection = _PinnedHTTPSConnection(hostname, port, pinned_ip, timeout)
|
|
path = parsed.path or "/"
|
|
if parsed.query:
|
|
path = f"{path}?{parsed.query}"
|
|
try:
|
|
connection.request("GET", path, headers=headers)
|
|
response = connection.getresponse()
|
|
if response.status != 200:
|
|
raise ProviderError("http_status_rejected", f"授權 HTTPS 報表來源回應 {response.status}，未執行匯入。")
|
|
content_length = response.getheader("Content-Length")
|
|
if content_length and int(content_length) > max_bytes:
|
|
raise ProviderError("http_file_too_large", "授權報表超過檔案大小上限，未執行匯入。")
|
|
disposition = response.getheader("Content-Disposition", "")
|
|
match = re.search(r"filename\*?=(?:UTF-8''|\")?([^\";]+)", disposition, re.I)
|
|
raw_name = unquote(match.group(1).strip()) if match else os.path.basename(parsed.path)
|
|
file_name = _safe_file_name(raw_name)
|
|
suffix = Path(file_name).suffix.lower()
|
|
if suffix not in VALID_EXTENSIONS:
|
|
suffix = ".xlsx"
|
|
file_name = f"{Path(file_name).stem or 'pchome-sales'}{suffix}"
|
|
temp_path = _make_temp_path(suffix)
|
|
written = 0
|
|
try:
|
|
with open(temp_path, "wb") as handle:
|
|
while True:
|
|
chunk = response.read(1024 * 1024)
|
|
if not chunk:
|
|
break
|
|
written += len(chunk)
|
|
if written > max_bytes:
|
|
raise ProviderError("http_file_too_large", "授權報表超過檔案大小上限，未執行匯入。")
|
|
handle.write(chunk)
|
|
_validate_excel_file(temp_path, file_name)
|
|
except Exception:
|
|
Path(temp_path).unlink(missing_ok=True)
|
|
raise
|
|
except ProviderError:
|
|
raise
|
|
except (OSError, ssl.SSLError, http.client.HTTPException, ValueError) as exc:
|
|
raise ProviderError("http_fetch_failed", "授權 HTTPS 報表來源目前無法連線。") from exc
|
|
finally:
|
|
connection.close()
|
|
fingerprint = _sha256_file(temp_path)
|
|
return [AcquisitionCandidate(
|
|
source_type="authorized_https",
|
|
file_path=temp_path,
|
|
file_name=file_name,
|
|
fingerprint=fingerprint,
|
|
source_ref_hash=_hash_text(f"https://{hostname}{parsed.path}"),
|
|
finalize_success=lambda: None,
|
|
finalize_rejected=lambda: None,
|
|
cleanup=lambda candidate_path=temp_path: Path(candidate_path).unlink(missing_ok=True),
|
|
)]
|
|
|
|
def _mark_imap_seen(self, uid: bytes) -> None:
|
|
host = os.getenv("PCHOME_SALES_IMAP_HOST", "").strip()
|
|
user = os.getenv("PCHOME_SALES_IMAP_USER", "").strip()
|
|
password = os.getenv("PCHOME_SALES_IMAP_PASSWORD", "")
|
|
port = env_int("PCHOME_SALES_IMAP_PORT", 993, 1, 65535)
|
|
mailbox = os.getenv("PCHOME_SALES_IMAP_MAILBOX", "INBOX").strip() or "INBOX"
|
|
client = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=20)
|
|
try:
|
|
client.login(user, password)
|
|
client.select(mailbox, readonly=False)
|
|
client.uid("store", uid, "+FLAGS", "(\\Seen)")
|
|
finally:
|
|
try:
|
|
client.logout()
|
|
except Exception:
|
|
pass
|
|
|
|
def imap_candidates(self) -> List[AcquisitionCandidate]:
|
|
if not env_bool("PCHOME_SALES_IMAP_ENABLED"):
|
|
return []
|
|
host = os.getenv("PCHOME_SALES_IMAP_HOST", "").strip()
|
|
user = os.getenv("PCHOME_SALES_IMAP_USER", "").strip()
|
|
password = os.getenv("PCHOME_SALES_IMAP_PASSWORD", "")
|
|
if not host or not user or not password:
|
|
raise ProviderError("imap_not_configured", "授權信箱來源已啟用但連線設定不完整。")
|
|
port = env_int("PCHOME_SALES_IMAP_PORT", 993, 1, 65535)
|
|
mailbox = os.getenv("PCHOME_SALES_IMAP_MAILBOX", "INBOX").strip() or "INBOX"
|
|
subject_pattern = os.getenv("PCHOME_SALES_IMAP_SUBJECT", "即時業績").strip()
|
|
max_messages = env_int("PCHOME_SALES_IMAP_MAX_MESSAGES", 20, 1, 100)
|
|
candidates: List[AcquisitionCandidate] = []
|
|
client = None
|
|
try:
|
|
client = imaplib.IMAP4_SSL(host, port, ssl_context=ssl.create_default_context(), timeout=20)
|
|
client.login(user, password)
|
|
status, _ = client.select(mailbox, readonly=True)
|
|
if status != "OK":
|
|
raise ProviderError("imap_mailbox_failed", "授權信箱資料夾目前無法開啟。")
|
|
status, data = client.uid("search", None, "UNSEEN")
|
|
if status != "OK":
|
|
raise ProviderError("imap_search_failed", "授權信箱目前無法搜尋新報表。")
|
|
uids = list(reversed((data[0] or b"").split()))[:max_messages]
|
|
for uid in uids:
|
|
status, payload = client.uid("fetch", uid, "(RFC822)")
|
|
if status != "OK" or not payload or not isinstance(payload[0], tuple):
|
|
continue
|
|
message = BytesParser(policy=policy.default).parsebytes(payload[0][1])
|
|
if subject_pattern and subject_pattern not in _decode_mail_header(message.get("Subject")):
|
|
continue
|
|
for attachment in message.iter_attachments():
|
|
file_name = _safe_file_name(_decode_mail_header(attachment.get_filename()))
|
|
suffix = Path(file_name).suffix.lower()
|
|
if suffix not in VALID_EXTENSIONS:
|
|
continue
|
|
temp_path = _make_temp_path(suffix)
|
|
try:
|
|
with open(temp_path, "wb") as handle:
|
|
handle.write(attachment.get_payload(decode=True) or b"")
|
|
_validate_excel_file(temp_path, file_name)
|
|
except Exception:
|
|
Path(temp_path).unlink(missing_ok=True)
|
|
raise
|
|
fingerprint = _sha256_file(temp_path)
|
|
candidates.append(AcquisitionCandidate(
|
|
source_type="authorized_imap",
|
|
file_path=temp_path,
|
|
file_name=file_name,
|
|
fingerprint=fingerprint,
|
|
source_ref_hash=_hash_text(f"{host}:{mailbox}:{uid.decode(errors='ignore')}"),
|
|
finalize_success=lambda message_uid=uid: self._mark_imap_seen(message_uid),
|
|
finalize_rejected=lambda message_uid=uid: self._mark_imap_seen(message_uid),
|
|
cleanup=lambda candidate_path=temp_path: Path(candidate_path).unlink(missing_ok=True),
|
|
))
|
|
except ProviderError:
|
|
for candidate in candidates:
|
|
candidate.cleanup()
|
|
raise
|
|
except (imaplib.IMAP4.error, OSError, ssl.SSLError) as exc:
|
|
for candidate in candidates:
|
|
candidate.cleanup()
|
|
raise ProviderError("imap_connection_failed", "授權信箱來源目前無法連線或驗證。") from exc
|
|
finally:
|
|
if client is not None:
|
|
try:
|
|
client.logout()
|
|
except Exception:
|
|
pass
|
|
return candidates
|
|
|
|
def local_candidates(self) -> List[AcquisitionCandidate]:
|
|
configured = os.getenv("PCHOME_SALES_LOCAL_DROP_DIR", "").strip()
|
|
if not configured:
|
|
return []
|
|
root = Path(configured).expanduser().resolve()
|
|
if not root.is_dir():
|
|
raise ProviderError("local_drop_missing", "受控落地目錄不存在，未執行檔案存取。")
|
|
max_files = env_int("PCHOME_SALES_MAX_FILES_PER_RUN", 5, 1, 25)
|
|
sources = sorted(
|
|
(
|
|
path for path in root.iterdir()
|
|
if path.is_file() and not path.is_symlink() and path.suffix.lower() in VALID_EXTENSIONS
|
|
),
|
|
key=lambda path: path.stat().st_mtime,
|
|
reverse=True,
|
|
)[:max_files]
|
|
candidates: List[AcquisitionCandidate] = []
|
|
for source in sources:
|
|
temp_path = _make_temp_path(source.suffix.lower())
|
|
try:
|
|
shutil.copyfile(str(source), temp_path)
|
|
_validate_excel_file(temp_path, source.name)
|
|
except Exception:
|
|
Path(temp_path).unlink(missing_ok=True)
|
|
for candidate in candidates:
|
|
candidate.cleanup()
|
|
raise
|
|
fingerprint = _sha256_file(temp_path)
|
|
candidates.append(AcquisitionCandidate(
|
|
source_type="controlled_local_drop",
|
|
file_path=temp_path,
|
|
file_name=_safe_file_name(source.name),
|
|
fingerprint=fingerprint,
|
|
source_ref_hash=_hash_text(str(source)),
|
|
finalize_success=lambda item=source, fp=fingerprint: _archive_file(item, "archive", fp),
|
|
finalize_rejected=lambda item=source, fp=fingerprint: _archive_file(item, "quarantine", fp),
|
|
cleanup=lambda candidate_path=temp_path: Path(candidate_path).unlink(missing_ok=True),
|
|
))
|
|
return candidates
|