修復 Wave 0 阻塞與 market intel 入庫
This commit is contained in:
OoO
237
migrations/032_market_intel_core_schema.sql
Normal file
237
migrations/032_market_intel_core_schema.sql
Normal file
@@ -0,0 +1,237 @@
|
||||
-- =============================================================================
|
||||
-- Migration 032: market_intel core schema
|
||||
-- MOMO PRO - Cross-platform market campaign intelligence
|
||||
-- 2026-05-07 Taipei
|
||||
-- =============================================================================
|
||||
-- Notes:
|
||||
-- Creates the ADR-035 market_* schema. This migration is additive only:
|
||||
-- it creates tables, indexes, and grants. It does not drop or alter existing
|
||||
-- sales tables, and it does not touch the momo-db container lifecycle.
|
||||
-- =============================================================================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS market_platforms (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
code VARCHAR(50) NOT NULL UNIQUE,
|
||||
name VARCHAR(120) NOT NULL,
|
||||
base_url VARCHAR(500),
|
||||
enabled BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
crawl_policy_json TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_market_platforms_code
|
||||
ON market_platforms (code);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS market_campaigns (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
platform_code VARCHAR(50) NOT NULL REFERENCES market_platforms(code),
|
||||
campaign_key VARCHAR(200) NOT NULL,
|
||||
campaign_name VARCHAR(500) NOT NULL,
|
||||
campaign_type VARCHAR(80),
|
||||
campaign_url TEXT,
|
||||
start_at TIMESTAMP,
|
||||
end_at TIMESTAMP,
|
||||
status VARCHAR(30) NOT NULL DEFAULT 'unknown',
|
||||
discovered_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
last_seen_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
metadata_json TEXT,
|
||||
CONSTRAINT uq_market_campaign_platform_key
|
||||
UNIQUE (platform_code, campaign_key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaigns_platform_code
|
||||
ON market_campaigns (platform_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaigns_campaign_type
|
||||
ON market_campaigns (campaign_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaigns_status
|
||||
ON market_campaigns (status);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_status_time
|
||||
ON market_campaigns (status, start_at, end_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS market_campaign_snapshots (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
campaign_id BIGINT NOT NULL REFERENCES market_campaigns(id),
|
||||
batch_id VARCHAR(80) NOT NULL,
|
||||
crawled_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
title VARCHAR(500),
|
||||
hero_text TEXT,
|
||||
coupon_text TEXT,
|
||||
raw_discount_text TEXT,
|
||||
page_hash VARCHAR(128),
|
||||
raw_snapshot_path TEXT,
|
||||
status VARCHAR(30) NOT NULL DEFAULT 'success',
|
||||
error_message TEXT,
|
||||
metadata_json TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_snapshots_campaign_id
|
||||
ON market_campaign_snapshots (campaign_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_snapshots_batch_id
|
||||
ON market_campaign_snapshots (batch_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_snapshots_crawled_at
|
||||
ON market_campaign_snapshots (crawled_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_snapshots_page_hash
|
||||
ON market_campaign_snapshots (page_hash);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_snapshots_status
|
||||
ON market_campaign_snapshots (status);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_snapshot_campaign_time
|
||||
ON market_campaign_snapshots (campaign_id, crawled_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS market_campaign_products (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
campaign_id BIGINT NOT NULL REFERENCES market_campaigns(id),
|
||||
platform_code VARCHAR(50) NOT NULL,
|
||||
platform_product_id VARCHAR(200) NOT NULL,
|
||||
product_url TEXT,
|
||||
name VARCHAR(500) NOT NULL,
|
||||
brand VARCHAR(200),
|
||||
image_url TEXT,
|
||||
category_text VARCHAR(300),
|
||||
price DOUBLE PRECISION,
|
||||
original_price DOUBLE PRECISION,
|
||||
discount_text VARCHAR(200),
|
||||
discount_rate DOUBLE PRECISION,
|
||||
coupon_text TEXT,
|
||||
stock_text VARCHAR(200),
|
||||
sold_count INTEGER,
|
||||
rating DOUBLE PRECISION,
|
||||
review_count INTEGER,
|
||||
rank_position INTEGER,
|
||||
is_active BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
first_seen_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
last_seen_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
metadata_json TEXT,
|
||||
CONSTRAINT uq_market_campaign_product
|
||||
UNIQUE (campaign_id, platform_code, platform_product_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_products_campaign_id
|
||||
ON market_campaign_products (campaign_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_products_platform_code
|
||||
ON market_campaign_products (platform_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_products_platform_product_id
|
||||
ON market_campaign_products (platform_product_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_products_brand
|
||||
ON market_campaign_products (brand);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_products_category_text
|
||||
ON market_campaign_products (category_text);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_products_is_active
|
||||
ON market_campaign_products (is_active);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_campaign_products_last_seen_at
|
||||
ON market_campaign_products (last_seen_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_platform_seen
|
||||
ON market_campaign_products (platform_code, last_seen_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_discount
|
||||
ON market_campaign_products (discount_rate, price);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS market_product_price_history (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
market_product_id BIGINT NOT NULL REFERENCES market_campaign_products(id),
|
||||
campaign_id BIGINT NOT NULL REFERENCES market_campaigns(id),
|
||||
platform_code VARCHAR(50) NOT NULL,
|
||||
platform_product_id VARCHAR(200) NOT NULL,
|
||||
price DOUBLE PRECISION,
|
||||
original_price DOUBLE PRECISION,
|
||||
discount_rate DOUBLE PRECISION,
|
||||
stock_text VARCHAR(200),
|
||||
sold_count INTEGER,
|
||||
rank_position INTEGER,
|
||||
crawled_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
batch_id VARCHAR(80) NOT NULL,
|
||||
metadata_json TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_price_history_market_product_id
|
||||
ON market_product_price_history (market_product_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_price_history_campaign_id
|
||||
ON market_product_price_history (campaign_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_price_history_platform_code
|
||||
ON market_product_price_history (platform_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_price_history_platform_product_id
|
||||
ON market_product_price_history (platform_product_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_price_history_crawled_at
|
||||
ON market_product_price_history (crawled_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_price_history_batch_id
|
||||
ON market_product_price_history (batch_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_price_platform_time
|
||||
ON market_product_price_history (platform_code, platform_product_id, crawled_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_price_campaign_time
|
||||
ON market_product_price_history (campaign_id, crawled_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS market_product_matches (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
market_product_id BIGINT NOT NULL REFERENCES market_campaign_products(id),
|
||||
momo_product_id INTEGER REFERENCES products(id),
|
||||
momo_i_code VARCHAR(50),
|
||||
match_score DOUBLE PRECISION NOT NULL DEFAULT 0.0,
|
||||
match_status VARCHAR(30) NOT NULL DEFAULT 'needs_review',
|
||||
match_reason_json TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
reviewed_at TIMESTAMP,
|
||||
reviewed_by VARCHAR(120),
|
||||
CONSTRAINT uq_market_product_momo_match
|
||||
UNIQUE (market_product_id, momo_i_code)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_matches_market_product_id
|
||||
ON market_product_matches (market_product_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_matches_momo_product_id
|
||||
ON market_product_matches (momo_product_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_matches_momo_i_code
|
||||
ON market_product_matches (momo_i_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_product_matches_match_status
|
||||
ON market_product_matches (match_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_match_status_score
|
||||
ON market_product_matches (match_status, match_score);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS market_crawler_runs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
platform_code VARCHAR(50),
|
||||
crawler_name VARCHAR(120) NOT NULL,
|
||||
campaign_id BIGINT REFERENCES market_campaigns(id),
|
||||
batch_id VARCHAR(80) NOT NULL,
|
||||
started_at TIMESTAMP NOT NULL DEFAULT NOW(),
|
||||
finished_at TIMESTAMP,
|
||||
status VARCHAR(30) NOT NULL DEFAULT 'started',
|
||||
dry_run BOOLEAN NOT NULL DEFAULT TRUE,
|
||||
pages_found INTEGER NOT NULL DEFAULT 0,
|
||||
products_found INTEGER NOT NULL DEFAULT 0,
|
||||
products_changed INTEGER NOT NULL DEFAULT 0,
|
||||
error_count INTEGER NOT NULL DEFAULT 0,
|
||||
error_message TEXT,
|
||||
metadata_json TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_market_crawler_runs_platform_code
|
||||
ON market_crawler_runs (platform_code);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_crawler_runs_crawler_name
|
||||
ON market_crawler_runs (crawler_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_crawler_runs_campaign_id
|
||||
ON market_crawler_runs (campaign_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_crawler_runs_batch_id
|
||||
ON market_crawler_runs (batch_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_crawler_runs_started_at
|
||||
ON market_crawler_runs (started_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_crawler_runs_status
|
||||
ON market_crawler_runs (status);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_crawler_run_platform_time
|
||||
ON market_crawler_runs (platform_code, started_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_market_crawler_run_status_time
|
||||
ON market_crawler_runs (status, started_at);
|
||||
|
||||
GRANT ALL PRIVILEGES ON market_platforms TO momo;
|
||||
GRANT ALL PRIVILEGES ON market_campaigns TO momo;
|
||||
GRANT ALL PRIVILEGES ON market_campaign_snapshots TO momo;
|
||||
GRANT ALL PRIVILEGES ON market_campaign_products TO momo;
|
||||
GRANT ALL PRIVILEGES ON market_product_price_history TO momo;
|
||||
GRANT ALL PRIVILEGES ON market_product_matches TO momo;
|
||||
GRANT ALL PRIVILEGES ON market_crawler_runs TO momo;
|
||||
|
||||
GRANT USAGE, SELECT ON SEQUENCE market_platforms_id_seq TO momo;
|
||||
GRANT USAGE, SELECT ON SEQUENCE market_campaigns_id_seq TO momo;
|
||||
GRANT USAGE, SELECT ON SEQUENCE market_campaign_snapshots_id_seq TO momo;
|
||||
GRANT USAGE, SELECT ON SEQUENCE market_campaign_products_id_seq TO momo;
|
||||
GRANT USAGE, SELECT ON SEQUENCE market_product_price_history_id_seq TO momo;
|
||||
GRANT USAGE, SELECT ON SEQUENCE market_product_matches_id_seq TO momo;
|
||||
GRANT USAGE, SELECT ON SEQUENCE market_crawler_runs_id_seq TO momo;
|
||||
38
migrations/033_fix_host_health_probe_labels.sql
Normal file
38
migrations/033_fix_host_health_probe_labels.sql
Normal file
@@ -0,0 +1,38 @@
|
||||
-- =============================================================================
|
||||
-- Migration 033: host_health_probes host_label CHECK alignment
|
||||
-- 2026-05-12 Taipei
|
||||
-- =============================================================================
|
||||
-- Context:
|
||||
-- Migration 029 only allowed the original dashboard labels:
|
||||
-- Primary (GCP), Secondary (GCP), Fallback (111)
|
||||
-- services/ollama_service.py::get_host_label() later added operational labels:
|
||||
-- GCP-SSD, GCP-SSD-2, 111 備援, GCP-SSD（via Nginx 110）,
|
||||
-- GCP-SSD-2（via Nginx 110）
|
||||
--
|
||||
-- Keep the existing constraint name to avoid downstream drift, but widen the
|
||||
-- allowed set. NOT VALID avoids failing on any historical probe rows while still
|
||||
-- enforcing the widened CHECK for future inserts.
|
||||
-- =============================================================================
|
||||
|
||||
ALTER TABLE IF EXISTS host_health_probes
|
||||
DROP CONSTRAINT IF EXISTS chk_host_label_029;
|
||||
|
||||
ALTER TABLE IF EXISTS host_health_probes
|
||||
ADD CONSTRAINT chk_host_label_029
|
||||
CHECK (
|
||||
host_label IN (
|
||||
'Primary (GCP)',
|
||||
'Secondary (GCP)',
|
||||
'Fallback (111)',
|
||||
'GCP-SSD',
|
||||
'GCP-SSD-2',
|
||||
'111 備援',
|
||||
'GCP-SSD（via Nginx 110）',
|
||||
'GCP-SSD-2（via Nginx 110）',
|
||||
'188 本地',
|
||||
'未知'
|
||||
)
|
||||
) NOT VALID;
|
||||
|
||||
COMMENT ON CONSTRAINT chk_host_label_029 ON host_health_probes IS
|
||||
'Accepted labels from migration 029 dashboard probes and services/ollama_service.py::get_host_label()';
|
||||
Reference in New Issue
Block a user