Add database migrations + README
- migrations/001_initial.sql: full schema with 12 tables, triggers, indexes - migrate.py: idempotent migration runner with version tracking - README.md: deploy instructions, architecture overview - .gitignore: exclude migration tracking data
This commit is contained in:
Jetour BI
1
.gitignore
vendored
1
.gitignore
vendored
@@ -5,3 +5,4 @@ __pycache__/
|
||||
.env
|
||||
*.log
|
||||
.DS_Store
|
||||
migrations/_migrations*
|
||||
|
||||
78
README.md
Normal file
78
README.md
Normal file
@@ -0,0 +1,78 @@
|
||||
# Jetour BI Dashboard
|
||||
|
||||
Dark-themed sales & CRM pipeline dashboard for Jetour Mauritius, powered by CMS BI data.
|
||||
|
||||
## Quick Deploy
|
||||
|
||||
```bash
|
||||
# 1. Clone
|
||||
git clone https://gitea.iseva.net.za/jjsm01/jetour-bi.git
|
||||
cd jetour-bi
|
||||
|
||||
# 2. Create database
|
||||
createdb jetour-bi
|
||||
|
||||
# 3. Run migrations
|
||||
pip install psycopg bcrypt
|
||||
DATABASE_URL="postgresql://postgres:***@localhost:5432/jetour-bi" python migrate.py
|
||||
|
||||
# 4. Start dashboard
|
||||
pip install fastapi uvicorn
|
||||
uvicorn app:app --host 0.0.0.0 --port 8765
|
||||
|
||||
# 5. Open http://localhost:8765
|
||||
# Default login: admin / admin
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
Browser → FastAPI (:8765) → PostgreSQL (jetour-bi)
|
||||
↑
|
||||
n8n workflows (02:00-02:50) ───┘ (populate BI data from CMS)
|
||||
n8n report triggers (07:00) ───┘ (daily/weekly/monthly emails)
|
||||
```
|
||||
|
||||
## Database
|
||||
|
||||
| Table | Purpose |
|
||||
|-------|---------|
|
||||
| `prospects` | CMS BI prospect records |
|
||||
| `prospect_otps` | Vehicle orders (model, color, amount, stock) |
|
||||
| `status_logs` | Status change history |
|
||||
| `change_log` | Field-level audit trail (auto-triggered) |
|
||||
| `appointments` | Delivery/test-drive appointments |
|
||||
| `report_recipients` | Email report recipients per type |
|
||||
| `app_settings` | SMTP config, app preferences |
|
||||
| `dashboard_users` | Auth users |
|
||||
|
||||
### Migrations
|
||||
|
||||
```bash
|
||||
# Apply all pending migrations (idempotent)
|
||||
DATABASE_URL="postgresql://user:***@host:5432/dbname" python migrate.py
|
||||
|
||||
# Or pass URL directly
|
||||
python migrate.py postgresql://user:***@host:5432/jetour-bi
|
||||
```
|
||||
|
||||
Migrations are tracked in the `_migrations` table. Safe to run repeatedly.
|
||||
|
||||
## Dashboard Pages
|
||||
|
||||
| URL | Description |
|
||||
|-----|-------------|
|
||||
| `/` | Main dashboard (pipeline, velocity, models, vehicle orders) |
|
||||
| `/settings.html` | Email report settings (recipients + SMTP config) |
|
||||
|
||||
## n8n Integration
|
||||
|
||||
5 workflows manage the data pipeline. n8n just calls dashboard endpoints:
|
||||
|
||||
| Workflow | Schedule | Endpoint |
|
||||
|----------|----------|----------|
|
||||
| Daily Report | 07:00 SAST | `POST /api/send-report/daily` |
|
||||
| Weekly Report | Mon 07:00 | `POST /api/send-report/weekly` |
|
||||
| Monthly Report | 1st 07:00 | `POST /api/send-report/monthly` |
|
||||
|
||||
Dashboard handles all SMTP send + recipient resolution.
|
||||
85
migrate.py
Executable file
85
migrate.py
Executable file
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Jetour BI — Database Migration Runner
|
||||
|
||||
Usage:
|
||||
python migrate.py [DB_URL]
|
||||
|
||||
Defaults to DATABASE_URL env var, or:
|
||||
postgresql://postgres:password@host:5432/jetour-bi
|
||||
|
||||
Runs all .sql files in migrations/ in order, tracking applied
|
||||
migrations in a _migrations table. Safe to run repeatedly —
|
||||
only unapplied migrations are executed.
|
||||
"""
|
||||
|
||||
import os, sys, re, psycopg
|
||||
from pathlib import Path
|
||||
|
||||
MIGRATIONS_DIR = Path(__file__).parent / "migrations"
|
||||
DB_URL = sys.argv[1] if len(sys.argv) > 1 else os.environ.get(
|
||||
"DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/jetour-bi"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
conn = psycopg.connect(DB_URL)
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
|
||||
# Ensure tracking table exists
|
||||
cur.execute("""
|
||||
CREATE TABLE IF NOT EXISTS _migrations (
|
||||
id SERIAL PRIMARY KEY,
|
||||
filename VARCHAR(200) NOT NULL UNIQUE,
|
||||
applied_at TIMESTAMPTZ DEFAULT NOW()
|
||||
)
|
||||
""")
|
||||
|
||||
# Get list of applied migrations
|
||||
cur.execute("SELECT filename FROM _migrations ORDER BY id")
|
||||
applied = {r[0] for r in cur.fetchall()}
|
||||
|
||||
# Find and run pending .sql files
|
||||
sql_files = sorted(MIGRATIONS_DIR.glob("*.sql"))
|
||||
if not sql_files:
|
||||
print("No migration files found in", MIGRATIONS_DIR)
|
||||
return
|
||||
|
||||
pending = 0
|
||||
for f in sql_files:
|
||||
if f.name in applied:
|
||||
continue
|
||||
print(f" Applying: {f.name} ...", end=" ", flush=True)
|
||||
try:
|
||||
sql = f.read_text()
|
||||
cur.execute(sql)
|
||||
cur.execute("INSERT INTO _migrations (filename) VALUES (%s)", (f.name,))
|
||||
print("✓")
|
||||
pending += 1
|
||||
except Exception as e:
|
||||
print(f"✗ {e}")
|
||||
conn.rollback()
|
||||
sys.exit(1)
|
||||
|
||||
if pending:
|
||||
print(f"\n✅ {pending} migration(s) applied")
|
||||
else:
|
||||
print("\n✓ All migrations already applied")
|
||||
|
||||
# Create a default admin user if none exists
|
||||
cur.execute("SELECT count(*) FROM dashboard_users")
|
||||
if cur.fetchone()[0] == 0:
|
||||
import bcrypt
|
||||
pw = bcrypt.hashpw(b"admin", bcrypt.gensalt()).decode()
|
||||
cur.execute(
|
||||
"INSERT INTO dashboard_users (username, email, password_hash, role) VALUES (%s, %s, %s, %s)",
|
||||
("admin", "admin@jetour.mu", pw, "admin"),
|
||||
)
|
||||
print("✓ Default admin user created (username: admin, password: admin)")
|
||||
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
245
migrations/001_initial.sql
Normal file
245
migrations/001_initial.sql
Normal file
@@ -0,0 +1,245 @@
|
||||
-- ============================================================================
|
||||
-- Jetour BI Dashboard — Initial Schema
|
||||
-- Creates all tables, triggers, indexes, and default settings
|
||||
-- ============================================================================
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- ── Auth ───────────────────────────────────────────────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dashboard_users (
|
||||
id SERIAL PRIMARY KEY,
|
||||
username VARCHAR(100) NOT NULL UNIQUE,
|
||||
email VARCHAR(200),
|
||||
password_hash TEXT NOT NULL,
|
||||
role VARCHAR(20) DEFAULT 'viewer',
|
||||
active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
last_login TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS password_reset_tokens (
|
||||
id SERIAL PRIMARY KEY,
|
||||
user_id INT NOT NULL REFERENCES dashboard_users(id) ON DELETE CASCADE,
|
||||
token VARCHAR(200) NOT NULL UNIQUE,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
used BOOLEAN DEFAULT false,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_reset_token ON password_reset_tokens(token) WHERE NOT used;
|
||||
|
||||
-- ── BI Data Tables (populated by CMS BI via n8n) ───────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS prospects (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
prospect_id BIGINT NOT NULL,
|
||||
cms_prospect_id INTEGER,
|
||||
name VARCHAR(100),
|
||||
surname VARCHAR(100),
|
||||
id_number VARCHAR(50),
|
||||
cell_number VARCHAR(50),
|
||||
email_addr VARCHAR(200),
|
||||
prospect_status VARCHAR(50),
|
||||
sales_person VARCHAR(150),
|
||||
source_of_initial_contact VARCHAR(100),
|
||||
referral_source VARCHAR(150),
|
||||
lost_reason TEXT,
|
||||
dt_prospect_created TIMESTAMPTZ,
|
||||
dt_updated TIMESTAMPTZ,
|
||||
export_guid UUID,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_prospects_status ON prospects(prospect_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_prospects_created ON prospects(dt_prospect_created);
|
||||
CREATE INDEX IF NOT EXISTS idx_prospects_sales ON prospects(sales_person);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_prospects_cms ON prospects(cms_prospect_id) WHERE cms_prospect_id IS NOT NULL;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS prospect_otps (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
prospect_id BIGINT REFERENCES prospects(prospect_id),
|
||||
cms_prospect_id INTEGER,
|
||||
mm_code VARCHAR(50),
|
||||
make VARCHAR(100),
|
||||
model VARCHAR(200),
|
||||
model_range VARCHAR(100),
|
||||
model_year INTEGER,
|
||||
colour VARCHAR(100),
|
||||
vin VARCHAR(50),
|
||||
stock_no VARCHAR(50),
|
||||
amount NUMERIC(18,2),
|
||||
otp_new_used VARCHAR(20),
|
||||
dt_otp_created TIMESTAMPTZ,
|
||||
dt_sold TIMESTAMPTZ,
|
||||
dt_invoiced TIMESTAMPTZ,
|
||||
export_guid UUID,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_otps_prospect ON prospect_otps(prospect_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_otps_model ON prospect_otps(model);
|
||||
CREATE INDEX IF NOT EXISTS idx_otps_range ON prospect_otps(model_range);
|
||||
CREATE INDEX IF NOT EXISTS idx_otps_created ON prospect_otps(dt_otp_created);
|
||||
CREATE INDEX IF NOT EXISTS idx_otps_invoiced ON prospect_otps(dt_invoiced);
|
||||
CREATE INDEX IF NOT EXISTS idx_otps_stock ON prospect_otps(stock_no);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS status_logs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
prospect_id BIGINT REFERENCES prospects(prospect_id),
|
||||
old_status VARCHAR(50),
|
||||
new_status VARCHAR(50),
|
||||
changed_by VARCHAR(100),
|
||||
dt_changed TIMESTAMPTZ DEFAULT NOW(),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_slogs_prospect ON status_logs(prospect_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_slogs_date ON status_logs(dt_changed);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS change_log (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
table_name VARCHAR(100) NOT NULL,
|
||||
record_id BIGINT NOT NULL,
|
||||
field_name VARCHAR(100) NOT NULL,
|
||||
old_value TEXT,
|
||||
new_value TEXT,
|
||||
changed_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_clog_table ON change_log(table_name, record_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_clog_date ON change_log(changed_at);
|
||||
|
||||
-- Change tracking trigger function
|
||||
CREATE OR REPLACE FUNCTION track_changes() RETURNS TRIGGER AS $$
|
||||
DECLARE
|
||||
col_name TEXT;
|
||||
old_val TEXT;
|
||||
new_val TEXT;
|
||||
BEGIN
|
||||
FOR col_name IN
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_schema = TG_TABLE_SCHEMA AND table_name = TG_TABLE_NAME
|
||||
AND column_name NOT IN ('id', 'created_at', 'updated_at', 'export_guid')
|
||||
LOOP
|
||||
EXECUTE format('SELECT ($1).%I::text, ($2).%I::text', col_name, col_name)
|
||||
INTO old_val, new_val USING OLD, NEW;
|
||||
IF old_val IS DISTINCT FROM new_val THEN
|
||||
INSERT INTO change_log (table_name, record_id, field_name, old_value, new_value)
|
||||
VALUES (TG_TABLE_NAME, NEW.id, col_name, old_val, new_val);
|
||||
END IF;
|
||||
END LOOP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_prospects_changes ON prospects;
|
||||
CREATE TRIGGER trg_prospects_changes AFTER UPDATE ON prospects
|
||||
FOR EACH ROW EXECUTE FUNCTION track_changes();
|
||||
|
||||
DROP TRIGGER IF EXISTS trg_otps_changes ON prospect_otps;
|
||||
CREATE TRIGGER trg_otps_changes AFTER UPDATE ON prospect_otps
|
||||
FOR EACH ROW EXECUTE FUNCTION track_changes();
|
||||
|
||||
-- ── Appointments ───────────────────────────────────────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS appointments (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
prospect_id BIGINT REFERENCES prospects(prospect_id),
|
||||
cms_prospect_id INTEGER,
|
||||
dt_scheduled TIMESTAMPTZ,
|
||||
appointment_type VARCHAR(100),
|
||||
appointment_status VARCHAR(50),
|
||||
confirmed BOOLEAN,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_appt_prospect ON appointments(prospect_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_appt_type ON appointments(appointment_type);
|
||||
|
||||
-- ── Leads ──────────────────────────────────────────────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS leads (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
lead_id INTEGER,
|
||||
prospect_id BIGINT REFERENCES prospects(prospect_id),
|
||||
cms_prospect_id INTEGER,
|
||||
name VARCHAR(100),
|
||||
surname VARCHAR(100),
|
||||
cell_number VARCHAR(50),
|
||||
email_addr VARCHAR(200),
|
||||
prospect_status VARCHAR(50),
|
||||
lead_source VARCHAR(50),
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- ── Report Snapshots (pre-computed aggregations) ───────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS report_daily_kpis (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
report_date DATE NOT NULL UNIQUE,
|
||||
total_prospects INT DEFAULT 0,
|
||||
new_prospects INT DEFAULT 0,
|
||||
total_otps INT DEFAULT 0,
|
||||
new_otps INT DEFAULT 0,
|
||||
sold_count INT DEFAULT 0,
|
||||
pipeline_value NUMERIC(18,2) DEFAULT 0,
|
||||
avg_otp_value NUMERIC(18,2) DEFAULT 0,
|
||||
pending_count INT DEFAULT 0,
|
||||
finance_count INT DEFAULT 0,
|
||||
lost_count INT DEFAULT 0,
|
||||
conversion_rate NUMERIC(5,2) DEFAULT 0,
|
||||
generated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS report_model_performance (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
report_date DATE NOT NULL,
|
||||
model_group VARCHAR(100) NOT NULL,
|
||||
model VARCHAR(200) NOT NULL,
|
||||
color VARCHAR(100),
|
||||
otp_type VARCHAR(20),
|
||||
otp_count INT DEFAULT 0,
|
||||
total_amount NUMERIC(18,2) DEFAULT 0,
|
||||
avg_amount NUMERIC(18,2) DEFAULT 0,
|
||||
sold_count INT DEFAULT 0,
|
||||
generated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(report_date, model_group, model, color, otp_type)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS report_salesperson_performance (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
report_date DATE NOT NULL,
|
||||
sales_person VARCHAR(150) NOT NULL,
|
||||
total_prospects INT DEFAULT 0,
|
||||
otp_count INT DEFAULT 0,
|
||||
sold_count INT DEFAULT 0,
|
||||
finance_count INT DEFAULT 0,
|
||||
lost_count INT DEFAULT 0,
|
||||
total_pipeline_value NUMERIC(18,2) DEFAULT 0,
|
||||
generated_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(report_date, sales_person)
|
||||
);
|
||||
|
||||
-- ── App Settings & Recipients ──────────────────────────────────────────────
|
||||
|
||||
CREATE TABLE IF NOT EXISTS report_recipients (
|
||||
id SERIAL PRIMARY KEY,
|
||||
report_type VARCHAR(20) NOT NULL,
|
||||
email VARCHAR(200) NOT NULL,
|
||||
name VARCHAR(100),
|
||||
active BOOLEAN DEFAULT true,
|
||||
added_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(report_type, email)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS app_settings (
|
||||
key VARCHAR(100) PRIMARY KEY,
|
||||
value TEXT,
|
||||
updated_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- ── Default settings ───────────────────────────────────────────────────────
|
||||
|
||||
INSERT INTO app_settings (key, value) VALUES
|
||||
('smtp_port', '587'),
|
||||
('smtp_use_tls', 'true')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
|
||||
COMMIT;
|
||||
Reference in New Issue
Block a user