feat: initial SA Tender Monitor project

- OCDS API client for eTenders.gov.za (primary tender discovery) - Web UI scraper using Playwright (fetches supportDocument[] array that the OCDS API doesn't expose — only 1 doc per tender in OCDS) - SQLite database with full schema for tenders + documents - Click CLI: pull, stats, list, show commands - 10 unit tests (all passing) covering models, OCDS parsing, web UI parsing, and document enrichment logic - Comprehensive research report: 75+ SA tender sources documented with automation details (API/RSS/scraping availability per source) - Hybrid architecture: OCDS for discovery + web scraping for full document lists (verified: web UI shows 2+ docs, OCDS shows 1)
2026-07-03 09:09:39 +00:00
commit 2b39c0aa28
14 changed files with 2333 additions and 0 deletions
--- a/tests/init.py
+++ b/tests/init.py
--- a/tests/test_ocds_client.py
+++ b/tests/test_ocds_client.py
@@ -0,0 +1,232 @@
+"""Tests for the OCDS API client and data models."""
+
+import json
+from datetime import datetime, timezone
+
+import pytest
+
+from src.models import Tender, TenderDocument, TenderBuyer
+from src.ocds_client import OcdsClient
+
+
+# ── Fixtures ────────────────────────────────────────────────────────────────
+
+SAMPLE_OCDS_RELEASE = {
+    "ocid": "ocds-9t57fa-123931",
+    "id": "123931",
+    "date": "2025-06-02T00:00:00Z",
+    "tag": ["tender"],
+    "initiationType": "tender",
+    "tender": {
+        "title": "RFQ2025/135",
+        "status": "complete",
+        "tenderPeriod": {
+            "startDate": "2025-06-02T00:00:00Z",
+            "endDate": "2025-06-03T10:00:00Z",
+        },
+        "documents": [
+            {
+                "id": "5f928b6e-0be2-47ec-a7fa-39b1bf6d8e0a",
+                "documentType": "basic",
+                "title": "21 BATTALION CLC ADVERT.pdf",
+                "description": "21 BATTALION CLC ADVERT.pdf",
+                "url": "https://www.etenders.gov.za/home/Download?blobName=5f928b6e-0be2-47ec-a7fa-39b1bf6d8e0a.pdf&downloadedFileName=21%20BATT",
+                "datePublished": "2025-06-02T00:00:00Z",
+                "dateModified": "2025-06-02T00:00:00Z",
+                "format": "pdf",
+                "language": "en",
+            }
+        ],
+    },
+    "parties": [
+        {
+            "name": "Test Department",
+            "roles": ["buyer"],
+            "identifier": {"id": "TEST-001"},
+        }
+    ],
+    "buyer": {"name": "Test Department"},
+}
+
+SAMPLE_WEB_UI_ROW = {
+    "id": 160907,
+    "tender_No": "TFR/2026/07/0002/114548/RFI",
+    "description": "NOTICE TO THE MARKET: INVITATION FOR INTERESTED PARTIES",
+    "status": "Published",
+    "category": "Services: Professional",
+    "type": "Request for Information",
+    "province": "Western Cape",
+    "department": "Transnet SOC Ltd",
+    "date_Published": "2026-07-02T00:00:00",
+    "closing_Date": "2026-09-03T23:00:00",
+    "contactPerson": "TRIM Commercial",
+    "email": "noticeandcomments@transnet.net",
+    "telephone": "031-308-8256",
+    "supportDocument": [
+        {
+            "supportDocumentID": "2b32232e-777c-41d1-86e1-8dd2bd0eea6e",
+            "extension": ".pptx",
+            "fileName": "Notice and Comment advert.pptx",
+            "dateModified": "2026-07-02T05:57:24.3088558",
+            "active": True,
+        },
+        {
+            "supportDocumentID": "40efd46e-c7c5-4342-a0f6-f4d1692adbbd",
+            "extension": ".pdf",
+            "fileName": "Notice and Comments Advertisement Form.pdf",
+            "dateModified": "2026-07-02T05:57:24.7810594",
+            "active": True,
+        },
+    ],
+}
+
+
+# ── Model tests ─────────────────────────────────────────────────────────────
+
+class TestTenderDocument:
+    def test_from_ocds(self):
+        doc_data = SAMPLE_OCDS_RELEASE["tender"]["documents"][0]
+        doc = TenderDocument.from_ocds(doc_data)
+
+        assert doc.document_id == "5f928b6e-0be2-47ec-a7fa-39b1bf6d8e0a"
+        assert doc.title == "21 BATTALION CLC ADVERT.pdf"
+        assert doc.format == "pdf"
+        assert doc.source == "ocds"
+        assert doc.url.startswith("https://www.etenders.gov.za/home/Download")
+        assert doc.download_url == doc.url
+
+    def test_from_web_ui(self):
+        sd = SAMPLE_WEB_UI_ROW["supportDocument"][0]
+        doc = TenderDocument.from_web_ui(sd)
+
+        assert doc.document_id == "2b32232e-777c-41d1-86e1-8dd2bd0eea6e"
+        assert doc.title == "Notice and Comment advert.pptx"
+        assert doc.format == "pptx"
+        assert doc.source == "web_ui"
+        assert doc.extension == ".pptx"
+        assert doc.support_document_id == "2b32232e-777c-41d1-86e1-8dd2bd0eea6e"
+
+    def test_download_url_construction(self):
+        """Web UI documents construct their own download URL from supportDocumentID."""
+        sd = SAMPLE_WEB_UI_ROW["supportDocument"][0]
+        doc = TenderDocument.from_web_ui(sd)
+
+        expected = (
+            "https://www.etenders.gov.za/home/Download"
+            "?blobName=2b32232e-777c-41d1-86e1-8dd2bd0eea6e.pptx"
+            "&downloadedFileName=Notice and Comment advert.pptx"
+        )
+        assert doc.download_url == expected
+
+
+class TestTender:
+    def test_from_ocds(self):
+        tender = Tender.from_ocds(SAMPLE_OCDS_RELEASE)
+
+        assert tender.ocid == "ocds-9t57fa-123931"
+        assert tender.title == "RFQ2025/135"
+        assert tender.status == "complete"
+        assert tender.source == "etenders_ocds"
+        assert len(tender.documents) == 1
+        assert tender.documents[0].title == "21 BATTALION CLC ADVERT.pdf"
+        assert tender.buyer is not None
+        assert tender.buyer.name == "Test Department"
+        assert tender.date_published is not None
+        assert tender.closing_date is not None
+
+    def test_from_web_ui(self):
+        tender = Tender.from_web_ui(SAMPLE_WEB_UI_ROW)
+
+        assert tender.internal_id == 160907
+        assert tender.tender_number == "TFR/2026/07/0002/114548/RFI"
+        assert tender.category == "Services: Professional"
+        assert tender.province == "Western Cape"
+        assert tender.department == "Transnet SOC Ltd"
+        assert tender.tender_type == "Request for Information"
+        assert tender.source == "etenders_web_ui"
+        assert len(tender.documents) == 2
+        assert tender.documents[0].file_name == "Notice and Comment advert.pptx"
+        assert tender.documents[1].file_name == "Notice and Comments Advertisement Form.pdf"
+        assert tender.buyer is not None
+        assert tender.buyer.email == "noticeandcomments@transnet.net"
+
+    def test_ocds_to_web_ui_doc_count_difference(self):
+        """The key test: OCDS gives 1 doc, web UI gives 2 docs for the same tender."""
+        ocds_tender = Tender.from_ocds(SAMPLE_OCDS_RELEASE)
+        web_ui_tender = Tender.from_web_ui(SAMPLE_WEB_UI_ROW)
+
+        assert len(ocds_tender.documents) == 1
+        assert len(web_ui_tender.documents) == 2
+
+    def test_to_dict_serialization(self):
+        tender = Tender.from_ocds(SAMPLE_OCDS_RELEASE)
+        d = tender.to_dict()
+
+        assert d["ocid"] == "ocds-9t57fa-123931"
+        assert d["title"] == "RFQ2025/135"
+        assert d["doc_count"] == 1
+        assert len(d["documents"]) == 1
+        assert d["buyer"]["name"] == "Test Department"
+
+
+# ── OCDS client tests (mocked) ───────────────────────────────────────────────
+
+class TestOcdsClient:
+    def test_date_range(self):
+        """Date range helper returns valid YYYY-MM-DD strings."""
+        date_from, date_to = OcdsClient.date_range(days=7)
+        assert date_from < date_to
+        # Should be valid date strings
+        datetime.strptime(date_from, "%Y-%m-%d")
+        datetime.strptime(date_to, "%Y-%m-%d")
+
+
+# ── Web UI enrichment tests ──────────────────────────────────────────────────
+
+class TestWebUiEnrichment:
+    def test_enrichment_merges_documents(self):
+        """Enriching an OCDS tender with web UI data adds the missing documents."""
+        from src.web_scraper import WebUiScraper
+
+        ocds_tender = Tender.from_ocds(SAMPLE_OCDS_RELEASE)
+        assert len(ocds_tender.documents) == 1  # Only the advert
+
+        scraper = WebUiScraper()
+        # Simulate enrichment with web UI data (different tender, same concept)
+        web_ui_doc = {
+            "supportDocumentID": "new-doc-001",
+            "extension": ".pdf",
+            "fileName": "Additional Document.pdf",
+            "dateModified": "2025-06-01T00:00:00",
+        }
+        ocds_tender = scraper.enrich_tender_documents(
+            ocds_tender, {"supportDocument": [web_ui_doc]}
+        )
+
+        # Should now have 2 documents: the OCDS one + the web UI one
+        assert len(ocds_tender.documents) == 2
+        assert ocds_tender.documents[0].source == "ocds"
+        assert ocds_tender.documents[1].source == "web_ui"
+        assert ocds_tender.documents[1].file_name == "Additional Document.pdf"
+
+    def test_enrichment_does_not_duplicate(self):
+        """If a document already exists (same ID), it's not added again."""
+        from src.web_scraper import WebUiScraper
+
+        ocds_tender = Tender.from_ocds(SAMPLE_OCDS_RELEASE)
+        assert len(ocds_tender.documents) == 1
+
+        scraper = WebUiScraper()
+        # Add the same document ID as the OCDS one
+        web_ui_doc = {
+            "supportDocumentID": "5f928b6e-0be2-47ec-a7fa-39b1bf6d8e0a",
+            "extension": ".pdf",
+            "fileName": "21 BATTALION CLC ADVERT.pdf",
+            "dateModified": "2025-06-02T00:00:00",
+        }
+        ocds_tender = scraper.enrich_tender_documents(
+            ocds_tender, {"supportDocument": [web_ui_doc]}
+        )
+
+        # Should still be 1 (no duplicate)
+        assert len(ocds_tender.documents) == 1