jjsm01/sa-tender-monitor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
sa-tender-monitor/tests/test_ocds_client.py
jjsm01 2b39c0aa28 feat: initial SA Tender Monitor project 
- OCDS API client for eTenders.gov.za (primary tender discovery)
- Web UI scraper using Playwright (fetches supportDocument[] array
  that the OCDS API doesn't expose — only 1 doc per tender in OCDS)
- SQLite database with full schema for tenders + documents
- Click CLI: pull, stats, list, show commands
- 10 unit tests (all passing) covering models, OCDS parsing,
  web UI parsing, and document enrichment logic
- Comprehensive research report: 75+ SA tender sources documented
  with automation details (API/RSS/scraping availability per source)
- Hybrid architecture: OCDS for discovery + web scraping for full
  document lists (verified: web UI shows 2+ docs, OCDS shows 1)
2026-07-03 09:09:39 +00:00

232 lines
9.0 KiB
Python
Raw Permalink Blame History

"""Tests for the OCDS API client and data models."""
import json
from datetime import datetime, timezone
import pytest
from src.models import Tender, TenderDocument, TenderBuyer
from src.ocds_client import OcdsClient
# ── Fixtures ────────────────────────────────────────────────────────────────
SAMPLE_OCDS_RELEASE = {
"ocid": "ocds-9t57fa-123931",
"id": "123931",
"date": "2025-06-02T00:00:00Z",
"tag": ["tender"],
"initiationType": "tender",
"tender": {
"title": "RFQ2025/135",
"status": "complete",
"tenderPeriod": {
"startDate": "2025-06-02T00:00:00Z",
"endDate": "2025-06-03T10:00:00Z",
},
"documents": [
{
"id": "5f928b6e-0be2-47ec-a7fa-39b1bf6d8e0a",
"documentType": "basic",
"title": "21 BATTALION CLC ADVERT.pdf",
"description": "21 BATTALION CLC ADVERT.pdf",
"url": "https://www.etenders.gov.za/home/Download?blobName=5f928b6e-0be2-47ec-a7fa-39b1bf6d8e0a.pdf&downloadedFileName=21%20BATT",
"datePublished": "2025-06-02T00:00:00Z",
"dateModified": "2025-06-02T00:00:00Z",
"format": "pdf",
"language": "en",
}
],
},
"parties": [
{
"name": "Test Department",
"roles": ["buyer"],
"identifier": {"id": "TEST-001"},
}
],
"buyer": {"name": "Test Department"},
}
SAMPLE_WEB_UI_ROW = {
"id": 160907,
"tender_No": "TFR/2026/07/0002/114548/RFI",
"description": "NOTICE TO THE MARKET: INVITATION FOR INTERESTED PARTIES",
"status": "Published",
"category": "Services: Professional",
"type": "Request for Information",
"province": "Western Cape",
"department": "Transnet SOC Ltd",
"date_Published": "2026-07-02T00:00:00",
"closing_Date": "2026-09-03T23:00:00",
"contactPerson": "TRIM Commercial",
"email": "noticeandcomments@transnet.net",
"telephone": "031-308-8256",
"supportDocument": [
{
"supportDocumentID": "2b32232e-777c-41d1-86e1-8dd2bd0eea6e",
"extension": ".pptx",
"fileName": "Notice and Comment advert.pptx",
"dateModified": "2026-07-02T05:57:24.3088558",
"active": True,
},
{
"supportDocumentID": "40efd46e-c7c5-4342-a0f6-f4d1692adbbd",
"extension": ".pdf",
"fileName": "Notice and Comments Advertisement Form.pdf",
"dateModified": "2026-07-02T05:57:24.7810594",
"active": True,
},
],
}
# ── Model tests ─────────────────────────────────────────────────────────────
class TestTenderDocument:
def test_from_ocds(self):
doc_data = SAMPLE_OCDS_RELEASE["tender"]["documents"][0]
doc = TenderDocument.from_ocds(doc_data)
assert doc.document_id == "5f928b6e-0be2-47ec-a7fa-39b1bf6d8e0a"
assert doc.title == "21 BATTALION CLC ADVERT.pdf"
assert doc.format == "pdf"
assert doc.source == "ocds"
assert doc.url.startswith("https://www.etenders.gov.za/home/Download")
assert doc.download_url == doc.url
def test_from_web_ui(self):
sd = SAMPLE_WEB_UI_ROW["supportDocument"][0]
doc = TenderDocument.from_web_ui(sd)
assert doc.document_id == "2b32232e-777c-41d1-86e1-8dd2bd0eea6e"
assert doc.title == "Notice and Comment advert.pptx"
assert doc.format == "pptx"
assert doc.source == "web_ui"
assert doc.extension == ".pptx"
assert doc.support_document_id == "2b32232e-777c-41d1-86e1-8dd2bd0eea6e"
def test_download_url_construction(self):
"""Web UI documents construct their own download URL from supportDocumentID."""
sd = SAMPLE_WEB_UI_ROW["supportDocument"][0]
doc = TenderDocument.from_web_ui(sd)
expected = (
"https://www.etenders.gov.za/home/Download"
"?blobName=2b32232e-777c-41d1-86e1-8dd2bd0eea6e.pptx"
"&downloadedFileName=Notice and Comment advert.pptx"
)
assert doc.download_url == expected
class TestTender:
def test_from_ocds(self):
tender = Tender.from_ocds(SAMPLE_OCDS_RELEASE)
assert tender.ocid == "ocds-9t57fa-123931"
assert tender.title == "RFQ2025/135"
assert tender.status == "complete"
assert tender.source == "etenders_ocds"
assert len(tender.documents) == 1
assert tender.documents[0].title == "21 BATTALION CLC ADVERT.pdf"
assert tender.buyer is not None
assert tender.buyer.name == "Test Department"
assert tender.date_published is not None
assert tender.closing_date is not None
def test_from_web_ui(self):
tender = Tender.from_web_ui(SAMPLE_WEB_UI_ROW)
assert tender.internal_id == 160907
assert tender.tender_number == "TFR/2026/07/0002/114548/RFI"
assert tender.category == "Services: Professional"
assert tender.province == "Western Cape"
assert tender.department == "Transnet SOC Ltd"
assert tender.tender_type == "Request for Information"
assert tender.source == "etenders_web_ui"
assert len(tender.documents) == 2
assert tender.documents[0].file_name == "Notice and Comment advert.pptx"
assert tender.documents[1].file_name == "Notice and Comments Advertisement Form.pdf"
assert tender.buyer is not None
assert tender.buyer.email == "noticeandcomments@transnet.net"
def test_ocds_to_web_ui_doc_count_difference(self):
"""The key test: OCDS gives 1 doc, web UI gives 2 docs for the same tender."""
ocds_tender = Tender.from_ocds(SAMPLE_OCDS_RELEASE)
web_ui_tender = Tender.from_web_ui(SAMPLE_WEB_UI_ROW)
assert len(ocds_tender.documents) == 1
assert len(web_ui_tender.documents) == 2
def test_to_dict_serialization(self):
tender = Tender.from_ocds(SAMPLE_OCDS_RELEASE)
d = tender.to_dict()
assert d["ocid"] == "ocds-9t57fa-123931"
assert d["title"] == "RFQ2025/135"
assert d["doc_count"] == 1
assert len(d["documents"]) == 1
assert d["buyer"]["name"] == "Test Department"
# ── OCDS client tests (mocked) ───────────────────────────────────────────────
class TestOcdsClient:
def test_date_range(self):
"""Date range helper returns valid YYYY-MM-DD strings."""
date_from, date_to = OcdsClient.date_range(days=7)
assert date_from < date_to
# Should be valid date strings
datetime.strptime(date_from, "%Y-%m-%d")
datetime.strptime(date_to, "%Y-%m-%d")
# ── Web UI enrichment tests ──────────────────────────────────────────────────
class TestWebUiEnrichment:
def test_enrichment_merges_documents(self):
"""Enriching an OCDS tender with web UI data adds the missing documents."""
from src.web_scraper import WebUiScraper
ocds_tender = Tender.from_ocds(SAMPLE_OCDS_RELEASE)
assert len(ocds_tender.documents) == 1 # Only the advert
scraper = WebUiScraper()
# Simulate enrichment with web UI data (different tender, same concept)
web_ui_doc = {
"supportDocumentID": "new-doc-001",
"extension": ".pdf",
"fileName": "Additional Document.pdf",
"dateModified": "2025-06-01T00:00:00",
}
ocds_tender = scraper.enrich_tender_documents(
ocds_tender, {"supportDocument": [web_ui_doc]}
)
# Should now have 2 documents: the OCDS one + the web UI one
assert len(ocds_tender.documents) == 2
assert ocds_tender.documents[0].source == "ocds"
assert ocds_tender.documents[1].source == "web_ui"
assert ocds_tender.documents[1].file_name == "Additional Document.pdf"
def test_enrichment_does_not_duplicate(self):
"""If a document already exists (same ID), it's not added again."""
from src.web_scraper import WebUiScraper
ocds_tender = Tender.from_ocds(SAMPLE_OCDS_RELEASE)
assert len(ocds_tender.documents) == 1
scraper = WebUiScraper()
# Add the same document ID as the OCDS one
web_ui_doc = {
"supportDocumentID": "5f928b6e-0be2-47ec-a7fa-39b1bf6d8e0a",
"extension": ".pdf",
"fileName": "21 BATTALION CLC ADVERT.pdf",
"dateModified": "2025-06-02T00:00:00",
}
ocds_tender = scraper.enrich_tender_documents(
ocds_tender, {"supportDocument": [web_ui_doc]}
)
# Should still be 1 (no duplicate)
assert len(ocds_tender.documents) == 1
Reference in New Issue View Git Blame Copy Permalink