diff --git a/src/web_scraper.py b/src/web_scraper.py index 1797cdc..2b39acd 100644 --- a/src/web_scraper.py +++ b/src/web_scraper.py @@ -21,76 +21,16 @@ from .config import WebUiConfig from .models import Tender, TenderDocument -# JavaScript injected into the page to intercept DataTables AJAX responses -# and extract the raw JSON data (which includes supportDocument[]). -INTERCEPT_JS = """ -(async () => { - // Intercept the next DataTables AJAX response - return new Promise((resolve) => { - // Override XMLHttpRequest to capture the DataTables response - const origOpen = XMLHttpRequest.prototype.open; - const origSend = XMLHttpRequest.prototype.send; - - XMLHttpRequest.prototype.open = function(method, url) { - this._url = url; - return origOpen.apply(this, arguments); - }; - - XMLHttpRequest.prototype.send = function() { - const xhr = this; - const origOnReady = xhr.onreadystatechange; - xhr.onreadystatechange = function() { - if (xhr.readyState === 4 && xhr._url && - xhr._url.includes('PaginatedTenderOpportunities')) { - try { - const data = JSON.parse(xhr.responseText); - window.__tenderData = data; - } catch(e) { - window.__tenderData = null; - } - } - if (origOnReady) origOnReady.apply(xhr, arguments); - }; - return origSend.apply(xhr, arguments); - }; - - // Wait for the data to arrive, with a timeout - let attempts = 0; - const checkInterval = setInterval(() => { - attempts++; - if (window.__tenderData || attempts > 100) { - clearInterval(checkInterval); - resolve(JSON.stringify(window.__tenderData || {})); - } - }, 200); - }); -})() -""" - -# Alternative: directly read the DataTable API data (if table is already loaded) -READ_TABLE_DATA_JS = """ -(() => { - if (typeof table === 'undefined' || !table) { - return JSON.stringify({error: 'table not initialized'}); - } - const data = table.rows().data(); - const rows = []; - for (let i = 0; i < data.length; i++) { - const row = data[i]; - if (row) rows.push(row); - } - return JSON.stringify({rows: rows, count: rows.length}); -})() -""" - - class WebUiScraper: """Scrapes the eTenders web UI for full tender document lists. Uses Playwright to load the opportunities page, which triggers the DataTables AJAX call to /Home/PaginatedTenderOpportunities. The response - contains each tender row with a supportDocument[] array holding all - attached files — something the OCDS API doesn't provide. + is intercepted at the network layer and parsed for the supportDocument[] + array — something the OCDS API doesn't provide. + + For pagination, subsequent pages are fetched by triggering + table.page('next').draw() and intercepting the next AJAX response. """ def __init__(self, config: WebUiConfig | None = None) -> None: @@ -98,33 +38,58 @@ class WebUiScraper: self._playwright = None self._browser = None self._page = None + self._context = None + self._captured_responses: list[dict[str, Any]] = [] def start(self) -> None: - """Launch the browser and open the opportunities page.""" + """Launch the browser.""" from playwright.sync_api import sync_playwright self._playwright = sync_playwright().start() self._browser = self._playwright.chromium.launch( headless=self.config.headless ) - context = self._browser.new_context( + self._context = self._browser.new_context( user_agent=( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/126.0.0.0 Safari/537.36" ) ) - self._page = context.new_page() + self._page = self._context.new_page() + + # Intercept DataTables AJAX responses at the network layer + self._page.on( + "response", + lambda resp: self._capture_response(resp), + ) + + def _capture_response(self, resp: Any) -> None: + """Capture DataTables AJAX responses.""" + if "PaginatedTenderOpportunities" not in resp.url: + return + try: + if resp.status == 200: + body = resp.text() + data = json.loads(body) + if "data" in data: + self._captured_responses.append(data) + except Exception: + pass def stop(self) -> None: """Close the browser.""" + if self._context: + self._context.close() if self._browser: self._browser.close() if self._playwright: self._playwright.stop() self._page = None + self._context = None self._browser = None self._playwright = None + self._captured_responses.clear() def __enter__(self) -> WebUiScraper: self.start() @@ -133,30 +98,32 @@ class WebUiScraper: def __exit__(self, *args: Any) -> None: self.stop() - def _load_page(self) -> None: - """Navigate to the opportunities page and wait for DataTables to load.""" + def _load_and_wait(self, timeout_ms: int = 15_000) -> None: + """Load the page and wait for the first DataTables AJAX response.""" url = f"{self.config.base_url}{self.config.opportunities_path}" - self._page.goto(url, wait_until="networkidle", timeout=self.config.timeout_ms) - # Wait for the DataTable to be populated - self._page.wait_for_function( - "typeof table !== 'undefined' && table && table.rows().count() > 0", - timeout=self.config.timeout_ms, - ) + # Use 'domcontentloaded' — the site keeps connections open so + # 'networkidle' never fires. + self._page.goto(url, wait_until="domcontentloaded", timeout=self.config.timeout_ms) - def _get_table_rows(self) -> list[dict[str, Any]]: - """Extract all tender row data from the loaded DataTable.""" - # Execute JavaScript to read all row data directly from the DataTable API - result = self._page.evaluate(READ_TABLE_DATA_JS) - data = json.loads(result) if isinstance(result, str) else result + # Wait for the first DataTables response to be captured + deadline = time.time() + (timeout_ms / 1000) + while time.time() < deadline: + if self._captured_responses: + return + self._page.wait_for_timeout(500) - if "error" in data: - raise RuntimeError(f"DataTable not available: {data['error']}") - - return data.get("rows", []) + # Fallback: check if the table JS variable exists + try: + self._page.wait_for_function( + "typeof table !== 'undefined' && table && table.rows().count() > 0", + timeout=5_000, + ) + except Exception: + pass def fetch_tender_rows(self, status: int = 1) -> list[dict[str, Any]]: - """Fetch tender rows from the web UI. + """Fetch all tender rows from the web UI. Args: status: 1=Advertised, 2=Awarded, 3=Closed, 4=Cancelled @@ -167,30 +134,52 @@ class WebUiScraper: if not self._page: raise RuntimeError("Scraper not started. Call start() or use context manager.") - self._load_page() + self._captured_responses.clear() - # If a different status is requested, click the corresponding tab - if status != 1: - tab_selectors = {2: "a:has-text('Awarded')", 3: "a:has-text('Closed')", 4: "a:has-text('Cancelled')"} - selector = tab_selectors.get(status) - if selector: - self._page.click(selector) - self._page.wait_for_timeout(2000) # Wait for DataTables to reload + # Load page and wait for first AJAX response + self._load_and_wait() - rows = self._get_table_rows() + all_rows: list[dict[str, Any]] = [] - # If there are more pages, paginate through them - all_rows = list(rows) - total_pages = self._page.evaluate( - "() => typeof table !== 'undefined' && table ? table.page.info().pages : 1" - ) + # Collect rows from captured responses + for resp_data in self._captured_responses: + all_rows.extend(resp_data.get("data", [])) + # Determine total pages from first response + total_pages = 1 + if self._captured_responses: + # DataTables server-side: recordsTotal / length = pages + total = self._captured_responses[0].get("recordsTotal", 0) + length = self._captured_responses[0].get("data", []) + page_len = len(length) if length else 10 + if total and page_len: + import math + total_pages = math.ceil(total / page_len) + + # Paginate through remaining pages current_page = 1 - while current_page < total_pages: - self._page.evaluate("table.page('next').draw(false)") - self._page.wait_for_timeout(1500) # Wait for AJAX to complete - page_rows = self._get_table_rows() - all_rows.extend(page_rows) + while current_page < total_pages and current_page < 50: # safety cap + self._captured_responses.clear() + + # Trigger next page load via DataTables API + try: + self._page.evaluate("table.page('next').draw(false)") + except Exception: + break + + # Wait for the next AJAX response + deadline = time.time() + 15 + while time.time() < deadline: + if self._captured_responses: + break + self._page.wait_for_timeout(500) + + if not self._captured_responses: + break + + for resp_data in self._captured_responses: + all_rows.extend(resp_data.get("data", [])) + current_page += 1 return all_rows