import sys
import os
import time
import re
import requests
from datetime import datetime
from urllib.parse import quote
if sys.platform == 'win32':
sys.stdout.reconfigure(encoding='utf-8')
from dotenv import load_dotenv
load_dotenv(os.path.join(os.path.dirname(__file__), '../.env'))
import mysql.connector
def get_db_connection():
return mysql.connector.connect(
host=os.getenv('DB_HOST'),
port=int(os.getenv('DB_PORT', 3306)),
user=os.getenv('DB_USER'),
password=os.getenv('DB_PASSWORD'),
database=os.getenv('DB_NAME'),
)
from playwright.sync_api import sync_playwright
SOURCE = 'ezamowienia.gov.pl'
DETAIL_URL = 'https://ezamowienia.gov.pl/mp-client/search/list/{source_id}'
DETAIL_BASE = 'https://ezamowienia.gov.pl/mo-client-board/bzp/notice-details/'
BZP_API = 'https://ezamowienia.gov.pl/mo-board/api/v1/Board/GetNoticeHtmlBody'
TIMEOUT = 30000
POLISH_MONTHS = {
'stycznia': 1, 'lutego': 2, 'marca': 3, 'kwietnia': 4,
'maja': 5, 'czerwca': 6, 'lipca': 7, 'sierpnia': 8,
'września': 9, 'października': 10, 'listopada': 11, 'grudnia': 12,
}
def parse_polish_datetime(raw):
if not raw or not raw.strip():
return None, None
raw = raw.strip()
m = re.match(r'(\d{1,2})\s+(\w+)\s+(\d{4}),\s*godz\s+(\d{1,2}):(\d{2})', raw, re.UNICODE)
if m:
day, month_name, year = int(m.group(1)), m.group(2).lower(), int(m.group(3))
hour, minute = int(m.group(4)), int(m.group(5))
month = POLISH_MONTHS.get(month_name)
if month:
try:
dt = datetime(year, month, day, hour, minute)
return dt.date(), dt
except ValueError:
pass
return None, None
def fetch_bzp_data(reference_number):
"""
Call GetNoticeHtmlBody API and parse fields from the HTML.
Returns a dict with extracted values, or {} on failure.
The HTML uses tags:
- Inline value: 1.2.) Label: VALUE
- Block value: 4.2.2.) Label
followed by
VALUE
"""
try:
resp = requests.get(
BZP_API,
params={'noticeNumber': reference_number},
headers={'User-Agent': 'Mozilla/5.0', 'Referer': 'https://ezamowienia.gov.pl/'},
timeout=30,
)
if resp.status_code != 200:
print(f' BZP API {resp.status_code} for {reference_number}')
return {}
html = resp.text
except Exception as e:
print(f' BZP API error: {e}')
return {}
out = {}
# --- Inline fields: value inside within ---
inline_map = {
r'1\.2\.\).*Nazwa zamawiającego': 'bzp_name',
r'1\.5\.1\.\).*Ulica': 'bzp_address',
r'1\.5\.2\.\).*Miejscowość': 'bzp_city',
r'1\.5\.3\.\).*Kod pocztowy': 'bzp_postal',
r'1\.5\.4\.\).*Województwo': 'bzp_province',
r'1\.5\.5\.\).*Kraj': 'bzp_country',
r'1\.5\.9\.\).*poczty elektronicznej': 'bzp_email',
r'4\.2\.6\.\).*Główny kod CPV': 'bzp_cpv_code',
r'4\.2\.7\.\).*Dodatkowy kod CPV': 'bzp_cpv_additional',
}
h3_blocks = re.findall(r']*>(.*?)
', html, re.DOTALL)
for block in h3_blocks:
text = re.sub(r'<[^>]+>', '', block).strip()
span = re.search(r']*class="normal"[^>]*>(.*?)', block, re.DOTALL)
value = re.sub(r'<[^>]+>', '', span.group(1)).strip() if span else ''
for pattern, key in inline_map.items():
if re.search(pattern, text):
if value:
out[key] = value
break
# --- Block fields: value in immediately after matching
---
block_map = {
r'4\.2\.2\.\).*Krótki opis': 'bzp_summary',
r'8\.1\.\).*Termin składania': 'bzp_submission_end',
}
# Split HTML into segments on boundaries
segments = re.split(r'(]*>.*?
)', html, flags=re.DOTALL)
for i, seg in enumerate(segments):
if not seg.startswith(']+>', '', seg).strip()
for pattern, key in block_map.items():
if re.search(pattern, h3_text):
# Look for first in the following segment
if i + 1 < len(segments):
p_match = re.search(r']*>(.*?)
', segments[i + 1], re.DOTALL)
if p_match:
val = re.sub(r'<[^>]+>', '', p_match.group(1)).strip()
if val:
out[key] = val
break
return out
def _extract_fields_and_docs(page):
fields = page.eval_on_selector_all('tr', """
trs => trs.map(tr => {
const label = tr.querySelector('td.label');
const value = tr.querySelector('td.value');
if (!label || !value) return null;
return { label: label.innerText.trim(), value: value.innerText.trim() };
}).filter(r => r !== null && r.label !== '')
""")
docs = page.eval_on_selector_all('a.ng-star-inserted', """
anchors => anchors
.filter(a => a.href && a.href.includes('tenderdocument'))
.map(a => ({ name: a.innerText.trim(), url: a.href }))
""")
return fields, docs
def scrape_detail_page(page, source_id, reference_number=None):
data = {}
docs = []
# --- Original URL ---
url1 = DETAIL_URL.format(source_id=source_id)
print(f' URL1: {url1}')
page.goto(url1, wait_until='networkidle', timeout=60000)
try:
page.wait_for_selector('td.label', timeout=TIMEOUT)
fields, docs1 = _extract_fields_and_docs(page)
for f in fields:
data[f['label']] = f['value']
docs.extend(docs1)
except Exception:
print(f' td.label not found on URL1 for {source_id}')
# --- BZP API (additional fields) ---
if reference_number and reference_number.strip():
print(f' BZP API: {reference_number.strip()}')
bzp = fetch_bzp_data(reference_number.strip())
if bzp:
data.update(bzp)
print(f' BZP fields fetched: {list(bzp.keys())}')
if not data:
return None, []
return data, docs
def update_tender(cursor, tender_id, data):
status = data.get('Status', '').strip() or None
reference_number = (
next((v for k, v in data.items() if 'Numer og' in k), None)
or data.get('Numer referencyjny postępowania', '')
).strip() or None
cursor.execute(
"""UPDATE tenders
SET status=%s, detail=1, updated_at=NOW(),
reference_number = COALESCE(reference_number, %s)
WHERE id=%s""",
(status, reference_number, tender_id)
)
def update_tender_detail(cursor, tender_id, data):
# --- From URL1 (td.label) ---
solicitation_number = data.get('Numer referencyjny postępowania', '').strip() or None
specific_procedure = data.get('Procedura', '').strip() or None
kind = data.get('Faza postępowania', '').strip() or None
procedure_type = data.get('Tryb postępowania', '').strip() or None
profile_code = data.get('Pozycja w planie postępowań', '').strip() or None
_, open_date_dt = parse_polish_datetime(data.get('Termin otwarcia', ''))
language = 'Polish'
# --- From BZP API ---
contracting_authority_name = (data.get('bzp_name') or '').strip() or None
contracting_org_address = (data.get('bzp_address') or '').strip() or None
contracting_org_city = (data.get('bzp_city') or '').strip() or None
contracting_org_province = (data.get('bzp_province') or '').strip() or None
contracting_org_postal = (data.get('bzp_postal') or '').strip() or None
contracting_org_country = (data.get('bzp_country') or '').strip() or None
contracting_authority_email = (data.get('bzp_email') or '').strip() or None
cpv_code = (data.get('bzp_cpv_code') or '').strip() or None
cpv_additional = (data.get('bzp_cpv_additional') or '').strip() or None
summary = (data.get('bzp_summary') or '').strip() or None
submission_end_date = None
submission_end_time = None
raw_sub = (data.get('bzp_submission_end') or '').strip()
if raw_sub:
m = re.match(r'(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2})', raw_sub)
if m:
submission_end_date = m.group(1)
submission_end_time = m.group(2)
cursor.execute("""
UPDATE tender_details
SET solicitation_number = %s,
specific_procedure = %s,
kind = %s,
procedure_type = %s,
profile_code = %s,
open_date = %s,
languages = %s,
contracting_authority_name = %s,
contracting_org_address = %s,
contracting_org_city = %s,
contracting_org_province = %s,
contracting_org_postal = %s,
contracting_org_country = %s,
contracting_authority_email = %s,
cpv_code = %s,
cpv_additional = %s,
summary = %s,
submission_end = COALESCE(%s, submission_end),
submission_end_time = %s,
updated_at = NOW()
WHERE tender_id = %s
""", (
solicitation_number,
specific_procedure,
kind,
procedure_type,
profile_code,
open_date_dt,
language,
contracting_authority_name,
contracting_org_address,
contracting_org_city,
contracting_org_province,
contracting_org_postal,
contracting_org_country,
contracting_authority_email,
cpv_code,
cpv_additional,
summary,
submission_end_date,
submission_end_time,
tender_id,
))
def insert_documents(cursor, tender_id, docs):
count = 0
for doc in docs:
name = doc.get('name', '').strip()
url = doc.get('url', '').strip()
if not name and not url:
continue
# Skip if already exists
cursor.execute(
"SELECT id FROM tender_documents WHERE tender_id=%s AND document_url=%s LIMIT 1",
(tender_id, url)
)
if cursor.fetchone():
continue
cursor.execute(
"""INSERT INTO tender_documents (tender_id, document_name, document_url, created_at)
VALUES (%s, %s, %s, NOW())""",
(tender_id, name, url)
)
count += 1
return count
def run(limit=None, tender_id=None):
print('=' * 70)
print('EZamowienia.gov.pl — Detail Scraper')
print('=' * 70)
conn = get_db_connection()
cursor = conn.cursor()
if tender_id:
query = "SELECT id, source_id, reference_number FROM tenders WHERE source = %s AND id = %s"
cursor.execute(query, (SOURCE, tender_id))
else:
query = """
SELECT id, source_id, reference_number FROM tenders
WHERE source = %s AND detail = 0
ORDER BY id ASC
"""
if limit:
query += f' LIMIT {limit}'
cursor.execute(query, (SOURCE,))
records = cursor.fetchall()
print(f'Records to process: {len(records)}')
total_updated = 0
total_docs = 0
total_failed = 0
with sync_playwright() as pw:
browser = pw.chromium.launch(headless=True)
context = browser.new_context(user_agent=(
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'
))
page = context.new_page()
for tender_id, source_id, reference_number in records:
print(f'\n [{tender_id}] {source_id} | ref: {reference_number}')
try:
data, docs = scrape_detail_page(page, source_id, reference_number)
if data is None:
total_failed += 1
continue
update_tender(cursor, tender_id, data)
update_tender_detail(cursor, tender_id, data)
doc_count = insert_documents(cursor, tender_id, docs)
conn.commit()
print(f' Status : {data.get("Status")}')
print(f' Ref number : {data.get("Numer referencyjny postępowania")}')
print(f' Procedura : {data.get("Procedura")}')
print(f' Faza : {data.get("Faza postępowania")}')
print(f' Open date : {data.get("Termin otwarcia")}')
print(f' Profile code : {data.get("Pozycja w planie postępowań")}')
print(f' Name : {data.get("bzp_name")}')
print(f' Address : {data.get("bzp_address")}')
print(f' City : {data.get("bzp_city")}')
print(f' Province : {data.get("bzp_province")}')
print(f' Postal : {data.get("bzp_postal")}')
print(f' Country : {data.get("bzp_country")}')
print(f' Email : {data.get("bzp_email")}')
print(f' CPV code : {data.get("bzp_cpv_code")}')
print(f' CPV additional : {data.get("bzp_cpv_additional")}')
print(f' Summary : {str(data.get("bzp_summary",""))[:80]}')
print(f' Submission end : {data.get("bzp_submission_end")}')
print(f' Language : Polish')
print(f' Documents : {doc_count}')
total_updated += 1
total_docs += doc_count
except Exception as e:
import traceback
print(f' ERROR: {e}')
traceback.print_exc()
total_failed += 1
conn.rollback()
browser.close()
cursor.close()
conn.close()
print('\n' + '=' * 70)
print('Detail Scraping Complete!')
print('=' * 70)
print(f' Updated : {total_updated}')
print(f' Documents: {total_docs}')
print(f' Failed : {total_failed}')
print('=' * 70)
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser(description='Scrape ezamowienia.gov.pl detail pages')
parser.add_argument('--limit', type=int, default=None, help='Limit records to process')
parser.add_argument('--tender-id', type=int, default=None, help='Process a specific tender by id')
args = parser.parse_args()
try:
run(limit=args.limit, tender_id=args.tender_id)
except KeyboardInterrupt:
print('\n\nInterrupted by user.')
except Exception as e:
import traceback
print(f'\nFatal error: {e}')
traceback.print_exc()