import requests import json import time import sys import os from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime if sys.platform == 'win32': sys.stdout.reconfigure(encoding='utf-8') from dotenv import load_dotenv load_dotenv(os.path.join(os.path.dirname(__file__), '../.env')) import mysql.connector def get_db_connection(): return mysql.connector.connect( host=os.getenv('DB_HOST'), port=int(os.getenv('DB_PORT', 3306)), user=os.getenv('DB_USER'), password=os.getenv('DB_PASSWORD'), database=os.getenv('DB_NAME'), ) API_URL = "https://nen.nipez.cz/api/datarows?className=Zadavaci_postup_hist" MISTO_API_URL = "https://nen.nipez.cz/api/datarows/datawithcount?className=Misto_NUTS_hist" PUB_API_URL = "https://nen.nipez.cz/api/datarows/datawithcount?className=Info_uverejneni" DOC_API_URL = "https://nen.nipez.cz/api/datarows/datawithcount?className=Uverejneny_dokum" SOURCE = 'nipez' DELAY = 0.3 # Attribute mapping extracted from the site's JS bundle ATTRIBUTES = [ {"clientName": "id", "serverName": "ID"}, {"clientName": "kod", "serverName": "Kod"}, {"clientName": "nazev", "serverName": "Nazev"}, {"clientName": "popisPredmet", "serverName": "Popis_predmetu"}, {"clientName": "typVZ", "serverName": "Typ_VZ_uziv"}, {"clientName": "druhVZ", "serverName": "Druh_VZ"}, {"clientName": "druhZRNazev", "serverName": "Druh_ZR.Nazev"}, {"clientName": "specifZRNazev", "serverName": "Specif_ZR.Nazev"}, {"clientName": "predpokladHodnota", "serverName": "Predpokl_hodnota"}, {"clientName": "predpokladMenaNazev", "serverName": "Predpoklada_mena.Nazev"}, {"clientName": "datumProfil", "serverName": "Datum_uver_profil"}, {"clientName": "datumUkonceni", "serverName": "Datum_ukonceni"}, {"clientName": "datumZruseni", "serverName": "Datum_zruseni_ZR"}, {"clientName": "osobaJmeno", "serverName": "Osoba_zadavat.Jmeno"}, {"clientName": "osobaPrijmeni", "serverName": "Osoba_zadavat.Prijmeni"}, {"clientName": "osobaEmail", "serverName": "Osoba_zadavat.Email"}, {"clientName": "osobaTelefon", "serverName": "Osoba_zadavat.Telefon_zam"}, {"clientName": "osobaMobil", "serverName": "Osoba_zadavat.Mobil"}, {"clientName": "cpvPredmetuKod", "serverName": "CPV_predmetu.Kod"}, {"clientName": "cpvPredmetuNazev", "serverName": "CPV_predmetu.Nazev"}, {"clientName": "nipezPredmetuKod", "serverName": "NIPEZ_predmetu.Kod"}, {"clientName": "nipezPredmetuNazev", "serverName": "NIPEZ_predmetu.Nazev"}, {"clientName": "hlavniMistoNUTS", "serverName": "Hlavni_misto_nuts.Nazev"}, {"clientName": "mistoPlneni", "serverName": "Misto_plneni"}, {"clientName": "rozdeleniNaCasti", "serverName": "Rozdeleni_na_casti"}, {"clientName": "jeToRD", "serverName": "Jedna_se_o_RD"}, {"clientName": "zadavanaDNS", "serverName": "Zadavana_v_DNS"}, {"clientName": "importovanaZakazka", "serverName": "Importovana_zakazka"}, {"clientName": "kodEU", "serverName": "Evid_c_Uv_EU"}, {"clientName": "identifEU", "serverName": "Identifikator_EU"}, {"clientName": "kodZakazkaProfil", "serverName": "Kod_na_profil"}, {"clientName": "nazevPredmetu", "serverName": "Nazev_predmetu"}, {"clientName": "zadavatelID", "serverName": "Zadavatel"}, {"clientName": "zadavatelNazev", "serverName": "Zadavatel.Nazev"}, {"clientName": "kodVestnik", "serverName": "Evid_c_ve_Vestniku"}, {"clientName": "datumVestnik", "serverName": "Datum_uver_vestnik"}, {"clientName": "stavZP", "serverName": "Stav_ZP"}, {"clientName": "naZakladeRSRD", "serverName": "Na_zaklade_RS"}, {"clientName": "zavedeniDNS", "serverName": "Zavedeni_DNS"}, {"clientName": "histId", "serverName": "Hist"}, # deadline datetime (same field used by keywords scraper) {"clientName": "podaniLhuta", "serverName": "Podani_nabidka.Lhuta"}, ] CONDITION = "Kod=@0 AND aktualni = true and not exists(Zadavaci_postup_hist,Hist = &Hist and create_cas>&create_cas and aktualni = true)" def get_session(): session = requests.Session() session.headers.update({ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', }) session.get('https://nen.nipez.cz/en/verejne-zakazky', timeout=30) return session def get_api_headers(session): xsrf_client = session.cookies.get('XSRF-TOKEN-Client', '') return { 'Accept': 'application/json, text/plain, */*', 'Content-Type': 'application/json', 'Origin': 'https://nen.nipez.cz', 'Referer': 'https://nen.nipez.cz/en/verejne-zakazky', 'X-XSRF-TOKEN': xsrf_client, } def fetch_detail_api(session, kod): payload = { "className": "Zadavaci_postup_hist", "attributes": ATTRIBUTES, "condition": CONDITION, "conditionParams": [kod] } try: r = session.post(API_URL, json=payload, headers=get_api_headers(session), timeout=30) if r.status_code == 200: data = r.json() if data and len(data) > 0: return data[0] return None else: print(f" HTTP {r.status_code}") return None except requests.RequestException as e: print(f" Request Error: {e}") return None def fetch_place_of_performance(session, hist_id): payload = { "className": "Misto_NUTS_hist", "attributes": [ {"clientName": "id", "serverName": "ID"}, {"clientName": "nutsKod", "serverName": "vazba_NUTS.Kod"}, {"clientName": "nutsNazev", "serverName": "vazba_NUTS.Nazev"}, ], "condition": "ZP=@0", "conditionParams": [str(hist_id)], "count": 10, "startIndex": 0, "isLoadMore": False, "abortSignal": {} } try: r = session.post(MISTO_API_URL, json=payload, headers=get_api_headers(session), timeout=30) if r.status_code == 200: collection = r.json().get('collection', []) if collection: codes = [str(item.get('nutsKod', '')) for item in collection if item.get('nutsKod')] return ', '.join(codes) return '' except requests.RequestException: return '' def fetch_publications(session, hist_id): payload = { "className": "Info_uverejneni", "attributes": [ {"clientName": "id", "serverName": "ID"}, {"clientName": "datumUverejneni", "serverName": "Datum_uverejneni"}, {"clientName": "text", "serverName": "Text"}, {"clientName": "datumOduver", "serverName": "Datum_oduverejneni"}, {"clientName": "oduverejnil", "serverName": "Oduverejnil"}, ], "condition": "ZP = @0", "conditionParams": [str(hist_id)], "count": 100, "startIndex": 0, "isLoadMore": False, "orderBy": ["DESC Datum_uverejneni"], "abortSignal": {} } try: r = session.post(PUB_API_URL, json=payload, headers=get_api_headers(session), timeout=30) if r.status_code == 200: return r.json().get('collection', []) return [] except requests.RequestException: return [] def fetch_documents(session, hist_id): payload = { "className": "Uverejneny_dokum", "attributes": [ {"clientName": "id", "serverName": "ID"}, {"clientName": "nazev", "serverName": "Nazev"}, {"clientName": "format", "serverName": "Format"}, {"clientName": "velikost", "serverName": "Velikost"}, {"clientName": "datumVlozeni", "serverName": "Datum_vlozeni"}, {"clientName": "datumUver", "serverName": "Info_uver.Datum_uverejneni"}, {"clientName": "typDokument", "serverName": "typdokument.Nazev"}, {"clientName": "odkazLW", "serverName": "Odkaz_LW"}, ], "condition": "Info_uver.ZP = @0 AND Datum_oduverejneni = null", "conditionParams": [str(hist_id)], "count": 100, "startIndex": 0, "isLoadMore": False, "orderBy": ["DESC Info_uver.Datum_uverejneni"], "abortSignal": {} } try: r = session.post(DOC_API_URL, json=payload, headers=get_api_headers(session), timeout=30) if r.status_code == 200: return r.json().get('collection', []) return [] except requests.RequestException: return [] def save_publications(cursor, tender_id, publications): if not publications: return 0 count = 0 for pub in publications: pub_id = pub.get('id') cursor.execute( "SELECT id FROM tender_publications WHERE tender_id = %s AND publication_id = %s LIMIT 1", (tender_id, pub_id) ) if cursor.fetchone(): continue cursor.execute( """INSERT INTO tender_publications (tender_id, publication_id, publication_date, publication_text, withdrawal_date, withdrawn_by) VALUES (%s, %s, %s, %s, %s, %s)""", ( tender_id, pub_id, str(pub.get('datumUverejneni') or '') or None, str(pub.get('text') or ''), str(pub.get('datumOduver') or '') or None, str(pub.get('oduverejnil') or ''), ) ) count += 1 return count def save_documents(cursor, tender_id, documents): if not documents: return 0 count = 0 for doc in documents: doc_url = str(doc.get('odkazLW') or '') if not doc_url: continue cursor.execute( "SELECT id FROM tender_documents WHERE tender_id = %s AND document_url = %s LIMIT 1", (tender_id, doc_url) ) if cursor.fetchone(): continue cursor.execute( """INSERT INTO tender_documents (tender_id, document_name, document_url) VALUES (%s, %s, %s)""", (tender_id, str(doc.get('nazev') or ''), doc_url) ) count += 1 return count def parse_deadline(val): """Return (date, datetime_str) from e.g. '2020-01-09T14:00:00'. datetime_str is formatted as '2020-01-09 14:00:00' for tender_details.deadline. """ if not val: return None, None s = str(val).strip()[:19] for fmt in ('%Y-%m-%dT%H:%M:%S', '%Y-%m-%d', '%d-%m-%YT%H:%M:%S', '%d-%m-%Y', '%d/%m/%Y'): try: dt = datetime.strptime(s, fmt) d = dt.date() dt_str = dt.strftime('%Y-%m-%d %H:%M:%S') return d, dt_str except ValueError: continue return None, None def _combine_address(nuts_place, place_of_performance): parts = [p.strip() for p in [nuts_place, place_of_performance] if p.strip()] if len(parts) == 2 and parts[0] == parts[1]: return parts[0] return ', '.join(parts) def extract_detail_data(obj, place_code=''): contact_parts = [] if obj.get('osobaJmeno'): contact_parts.append(str(obj['osobaJmeno'])) if obj.get('osobaPrijmeni'): contact_parts.append(str(obj['osobaPrijmeni'])) phone_parts = [] if obj.get('osobaTelefon'): phone_parts.append(str(obj['osobaTelefon'])) if obj.get('osobaMobil'): phone_parts.append(str(obj['osobaMobil'])) closing_date, deadline_dt = parse_deadline(obj.get('podaniLhuta')) # Language: NIPEZ is a Czech platform — default to CS language = 'CS' return { 'description': str(obj.get('popisPredmet') or ''), 'organization': str(obj.get('zadavatelNazev') or ''), 'type_vz': str(obj.get('typVZ') or ''), 'kind': str(obj.get('druhVZ') or ''), 'procedure_type': str(obj.get('druhZRNazev') or ''), 'specific_procedure': str(obj.get('specifZRNazev') or ''), 'estimated_value': str(obj.get('predpokladHodnota') or ''), 'currency': str(obj.get('predpokladMenaNazev') or ''), 'profile_date': str(obj.get('datumProfil') or ''), 'end_date': str(obj.get('datumUkonceni') or ''), 'cancellation_date': str(obj.get('datumZruseni') or ''), 'contact_name': ' '.join(contact_parts), 'contact_email': str(obj.get('osobaEmail') or ''), 'contact_phone': ', '.join(phone_parts), 'cpv_code': str(obj.get('cpvPredmetuKod') or ''), 'cpv_name': str(obj.get('cpvPredmetuNazev') or ''), 'nipez_code': str(obj.get('nipezPredmetuKod') or ''), 'nipez_name': str(obj.get('nipezPredmetuNazev') or ''), 'nuts_place': str(obj.get('hlavniMistoNUTS') or ''), 'place_of_performance': str(obj.get('mistoPlneni') or ''), 'place_of_performance_code': place_code, 'divided_into_lots': str(obj.get('rozdeleniNaCasti') or ''), 'framework_agreement': str(obj.get('jeToRD') or ''), 'awarded_in_dns': str(obj.get('zadavanaDNS') or ''), 'awarded_on_framework': str(obj.get('naZakladeRSRD') or ''), 'result_dns': str(obj.get('zavedeniDNS') or ''), 'imported_contract': str(obj.get('importovanaZakazka') or ''), 'eu_code': str(obj.get('kodEU') or ''), 'eu_identifier': str(obj.get('identifEU') or ''), 'profile_code': str(obj.get('kodZakazkaProfil') or ''), 'subject_name': str(obj.get('nazevPredmetu') or ''), 'authority_id': str(obj.get('zadavatelID') or ''), 'bulletin_code': str(obj.get('kodVestnik') or ''), 'bulletin_date': str(obj.get('datumVestnik') or ''), 'contracting_org_country': place_code[:2] if place_code else 'CZ', 'contracting_org_address': _combine_address( str(obj.get('hlavniMistoNUTS') or ''), str(obj.get('mistoPlneni') or '') ), 'closing_date': closing_date, 'deadline': deadline_dt, 'languages': language, 'deadline_timezone': 'CET', # NIPEZ is Czech platform — always Prague time } def save_detail(cursor, tender_id, kod, detail_data): # Update tenders: description, organization, deadline, mark done cursor.execute( """UPDATE tenders SET description = %s, organization = %s, closing_date = COALESCE(%s, closing_date), detail = 1, updated_at = NOW() WHERE id = %s""", (detail_data['description'], detail_data['organization'], detail_data['closing_date'], tender_id) ) # Insert or update tender_details cursor.execute( "SELECT id FROM tender_details WHERE tender_id = %s LIMIT 1", (tender_id,) ) row = cursor.fetchone() fields = ( detail_data['organization'], detail_data['type_vz'], detail_data['kind'], detail_data['procedure_type'], detail_data['specific_procedure'], detail_data['estimated_value'], detail_data['currency'], detail_data['profile_date'] or None, detail_data['end_date'] or None, detail_data['cancellation_date'] or None, detail_data['contact_name'], detail_data['contact_email'], detail_data['contact_phone'], detail_data['cpv_code'], detail_data['cpv_name'], detail_data['nipez_code'], detail_data['nipez_name'], detail_data['nuts_place'], detail_data['place_of_performance'], detail_data['place_of_performance_code'], detail_data['divided_into_lots'], detail_data['framework_agreement'], detail_data['awarded_in_dns'], detail_data['awarded_on_framework'], detail_data['result_dns'], detail_data['imported_contract'], detail_data['eu_code'], detail_data['eu_identifier'], detail_data['profile_code'], detail_data['subject_name'], detail_data['authority_id'], detail_data['bulletin_code'], detail_data['bulletin_date'] or None, detail_data['profile_date'] or None, detail_data['contracting_org_country'], detail_data['contracting_org_address'], detail_data['languages'], detail_data['deadline_timezone'], detail_data['deadline'], ) if row: cursor.execute( """UPDATE tender_details SET contracting_authority_name = %s, type_vz = %s, kind = %s, procedure_type = %s, specific_procedure = %s, estimated_value = %s, currency = %s, profile_date = %s, end_date = %s, cancellation_date = %s, contact_name = %s, contact_email = %s, contact_phone = %s, cpv_code = %s, cpv_name = %s, nipez_code = %s, nipez_name = %s, nuts_place = %s, place_of_performance = %s, place_of_performance_code = %s, divided_into_lots = %s, framework_agreement = %s, awarded_in_dns = %s, awarded_on_framework = %s, result_dns = %s, imported_contract = %s, eu_code = %s, eu_identifier = %s, profile_code = %s, subject_name = %s, authority_id = %s, bulletin_code = %s, bulletin_date = %s, publication_date = %s, contracting_org_country = %s, contracting_org_address = %s, languages = %s, deadline_timezone = %s, deadline = %s, updated_at = NOW() WHERE tender_id = %s""", fields + (tender_id,) ) else: cursor.execute( """INSERT INTO tender_details (tender_id, nen_number, reference_number, contracting_authority_name, type_vz, kind, procedure_type, specific_procedure, estimated_value, currency, profile_date, end_date, cancellation_date, contact_name, contact_email, contact_phone, cpv_code, cpv_name, nipez_code, nipez_name, nuts_place, place_of_performance, place_of_performance_code, divided_into_lots, framework_agreement, awarded_in_dns, awarded_on_framework, result_dns, imported_contract, eu_code, eu_identifier, profile_code, subject_name, authority_id, bulletin_code, bulletin_date, publication_date, contracting_org_country, contracting_org_address, languages, deadline_timezone, deadline, created_at, updated_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW())""", (tender_id, kod, kod) + fields ) return True FETCH_BATCH = 200 # rows loaded from DB per iteration def scrape_details(batch_size=None): print("=" * 70) print("NEN.NIPEZ.CZ - Detail Scraper (API)") print("=" * 70) print("Getting session...", end=' ') session = get_session() print("OK") conn = get_db_connection() cursor = conn.cursor(dictionary=True) cursor.execute("SELECT COUNT(*) as total FROM tenders WHERE source = %s AND detail = 0", (SOURCE,)) total = cursor.fetchone()['total'] cursor.close() conn.close() print(f"Total records to process: {total:,}") print("=" * 70) success_count = 0 error_count = 0 consecutive_errors = 0 processed = 0 offset = 0 limit = min(FETCH_BATCH, batch_size) if batch_size else FETCH_BATCH conn = get_db_connection() cursor = conn.cursor() while True: fetch_conn = get_db_connection() fetch_cursor = fetch_conn.cursor(dictionary=True) fetch_cursor.execute( "SELECT id, url, title, reference_number FROM tenders WHERE source = %s AND detail = 0 ORDER BY id LIMIT %s OFFSET %s", (SOURCE, limit, offset) ) records = fetch_cursor.fetchall() fetch_cursor.close() fetch_conn.close() if not records: break for record in records: tender_id = record['id'] kod = record['reference_number'] name = record['title'] processed += 1 print(f"\n[{processed}] {kod}") print(f" Name: {(name or 'N/A')[:70]}") if not kod: print(" Skipped: no kod") error_count += 1 continue obj = fetch_detail_api(session, kod) if not obj: consecutive_errors += 1 error_count += 1 if consecutive_errors >= 3: print(" Refreshing session...") session = get_session() consecutive_errors = 0 time.sleep(DELAY) continue try: hist_id = obj.get('histId') place_code = '' publications = [] documents = [] pub_count = 0 doc_count = 0 if hist_id: with ThreadPoolExecutor(max_workers=3) as ex: f_place = ex.submit(fetch_place_of_performance, session, hist_id) f_pubs = ex.submit(fetch_publications, session, hist_id) f_docs = ex.submit(fetch_documents, session, hist_id) place_code = f_place.result() publications = f_pubs.result() documents = f_docs.result() detail_data = extract_detail_data(obj, place_code) consecutive_errors = 0 fields_found = sum(1 for v in detail_data.values() if v) try: pub_count = save_publications(cursor, tender_id, publications) doc_count = save_documents(cursor, tender_id, documents) save_detail(cursor, tender_id, kod, detail_data) conn.commit() success_count += 1 print(f" Parsed: {fields_found} fields | Pubs: {pub_count} | Docs: {doc_count}") print(f" Saved & detail set to 1") except Exception as e: conn.rollback() print(f" DB Error: {e}") error_count += 1 except Exception as e: print(f" Parse error: {e}") error_count += 1 if processed % 10 == 0: print(f"\n{'─' * 70}") print(f" Progress: {processed} | Success: {success_count} | Errors: {error_count}") print(f"{'─' * 70}") time.sleep(DELAY) if batch_size and processed >= batch_size: break offset += len(records) if batch_size and processed >= batch_size: break cursor.close() conn.close() print("\n" + "=" * 70) print("Scraping Complete!") print("=" * 70) print(f" Total processed: {processed}") print(f" Successful: {success_count}") print(f" Errors: {error_count}") print("=" * 70) if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description='Scrape NEN.NIPEZ tender details via API') parser.add_argument('--batch-size', type=int, default=None, help='Number of records to process') args = parser.parse_args() try: scrape_details(batch_size=args.batch_size) except KeyboardInterrupt: print("\n\nInterrupted by user") except Exception as e: print(f"\nFatal Error: {e}") import traceback traceback.print_exc()