import requests import time import sys import os from datetime import datetime if sys.platform == 'win32': sys.stdout.reconfigure(encoding='utf-8') from dotenv import load_dotenv load_dotenv(os.path.join(os.path.dirname(__file__), '../.env')) import mysql.connector sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) from keyword_tracker import is_keyword_done, mark_keyword_done def get_db_connection(): return mysql.connector.connect( host=os.getenv('DB_HOST'), port=int(os.getenv('DB_PORT', 3306)), user=os.getenv('DB_USER'), password=os.getenv('DB_PASSWORD'), database=os.getenv('DB_NAME'), ) KEYWORDS = [k.strip() for k in os.getenv('KEYWORDS', '').split(',') if k.strip()] API_URL = "https://nen.nipez.cz/api/datarows/datawithcount?className=Zadavaci_postup_hist" DETAIL_BASE = "https://nen.nipez.cz/en/verejne-zakazky/detail-zakazky/{}" PAGE_SIZE = 200 SOURCE = 'nipez' MAX_PAGES = int(os.getenv('NIPEZ_MAX_PAGES', 3)) def get_session(): session = requests.Session() session.headers.update({ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36', 'Accept': 'application/json, text/plain, */*', }) session.get('https://nen.nipez.cz/en/verejne-zakazky', timeout=30) xsrf = session.cookies.get('XSRF-TOKEN-Client', '') if not xsrf: print("ERROR: Could not get XSRF token") sys.exit(1) return session, xsrf def fetch_page(session, xsrf, start_index, keyword): payload = { 'className': 'Zadavaci_postup_hist', 'attributes': [ {'clientName': 'id', 'serverName': 'ID'}, {'clientName': 'kod', 'serverName': 'Kod'}, {'clientName': 'nazev', 'serverName': 'Nazev'}, {'clientName': 'stavZP', 'serverName': 'Stav_ZP'}, {'clientName': 'zadavatelNazev', 'serverName': 'Zadavatel.Nazev'}, {'clientName': 'podaniLhuta', 'serverName': 'Podani_nabidka.Lhuta'}, {'clientName': 'createCas', 'serverName': 'create_cas'}, ], 'condition': '(Hierarchie = null and Posledni_uverejneni = true)', 'conditionParams': [], 'count': PAGE_SIZE, 'isLoadMore': True, 'startIndex': start_index, 'orderBy': ['DESC create_cas'], 'query': keyword, 'abortSignal': {} } try: r = session.post(API_URL, json=payload, headers={ 'X-XSRF-TOKEN': xsrf, 'Language': 'EN', 'Origin': 'https://nen.nipez.cz', 'Content-Type': 'application/json', }, timeout=60) if r.status_code == 200: return r.json() else: print(f" HTTP {r.status_code}") return None except requests.RequestException as e: print(f" Request Error: {e}") return None def parse_deadline(val): """Return (date, time_str) parsed from podaniLhuta e.g. '2020-01-09T14:00:00'.""" if not val: return None, None s = str(val).strip()[:19] for fmt in ('%Y-%m-%dT%H:%M:%S', '%Y-%m-%d', '%d-%m-%YT%H:%M:%S', '%d-%m-%Y', '%d/%m/%Y'): try: dt = datetime.strptime(s, fmt) d = dt.date() t = dt.strftime('%H:%M:%S') if 'T' in fmt or '%H' in fmt else None return d, t except ValueError: continue return None, None def insert_tenders(tenders, keyword): if not tenders: return 0, 0 conn = get_db_connection() cursor = conn.cursor() inserted = 0 updated = 0 for t in tenders: source_id = str(t.get('id', '')) kod = t.get('kod', '') title = t.get('nazev', '') status = t.get('stavZP', '') authority = t.get('zadavatelNazev', '') deadline, closing_time = parse_deadline(t.get('podaniLhuta')) url = DETAIL_BASE.format(kod.replace('/', '-')) if kod else '' try: cursor.execute( "SELECT id FROM tenders WHERE source = %s AND source_id = %s LIMIT 1", (SOURCE, source_id) ) row = cursor.fetchone() if row: cursor.execute( """UPDATE tenders SET title = %s, organization = %s, status = %s, closing_date = %s, closing_time = %s, keyword = %s, updated_at = NOW() WHERE id = %s""", (title, authority, status, deadline, closing_time, keyword, row[0]) ) updated += 1 else: cursor.execute( """INSERT INTO tenders (source, source_id, title, reference_number, url, organization, status, closing_date, closing_time, keyword, detail, created_at, updated_at) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 0, NOW(), NOW())""", (SOURCE, source_id, title, kod, url, authority, status, deadline, closing_time, keyword) ) inserted += 1 except Exception as e: print(f" DB Error: {e}") continue conn.commit() cursor.close() conn.close() return inserted, updated def get_current_count(): conn = get_db_connection() cursor = conn.cursor() cursor.execute("SELECT COUNT(*) FROM tenders WHERE source = %s", (SOURCE,)) count = cursor.fetchone()[0] cursor.close() conn.close() return count def scrape_keyword(session, xsrf, keyword): print(f"\n Keyword: \"{keyword}\"") start_index = 0 page = 1 total_fetched = 0 total_inserted = 0 total_updated = 0 consecutive_errors = 0 while True: print(f" [Page {page}] startIndex={start_index}...", end=' ') data = fetch_page(session, xsrf, start_index, keyword) if data is None: consecutive_errors += 1 print(f"Failed (#{consecutive_errors})") if consecutive_errors >= 3: print(" Refreshing session...") session, xsrf = get_session() consecutive_errors = 0 time.sleep(2) continue collection = data.get('collection', []) count = len(collection) if count == 0: print("No results.") break consecutive_errors = 0 inserted, updated = insert_tenders(collection, keyword) total_fetched += count total_inserted += inserted total_updated += updated print(f"Got {count} (New: {inserted}, Updated: {updated})") if page >= MAX_PAGES: print(f" Reached max pages ({MAX_PAGES}) — stopping.") break if count < PAGE_SIZE: break start_index += PAGE_SIZE page += 1 time.sleep(1) print(f" => Fetched: {total_fetched}, New: {total_inserted}, Updated: {total_updated}") return session, xsrf, total_fetched, total_inserted, total_updated def scrape_tenders(): print("=" * 70) print("NEN.NIPEZ.CZ - Keyword Tender Scraper") print("=" * 70) print(f"Keywords to search: {len(KEYWORDS)}") print("\nGetting session...", end=' ') session, xsrf = get_session() print("OK") grand_fetched = 0 grand_inserted = 0 grand_updated = 0 tracker_conn = get_db_connection() for i, keyword in enumerate(KEYWORDS, 1): print(f"\n[{i}/{len(KEYWORDS)}]", end='') if is_keyword_done(tracker_conn, keyword, SOURCE): print(f" [SKIP] Already ran today: \"{keyword}\"") continue session, xsrf, fetched, inserted, updated = scrape_keyword(session, xsrf, keyword) grand_fetched += fetched grand_inserted += inserted grand_updated += updated mark_keyword_done(tracker_conn, keyword, SOURCE) print(f" [SAVED] Marked keyword done: \"{keyword}\"") time.sleep(1) tracker_conn.close() print("\n" + "=" * 70) print("Scraping Complete!") print("=" * 70) print(f" Keywords searched : {len(KEYWORDS)}") print(f" Total fetched : {grand_fetched:,}") print(f" New : {grand_inserted:,}") print(f" Updated : {grand_updated:,}") print(f" DB total (nipez) : {get_current_count():,}") print("=" * 70) if __name__ == "__main__": try: scrape_tenders() except KeyboardInterrupt: print(f"\n\nInterrupted. DB total: {get_current_count():,}") except Exception as e: print(f"\nFatal Error: {e}") import traceback traceback.print_exc()