import requests
import json
import time
import sys
import os
from urllib.parse import unquote
from html.parser import HTMLParser

if sys.platform == 'win32':
    sys.stdout.reconfigure(encoding='utf-8')

from dotenv import load_dotenv
load_dotenv(os.path.join(os.path.dirname(__file__), '../../.env'))
import mysql.connector

def get_db_connection():
    return mysql.connector.connect(
        host=os.getenv('DB_HOST'),
        port=int(os.getenv('DB_PORT', 3306)),
        user=os.getenv('DB_USER'),
        password=os.getenv('DB_PASSWORD'),
        database=os.getenv('DB_NAME'),
    )

DETAIL_URL = "https://nen.nipez.cz/en/verejne-zakazky/detail-zakazky/{}"
SOURCE     = 'nipez'
DELAY      = 5


class MetaExtractor(HTMLParser):
    def __init__(self):
        super().__init__()
        self.redux_state = None

    def handle_starttag(self, tag, attrs):
        d = dict(attrs)
        if tag == 'meta' and d.get('name') == 'initialReduxState':
            self.redux_state = d.get('content', '')


def get_session():
    session = requests.Session()
    session.headers.update({
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    })
    session.get('https://nen.nipez.cz/en/verejne-zakazky', timeout=30)
    return session


def fetch_detail_page(session, url):
    try:
        r = session.get(url, timeout=30)
        if r.status_code == 200:
            return r.text
        else:
            print(f"    HTTP {r.status_code}")
            return None
    except requests.RequestException as e:
        print(f"    Request Error: {e}")
        return None


def extract_detail_data(html):
    parser = MetaExtractor()
    parser.feed(html)

    if not parser.redux_state:
        return None

    state = json.loads(unquote(parser.redux_state))
    objects = state.get('detailObjectStore', {}).get('objects', {})

    for key, val in objects.items():
        obj = val.get('object', {})
        if obj:
            contact_parts = []
            if obj.get('osobaJmeno'):
                contact_parts.append(str(obj['osobaJmeno']))
            if obj.get('osobaPrijmeni'):
                contact_parts.append(str(obj['osobaPrijmeni']))

            phone_parts = []
            if obj.get('osobaTelefon'):
                phone_parts.append(str(obj['osobaTelefon']))
            if obj.get('osobaMobil'):
                phone_parts.append(str(obj['osobaMobil']))

            return {
                'description': str(obj.get('popisPredmet') or ''),
                'type_vz': str(obj.get('typVZ') or ''),
                'kind': str(obj.get('druhVZ') or ''),
                'procedure_type': str(obj.get('druhZRNazev') or ''),
                'specific_procedure': str(obj.get('specifZRNazev') or ''),
                'estimated_value': str(obj.get('predpokladHodnota') or ''),
                'currency': str(obj.get('predpokladMenaNazev') or ''),
                'profile_date': str(obj.get('datumProfil') or ''),
                'end_date': str(obj.get('datumUkonceni') or ''),
                'cancellation_date': str(obj.get('datumZruseni') or ''),
                'contact_name': ' '.join(contact_parts),
                'contact_email': str(obj.get('osobaEmail') or ''),
                'contact_phone': ', '.join(phone_parts),
                'cpv_code': str(obj.get('cpvPredmetuKod') or ''),
                'cpv_name': str(obj.get('cpvPredmetuNazev') or ''),
                'nipez_code': str(obj.get('nipezPredmetuKod') or ''),
                'nipez_name': str(obj.get('nipezPredmetuNazev') or ''),
                'nuts_place': str(obj.get('hlavniMistoNUTS') or ''),
                'place_of_performance': str(obj.get('mistoPlneni') or ''),
                'divided_into_lots': str(obj.get('rozdeleniNaCasti') or ''),
                'framework_agreement': str(obj.get('jeToRD') or ''),
                'awarded_in_dns': str(obj.get('zadavanaDNS') or ''),
                'imported_contract': str(obj.get('importovanaZakazka') or ''),
                'eu_code': str(obj.get('kodEU') or ''),
                'eu_identifier': str(obj.get('identifEU') or ''),
                'profile_code': str(obj.get('kodZakazkaProfil') or ''),
                'subject_name': str(obj.get('nazevPredmetu') or ''),
                'authority_id': str(obj.get('zadavatelID') or ''),
                'bulletin_code': str(obj.get('kodVestnik') or ''),
                'bulletin_date': str(obj.get('datumVestnik') or ''),
            }

    return None


def save_detail(tender_id, kod, detail_data):
    try:
        conn = get_db_connection()
        cursor = conn.cursor()

        # Update tenders: description + mark done
        cursor.execute(
            "UPDATE tenders SET description = %s, detail = 1, updated_at = NOW() WHERE id = %s",
            (detail_data['description'], tender_id)
        )

        # Insert or update tender_details
        cursor.execute(
            "SELECT id FROM tender_details WHERE tender_id = %s LIMIT 1",
            (tender_id,)
        )
        row = cursor.fetchone()

        fields = (
            detail_data['type_vz'], detail_data['kind'],
            detail_data['procedure_type'], detail_data['specific_procedure'],
            detail_data['estimated_value'], detail_data['currency'],
            detail_data['profile_date'] or None, detail_data['end_date'] or None,
            detail_data['cancellation_date'] or None,
            detail_data['contact_name'], detail_data['contact_email'],
            detail_data['contact_phone'],
            detail_data['cpv_code'], detail_data['cpv_name'],
            detail_data['nipez_code'], detail_data['nipez_name'],
            detail_data['nuts_place'], detail_data['place_of_performance'],
            detail_data['divided_into_lots'], detail_data['framework_agreement'],
            detail_data['awarded_in_dns'], detail_data['imported_contract'],
            detail_data['eu_code'], detail_data['eu_identifier'],
            detail_data['profile_code'], detail_data['subject_name'],
            detail_data['authority_id'], detail_data['bulletin_code'],
            detail_data['bulletin_date'] or None,
        )

        if row:
            cursor.execute(
                """UPDATE tender_details SET
                       type_vz = %s, kind = %s, procedure_type = %s, specific_procedure = %s,
                       estimated_value = %s, currency = %s,
                       profile_date = %s, end_date = %s, cancellation_date = %s,
                       contact_name = %s, contact_email = %s, contact_phone = %s,
                       cpv_code = %s, cpv_name = %s, nipez_code = %s, nipez_name = %s,
                       nuts_place = %s, place_of_performance = %s,
                       divided_into_lots = %s, framework_agreement = %s,
                       awarded_in_dns = %s, imported_contract = %s,
                       eu_code = %s, eu_identifier = %s, profile_code = %s,
                       subject_name = %s, authority_id = %s,
                       bulletin_code = %s, bulletin_date = %s,
                       updated_at = NOW()
                   WHERE tender_id = %s""",
                fields + (tender_id,)
            )
        else:
            cursor.execute(
                """INSERT INTO tender_details
                       (tender_id, nen_number, reference_number,
                        type_vz, kind, procedure_type, specific_procedure,
                        estimated_value, currency,
                        profile_date, end_date, cancellation_date,
                        contact_name, contact_email, contact_phone,
                        cpv_code, cpv_name, nipez_code, nipez_name,
                        nuts_place, place_of_performance,
                        divided_into_lots, framework_agreement,
                        awarded_in_dns, imported_contract,
                        eu_code, eu_identifier, profile_code,
                        subject_name, authority_id,
                        bulletin_code, bulletin_date,
                        created_at, updated_at)
                   VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
                           %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
                           %s, %s, NOW(), NOW())""",
                (tender_id, kod, kod) + fields
            )

        conn.commit()
        cursor.close()
        conn.close()
        return True
    except Exception as e:
        print(f"    DB Error: {e}")
        return False


def scrape_details(batch_size=None):
    print("=" * 70)
    print("NEN.NIPEZ.CZ - Detail Scraper")
    print("=" * 70)

    print("Getting session...", end=' ')
    session = get_session()
    print("OK")

    conn = get_db_connection()
    cursor = conn.cursor(dictionary=True)

    cursor.execute("SELECT COUNT(*) as total FROM tenders WHERE source = %s AND detail = 0", (SOURCE,))
    total = cursor.fetchone()['total']
    print(f"Total records to process: {total:,}")

    query = "SELECT id, url, title, reference_number FROM tenders WHERE source = %s AND detail = 0 ORDER BY id"
    params = [SOURCE]
    if batch_size:
        query += " LIMIT %s"
        params.append(batch_size)

    cursor.execute(query, params)
    records = cursor.fetchall()
    cursor.close()
    conn.close()

    print(f"Processing {len(records)} records...")
    print("=" * 70)

    success_count = 0
    error_count = 0
    consecutive_errors = 0

    for i, record in enumerate(records, 1):
        tender_id = record['id']
        url       = record['url']
        kod       = record['reference_number']
        name      = record['title']

        print(f"\n[{i}/{len(records)}] {kod}")
        print(f"  Name: {(name or 'N/A')[:70]}")
        print(f"  URL: {url}")

        if not url:
            print("    Skipped: no URL")
            error_count += 1
            continue

        html = fetch_detail_page(session, url)
        if not html:
            consecutive_errors += 1
            error_count += 1
            if consecutive_errors >= 3:
                print("  Refreshing session...")
                session = get_session()
                consecutive_errors = 0
            time.sleep(DELAY)
            continue

        try:
            detail_data = extract_detail_data(html)
            if not detail_data:
                print("    No detail data found in page")
                error_count += 1
                consecutive_errors += 1
                time.sleep(DELAY)
                continue

            consecutive_errors = 0
            fields_found = sum(1 for v in detail_data.values() if v)
            print(f"    Parsed: {fields_found} fields")

            if save_detail(tender_id, kod, detail_data):
                success_count += 1
                print(f"    Saved & detail set to 1")
            else:
                error_count += 1

        except Exception as e:
            print(f"    Parse error: {e}")
            error_count += 1

        if i % 10 == 0:
            print(f"\n{'─' * 70}")
            print(f"  Progress: {i}/{len(records)} | Success: {success_count} | Errors: {error_count}")
            print(f"{'─' * 70}")

        time.sleep(DELAY)

    print("\n" + "=" * 70)
    print("Scraping Complete!")
    print("=" * 70)
    print(f"  Total processed: {len(records)}")
    print(f"  Successful: {success_count}")
    print(f"  Errors: {error_count}")
    print("=" * 70)


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description='Scrape NEN.NIPEZ tender details')
    parser.add_argument('--batch-size', type=int, default=None, help='Number of records to process')
    args = parser.parse_args()

    try:
        scrape_details(batch_size=args.batch_size)
    except KeyboardInterrupt:
        print("\n\nInterrupted by user")
    except Exception as e:
        print(f"\nFatal Error: {e}")
        import traceback
        traceback.print_exc()