
     wi%                     n   S SK r S SKrS SKrS SKrS SKJrJr  S SKJr  S SKr	S SK
Jr  \" \R                  R                  \R                  R                  \5      S5      5        \R                   S:X  a  \R"                  R%                  SS9  \R&                  " S	5      \" \R&                  " S
S5      5      \R&                  " S5      \R&                  " S5      \R&                  " S5      S.r/ SQrSrSr\R2                  " 5       \" SS9-
  rS rS rS rS r\S:X  a  \ R@                  " \" 5       5        gg)    N)datetime	timedelta)load_dotenv)async_playwrightz../.envwin32zutf-8)encodingDB_HOSTDB_PORTi  DB_USERDB_PASSWORDDB_NAME)hostportuserpassworddatabase)0zsmall satellitezLEO satellitezsatellite platformzsatellite buszspacecraft platformzspace vehiclezinstitutional satellitezoperational missionzdemonstration missionzin-orbit demonstrationz	IOD / IoDconstellationzInspector satellitezsatellite manufacturingzspacecraft manufacturingzsatellite design and buildzsatellite deliveryzturnkey satellitezrapid manufacturingzshort lead-timezagile satellite developmentzmass customizationzlow-volume productionzscalable productionzpayload accommodationzhosted payloadzcustomer-furnished payloadzpayload integrationzpayload interfacezpayload mass / power / volumezEO (payload)zEarth observationzRemote sensingzoptical payloadzSAR payloadzSynthetic aperture radarz
RF payloadzISR payloadzdual-use payloadzdefence satellitezmilitary satellitezsovereign capabilityISRz	SSA / SST
classifiedzdual-usezgovernmental missionzinstitutional customeri  i  Z   )daysc                 P    SSK Jn  U" U 5      R                  SS5      nSU SU S3$ )Nr   )quotez%20+z*https://ted.europa.eu/en/search/result?FT=zg&search-scope=ALL&scope=ALL&onlyLatestVersions=false&sortColumn=publication-number&sortOrder=DESC&page=z&simpleSearchRef=true)urllib.parser   replace)keywordpager   fts       europa_all.py	build_urlr!   S   s:    "	w		s	+B
4RD>>BVCX	Z    c                    U (       d  g U R                  5       nU(       d  g [        R                  " SU5      (       a  US S S-   $ [        R                  " SU5      nU(       ay  UR                  S5      UR                  S5      UR                  S5      UR                  S5      4u  p4pVU S	UR	                  S5       S	UR	                  S5       S
U=(       d    S 3$ g )Nz^\d{4}-\d{2}-\d{2}
   z	 00:00:00zG^(\d{1,2})[\/\.\-](\d{1,2})[\/\.\-](\d{4})(?:\s+(\d{1,2}:\d{2}:\d{2}))?            - z00:00:00)striprematchgroupzfill)rawsmddmmyyyytime_s          r    
parse_dater7   ]   s    		A	xx%q))"v##
[]^_Aggaj!''!*aggaj!''!*Lq!Qrxx{m1U5Hj4IJJr"   c                 l   U R                  5       nUR                  SSU45        UR                  5       (       a   [        SU S35        UR	                  5         g SU 3n UR                  SSXX(U[        U5      45        UR                  n	UR                  SX=(       d    S [        U5      [        U5      U=(       d    S 45        U R                  5         [        SU	 S	U S
U S35        U	UR	                  5         $ ! [         a  n
U R                  5         U
eS n
A
ff = f! UR	                  5         f = f)Nz;SELECT id FROM tenders WHERE source = %s AND source_id = %seuropau     ⤷  Skip duplicate []z)https://ted.europa.eu/en/notice/-/detail/zINSERT INTO tenders (source, source_id, title, description, url, keyword, detail, closing_date, created_at)
               VALUES (%s, %s, %s, %s, %s, %s, 0, %s, NOW())zINSERT INTO tender_details
                 (tender_id, deadline_text, deadline, publication_date, contracting_org_country, created_at)
               VALUES (%s, %s, %s, %s, %s, NOW())u     ✔  Inserted id=z  [z]  keyword="")
cursorexecutefetchoneprintcloser7   	lastrowidcommit	Exceptionrollback)dbnotice_numberdescriptioncountrypublication_datedeadliner   r<   url	tender_ides              r    insert_tenderrN   l   s0   YY[F
NNE	=! 'a895m_
EC@}[wPZ[cPde	

 $$	5 (D*X*>
K[@\^e^mimn		
 			#I;c-WIUVWX
 		  
 	s%    BC= =
DDDD! !D3c                  \  #    [         R                  R                  " S/0 [        D6n [	        S5        [        5        IS h  vN nUR                  R                  SS9I S h  vN nUR                  SS9I S h  vN nUR                  5       I S h  vN n[        SS  GHc  n[	        SS	 35        [	        S
U S35        [	        S	5        SnSnU(       a  M9  [        XV5      n[	        SU S35        [	        SU 35         UR                  USSS9I S h  vN   [        R                  " ["        S-  5      I S h  vN    UR%                  SSS9I S h  vN   UR'                  SS5      I S h  vN n	[	        S[)        U	5       S35        Sn
U	 H  nUR+                  S5      (       d  M  [-        UR+                  SS 5      5      nU(       aB   [.        R0                  " US S S!5      nU[2        :  a  [	        S"US    S#US    S$35        Sn
  OZ  [7        U US   UR+                  S%S 5      UR+                  S&S 5      UR+                  SS 5      UR+                  S'S 5      U5        M     U
(       a  SnGM  UR9                  S*5      I S h  vN nU(       d  [	        S+5        SnOXUR;                  S,5      I S h  vN nU(       a  [	        S-5        SnO*US-  n[        R                  " [         S-  5      I S h  vN   U(       d  GM*  GMf     [	        S.5        UR=                  5       I S h  vN   S S S 5      IS h  vN   U R=                  5         g  GN
 GN GN GN GNB! [         a;    [	        S5        US-  n[        R                  " [         S-  5      I S h  vN     GM  f = f GNh GNR! [         a    [	        S5         GM"  f = f GNX! [4         a     GNf = f! [         a   n[	        S(US    S)U 35         S nAGMp  S nAff = f GN GNU GN N N! , IS h  vN  (       d  f       N= f7f)0Nu   ✔  Database connectedT)headlesszoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36)
user_agentr'   r$   
u   ════════════════════════════════════════════════════════════z  KEYWORD: "r;   r%   Fu   
  ─── Page u
    ───z  URL: networkidlei`  )
wait_untiltimeoutu-     ⚠  Page load timed out – skipping page.i  z,tbody.custom-react-classes-MuiTableBody-rooti0u  )rU   u;     ⚠  Table body not found – no results or end of pages.z/tbody.custom-react-classes-MuiTableBody-root traA  (trs) => trs.map(tr => {
                        const tds = Array.from(tr.querySelectorAll('td'));
                        return {
                            notice_number:    tds[1]?.innerText?.trim() || '',
                            description:      tds[2]?.innerText?.trim() || '',
                            country:          tds[3]?.innerText?.trim() || '',
                            publication_date: tds[4]?.innerText?.trim() || '',
                            deadline:         tds[5]?.innerText?.trim() || '',
                        };
                    })z  Found z row(s)rF   rI    z%Y-%m-%du     ⏹  [z] published u#    — older than 3 months, stopping.rG   rH   rJ   u     ✖  DB error for [z]: z(button[aria-label="Go to the last page"]u0     ✔  No "last page" button – end of results.zbtn => btn.disabledu6     ✔  "Last page" button disabled – end of results.u&   
═══ Scraping complete ═══ )mysql	connectorconnect	DB_CONFIGr?   r   chromiumlaunchnew_contextnew_pageKEYWORDSr!   gotorC   asynciosleepBETWEEN_PAGE_MSPAGE_WAIT_MSwait_for_selectoreval_on_selector_alllengetr7   r   strptimeCUTOFF_DATE
ValueErrorrN   query_selectorevaluater@   )rE   pbrowsercontextr   r   current_pageis_last_pagerK   rowshit_old_recordrowpub_date_strpub_daterM   last_page_btnis_disableds                    r    mainr{      s    		 	 	-9	-B	
#$!!Q

))4)88++E , 
 
 %%''"~GBzl#$L	+,*L L"l6+L>DEuo&))CM5)QQQ mmL4$788800F % 1    "66E	  T734!&C77?33 #-cgg6H".M#NL#!'/'8'8cr9JJ'WH'+5 %_1E0FlSVWiSjRk  lO  'P  !Q15 %  6T%0GGM26GGIr2GG$6;GGJ3#!  8 "#'L&*&9&9:d&e e$LM#'L(5(>(>?T(U"UK"VW'+$)%mmOd,BCCCi #l &| 	78mmoS "!V HHJW "8
 (  R  IJ A%L!--$(>???	 9 ! WX:  * ! ! % T 5c/6J5K3qcRSST !f
 #V D 	S "!!!s  >P, M
P,P!M"P8M9PMAP(P MMM P9N!:P?N'N$N'P,O-AP>O	PAO!$PP-P3P4>P2P3P#P$P%P)P,4P5P,PPPM:NN
NPNP$N''O<P OP
O	PO	P
P	"O=	6P=P	PPPPP,P)PP)%P,__main__)!rb   osr,   sysr   r   dotenvr   mysql.connectorrX   playwright.async_apir   pathjoindirname__file__platformstdoutreconfiguregetenvintr[   r`   re   rd   nowrk   r!   r7   rN   r{   __name__runrW   r"   r    <module>r      s    	 	 
 (   1 BGGLL2I> ?<<7JJG, 		)$BIIi./		)$		-(		)$	1f llnyb11!Jod zKK r"   