
    HziF!              	       L   S SK r S SKrS SKrS SKrS SKJrJr  S SKJr  S SKr	S SK
Jr  \" \R                  R                  \R                  R                  \5      S5      5        \R                   S:X  a  \R"                  R%                  SS9  \R&                  " S	5      \" \R&                  " S
S5      5      \R&                  " S5      \R&                  " S5      \R&                  " S5      S.r\R&                  " SS5      R-                  S5       V s/ s H)  o R/                  5       (       d  M  U R/                  5       PM+     sn rSrSr\R6                  " 5       \" \" \R&                  " SS5      5      S9-
  rS rS rS rS r \!S:X  a  \ RD                  " \ " 5       5        ggs  sn f )    N)datetime	timedelta)load_dotenv)async_playwrightz../.envwin32zutf-8)encodingDB_HOSTDB_PORTi  DB_USERDB_PASSWORDDB_NAME)hostportuserpassworddatabaseKEYWORDS ,i  i  EUROPA_DAYS   )daysc                 P    SSK Jn  U" U 5      R                  SS5      nSU SU S3$ )Nr   )quotez%20+z*https://ted.europa.eu/en/search/result?FT=zg&search-scope=ALL&scope=ALL&onlyLatestVersions=false&sortColumn=publication-number&sortOrder=DESC&page=z&simpleSearchRef=true)urllib.parser   replace)keywordpager   fts        europa-python/europa_keywords.py	build_urlr"   "   s:    "	w		s	+B
4RD>>BVCX	Z    c                    U (       d  g U R                  5       nU(       d  g [        R                  " SU5      (       a  US S S-   $ [        R                  " SU5      nU(       ay  UR                  S5      UR                  S5      UR                  S5      UR                  S5      4u  p4pVU S	UR	                  S5       S	UR	                  S5       S
U=(       d    S 3$ g )Nz^\d{4}-\d{2}-\d{2}
   z	 00:00:00zG^(\d{1,2})[\/\.\-](\d{1,2})[\/\.\-](\d{4})(?:\s+(\d{1,2}:\d{2}:\d{2}))?            - z00:00:00)striprematchgroupzfill)rawsmddmmyyyytime_s          r!   
parse_dater8   ,   s    		A	xx%q))"v##
[]^_Aggaj!''!*aggaj!''!*Lq!Qrxx{m1U5Hj4IJJr#   c                 l   U R                  5       nUR                  SSU45        UR                  5       (       a   [        SU S35        UR	                  5         g SU 3n UR                  SSXX(U[        U5      45        UR                  n	UR                  SX=(       d    S [        U5      [        U5      U=(       d    S 45        U R                  5         [        SU	 S	U S
U S35        U	UR	                  5         $ ! [         a  n
U R                  5         U
eS n
A
ff = f! UR	                  5         f = f)Nz;SELECT id FROM tenders WHERE source = %s AND source_id = %seuropau     ⤷  Skip duplicate []z)https://ted.europa.eu/en/notice/-/detail/zINSERT INTO tenders (source, source_id, title, description, url, keyword, detail, closing_date, created_at)
               VALUES (%s, %s, %s, %s, %s, %s, 0, %s, NOW())zINSERT INTO tender_details
                 (tender_id, deadline_text, deadline, publication_date, contracting_org_country, created_at)
               VALUES (%s, %s, %s, %s, %s, NOW())u     ✔  Inserted id=z  [z]  keyword="")
cursorexecutefetchoneprintcloser8   	lastrowidcommit	Exceptionrollback)dbnotice_numberdescriptioncountrypublication_datedeadliner   r=   url	tender_ides              r!   insert_tenderrO   ;   s0   YY[F
NNE	=! 'a895m_
EC@}[wPZ[cPde	

 $$	5 (D*X*>
K[@\^e^mimn		
 			#I;c-WIUVWX
 		  
 	s%    BC= =
DDDD! !D3c                    #    [         (       d  [        S5        g [        R                  R                  " S10 [
        D6n [        S5        [        S[        [         5       S35        [        5        IS h  vN nUR                  R                  SS9I S h  vN nUR                  SS9I S h  vN nUR                  5       I S h  vN n[          GHc  n[        S	S
 35        [        SU S35        [        S
5        SnSnU(       a  M9  [        XV5      n[        SU S35        [        SU 35         UR                  USSS9I S h  vN   [        R                   " [$        S-  5      I S h  vN    UR'                  SSS9I S h  vN   UR)                  SS5      I S h  vN n	[        S[        U	5       S35        Sn
U	 H  nUR+                  S5      (       d  M  [-        UR+                  S S!5      5      nU(       aB   [.        R0                  " US S" S#5      nU[2        :  a  [        S$US    S%US     S&35        Sn
  OZ  [7        U US   UR+                  S'S!5      UR+                  S(S!5      UR+                  S S!5      UR+                  S)S!5      U5        M     U
(       a  SnGM  UR9                  S,5      I S h  vN nU(       d  [        S-5        SnOXUR;                  S.5      I S h  vN nU(       a  [        S/5        SnO*US-  n[        R                   " ["        S-  5      I S h  vN   U(       d  GM*  GMf     [        S05        UR=                  5       I S h  vN   S S S 5      IS h  vN   U R=                  5         g  GN GN GN GN GNB! [         a;    [        S5        US-  n[        R                   " ["        S-  5      I S h  vN     GM  f = f GNh GNR! [         a    [        S5         GM"  f = f GNX! [4         a     GNf = f! [         a   n[        S*US    S+U 35         S nAGMp  S nAff = f GN GNU GN N N! , IS h  vN  (       d  f       N= f7f)2Nz+No keywords found in KEYWORDS env variable.u   ✔  Database connectedz  Keywords: z loaded from .envT)headlesszoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36)
user_agent
u   ════════════════════════════════════════════════════════════z  KEYWORD: "r<   r&   Fu   
  ─── Page u
    ───z  URL: networkidlei`  )
wait_untiltimeoutu-     ⚠  Page load timed out – skipping page.i  z,tbody.custom-react-classes-MuiTableBody-rooti0u  )rV   u;     ⚠  Table body not found – no results or end of pages.z/tbody.custom-react-classes-MuiTableBody-root traA  (trs) => trs.map(tr => {
                        const tds = Array.from(tr.querySelectorAll('td'));
                        return {
                            notice_number:    tds[1]?.innerText?.trim() || '',
                            description:      tds[2]?.innerText?.trim() || '',
                            country:          tds[3]?.innerText?.trim() || '',
                            publication_date: tds[4]?.innerText?.trim() || '',
                            deadline:         tds[5]?.innerText?.trim() || '',
                        };
                    })z  Found z row(s)rG   rJ   r   r%   z%Y-%m-%du     ⏹  [z] published u#    — older than 3 months, stopping.rH   rI   rK   u     ✖  DB error for [z]: z(button[aria-label="Go to the last page"]u0     ✔  No "last page" button – end of results.zbtn => btn.disabledu6     ✔  "Last page" button disabled – end of results.u&   
═══ Scraping complete ═══ )r   r@   mysql	connectorconnect	DB_CONFIGlenr   chromiumlaunchnew_contextnew_pager"   gotorD   asynciosleepBETWEEN_PAGE_MSPAGE_WAIT_MSwait_for_selectoreval_on_selector_allgetr8   r   strptimeCUTOFF_DATE
ValueErrorrO   query_selectorevaluaterA   )rF   pbrowsercontextr   r   current_pageis_last_pagerL   rowshit_old_recordrowpub_date_strpub_daterN   last_page_btnis_disableds                    r!   mainrz   `   s4    8;<		 	 	-9	-B	
#$	LX'8
9:!!Q

))4)88++E , 
 
 %%''xGBzl#$L	+,*L L"l6+L>DEuo&))CM5)QQQ mmL4$788800F % 1    "66E	  T734!&C77?33 #-cgg6H".M#NL#!'/'8'8cr9JJ'WH'+5 %_1E0FlSVWiSjRk  lO  'P  !Q15 %  6T%0GGM26GGIr2GG$6;GGJ3#!  8 "#'L&*&9&9:d&e e$LM#'L(5(>(>?T(U"UK"VW'+$)%mmOd,BCCCi #l  | 	78mmoS "!V HHJW "8
 (  R  IJ A%L!--$(>???	 9 ! WX:  * ! ! % T 5c/6J5K3qcRSST !f
 #V D 	S "!!!s  A1Q3M:4Q7QM=Q+N ,QN?Q(Q0N	NN		 Q)O*Q/OOOQO4AQ;>O79Q=AP$Q5P56-Q#P8$>Q"P;#Q1#QP>QQ$Q %Q=Q QQN		:OO
O	QOQOO1,Q0O11Q7
P	QP	Q
P2	P-	&Q-P2	2Q8Q;Q>Q QQQ	QQ__main__)#rb   osr-   sysr   r   dotenvr   mysql.connectorrX   playwright.async_apir   pathjoindirname__file__platformstdoutreconfiguregetenvintr[   splitr,   r   re   rd   nowrj   r"   r8   rO   rz   __name__run)ks   0r!   <module>r      sT    	 	 
 (   1 BGGLL2I> ?<<7JJG, 		)$BIIi./		)$		-(		)$	  "yyR8>>sCQC!wwyIAGGICQllnyc"))M22N.OPP!Jtn zKK } Rs   F!,F!