
    |ziQ*                        S SK r S SKrS SKrS SKrS SKrS SKJrJr  \ R                  S:X  a  \ R                  R                  SS9  S SK
Jr  \" \R                  R                  \R                  R                  \5      S5      5        S SKr\R$                  " S5      \" \R$                  " S	S
5      5      \R$                  " S5      \R$                  " S5      \R$                  " S5      S.r\R*                  " \R$                  " SS5      5      r\" \R$                  " SS5      5      r\R0                  " 5       \" \S9-
  rS rS SKJr  SrSrSrSr Sr!Sr"S r#S r$S r%S)S jr&S  r'S! r(S" r)S# r*S$ r+S% r,\-S&:X  a	   \," 5         gg! \. a    \/" S'5         g\0 a*  r1S SK2r2\/" S(\1 35        \2Rf                  " 5          Sr1C1gSr1C1ff = f)*    N)datetime	timedeltawin32zutf-8)encoding)load_dotenvz../.envDB_HOSTDB_PORTi  DB_USERDB_PASSWORDDB_NAME)hostportuserpassworddatabase	CPV_CODESz{}EVERGABE_DAYS   )daysc                  J    [         R                  R                  " S0 [        D6$ )N )mysql	connectorconnect	DB_CONFIGr       evergabe/evergabe_keywords.pyget_db_connectionr      s    ??""/Y//r   )sync_playwrightz,https://www.evergabe-online.de/search.html?4zhttps://www.evergabe-online.dezwww.evergabe-online.de
   i0u  i  c                     U (       d  g  [         R                  " U R                  5       S5      R                  S5      $ ! [         a     g f = f)Nz%d.%m.%y, %H:%M%Y-%m-%dr   strptimestripstrftime
ValueErrorraws    r   parse_deadliner*   )   sD      .?@II*UU    3> 
A
Ac                     U (       d  g  [         R                  " U R                  5       S5      R                  S5      $ ! [         a     g f = f)Nz%d.%m.%yr"   r#   r(   s    r   parse_publication_dater-   2   sC      j9BB:NN r+   c                 v    [         R                  " SU 5      nU(       a  [        UR                  S5      5      $ S $ )Nz[?&]id=(\d+)   )researchintgroup)urlms     r   extract_source_idr6   ;   s+    
		/3'A3qwwqz?)T)r   c                     U R                  SU[        45        U R                  5       nU(       a  g [        U5      nU R                  S[        XX#XEU45        U R                  $ )NzJSELECT id FROM tenders WHERE reference_number = %s AND source = %s LIMIT 1zINSERT INTO tenders (source, source_id, title, url, reference_number, organization, keyword, detail, closing_date, created_at)
           VALUES (%s, %s, %s, %s, %s, %s, %s, 0, %s, NOW()))executeSOURCEfetchoner6   	lastrowid)	cursortitler4   reference_numberorganizationcpv_codeclosing_daterow	source_ids	            r   insert_tenderrD   @   sf    
NNT	6" //
C
!#&I
NN	@	3,R^_
 r   c           
      0    U R                  SXUXEXg45        g )NzINSERT INTO tender_details
               (tender_id, contracting_authority_name, contracting_org_address,
                procedure_type, deadline, deadline_text, publication_date, created_at)
           VALUES (%s, %s, %s, %s, %s, %s, %s, NOW()))r8   )r<   	tender_idcontracting_authority_namecontracting_org_addressprocedure_typedeadlinedeadline_textpublication_dates           r   insert_tender_detailrM   R   s&    
NN	9 
0G	=	Dr   c                 *    U R                  SS5      nU$ )Ntable#datatable tbody tra  trs => trs.map(tr => {
            const tds = Array.from(tr.querySelectorAll('td'));
            if (tds.length < 4) return null;
            const anchor = tds[0].querySelector('a');
            return {
                title            : tds[0].innerText.trim(),
                url              : anchor ? anchor.href : '',
                reference_number : tds[1].innerText.trim(),
                organization     : tds[2].innerText.trim(),
                address          : tds[3].innerText.trim(),
                procedure_type   : tds[4] ? tds[4].innerText.trim() : '',
                deadline         : tds[5] ? tds[5].innerText.trim() : '',
                publication_date : tds[6] ? tds[6].innerText.trim() : '',
            };
        }).filter(r => r !== null))eval_on_selector_all)pagerowss     r   scrape_rowsrS   ^   s!    $$"	&D$ Kr   c                 l   SnSnU  H  nUR                  SS5      R                  5       nU(       d  US-  nM1  [        UUS   US   UUS   U[        US   5      S	9nUc  US-  nM_  [	        UUUS   US
   US   [        US   5      US   [        US   5      S9  US-  nM     UR                  5         XE4$ )Nr   r>    r/   r=   r4   r?   rJ   )r=   r4   r>   r?   r@   rA   addressrI   rL   )rF   rG   rH   rI   rJ   rK   rL   )getr%   rD   r*   rM   r-   commit)	rR   connr<   r@   insertedskippedrB   refrF   s	            r   process_rowsr]   t   s    HGgg("-335qLG!"7|"5z"">2'-c*o>
	 qLG)2),^)<),Y),-=)>)7J)H),Z)?DV@W)X		
 	A= @ 	KKMr   c                 n     U R                  SS5      nSUR                  5       ;   $ ! [         a     gf = f)Na.next.iconzel => el.classNamedisabledT)eval_on_selectorsplit	Exception)rQ   classess     r   is_next_disabledre      s=    ''7KLW]]_,, s   $' 
44c           
      ^   [        SU SU 35        U R                  [        SSS9   U R                  S5      nUR	                  5       (       d{  [        S5        S Hj  n U R                  U5      R
                  nUR	                  5       (       a6  UR                  5         [        S	U 35        [        R                  " S
5          OMl      U R                  SSS9   U R                  S5      R                  USS9  [        SU 35         U R                  S5        [        S5        [        S[         S35        [        R                  " [        5        Sn	Sn
S
n [        SU S35         U R                  S[        S9  [        U 5      n[        S[!        U5       S35        U(       d  [        S5         X4$ S n/ nU H  n[#        UR%                  S!S"5      5      nU(       aS   [&        R(                  " US#5      nU[*        :  a0  [        S$UR%                  S%S"5       S&US!    S'[,         S(35        Sn  O UR1                  U5        M     [3        XX#5      u  nnU	U-  n	U
U-  n
[        S)U S*U 35        U(       d  [5        U 5      (       a  [        S+5         X4$ [        S,5         U R                  S-5        [        R                  " [6        S0-  5        US
-  nGM  ! [         a     GM  f = f! [         a     GN1f = f! [         a     GN1f = f! [         a  n[        SU 35         S nAgS nAff = f! [         a  n[        SU 35         S nAgS nAff = f! [         a    [        S5         X4$ f = f! [.         a     GNOf = f! [         a  n[        S.U S/35         S nAX4$ S nAff = f)1Nz
  CPV: u    — networkidlei`  )
wait_untiltimeoutz#cpvCodeu<     CPV field hidden — looking for advanced search toggle...)za[data-toggle="collapse"]zbutton[data-toggle="collapse"]z.advancedSearchz#advancedSearchza:has-text("Erweiterte")za:has-text("Advanced")zbutton:has-text("Erweiterte")zspan:has-text("Erweiterte")z  Clicked toggle: r/   z&#cpvCode:not([style*="display: none"])i  )ri   T)forcez  Filled #cpvCode with: z(  ERROR: Could not fill #cpvCode field: )r   r   z7document.querySelector('[name="submitButton"]').click()z  Clicked submitButton (JS)z'  ERROR: Could not click submitButton: z
  Waiting zs for results...r   z  --- Page z ---rO   u)     table#datatable not found — stopping.z  Found z row(s)u     No rows — stopping.FrL   rU   r"   u     ⏹  [r>   z] published u    — older than z days, stopping.z  Inserted: z!  |  Skipped (duplicates/empty): u%     Next button disabled — last page.z  Clicking next page...r_   z  Could not click next: u    — stopping.i  )printgotoBASE_URLlocator
is_visiblefirstclicktimesleeprc   wait_for_selectorfillevaluateWAIT_AFTER_SEARCH_STABLE_TIMEOUT_MSrS   lenr-   rW   r   r$   CUTOFF_DATECUTOFF_DAYSr'   appendr]   re   PAGE_LOAD_WAIT_MS)pgrY   r<   r@   	cpv_label	cpv_inputselectortoggleetotal_insertedtotal_skippedpage_numrR   
hit_cutofffiltered_rowsrB   pubpub_dtinsskps                       r   
scrape_cpvr      s   	IhZuYK
01 GGHG>JJz*	##%%PQ	ZZ177F((** 28*=>

1	 +	.
EtT


:##HD#9(
34
OP+- 
J*++;
<=JJ"#NMH
H:T*+	  !;EU V
 2T7+,+,J ((E 
C(1CR)HIC%..sJ?F+1CR)H(IVYZlVmUnn~  @K  L  L\  ]  ^%)
 ,   %   VFS##SE!B3%HI)"--9: (( 	'(	HH]#
 	

$t+,Ac E !      8<=  7s;<"  	=>X ((]	. "    	,QC~>?
 ((	s   6K- A$KK- K- 
K> -L 	L4 )M "AM6%N 
K*%K- )K**K- -
K;:K;>
LL
L1L,,L14
M>MMM32M36
NN
N,N''N,c            
         [        S5        [        S5        [        S5        [        S[        [        5       35        [        S5        [        (       d  [        S5        g [        5       n U R	                  5       nSnSn[        5        nUR                  R                  SS9nUR                  SS	9nUR                  5       n[        [        R                  5       S
5       HS  u  nu  p[        SU S[        [        5       S3SS9  [        XpXU
5      u  pX+-  nX<-  n[        R                  " S5        MU     UR                  5         S S S 5        UR                  5         U R                  5         [        S5        [        S5        [        S5        [        S[        [        5       35        [        SUS 35        [        SUS 35        [        S5        g ! , (       d  f       N= f)NzF======================================================================z"Evergabe-Online - CPV Code ScraperzCPV codes to search: z!ERROR: No CPV_CODES found in .envr   T)headlesszoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36)
user_agentr/   z
[/]rU   )end   zG
======================================================================zScraping Complete!z  CPV codes searched : z  Rows inserted      : ,z  Rows skipped       : )rk   ry   r   r   r<   r   chromiumlaunchnew_contextnew_page	enumerateitemsr   rr   rs   close)rY   r<   grand_insertedgrand_skippedpwbrowsercontextr~   ir@   r   r   r   s                r   runr     s   	(O	
./	(O	!#i.!1
23	(O912 D[[]FNM		b++$$d$3%%1 & 
 (1)//2CQ(G$A$Cs!C	N+1-26!"FiHHC!N!MJJqM )H 	% 
( LLNJJL	/	
	(O	#C	N#3
45	#N1#5
67	#M!#4
56	(O; 
	s   B>G
G!__main__z

Interrupted by user.z
Fatal error: )N)4sysosrr   r0   jsonr   r   platformstdoutreconfiguredotenvr   pathjoindirname__file__mysql.connectorr   getenvr2   r   loadsr   r{   nowrz   r   playwright.sync_apir   rm   	SITE_ROOTr9   rw   rx   r}   r*   r-   r6   rD   rM   rS   r]   re   r   r   __name__KeyboardInterruptrk   rc   r   	traceback	print_excr   r   r   <module>r      s   
 	  	  (<<7JJG,  BGGLL2I> ?  IIi 		)T*+IIi 		-(		)$	 jj;56	"))OR01llnyk::0 0:,	$   *
$	,$Nr)j.b z   *() s#$s   :F GG F==G