
    hzi#                        S SK r S SKrS SKrS SKrS SKJrJr  \R                  S:X  a  \R                  R                  SS9  S SK	J
r
  \
" \R                  R                  \R                  R                  \5      S5      5        S SKrS r\R$                  " S	S
5      R'                  S5       V s/ s H)  o R)                  5       (       d  M  U R)                  5       PM+     sn rSrSrSrSr\" \R$                  " SS5      5      r\R8                  " 5       \" \S9-
  rS rS rS r S r!S r"S r#S r$\%S:X  a	   \$" 5         ggs  sn f ! \& a    \'" S\"" 5       S 35         g\( a*  r)\'" S\) 35        S SK*r*\*RV                  " 5          Sr)C)gSr)C)ff = f)    N)datetime	timedeltawin32zutf-8)encoding)load_dotenvz../.envc            	          [         R                  R                  [        R                  " S5      [        [        R                  " SS5      5      [        R                  " S5      [        R                  " S5      [        R                  " S5      S9$ )NDB_HOSTDB_PORTi  DB_USERDB_PASSWORDDB_NAME)hostportuserpassworddatabase)mysql	connectorconnectosgetenvint     nen.nipez/nipez_keywords.pyget_db_connectionr      sa    ??""YYy!9d+,YYy!=)9% #  r   KEYWORDS ,zNhttps://nen.nipez.cz/api/datarows/datawithcount?className=Zadavaci_postup_histz9https://nen.nipez.cz/en/verejne-zakazky/detail-zakazky/{}   nipez
NIPEZ_DAYS   )daysc                     [         R                  " 5       n U R                  R                  SSS.5        U R	                  SSS9  U R
                  R	                  SS5      nU(       d!  [        S	5        [        R                  " S
5        X4$ )NzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36z!application/json, text/plain, */*)z
User-AgentAcceptz'https://nen.nipez.cz/en/verejne-zakazkyr#   )timeoutzXSRF-TOKEN-Clientr   zERROR: Could not get XSRF token   )	requestsSessionheadersupdategetcookiesprintsysexit)sessionxsrfs     r   get_sessionr4   "   sy     GOO H5  KK92KF??2B7D/0=r   c                 f   SSSS.SSS.SSS.S	S
S.SSS.SSS.SSS./S/ [         SUS/U0 S.
n U R                  [        UUSSSS.SS9nUR                  S:X  a  UR	                  5       $ [        SUR                   35        g ! [        R                   a  n[        SU 35         S nAg S nAff = f)NZadavaci_postup_histidID)
clientName
serverNamekodKodnazevNazevstavZPStav_ZPzadavatelNazevzZadavatel.NazevpodaniLhutazPodani_nabidka.Lhuta	createCas
create_casz2(Hierarchie = null and Posledni_uverejneni = true)TzDESC create_cas)
	className
attributes	conditionconditionParamscount
isLoadMore
startIndexorderByqueryabortSignalENzhttps://nen.nipez.czzapplication/json)zX-XSRF-TOKENLanguageOriginzContent-Type<   )jsonr+   r'   r    z  HTTP z  Request Error: )	PAGE_SIZEpostAPI_URLstatus_coderS   r/   r)   RequestException)r2   r3   start_indexkeywordpayloadres          r   
fetch_pager^   0   s    +t4 6"':#9=+;LM(8NO&lC
 J!%&%G(LLw ,.	9

    ==C668OGAMM?+,$$ !!%&s   :B +B B0B++B0c                     U (       d  g  S HC  n [         R                  " [        U 5      R                  5       S S U5      R	                  5       s  $    g ! [
         a     MU  f = f! [         a     g f = f)N)%Y-%m-%dT%H:%M:%S%Y-%m-%dz%d-%m-%Yz%d/%m/%Y   )r   strptimestrstripdate
ValueError	Exception)valfmts     r   	safe_daterk   V   sx    LC((S)9#2)>DIIKK M 	   s:   A% >AA% A% 
A"A% !A""A% %
A21A2c                 L   U (       d  g[        5       nUR                  5       nSnSnU  GH  n[        UR                  SS5      5      nUR                  SS5      nUR                  SS5      n	UR                  SS5      n
UR                  SS5      n[	        UR                  S	5      5      nU(       a%  [
        R                  UR                  S
S5      5      OSn UR                  S[        U45        UR                  5       nU(       a  UR                  SXXXS   45        US-  nM  UR                  S[        XyXXX4	5        US-  nGM!     UR                  5         UR                  5         UR                  5         XE4$ ! [         a  n[        SU 35         S nAGMu  S nAff = f)N)r   r   r   r7   r   r;   r=   r?   rA   rB   /-zCSELECT id FROM tenders WHERE source = %s AND source_id = %s LIMIT 1zUPDATE tenders SET title = %s, organization = %s, status = %s,
                       closing_date = %s, keyword = %s, updated_at = NOW() WHERE id = %sr(   a  INSERT INTO tenders
                           (source, source_id, title, reference_number, url, organization,
                            status, closing_date, keyword, detail, created_at, updated_at)
                       VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 0, NOW(), NOW())z  DB Error: )r   cursorrd   r-   rk   DETAIL_BASEformatreplaceexecuteSOURCEfetchonerh   r/   commitclose)tendersrZ   connro   insertedupdatedt	source_idr;   titlestatus	authoritydeadlineurlrowr]   s                   r   insert_tendersr   d   s   D[[]FHGtR)	UU5"%UU7B'UU8R(UU+R0	quu]34BE[''C(=>2	NNU# //#C\va&I
 1W YsH^ A? J 	KKM
LLNJJL  	L$%	s   AE?' E??
F#	FF#c                      [        5       n U R                  5       nUR                  S[        45        UR	                  5       S   nUR                  5         U R                  5         U$ )Nz.SELECT COUNT(*) FROM tenders WHERE source = %sr   )r   ro   rs   rt   ru   rw   )ry   ro   rI   s      r   get_current_countr      sN    D[[]F
NNCfYOOOa E
LLNJJLLr   c           
      B   [        SU S35        SnSnSnSnSnSn [        SU SU S3S	S
9  [        XX25      n	U	cK  US-  n[        SU S35        US:  a  [        S5        [        5       u  pSn[        R                  " S5        Ml  U	R                  S/ 5      n
[        U
5      nUS:X  a  [        S5        GOSnSn/ nU
 H  nUR                  S5      nU(       ae   [        R                  " [        U5      S S S5      nU(       a:  U[        :  a0  [        SUR                  SS5       SUS S  S[         S35        Sn  OUR                  U5        M     [        X5      u  nnX[-  nUU-  nUU-  n[        SU SU S U S35        U(       d
  U[        :  a  O'U[        -  nUS-  n[        R                  " S5        GM  [        S!U S"U S U 35        XXVU4$ ! [         a<     [        R                  " [        U5      S S S5      n GN
! [         a    S n  GNf = ff = f)#Nz
  Keyword: ""r   r(   Tz
    [Page z] startIndex=z... endz	Failed (#)   z    Refreshing session...   
collectionzNo results.FrC   rb   r`   
   ra   u
       ⏹  [r;   r   z
] created u    — older than z days, stopping.zGot z (New: z, Updated: z    => Fetched: z, New: )r/   r^   r4   timesleepr-   lenr   rc   rd   rg   CUTOFF_DATECUTOFF_DAYSappendr   rT   )r2   r3   rZ   rY   pagetotal_fetchedtotal_insertedtotal_updatedconsecutive_errorsdatar   rI   
hit_cutofffiltereditemrD   	item_daterz   r{   s                      r   scrape_keywordr      su   	OG9B
'(KDMNM

4&k]#>CH'><!#I0134!Q&12 +%&"JJqMXXlB/
JA:-  
D+.J) ( 1 1#j/#22FH[ \I [!8Jtxxr':&;:jQTRToEVVfgrfs  tD  E  F!%JOOD!   +8='(" UG78*KyBC*y 	

1o r 
]O7>2B+m_
]^-FF5 " ))$,$5$5c*ocr6JJ$W	% )$(	))s*   #G
H##H		HHHHc            	         [        S5        [        S5        [        S5        [        S[        [        5       35        [        SSS9  [        5       u  p[        S5        SnSnSn[	        [        S	5       HU  u  pV[        S
U S[        [        5       S3SS9  [        XU5      u  ppxn	X'-  nX8-  nXI-  n[        R                  " S	5        MW     [        S5        [        S5        [        S5        [        S[        [        5       35        [        SUS 35        [        SUS 35        [        SUS 35        [        S[        5       S 35        [        S5        g )NzF======================================================================z%NEN.NIPEZ.CZ - Keyword Tender ScraperzKeywords to search: z
Getting session...r   r   OKr   r(   z
[rm   ]r   zG
======================================================================zScraping Complete!z  Keywords searched : z  Total fetched     : r   z  New               : z  Updated           : z  DB total (nipez)  : )	r/   r   r   r4   	enumerater   r   r   r   )
r2   r3   grand_fetchedgrand_insertedgrand_updatedirZ   fetchedrz   r{   s
             r   scrape_tendersr      sC   	(O	
12	(O	 X
01	
 c*MMG	$KMNM!,
A3aHa(b14B7RY4Z1w' " 

1 - 
/	
	(O	"3x=/
23	"="3
45	">!"4
56	"="3
45	"#4#6q"9
:;	(Or   __main__z

Interrupted. DB total: z
Fatal Error: ),r)   r   r0   r   r   r   platformstdoutreconfiguredotenvr   pathjoindirname__file__mysql.connectorr   r   r   splitre   r   rV   rp   rT   rt   r   r   nowr   r4   r^   rk   r   r   r   r   __name__KeyboardInterruptr/   rh   r]   	traceback	print_exc)ks   0r   <module>r      s|     
 	 (<<7JJG,  BGGLL2I> ?  #%))J";"A"A#"FT"FQ'')yqwwy"FT^I	"))L"-.llnyk::#L2jCGLB z e Uj  E+,=,?+BCD s#$s*   .D?
D?6E FF" FF