
    6ibT                     *   S SK r S SKJr  S SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKr\R                  S:X  a  \R                  R                  SS9  \R                  R!                  S \R                  R#                  \R                  R%                  \5      S5      5        S SKJrJr  SrS	S
SS.rS	SSS
SSS.rSrS rS-S jrS rS rS rS r S r!S r"S r#S r$S r%S r&S r'S.S jr(\)S :X  a`  \RT                  " S!S"9r+\+RY                  S#\-SS$S%9  \+RY                  S&S'S(S)9  \+R]                  5       r/ \(" \/R`                  \/Rb                  S*9  gg! \2 a    \3" S+5         g\4 a*  r5\3" S,\5 35        S SK6r6\6Rn                  " 5          Sr5C5gSr5C5ff = f)/    N)BeautifulSoupwin32utf-8encodingz..)	DB_CONFIGget_db_connectionzPhttps://nepps-search.eprocurement.gov.gr/actSearch/faces/active_search_main.jspxzoMozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36z?text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8zel-GR,el;q=0.9,en;q=0.8)
User-AgentAcceptzAccept-Languagez0application/x-www-form-urlencoded; charset=UTF-8true(https://nepps-search.eprocurement.gov.gr)r
   zContent-TypezAdf-Rich-Messager   OriginReferer   c                     [        5       n U R                  5       nUR                  S5        UR                  5        Vs1 s H  o"S   iM	     nnSU;  a  UR                  S5        [	        S5        UR                  S5        UR                  5       (       d  UR                  S5        [	        S5        U R                  5         UR                  5         U R                  5         g	s  snf )
z&Add detail column if it doesn't exist.zSHOW COLUMNS FROM neppsr   detailz:ALTER TABLE nepps ADD COLUMN detail INT NOT NULL DEFAULT 0z  Added column: detailz3SHOW INDEX FROM nepps WHERE Key_name = 'idx_detail'z)CREATE INDEX idx_detail ON nepps (detail)z  Added index: idx_detailN)r	   cursorexecutefetchallprintcommitclose)connr   rowexistings       nepps/scrape_detail.pyadd_detail_columnr   &   s    D[[]F
NN,-"(//"34"33A"3H4xST&'
NNHI??BC)*KKM
LLNJJL 5s   C#c                 &   [        5       nUR                  SS9nUR                  S5        UR                  5       S   nSnU (       a  USU  3-  nUR                  U5        UR	                  5       nUR                  5         UR                  5         XS4$ )zGet records where detail=0.T)
dictionaryz4SELECT COUNT(*) as total FROM nepps WHERE detail = 0totalzFSELECT id, procedure_id, title FROM nepps WHERE detail = 0 ORDER BY idz LIMIT )r	   r   r   fetchoner   r   )
batch_sizer   r   r    queryrecordss         r   get_pending_recordsr%   <   s    D[[D[)F
NNIJOOg&ETE7:,''
NN5ooG
LLNJJL>    c                    U R                  [        [        SS9nUR                  5         UR                  nSn[
        R                  " SU5      nU(       a  UR                  S5      nSn[
        R                  " SU5      nU(       a  UR                  S5      nSSR                  [        R                  " [        R                  [        R                  -   S	S
95      -   n0 SU=(       d    S_SS_SU_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_S S_S!S_S"S_n[        nU(       a  [        S#-   U-   nU R                  X[        SS$9nUR                  5         UR                  nS%U;   a  [
        R                  " SU5      nU(       a  UR                  S5      n[
        R                  " SU5      nU(       a  UR                  S5      n[        S#-   U-   nU=(       d    S&US'   U R                  X[        SS$9nUR                  5         UR                  nSn	[
        R                  " S'U5      nU(       a  UR                  S5      n	U	(       d/  [
        R                  " S(U5      nU(       a  UR                  S5      n	U	(       d/  [
        R                  " S)U5      nU(       a  UR                  S5      n	[        S*U 35        [        S+U	(       a  U	SS, OS- S.35        X4$ )/zdHandle ADF loopback flow and extract ViewState + Adf-Window-Id.
Reused from scrape_tenders_ajax.py.
   headerstimeoutNz'_afrLoop',\s*'(\d+)'   z;jsessionid=([^'\"]+)'w 
   )k_afrLoop1_afrWindowMode0Adf-Window-Id_afrFS16_afrMTscreen_afrMFW1920_afrMFH1080_afrMFDW_afrMFDH_afrMFC24_afrMFCI_afrMFM_afrMFR96_afrMFG_afrMFS_afrMFOz;jsessionid=)paramsr*   r+   zAdfLoopbackUtils.runLoopback20name="javax\.faces\.ViewState"\s+value="([^"]+)"z-javax\.faces\.ViewState["\s>]*value="([^"]+)"z)ViewState.*?(?:value|CDATA)\W+([^\s"<>]+)z  Window-Id: z  ViewState: 2   Nonez...)getURLHEADERSraise_for_statustextresearchgroupjoinrandomchoicesstringascii_lowercasedigitsr   )
sessionresprR   afr_loopm
jsessionid	window_idloopback_paramsloopback_url
view_states
             r   get_initial_statere   P   s    ;;sGR;8D99DH
		*D1A771:J
		+T2AWWQZ
bggfnnV-C-Cfmm-SWYZ[[IHO# 	 	$	
 	( 	6 	6 	F 	F 	4 	C 	3 	4 	3 	3  	3!O& L^+j8;;|WVX;YD99D%-II.5wwqzHII/6J/*<L&.o#
#{{<Z\{]yyJ
		EtLAWWQZ
IIFMJIIBDIJ	M)
%&	MZ*Sb/VDC
HI  r&   c                     [          SU SU 3nU R                  U[        SS9nUR                  5         UR                  $ )z?Try direct GET with procedureId parameter using active session.z?procedureId=z&Adf-Window-Id=r(   r)   )rO   rN   rP   rQ   rR   )r\   ra   procedure_id
detail_urlr]   s        r   try_direct_detailri      sB    5l^?9+NJ;;z7B;?D99r&   c                     0 SS_SS_SS_SS_SU_S	S_S
S_SS_SS_SS_SS_SS_SS_SS_SS_SS_SS_SUSUSSSSSS.	En[          SU S3nU R                  XT[        SS 9nUR                  $ )!zFPOST search filtered by specific procedure_id to get exactly 1 result.socLangr4   qryId1:oper0=qryId1:val00r2   qryId1:oper1qryId1:val10qryId1:oper2qryId1:val20r.   qryId1:oper3qryId1:val30qryId1:oper4qryId1:val40qryId1:oper5qryId1:val50qryId1:oper6CONTAINSqryId1:val60qryId1:oper7qryId1:val70f1z8{pc1:t1={viewportSize=26,rows=50},pc1:_clmCxt={_shown=}}zpc1:t1,qryId1z<m xmlns="http://oracle.com/richClient/comm"><k v="operation"><s>RESULT_COMPONENT_DECODE</s></k><k v="type"><s>queryInternal</s></k></m>z^<m xmlns="http://oracle.com/richClient/comm"><k v="clearAll"/><k v="type"><s>query</s></k></m>)	&org.apache.myfaces.trinidad.faces.FORMr5   Adf-Page-Idjavax.faces.ViewStatezoracle.adf.view.rich.DELTASeventzevent.pc1:t1zevent.qryId1oracle.adf.view.rich.PROCESS?Adf-Window-Id=z&Adf-Page-Id=0<   datar*   r+   rO   postAJAX_HEADERSrR   )r\   rd   ra   rg   	form_datapost_urlr]   s          r   post_filtered_searchr      s)   3 	 		
 	 	 	 	 	 	 	 	 	 	
 	  	
!" 	#$ 37"!+'a  cx(75I: oi[?H<<,PR<SD99r&   c                     0 SS_SS_SS_SS_SS	_S
S_SS	_SS_SS	_SS_SS	_SS_SS	_SS_SS	_SS_SS	_SSSUSSSUSUSU 3SSU0En[          S U S!U 3nU R                  Xe[        S"S#9nUR                  $ )$z8Click a row's procedure link to navigate to detail page.rk   r4   rl   rm   rn   r2   ro   rp   r.   rq   rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   r|   r}   r   r~   r5   r   r   r   zevent.zN<m xmlns="http://oracle.com/richClient/comm"><k v="type"><s>action</s></k></m>r   r   z&Adf-Page-Id=0&_afrRK=r   r   r   )r\   rd   ra   row_keylink_idr   r   r]   s           r   click_row_linkr      s;   3 	 		
 	 	 	 	 	 	 	 	 	 	
 	  	
!" 	#$ 	1$s
	l&1I6 oi[0FwiPH<<,PR<SD99r&   c                    [         R                  " SU [         R                  5      nU(       d  gUR                  S5      n[	        US5      nUR                  SSS0S9nU Hn  nUR                  S	5      nUc  M  UR                  S
SS9nU(       d  M2  US   R                  S5      nU(       d  MO  UR                  S5      (       d  Mg  US   U4s  $    g)zBExtract link ID and row key from ADF search response for clicking.z2<update id="pc1:t1"><!\[CDATA\[(.*?)\]\]></update>)NNr,   html.parsertrroler   )attrs_afrrktdgridcell)r   r   aid)rS   rT   DOTALLrU   r   find_allrN   find)	xml_texttable_match
table_htmlsouprowsr   afrrkcellslinks	            r   extract_link_infor      s    ))="))K ""1%J]3D==fe_=5D!=T
358==%DtDz5((  r&   c                 b    [         R                  " SU 5      nU(       a  UR                  S5      $ g)z,Extract updated ViewState from ADF response.zC<update id="javax\.faces\.ViewState"><!\[CDATA\[(.*?)\]\]></update>r,   N)rS   rT   rU   )r   vs_matchs     r   extract_new_view_stater   	  s-    yyNH ~~a  r&   c                 t   [        U S5      n0 nUR                  S5      nU Hj  nUR                  SS9nUR                  SS5      nU(       d  M-  UR	                  US9nU(       d  ME  UR                  SS9nU(       d  M]  U(       d  Mf  XU'   Ml     UR                  SSS9 H  n	U	R                  S	S5      n
S
U
;   d  M  U	R                  SS9nU(       d  M5  U	R                  S5      nU(       d  MO  UR                  S5      nU(       d  Mi  UR                  SS9R                  S5      nU(       d  M  XU'   M     U$ )u   Parse detail page HTML to extract tender detail fields.

This is a placeholder — after running discovery mode and inspecting
the debug HTML files, fill in the actual field extraction logic.
r   labelTstripforr.   r   spanr   z	::contentr   :)r   r   get_textrN   r   find_parentfind_previous_siblingrstrip)htmlr   r   labelsr   
label_text	label_for
value_elemvaluer   span_id	parent_tdkeys                r   parse_detail_pager     s+    }-DD ]]7#F^^$^/
IIeR(	9i0Jz"++$+7:%%',$  f.((4$'!MMM-Eu ,,T2	9%;;DAEu#nn4n8??D3(-I / Kr&   c                 ,   [         R                  R                  [         R                  R                  [        5      U5      n[        USSS9 nUR                  U 5        SSS5        [        SU S[        U 5      S S35        U$ ! , (       d  f       N,= f)	z'Save HTML to debug file for inspection.r-   r   r   Nz  Saved debug HTML to z (, chars))	ospathrV   dirname__file__openwriter   len)r   filename
debug_pathfs       r   save_debug_htmlr   <  sn    bggooh7BJ	j#	0A	 
1	"8*Bs4ym7
CD 
1	0s   B
Bc                 
    [        5       nUR                  5       nUR                  SU 45        UR                  5         UR	                  5         UR	                  5         g! [
         a  n[        SU 35         SnAgSnAff = f)z"Save detail data and set detail=1.z3UPDATE nepps SET detail = 1 WHERE procedure_id = %sTz    DB error: NF)r	   r   r   r   r   	Exceptionr   )rg   detail_datar   r   es        r   save_detailr   E  sp     "AO	
 	

 qc"#s   AA   
B*A==Bc                     U (       a  SU ;  a  g[        U S5      nUR                  [        R                  " S5      S9nUSL $ )z5Check if HTML is a detail page (not the search page).	ViewStateFr   zpc1:t1r   N)r   r   rS   compile)r   r   search_tables      r   is_detail_pager   W  s?    ;d*}-D99

9 596L4r&   c                    [        SU S35        [        S5         [        XU5      n[        US5        [        U5      (       a  [        S5        USU4$ [	        US5      nUR                  S5      n[        S	U(       a  UR                  S
S9OS 35        [        S5        [        R                  " SU5      nU(       a  UR                  S5      n[        S5         [        XX#5      n	[        U	S5        [        U	5      n
U
(       a  U
n[        U	5      u  p[        SU SU 35        U(       GaX  U(       GaP  [        S5        [        XX,U5      n[        US5        [        U5      n
U
(       a  U
n[        R                  " SU5      nU(       at  UR                  S5      nUR                  S5      (       d  SU-   n[        SU 35        U R!                  U["        SS9n[        UR$                  S5        UR$                  SU4$ [        S ['        U5      S! S"35        S#U;   ab  [        R(                  " S$U[        R*                  5      nU(       a5  S%R-                  U5      n[        US&5        [        S'['        U5       S(35        US)U4$ [        S*5         SSU4$ ! [         a  n[        SU 35         SnAGNSnAff = f! [         a  n[        SU 35         SnANHSnAff = f)+z?Discovery: try both approaches and save results for inspection.z&
--- Discovery Mode for procedure_id: z ---z.
[Approach 1] Direct GET with ?procedureId=...zdebug_direct_detail.htmlz  Result: Got detail page!directr   titlez  Page title: Tr   rM   z&  Result: Got search page (not detail)rK   r,   	  Error: Nz0
[Approach 2] Filtered search by procedure_id...zdebug_filtered_search.htmlz  Link ID: z, Row key: z  Clicking row link...zdebug_click_detail.html<redirect url="([^"]+)"httpr   z  Following redirect: r(   r)   zdebug_redirect_detail.htmlfiltered_redirectz  Got ADF response (r   r   CDATAz<!\[CDATA\[(.*?)\]\]>z
<!-- CDATA BLOCK -->
zdebug_cdata_blocks.htmlz  Found z CDATA blocksfiltered_clickz0  Could not find link to click in search results)r   ri   r   r   r   r   r   rS   rT   rU   r   r   r   r   r   
startswithrN   rP   rR   r   findallr   rV   )r\   rd   ra   rg   r   r   r   r_   r   search_respnew_vsr   r   detail_respredirect_matchredirect_url
redir_respcdata_blockscombineds                      r   discover_detail_pager   a  s   	3L>
FG 

;< \B89$./:-- }5DIIg&EN5>>>#=F"STU:; 		MtTAWWQZ

 

=>.*7	X%AB'4J,[9G9Ky9:w*+(iRYZKK)BC+K8F#
  YY'A;ON-33A6#..v66#MP\#\L.|n=>$[[wPR[S

1MN!(;ZGG([)9!(<GDE +%!zz*BKQSQZQZ[9>>|LH#H.GHHS%6$7}EF 0*<<DE z!!k  	!od  	!osD   7J A;J D#J< ?BJ< J< 
J9 J44J9<
KKKc                 >   [        S5        [        S5        [        S5        [        S5        [        5         [        U 5      u  p#[        SUS 35        [        S[        U5       S35        [        S5        U(       d  [        S5        g	[        S
5        [        R
                  " 5       n[        U5      u  pVU(       d  [        S5        g	[        S5        U(       Ga  US   nUS   n[        SU SUS   =(       d    SS	S  S35        [        XEXh5      u  pnU	(       a  [        S5        [        SU
 35        [        S[        U	5      S S35        [        U	5      nU(       a[  [        S[        U5       S35        [        UR                  5       5      S	S  H"  u  p[        SU S[        U5      S	S  35        M$     g	[        S5         g	[        S 5        [        S!5        [        S"5        g	SnSnSn[        US#5       GH  u  nnUS   nUS   n[        S$U S%[        U5       S&U 35        [        S'U=(       d    S(S	S)  35         S*nS	n	 [        XFU5      n	[        U	5      (       a  S+nO/[        R                   " S,U	5      nU(       a  UR#                  S#5      nU(       d   ['        XEXh5      n[)        U5      nU(       a  Un[+        U5      u  nnU(       a  U(       a  [-        XEUUU5      n[)        U5      nU(       a  Un[        R                   " S.U5      nU(       aP  UR#                  S#5      nUR/                  S/5      (       d  S0U-   nUR1                  U[2        S1S29nUR4                  n	S+nOUn	S+nO[        S35         U(       a^  U	(       aW  [        U	5      nU(       a  [        U5      OSn[        S5U S635        [7        X5      (       a  US#-  nSn[        S75        OUS#-  nO[        S85        US#-  nUS#-  n US::  a;  [        S;5         [        R
                  " 5       n[        U5      u  pVSn[        S<5        US>-  S:X  a:  [        S?S@ 35        [        SAU S%[        U5       SBU SCU 35        [        S@ 5        [8        R:                  " [<        5        GM     [        SD5        [        SE5        [        S5        [        SF[        U5       35        [        SGU 35        [        SHU 35        [        S5        g	! [$         a  n[        S-U 35         S	nAGNS	nAff = f! [$         a  n[        S4U 35         S	nAGNS	nAff = f! [$         a#  n[        S9U 35        US#-  nUS#-  n S	nAGNwS	nAff = f! [$         a  n[        S=U 35         S	nA  M  S	nAff = f)Iz'Main function to scrape tender details.zF======================================================================z&NEPPS (ESIDIS) Greece - Detail ScraperzChecking table schema...zTotal records to process: r   zProcessing z records...zNo pending records.Nz&
[Session] Establishing ADF session...z"ERROR: Could not extract ViewStatez  Session established.r   rg   z
Discovery target: z ('r   r.   r   z...')z
--- Discovery Result ---zMethod that worked: zResponse length: z charsz
Extracted z fields:   z  z: P   z6No fields extracted by parser (check debug HTML files)z
--- Discovery Failed ---z(Neither approach returned a detail page.z+Check debug HTML files in nepps/ directory.r,   z
[/z] z	  Title: zN/AF   FTrK   z  Direct GET failed: r   r   r   r(   r)   z*  No link found in filtered search resultsz  Filtered search failed: z
  Parsed: z fieldsz  Saved & detail set to 1z  Could not get detail pager   r   z
  Re-establishing session...z  Session refreshed.z  Session refresh failed: r/   
u   ──────────────────────────────────────────────────────────────────────z  Progress: z | Success: z | Errors: zG
======================================================================zScraping Complete!z  Total processed: z  Successful: z
  Errors: )r   r   r%   r   requestsSessionre   r   r   listitemsstr	enumerateri   r   rS   rT   rU   r   r   r   r   r   r   rN   rP   rR   r   timesleepDELAY)r"   	discoveryr$   r    r\   rd   ra   recordrg   r   methodr   r0   vsuccess_counterror_countconsecutive_errorsir   
got_detailr_   r   r   r   r   r   r   r   r   r   fields_founds                                  r   scrape_detailsr    s}   	(O	
23	(O	
$%(4NG	&uQi
01	KG~[
12	(O#$ 

34 G-g6J23	
"#n-$\N#vg7L"cr6R5SSXYZ#7Y#e j.0(12%c$i]&9:+D1K
3{#3"4H=> !2!2!45cr:DABqcCF3BK=12 ; 	 NO
 	 ./<=?@ MKw*	6n-wA3aG~R~67	5>E3B/012E	$JD
3(\J!$''!%J 		"UW[\A%&WWQZ

 <"6wI"dK3K@F%+
'8'E$GW7&4W)U\^e&f!7!D!)/J *,3M{)[)+9+?+?+BL#/#:#:6#B#B/Y\h/h)0\7\^)_J#-??D)-J#.D)-JJK d/53>s;/A
<.89|99!Q&M)*&571$K34q "a'" "23"**,(9'(B%
%&",-
 r6Q;Bzl#$L1S\N,}o[YdXefgZL"

5 +B 
/	
	(O	G~
./	N=/
*+	J{m
$%	(Oe  3-aS1223B ! <6qc:;;<&  	$IaS/"1K!#	$  21#67s   US<)U*/S<U!CT",U-T"1U2T"=AUU#U/U8<
TTUTU"
U,U :U UU
U5U00U58
VVV__main__zScrape NEPPS tender details)descriptionz--batch-sizezNumber of records to process)typedefaulthelpz--discovery
store_truez,Run in discovery mode (inspect first record))actionr  )r"   r   z

Interrupted by userz
Fatal Error: )N)NF)8r   bs4r   mysql.connectormysqlrS   r   sysr   rW   rY   argparseplatformstdoutreconfigurer   insertrV   r   r   	db_configr   r	   rO   rP   r   r   r   r%   re   ri   r   r   r   r   r   r   r   r   r   r  __name__ArgumentParserparseradd_argumentint
parse_argsargsr"   r   KeyboardInterruptr   r   r   	traceback	print_exc r&   r   <module>r#     s      	  
 	   <<7JJG, 277<< 94@ A 2X DO0 DFO8a 	
,(M!`!HD6%P$ L"^cL z$$1NOF
S$Ecd
lAopD$//T^^L   )'( s#$s   6E F#F( FF