
    >i-                    d   d Z ddlmZ ddlZddlZddlZddlmZ ddlm	Z	m
Z
mZmZmZmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ e	rddlmZ  ej6                  e      Z edd      Z G d d      Z	 	 	 	 	 	 	 	 ddZ 	 	 	 	 	 	 ddZ!	 	 	 	 	 	 ddZ"	 	 	 	 	 	 ddZ#ddZ$ddZ%ddZ&ddZ'y)zDExtraction engine that orchestrates DOM querying and model building.    )annotationsN)	Coroutine)TYPE_CHECKINGOptionalTypeVarUnionget_args
get_origin)FindElementsMixin)
WebElement)FieldExtractionFailed)ExtractionMetadata)ExtractionModel)TabTr   )boundc                      e Zd ZdZddZddd	 	 	 	 	 	 	 ddZddd	 	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 dd	Z	 	 	 	 	 	 	 	 	 	 dd
Z	 	 	 	 	 	 	 	 	 	 ddZ		 	 	 	 	 	 	 	 	 	 ddZ
y)ExtractionEnginezOrchestrates extraction by querying the DOM and building model instances.

    Internal engine used by Tab.extract() and Tab.extract_all().
    Users do not interact with it directly.
    c                    || _         y N)_tab)selftabs     ]/var/www/python/tenders/Scrapper/venv/lib/python3.12/site-packages/pydoll/extractor/engine.py__init__zExtractionEngine.__init__    s	    	    Nr   )scopetimeoutc                 K   | j                   }|Q| j                   j                  ||       d{   }t        |t              st	        d| dt        |             |}| j                  |||       d{   }t        ||      S 7 W7 w)a  Extract a single model instance from the page.

        Args:
            model: ExtractionModel subclass to populate.
            scope: Optional CSS/XPath selector to limit extraction region.
            timeout: Seconds to wait for elements to appear (0 = no wait).

        Returns:
            Populated model instance.

        Raises:
            FieldExtractionFailed: If a required field cannot be extracted.
        N)r   z%Expected a single element for scope "", got )r   query
isinstancer   
ValueErrortype_extract_fields_build_instance)r   modelr   r   contextresultvaluess          r   extractzExtractionEngine.extract#   s     ( &*YY99??5'?BBFfj1 ;E7'$v,X  G++E7GDDuf-- C Es"   /BB	AB8B9BB)r   limitc               `  K   | j                   j                  |d|d       d{   }||sg S t        |t              r|n|g}||d| }|D cg c]  }| j	                  |||       }}t        j                  |  d{   }	|	D 
cg c]  }
t        ||
       c}
S 7 zc c}w 7 %c c}
w w)a  Extract multiple model instances from repeated containers.

        Each element matching scope generates one model instance.

        Args:
            model: ExtractionModel subclass to populate.
            scope: CSS/XPath selector for the repeated container (required).
            timeout: Seconds to wait for elements to appear (0 = no wait).
            limit: Maximum number of items to extract (None = all).

        Returns:
            List of populated model instances.
        TFfind_allr   	raise_excN)r   r!   r"   listr%   asynciogatherr&   )r   r'   r   r   r,   found
containers	containerextraction_tasks
all_valuesr*   s              r   extract_allzExtractionEngine.extract_allC   s     * iiooedGW\o]]=I0:5$0GueW
#FU+J NX
@ID  	7;
 
 #>>+;<<
=GH6v.HH ^
 =Hs?   #B.B *B.B"*B.B'B.
B)B."B.)B.c           	       K   g }g }|j                         j                         D ]}  \  }}|j                  st        j	                  d| d       ,|j
                  |   }|j                  }	|	J|j                  |       |j                  | j                  ||	||              t        j                  |ddi d{   }
i }t        ||
      D ]g  \  }}t        |t              rM|j
                  |   }|j                         st        j	                  d| d|        Qt        d| d	|       ||||<   i |S 7 ~w)
a  Extract all fields from the DOM concurrently.

        Launches all field extractions in parallel using asyncio.gather,
        then collects results and handles errors per field.

        Args:
            model: ExtractionModel subclass with extraction fields.
            context: Tab or WebElement to scope queries within.
            timeout: Seconds to wait for each element to appear.

        Returns:
            Dictionary of field name -> extracted value.
        zSkipping field "z" (no selector)Nreturn_exceptionsTzOptional field "z" extraction failed: zRequired field "z" could not be extracted: )get_extraction_fieldsitemshas_selectorloggerdebugmodel_fields
annotationappend_extract_fieldr2   r3   zipr"   BaseExceptionis_requiredr   )r   r'   r(   r   field_names
coroutinesnamemetadata
field_inforB   resultsr*   r)   s                r   r%   z ExtractionEngine._extract_fieldsg   sh    & "$  	 $99;AAC 	[ND(((/v_EF++D1J#..J!t$d11(JQXYZ	[  
KdKKMOW5 		"LD&&-0"//5
!--/LL#3D69Nvh!WX+&tf,FvhO "F4L		"  Ls   B;D>=D<>A?D>c                   K   t        |      }t        |      r| j                  ||||       d{   S t        |      r| j	                  ||||       d{   S t        |||       d{   S 7 @7 7 w)a  Extract a single field value from the DOM.

        Handles scalar types, list types, nested ExtractionModel,
        and list[ExtractionModel].

        Args:
            metadata: Extraction metadata with selector/attribute/transform.
            annotation: The field's resolved type annotation.
            context: Tab or WebElement to query within.
            timeout: Seconds to wait for the element to appear.

        Returns:
            Extracted and optionally transformed value.
        N)_unwrap_optional_is_list_type_extract_list_field_is_extraction_model_extract_nested_model_extract_scalar_field)r   rK   rB   r(   r   	unwrappeds         r   rD   zExtractionEngine._extract_field   s{     * %Z0	#11(IwPWXXX	*33HiRYZZZ*8WgFFF Y [Fs3   .A7A1&A7A3A7,A5-A73A75A7c                   K   j                   }|g S |j                  |dd       d{   }||sg S t        |t              r|n|g}t	        |      t              r?t        j                   fd|D          d{   }|D 	cg c]  }	t        |	       c}	S t        j                  fd|D          d{   }
|
D cg c]  }t        |       c}S 7 7 ]c c}	w 7 'c c}w w)z9Extract a list of values from multiple matching elements.NTFr.   c              3  D   K   | ]  }j                  |        y wr   )r%   ).0el
inner_typer   r   s     r   	<genexpr>z7ExtractionEngine._extract_list_field.<locals>.<genexpr>   s      SB$&&z2w?Ss    c              3  6   K   | ]  }t        |        y wr   )_extract_value)rX   rY   rK   s     r   r[   z7ExtractionEngine._extract_list_field.<locals>.<genexpr>   s     (Y"H)E(Ys   )
selectorr!   r"   r1   _get_inner_typerR   r2   r3   r&   _apply_transform)r   rK   rB   r(   r   r^   r4   elementsall_field_valuesfvall_rawrawrZ   s   ``  `       @r   rQ   z$ExtractionEngine._extract_list_field   s      $$ImmHtWX]m^^=I.8.EUE7$Z0

+%,^^S(S&   ?OOOJ3OO(YPX(YZZ;BCC h/CC _  PZCsR   )C1C!AC1C#C1C%!!C1C*C1C,C1#C1%C1,C1c                  K   |j                   }|t        d      |j                  ||d       d{   }t        |t              st        d| dt        |             | j                  |||       d{   }t        ||      S 7 U7 w)zDExtract a nested ExtractionModel by scoping to the selector element.Nz"Nested model field has no selectorTr   r0   Expected a single element for "r    )	r^   r   r!   r"   r   r#   r$   r%   r&   )r   rK   r'   r(   r   r^   r)   r*   s           r   rS   z&ExtractionEngine._extract_nested_model   s      $$'(LMM}}Xw$}OO&*->xjPTU[P\~^__++E67CCuf--	 P Ds"   1BB	AB8B9BB)r   r   returnNone)r'   type[T]r   zOptional[str]r   intri   r   )
r'   rk   r   strr   rl   r,   zOptional[int]ri   zlist[T])r'   rk   r(   r   r   rl   ri   :dict[str, Union[str, int, float, bool, list[str], object]])
rK   r   rB   r$   r(   r   r   rl   ri   z/Union[str, int, float, bool, list[str], object])
rK   r   rB   r$   r(   r   r   rl   ri   z*list[Union[str, int, float, bool, object]])
rK   r   r'   rk   r(   r   r   rl   ri   r   )__name__
__module____qualname____doc__r   r+   r9   r%   rD   rQ   rS    r   r   r   r      s\     $.. 	.
 . 
.J #"I"I 	"I
 "I "I 
"IH33 #3 	3
 
D3jG$G G #	G
 G 
9G>D$D D #	D
 D 
4D8.$. . #	.
 . 
.r   r   c                  K   | j                   }|t        d      |j                  ||d       d{   }t        |t              st        d| dt        |             t        ||        d{   }t        ||       S 7 N7 w)z+Extract a single scalar value from the DOM.NzScalar field has no selectorTrg   rh   r    )	r^   r   r!   r"   r   r#   r$   r]   r`   )rK   r(   r   r^   r)   re   s         r   rT   rT      s        H#$BCC==7d=KKFfj):8*GDQWL>Z[[vx0
0CC**	 L 1s!   1BB=B1B2BBc                   K   |j                   | j                  |j                         xs dS | j                   d{   S 7 w)aF  Read raw string value from a WebElement.

    If metadata.attribute is set, reads that HTML attribute.
    Otherwise reads element.text (innerText).

    Args:
        element: WebElement to read from.
        metadata: Field metadata with optional attribute name.

    Returns:
        Raw string value before transform.
    N )	attributeget_attributetext)elementrK   s     r   r]   r]      s@       %$$X%7%78>B>s   ;AAAc                @    |j                   |j                  |       S | S )zApply metadata.transform to the raw extracted string.

    Args:
        raw: Raw string from the DOM.
        metadata: Field metadata with optional transform callable.

    Returns:
        Transformed value, or raw string if no transform.
    )	transform)re   rK   s     r   r`   r`     s&     %!!#&&Jr   c                n    	  | di |S # t         $ r!}t        d| j                   d|       |d}~ww xY w)aJ  Build model instance from extracted values.

    Pydantic handles validation, type coercion, and defaults.

    Args:
        model: ExtractionModel subclass.
        values: Field name -> value mapping.

    Returns:
        Populated model instance.

    Raises:
        FieldExtractionFailed: If pydantic validation fails.
    zFailed to build z: Nrs   )	Exceptionr   ro   )r'   r*   excs      r   r&   r&   "  sG    $Yv Y#&6u~~6Fb$NOUXXYs   
 	4/4c                    t        |       }|t        u st        | t        j                        r;t        |       }|D cg c]  }|t        d      us| }}t        |      dk(  r|d   S | S c c}w )zUnwrap Optional[X] or X | None to X. Returns annotation unchanged otherwise.

    Handles both typing.Optional (Union) and PEP 604 syntax (types.UnionType).
    N   r   )r
   r   r"   types	UnionTyper	   r$   len)rB   originargsanon_nones        r   rO   rO   :  sk    
 
#F*ZA
##;!qT
':A;;x=AA; <s   A+A+c                $    t        |       t        u S )zCheck if annotation is list[X].)r
   r1   rB   s    r   rP   rP   H  s    j!T))r   c                2    t        |       }|r|d   S t        S )zGet X from list[X].r   )r	   rm   )rB   r   s     r   r_   r_   M  s    JDAwJr   c                f    	 t        | t              xr t        | t              S # t        $ r Y yw xY w)z3Check if annotation is an ExtractionModel subclass.F)r"   r$   
issubclassr   	TypeErrorr   s    r   rR   rR   U  s2    *d+W
:0WW s   !$ 	00)rK   r   r(   r   r   rl   ri   $Union[str, int, float, bool, object])rz   r   rK   r   ri   rm   )re   rm   rK   r   ri   r   )r'   rk   r*   rn   ri   r   )rB   r$   ri   r$   )rB   r$   ri   bool)(rr   
__future__r   r2   loggingr   collections.abcr   typingr   r   r   r   r	   r
   *pydoll.elements.mixins.find_elements_mixinr   pydoll.elements.web_elementr   pydoll.extractor.exceptionsr   pydoll.extractor.fieldr   pydoll.extractor.modelr   pydoll.browser.tabr   	getLoggerro   r?   r   r   rT   r]   r`   r&   rO   rP   r_   rR   rs   r   r   <module>r      s    J "    % P P H 2 = 5 2&			8	$C()N. N.b+ ++ + *	+"  	*	  *$YYFY Y0*
r   