
    Z j                        S SK r SSKJrJrJr  SSKJr  \R                  " \5      r	\" SSS9r
\" SSS9r\" 5       r\" 5       rS	\ R                  S
\S\ R                  4S jrS\ R                  S-  S\ R                  S\4S jr   SS\ R&                  R(                  S\ R                  S\ R                  S\ R                  S\ R                  S-  S\S\S-  S\S-  S\\ R                  S4   4S jjrg)    N   )is_torch_npu_availableis_torch_xpu_availablelogging)is_torch_greater_or_equalz2.5T)
accept_devz2.8hidden_statesn_repreturnc                     U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )z
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
   N)shapeexpandreshape)r	   r
   batchnum_key_value_headsslenhead_dims         y/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/integrations/sdpa_attention.py	repeat_kvr      s_    
 2?1D1D.Ez!!Qa"23::5W\dlmM  e(CTTT    attention_maskkeyc                 F    [         (       a  [        $ [        =(       a    U S L $ )N)_is_torch_xpu_available#_is_torch_greater_or_equal_than_2_8#_is_torch_greater_or_equal_than_2_5)r   r   s     r   use_gqa_in_sdpar      s      22.I>T3IIr   modulequeryvaluedropoutscaling	is_causalc                    UR                  SS5      (       a  [        R                  S5        0 n	[        U S5      (       a?  [	        XB5      (       d+  [        X R                  5      n[        X0R                  5      nOSS0n	Ub  UO[        U SS5      nUR                  S   S	:  =(       a    US L =(       a    Un[        R                  R                  5       (       a/  [        U[        R                  5      (       a  UR                  5       n[        (       a^  Ub[  UR                   [        R"                  :w  a=  [        R$                  " UR#                  5       5      R'                  UR(                  5      n[        R*                  R,                  R.                  " UUU4UUUUS
.U	D6n
U
R1                  S	S5      R3                  5       n
U
S 4$ )Noutput_attentionsFz`sdpa` attention does not support `output_attentions=True`. Please set your attention to `eager` if you want any of these features.num_key_value_groups
enable_gqaTr$   r   r   )	attn_mask	dropout_pscaler$   )getloggerwarning_oncehasattrr   r   r'   getattrr   torchjit
is_tracing
isinstanceTensoritem_is_torch_npu_availabledtypeboollogical_nottodevicenn
functionalscaled_dot_product_attention	transpose
contiguous)r   r    r   r!   r   r"   r#   r$   kwargssdpa_kwargsattn_outputs              r   sdpa_attention_forwardrE   (   s    zz%u--W	
 Kv-..~33C!<!<=Ce%@%@AE'.K '2	UY8ZI A"K~'=K)I yy*Y"E"ENN$	
 %.*>*>%***L"..~/B/B/DEHHVN((%%BB	 !	 	K ''1-88:Kr   )g        NN)r1   utilsr   r   r   utils.import_utilsr   
get_logger__name__r-   r   r   r   r7   r5   intr   r9   r   r=   ModulefloattuplerE    r   r   <module>rO      sH    K K : 
		H	% '@RV&W #&?RV&W #02 02 	UU\\ 	U# 	U%,, 	U	JELL4$7 	Jell 	Jt 	J$  !@HHOO@<<@ 
@ <<	@
 LL4'@ @ T\@ d{@ 5<<@r   