
    Z j(                        S r SSKJr  SSKrSSKJr  SSKJr  SSKJr  SSK	J
r
  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJrJrJrJrJrJrJr  SSKJr  SSKJr  \R>                  " \ 5      r!Sr"Sr# " S S\RH                  5      r% " S S\5      r&S&S jr' " S S\RH                  5      r( " S S\5      r) " S S\5      r* " S S \5      r+ " S! S"\5      r, " S# S$\5      r-/ S%Qr.g)'zPyTorch Phi-3 model.    )CallableN)nn   )ACT2FN)Cache)GenerationMixin)FlashAttentionKwargs)ALL_ATTENTION_FUNCTIONS)Unpack)logging   )MistralDecoderLayerMistralForCausalLM MistralForSequenceClassificationMistralForTokenClassificationMistralPreTrainedModeleager_attention_forwardrotate_half)PhiRotaryEmbedding   )
Phi3Configz microsoft/Phi-3-mini-4k-instructr   c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )Phi3MLP0   c                    > [         TU ]  5         Xl        [        R                  " UR
                  SUR                  -  SS9U l        [        R                  " UR                  UR
                  SS9U l        [        UR                     U l        g )Nr   Fbias)super__init__configr   Linearhidden_sizeintermediate_sizegate_up_proj	down_projr   
hidden_actactivation_fn)selfr    	__class__s     v/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/phi3/modular_phi3.pyr   Phi3MLP.__init__1   sn    IIf&8&8!f>V>V:V]bc6#;#;V=O=OV[\#F$5$56    hidden_statesreturnc                     U R                  U5      nUR                  SSS9u  p2X R                  U5      -  nU R                  U5      $ )Nr   dim)r$   chunkr'   r%   )r(   r-   	up_statesgates       r*   forwardPhi3MLP.forward9   sH    %%m4	#//!/4 2 24 88	~~i((r,   )r'   r    r%   r$   )
__name__
__module____qualname____firstlineno__r   torchFloatTensorr6   __static_attributes____classcell__r)   s   @r*   r   r   0   s,    7)U%6%6 )5;L;L ) )r,   r   c                       \ rS rSrSrg)Phi3RotaryEmbeddingB    Nr8   r9   r:   r;   r>   rD   r,   r*   rB   rB   B       r,   rB   c                 N   UR                  U5      nUR                  U5      nUR                  S   nU SSU24   U SUS24   pvUSSU24   USUS24   p[        R                  " Xb-  [	        U5      U-  -   U/SS9n
[        R                  " X-  [	        U5      U-  -   U	/SS9nX4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
r0   .Nr1   )	unsqueezeshaper<   catr   )qkcossinunsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embeds               r*   apply_rotary_pos_embrW   F   s    $ --
&C
--
&C2Jc;J;&'3
+;)<6c;J;&'3
+;)<6ii%++e*<s*BCVLRTUGii%++e*<s*BCVLRTUGr,   c                   0  ^  \ rS rSrSrSS\S\S-  4U 4S jjjr SS\R                  S\
\R                  \R                  4   S	\R                  S-  S
\S-  S\\   S\
\R                  \R                  S-  \
\R                     S-  4   4S jjrSrU =r$ )Phi3Attentiond   z=Multi-headed attention from 'Attention Is All You Need' paperNr    	layer_idxc                 p  > [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        UR                  U l        U R                  S-  U l
        UR                  U l        SU l        UR                  U R                  -  SUR                  U R                  -  -  -   n[        R                  " UR                  U R                  -  UR
                  SS9U l        [        R                  " UR
                  USS9U l        g )Nhead_dimg      Tr   Fr   )r   r   r    r[   getattrr"   num_attention_headsr]   num_key_value_headsnum_key_value_groupsscalingattention_dropout	is_causalr   r!   o_projqkv_proj)r(   r    r[   op_sizer)   s       r*   r   Phi3Attention.__init__g   s    "
F4F4F&JdJd4de$*$>$>&B\B\$\!#)#=#= }}d*!'!9!9,,t}}<qFD^D^aeananDn?ooii : :T]] JFL^L^ejk		&"4"4gEJr,   r-   position_embeddingsattention_maskpast_key_valueskwargsr.   c           
         UR                   S S n/ UQSPU R                  P7nU R                  U5      nU R                  R                  U R                  -  n	USS U	24   n
USXU R
                  U R                  -  -   24   nUSXR
                  U R                  -  -   S 24   nU
R                  U5      R                  SS5      n
UR                  U5      R                  SS5      nUR                  U5      R                  SS5      nUu  p[        XX5      u  pUb  UR                  XU R                  5      u  p[        R                  " U R                  R                  [        5      nU" U U
UUU4U R                  (       d  SOU R                   U R"                  [%        U R                  SS 5      S.UD6u  nnUR&                  " / UQSP76 R)                  5       nU R+                  U5      nUU4$ )Nr0   .r   r   g        sliding_window)dropoutrb   rn   )rI   r]   rf   r    r_   r`   view	transposerW   updater[   r
   get_interface_attn_implementationr   trainingrc   rb   r^   reshape
contiguousre   )r(   r-   ri   rj   rk   rl   input_shapehidden_shapeqkv	query_posquery_states
key_statesvalue_statesrM   rN   attention_interfaceattn_outputattn_weightss                     r*   r6   Phi3Attention.forwardv   s    $))#2.88b8$--8mmM*KK33dmmC	3

?+id6N6NQUQ^Q^6^*^^^_
3	,D,Dt}},T T VVW#((6@@AF__\2<<QB
#((6@@AF&#7RU#[ &'6'='=jX\XfXf'g$J(?(M(MKK,,.E)
 %8
%
  $}}C$2H2HLL"4;;0@$G
%
 
%
!\ "));;;;FFHkk+.L((r,   )
rc   r    r]   rd   r[   ra   r`   re   rf   rb   )N)r8   r9   r:   r;   __doc__r   intr   r<   Tensortupler   r   r	   r6   r>   r?   r@   s   @r*   rY   rY   d   s    GKz KcDj K K( )--)||-) #5<<#=>-) t+	-)
 -) -.-) 
u||U\\D0%2E2LL	M-) -)r,   rY   c                   T  ^  \ rS rSrS\S\4U 4S jjr     SS\R                  S\R                  S-  S\R                  S-  S	\
S-  S
\S-  S\\R                  \R                  4   S-  S\\   S\\R                  \\R                  \R                  4   S-  4   4S jjrSrU =r$ )Phi3DecoderLayer   r    r[   c                    > [         TU ]  X5        Xl        [        XS9U l        [        U5      U l        [        R                  " UR                  5      U l
        [        R                  " UR                  5      U l        g )N)r    r[   )r   r   r    rY   	self_attnr   mlpr   Dropoutresid_pdropresid_attn_dropoutresid_mlp_dropout)r(   r    r[   r)   s      r*   r   Phi3DecoderLayer.__init__   sZ    +&fJ6?"$**V-?-?"@!#F,>,>!?r,   Nr-   rj   position_idsrk   	use_cacheri   rl   r.   c           
          UnU R                  U5      nU R                  " SUUUUUUS.UD6u  pXR                  U5      -   nUnU R                  U5      nU R	                  U5      nXR                  U5      -   nU$ )N)r-   rj   r   rk   r   ri   rD   )input_layernormr   r   post_attention_layernormr   r   )
r(   r-   rj   r   rk   r   ri   rl   residualself_attn_weightss
             r*   r6   Phi3DecoderLayer.forward   s     !,,];+/>> ,
')%+ 3,
 ,
( !#:#:=#II 55mD/ #9#9-#HHr,   )r    r   r   r   r   )NNNFN)r8   r9   r:   r;   r   r   r   r<   r   
LongTensorr   boolr   r   r	   r=   r6   r>   r?   r@   s   @r*   r   r      s    @z @c @ /304(,!&HL|| t+ &&-	
  $; #5<<#=>E -. 
u  %(9(95;L;L(L"MPT"TT	U r,   r   c                       \ rS rSrSrSrg)Phi3PreTrainedModel   z0.0.5rD   N)r8   r9   r:   r;   _versionr>   rD   r,   r*   r   r      s    Hr,   r   c                   *    \ rS rSr      SS jrSrg)Phi3ForCausalLM   Nc                 2   U(       ap  [        U R                  S5      (       aU  UR                  S   U R                  R                  S-   :  a+  UR	                  5       n	XR                  R                  ::  a  S n[
        R                  " U 4UUUUUUUS.UD6n
U
$ )N original_max_position_embeddingsr   )	input_idsrk   rj   inputs_embedsr   r   logits_to_keep)hasattrr    rI   r   get_seq_lengthr   prepare_inputs_for_generation)r(   r   rk   rj   r   r   r   r   rl   past_lengthmodel_inputss              r*   r   -Phi3ForCausalLM.prepare_inputs_for_generation   s    " %GHH"dkk&R&RUV&VV)88:KkkJJJ"&&DD

+)'%)

 

 r,   rD   )NNNNTN)r8   r9   r:   r;   r   r>   rD   r,   r*   r   r      s     $r,   r   c                       \ rS rSrSrg)Phi3ForSequenceClassification   rD   NrE   rD   r,   r*   r   r      rF   r,   r   c                       \ rS rSrSrg)Phi3ForTokenClassification   rD   NrE   rD   r,   r*   r   r      rF   r,   r   )r   	Phi3Modelr   r   r   )r   )/r   collections.abcr   r<   r   activationsr   cache_utilsr   
generationr   modeling_flash_attention_utilsr	   modeling_utilsr
   processing_utilsr   utilsr   mistral.modeling_mistralr   r   r   r   r   r   r   phi.modeling_phir   configuration_phi3r   
get_loggerr8   logger_CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCModuler   rB   rW   rY   r   r   r   r   r   __all__rD   r,   r*   <module>r      s     $   !   ) B 5 &    2 * 
		H	%8 )bii )$	, 	<?)BII ?)D%* %P0 %( %P	$D 		!> 	r,   