
    Z j                        S SK Jr  S SKrS SKJr  SSKJr  SSKJrJ	r	  SSK
Jr  SSKJrJr  SS	KJrJrJrJr  SS
KJr  SSKJrJrJr  SSKJr  SSKJrJr  SSKJ r J!r!J"r"J#r#J$r$J%r%J&r&J'r'J(r(J)r)  SSK*J+r+J,r,J-r-J.r.J/r/J0r0  SSK1J2r2J3r3  \Rh                  " \55      r6 " S S\!5      r7 " S S\'5      r8 " S S\(5      r9 " S S\)5      r: " S S\ 5      r; " S S\#5      r< " S S \%5      r= " S! S"\$5      r> " S# S$\"5      r? " S% S&\&5      r@\ " S' S(\5      5       rA " S) S*\A5      rB " S+ S,\R                  5      rD " S- S.\R                  5      rE " S/ S0\R                  5      rF\\ " S1 S2\5      5       5       rG " S3 S4\R                  5      rH " S5 S6\R                  5      rI " S7 S8\/5      rJ " S9 S:\05      rK " S; S<\-5      rL " S= S>\+5      rM " S? S@\,5      rN " SA SB\.5      rO " SC SD\O5      rP " SE SF\O\5      rQ/ SGQrRg)H    )	dataclassN)nn   )initialization)CacheDynamicCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)BaseModelOutputWithPast,BaseModelOutputWithPoolingAndCrossAttentionsCausalLMOutputWithPastModelOutput)PreTrainedModel)auto_docstringcan_return_tuplelogging)merge_with_config_defaults)OutputRecordercapture_outputs   )
EsmAttentionEsmEmbeddings
EsmEncoderEsmIntermediateEsmLayer	EsmOutput	EsmPoolerEsmRotaryEmbeddingEsmSelfAttentionEsmSelfOutput)LlamaAttentionLlamaDecoderLayerLlamaMLPLlamaPreTrainedModelLlamaRMSNormLlamaRotaryEmbedding   )EvollaConfigSaProtConfigc                   (   ^  \ rS rSrU 4S jrSrU =r$ )EvollaSaProtEmbeddings@   c                 2   > [         TU ]  U5        S U l        g N)super__init__position_idsselfconfig	__class__s     z/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/evolla/modular_evolla.pyr1   EvollaSaProtEmbeddings.__init__A   s          r2   )__name__
__module____qualname____firstlineno__r1   __static_attributes____classcell__r6   s   @r7   r,   r,   @   s    ! !r9   r,   c                   ~   ^  \ rS rSrSS\4U 4S jjjr\   SS\S-  SSS\S-  S\S	\	4   4U 4S
 jjj5       r
SrU =r$ )EvollaSaProtRotaryEmbeddingG   Nr5   c                 $   > [         TU ]  X5        g r/   )r0   r1   )r4   r5   devicer6   s      r7   r1   $EvollaSaProtRotaryEmbedding.__init__H   s    (r9   rF   ztorch.device | Noneseq_lenreturnztorch.Tensorc                 $   > [         TU ]  XU5      $ r/   )r0   compute_default_rope_parameters)r5   rF   rH   r6   s      r7   rK   ;EvollaSaProtRotaryEmbedding.compute_default_rope_parametersK   s     w6vwOOr9    r/   NNN)r;   r<   r=   r>   r*   r1   staticmethodinttuplefloatrK   r?   r@   rA   s   @r7   rC   rC   G   sn    )| ) ) &*(,"Pt#P%P tP 
~u$	%	P Pr9   rC   c                       \ rS rSrSrg)EvollaSaProtSelfAttentionT   rM   Nr;   r<   r=   r>   r?   rM   r9   r7   rT   rT   T       r9   rT   c                       \ rS rSrSrg)EvollaSaProtSelfOutputX   rM   NrV   rM   r9   r7   rY   rY   X   rW   r9   rY   c                       \ rS rSrSrg)EvollaSaProtAttention\   rM   NrV   rM   r9   r7   r\   r\   \   rW   r9   r\   c                       \ rS rSrSrg)EvollaSaProtIntermediate`   rM   NrV   rM   r9   r7   r_   r_   `   rW   r9   r_   c                       \ rS rSrSrg)EvollaSaProtOutputd   rM   NrV   rM   r9   r7   rb   rb   d   rW   r9   rb   c                       \ rS rSrSrg)EvollaSaProtLayerh   rM   NrV   rM   r9   r7   re   re   h   rW   r9   re   c                       \ rS rSrSrg)EvollaSaProtEncoderl   rM   NrV   rM   r9   r7   rh   rh   l   rW   r9   rh   c                       \ rS rSrSrg)EvollaSaProtPoolerp   rM   NrV   rM   r9   r7   rk   rk   p   rW   r9   rk   c                      ^  \ rS rSr% \\S'   S/rSrSrSr	Sr
\\" \SSS9/\" \SSS9/S	.r\R                   " 5       U 4S
 j5       rSrU =r$ )EvollaSaProtPreTrainedModelt   r5   re   Tr(   	attention)index
layer_namecrossattention)hidden_states
attentionscross_attentionsc                    > [         TU ]  U5        [        U[        5      (       a?  UR	                  UR
                  5      u  p#[        R                  " [        US5      U5        g g )Ninv_freq)	r0   _init_weights
isinstancerC   rK   r5   initcopy_getattr)r4   modulecurr_inv_freq_r6   s       r7   ry   )EvollaSaProtPreTrainedModel._init_weights   sO    f%f9::%EEfmmTMJJwvz2MB ;r9   rM   )r;   r<   r=   r>   r*   __annotations___no_split_modules_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendre   r   rT   _can_record_outputstorchno_gradry   r?   r@   rA   s   @r7   rn   rn   t   sv    ,-N"& +%&?qU`ab4AJZ[
 ]]_C Cr9   rn   c            
          ^  \ rS rSrS\4U 4S jjrS rS r\\	 SS\
R                  S-  S\
R                  S-  S	\\
R                     \-  4S
 jj5       5       rSrU =r$ )EvollaSaProtProteinEncoder   r5   c                    > [         TU ]  U5        [        U5      U l        [	        US9U l        [        U5      U l        U R                  5         g Nr5   )	r0   r1   r,   
embeddingsrC   rotary_embeddingsrh   encoder	post_initr3   s     r7   r1   #EvollaSaProtProteinEncoder.__init__   s?     08!<F!K*62r9   c                 .    U R                   R                  $ r/   r   word_embeddingsr4   s    r7   get_input_embeddings/EvollaSaProtProteinEncoder.get_input_embeddings   s    ...r9   c                 $    XR                   l        g r/   r   r4   values     r7   set_input_embeddings/EvollaSaProtProteinEncoder.set_input_embeddings   s    */'r9   N	input_idsattention_maskrI   c                    UR                  5       nUu  pVUR                  nUc  [        R                  " XV4US9nU R	                  XS9n[        U R                  UUS9n[        R                  " XgS9R                  S5      n	U R                  X5      n
U R                  " U4X*S.UD6nUS   n[        UUR                  UR                  UR                  S9$ )NrF   r   r   )r5   inputs_embedsr   r   )r   position_embeddings)last_hidden_statert   ru   rv   )sizerF   r   onesr   r
   r5   arange	unsqueezer   r   r   rt   ru   rv   )r4   r   r   kwargsinput_shape
batch_size
seq_lengthrF   r   r2   r   encoder_outputssequence_outputs                r7   forward"EvollaSaProtProteinEncoder.forward   s      nn&!,
!!!"ZZ*)A6RN)[2;;')
 ||J>HHK"44]Q,,
*8
ek
 *!,;-)77&11,==	
 	
r9   )r   r   r   r/   )r;   r<   r=   r>   r*   r1   r   r   r   r   r   TensorrQ   r   r   r?   r@   rA   s   @r7   r   r      sw    | /0   /3!
<<$&!
 t+!

 
u||	K	K!
   !
r9   r   c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )!EvollaSequenceCompressorAttention   c                 X  > [         TU ]  5         US-  U l        X0l        X#-  n[        R
                  " U5      U l        [        R
                  " U5      U l        [        R                  " XSS9U l	        [        R                  " XS-  SS9U l
        [        R                  " XASS9U l        g )N      Fbiasr   )r0   r1   scaleheadsr   	LayerNorm
norm_medianorm_latentsLinearto_qto_kvto_out)r4   dimdim_headr   	inner_dimr6   s        r7   r1   *EvollaSequenceCompressorAttention.__init__   s    t^

$	,,s+LL-IIc59	YYsM>
ii	U;r9   c                 &   U R                  U5      nU R                  U5      nU R                  nU R                  U5      n[        R
                  " X4SS9nU R                  U5      R                  SSS9u  pxUR                  UR                  S5      UR                  S5      US5      R                  SSSS5      nUR                  UR                  S5      UR                  S5      US5      R                  SSSS5      nUR                  UR                  S5      UR                  S5      US5      R                  SSSS5      nXPR                  -  n[        R                  " XWR                  SS5      5      n	XR                  SSS	9R                  5       -
  n	U	R                   u  pp[        R"                  " X5      R%                  UR&                  5      nUS
S
2S
S
S
S
24   nUS
S
S
2S
S
2S
4   nUU-  nU	R)                  SU-
  R+                  5       S5      n	U	R-                  SS9n[        R                  " UU5      nUR                  SSSS5      nUR/                  UR                  S5      UR                  S5      S5      nU R1                  U5      $ )z
Args:
    x (torch.Tensor): image features
        shape (b, n1, D)
    latent (torch.Tensor): latent features
        shape (b, n2, D);  n2: num of latent tokens
r   r   r   r(   r   Tr   keepdimNg     )r   r   r   r   r   catr   chunkviewr   permuter   matmul	transposeamaxdetachshaper   torF   masked_fillboolsoftmaxreshaper   )r4   xlatentsmaskhqkv_inputkvsimbsnhskdokdr   mask_expones_expattnouts                      r7   r   )EvollaSequenceCompressorAttention.forward   s2    OOA##G,JJIIg99a\r2zz(#))2 * 
 FF166!9affQiB/771aCFF166!9affQiB/771aCFF166!9affQiB/771aC

N ll1kk"b12HHTH299;;99zz""%%dkk24q()aD()("ooq4xoo/6{{r{"ll4#kk!Q1% kk#((1+sxx{B7{{3r9   )r   r   r   r   r   r   r   )r-      r;   r<   r=   r>   r1   r   r?   r@   rA   s   @r7   r   r      s    <)  ) r9   r   c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )EvollaFeedForward   c                   > [         TU ]  5         [        X-  5      n[        R                  " U5      U l        [        R                  " XSS9U l        [        R                  " 5       U l	        [        R                  " X1SS9U l
        g NFr   )r0   r1   rP   r   r   normr   fc1GELU
activationfc2)r4   r   multr   r6   s       r7   r1   EvollaFeedForward.__init__   sZ    
O	LL%	99S%8'')99Y%8r9   c           	      ~    U R                  U R                  U R                  U R                  U5      5      5      5      $ r/   )r   r   r   r   )r4   r   s     r7   r   EvollaFeedForward.forward  s+    xx1(>?@@r9   )r   r   r   r   )   r   rA   s   @r7   r   r      s    9A Ar9   r   c                   6   ^  \ rS rSrS\4U 4S jjrS rSrU =r$ )!EvollaSequenceCompressorResampleri	  r5   c                   > [         TU ]  5         UR                  R                  nUR                  U l        [        R                  " [        R                  " U R
                  U5      SS9U l
        [        R                  " / 5      U l        [        UR                  5       Ha  nU R                  R                  [        R                  " [!        X!R"                  UR$                  S9['        X!R(                  S9/5      5        Mc     [        R*                  " UR                  5      U l        [        R.                  " X!R                  5      U l        g )NT)requires_grad)r   r   r   )r   r   )r0   r1   protein_encoder_confighidden_sizeresampler_num_latentsnum_latentsr   	Parameterr   randnr   
ModuleListlayersrangeresampler_depthappendr   resampler_dim_headresampler_headsr   resampler_ff_multr   r   r   protein_projector)r4   r5   protein_repr_dimr   r6   s       r7   r1   *EvollaSequenceCompressorResampler.__init__
  s    !88DD!77||EKK0@0@BR$ScghmmB'v--.AKK9 0;T;T\b\r\r *.>E]E]^		 / LL!3!34	!#+;=O=O!Pr9   c                 d   UR                   S   nUR                   u  pE[        R                  " X@R                  5      R	                  UR
                  5      n[        R                  " X&4SS9n[        R                  " U5      R	                  U R                  R
                  5      nU R                  S    UR                  SSS5      -  nUR	                  UR                  5      nU R                   H  u  pU	" XU5      U-   nU
" U5      U-   nM     U R                  U5      nU R                  U5      $ )Nr   r(   r   r   )r   r   r   r  r   rF   r   r   r   dtyper  r  r   )r4   embedsr   br   r   latent_maskr   r   r   fftransformed_features               r7   r   )EvollaSequenceCompressorResampler.forward  s    LLO

jj%5%5699$++Fyy$,!4 zz!} 3 34,,t$tyyQ'::**V\\*HD6D1G;GkG+G $ #44W=yy,--r9   )r   r  r   r  r  )	r;   r<   r=   r>   r)   r1   r   r?   r@   rA   s   @r7   r  r  	  s    Q| Q*. .r9   r  c                       \ rS rSr% Sr\R                  S-  \S'   Sr\R                  S-  \S'   Sr	\
\R                  S4   S-  \S'   Sr\
\R                  S4   S-  \S'   Srg)	EvollaProteinEncoderModelOutputi3  Nsequence_compressor_outputr   .rt   ru   rM   )r;   r<   r=   r>   r   r   FloatTensorr   r   rt   rQ   ru   r?   rM   r9   r7   r  r  3  so     <@ 1 1D 8?26u((4/6:>M5**C/047>7;Je'',-4;r9   r  c                   t   ^  \ rS rSrS\4U 4S jjr\S\R                  S\R                  4S j5       r
SrU =r$ )EvollaProteinEncoderi<  r5   c                 n   > [         TU ]  5         [        UR                  S9U l        [        US9U l        g r   )r0   r1   r   r  modelr  sequence_compressor_resamplerr3   s     r7   r1   EvollaProteinEncoder.__init__=  s.    /v7T7TU
-NV\-]*r9   r   r   c                     U R                  XS9nUR                  nU R                  XR5      n[        UUR                  S9$ )Nr   )r   r   )r%  r   r&  r  )r4   r   r   r   protein_outputprotein_embedssequence_reprs          r7   r   EvollaProteinEncoder.forwardB  sF    iW'99::>Z.'4,>>
 	
r9   )r%  r&  )r;   r<   r=   r>   r)   r1   r   r   
LongTensorr!  r   r?   r@   rA   s   @r7   r#  r#  <  s?    ^| ^
 
!1!1 
5CTCT 
 
r9   r#  c                   r   ^  \ rS rSr   S
S\S-  S\S-  S\S-  4U 4S jjjrS r       SS jrS	rU =r	$ )#EvollaSequenceAlignerCrossAttentioniN  Nprotein_encoder_dimstructure_encoder_dimmsa_encoder_dimc                   > [         TU ]  5         UR                  U l        UR                  U l        U R                  S-  U l        [        U R                  U R                  -  5      U l        U R                  U R                  -  U l        UR                  nUR                  nUR                  n[        R                  " U R                  U R                  5      U l        UbK  [        R                  " X R                  5      U l        [        R                  " X R                  5      U l        OS U l        S U l        UbK  [        R                  " X0R                  5      U l        [        R                  " X0R                  5      U l        OS U l        S U l        UbK  [        R                  " X@R                  5      U l        [        R                  " X@R                  5      U l        OS U l        S U l        [)        U R                  5      U l        [        R,                  " U5      U l        [        R                  " U R                  U R                  US9U l        [3        U R                  U5      U l        [        R6                  " [8        R:                  " S/5      5      U l        [        R6                  " [8        R:                  " S/5      5      U l        g )Nr   r           ) r0   r1   r  num_attention_headsr   rP   attention_head_sizeall_head_size$aligner_attention_probs_dropout_probaligner_enable_biasaligner_ffn_multr   r   querykey_proteinvalue_proteinkey_structurevalue_structurekey_msa	value_msaEvollaRMSNormattention_normDropoutdropoutout_projr   r  r	  r   tensorgate_attentiongate_ffw)	r4   r5   r0  r1  r2  attention_probs_dropout_probenable_biasffn_multr6   s	           r7   r1   ,EvollaSequenceAlignerCrossAttention.__init__O  s    	!--#)#=#= --t3
#&t'7'7$:R:R'R#S !558P8PP'-'R'R$00**YYt//1C1CD
*!yy)<>P>PQD!#+>@R@R!SD#D!%D ,!#+@BTBT!UD#%99-BDVDV#WD !%D#'D &99_6H6HIDLYY8J8JKDNDL!DN+D,<,<=zz">?		$"2"2D4D4D;W#D$4$4h? ll5<<+>?U\\3%%89r9   c	                    XgU/n	U	 V
s/ s H	  oc  M  U
PM     n	n
U	(       d  [        S5      e[        R                  " U	SS9n	U R                  U5      nU R	                  U5      nU R
                  bA  U R                  b4  UR                  U5      nU R                  U5      nU R                  U5      nOSnSnU R                  bA  U R                  b4  UR                  U5      nU R                  U5      nU R                  U5      nOSnSnU R                  bA  U R                  b4  UR                  U5      nU R                  U5      nU R                  U5      nOSnSnXU/nU V
s/ s H	  oc  M  U
PM     nn
[        R                  " USS9nXU/nU V
s/ s H	  oc  M  U
PM     nn
[        R                  " USS9nUR                  5       SS U R                  U R                  4-   nUR                  " U6 R!                  SSSS5      nUR                  5       SS U R                  U R                  4-   nUR                  " U6 R!                  SSSS5      nUR                  5       SS U R                  U R                  4-   nUR                  " U6 R!                  SSSS5      nXR"                  -  nUcN  [        R$                  " UR                  S5      UR                  S5      5      R                  UR&                  5      nUSS2SSS2S4   U	SS2SSSS24   -  n[        R(                  " UUR+                  SS	5      5      nUUR-                  SS
S9R/                  5       -
  nUR1                  SU-
  R3                  5       [        R4                  " UR6                  5      R8                  5      n[:        R<                  " SS9" U5      n[        R(                  " UU5      nUR!                  SSSS5      R?                  5       nUR                  5       SS	 U R@                  4-   nUR                  " U6 nU RC                  U5      nU$ s  sn
f s  sn
f s  sn
f )z
query_states: text
key_value_states: protein
query_states: [bs, query_seq_len, dim]
key_value_states: [bs, kv_seq_len, dim]
query_attn_mask: [bs, query_seq_len]
kv_attn_mask: [bs, kv_seq_len]
Nz=At least one modality should be provided for cross attention.r(   r   r   r   r   r   r   Tr   )"
ValueErrorr   r   rC  r;  r<  r=  r   r>  r?  r@  rA  r   r5  r6  r   r   r   r   rF   r   r   r   r   r   r   finfor  minr   Softmax
contiguousr7  rF  )r4   query_statesprotein_key_value_statesstructure_key_value_statesmsa_key_value_statesquery_attn_maskprotein_kv_attn_maskstructure_kv_attn_maskmsa_kv_attn_maskkv_attn_maskr   query_layerkey_layer_proteinvalue_layer_proteinkey_layer_structurevalue_layer_structurekey_layer_msavalue_layer_msa	key_layervalue_layernew_query_layer_shapenew_key_layer_shapenew_value_layer_shaper   attn_weightsattention_scoresattention_probscontext_layernew_context_layer_shapes                                r7   cross_attention3EvollaSequenceAlignerCrossAttention.cross_attention  sK   * -FVW#/A<a<A\]]yy15)),7 jj-'D,>,>,J'?'B'B<'P$ $ 0 01I J"&"4"45M"N $"&)d.B.B.N)C)F)F|)T&"&"4"45O"P$($8$89S$T!"&$(!<<#(B#7#:#:<#H  LL)=>M"nn-ABO M"O&]K	 );	1Q		;IIiQ/	*?S"-?+Qq+?ii3 + 0 0 23B 7$$$$;
 !
 "&&(=>FFq!QPQR'nn.s3$$$$7
 
 NN$78@@Aq!L	 + 0 0 23B 7$$$$;
 !
 "&&(=>FFq!QPQR!JJ. "#jj):):1)=|?P?PQR?STWWXdXkXklO(D!T)9:\!TSWYZJZ=[[||K1D1DR1LM#l&7&7B&7&M&T&T&VV'33%%'\5G5G)H)L)L
 **,-=> _kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**,CDm4q BL < @s"   QQ"Q,QQQc                 z   Ubv  UR                   u  pnUcc  [        R                  " X5      R                  U	R                  5      U	R                  X4S9R                  -  R                  UR                  5      nOS nUby  UR                   u  nnnUce  [        R                  " UU5      R                  U	R                  5      U
R                  UU4S9R                  -  R                  UR                  5      nOS nUby  UR                   u  nnnUce  [        R                  " UU5      R                  U	R                  5      UR                  UU4S9R                  -  R                  UR                  5      nOS nUnUb  UR                  5       (       d0  Ub  UR                  5       (       d  Ub  UR                  5       (       ay  UnU R                  UUUUUUUUS9n[        R                  " U R                  5      U-  nUU-   nUnU R                  U5      [        R                  " U R                  5      -  nUU-   nU$ )N)r   )rT  rU  rV  rW  rX  rY  rZ  r[  )r   r   r   r   rF   expandTanyrn  tanhrH  r  rI  )r4   rT  protein_kv_statesstructure_kv_statesmsa_kv_statesrX  rY  rZ  r[  protein_batch_maskstructure_batch_maskmsa_batch_maskpast_key_valuesr   protein_kv_seq_lenr   structure_kv_seq_lenmsa_kv_seq_lenrt   residuals                       r7   r   +EvollaSequenceAlignerCrossAttention.forward  sP    (*;*A*A'BC#+JJr699:L:S:ST(//6H5M/NPPQ"&--. %
 $( *,?,E,E)B$c%-JJr#78;;<N<U<UV*118Lb7Q1RTTU"(//0 '
 &*"$&3&9&9#B'JJr>2556H6O6OP$++."1E+FHHI"]))* !
  $$ */C/G/G/I/I#/4J4N4N4P4P).>.B.B.D.D$H 00*):+>%2 /%9'=!1 1 	M "JJt':':;mKM$}4M$H GGM2UZZ5NNM$}4Mr9   )r7  r6  rC  rE  r  rH  rI  r  r@  r<  r>  r5  rF  r;  r   rA  r=  r?  rN   )NNNNNNN)
r;   r<   r=   r>   rP   r1   rn  r   r?   r@   rA   s   @r7   r/  r/  N  sm     +/,0&*1: !4Z1:  #Tz	1:
 t1: 1:fnn "#!G Gr9   r/  c                       \ rS rSrSrg)rB  i<  rM   NrV   rM   r9   r7   rB  rB  <  rW   r9   rB  c                       \ rS rSrSrg)EvollaRotaryEmbeddingi@  rM   NrV   rM   r9   r7   r  r  @  rW   r9   r  c                       \ rS rSrSrg)	EvollaMLPiD  rM   NrV   rM   r9   r7   r  r  D  rW   r9   r  c                       \ rS rSrSrg)EvollaAttentioniH  rM   NrV   rM   r9   r7   r  r  H  rW   r9   r  c                     ^  \ rS rSrS\S\4U 4S jjr            SS\R                  S\	\R                  \R                  4   S-  S\R                  S-  S	\R                  S-  S
\S-  S\S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  4S jjrSrU =r$ )EvollaDecoderLayeriL  r5   	layer_idxc                    > [         TU ]  X5        US-   [        UR                  UR                  -  S5      -  S:X  a  [        UUR                  S9U l        g g )Nr(   r   )r0  )r0   r1   maxnum_hidden_layersaligner_num_add_layersr/  r  adapterr4   r5   r  r6   s      r7   r1   EvollaDecoderLayer.__init__M  sY    +MS!9!9V=Z=Z!Z\]^^bcc>$*$6$6DL dr9   Nrt   r   r   r2   r{  	use_cacheru  rv  rw  rx  ry  rz  rX  c                 
   UnU R                  U5      nU R                  " SUUUUUUS.UD6u  nnX-   nUnU R                  U5      nU R                  U5      nX-   n[	        U S5      (       a  U R                  UUUU	UU
UUS9nU$ )N)rt   r   r2   r{  r  r   r  )rT  ru  rv  rw  rX  rx  ry  rz  rM   )input_layernorm	self_attnpost_attention_layernormmlphasattrr  )r4   rt   r   r   r2   r{  r  ru  rv  rw  rx  ry  rz  rX  r   r  r   s                    r7   r   EvollaDecoderLayer.forwardU  s    " !,,];  >> 
')%+ 3
 
q !0 !55mD/ 04## LL*"3$7+ /#5%9- ) 	M r9   )r  )NNNNFNNNNNNN)r;   r<   r=   r>   r)   rP   r1   r   r   rQ   r-  r   r   r   r?   r@   rA   s   @r7   r  r  L  s>   |   IM.204(,!&1537-12648.2/33||3 #5<<#=>E3 t+	3
 &&-3 3 $;3 !<<$.3 #\\D03 ||d*3 "LL4/3 $llT13 t+3 ,3 3r9   r  c                   V    \ rS rSrSrSrSr/ SQr\R                  " 5       S 5       r
Srg)EvollaPreTrainedModeli  F)r  re   r  r/  c                    U R                   R                  n[        R                  " X5        [	        U[
        5      (       ak  [        R                  " UR                  5        [        R                  " UR                  5        [        R                  " UR                  R                  5        g [	        U[        5      (       a!  [        R                  " UR                  SUS9  g g )Nr4  )meanstd)r5   initializer_ranger   ry   rz   r/  r{   zeros_rH  rI  ones_rC  weightr  normal_r   )r4   r~   r  s      r7   ry   #EvollaPreTrainedModel._init_weights  s    kk++%%d3fABBKK--.KK(JJv,,334 ABBLLcs; Cr9   rM   N)r;   r<   r=   r>   r   r   r   r   r   r   ry   r?   rM   r9   r7   r  r    s6     "' ]]_< <r9   r  c                     ^  \ rS rSrS\4U 4S jjrS rS r\\	\
            SS\R                  S-  S\R                  S-  S	\R                  S-  S
\S-  S\R                  S-  S\S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\\-  4S jj5       5       5       rSrU =r$ )EvollaModeli  r5   c           
      8  > [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " U R                  UR                  U R                  5      U l        [        US9U l
        [
        R                  " [        UR                  5       Vs/ s H  n[        UUS9PM     sn5      U l        [!        UR                  UR"                  S9U l        ['        USS5      U l        [+        US9U l        U R/                  5         g s  snf )Nr   )r5   r  )epsgradient_checkpointingF)r0   r1   pad_token_idpadding_idx
vocab_sizer   	Embeddingr  embed_tokensr#  protein_encoderr  r  r  r  r  rB  rms_norm_epsr   r}   r  r  
rotary_embr   r  s      r7   r1   EvollaModel.__init__  s     !.. ++LL&:L:LdN^N^_36Bmm "'v'?'?!@
 "AI	 #!' "A
 "&"4"4&:M:MN	&-f6NPU&V#/v>s   #Dc                     U R                   $ r/   r  r   s    r7   r    EvollaModel.get_input_embeddings  s       r9   c                     Xl         g r/   r  r   s     r7   r    EvollaModel.set_input_embeddings  s    !r9   Nr   r   r2   r{  r   r  protein_input_idsprotein_attention_maskstructure_feats	msa_featsry  rz  rI   c                    USL USL-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      nSnSnUbZ  UbW  U R                  UUS9nUR                  n[
        R                  " UR                  S   UR                  [
        R                  S9n[        U R                  UUUS	9nUnU R                  UUS
9nU R                    H  nU" U4UUUUUU	U
UUUUUS.UD6nM     U R#                  U5      n[%        UUS9nU$ )a  
protein_input_ids (torch.LongTensor):
    The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
protein_attention_mask (torch.Tensor):
    The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
structure_feats (torch.FloatTensor):
    The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
msa_feats (torch.FloatTensor):
    The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
structure_batch_mask (torch.Tensor):
    The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummpy input for now.
msa_batch_mask (torch.Tensor):
    The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummpy input for now.
Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r(   r   r   )rF   r  )r5   r   r   r{  r:   )r   r2   r{  r  ru  rv  rw  rx  ry  rz  rX  r   )r   r{  )rO  r  r   r5   get_seq_lengthr   r   r   rF   r   r  r   r   r   r   r  r  r   r   )r4   r   r   r2   r{  r   r  r  r  r  r  ry  rz  r   past_seen_tokensprotein_featsrx  protein_outputscausal_maskrt   r   decoder_layeroutputs                          r7   r   EvollaModel.forward  s   B -t";<YZZ  --i8M0*$++>OCRC^==?de <<(;(;A(>}G[G[\_ooL'11!4L!(-C-O"22+5 3 O ,FFM!&!''*(//jj" );;')+	
 &"oom,oW![[M)*) /#"/$3'#5%9- .$7 M )$ 		-0(++
 r9   )r  r  r  r   r  r  r  r  )NNNNNNNNNNNN)r;   r<   r=   r>   r)   r1   r   r   r   r   r   r   r-  r   r   r!  r   rQ   r   r   r?   r@   rA   s   @r7   r  r    sf   | *!"  .2.204(,26!%596:48.248.2]##d*] t+] &&-	]
 ] ((4/] $;] !++d2] !&t 3] **T1] $$t+] $llT1] t+] 
(	(]    ]r9   r  c                   B  ^  \ rS rSrU 4S jrS rS r\\        SS\	R                  S-  S\	R                  S-  S\	R                  S-  S	\	R                  S-  S
\	R                  S-  S\	R                  S-  S\S-  S\\	R                  -  4S jj5       5       rSrU =r$ )EvollaForProteinText2Texti!  c                    > [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  U R                  SS9U l        U R                  5         g r   )
r0   r1   r  r%  r  r   r   r  lm_headr   r3   s     r7   r1   "EvollaForProteinText2Text.__init__"  sQ      (
 ++yy!3!3T__5Qr9   c                 6    U R                   R                  5       $ r/   )r%  r   r   s    r7   r   .EvollaForProteinText2Text.get_input_embeddings*  s    zz..00r9   c                 8    U R                   R                  U5      $ r/   )r%  r   r   s     r7   r   .EvollaForProteinText2Text.set_input_embeddings-  s    zz..u55r9   Nr   r   r   labelsr  r  r  logits_to_keepc	           
      l   U R                   " SUUUUUUS.U	D6n
U
R                  n[        U[        5      (       a  [	        U* S5      OUnU R                  USS2USS24   5      nSnUb  U R                  " SXU R                  S.U	D6n[        UUU
R                  U
R                  U
R                  S9nU$ )a|  
protein_input_ids (torch.LongTensor):
    The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
protein_attention_mask (torch.Tensor):
    The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

Example:

```python
>>> from transformers import EvollaProcessor, EvollaForProteinText2Text
>>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
>>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

>>> protein_information = {
    "aa_seq": "your amino acid sequence",
    "foldseek": "your foldseek sequence",
}
>>> question = "What is the function of this protein?"
>>> message = [
    {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
    {"role": "user", "content": question},
]

>>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
>>> outputs = model.generate(**inputs)

>>> print(processor.batch_decode(outputs, skip_special_tokens=True))
```)r   r   r   r  r  r  N)logitsr  r  )lossr  r{  rt   ru   rM   )r%  r   rz   rP   slicer  loss_functionr  r   r{  rt   ru   )r4   r   r   r   r  r  r  r  r  r   outputsrt   slice_indicesr  r  
lm_outputss                   r7   r   !EvollaForProteinText2Text.forward0  s    T ,0:: ,
)'/#9,
 ,
  118B>SV8W8W~ot4]kmA}a,?@A%%iVtibhiD+#33!//))

 r9   )r  r%  r  )NNNNNNNr   )r;   r<   r=   r>   r1   r   r   r   r   r   r-  r   r!  r   rP   r   r?   r@   rA   s   @r7   r  r  !  s    16  .2.226*.596:!%-.B##d*B t+B ((4/	B
   4'B !++d2B !&t 3B $;B ell*B  Br9   r  )r  r  r  )Sdataclassesr   r   r    r   r{   cache_utilsr   r   
generationr	   masking_utilsr
   r   modeling_outputsr   r   r   r   modeling_utilsr   utilsr   r   r   utils.genericr   utils.output_capturingr   r   esm.modeling_esmr   r   r   r   r   r   r   r   r    r!   llama.modeling_llamar"   r#   r$   r%   r&   r'   configuration_evollar)   r*   
get_loggerr;   loggerr,   rC   rT   rY   r\   r_   rb   re   rh   rk   rn   r   Moduler   r   r  r  r#  r/  rB  r  r  r  r  r  r  r  __all__rM   r9   r7   <module>r     s   "   & . ) J  . 
 8 E    = 
		H	%!] !
P"4 
P	 0 		] 		L 		 		 		 		* 		 	 C/ C C01
!< 1
h7 		 7 tA		 A'.		 '.T <k <  <
299 
$k")) k\	L 		0 		 		n 	<* <~<0 <.|' |~S 5 Sl Pr9   