
    Z j'                         S SK r S SKrS SKJr  SSKJr  SSKJr  SSKJ	r	  SSK
JrJr  SSKJrJr  SS	KJr  S
SKJr  \" 5       (       a
  S SKrS SKJr  SrSrSrSrSr SS jr " S S5      rg)    N)Image   )create_causal_mask)_get_model_class)
AutoConfig)MODEL_FOR_PRETRAINING_MAPPINGMODEL_MAPPING)PROCESSOR_MAPPING_NAMESAutoProcessor)AutoTokenizer   )is_torch_availablez[92mz[93mz[0mu   ■u   ⬚c                   ^ ^^^^^^ TR                  5       mTR                  S:X  a  TSSS2SS24   mTR                  S:X  a  TSSSS2SS24   m[        T 5      n[        S T  5       5      nSn/ n	[	        T 5       HP  u  mn
U
T:X  a  U(       d	  TnSTTT4'   US:  d  M$  U
T:w  d  TUS-
  :X  d  M5  TUS-
  :X  a  TS-  mSTUT2UT24'   SnMR     SmUbL  [        U5       VVs/ s H3  n[        U5       Vs/ s H  nSX-
  s=::  a  U:  a  O  OSOSPM     snPM5     snnmSR                  U4S	 j[        U5       5       5      nUb  US:H  n[        R                  " UR                  S
5      S-  U-   R                  5       UR                  S
5      S5      m[        R                  " SUS-   U5      n[        R                  " TUS9m[         [         [         S[          [         [         S3nU	R#                  SU-   5        SUS-   -  SR%                  [        U5      S-  5      -   nUb  US-  nU	R#                  U5        / n[	        T 5       H  u  nmTUU4   S:X  ae  UR#                  ['        [)        U5      R+                  [        [)        U5      5      5      5       V
s/ s H  n
[          U
 [         3PM     sn
5        Mv  UR#                  ['        [)        U5      R+                  [        [)        U5      5      5      5      5        M     ['        [-        [&        [/        U6 5      5      nU HF  nU	R#                  Ub/  US-   S-  SR                  U5      -   S-   SR                  U5      -   OS5        MH     [	        T 5       H  u  mm[1        T5      R%                  U5      nTT;   a  [          U [         3OUnSR                  UUUUU 4S j[        U5       5       5      nSnUb)  SR                  UUUUUU 4S j[        U5       5       5      nU	R#                  U S[)        T5      R+                  S5       SU SU 35        M     SR                  U	5      $ s  snf s  snnf s  sn
f )z
Generates an attention matrix from a given attention mask.

Optionally applies a sliding window mask (e.g., for Gemma2/3) and
marks regions where image tokens occur based on the specified `img_token`.
   r   N   c              3   J   #    U  H  n[        [        U5      5      v   M     g 7fN)lenrepr).0words     x/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/utils/attention_visualizer.py	<genexpr>6generate_attention_matrix_from_mask.<locals>.<genexpr>8   s     <ed#d4j//es   !#r   r    c              3      >#    U  HW  nTS U4   (       a  [          [         [         3O1US :X  a  [         [         [         3OTS U4   (       a  [        O[        v   MY     g7fr   NYELLOWBLACK_SQUARERESETGREENWHITE_SQUARE)r   jmasks     r   r   r   K   sk      	 A 1: (<.( 6 w|nUG, 1: 	 s   AA"   )
boundariesz: i == j (diagonal)   z: token_type_idszAttention MatrixzSliding Window Maskz	    |     c              3      >#    U  Hf  nTTU   ;   a&  TTU4   (       a  TT;   a  [          [         [         3O1TU:X  a  [         [         [         3OTTU4   (       a  [        O[        v   Mh     g 7fr   r   )r   r$   i	img_tokenr%   r   wordss     r   r   r   x   s}      	
  E!H$ad	T8I h|nUG, Av 7<.0 AqDz  s   A.A1c              3      >#    U  Hk  nTTU   ;   a*  TT;   a$  TS T4   TS U4   :X  a  [          [         [         3O2TU:X  a  [         [         [         3OTT   U   (       a  [        O[        v   Mm     g7fr   r   )r   r$   r+   r,   sliding_window_masktoken_type_bucketsr   r-   s     r   r   r      s      	* "A a(Y$->CUVWYZVZC[_qrsuvrv_wCw (<.0 6 w|nUG4 'q)!, "!" "s   A3A6z: 
)intndimr   max	enumeraterangejointorchwherecumsumboolarange	bucketizer"   r    r!   r   appendljustliststrrjustmapzipr   )r-   r%   r,   sliding_windowtoken_type_idsimage_seq_lengthnmax_word_lengthfirst_img_idxoutputkr+   r$   	row_dummy
is_specialr(   legendf_stringvertical_headeridxrow	word_reprcolored_wordrow_displaysliding_window_rowr/   r0   r   s   ```        `             @@@r   #generate_attention_matrix_from_maskrX   (   sM    88:DyyA~Aq!G}yyA~Aq!QJE
A<e<<OMF% 1	>-MDAJ1!y.AQJAEzQ56Dq-/12M ! !fklmfnofnabSXYZS[\S[aa15&A>&A IS[\fno 	 q	 	I !#q(
"[[""2&*Z7==?AVAVWYAZ\]
 \\!%5%9;KL
"__-?JW w|nUG+A&,X]W^^noF
MM#,o)*-?-E-Ec)nXYFY-ZZH!))
MM(Ou%	TS>Q""4CWZ[^_`[aWbHcCd#eCdavhqc%$9Cd#ef""4Cs3q6{(C#DE	 & 3tS/%:;<O) q C'#((3-7+EQTU	
  U#4J$$_5	8AT8I&)UG4yhh 	
 	
 1X	
 	
  %!$ 	* 	* q	* 	" 	bQa(9;-yQcPdef5 $8 99VQ  ]o@ $fs   $Q 7 QQ #Q&
Q c                   F    \ rS rSrS\4S jrS	S\4S jjrS	S\4S jjrSrg)
AttentionMaskVisualizer   
model_namec                    [         R                  " U5      nSU l        [        UR	                  5       S5      (       a   [        UR	                  5       SS 5      U l         [        U[        5      nUc  [        SU S35      eX0l         " S SU[        R                  5      nU" X!5      U l        U R                  R!                  UR"                  5        Xl        X l        g ! [         a    [        U[        5      n Nf = f)N<img>rE   zModel name z- is not supported for attention visualizationc                       \ rS rSrS rSrg)7AttentionMaskVisualizer.__init__.<locals>._ModelWrapper   c                     [         R                  R                  U 5        [         R                  " SS5      U l        Xl        g )Nr   )nnModule__init__Lineardummy_moduleconfig)selfrh   r\   s      r   re   @AttentionMaskVisualizer.__init__.<locals>._ModelWrapper.__init__   s+    		""4($&IIaO!$    )rh   rg   N)__name__
__module____qualname____firstlineno__re   __static_attributes__ rk   r   _ModelWrapperr`      s    %rk   rr   )r   from_pretrainedimage_tokenhasattrget_text_configgetattrrE   r   r	   	Exceptionr   
ValueError
mapped_clsrc   rd   modeltodtyperepo_idrh   )ri   r\   rh   rz   rr   s        r   re    AttentionMaskVisualizer.__init__   s    ++J7"6))+-=>>")&*@*@*BDTVZ"[D	Q)&-@J {:,6cdee$	%J		 	% #66


fll#!!  	Q)&2OPJ	Qs   C C<;C<input_sentencec                 "    U R                  XS9  g )N)suffix)visualize_attention_mask)ri   r   r   s      r   __call__ AttentionMaskVisualizer.__call__   s    %%n%Drk   c           
         U R                   n0 nS nU R                  R                  [        ;   Ga5  Sn[        R
                  " [        R                  " [        R                  " USS9R                  5      5      nSn[        R                  " U R                  US9n[        US5      (       a  UR                  nO)UR                   R#                  UR$                  /5      S   nU(       a  UR'                  SU5      nU" XaUS	S
9n	UR                   R#                  UR$                  /5      S   U l        U	S   n
SU	;   a  U	S   US'   UR                   R#                  U	S   S   5      nO|[(        R                  " U R                  5      nUc  [+        SU R                   35      eUR-                  U5      nU" US	S9S   n
U
c#  [+        SU R                  R                   S35      eSUR                  l        UR1                  5         U
R2                  u  p[4        R6                  " XUR                  R8                  4U R                   R:                  S9n[=        UR                  UU
S S9nUc2  U
R?                  S5      R?                  S5      RA                  USX5      n
OA[C        U[4        RD                  5      (       a  URG                  [4        RH                  S9) n
OU) n
S[K        SU R                  R                   SU RL                   35      S-   -  nSn[O        SU 35        [O        SSU R                  R                   SU R                   SU RL                  RP                   3RS                  [K        U5      5      -   S-   U-   5        [O        U 5        [U        UU
U R                  [W        U R                  SS 5      UR                  S5      US 9n[O        U5        [O        U 5        g )!Nzchttps://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg?download=trueT)follow_redirectsr'   )rG   rt   r   r^   pt)imagestextr   return_tensorsattention_maskrF   	input_idszCould not load tokenizer for )r   zModel type z) does not support attention visualizationeager)r}   )rh   inputs_embedsr   past_key_valuesr   z##zAttention visualization for z | r   r1   z"  Attention visualization for [1m:z[0m z    rE   )r,   rE   rF   rG   ),r{   rh   
model_typer
   r   openioBytesIOhttpxgetcontentr   rs   r~   ru   rt   	tokenizerconvert_ids_to_tokensimage_token_idreplacer   ry   tokenize_attn_implementationtrainshaper8   zeroshidden_sizer}   r   	unsqueezeexpand
isinstanceTensorr|   r;   r   rz   printrl   centerrX   rw   )ri   r   r   r{   kwargsrG   img	processorrt   inputsr   tokensr   
batch_size
seq_lengthr   causal_masktop_bottom_borderside_borderrP   s                       r   r   0AttentionMaskVisualizer.visualize_attention_mask   s   

;;!!%<<wC**RZZ		#(M(U(UVWC %55dllUefIy-00'33'11GGIaIaHbcdef!/!7!7!Mcv^bcF(22HH)JbJbIcdefgD#$45N6)+12B+C'(((>>vk?RST?UVF%55dllCI  #@!OPP''7F&~dKL\]N% ;t{{/E/E.FFo!pqq,3)!/!5!5
ZU\\=U=U$V^b^h^h^n^no(<<') 	
 +55a8BB1ELLZYZ\fsNU\\22)nn5::n>>N)\N .t{{/E/E.Fc$//IZ[\_``
 $%&'5dkk6L6L5MQt||n\deietete}e}d~  G  G%& 	
 	
 	"#%6&&"4;;0@$G!::&67-
 	h"#%rk   )rh   rt   rz   r{   r~   rE   N)r)   )	rl   rm   rn   ro   rA   re   r   r   rp   rq   rk   r   rZ   rZ      s0    3 2Es EN&s N& N&rk   rZ   )r^   NNN)r   r   PILr   masking_utilsr   models.auto.auto_factoryr   models.auto.configuration_autor   models.auto.modeling_autor   r	   models.auto.processing_autor
   r   models.auto.tokenization_autor   import_utilsr   r8   torch.nnrc   r"   r   r!   r    r#   rX   rZ   rq   rk   r   <module>r      sk    
   . 7 7 T P 9 ,  		 `diXk& k&rk   