
    Z j
                     f    S r SSKrSSKJr  SSKJr  \R                  " \5      r " S S5      r	S/r
g)	zTokenization classes for RAG.    N   )logging   )	RagConfigc                   N    \ rS rSrS rS r\S 5       rS rS r	S r
S rS	 rS
rg)RagTokenizer   c                 >    Xl         X l        U R                   U l        g N)question_encoder	generatorcurrent_tokenizer)selfr   r   s      y/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/rag/tokenization_rag.py__init__RagTokenizer.__init__   s     0"!%!6!6    c                    [         R                  R                  U5      (       a  [        SU S35      e[         R                  " USS9  [         R                  R                  US5      n[         R                  R                  US5      nU R                  R                  U5        U R                  R                  U5        g )NzProvided path (z#) should be a directory, not a fileT)exist_okquestion_encoder_tokenizergenerator_tokenizer)	ospathisfile
ValueErrormakedirsjoinr   save_pretrainedr   )r   save_directoryquestion_encoder_pathgenerator_paths       r   r   RagTokenizer.save_pretrained   s    77>>.))~.>>abcc
NT2 "^=Y Zn6KL--.CD&&~6r   c                     SSK Jn  UR                  SS 5      nUc  [        R                  " U5      nUR	                  XR
                  SS9nUR	                  XR                  SS9nU " XVS9$ )N   )AutoTokenizerconfigr   )r&   	subfolderr   )r   r   )auto.tokenization_autor%   popr   from_pretrainedr   r   )clspretrained_model_name_or_pathkwargsr%   r&   r   r   s          r   r*   RagTokenizer.from_pretrained(   s     	;Hd+>../LMF(88)2I2IUq 9 
 "11)2B2BNc 2 
	 $4JJr   c                 &    U R                   " U0 UD6$ r   )r   r   argsr-   s      r   __call__RagTokenizer.__call__:   s    %%t6v66r   c                 :    U R                   R                  " U0 UD6$ r   )r   batch_decoder0   s      r   r5   RagTokenizer.batch_decode=   s    ~~**D;F;;r   c                 :    U R                   R                  " U0 UD6$ r   )r   decoder0   s      r   r8   RagTokenizer.decode@   s    ~~$$d5f55r   c                 &    U R                   U l        g r   )r   r   r   s    r   _switch_to_input_mode"RagTokenizer._switch_to_input_modeC   s    !%!6!6r   c                 &    U R                   U l        g r   )r   r   r;   s    r   _switch_to_target_mode#RagTokenizer._switch_to_target_modeF   s    !%r   )r   r   r   N)__name__
__module____qualname____firstlineno__r   r   classmethodr*   r2   r5   r8   r<   r?   __static_attributes__ r   r   r   r      s;    7
7 K K"7<670r   r   )__doc__r   utilsr   configuration_ragr   
get_loggerrA   loggerr   __all__rG   r   r   <module>rN      s:    $ 	  ( 
		H	%.0 .0b 
r   