
    Z j                        S SK Jr  S SKJr  SSKJrJr  SSKJr  SSK	J
r
JrJrJr  SSKJr  \" S	S
9\ " S S\5      5       5       r " S S\5      r " S S\
5      r " S S\5      r " S S\5      r " S S\5      r/ SQrg)    N)strict   )auto_docstringcan_return_tuple   )LlamaConfig)LlamaDecoderLayerLlamaForCausalLM
LlamaModelLlamaPreTrainedModel)NemotronMLPzinceptionai/Jais-2-8B-Chat)
checkpointc                       \ rS rSr% SSSSSSS.rSr\\S'   Sr\\S'   S	r	\\S
'   Sr
\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\S-  \S'   Sr\\\   -  S-  \S'   Sr\\S'   Sr\\S'   \" 5       r\" 5       rSrg)Jais2Config   colwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proji K 
vocab_sizei   hidden_sizei h  intermediate_size   num_attention_headsrelu2
hidden_acti    max_position_embeddingsgh㈵>layer_norm_epsr   Nbos_token_idiJ eos_token_idTattention_biasmlp_bias )__name__
__module____qualname____firstlineno__base_model_tp_planr   int__annotations__r   r   r   r   strr   r   floatr   r   listr   boolr    AttributeErrorrms_norm_epspretraining_tp__static_attributes__r!       x/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/jais2/modular_jais2.pyr   r      s     &/%.%.%. )"+ JK"s"!!J#'S' NE  L#* +1L#S	/D(1NDHd!#L#%Nr1   r   c                       \ rS rSrSrg)Jais2MLP9   r!   Nr"   r#   r$   r%   r0   r!   r1   r2   r4   r4   9       r1   r4   c                   4   ^  \ rS rSrS\S\4U 4S jjrSrU =r$ )Jais2DecoderLayer=   config	layer_idxc                    > [         TU ]  X5        [        R                  " UR                  UR
                  S9U l        [        R                  " UR                  UR
                  S9U l        g N)eps)super__init__nn	LayerNormr   r   input_layernormpost_attention_layernorm)selfr;   r<   	__class__s      r2   rA   Jais2DecoderLayer.__init__>   sN    +!||F,>,>FDYDYZ(*V5G5GVMbMb(c%r1   )rD   rE   )	r"   r#   r$   r%   r   r'   rA   r0   __classcell__rG   s   @r2   r9   r9   =   s    d{ ds d dr1   r9   c                       \ rS rSrSrg)Jais2PreTrainedModelD   r!   Nr6   r!   r1   r2   rL   rL   D   r7   r1   rL   c                   0   ^  \ rS rSrS\4U 4S jjrSrU =r$ )
Jais2ModelH   r;   c                    > [         TU ]  U5        [        R                  " UR                  UR
                  S9U l        g r>   )r@   rA   rB   rC   r   r   norm)rF   r;   rG   s     r2   rA   Jais2Model.__init__I   s.     LL!3!39N9NO	r1   )rR   )r"   r#   r$   r%   r   rA   r0   rI   rJ   s   @r2   rO   rO   H   s    P{ P Pr1   rO   c                   <   ^  \ rS rSr\\U 4S j5       5       rSrU =r$ )Jais2ForCausalLMN   c                 $   > [         TU ]  " S0 UD6$ )as  
Example:

```python
>>> from transformers import AutoTokenizer, Jais2ForCausalLM

>>> model = Jais2ForCausalLM.from_pretrained("inceptionai/Jais-2-8B-Chat")
>>> tokenizer = AutoTokenizer.from_pretrained("inceptionai/Jais-2-8B-Chat")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```r!   )r@   forward)rF   super_kwargsrG   s     r2   rX   Jais2ForCausalLM.forwardO   s    ( w...r1   r!   )	r"   r#   r$   r%   r   r   rX   r0   rI   rJ   s   @r2   rU   rU   N   s    /  /r1   rU   )r   rO   rU   rL   )torch.nnrB   huggingface_hub.dataclassesr   utilsr   r   llama.configuration_llamar   llama.modeling_llamar	   r
   r   r   nemotron.modeling_nemotronr   r   r4   r9   rL   rO   rU   __all__r!   r1   r2   <module>rb      s      . 5 3  5 78&+ &  9&2	{ 	d) d	/ 	P P/' /0r1   