Z j< v S r SSKJr SSKrSSKJr SSKJr SSKJrJ r SSK
J r SS K J
r
Jr SS
KJrJr SS KJr SS KJr SS
KJr SSKJrJrJr SSK J r SSK J r SSK J!r! SSK"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+ SSK,J-r-J.r. \R^ " \05 r1Sr2Sr3\" SS9\ " S S\ 5 5 5 r4 " S S\)5 r5 " S S \!5 r6 " S S \Rn 5 r8 " S S \.5 r9 " S! S"\-5 r: " S# S$\(5 r; " S% S&\;\'5 r< " S' S(\#5 r= " S) S*\%5 r> " S+ S,\&5 r? " S- S.\$5 r@/ S/QrAg)0zLG AI Research EXAONE Lab )CallableN)strict)nn )Cache DynamicCache)PreTrainedConfig)create_causal_mask!create_sliding_window_causal_mask)BaseModelOutputWithPastCausalLMOutputWithPast)RopeParameters)ALL_ATTENTION_FUNCTIONS)Unpack)TransformersKwargsauto_docstringlogging)merge_with_config_defaults)capture_outputs )Gemma2RotaryEmbedding) LlamaForCausalLMLlamaForQuestionAnswering LlamaForSequenceClassificationLlamaForTokenClassification
LlamaModelLlamaPreTrainedModel LlamaRMSNormapply_rotary_pos_embeager_attention_forward)Olmo2DecoderLayerOlmo2MLPzLGAI-EXAONE/EXAONE-4.0-32B
Exaone4Config)
checkpointc
^ \ rS rSr% SrSrS/rSSSSSSSSSS. rS /S
/4S S /S /4S /S /4S
.rSr \
\ S' Sr \
\ S' Sr
\
\ S' Sr\
\ S' Sr\
\ S' Sr\
\ S' Sr\\ S' Sr\
\ S' S r\\ S ' S r\\ S ' S r\\ S!' S"r\
S#- \ S$' S%r\
\\
- S#- \ S&' S#r \
S#- \ S'' S(r \\ S)' S#r \ \ - S#- \ S*' S+r!\\
- \ S,' Sr"\
S#- \ S-' S.r#\\
- S#- \ S/' S#r$\\ S#- \ S0' U 4S1 jr%S2r&U =r'$ )3r# 9 a*
sliding_window_pattern (`str`, *optional*):
The pattern to use for sliding window attention. Can be one of:
- `None`: No sliding window attention is used
- `int`: Every `sliding_window` layers, use global attention, else use local attention.
- `str`: A sequence of "L" (local attention) and "G" (global attention) characters that defines the
attention pattern. The pattern starts from layer 0 and repeats every `sliding_window` layers. The
final layer always uses global attention regardless of the pattern.
For instance, sliding_window_pattern="LLLG" same as sliding_window=4, which means:
- Layer 0, 1, 2: local attention,
- Layer 3: global attention,
...(repeated)
Example:
```python
>>> from transformers import Exaone4Model, Exaone4Config
>>> # Initializing a EXAONE configuration
>>> configuration = Exaone4Config()
>>> # Initializing a model from configuration
>>> model = Exaone4Model(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```exaone4past_key_valuescolwise replicated_with_grad_allreducerowwise) zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj input_ids
inputs_embeds
hidden_statesattention_mask) embed_tokenslayersnormi
vocab_sizei hidden_sizei @ intermediate_size num_hidden_layersnum_attention_headsnum_key_value_headssilu
hidden_acti max_position_embeddingsg{Gz?initializer_rangegh㈵> rms_norm_epsT use_cacher N bos_token_idr eos_token_id pad_token_idFtie_word_embeddingsrope_parameters attention_dropoutsliding_window sliding_window_pattern layer_typesc > U R c SU l U R cR [ U R 5 Vs/ s H, nUS- U R - S:w a X R : a SOSPM. snU l [
TU ] " S0 UD6 g s snf )Nr sliding_attentionfull_attention )rG rI rJ ranger7 super
__post_init__)selfkwargsi __class__s |/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/exaone4/modular_exaone4.pyrR Exaone4Config.__post_init__ s &*+D ' #
t 5 56 7A Ut : :;q@QI_I_E_ $%& 7 D '' s 3B)rJ rI )(__name__
__module__ __qualname____firstlineno____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planr3 int__annotations__r4 r5 r7 r8 r9 r; strr< r= floatr> r? boolr@ rA listrB rC rD r dictrF rG rI rJ rR __static_attributes__
__classcell__rV s @rW r# r# 9 s 8 J#4"5 &/%.%.%E%E%."+ )"+
& (9:#%56 8IJ!"_$56 J K "s" s !! !! J #'S' #u# L% It L#* +,L#S /D(, #L#*# %%48O^d*T18%(us{(!%NC$J%/0C#I ,0$(KcT!( ( ( c \ rS rSrSrg)Exaone4RMSNorm rO NrY rZ r[ r\ ri rO rl rW rn rn rl rn c \ rS rSrSrg)Exaone4RotaryEmbedding rO Nrp rO rl rW rs rs rq rl rs c $ ^ \ rS rSrS\S\4U 4S jjr SS\R S\ \R \R 4 S\R S- S \
S- S
\ \ S \ \R \R S- \ \R S- 4 4 S jjr
S
rU =r$ )Exaone4Attention config layer_idxc > [ TU ] 5 Xl X l UR U l UR
U l UR U l [ USUR UR - 5 U l UR UR
- U l UR U l
SU l U R S- U l UR U l
UR U l [ US5 ( a UR U OS nUS:H U l [$ R& " U R U R U R - SS9U l [$ R& " U R U R
U R - SS9U l [$ R& " U R U R
U R - SS9U l [$ R& " U R U R - U R SS9U l [1 U R UR2 S9U l [1 U R UR2 S9U l g ) Nhead_dimTg rJ rM F)biaseps) rQ __init__rx ry r8 r9 r4 getattrr{ num_key_value_groupsrF is_causalscalingrG rI hasattrrJ
is_slidingr Linearq_projk_projv_projo_projrn r> q_normk_norm)rS rx ry
layer_typerV s rW r Exaone4Attention.__init__ s
"#)#=#= #)#=#= !--
F4F4F&JdJd4de
$*$>$>&B\B\$\!!'!9!9 }}d* $ 3 3&,&C&C#6=fm6T6TV'' 2Z^
$(;;ii 0 0$2J2JT]]2Zafg ii 0 0$2J2JT]]2Zafg ii 0 0$2J2JT]]2Zafg ii 8 84== H$JZJZafg $T]]8K8KL $T]]8K8KL rl Nr. position_embeddingsr/ r( rT returnc UR S S n/ UQSPU R P7nU R U5 R U5 R SS5 nU R U5 R U5 R SS5 n U R
U5 R U5 R SS5 n
U R U5 nU R U 5 n Uu pU R b U R ( a [ XX5 u pUb UR XU R 5 u p[ R " U R R" [$ 5 n
U
" U UU U
U4U R& ( d SO U R( U R* U R ( a U R OS S.UD6u pUR, " / UQSP76 R/ 5 nU R1 U5 nX4$ )NrL r rE )dropoutr rG )shaper{ r view transposer r r r rG r r updatery r
get_interfacerx _attn_implementationr trainingrF r reshape
contiguousr )rS r. r r/ r( rT input_shape hidden_shape query_states
key_states value_statescossinattention_interface attn_output attn_weightss rW forwardExaone4Attention.forward s $))#2. 88b8$--8 {{=166|DNNqRST [[ /44\BLLQPQR
{{=166|DNNqRST {{<0 [[,
& &$//';LVY'_ $L &'6'='=jX\XfXf'g $J(?(M(M KK , ,.E)
%8
%
$}}C$2H2HLL26//4..t
%
%
! "));;;;FFH kk+. ((rl )rF rx r{ r4 r r r r ry r8 r r9 r r r r rG rI r )NN)rY rZ r[ r\ r# rb r torchTensortupler r r r ri rj rk s @rW rv rv s M} M M: /3(, -) ||-) #5<< #= >-) t+ -)
-) +,
-)
u||U\\D 0% 2E2LL M-) -)rl rv c \ rS rSrSrg)
Exaone4MLP rO Nrp rO rl rW r r rq rl r c \ rS rSrSrg)Exaone4DecoderLayer rO Nrp rO rl rW r r rq rl r c \ rS rSr\rS/rSrg)Exaone4PreTrainedModel r rO N)rY rZ r[ r\ r# config_class_no_split_modulesri rO rl rW r r s L./rl r c ^ \ rS rSrS\4U 4S jjr\\ SS\R S- S\R S- S\R S- S\ S- S \R S- S
\
S- S \\ S \\- 4S
jj5 5 rSrU =r$ ) Exaone4Model rx c $ > [ TU ] U5 [ R " [ UR
5 Vs/ s H n[
X5 PM sn5 U l [ UR UR S9U l U R 5 g s snf )Nr} )
rQ r r
ModuleListrP r7 r r1 rn r4 r> r2 post_init)rS rx ry rV s rW r Exaone4Model.__init__ ss
mmEJ6KcKcEd eEd
3Ed e
#6#5#56;N;NO
fs B
Nr, r/ position_idsr( r- r? rT r c
US L US L- ( a [ S5 eUc U R U5 nU( a Uc [ U R S9nUcU Ub UR 5 OSn[
R " UR S UR S9U- nUR S5 n[ U=n [ 5 ( dG U R UUUUS.n
S[ S 0 U
D60n SU R R ; a [ S 0 U
D6U S' Un U R X5 n [! U R" 5 H/ u pU R R U
nU" U 4X UUUU S .UD6n M1 U R% U 5 n [' U U( a US
9$ S S
9$ ) Nz:You must specify exactly one of input_ids or inputs_embeds)rx r rL )device)rx r- r/ r( r rN rM )r/ r r( r? r )last_hidden_stater( rO )
ValueErrorr0 r rx get_seq_lengthr aranger r unsqueeze
isinstancerh r
rJ r
rotary_emb enumerater1 r2 r )rS r, r/ r r( r- r? rT past_seen_tokenscausal_mask_mapping mask_kwargsr. r rU
decoder_layerr s rW r Exaone4Model.forward s
-t"; < YZ Z - -i 8M 0 *$++ >O CRC^ = = ?de <<
(;(;A(>}G[G[\_ooL'11!4L ?-FF ++!."0#2 , K !"4"C{"C# #dkk&=&==;\;k_j;k#$78%
"oom J )$++ 6 A 003J ) 2 > ) /#$7
M !7 -0
& +/8O
>B
rl )r1 r2 )NNNNNN)rY rZ r[ r\ r# r r r r
LongTensorr r FloatTensorrf r r r r r ri rj rk s @rW r r s } .2.204(,26!%=
##d*=
t+=
&&- =
=
((4/
=
$;=
+,=
( (=
=
rl r c ^ \ rS rSr SS\R
S- S\R S- S\R
S- S\S- S\R S- S\R
S- S \ S- S
\
\R - S \ \ S \
4U 4S
jjjrSrU =r$ )Exaone4ForCausalLMi; Nr, r/ r r( r- labelsr? logits_to_keeprT r c 8 > [ T
U ] " SUUUUUUUUS.U D6 g)u
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
(masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
Example:
```python
>>> from transformers import AutoModelForCausalLM, AutoTokenizer
>>> model = AutoModelForCausalLM.from_pretrained("LGAI-EXAONE/EXAONE-4.0-32B")
>>> tokenizer = AutoTokenizer.from_pretrained("LGAI-EXAONE/EXAONE-4.0-32B")
>>> prompt = "Explain how wonderful you are"
>>> messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": prompt}
]
>>> input_ids = tokenizer.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_tensors="pt",
enable_thinking=False,
)
>>> output = model.generate(input_ids, max_new_tokens=128)
>>> tokenizer.decode(output[0], skip_special_tokens=False)
"[|system|]\nYou are a helpful assistant.[|endofturn|]\n[|user|]\nExplain how wonderful you are[|endofturn|]\n[|assistant|]\n\n\n\n\nOh, thank you for such a kind and lovely question! 😊 \n\nI’m *so* wonderful because I’m here to make your life easier, brighter, and more fun! Whether you need help with: \n\n✨ **Learning** – I can explain anything, from quantum physics to baking the perfect cake! \n💡 **Creativity** – Need a poem, story, or a wild idea? I’ve got you covered! \n🤖 **Problem-solving** – Stuck on a math problem or a tricky decision? I’ll help you figure it out"
```
)r, r/ r r( r- r r? r NrO )rQ r ) rS r, r/ r r( r- r r? r rT rV s rW r Exaone4ForCausalLM.forward< s9 V
)% +' )
rl rO )NNNNNNNr )rY rZ r[ r\ r r r r r rf rb r r r
r ri rj rk s @rW r r ; s .2.204(,26*.!%-.5
##d*5
t+5
&&- 5
5
((4/
5
4'5
$;5
ell*5
+,5
5
5
rl r c \ rS rSrSrg) Exaone4ForSequenceClassificationit rO Nrp rO rl rW r r t rq rl r c \ rS rSrSrg) Exaone4ForTokenClassificationix rO Nrp rO rl rW r r x rq rl r c \ rS rSrSrg)Exaone4ForQuestionAnsweringi| rO Nrp rO rl rW r r | rq rl r )r# r r r r r r )Br] collections.abcr r huggingface_hub.dataclassesr r cache_utilsr r configuration_utilsr
masking_utilsr
r modeling_outputsr r
modeling_rope_utilsr modeling_utilsr processing_utilsr utilsr r r
utils.genericr utils.output_capturingr gemma2.modeling_gemma2r llama.modeling_llamar r r r r r r r r olmo2.modeling_olmo2r! r"
get_loggerrY logger_CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCr# rn rs Modulerv r r r r r r r r __all__rO rl rW r s= $ . . 3 R 2 5 & @ @ 7 5 :
?
H %2 ! 78Q($ Q( 9Q(h \ 2 G)ryy G)T + 0 1 0
J
): J
Z6
) 6
r 'E $? "; rl