ZjzSSKJr SSKrSSKJr SSKJs Jr SSKJ r SSK J r SSK J r SSKJr SSKJr S S KJrJrJrJrJrJrJrJrJr S S KJr \R<"\5r "S S\RB5r""SS\5r#"SS\5r$SSjr%"SS\5r&"SS\5r'"SS\5r("SS\5r)"SS\5r*/SQr+g) )CallableN)Cache)dynamic_rope_update)ALL_ATTENTION_FUNCTIONS)logging)maybe_autocast) LlamaAttentionLlamaDecoderLayerLlamaForCausalLMLlamaForSequenceClassificationLlamaMLP LlamaModelLlamaRotaryEmbeddingeager_attention_forward rotate_half) OlmoConfigcr^\rSrSrSrS\SS4U4SjjrS\RS\R4Sjr S r U=r $) OlmoLayerNorm0z/LayerNorm but with no learnable weight or bias. hidden_sizereturnNc2>[TU]5 U4UlgN)super__init__normalized_shape)selfr __class__s v/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/olmo/modular_olmo.pyrOlmoLayerNorm.__init__3s !, hidden_statescURn[R"UR[R S9UR SSSS9RU5$)N)dtypegh㈵>)eps)r'F layer_normtotorchfloat32r)r r% orig_dtypes r"forwardOlmoLayerNorm.forward7sO"(( ||M,,5==,A4CXCXZ^`djnorr   r$)r) __name__ __module__ __qualname____firstlineno____doc__intrr,Tensorr/__static_attributes__ __classcell__r!s@r"rr0s99/C/D/ U\\ ell  r$rc(^\rSrSrU4SjrSrU=r$)OlmoMLP>c>>[TU]U5 [R"URUR SS9Ul[R"URUR SS9Ul[R"UR URSS9Ulg)NF)bias) rrnnLinearrintermediate_size gate_projup_proj down_proj)r configr!s r"rOlmoMLP.__init__?ss  4#3#3T5K5KRWXyy!1!143I3IPUV 4#9#94;K;KRWXr$)rErCrD)r1r2r3r4rr8r9r:s@r"r<r<>sYYr$r<cL\rSrSr\R "5\S55rSrg)OlmoRotaryEmbeddingHcURSSS2S4R5RURSSS5R UR 5nUSS2SSS24R5n[ UR R[5(a0UR RS:waUR ROSn[USS9 UR5UR5-RSS5n[R"Xf4SS 9nUR5UR-nUR5UR-n SSS5 X4$!,(df  WW 4$=f) NrrmpscpuF) device_typeenabledr )dim)inv_freqfloatexpandshaper+device isinstancetypestrr transposer,catcosattention_scalingsin) r x position_idsinv_freq_expandedposition_ids_expandedrOfreqsembr\r^s r"r/OlmoRotaryEmbedding.forwardIs7!MM$4-8>>@GG HZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E'E!((--[`J`ahhmmfk  U C&,,.1F1L1L1NNYYZ[]^_E))UN3C'')d444C'')d444C D x D C Cxs BE&& E7N) r1r2r3r4r,no_gradrr/r8rfr$r"rIrIHs" ]]_  r$rIcURURpeURU5nURU5nX-[U5U--nX-[U5U--nURU5URU54$)aIApplies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. )r' unsqueezerr+) qkr\r^ unsqueeze_dimq_typek_typeq_embedk_embeds r"apply_rotary_pos_embrqXsv$WWaggF -- &C -- &Cw;q>C/0Gw;q>C/0G ::f wzz&1 11r$c\rSrSrS S\R S\\R \R 4S\R S-S\S-S\\R \R S-44 SjjrS r g) OlmoAttentionrNr%position_embeddingsattention_maskpast_key_valuesrc URSSn/UQSPURP7nURU5nURU5n UR U5n UR R bURUR R *UR R S9 U RUR R *UR R S9 U RUR R *UR R S9 URU5RSS5nU RU5RSS5n U RU5RSS5n Uup[XX5upUbURXUR5up[R"UR R[ 5n U "UUU U U4UR"(dSO UR$UR&S.UD6upUR("/UQSP76R+5nUR-U5nX4$)NrL)minmaxrr g)dropoutscaling)rUhead_dimq_projk_projv_projrFclip_qkvclamp_viewrZrqupdate layer_idxr get_interface_attn_implementationrtrainingattention_dropoutr|reshape contiguouso_proj)r r%rurvrwkwargs input_shape hidden_shape query_states key_states value_statesr\r^attention_interface attn_output attn_weightss r"r/OlmoAttention.forwardss$))#2. 88b8$--8 {{=1 [[/ {{=1 ;;   +   T[[%9%9$9t{{?S?S  T   4;;#7#7"7T[[=Q=Q  R   T[[%9%9$9t{{?S?S  T#((6@@AF __\2</) t+ /)  /) u||U\\D00 1/)/)r$rsc4^\rSrSrS\S\4U4SjjrSrU=r$)OlmoDecoderLayerrFrc>[TU]X5 [UR5Ul[UR5Ul[ XS9Ulg)N)rFr)rrrrinput_layernormpost_attention_layernormrs self_attnr rFrr!s r"rOlmoDecoderLayer.__init__sB +,V-?-?@(5f6H6H(I%&fJr$)rrr) r1r2r3r4rr6rr8r9r:s@r"rrsKzKcKKr$rc0^\rSrSrS\4U4SjjrSrU=r$) OlmoModelrFc >[TU]U5 [R"[ UR 5Vs/sHn[ X5PM sn5Ul[UR5Ul gs snfr) rrr@ ModuleListrangenum_hidden_layersrlayersrrnormrs r"rOlmoModel.__init__s_  mmBGH`H`Ba bBaY f 0Ba b  "&"4"45  csA4)rr)r1r2r3r4rrr8r9r:s@r"rrs6z66r$rc\rSrSrSrg)OlmoForCausalLMrfNr1r2r3r4r8rfr$r"rrr$rc\rSrSrSrg)OlmoForSequenceClassificationrfNrrfr$r"rrrr$r)rrrOlmoPreTrainedModel)r),collections.abcrr,torch.nnr@torch.nn.functional functionalr) cache_utilsrmodeling_rope_utilsrmodeling_utilsrutilsr utils.genericr llama.modeling_llamar r r rrrrrrconfiguration_olmor get_loggerr1loggerModulerr<rIrqrsrrrr__all__rfr$r"rs(%  65+   +   H %  BII  YhY .  240)N0)fK(K6 6 &  $B  r$