Zj*PSrSSKJr SSKrSSKJr SSKJr SSKJ r SSK J r SS K J r SS KJr S S KJr S S KJrJrJrJrJrJrJr SSKJr \R8"\5r"SS\5r"SS\5r "SS\5r!"SS\5r""SS\5r#"SS\5r$/SQr%g)zPyTorch BitNet model.)CallableN)Cache)FlashAttentionKwargs)CausalLMOutputWithPast)ALL_ATTENTION_FUNCTIONS)Unpack)logging)GemmaMLP)LlamaAttentionLlamaDecoderLayerLlamaForCausalLM LlamaModel LlamaRMSNormapply_rotary_pos_embeager_attention_forward) BitNetConfigc\rSrSrSrg) BitNetRMSNorm)N__name__ __module__ __qualname____firstlineno____static_attributes__rz/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/bitnet/modular_bitnet.pyrr)r rc6^\rSrSrS\4U4SjjrSrSrU=r$) BitNetMLP-configcj>[TU]U5 [URURS9UlgN)eps)super__init__rintermediate_size rms_norm_eps ffn_sub_norm)selfr& __class__s r!r+BitNetMLP.__init__.s+  )&*B*BH[H[\r c URURURURU55UR U5-55nU$N) down_projr.act_fn gate_projup_proj)r/xr4s r!forwardBitNetMLP.forward2sFNN4#4#4T[[PQAR5SVZVbVbcdVe5e#fg r )r.) rrrrrr+r9r __classcell__r0s@r!r$r$-s]|]r r$c^\rSrSrS\S\4U4SjjrSS\RS\ \R\R4S\RS-S \ S-S \ \ S \ \R\RS-44 S jjr S rU=r$)BitNetAttention7r& layer_idxcj>[TU]X5 [URURS9Ulgr()r*r+r hidden_sizer- attn_sub_norm)r/r&r@r0s r!r+BitNetAttention.__init__8s+ +*6+=+=6CVCVWr N hidden_statesposition_embeddingsattention_maskpast_key_valueskwargsreturnc *URSSn/UQSPURP7nURU5RU5R SS5nUR U5RU5R SS5n UR U5RU5R SS5n Uup[XX5upUbURXUR5up[R"URR[5n U "UUU U U4UR(dSO UR UR"S.UD6upUR$"/UQSP76R'5nUR)U5nUR+U5nX4$)Nrr g)dropoutscaling)shapehead_dimq_projview transposek_projv_projrupdater@r get_interfacer&_attn_implementationrtrainingattention_dropoutrNreshape contiguousrCo_proj)r/rErFrGrHrI input_shape hidden_shape query_states key_states value_statescossinattention_interface attn_output attn_weightss r!r9BitNetAttention.forward<s$))#2. 88b8$--8 {{=166|DNNqRST [[/44\BLLQPQR {{=166|DNNqRST &#7 RU#[  &'6'='=jX\XfXf'g $J(?(M(M KK , ,.E) %8      % $}}C$2H2HLL %  % ! "));;;;FFH ((5 kk+. ((r )rCr3)rrrrrintr+torchTensortuplerr rr9rr;r<s@r!r>r>7sX|XX)- ')||')#5<<#=>') t+ ')  ') -. ') u||U\\D00 1')')r r>c\rSrSrSrg)BitNetDecoderLayerfrNrrr r!rnrnfr"r rnc\rSrSrSrg) BitNetModeljrNrrr r!rqrqjr"r rqc@^\rSrSrSS0rSrSrS\4U4SjjrSr U=r $)BitNetForCausalLMnzlm_head.weightzmodel.embed_tokens.weightNrJc $>[TU]"S0UD6$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, transformers., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, transformers., config.vocab_size]`. Example: ```python >>> from transformers import AutoTokenizer, BitNetForCausalLM >>> model = BitNetForCausalLM.from_pretrained("microsoft/bitnet-b1.58-2B-4T") >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/bitnet-b1.58-2B-4T") >>> prompt = f'<|begin_of_text|>User: Hey, are you conscious? Can you talk to me?<|eot_id|>Assistant: ' >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=100) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "User: Hey, are you conscious? Can you talk to me?Assistant: No, I'm not conscious. I'm an artificial intelligence designed to assist with information and tasks. How can I help you today?" ```r)r*r9)r/ super_kwargsr0s r!r9BitNetForCausalLM.forwardss4w...r r) rrrr_tied_weights_keys_tp_plan_pp_planrr9rr;r<s@r!rtrtns-*,GHHH/ //r rt)rtrqBitNetPreTrainedModel)&__doc__collections.abcrrj cache_utilsrmodeling_flash_attention_utilsrmodeling_outputsrmodeling_utilsrprocessing_utilsr utilsr gemma.modeling_gemmar llama.modeling_llamar rrrrrrconfiguration_bitnetr get_loggerrloggerrr$r>rnrqrt__all__rr r!rs$ B65&+/   H % L ,)n,)^ *  * /(/D r