ZjSSKJr SSKJr SSKrSSKJr SSKJrJrJ r SSK J r SSK Jr SS KJr SS KJr SS KJrJrJrJrJrJrJrJr SS KJrJr SS KJ r SSK!J"r"J#r#J$r$J%r% SSK&J'r'J(r( SSK)J*r* SSK+J,r, \%RZ"\.5r/"SS\R`5r1\Rd\1S.r3"SS\R`5r4S_S\R`S\RjS\RjS\RjS\RjS-S\6S-S\6S\ \#4S jjr7"S!S"\R`5r8"S#S$\R`5r9"S%S&\R`5r:"S'S(\R`5r;"S)S*\R`5r<"S+S,\R`5r="S-S.\R`5r>"S/S0\R`5r?"S1S2\R`5r@"S3S4\R`5rA"S5S6\5rB"S7S8\R`5rC"S9S:\R`5rD"S;S<\R`5rE"S=S>\R`5rF"S?S@\R`5rG"SASB\R`5rH\$"SCSD\55rI\$"SESF9\"SGSH\"555rJ\$"SISJ\I55rK\$"SKSF9"SLSM\I55rL\$"SNSO\I55rM"SPSQ\R`5rN\$"SRSF9"SSST\I55rO\$"SUSF9"SVSW\I55rP\$"SXSY\I55rQ\$"SZS[\I55rR\$"S\S]\I55rS/S^QrTg)`)Callable) dataclassN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FN)create_bidirectional_mask)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingMaskedLMOutputMultipleChoiceModelOutputNextSentencePredictorOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack) ModelOutputTransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs)MobileBertConfigcf^\rSrSrSU4SjjrS\R S\R 4SjrSrU=r $)NoNorm7c>[TU]5 [R"[R "U55Ul[R"[R"U55UlgN) super__init__r Parametertorchzerosbiasonesweight)self feat_sizeeps __class__s ڃ/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/mobilebert/modeling_mobilebert.pyr(NoNorm.__init__8s@ LLY!78 ll5::i#89 input_tensorreturnc8XR-UR-$r&)r.r,)r/r6s r3forwardNoNorm.forward=skk)DII55r5)r,r.r& __name__ __module__ __qualname____firstlineno__r(r*Tensorr9__static_attributes__ __classcell__r2s@r3r#r#7s(: 6ELL6U\\66r5r#) layer_normno_normc ^\rSrSrSrU4SjrS S\RS-S\RS-S\RS-S\RS-S \R4 S jjr S r U=r $) MobileBertEmbeddingsDzGConstruct the embeddings from word, position and token_type embeddings.cb>[TU]5 URUlURUlURUl[ R "URURURS9Ul [ R "URUR5Ul [ R "URUR5Ul UR(aSOSnURU-n[ R"X1R5Ul[ UR""UR5Ul[ R&"UR(5UlUR-S[.R0"UR5R3S5SS9 g)N) padding_idxr r position_idsr F) persistent)r'r( trigram_inputembedding_size hidden_sizer Embedding vocab_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddingsLinearembedding_transformationNORM2FNnormalization_type LayerNormDropouthidden_dropout_probdropoutregister_bufferr*arangeexpand)r/configembed_dim_multiplierembedded_input_sizer2s r3r(MobileBertEmbeddings.__init__GsD #11$33!--!||F,=,=v?T?Tbhbubuv#%<<0N0NPVPbPb#c %'\\&2H2H&J\J\%]"$($6$6qA"114HH(* 2EGYGY(Z% !:!:;F J"8;PP ^^J/ \\*- r5) r^rarPr[rQrWrYrOrU)NNNN) r<r=r>r?__doc__r(r* LongTensor FloatTensorr@r9rArBrCs@r3rGrGDsQ 0.2260426 0##d*0((4/0&&- 0 ((4/ 0  00r5rGmodulequerykeyrpattention_maskscalingrakwargsc UcURS5S-n[R"XRSS55U-nUbX-n[R R USS9n[R RXURS9n[R"X5n U RSS5R5n X4$)NrMrqr rr)ptrainingr ) rtr*matmul transposerrwsoftmaxrar contiguous) rrrrprrrar attn_weights attn_outputs r3eager_attention_forwardrs**R.D(<<}}Q':;gEL!#4 ==((2(>L==((6??([L,,|3K''1-88:K  $$r5c^\rSrSrU4SjrS S\R S\R S\R S\RS-S\\ S \ \R 4 S jjr S r U=r $) MobileBertSelfAttentionc>[TU]5 XlURUl[ UR UR- 5UlURUR -UlUR S-Ul[R"UR UR5Ul [R"UR UR5Ul [R"UR(a UR O URUR5Ul[R "UR"5UlSUlg)NrF)r'r(renum_attention_headsinttrue_hidden_sizeattention_head_size all_head_sizerrrZrruse_bottleneck_attentionrQrpr_attention_probs_dropout_probra is_causalr/rer2s r3r( MobileBertSelfAttention.__init__s  #)#=#= #&v'>'>A[A['[#\ !558P8PP//5 YYv668J8JK 99V44d6H6HIYY'-'F'FF # #FL^L^`d`r`r  zz&"E"EF r5N query_tensor key_tensor value_tensorrrr7c URSSn/UQSPURP7nURU5R"U6R SS5nUR U5R"U6R SS5n UR U5R"U6R SS5n [R"URR[5n U "UUU U U4UR(dSOURRURS.UD6upU R "/UQSP76R#5n X4$)NrMr rqro)rar)shaperrviewrrrpr get_interfacere_attn_implementationrrrarrreshaper)r/rrrrrry hidden_shape query_layer key_layer value_layerattention_interfacerrs r3r9MobileBertSelfAttention.forwards<#(("- CCbC$*B*BC jj.33\BLLQPQR HHZ(--|<FFq!L jj.33\BLLQPQR (?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! "));;;;FFH ((r5) rrrerarrrrrrpr&r<r=r>r?r(r*r@rrrtupler9rArBrCs@r3rrsw,48 )ll)LL)ll ) ))D0 ) +, ) u|| ))r5rcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $)MobileBertSelfOutputct>[TU]5 URUl[R"UR UR 5Ul[UR"UR URS9Ul UR(d&[R"UR5Ul ggNr1)r'r(use_bottleneckrrZrdenser\r]layer_norm_epsr^r_r`rars r3r(MobileBertSelfOutput.__init__s $33YYv668O8OP  !:!:;F =1 "" LL7M}'FG r5)r^rrarr;rCs@r3rrs7BU\\ELLUZUaUar5rc^\rSrSrU4SjrS S\R S\R S\R S\R S\RS-S \\ S \ \R 4S jjr S r U=r $)MobileBertAttentioncb>[TU]5 [U5Ul[ U5Ulgr&)r'r(rr/routputrs r3r(MobileBertAttention.__init__s& +F3 *62 r5Nrrr layer_inputrrr7c ZUR"UUUU40UD6upxURXt5nXx4$r&)r/r) r/rrrrrrattention_outputrs r3r9MobileBertAttention.forwardsG*.     *  * & ;;'7E--r5)rr/r&rrCs@r3rrs348 .ll.LL.ll . \\ . ))D0 .+,. u|| ..r5rcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)MobileBertIntermediateic>[TU]5 [R"URUR 5Ul[UR[5(a[URUl gURUl gr&) r'r(rrZrintermediate_sizer isinstance hidden_actstrr intermediate_act_fnrs r3r(MobileBertIntermediate.__init__s` YYv668P8PQ f'' - -'-f.?.?'@D $'-'8'8D $r5rr7cJURU5nURU5nU$r&rrr/rs r3r9MobileBertIntermediate.forwards& =1 00? r5rr;rCs@r3rrs(9U\\ellr5rcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $)OutputBottleneckic.>[TU]5 [R"URUR 5Ul[UR"UR URS9Ul [R"UR5Ul gr)r'r(rrZrrQrr\r]rr^r_r`rars r3r(OutputBottleneck.__init__sh YYv668J8JK  !:!:;F U\\ELLUZUaUar5rc^\rSrSrU4SjrS\R S\R S\R S\R 4SjrSrU=r $) MobileBertOutputi+c>[TU]5 URUl[R"UR UR 5Ul[UR"UR 5Ul UR(d&[R"UR5Ul g[U5Ulgr&)r'r(rrrZrrrr\r]r^r_r`rar bottleneckrs r3r(MobileBertOutput.__init__,s $33YYv779P9PQ  !:!:;F>,*JKL >>,*JKL??<KLr5)r^rrrarr;rCs@r3rr+sD7 #(<< DILL ejeqeq    r5rcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)BottleneckLayeriCc>[TU]5 [R"URUR 5Ul[UR"UR URS9Ul gr) r'r(rrZrQintra_bottleneck_sizerr\r]rr^rs r3r(BottleneckLayer.__init__DsR YYv1163O3OP  !:!:;F[TU]5 URUlURUl[ U5UlUR(a[ U5Ulggr&)r'r(key_query_shared_bottleneckrrinput attentionrs r3r(Bottleneck.__init__PsP +1+M+M((.(G(G%$V,  + +,V4DN ,r5rr7cURU5nUR(aU4S-$UR(aURU5nX3X4$XX4$)N)rrrr)r/rbottlenecked_hidden_statesshared_attention_inputs r3r9Bottleneck.forwardXsX"&*ZZ %>"  ( (.014 4  - -%)^^M%B "*Mn n!-\ \r5)rrrr r<r=r>r?r(r*r@rr9rArBrCs@r3rrOs15]U\\]eELL6I]]r5rcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $) FFNOutputisc>[TU]5 [R"URUR 5Ul[UR"UR URS9Ul gr) r'r(rrZrrrr\r]rr^rs r3r(FFNOutput.__init__tsR YYv779P9PQ  !:!:;F[TU]5 [U5Ul[ U5Ulgr&)r'r(r intermediaterrrs r3r(FFNLayer.__init__s' 26:' r5rr7cJURU5nURX!5nU$r&rr)r/rintermediate_outputrs r3r9FFNLayer.forwards("// > $7G r5r r;rCs@r3rrs(( U\\ellr5rc ^\rSrSrU4SjrS S\R S\RS-S\\ S\R 4Sjjr S r U=r $) MobileBertLayeric>[TU]5 URUlURUl[ U5Ul[ U5Ul[U5Ul UR(a[U5Ul URS:aL[R"[URS- 5Vs/sHn[U5PM sn5Ulggs snfNr )r'r(rnum_feedforward_networksrrrrrrrrr ModuleListrangerffnr/re_r2s r3r(MobileBertLayer.__init__s $33(.(G(G%,V426:&v.   (0DO  * *Q .}}fFeFehiFi@j%k@j1hv&6@j%klDH /%ks=C Nrrrr7c 6UR(aURU5upEpgO U/S-upEpgUR"UUUUU40UD6upUn URS:waURH n U "U 5n M UR U 5n UR XU5n U $)Nrr )rrrrrrr)r/rrrrrrrself_attention_outputrr ffn_moduler rs r3r9MobileBertLayer.forwards   BF//R_B` ?LlKCP/TUBU ?Ll#'>>      $  $  1  ( (A -"hh #-.>#? '#//0@A{{#6-X r5)rrrrrrrr&) r<r=r>r?r(r*r@rrrr9rArBrCs@r3rrsW m 48||))D0+,   r5rc ^\rSrSrU4SjrS S\R S\RS-S\\ S\ \ -4Sjjr S r U=r$) MobileBertEncoderic>[TU]5 [R"[ UR 5Vs/sHn[ U5PM sn5Ulgs snfr&)r'r(rrrnum_hidden_layersrlayerrs r3r(MobileBertEncoder.__init__sB ]]U6KcKcEd#eEdOF$;Ed#ef #esANrrrr7c d[UR5HupEU"UU40UD6nM [US9$)N)last_hidden_state) enumerater r)r/rrri layer_modules r3r9MobileBertEncoder.forwards@ )4OA(M 5 ??r5)r r&)r<r=r>r?r(r*r@rrrrrr9rArBrCs@r3rrs^g48 @|| @))D0 @+, @  @ @r5rcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)MobileBertPooleric>[TU]5 URUlUR(a1[R "UR UR 5Ulggr&)r'r(classifier_activation do_activaterrZrQrrs r3r(MobileBertPooler.__init__sH !77   6#5#5v7I7IJDJ r5rr7cUSS2S4nUR(dU$URU5n[R"U5nU$)Nr)r,rr*tanh)r/rfirst_token_tensor pooled_outputs r3r9MobileBertPooler.forwardsE+1a40% % JJ'9:M!JJ}5M r5)rr,r;rCs@r3r)r)s)K !U\\ !ell ! !r5r)cb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)!MobileBertPredictionHeadTransformicb>[TU]5 [R"URUR5Ul[ UR[5(a[URUl OURUl [S"URURS9Ul g)NrDr)r'r(rrZrQrrrrr transform_act_fnr\rr^rs r3r(*MobileBertPredictionHeadTransform.__init__s YYv1163E3EF f'' - -$*6+<+<$=D !$*$5$5D ! .v/A/AvG\G\]r5rr7clURU5nURU5nURU5nU$r&)rr6r^rs r3r9)MobileBertPredictionHeadTransform.forwards4 =1 --m< }5 r5)r^rr6r;rCs@r3r4r4s)^U\\ellr5r4cb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)MobileBertLMPredictionHeadic>[TU]5 [U5Ul[R "UR URUR- SS9Ul [R "URUR SS9Ul [R"[R"UR 55Ulg)NF)r,T)r'r(r4 transformrrZrSrQrPrdecoderr)r*r+r,rs r3r(#MobileBertLMPredictionHead.__init__s :6BYYv00&2D2DvG\G\2\chi yy!6!68I8IPTU LLV->->!?@ r5rr7cURU5nUR[R"URR R 5URR /SS95nXRR- nU$)Nrrr) r=rr*rvr>r.trr,rs r3r9"MobileBertLMPredictionHead.forwardsi}5 %,,UYY 8K8K8M8M8OQUQ[Q[QbQb7cij-kl *** r5)r,r>rr=r;rCs@r3r;r;s)AU\\ellr5r;cb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)MobileBertOnlyMLMHeadicB>[TU]5 [U5Ulgr&)r'r(r; predictionsrs r3r(MobileBertOnlyMLMHead.__init__s 5f=r5sequence_outputr7c(URU5nU$r&rF)r/rHprediction_scoress r3r9MobileBertOnlyMLMHead.forwards ,,_=  r5rJr;rCs@r3rDrDs(>!u||! !!r5rDc^\rSrSrU4SjrS\R S\R S\\R 4SjrSr U=r $)MobileBertPreTrainingHeadsi c>[TU]5 [U5Ul[R "UR S5UlgNrq)r'r(r;rFrrZrQseq_relationshiprs r3r(#MobileBertPreTrainingHeads.__init__ s4 5f= " &*<*[TU]U5 [U[5(aA[R "UR 5 [R"UR5 g[U[5(a![R "UR 5 g[U[5(a\[R"UR[R"URRS5R!S55 gg)zInitialize the weightsrMrLN)r' _init_weightsrr#initzeros_r,ones_r.r;rGcopy_rKr*rcrrd)r/rr2s r3r\'MobileBertPreTrainedModel._init_weights$s f% ff % % KK $ JJv}} %  : ; ; KK $  4 5 5 JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h6r5)r<r=r>r?r!__annotations__base_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrr_can_record_outputsr*no_gradr\rArBrCs@r3rXrXsV $&*#N"&(-  ]]_ i ir5rXz6 Output type of [`MobileBertForPreTraining`]. ) custom_introc\rSrSr%SrSr\RS-\S'Sr \RS-\S'Sr \RS-\S'Sr \ \RS-\S'Sr \ \RS-\S'S rg) MobileBertForPreTrainingOutputi1ar loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). seq_relationship_logits (`torch.FloatTensor` of shape `(batch_size, 2)`): Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). Nlossprediction_logitsseq_relationship_logitsrrZrb)r<r=r>r?r}ror*rrcrprqrrrZrArbr5r3rnrn1s~ &*D%  d ")26u((4/68<U..5<59M5**+d2926Je''(4/6r5rnc^\rSrSrSrSU4SjjrSrSr\\ \ SS\ RS-S\ RS-S \ RS-S \ RS-S \ RS-S \\S \\-4Sjj555rSrU=r$)MobileBertModeliJz* https://huggingface.co/papers/2004.02984 c>[TU]U5 XlSUl[ U5Ul[ U5UlU(a [U5OSUl UR5 g)z^ add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer FN) r'r(regradient_checkpointingrGr{rencoderr)pooler post_init)r/readd_pooling_layerr2s r3r(MobileBertModel.__init__PsT   &+#.v6(0 2C&v.  r5c.URR$r&r{rUr/s r3get_input_embeddings$MobileBertModel.get_input_embeddingsas...r5c$XRlgr&r|)r/rps r3set_input_embeddings$MobileBertModel.set_input_embeddingsds */'r5NrirrjrKrkrr7c USLUSL- (a [S5eURUUUUS9n[URUUS9nUR"U4SU0UD6nUSn UR bUR U 5OSn [ U U S9$)Nz:You must specify exactly one of input_ids or inputs_embeds)rirKrjrk)rerkrrr)r# pooler_output) ValueErrorr{r rervrwr) r/rirrjrKrkrembedding_outputencoder_outputsrHr1s r3r9MobileBertModel.forwardgs  -t"; <YZ Z??%)' + 3;;*)  ,,  )   *!,8< 8O O4UY )-'  r5)rer{rvrurw)T)NNNNN)r<r=r>r?r}r(r~rrrrr*r~rrrrrr9rArBrCs@r3rsrsJs"/0 .237260426 $ ##d*$ ))D0$ ((4/ $ &&- $ ((4/ $ +,$  + +$  $ r5rsz MobileBert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next sentence prediction (classification)` head. c^\rSrSrSSS.rU4SjrSrSrSS \S-S \ R4U4S jjjr \ \ SS \RS-S \R S-S\RS-S\RS-S\R S-S\RS-S\RS-S\\S \\-4Sjj55rSrU=r$)MobileBertForPreTrainingicls.predictions.bias,mobilebert.embeddings.word_embeddings.weightzcls.predictions.decoder.biaszcls.predictions.decoder.weightc>[TU]U5 [U5Ul[ U5UlUR 5 gr&)r'r(rsrYrNclsrxrs r3r(!MobileBertForPreTraining.__init__s4  )&1-f5 r5cBURRR$r&rrFr>r}s r3get_output_embeddings.MobileBertForPreTraining.get_output_embeddingsxx##+++r5cXRRlURURRlgr&rrFr>r,r/new_embeddingss r3set_output_embeddings.MobileBertForPreTraining.set_output_embeddings*'5$$2$7$7!r5Nnew_num_tokensr7c>URURRRUSS9URRl[TU]US9$NT)r transposed)r_get_resized_lm_headrrFrr'resize_token_embeddingsr/rr2s r3r0MobileBertForPreTraining.resize_token_embeddingssR%)%>%> HH & &~RV&?& "w.n.MMr5rirrjrKrklabelsnext_sentence_labelrc UR"U4UUUUSS.UD6n U SSupURX5upSnUbvUbs[5nU"U RSURR 5URS55nU"U RSS5URS55nUU-n[ UU U U RU RS9$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring) Indices should be in `[0, 1]`: - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Examples: ```python >>> from transformers import AutoTokenizer, MobileBertForPreTraining >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased") >>> model = MobileBertForPreTraining.from_pretrained("google/mobilebert-uncased") >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) >>> # Batch size 1 >>> outputs = model(input_ids) >>> prediction_logits = outputs.prediction_logits >>> seq_relationship_logits = outputs.seq_relationship_logits ```TrrjrKrk return_dictNrqrM)rorprqrrZ) rYrrrrerSrnrrZ)r/rirrjrKrkrrroutputsrHr1rKrU total_lossloss_fctmasked_lm_lossnext_sentence_losss r3r9 MobileBertForPreTraining.forwardsR//  ))%'   *1!&48HH_4\1  "5"A')H%&7&<&r?_tied_weights_keysr(rrrrrRrrrr*r~rrrrrnr9rArBrCs@r3rrs5)?*X ,8NcDjNBLLNN.237260426*.7;@ ##d*@ ))D0@ ((4/ @ &&- @ ((4/ @   4'@ #--4@ +,@  / /@ @ r5rcj^\rSrSrSSS.rU4SjrSrSrSS \S-S \ R4U4S jjjr \ \ SS \RS-S \R S-S\RS-S\RS-S\R S-S\RS-S\\S \\-4Sjj55rSrU=r$)MobileBertForMaskedLMirrrc>[TU]U5 [USS9Ul[ U5UlXlUR5 gNF)ry)r'r(rsrYrDrrerxrs r3r(MobileBertForMaskedLM.__init__s;  )&EJ(0  r5cBURRR$r&rr}s r3r+MobileBertForMaskedLM.get_output_embeddings rr5cXRRlURURRlgr&rrs r3r+MobileBertForMaskedLM.set_output_embeddings rr5Nrr7c>URURRRUSS9URRl[TU]US9$rrrs r3r-MobileBertForMaskedLM.resize_token_embeddingssR%)%>%> HH & &~RV&?& "w.n.MMr5rirrjrKrkrrc 6UR"U4UUUUSS.UD6nUSn URU 5n Sn UbF[5n U "U RSURR 5URS55n [ U U URURS9$)az labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` TrrNrMrologitsrrZ) rYrrrrerSrrrZ) r/rirrjrKrkrrrrHrKrrs r3r9MobileBertForMaskedLM.forwards$//  ))%'   "!* HH_5  ')H%&7&<&r?rr(rrrrrRrrrr*r~rrrrrr9rArBrCs@r3rrs)?*X ,8NcDjNBLLNN.237260426*.' ##d*' ))D0' ((4/ ' &&- ' ((4/ '   4'' +,'   ' ' r5rcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)MobileBertOnlyNSPHeadiCcn>[TU]5 [R"URS5UlgrP)r'r(rrZrQrQrs r3r(MobileBertOnlyNSPHead.__init__Ds'  " &*<*[TU]U5 [U5Ul[ U5UlUR 5 gr&)r'r(rsrYrrrxrs r3r(,MobileBertForNextSentencePrediction.__init__Ss4  )&1(0 r5NrirrjrKrkrrr7c UR"U4UUUUSS.UD6nUSn URU 5n Sn Ub2[5n U "U RSS5URS55n [ U U UR UR S9$)a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring) Indices should be in `[0, 1]`. - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Examples: ```python >>> from transformers import AutoTokenizer, MobileBertForNextSentencePrediction >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google/mobilebert-uncased") >>> model = MobileBertForNextSentencePrediction.from_pretrained("google/mobilebert-uncased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light." >>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt") >>> outputs = model(**encoding, labels=torch.LongTensor([1])) >>> loss = outputs.loss >>> logits = outputs.logits ```Trr NrMrqr)rYrrrrrrZ) r/rirrjrKrkrrrr1rUrrs r3r9+MobileBertForNextSentencePrediction.forward\sL//  ))%'     !%-!8!  ')H!)*@*E*Eb!*LfkkZ\o!^ *#)!//))   r5rr)r<r=r>r?r(rrr*r~rrrrrr9rArBrCs@r3rrMs .237260426*.; ##d*; ))D0; ((4/ ; &&- ; ((4/ ;   4'; +,;  , ,; ; r5rz MobileBert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. c8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)#MobileBertForSequenceClassificationicr>[TU]U5 URUlXl[ U5UlUR b UR O URn[R"U5Ul [R"URUR5Ul UR5 gr&)r'r( num_labelsrersrYclassifier_dropoutr`rr_rarZrQ classifierrxr/rerr2s r3r(,MobileBertForSequenceClassification.__init__s   ++ )&1)/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ r5NrirrjrKrkrrr7c UR"U4UUUUSS.UD6nUSn URU 5n URU 5n Sn UGbURRcUR S:XaSURlOoUR S:aNUR [R:XdUR [R:XaSURlOSURlURRS:XaI[5n UR S:Xa&U "U R5UR55n OU "X5n OURRS:Xa=[5n U "U RSUR 5URS55n O,URRS:Xa[5n U "X5n [U U UR UR"S 9$) ae labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Trr N regressionsingle_label_classificationmulti_label_classificationrMr)rYrarre problem_typerrmr*rurrsqueezerrrrrrZ) r/rirrjrKrkrrrr1rrors r3r9+MobileBertForSequenceClassification.forwards$//  ))%'      ]3 /  {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#F3D))-JJ+- B @&++b/R))-II,./'!//))   r5)rrerarYrr)r<r=r>r?r(rrr*r@rrrrr9rArBrCs@r3rrs *..2.2,0-1&*; <<$&;  t+;  t+ ; llT) ; ||d* ;  t#; +,;  u|| 7 7; ; r5rcX^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \RS-S \ \ S \ \R\ -4S jj55r SrU=r$)MobileBertForQuestionAnsweringic>[TU]U5 URUl[USS9Ul[ R "URUR5UlUR5 gr) r'r(rrsrYrrZrQ qa_outputsrxrs r3r('MobileBertForQuestionAnswering.__init__sU   ++)&EJ))F$6$68I8IJ r5NrirrjrKrkstart_positions end_positionsrr7c UR"U4UUUUSS.UD6n U Sn URU 5n U RSSS9upU RS5R 5n U RS5R 5n SnUbUb[ UR 55S:aURS5n[ UR 55S:aURS5nU R S5nURSU5nURSU5n[US9nU"X5nU"X5nUU-S- n[UU U U RU RS 9$) NTrrr rMrr) ignore_indexrq)ro start_logits end_logitsrrZ) rYrsplitrrlenrtclamprrrrZ)r/rirrjrKrkrrrrrHrrrr ignored_indexr start_lossend_losss r3r9&MobileBertForQuestionAnswering.forwardsx//  ))%'   "!*1#)<<r<#: #++B/::< ''+668   &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M-33A}EO)//=AM']CH!,@J :H$x/14J+%!!//))   r5)rYrrr)r<r=r>r?r(rrr*r@rrrrr9rArBrCs@r3rrs*..2.2,0-1/3-13 <<$&3  t+3  t+ 3 llT) 3 ||d* 3 ,3 ||d*3 +,3  u|| ; ;3 3 r5rc8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)MobileBertForMultipleChoicei8c0>[TU]U5 [U5UlURb URO UR n[ R"U5Ul[ R"URS5Ul UR5 gr) r'r(rsrYrr`rr_rarZrQrrxrs r3r($MobileBertForMultipleChoice.__init__;su  )&1)/)B)B)NF % %TZTnTn zz"45 ))F$6$6: r5NrirrjrKrkrrr7c UbURSOURSnUb!URSURS55OSnUb!URSURS55OSnUb!URSURS55OSnUb!URSURS55OSnUb1URSURS5URS55OSnUR"U4UUUUSS.UD6n U Sn UR U 5n UR U 5n U RSU5n Sn Ub[ 5nU"X5n [U U U RU RS9$)a input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr rMTrr) rrrtrYrarrrrrZ)r/rirrjrKrkrr num_choicesrr1rreshaped_logitsrors r3r9#MobileBertForMultipleChoice.forwardHsT-6,Aiooa(}GZGZ[\G] >G>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei (   r=#5#5b#9=;M;Mb;Q R  //  ))%'      ]3 / ++b+6  ')HO4D("!//))   r5)rrarYr)r<r=r>r?r(rrr*r@rrrrr9rArBrCs@r3rr8s *..2.2,0-1&*N <<$&N  t+N  t+ N llT) N ||d* N  t#N +,N  u|| 8 8N N r5rc8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$) MobileBertForTokenClassificationicd>[TU]U5 URUl[USS9UlUR b UR O UR n[R"U5Ul [R"URUR5Ul UR5 gr)r'r(rrsrYrr`rr_rarZrQrrxrs r3r()MobileBertForTokenClassification.__init__s   ++)&EJ)/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ r5NrirrjrKrkrrr7c DUR"U4UUUUSS.UD6nUSn URU 5n URU 5n Sn Ub<[5n U "U R SUR 5UR S55n [ U U URURS9$)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. TrrNrMr) rYrarrrrrrrZ) r/rirrjrKrkrrrrHrrors r3r9(MobileBertForTokenClassification.forwards //  ))%'   "!*,,71  ')HFKKDOOr?r(rrr*r@rrrrr9rArBrCs@r3rrs *..2.2,0-1&*' <<$&'  t+'  t+ ' llT) ' ||d* '  t#' +,'  u|| 4 4' ' r5r) rrrrrrrrrsrX)Nro)Ucollections.abcr dataclassesrr*rtorch.nnrrrr r] activationsr masking_utilsr modeling_layersr modeling_outputsrrrrrrrrmodeling_utilsrrprocessing_utilsrutilsrrrr utils.genericrrutils.output_capturingrconfiguration_mobilebertr! get_loggerr<loggerModuler#r^r\rGr@floatrrrrrrrrrrrrrr)r4r;rDrNrXrnrsrrrrrrrr__all__rbr5r3rs.%! AA&!69   G&MMI56   H %6RYY6& 9I299If!% II% <<% % << % LL4' % T\ %%'(%81)bii1)h299".")).8 RYY  ryy ryy0 bii !]!]H   ryy )0)X@ @(!ryy!& ""!BII! 9 9iii4  7[7  7&C /C C L _ 8_  _ DF 5F F R&BII& G *CG  G TM *CM M `@ %>@ @ F^ ";^ ^ B8 '@8 8 v r5