ZjSrSSKJr SSKJr SSKrSSKJr SSKJrJ r J r SSK J r SS KJr SS KJrJrJr SS KJr SS KJrJr SS KJr SSKJrJrJrJrJ r J!r!J"r"J#r#J$r$ SSK%J&r&J'r' SSK(J)r) SSK*J+r+ SSK,J-r-J.r.J/r/J0r0 SSK1J2r2J3r3 SSK4J5r5 SSK6J7r7 \0Rp"\95r:"SS\Rv5r<S\S\RvS\RzS\RzS\RzS\RzS-S\>S-S\>S \)\.4S!jjr?"S"S#\Rv5r@"S$S%\Rv5rA"S&S'\Rv5rB"S(S)\Rv5rC"S*S+\Rv5rD"S,S-\Rv5rE"S.S/\5rF"S0S1\Rv5rG"S2S3\Rv5rH"S4S5\Rv5rI"S6S7\Rv5rJ"S8S9\Rv5rK"S:S;\Rv5rL"S<S=\Rv5rM\/"S>S?\'55rN\/"S@SA9\"SBSC\-555rO\/"SDSA9"SESF\N55rP\/"SGSA9"SHSI\N55rQ\/"SJSA9"SKSL\N\55rR\/"SMSN\N55rS\/"SOSA9"SPSQ\N55rT\/"SRSA9"SSST\N55rU\/"SUSV\N55rV\/"SWSX\N55rW\/"SYSZ\N55rX/S[QrYg)]zPyTorch BERT model.)Callable) dataclassN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FN)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer) )BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputNextSentencePredictorOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward) ModelOutputTransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs) BertConfigc ^\rSrSrSrU4SjrS S\RS-S\RS-S\RS-S\RS-S \ S \R4 S jjr S r U=r $)BertEmbeddings5zGConstruct the embeddings from word, position and token_type embeddings.c >[TU]5 [R"URUR UR S9Ul[R"URUR 5Ul [R"URUR 5Ul [R"UR URS9Ul [R"UR5UlUR#S[$R&"UR5R)S5SS9 UR#S[$R*"UR,R/5[$R0S9SS9 g) N) padding_idxeps position_idsr'F) persistenttoken_type_ids)dtype)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandzerosr0sizelongselfconfig __class__s w/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/bert/modeling_bert.pyr7BertEmbeddings.__init__8s !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c %'\\&2H2H&J\J\%]"f&8&8f>S>STzz&"<"<=   ELL)G)GHOOPWXej    ekk$*;*;*@*@*B%**Ubg  N input_idsr4r0 inputs_embedspast_key_values_lengthreturncUbUR5nOUR5SSnUupxUcURSS2XXU-24nUc[US5(aQURR UR SS5n [ R"U SUS9n U R Xx5nO8[ R"U[ RURRS9nUcURU5nURU5n XJ-n URU5n X-n URU 5n URU 5n U $)Nr2r4rr')dimindex)r5device)rKr0hasattrr4rIshaperGgatherrJrLr[r<r@r>rArE) rNrTr4r0rUrV input_shape batch_size seq_lengthbuffered_token_type_idsr@ embeddingsr>s rQforwardBertEmbeddings.forwardHsG  #..*K',,.s3K!,  ,,Q0FVlIl0l-lmL  !t-..*.*=*=*D*D\EWEWXYEZ\^*_'*/,,7NTU]i*j'!8!?!? !W!&[ SWSdSdSkSk!l   00;M $ : :> J": "66|D5 ^^J/ \\*- rS)rArEr>r@r<)NNNNr)__name__ __module__ __qualname____firstlineno____doc__r7rG LongTensor FloatTensorintTensorrd__static_attributes__ __classcell__rPs@rQr*r*5sQ $.2260426&' (##d*(((4/(&&- ( ((4/ ( !$ ( ((rSr*modulequerykeyvalueattention_maskscalingrEkwargsc UcURS5S-n[R"XRSS55U-nUbX-n[R R USS9n[R RXURS9n[R"X5n U RSS5R5n X4$)Nr2r rY)ptrainingr') rKrGmatmul transposer functionalsoftmaxrEr~ contiguous) rrrsrtrurvrwrErx attn_weights attn_outputs rQeager_attention_forwardrss**R.D(<<}}Q':;gEL!#4 ==((2(>L==((6??([L,,|3K''1-88:K  $$rSc ^\rSrSrS U4SjjrS S\R S\RS-S\S-S\ \ S\ \R 4 S jjr S r U=r$) BertSelfAttentionNcN>[TU]5 URUR-S:wa7[ US5(d&[ SURSURS35eXlURUl[URUR- 5UlURUR-Ul URS-Ul [R"URUR5Ul [R"URUR5Ul[R"URUR5Ul[R "UR"5UlUR&UlX lX0lgNrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()rz)r6r7r:num_attention_headsr\ ValueErrorrOrmattention_head_size all_head_sizerwrLinearrsrtrurCattention_probs_dropout_probrE is_decoder is_causal layer_idxrNrOrrrPs rQr7BertSelfAttention.__init__sM     : : :a ?PVXhHiHi#F$6$6#78 445Q8  #)#=#= #&v'9'9F??*9*N*N'&=%C%CI\`\j\j%k "I(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! "));;;;FFH ((rS) rrrOrErrrtrrrsrwruFN)NNrfrgrhrir7rGrnrlr rr!tuplerdrorprqs@rQrrsl#648(, ')||')))D0') ') +, ') u||  ')')rSrc^\rSrSrS U4SjjrS S\R S\RS-S\RS-S\S-S\ \ S \ \R 4 S jjr S r U=r$)BertCrossAttentionNc,>[TU]5 URUR-S:wa7[ US5(d&[ SURSURS35eXlURUl[URUR- 5UlURUR-Ul URS-Ul [R"URUR5Ul [R"URUR5Ul[R"URUR5Ul[R "UR"5UlX lX0lgr)r6r7r:rr\rrOrmrrrwrrrsrtrurCrrErrrs rQr7BertCrossAttention.__init__s@     : : :a ?PVXhHiHi#F$6$6#78 445Q8  #)#=#= #&v'9'9FB**4>>:(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! "));;;;FFH ((rS) rrrOrErrtrrrsrwrur)NNN)rfrgrhrir7rGrnrlrrr!rrdrorprqs@rQrrs#4;?376: 1)||1) %00471)))D0 1) -t3 1) +, 1) u|| 1)1)rSrcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $)BertSelfOutputic(>[TU]5 [R"URUR5Ul[R "URURS9Ul[R"UR5Ul gNr.) r6r7rrr:denserArBrCrDrErMs rQr7BertSelfOutput.__init__s` YYv1163E3EF f&8&8f>S>STzz&"<"<= rSr input_tensorrWcpURU5nURU5nURX-5nU$NrrErArNrrs rQrdBertSelfOutput.forward%5 =1  ]3 }'CD rSrArrE rfrgrhrir7rGrnrdrorprqs@rQrr6> U\\RWR^R^rSrc^\rSrSrS U4SjjrSS\R S\RS-S\RS-S\RS-S\S-S \ \ S \ \R 4S jjr S r U=r$) BertAttentioni,Nc>[TU]5 X@lU(a[O[nU"XUS9Ul[ U5Ulg)Nrr)r6r7is_cross_attentionrrrNroutput)rNrOrrrattention_classrPs rQr7BertAttention.__init__-s9 "40B,HY#F9U $V, rSrrvrencoder_attention_maskrrxrWc UR(dUOUnUR"U4UUUS.UD6upxURXq5nXx4$)N)rrvr)rrNr) rNrrvrrrrxattention_outputrs rQrdBertAttention.forward4s\04/F/FLb)- * "7)+ *  * & ;;'7G--rS)rrrN)FNFNNNNrrqs@rQrr,s-48:>;?(, .||.))D0. %0047 . !& 1 1D 8 .  .+,. u|| ..rSrcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)BertIntermediateiIc>[TU]5 [R"URUR 5Ul[UR[5(a[URUl gURUl gr) r6r7rrr:intermediate_sizerr hidden_actstrr intermediate_act_fnrMs rQr7BertIntermediate.__init__Js` YYv1163K3KL f'' - -'-f.?.?'@D $'-'8'8D $rSrrWcJURU5nURU5nU$rrrrNrs rQrdBertIntermediate.forwardRs& =1 00? rSrrrqs@rQrrIs(9U\\ellrSrcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $) BertOutputiXc(>[TU]5 [R"URUR 5Ul[R"UR URS9Ul[R"UR5Ul gr) r6r7rrrr:rrArBrCrDrErMs rQr7BertOutput.__init__Ys` YYv779K9KL f&8&8f>S>STzz&"<"<= rSrrrWcpURU5nURU5nURX-5nU$rrrs rQrdBertOutput.forward_rrSrrrqs@rQrrXrrSrc^\rSrSrSU4SjjrSS\R S\RS-S\RS-S\RS-S\S-S \ \ S \R 4S jjr S r S r U=r$) BertLayerifNc>[TU]5 URUlSUl[ XR US9UlUR UlURUlUR(a0UR (d[US35e[ USUSS9Ul [U5Ul [U5Ul g)Nr'rz> should be used as a decoder model if cross attention is addedFT)rrr)r6r7chunk_size_feed_forward seq_len_dimrr attentionadd_cross_attentionrcrossattentionr intermediaterr)rNrOrrPs rQr7BertLayer.__init__gs '-'E'E$&v9J9JV_` ++#)#=#=  # #?? D6)g!hii"/##' #D  -V4 ( rSrrvrrrrxrWc 2UR"UU4SU0UD6upxUn UR(a?Ub<[US5(d[SUS35eUR"USUU4SU0UD6upU n [ UR URURU 5n U $)Nrrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`) rrr\rrrfeed_forward_chunkrr) rNrrvrrrrxself_attention_output_rcross_attention_output layer_outputs rQrdBertLayer.forwardzs$(>>  $ ,$  $   1 ??4@4!122 =dVDDD )-(;(;%%& ) !0 )  ) % " 6 0  # #T%A%A4CSCSUe rScJURU5nURX!5nU$r)rr)rNrintermediate_outputrs rQrBertLayer.feed_forward_chunks)"//0@A{{#6I rS)rrrrrrrrrr)rfrgrhrir7rGrnrlr rr!rdrrorprqs@rQrrfs),48:>;?(, %||%))D0% %0047 % !& 1 1D 8 %  %+,% %NrSrc^\rSrSrU4SjrSS\R S\RS-S\RS-S\RS-S\S-S \ S-S \ \ S \ \R \ -4S jjrS rU=r$) BertEncoderic >[TU]5 Xl[R"[ UR 5Vs/sH n[XS9PM sn5Ulgs snf)N)r) r6r7rOr ModuleListrangenum_hidden_layersrlayer)rNrOirPs rQr7BertEncoder.__init__sH  ]]ERXRjRjLk#lLkqIf$BLk#lm #lsANrrvrrr use_cacherxrWc [UR5HupU "UUU4UUS.UD6nM [UU(aUS9$SS9$)N)rr)last_hidden_stater) enumeraterr) rNrrvrrrr rxr layer_modules rQrdBertEncoder.forwardsg )4OA(%(> /   M 59+/8O  >B  rS)rOr)NNNNN)rfrgrhrir7rGrnrlr boolrr!rrrdrorprqs@rQrrsn48:>;?(,!% || ))D0  %0047  !& 1 1D 8    $; +,  u|| H H  rSrcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $) BertPooleric>[TU]5 [R"URUR5Ul[R "5Ulgr)r6r7rrr:rTanh activationrMs rQr7BertPooler.__init__s9 YYv1163E3EF '')rSrrWc\USS2S4nURU5nURU5nU$)Nr)rr)rNrfirst_token_tensor pooled_outputs rQrdBertPooler.forwards6+1a40 #56  6 rS)rrrrqs@rQrrs($ U\\ellrSrcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)BertPredictionHeadTransformicp>[TU]5 [R"URUR5Ul[ UR[5(a[URUl OURUl [R"URURS9Ul gr) r6r7rrr:rrrrr transform_act_fnrArBrMs rQr7$BertPredictionHeadTransform.__init__s~ YYv1163E3EF f'' - -$*6+<+<$=D !$*$5$5D !f&8&8f>S>STrSrrWclURU5nURU5nURU5nU$r)rrrArs rQrd#BertPredictionHeadTransform.forwards4 =1 --m< }5 rS)rArrrrqs@rQrrs)UU\\ellrSrc.^\rSrSrU4SjrSrSrU=r$)BertLMPredictionHeadic>[TU]5 [U5Ul[R "UR URSS9Ul[R"[R"UR55Ul g)NT)bias) r6r7r transformrrr:r9decoder ParameterrGrJr%rMs rQr7BertLMPredictionHead.__init__s[ 4V<yy!3!3V5F5FTR LLV->->!?@ rScJURU5nURU5nU$r)r&r'rs rQrdBertLMPredictionHead.forwards$}5  ]3 rS)r%r'r&rfrgrhrir7rdrorprqs@rQr#r#sArSr#cb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)BertOnlyMLMHeadicB>[TU]5 [U5Ulgr)r6r7r# predictionsrMs rQr7BertOnlyMLMHead.__init__s /7rSsequence_outputrWc(URU5nU$rr0)rNr2prediction_scoress rQrdBertOnlyMLMHead.forwards ,,_=  rSr4rrqs@rQr.r.s(8!u||! !!rSr.c.^\rSrSrU4SjrSrSrU=r$)BertOnlyNSPHeadicn>[TU]5 [R"URS5UlgNr{)r6r7rrr:seq_relationshiprMs rQr7BertOnlyNSPHead.__init__s'  " &*<*r,rqs@rQr8r8sA&&rSr8c.^\rSrSrU4SjrSrSrU=r$)BertPreTrainingHeadsi c>[TU]5 [U5Ul[R "UR S5Ulgr:)r6r7r#r0rrr:r;rMs rQr7BertPreTrainingHeads.__init__ s4 /7 " &*<*[TU]U5 [U[5(a![R "UR 5 g[U[5(a|[R"UR[R"URRS5RS55 [R "UR5 gg)zInitialize the weightsr2r1N)r6 _init_weightsrr#initzeros_r%r*copy_r0rGrHr]rIr4)rNrrrPs rQrN!BertPreTrainedModel._init_weights&s f% f2 3 3 KK $  / / JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h KK-- .0rS)rfrgrhrir( config_classbase_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrrr_can_record_outputsrGno_gradrNrorprqs@rQrIrIsVL&*#N"&"'.  ]]_//rSrIz0 Output type of [`BertForPreTraining`]. ) custom_introc\rSrSr%SrSr\RS-\S'Sr \RS-\S'Sr \RS-\S'Sr \ \RS-\S'Sr \ \RS-\S'S rg) BertForPreTrainingOutputi1ar loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). seq_relationship_logits (`torch.FloatTensor` of shape `(batch_size, 2)`): Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). Nlossprediction_logitsseq_relationship_logitsrrKrS)rfrgrhrirjr`rGrl__annotations__rarbrrrKrorSrSrQr_r_1s~ &*D%  d ")26u((4/68<U..5<59M5**+d2926Je''(4/6rSr_a  The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. c^\rSrSrSS/rSU4SjjrSrSr\\ \ SS\ RS-S \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ RS-S\ S-S\S-S\\S\\ R\-4Sjj555rSrSrU=r$) BertModeliJr*rc>[TU]U5 XlSUl[ U5Ul[ U5UlU(a [U5OSUl UR5 g)z^ add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer FN) r6r7rOgradient_checkpointingr*rcrencoderrpooler post_init)rNrOadd_pooling_layerrPs rQr7BertModel.__init__YsS   &+#(0"6* ,=j(4  rSc.URR$rrcr<rNs rQget_input_embeddingsBertModel.get_input_embeddingsjs...rSc$XRlgrrn)rNrus rQset_input_embeddingsBertModel.set_input_embeddingsms */'rSNrTrvr4r0rUrrrr rxrWc USLUSL- (a [S5eURR(aU bU OURRn OSn U (abUc_UcURR(a.[ [ URS9[ URS95O[ URS9nUbUR5OSn URUUUUU S9n URUUU UUS9up'UR"U 4UUUUU US.U D6n U RnURbURU5OSn[UUU RS9$) Nz:You must specify exactly one of input_ids or inputs_embedsF)rOr)rTr0r4rUrV)rvrembedding_outputrr)rvrrrr r0)r  pooler_outputr)rrOrr is_encoder_decoderrr get_seq_lengthrc_create_attention_masksrhr rirr)rNrTrvr4r0rUrrrr rxrVrvencoder_outputsr2rs rQrdBertModel.forwardpsw  -t"; <YZ Z ;; ! !%.%:  @U@UII 0)4 8V8V$L $DlZ^ZeZeFfg!5  FUE`!?!?!Afg??%)'#9 + 261M1M)#9-"7+ 2N2 .,,   )"7#9+%    *;;8< 8O O4UY ;-'+;;  rScURR(a[URUUUS9nO[URUUS9nUb[URUUUS9nX4$)N)rOrUrvr)rOrUrv)rOrUrvr)rOrrr)rNrvrrvrrs rQrz!BertModel._create_attention_maskssr ;; ! !/{{.- / N7{{.-N " -%>{{.5&; & "55rS)rOrcrhrgri)T) NNNNNNNNN)rfrgrhri_no_split_modulesr7rprsr%r&r"rGrnr rrr!rrrdrzrorprqs@rQrereJs3*;7"/0 *..2.2,0-1596:(,!%? <<$&?  t+?  t+ ? llT) ? ||d* ?  %||d2? !& t 3? ? $;? +,?  u|| K K?  ? B66rSrez Bert Model with two heads on top as done during the pretraining: a `masked language modeling` head and a `next sentence prediction (classification)` head. cn^\rSrSrSSS.rU4SjrSrSr\\ SS \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ RS-S\ RS-S\ \ S\\ R\-4Sjj55rSrU=r$)BertForPreTrainingi&bert.embeddings.word_embeddings.weightcls.predictions.biaszcls.predictions.decoder.weightzcls.predictions.decoder.biasc>[TU]U5 [U5Ul[ U5UlUR 5 gr)r6r7rerJrBclsrjrMs rQr7BertForPreTraining.__init__s4  f% '/ rScBURRR$rrr0r'ros rQget_output_embeddings(BertForPreTraining.get_output_embeddingsxx##+++rScXRRlURURRlgrrr0r'r%rNnew_embeddingss rQset_output_embeddings(BertForPreTraining.set_output_embeddings*'5$$2$7$7!rSNrTrvr4r0rUlabelsnext_sentence_labelrxrWc UR"U4UUUUSS.UD6n U SSupURX5upSnUbvUbs[5nU"U RSURR 5URS55nU"U RSS5URS55nUU-n[ UU U U RU RS9$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring) Indices should be in `[0, 1]`: - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Example: ```python >>> from transformers import AutoTokenizer, BertForPreTraining >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased") >>> model = BertForPreTraining.from_pretrained("google-bert/bert-base-uncased") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.prediction_logits >>> seq_relationship_logits = outputs.seq_relationship_logits ``` Trvr4r0rU return_dictNr{r2)r`rarbrrK) rJrrrrOr9r_rrK)rNrTrvr4r0rUrrrxoutputsr2rr5r? total_lossloss_fctmasked_lm_lossnext_sentence_losss rQrdBertForPreTraining.forwardsR))  ))%'   *1!&48HH_4\1  "5"A')H%&7&<& ,8*..2.2,0-1&*37A <<$&A  t+A  t+ A llT) A ||d* A  t#A #\\D0A +,A  u|| 7 7A A rSrzP Bert Model with a `language modeling` head on top for CLM fine-tuning. c^\rSrSrSSS.rU4SjrSrSr\\ SS \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ RS-S\ RS-S\ RS-S\ S-S\ S-S\\ R-S\\S\\ R\-4Sjj55rSrU=r$)BertLMHeadModeli7rrrc>[TU]U5 UR(d[R S5 [ USS9Ul[U5UlUR5 g)NzLIf you want to use `BertLMHeadModel` as a standalone, add `is_decoder=True.`Frk r6r7rloggerwarningrerJr.rrjrMs rQr7BertLMHeadModel.__init__BsL    NNi jf> "6* rScBURRR$rrros rQr%BertLMHeadModel.get_output_embeddingsNrrScXRRlURURRlgrrrs rQr%BertLMHeadModel.set_output_embeddingsQrrSNrTrvr4r0rUrrrrr logits_to_keeprxrWc UbSn UR"U4UUUUUUU U SS. U D6n U Rn[U [5(a [ U *S5OU nUR USS2USS245nSnUb)UR "SUXRRS.U D6n[UUU RU RU RU RS9$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]` NFT) rvr4r0rUrrrr r)logitsrr9)r`rrrrKrLrS)rJr rrmslicer loss_functionrOr9rrrrKrL)rNrTrvr4r0rUrrrrr rrxrr slice_indicesrr`s rQrdBertLMHeadModel.forwardUs.  I@D  A ))%'"7#9+ A  A  11 8B>SV8W8W~ot4]k -=!(;<=  %%pVF{{OeOepiopD0#33!//))$55   rSr) NNNNNNNNNNr)rfrgrhrirr7rrr$r"rGrnr rrmrr!rrrdrorprqs@rQrr7sP+S(> ,8*..2.2,0-1596:&*(,!%-.6 <<$&6  t+6  t+ 6 llT) 6 ||d* 6  %||d26 !& t 36  t#6 6 $;6 ell*6 +,6  u|| @ @6 6 rSrc^\rSrSrSSS.rU4SjrSrSr\\ SS \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ RS-S\ RS-S\ RS-S\ \ S\\ R\-4Sjj55rSrU=r$)BertForMaskedLMirrrc>[TU]U5 UR(a[R S5 [ USS9Ul[U5UlUR5 g)NzkIf you want to use `BertForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.FrrrMs rQr7BertForMaskedLM.__init__sR     NN1  f> "6* rScBURRR$rrros rQr%BertForMaskedLM.get_output_embeddingsrrScXRRlURURRlgrrrs rQr%BertForMaskedLM.set_output_embeddingsrrSNrTrvr4r0rUrrrrxrWc :UR"U4UUUUUUSS.U D6n U Sn URU 5n Sn UbF[5nU"U RSURR 5URS55n [ U U U RU RS9$)az labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` T)rvr4r0rUrrrrNr2r`rrrK) rJrrrrOr9rrrK)rNrTrvr4r0rUrrrrxrr2r5rrs rQrdBertForMaskedLM.forwards())   ))%'"7#9    "!* HH_5  ')H%&7&<& ,8*..2.2,0-1596:&*+ <<$&+  t++  t+ + llT) + ||d* +  %||d2+ !& t 3+  t#+ +,+  u|| ~ -+ + rSrzT Bert Model with a `next sentence prediction (classification)` head on top. c8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)BertForNextSentencePredictionic>[TU]U5 [U5Ul[ U5UlUR 5 gr)r6r7rerJr8rrjrMs rQr7&BertForNextSentencePrediction.__init__s4  f% "6* rSNrTrvr4r0rUrrxrWc UR"U4UUUUSS.UD6nUSn URU 5n Sn Ub2[5n U "U RSS5URS55n [ U U UR UR S9$)a" labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring). Indices should be in `[0, 1]`: - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Example: ```python >>> from transformers import AutoTokenizer, BertForNextSentencePrediction >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased") >>> model = BertForNextSentencePrediction.from_pretrained("google-bert/bert-base-uncased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light." >>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt") >>> outputs = model(**encoding, labels=torch.LongTensor([1])) >>> logits = outputs.logits >>> assert logits[0, 0] < logits[0, 1] # next sentence was random ``` Trr'Nr2r{r)rJrrrrrrK) rNrTrvr4r0rUrrxrrseq_relationship_scoresrrs rQrd%BertForNextSentencePrediction.forwardsN))  ))%'     "&((="9!  ')H!)*A*F*Fr1*Mv{{[]!_ *#*!//))   rSrNNNNNN)rfrgrhrir7r$r"rGrnrr!rrrdrorprqs@rQrrs *..2.2,0-1&*= <<$&=  t+=  t+ = llT) = ||d* =  t#= +,=  u|| : := = rSrz Bert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. c8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)BertForSequenceClassificationi.cr>[TU]U5 URUlXl[ U5UlUR b UR O URn[R"U5Ul [R"URUR5Ul UR5 gr)r6r7 num_labelsrOrerJclassifier_dropoutrDrrCrErr: classifierrjrNrOrrPs rQr7&BertForSequenceClassification.__init__5s   ++ f% )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ rSNrTrvr4r0rUrrxrWc UR"U4UUUUSS.UD6nUSn URU 5n URU 5n Sn UGbURRcUR S:XaSURlOoUR S:aNUR [R:XdUR [R:XaSURlOSURlURRS:XaI[5n UR S:Xa&U "U R5UR55n OU "X5n OURRS:Xa=[5n U "U RSUR 5URS55n O,URRS:Xa[5n U "X5n [U U UR UR"S 9$) ae labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Trr'N regressionsingle_label_classificationmulti_label_classificationr2r)rJrErrO problem_typerr5rGrLrmrsqueezerrrrrrK) rNrTrvr4r0rUrrxrrrr`rs rQrd%BertForSequenceClassification.forwardDs$))  ))%'      ]3 /  {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#F3D))-JJ+- B @&++b/R))-II,./'!//))   rS)rJrrOrErr)rfrgrhrir7r$r"rGrnrr!rrrdrorprqs@rQrr.s *..2.2,0-1&*; <<$&;  t+;  t+ ; llT) ; ||d* ;  t#; +,;  u|| 7 7; ; rSrc8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)BertForMultipleChoiceic0>[TU]U5 [U5UlURb URO UR n[ R"U5Ul[ R"URS5Ul UR5 g)Nr') r6r7rerJrrDrrCrErr:rrjrs rQr7BertForMultipleChoice.__init__su  f% )/)B)B)NF % %TZTnTn zz"45 ))F$6$6: rSNrTrvr4r0rUrrxrWc UbURSOURSnUb!URSURS55OSnUb!URSURS55OSnUb!URSURS55OSnUb!URSURS55OSnUb1URSURS5URS55OSnUR"U4UUUUSS.UD6n U Sn UR U 5n UR U 5n U RSU5n Sn Ub[ 5nU"X5n [U U U RU RS9$)a input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr'r2Trr) r]rrKrJrErrrrrK)rNrTrvr4r0rUrrx num_choicesrrrreshaped_logitsr`rs rQrdBertForMultipleChoice.forwardsT-6,Aiooa(}GZGZ[\G] >G>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei (   r=#5#5b#9=;M;Mb;Q R  ))  ))%'      ]3 / ++b+6  ')HO4D("!//))   rS)rJrrEr)rfrgrhrir7r$r"rGrnrr!rrrdrorprqs@rQrrs *..2.2,0-1&*N <<$&N  t+N  t+ N llT) N ||d* N  t#N +,N  u|| 8 8N N rSrc8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)BertForTokenClassificationicd>[TU]U5 URUl[USS9UlUR b UR O UR n[R"U5Ul [R"URUR5Ul UR5 gNFr)r6r7rrerJrrDrrCrErr:rrjrs rQr7#BertForTokenClassification.__init__s   ++f> )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ rSNrTrvr4r0rUrrxrWc DUR"U4UUUUSS.UD6nUSn URU 5n URU 5n Sn Ub<[5n U "U R SUR 5UR S55n [ U U URURS9$)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. TrrNr2r) rJrErrrrrrrK) rNrTrvr4r0rUrrxrr2rr`rs rQrd"BertForTokenClassification.forwards ))  ))%'   "!*,,71  ')HFKKDOO[TU]U5 URUl[USS9Ul[ R "URUR5UlUR5 gr) r6r7rrerJrrr: qa_outputsrjrMs rQr7!BertForQuestionAnswering.__init__$sU   ++f> ))F$6$68I8IJ rSNrTrvr4r0rUstart_positions end_positionsrxrWc UR"U4UUUUSS.UD6n U Sn URU 5n U RSSS9upU RS5R 5n U RS5R 5n SnUbUb[ UR 55S:aURS5n[ UR 55S:aURS5nU R S5nURSU5nURSU5n[US9nU"X5nU"X5nUU-S- n[UU U U RU RS 9$) NTrrr'r2r|) ignore_indexr{)r` start_logits end_logitsrrK) rJrsplitrrlenrKclamprrrrK)rNrTrvr4r0rUrrrxrr2rrrr ignored_indexr start_lossend_losss rQrd BertForQuestionAnswering.forward.sx))  ))%'   "!*1#)<<r<#: #++B/::< ''+668   &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M-33A}EO)//=AM']CH!,@J :H$x/14J+%!!//))   rS)rJrrr)rfrgrhrir7r$r"rGrnrr!rrrdrorprqs@rQrr"s*..2.2,0-1/3-13 <<$&3  t+3  t+ 3 llT) 3 ||d* 3 ,3 ||d*3 +,3  u|| ; ;3 3 rSr) rrrrrrrrrrerI)Nr)Zrjcollections.abcr dataclassesrrGrtorch.nnrrrr rO activationsr cache_utilsr r r generationr masking_utilsrrmodeling_layersrmodeling_outputsrrrrrrrrrmodeling_utilsrrprocessing_utilsr pytorch_utilsrutilsr r!r"r# utils.genericr$r%utils.output_capturingr&configuration_bertr( get_loggerrfrModuler*rnfloatrrrrrrrrrrrr#r.r8rBrIr_rerrrrrrrr__all__rSrSrQrs$! AA&!CC)J9   G&6MMI5*   H %;RYY;H!% II% <<% % << % LL4' % T\ %%'(%8@) @)FI)I)X RYY .BII.: ryy   >*>B ")) @  "))" 299  !bii!&bii& 9299 9////2  7{7  7&  |6#|6 |6~ Y ,Y  Y x Q )?Q  Q hI )I I X I $7I  I X M $7M  M `^ /^ ^ B8 !48 8 v@ 2@ @ F rS