ZjSSKJr SSKJr SSKrSSKJr SSKJrJrJ r SSK J r SSK Jr SSKJrJrJr SS KJr SS KJrJr SS KJr SS KJrJrJrJrJrJ r J!r!J"r"J#r# SS K$J%r%J&r& SSK'J(r( SSK)J*r* SSK+J,r,J-r-J.r.J/r/ SSK0J1r1J2r2 SSK3J4r4 SSK5J6r6 \/Rn"\85r9"SS\Rt5r;SZS\RtS\RxS\RxS\RxS\RxS-S\=S-S\=S\(\-4Sjjr>"S S!\Rt5r?"S"S#\Rt5r@"S$S%\Rt5rA"S&S'\Rt5rB"S(S)\Rt5rC"S*S+\Rt5rD"S,S-\5rE"S.S/\Rt5rF"S0S1\Rt5rG"S2S3\Rt5rH"S4S5\Rt5rI\."S6S7\&55rJ\."S8S99"S:S;\J55rK\."S\,555rL"S?S@\Rt5rM\."SAS99"SBSC\J55rN"SDSE\Rt5rO\."SFS99"SGSH\J\55rP\."SISJ\J55rQ"SKSL\Rt5rR\."SMS99"SNSO\J55rS\."SPS99"SQSR\J55rT\."SSST\J55rU\."SUSV\J55rV\."SWSX\J55rW/SYQrXg)[)Callable) dataclassN)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FN)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer) )BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputNextSentencePredictorOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward) ModelOutputTransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs) ErnieConfigc^\rSrSrSrU4SjrSS\RS-S\RS-S\RS-S\RS-S \RS-S \ S \R4S jjr S r U=r $)ErnieEmbeddings9zGConstruct the embeddings from word, position and token_type embeddings.c>[TU]5 [R"URUR UR S9Ul[R"URUR 5Ul [R"URUR 5Ul [R"UR URS9Ul [R"UR5UlUR#S[$R&"UR5R)S5SS9 UR#S[$R*"UR,R/5[$R0S9SS9 UR2UlUR2(a1[R"UR4UR 5Ulgg) N) padding_idxeps position_idsr&F) persistenttoken_type_ids)dtype)super__init__nn Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandzerosr/sizelong use_task_idtask_type_vocab_sizetask_type_embeddingsselfconfig __class__s y/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/ernie/modeling_ernie.pyr6ErnieEmbeddings.__init__<sX !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c %'\\&2H2H&J\J\%]"f&8&8f>S>STzz&"<"<=   ELL)G)GHOOPWXej    ekk$*;*;*@*@*B%**Ubg  "--   (* V5P5PRXRdRd(eD % N input_idsr3 task_type_idsr/ inputs_embedspast_key_values_lengthreturncUbUR5nOUR5SSnUupUcURSS2XiU-24nUc[US5(aQURR UR SS5n [ R"U SUS9n U R X5nO8[ R"U[ RURRS9nUcURU5nURU5n URU R5nX[-n URU5n X-n UR(aPUc8[ R"U[ RURRS9nUR!U5nX- n UR#U 5n UR%U 5n U $)Nr1r3rr&)dimindex)r4device)rKr/hasattrr3rIshaperGgatherrJrLr_r<r@tor>rMrOrArE)rQrWr3rXr/rYrZ input_shape batch_size seq_lengthbuffered_token_type_idsr@ embeddingsr>rOs rTforwardErnieEmbeddings.forwardPs  #..*K',,.s3K!,  ,,Q0FVlIl0l-lmL  !t-..*.*=*=*D*D\EWEWXYEZ\^*_'*/,,7NTU]i*j'!8!?!? !W!&[ SWSdSdSkSk!l   00;M $ : :> J&(()>)E)EF ": "66|D5    $ % KuzzRVRcRcRjRj k #'#<#<]#K  .J^^J/ \\*- rV)rArEr>rOr@rMr<)NNNNNr)__name__ __module__ __qualname____firstlineno____doc__r6rG LongTensor FloatTensorintTensorri__static_attributes__ __classcell__rSs@rTr)r)9sQf,.226150426&'3##d*3((4/3''$. 3 &&- 3 ((4/ 3!$3 33rVr)modulequerykeyvalueattention_maskscalingrEkwargsc UcURS5S-n[R"XRSS55U-nUbX-n[R R USS9n[R RXURS9n[R"X5n U RSS5R5n X4$)Nr1rr])ptrainingr&) rKrGmatmul transposer7 functionalsoftmaxrEr contiguous) rwrxryrzr{r|rEr} attn_weights attn_outputs rTeager_attention_forwardrs**R.D(<<}}Q':;gEL!#4 ==((2(>L==((6??([L,,|3K''1-88:K  $$rVc ^\rSrSrS U4SjjrS S\R S\RS-S\S-S\ \ S\ \R 4 S jjr S r U=r$) ErnieSelfAttentionNcN>[TU]5 URUR-S:wa7[ US5(d&[ SURSURS35eXlURUl[URUR- 5UlURUR-Ul URS-Ul [R"URUR5Ul [R"URUR5Ul[R"URUR5Ul[R "UR"5UlUR&UlX lX0lgNrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r)r5r6r:num_attention_headsr` ValueErrorrRrrattention_head_size all_head_sizer|r7LinearrxryrzrCattention_probs_dropout_probrE is_decoder is_causal layer_idxrQrRrrrSs rTr6ErnieSelfAttention.__init__sM     : : :a ?PVXhHiHi#F$6$6#78 445Q8  #)#=#= #&v'9'9F??*9*N*N'&=%C%CI\`\j\j%k "I(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! "));;;;FFH ((rV) rrrRrErrryrrrxr|rzFN)NNrkrlrmrnr6rGrsrqr rr tuplerirtrurvs@rTrrsl#648(, ')||')))D0') ') +, ') u||  ')')rVrc^\rSrSrS U4SjjrS S\R S\RS-S\RS-S\S-S\ \ S \ \R 4 S jjr S r U=r$)ErnieCrossAttentionNc,>[TU]5 URUR-S:wa7[ US5(d&[ SURSURS35eXlURUl[URUR- 5UlURUR-Ul URS-Ul [R"URUR5Ul [R"URUR5Ul[R"URUR5Ul[R "UR"5UlX lX0lgr)r5r6r:rr`rrRrrrrr|r7rrxryrzrCrrErrrs rTr6ErnieCrossAttention.__init__s@     : : :a ?PVXhHiHi#F$6$6#78 445Q8  #)#=#= #&v'9'9FB**4>>:(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! "));;;;FFH ((rV) rrrRrErryrrrxr|rzr)NNN)rkrlrmrnr6rGrsrqr rr rrirtrurvs@rTrrs#4;?376: 1)||1) %00471)))D0 1) -t3 1) +, 1) u|| 1)1)rVrcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $)ErnieSelfOutputi1c(>[TU]5 [R"URUR5Ul[R "URURS9Ul[R"UR5Ul gNr-) r5r6r7rr:denserArBrCrDrErPs rTr6ErnieSelfOutput.__init__2s` YYv1163E3EF f&8&8f>S>STzz&"<"<= rVr input_tensorr[cpURU5nURU5nURX-5nU$NrrErArQrrs rTriErnieSelfOutput.forward85 =1  ]3 }'CD rVrArrE rkrlrmrnr6rGrsrirtrurvs@rTrr16> U\\RWR^R^rVrc^\rSrSrS U4SjjrSS\R S\RS-S\RS-S\RS-S\S-S \ \ S \ \R 4S jjr S r U=r$)ErnieAttentioni?Nc>[TU]5 X@lU(a[O[nU"XUS9Ul[ U5Ulg)Nrr)r5r6is_cross_attentionrrrQroutput)rQrRrrrattention_classrSs rTr6ErnieAttention.__init__@s9 "41C-I[#F9U %f- rVrr{rencoder_attention_maskrr}r[c UR(dUOUnUR"U4UUUS.UD6upxURXq5nXx4$)N)rr{r)rrQr) rQrr{rrrr}attention_outputrs rTriErnieAttention.forwardGs\04/F/FLb)- * "7)+ *  * & ;;'7G--rV)rrrQ)FNFNNNNrrvs@rTrr?s.48:>;?(, .||.))D0. %0047 . !& 1 1D 8 .  .+,. u|| ..rVrcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)ErnieIntermediatei\c>[TU]5 [R"URUR 5Ul[UR[5(a[URUl gURUl gr) r5r6r7rr:intermediate_sizerr hidden_actstrr intermediate_act_fnrPs rTr6ErnieIntermediate.__init__]s` YYv1163K3KL f'' - -'-f.?.?'@D $'-'8'8D $rVrr[cJURU5nURU5nU$rrrrQrs rTriErnieIntermediate.forwardes& =1 00? rVrrrvs@rTrr\s(9U\\ellrVrcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $) ErnieOutputikc(>[TU]5 [R"URUR 5Ul[R"UR URS9Ul[R"UR5Ul gr) r5r6r7rrr:rrArBrCrDrErPs rTr6ErnieOutput.__init__ls` YYv779K9KL f&8&8f>S>STzz&"<"<= rVrrr[cpURU5nURU5nURX-5nU$rrrs rTriErnieOutput.forwardrrrVrrrvs@rTrrkrrVrc^\rSrSrSU4SjjrSS\R S\RS-S\RS-S\RS-S\S-S \ \ S \R 4S jjr S r S r U=r$) ErnieLayeriyNc>[TU]5 URUlSUl[ XR US9UlUR UlURUlUR(a0UR (d[US35e[ USUSS9Ul [U5Ul [U5Ul g)Nr&rz> should be used as a decoder model if cross attention is addedFT)rrr)r5r6chunk_size_feed_forward seq_len_dimrr attentionadd_cross_attentionrcrossattentionr intermediaterr)rQrRrrSs rTr6ErnieLayer.__init__zs '-'E'E$':K:KW`a ++#)#=#=  # #?? D6)g!hii"0##' #D  .f5!&) rVrr{rrrr}r[c 2UR"UU4SU0UD6upxUn UR(a?Ub<[US5(d[SUS35eUR"USUU4SU0UD6upU n [ UR URURU 5n U $)Nrrz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`) rrr`rrrfeed_forward_chunkrr) rQrr{rrrr}self_attention_output_rcross_attention_output layer_outputs rTriErnieLayer.forwards$(>>  $ ,$  $   1 ??4@4!122 =dVDDD )-(;(;%%& ) !0 )  ) % " 6 0  # #T%A%A4CSCSUe rVcJURU5nURX!5nU$r)rr)rQrintermediate_outputrs rTrErnieLayer.feed_forward_chunks)"//0@A{{#6I rV)rrrrrrrrrr)rkrlrmrnr6rGrsrqr rr rirrtrurvs@rTrrys*,48:>;?(, %||%))D0% %0047 % !& 1 1D 8 %  %+,% %NrVrcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $) ErniePooleric>[TU]5 [R"URUR5Ul[R "5Ulgr)r5r6r7rr:rTanh activationrPs rTr6ErniePooler.__init__s9 YYv1163E3EF '')rVrr[c\USS2S4nURU5nURU5nU$)Nr)rr )rQrfirst_token_tensor pooled_outputs rTriErniePooler.forwards6+1a40 #56  6 rV)r rrrvs@rTrrs($ U\\ellrVrcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)ErniePredictionHeadTransformicp>[TU]5 [R"URUR5Ul[ UR[5(a[URUl OURUl [R"URURS9Ul gr) r5r6r7rr:rrrrr transform_act_fnrArBrPs rTr6%ErniePredictionHeadTransform.__init__s~ YYv1163E3EF f'' - -$*6+<+<$=D !$*$5$5D !f&8&8f>S>STrVrr[clURU5nURU5nURU5nU$r)rrrArs rTri$ErniePredictionHeadTransform.forwards4 =1 --m< }5 rV)rArrrrvs@rTrrs)UU\\ellrVrc.^\rSrSrU4SjrSrSrU=r$)ErnieLMPredictionHeadic>[TU]5 [U5Ul[R "UR URSS9Ul[R"[R"UR55Ul g)NT)bias) r5r6r transformr7rr:r9decoder ParameterrGrJrrPs rTr6ErnieLMPredictionHead.__init__s[ 5f=yy!3!3V5F5FTR LLV->->!?@ rVcJURU5nURU5nU$r)rrrs rTriErnieLMPredictionHead.forwards$}5  ]3 rV)rrrrkrlrmrnr6rirtrurvs@rTrrsArVrc^\rSrSrU4SjrSS\R S\RS-S\RS-S\RS-S\S-S \ S-S \ \ S \ \R \ -4S jjrS rU=r$) ErnieEncoderic >[TU]5 Xl[R"[ UR 5Vs/sH n[XS9PM sn5Ulgs snf)N)r) r5r6rRr7 ModuleListrangenum_hidden_layersrlayer)rQrRirSs rTr6ErnieEncoder.__init__sH  ]]USYSkSkMl#mMlJv$CMl#mn #msANrr{rrr use_cacher}r[c [UR5HupU "UUU4UUS.UD6nM [UU(aUS9$SS9$)N)rr)last_hidden_stater) enumerater(r) rQrr{rrrr+r}r) layer_modules rTriErnieEncoder.forwardsg )4OA(%(> /   M 59+/8O  >B  rV)rRr()NNNNN)rkrlrmrnr6rGrsrqr boolrr rrrirtrurvs@rTr#r#so48:>;?(,!% || ))D0  %0047  !& 1 1D 8    $; +,  u|| H H  rVr#cx^\rSrSr\rSrSrSrSr Sr Sr \ \ \S.r\R""5U4Sj5rSrU=r$)ErniePreTrainedModeli ernieT)r attentionscross_attentionsc>[TU]U5 [U[5(a![R "UR 5 g[U[5(a|[R"UR[R"URRS5RS55 [R "UR5 gg)zInitialize the weightsr1r0N)r5 _init_weightsrrinitzeros_rr)copy_r/rGrHrarIr3)rQrwrSs rTr8"ErniePreTrainedModel._init_weightss f% f3 4 4 KK $  0 0 JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h KK-- .1rV)rkrlrmrnr' config_classbase_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrrr_can_record_outputsrGno_gradr8rtrurvs@rTr3r3 sVL&*#N"&#(/  ]]_//rVr3a  The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. ) custom_introc^\rSrSrS/rSU4SjjrSrSr\\ \ SS\ RS-S\ RS-S \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ RS-S\ S-S\S-S\\S\\ R\-4Sjj555rSrSrU=r$) ErnieModeli$rc>[TU]U5 XlSUl[ U5Ul[ U5UlU(a [U5OSUl UR5 g)z^ add_pooling_layer (bool, *optional*, defaults to `True`): Whether to add a pooling layer FN) r5r6rRgradient_checkpointingr)rhr#encoderrpooler post_init)rQrRadd_pooling_layerrSs rTr6ErnieModel.__init__3sS   &+#)&1#F+ ->k&)D  rVc.URR$rrhr<rQs rTget_input_embeddingsErnieModel.get_input_embeddingsDs...rVc$XRlgrrR)rQrzs rTset_input_embeddingsErnieModel.set_input_embeddingsGs */'rVNrWr{r3rXr/rYrrrr+r}r[c USLUSL- (a [S5eURR(aU bU OURRn OSn U (abU c_UcURR(a.[ [ URS9[ URS95O[ URS9n U bU R5OSn URUUUUUU S9n URUUU UU S9up(UR"U 4UUUU U US.U D6nUSnURbURU5OSn[UUURS 9$)  task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] Nz:You must specify exactly one of input_ids or inputs_embedsF)rRr)rWr/r3rXrYrZ)r{rembedding_outputrr)r{rrrr+r/)r- pooler_outputr)rrRrr+is_encoder_decoderr r get_seq_lengthrh_create_attention_masksrLrMrr)rQrWr{r3rXr/rYrrrr+r}rZr[encoder_outputssequence_outputrs rTriErnieModel.forwardJsy0  -t"; <YZ Z ;; ! !%.%:  @U@UII 0)4 8V8V$L $DlZ^ZeZeFfg!5  FUE`!?!?!Afg??%)''#9+ 261M1M)#9-"7+ 2N2 .,,   )"7#9+%    *!,8< 8O O4UY ;-'+;;  rVcURR(a[URUUUS9nO[URUUS9nUb[URUUUS9nX4$)N)rRrYr{r)rRrYr{)rRrYr{r)rRrrr)rQr{rr[rrs rTr_"ErnieModel._create_attention_maskssr ;; ! !/{{.- / N7{{.-N " -%>{{.5&; & "55rV)rRrhrLrKrM)T) NNNNNNNNNN)rkrlrmrn_no_split_modulesr6rTrWr$r%r!rGrsr r1rr rrrir_rtrurvs@rTrIrI$sT&"/0 *..2.2-1,0-1596:(,!%H <<$&H  t+H  t+ H ||d* H llT) H ||d*H  %||d2H !& t 3H H $;H +,H  u|| K KH  H T66rVrIz1 Output type of [`ErnieForPreTraining`]. c\rSrSr%SrSr\RS-\S'Sr \RS-\S'Sr \RS-\S'Sr \ \RS-\S'Sr \ \RS-\S'S rg) ErnieForPreTrainingOutputiar loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss. prediction_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). seq_relationship_logits (`torch.FloatTensor` of shape `(batch_size, 2)`): Prediction scores of the next sequence prediction (classification) head (scores of True/False continuation before SoftMax). Nlossprediction_logitsseq_relationship_logitsrr5r=)rkrlrmrnrorhrGrq__annotations__rirjrrr5rtr=rVrTrgrgs~ &*D%  d ")26u((4/68<U..5<59M5**+d2926Je''(4/6rVrgc.^\rSrSrU4SjrSrSrU=r$)ErniePreTrainingHeadsic>[TU]5 [U5Ul[R "UR S5UlgNr)r5r6r predictionsr7rr:seq_relationshiprPs rTr6ErniePreTrainingHeads.__init__s4 08 " &*<*[TU]U5 [U5Ul[ U5UlUR 5 gr)r5r6rIr4rmclsrNrPs rTr6ErnieForPreTraining.__init__s4  ' (0 rVcBURRR$rrrprrSs rTget_output_embeddings)ErnieForPreTraining.get_output_embeddingsxx##+++rVcXRRlURURRlgrrrprrrQnew_embeddingss rTset_output_embeddings)ErnieForPreTraining.set_output_embeddings*'5$$2$7$7!rVNrWr{r3rXr/rYlabelsnext_sentence_labelr}r[c UR"U4UUUUUSS.U D6n U SSupURX5upSnUbvUbs[5nU"U RSURR 5URS55nU"URSS5URS55nUU-n[ UU UU RU RS9$)aj task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` next_sentence_label (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring) Indices should be in `[0, 1]`: - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Example: ```python >>> from transformers import AutoTokenizer, ErnieForPreTraining >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> model = ErnieForPreTraining.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.prediction_logits >>> seq_relationship_logits = outputs.seq_relationship_logits ``` Tr{r3rXr/rY return_dictNrr1)rhrirjrr5) r4rrrrRr9rgrr5)rQrWr{r3rXr/rYrrr}outputsrarrurv total_lossloss_fctmasked_lm_lossnext_sentence_losss rTriErnieForPreTraining.forwards^**   ))'%'    *1!&48HH_4\1  "5"A')H%&7&<&[TU]5 [U5Ulgr)r5r6rrprPs rTr6ErnieOnlyMLMHead.__init__Gs 08rVrar[c(URU5nU$rrp)rQrarus rTriErnieOnlyMLMHead.forwardKs ,,_=  rVrrrvs@rTrrFs(9!u||! !!rVrzQ Ernie Model with a `language modeling` head on top for CLM fine-tuning. c ^\rSrSrSSS.rU4SjrSrSr\\ SS \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ RS-S\ RS-S\ RS-S\ RS-S\ \ RS-S\ S-S\\ R-S\\S\\ R\-4Sjj55rSrU=r$)ErnieForCausalLMiPr{rz)r~r}c>[TU]U5 UR(d[R S5 [ USS9Ul[U5UlUR5 g)NzMIf you want to use `ErnieForCausalLM` as a standalone, add `is_decoder=True.`FrO r5r6rloggerwarningrIr4rrrNrPs rTr6ErnieForCausalLM.__init__[sL    NNj k%@ #F+ rVcBURRR$rrrSs rTr&ErnieForCausalLM.get_output_embeddingsgrrVcXRRlURURRlgrrrs rTr&ErnieForCausalLM.set_output_embeddingsjrrVNrWr{r3rXr/rYrrrrr+logits_to_keepr}r[c  U bSn UR"U4UUUUUUUU U SS. U D6nURn[U [5(a [ U *S5OU nUR USS2USS245nSnU b)UR "SUXRRS.U D6n[UUURURURURS9$)a@ task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]` NFT) r{r3rXr/rYrrrr+r)logitsrr9)rhrrrr5r6r=)r4r-rrrslicer loss_functionrRr9rrrr5r6)rQrWr{r3rXr/rYrrrrr+rr}rr slice_indicesrrhs rTriErnieForCausalLM.forwardns:  I@D  A ))'%'"7#9+ A  A  11 8B>SV8W8W~ot4]k -=!(;<=  %%pVF{{OeOepiopD0#33!//))$55   rVr) NNNNNNNNNNNr)rkrlrmrnrr6rrr#r!rGrslistr1rrrr rrrirtrurvs@rTrrPsp+T(> ,8*..2.2-1,0-1596:&*59!%-.= <<$&=  t+=  t+ = ||d* = llT) = ||d*=  %||d2= !& t 3=  t#= ell+d2= $;= ell*= +,=  u|| @ @= = rVrc^\rSrSrSSS.rU4SjrSrSr\\ SS \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ RS-S\ RS-S\ RS-S\ RS-S\ \ S\\ R\-4Sjj55rSrU=r$)ErnieForMaskedLMirzr{r|c>[TU]U5 UR(a[R S5 [ USS9Ul[U5UlUR5 g)NzlIf you want to use `ErnieForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.FrrrPs rTr6ErnieForMaskedLM.__init__sR     NN1   %@ #F+ rVcBURRR$rrrSs rTr&ErnieForMaskedLM.get_output_embeddingsrrVcXRRlURURRlgrrrs rTr&ErnieForMaskedLM.set_output_embeddingsrrVNrWr{r3rXr/rYrrrr}r[c <UR"U4UUUUUUUSS.U D6n U Sn URU 5n SnU bF[5nU"U RSURR 5U RS55n[ UU U RU RS9$)a# task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` T)r{r3rXr/rYrrrrNr1rhrrr5) r4rrrrRr9rrr5)rQrWr{r3rXr/rYrrrr}rrarurrs rTriErnieForMaskedLM.forwards4**   ))'%'"7#9    "!* HH_5  ')H%&7&<&[TU]5 [R"URS5Ulgro)r5r6r7rr:rqrPs rTr6ErnieOnlyNSPHead.__init__s'  " &*<*[TU]U5 [U5Ul[ U5UlUR 5 gr)r5r6rIr4rrrNrPs rTr6'ErnieForNextSentencePrediction.__init__s4  ' #F+ rVNrWr{r3rXr/rYrr}r[c UR"U4UUUUUSS.UD6n U Sn URU 5n Sn Ub2[5n U "U RSS5URS55n [ U U U R U R S9$)a task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the next sequence prediction (classification) loss. Input should be a sequence pair (see `input_ids` docstring). Indices should be in `[0, 1]`: - 0 indicates sequence B is a continuation of sequence A, - 1 indicates sequence B is a random sequence. Example: ```python >>> from transformers import AutoTokenizer, ErnieForNextSentencePrediction >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> model = ErnieForNextSentencePrediction.from_pretrained("nghuyong/ernie-1.0-base-zh") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." >>> next_sentence = "The sky is blue due to the shorter wavelength of blue light." >>> encoding = tokenizer(prompt, next_sentence, return_tensors="pt") >>> outputs = model(**encoding, labels=torch.LongTensor([1])) >>> logits = outputs.logits >>> assert logits[0, 0] < logits[0, 1] # next sentence was random ``` Trr&Nr1rr)r4rrrrrr5)rQrWr{r3rXr/rYrr}rrseq_relationship_scoresrrs rTri&ErnieForNextSentencePrediction.forwardsZ**   ))'%'      "&((="9!  ')H!)*A*F*Fr1*Mv{{[]!_ *#*!//))   rVrNNNNNNN)rkrlrmrnr6r#r!rGrsrr rrrirtrurvs@rTrrs *..2.2-1,0-1&*D <<$&D  t+D  t+ D ||d* D llT) D ||d*D  t#D +,D  u|| : :D D rVrz Ernie Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. cX^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \RS-S \ \ S \ \R\ -4S jj55r SrU=r$)ErnieForSequenceClassificationifcr>[TU]U5 URUlXl[ U5UlUR b UR O URn[R"U5Ul [R"URUR5Ul UR5 gr)r5r6 num_labelsrRrIr4classifier_dropoutrDr7rCrErr: classifierrNrQrRrrSs rTr6'ErnieForSequenceClassification.__init__ms   ++ ' )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ rVNrWr{r3rXr/rYrr}r[c UR"U4UUUUUSS.UD6n U Sn URU 5n URU 5n Sn UGbURRcUR S:XaSURlOoUR S:aNUR [R:XdUR [R:XaSURlOSURlURRS:XaI[5n UR S:Xa&U "U R5UR55n OU "X5n OURRS:Xa=[5n U "U RSUR 5URS55n O,URRS:Xa[5n U "X5n [U U U R U R"S 9$) a task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Trr&N regressionsingle_label_classificationmulti_label_classificationr1r)r4rErrR problem_typerr4rGrLrrrsqueezerrrrrr5)rQrWr{r3rXr/rYrr}rrrrhrs rTri&ErnieForSequenceClassification.forward|s0**   ))'%'       ]3 /  {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#F3D))-JJ+- B @&++b/R))-II,./'!//))   rV)rrRrEr4rr)rkrlrmrnr6r#r!rGrsrr rrrirtrurvs@rTrrfs *..2.2-1,0-1&*B <<$&B  t+B  t+ B ||d* B llT) B ||d*B  t#B +,B  u|| 7 7B B rVrcX^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \RS-S \ \ S \ \R\ -4S jj55r SrU=r$)ErnieForMultipleChoiceic0>[TU]U5 [U5UlURb URO UR n[ R"U5Ul[ R"URS5Ul UR5 g)Nr&) r5r6rIr4rrDr7rCrErr:rrNrs rTr6ErnieForMultipleChoice.__init__su  ' )/)B)B)NF % %TZTnTn zz"45 ))F$6$6: rVNrWr{r3rXr/rYrr}r[c UbURSOURSn Ub!URSURS55OSnUb!URSURS55OSnUb!URSURS55OSnUb!URSURS55OSnUb1URSURS5URS55OSnUR"U4UUUUUSS.UD6n U Sn UR U 5n UR U 5n U RSU 5n SnUb[ 5nU"X5n[UU U RU RS9$)aQ input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) task_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr&r1Trr) rarrKr4rErrrrr5)rQrWr{r3rXr/rYrr} num_choicesrrrreshaped_logitsrhrs rTriErnieForMultipleChoice.forwards`-6,Aiooa(}GZGZ[\G] >G>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei (   r=#5#5b#9=;M;Mb;Q R  **   ))'%'       ]3 / ++b+6  ')HO4D("!//))   rV)rrEr4r)rkrlrmrnr6r#r!rGrsrr rrrirtrurvs@rTrrs *..2.2-1,0-1&*U <<$&U  t+U  t+ U ||d* U llT) U ||d*U  t#U +,U  u|| 8 8U U rVrcX^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \RS-S \ \ S \ \R\ -4S jj55r SrU=r$)ErnieForTokenClassificationi,cd>[TU]U5 URUl[USS9UlUR b UR O UR n[R"U5Ul [R"URUR5Ul UR5 gNFr)r5r6rrIr4rrDr7rCrErr:rrNrs rTr6$ErnieForTokenClassification.__init__.s   ++%@ )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ rVNrWr{r3rXr/rYrr}r[c FUR"U4UUUUUSS.UD6n U Sn URU 5n URU 5n Sn Ub<[5n U "U R SUR 5UR S55n [ U U U RU RS9$)al task_type_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Task type embedding is a special embedding to represent the characteristic of different tasks, such as word-aware pre-training task, structure-aware pre-training task and semantic-aware pre-training task. We assign a `task_type_id` to each task and the `task_type_id` is in the range `[0, config.task_type_vocab_size-1] labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. TrrNr1r) r4rErrrrrrr5)rQrWr{r3rXr/rYrr}rrarrhrs rTri#ErnieForTokenClassification.forward<s,**   ))'%'    "!*,,71  ')HFKKDOO[TU]U5 URUl[USS9Ul[ R "URUR5UlUR5 gr) r5r6rrIr4r7rr: qa_outputsrNrPs rTr6"ErnieForQuestionAnswering.__init__qsU   ++%@ ))F$6$68I8IJ rVNrWr{r3rXr/rYstart_positions end_positionsr}r[c UR"U4UUUUUSS.U D6n U Sn URU 5n U RSSS9upU RS5R 5n URS5R 5nSnUbUb[ UR 55S:aURS5n[ UR 55S:aURS5nU R S5nURSU5nURSU5n[US9nU"X5nU"X5nUU-S - n[UU UU RU RS 9$) rZTrrr&r1rN) ignore_indexr)rh start_logits end_logitsrr5) r4rsplitrrlenrKclamprrrr5)rQrWr{r3rXr/rYrrr}rrarrrr ignored_indexr start_lossend_losss rTri!ErnieForQuestionAnswering.forward{s{***   ))'%'    "!*1#)<<r<#: #++B/::< ''+668   &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M-33A}EO)//=AM']CH!,@J :H$x/14J+%!!//))   rV)r4rrr)rkrlrmrnr6r#r!rGrsrr rrrirtrurvs@rTrros*..2.2-1,0-1/3-1< <<$&<  t+<  t+ < ||d* < llT) < ||d*< ,< ||d*< +,<  u|| ; ;< < rVr) rrrrryrrrrIr3)Nr)Ycollections.abcr dataclassesrrGtorch.nnr7rrrr r9 activationsr cache_utilsr r r generationr masking_utilsrrmodeling_layersrmodeling_outputsrrrrrrrrrmodeling_utilsrrprocessing_utilsr pytorch_utilsrutilsrr r!r" utils.genericr#r$utils.output_capturingr%configuration_ernier' get_loggerrkrModuler)rsfloatrrrrrrrrrrrr#r3rIrgrmryrrrrrrrrr__all__r=rVrTrs*%! AA&!CC)J9   G&6MMI5,   H %JbiiJf!% II% <<% % << % LL4' % T\ %%'(%8@)@)FI)"))I)X bii .RYY.:   ")) >+>B ")) 299" BII   299 @/?//2  E6%E6 E6P  7 7  7& 9BII 9 ` .`  ` F!ryy! X +_X  X vP +P P f&ryy& P %9P  P f T %9T  T ne 1e e P? "6? ? DI 4I I X rV