ZjSrSSKJr SSKJr SSKrSSKJr SSKJrJ r J r SSK J r SS KJrJr SS KJrJrJr SS KJr SS KJrJr SS KJr SSKJrJrJrJ r J!r!J"r"J#r#J$r$ SSK%J&r&J'r' SSK(J)r) SSK*J+r+ SSK,J-r-J.r.J/r/J0r0 SSK1J2r2J3r3 SSK4J5r5 SSK6J7r7 \0Rp"\95r:"SS\Rv5r<SVS\RvS\RzS\RzS\RzS\RzS-S\>S-S\>S \)\.4S!jjr?"S"S#\Rv5r@"S$S%\Rv5rA"S&S'\Rv5rB"S(S)\Rv5rC"S*S+\Rv5rD"S,S-\Rv5rE"S.S/\5rF"S0S1\Rv5rG"S2S3\Rv5rH"S4S5\Rv5rI\/"S6S7\'55rJ\/"S8S99\"S:S;\-555rK\/"S<S=\J55rL"S>S?\Rv5rM"S@SA\Rv5rN\/"SBS99"SCSD\J55rO\/"SES99"SFSG\J55rP\/"SHS99"SISJ\J55rQ\/"SKS99"SLSM\J55rR\/"SNSO\J55rS\/"SPSQ\J55rT\/"SRS99"SSST\J\55rU/SUQrVg)WzPyTorch ELECTRA model.)Callable) dataclassN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FNget_activation)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer)"BaseModelOutputWithCrossAttentions)BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward) ModelOutputTransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs) ElectraConfigc ^\rSrSrSrU4SjrS S\RS-S\RS-S\RS-S\RS-S \ S \R4 S jjr S r U=r $)ElectraEmbeddings8zGConstruct the embeddings from word, position and token_type embeddings.c >[TU]5 [R"URUR UR S9Ul[R"URUR 5Ul [R"URUR 5Ul [R"UR URS9Ul [R"UR5UlUR#S[$R&"UR5R)S5SS9 UR#S[$R*"UR,R/5[$R0S9SS9 g) N) padding_idxeps position_idsr'F) persistenttoken_type_idsdtype)super__init__r Embedding vocab_sizeembedding_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandzerosr0sizelongselfconfig __class__s }/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/electra/modeling_electra.pyr8ElectraEmbeddings.__init__;s !||F,=,=v?T?Tbhbubuv#%<<0N0NPVPePe#f %'\\&2H2H&J_J_%`"f&;&;AVAVWzz&"<"<=   ELL)G)GHOOPWXej    ekk$*;*;*@*@*B%**Ubg  N input_idsr4r0 inputs_embedspast_key_values_lengthreturncUbUR5nOUR5SSnUupxUcURSS2XXU-24nUc[US5(aQURR UR SS5n [ R"U SUS9n U R Xx5nO8[ R"U[ RURRS9nUcURU5nURU5n XJ-n URU5n X-n URU 5n URU 5n U $)Nr2r4rr')dimindex)r6device)rLr0hasattrr4rJshaperHgatherrKrMr\r=rAr?rBrF) rOrUr4r0rVrW input_shape batch_size seq_lengthbuffered_token_type_idsrA embeddingsr?s rRforwardElectraEmbeddings.forwardMsG  #..*K',,.s3K!,  ,,Q0FVlIl0l-lmL  !t-..*.*=*=*D*D\EWEWXYEZ\^*_'*/,,7NTU]i*j'!8!?!? !W!&[ SWSdSdSkSk!l   00;M $ : :> J": "66|D5 ^^J/ \\*- rT)rBrFr?rAr=)NNNNr)__name__ __module__ __qualname____firstlineno____doc__r8rH LongTensor FloatTensorintTensorre__static_attributes__ __classcell__rQs@rRr*r*8sQ (.2260426&' (##d*(((4/(&&- ( ((4/ ( !$ ( ((rTr*modulequerykeyvalueattention_maskscalingrFkwargsc UcURS5S-n[R"XRSS55U-nUbX-n[R R USS9n[R RXURS9n[R"X5n U RSS5R5n X4$)Nr2r rZ)ptrainingr') rLrHmatmul transposer functionalsoftmaxrFr contiguous) rsrtrurvrwrxrFry attn_weights attn_outputs rReager_attention_forwardrys**R.D(<<}}Q':;gEL!#4 ==((2(>L==((6??([L,,|3K''1-88:K  $$rTc ^\rSrSrS U4SjjrS S\R S\RS-S\S-S\ \ S\ \R 4 S jjr S r U=r$) ElectraSelfAttentionNcN>[TU]5 URUR-S:wa7[ US5(d&[ SURSURS35eXlURUl[URUR- 5UlURUR-Ul URS-Ul [R"URUR5Ul [R"URUR5Ul[R"URUR5Ul[R "UR"5UlUR&UlX lX0lgNrr;zThe hidden size (z6) is not a multiple of the number of attention heads ()r{)r7r8 hidden_sizenum_attention_headsr] ValueErrorrPrnattention_head_size all_head_sizerxrLinearrtrurvrDattention_probs_dropout_probrF is_decoder is_causal layer_idxrOrPrrrQs rRr8ElectraSelfAttention.__init__sM     : : :a ?PVXhHiHi#F$6$6#78 445Q8  #)#=#= #&v'9'9F??*9*N*N'&=%C%CI\`\j\j%k "I(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! "));;;;FFH ((rT) rrrPrFrrrurrrtrxrvFN)NNrgrhrirjr8rHrormr rr!tuplererprqrrs@rRrrsl#648(, ')||')))D0') ') +, ') u||  ')')rTrc^\rSrSrS U4SjjrS S\R S\RS-S\RS-S\S-S\ \ S \ \R 4 S jjr S r U=r$)ElectraCrossAttentionNc,>[TU]5 URUR-S:wa7[ US5(d&[ SURSURS35eXlURUl[URUR- 5UlURUR-Ul URS-Ul [R"URUR5Ul [R"URUR5Ul[R"URUR5Ul[R "UR"5UlX lX0lgr)r7r8rrr]rrPrnrrrxrrrtrurvrDrrFrrrs rRr8ElectraCrossAttention.__init__s@     : : :a ?PVXhHiHi#F$6$6#78 445Q8  #)#=#= #&v'9'9FB**4>>:(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! "));;;;FFH ((rT) rrrPrFrrurrrtrxrvr)NNN)rgrhrirjr8rHrormrrr!rrerprqrrs@rRrrs#4;?376: 1)||1) %00471)))D0 1) -t3 1) +, 1) u|| 1)1)rTrcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $)ElectraSelfOutputi'c(>[TU]5 [R"URUR5Ul[R "URURS9Ul[R"UR5Ul gNr.) r7r8rrrdenserBrCrDrErFrNs rRr8ElectraSelfOutput.__init__(s` YYv1163E3EF f&8&8f>S>STzz&"<"<= rTr input_tensorrXcpURU5nURU5nURX-5nU$NrrFrBrOrrs rRreElectraSelfOutput.forward.5 =1  ]3 }'CD rTrBrrF rgrhrirjr8rHrorerprqrrs@rRrr'6> U\\RWR^R^rTrc^\rSrSrS U4SjjrSS\R S\RS-S\RS-S\RS-S\S-S \ \ S \ \R 4S jjr S r U=r$)ElectraAttentioni6Nc>[TU]5 X@lU(a[O[nU"XUS9Ul[ U5Ulg)Nrr)r7r8is_cross_attentionrrrOroutput)rOrPrrrattention_classrQs rRr8ElectraAttention.__init__7s9 "43E/K_#F9U '/ rTrrwrencoder_attention_maskrryrXc UR(dUOUnUR"U4UUUS.UD6upxURXq5nXx4$)N)rrwr)rrOr) rOrrwrrrryattention_outputrs rRreElectraAttention.forward>s\04/F/FLb)- * "7)+ *  * & ;;'7G--rT)rrrO)FNFNNNNrrrs@rRrr6s048:>;?(, .||.))D0. %0047 . !& 1 1D 8 .  .+,. u|| ..rTrcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)ElectraIntermediateiTc>[TU]5 [R"URUR 5Ul[UR[5(a[URUl gURUl gr) r7r8rrrintermediate_sizerr hidden_actstrr intermediate_act_fnrNs rRr8ElectraIntermediate.__init__Us` YYv1163K3KL f'' - -'-f.?.?'@D $'-'8'8D $rTrrXcJURU5nURU5nU$rrr)rOrs rRreElectraIntermediate.forward]s& =1 00? rTrrrrs@rRrrTs(9U\\ellrTrcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $) ElectraOutputidc(>[TU]5 [R"URUR 5Ul[R"UR URS9Ul[R"UR5Ul gr) r7r8rrrrrrBrCrDrErFrNs rRr8ElectraOutput.__init__es` YYv779K9KL f&8&8f>S>STzz&"<"<= rTrrrXcpURU5nURU5nURX-5nU$rrrs rRreElectraOutput.forwardkrrTrrrrs@rRrrdrrTrc^\rSrSrSU4SjjrSS\R S\RS-S\RS-S\RS-S\S-S \ \ S \R 4S jjr S r S r U=r$) ElectraLayerisNc>[TU]5 URUlSUl[ XR US9UlUR UlURUlUR(a0UR (d[US35e[ USUSS9Ul [U5Ul [U5Ul g)Nr'rz> should be used as a decoder model if cross attention is addedFT)rrr)r7r8chunk_size_feed_forward seq_len_dimrr attentionadd_cross_attentionrcrossattentionr intermediaterr)rOrPrrQs rRr8ElectraLayer.__init__ts '-'E'E$)&>  $ ,$  $   1 ??4@4!122 =dVDDD )-(;(;%%& ) !0 )  ) % " 6 0  # #T%A%A4CSCSUe rTcJURU5nURX!5nU$r)rr)rOrintermediate_outputrs rRrElectraLayer.feed_forward_chunks)"//0@A{{#6I rT)rrrrrrrrrr)rgrhrirjr8rHrormr rr!rerrprqrrs@rRrrss,,48:>;?(, %||%))D0% %0047 % !& 1 1D 8 %  %+,% %NrTrc^\rSrSrU4SjrSS\R S\RS-S\RS-S\RS-S\S-S \ S-S \ \ S \ \R \ -4S jjrS rU=r$)ElectraEncoderic >[TU]5 Xl[R"[ UR 5Vs/sH n[XS9PM sn5Ulgs snf)N)r) r7r8rPr ModuleListrangenum_hidden_layersrlayer)rOrPirQs rRr8ElectraEncoder.__init__sH  ]]uU[UmUmOn#oOn!L$EOn#op #osANrrwrrr use_cacheryrXc [UR5HupU "UUU4UUS.UD6nM [UU(aUS9$SS9$)N)rrlast_hidden_stater) enumeraterr) rOrrwrrrr ryr layer_modules rRreElectraEncoder.forwardsg )4OA(%(> /   M 59+/8O  >B  rT)rPr)NNNNN)rgrhrirjr8rHrormr boolrr!rrrerprqrrs@rRrrsq48:>;?(,!% || ))D0  %0047  !& 1 1D 8    $; +,  u|| H H  rTrc2^\rSrSrSrU4SjrSrSrU=r$)ElectraDiscriminatorPredictionsizEPrediction module for the discriminator, made up of two dense layers.c>[TU]5 [R"URUR5Ul[ UR5Ul[R"URS5Ul Xl gNr') r7r8rrrrr r activationdense_predictionrPrNs rRr8(ElectraDiscriminatorPredictions.__init__s\ YYv1163E3EF ():):; " &*<*?  6 &&}5==bA rT)rrPrr rgrhrirjrkr8rerprqrrs@rRrrsOrTrc2^\rSrSrSrU4SjrSrSrU=r$)ElectraGeneratorPredictionsizAPrediction module for the generator, made up of two dense layers.c>[TU]5 [S5Ul[R "UR URS9Ul[R"URUR 5Ul g)Ngelur.) r7r8r rrrBr;rCrrrrNs rRr8$ElectraGeneratorPredictions.__init__sV (0f&;&;AVAVWYYv1163H3HI rTclURU5nURU5nURU5nU$r)rrrB)rOgenerator_hidden_statesrs rRre#ElectraGeneratorPredictions.forwards3 #:;  6 }5 rT)rBrrrrrs@rRr r sKJrTr cP^\rSrSr\rSrSrSrSr Sr Sr \ \ \S.rU4SjrSrU=r$)ElectraPreTrainedModelielectraT)r attentionscross_attentionscF>[TU]U5 [U[5(a|[R "UR [R"UR RS5RS55 [R"UR5 gg)Nr2r1) r7 _init_weightsrr*initcopy_r0rHrIr^rJzeros_r4)rOrsrQs rRr-$ElectraPreTrainedModel._init_weights sp f% f/ 0 0 JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h KK-- . 1rT)rgrhrirjr( config_classbase_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrrr_can_record_outputsr-rprqrrs@rRr(r(sF L!&*#N"&%*1 //rTr(z3 Output type of [`ElectraForPreTraining`]. ) custom_introc\rSrSr%SrSr\RS-\S'Sr \RS-\S'Sr \ \RS-\S'Sr \ \RS-\S'Sr g) ElectraForPreTrainingOutputia loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Total loss of the ELECTRA objective. logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Prediction scores of the head (scores for each token before SoftMax). Nlossrrr*r2)rgrhrirjrkr>rHrm__annotations__rrrr*rpr2rTrRr=r=sg&*D%  d ")'+FE   $+59M5**+d2926Je''(4/6rTr=c^\rSrSrU4SjrSrSr\\\ SS\ RS-S\ RS-S\ RS-S \ RS-S \ RS-S \ RS-S \ RS-S \ \ RS-S\S-S\\S\\ R\-4Sjj555rSrSrU=r$) ElectraModeli$c2>[TU]U5 [U5UlURUR :wa0[ R"URUR 5Ul[U5Ul Xl SUl UR5 g)NF)r7r8r*rdr;rrrembeddings_projectrencoderrPgradient_checkpointing post_initrNs rRr8ElectraModel.__init__&sr  +F3  F$6$6 6&(ii0E0EvGYGY&ZD #%f-  &+# rTc.URR$rrdr=rOs rRget_input_embeddings!ElectraModel.get_input_embeddings3s...rTc$XRlgrrI)rOrvs rRset_input_embeddings!ElectraModel.set_input_embeddings6s */'rTNrUrwr4r0rVrrrr ryrXc USLUSL- (a [S5eURR(aU bU OURRn OSn U (abUc_UcURR(a.[ [ URS9[ URS95O[ URS9nUbUR5OSn URUUUUU S9n [US5(aURU 5n URUUU UUS9up'UR"U 4UUUUU US.U D6n [U RU RS 9$) Nz:You must specify exactly one of input_ids or inputs_embedsF)rPr)rUr0r4rVrWrC)rwrembedding_outputrr)rwrrrr r0r )rrPrr is_encoder_decoderrrget_seq_lengthrdr]rC_create_attention_masksrDrr r)rOrUrwr4r0rVrrrr ryrWrQencoder_outputss rRreElectraModel.forward9su  -t"; <YZ Z ;; ! !%.%:  @U@UII 0)4 8V8V$L $DlZ^ZeZeFfg!5  FUE`!?!?!Afg??%)'#9 +  4- . .#667GH 151M1M)#9-"7+ 2N2 .,,   )"7#9+%    9-??+;;  rTcURR(a[URUUUS9nO[URUUS9nUb[URUUUS9nX4$)N)rPrVrwr)rPrVrw)rPrVrwr)rPrrr)rOrwrrQrrs rRrT$ElectraModel._create_attention_masks}sr ;; ! !/{{.- / N7{{.-N " -%>{{.5&; & "55rT)rPrdrCrDrE) NNNNNNNNN)rgrhrirjr8rKrNr%r&r"rHrolistrmrrr!rrrerTrprqrrs@rRrArA$s2 /0 *..2.2,0-1596::>!%> <<$&>  t+>  t+ > llT) > ||d* >  %||d2> !& t 3> e//047> $;> +,>  u|| A A>  > B66rTrAc2^\rSrSrSrU4SjrSrSrU=r$)ElectraClassificationHeadiz-Head for sentence-level classification tasks.c>[TU]5 [R"URUR5UlUR b UR O URn[S5Ul [R"U5Ul [R"URUR5Ul g)Nr")r7r8rrrrclassifier_dropoutrEr rrDrF num_labelsout_projrOrPr]rQs rRr8"ElectraClassificationHead.__init__s YYv1163E3EF )/)B)B)NF % %TZTnTn )0zz"45  &"4"4f6G6GH rTc USS2SSS24nURU5nURU5nURU5nURU5nURU5nU$)Nr)rFrrr_)rOfeaturesryxs rRre!ElectraClassificationHead.forwardsZ Q1W  LLO JJqM OOA  LLO MM! rT)rrrFr_rrrs@rRr[r[s7IrTr[c^\rSrSrSrS\4U4SjjrS S\RS\RS-S\R4S jjr S r U=r $) ElectraSequenceSummaryia Compute a single vector summary of a sequence hidden states. Args: config ([`ElectraConfig`]): The config used by the model. Relevant arguments in the config class of the model are (refer to the actual config class of your model for the default values it uses): - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are: - `"last"` -- Take the last token hidden state (like XLNet) - `"first"` -- Take the first token hidden state (like Bert) - `"mean"` -- Take the mean of all tokens hidden states - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2) - `"attn"` -- Not implemented now, use multi-head attention - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction. - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes (otherwise to `config.hidden_size`). - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output, another string or `None` will add no activation. - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation. - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation. rPc>[TU]5 [USS5UlURS:Xa[e[ R "5Ul[US5(aUR(aq[US5(a.UR(aURS:a URnO URn[ R"URU5Ul[USS5nU(a [U5O[ R "5Ul[ R "5Ul[US5(a5UR"S:a%[ R$"UR"5Ul[ R "5Ul[US 5(a7UR(S:a&[ R$"UR(5Ulggg) N summary_typelastattnsummary_use_projsummary_proj_to_labelsrsummary_activationsummary_first_dropoutsummary_last_dropout)r7r8getattrriNotImplementedErrorrIdentitysummaryr]rlrmr^rrr r first_dropoutrorD last_dropoutrp)rOrP num_classesactivation_stringrQs rRr8ElectraSequenceSummary.__init__sa #FNFC    && %{{} 6- . .63J3Jv788V=Z=Z_e_p_pst_t$// $00 99V%7%7EDL#F,@$GIZN3D$E`b`k`k`m[[] 62 3 38T8TWX8X!#F,H,H!ID KKM 61 2 2v7R7RUV7V " 6+F+F GD 8W 2rTNr cls_indexrXcURS:Xa USS2S4nGOURS:Xa USS2S4nGOURS:XaURSS9nOURS :XaUc?[R"US SS2SS24URS S- [R S 9nOXUR S5R S5nURS UR5S- -URS54-5nURS U5RS 5nOURS:Xa[eURW5nURU5nURU5nUR!U5nU$)a# Compute a single vector summary of a sequence hidden states. Args: hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`): The hidden states of the last layer. cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*): Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token. Returns: `torch.FloatTensor`: The summary of the sequence hidden states. rjNr2firstrmeanr'r}rz.r5)r2rk)rir}rH full_liker^rM unsqueezerJrZrLr_rrrrurtrrv)rOrrzrs rRreElectraSequenceSummary.forwardsn    &"1b5)F   ' )"1a4(F   & ("''A'.F   + - !OO!#rr1*-!''+a/** &//3==bA %,,Uimmo6I-JmN`N`acNdMf-fg "))"i8@@DF   & (% %##F+f%(""6* rT)rrurvrtrir) rgrhrirjrkr(r8rHrmrlrerprqrrs@rRrgrgsV2H}H<VZ)"..);@;K;Kd;R)   ))rTrgz ELECTRA Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. c8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$) ElectraForSequenceClassificationic>[TU]U5 URUlXl[ U5Ul[ U5UlUR5 gr) r7r8r^rPrAr)r[ classifierrFrNs rRr8)ElectraForSequenceClassification.__init__ sF   ++ #F+ 3F; rTNrUrwr4r0rVlabelsryrXc UR"U4UUUUSS.UD6nUSn URU 5n Sn UGbURRcURS:XaSURlOoURS:aNUR [ R:XdUR [ R:XaSURlOSURlURRS:XaI[5n URS:Xa&U "U R5UR55n OU "X5n OURRS:Xa=[5n U "U RS UR5URS 55n O,URRS:Xa[5n U "X5n [U U URUR S 9$) ae labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Trwr4r0rV return_dictrNr' regressionsingle_label_classificationmulti_label_classificationr2r>rrr*)r)rrP problem_typer^r6rHrMrnrrrrrrrr*) rOrUrwr4r0rVrryrsequence_outputrr>loss_fcts rRre(ElectraForSequenceClassification.forward*s$'+ll ' ))%' ' ' #6a81  {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#F3D))-JJ+- B @&++b/R))-II,./'5CC2==   rT)rrPr)r^NNNNNN)rgrhrirjr8r$r"rHrorr!rrrerprqrrs@rRrrs*..2.2,0-1&*9 <<$&9  t+9  t+ 9 llT) 9 ||d* 9  t#9 +,9  u|| 7 79 9 rTrz Electra model with a binary classification head on top as used during pretraining for identifying generated tokens. It is recommended to load the discriminator checkpoint into that model. c8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)ElectraForPreTrainingihc>[TU]U5 [U5Ul[ U5UlUR 5 gr)r7r8rAr)rdiscriminator_predictionsrFrNs rRr8ElectraForPreTraining.__init__ps3  #F+ )H)P& rTNrUrwr4r0rVrryrXc UR"U4UUUUSS.UD6nUSn URU 5n Sn Ub[R"5n Ub_UR SU R S5S:Hn U R SU R S5U nXmnU "XR 55n O4U "U R SU R S5UR 55n [U U URURS9$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the ELECTRA loss. Input should be a sequence of tokens (see `input_ids` docstring) Indices should be in `[0, 1]`: - 0 indicates the token is an original token, - 1 indicates the token was replaced. Examples: ```python >>> from transformers import ElectraForPreTraining, AutoTokenizer >>> import torch >>> discriminator = ElectraForPreTraining.from_pretrained("google/electra-base-discriminator") >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-discriminator") >>> sentence = "The quick brown fox jumps over the lazy dog" >>> fake_sentence = "The quick brown fox fake over the lazy dog" >>> fake_tokens = tokenizer.tokenize(fake_sentence, add_special_tokens=True) >>> fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt") >>> discriminator_outputs = discriminator(fake_inputs) >>> predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2) >>> fake_tokens ['[CLS]', 'the', 'quick', 'brown', 'fox', 'fake', 'over', 'the', 'lazy', 'dog', '[SEP]'] >>> predictions.squeeze().tolist() [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0] ```TrrNr2r'r) r)rrrrr^floatr=rr*)rOrUrwr4r0rVrryrdiscriminator_sequence_outputrr>r active_loss active_logits active_labelss rRreElectraForPreTraining.forwardxs)V'+ll ' ))%' ' ' #)DA(F%//0MN  ++-H),11"6S6Y6YZ[6\]abb & B0M0S0STU0V WXc d & 3  /B/B/DE B0M0S0STU0V WY_YeYeYgh*5CC2==   rT)rr)r)rgrhrirjr8r$r"rHrorr!rr=rerprqrrs@rRrrhs*..2.2,0-1&*F <<$&F  t+F  t+ F llT) F ||d* F  t#F +,F  u|| : :F F rTrz Electra model with a language modeling head on top. Even though both the discriminator and generator may be loaded into this model, the generator is the only model of the two to have been trained for the masked language modeling task. cL^\rSrSrSS0rU4SjrSrSr\\ SS\ RS-S \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ \ S\\ R\-4Sjj55rSrU=r$)ElectraForMaskedLMigenerator_lm_head.weight)electra.embeddings.word_embeddings.weightc>[TU]U5 [U5Ul[ U5Ul[ R"URUR5Ul UR5 gr) r7r8rAr)r generator_predictionsrrr;r:generator_lm_headrFrNs rRr8ElectraForMaskedLM.__init__sR  #F+ %@%H"!#6+@+@&BSBS!T rTcUR$rrrJs rRget_output_embeddings(ElectraForMaskedLM.get_output_embeddings%%%rTcXlgrr)rOr=s rRset_output_embeddings(ElectraForMaskedLM.set_output_embeddingss!0rTNrUrwr4r0rVrryrXc nUR"U4UUUUSS.UD6nUSn URU 5n URU 5n Sn UbQ[R"5n U "U R SUR R5UR S55n [U U URURS9$)az labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` TrrNr2r) r)rrrrrrPr:rrr*) rOrUrwr4r0rVrryr%generator_sequence_outputprediction_scoresr>rs rRreElectraForMaskedLM.forwards$#',, # ))%' # # %! 667PQ 223DE  **,H-222t{{7M7MNPVP[P[\^P_`D$1??.99   rTr)rrr)rgrhrirj_tied_weights_keysr8rrr$r"rHrorr!rrrerprqrrs@rRrrs56ab&1*..2.2,0-1&*) <<$&)  t+)  t+ ) llT) ) ||d* )  t#) +,)  u|| ~ -) ) rTrz Electra model with a token classification head on top. Both the discriminator and generator may be loaded into this model. c8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)ElectraForTokenClassificationi cf>[TU]U5 URUl[U5UlUR b UR O UR n[R"U5Ul [R"URUR5Ul UR5 gr)r7r8r^rAr)r]rErrDrFrrrrFr`s rRr8&ElectraForTokenClassification.__init__s   ++#F+ )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ rTNrUrwr4r0rVrryrXc DUR"U4UUUUSS.UD6nUSn URU 5n URU 5n Sn Ub<[5n U "U R SUR 5UR S55n [ U U URURS9$)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. TrrNr2r) r)rFrrrr^rrr*) rOrUrwr4r0rVrryrrrr>rs rRre%ElectraForTokenClassification.forward!s '+ll ' ))%' ' ' #)DA(F%(, 5R(S%!>?  ')HFKKDOO[TU]U5 URUl[U5Ul[ R "URUR5UlUR5 gr) r7r8r^rAr)rrr qa_outputsrFrNs rRr8$ElectraForQuestionAnswering.__init__QsS   ++#F+ ))F$6$68I8IJ rTNrUrwr4r0rVstart_positions end_positionsryrXc UR"U4UUUUSS.UD6n U Sn URU 5n U RSSS9upU RS5R 5n U RS5R 5n SnUbUb[ UR 55S:aURS5n[ UR 55S:aURS5nU R S5nURSU5nURSU5n[US9nU"X5nU"X5nUU-S- n[UU U U RU RS 9$) NTrrr'r2r}) ignore_indexr|)r> start_logits end_logitsrr*) r)rsplitrrlenrLclamprrrr*)rOrUrwr4r0rVrrryrrrrr total_loss ignored_indexr start_lossend_losss rRre#ElectraForQuestionAnswering.forward[sz'+ll ' ))%' ' ' #6a81#)<<r<#: #++B/::< ''+668   &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M-33A}EO)//=AM']CH!,@J :H$x/14J+%!5CC2==   rT)r)r^r)NNNNNNN)rgrhrirjr(r3r4r8r$r"rHrorr!rrrerprqrrs@rRrrLs L!*..2.2,0-1/3-13 <<$&3  t+3  t+ 3 llT) 3 ||d* 3 ,3 ||d*3 +,3  u|| ; ;3 3 rTrc8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)ElectraForMultipleChoiceic>[TU]U5 [U5Ul[ U5Ul[ R"URS5Ul UR5 gr) r7r8rAr)rgsequence_summaryrrrrrFrNs rRr8!ElectraForMultipleChoice.__init__sM  #F+ 6v >))F$6$6: rTNrUrwr4r0rVrryrXc UbURSOURSnUb!URSURS55OSnUb!URSURS55OSnUb!URSURS55OSnUb!URSURS55OSnUb1URSURS5URS55OSnUR"U4UUUUSS.UD6n U Sn UR U 5n UR U 5n U RSU5n SnUb[ 5nU"X5n[UU U RU RS9$) a input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr'r2r~Trrr) r^rrLr)rrrrrr*)rOrUrwr4r0rVrry num_choicesrr pooled_outputrreshaped_logitsr>rs rRre ElectraForMultipleChoice.forwardsT-6,Aiooa(}GZGZ[\G] >G>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei (   r=#5#5b#9=;M;Mb;Q R  '+ll ' ))%' ' ' #6a8--o> / ++b+6  ')HO4D("5CC2==   rT)rr)rr)rgrhrirjr8r$r"rHrorr!rrrerprqrrs@rRrrs*..2.2,0-1&*N <<$&N  t+N  t+ N llT) N ||d* N  t#N +,N  u|| 8 8N N rTrzS ELECTRA Model with a `language modeling` head on top for CLM fine-tuning. c^\rSrSrSS0rU4SjrSrSr\\ SS\ RS-S \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ RS-S\ RS-S\ S-S\ S-S\\ R-S\\S\\ R\-4Sjj55rSrU=r$)ElectraForCausalLMirrc0>[TU]U5 UR(d[R S5 [ U5Ul[U5Ul[R"URUR5Ul UR5 g)NzOIf you want to use `ElectraForCausalLM` as a standalone, add `is_decoder=True.`)r7r8rloggerwarningrAr)r rrrr;r:rrFrNs rRr8ElectraForCausalLM.__init__sh    NNl m#F+ %@%H"!#6+@+@&BSBS!T rTcUR$rrrJs rRr(ElectraForCausalLM.get_output_embeddingsrrTcXlgrr)rOnew_embeddingss rRr(ElectraForCausalLM.set_output_embeddings s!/rTNrUrwr4r0rVrrrrr logits_to_keepryrXc UbSn UR"U4UUUUUUU U SS. U D6n U Rn[U [5(a [ U *S5OU nUR UR USS2USS2455nSnUb)UR"SUXRRS.U D6n[UUU RU RU RU RS9$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Example: ```python >>> from transformers import AutoTokenizer, ElectraForCausalLM, ElectraConfig >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-generator") >>> config = ElectraConfig.from_pretrained("google/electra-base-generator") >>> config.is_decoder = True >>> model = ElectraForCausalLM.from_pretrained("google/electra-base-generator", config=config) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits ```NFT) rwr4r0rVrrrr r)rrr:)r>rrrr*r+r2)r)r rrnslicerr loss_functionrPr:rrrr*r+)rOrUrwr4r0rVrrrrr rryoutputsr slice_indicesrr>s rRreElectraForCausalLM.forward sN  I=A\\  > ))%'"7#9+ >  >  11 8B>SV8W8W~ot4]k ''(B(B=QRTacdQdCe(fg  %%pVF{{OeOepiopD0#33!//))$55   rTr) NNNNNNNNNNr)rgrhrirjrr8rrr$r"rHror rrnrr!rrrerprqrrs@rRrrs\ 56ab &0*..2.2,0-1596:&*(,!%-.F <<$&F  t+F  t+ F llT) F ||d* F  %||d2F !& t 3F  t#F F $;F ell*F +,F  u|| @ @F F rTr) rrrrrrrrAr()Nr)Wrkcollections.abcr dataclassesrrHrtorch.nnrrrr r. activationsr r cache_utilsr rr generationr masking_utilsrrmodeling_layersrmodeling_outputsrrrrrrrrmodeling_utilsrrprocessing_utilsr pytorch_utilsrutilsr r!r"r# utils.genericr$r%utils.output_capturingr&configuration_electrar( get_loggerrgrModuler*rorrrrrrrrrrrr r(r=rAr[rgrrrrrrr__all__r2rTrRrsH$! AA&1CC)J9   G&6 J50   H %= =N!% II% <<% % << % LL4' % T\ %%'(%:@)299@)HI)BIII)Z  .ryy.< "))  BII >->D RYY @bii&"))$/_//*   7+ 7  7v6)v6v6r 0`RYY`F F '=F  F RQ 2Q Q h> /> > B6 $:6 6 rC "8C C L[ 5[ [ | ] /]  ] @ rT