ZjaSrSSKJr SSKrSSKrSSKJr SSKJrJ r J r SSK J r SSKJr SS KJr SS KJr SS KJr SS KJr SS KJrJrJrJrJrJr SSKJ r J!r! SSK"J#r# SSK$J%r% SSK&J'r'J(r(J)r) SSK*J+r+ SSK,J-r-J.r. SSK/J0r0 SSK1J2r2 \)Rf"\45r5S\6S\6S\Rn4Sjr8S\6S\6S\Rn4Sjr9"SS\Rt5r;SAS\RtS\RnS \RnS!\RnS"\RnS-S#\"S)S*\Rt5r?"S+S,\5r@"S-S.\Rt5rA\("S/S0\!55rB\("S1S2\B55rC\("S3S49"S5S6\B55rD\("S7S49"S8S9\B55rE\("S:S;\B55rF\("S<S=\B55rG\("S>S?\B55rH/S@QrIg)Bz PyTorch DistilBERT model adapted in part from Facebook, Inc XLM model (https://github.com/facebookresearch/XLM) and in part from HuggingFace PyTorch version of Google AI Bert model (https://github.com/google-research/bert) )CallableN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)get_activation)PreTrainedConfig)is_deepspeed_zero3_enabled)create_bidirectional_mask)GradientCheckpointingLayer)BaseModelOutputMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward)TransformersKwargsauto_docstringlogging)deprecate_kwarg)can_return_tuplemerge_with_config_defaults)capture_outputs)DistilBertConfign_posdimoutc[5(a^SSKnURRUSS9 [R R 5S:Xa[XUS9sSSS5 $SSS5 g[XUS9$!,(df  g=f)Nr) modifier_rankr"r#r$)r deepspeedzeroGatheredParameterstorch distributedget_rank_create_sinusoidal_embeddings)r"r#r$r(s ڃ/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/distilbert/modeling_distilbert.pycreate_sinusoidal_embeddingsr0?st!## ^^ . .s! . D  ))+q045sSE D0E D-5sKK E Ds ,A88 Bc[R"[U5VVs/sH?n[U5Vs/sH%oC[R"SSUS--U- 5- PM' snPMA snn5nSUl[ R "[R"USS2SSS2455USS2SSS24'[ R "[R"USS2SSS2455USS2SSS24'UR5 U$s snfs snnf)Ni'Frr ) nparrayrangepower requires_gradr+ FloatTensorsincosdetach_)r"r#r$posj position_encs r/r.r.Js88hmnshtuhtadQVWZQ[\Q[ABHHUAaL34F$GGQ[\htuvLC$$RVVLADqD,A%BCC14a4L$$RVVLADqD,A%BCC14a4LKKM J ]usD ,DD D c ^\rSrSrS\4U4Sjjr\"SSSS9SS \RS\RS-S \RS-S \R4S jj5r S r U=r $) EmbeddingsSconfigc>[TU]5 [R"URUR UR S9Ul[R"URUR 5Ul [R"UR SS9Ul [R"UR5Ul URS[R"UR5R!S5SS9 g)N) padding_idx-q=eps position_idsr F) persistent)super__init__r Embedding vocab_sizer# pad_token_idword_embeddingsmax_position_embeddingsposition_embeddings LayerNormDropoutdropoutregister_bufferr+arangeexpandselfrB __class__s r/rMEmbeddings.__init__Ts !||F,=,=vzzW]WjWjk#%<<0N0NPVPZPZ#[ fjje<zz&..1   ELL)G)GHOOPWXej  input_embedsz5.6.0 inputs_embeds)versionnew_nameN input_idsrHreturncUbURU5nURS5nUcu[US5(aURSS2SU24nON[R "U[R URS9nURS5RU5nURU5nX%-nURU5nURU5nU$)Nr rH)dtypedevicer) rQsizehasattrrHr+rXlongrg unsqueeze expand_asrSrTrV)r[rcr`rH seq_lengthrS embeddingss r/forwardEmbeddings.forward_s   00;M"''*  t^,,#00KZK@ $||JejjQZQaQab +55a8BB9M "66|D"8 ^^J/ \\*- r^)rTrVrSrQ)NN) __name__ __module__ __qualname____firstlineno__r rMrr+Tensor LongTensorro__static_attributes__ __classcell__r\s@r/r@r@Ssx  /  ^WO.204 <<||d*&&-   Pr^r@modulequerykeyvalueattention_maskscalingrVkwargsc UcURS5S-n[R"XRSS55U-nUbX-n[R R USS9n[R RXURS9n[R"X5n U RSS5R5n X4$)NrJr2rr#)ptrainingr ) rhr+matmul transposer functionalsoftmaxrVr contiguous) rzr{r|r}r~rrVr attn_weights attn_outputs r/eager_attention_forwardr~s**R.D(<<}}Q':;gEL!#4 ==((2(>L==((6??([L,,|3K''1-88:K  $$r^c ^\rSrSrS\4U4SjjrS S\RS\RS-S\ \ S\ \R4S jjr S r U=r$) DistilBertSelfAttentionrBc >[TU]5 XlURUlURUlURUR-UlUR S-UlURUR-S:wa&[SURSURS35e[R"URURS9Ul [R"URURS9Ul [R"URURS9Ul [R"URURS9Ul [R"URS9UlSUlg) Nrrzself.n_heads: z must divide self.dim:  evenly in_features out_featuresrF)rLrMrBn_headsr#attention_head_sizer ValueErrorrLinearq_link_linv_linout_linrUattention_dropoutrV is_causalrZs r/rM DistilBertSelfAttention.__init__s  ~~ ::#'88t||#; //5  88dll "a '~dll^;RSWS[S[R\\cde eYY6::FJJO YY6::FJJO YY6::FJJO yyVZZfjjQ zzF$<$<= r^N hidden_statesr~rrdc URSSn/UQSPURP7nURU5R"U6R SS5nUR U5R"U6R SS5nUR U5R"U6R SS5n[R"URR[5n U "UUUUU4UR(dSOURRURS.UD6upU R "/UQSP76R#5n UR%U 5n X4$)NrJr r2)rVr)shaperrviewrrrr get_interfacerB_attn_implementationrrrVrrreshaperr) r[rr~r input_shape hidden_shape query_layer key_layer value_layerattention_interfacerrs r/roDistilBertSelfAttention.forwardsJ $))#2. CCbC$*B*BC jj/44lCMMaQRS JJ}-22LAKKAqQ jj/44lCMMaQRS (?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! "));;;;FFH ll;/ ((r^) rrBr#rVrrrrrrrN)rqrrrsrtr rMr+rur8rrtuplerorwrxrys@r/rrsc/248)||)))D0)+, ) u||  ))r^rc^\rSrSrS\4U4SjjrS\RS\R4SjrS\RS\R4Sjr Sr U=r $) FFNrBc>[TU]5 [R"URS9UlUR UlSUl[R"URURS9Ul [R"URURS9Ul [UR5Ul g)Nrr r)rLrMrrUrVchunk_size_feed_forward seq_len_dimrr# hidden_dimlin1lin2r activationrZs r/rM FFN.__init__s zzFNN3 '-'E'E$II&**6CTCTU II&*;*;&**U ():):;r^inputrdcZ[URURURU5$r)rff_chunkrr)r[rs r/ro FFN.forwards%(8T8TVZVfVfhmnnr^cURU5nURU5nURU5nURU5nU$r)rrrrV)r[rxs r/r FFN.ff_chunks= IIe  OOA  IIaL LLOr^)rrrVrrr) rqrrrsrtr rMr+rurorrwrxrys@r/rrsN</<oU\\oelloellu||r^rc ^\rSrSrS\4U4SjjrS S\RS\RS-S\\ S\ \RS 44S jjr S r U=r $) TransformerBlockrBcx>[TU]5 URUR-S:wa&[ SURSURS35e[ U5Ul[R"URSS9Ul [U5Ul [R"URSS9Ul g)Nrzconfig.n_heads z must divide config.dim rrE)normalized_shaperG) rLrMr#rrr attentionrrT sa_layer_normrffnoutput_layer_normrZs r/rMTransformerBlock.__init__s  :: &! +v~~.>>VW]WaWaVbbijk k08\\6::5Qv;!#vzzu!Ur^Nrr~rrd.c UR"U4SU0UD6upEURXA-5nURU5nURXd-5nU$)Nr~)rrrr)r[rr~rattention_output_ ffn_outputs r/roTransformerBlock.forwardsl#nn  )    --.>.NOXX./ ++J,IJ r^)rrrrr)rqrrrsrtr rMr+rurrrrorwrxrys@r/rrsh V/ V /3|| t++,  u||S ! r^rc ^\rSrSrS\4U4SjjrS S\RS\RS-S\\ S\ 4S jjr S r U=r $) Transformeri rBc>[TU]5 URUl[R"[ UR5Vs/sHn[ U5PM sn5UlSUlgs snf)NF) rLrMn_layersr ModuleListr5rlayergradient_checkpointing)r[rBrr\s r/rMTransformer.__init__ sW  ]]eFOOF\#]F\$4V$[TU]U5 [U[5(aURR (a[ R"URR[URRURR[R"URR555 [ R"UR[R "URR"S5R%S55 gg)zInitialize the weights.rJrIN)rL _init_weights isinstancer@rBsinusoidal_pos_embdsinitcopy_rSweightr0rRr#r+ empty_likerHrXrrY)r[rzr\s r/r'DistilBertPreTrainedModel._init_weights0s f% fj ) ){{// ..550 ;; (()C)C)J)JK JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h *r^)rqrrrsrtr!__annotations__base_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrr_can_record_outputsr+no_gradrModulerrwrxrys@r/rr"sb $&*#N"&)-  ]]_ iBII i ir^rc^\rSrSrS\4U4SjjrS\R4SjrS\ 4Sjr S\R4Sjr S \R4S jr \ \\SS \R"S -S \R"S -S\R"S -S\R"S -S\\S\\\R"S4-4 Sjj555rSrU=r$)DistilBertModeliArBc>[TU]U5 [U5Ul[ U5UlUR 5 gr)rLrMr@rnr transformer post_initrZs r/rMDistilBertModel.__init__Cs5  $V,&v. r^rdc.URR$z! Returns the position embeddings )rnrSr[s r/get_position_embeddings'DistilBertModel.get_position_embeddingsLs222r^new_num_position_embeddingscXRR- nUS:Xag[RSUS35 XRlURR R R5n[R"URRURR5URlURR(aH[URRURRUR R S9 O[R"5 US:a9[R"U5URR R SU*&O2[R"USU5URR lSSS5 URR R!UR"5 g!,(df  N>=f) Resizes position embeddings of the model if `new_num_position_embeddings != config.max_position_embeddings`. Arguments: new_num_position_embeddings (`int`): The number of new position embedding matrix. If position embeddings are learned, increasing the size will add newly initialized vectors at the end, whereas reducing the size will remove vectors from the end. If position embeddings are not learned (*e.g.* sinusoidal position embeddings), increasing the size will add correct vectors at the end following the position encoding algorithm, whereas reducing the size will remove vectors from the end. rNz(Setting `config.max_position_embeddings=z`...r')rBrRloggerinfornrSrclonerrNr#rr0r+r Parametertorg)r[rnum_position_embeds_diffold_position_embeddings_weights r/resize_position_embeddings*DistilBertModel.resize_position_embeddingsRsi$?AdAd#d  $q (  >?Z>[[_`a.I +)-)L)L)S)S)Y)Y)[&.0ll4;;;^;^`d`k`k`o`o.p+ ;; + + (kk99t{{TXTlTlTsTs +a/]_]i]i6^DOO77>>?YAY@YZBD67P8PQBDOO77> ! ++..t{{;!s =A2G'' G5c.URR$rrnrQrs r/get_input_embeddings$DistilBertModel.get_input_embeddings|s...r^new_embeddingsc$XRlgrrr[rs r/set_input_embeddings$DistilBertModel.set_input_embeddingss *8'r^Nrcr~r`rHr.c USLUSL- (a [S5eURXU5n[URUUS9nUR"SUUS.UD6$) input_ids (`torch.LongTensor` of shape `(batch_size, num_choices)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. Nz:You must specify exactly one of input_ids or inputs_embeds)rBr`r~)rr~r)rrnr rBr)r[rcr~r`rHrrns r/roDistilBertModel.forwardss0  -t"; <YZ Z__Y|L 2;;$)   $)   r^)rnr)NNNN)rqrrrsrtr rMrrNrintr rrrrrr+rurrrrrorwrxrys@r/rrAs/33 ([TU]U5 [UR5Ul[ U5Ul[ R"URUR5Ul [ R"URSS9Ul [ R"URUR5Ul UR5 [ R"5Ulg)NrErF)rLrMr rrrrrr#vocab_transformrTvocab_layer_normrOvocab_projectorrr mlm_loss_fctrZs r/rMDistilBertForMaskedLM.__init__s  ():):;)&1!yyVZZ@ " VZZU C!yyV5F5FG //1r^rdc6URR5$rrrrs r/r-DistilBertForMaskedLM.get_position_embeddings6688r^rc:URRU5 grNrr r[rs r/r 0DistilBertForMaskedLM.resize_position_embeddings 223NOr^cUR$rr"rs r/get_output_embeddings+DistilBertForMaskedLM.get_output_embeddingss###r^rcXlgrr0rs r/set_output_embeddings+DistilBertForMaskedLM.set_output_embeddingss-r^Nrcr~r`labelsrHr.c UR"SUUUUSS.UD6nUSnURU5n URU 5n URU 5n UR U 5n Sn Ub@UR U R SU RS55UR S55n [U U URURS9$)a* input_ids (`torch.LongTensor` of shape `(batch_size, num_choices)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Trcr~r`rH return_dictrNrJlosslogitsrrr) rr rr!r"r#rrhrrr) r[rcr~r`r6rHr dlbrt_outputrprediction_logitsmlm_losss r/roDistilBertForMaskedLM.forwards8 )'%    %Q  00? OO,=> 112CD 001BC  (():)?)?DUDZDZ[]D^)_agalalmoapqH$&44#..   r^)rrr#r!r"r NNNNN)rqrrrsrt_tied_weights_keysr rMrrNrrr rr1r4rrr+rurvrrrrrorwrxrys@r/rrs 34bc 2/ 299 Pc P$ryy$.BII.*..2-1*.,0 1 <<$&1  t+1 ||d* 1   4' 1 llT) 1 +,1  % c 12 21 1 r^rz DistilBert Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. cT^\rSrSrS\4U4SjjrS\R4SjrS\ 4Sjr \ \ SS \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\\S\\\ RS4-4Sjj55rSrU=r$)#DistilBertForSequenceClassificationirBc>[TU]U5 URUlXl[ U5Ul[ R"URUR5Ul [ R"URUR5Ul [ R"UR5Ul UR5 gr)rLrM num_labelsrBrrrrr#pre_classifier classifierrUseq_classif_dropoutrVrrZs r/rM,DistilBertForSequenceClassification.__init__s   ++ )&1 ii FJJ?))FJJ0A0ABzz&"<"<=  r^rdc6URR5$rr&rs r/r;DistilBertForSequenceClassification.get_position_embeddings'r(r^rc:URRU5 gr*r+r,s r/r >DistilBertForSequenceClassification.resize_position_embeddings-r.r^Nrcr~r`r6rHr.c RUR"S UUUUSS.UD6nUSnUSS2S4n URU 5n [R"5"U 5n UR U 5n UR U 5n Sn UGbUR RcURS:XaSUR lOoURS:aNUR[R:XdUR[R:XaSUR lOSUR lUR RS:XaI[5n URS:Xa&U "U R5UR55n OU "X5n OUR RS:Xa=[5n U "U R!S UR5UR!S 55n O,UR RS:Xa[#5n U "X5n [%U U UR&UR(S 9$) ae labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Tr8rNr regressionsingle_label_classificationmulti_label_classificationrJr:r)rrGrReLUrVrHrB problem_typerFrfr+rjrrsqueezerrrrrr) r[rcr~r`r6rHrdistilbert_output hidden_state pooled_outputr<r;loss_fcts r/ro+DistilBertForSequenceClassification.forward;s"!OO )'%    )+ $QT* ++M:  -0  ]3 /  {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#F3D))-JJ+- B @&++b/R))-II,./'+99(33   r^)rHrBrrVrFrGrA)rqrrrsrtr rMrrNrrr rrr+rurvrrrrrorwrxrys@r/rDrDs / 99 Pc P*..2-1*.,0 : <<$&:  t+: ||d* :   4' : llT) : +,:  "E%,,*;$< <: : r^rDct^\rSrSrS\4U4SjjrS\R4SjrS\ 4Sjr \ \ SS \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ RS-S\\S\\\ RS4-4Sjj55rSrU=r$)DistilBertForQuestionAnsweringizrBc^>[TU]U5 [U5Ul[R "UR UR5UlURS:wa[SUR35e[R"UR5Ul UR5 g)Nr2z)config.num_labels should be 2, but it is )rLrMrrrrr#rF qa_outputsrrU qa_dropoutrVrrZs r/rM'DistilBertForQuestionAnswering.__init__|s  )&1))FJJ0A0AB    !HIZIZH[\] ]zz&"3"34  r^rdc6URR5$rr&rs r/r6DistilBertForQuestionAnswering.get_position_embeddingsr(r^rc:URRU5 gr*r+r,s r/r 9DistilBertForQuestionAnswering.resize_position_embeddingsr.r^Nrcr~r`start_positions end_positionsrHr.c UR"S UUUUSS.UD6nUSn URU 5n URU 5n U RSSS9upU R S5R 5n U R S5R 5n Sn UbUb[ UR55S:aUR S5n[ UR55S:aUR S5nU RS5nURSU5nURSU5n[R"US9nU"X5nU"X5nUU-S - n [U U U URURS 9$) rTr8rr rJrN) ignore_indexr2)r; start_logits end_logitsrrr)rrVr^splitrUrlenrhclamprrrrr)r[rcr~r`rerfrHrrVrr<rirj total_loss ignored_indexrY start_lossend_losss r/ro&DistilBertForQuestionAnswering.forwards2!OO )'%    *!,  ]3 /#)<<r<#: #++B/::< ''+668   &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M-33A}EO)//=AM** FH!,@J :H$x/14J+%!+99(33   r^)rrVr^)NNNNNN)rqrrrsrtr rMrrNrrr rrr+rurrrrrorwrxrys@r/r\r\zs / 99 Pc P*..2-1/3-1,0> <<$&>  t+> ||d* > , > ||d* > llT)> +,>  &ellC.?(@ @> > r^r\cT^\rSrSrS\4U4SjjrS\R4SjrS\ 4Sjr \ \ SS \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\\S\\\ RS4-4Sjj55rSrU=r$) DistilBertForTokenClassificationirBc0>[TU]U5 URUl[U5Ul[ R "UR5Ul[ R"URUR5Ul UR5 gr) rLrMrFrrrrUrVr hidden_sizerHrrZs r/rM)DistilBertForTokenClassification.__init__sg   ++)&1zz&..1 ))F$6$68I8IJ r^rdc6URR5$rr&rs r/r8DistilBertForTokenClassification.get_position_embeddingsr(r^rc:URRU5 gr*r+r,s r/r ;DistilBertForTokenClassification.resize_position_embeddingsr.r^Nrcr~r`r6rHr.c BUR"U4UUUSS.UD6nUSnURU5nURU5n Sn Ub<[5n U "U R SUR 5UR S55n [ U U URURS9$)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Tr~r`rHr9rNrJr:) rrVrHrrrFrrr) r[rcr~r`r6rHroutputssequence_outputr<r;rYs r/ro(DistilBertForTokenClassification.forwards//  )'%    "!*,,71  ')HFKKDOO[TU]U5 [U5Ul[R "UR UR 5Ul[R "UR S5Ul[R"UR5Ul UR5 g)Nr ) rLrMrrrrr#rGrHrUrIrVrrZs r/rM$DistilBertForMultipleChoice.__init__-so  )&1 ii FJJ?))FJJ2zz&"<"<=  r^rdc6URR5$rr&rs r/r3DistilBertForMultipleChoice.get_position_embeddings8r(r^rc:URRU5 g)a Resizes position embeddings of the model if `new_num_position_embeddings != config.max_position_embeddings`. Arguments: new_num_position_embeddings (`int`) The number of new position embeddings. If position embeddings are learned, increasing the size will add newly initialized vectors at the end, whereas reducing the size will remove vectors from the end. If position embeddings are not learned (*e.g.* sinusoidal position embeddings), increasing the size will add correct vectors at the end following the position encoding algorithm, whereas reducing the size will remove vectors from the end. Nr+r,s r/r 6DistilBertForMultipleChoice.resize_position_embeddings>r.r^Nrcr~r`r6rHr.c UbURSOURSnUb!URSURS55OSnUb!URSURS55OSnUb1URSURS5URS55OSnUR"U4UUUSS.UD6nUSn U SS2S4n UR U 5n [ R "5"U 5n URU 5n URU 5n U RSU5n Sn Ub[5nU"X5n [U U URURS9$) av input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Examples: ```python >>> from transformers import AutoTokenizer, DistilBertForMultipleChoice >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("distilbert-base-cased") >>> model = DistilBertForMultipleChoice.from_pretrained("distilbert-base-cased") >>> prompt = "In Italy, pizza served in formal settings, such as at a restaurant, is presented unsliced." >>> choice0 = "It is eaten with a fork and a knife." >>> choice1 = "It is eaten while held in the hand." >>> labels = torch.tensor(0).unsqueeze(0) # choice0 is correct (according to Wikipedia ;)), batch size 1 >>> encoding = tokenizer([[prompt, choice0], [prompt, choice1]], return_tensors="pt", padding=True) >>> outputs = model(**{k: v.unsqueeze(0) for k, v in encoding.items()}, labels=labels) # batch size is 1 >>> # the linear classifier still needs to be trained >>> loss = outputs.loss >>> logits = outputs.logits ```Nr rJTr}rr:) rrrhrrGrrSrVrHrrrr)r[rcr~r`r6rHr num_choicesr~rWrXr<reshaped_logitsr;rYs r/ro#DistilBertForMultipleChoice.forwardLs{b-6,Aiooa(}GZGZ[\G] >G>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImq(   r=#5#5b#9=;M;Mb;Q R  //  )'%    qz $QT* ++M:  -0  ]3 / ++b+6  ')HO4D("!//))   r^)rHrrVrGrA)rqrrrsrtr rMrrNrrr rrr+rurvrrrrrorwrxrys@r/rr+s / 99 Pc P*..2-1*.,0 U <<$&U  t+U ||d* U   4' U llT) U +,U  #U5<<+<%= =U U r^r)rrr\rDrtrr)Nr)J__doc__collections.abcrnumpyr3r+rtorch.nnrrrr r activationsr configuration_utilsr integrations.deepspeedr masking_utilsr modeling_layersrmodeling_outputsrrrrrrmodeling_utilsrrprocessing_utilsr pytorch_utilsrutilsrrrutils.deprecationr utils.genericrrutils.output_capturingrconfiguration_distilbertr! get_loggerrqrrrur0r.rr@floatrrrrrrrrrDr\rtr__all__rr^r/rsy % AA&)3@69G& 1I56   H % LL#LELLL3U\\''b!% II% <<% % << % LL4' % T\ %%'(%85)bii5)p"))* 1 F@"))@0iii<g /g g T _ 5_  _ D ^ *C^  ^ Bb %>b b JG '@G G Tw ";w w t r^