ZjiSrSSKrSSKrSSKJr SSKJrJrJr SSK J r SSK J r Jr SSKJr SS KJrJrJr SS KJr SS KJrJr S S KJr \R8"\5rSrSr S"Sjr!"SS\RD5r#Sr$"SS\RD5r%\"SS\55r&\"SS\&55r'\"SS9"SS\&\55r(\"SS9"SS \&55r)/S!Qr*g)#zPyTorch CTRL model.N)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)Cache DynamicCache)GenerationMixin)BaseModelOutputWithPastCausalLMOutputWithPastSequenceClassifierOutput)PreTrainedModel)auto_docstringlogging) CTRLConfigcPS[R"SSUS--U- 5- nX-$)Nri')torchpow)posi d_model_size angle_ratess w/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/ctrl/modeling_ctrl.py angle_defnr%s-eiiQ!V 'DEEK  c[[R"U[RS9R U5R S5[R"U[RS9R U5R S5U5n[R "USS2SSS245n[R"USS2SSS245n[R"XE/SS9nU$)Ndtyperrrdim) rrarangeint64to unsqueezesincoscat)positionrr! angle_radssinescosines pos_encodings rpositional_encodingr1*s XU[[144U;EEaH \588?II!LJ IIjADqD) *Eii 1add7+,G99e-26L rc x[R"XRSSSS55nURSnU[R "U5- nUb3UR S5UR S5pXsX- U 2SU 24S-- nUbXt-n[R"USS9n [R"X5n X4$) Nrrrrr"gr#)rmatmulpermuteshapenpsqrtsizesoftmax) qkvmaskattention_mask matmul_qkdkscaled_attention_logitsndnsattention_weightsoutputs rscaled_dot_product_attentionrG9s Q !Q1 56I B'"''"+5 (--b13J3O3OPR3SB" crc(9#:T#AA!"9"J &=2F \\+ /F  $$rcD^\rSrSrSU4SjjrSrSSjrSrU=r$)MultiHeadAttentionOcV>[TU]5 X lXlX0l[ XR- 5Ul[R"X5Ul [R"X5Ul [R"X5Ul [R"X5Ul gN) super__init__ num_headsr layer_idxintdepthrLinearWqWkWvdense)selfrrOrP __class__s rrNMultiHeadAttention.__init__Psq "("67 ))L7))L7))L7YY|: rcxURUSURUR5nUR/SQ5$)Nr"rrrr)reshaperOrRr5)rXx batch_sizes rsplit_into_heads#MultiHeadAttention.split_into_heads^s- IIj"dnndjj Ayy&&rc  URSn URU5nURU5nURU5nUR X:5nUR X*5nUR X5nUbUR X!UR 5up![X2XU5n U SR/SQ5n U Sn U RU SUR5nURU5n X4$)Nrr\rr") r6rTrUrVr`updaterPrGr5r]rrW)rXr=r<r;r> layer_pastr? use_cacheoutput_attentionskwargsr_rFscaled_attentionattnoriginal_size_attentions rforwardMultiHeadAttention.forwardbsWWQZ GGAJ GGAJ GGAJ  ! !! 0  ! !! 0  ! !! 0  !$$Q4>>:DA-aA^L!!9,,\:ay"2":"::r4K\K\"]34|r)rUrTrVrrWrRrPrOrLNNFF) __name__ __module__ __qualname____firstlineno__rNr`rk__static_attributes__ __classcell__rYs@rrIrIOs% ;'rrIc[R"[R"X5[R"5[R"X55$rL)r SequentialrSReLU)rdffs rpoint_wise_feed_forward_networkrys- ==<5rwwy"))CB^ __rc>^\rSrSrSU4SjjrSSjrSrU=r$) EncoderLayerc4>[TU]5 [XUS9Ul[ X5Ul[ R"USS9Ul[ R"USS9Ul [ R"U5Ul [ R"U5Ul g)NrPgư>eps) rMrNrImulti_head_attentionryffnr LayerNorm layernorm1 layernorm2Dropoutdropout1dropout2)rXrrOrxraterPrYs rrNEncoderLayer.__init__sn $6|Zc$d!2<E,,|>,,|> 4(  4( rc URU5nURUUUUUUUUS9n U Sn URU 5n X-n URU 5n UR U 5n UR U 5n X-n U 4U SS-nU$)Nrdr?rerfrr)rrrrrr)rXr^r>rdr?rerfrgnormed attn_outputs attn_outputout1out2 ffn_outputoutputss rrkEncoderLayer.forwards#00    !)/1  #1o mmK0 t$XXd^ ]]:.  'L,,r)rrrrrr)g?Nrm)rnrorprqrNrkrrrsrts@rr{r{s  ) rr{c8^\rSrSr%\\S'SrU4SjrSrU=r $)CTRLPreTrainedModelconfig transformerc>[TU]U5 [U[5(aY[R "UR [URRUR[R55 ggrL) rM _init_weights isinstance CTRLModelinitcopy_r0r1r n_positionsrrfloat)rXmodulerYs rr!CTRLPreTrainedModel._init_weightss[ f% fi ( ( JJ##%89R9RTZTgTginitit%u  )r) rnrorprqr__annotations__base_model_prefixrrrrsrts@rrrs %rrcL^\rSrSrU4SjrSrSr\SS\RS-S\ S-S\RS-S \RS-S \RS-S \RS-S \ S-S \ S-S\ S-S\ S-S\ \R\-4Sjj5rSrU=r$)rc >[TU]U5 URUlURUl[ R"URUR5Ul [ R"UR5Ul [ R"[UR5Vs/sH8n[URUR UR"UR$US9PM: sn5Ul[ R("URUR*S9UlUR/S[1UR2UR[4R65SS9 UR95 gs snf)Nr~rr0F) persistent)rMrNn_embdrn_layer num_layersr Embedding vocab_sizewr embd_pdropdropout ModuleListranger{n_headrx resid_pdrophrlayer_norm_epsilon layernormregister_bufferr1rrr post_init)rXrrrYs rrNCTRLModel.__init__s  "MM ..f//?zz&"3"34 v~~. .AV]]FMM6::vGYGYefg.   fmm9R9RS  /0B0BDDUDUW\WbWbcpu   s/?E7cUR$rLr)rXs rget_input_embeddingsCTRLModel.get_input_embeddingss vv rcXlgrLr)rXnew_embeddingss rset_input_embeddingsCTRLModel.set_input_embeddingssrN input_idspast_key_valuesr?token_type_ids position_ids inputs_embedsrerfoutput_hidden_states return_dictreturnc UbUOURRnUbUOURRnU bU OURRn U bU OURRn UbUb [ S5eUbFUR X5 UR5n URSU S5nURSn O1Ub#UR5SSn URSn O [ S5eUb URO URnU(aUc[URS9nUbUR5OSnUc;[R"XSU-[RUS9nUR!S5nUbU S::a [ S5eURU S5nUR!S 5R!S 5nUR#UR$S 9nS U- [R&"UR$5R(-nUbJURSU S5nUR+U5nU[,R."UR05-nOSnUcUR+U5nU Sn[R2"[R4"UU-UU-5S 5R#U5nU[,R."UR05-nUR6R#U5UlUR6USS24nUU-U-nUR9U5nU (aS OSnU(aS OSn[;UR<5H5unnU (aUU4-nU"UUUUUUS9nUSnU(dM,UUS 4- nM7 UR?U5nU (aUU4-nU (d[ASUUUU455$[CUUUUS9$)a1 Example: ```python >>> from transformers import AutoTokenizer, CTRLModel >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLModel.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> outputs = model(**inputs) >>> last_hidden_states = outputs.last_hidden_state >>> list(last_hidden_states.shape) [1, 5, 1280] ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer"rz5You have to specify either input_ids or inputs_embeds)r)r!devicez$batch_size has to be defined and > 0rrr g?rrc3.# UH ocMUv M g7frLr).0r=s r $CTRLModel.forward..es^a^s )last_hidden_stater hidden_states attentions)"rrfrerr ValueError%warn_if_padding_and_no_attention_maskr9viewr6rr get_seq_lengthrr%longr(r'r!finfominrr7r8rtriuonesr0r enumeraterrtupler )rXrrr?rrrrerfrrrg input_shaper_r past_lengthtoken_type_embedsseq_lenr> pos_embedsrall_hidden_statesall_attentionsrrrs rrkCTRLModel.forwardsF2C1N-TXT_T_TqTq!*!6IDKKcd d  "  6 6y Q#..*K!r;r?;I"+J  &',,.s3K&,,Q/JTU U%.%:!! @T@T 0*$++>O:I:Uo446[\   << _{5RZ_ZdZdmstL'11!4L  %Q !GHH+00R@N ,55a8BB1EN,..TZZ.@N!N2ekk$**6M6Q6QQN  %+00[_EN $~ 6  ):):!; ;  !    FF9-Mb/zz%**W{%:Gk[TU]U5 [U5Ul[R "UR URSS9UlUR5 g)NTbias) rMrNrrrrSrrlm_headrrXrrYs rrNCTRLLMHeadModel.__init__zsG  $V,yy0A0AM  rNrrr?rrrlabelsrerfrrlogits_to_keeprc U bU OURRn URUUUUUUUU U U S9 nUSn[U [5(a [ U *S5OU nUR USS2USS245nSnUb*UR"UU4SURR0U D6nU (dU4USS-nUbU4U-$U$[UUURURURS9$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` Example: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLLMHeadModel >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLLMHeadModel.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Wikipedia The llama is", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> sequence_ids = model.generate(inputs["input_ids"]) >>> sequences = tokenizer.batch_decode(sequence_ids) >>> sequences ['Wikipedia The llama is a member of the family Bovidae. It is native to the Andes of Peru,'] >>> outputs = model(**inputs, labels=inputs["input_ids"]) >>> round(outputs.loss.item(), 2) 9.21 >>> list(outputs.logits.shape) [1, 5, 246534] ```N rr?rrrrerfrrrrr)losslogitsrrr) rrrrrQslicer loss_functionrr rrr)rXrrr?rrrrrerfrrrrgtransformer_outputsr slice_indicesrrrFs rrkCTRLLMHeadModel.forwards0`&1%SV8W8W~ot4]k mA}a,?@A  %% ;;11 DY!4QR!88F)-)9TGf$ Ev E%/??-;;*55   rc V>[TU]"U4UUUS.UD6nURSS5 U$)N)rreis_first_iterationr)rMprepare_inputs_for_generationpop)rXrrrerrg model_inputsrYs rr-CTRLLMHeadModel.prepare_inputs_for_generationsH w<  +1     )40r)rr) NNNNNNNNNNNr)NNF)rnrorprq_tied_weights_keysrNrrrr rrrQrrr rkrrrrsrts@rrrqs_+,BC.2(,37260426*.!%)-,0#'-.V ##d*V V ))D0 V ((4/ V &&- V ((4/V   4'V $;V  $;V #TkV D[V ell*V  u|| 5 5V V rSXrra The CTRL Model transformer with a sequence classification head on top (linear layer). [`CTRLForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). c`^\rSrSrU4Sjr\SS\RS-S\S-S\RS-S\RS-S\RS-S \RS-S \RS-S \ S-S \ S-S \ S-S\ S-S\ \R\ -4Sjj5rSrU=r$)CTRLForSequenceClassificationic>[TU]U5 URUl[U5Ul[ R "URURSS9UlUR5 g)NFr) rMrN num_labelsrrrrSr classifierrrs rrN&CTRLForSequenceClassification.__init__sR   ++$V,))FMM4??O rNrrr?rrrrrerfrrrc U bU OURRn URUUUUUUUU U U S9 n U SnURU5nUbURSSunnOURSSunnURR cUS:wa [ S5eURR cSnOUbXRR :gRUR[R5n[R"URSUR[RS9nUU-RS5nO.Sn[RURR S 35 U[R"UURS 9U4nSnUGbURR"cUR$S:XaS URlOoUR$S:aNUR&[R(:XdUR&[R*:XaS URlOS URlURR"S :XaJ[-5nUR$S:Xa&U"UR/5UR/55nOU"UU5nOURR"S :Xa=[15nU"UR3SUR$5UR3S55nO-URR"S :Xa[55nU"UU5nU (dU4U SS-nUbU4U-$U$[7UUU R8U R:S9$)a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Example of single-label classification: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLForSequenceClassification >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl") >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> with torch.no_grad(): ... logits = model(**inputs).logits >>> predicted_class_id = logits.argmax().item() >>> model.config.id2label[predicted_class_id] 'LABEL_0' ``` ```python >>> import torch >>> torch.manual_seed(42) # doctest: +IGNORE_RESULT >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)` >>> num_labels = len(model.config.id2label) >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl", num_labels=num_labels) >>> labels = torch.tensor(1) >>> loss = model(**inputs, labels=labels).loss >>> round(loss.item(), 2) 0.93 ``` Example of multi-label classification: ```python >>> import torch >>> from transformers import AutoTokenizer, CTRLForSequenceClassification >>> tokenizer = AutoTokenizer.from_pretrained("Salesforce/ctrl") >>> model = CTRLForSequenceClassification.from_pretrained( ... "Salesforce/ctrl", problem_type="multi_label_classification" ... ) >>> # CTRL was trained with control codes as the first token >>> inputs = tokenizer("Opinion My dog is cute", return_tensors="pt") >>> assert inputs["input_ids"][0, 0].item() in tokenizer.control_codes.values() >>> with torch.no_grad(): ... logits = model(**inputs).logits >>> predicted_class_id = logits.argmax().item() >>> model.config.id2label[predicted_class_id] 'LABEL_0' ``` ```python >>> # To train a model on `num_labels` classes, you can pass `num_labels=num_labels` to `.from_pretrained(...)` >>> num_labels = len(model.config.id2label) >>> model = CTRLForSequenceClassification.from_pretrained("Salesforce/ctrl", num_labels=num_labels) >>> num_labels = len(model.config.id2label) >>> labels = torch.nn.functional.one_hot(torch.tensor([predicted_class_id]), num_classes=num_labels).to( ... torch.float ... ) >>> loss = model(**inputs, labels=labels).loss >>> loss.backward() # doctest: +IGNORE_RESULT ```Nrrrrz=Cannot handle batch sizes > 1 if no padding token is defined.r")rr!z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`)r regressionsingle_label_classificationmulti_label_classification)rrrr)rrrr r6 pad_token_idrr'rrint32r%argmaxlogger warning_oncerYrn problem_typer r!rrQrsqueezerrrrrr)rXrrr?rrrrrerfrrrgrrrr_sequence_lengthlast_non_pad_token non_pad_mask token_indices pooled_logitsrloss_fctrFs rrk%CTRLForSequenceClassification.forwardsz&1%!F!Fr!J !#    >>**+,ZZ  u||Jv}}MOaab   {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#M$9$9$;V^^=MND#M6:D))-JJ+- 2 22t GUWY))-II,. v6#%(;AB(??F)-)9TGf$ Ev E' -;;*55   r)r r r) NNNNNNNNNNN)rnrorprqrNrrrr rrrrrrkrrrsrts@rr r s(.2(,37260426*.!%)-,0#'d ##d*d d ))D0 d ((4/ d &&- d ((4/d   4'd $;d  $;d #Tkd D[d  u|| 7 7d d rr )r rrrrL)+__doc__numpyr7rrtorch.nnrrrrr cache_utilsr r generationr modeling_outputsr r rmodeling_utilsrutilsrrconfiguration_ctrlr get_loggerrnrrr1rGModulerIryr{rrrr __all__rrrr-s AA&.)ii-+   H % %,11h`,299,^ /  j #j j Z t)?t  tn o $7o  o d cr