Zj|SrSSKJr SSKJr SSKrSSKJr SSKJr SSK J r SS K J r SS KJr SS KJrJrJr SS KJrJr SS KJr SSKJr SSKJrJrJrJrJ r SSK!J"r" SSK#J$r$ SSK%J&r& \RN"\(5r)"SS\RT5r+S>S\RTS\RXS\RXS\RXS\RXS-S\-S\-4Sjjr."SS\RT5r/"S S!\RT5r0"S"S#\RT5r1"S$S%\RT5r2"S&S'\RT5r3"S(S)\5r4"S*S+\RT5r5\"S,S-\55r6\"S.S/\655r7"S0S1\RT5r8"S2S3\RT5r9\"S4S5\655r:\"S6S79\"S8S9\555r;\"S:S79"S;S<\655r[TU]5 [R"URUR UR S9Ul[R"URUR 5Ul [R"URUR 5Ul [R"UR URS9Ul [R"UR5UlUR#S[$R&"UR5R)S5SS9 g)N) padding_idxeps position_idsrF) persistent)super__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandselfconfig __class__s /root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/splinter/modeling_splinter.pyr'SplinterEmbeddings.__init__*s !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c %'\\&2H2H&J\J\%]"f&8&8f>S>STzz&"<"<=   ELL)G)GHOOPWXej  N input_idstoken_type_idsr" inputs_embedsreturncUbUR5nOUR5SSnUSnUcURSS2SU24nUc8[R"U[RURR S9nUcUR U5nURU5nXG-nURU5n X- nURU5nURU5nU$)Nr$rdtypedevice) sizer"r7zeroslongrHr,r0r.r1r5) r;rArBr"rC input_shape seq_lengthr0 embeddingsr.s r>forwardSplinterEmbeddings.forward8s  #..*K',,.s3K ^  ,,Q  ^ J": "66|D) ^^J/ \\*- r@)r1r5r.r0r,)NNNN) __name__ __module__ __qualname____firstlineno____doc__r'r7 LongTensor FloatTensortuplerO__static_attributes__ __classcell__r=s@r>rr'sQ  .2260426 ##d*((4/&&-  ((4/   r@rmodulequerykeyvalueattention_maskscalingr5c [R"XRSS55U-nUbX-n[RR US[R S9RUR5n[RRXURS9n[R"X5n U RSS5R5n X4$)Nrr$)dimrG)ptrainingr) r7matmul transposer functionalsoftmaxfloat32torGr5rf contiguous) r\r]r^r_r`rar5kwargs attn_weights attn_outputs r>eager_attention_forwardrqZs<<}}Q':;gEL!#4 ==((2U]](SVVW\WbWbcL==((6??([L,,|3K''1-88:K  $$r@c ^\rSrSrU4SjrS S\R S\RS-S\\ S\ \R \R S-44Sjjr S r U=r $) SplinterSelfAttentionqc6>[TU]5 URUR-S:wa7[ US5(d&[ SURSURS35eXlURUl[URUR- 5UlURUR-Ul [R"URUR5Ul [R"URUR5Ul [R"URUR5Ul[R"UR 5UlUR UlURS-Ulg)Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()g)r&r'r*num_attention_headshasattr ValueErrorr<intattention_head_size all_head_sizerLinearr]r^r_r3attention_probs_dropout_probr5attention_dropoutrar:s r>r'SplinterSelfAttention.__init__rsD     : : :a ?PVXhHiHi#F$6$6#78 445Q8   #)#=#= #&v'9'9FrOSplinterSelfAttention.forwards8 $))#2. CCbC$*B*BC zz-055lCMMaQRS XXm,11,?II!QO zz-055lCMMaQRS (?(M(M KK , ,.E) %8      % $}}C$2H2HLL %  % ! "));;;;FFH ((r@) r}rr|r<r5r^rxr]rar_N)rQrRrSrTr'r7TensorrWrrrXrOrYrZr[s@r>rsrsqsi6048)||)))D0)+, ) u||U\\D00 1 ))r@rscz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $)SplinterSelfOutputc(>[TU]5 [R"URUR5Ul[R "URURS9Ul[R"UR5Ul gNr ) r&r'rr~r*denser1r2r3r4r5r:s r>r'SplinterSelfOutput.__init__s` YYv1163E3EF f&8&8f>S>STzz&"<"<= r@r input_tensorrDcpURU5nURU5nURX-5nU$rrr5r1r;rrs r>rOSplinterSelfOutput.forward5 =1  ]3 }'CD r@r1rr5 rQrRrSrTr'r7rrOrYrZr[s@r>rr6> U\\RWR^R^r@rc ^\rSrSrU4SjrS S\R S\RS-S\\ S\R 4Sjjr S r U=r $) SplinterAttentioncb>[TU]5 [U5Ul[ U5Ulgr)r&r'rsr;routputr:s r>r'SplinterAttention.__init__s& )&1 (0 r@Nrr`rnrDc ZUnUR"U4SU0UD6upURX5nU$Nr`)r;r)r;rr`rnresidual_s r>rOSplinterAttention.forwardsE !99  )    M< r@)rr;r) rQrRrSrTr'r7rrWrrrOrYrZr[s@r>rrsV148 || ))D0 +,    r@rcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)SplinterIntermediatec>[TU]5 [R"URUR 5Ul[UR[5(a[URUl gURUl gr) r&r'rr~r*intermediate_sizer isinstance hidden_actstrr intermediate_act_fnr:s r>r'SplinterIntermediate.__init__s` YYv1163K3KL f'' - -'-f.?.?'@D $'-'8'8D $r@rrDcJURU5nURU5nU$rrr)r;rs r>rOSplinterIntermediate.forwards& =1 00? r@rrr[s@r>rrs(9U\\ellr@rcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $)SplinterOutputc(>[TU]5 [R"URUR 5Ul[R"UR URS9Ul[R"UR5Ul gr) r&r'rr~rr*rr1r2r3r4r5r:s r>r'SplinterOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= r@rrrDcpURU5nURU5nURX-5nU$rrrs r>rOSplinterOutput.forwardrr@rrr[s@r>rrrr@rc ^\rSrSrU4SjrS S\R S\RS-S\\ S\R 4Sjjr S r S r U=r $) SplinterLayerc>[TU]5 URUlSUl[ U5Ul[ U5Ul[U5Ul g)Nr) r&r'chunk_size_feed_forward seq_len_dimr attentionr intermediaterrr:s r>r'SplinterLayer.__init__sI '-'E'E$*6208$V, r@Nrr`rnrDc UR"U4SU0UD6n[URURURU5nU$r)rrfeed_forward_chunkrr)r;rr`rns r>rOSplinterLayer.forwardsW   )  2  # #T%A%A4CSCSUb r@cJURU5nURX!5nU$r)rr)r;attention_outputintermediate_output layer_outputs r>r SplinterLayer.feed_forward_chunks)"//0@A{{#6I r@)rrrrrr)rQrRrSrTr'r7rrWrrrOrrYrZr[s@r>rrs[-48||))D0+,   $r@rc |^\rSrSrU4SjrS S\R S\RS-S\\ S\ 4Sjjr S r U=r $) SplinterEncoderic>[TU]5 Xl[R"[ UR 5Vs/sHn[U5PM sn5UlSUl gs snf)NF) r&r'r<r ModuleListrangenum_hidden_layersrlayergradient_checkpointing)r;r<ir=s r>r'SplinterEncoder.__init__sR  ]]5IaIaCb#cCbaM&$9Cb#cd &+#$dsA&Nrr`rnrDc NURH nU"UU40UD6nM [US9$)Nlast_hidden_state)rr )r;rr`rn layer_modules r>rOSplinterEncoder.forwards> !JJL(M'+  r@)r<rrr)rQrRrSrTr'r7rrWrrr rOrYrZr[s@r>rrsR,48 || ))D0 +,     r@rcF^\rSrSr%\\S'SrSr\\ S.r U4Sjr Sr U=r $)SplinterPreTrainedModeli(r<splinterT)r attentionsc>[TU]U5 [U[5(a\[R "UR [R"UR RS5RS55 gg)Nr$r#) r& _init_weightsrrinitcopy_r"r7r8rr9)r;r\r=s r>r%SplinterPreTrainedModel._init_weights2s^ f% f0 1 1 JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h 2r@)rQrRrSrTr__annotations__base_model_prefixsupports_gradient_checkpointingrrs_can_record_outputsrrYrZr[s@r>rr(s/ "&*#&+ iir@rc^\rSrSrSrU4SjrSrSr\\ \ SS\ RS-S\ RS-S \ RS-S \ RS-S \ RS-S \ \S \\-4Sjj555rSrU=r$) SplinterModeli8a" The model is an encoder (with only self-attention) following the architecture described in [Attention is all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. c>[TU]U5 Xl[U5Ul[ U5UlUR5 gr)r&r'r<rrNrencoder post_initr:s r>r'SplinterModel.__init__@s9   ,V4&v.  r@c.URR$rrNr,)r;s r>get_input_embeddings"SplinterModel.get_input_embeddingsJs...r@c$XRlgrr)r;r_s r>set_input_embeddings"SplinterModel.set_input_embeddingsMs */'r@NrAr`rBr"rCrnrDc UbUb [S5eUb"URX5 UR5nO"UbUR5SSnO [S5eUupUb URO URn Uc[R "X4U S9nUc$[R "U[RU S9nURX'5n URUUUUS9n UR"U 4SU 0UD6n U S n[US 9$) a token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) NzDYou cannot specify both input_ids and inputs_embeds at the same timer$z5You have to specify either input_ids or inputs_embeds)rHrF)rAr"rBrCr`rr) rz%warn_if_padding_and_no_attention_maskrIrHr7onesrJrKget_extended_attention_maskrNrr )r;rAr`rBr"rCrnrL batch_sizerMrHextended_attention_maskembedding_outputencoder_outputssequence_outputs r>rOSplinterModel.forwardPs,6  ]%>cd d  "  6 6y Q#..*K  &',,.s3KTU U!, %.%:!! @T@T  !"ZZ*)A6RN  !"[[EJJvVN150P0PQ_0m??%)' +  ,,  2   *!,-  r@)r<rNr)NNNNN)rQrRrSrTrUr'rrrrrr7rrrrXr rOrYrZr[s@r>rr8s /0 *..2.2,0-1 = <<$&=  t+=  t+ = llT) = ||d* = +,=   =  = r@rcf^\rSrSrSU4SjjrS\R S\R 4SjrSrU=r $)SplinterFullyConnectedLayeric>[TU]5 XlX l[R "URUR5Ul[UUl[R"UR5Ul gr) r&r' input_dim output_dimrr~rr act_fnr1)r;r r rr=s r>r'$SplinterFullyConnectedLayer.__init__sR "$YYt~~t? Z( doo6r@inputsrDclURU5nURU5nURU5nU$r)rr r1)r;r rs r>rO#SplinterFullyConnectedLayer.forwards2 6*  M2 }5 r@)r1r rr r )gelurr[s@r>rrs(7ellu||r@rc2^\rSrSrSrU4SjrSrSrU=r$)QuestionAwareSpanSelectionHeadiz^ Implementation of Question-Aware Span Selection (QASS) head, described in Splinter's paper: c>[TU]5 [URUR5Ul[URUR5Ul[URUR5Ul[URUR5Ul[R"URURSS9Ul [R"URURSS9Ul g)NF)bias) r&r'rr*query_start_transformquery_end_transformstart_transform end_transformrr~start_classifierend_classifierr:s r>r''QuestionAwareSpanSelectionHead.__init__s %@ASASU[UgUg%h"#>v?Q?QSYSeSe#f :6;M;MvOaOab89K9KVM_M_` " &*<*P>PW\ ] ii(:(:FrO&QuestionAwareSpanSelectionHead.forwardsKKM 1##B'..q!S9 V%@ 55mD11-@))&1 %%f---.>? ''1a0 ||M> ++N; ##Aq!,\\-: ''r@)rrrrrr) rQrRrSrTrUr'rOrYrZr[s@r>rrs \((r@rc^^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \RS-S \RS-S \ \ S \ \ -4Sjj55rSrU=r$)SplinterForQuestionAnsweringic>[TU]U5 [U5Ul[ U5UlUR UlUR5 grr&r'rrr splinter_qassquestion_token_idrr:s r>r'%SplinterForQuestionAnswering.__init__C  %f- ;FC!'!9!9 r@NrAr`rBr"rCstart_positions end_positionsquestion_positionsrnrDc  Sn UcUbB[R"[R"XR5R 5SS9n OH[R "UR S5[RURURS9n U RS5nSn UR"U4UUUUS.U D6n U Sn URX5upU (a!URS 5URS 5pUbfUS U- [R"UR5R --nUS U- [R"UR5R --nSnUbUb[#UR 55S :aURS5n[#UR 55S :aURS5nUR S 5nUR%SU5 UR%SU5 ['US 9nU"X5nU"X5nUU-S - n[)UUUU R*U R,S 9$) aI token_type_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) question_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): The positions of all question tokens. If given, start_logits and end_logits will be of shape `(batch_size, num_questions, sequence_length)`. If None, the first question token in each sequence in the batch will be the only one for which start_logits and end_logits are calculated and they will be of shape `(batch_size, sequence_length)`. FNr$)rdr)rGlayoutrHTr`rBr"rCr ignore_indexrclossr(r)rr)r7argmaxeqr0r{rJrIrKr7rHrrr/squeezefinforGminlenclamp_rr rr)r;rAr`rBr"rCr3r4r5rnquestion_positions_were_none"question_position_for_each_exampleoutputsrr(r) total_loss ignored_indexloss_fct start_lossend_losss r>rO$SplinterForQuestionAnswering.forwards7D(-$  %$5:\\XXi)?)?@EEGR626;[[!&&q)MDXDXanauau62"D!M!Mb!Q +/ (--  ))%'    "!*#'#5#5o#Z ''3';';A'> @R@RST@U*  %'1~+=\M_M_A`AdAd*ddL#q>'9U[[IYIY=Z=^=^&^^J  &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M  " "1m 4  M 2']CH!,@J :H$x/14J+%!!//))   r@r0rr/NNNNNNNN)rQrRrSrTr'rrr7rrVrrrXr rOrYrZr[s@r>r,r,s*..2.2,0-137156:W <<$&W  t+W  t+ W llT) W ||d* W ))D0W ''$.W ",,t3W +,W  - -W W r@r,zB Class for outputs of Splinter as a span selection model. ) custom_introc\rSrSr%SrSr\RS-\S'Sr \RS-\S'Sr \RS-\S'Sr \ \RS-\S'Sr \ \RS-\S'S rg) SplinterForPreTrainingOutputi3a loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when start and end positions are provided): Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. start_logits (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length)`): Span-start scores (before SoftMax). end_logits (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length)`): Span-end scores (before SoftMax). Nr<r(r)rrr)rQrRrSrTrUr<r7rWrr(r)rrXrrYrr@r>rQrQ3s|&*D%  d ")-1L%##d*1+/J!!D(/59M5**+d2926Je''(4/6r@rQz Splinter Model for the recurring span selection task as done during the pretraining. The difference to the QA task is that we do not have a question, but multiple question tokens that replace the occurrences of recurring spans instead. c^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \RS-S \RS-S \ \ S \ \ -4Sjj55rS\RS \R4SjrSrU=r$)SplinterForPreTrainingiJc>[TU]U5 [U5Ul[ U5UlUR UlUR5 grr.r:s r>r'SplinterForPreTraining.__init__Rr2r@NrAr`rBr"rCr3r4r5rnrDc  UcUbUb [S5eUcUc [S5eUcURU5nUR"U4UUUUS.U D6n U Sn U R5upnUR X5unnURS5nUbUR S5R U UU 5nUSU- [R"UR5R--nUSU- [R"UR5R--nSnUbUbURS[SU S- 55 URS[SU S- 55 [URRS9nU"UR!U U-U 5UR!U U-55nU"UR!U U-U 5UR!U U-55nUU-S- n[#UUUU R$U R&S 9$) a input_ids (`torch.LongTensor` of shape `(batch_size, num_questions, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `batch_size, num_questions, sequence_length`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `batch_size, num_questions, sequence_length`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_questions, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. start_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. question_positions (`torch.LongTensor` of shape `(batch_size, num_questions)`, *optional*): The positions of all question tokens. If given, start_logits and end_logits will be of shape `(batch_size, num_questions, sequence_length)`. If None, the first question token in each sequence in the batch will be the only one for which start_logits and end_logits are calculated and they will be of shape `(batch_size, sequence_length)`. NzCquestion_positions must be specified in order to calculate the lossz?question_positions must be specified when inputs_embeds is usedr8rrr9rcr;) TypeError_prepare_question_positionsrrIr/rr9r7r@rGrArCmaxrr<r+rrQrr)r;rAr`rBr"rCr3r4r5rnrFrrsequence_lengthrdr(r) num_questions attention_mask_for_each_questionrGrIrJrKs r>rOSplinterForPreTraining.forward\s=j  %/*E-Jcab b  'I,=]^ ^  '!%!A!A)!L --  ))%'    "!*+:+?+?+A( S#'#5#5o#Z j*//2  %/=/G/G/J/Q/QM?0 ,(1/O+OSXS^S^_k_q_qSrSvSv*vvL#q+K'Ku{{[e[k[kOlOpOp&ppJ  &=+D  " "1c!_q-@&A B  C?Q+>$? @ (T[[5M5MNH!!!*}"H%x/14J+%!!//))   r@c6[R"XRR:H5up#[R"U5n[R "UR S5UR54URR[RURS9n[UR S5UR S5:HS5 [R"UVs/sHn[R"U5PM sn5nX5X'4'U$s snf)NrrFz?All samples in the batch must have at least one question token.)r7wherer<r0bincountfullrIrYr+rKrHrcatr8)r;rArowsflat_positionsr[r"ncolss r>rX2SplinterForPreTraining._prepare_question_positionss${{9 8U8U+UVt, JJ ^^A  1 1 3 4 KK $ $**##     q !Y^^A%6 6 M yy=A=a%,,q/=AB .$*Bs( DrMrN)rQrRrSrTr'rrr7rrVrrrXrQrOrXrYrZr[s@r>rSrSJs'*..2.2,0-137156:n <<$&n  t+n  t+ n llT) n ||d* n ))D0n ''$.n ",,t3n +,n  - -n n `U\\ellr@rS)r,rSrrr)r)>rUcollections.abcr dataclassesrr7rtorch.nnrrr activationsr modeling_layersr modeling_outputsr r r modeling_utilsrrprocessing_utilsr pytorch_utilsrutilsrrrrr utils.genericrutils.output_capturingrconfiguration_splinterr get_loggerrQloggerModulerrfloatrqrsrrrrrrrrrrr,rQrS__all__rr@r>r{s$! %&!9ZZF&6jj752   H %//t% II% <<% % << % LL4' %  %%.3)BII3)n   . 299  RYY .D bii 2 io i iW +W W t"))$#(RYY#(Ld #:d d N  7;7  7"L4LL^ r@