ZjSSSKrSSKJr SSKrSSKJr SSKJr SSKJr SSK J r SSK J r SS K JrJrJr SS KJrJr SS KJr SS KJrJrJrJr SS KJr SSKJr SSKJ r J!r! SSK"J#r#J$r$ \RJ"\&5r'S+S\RPS\RRS\RRS\RRS\RRS-S\*S-S\*4Sjjr+"SS\RP5r,"SS\ 5r-\"SS \55r.\"S!S"9"S#S$\.55r/"S%S&\RP5r0\"S'S"9"S(S)\.\ 55r1/S*Qr2g),N)Callable)nn)ACT2FN)Cache)GenerationMixin)GradientCheckpointingLayer)BaseModelOutputWithPastBaseModelOutputWithPoolingCausalLMOutputWithPast)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)merge_with_config_defaults)capture_outputs) AutoModelAutoModelForCausalLM) VoxtralConfigVoxtralEncoderConfigmodulequerykeyvalueattention_maskscalingdropoutc UcURS5S-n[R"XRSS55U-nUbX-n[R R USS9n[R RXURS9n[R"X5n U RSS5R5n X4$)Nrr)dimptrainingr) sizetorchmatmul transposer functionalsoftmaxr"r) contiguous) rrrrr r!r"kwargs attn_weights attn_outputs }/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/voxtral/modeling_voxtral.pyeager_attention_forwardr5-s**R.D(<<}}Q':;gEL!#4 ==((2(>L==((6??([L,,|3K''1-88:K  $$c@^\rSrSrSrSS\S\S\S\S\S \S \S-S \S-4U4S jjjr S \ RS\S\4Sjr SS\ RS\ RS-S\S\ \ R\ RS-\ \ RS-44SjjrSrU=r$)VoxtralAttentionGz=Multi-headed attention from 'Attention Is All You Need' paperN embed_dim num_headsr" is_decoderbias is_causal layer_idxconfigc `>[T U]5 XlX lX0lX-UlXlUR U-UR:wa[SURSUS35eUR S-UlX@l X`l Uc4U(a-[RSURRS35 Xpl[ R""XSS9Ul[ R""XUS9Ul[ R""XUS9Ul[ R""XUS9Ulg) Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).r%zInstantiating a decoder z without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.Fr=)super__init__r:r;r"head_dimr@ ValueErrorr!r<r>logger warning_once __class____name__r?rLineark_projv_projq_projout_proj) selfr:r;r"r<r=r>r?r@rIs r4rDVoxtralAttention.__init__Js "" !.  MMI %$.. 8MdnnM]$YKr3 }}d* $"     *4>>+B+B*CD,,  #ii 5A ii 4@ ii 4@  )TB r6tensorseq_lenbszcURX2URUR5RSS5R 5$)Nrr)viewr;rEr-r0)rPrRrSrTs r4_shapeVoxtralAttention._shapers5{{3GQQRSUVWbbddr6 hidden_statesr output_attentionsreturnc 2UR5upVnURURU5UR-Xe5nURUR U5SU5n URUR U5SU5n [ R"URR[5n U "UUU U U4UR(dSO URSUS.UD6upU RXVS5R5n URU 5n X4$)z#Input shape: Batch x Time x Channelr$?)r"r!rZ)r*rWrNr!rLrMr get_interfacer@_attn_implementationr5r)r"reshaper0rO)rPrYr rZr1rTtgt_len_ query_states key_states value_statesattention_interfacer3r2s r4forwardVoxtralAttention.forwardus (,,.a{{4;;}#= #Lg[ [[]!;RE {{4;;}#=r3G (?(M(M KK , ,.E) %8      % $}}C$,,/ %  % ! "))#;FFH mmK0 ((r6) r@r"r:rEr>r<rLr?r;rOrNr!rM)r]FTFNNNF)rJ __module__ __qualname____firstlineno____doc__intfloatboolrrDr+TensorrWtuplerh__static_attributes__ __classcell__rIs@r4r8r8GsG   $'+&C&C&C &C  &C  &C&C:&C$&C&CPeU\\eCece /3"' ')||') t+') ') u||U\\D0% 2E2LL M ')')r6r8c^\rSrSrS\4U4SjjrS\RS\RS\\ S\R4Sjr S r U=r $) VoxtralEncoderLayerr@cj>[TU]5 URUl[ URUR UR US9Ul[R"UR5Ul URUl [URUlURUl[R "URUR"5Ul[R "UR"UR5Ul[R"UR5Ulg)N)r:r;r"r@)rCrDd_modelr:r8encoder_attention_headsattention_dropout self_attnr LayerNormself_attn_layer_normr"ractivation_function activation_fnactivation_dropoutrKencoder_ffn_dimfc1fc2final_layer_normrPr@rIs r4rDVoxtralEncoderLayer.__init__s )nn44,,   %'LL$@!~~ #F$>$>?"(";";99T^^V-C-CD99V33T^^D " T^^ <r6rYr r1r[c UnURU5nUR"SUUS.UD6up[RR XRUR S9nXA-nUnUR U5nURURU55n[RR XRUR S9nURU5n[RR XRUR S9nXA-nUR[R:XaC[R"UR5RS- n[R "X*US9nU$)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. )rYr r'i)minmax)rr~rr.r"r)rrrrrdtyper+float16finforclamp)rPrYr r1residualrc clamp_values r4rhVoxtralEncoderLayer.forwardsD!11-@ >> ')    --m||VZVcVc-d  0  --m< **488M+BC  --m?V?Vaeanan-o /  --m||VZVcVc-d  0   %-- /++m&9&9:>>EK!KK <[YMr6) rrr"r:rrrr~r) rJrkrlrmrrDr+rrrrrhrtrurvs@r4rxrxsQ=}=$"||" "+, "  ""r6rxcL\rSrSr%\\S'SrSrSrSr Sr Sr Sr Sr SrSrSrSrg) VoxtralPreTrainedModelr@model)audiotextTNpast_key_valuesr)rJrkrlrmr__annotations__base_model_prefixinput_modalitiessupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_cache_class_supports_attention_backend_can_compile_fullgraphrtrr6r4rrsI (&*#"3N "&!r6rz: The Voxtral encoder, which is a Whisper encoder.  custom_introc^\rSrSr%Sr\\S'SrSrS/r \ \ S.r S\4U4Sjjr S rS \R 4S jrS \R 4S jr\\SS\\S \\-4Sjj55rS\R64SjrSrU=r$)VoxtralEncoderz Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a [`VoxtralEncoderLayer`]. Args: config: VoxtralEncoderConfig r@input_featuresrrx) attentionsrYcl>[TU]U5 URUlURUlUR nUR UlURUlUR(a[R"U5OSUl [R"UR USSS9Ul[R"X"SSSS9Ul[R "URU5UlUR"R%S5 [R&"[)UR*5Vs/sHn[-U5PM sn5Ul[R0"UR 5Ul[R4"SSS9UlSUlUR;5 gs snf) Nr^rr) kernel_sizepaddingr)rstriderF)r)rCrDr"encoder_layerdrop layerdropr{ num_mel_binsmax_source_positionsscale_embeddingmathsqrt embed_scalerConv1dconv1conv2 Embeddingembed_positionsrequires_grad_ ModuleListrangeencoder_layersrxlayersr layer_norm AvgPool1d avg_poolergradient_checkpointing post_init)rPr@r:rcrIs r4rDVoxtralEncoder.__init__s:  ~~ 11NN "//$*$?$?!393I3I499Y/sYYt00)TUV YYy1VWX !||D,E,EyQ ++E2mm%PVPePeJf$gJfQ%8%@Jf$gh ,,v~~6,,q3&+# %hs9F1cNUR5H nSUlM SUlgrj) parameters requires_grad_requires_grad)rPparams r4_freeze_parameters!VoxtralEncoder._freeze_parameterss#__&E"'E '#r6r[cUR$NrrPs r4get_input_embeddings#VoxtralEncoder.get_input_embeddingss zzr6rcXlgrrrPrs r4set_input_embeddings#VoxtralEncoder.set_input_embeddings s r6r1c URRURRS-URRS-nUR SU:wa"[ SUSUR SSUS35eURURRRURRRS9n[RRURU55n[RRUR U55nURSSS 5nURRnXV-RUR5n[RR!XpR UR"S 9n[%UR&5H upU "UUS 9nM UR)U5n[+US 9$) a Args: input_features (`torch.LongTensor` of shape `(batch_size, feature_size, sequence_length)`): Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] attention_mask (`torch.Tensor`)`, *optional*): Voxtral does not support masking of the `input_features`, this argument is preserved for compatibility, but it is not used. By default the silence in the input log mel spectrogram are ignored. rr$z7Voxtral expects the mel input features to be of length z , but found z-. Make sure to pad the input mel features to .)rdevicerrr')r )last_hidden_state)r@rrrrshaperFtoweightrrrr.gelupermuterr"r) enumeraterrr ) rPrr r1expected_seq_length inputs_embeds embed_posrYidx encoder_layers r4rhVoxtralEncoder.forward#s(#kk>>ARARSTAUUX\XbXbXiXijkXll    #': :IJ]I^^jkykkAClDkEErsFrGGHI (**1B1B1H1HQUQ[Q[QbQbQiQi*j **4::n+EF  **4::m+DE %--aA6 ((// &266}7J7JK  --m||VZVcVc-d "+DKK"8 C)-M#9  6 )+  r6 input_lengthsc4US- S-S-nUS- S-S-nX4$)zc Computes the output length of the convolutional layers and the output length of the audio encoder rrr)rProutput_lengthss r4 _get_feat_extract_output_lengths/VoxtralEncoder._get_feat_extract_output_lengthsSs5'*q014 '!+1A5,,r6) rrrrr"rrrrrrrrr)rJrkrlrmrnrrmain_input_namerrr8rx_can_record_outputsrDrrModulerrrrrrrsr rhr+ LongTensorrrtrurvs@r4rrs  ! &O./&, 32$ bii")) + +, + + + +  + \-e>N>N--r6rc6^\rSrSrS\4U4SjjrSrSrU=r$)VoxtralMultiModalProjectori\r@c^>[TU]5 [R"URR UR RSS9Ul[URUl [R"UR RUR RSS9Ul g)NFrB) rCrDrrK audio_configintermediate_size text_config hidden_sizelinear_1rprojector_hidden_actactlinear_2rs r4rD#VoxtralMultiModalProjector.__init__]sz  &"5"5"G"GI[I[IgIgnst &556 &"4"4"@"@&BTBTB`B`glm r6clURU5nURU5nURU5nU$r)rrr)rPaudio_featuresrYs r4rh"VoxtralMultiModalProjector.forwardcs2 n5 /  m4 r6)rrr) rJrkrlrmrrDrhrtrurvs@r4rr\sn}n r6rzs The Voxtral model, which consists of Whisper encoder, a multi-modal projector and a LLama language model. c^\rSrSrS/rU4SjrSrSrSrSr Sr S r \ \ "S S 9S \RS \\S\\-4Sj55r\ \ SS\R*S-S \RS-S\R,S-S\R*S-S\S-S\RS-S\R*S-S\S-S\\R,-S \\S\4Sjj55rU4SjrSrU=r$)VoxtralForConditionalGenerationijrc.>[TU]U5 URRUl[R "UR 5Ul[R "UR5Ul [U5Ul UR5 gr) rCrDr vocab_sizer from_configr audio_towerrlanguage_modelrmulti_modal_projectorrrs r4rD(VoxtralForConditionalGeneration.__init__rsn   ,,77$001D1DE2>>v?Q?QR%?%G" r6c6URR5$r)r rrs r4r4VoxtralForConditionalGeneration.get_input_embeddings|s""7799r6c:URRU5 gr)r rrs r4r4VoxtralForConditionalGeneration.set_input_embeddingss 007r6c6URR5$r)r get_output_embeddingsrs r4r5VoxtralForConditionalGeneration.get_output_embeddingss""88::r6c:URRU5 gr)r set_output_embeddings)rPnew_embeddingss r4r5VoxtralForConditionalGeneration.set_output_embeddingss 11.Ar6c:URRU5 gr)r  set_decoder)rPdecoders r4r+VoxtralForConditionalGeneration.set_decoders ''0r6c6URR5$r)r  get_decoderrs r4r+VoxtralForConditionalGeneration.get_decoders""..00r6zThis method is used to get the audio embeddings from input features (a log mel spectrogram), meaning inferring the audio encoder and the multi-modal projector.rrr1r[c UR"U4SS0UD6nURnURSURRR 5nUR U5nXSlU$)a) input_features (`torch.FloatTensor`): Float values of mel features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the mel features, padding and conversion into a tensor of type `torch.FloatTensor`. See [`~WhisperFeatureExtractor.__call__`] return_dictTr$)r rrar@rrr  pooler_output)rPrr1 audio_outputsaudio_hidden_states audio_embedss r4get_audio_features2VoxtralForConditionalGeneration.get_audio_featuressj((TTTVT +==199"dkk>V>V>h>hi112EF &2#r6N input_idsr position_idsrrlabels use_cachelogits_to_keepc UcUR5"U5nUbUbURUSS9Rn XRR:HR S5n UR U RUR5U RUR55nUR"SUUUUUUU S.U D6n U $)a Example: ```python >>> from transformers import VoxtralForConditionalGeneration, AutoProcessor >>> import torch >>> device = "cuda" if torch.cuda.is_available() else "cpu" >>> repo_id = "mistralai/Voxtral-Mini-3B-2507" >>> processor = AutoProcessor.from_pretrained(repo_id) >>> model = VoxtralForConditionalGeneration.from_pretrained(repo_id, dtype=torch.bfloat16, device_map=device) >>> conversation = [ { "role": "user", "content": [ { "type": "audio", "url": "https://huggingface.co/datasets/hf-internal-testing/dummy-audio-samples/resolve/main/dude_where_is_my_car.wav", }, {"type": "text", "text": "What can you tell me about this audio?"}, ], } ] >>> inputs = processor.apply_chat_template(conversation) >>> inputs = inputs.to(device, dtype=torch.bfloat16) >>> outputs = model.generate(**inputs, max_new_tokens=30) >>> processor.batch_decode(outputs[:, inputs.input_ids.shape[1]:], skip_special_tokens=True) ["This audio is a humorous conversation between two friends, likely in English, where one of them is trying to figure out what the other's tattoo says."] ```T)r r$)r r(rrr)r*r+r) rr%r!r@audio_token_id unsqueezemasked_scatterrrr )rPr'rr r(rrr)r*r+r1r$audio_token_maskoutputss r4rh'VoxtralForConditionalGeneration.forwards`   557 BM  %)*?22>t2TbbL!*[[-G-G GRRSUV )88 ##M$8$89+> , )%+') ,  , r6c>URSS5nURSS5n[TU] "U0UD6nU(dURSS5(dX5S'U$)Nris_first_iterationFr*T)popgetrCprepare_inputs_for_generation)rPargsr1rr4 model_inputsrIs r4r7=VoxtralForConditionalGeneration.prepare_inputs_for_generations^ $4d;#ZZ(rLse, $ ! )9kkF&RR752F   H %!% II% <<% % << % LL4' % T\ %%4U)ryyU)p545p "_ " " m-+m-  m-`   J&