
    Z j                     >   S SK Jr  S SKJr  S SKrS SKrS SKJr  S SKJ	r	  SSK
Jr  SSKJr  SS	KJr  SS
KJrJr  SSKJr  SSKJrJrJrJr  SSKJr  SSKJr  \" SS9\ " S S\5      5       5       r " S S\R>                  5      r S r!S@S jr"S\RF                  S\$S\RF                  4S jr% SAS\R>                  S\RF                  S\RF                  S \RF                  S!\RF                  S-  S"\&S#\&S$\\   4S% jjr'\" \"5       " S& S'\R>                  5      5       r( " S( S)\R>                  5      r) " S* S+\R>                  5      r*S,\RF                  S-\RF                  S.\RF                  S\RF                  4S/ jr+ " S0 S1\R>                  5      r, " S2 S3\R>                  5      r-\ " S4 S5\5      5       r.S6\RF                  S7\&S\/\RF                  \RF                  4   4S8 jr0S9\RF                  S:\$S;\$S\RF                  4S< jr1\" S=S9 " S> S?\.5      5       r2S5S?/r3g)B    )Callable)	dataclassN)nnpad_sequence   )ACT2FN)use_kernelized_func)FlashAttentionKwargs)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstringcan_return_tuple   )AutoModelForKeypointDetection   )LightGlueConfiga  
    Base class for outputs of LightGlue keypoint matching models. Due to the nature of keypoint detection and matching,
    the number of keypoints is not fixed and can vary from image to image, which makes batching non-trivial. In the
    batch of images, the maximum number of matches is set as the dimension of the matches and matching scores. The mask
    tensor is used to indicate which values in the keypoints, matches, matching_scores and prune tensors are keypoint
    matching information.
    )custom_introc                   f   \ rS rSr% SrSr\R                  S-  \S'   Sr	\R                  S-  \S'   Sr
\R                  S-  \S'   Sr\R                  S-  \S'   Sr\R                  S-  \S'   Sr\R                  S-  \S	'   Sr\\R                     S-  \S
'   Sr\\R                     S-  \S'   Srg)LightGlueKeypointMatchingOutput&   aW  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*):
    Loss computed during training.
matches (`torch.FloatTensor` of shape `(batch_size, 2, num_matches)`):
    Index of keypoint matched in the other image.
matching_scores (`torch.FloatTensor` of shape `(batch_size, 2, num_matches)`):
    Scores of predicted matches.
keypoints (`torch.FloatTensor` of shape `(batch_size, num_keypoints, 2)`):
    Absolute (x, y) coordinates of predicted keypoints in a given image.
prune (`torch.IntTensor` of shape `(batch_size, num_keypoints)`):
    Pruning mask indicating which keypoints are removed and at which layer.
mask (`torch.BoolTensor` of shape `(batch_size, num_keypoints)`):
    Mask indicating which values in matches, matching_scores, keypoints and prune are keypoint matching
    information.
hidden_states (`Tuple[torch.FloatTensor, ...]`, *optional*):
    Tuple of `torch.FloatTensor` (one for the output of each stage) of shape `(batch_size, 2, num_channels,
    num_keypoints)` returned when `output_hidden_states=True` is passed or when
    `config.output_hidden_states=True`
attentions (`Tuple[torch.FloatTensor, ...]`, *optional*):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, 2, num_heads, num_keypoints,
    num_keypoints)` returned when `output_attentions=True` is passed or when
    `config.output_attentions=True`
Nlossmatchesmatching_scores	keypointsprunemaskhidden_states
attentions )__name__
__module____qualname____firstlineno____doc__r   torchFloatTensor__annotations__r   r   r   r   	IntTensorr    r!   tupler"   __static_attributes__r#       ځ/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/lightglue/modeling_lightglue.pyr   r   &   s    0 &*D%

d
")(,GU%,04OU&&-4*.Iu  4'.$(E5??T!(%)D%

d
")59M5**+d2926Je''(4/6r/   r   c            
          ^  \ rS rSrS\4U 4S jjr S
S\R                  S\S-  S\	\R                     \	\R                  \R                  4   -  4S jjr
S	rU =r$ )LightGluePositionalEncoderS   configc                    > [         TU ]  5         [        R                  " SUR                  UR
                  -  S-  SS9U l        g )Nr   Fbias)super__init__r   Lineardescriptor_dimnum_attention_heads	projectorselfr4   	__class__s     r0   r9   #LightGluePositionalEncoder.__init__T   s:    1f&;&;v?Y?Y&Y]^&^ejkr/   r   output_hidden_statesNreturnc                     U R                  U5      nUR                  SSS9n[        R                  " U5      n[        R                  " U5      nXV4nU(       a  XC4nU$ U4nU$ )Nr   dim)r=   repeat_interleaver)   cossin)r?   r   rB   projected_keypoints
embeddingscosinessinesoutputs           r0   forward"LightGluePositionalEncoder.forwardX   sl     #nnY7(::1":E
))J'		*%%
6J*2 R\P]r/   )r=   F)r$   r%   r&   r'   r   r9   r)   Tensorboolr-   rP   r.   __classcell__r@   s   @r0   r2   r2   S   sb    l l
 LQ		=AD[		u||	uU\\5<<%?@	@	 	r/   r2   c                 |    U SS S S24   nU SSS S24   n[         R                  " U* U/SS9R                  S5      nU$ )N.r   r   rE   rF   )r)   stackflatten)xx1x2rot_xs       r0   rotate_halfr_   d   sL    	
3!8B	
319BKK"b	r*2226ELr/   c                 &   U R                   nU R                  5       n UR                  5       nUR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nUR	                  US9UR	                  US94$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
dtype)rb   float	unsqueezer_   to)qkrI   rJ   unsqueeze_dimrb   q_embedk_embeds           r0   apply_rotary_pos_embrk   l   s    $ GGE		A		A
--
&C
--
&Cw;q>C/0Gw;q>C/0G::E:"GJJUJ$;;;r/   r!   n_reprC   c                     U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )z
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r   N)shapeexpandreshape)r!   rl   batchnum_key_value_headsslenhead_dims         r0   	repeat_kvru      s_    
 2?1D1D.Ez!!Qa"23::5W\dlmM  e(CTTTr/   modulequerykeyvalueattention_maskscalingdropoutkwargsc                    [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub  X-   n
[
        R                  R                  U
S[        R                  S9R                  UR                  5      n
[
        R                  R                  XU R                  S9n
[        R                  " X5      nUR	                  SS5      R                  5       nX4$ )Nr   r   rE   )rG   rb   )ptrainingr   )ru   num_key_value_groupsr)   matmul	transposer   
functionalsoftmaxfloat32re   rb   r|   r   
contiguous)rv   rw   rx   ry   rz   r{   r|   r}   
key_statesvalue_statesattn_weightsattn_outputs               r0   eager_attention_forwardr      s     3 ; ;<JU$?$?@L<<';';Aq'ABWLL!#4==((2U]](SVVW\WbWbcL==((6??([L,,|:K''1-88:K$$r/   c                   B  ^  \ rS rSrSrS\S\4U 4S jjr    SS\R                  S\
\R                  \R                  4   S-  S	\R                  S-  S
\R                  S-  S\R                  S-  S\\   S\
\R                  \R                  S-  4   4S jjrSrU =r$ )LightGlueAttention   z=Multi-headed attention from 'Attention Is All You Need' paperr4   	layer_idxc                 P  > [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR                  U R                  -  UR
                  UR                  S9U l        g )Nrt   g      Tr6   )r8   r9   r4   r   getattrhidden_sizer<   rt   rr   r   r{   attention_dropout	is_causalr   r:   attention_biasq_projk_projv_projo_projr?   r4   r   r@   s      r0   r9   LightGlueAttention.__init__   sI   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii&&68J8JQWQfQf
r/   Nr!   position_embeddingsrz   encoder_hidden_statesencoder_attention_maskr}   rC   c                    UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      n	US Ln
U
(       a  UOUnU
(       a  UOUnU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      nUb  Uu  nn[        XUU5      u  p[        R                  " U R                  R                  [        5      nU" U U	UUU4U R                  (       d  SOU R                  U R                  S.UD6u  nnUR                   " / UQSP76 R#                  5       nU R%                  U5      nUU4$ )NrE   r   r           )r|   r{   )rn   rt   r   viewr   r   r   rk   r   get_interfacer4   _attn_implementationr   r   r   r{   rp   r   r   )r?   r!   r   rz   r   r   r}   input_shapehidden_shapequery_statesis_cross_attentioncurrent_statescurrent_attention_maskr   r   rI   rJ   attention_interfacer   r   s                       r0   rP   LightGlueAttention.forward   s    $))#2.88b8$--8{{=166|DNNqRST2$>2D.-;M!7Sa[[055lCMMaQRS
{{>277EOOPQSTU**HC';LVY[^'_$L(?(M(MKK,,.E)
 %8"	%
  $}}C$2H2HLL	%
 	%
!\ "));;;;FFHkk+.L((r/   )r   r4   rt   r   r   r   r   r   r   r{   r   )NNNN)r$   r%   r&   r'   r(   r   intr9   r)   rS   r-   r   r   rP   r.   rU   rV   s   @r0   r   r      s    G
 
3 
4 IM.2596:*)||*) #5<<#=>E*) t+	*)
  %||d2*) !&t 3*) -.*) 
u||U\\D00	1*) *)r/   r   c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )LightGlueMLP   r4   c                 f  > [         TU ]  5         Xl        [        UR                     U l        [        R                  " UR                  UR                  5      U l	        [        R                  " UR                  UR                  5      U l        [        R                  " UR                  SS9U l        g )NT)elementwise_affine)r8   r9   r4   r	   
hidden_actactivation_fnr   r:   intermediate_sizefc1r   fc2	LayerNorm
layer_normr>   s     r0   r9   LightGlueMLP.__init__   s{    #F$5$5699V55v7O7OP99V55v7I7IJ,,v'?'?TXYr/   r!   rC   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ N)r   r   r   r   )r?   r!   s     r0   rP   LightGlueMLP.forward   sB    /6**=9/r/   )r   r4   r   r   r   r$   r%   r&   r'   r   r9   r)   rS   rP   r.   rU   rV   s   @r0   r   r      s1    Z ZU\\ ell  r/   r   c                     ^  \ rS rSrS\S\4U 4S jjr  SS\R                  S\R                  S\R                  S\	S	-  S
\	S	-  S\
\R                  \
\R                     S	-  \
\R                     S	-  4   4S jjrSrU =r$ )LightGlueTransformerLayeri  r4   r   c                    > [         TU ]  5         [        X5      U l        [	        U5      U l        [        X5      U l        [	        U5      U l        g r   )r8   r9   r   self_attentionr   self_mlpcross_attention	cross_mlpr   s      r0   r9   "LightGlueTransformerLayer.__init__  s@    0C$V,1&D%f-r/   descriptorsr   rz   rB   Noutput_attentionsrC   c                    U(       a  SOS nU(       a  SOS nU(       a  Xa4-   nUR                   u  pn
U R                  UUUUS9u  p[        R                  " X/SS9nU R	                  U5      nX-   nU(       a  X4nUR                  SSX5      R                  S5      R                  XU
5      nUb6  UR                  SSSSU	5      R                  S5      R                  USSU	5      OS nU R                  UUUUS9u  nn[        R                  " UU/SS9nU R                  U5      nUU-   nU(       a4  UU4nUUR                  XU
5      4-   W-   UR                  XU
5      4-   U-   nU(       a	  X|4-   U4-   nXU4$ )Nr#   )r   rz   r   rE   rF   r   r   )r   r   r   )	rn   r   r)   catr   rp   flipr   r   )r?   r   r   rz   rB   r   all_hidden_statesall_attentions
batch_sizenum_keypointsr;   attention_outputself_attentionsintermediate_statesoutput_statesself_attention_descriptorsself_attention_hidden_statesr   r   cross_attention_outputcross_attentionscross_intermediate_statescross_output_statescross_attention_hidden_statess                           r0   rP   !LightGlueTransformerLayer.forward  s    #7BD0d 1N B4?4E4E1
> -1,?,? ))/	 -@ -
) $ii(GRP&9:%0%@",?+O( '..r1mTT!WWZ? 	 ) ""2q!Q>CCAFNNz[\^_ano 	 483G3G&"7#9/	 4H 4
0 0 %*II/IKa.bhj$k!"nn-FG03FF-FH[,\)!-55jQ_`bc./ &&z.QST 0	0  +.@@DTCVVN~==r/   )r   r   r   r   )FF)r$   r%   r&   r'   r   r   r9   r)   rS   rT   r-   rP   r.   rU   rV   s   @r0   r   r     s    . .3 . -2).H>\\H> <<H> 	H>
 #TkH>  $;H> 
u||U5<<047u||9Lt9SS	TH> H>r/   r   
similaritymatchability0matchability1c                    U R                   u  p4n[        R                  R                  U5      [        R                  R                  U5      R	                  SS5      -   n[        R                  R                  U S5      n[        R                  R                  U R	                  SS5      R                  5       S5      R	                  SS5      nU R                  X4S-   US-   4S5      n	Xx-   U-   U	SS2SU2SU24'   [        R                  R                  UR                  S5      * 5      U	SS2SS2S4'   [        R                  R                  UR                  S5      * 5      U	SS2SSS24'   U	$ )z;create the log assignment matrix from logits and similarityr   r   rE   rX   r   N)	rn   r   r   
logsigmoidr   log_softmaxr   new_fullsqueeze)
r   r   r   r   num_keypoints_0num_keypoints_1certaintiesscores0scores1scoress
             r0   sigmoid_log_double_softmaxr   Y  sM    4>3C3C0J--**=9BMM<T<TUb<c<m<mnoqr<ssKmm''
A6Gmm''
(<(<R(D(O(O(QSTU__`bdfgG  *.A?UVCV!WYZ[F4;4E4SF1 0 00111=3H3H3L2LMF1crc2:11=3H3H3L2LMF1b#2#:Mr/   c                      ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  S\R                  4S jrS\R                  S\R                  4S jr	S	r
U =r$ )
LightGlueMatchAssignmentLayerih  r4   c                    > [         TU ]  5         UR                  U l        [        R                  " U R                  U R                  SS9U l        [        R                  " U R                  SSS9U l        g )NTr6   r   )r8   r9   r;   r   r:   final_projectionmatchabilityr>   s     r0   r9   &LightGlueMatchAssignmentLayer.__init__i  sY    $33 "		$*=*=t?R?RY] ^IId&9&914Hr/   r   r    rC   c                    UR                   u  p4nU R                  U5      nU[        R                  " U R                  UR
                  S9S-  -  nUR                  US-  SXE5      nUS S 2S4   nUS S 2S4   nXxR                  SS5      -  n	Ub  UR                  US-  SU5      nUS S 2S4   R                  S5      n
US S 2S4   R                  S5      R                  SS5      nX-  nU	R                  US:H  [        R                  " U	R                  5      R                  5      n	U R                  U5      nUR                  US-  SUS5      nUS S 2S4   nUS S 2S4   n[        XU5      nU$ )Ndeviceg      ?r   r   r   rE   rX   )rn   r   r)   tensorr;   r   rp   r   rd   masked_fillfinforb   minr   r   )r?   r   r    r   r   r;   m_descriptorsm_descriptors0m_descriptors1r   mask0mask1r   matchability_0matchability_1r   s                   r0   rP   %LightGlueMatchAssignmentLayer.forwardp  sy   4?4E4E1
>--k:%T5H5HQ^QeQe(fjn(nn%--jAoq-`&q!t,&q!t,#&>&>r2&FF
<<
aMBDAJ((,EAJ((,66r2>E=D#//	5;;zGWGW;X;\;\]J ((5#++J!OQqQ%ad+%ad+ ,JWr/   c                     U R                  U5      n[        R                  R                  U5      R	                  S5      nU$ )z0Get matchability of descriptors as a probabilityrE   )r   r   r   sigmoidr   )r?   r   r   s      r0   get_matchability.LightGlueMatchAssignmentLayer.get_matchability  s7    ((5}},,\:BB2Fr/   )r;   r   r   )r$   r%   r&   r'   r   r9   r)   rS   rP   r  r.   rU   rV   s   @r0   r   r   h  sW    I I5<< u||  4ELL U\\  r/   r   c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )LightGlueTokenConfidenceLayeri  r4   c                 n   > [         TU ]  5         [        R                  " UR                  S5      U l        g )Nr   )r8   r9   r   r:   r;   tokenr>   s     r0   r9   &LightGlueTokenConfidenceLayer.__init__  s&    YYv44a8
r/   r   rC   c                     U R                  UR                  5       5      n[        R                  R	                  U5      R                  S5      nU$ )NrE   )r  detachr   r   r  r   )r?   r   r  s      r0   rP   %LightGlueTokenConfidenceLayer.forward  s=    

;--/0%%e,44R8r/   )r  r   rV   s   @r0   r  r    s/    9 9
5<< ELL  r/   r  c                   <    \ rS rSr% Sr\\S'   SrSrSr	Sr
SrSrS	rg
)LightGluePreTrainedModeli  zz
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
models.
r4   	lightgluepixel_values)imageFTr#   N)r$   r%   r&   r'   r(   r   r+   base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_supports_flash_attn_supports_sdpar.   r#   r/   r0   r  r    s1    
 #$O!&+#Nr/   r  r   	thresholdc                 (   U R                   u  n  nU SS2SS2SS24   R                  S5      nU SS2SS2SS24   R                  S5      nUR                  nUR                  n[        R                  " UR                   S   UR
                  S9S   n[        R                  " UR                   S   UR
                  S9S   n	XR                  SU5      :H  n
XR                  SU5      :H  nUR                  R                  5       nUR                  S5      n[        R                  " XU5      n[        R                  " XR                  SU5      U5      nXU:  -  nXR                  SU5      -  n[        R                  " XS5      n[        R                  " UUS5      n[        R                  " Xg/5      R                  SS5      R                  US-  S5      n[        R                  " X/5      R                  SS5      R                  US-  S5      nUU4$ )z1obtain matches from a score matrix [Bx M+1 x N+1]NrE   r   r   r   r   )rn   maxindicesr)   aranger   gathervaluesexp
new_tensorwhererY   r   rp   )r   r  r   _max0max1matches0matches1indices0indices1mutual0mutual1zeromatching_scores0matching_scores1valid0valid1r   r   s                      r0   get_matches_from_scoresr/    s   ||J1!SbS#2#+""1%D!SbS#2#+""1%D||H||H ||HNN1-hooFtLH||HNN1-hooFtLH//!X66G//!X66G ;;??D??1D{{7$7{{7,C,CAx,PRVW945F}}Q11F {{6R0H{{68R0Hkk8./99!Q?GG
UVXZ[Gkk#3"FGQQRSUVW__`jmn`nprsOO##r/   r   heightwidthc                     [         R                  " X!/U R                  U R                  S9S   nUS-  nUR	                  S5      R
                  S-  nXSSSS24   -
  US   -  n U $ )a}  
Normalize keypoints locations based on image image_shape

Args:
    keypoints (`torch.Tensor` of shape `(batch_size, num_keypoints, 2)`):
        Keypoints locations in (x, y) format.
    height (`int`):
        Image height.
    width (`int`):
        Image width.

Returns:
    Normalized keypoints locations of shape (`torch.Tensor` of shape `(batch_size, num_keypoints, 2)`).
r   rb   Nr   rE   .).NN)r)   r   r   rb   r  r  )r   r0  r1  sizeshiftscales         r0   normalize_keypointsr7    sl     <<	0@0@	XY]^D1HEHHRL!#E3a<00E/4JJIr/   zV
    LightGlue model taking images as inputs and outputting the matching of them.
    c                   @  ^  \ rS rSrSrS\4U 4S jjrS\S\4S jr	 S%S\
R                  S	\
R                  S
\S-  S\\
R                  \\
R                  \
R                  4   4   4S jjrS\
R                  S\S\
R                  S\
R                  S\
R                  4
S jrS&S jrS\
R                  S\
R                  S\S\
R                  4S jrS\
R                  S	\
R                  S\
R                  S\
R                  S\
R                  S\
R                  S\4S jrS rS\
R                  S\
R                  S\
R                  S\
R                  S\\
R                  \
R                  4   4
S jr   S'S	\
R                  S\
R                  S\S\S\
R                  S-  S\S-  S
\S-  S\\
R                  \
R                  \
R                  \\4   4S  jjr\\   S'S!\
R0                  S"\
R2                  S-  S\S-  S
\S-  S\\-  4
S# jj5       5       rS$rU =r$ )(LightGlueForKeypointMatchingi  aJ  
LightGlue is a model matching keypoints in images by leveraging detections from a keypoint detector such as
SuperPoint. It is based on the SuperGlue architecture and is designed to be lightweight and efficient.
It consists of :
    1. Keypoint Encoder
    2. A Graph Neural Network with self and cross attention layers
    3. Matching Assignment layers

The correspondence ids use -1 to indicate non-matching points.

Philipp Lindenberger, Paul-Edouard Sarlin and Marc Pollefeys. LightGlue: Local Feature Matching at Light Speed.
In ICCV 2023. https://huggingface.co/papers/2306.13643
r4   c           
      &  > [         TU ]  U5        [        R                  " UR                  5      U l        UR                  R                  U l        UR                  U l        UR                  U l
        UR                  U l        UR                  U l        UR                  U l        U R                  U R                  :w  a0  [        R                  " U R                  U R                  SS9U l        O[        R"                  " 5       U l        [%        U5      U l        [        R(                  " [+        UR                  5       Vs/ s H  n[-        XS9PM     sn5      U l        [        R(                  " [+        UR                  5       Vs/ s H  n[1        U5      PM     sn5      U l        [        R(                  " [+        UR                  S-
  5       Vs/ s H  n[5        U5      PM     sn5      U l        U R9                  5         g s  snf s  snf s  snf )NTr6   )r   r   )r8   r9   r   from_configkeypoint_detector_configkeypoint_detectordescriptor_decoder_dim keypoint_detector_descriptor_dimr;   num_hidden_layers
num_layersfilter_thresholddepth_confidencewidth_confidencer   r:   input_projectionIdentityr2   positional_encoder
ModuleListranger   transformer_layersr   match_assignment_layersr  token_confidence	post_init)r?   r4   ir!  r@   s       r0   r9   %LightGlueForKeypointMatching.__init__  s    !>!J!J6KjKj!k060O0O0f0f-$33 22 & 7 7 & 7 7 & 7 7$"G"GG$&IId.S.SUYUhUhos$tD!$&KKMD!"<V"D"$--EJ6KcKcEdeEd&v;Ede#
 (*}}<A&BZBZ<[\<[q*62<[\(
$ !#<A&BZBZ]^B^<_`<_q*62<_`!
 	 f ] as   HH	Hlayer_indexrC   c                     SS[         R                  " SU-  U R                  -  5      -  -   n[         R                  " USS5      $ )z-scaled confidence threshold for a given layerg?g?g      r   r   )npr  rA  clip)r?   rP  r  s      r0   _get_confidence_threshold6LightGlueForKeypointMatching._get_confidence_threshold  s;    #tk'9DOO'K LLL	wwy!Q''r/   r   r   rB   Nc                     UR                  5       R                  5       nU R                  U5      nU R                  X#S9nXE4$ )NrB   )r
  r   rE  rG  )r?   r   r   rB   projected_descriptorskeypoint_encoding_outputs         r0   _keypoint_processing1LightGlueForKeypointMatching._keypoint_processing  sI     "((*557 $ 5 5k B#'#:#:9#:#p $>>r/   keypoint_confidencesr    
num_pointsc                 l   UR                   u  pVX R                  S-
  :  ap  UR                  US:H  S5      nUR                  US-  S5      nU R	                  U5      nSX:  R                  5       R                  SS9U-  -
  nXR                  :  n	U	$ [        R                  " U[        R                  S9n	U	$ )zRevaluate whether we should stop inference based on the confidence of the keypointsr   r   r   rE   g      ?rF   ra   )rn   rA  r   rp   rT  rc   sumrC  r)   onesrT   )
r?   r\  rP  r    r]  r   r!  r  ratio_confidentearly_stopped_pairss
             r0   _get_early_stopped_image_pairs;LightGlueForKeypointMatching._get_early_stopped_image_pairs"  s     


1,, $8#C#CDAIq#Q #7#?#?
aQS#T 66{CI!%9%E$L$L$N$R$RWX$R$Y\f$ffO"14I4I"I
 #" #(**Zuzz"J""r/   c                 v    Ub  X   nX$   nU R                   U   " X5      n[        XPR                  5      u  pgXg4$ r   )rK  r/  rB  )r?   r   r    rP  early_stopsr   r   r   s           r0   _get_keypoint_matching3LightGlueForKeypointMatching._get_keypoint_matching5  sI    "%2K$D--k:;M#:6CXCX#Y ''r/   confidencesr   c                 \    USU R                   -
  :  nUb  XAU R                  U5      :*  -  nU$ )z#mask points which should be removedr   )rD  rT  )r?   ri  r   rP  keeps        r0   _get_pruning_mask.LightGlueForKeypointMatching._get_pruning_mask=  s:    T2223"4#A#A+#NNNDr/   r  prune_outputc                   ^ UR                   u  n  n	U R                  U   R                  U5      n
U R                  XjU5      mTR	                  US:H  [
        R                  " S5      5      mU4S jXS   US   TU4 5       u  ppn[        U5       H  nUUUU   4==   S-  ss'   M     S XX4 5       u  ppX4n[        USSS9nUUXU4$ )	zv
For a given layer, prune keypoints based on the confidence of the keypoints and the matchability of the
descriptors.
r   Fc              3   v   >#    U  H(  n[        UT5       VVs/ s H	  u  p#X#   PM     snnv   M*     g s  snnf 7fr   )zip).0r   tr    pruned_keypoints_masks       r0   	<genexpr>JLightGlueForKeypointMatching._do_layer_keypoint_pruning.<locals>.<genexpr>X  s<      c
c %(0E$FG$FQW$FGc Hs   939r   c              3   6   #    U  H  n[        US S9v   M     g7f)T)batch_firstNr   )rr  pruned_tensors     r0   ru  rv  `  s      S
!j D9!js   TrE   rx  padding_value)	rn   rK  r  rl  r   r)   r   rI  r   )r?   r   r   r    r  rn  r\  rP  r   r!  descriptors_matchabilitypruned_descriptorspruned_keypoints_0pruned_keypoints_1pruned_maskpruned_indicesrN  pruned_keypointsrt  s                     @r0   _do_layer_keypoint_pruning7LightGlueForKeypointMatching._do_layer_keypoint_pruningD  s    ',,
Aq#'#?#?#L#]#]^i#j  $ 6 67Kgr s 5 A A$!)U\\Z_M` ac
&!ilDY[bcc
_0BQ_ z"AN1--.!3. #S
"4J\!jS
O0B /C%n$VXY!#3^R^^^r/   c                    ^ [         R                  " T5      m[         R                  " TR                  S   5      nTU   nTU   mS XB4 5       u  pBS XS4 5       u  pSU4S jUUUU4 5       u  pEp#X#XE4$ )Nr   c              3   8   #    U  H  n[        US SS9v   M     g7f)TrE   rz  Nr   rr  r   s     r0   ru  MLightGlueForKeypointMatching._concat_early_stopped_outputs.<locals>.<genexpr>v  s       3
C TDC   c              3   8   #    U  H  n[        US SS9v   M     g7f)Tr   rz  Nr   r  s     r0   ru  r  z  s       >
N TCNr  c              3   .   >#    U  H
  nUT   v   M     g 7fr   r#   )rr  r   early_stops_indicess     r0   ru  r  ~  s#      g
 &'s   )r)   rY   r  rn   )r?   r  final_pruned_keypoints_indices!final_pruned_keypoints_iterationsr   r   idsorder_indicess    `      r0   _concat_early_stopped_outputs:LightGlueForKeypointMatching._concat_early_stopped_outputsi  s     $kk*=>ll.44Q78+C01-@3
"C3
/>
*N>
:g
 .1	g
c"@ .RYjjr/   r   r   r   c                   ^ UR                   u  mnU4S jXU4 5       u  pnUS S 2S4   nUS S 2S4   nUS S 2S4   nUS S 2S4   n	US S 2S4   n
US S 2S4   n[        R                  " TS-  SU4SUR                  UR                  S9n[        R
                  " TS-  SU4UR                  UR                  S9n[        TS-  5       H  n[        R                  " X   S:H  SX~   R                  SX   R                  SS95      5      XSXn   4'   [        R                  " X   S:H  SXn   R                  SX   R                  SS95      5      XSX~   4'   X   XSXn   4'   X   XSX~   4'   M     X4$ )Nc              3   N   >#    U  H  oR                  TS -  S S5      v   M     g7f)r   rE   N)rp   )rr  r   r   s     r0   ru  JLightGlueForKeypointMatching._do_final_keypoint_pruning.<locals>.<genexpr>  s'      -
AdvNN:?Ar22Ads   "%r   r   r   rE   r3  )r   )
rn   r)   fullr   rb   zerosrI  r   r  clamp)r?   r  r   r   r   r!  r&  r'  r$  r%  r+  r,  _matches_matching_scoresrN  r   s                  @r0   _do_final_keypoint_pruning7LightGlueForKeypointMatching._do_final_keypoint_pruning  s     
A-
BITcAd-
)/ 1a4=1a4=1a4=1a4=*1a40*1a40 ::zQ=A2gnndkdqdqr ;;1_a/oNcNc
 zQ'A*/++r!2x{'9'9!X[=N=NST=N=U'V+H8;&' +0++r!2x{'9'9!X[=N=NST=N=U'V+H8;&' 3C2E8;./2B2E8;./ ( ))r/   r0  r1  r   c           
      p	  ^( U(       a  SOS nU(       a  SOS n	UR                   S   S:X  aQ  UR                   S S n
UR                  U
S[        R                  S9UR	                  U
5      UR	                  U
5      UU	4$ UR
                  nUR                   u  pp[        R                  " UR                  US5      SS9nUR                  US-  US5      nUb  UR                  US-  U5      OS nUR                  US-  XR                  5      n[        R                  " US-  US9n[        XU5      nU R                  X!US	9u  nnUS   nU R                  S:  nU R                  S:  n/ n/ n/ n/ n/ n[        R                  " SXS9R                  US-  S5      n[        R                  " U5      n[!        U R"                  5       GHR  nUR%                  5       nUb  U R'                  UU5      nO$[        R(                  " UUS
   4UR
                  S9nU R*                  U   " UUUUUS9nUu  nnn U(       a  UU-   nU(       a  U	U -   n	U(       Ga  UU R"                  S-
  :  a'  U R,                  U   " U5      n!U R/                  U!UX_S9n"O#[        R(                  " U[        R0                  S9n"[        R2                  " U"5      (       Ga  U"R5                  S5      m(UT(   n#U R7                  X%UT(S9u  n$n%UR9                  [;        U#5      5        UR9                  [;        U$5      5        UR9                  [;        U%5      5        U(       a:  UR9                  [;        UT(   5      5        UR9                  [;        UT(   5      5        UU")    n[=        U(4S jX!S   US   UU4 5       5      u  nn&n'nnU&U'4nU(       a  [=        U(4S jUUW!4 5       5      u  nnn![        R>                  " U"5      (       a    O+U(       d  GM6  U RA                  UUUUUW!U5      u  p!nnnGMU     U(       a9  U(       a2  U RC                  UUUUU5      u  nnnnU RE                  UUUU5      u  nnOEU R7                  X%U R"                  S-
  5      u  nn[        R                  " U5      U R"                  -  nUR                  USU5      nUUUUU	4$ )Nr#   r   r   rE   ra   r   rF   r   rW  rX   )rz   rB   r   )r]  )rf  c              3   0   >#    U  H  nUT)    v   M     g 7fr   r#   rr  r   rf  s     r0   ru  ALightGlueForKeypointMatching._match_image_pair.<locals>.<genexpr>  s       V&dF |,&d   c              3   0   >#    U  H  nUT)    v   M     g 7fr   r#   r  s     r0   ru  r    s$      l+ #K<0+r  )#rn   r   r)   r   	new_zerosr   r_  rp   r?  r  r7  rZ  rC  rD  ro   	ones_likerI  rA  r4  get_extended_attention_maskr`  rJ  rL  rc  rT   anyrH   rg  extendlistr-   allr  r  r  ))r?   r   r   r0  r1  r    r   rB   r   r   rn   r   r   r!  initial_num_keypointsnum_points_per_pairimage_indicesrY  do_early_stopdo_keypoint_pruningr  r   r   r  r  pruned_keypoints_indicespruned_keypoints_iterationsrP  r   extended_attention_masklayer_outputr!   	attentionr\  rb  early_stopped_image_indicesearly_stopped_matchesearly_stopped_matching_scoreskeypoints_0
keypoint_1rf  s)                                           @r0   _match_image_pair.LightGlueForKeypointMatching._match_image_pair  sK    #7BD0d??1"OOCR(E""5"EII">##E*##E*!  !!2;///
,#iiZ(D!L%%j1n6KQO	FJFVt||JN,AB\`!))*q.:OQvQvwZ!^FC'	5A	040I0I9M 1J 1
-- -Q/	 --1 #33a7 )+&,.)#(<<3H#X#_#_`jmn`npr#s &+oo6N&O# 1K%**,K*.*J*J4Q\*]'*/**j+b/5R[d[k[k*l'22;?6%9"3L 5A1K	#$5$E! !/)!;1!44+/+@+@+Mk+Z( +/*M*M,k4 +N +'
 +0**Zuzz*R'99011 #6"G"G"JK2?2L/KOKfKf#;K Lg LH)+H (..t4O/PQNN4(=#>?#**40M+NO*6==dC[\gCh>ij9@@FabmFnAop +>?R>R*S'PU V'2aL)A,PTVc&dV QMKj$ "-j 9I*fk l !9 ; 4+l gc02MOc 99011"" 33#!03,# d(@$HcQ 2h 0 22'25# h*,MwXg (,'F'F.%	($G_ (,'B'B;VZVeVehiVi'j$G_050PSWSbSb0b-,M,U,U0-
)
 -
 	
r/   r  labelsc                 $   S nUb  [        S5      eUb  UOU R                  R                  nUb  UOU R                  R                  nUR                  S:w  d  UR                  S5      S:w  a  [        S5      eUR                  u  pxpnUR                  US-  XU5      nU R                  U5      nUS S u  ppUR                  USSS5      R                  U5      nUR                  USSU R                  5      R                  U5      nUR                  USS5      nUR                  5       nUS S 2S S 2S S 2S4   U-  US S 2S S 2S S 2S4'   US S 2S S 2S S 2S4   U
-  US S 2S S 2S S 2S4'   U R                  UUU
UUUUS	9u  nnnnn[        UUUUUUUUS
9$ )Nz9LightGlue is not trainable, no labels should be provided.   r   r   zOInput must be a 5D tensor of shape (batch_size, 2, num_channels, height, width)   rE   r   )r    r   rB   )r   r   r   r   r   r    r!   r"   )
ValueErrorr4   r   rB   ndimr4  rn   rp   r=  re   r?  cloner  r   )r?   r  r  r   rB   r}   r   r   r!  channelsr0  r1  keypoint_detectionsr   r   r    absolute_keypointsr   r   r   r!   r"   s                         r0   rP   $LightGlueForKeypointMatching.forward[  s    XYY1B1N-TXT_T_TqTq$8$D $++JjJj 	 !\%6%6q%9Q%>noo1=1C1C.
x#++JNHeT"44\B*=bq*A'	k%%j!R;>>|L	!))*aT=b=bcffgst||J2.&__.);Aq!QJ)G%)O1aA:&);Aq!QJ)G&)P1aA:&EIE[E[/!5 F\ F
B%
 /+'!	
 		
r/   )rC  r;   rB  rE  r=  r?  rK  rA  rG  rL  rJ  rD  rR   r   )NNN)r$   r%   r&   r'   r(   r   r9   r   rc   rT  r)   rS   rT   r-   rZ  rc  rg  rl  r  r  r  r  r   r   r*   
LongTensorr   rP   r.   rU   rV   s   @r0   r9  r9    s    <(S (U ( gl? <<?49LL?X\_cXc?	u||U5<<#=>>	??#$)LL#?B#JO,,#didpdp#	#&(U\\ 5<< ^a fkfrfr #_\\#_ <<#_ ll	#_
 #_ ll#_ $ll#_ #_Jk@#*#* #* 	#*
 ||#* 
u||U\\)	*#*V %))-,0k
<<k
 \\k
 	k

 k
 llT!k
  $;k
 #Tkk
 
u||U\\5<<E	Fk
Z  +/)-,04
''4
   4'4
  $;	4

 #Tk4
 
0	04
  4
r/   r9  )r   )r   )4collections.abcr   dataclassesr   numpyrR  r)   r   torch.nn.utils.rnnr   activationsr	   integrationsr
   modeling_flash_attention_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   auto.modeling_autor   configuration_lightgluer   r   Moduler2   r_   rk   rS   r   ru   rc   r   r   r   r   r   r   r  r  r-   r/  r7  r9  __all__r#   r/   r0   <module>r     sj  ( % !    + ! / B F & V V > 4   7k  7  7F "<8	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 LL4'% % % '(%2 )*D) D) +D)N299 "P>		 P>f-2\\JO,,
\\&BII &R	BII 	   $ELL $U $uU\\[`[g[gMgGh $@5<<  S U\\ , 
i
#; i

i
X &'E
Fr/   