
    Z jҥ                         S SK Jr  S SKJr  S SKrS SKrS SKJr  S SKJ	r	  S SK
Jr  SSKJr  SS	KJr  SS
KJrJr  SSKJr  SSKJrJrJrJrJr  SSKJr  SSKJrJr  SSK J!r!  SSK"J#r#  SSK$J%r%  SSK&J'r'J(r(  SSK)J*r*  SSK+J,r,J-r-  SSK.J/r/  \R`                  " \15      r2\" SS9\ " S S\5      5       5       r3\" SS9\ " S S\5      5       5       r4 " S S \-5      r5 " S! S"\,5      r6\" S#S$9 " S% S&\*5      5       r7 " S' S(\	Rp                  5      r9 " S) S*\'5      r: " S+ S,\#5      r; " S- S.\	Rp                  5      r<S/\Rz                  S0\Rz                  S1\Rz                  S2\Rz                  4S3 jr> " S4 S5\	Rp                  5      r? " S6 S7\	Rp                  5      r@\ " S8 S9\5      5       rAS:\Rz                  S;\BS2\C\Rz                  \Rz                  4   4S< jrDS=\Rz                  S>\ES?\ES2\Rz                  4S@ jrF\" SAS9 " SB SC\A5      5       rG/ SDQrHg)E    )Callable)	dataclassN)strict)nnpad_sequence   )PreTrainedConfig)FlashAttentionKwargs)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutput
TensorTypeauto_docstringcan_return_tuplelogging)requires   )CONFIG_MAPPING
AutoConfig)AutoModelForKeypointDetection)CLIPMLP)apply_rotary_pos_emb)LlamaAttentioneager_attention_forward)SuperGlueImageProcessorPil)SuperGlueImageProcessorSuperGlueImageProcessorKwargs)SuperPointConfigzETH-CVG/lightglue_superpoint)
checkpointc                      ^  \ rS rSr% SrSrS\0rSr\	\
-  S-  \S'   Sr\\S'   Sr\\S	'   S
r\\S'   Sr\S-  \S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\-  \S'   Sr\\S'   U 4S jrS rSrU =r$ )LightGlueConfig*   a  
keypoint_detector_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `SuperPointConfig`):
    The config object or dictionary of the keypoint detector.
descriptor_dim (`int`, *optional*, defaults to 256):
    The dimension of the descriptors.
depth_confidence (`float`, *optional*, defaults to 0.95):
    The confidence threshold used to perform early stopping
width_confidence (`float`, *optional*, defaults to 0.99):
    The confidence threshold used to prune points
filter_threshold (`float`, *optional*, defaults to 0.1):
    The confidence threshold used to filter matches

Examples:
    ```python
    >>> from transformers import LightGlueConfig, LightGlueForKeypointMatching

    >>> # Initializing a LightGlue style configuration
    >>> configuration = LightGlueConfig()

    >>> # Initializing a model from the LightGlue style configuration
    >>> model = LightGlueForKeypointMatching(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
	lightgluekeypoint_detector_configN   descriptor_dim	   num_hidden_layers   num_attention_headsnum_key_value_headsgffffff?depth_confidencegGz?width_confidence皙?filter_thresholdg{Gz?initializer_rangegelu
hidden_act        attention_dropoutTattention_biasc                   > U R                   c  U R                  U l         [        U R                  [        5      (       aY  U R                  R                  SS5      U R                  S'   [        U R                  S      " S0 U R                  DSS0D6U l        OU R                  c  [        S   " SS9U l        U R                  S-  U l        U R                  U l	        [        TU ],  " S0 UD6  g )N
model_type
superpointattn_implementationeager)r;   r    )r-   r,   
isinstancer&   dictgetr   r(   intermediate_sizehidden_sizesuper__post_init__)selfkwargs	__class__s     ڀ/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/models/lightglue/modular_lightglue.pyrD   LightGlueConfig.__post_init__X   s    ##+'+'?'?D$ d33T:::>:W:W:[:[\hjv:wD)),7,:4;X;XYe;f,g -//-EL-D) **2,:<,H]d,eD)!%!4!4q!8..''    c                 T    U R                   U R                  -  S:w  a  [        S5      eg)zOPart of `@strict`-powered validation. Validates the architecture of the config.r   z1descriptor_dim % num_heads is different from zeroN)r(   r,   
ValueError)rE   s    rH   validate_architecture%LightGlueConfig.validate_architecturej   s,    !9!99Q>PQQ ?rJ   )rB   rA   r&   r-   )__name__
__module____qualname____firstlineno____doc__r9   r   sub_configsr&   r?   r    __annotations__r(   intr*   r,   r-   r.   floatr/   r1   r2   r4   strr6   r7   boolrD   rM   __static_attributes____classcell__rG   s   @rH   r#   r#   *   s    6 J-z:K?Cd%55<CNCs  &*t*"e""e"!e!#u#J%(us{(ND($R RrJ   r#   a  
    Base class for outputs of LightGlue keypoint matching models. Due to the nature of keypoint detection and matching,
    the number of keypoints is not fixed and can vary from image to image, which makes batching non-trivial. In the
    batch of images, the maximum number of matches is set as the dimension of the matches and matching scores. The mask
    tensor is used to indicate which values in the keypoints, matches, matching_scores and prune tensors are keypoint
    matching information.
    )custom_introc                   f   \ rS rSr% SrSr\R                  S-  \S'   Sr	\R                  S-  \S'   Sr
\R                  S-  \S'   Sr\R                  S-  \S'   Sr\R                  S-  \S'   Sr\R                  S-  \S	'   Sr\\R                     S-  \S
'   Sr\\R                     S-  \S'   Srg)LightGlueKeypointMatchingOutputp   aW  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*):
    Loss computed during training.
matches (`torch.FloatTensor` of shape `(batch_size, 2, num_matches)`):
    Index of keypoint matched in the other image.
matching_scores (`torch.FloatTensor` of shape `(batch_size, 2, num_matches)`):
    Scores of predicted matches.
keypoints (`torch.FloatTensor` of shape `(batch_size, num_keypoints, 2)`):
    Absolute (x, y) coordinates of predicted keypoints in a given image.
prune (`torch.IntTensor` of shape `(batch_size, num_keypoints)`):
    Pruning mask indicating which keypoints are removed and at which layer.
mask (`torch.BoolTensor` of shape `(batch_size, num_keypoints)`):
    Mask indicating which values in matches, matching_scores, keypoints and prune are keypoint matching
    information.
hidden_states (`Tuple[torch.FloatTensor, ...]`, *optional*):
    Tuple of `torch.FloatTensor` (one for the output of each stage) of shape `(batch_size, 2, num_channels,
    num_keypoints)` returned when `output_hidden_states=True` is passed or when
    `config.output_hidden_states=True`
attentions (`Tuple[torch.FloatTensor, ...]`, *optional*):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, 2, num_heads, num_keypoints,
    num_keypoints)` returned when `output_attentions=True` is passed or when
    `config.output_attentions=True`
Nlossmatchesmatching_scores	keypointsprunemaskhidden_states
attentionsr=   )rO   rP   rQ   rR   rS   ra   torchFloatTensorrU   rb   rc   rd   re   	IntTensorrf   rg   tuplerh   rZ   r=   rJ   rH   r_   r_   p   s    0 &*D%

d
")(,GU%,04OU&&-4*.Iu  4'.$(E5??T!(%)D%

d
")59M5**+d2926Je''(4/6rJ   r_   c                       \ rS rSrSrg)LightGlueImageProcessorKwargs   r=   N)rO   rP   rQ   rR   rZ   r=   rJ   rH   rn   rn      s    rJ   rn   c                   r   ^  \ rS rSr S	SSS\\\   -  S\S\\\	\
R                  4      4U 4S jjjrSrU =r$ )
LightGlueImageProcessor   outputsr_   target_sizes	thresholdreturnc                 $   > [         TU ]  XU5      $ NrC   post_process_keypoint_matchingrE   rs   rt   ru   rG   s       rH   rz   6LightGlueImageProcessor.post_process_keypoint_matching   s     w5gYWWrJ   r=   r5   )rO   rP   rQ   rR   r   listrl   rW   r?   rX   ri   Tensorrz   rZ   r[   r\   s   @rH   rq   rq      sZ    
 	X2X !4;.X 	X
 
d3$%	&X XrJ   rq   ri   backendsc                   p   ^  \ rS rSr\" SS9 SSSS\\\   -  S\S\\	\
S	4      4U 4S
 jjj5       rSrU =r$ )LightGlueImageProcessorPil   r   r   rs   r_   rt   ru   rv   ztorch.Tensorc                 $   > [         TU ]  XU5      $ rx   ry   r{   s       rH   rz   9LightGlueImageProcessorPil.post_process_keypoint_matching   s     w5gYWWrJ   r=   r}   )rO   rP   rQ   rR   r   r   r~   rl   rW   r?   rX   rz   rZ   r[   r\   s   @rH   r   r      sc    z"
 	X2X !4;.X 	X
 
d3&'	(X #XrJ   r   c            
          ^  \ rS rSrS\4U 4S jjr S
S\R                  S\S-  S\	\R                     \	\R                  \R                  4   -  4S jjr
S	rU =r$ )LightGluePositionalEncoder   configc                    > [         TU ]  5         [        R                  " SUR                  UR
                  -  S-  SS9U l        g )Nr   Fbias)rC   __init__r   Linearr(   r,   	projectorrE   r   rG   s     rH   r   #LightGluePositionalEncoder.__init__   s:    1f&;&;v?Y?Y&Y]^&^ejkrJ   rd   output_hidden_statesNrv   c                     U R                  U5      nUR                  SSS9n[        R                  " U5      n[        R                  " U5      nXV4nU(       a  XC4nU$ U4nU$ )Nr   dim)r   repeat_interleaveri   cossin)rE   rd   r   projected_keypoints
embeddingscosinessinesoutputs           rH   forward"LightGluePositionalEncoder.forward   sl     #nnY7(::1":E
))J'		*%%
6J*2 R\P]rJ   )r   F)rO   rP   rQ   rR   r#   r   ri   r   rY   rl   r   rZ   r[   r\   s   @rH   r   r      sb    l l
 LQ		=AD[		u||	uU\\5<<%?@	@	 	rJ   r   c                   >  ^  \ rS rSrS\S\4U 4S jjr    SS\R                  S\	\R                  \R                  4   S-  S\R                  S-  S	\R                  S-  S
\R                  S-  S\
\   S\	\R                  \R                  S-  4   4S jjrSrU =r$ )LightGlueAttention   r   	layer_idxc                 &   > [         TU ]  5         U ?g rx   )rC   r   
rotary_embrE   r   r   rG   s      rH   r   LightGlueAttention.__init__   s    OrJ   Nrg   position_embeddingsattention_maskencoder_hidden_statesencoder_attention_maskrF   rv   c                    UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      n	US Ln
U
(       a  UOUnU
(       a  UOUnU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      nUb  Uu  nn[        XUU5      u  p[        R                  " U R                  R                  [        5      nU" U U	UUU4U R                  (       d  SOU R                  U R                  S.UD6u  nnUR                   " / UQSP76 R#                  5       nU R%                  U5      nUU4$ )Nr      r   r5   )dropoutscaling)shapehead_dimq_projview	transposek_projv_projr   r   get_interfacer   _attn_implementationr   trainingr6   r   reshape
contiguouso_proj)rE   rg   r   r   r   r   rF   input_shapehidden_shapequery_statesis_cross_attentioncurrent_statescurrent_attention_mask
key_statesvalue_statesr   r   attention_interfaceattn_outputattn_weightss                       rH   r   LightGlueAttention.forward   s    $))#2.88b8$--8{{=166|DNNqRST2$>2D.-;M!7Sa[[055lCMMaQRS
{{>277EOOPQSTU**HC';LVY[^'_$L(?(M(MKK,,.E)
 %8"	%
  $}}C$2H2HLL	%
 	%
!\ "));;;;FFHkk+.L((rJ   r=   )NNNN)rO   rP   rQ   rR   r#   rV   r   ri   r   rl   r   r   r   rZ   r[   r\   s   @rH   r   r      s     3  IM.2596:*)||*) #5<<#=>E*) t+	*)
  %||d2*) !&t 3*) -.*) 
u||U\\D00	1*) *)rJ   r   c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )LightGlueMLP   r   c                    > [         TU ]  U5        [        R                  " UR                  UR                  5      U l        [        R                  " UR                  SS9U l        g )NT)elementwise_affine)rC   r   r   r   rA   fc1	LayerNorm
layer_normr   s     rH   r   LightGlueMLP.__init__   sG     99V55v7O7OP,,v'?'?TXYrJ   rg   rv   c                     U R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nU$ rx   )r   r   activation_fnfc2)rE   rg   s     rH   r   LightGlueMLP.forward   sB    /6**=9/rJ   )r   r   rO   rP   rQ   rR   r#   r   ri   r   r   rZ   r[   r\   s   @rH   r   r      s1    Z Z
U\\ ell  rJ   r   c                     ^  \ rS rSrS\S\4U 4S jjr  SS\R                  S\R                  S\R                  S\	S	-  S
\	S	-  S\
\R                  \
\R                     S	-  \
\R                     S	-  4   4S jjrSrU =r$ )LightGlueTransformerLayeri  r   r   c                    > [         TU ]  5         [        X5      U l        [	        U5      U l        [        X5      U l        [	        U5      U l        g rx   )rC   r   r   self_attentionr   self_mlpcross_attention	cross_mlpr   s      rH   r   "LightGlueTransformerLayer.__init__	  s@    0C$V,1&D%f-rJ   descriptorsrd   r   r   Noutput_attentionsrv   c                    U(       a  SOS nU(       a  SOS nU(       a  Xa4-   nUR                   u  pn
U R                  UUUUS9u  p[        R                  " X/SS9nU R	                  U5      nX-   nU(       a  X4nUR                  SSX5      R                  S5      R                  XU
5      nUb6  UR                  SSSSU	5      R                  S5      R                  USSU	5      OS nU R                  UUUUS9u  nn[        R                  " UU/SS9nU R                  U5      nUU-   nU(       a4  UU4nUUR                  XU
5      4-   W-   UR                  XU
5      4-   U-   nU(       a	  X|4-   U4-   nXU4$ )Nr=   )r   r   r   r   r   r   r   )r   r   r   )	r   r   ri   catr   r   flipr   r   )rE   r   rd   r   r   r   all_hidden_statesall_attentions
batch_sizenum_keypointsr(   attention_outputself_attentionsintermediate_statesoutput_statesself_attention_descriptorsself_attention_hidden_statesr   r   cross_attention_outputcross_attentionscross_intermediate_statescross_output_statescross_attention_hidden_statess                           rH   r   !LightGlueTransformerLayer.forward  s    #7BD0d 1N B4?4E4E1
> -1,?,? ))/	 -@ -
) $ii(GRP&9:%0%@",?+O( '..r1mTT!WWZ? 	 ) ""2q!Q>CCAFNNz[\^_ano 	 483G3G&"7#9/	 4H 4
0 0 %*II/IKa.bhj$k!"nn-FG03FF-FH[,\)!-55jQ_`bc./ &&z.QST 0	0  +.@@DTCVVN~==rJ   )r   r   r   r   )FF)rO   rP   rQ   rR   r#   rV   r   ri   r   rY   rl   r   rZ   r[   r\   s   @rH   r   r     s    . .3 . -2).H>\\H> <<H> 	H>
 #TkH>  $;H> 
u||U5<<047u||9Lt9SS	TH> H>rJ   r   
similaritymatchability0matchability1rv   c                    U R                   u  p4n[        R                  R                  U5      [        R                  R                  U5      R	                  SS5      -   n[        R                  R                  U S5      n[        R                  R                  U R	                  SS5      R                  5       S5      R	                  SS5      nU R                  X4S-   US-   4S5      n	Xx-   U-   U	SS2SU2SU24'   [        R                  R                  UR                  S5      * 5      U	SS2SS2S4'   [        R                  R                  UR                  S5      * 5      U	SS2SSS24'   U	$ )z;create the log assignment matrix from logits and similarityr   r   r   r   N)	r   r   
functional
logsigmoidr   log_softmaxr   new_fullsqueeze)
r   r   r   r   num_keypoints_0num_keypoints_1certaintiesscores0scores1scoress
             rH   sigmoid_log_double_softmaxr  [  sM    4>3C3C0J--**=9BMM<T<TUb<c<m<mnoqr<ssKmm''
A6Gmm''
(<(<R(D(O(O(QSTU__`bdfgG  *.A?UVCV!WYZ[F4;4E4SF1 0 00111=3H3H3L2LMF1crc2:11=3H3H3L2LMF1b#2#:MrJ   c                      ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  S\R                  4S jrS\R                  S\R                  4S jr	S	r
U =r$ )
LightGlueMatchAssignmentLayerij  r   c                    > [         TU ]  5         UR                  U l        [        R                  " U R                  U R                  SS9U l        [        R                  " U R                  SSS9U l        g )NTr   r   )rC   r   r(   r   r   final_projectionmatchabilityr   s     rH   r   &LightGlueMatchAssignmentLayer.__init__k  sY    $33 "		$*=*=t?R?RY] ^IId&9&914HrJ   r   rf   rv   c                    UR                   u  p4nU R                  U5      nU[        R                  " U R                  UR
                  S9S-  -  nUR                  US-  SXE5      nUS S 2S4   nUS S 2S4   nXxR                  SS5      -  n	Ub  UR                  US-  SU5      nUS S 2S4   R                  S5      n
US S 2S4   R                  S5      R                  SS5      nX-  nU	R                  US:H  [        R                  " U	R                  5      R                  5      n	U R                  U5      nUR                  US-  SUS5      nUS S 2S4   nUS S 2S4   n[        XU5      nU$ )Ndeviceg      ?r   r   r   r   r   )r   r  ri   tensorr(   r  r   r   	unsqueezemasked_fillfinfodtypeminr  r  )rE   r   rf   r   r   r(   m_descriptorsm_descriptors0m_descriptors1r   mask0mask1r  matchability_0matchability_1r  s                   rH   r   %LightGlueMatchAssignmentLayer.forwardr  sy   4?4E4E1
>--k:%T5H5HQ^QeQe(fjn(nn%--jAoq-`&q!t,&q!t,#&>&>r2&FF
<<
aMBDAJ((,EAJ((,66r2>E=D#//	5;;zGWGW;X;\;\]J ((5#++J!OQqQ%ad+%ad+ ,JWrJ   c                     U R                  U5      n[        R                  R                  U5      R	                  S5      nU$ )z0Get matchability of descriptors as a probabilityr   )r  r   r   sigmoidr  )rE   r   r  s      rH   get_matchability.LightGlueMatchAssignmentLayer.get_matchability  s7    ((5}},,\:BB2FrJ   )r(   r  r  )rO   rP   rQ   rR   r#   r   ri   r   r   r"  rZ   r[   r\   s   @rH   r
  r
  j  sW    I I5<< u||  4ELL U\\  rJ   r
  c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )LightGlueTokenConfidenceLayeri  r   c                 n   > [         TU ]  5         [        R                  " UR                  S5      U l        g )Nr   )rC   r   r   r   r(   tokenr   s     rH   r   &LightGlueTokenConfidenceLayer.__init__  s&    YYv44a8
rJ   r   rv   c                     U R                  UR                  5       5      n[        R                  R	                  U5      R                  S5      nU$ )Nr   )r'  detachr   r   r!  r  )rE   r   r'  s      rH   r   %LightGlueTokenConfidenceLayer.forward  s=    

;--/0%%e,44R8rJ   )r'  r   r\   s   @rH   r%  r%    s/    9 9
5<< ELL  rJ   r%  c                   <    \ rS rSr% Sr\\S'   SrSrSr	Sr
SrSrS	rg
)LightGluePreTrainedModeli  zz
An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
models.
r   r%   pixel_values)imageFTr=   N)rO   rP   rQ   rR   rS   r#   rU   base_model_prefixmain_input_nameinput_modalitiessupports_gradient_checkpointing_supports_flash_attn_supports_sdparZ   r=   rJ   rH   r-  r-    s1    
 #$O!&+#NrJ   r-  r  ru   c                 (   U R                   u  n  nU SS2SS2SS24   R                  S5      nU SS2SS2SS24   R                  S5      nUR                  nUR                  n[        R                  " UR                   S   UR
                  S9S   n[        R                  " UR                   S   UR
                  S9S   n	XR                  SU5      :H  n
XR                  SU5      :H  nUR                  R                  5       nUR                  S5      n[        R                  " XU5      n[        R                  " XR                  SU5      U5      nXU:  -  nXR                  SU5      -  n[        R                  " XS5      n[        R                  " UUS5      n[        R                  " Xg/5      R                  SS5      R                  US-  S5      n[        R                  " X/5      R                  SS5      R                  US-  S5      nUU4$ )z1obtain matches from a score matrix [Bx M+1 x N+1]Nr   r   r   r  r   )r   maxindicesri   aranger  gathervaluesexp
new_tensorwherestackr   r   )r  ru   r   _max0max1matches0matches1indices0indices1mutual0mutual1zeromatching_scores0matching_scores1valid0valid1rb   rc   s                      rH   get_matches_from_scoresrN    s   ||J1!SbS#2#+""1%D!SbS#2#+""1%D||H||H ||HNN1-hooFtLH||HNN1-hooFtLH//!X66G//!X66G ;;??D??1D{{7$7{{7,C,CAx,PRVW945F}}Q11F {{6R0H{{68R0Hkk8./99!Q?GG
UVXZ[Gkk#3"FGQQRSUVW__`jmn`nprsOO##rJ   rd   heightwidthc                     [         R                  " X!/U R                  U R                  S9S   nUS-  nUR	                  S5      R
                  S-  nXSSSS24   -
  US   -  n U $ )a}  
Normalize keypoints locations based on image image_shape

Args:
    keypoints (`torch.Tensor` of shape `(batch_size, num_keypoints, 2)`):
        Keypoints locations in (x, y) format.
    height (`int`):
        Image height.
    width (`int`):
        Image width.

Returns:
    Normalized keypoints locations of shape (`torch.Tensor` of shape `(batch_size, num_keypoints, 2)`).
r  r  Nr   r   .).NN)ri   r  r  r  r7  r;  )rd   rO  rP  sizeshiftscales         rH   normalize_keypointsrV    sl     <<	0@0@	XY]^D1HEHHRL!#E3a<00E/4JJIrJ   zV
    LightGlue model taking images as inputs and outputting the matching of them.
    c                   @  ^  \ rS rSrSrS\4U 4S jjrS\S\4S jr	 S%S\
R                  S	\
R                  S
\S-  S\\
R                  \\
R                  \
R                  4   4   4S jjrS\
R                  S\S\
R                  S\
R                  S\
R                  4
S jrS&S jrS\
R                  S\
R                  S\S\
R                  4S jrS\
R                  S	\
R                  S\
R                  S\
R                  S\
R                  S\
R                  S\4S jrS rS\
R                  S\
R                  S\
R                  S\
R                  S\\
R                  \
R                  4   4
S jr   S'S	\
R                  S\
R                  S\S\S\
R                  S-  S\S-  S
\S-  S\\
R                  \
R                  \
R                  \\4   4S  jjr\\   S'S!\
R0                  S"\
R2                  S-  S\S-  S
\S-  S\\-  4
S# jj5       5       rS$rU =r$ )(LightGlueForKeypointMatchingi  aJ  
LightGlue is a model matching keypoints in images by leveraging detections from a keypoint detector such as
SuperPoint. It is based on the SuperGlue architecture and is designed to be lightweight and efficient.
It consists of :
    1. Keypoint Encoder
    2. A Graph Neural Network with self and cross attention layers
    3. Matching Assignment layers

The correspondence ids use -1 to indicate non-matching points.

Philipp Lindenberger, Paul-Edouard Sarlin and Marc Pollefeys. LightGlue: Local Feature Matching at Light Speed.
In ICCV 2023. https://huggingface.co/papers/2306.13643
r   c           
      &  > [         TU ]  U5        [        R                  " UR                  5      U l        UR                  R                  U l        UR                  U l        UR                  U l
        UR                  U l        UR                  U l        UR                  U l        U R                  U R                  :w  a0  [        R                  " U R                  U R                  SS9U l        O[        R"                  " 5       U l        [%        U5      U l        [        R(                  " [+        UR                  5       Vs/ s H  n[-        XS9PM     sn5      U l        [        R(                  " [+        UR                  5       Vs/ s H  n[1        U5      PM     sn5      U l        [        R(                  " [+        UR                  S-
  5       Vs/ s H  n[5        U5      PM     sn5      U l        U R9                  5         g s  snf s  snf s  snf )NTr   )r   r   )rC   r   r   from_configr&   keypoint_detectordescriptor_decoder_dim keypoint_detector_descriptor_dimr(   r*   
num_layersr1   r.   r/   r   r   input_projectionIdentityr   positional_encoder
ModuleListranger   transformer_layersr
  match_assignment_layersr%  token_confidence	post_init)rE   r   ir@  rG   s       rH   r   %LightGlueForKeypointMatching.__init__  s    !>!J!J6KjKj!k060O0O0f0f-$33 22 & 7 7 & 7 7 & 7 7$"G"GG$&IId.S.SUYUhUhos$tD!$&KKMD!"<V"D"$--EJ6KcKcEdeEd&v;Ede#
 (*}}<A&BZBZ<[\<[q*62<[\(
$ !#<A&BZBZ]^B^<_`<_q*62<_`!
 	 f ] as   HH	Hlayer_indexrv   c                     SS[         R                  " SU-  U R                  -  5      -  -   n[         R                  " USS5      $ )z-scaled confidence threshold for a given layerg?r0   g      r   r   )npr<  r^  clip)rE   rj  ru   s      rH   _get_confidence_threshold6LightGlueForKeypointMatching._get_confidence_threshold  s;    #tk'9DOO'K LLL	wwy!Q''rJ   r   rd   r   Nc                     UR                  5       R                  5       nU R                  U5      nU R                  X#S9nXE4$ )Nr   )r*  r   r_  ra  )rE   r   rd   r   projected_descriptorskeypoint_encoding_outputs         rH   _keypoint_processing1LightGlueForKeypointMatching._keypoint_processing  sI     "((*557 $ 5 5k B#'#:#:9#:#p $>>rJ   keypoint_confidencesrf   
num_pointsc                 l   UR                   u  pVX R                  S-
  :  ap  UR                  US:H  S5      nUR                  US-  S5      nU R	                  U5      nSX:  R                  5       R                  SS9U-  -
  nXR                  :  n	U	$ [        R                  " U[        R                  S9n	U	$ )zRevaluate whether we should stop inference based on the confidence of the keypointsr   r   r   r   g      ?r   r  )r   r^  r  r   rn  rW   sumr.   ri   onesrY   )
rE   rv  rj  rf   rw  r   r@  ru   ratio_confidentearly_stopped_pairss
             rH   _get_early_stopped_image_pairs;LightGlueForKeypointMatching._get_early_stopped_image_pairs$  s     


1,, $8#C#CDAIq#Q #7#?#?
aQS#T 66{CI!%9%E$L$L$N$R$RWX$R$Y\f$ffO"14I4I"I
 #" #(**Zuzz"J""rJ   c                 v    Ub  X   nX$   nU R                   U   " X5      n[        XPR                  5      u  pgXg4$ rx   )re  rN  r1   )rE   r   rf   rj  early_stopsr  rb   rc   s           rH   _get_keypoint_matching3LightGlueForKeypointMatching._get_keypoint_matching7  sI    "%2K$D--k:;M#:6CXCX#Y ''rJ   confidencesr  c                 \    USU R                   -
  :  nUb  XAU R                  U5      :*  -  nU$ )z#mask points which should be removedr   )r/   rn  )rE   r  r  rj  keeps        rH   _get_pruning_mask.LightGlueForKeypointMatching._get_pruning_mask?  s:    T2223"4#A#A+#NNNDrJ   r8  prune_outputc                   ^ UR                   u  n  n	U R                  U   R                  U5      n
U R                  XjU5      mTR	                  US:H  [
        R                  " S5      5      mU4S jXS   US   TU4 5       u  ppn[        U5       H  nUUUU   4==   S-  ss'   M     S XX4 5       u  ppX4n[        USSS9nUUXU4$ )	zv
For a given layer, prune keypoints based on the confidence of the keypoints and the matchability of the
descriptors.
r   Fc              3   v   >#    U  H(  n[        UT5       VVs/ s H	  u  p#X#   PM     snnv   M*     g s  snnf 7frx   )zip).0r  trf   pruned_keypoints_masks       rH   	<genexpr>JLightGlueForKeypointMatching._do_layer_keypoint_pruning.<locals>.<genexpr>Z  s<      c
c %(0E$FG$FQW$FGc Hs   939r   c              3   6   #    U  H  n[        US S9v   M     g7f)T)batch_firstNr   )r  pruned_tensors     rH   r  r  b  s      S
!j D9!js   Tr   r  padding_value)	r   re  r"  r  r  ri   r  rc  r   )rE   r   rd   rf   r8  r  rv  rj  r   r@  descriptors_matchabilitypruned_descriptorspruned_keypoints_0pruned_keypoints_1pruned_maskpruned_indicesrh  pruned_keypointsr  s                     @rH   _do_layer_keypoint_pruning7LightGlueForKeypointMatching._do_layer_keypoint_pruningF  s    ',,
Aq#'#?#?#L#]#]^i#j  $ 6 67Kgr s 5 A A$!)U\\Z_M` ac
&!ilDY[bcc
_0BQ_ z"AN1--.!3. #S
"4J\!jS
O0B /C%n$VXY!#3^R^^^rJ   c                    ^ [         R                  " T5      m[         R                  " TR                  S   5      nTU   nTU   mS XB4 5       u  pBS XS4 5       u  pSU4S jUUUU4 5       u  pEp#X#XE4$ )Nr   c              3   8   #    U  H  n[        US SS9v   M     g7f)Tr   r  Nr   r  r  s     rH   r  MLightGlueForKeypointMatching._concat_early_stopped_outputs.<locals>.<genexpr>x  s       3
C TDC   c              3   8   #    U  H  n[        US SS9v   M     g7f)Tr   r  Nr   r  s     rH   r  r  |  s       >
N TCNr  c              3   .   >#    U  H
  nUT   v   M     g 7frx   r=   )r  r  early_stops_indicess     rH   r  r    s#      g
 &'s   )ri   r?  r9  r   )rE   r  final_pruned_keypoints_indices!final_pruned_keypoints_iterationsrb   rc   idsorder_indicess    `      rH   _concat_early_stopped_outputs:LightGlueForKeypointMatching._concat_early_stopped_outputsk  s     $kk*=>ll.44Q78+C01-@3
"C3
/>
*N>
:g
 .1	g
c"@ .RYjjrJ   rb   rc   r   c                   ^ UR                   u  mnU4S jXU4 5       u  pnUS S 2S4   nUS S 2S4   nUS S 2S4   nUS S 2S4   n	US S 2S4   n
US S 2S4   n[        R                  " TS-  SU4SUR                  UR                  S9n[        R
                  " TS-  SU4UR                  UR                  S9n[        TS-  5       H  n[        R                  " X   S:H  SX~   R                  SX   R                  SS95      5      XSXn   4'   [        R                  " X   S:H  SXn   R                  SX   R                  SS95      5      XSX~   4'   X   XSXn   4'   X   XSX~   4'   M     X4$ )Nc              3   N   >#    U  H  oR                  TS -  S S5      v   M     g7f)r   r   N)r   )r  r  r   s     rH   r  JLightGlueForKeypointMatching._do_final_keypoint_pruning.<locals>.<genexpr>  s'      -
AdvNN:?Ar22Ads   "%r   r   r   r   rR  )r  )
r   ri   fullr  r  zerosrc  r>  r:  clamp)rE   r8  rb   rc   r   r@  rE  rF  rC  rD  rJ  rK  _matches_matching_scoresrh  r   s                  @rH   _do_final_keypoint_pruning7LightGlueForKeypointMatching._do_final_keypoint_pruning  s     
A-
BITcAd-
)/ 1a4=1a4=1a4=1a4=*1a40*1a40 ::zQ=A2gnndkdqdqr ;;1_a/oNcNc
 zQ'A*/++r!2x{'9'9!X[=N=NST=N=U'V+H8;&' +0++r!2x{'9'9!X[=N=NST=N=U'V+H8;&' 3C2E8;./2B2E8;./ ( ))rJ   rO  rP  r   c           
      p	  ^( U(       a  SOS nU(       a  SOS n	UR                   S   S:X  aQ  UR                   S S n
UR                  U
S[        R                  S9UR	                  U
5      UR	                  U
5      UU	4$ UR
                  nUR                   u  pp[        R                  " UR                  US5      SS9nUR                  US-  US5      nUb  UR                  US-  U5      OS nUR                  US-  XR                  5      n[        R                  " US-  US9n[        XU5      nU R                  X!US	9u  nnUS   nU R                  S:  nU R                  S:  n/ n/ n/ n/ n/ n[        R                  " SXS9R                  US-  S5      n[        R                  " U5      n[!        U R"                  5       GHR  nUR%                  5       nUb  U R'                  UU5      nO$[        R(                  " UUS
   4UR
                  S9nU R*                  U   " UUUUUS9nUu  nnn U(       a  UU-   nU(       a  U	U -   n	U(       Ga  UU R"                  S-
  :  a'  U R,                  U   " U5      n!U R/                  U!UX_S9n"O#[        R(                  " U[        R0                  S9n"[        R2                  " U"5      (       Ga  U"R5                  S5      m(UT(   n#U R7                  X%UT(S9u  n$n%UR9                  [;        U#5      5        UR9                  [;        U$5      5        UR9                  [;        U%5      5        U(       a:  UR9                  [;        UT(   5      5        UR9                  [;        UT(   5      5        UU")    n[=        U(4S jX!S   US   UU4 5       5      u  nn&n'nnU&U'4nU(       a  [=        U(4S jUUW!4 5       5      u  nnn![        R>                  " U"5      (       a    O+U(       d  GM6  U RA                  UUUUUW!U5      u  p!nnnGMU     U(       a9  U(       a2  U RC                  UUUUU5      u  nnnnU RE                  UUUU5      u  nnOEU R7                  X%U R"                  S-
  5      u  nn[        R                  " U5      U R"                  -  nUR                  USU5      nUUUUU	4$ )Nr=   r   r   r   ry  r   r   r  rq  r   )r   r   r   )rw  )r  c              3   0   >#    U  H  nUT)    v   M     g 7frx   r=   r  r  r  s     rH   r  ALightGlueForKeypointMatching._match_image_pair.<locals>.<genexpr>  s       V&dF |,&d   c              3   0   >#    U  H  nUT)    v   M     g 7frx   r=   r  s     rH   r  r     s$      l+ #K<0+r  )#r   r   ri   rV   	new_zerosr  rz  r   r]  r9  rV  rt  r.   r/   expand	ones_likerc  r^  rS  get_extended_attention_maskr{  rd  rf  r~  rY   anyr   r  extendr~   rl   allr  r  r  ))rE   rd   r   rO  rP  rf   r   r   r   r   r   r  r   r@  initial_num_keypointsnum_points_per_pairimage_indicesrs  do_early_stopdo_keypoint_pruningr  rb   rc   r  r  pruned_keypoints_indicespruned_keypoints_iterationsrj  r   extended_attention_masklayer_outputrg   	attentionrv  r}  early_stopped_image_indicesearly_stopped_matchesearly_stopped_matching_scoreskeypoints_0
keypoint_1r  s)                                           @rH   _match_image_pair.LightGlueForKeypointMatching._match_image_pair  sK    #7BD0d??1"OOCR(E""5"EII">##E*##E*!  !!2;///
,#iiZ(D!L%%j1n6KQO	FJFVt||JN,AB\`!))*q.:OQvQvwZ!^FC'	5A	040I0I9M 1J 1
-- -Q/	 --1 #33a7 )+&,.)#(<<3H#X#_#_`jmn`npr#s &+oo6N&O# 1K%**,K*.*J*J4Q\*]'*/**j+b/5R[d[k[k*l'22;?6%9"3L 5A1K	#$5$E! !/)!;1!44+/+@+@+Mk+Z( +/*M*M,k4 +N +'
 +0**Zuzz*R'99011 #6"G"G"JK2?2L/KOKfKf#;K Lg LH)+H (..t4O/PQNN4(=#>?#**40M+NO*6==dC[\gCh>ij9@@FabmFnAop +>?R>R*S'PU V'2aL)A,PTVc&dV QMKj$ "-j 9I*fk l !9 ; 4+l gc02MOc 99011"" 33#!03,# d(@$HcQ 2h 0 22'25# h*,MwXg (,'F'F.%	($G_ (,'B'B;VZVeVehiVi'j$G_050PSWSbSb0b-,M,U,U0-
)
 -
 	
rJ   r.  labelsc                 $   S nUb  [        S5      eUb  UOU R                  R                  nUb  UOU R                  R                  nUR                  S:w  d  UR                  S5      S:w  a  [        S5      eUR                  u  pxpnUR                  US-  XU5      nU R                  U5      nUS S u  ppUR                  USSS5      R                  U5      nUR                  USSU R                  5      R                  U5      nUR                  USS5      nUR                  5       nUS S 2S S 2S S 2S4   U-  US S 2S S 2S S 2S4'   US S 2S S 2S S 2S4   U
-  US S 2S S 2S S 2S4'   U R                  UUU
UUUUS	9u  nnnnn[        UUUUUUUUS
9$ )Nz9LightGlue is not trainable, no labels should be provided.   r   r   zOInput must be a 5D tensor of shape (batch_size, 2, num_channels, height, width)r+   r   r   )rf   r   r   )ra   rb   rc   rd   re   rf   rg   rh   )rL   r   r   r   ndimrS  r   r   r[  tor]  cloner  r_   )rE   r.  r  r   r   rF   ra   r   r@  channelsrO  rP  keypoint_detectionsrd   r   rf   absolute_keypointsrb   rc   re   rg   rh   s                         rH   r   $LightGlueForKeypointMatching.forward]  s    XYY1B1N-TXT_T_TqTq$8$D $++JjJj 	 !\%6%6q%9Q%>noo1=1C1C.
x#++JNHeT"44\B*=bq*A'	k%%j!R;>>|L	!))*aT=b=bcffgst||J2.&__.);Aq!QJ)G%)O1aA:&);Aq!QJ)G&)P1aA:&EIE[E[/!5 F\ F
B%
 /+'!	
 		
rJ   )r.   r(   r1   r_  r[  r]  re  r^  ra  rf  rd  r/   r   rx   )NNN)rO   rP   rQ   rR   rS   r#   r   rV   rW   rn  ri   r   rY   rl   rt  r~  r  r  r  r  r  r  r   r   rj   
LongTensorr_   r   rZ   r[   r\   s   @rH   rX  rX    s    <(S (U ( gl? <<?49LL?X\_cXc?	u||U5<<#=>>	??#$)LL#?B#JO,,#didpdp#	#&(U\\ 5<< ^a fkfrfr #_\\#_ <<#_ ll	#_
 #_ ll#_ $ll#_ #_Jk@#*#* #* 	#*
 ||#* 
u||U\\)	*#*V %))-,0k
<<k
 \\k
 	k

 k
 llT!k
  $;k
 #Tkk
 
u||U\\5<<E	Fk
Z  +/)-,04
''4
   4'4
  $;	4

 #Tk4
 
0	04
  4
rJ   rX  )r-  rX  r#   rq   r   )Icollections.abcr   dataclassesr   numpyrl  ri   huggingface_hub.dataclassesr   r   torch.nn.utils.rnnr   configuration_utilsr
   modeling_flash_attention_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   r   utils.import_utilsr   autor   r   auto.modeling_autor   clip.modeling_clipr   cohere.modeling_coherer   llama.modeling_llamar   r   (superglue.image_processing_pil_supergluer   $superglue.image_processing_supergluer   r   r:   r    
get_loggerrO   loggerr#   r_   rn   rq   r   Moduler   r   r   r   r   r  r
  r%  r-  rW   rl   rN  rV   rV  rX  __all__r=   rJ   rH   <module>r     s7   % !   .  + 3 B F & W W * - > ( 9 J Q i ) 
		H	% 9:AR& AR  ;ARH   7k  7  7F	$A 	X5 X 
:X!; X X "/) /)d7 P>		 P>f-2\\JO,,
\\&BII &R	BII 	   $ELL $U $uU\\[`[g[gMgGh $@5<<  S U\\ , 
i
#; i

i
XrJ   