
    Z jxS                        S r SSKrSSKrSSKJr  SSKJr  SSKJr  SSKJ	r	J
r
Jr  \R                  " \5      rS	r    S*S
\\
   S\S\S-  4S jjr    S*S
\\
   S\S\S-  4S jjr " S S\5      r " S S\	5      r " S S\	5      r " S S\5      r " S S\	5      r " S S\	5      r " S S\	5      r " S S\	5      r " S S \	5      r " S! S"\	5      r " S# S$\	5      rS%SS%S%SS%S%S%S%S&.	r\\\\\\\\\\S'.
r S(S(S(S(S(S)S(S(S(S(S'.
r!g)+zGLUE processors and helpers    N)Enum   )PreTrainedTokenizer)logging   )DataProcessorInputExampleInputFeaturesa  This {0} will be removed from the library soon, preprocessing should be handled with the Hugging Face Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/main/examples/pytorch/text-classification/run_glue.pyexamples	tokenizer
max_lengthc           	      t    [         R                  " [        R                  S5      [        5        [        XX#XES9$ )aJ  
Loads a data file into a list of `InputFeatures`

Args:
    examples: List of `InputExamples` containing the examples.
    tokenizer: Instance of a tokenizer that will tokenize the examples
    max_length: Maximum example length. Defaults to the tokenizer's max_len
    task: GLUE task
    label_list: List of labels. Can be obtained from the processor using the `processor.get_labels()` method
    output_mode: String indicating the output mode. Either `regression` or `classification`

Returns:
    Will return a list of task-specific `InputFeatures` which can be fed to the model.

function)r   task
label_listoutput_mode)warningswarnDEPRECATION_WARNINGformatFutureWarning"_glue_convert_examples_to_features)r   r   r   r   r   r   s         r/root/GenerationalWealth/GenerationalWealth/venv/lib/python3.13/site-packages/transformers/data/processors/glue.py!glue_convert_examples_to_featuresr   #   s2    . MM%,,Z8-H-
*     c                   ^^ Uc  UR                   nUbc  [        U   " 5       nUc+  UR                  5       n[        R	                  SU SU 35        Tc$  [
        U   m[        R	                  ST SU 35        [        U5       VVs0 s H  u  pxX_M	     snnmS[        S[        [        -  S -  4UU4S jjn	U  V
s/ s H
  o" U
5      PM     nn
U" U  V
s/ s H  oR                  U
R                  4PM     sn
USSS	9n/ n[        [        U 5      5       H<  nU Vs0 s H
  oX   U   _M     nn[        S0 UDS
X   0D6nUR                  U5        M>     [        U S S 5       HV  u  pz[        R	                  S5        [        R	                  SU
R                    35        [        R	                  SX    35        MX     U$ s  snnf s  sn
f s  sn
f s  snf )NzUsing label list z
 for task zUsing output mode examplereturnc                    > U R                   c  g TS:X  a  TU R                      $ TS:X  a  [        U R                   5      $ [        T5      e)Nclassification
regression)labelfloatKeyError)r   	label_mapr   s    r   label_from_example>_glue_convert_examples_to_features.<locals>.label_from_exampleV   sJ    == **W]]++L(''{##r   r   T)r   padding
truncationr"      z*** Example ***zguid: z
features:  )model_max_lengthglue_processors
get_labelsloggerinfoglue_output_modes	enumerater	   intr#   text_atext_brangelenr
   appendguid)r   r   r   r   r   r   	processorir"   r&   r   labelsbatch_encodingfeatureskinputsfeaturer%   s        `           @r   r   r   @   s    //
#D)+	"--/JKK+J<z$HI+D1KKK,[MD6JK*3J*?@*?ha*?@I$L $S5[45G $ $ :BBg )FB9ABg..'..	)B	N H3x=!3AB>a^&q))>B:&:	: 	 "  !-
%&fW\\N+,j./ .
 OA A C 	C Cs   F=;G!GGc                       \ rS rSrSrSrSrg)
OutputModew   r    r!   r+   N)__name__
__module____qualname____firstlineno__r    r!   __static_attributes__r+   r   r   rC   rC   w   s    %NJr   rC   c                   P   ^  \ rS rSrSrU 4S jrS rS rS rS r	S r
S	 rS
rU =r$ )MrpcProcessor|   z/Processor for the MRPC data set (GLUE version).c                    > [         TU ]  " U0 UD6  [        R                  " [        R                  S5      [        5        g Nr:   super__init__r   r   r   r   r   selfargskwargs	__class__s      r   rQ   MrpcProcessor.__init__   /    $)&))00=}Mr   c           	          [        US   R                  5       US   R                  5       R                  S5      US   R                  5       R                  S5      [        US   R                  5       5      5      $ See base class.idx	sentence1utf-8	sentence2r"   r	   numpydecodestrrS   tensor_dicts     r   get_example_from_tensor_dict*MrpcProcessor.get_example_from_tensor_dict   n    $$&$**,33G<$**,33G<G$**,-	
 	
r   c                     [         R                  S[        R                  R	                  US5       35        U R                  U R                  [        R                  R	                  US5      5      S5      $ )r[   zLOOKING AT 	train.tsvtrain)r/   r0   ospathjoin_create_examples	_read_tsvrS   data_dirs     r   get_train_examples MrpcProcessor.get_train_examples   sQ    k"'',,x"E!FGH$$T^^BGGLL;4W%XZabbr   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r[   zdev.tsvdevro   rp   rl   rm   rn   rq   s     r   get_dev_examplesMrpcProcessor.get_dev_examples   .    $$T^^BGGLL94U%VX]^^r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r[   ztest.tsvtestrx   rq   s     r   get_test_examplesMrpcProcessor.get_test_examples   .    $$T^^BGGLL:4V%WY_``r   c                 
    SS/$ r[   01r+   rS   s    r   r.   MrpcProcessor.get_labels       Szr   c           
          / n[        U5       HD  u  pEUS:X  a  M  U SU 3nUS   nUS   nUS:X  a  SOUS   n	UR                  [        XgXS95        MF     U$ )5Creates examples for the training, dev and test sets.r   -r      r~   Nr9   r4   r5   r"   r2   r8   r	   
rS   linesset_typer   r;   liner9   r4   r5   r"   s
             r   ro   MrpcProcessor._create_examples   sp     'GAAvZq$D!WF!WF$.DDGEOOLd&^_ ( r   r+   rE   rF   rG   rH   __doc__rQ   rf   rs   ry   r   r.   ro   rI   __classcell__rV   s   @r   rK   rK   |   s2    9N
c
_a r   rK   c                   P   ^  \ rS rSrSrU 4S jrS rS rS rS r	S r
S	 rS
rU =r$ )MnliProcessor   z3Processor for the MultiNLI data set (GLUE version).c                    > [         TU ]  " U0 UD6  [        R                  " [        R                  S5      [        5        g rN   rO   rR   s      r   rQ   MnliProcessor.__init__   rX   r   c           	          [        US   R                  5       US   R                  5       R                  S5      US   R                  5       R                  S5      [        US   R                  5       5      5      $ )r[   r\   premiser^   
hypothesisr"   r`   rd   s     r   rf   *MnliProcessor.get_example_from_tensor_dict   sn    $$&	"((*11':%++-44W=G$**,-	
 	
r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r[   rj   rk   rx   rq   s     r   rs    MnliProcessor.get_train_examples   .    $$T^^BGGLL;4W%XZabbr   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ )r[   zdev_matched.tsvdev_matchedrx   rq   s     r   ry   MnliProcessor.get_dev_examples   s/    $$T^^BGGLLK\4]%^`mnnr   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ )r[   ztest_matched.tsvtest_matchedrx   rq   s     r   r   MnliProcessor.get_test_examples   s/    $$T^^BGGLLK]4^%_aoppr   c                 
    / SQ$ )r[   )contradiction
entailmentneutralr+   r   s    r   r.   MnliProcessor.get_labels   s    99r   c           
          / n[        U5       HW  u  pEUS:X  a  M  U SUS    3nUS   nUS   nUR                  S5      (       a  SOUS   n	UR                  [        XgXS95        MY     U$ )	r   r   r      	   r~   Nr   )r2   
startswithr8   r	   r   s
             r   ro   MnliProcessor._create_examples   s}     'GAAvZqa	*D!WF!WF$//77DT"XEOOLd&^_ ( r   r+   r   r   s   @r   r   r      s2    =N
coq: r   r   c                   8   ^  \ rS rSrSrU 4S jrS rS rSrU =r	$ )MnliMismatchedProcessor   z>Processor for the MultiNLI Mismatched data set (GLUE version).c                    > [         TU ]  " U0 UD6  [        R                  " [        R                  S5      [        5        g rN   rO   rR   s      r   rQ    MnliMismatchedProcessor.__init__   rX   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ )r[   zdev_mismatched.tsvdev_mismatchedrx   rq   s     r   ry   (MnliMismatchedProcessor.get_dev_examples   s/    $$T^^BGGLLK_4`%acsttr   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ )r[   ztest_mismatched.tsvtest_mismatchedrx   rq   s     r   r   )MnliMismatchedProcessor.get_test_examples   s/    $$T^^BGGLLK`4a%bduvvr   r+   )
rE   rF   rG   rH   r   rQ   ry   r   rI   r   r   s   @r   r   r      s    HNuw wr   r   c                   P   ^  \ rS rSrSrU 4S jrS rS rS rS r	S r
S	 rS
rU =r$ )ColaProcessor   z/Processor for the CoLA data set (GLUE version).c                    > [         TU ]  " U0 UD6  [        R                  " [        R                  S5      [        5        g rN   rO   rR   s      r   rQ   ColaProcessor.__init__   rX   r   c           	          [        US   R                  5       US   R                  5       R                  S5      S[        US   R                  5       5      5      $ r[   r\   sentencer^   Nr"   r`   rd   s     r   rf   *ColaProcessor.get_example_from_tensor_dict   U    $$&
#))+227;G$**,-	
 	
r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r   rx   rq   s     r   rs    ColaProcessor.get_train_examples   r   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ rv   rx   rq   s     r   ry   ColaProcessor.get_dev_examples   r{   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r}   rx   rq   s     r   r   ColaProcessor.get_test_examples  r   r   c                 
    SS/$ r   r+   r   s    r   r.   ColaProcessor.get_labels  r   r   c           
          US:H  nU(       a  USS nU(       a  SOSn/ n[        U5       H8  u  pgU SU 3nXt   n	U(       a  SOUS   n
UR                  [        XSU
S95        M:     U$ )r   r~   r   Nr   r   r   r   )rS   r   r   	test_mode
text_indexr   r;   r   r9   r4   r"   s              r   ro   ColaProcessor._create_examples	  s{    &	!"IE#Q
 'GAZq$D%F%D47EOOLd$V[\]	 (
 r   r+   r   r   s   @r   r   r      s2    9N
c_a r   r   c                   P   ^  \ rS rSrSrU 4S jrS rS rS rS r	S r
S	 rS
rU =r$ )Sst2Processori  z0Processor for the SST-2 data set (GLUE version).c                    > [         TU ]  " U0 UD6  [        R                  " [        R                  S5      [        5        g rN   rO   rR   s      r   rQ   Sst2Processor.__init__  rX   r   c           	          [        US   R                  5       US   R                  5       R                  S5      S[        US   R                  5       5      5      $ r   r`   rd   s     r   rf   *Sst2Processor.get_example_from_tensor_dict  r   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r   rx   rq   s     r   rs    Sst2Processor.get_train_examples(  r   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ rv   rx   rq   s     r   ry   Sst2Processor.get_dev_examples,  r{   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r}   rx   rq   s     r   r   Sst2Processor.get_test_examples0  r   r   c                 
    SS/$ r   r+   r   s    r   r.   Sst2Processor.get_labels4  r   r   c           
          / nUS:X  a  SOSn[        U5       H?  u  pVUS:X  a  M  U SU 3nXd   nUS:X  a  SOUS   n	UR                  [        XxSU	S95        MA     U$ )r   r~   r   r   r   Nr   r   )
rS   r   r   r   r   r;   r   r9   r4   r"   s
             r   ro   Sst2Processor._create_examples8  sw    "f,Q!
 'GAAvZq$D%F$.DDGEOOLd$V[\] ( r   r+   r   r   s   @r   r   r     s2    :N
c_a r   r   c                   P   ^  \ rS rSrSrU 4S jrS rS rS rS r	S r
S	 rS
rU =r$ )StsbProcessoriF  z0Processor for the STS-B data set (GLUE version).c                    > [         TU ]  " U0 UD6  [        R                  " [        R                  S5      [        5        g rN   rO   rR   s      r   rQ   StsbProcessor.__init__I  rX   r   c           	          [        US   R                  5       US   R                  5       R                  S5      US   R                  5       R                  S5      [        US   R                  5       5      5      $ rZ   r`   rd   s     r   rf   *StsbProcessor.get_example_from_tensor_dictM  rh   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r   rx   rq   s     r   rs    StsbProcessor.get_train_examplesV  r   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ rv   rx   rq   s     r   ry   StsbProcessor.get_dev_examplesZ  r{   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r}   rx   rq   s     r   r   StsbProcessor.get_test_examples^  r   r   c                     S/$ )r[   Nr+   r   s    r   r.   StsbProcessor.get_labelsb  s	    vr   c           
          / n[        U5       HG  u  pEUS:X  a  M  U SUS    3nUS   nUS   nUS:X  a  SOUS   n	UR                  [        XgXS95        MI     U$ )	r   r   r      r   r~   Nr   r   r   r   s
             r   ro   StsbProcessor._create_examplesf  t     'GAAvZqa	*D!WF!WF$.DDHEOOLd&^_ ( r   r+   r   r   s   @r   r   r   F  s2    :N
c_a r   r   c                   P   ^  \ rS rSrSrU 4S jrS rS rS rS r	S r
S	 rS
rU =r$ )QqpProcessorit  z.Processor for the QQP data set (GLUE version).c                    > [         TU ]  " U0 UD6  [        R                  " [        R                  S5      [        5        g rN   rO   rR   s      r   rQ   QqpProcessor.__init__w  rX   r   c           	          [        US   R                  5       US   R                  5       R                  S5      US   R                  5       R                  S5      [        US   R                  5       5      5      $ )r[   r\   	question1r^   	question2r"   r`   rd   s     r   rf   )QqpProcessor.get_example_from_tensor_dict{  rh   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r   rx   rq   s     r   rs   QqpProcessor.get_train_examples  r   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ rv   rx   rq   s     r   ry   QqpProcessor.get_dev_examples  r{   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r}   rx   rq   s     r   r   QqpProcessor.get_test_examples  r   r   c                 
    SS/$ r   r+   r   s    r   r.   QqpProcessor.get_labels  r   r   c           
         US:H  nU(       a  SOSnU(       a  SOSn/ n[        U5       HG  u  pxUS:X  a  M  U SUS    3n	 X   n
X   nU(       a  SOUS	   nUR                  [        XXS
95        MI     U$ ! [         a     MZ  f = f)r   r~   r   r      r   r   r   Nr*   r   )r2   
IndexErrorr8   r	   )rS   r   r   r   q1_indexq2_indexr   r;   r   r9   r4   r5   r"   s                r   ro   QqpProcessor._create_examples  s    &	!1q!1q 'GAAvZqa	*D )tAw OOLd&^_ (   s   A66
BBr+   r   r   s   @r   r   r   t  s2    8N
c_a r   r   c                   P   ^  \ rS rSrSrU 4S jrS rS rS rS r	S r
S	 rS
rU =r$ )QnliProcessori  z/Processor for the QNLI data set (GLUE version).c                    > [         TU ]  " U0 UD6  [        R                  " [        R                  S5      [        5        g rN   rO   rR   s      r   rQ   QnliProcessor.__init__  rX   r   c           	          [        US   R                  5       US   R                  5       R                  S5      US   R                  5       R                  S5      [        US   R                  5       5      5      $ )r[   r\   questionr^   r   r"   r`   rd   s     r   rf   *QnliProcessor.get_example_from_tensor_dict  sn    $$&
#))+227;
#))+227;G$**,-	
 	
r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r   rx   rq   s     r   rs    QnliProcessor.get_train_examples  r   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ rv   rx   rq   s     r   ry   QnliProcessor.get_dev_examples  r{   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r}   rx   rq   s     r   r   QnliProcessor.get_test_examples  r   r   c                 
    SS/$ r[   r   not_entailmentr+   r   s    r   r.   QnliProcessor.get_labels      .//r   c           
          / n[        U5       HG  u  pEUS:X  a  M  U SUS    3nUS   nUS   nUS:X  a  SOUS   n	UR                  [        XgXS95        MI     U$ 	r   r   r   r   r
  r~   Nr   r   r   r   s
             r   ro   QnliProcessor._create_examples  r   r   r+   r   r   s   @r   r  r    s2    9N
c_a0 r   r  c                   P   ^  \ rS rSrSrU 4S jrS rS rS rS r	S r
S	 rS
rU =r$ )RteProcessori  z.Processor for the RTE data set (GLUE version).c                    > [         TU ]  " U0 UD6  [        R                  " [        R                  S5      [        5        g rN   rO   rR   s      r   rQ   RteProcessor.__init__  rX   r   c           	          [        US   R                  5       US   R                  5       R                  S5      US   R                  5       R                  S5      [        US   R                  5       5      5      $ rZ   r`   rd   s     r   rf   )RteProcessor.get_example_from_tensor_dict  rh   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r   rx   rq   s     r   rs   RteProcessor.get_train_examples  r   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ rv   rx   rq   s     r   ry   RteProcessor.get_dev_examples  r{   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r}   rx   rq   s     r   r   RteProcessor.get_test_examples  r   r   c                 
    SS/$ r  r+   r   s    r   r.   RteProcessor.get_labels  r   r   c           
          / n[        U5       HG  u  pEUS:X  a  M  U SUS    3nUS   nUS   nUS:X  a  SOUS   n	UR                  [        XgXS95        MI     U$ r"  r   r   s
             r   ro   RteProcessor._create_examples  r   r   r+   r   r   s   @r   r%  r%    s2    8N
c_a0 r   r%  c                   P   ^  \ rS rSrSrU 4S jrS rS rS rS r	S r
S	 rS
rU =r$ )WnliProcessori  z/Processor for the WNLI data set (GLUE version).c                    > [         TU ]  " U0 UD6  [        R                  " [        R                  S5      [        5        g rN   rO   rR   s      r   rQ   WnliProcessor.__init__  rX   r   c           	          [        US   R                  5       US   R                  5       R                  S5      US   R                  5       R                  S5      [        US   R                  5       5      5      $ rZ   r`   rd   s     r   rf   *WnliProcessor.get_example_from_tensor_dict  rh   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r   rx   rq   s     r   rs    WnliProcessor.get_train_examples  r   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ rv   rx   rq   s     r   ry   WnliProcessor.get_dev_examples  r{   r   c                     U R                  U R                  [        R                  R	                  US5      5      S5      $ r}   rx   rq   s     r   r   WnliProcessor.get_test_examples  r   r   c                 
    SS/$ r   r+   r   s    r   r.   WnliProcessor.get_labels   r   r   c           
          / n[        U5       HG  u  pEUS:X  a  M  U SUS    3nUS   nUS   nUS:X  a  SOUS   n	UR                  [        XgXS95        MI     U$ r"  r   r   s
             r   ro   WnliProcessor._create_examples$  r   r   r+   r   r   s   @r   r5  r5    s2    9N
c_a r   r5  r
  )	colamnlimrpcsst-2sts-bqqpqnlirtewnli)
rD  rE  zmnli-mmrF  rG  rH  rI  rJ  rK  rL  r    r!   )NNNN)"r   rl   r   enumr   tokenization_pythonr   utilsr   r   r	   r
   
get_loggerrE   r/   r   listr3   r   r   rC   rK   r   r   r   r   r   r   r  r%  r5  glue_tasks_num_labelsr-   r1   r+   r   r   <module>rS     s   " 	   6  = = 
		H	%m  "	< " d
@ "	4< 4"4 d
4n 
,M ,^+M +\wm w ,M ,^+M +\+M +\1= 1h+M +\+= +\+M +^ 
  &  r   