
    fd                     $   U d Z ddlmZ ddlZddlZddlZddlmZ ddl	m
Z
mZmZmZmZmZmZmZ ddlmZ ddlmZmZmZ dZej2                  Zej4                  Zej6                  Zej8                  Zej:                  Zej<                  Zej>                  Zej@                  Z ejB                  Z!ejD                  Z"ejF                  Z#ejH                  Z$ejJ                  Z% G d	 d
e
      Z& ejN                  d      Z(i Z)eee&f   e*d<   dddZ+d Z,ddddZ-d Z.d Z/dZ0 e.de0z   dz         Z1 e.de0z   dz         Z2 e.d      Z3 e.d      Z4d Z5 G d de
      Z6 G d de6      Z7 G d  d!      Z8d" Z9d# Z:d$d%d&e;d'ed(ee<e<f   d)ee7   fd*Z=d+ Z>dd$d,d-d.ee;   d'ed/ee<   d(ee<e<f   d)ee7   f
d0Z?d1 Z@eAd2k(  rNej                  d3   ZC eDeC      5 ZEeEj                         ZGddd        e=eG ed4      5      D ]
  ZH eIeH        yy# 1 sw Y   (xY w)6a  
This tokenizer has been copied from the ``tokenize.py`` standard library
tokenizer. The reason was simple: The standard library tokenizer fails
if the indentation is not right. To make it possible to do error recovery the
    tokenizer needed to be rewritten.

Basically this is a stripped down version of the standard library module, so
you can read the documentation there. Additionally we included some speed and
memory optimizations here.
    )absolute_importN)BOM_UTF8)
NamedTupleTupleIteratorIterableListDictPatternSet)PythonTokenTypes)split_linesPythonVersionInfoparse_version_stringu   􏿿c                   z    e Zd ZU eed<   ee   ed<   ee   ed<   eeef   ed<   eed<   eeef   ed<   ee   ed<   y)	TokenCollectionpseudo_tokensingle_quotedtriple_quotedendpats
whitespacefstring_pattern_mapalways_break_tokensN)	__name__
__module____qualname__r   __annotations__r   strr
   r        P/var/www/cvtools/html/venv/lib/python3.12/site-packages/parso/python/tokenize.pyr   r   +   sI    s8s8#w,c3h's#r    r   zutf-8_token_collection_cacheFcapturec                 J    |rJ d}| s|dz  }|dj                  |      z   dz   S )N(z?:|))join)r$   choiceskwargsstarts       r!   groupr-   :   s4    :E388G$$s**r    c                      t        |  dz   S )N?)r-   )r*   s    r!   mayber0   C   s    '?S  r    include_fstringonly_fstringc                     d }g d}dh}| rddg}|r|}t               }n||z  }n|r
t               S |D ]3  }t        j                  |      D ]  }|j                   ||              5 |S )Nc              3      K   t        j                  | D cg c]  }||j                         f c} D ]  }dj                  |        y c c}w w)N )
_itertoolsproductupperr)   )prefixcss      r!   different_case_versionsz5_all_string_prefixes.<locals>.different_case_versionsI   sF     ##f%Eq!'')n%EF 	A''!*	%Es   AA"A)brubrr6   ffr)setr7   permutationsupdate)r2   r3   r=   valid_string_prefixesresultrB   r:   ts           r!   _all_string_prefixesrJ   H   s     2TF$K$%!UF!Q&!	u ( 6((0 	6A MM1!45	66
 Mr    c                 J    t        j                  | t         j                        S N)recompileUNICODE)exprs    r!   _compilerQ   e   s    ::dBJJ''r    c                     	 t         t        |          S # t        $ r" t        |       xt         t        |       <   }|cY S w xY wrL   )r"   tupleKeyError_create_token_collection)version_inforH   s     r!   _get_token_collectionrW   i   sG    &u\':;; $\2	3l 34vs    (??z"[A-Za-z0-9\-]+(?: [A-Za-z0-9\-]+)*z(?:\{\{|\}\}|\\N\{z*\}|\\(?:\r\n?|\n)|\\[^\r\nN]|[^{}\r\n\\])+z\}|\\[^N]|[^{}\\])+z(?:\\(?:\r\n?|\n)|[^{}\r\n])+z[^{}]+c                    d}t        |      }d}dt        z   dz   }d}d}d}d}t        ||||      }	d	}
t        d
d      t        |
      z   }d|
z   }t        ||      }t        d|dz         }t        |||	      }t	               }t        | }t        t	        d       }t	        dd      }t        | }d}d}d}d}t        |dz   |dz         }t        ddddddd      }d}g d }| d!k\  r|j                  d"d#       n|j                  d"d$       t        | }t        |||      }t        |d%z   t        d&d'      z   |d(z   t        d)d'      z         }||g} d*}!|r| j                  |t        |! z          t        d+g|  }"t        |d,      t        |"||||d,      z   }#i }$|D ]F  }%t        |      |$|%d&z   <   t        |      |$|%d)z   <   t        |      |$|%dz   <   t        |      |$|%dz   <   H t               }&t               }'i }(|D ]8  })d-D ]  }*|&j                  |)|*z           d.D ]  }*|'j                  |)|*z           : |D ]  })|!D ]
  }*|*|(|)|*z   <     d/}+t        |#      },t        |,|&|'|$||(t        |+            S )0Nz[ \f\t]*z	#[^\r\n]*u   ([A-Za-z_0-9-z]+)z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z(?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]T)r2   r1   z(?:\\.|[^'\\])*'z(?:\\.|[^"\\])*"z(?:\\.|'(?!'')|[^'\\])*'''z(?:\\.|"(?!"")|[^"\\])*"""'''"""z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@`|^!=<>]=?~z[][(){}])z\.\.\.z\r\n?z\nz[;.,@])      r   z:=?:z'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*'z\\(?:\r\n?|\n)z"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*")r`   r_   rZ   rY   z\\(?:\r\n?|\n)|\Zr#   )r`   r_   )rZ   rY   );importclassdeftryexceptfinallywhilewithreturncontinuebreakdelpassglobalassertnonlocal)
rQ   MAX_UNICODEr-   r0   rJ   insertappendrD   addr   )-rV   
Whitespacer   CommentName	Hexnumber	Binnumber	Octnumber	Decnumber	IntnumberExponent
PointfloatExpfloatFloatnumber
ImagnumberNumberpossible_prefixesStringPrefixStringPrefixWithFfstring_prefixesFStringStartSingleDoubleSingle3Double3TripleOperatorBracketspecial_argsSpecialFunnyContStrpseudo_extra_pool
all_quotesPseudoExtrasPseudoTokenr   _prefixr   r   r   rI   quoteALWAYS_BREAK_TOKENSpseudo_token_compileds-                                                r!   rU   rU   ~   sL    J*%JG ;.6D*I#I$I2IiIyAI,HC-/16xAJ#h.H
H-K/w1FGJ:{I6F -.+,L3DIJ+DtT*+L !F F+G+G$u,.?%.GHF
 Ye+H
 G:LvAu%As#\"G(GW-E L#EEC!234 #EEC!2345G !&)'J  z0B!BC-B0ABL
D1lFE7D$GHK G$ 5!)&!1#!)&!1##+G#4% #+G#4% 	5 EMEM ) 	)Ea%i(	) " 	)Ea%i(	)	)  3 	3E-2E	*	33S %[1}mW'-@)A r    c                   `    e Zd ZU eed<   eed<   eeef   ed<   eed<   edeeef   fd       Z	y)Tokentypestring	start_posr:   rj   c                     t        | j                        }t        |      dkD  r | j                  d   t        |      z   dz
  dfS | j                  d   | j                  d   t        | j                        z   fS N   r   )r   r   lenr   )selfliness     r!   end_poszToken.end_pos   sh    DKK(u:>>>!$s5z1A5q88>>!$dnnQ&7#dkk:J&JJJr    N)
r   r   r   r   r   r   r   intpropertyr   r   r    r!   r   r      sD    
KS#XKKsCx K Kr    r   c                       e Zd Zd Zy)PythonTokenc                 T    d| j                  | j                  j                        z  S )Nz6TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r))r   )_replacer   namer   s    r!   __repr__zPythonToken.__repr__   s%    H499>>23 	4r    N)r   r   r   r   r   r    r!   r   r      s    4r    r   c                   0    e Zd Zd Zd Zd Zd Zd Zd Zy)FStringNodec                 J    || _         d| _        d| _        d | _        d| _        y )Nr   r6   )r   parentheses_countprevious_lineslast_string_start_posformat_spec_count)r   r   s     r!   __init__zFStringNode.__init__   s+    
!" %)" "#r    c                 .    | xj                   dz  c_         y )Nr   )r   r   	characters     r!   open_parentheseszFStringNode.open_parentheses	  s    !#r    c                 \    | xj                   dz  c_         | j                   dk(  rd| _        y y r   r   r   r   s     r!   close_parentheseszFStringNode.close_parentheses  s.    !#!!Q&%&D" 'r    c                 2    t        | j                        dk(  S )Nr\   )r   r   r   s    r!   allow_multilinezFStringNode.allow_multiline  s    4::!##r    c                 4    | j                   | j                  kD  S rL   r   r   s    r!   
is_in_exprzFStringNode.is_in_expr  s    %%(>(>>>r    c                 @    | j                          xr | j                  S rL   )r   r   r   s    r!   is_in_format_speczFStringNode.is_in_format_spec  s    ??$$?)?)??r    N)	r   r   r   r   r   r   r   r   r   r   r    r!   r   r      s!    #$'$?@r    r   c           
      j   t        |       D ]  \  }}|j                         }t        |      t        |      z
  }|j                  |j                        sIt        t        |j                  |||z   f||d | z         }	d}|j                  rJ | |d = |	dt        |j                        |z   fc S  d |dfS )Nr:   r6   r   )	enumeratelstripr   
startswithr   r   FSTRING_ENDr   )
fstring_stackr   line_nrcolumnadditional_prefixfstring_stack_indexnodelstripped_string
len_lstriptokens
             r!   _close_fstring_if_necessaryr     s    %.}%= ;!T!==?[3'7#88
&&tzz2

&:-.()<<	E !#****123"c$**o
:::; "A%%r    c                 z   |d   }|j                         }|j                         r|rt        }nt        }n|rt        }nt
        }|j                  ||      }||j                  |fS |j                  s	||f|_        |j                  d      }	|D ]K  }
| |
j                     j                  |	      }|$|j                  d      d t        |
j                          }	M |}|t        |	      z  }|	j                  d      s|	j                  d      r|xj                  |	z  c_        d}	|	|fS |j                  |	z   }	|	|fS )Nr   
r6   )r   r   fstring_format_spec_multi_linefstring_format_spec_single_linefstring_string_multi_linefstring_string_single_linematchr   r   r-   r   r   endswith)r   r   linelnumpostosr   regexr   r   fstring_stack_node	end_matchnew_poss                r!   _find_fstring_stringr   .  sM   

C))+O
2E3E-E.EKKc"E}!!3&&%)3K![[^F+ I.445;;FC	 __Q'(G#.@.F.F*G)GHFI
 Gs6{G t 5f$ 7? ##f,7?r    )r   r   )r   coderV   r   rj   c                8    t        | d      }t        |||      S )z0Generate tokens from a the source code (string).T)keepends)rV   r   )r   tokenize_lines)r   rV   r   r   s       r!   tokenizer   V  s     t,E%liPPr    c                       fd}|S )zL
    A small helper function to help debug the tokenize_lines function.
    c               ?   F   K    | i |D ]  }t        |       |  y wrL   )print)argsr+   r   funcs      r!   wrapperz_print_tokens.<locals>.wrapperb  s,     4*6* 	E%LK	s   !r   )r   r   s   ` r!   _print_tokensr   ^  s    
 Nr    T)indentsr   is_first_tokenr   r   c          
   #     ./K   ./fd}t        |      \  }}}}	}
}}d}dgd}d}d}d}d}d}|d   dz
  .g }| D ]m  }.dz  .d}t        |      }|rE|j                  t              rt        }|dd }t        |      }d|d   z  |z   }|d   }||d   z  }d	}|rNj	                  |      }|r0|j                  d      }t        t        ||d| z   |       d}d}n||z   }|z   }||k  s|r{|d
   }|j                         sBt        |	||.|      \  }}|r't        t        ||j                  d       d|_        X||k(  r||d }t        ||.||      \  }}}||z  }|| |r|} |D ]s  }!|!j                  }"|	|"   j	                  ||      }#|#'|#j                  d      }$t        |$      t        |"      z
  |z   t        |       k  s_|d| |$dt        |"        z   } u |j	                  | |      }%n|j	                  ||      }%|%rR||%j                  d      z   }d}|%j!                  d      \  }&}.|&f/|%j                  d      }'|'dk(  r|sJ |}|'d   }(n9|
j	                  ||      })||)j                            }(|)j                         }&.|&f/|rS|(dvrO|(dk7  s|%Hd	}|dk(  rA|s?|&}*|*d
   kD  r%t        t"        d/d       j%                  |*        ||*      E d{    |%s|
j	                  ||      })|r%|dk(  r |s ||)j                               E d{    |)j                         }d	}t        t&        ||   .|f||)j                  d      z          d}|dz  }|(|v s|(dk(  r 'dk7  r|'dk7  rt        t(        '/|       n
|%j                  d      '|v rD|s|r@g |dd d}t+        j                  d|d|&       }+|+ ||+j                               E d{    |'j-                         rt        t.        |'/|       nt1        |'/|      E d{    nt|(dv rIt3        d |D              r|j5                          |s|dk(  r|st        t6        '/|       n|'z   }d}n'|(dk(  r['j9                  d      s|'j9                  d      rJ |r.|d
   j                         rt        t&        |(/|       |&dz   }n||'z   }n'|v rP|	|'   }|j	                  ||      }|r,|j                  d      }||&| }'t        t        |'/|       n~/}||&d }|}|(|v s|'dd |v s|'dd |v rf|'d
   dv rJ.|&f}|	j;                  |(      xs* |	j;                  |'d         xs |	j;                  |'d         }||&d }|}Wt        t        |'/|       n|'|v r2|j%                  t=        ||'                t        t>        |'/|       n|(dk(  r||&d dv r||||&d z   z  }|'dv r|r|d
   jA                  |'       n}|dz  }nw|'dv r|r|d
   jC                  |'       n\|rZ|dz  }nT|'j                  d      rC|rA|d
   jD                  |d
   jF                  z
  dk(  r|d
   xjF                  dz  c_#        d}'|&dz   }t        tH        |'/|       ||k  rp |r8t        t&        ||       |j9                  d      s|j9                  d      rd}|r:|d
   }|j                  r)t        t        |j                  |j                  d       .|f},dd D ]&  }-jK                          t        tL        d|,d       ( t        tN        d|,|       y7 7 H7 7 Kw)a)  
    A heavily modified Python standard library tokenizer.

    Additionally to the default information, yields also the prefix of each
    token. This idea comes from lib2to3. The prefix contains all information
    that is irrelevant for the parser like newlines in parentheses or comments.
    c              3      K   | d   k  rR| d   kD  rt        t        d| fd       | d<   y j                          t        t        dd       | d   k  rQy y w)Nr   r6   )r   ERROR_DEDENTpopDEDENT)r,   r   r   sposs    r!   dedent_if_necessaryz+tokenize_lines.<locals>.dedent_if_necessaryz  sh     gbk!wr{"!,T5M2FF#KKMfb$33 gbk!s   AAAr   N
0123456789r6   Tr   ^Fr   r      z
#\.z...r\   z	[ \f\t]*$z
c              3   >   K   | ]  }|j                            y wrL   )r   ).0rB   s     r!   	<genexpr>z!tokenize_lines.<locals>.<genexpr>  s     F11,,..Fs   #r   r   )z\
z\
z\z([{z)]}r^   )(rW   r   r   BOM_UTF8_STRINGr   endr   STRINGr   r   FSTRING_STRINGr   r   r   r   r-   spanINDENTrt   
ERRORTOKENNUMBERrM   isidentifierNAME_split_illegal_unicode_nameanyclearNEWLINEr   getr   FSTRING_STARTr   r   r   r   OPr   r   	ENDMARKER)0r   rV   r   r   r   r   r   r   r   r   r   r   r   paren_levelmax_numcharscontstrnew_liner:   r   r   r   r   endprogendmatchcontstr_startcontliner   r   restfstring_end_tokenquote_lengthstring_liner   r   r   end_match_stringpseudomatchr,   r   initialr   indent_startmr   indentr   r   s0     `                                           @@r!   r   r   k  s    4 	l+2L-0K#DHG HFQ<!D')M U;	4y/$3!ABx4y 1%,DA,CIaL D"N}}T*Hll1o!Gd4Cj0!6+ + !D.#d?Dj#B'~~'"6wtUY[^"_KFC)*F55 $&  .0* d{CDzE`!%FB!#4l |#$0++ "*7 V&.44E ' 4 4T3 ?I ,+4??1+=(/03u:=Cc+FVV*.t*7G#e*7U*UKV +00cB*00s;*[->->q-AA$&!(--a0
se}#))!,B;!M6(.% ("((s3uyy{+		e}G724;K^ !#M#(L#gbk1)&"dB??|42<@@@"((s3q 02599;???iik !S	D#;%A6  %'!q8#35C<EUN!&%v>>""1%1//]k')M!$"#KtFU|<A}#6quuw#???%%'%dE4@@:5$OOOF"FFF!'')K1$4]%gudFCC(.%C >>$/t8LLL ]2%6%A%A%C%j'4HH!)C(.%-'!%."==s3",,q/C sOE%feT6BB$(M"56lG#H M)"1I."1I.9& %)%KM&{{73  8w{{587L  8")++eAh"7 "56lG#H%feT6BB--$$[1DU1K%LM!-fEEDT%&\5O%O!Vd56l%::!E>$%b)::5A#q(e^$%b);;EB&'1,K%%c*})"-??'+==>ABC ""%771<7E!)C!"eT6::m Dj?U;n *g}fEED!W%5%5d%;HB 2 2))   DjG !"+ 3&"gr223 iW.?
@@O A
 @* @ Pse   C,]92B)]97]9D]9-]-.:]9(]0)C]90]31:]9+]6,I?]9-C]90]93]96]9c              #   D  	K   	fd}dd|	t        |       D ]p  \  }}r3|j                         r |        |dd|d   |d   |z   f	5|z  ;|z   }|j                         r|Sr |        d|d   |d   |z   f	|dr r
 |        y y w)Nc                  8    t        rt        nt               S rL   )r   r  r  )found
is_illegalr   r:   s   r!   create_tokenz1_split_illegal_unicode_name.<locals>.create_token  s    :uc6RRr    r6   Fr   r   T)r   r  )
r   r   r:   r1  ichar	new_foundr/  r0  r   s
     `    @@@r!   r  r    s     S EJ
CU# "4  ""n$"
lIaL1$44I%%'!&.(F#A,	!q(88C!
)", n s   BB __main__r   z3.10)rV   )J__doc__
__future__r   sysrM   	itertoolsr7   codecsr   typingr   r   r   r   r	   r
   r   r   parso.python.tokenr   parso.utilsr   r   r   rr   r
  r  r  r  r  r  r   r  r  r   r  r  r   r   decoder  r"   r   r-   r0   rJ   rQ   rW   unicode_character_namer   r   r   r   rU   r   r   r   r   r   r   r   r   r   r   r  r   argvpathopenrB   readr   r   r   r   r    r!   <module>rD     s  	 ' 
 	     0 L L 		 	 		 	 

"
"		 	 		 	 &&	((
,, ..!00**$j $ "(//'*DF /@A F " +!
 -2 :( ? %2234  %225KK  #++K"L !))!4 iXKJ K4% 4@ @:&$%R QWQ
Q 1Q>CCHoQkQ	" !'ZAC=ZA $ZA #Y	ZA
 S#XZA kZAzB z88A;D	d qvvx $-A&-IJ e  s   HH