+
    Bi                    l   ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	H
t
Ht ^ RIHt ^ RIHtHt ^ RIHt ^ RIHt ^ RIHtHtHt ^ RIHtHt R	 t ! R
 R]4      t ! R R]4      tR t ! R R]4      t ! R R]4      t  ! R R]4      t! ! R R]4      t" ! R R]4      t# ! R R]4      t$R t% ! R R]4      t& ! R R]4      t' ! R  R!]4      t( ! R" R#]P                  4      t) ! R$ R%]P                  4      t* ! R& R']P                  4      t+],R(8X  d   ]PZ                  ! 4        R# R# ))    N)BytesIOStringIO)dedent)TestCasemock)support)	os_helper)run_test_scriptmake_scriptrun_python_until_end)VALID_UNDERSCORE_LITERALSINVALID_UNDERSCORE_LITERALSc                 f   . p\        VP                  4       4      pVR,          R9  pV  F  w  rVrxp	V\        P                  8X  d    V# V'       d&   V\        P                  8X  d   V^ ,          V8X  d   KL  \        P
                  V,          pVP                  RVR RV:R RV RV 24       K  	  V# )   z
    10 13)len
splitlinestokenize	ENDMARKERNEWLINEtok_nameappend)
token_generatorsource_stringresult	num_linesmissing_trailing_nltypetokenstartendlines
   &&        )/usr/lib/python3.14/test/test_tokenize.pystringify_tokens_from_sourcer(      s    FM,,./I'+69)8%U8%%% M 48+;+;#;A)@S  &T"IQulrl!E7!C5AB *9 M    c                      a  ] tR t^*t o R tR tR tR tR tR t	R t
R tR	 tR
 tR tR tR tR tR tR tR tR tR tR tR tR tR tR tR tRtV tR# )TokenizeTestc                    \        VP                  R 4      4      p\        \        P                  ! VP                  4      V4      pV P                  VR.VP                  4       P                  4       ,           4       R# )utf-8z*    ENCODING   'utf-8'       (0, 0) (0, 0)N)r   encoder(   r   readlineassertEqualrstripr   selfsexpectedfr   s   &&&  r'   check_tokenizeTokenizeTest.check_tokenize1   s`     AHHW%&-h.?.?

.KQOFG!*5578	9r)   c                   R  pV P                  \        4      ;_uu_ 4        \        \        P                  ! V! 4       P                  4      4       RRR4       R pV P                  \        4      ;_uu_ 4        \        \        P
                  ! V! 4       P                  4      4       RRR4       R pV P                  \        4      ;_uu_ 4        \        \        P
                  ! V! 4       P                  4      4       RRR4       R#   + '       g   i     L; i  + '       g   i     L|; i  + '       g   i     R# ; i)c               3      "   R x  R x  R# 5isdfosdgN r=   r)   r'   gen/TokenizeTest.test_invalid_readline.<locals>.gen;   s     OO   Nc               3      "   R x  R x  R# 5i)s   sdfosdgNr=   r=   r)   r'   r>   r?   A   s     r@   c               3   (   "   R x  ^^ ,           R# 5ir;   r=   r=   r)   r'   r>   r?   G   s     OaCCs   )assertRaises	TypeErrorlistr   __next__generate_tokensZeroDivisionError)r3   r>   s   & r'   test_invalid_readline"TokenizeTest.test_invalid_readline:   s    	 y))""35>>23 *	 y))))#%..9: *	 011))#%..9: 21 *) *) 211s#   /D8/D/D.D	D+	.D?	c                P   \        R P                  R4      4      p\        \        P                  ! VP                  4      4      pV P                  VR,          P                  \        P                  4       V P                  VR,          P                  \        P                  4       R# )xr-   Nr   )	r   r.   rE   r   r/   r0   r"   r   r   )r3   r6   tokenss   &  r'   test_implicit_newline"TokenizeTest.test_implicit_newlineM   sl     CJJw'(h''

34(*:*:;(*<*<=r)   c                   V P                  R R4       V P                  RR4       V P                  RR4       V P                  RR4       Rp\        V4      P                  pV P                  \        R	4      ;_uu_ 4       p\
        P
                  ! V4       F  pK  	  R
R
R
4       V P                  XP                  P                  ^4       V P                  VP                  P                  R4       V P                  VP                  P                  R
4       V P                  VP                  P                  R
4       V P                  VP                  P                  R	4       V P                  VP                  P                  ^	4       V P                  VP                  P                  R4       R
#   + '       g   i     EL; i)z1 + 1z    NUMBER     '1'           (1, 0) (1, 1)
    OP         '+'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    z3if False:
    # NL
    
    True = False # NEWLINE
af      NAME       'if'          (1, 0) (1, 2)
    NAME       'False'       (1, 3) (1, 8)
    OP         ':'           (1, 8) (1, 9)
    NEWLINE    '\n'          (1, 9) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NL         '\n'          (3, 4) (3, 5)
    INDENT     '    '        (4, 0) (4, 4)
    NAME       'True'        (4, 4) (4, 8)
    OP         '='           (4, 9) (4, 10)
    NAME       'False'       (4, 11) (4, 16)
    COMMENT    '# NEWLINE'   (4, 17) (4, 26)
    NEWLINE    '\n'          (4, 26) (4, 27)
    DEDENT     ''            (5, 0) (5, 0)
    z%if True:
    # NL
    foo='bar'

a@      NAME       'if'          (1, 0) (1, 2)
    NAME       'True'        (1, 3) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\r\n'        (1, 8) (1, 10)
    COMMENT    '# NL'        (2, 4) (2, 8)
    NL         '\r\n'        (2, 8) (2, 10)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'foo'         (3, 4) (3, 7)
    OP         '='           (3, 7) (3, 8)
    STRING     "'bar'"       (3, 8) (3, 13)
    NEWLINE    '\r\n'        (3, 13) (3, 15)
    NL         '\r\n'        (4, 0) (4, 2)
    DEDENT     ''            (5, 0) (5, 0)
            zx = 1 + \
1
a      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (2, 0) (2, 1)
    NEWLINE    '\r\n'        (2, 1) (2, 3)
            s   def k(x):
    x += 2
  x += 5
z3unindent does not match any outer indentation levelN<string>z  x += 5)r7   r   r/   assertRaisesRegexIndentationErrorr   r0   	exceptionlinenofilename
end_lineno
end_offsetmsgoffsettext)r3   indent_error_filer/   etoks   &    r'   
test_basicTokenizeTest.test_basicU   s   G & 		
 	 ;=		( 	K N 	  	1 4 	
 ,-66##$4%>? ?BC((2 3?
 	++Q/--z://6//6KKOOA	C 	++Q/)):6? ? ?s   >F00G	c                J   V P                  R R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR	4       V P                  R
R4       V P                  RR4       V P                  RR4       V P                  RR4       R# )0xff <= 255z    NUMBER     '0xff'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    0b10 <= 255z    NUMBER     '0b10'        (1, 0) (1, 4)
    OP         '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    0o123 <= 0O123z    NUMBER     '0o123'       (1, 0) (1, 5)
    OP         '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    1234567 > ~0x15z    NUMBER     '1234567'     (1, 0) (1, 7)
    OP         '>'           (1, 8) (1, 9)
    OP         '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    2134568 != 1231515z    NUMBER     '2134568'     (1, 0) (1, 7)
    OP         '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    (-124561-1) & 200000000ac      OP         '('           (1, 0) (1, 1)
    OP         '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    OP         '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         ')'           (1, 10) (1, 11)
    OP         '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    0xdeadbeef != -1z    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    OP         '!='          (1, 11) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    0xdeadc0de & 12345z    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    OP         '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    0xFF & 0x15 | 1234z    NUMBER     '0xFF'        (1, 0) (1, 4)
    OP         '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    OP         '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    Nr7   r3   s   &r'   test_intTokenizeTest.test_int   s    M , 		
 	M , 		
 	, / 		
 	- 0 		 	0 3 		
 	5 	8 			 	. 1 		 	0 3 		
 	0 3 		r)   c                    V P                  R R4       V P                  RR4       V P                  RR4       V P                  RR4       R# )	zx = 0z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0'           (1, 4) (1, 5)
    zx = 0xfffffffffffz    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '0xfffffffffff' (1, 4) (1, 17)
    zx = 123141242151251616110z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '123141242151251616110' (1, 4) (1, 25)
    zx = -15921590215012591z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '-'           (1, 4) (1, 5)
    NUMBER     '15921590215012591' (1, 5) (1, 22)
    Nrl   rm   s   &r'   	test_longTokenizeTest.test_long   s_    G & 		
 	/ 2 		
 	7 : 		
 	4 7 		r)   c                   V P                  R R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR	4       V P                  R
R4       V P                  RR4       R# )x = 3.14159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    x = 314159.z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    x = .314159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    x = 3e14159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    	x = 3E123z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    x+y = 3e-1230z    NAME       'x'           (1, 0) (1, 1)
    OP         '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    OP         '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    x = 3.14e159z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    Nrl   rm   s   &r'   
test_floatTokenizeTest.test_float   s    M , 		
 	M , 		
 	M , 		
 	M , 		
 	K * 		
 	O . 		 	N - 		r)   c                   R  p\          F$  pRV9   d   K  V P                  V! V4      V4       K&  	  RR0p\         F,  pW#9   d   K   V! V4       T P                  T! T4      T4       K.  	  R#   \        P                   d     KJ  i ; i)c                     \        V P                  R 4      4      p\        P                  ! VP                  4       F   w  r#rEpV\        P                  8X  g   K  Vu # 	  R# )r-   zinvalid token)r   r.   r   r/   NUMBER)r4   r6   toktyper#   r$   r%   r&   s   &      r'   number_token;TokenizeTest.test_underscore_literals.<locals>.number_token  sL    )*A4<4E4Eajj4Q0Dhoo- L 5R #r)   (0_709_99N)r   r0   r   r   
TokenErrorassertNotEqual)r3   r   litextra_valid_casess   &   r'   test_underscore_literals%TokenizeTest.test_underscore_literals  s    	# -Ccz\#.4	 - #G,.C'S! S 137 /
 && s   A..BBc                   V P                  R R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR	4       V P                  R
R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  R R!4       V P                  R"R#4       V P                  R$R%4       V P                  R&R'4       V P                  R(R)4       V P                  R*R+4       V P                  R,R-4       V P                  R.R/4       V P                  R0R14       V P                  R2R34       V P                  R4R54       V P                  R6R74       V P                  R8R94       V P                  R:R;4       V P                  R<R=4       V P                  R>R?4       V P                  R@RA4       V P                  RBRC4       V P                  RDRE4       V P                  RFRG4       RH# )Ix = ''; y = ""a5      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    OP         ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    x = '"'; y = "'"a6      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    OP         ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    OP         '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    x = "doesn't "shrink", does it"z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    x = 'abc' + 'ABC'z    NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    y = "ABC" + "ABC"z    NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    %x = r'abc' + r'ABC' + R'ABC' + R'ABC'a      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    %y = r"abc" + r"ABC" + R"ABC" + R"ABC"a      NAME       'y'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    OP         '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    OP         '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    OP         '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    u'abc' + U'abc'z    STRING     "u'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    u"abc" + U"abc"z    STRING     'u"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    b'abc' + B'abc'z    STRING     "b'abc'"      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    b"abc" + B"abc"z    STRING     'b"abc"'      (1, 0) (1, 6)
    OP         '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    %br'abc' + bR'abc' + Br'abc' + BR'abc'a;      STRING     "br'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    %br"abc" + bR"abc" + Br"abc" + BR"abc"a;      STRING     'br"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    %rb'abc' + rB'abc' + Rb'abc' + RB'abc'a;      STRING     "rb'abc'"     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    %rb"abc" + rB"abc" + Rb"abc" + RB"abc"a;      STRING     'rb"abc"'     (1, 0) (1, 7)
    OP         '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    OP         '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    OP         '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    "a\
de\
fg"3    STRING     '"a\\\nde\\\nfg"' (1, 0) (3, 3)
    u"a\
de"/    STRING     'u"a\\\nde"'  (1, 0) (2, 3)
    rb"a\
d"/    STRING     'rb"a\\\nd"'  (1, 0) (2, 2)
    
"""a\
b"""0    STRING     '"""a\\\nb"""' (1, 0) (2, 4)
    u"""a\
b"""1    STRING     'u"""a\\\nb"""' (1, 0) (2, 4)
    rb"""a\
b\
c"""7    STRING     'rb"""a\\\nb\\\nc"""' (1, 0) (3, 4)
    f"abc"    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'abc'         (1, 2) (1, 5)
    FSTRING_END '"'           (1, 5) (1, 6)
    	fR"a{b}c"a=      FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a'           (1, 3) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    NAME       'b'           (1, 5) (1, 6)
    OP         '}'           (1, 6) (1, 7)
    FSTRING_MIDDLE 'c'           (1, 7) (1, 8)
    FSTRING_END '"'           (1, 8) (1, 9)
    zfR"a{{{b!r}}}c"a      FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a{'          (1, 3) (1, 5)
    OP         '{'           (1, 6) (1, 7)
    NAME       'b'           (1, 7) (1, 8)
    OP         '!'           (1, 8) (1, 9)
    NAME       'r'           (1, 9) (1, 10)
    OP         '}'           (1, 10) (1, 11)
    FSTRING_MIDDLE '}'           (1, 11) (1, 12)
    FSTRING_MIDDLE 'c'           (1, 13) (1, 14)
    FSTRING_END '"'           (1, 14) (1, 15)
    zf"{{{1+1}}}"a      FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE '{'           (1, 2) (1, 3)
    OP         '{'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '}'           (1, 8) (1, 9)
    FSTRING_MIDDLE '}'           (1, 9) (1, 10)
    FSTRING_END '"'           (1, 11) (1, 12)
    zf"""{f'''{f'{f"{1+1}"}'}'''}"""ad      FSTRING_START 'f"""'        (1, 0) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    FSTRING_START "f'''"        (1, 5) (1, 9)
    OP         '{'           (1, 9) (1, 10)
    FSTRING_START "f'"          (1, 10) (1, 12)
    OP         '{'           (1, 12) (1, 13)
    FSTRING_START 'f"'          (1, 13) (1, 15)
    OP         '{'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '+'           (1, 17) (1, 18)
    NUMBER     '1'           (1, 18) (1, 19)
    OP         '}'           (1, 19) (1, 20)
    FSTRING_END '"'           (1, 20) (1, 21)
    OP         '}'           (1, 21) (1, 22)
    FSTRING_END "'"           (1, 22) (1, 23)
    OP         '}'           (1, 23) (1, 24)
    FSTRING_END "'''"         (1, 24) (1, 27)
    OP         '}'           (1, 27) (1, 28)
    FSTRING_END '"""'         (1, 28) (1, 31)
    z.f"""     x
str(data, encoding={invalid!r})
"""a      FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE '     x\nstr(data, encoding=' (1, 4) (2, 19)
    OP         '{'           (2, 19) (2, 20)
    NAME       'invalid'     (2, 20) (2, 27)
    OP         '!'           (2, 27) (2, 28)
    NAME       'r'           (2, 28) (2, 29)
    OP         '}'           (2, 29) (2, 30)
    FSTRING_MIDDLE ')\n'         (2, 30) (3, 0)
    FSTRING_END '"""'         (3, 0) (3, 3)
    z#f"""123456789
something{None}bad"""aO      FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE '123456789\nsomething' (1, 4) (2, 9)
    OP         '{'           (2, 9) (2, 10)
    NAME       'None'        (2, 10) (2, 14)
    OP         '}'           (2, 14) (2, 15)
    FSTRING_MIDDLE 'bad'         (2, 15) (2, 18)
    FSTRING_END '"""'         (2, 18) (2, 21)
    
f"""abc"""    FSTRING_START 'f"""'        (1, 0) (1, 4)
    FSTRING_MIDDLE 'abc'         (1, 4) (1, 7)
    FSTRING_END '"""'         (1, 7) (1, 10)
    f"abc\
def"    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'abc\\\ndef'  (1, 2) (2, 3)
    FSTRING_END '"'           (2, 3) (2, 4)
    Rf"abc\
def"    FSTRING_START 'Rf"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'abc\\\ndef'  (1, 3) (2, 3)
    FSTRING_END '"'           (2, 3) (2, 4)
    z5f'some words {a+b:.3f} more words {c+d=} final words'aA      FSTRING_START "f'"          (1, 0) (1, 2)
    FSTRING_MIDDLE 'some words ' (1, 2) (1, 13)
    OP         '{'           (1, 13) (1, 14)
    NAME       'a'           (1, 14) (1, 15)
    OP         '+'           (1, 15) (1, 16)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    FSTRING_MIDDLE '.3f'         (1, 18) (1, 21)
    OP         '}'           (1, 21) (1, 22)
    FSTRING_MIDDLE ' more words ' (1, 22) (1, 34)
    OP         '{'           (1, 34) (1, 35)
    NAME       'c'           (1, 35) (1, 36)
    OP         '+'           (1, 36) (1, 37)
    NAME       'd'           (1, 37) (1, 38)
    OP         '='           (1, 38) (1, 39)
    OP         '}'           (1, 39) (1, 40)
    FSTRING_MIDDLE ' final words' (1, 40) (1, 52)
    FSTRING_END "'"           (1, 52) (1, 53)
    zf'''{
3
=}'''a`      FSTRING_START "f'''"        (1, 0) (1, 4)
    OP         '{'           (1, 4) (1, 5)
    NL         '\n'          (1, 5) (1, 6)
    NUMBER     '3'           (2, 0) (2, 1)
    NL         '\n'          (2, 1) (2, 2)
    OP         '='           (3, 0) (3, 1)
    OP         '}'           (3, 1) (3, 2)
    FSTRING_END "'''"         (3, 2) (3, 5)
    f'''__{
    x:a
}__'''a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    OP         '{'           (1, 6) (1, 7)
    NL         '\n'          (1, 7) (1, 8)
    NAME       'x'           (2, 4) (2, 5)
    OP         ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n'         (2, 6) (3, 0)
    OP         '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'''"         (3, 3) (3, 6)
    +f'''__{
    x:a
    b
     c
      d
}__'''a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    OP         '{'           (1, 6) (1, 7)
    NL         '\n'          (1, 7) (1, 8)
    NAME       'x'           (2, 4) (2, 5)
    OP         ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n    b\n     c\n      d\n' (2, 6) (6, 0)
    OP         '}'           (6, 0) (6, 1)
    FSTRING_MIDDLE '__'          (6, 1) (6, 3)
    FSTRING_END "'''"         (6, 3) (6, 6)
    uz       '''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli
    aktualni pracownicy, obecni pracownicy'''
u      INDENT     '    '        (1, 0) (1, 4)
    STRING     "'''Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli\n    aktualni pracownicy, obecni pracownicy'''" (1, 4) (2, 45)
    NEWLINE    '\n'          (2, 45) (2, 46)
    DEDENT     ''            (3, 0) (3, 0)
    Nrl   rm   s   &r'   test_stringTokenizeTest.test_string.  s   . 1 		 	1 4 		 	A D 		 	/ 2 		 	/ 2 		 	C 
F 
		 	C 
F 
		 	- 0 		
 	- 0 		 	- 0 		
 	- 0 		
 	C F 		 	C F 		 	C F 		 	C F 		 	 		
 	 		 	 		 	 		 	 		 	 		
 	H ' 		
 	K * 		 	- 0 		 	N 
- 
		 	E H 		* 	N 
Q 
		 	B E 		 	L + 		
 	 		 	 		 	S V 		( 	 				 	 
		 	 
		& 	 		r)   c                r    V P                  R R4       V P                  RR4       V P                  RR4       R# )!def d22(a, b, c=2, d=2, *k): passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    OP         ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    OP         '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    OP         '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    OP         ','           (1, 22) (1, 23)
    OP         '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    OP         ')'           (1, 26) (1, 27)
    OP         ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    def d01v_(a=1, *k, **w): passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    OP         '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         ','           (1, 13) (1, 14)
    OP         '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    OP         ','           (1, 17) (1, 18)
    OP         '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    OP         ')'           (1, 22) (1, 23)
    OP         ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    &def d23(a: str, b: int=3) -> int: passa      NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    OP         '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    OP         ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    OP         ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    OP         '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    OP         ')'           (1, 24) (1, 25)
    OP         '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    OP         ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    Nrl   rm   s   &r'   test_functionTokenizeTest.test_functionn  sJ    ? B 		, 	; > 		" 	D G 		r)   c                *    V P                  R R4       R# )Xif 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != 1 and 5 in 1 not in 1 is 1 or 5 is not 1: passa      NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    OP         '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    OP         '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    OP         '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    OP         '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    OP         '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    OP         ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    Nrl   rm   s   &r'   test_comparisonTokenizeTest.test_comparison       M!O"		r)   c                *    V P                  R R4       R# )zx = 1 << 1 >> 5a6      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '<<'          (1, 6) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '>>'          (1, 11) (1, 13)
    NUMBER     '5'           (1, 14) (1, 15)
    Nrl   rm   s   &r'   
test_shiftTokenizeTest.test_shift  s    - 0 		r)   c                *    V P                  R R4       R# )%x = 1 - y + 15 - 1 + 0x124 + z + a[5]a$      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    OP         '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    OP         '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    OP         '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    OP         '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    OP         '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    OP         ']'           (1, 36) (1, 37)
    Nrl   rm   s   &r'   test_additiveTokenizeTest.test_additive      C F 		r)   c                *    V P                  R R4       R# )x = 1//1*1/5*12%0x12@42a      NAME       'x'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    OP         '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    OP         '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    OP         '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    OP         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    Nrl   rm   s   &r'   test_multiplicative TokenizeTest.test_multiplicative      5 8 		r)   c                N    V P                  R R4       V P                  RR4       R# )~1 ^ 1 & 1 |1 ^ -1a      OP         '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '^'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    -1*1/1+1*1//1 - ---1**1aH      OP         '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    OP         '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    OP         '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    OP         '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    OP         '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    OP         '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    OP         '-'           (1, 14) (1, 15)
    OP         '-'           (1, 16) (1, 17)
    OP         '-'           (1, 17) (1, 18)
    OP         '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    OP         '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    Nrl   rm   s   &r'   
test_unaryTokenizeTest.test_unary  s0    0 3 		 	5 8 		r)   c                *    V P                  R R4       R# )/import sys, time
x = sys.modules['time'].time()a      NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    OP         ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'x'           (2, 0) (2, 1)
    OP         '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    OP         '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    OP         '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    OP         ']'           (2, 22) (2, 23)
    OP         '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    OP         '('           (2, 28) (2, 29)
    OP         ')'           (2, 29) (2, 30)
    Nrl   rm   s   &r'   test_selectorTokenizeTest.test_selector,      N Q 		r)   c                *    V P                  R R4       R# ) @staticmethod
def foo(x,y): passa      OP         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    '\n'          (1, 13) (1, 14)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    OP         '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    OP         ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    OP         ')'           (2, 11) (2, 12)
    OP         ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    Nrl   rm   s   &r'   test_methodTokenizeTest.test_methodB      ? B 		r)   c                *    V P                  R R4       R# )zdef f():
	if x
        	passa_      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    INDENT     '\t'          (2, 0) (2, 1)
    NAME       'if'          (2, 1) (2, 3)
    NAME       'x'           (2, 4) (2, 5)
    NEWLINE    '\n'          (2, 5) (2, 6)
    INDENT     '        \t'  (3, 0) (3, 9)
    NAME       'pass'        (3, 9) (3, 13)
    DEDENT     ''            (4, 0) (4, 0)
    DEDENT     ''            (4, 0) (4, 0)
    Nrl   rm   s   &r'   	test_tabsTokenizeTest.test_tabsS  s     -/		r)   c                *    V P                  R R4       R# )u!   Örter = 'places'
grün = 'green'u7      NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "'places'"    (1, 8) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "'green'"     (2, 7) (2, 14)
    Nrl   rm   s   &r'   test_non_ascii_identifiers'TokenizeTest.test_non_ascii_identifiersh  s    @ C 		r)   c                *    V P                  R R4       R# )#   Örter = u'places'
grün = U'green'u7      NAME       'Örter'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    '\n'          (1, 17) (1, 18)
    NAME       'grün'        (2, 0) (2, 4)
    OP         '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    Nrl   rm   s   &r'   test_unicodeTokenizeTest.test_unicodet      B E 		r)   c                   V P                  R R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR	4       V P                  R
R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  R R!4       V P                  R"R#4       V P                  R$R%4       V P                  R&R'4       V P                  R(R)4       R*# )+	async = 1z    NAME       'async'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    a = (async = 1)a8      NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    OP         '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    OP         '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    async()z    NAME       'async'       (1, 0) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    class async(Bar):passa<      NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    OP         ')'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    class async:passz    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    OP         ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    	await = 1z    NAME       'await'       (1, 0) (1, 5)
    OP         '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    	foo.asyncz    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    async for a in b: passa;      NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    OP         ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    async with a as b: passa<      NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    OP         ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    	async.fooz    NAME       'async'       (1, 0) (1, 5)
    OP         '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    async/    NAME       'async'       (1, 0) (1, 5)
    async
#comment
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    COMMENT    '#comment'    (2, 0) (2, 8)
    NL         '\n'          (2, 8) (2, 9)
    NAME       'await'       (3, 0) (3, 5)
    async
...
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    OP         '...'         (2, 0) (2, 3)
    NEWLINE    '\n'          (2, 3) (2, 4)
    NAME       'await'       (3, 0) (3, 5)
    async
awaitz    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    '\n'          (1, 5) (1, 6)
    NAME       'await'       (2, 0) (2, 5)
    foo.async + 1z    NAME       'foo'         (1, 0) (1, 3)
    OP         '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    OP         '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    async def foo(): passa;      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    Nasync def foo():
  def foo(await):
    await = 1
  if 1:
    await
async += 1
a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    OP         '('           (2, 9) (2, 10)
    NAME       'await'       (2, 10) (2, 15)
    OP         ')'           (2, 15) (2, 16)
    OP         ':'           (2, 16) (2, 17)
    NEWLINE    '\n'          (2, 17) (2, 18)
    INDENT     '    '        (3, 0) (3, 4)
    NAME       'await'       (3, 4) (3, 9)
    OP         '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    '\n'          (3, 13) (3, 14)
    DEDENT     ''            (4, 2) (4, 2)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    OP         ':'           (4, 6) (4, 7)
    NEWLINE    '\n'          (4, 7) (4, 8)
    INDENT     '    '        (5, 0) (5, 4)
    NAME       'await'       (5, 4) (5, 9)
    NEWLINE    '\n'          (5, 9) (5, 10)
    DEDENT     ''            (6, 0) (6, 0)
    DEDENT     ''            (6, 0) (6, 0)
    NAME       'async'       (6, 0) (6, 5)
    OP         '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    '\n'          (6, 10) (6, 11)
    )async def foo():
  async for i in 1: passa      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    OP         ')'           (1, 14) (1, 15)
    OP         ':'           (1, 15) (1, 16)
    NEWLINE    '\n'          (1, 16) (1, 17)
    INDENT     '  '          (2, 0) (2, 2)
    NAME       'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    OP         ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (3, 0) (3, 0)
    async def foo(async): awaitah      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    OP         '('           (1, 13) (1, 14)
    NAME       'async'       (1, 14) (1, 19)
    OP         ')'           (1, 19) (1, 20)
    OP         ':'           (1, 20) (1, 21)
    NAME       'await'       (1, 22) (1, 27)
    @def f():

  def baz(): pass
  async def bar(): pass

  await = 2a      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    OP         '('           (1, 5) (1, 6)
    OP         ')'           (1, 6) (1, 7)
    OP         ':'           (1, 7) (1, 8)
    NEWLINE    '\n'          (1, 8) (1, 9)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    Fasync def f():

  def baz(): pass
  async def bar(): pass

  await = 2a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    OP         '('           (1, 11) (1, 12)
    OP         ')'           (1, 12) (1, 13)
    OP         ':'           (1, 13) (1, 14)
    NEWLINE    '\n'          (1, 14) (1, 15)
    NL         '\n'          (2, 0) (2, 1)
    INDENT     '  '          (3, 0) (3, 2)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    OP         '('           (3, 9) (3, 10)
    OP         ')'           (3, 10) (3, 11)
    OP         ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    '\n'          (3, 17) (3, 18)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    OP         '('           (4, 15) (4, 16)
    OP         ')'           (4, 16) (4, 17)
    OP         ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    '\n'          (4, 23) (4, 24)
    NL         '\n'          (5, 0) (5, 1)
    NAME       'await'       (6, 2) (6, 7)
    OP         '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (7, 0) (7, 0)
    Nrl   rm   s   &r'   
test_asyncTokenizeTest.test_async  s   K * 		 	- 0 		 	I ( 		 	3 6 		 	. 1 		 	K * 		 	K * 		 	4 7 		 	5 8 		 	K * 		 	G & 		 	4 7 		 	/ 2 		 	N - 		 	O . 		 	3 6 		 	 #*		X 	 		* 	= 	@ 			 	 #		J 	 $		r)   c                *    V P                  R R4       R# )z[
    # A comment here
    1
]
ad      OP         '['           (1, 0) (1, 1)
    NL         '\n'          (1, 1) (1, 2)
    COMMENT    '# A comment here' (2, 4) (2, 20)
    NL         '\n'          (2, 20) (2, 21)
    NUMBER     '1'           (3, 4) (3, 5)
    NL         '\n'          (3, 5) (3, 6)
    OP         ']'           (4, 0) (4, 1)
    NEWLINE    '\n'          (4, 1) (4, 2)
    Nrl   rm   s   &r'   3test_newline_after_parenthesized_block_with_comment@TokenizeTest.test_newline_after_parenthesized_block_with_comment  s     
			r)   c                *    V P                  R R4       R# )z); xz    OP         ')'           (1, 0) (1, 1)
    OP         ';'           (1, 1) (1, 2)
    NAME       'x'           (1, 3) (1, 4)
    Nrl   rm   s   &r'   ,test_closing_parenthesis_from_different_line9TokenizeTest.test_closing_parenthesis_from_different_line  s    F % 		r)   c                *    V P                  R R4       R# )uT   a = f'''
    Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli'''u%      NAME       'a'           (1, 0) (1, 1)
    OP         '='           (1, 2) (1, 3)
    FSTRING_START "f'''"        (1, 4) (1, 8)
    FSTRING_MIDDLE '\n    Autorzy, którzy tą jednostkę mają wpisani jako AKTUALNA -- czyli' (1, 8) (2, 68)
    FSTRING_END "'''"         (2, 68) (2, 71)
    Nrl   rm   s   &r'    test_multiline_non_ascii_fstring-TokenizeTest.test_multiline_non_ascii_fstring  s     OQ		r)   c                *    V P                  R R4       R# )u4   f'''
    🔗 This is a test {test_arg1}🔗
🔗'''uZ      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '\n    🔗 This is a test ' (1, 4) (2, 21)
    OP         '{'           (2, 21) (2, 22)
    NAME       'test_arg1'   (2, 22) (2, 31)
    OP         '}'           (2, 31) (2, 32)
    FSTRING_MIDDLE '🔗\n🔗'        (2, 32) (3, 1)
    FSTRING_END "'''"         (3, 1) (3, 4)
    Nrl   rm   s   &r'   *test_multiline_non_ascii_fstring_with_expr7TokenizeTest.test_multiline_non_ascii_fstring_with_expr  s     		r)   r=   N)__name__
__module____qualname____firstlineno__r7   rI   rO   r`   rn   rq   r{   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r
  __static_attributes____classdictcell____classdict__s   @r'   r+   r+   *   s     9;&>F7P7	r	0&	P80~	@
:	x$	L
		.	(#	J	,	"	*
	
	P	d	"				 	r)   r+   c                   &   a  ] tR tRt o R tRtV tR# )GenerateTokensTesti  c                    \        V4      p\        \        P                  ! VP                  4      V4      pV P                  WBP                  4       P                  4       4       R # N)r   r(   r   rG   r/   r0   r1   r   r2   s   &&&  r'   r7   !GenerateTokensTest.check_tokenize  sG     QK-h.F.Fqzz.RTUV!2!=!=!?@r)   r=   N)r  r  r  r  r7   r  r  r  s   @r'   r  r    s     A Ar)   r  c           	         . p\         P                   ! \        V P                  R 4      4      P                  4      pV F  w  r4  pV\         P                  8X  dh   RV9   da   VP                  \         P                  R3\         P                  R3\         P                  \        V4      3\         P                  R3.4       K  VP                  W434       K  	  \         P                  ! V4      P                  R 4      P                  4       # )r-   .Decimalr   ))r   r   r.   r/   r   extendNAMEOPSTRINGreprr   
untokenizedecodestrip)r4   r   gtoknumtokval_s   &     r'   decistmtr)    s    F'!((7"34==>A$%1aX__$MM	*c"$v,/c"	  MM6*+ %& v&--g6<<>>r)   c                   ,   a  ] tR tRt o R tR tRtV tR# )TestMisci  c                    ^ RI Hp RpV P                  \        V4      R4       V P	                  \        \        V4      4      R4       V P                  \        \        V4      4      V! R4      4       R# )r   )r  z+21.3e-5*-.1234/81.7z8+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')z-3.2171603427[0-9]*e-0+7z!-3.217160342717258261933904529E-7N)decimalr  r0   r)  assertRegexr!  eval)r3   r  r4   s   &  r'   test_decistmtTestMisc.test_decistmt  sd     	$"!S	U 	d1g(BC 	hqk* !DE	Gr)   c                x    \         P                  . RO,           pV P                  \        P                  V4       R# )	TokenInfoN)r3  r   rG   detect_encodingr"  openr   )r#   __all__assertCountEqualr   )r3   r5   s   & r'   test___all__TestMisc.test___all__  s-    == $
 
 	h..9r)   r=   N)r  r  r  r  r0  r8  r  r  r  s   @r'   r+  r+    s     G(: :r)   r+  c                   H   a  ] tR tRt o RtR tR tR tR tR t	R t
R	tV tR
# )TestTokenizerAdheresToPep0263i  zM
Test that tokenizer adheres to the coding behaviour stipulated in PEP 0263.
c                   \         P                  P                  \         P                  P                  \        4      R V4      p\        VR4      ;_uu_ 4       p\        P                  W4       RRR4       R#   + '       g   i     R# ; i)tokenizedatarbN)ospathjoindirname__file__r5  TestRoundtripcheck_roundtrip)r3   rW   r@  r6   s   &&  r'   	_testFile'TestTokenizerAdheresToPep0263._testFile  sQ    ww||BGGOOH5~xP$))$2 s   A77B	c                ,    R pV P                  V4       R# )z9tokenize_tests-utf8-coding-cookie-and-no-utf8-bom-sig.txtNrF  r3   r6   s   & r'   'test_utf8_coding_cookie_and_no_utf8_bomETestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_no_utf8_bom      Gqr)   c                L    RpV P                  \        V P                  V4       R# )aV  
As per PEP 0263, if a file starts with a utf-8 BOM signature, the only
allowed encoding for the comment is 'utf-8'.  The text file used in
this test starts with a BOM signature, but specifies latin1 as the
coding, so verify that a SyntaxError is raised, which matches the
behaviour of the interpreter when it encounters a similar condition.
z8tokenize_tests-latin1-coding-cookie-and-utf8-bom-sig.txtNrC   SyntaxErrorrF  rJ  s   & r'   &test_latin1_coding_cookie_and_utf8_bomDTestTokenizerAdheresToPep0263.test_latin1_coding_cookie_and_utf8_bom  s      G+t~~q9r)   c                ,    R pV P                  V4       R# )z9tokenize_tests-no-coding-cookie-and-utf8-bom-sig-only.txtNrI  rJ  s   & r'   "test_no_coding_cookie_and_utf8_bom@TestTokenizerAdheresToPep0263.test_no_coding_cookie_and_utf8_bom  rM  r)   c                ,    R pV P                  V4       R# )z6tokenize_tests-utf8-coding-cookie-and-utf8-bom-sig.txtNrI  rJ  s   & r'   $test_utf8_coding_cookie_and_utf8_bomBTestTokenizerAdheresToPep0263.test_utf8_coding_cookie_and_utf8_bom  s    Dqr)   c                    V P                  \        V P                  R 4       V P                  \        V P                  R4       R# )zbad_coding.pyzbad_coding2.pyNrO  rm   s   &r'   test_bad_coding_cookie4TestTokenizerAdheresToPep0263.test_bad_coding_cookie  s/    +t~~G+t~~7GHr)   r=   N)r  r  r  r  __doc__rF  rK  rQ  rT  rW  rZ  r  r  r  s   @r'   r;  r;    s/     3
	:I Ir)   r;  c                   &   a  ] tR tRt o R tRtV tR# )Test_Tokenizei  c                  aa R pVP                  R4      oRoVV3R lp\        \        P                  ! V! 4       P                  RRR7      4      RR p\        P
                  ! ^R R	R
R 4      .pV P                  W4R4       R# )u   "ЉЊЈЁЂ"r-   Fc               3   8   <"   S '       g	   R o Sx  R# Rx  R# 5i)Tr)   Nr=   )firstr&   s   r'   r/   NTest_Tokenize.test__tokenize_decodes_with_specified_encoding.<locals>.readline%  s     
	s   T)encodingextra_tokensNzbytes not decoded with encodingrM   r   r   r      )r.   rE   r   !_generate_tokens_from_c_tokenizerrF   r3  r0   )r3   literalr/   rN   expected_tokensra  r&   s   &    @@r'   .test__tokenize_decodes_with_specified_encoding<Test_Tokenize.test__tokenize_decodes_with_specified_encoding!  s     ~~g&	 h@@ATATJQNRT UUXVXZ $--aQ_`a:	<r)   r=   N)r  r  r  r  rk  r  r  r  s   @r'   r^  r^    s     < <r)   r^  c                      a  ] tR tRt o R tR tR tR tR tR t	R t
R	 tR
 tR tR tR tR tR tR tR tR t]P,                  R 4       tR tR tRtV tR# )TestDetectEncodingi6  c                   aa ^ oVV3R lpV# )r   c                  X   < S\        S4      8X  d   \        hSS,          p S^,          oV # r   )r   StopIteration)r&   indexliness    r'   r/   1TestDetectEncoding.get_readline.<locals>.readline:  s,    E
"##<DQJEKr)   r=   )r3   rt  r/   rs  s   &f @r'   get_readlineTestDetectEncoding.get_readline8  s    	 r)   c                    Rp\         P                  ! V P                  V4      4      w  r#V P                  VR4       V P                  V\	        VR,          4      4       R# )   # something
r-   N   NN)ry     print(something)
   do_something(else)
)r   r4  rv  r0   rE   r3   rt  rc  consumed_liness   &   r'   test_no_bom_no_encoding_cookie1TestDetectEncoding.test_no_bom_no_encoding_cookieC  sQ    

 $,#;#;D<M<Me<T#U 7+eBi9r)   c                    Rp\         P                  ! V P                  V4      4      w  r#V P                  VR4       V P                  VRR.4       R# )   ﻿# something
r|  	utf-8-sigry  N)r  r|  r}  r   r4  rv  r0   r~  s   &   r'   test_bom_no_cookie%TestDetectEncoding.test_bom_no_cookieM  sS    

 $,#;#;D<M<Me<T#U ;/*,AB	Dr)   c                    Rp\         P                  ! V P                  V4      4      w  r#V P                  VR4       V P                  VR .4       R# )   # -*- coding: latin-1 -*-

iso-8859-1N)r  r|  r}  r  r~  s   &   r'   test_cookie_first_line_no_bom0TestDetectEncoding.test_cookie_first_line_no_bomX  sM    

 $,#;#;D<M<Me<T#U <0*H)IJr)   c                    Rp\         P                  ! V P                  V4      4      w  r#V P                  VR4       V P                  VR.4       R# )   ﻿# coding=utf-8
r  s   # coding=utf-8
N)r  r|  r}  r  r~  s   &   r'   &test_matched_bom_and_cookie_first_line9TestDetectEncoding.test_matched_bom_and_cookie_first_lineb  sM    

 $,#;#;D<M<Me<T#U ;/*=)>?r)   c                v    RpV P                  V4      pV P                  \        \        P                  V4       R# )#   ﻿# vim: set fileencoding=ascii :
N)r  r|  r}  rv  rC   rP  r   r4  r3   rt  r/   s   &  r'   <test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorOTestDetectEncoding.test_mismatched_bom_and_cookie_first_line_raises_syntaxerrorl  s3    

 $$U++x'?'?Jr)   c                    Rp\         P                  ! V P                  V4      4      w  r#V P                  VR4       R R.pV P                  W44       R# )   #! something
    # vim: set fileencoding=ascii :
asciiN)r  r  r|  r}  r  r3   rt  rc  r  r5   s   &    r'   test_cookie_second_line_no_bom1TestDetectEncoding.test_cookie_second_line_no_bomu  sR    
 $,#;#;D<M<Me<T#U 7+%'KL2r)   c                    Rp\         P                  ! V P                  V4      4      w  r#V P                  VR4       V P                  VRR.4       R# )   ﻿#! something
   f# coding=utf-8
r  r  N)r  r  r|  r}  r  r~  s   &   r'   'test_matched_bom_and_cookie_second_line:TestDetectEncoding.test_matched_bom_and_cookie_second_line  sS    
 $,#;#;D<M<Me<T#U ;/+-AB	Dr)   c                v    RpV P                  V4      pV P                  \        \        P                  V4       R# )r  N)r  r  r|  r}  r  r  s   &  r'   =test_mismatched_bom_and_cookie_second_line_raises_syntaxerrorPTestDetectEncoding.test_mismatched_bom_and_cookie_second_line_raises_syntaxerror  s3    
 $$U++x'?'?Jr)   c                    Rp\         P                  ! V P                  V4      4      w  r#V P                  VR4       R .pV P                  W44       R# )   print('£')
r-   N)r  %   # vim: set fileencoding=iso8859-15 :
   print('€')
r  r  s   &    r'   /test_cookie_second_line_noncommented_first_lineBTestDetectEncoding.test_cookie_second_line_noncommented_first_line  sO    

 $,#;#;D<M<Me<T#U 7+*+2r)   c                    Rp\         P                  ! V P                  V4      4      w  r#V P                  VR4       R R.pV P                  W44       R# )   #print('£')
r  
iso8859-15N)r  r  r  r  r  s   &    r'   ,test_cookie_second_line_commented_first_line?TestDetectEncoding.test_cookie_second_line_commented_first_line  sR    

 $,#;#;D<M<Me<T#U <0+-VW2r)   c                    Rp\         P                  ! V P                  V4      4      w  r#V P                  VR4       R R.pV P                  W44       R# )   
r  r  N)r  r  r  r  r  s   &    r'   (test_cookie_second_line_empty_first_line;TestDetectEncoding.test_cookie_second_line_empty_first_line  sR    

 $,#;#;D<M<Me<T#U <0DE2r)   c                   R
pV F|  pR Fs  pVP                  RV4      pRRVP                  R4      ,           R,           RR3pV P                  V4      p\        P                  ! V4      w  rxV P                  VR4       Ku  	  K~  	  R	# )latin-1r  -   #!/usr/bin/python

   # coding: r  r  s   print(things)
s   do_something += 4
N)r  r  ziso-latin-1zlatin-1-unixziso-8859-1-unixziso-latin-1-macr  r(  replacer.   rv  r   r4  r0   	r3   	encodingsrc  repencrt  rlfoundr  s	   &        r'   test_latin1_normalization,TestDetectEncoding.test_latin1_normalization  s    ;	!H!&&sC0/&G)<<uD+/1 &&u-(0(@(@(D%  5 " "r)   c                v    RpV P                  V4      pV P                  \        \        P                  V4       R# )
   print("")Nr  r  r  s   &  r'   test_syntaxerror_latin1*TestDetectEncoding.test_syntaxerror_latin1  s3     $$U++x'?'?Jr)   c                   RpV F{  pR	 Fr  pVP                  RV4      pRRVP                  R4      ,           R,           R3pV P                  V4      p\        P                  ! V4      w  rxV P                  VR 4       Kt  	  K}  	  R# )
r-   r  r  r  r  r  s   1 + 3
N)r-   z	utf-8-macz
utf-8-unixr  r  r  s	   &        r'   test_utf8_normalization*TestDetectEncoding.test_utf8_normalization  s    8	!H!&&sC0/&G)<<uD#% &&u-(0(@(@(D%  0 " "r)   c                   V P                  R4      p\        P                  ! V4      w  r#V P                  VR4       V P                  VR .4       \        P                  ! V P                  R4      4      w  r#V P                  VR4       V P                  V. 4       V P                  R4      p\        P                  ! V4      w  r#V P                  VR4       V P                  VR .4       V P                  R4      p\        P                  ! V4      w  r#V P                  VR4       V P                  V. 4       V P                  R4      pV P	                  \
        \        P                  V4       R# )	r|  r-   r  N)r|  r=   )s   ﻿print(something)
)   ﻿)s   # coding: bad
)rv  r   r4  r0   rC   rP  r3   r/   rc  r  s   &   r'   test_short_files#TestDetectEncoding.test_short_files  s=   $$%=>#+#;#;H#E 7+*?)@A#+#;#;D<M<Mb<Q#R 7+,$$%IJ#+#;#;H#E ;/*?)@A$$%78#+#;#;H#E ;/,$$%:;+x'?'?Jr)   c                    V P                  R4      p\        P                  ! V4      w  r#V P                  VR4       V P                  VR .4       R# )   print("#coding=fake")r-   N)r  )rv  r   r4  r0   r  s   &   r'   test_false_encoding&TestDetectEncoding.test_false_encoding  sJ    $$%@A#+#;#;H#E 7+*B)CDr)   c                z   \         P                  R ,           pV P                  \         P                  V4       R	 F  p\	        VRVR7      ;_uu_ 4       p\        RV,          VR7       \        RVR7       RRR4       \        P                  ! V4      ;_uu_ 4       pV P                  VP                  V4       V P                  VP                  R4       RRR4       K  	  \	        VRRR7      ;_uu_ 4       p\        RVR7       RRR4       \        P                  ! V4      ;_uu_ 4       pV P                  VP                  R4       V P                  VP                  R4       RRR4       R#   + '       g   i     EL; i  + '       g   i     EKY  ; i  + '       g   i     L; i  + '       g   i     R# ; i)
z.pywrc  z# coding: %sfileu   print('euro:€')Nrr  )ziso-8859-15r-   )
r	   TESTFN
addCleanupunlinkr5  printr   r0   rc  mode)r3   rW   rc  fps   &   r'   	test_openTestDetectEncoding.test_open  s4   ##e+	(((3 1Hhh772nx/b9,26 8 x((B  h7  #. )(	 1 (C+66"(r2 7]]8$$R[[+6RWWc* %$ 877 )((
 76$$$s0   "E-9F6F*9F)-E>FF&	)F:	c                  a R pRo ! V3R lR4      pV P                  \        4      ;_uu_ 4        V! SV4      pV=\        P                  ! VP
                  4       RRR4       V P                  \        RP                  V4      4      ;_uu_ 4        V! SV4      p\        P                  ! VP
                  4       RRR4       R#   + '       g   i     Lq; i  + '       g   i     R# ; i)some_file_pathc                   4   <a  ] tR tRt o R tV3R ltRtV tR# );TestDetectEncoding.test_filename_in_exception.<locals>.Bunki  c                ,    W n         Wn        ^ V n        R# )r   N)name_lines_index)r3   rt  r@  s   &&&r'   __init__DTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.__init__  s     	#r)   c                   < V P                   \        S4      8X  d   \        hSV P                   ,          pV ;P                   ^,          un         V# rq  )r  r   rr  )r3   r&   rt  s   & r'   r/   DTestDetectEncoding.test_filename_in_exception.<locals>.Bunk.readline  s:    ;;#e*,''T[[)q r)   )r  r  r  N)r  r  r  r  r  r/   r  r  )r  rt  s   @r'   Bunkr    s      
 r)   r  Nz.*{}r  )rC   rP  r  r   r4  r/   rS   format)r3   r@  r  insrt  s   &   @r'   test_filename_in_exception-TestDetectEncoding.test_filename_in_exception  s    	 	 {++ud#C$$S\\2	 ,
 ##Kt1DEEud#C$$S\\2 FE ,+
 FEEs   ,C*CC	C)	c                   \        R 4      p\        P                  ! RVR7      ;_uu_ 4        V P                  \        \
        P                  R4       RRR4       V P                  VP                  4       R#   + '       g   i     L-; i)s   #coding:xxxztokenize._builtin_open)return_valuefoobarN)	r   r   patchrC   rP  r   r5  
assertTrueclosed)r3   ms   & r'   test_open_error"TestDetectEncoding.test_open_error-  sT    N#ZZ0qAAk8==(C B! BAs   &A77B	r=   N)r  r  r  r  rv  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   thread_unsafer  r  r  r  r  r  s   @r'   rn  rn  6  s     	:	DK@K
3
DK	3	3	36K1K.E + +(38" "r)   rn  c                   P   a  ] tR tRt o R tR tR tR tR tR t	R t
R	 tR
tV tR# )TestTokenizei5  c                `  aa	a
 ^ RI pRo	Ro
V	3R lpV
3R lp^ oV3R lpVP                  pVP                  pW!n        W1n         \         P                   ! V4      pV P                  \	        V4      R,          . RO4       WQn        Wan        V P                  S
S	4       R#   YQn        Yan        i ; i)r   Nr-   c                    < SR R.3# )   first   secondr=   )r/   rc  s   &r'   mock_detect_encoding8TestTokenize.test_tokenize.<locals>.mock_detect_encoding;  s    h	222r)   c                 ~   < Vo. p  V ! 4       pT'       d   TP                  T4       K$  T#   \          d    Tu # i ; i)T)rr  r   )r/   rc  kwargsout	next_lineencoding_useds   &&,  r'   mock__tokenize2TestTokenize.test_tokenize.<locals>.mock__tokenize>  sK    $MC (
I JJy)
 % Js   , <<c                  Z   < S ^,          o S ^8X  d   R# \        S 4      P                  4       # )r   r)   )strr.   )counters   r'   mock_readline1TestTokenize.test_tokenize.<locals>.mock_readlineM  s)    qLG!|w<&&((r)   :r   NN)r  r     1   2   3   4)r   r4  rh  r0   rE   )r3   tokenize_moduler  r  r  orig_detect_encodingorig_c_tokenresultsr  rc  r  s   &       @@@r'   test_tokenizeTestTokenize.test_tokenize7  s    *	3	 	)  />>&HH*>'<J9	M''6GT']2.JL /C+@L=1 /C+@L=s   :B B-c           	     |   . p\        R 4       F$  pVP                  RP                  VR7      4       K&  	  VP                  R4       RP                  V4      p\	        \
        P
                  ! \        VP                  R4      4      P                  4      4      pV P                  VR,          P                  R4       R# )i  zdef i{i}(): return {i})iOK
r-   N)ranger   r  rA  rE   r   r   r.   r/   r0   string)r3   bufr  tokss   &   r'   test_oneline_defsTestTokenize.test_oneline_defsb  s    sAJJ/666;< 

4iin H%%gcjj.A&B&K&KLMb$/r)   c           	        \        \        P                  ! \        VP                  R 4      4      P                  4      4      p\        V4      pV P                  \        V4      ^V,           4       V P                  \        P                  V^ ,          P                  ,          \        P                  \        P                  ,          4       \        V4       F\  pV P                  \        P                  W5^,           ,          P                  ,          \        P                  W%,          ,          4       K^  	  V P                  \        P                  V^V,           ,          P                  ,          \        P                  \        P                  ,          4       V P                  \        P                  V^V,           ,          P                  ,          \        P                  \        P                  ,          4       R# )r-   N)rE   r   r   r.   r/   r   r0   r   
exact_typeENCODINGr"  r#   r   r   )r3   opstroptypesrN   num_optypesr  s   &&*   r'   assertExactTypeEqual!TestTokenize.assertExactTypeEqualn  sE   h''W0E(F(O(OPQ'lVa+o6**6!9+?+?@!**8+<+<=	?{#AX..v!e}/G/GH%..wz:< $ 	**6!k/+B+M+MN!**5==9	;**6!k/+B+M+MN!**5??;	=r)   c                   V P                  R \        P                  \        P                  4       V P                  R\        P                  \        P
                  4       V P                  R\        P                  4       V P                  R\        P                  4       V P                  R\        P                  4       V P                  R\        P                  4       V P                  R\        P                  4       V P                  R\        P                  4       V P                  R\        P                  4       V P                  R	\        P                  4       V P                  R
\        P                  4       V P                  R\        P                  4       V P                  R\        P                   4       V P                  R\        P"                  4       V P                  R\        P$                  4       V P                  R\        P&                  4       V P                  R\        P(                  \        P*                  4       V P                  R\        P,                  4       V P                  R\        P.                  4       V P                  R\        P0                  4       V P                  R\        P2                  4       V P                  R\        P4                  4       V P                  R\        P6                  4       V P                  R\        P8                  4       V P                  R\        P:                  4       V P                  R\        P<                  4       V P                  R\        P>                  4       V P                  R\        P@                  4       V P                  R\        PB                  4       V P                  R\        PD                  4       V P                  R\        PF                  4       V P                  R\        PH                  4       V P                  R \        PJ                  4       V P                  R!\        PL                  4       V P                  R!\        PL                  4       V P                  R"\        PN                  4       V P                  R#\        PP                  4       V P                  R$\        PR                  4       V P                  R%\        PT                  4       V P                  R&\        PV                  4       V P                  R'\        PX                  4       V P                  R(\        PZ                  4       V P                  R)\        P\                  4       V P                  R*\        P^                  4       V P                  R+\        P`                  4       V P                  R,\b        Pd                  \        P<                  \b        Pf                  \        P                  \b        Pd                  \        P<                  \b        Pf                  \        P,                  \b        Pd                  \        P<                  \b        Pf                  4       V P                  R-\        P(                  \        Pf                  \        P                  \        Pf                  \        P                  \        Pf                  \        P*                  4       V P                  R.\        P6                  \        P                  \        Pd                  \        P                  \        Pf                  \        P                  4       R/# )0z()z[]:,;+r  */|&<>=r  %z{}z==z!=z<=z>=~^z<<z>>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=z:=z...z->@z@=za**2+b**2==c**2z	{1, 2, 3}z
^(x & 0x1)N)4r.  r#   LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL	LESSEQUALGREATEREQUALTILDE
CIRCUMFLEX	LEFTSHIFT
RIGHTSHIFT
DOUBLESTAR	PLUSEQUALMINEQUAL	STAREQUAL
SLASHEQUALPERCENTEQUAL
AMPEREQUAL	VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUALDOUBLESLASHDOUBLESLASHEQUAL
COLONEQUALELLIPSISRARROWATATEQUALr   r  r   rm   s   &r'   test_exact_typeTestTokenize.test_exact_type|  s   !!$

EJJ?!!$

EJJ?!!#u{{3!!#u{{3!!#uzz2!!#uzz2!!#u{{3!!#uzz2!!#u{{3!!#uzz2!!#u{{3!!#uzz2!!#u}}5!!#u{{3!!#uyy1!!#u}}5!!$ellC!!$6!!$7!!$8!!$(:(:;!!#u{{3!!#u'7'78!!$8!!$(8(89!!$(8(89!!$8!!$7!!$8!!$(8(89!!$(:(:;!!$(8(89!!$8!!$(=(=>!!$(=(=>!!%)=)=>!!%)>)>?!!%)>)>?!!$(9(9:!!%)?)?@!!$(8(89!!%8!!$5!!#uxx0!!$6!!"3"*--1A1A8??"'**"*--1A1A8??"'--"*--1A1A8??	T 	!!+"',,"',,"',,"',,"',,	0 	!!,"'"2"2"'**"'**ekk5<<"'**		.r)   c                F    V P                  R \        P                  4       R# )z@          N)r.  r#   rm  rm   s   &r'   %test_pathological_trailing_whitespace2TestTokenize.test_pathological_trailing_whitespace  s    !!-:r)   c                   R p\         P                  ! \        P                  RRRRR7      \         P                  ! \        P                  RRRRR7      \         P                  ! \        P
                  RRRRR7      \         P                  ! \        P                  RRRRR7      \         P                  ! \        P                  RRRRR7      \         P                  ! \        P                  RRRRR7      \         P                  ! \        P                  R	RRR	R7      \         P                  ! \        P                  RRRR	R7      \         P                  ! \        P                  RRRRR7      .	p\        \         P                   ! \        VP                  R4      4      P                  4      4      pV P                  W24       R
# )zb = 1

#testr-    r"   r#  r$   r%   r&   bzb = 1
r;  1r   z#testNr   r   re  r   r   r   r{  r      r      r      r      r{  r   r{  r   r}  r   )r}  r  )r}  r  )r  r   )r   r3  r#   r*  r  r  r   r   NLCOMMENTr   rE   r   r.   r/   r0   r3   sourcerj  rN   s   &   r'   5test_comment_at_the_end_of_the_source_without_newlineBTestTokenize.test_comment_at_the_end_of_the_source_without_newline  s?   !ENN7&V\cefEJJs&f[deEHHSFYbcELLFPV]fgEMM$fRX_hiEHHTVZ^_EMM'U[bijEHHRv6X_`EOOBfRX_ab

 h''g0F(G(P(PQR1r)   c                B   R p\         P                  ! \        P                  RR	R	RR7      \         P                  ! \        P                  RR
RRR7      \         P                  ! \        P
                  RRRRR7      \         P                  ! \        P                  RRRRR7      \         P                  ! \        P                  RRRRR7      .p\        \         P                   ! \        VP                  R4      4      P                  4      4      pV P                  W24       R# )za
 r-   ru  )r#  r$   r%   r&   aza
r   r   Nry  re  rz  r{  r  )r{  r{  r  )r   r3  r#   r*  r  r   r  r   rE   r   r.   r/   r0   r  s   &   r'   ?test_newline_and_space_at_the_end_of_the_source_without_newlineLTestTokenize.test_newline_and_space_at_the_end_of_the_source_without_newline  s    u~~gVQW^`auzz#VV[\u}}TVZ_`uxx&fSVWurVZ\]
 h''g0F(G(P(PQR1r)   c           	        R p\         P                  ! 4       ;_uu_ 4       p\        P                  P	                  VR4      p\        VR4      ;_uu_ 4       pVP                  V4       RRR4       \        V4      w  rVV P                  RVP                  4       RRR4       R#   + '       g   i     LC; i  + '       g   i     R# ; i)s   F"""
        """z	script.pywbNs   SyntaxError)
r	   temp_dirr?  r@  rA  r5  writer   assertInerr)r3   scriptr  rW   r  rsr(  s   &      r'   (test_invalid_character_in_fstring_middle5TestTokenize.test_invalid_character_in_fstring_middle  s     !!Xww||Hk:Hh%%

6" &(2EBMM."&&1 "!%% "!!s#   5B5B"	'1B5"B2-B55C	r=   N)r  r  r  r  r  r&  r.  ro  rr  r  r  r  r  r  r  s   @r'   r   r   5  s4     )2V	0=?.B;2$2
2 
2r)   r   c                   2   a  ] tR tRt o R tR tR tRtV tR# )UntokenizeTesti  c                |   \         P                  ! 4       p^Vn        ^Vn        V P	                  \
        4      ;_uu_ 4       pVP                  R4       RRR4       V P                  XP                  P                  ^ ,          R4       V P	                  \
        VP                  R4       R#   + '       g   i     L`; i)r{  Nz'start (1,3) precedes previous end (2,2)r|  r  )
r   Untokenizerprev_rowprev_colrC   
ValueErroradd_whitespacer0   rU   args)r3   ucms   &  r'   test_bad_input_order#UntokenizeTest.test_bad_input_order  s      "

z**bU# +**1-9	; 	*a&6&6> +*s   B++B;	c                N   \         P                  ! 4       p^Vn        ^Vn        . Vn        VP                  R4       V P                  VP                  R.4       ^Vn        VP                  R4       V P                  VP                  . RO4       \        P                  V R4       R# )r   \
za
  b
    c
  \
  c
Nr  )r  r  )r  z\
\
r   )	r   r  r  r  rN   r  r0   rD  rE  )r3   r  s   & r'   test_backslash_continuation*UntokenizeTest.test_backslash_continuation  s      "

	 F8,
	 #?@%%d,HIr)   c                   \         P                  ! 4       p\         P                  R 3p\         P                  R3V.pVP	                  V\        . 4      4       V P                  VP                  R.4       \         P                  ! 4       pV P                  VP                  \        V.4      4      R4       \         P                  ! 4       pV P                  VP                  \        V4      4      R4       V P                  VP                  R4       V P                  \         P                  ! \        V4      4      R4       R# )Hellor-   zHello s   Hello N)
r   r  r  r*  compatiterr0   rN   r"  rc  )r3   r  r#   rN   s   &   r'   test_iter_compatUntokenizeTest.test_iter_compat	  s      "($$g.6	R!H:.  "dE7m4h?  "d6l3X>W-,,T&\:IFr)   r=   N)	r  r  r  r  r  r  r  r  r  r  s   @r'   r  r    s     
?JG Gr)   r  c                T    \         P                  ! R4      pVP                  V 4      RJ# )a  Return `True` if the source contains a backslash on a
line by itself. For example:

a = (1
    \
)

Code like this cannot be untokenized exactly. This is because
the tokenizer does not produce any tokens for the line containing
the backslash and so there is no way to know its indent.
s   \n\s*\\\r?\nN)recompilesearch)r  patterns   & r'   contains_ambiguous_backslashr    s'     jj)*G>>&!--r)   c                   V   a  ] tR tRt o R tR tR tR tR tR t	R t
R	 tR
 tRtV tR# )rD  i'  c                .   \        V\        4      '       d   VP                  R4      pMVP                  4       p\	        VP                  RR7      4      P                  p\        \        P                  ! V4      4      pV Uu. uF  qUR,          NK  	  pp\        P                  ! V4      p\	        VP                  RR7      4      P                  p\        P                  ! V4       Uu. uF  qUR,          NK  	  p	pV P                  W4       \        P                  ! V4      p
\	        V
P                  RR7      4      P                  p\        P                  ! V4       Uu. uF  qUR,          NK  	  ppV P                  W4       \        V4      '       gt   VP                  R4      p\	        VP                  RR7      4      P                  p\        P                  ! \        P                  ! V4      4      pV P                  W4       R# R# u upi u upi u upi )aR  
Test roundtrip for `untokenize`. `f` is an open file or a string.
The source code in f is tokenized to both 5- and 2-tuples.
Both sequences are converted back to source code via
tokenize.untokenize(), and the latter tokenized again to 2-tuples.
The test fails if the 3 pair tokenizations do not match.

If the source code can be untokenized unambiguously, the
untokenized code must match the original code exactly.

When untokenize bugs are fixed, untokenize with 5-tuples should
reproduce code that does not contain a backslash continuation
following spaces.  A proper test should test this.
r-   Tkeependsrz  r  N)
isinstancer  r.   readr  r   rF   rE   r   r"  r0   r  removeprefix)r3   r6   coder/   tokens5r_   tokens2bytes_from2	readline2tokens2_from2bytes_from5	readline5tokens2_from5code_without_bomuntokenized_codes   &&             r'   rE  TestRoundtrip.check_roundtrip)  s     a88G$D668D67@@x((23&-.gsr77g.))'2///>?HH	,4,=,=i,HI,HSR,HI0))'2///>?HH	,4,=,=i,HI,HSR,HI0+D11#00A,777FGPPH'2283D3DX3NO-@ 2 / J
 Js   H'HHc                   \        V\        4      '       d   VP                  R 4      pMVP                  4       p\	        VP                  RR7      4      P                  p\        P                  ! V4       F  pVP                  \        P                  \        P                  09   d   K4  V P                  VP                  VP                  VP                  ^,          VP                  ^,           4       K  	  R# )r-   Tr  N)r  r  r.   r  r  r   rF   r   r"   r*  r   r0   r#  r&   r$   r%   )r3   r6   r  r/   r_   s   &&   r'   check_line_extraction#TestRoundtrip.check_line_extractionR  s    a88G$D668D67@@$$X.CxxX..0B0BCCSZZ#))A,
)KL /r)   c                h   V P                  R 4       V P                  R4       V P                  R4       \        P                  ! RRR7      p\        VR4      ;_uu_ 4       pV P                  V4       RRR4       V P                  R4       V P                  R	4       V P                  R
4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R4       V P                  R 4       V P                  R!4       V P                  R"4       V P                  R#4       V P                  R$4       V P                  R%4       V P                  R&4       V P                  R'4       V P                  R(4       V P                  R)4       V P                  R*4       . R.OpV F  pV P                  V4       K  	  V P                  R+4       V P                  R,4       V P                  R-4       R#   + '       g   i     EL; i)/zif x == 1:
    print(x)
z # This is a comment
# This also
zif x == 1 : 
  print(x)
ztokenize_tests.txtr=  )subdirr>  Nzoif x == 1:
    # A comment by itself.
    print(x) # Comment here, too.
    # Another comment.
after_if = True
zPif (x # The comments need to go in the right place
    == 1):
    print('x==1')
zclass Test: # A comment here
  # A comment with weird indent
  after_com = 5
  def x(m): return m*5 # a one liner
  def y(m): # A whitespace after the colon
     return y*4 # 3-space indent
zttry: import somemodule
except ImportError: # comment
    print('Can not import' # comment2
)else:   print('Loaded')
zf'\N{EXCLAMATION MARK}'zf'\\N{SNAKE}'zf'\\N{{SNAKE}}'zf'\N{SNAKE}'zf'\\\N{SNAKE}'zf'\\\\\N{SNAKE}'zf'\\\\\\\N{SNAKE}'z	f'\\N{1}'zf'\\\\N{2}'zf'\\\\\\N{3}'zf'\\\\\\\\N{4}'zf'\\N{{'z
f'\\\\N{{'zf'\\\\\\N{{'zf'\\\\\\\\N{{'zf'\n{{foo}}'zf'\\n{{foo}}'zf'\\\n{{foo}}'zf'\\\\n{{foo}}'zf'\t{{foo}}'zf'\\t{{foo}}'zf'\\\t{{foo}}'zf'\\\\t{{foo}}'zrf'\t{{foo}}'zrf'\\t{{foo}}'zrf'\\\t{{foo}}'zrf'\\\\t{{foo}}'zrf'\{{foo}}'zf'\\{{foo}}'zrf'\\\{{foo}}'zf'\\\\{{foo}}'zt'{ {}}'zt'{f'{ {}}'}{ {}}'zf'{t'{ {}}'}{ {}}')z
if 1:
    "foo"
"bar"
z
if 1:
    ("foo"
     "bar")
z
if 1:
    "foo"
    "bar"
)rE  r   findfiler5  )r3   fnr6   casescases   &    r'   test_roundtripTestRoundtrip.test_roundtrip]  s    	 . 	/ - 	. 	 , 	-2>J"d^^q  #  1 	2
 	 3 	4 	 B 	C 	 9 	:
 	78-./0_-./0123\*^,-./0[)]+_-./_--..//0_--..//0-..//001_-_-././  D  &  	[)2323c ^^s    L  L1	c                (    V P                  R 4       R# )zna = (3,4, 
5,6)
y = [3, 4,
5]
z = {'a': 5,
'b':15, 'c':True}
x = len(y) + 5 - a[
3] - a[2]
+ len(z) - z[
'b']
NrE  rm   s   &r'   test_continuationTestRoundtrip.test_continuation  s     	& 		'r)   c                J    V P                  R 4       V P                  R4       R# )z)x=1+\
1
# This is a comment\
# This also
z# Comment \
x = 0Nr  rm   s   &r'   r  )TestRoundtrip.test_backslash_continuation  s*     - 	. 	 % 	&r)   c                (    V P                  R 4       R# )z'' ''Nr  rm   s   &r'   test_string_concatenation'TestRoundtrip.test_string_concatenation  s    W%r)   c           
        ^ RI p^ RIp\        P                  P	                  \
        4      ;'       g    \        P                  pVP                  \        P                  P                  VP                  V4      R4      4      p\        P                  ! R4      '       g   VP                  V^
4      pV F  p\        P                  ^8  d   \        RV4       \        VR4      ;_uu_ 4       pV P                  VR7      ;_uu_ 4        V P!                  V4       V P#                  V4       RRR4       RRR4       K  	  R#   + '       g   i     L; i  + '       g   i     K  ; i)r   Nztest*.pycpur   r>  r  )globrandomr?  r@  rB  rC  curdirrA  escaper   is_resource_enabledsampleverboser  r5  subTestrE  r  )r3   r  r  tempdir	testfilestestfiler6   s   &      r'   test_random_filesTestRoundtrip.test_random_files  s     	''//(+88ryyIIbggll4;;w+?LM	**511i4I!H!#j(+h%%\\x\00((+..q1 1 &% " 10 &%%s$   *E#D>(E>E	EE"c                    \        V\        4      '       d   VP                  R 4      p\        P                  ! \        P                  ! \        V4      P                  4      4      P                  R 4      # )r-   )r  r  r.   r   r"  r   r/   r#  )r3   r  s   &&r'   	roundtripTestRoundtrip.roundtrip  sO    dC  ;;w'D""8#4#4WT]5K5K#LMTTU\]]r)   c                    RpV P                  V4      P                  R4      pV P                  V^,          V^,          4       V P                  V4       R# )z~
Ensure that although whitespace might be mutated in a roundtrip,
the semantic meaning of the indentation remains consistent.
zif False:
	x=3
	x=3
r   N)r  splitr0   rE  )r3   r  	codeliness   &  r'   #test_indentation_semantics_retained1TestRoundtrip.test_indentation_semantics_retained  sH    
 +NN4(..t4	1y|4T"r)   r=   N)r  r  r  r  rE  r  r  r  r  r  r  r  r  r  r  r  s   @r'   rD  rD  '  s=     'AR	Ma4H'&&2(^
# #r)   rD  c                   ,   a  ] tR tRt o R tR tRtV tR# )InvalidPythonTestsi  c                   R p\         P                  ! \        P                  RR	R
R R7      \         P                  ! \        P                  RR
RR R7      \         P                  ! \        P
                  RRRR R7      \         P                  ! \        P                  RRRR R7      \         P                  ! \        P
                  RRRR R7      \         P                  ! \        P                  RRRR R7      \         P                  ! \        P                  RRRRR7      .p\        \         P                  ! \        V4      P                  4      4      pV P                  W24       R# )z2sin(x)2rv  sinr   rL   r  ru  Nre  rz  r~  r  r  rf  )r      r  )r   r3  r#   r   r  r  r   r   rE   rG   r   r/   r0   r  s   &   r'   test_number_followed_by_name/InvalidPythonTests.test_number_followed_by_name   s    ELLFPV]fgEJJuFPV]fgEHHSFYbcEJJs&f[deEHHSFYbcEMM"FPV]fgEOOBfRX_ab
 h..x/?/H/HIJ1r)   c           
        R p\         P                  ! \        P                  R RRR R7      \         P                  ! \        P                  RRRR R7      \         P                  ! \        P
                  RRRRR7      .p\        \         P                  ! \        V4      P                  4      4      pV P                  W24       R# )01234rv  ru  Nre  r  r  r  )r   r3  r#   r   r   r   rE   rG   r   r/   r0   r  s   &   r'   test_number_starting_with_zero1InvalidPythonTests.test_number_starting_with_zero  s    ELLTZahiEMM"FPV]deEOOBfRX_ab
 h..x/?/H/HIJ1r)   r=   N)r  r  r  r  r  r   r  r  r  s   @r'   r  r    s     2 	2 	2r)   r  c                      a  ] tR tRt o R tR tR tR tR tR t	R t
R	 tR
 tR tR tR tR tR tR tR t]P*                  ! 4       R 4       tR tRtV tR# )CTokenizeTesti  c                8   \        V4      pV P                  VR 7      ;_uu_ 4        \        \        P                  ! VP
                  4      V4      pV P                  WBP                  4       P                  4       4       RRR4       R#   + '       g   i     R# ; i))r  N)	r   r  r(   r   rh  r/   r0   r1   r   r2   s   &&&  r'   r7   CTokenizeTest.check_tokenize  sk     QK\\\##1::1::FF V__%6%A%A%CD	 $###s   ABB	c                   R  p\         P                  ! \         P                  RR
RRR7      \         P                  ! \         P                  RRRRR7      \         P                  ! \         P                  RRRRR7      \         P                  ! \         P                  RRRRR7      \         P                  ! \         P
                  RRRRR7      .pR Fi  pV P                  VR7      ;_uu_ 4        \        \         P                  ! V! V4      P                  RVR7      4      pV P                  WB4       R	R	R	4       Kk  	  R	#   + '       g   i     K  ; i)c              3   4   "   R P                  V 4      x  R# 5i)1+1N)r.   r  s   &r'   r/   -CTokenizeTest.test_encoding.<locals>.readline'  s     ,,x((s   rx  r  rv  r4  ru  r  T)rd  rc  Nre  rz  r{  r|  r~  r  )r-   r  zutf-16)r   r3  r   r  r   r   r  rE   rh  rF   r0   )r3   r/   r5   rc  rN   s   &    r'   test_encodingCTokenizeTest.test_encoding&  s	   	) HOOCvSY`efHKK6v\abHOOCvSY`efH$4$4RvSY`efH$6$6rU[bde
 7Hx00hHHX&//!%% 
   2 10 7000s   -AD??Ec                J   V P                  R R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR	4       V P                  R
R4       V P                  RR4       V P                  RR4       V P                  RR4       R# )rc   z    NUMBER     '0xff'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    rd   z    NUMBER     '0b10'        (1, 0) (1, 4)
    LESSEQUAL  '<='          (1, 5) (1, 7)
    NUMBER     '255'         (1, 8) (1, 11)
    re   z    NUMBER     '0o123'       (1, 0) (1, 5)
    LESSEQUAL  '<='          (1, 6) (1, 8)
    NUMBER     '0O123'       (1, 9) (1, 14)
    rf   z    NUMBER     '1234567'     (1, 0) (1, 7)
    GREATER    '>'           (1, 8) (1, 9)
    TILDE      '~'           (1, 10) (1, 11)
    NUMBER     '0x15'        (1, 11) (1, 15)
    rg   z    NUMBER     '2134568'     (1, 0) (1, 7)
    NOTEQUAL   '!='          (1, 8) (1, 10)
    NUMBER     '1231515'     (1, 11) (1, 18)
    rh   ac      LPAR       '('           (1, 0) (1, 1)
    MINUS      '-'           (1, 1) (1, 2)
    NUMBER     '124561'      (1, 2) (1, 8)
    MINUS      '-'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    RPAR       ')'           (1, 10) (1, 11)
    AMPER      '&'           (1, 12) (1, 13)
    NUMBER     '200000000'   (1, 14) (1, 23)
    ri   z    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
    NOTEQUAL   '!='          (1, 11) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    NUMBER     '1'           (1, 15) (1, 16)
    rj   z    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
    AMPER      '&'           (1, 11) (1, 12)
    NUMBER     '12345'       (1, 13) (1, 18)
    rk   z    NUMBER     '0xFF'        (1, 0) (1, 4)
    AMPER      '&'           (1, 5) (1, 6)
    NUMBER     '0x15'        (1, 7) (1, 11)
    VBAR       '|'           (1, 12) (1, 13)
    NUMBER     '1234'        (1, 14) (1, 18)
    Nrl   rm   s   &r'   rn   CTokenizeTest.test_int:  s    M , 		 	M , 		 	, / 		 	- 0 		 	0 3 		 	5 	8 			 	. 1 		 	0 3 		 	0 3 		r)   c                   V P                  R R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR	4       V P                  R
R4       V P                  RR4       R# )rt   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14159'     (1, 4) (1, 11)
    ru   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '314159.'     (1, 4) (1, 11)
    rv   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '.314159'     (1, 4) (1, 11)
    rw   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3e14159'     (1, 4) (1, 11)
    rx   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3E123'       (1, 4) (1, 9)
    ry   z    NAME       'x'           (1, 0) (1, 1)
    PLUS       '+'           (1, 1) (1, 2)
    NAME       'y'           (1, 2) (1, 3)
    EQUAL      '='           (1, 4) (1, 5)
    NUMBER     '3e-1230'     (1, 6) (1, 13)
    rz   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '3.14e159'    (1, 4) (1, 12)
    Nrl   rm   s   &r'   r{   CTokenizeTest.test_float{  s    M , 		 	M , 		 	M , 		 	M , 		 	K * 		 	O . 		 	N - 		r)   c                b   V P                  R R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR	4       V P                  R
R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  R R!4       V P                  R"R#4       V P                  R$R%4       V P                  R&R'4       V P                  R(R)4       V P                  R*R+4       V P                  R,R-4       V P                  R.R/4       V P                  R0R14       V P                  R2R34       V P                  R4R54       V P                  R6R74       V P                  R8R94       V P                  R:R;4       V P                  R<R=4       R># )?r   a5      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "''"          (1, 4) (1, 6)
    SEMI       ';'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    EQUAL      '='           (1, 10) (1, 11)
    STRING     '""'          (1, 12) (1, 14)
    r   a6      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '\'"\''       (1, 4) (1, 7)
    SEMI       ';'           (1, 7) (1, 8)
    NAME       'y'           (1, 9) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    STRING     '"\'"'        (1, 13) (1, 16)
    r   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"doesn\'t "' (1, 4) (1, 14)
    NAME       'shrink'      (1, 14) (1, 20)
    STRING     '", does it"' (1, 20) (1, 31)
    r   z    NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "'abc'"       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     "'ABC'"       (1, 12) (1, 17)
    r   z    NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     '"ABC"'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    STRING     '"ABC"'       (1, 12) (1, 17)
    r   a      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     "r'abc'"      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     "r'ABC'"      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     "R'ABC'"      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     "R'ABC'"      (1, 31) (1, 37)
    r   a      NAME       'y'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    STRING     'r"abc"'      (1, 4) (1, 10)
    PLUS       '+'           (1, 11) (1, 12)
    STRING     'r"ABC"'      (1, 13) (1, 19)
    PLUS       '+'           (1, 20) (1, 21)
    STRING     'R"ABC"'      (1, 22) (1, 28)
    PLUS       '+'           (1, 29) (1, 30)
    STRING     'R"ABC"'      (1, 31) (1, 37)
    r   z    STRING     "u'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "U'abc'"      (1, 9) (1, 15)
    r   z    STRING     'u"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'U"abc"'      (1, 9) (1, 15)
    r   z    STRING     "b'abc'"      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     "B'abc'"      (1, 9) (1, 15)
    r   z    STRING     'b"abc"'      (1, 0) (1, 6)
    PLUS       '+'           (1, 7) (1, 8)
    STRING     'B"abc"'      (1, 9) (1, 15)
    r   a;      STRING     "br'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "bR'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Br'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "BR'abc'"     (1, 30) (1, 37)
    r   a;      STRING     'br"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'bR"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Br"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'BR"abc"'     (1, 30) (1, 37)
    r   a;      STRING     "rb'abc'"     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     "rB'abc'"     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     "Rb'abc'"     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     "RB'abc'"     (1, 30) (1, 37)
    r   a;      STRING     'rb"abc"'     (1, 0) (1, 7)
    PLUS       '+'           (1, 8) (1, 9)
    STRING     'rB"abc"'     (1, 10) (1, 17)
    PLUS       '+'           (1, 18) (1, 19)
    STRING     'Rb"abc"'     (1, 20) (1, 27)
    PLUS       '+'           (1, 28) (1, 29)
    STRING     'RB"abc"'     (1, 30) (1, 37)
    r   r   r   r   r   r   r   r   r   r   r   r   z"hola\\\r\ndfgf"z9    STRING     '"hola\\\\\\r\\ndfgf"' (1, 0) (1, 16)
    r   r   r   a=      FSTRING_START 'fR"'         (1, 0) (1, 3)
    FSTRING_MIDDLE 'a'           (1, 3) (1, 4)
    LBRACE     '{'           (1, 4) (1, 5)
    NAME       'b'           (1, 5) (1, 6)
    RBRACE     '}'           (1, 6) (1, 7)
    FSTRING_MIDDLE 'c'           (1, 7) (1, 8)
    FSTRING_END '"'           (1, 8) (1, 9)
    r   r   r   r   zf"{
a}"z    FSTRING_START 'f"'          (1, 0) (1, 2)
    LBRACE     '{'           (1, 2) (1, 3)
    NAME       'a'           (2, 0) (2, 1)
    RBRACE     '}'           (2, 1) (2, 2)
    FSTRING_END '"'           (2, 2) (2, 3)
    r   r   zf"hola\\\r\ndfgf"z    FSTRING_START 'f"'          (1, 0) (1, 2)
    FSTRING_MIDDLE 'hola\\\\\\r\\ndfgf' (1, 2) (1, 16)
    FSTRING_END '"'           (1, 16) (1, 17)
    r   a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    LBRACE     '{'           (1, 6) (1, 7)
    NAME       'x'           (2, 4) (2, 5)
    COLON      ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n'         (2, 6) (3, 0)
    RBRACE     '}'           (3, 0) (3, 1)
    FSTRING_MIDDLE '__'          (3, 1) (3, 3)
    FSTRING_END "'''"         (3, 3) (3, 6)
    r   a      FSTRING_START "f'''"        (1, 0) (1, 4)
    FSTRING_MIDDLE '__'          (1, 4) (1, 6)
    LBRACE     '{'           (1, 6) (1, 7)
    NAME       'x'           (2, 4) (2, 5)
    COLON      ':'           (2, 5) (2, 6)
    FSTRING_MIDDLE 'a\n    b\n     c\n      d\n' (2, 6) (6, 0)
    RBRACE     '}'           (6, 0) (6, 1)
    FSTRING_MIDDLE '__'          (6, 1) (6, 3)
    FSTRING_END "'''"         (6, 3) (6, 6)
    Nrl   rm   s   &r'   r   CTokenizeTest.test_string  s   . 1 		 	1 4 		 	> A 		 	/ 2 		 	/ 2 		 	C 
F 
		 	C 
F 
		 	- 0 		 	- 0 		 	- 0 		 	- 0 		 	C F 		 	C F 		 	C F 		 	C F 		 	- 0 		 	L + 		 	L + 		 	 		 	 		 	 		 	/ 2 		 	H ' 		 	K * 		 	L + 		 	 		 	 			 	 		 	0 3 		 	 

		 	 

		r)   c                r    V P                  R R4       V P                  RR4       V P                  RR4       R# )r   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd22'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COMMA      ','           (1, 9) (1, 10)
    NAME       'b'           (1, 11) (1, 12)
    COMMA      ','           (1, 12) (1, 13)
    NAME       'c'           (1, 14) (1, 15)
    EQUAL      '='           (1, 15) (1, 16)
    NUMBER     '2'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    NAME       'd'           (1, 19) (1, 20)
    EQUAL      '='           (1, 20) (1, 21)
    NUMBER     '2'           (1, 21) (1, 22)
    COMMA      ','           (1, 22) (1, 23)
    STAR       '*'           (1, 24) (1, 25)
    NAME       'k'           (1, 25) (1, 26)
    RPAR       ')'           (1, 26) (1, 27)
    COLON      ':'           (1, 27) (1, 28)
    NAME       'pass'        (1, 29) (1, 33)
    r   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd01v_'       (1, 4) (1, 9)
    LPAR       '('           (1, 9) (1, 10)
    NAME       'a'           (1, 10) (1, 11)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    COMMA      ','           (1, 13) (1, 14)
    STAR       '*'           (1, 15) (1, 16)
    NAME       'k'           (1, 16) (1, 17)
    COMMA      ','           (1, 17) (1, 18)
    DOUBLESTAR '**'          (1, 19) (1, 21)
    NAME       'w'           (1, 21) (1, 22)
    RPAR       ')'           (1, 22) (1, 23)
    COLON      ':'           (1, 23) (1, 24)
    NAME       'pass'        (1, 25) (1, 29)
    r   a      NAME       'def'         (1, 0) (1, 3)
    NAME       'd23'         (1, 4) (1, 7)
    LPAR       '('           (1, 7) (1, 8)
    NAME       'a'           (1, 8) (1, 9)
    COLON      ':'           (1, 9) (1, 10)
    NAME       'str'         (1, 11) (1, 14)
    COMMA      ','           (1, 14) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'int'         (1, 19) (1, 22)
    EQUAL      '='           (1, 22) (1, 23)
    NUMBER     '3'           (1, 23) (1, 24)
    RPAR       ')'           (1, 24) (1, 25)
    RARROW     '->'          (1, 26) (1, 28)
    NAME       'int'         (1, 29) (1, 32)
    COLON      ':'           (1, 32) (1, 33)
    NAME       'pass'        (1, 34) (1, 38)
    Nrl   rm   s   &r'   r   CTokenizeTest.test_function	  sJ    ? B 		. 	; > 		$ 	D G 		r)   c                *    V P                  R R4       R# )r   a      NAME       'if'          (1, 0) (1, 2)
    NUMBER     '1'           (1, 3) (1, 4)
    LESS       '<'           (1, 5) (1, 6)
    NUMBER     '1'           (1, 7) (1, 8)
    GREATER    '>'           (1, 9) (1, 10)
    NUMBER     '1'           (1, 11) (1, 12)
    EQEQUAL    '=='          (1, 13) (1, 15)
    NUMBER     '1'           (1, 16) (1, 17)
    GREATEREQUAL '>='          (1, 18) (1, 20)
    NUMBER     '5'           (1, 21) (1, 22)
    LESSEQUAL  '<='          (1, 23) (1, 25)
    NUMBER     '0x15'        (1, 26) (1, 30)
    LESSEQUAL  '<='          (1, 31) (1, 33)
    NUMBER     '0x12'        (1, 34) (1, 38)
    NOTEQUAL   '!='          (1, 39) (1, 41)
    NUMBER     '1'           (1, 42) (1, 43)
    NAME       'and'         (1, 44) (1, 47)
    NUMBER     '5'           (1, 48) (1, 49)
    NAME       'in'          (1, 50) (1, 52)
    NUMBER     '1'           (1, 53) (1, 54)
    NAME       'not'         (1, 55) (1, 58)
    NAME       'in'          (1, 59) (1, 61)
    NUMBER     '1'           (1, 62) (1, 63)
    NAME       'is'          (1, 64) (1, 66)
    NUMBER     '1'           (1, 67) (1, 68)
    NAME       'or'          (1, 69) (1, 71)
    NUMBER     '5'           (1, 72) (1, 73)
    NAME       'is'          (1, 74) (1, 76)
    NAME       'not'         (1, 77) (1, 80)
    NUMBER     '1'           (1, 81) (1, 82)
    COLON      ':'           (1, 82) (1, 83)
    NAME       'pass'        (1, 84) (1, 88)
    Nrl   rm   s   &r'   r   CTokenizeTest.test_comparison	  r   r)   c                *    V P                  R R4       R# )r   a$      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    MINUS      '-'           (1, 6) (1, 7)
    NAME       'y'           (1, 8) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '15'          (1, 12) (1, 14)
    MINUS      '-'           (1, 15) (1, 16)
    NUMBER     '1'           (1, 17) (1, 18)
    PLUS       '+'           (1, 19) (1, 20)
    NUMBER     '0x124'       (1, 21) (1, 26)
    PLUS       '+'           (1, 27) (1, 28)
    NAME       'z'           (1, 29) (1, 30)
    PLUS       '+'           (1, 31) (1, 32)
    NAME       'a'           (1, 33) (1, 34)
    LSQB       '['           (1, 34) (1, 35)
    NUMBER     '5'           (1, 35) (1, 36)
    RSQB       ']'           (1, 36) (1, 37)
    Nrl   rm   s   &r'   r   CTokenizeTest.test_additive
  r   r)   c                *    V P                  R R4       R# )r   a      NAME       'x'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    NUMBER     '1'           (1, 4) (1, 5)
    DOUBLESLASH '//'          (1, 5) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    SLASH      '/'           (1, 10) (1, 11)
    NUMBER     '5'           (1, 11) (1, 12)
    STAR       '*'           (1, 12) (1, 13)
    NUMBER     '12'          (1, 13) (1, 15)
    PERCENT    '%'           (1, 15) (1, 16)
    NUMBER     '0x12'        (1, 16) (1, 20)
    AT         '@'           (1, 20) (1, 21)
    NUMBER     '42'          (1, 21) (1, 23)
    Nrl   rm   s   &r'   r   !CTokenizeTest.test_multiplicative
  r   r)   c                N    V P                  R R4       V P                  RR4       R# )r   a      TILDE      '~'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    CIRCUMFLEX '^'           (1, 3) (1, 4)
    NUMBER     '1'           (1, 5) (1, 6)
    AMPER      '&'           (1, 7) (1, 8)
    NUMBER     '1'           (1, 9) (1, 10)
    VBAR       '|'           (1, 11) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    CIRCUMFLEX '^'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    NUMBER     '1'           (1, 17) (1, 18)
    r   aI      MINUS      '-'           (1, 0) (1, 1)
    NUMBER     '1'           (1, 1) (1, 2)
    STAR       '*'           (1, 2) (1, 3)
    NUMBER     '1'           (1, 3) (1, 4)
    SLASH      '/'           (1, 4) (1, 5)
    NUMBER     '1'           (1, 5) (1, 6)
    PLUS       '+'           (1, 6) (1, 7)
    NUMBER     '1'           (1, 7) (1, 8)
    STAR       '*'           (1, 8) (1, 9)
    NUMBER     '1'           (1, 9) (1, 10)
    DOUBLESLASH '//'          (1, 10) (1, 12)
    NUMBER     '1'           (1, 12) (1, 13)
    MINUS      '-'           (1, 14) (1, 15)
    MINUS      '-'           (1, 16) (1, 17)
    MINUS      '-'           (1, 17) (1, 18)
    MINUS      '-'           (1, 18) (1, 19)
    NUMBER     '1'           (1, 19) (1, 20)
    DOUBLESTAR '**'          (1, 20) (1, 22)
    NUMBER     '1'           (1, 22) (1, 23)
    Nrl   rm   s   &r'   r   CTokenizeTest.test_unary2
  s0    0 3 		 	5 8 		r)   c                *    V P                  R R4       R# )r   a      NAME       'import'      (1, 0) (1, 6)
    NAME       'sys'         (1, 7) (1, 10)
    COMMA      ','           (1, 10) (1, 11)
    NAME       'time'        (1, 12) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    NAME       'x'           (2, 0) (2, 1)
    EQUAL      '='           (2, 2) (2, 3)
    NAME       'sys'         (2, 4) (2, 7)
    DOT        '.'           (2, 7) (2, 8)
    NAME       'modules'     (2, 8) (2, 15)
    LSQB       '['           (2, 15) (2, 16)
    STRING     "'time'"      (2, 16) (2, 22)
    RSQB       ']'           (2, 22) (2, 23)
    DOT        '.'           (2, 23) (2, 24)
    NAME       'time'        (2, 24) (2, 28)
    LPAR       '('           (2, 28) (2, 29)
    RPAR       ')'           (2, 29) (2, 30)
    Nrl   rm   s   &r'   r   CTokenizeTest.test_selectorX
  r   r)   c                *    V P                  R R4       R# r   a      AT         '@'           (1, 0) (1, 1)
    NAME       'staticmethod' (1, 1) (1, 13)
    NEWLINE    ''            (1, 13) (1, 13)
    NAME       'def'         (2, 0) (2, 3)
    NAME       'foo'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'x'           (2, 8) (2, 9)
    COMMA      ','           (2, 9) (2, 10)
    NAME       'y'           (2, 10) (2, 11)
    RPAR       ')'           (2, 11) (2, 12)
    COLON      ':'           (2, 12) (2, 13)
    NAME       'pass'        (2, 14) (2, 18)
    Nrl   rm   s   &r'   r   CTokenizeTest.test_methodn
  r   r)   c                *    V P                  R R4       R# r  rl   rm   s   &r'   r   CTokenizeTest.test_tabs
  r   r)   c                   V P                  R R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR	4       V P                  R
R4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  RR4       V P                  R R!4       V P                  R"R#4       V P                  R$R%4       V P                  R&R'4       V P                  R(R)4       R*# )+r   z    NAME       'async'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r   a8      NAME       'a'           (1, 0) (1, 1)
    EQUAL      '='           (1, 2) (1, 3)
    LPAR       '('           (1, 4) (1, 5)
    NAME       'async'       (1, 5) (1, 10)
    EQUAL      '='           (1, 11) (1, 12)
    NUMBER     '1'           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    r   z    NAME       'async'       (1, 0) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    r   a<      NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    NAME       'Bar'         (1, 12) (1, 15)
    RPAR       ')'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 17) (1, 21)
    r   z    NAME       'class'       (1, 0) (1, 5)
    NAME       'async'       (1, 6) (1, 11)
    COLON      ':'           (1, 11) (1, 12)
    NAME       'pass'        (1, 12) (1, 16)
    r   z    NAME       'await'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    NUMBER     '1'           (1, 8) (1, 9)
    r   z    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    r   a;      NAME       'async'       (1, 0) (1, 5)
    NAME       'for'         (1, 6) (1, 9)
    NAME       'a'           (1, 10) (1, 11)
    NAME       'in'          (1, 12) (1, 14)
    NAME       'b'           (1, 15) (1, 16)
    COLON      ':'           (1, 16) (1, 17)
    NAME       'pass'        (1, 18) (1, 22)
    r   a<      NAME       'async'       (1, 0) (1, 5)
    NAME       'with'        (1, 6) (1, 10)
    NAME       'a'           (1, 11) (1, 12)
    NAME       'as'          (1, 13) (1, 15)
    NAME       'b'           (1, 16) (1, 17)
    COLON      ':'           (1, 17) (1, 18)
    NAME       'pass'        (1, 19) (1, 23)
    r   z    NAME       'async'       (1, 0) (1, 5)
    DOT        '.'           (1, 5) (1, 6)
    NAME       'foo'         (1, 6) (1, 9)
    r   r   r   z    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    NAME       'await'       (3, 0) (3, 5)
    r   z    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    ELLIPSIS   '...'         (2, 0) (2, 3)
    NEWLINE    ''            (2, 3) (2, 3)
    NAME       'await'       (3, 0) (3, 5)
    r   z    NAME       'async'       (1, 0) (1, 5)
    NEWLINE    ''            (1, 5) (1, 5)
    NAME       'await'       (2, 0) (2, 5)
    r   z    NAME       'foo'         (1, 0) (1, 3)
    DOT        '.'           (1, 3) (1, 4)
    NAME       'async'       (1, 4) (1, 9)
    PLUS       '+'           (1, 10) (1, 11)
    NUMBER     '1'           (1, 12) (1, 13)
    r   a;      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NAME       'pass'        (1, 17) (1, 21)
    r   a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    NAME       'def'         (2, 2) (2, 5)
    NAME       'foo'         (2, 6) (2, 9)
    LPAR       '('           (2, 9) (2, 10)
    NAME       'await'       (2, 10) (2, 15)
    RPAR       ')'           (2, 15) (2, 16)
    COLON      ':'           (2, 16) (2, 17)
    NEWLINE    ''            (2, 17) (2, 17)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'await'       (3, 4) (3, 9)
    EQUAL      '='           (3, 10) (3, 11)
    NUMBER     '1'           (3, 12) (3, 13)
    NEWLINE    ''            (3, 13) (3, 13)
    DEDENT     ''            (4, -1) (4, -1)
    NAME       'if'          (4, 2) (4, 4)
    NUMBER     '1'           (4, 5) (4, 6)
    COLON      ':'           (4, 6) (4, 7)
    NEWLINE    ''            (4, 7) (4, 7)
    INDENT     ''            (5, -1) (5, -1)
    NAME       'await'       (5, 4) (5, 9)
    NEWLINE    ''            (5, 9) (5, 9)
    DEDENT     ''            (6, -1) (6, -1)
    DEDENT     ''            (6, -1) (6, -1)
    NAME       'async'       (6, 0) (6, 5)
    PLUSEQUAL  '+='          (6, 6) (6, 8)
    NUMBER     '1'           (6, 9) (6, 10)
    NEWLINE    ''            (6, 10) (6, 10)
    r   a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    RPAR       ')'           (1, 14) (1, 15)
    COLON      ':'           (1, 15) (1, 16)
    NEWLINE    ''            (1, 16) (1, 16)
    INDENT     ''            (2, -1) (2, -1)
    NAME       'async'       (2, 2) (2, 7)
    NAME       'for'         (2, 8) (2, 11)
    NAME       'i'           (2, 12) (2, 13)
    NAME       'in'          (2, 14) (2, 16)
    NUMBER     '1'           (2, 17) (2, 18)
    COLON      ':'           (2, 18) (2, 19)
    NAME       'pass'        (2, 20) (2, 24)
    DEDENT     ''            (2, -1) (2, -1)
    r   ah      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'foo'         (1, 10) (1, 13)
    LPAR       '('           (1, 13) (1, 14)
    NAME       'async'       (1, 14) (1, 19)
    RPAR       ')'           (1, 19) (1, 20)
    COLON      ':'           (1, 20) (1, 21)
    NAME       'await'       (1, 22) (1, 27)
    r   a~      NAME       'def'         (1, 0) (1, 3)
    NAME       'f'           (1, 4) (1, 5)
    LPAR       '('           (1, 5) (1, 6)
    RPAR       ')'           (1, 6) (1, 7)
    COLON      ':'           (1, 7) (1, 8)
    NEWLINE    ''            (1, 8) (1, 8)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    NAME       'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    r   a      NAME       'async'       (1, 0) (1, 5)
    NAME       'def'         (1, 6) (1, 9)
    NAME       'f'           (1, 10) (1, 11)
    LPAR       '('           (1, 11) (1, 12)
    RPAR       ')'           (1, 12) (1, 13)
    COLON      ':'           (1, 13) (1, 14)
    NEWLINE    ''            (1, 14) (1, 14)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'def'         (3, 2) (3, 5)
    NAME       'baz'         (3, 6) (3, 9)
    LPAR       '('           (3, 9) (3, 10)
    RPAR       ')'           (3, 10) (3, 11)
    COLON      ':'           (3, 11) (3, 12)
    NAME       'pass'        (3, 13) (3, 17)
    NEWLINE    ''            (3, 17) (3, 17)
    NAME       'async'       (4, 2) (4, 7)
    NAME       'def'         (4, 8) (4, 11)
    NAME       'bar'         (4, 12) (4, 15)
    LPAR       '('           (4, 15) (4, 16)
    RPAR       ')'           (4, 16) (4, 17)
    COLON      ':'           (4, 17) (4, 18)
    NAME       'pass'        (4, 19) (4, 23)
    NEWLINE    ''            (4, 23) (4, 23)
    NAME       'await'       (6, 2) (6, 7)
    EQUAL      '='           (6, 8) (6, 9)
    NUMBER     '2'           (6, 10) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
    Nrl   rm   s   &r'   r   CTokenizeTest.test_async
  s   K * 		 	- 0 		 	I ( 		 	3 6 		 	. 1 		 	K * 		 	K * 		 	4 7 		 	5 8 		 	K * 		 	G & 		 	4 7 		 	/ 2 		 	N - 		 	O . 		 	3 6 		 	 #*		X 	H K 		& 	9 	< 			 	 !		F 	 "		r)   c                *    V P                  R R4       R# )r   u7      NAME       'Örter'       (1, 0) (1, 5)
    EQUAL      '='           (1, 6) (1, 7)
    STRING     "u'places'"   (1, 8) (1, 17)
    NEWLINE    ''            (1, 17) (1, 17)
    NAME       'grün'        (2, 0) (2, 4)
    EQUAL      '='           (2, 5) (2, 6)
    STRING     "U'green'"    (2, 7) (2, 15)
    Nrl   rm   s   &r'   r   CTokenizeTest.test_unicode  r   r)   c                H   R  p. RNRNRNRNRNRNRNRNR	NR
NRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRNRN FF  pV P                  VR7      ;_uu_ 4        V P                  \        P                  W4       RRR4       KH  	  R#   + '       g   i     K]  ; i)c                 j    \        V 4      p\        \        P                  ! VP                  4      4      # r  )r   rE   r   rh  r/   )r#  
the_strings   & r'   
get_tokens5CTokenizeTest.test_invalid_syntax.<locals>.get_tokens  s(    !&)JBB:CVCVWXXr)   z(1+2]z(1+2}z{1+2]1_z1.2_1e2_z1e+    u   €0b120b1_20b20b1_0b0o180o1_80o80o1_0o0x1_0x012z'sdfsdfz'''sdfsdf'']z6            f'__{
                x:d
            }__')r  Na  ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((a)))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))r  rC   r   r   )r3   r*  r  s   &  r'   test_invalid_syntax!CTokenizeTest.test_invalid_syntax  s   	Y$
$
$
 $
 	$

 $
 $
 $
 $
 $
 $
 $
 $
 $
 $
  !$
" #$
$ %$
& '$
( )$
* +$
, -$
. /$
0 1$
2 3$
4 5$
6 7$
8 9$
: ;$
< "=$
> ?$
@A$
DJ 4((!!("5"5zH )(K$
J )((s   !!BB!c                
  a ^dpR pV! V^,
          4      p\        V4      o\        \        P                  ! SP                  4      4      pV P                  VR,          P                  \        P                  4       V P                  VR,          P                  \        P                  4       \        VRR4       V! V4      p\        V4      oV P                  \        V3R l4       V P                  \        \        VRR4       R# )d   c                 v    R P                  R \        V 4       4       4      pVRV ,          R,           ,          pV# )ru  c              3   B   "   T F  pR V,          R,           x  K  	  R# 5i)  z	if True:
Nr=   ).0rL   s   & r'   	<genexpr>ICTokenizeTest.test_max_indent.<locals>.generate_source.<locals>.<genexpr>  s     O1dQh,66s   rC  zpass
)rA  r"  )indentsr  s   & r'   generate_source6CTokenizeTest.test_max_indent.<locals>.generate_source  s2    WWOgOOFdWnx//FMr)   rR   execc                  V   < \        \        P                  ! S P                  4      4      # r  )rE   r   rh  r/   )	the_inputs   r'   <lambda>/CTokenizeTest.test_max_indent.<locals>.<lambda>  s    D9c9cdmdvdv9w4xr)   NrM   r   )r   rE   r   rh  r/   r0   r"   DEDENTr   r  rC   rT   )r3   	MAXINDENTrH  validrN   invalidrL  s   &     @r'   test_max_indentCTokenizeTest.test_max_indent  s    		
  	A.UO	h@@ASASTU(//:(*<*<=z6*!),W%	*,xygw
F	
r)   c                   R  p\        R4      pV P                  VR4       \        R4      pV P                  V! V4      V! V4      4       \        R4      pV P                  VR4       \        R4      pV P                  V! V4      V! V4      4       \        R4      pV P                  VR4       \        R	4      pV P                  V! V4      V! V4      4       R
# )c                     \        V 4      p\        P                  ! VP                  4       UU Uu. uF  vr q2V 3NK
  	  upp p# u upp pi r  )r   r   rh  r/   )r#  r)  kindr(  s   &   r'   r*  ECTokenizeTest.test_continuation_lines_indentation.<locals>.get_tokens  sL    !&)JAA*BUBUVXV (:q6NVX X Xs   Az
            def fib(n):
                \
            '''Print a Fibonacci series up to n.'''
                \
            a, b = 0, 1
        ae      NAME       'def'         (2, 0) (2, 3)
    NAME       'fib'         (2, 4) (2, 7)
    LPAR       '('           (2, 7) (2, 8)
    NAME       'n'           (2, 8) (2, 9)
    RPAR       ')'           (2, 9) (2, 10)
    COLON      ':'           (2, 10) (2, 11)
    NEWLINE    ''            (2, 11) (2, 11)
    INDENT     ''            (4, -1) (4, -1)
    STRING     "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39)
    NEWLINE    ''            (4, 39) (4, 39)
    NAME       'a'           (6, 0) (6, 1)
    COMMA      ','           (6, 1) (6, 2)
    NAME       'b'           (6, 3) (6, 4)
    EQUAL      '='           (6, 5) (6, 6)
    NUMBER     '0'           (6, 7) (6, 8)
    COMMA      ','           (6, 8) (6, 9)
    NUMBER     '1'           (6, 10) (6, 11)
    NEWLINE    ''            (6, 11) (6, 11)
    DEDENT     ''            (6, -1) (6, -1)
        zu
            def fib(n):
                '''Print a Fibonacci series up to n.'''
                a, b = 0, 1
        z>
            pass
                \

            pass
        z    NAME       'pass'        (2, 0) (2, 4)
    NEWLINE    ''            (2, 4) (2, 4)
    NAME       'pass'        (5, 0) (5, 4)
    NEWLINE    ''            (5, 4) (5, 4)
        z+
            pass
            pass
        z
            if x:
                y = 1
                \
                        \
                    \
                \
                foo = 1
        aj      NAME       'if'          (2, 0) (2, 2)
    NAME       'x'           (2, 3) (2, 4)
    COLON      ':'           (2, 4) (2, 5)
    NEWLINE    ''            (2, 5) (2, 5)
    INDENT     ''            (3, -1) (3, -1)
    NAME       'y'           (3, 4) (3, 5)
    EQUAL      '='           (3, 6) (3, 7)
    NUMBER     '1'           (3, 8) (3, 9)
    NEWLINE    ''            (3, 9) (3, 9)
    NAME       'foo'         (8, 4) (8, 7)
    EQUAL      '='           (8, 8) (8, 9)
    NUMBER     '1'           (8, 10) (8, 11)
    NEWLINE    ''            (8, 11) (8, 11)
    DEDENT     ''            (8, -1) (8, -1)
        zI
            if x:
                y = 1
                foo = 1
        N)r   r7   r0   )r3   r*  r  code_no_conts   &   r'   #test_continuation_lines_indentation1CTokenizeTest.test_continuation_lines_indentation  s   	X
    	D # 	,    	D):l+CD   	D # 	   
 	D):l+CD   	D # 	"    	D):l+CDr)   r=   N)r  r  r  r  r7   r
  rn   r{   r   r   r   r   r   r   r   r   r   r   r   r=  r   skip_wasi_stack_overflowrS  rZ  r  r  r  s   @r'   r  r    s     E3(?	B,	\w	r=	~$	L	.	($	L	,	"	"H	T
	+IZ %%'
 (
,bE bEr)   r  c                   &   a  ] tR tRt o R tRtV tR# )CTokenizerBufferTestsiO  c                    R RR,           RRR,           2p\         P                  ! 4       ;_uu_ 4       p\        VRV4      p\        V4       RRR4       R#   + '       g   i     R# ; i)z"        #coding: latin-1
        #r  i'  z

        #i'  fooN)r	   r  r   r
   )r3   test_scriptr  	file_names   &   r'   !test_newline_at_the_end_of_buffer7CTokenizerBufferTests.test_newline_at_the_end_of_bufferP  s]    

e) 

e) !!X#He[AII& "!!!s   AA(	r=   N)r  r  r  r  rc  r  r  r  s   @r'   r^  r^  O  s     
' 
'r)   r^  c                   Z   a  ] tR tRt o R t]R 4       tR tR tR t	R t
R tR	 tR
tV tR# )CommandLineTesti]  c                    \         P                  ! 4       V n        V P                  \        P
                  V P                  4       R # r  )tempfilemktemprW   r  r	   r  rm   s   &r'   setUpCommandLineTest.setUp^  s(     )	(($--8r)   c                N    \         P                  ! RRV 4      P                  4       # )zDedent *string* and strip it from its surrounding whitespaces.

This method is used by the other utility functions so that any
string to write or to match against can be freely indented.
z\s+r   )r  subr$  )r#  s   &r'   text_normalizeCommandLineTest.text_normalizeb  s      vvfc6*0022r)   c                    \        V P                  R 4      ;_uu_ 4       pVP                  V4       RRR4       R#   + '       g   i     R# ; i)r  N)r5  rW   r  )r3   contentr  s   && r'   
set_sourceCommandLineTest.set_sourcek  s0    $--%%HHW &%%%s	   ;A	c                   \        4       p\        P                  ! V4      ;_uu_ 4        \        P                  ! . VOV P
                  NR 7       RRR4       V P                  VP                  4       4      #   + '       g   i     L/; i)r  N)r   
contextlibredirect_stdoutr   _mainrW   rn  getvalue)r3   flagsoutputs   &* r'   invoke_tokenizeCommandLineTest.invoke_tokenizeo  s[    ''//NN 7% 7 78 0""6??#455 0/s   &A66B	c                4   V P                  WR 7      ;_uu_ 4        V P                  V4       V P                  ! V!  pV P                  V4      pV P	                  VP                  4       VP                  4       4       RRR4       R#   + '       g   i     R# ; i))r  rz  N)r  rr  r|  rn  assertListEqualr   )r3   r  expectrz  ress   &&&* r'   check_outputCommandLineTest.check_outputu  sn    \\\55OOF#&&.C((0F  !163D3D3FG	 6555s   A!BB	c           	        RpV P                  R4       V F7  pV P                  VR7      ;_uu_ 4        V P                  V4      pRRR4       K9  	  V P                  \        4      ;_uu_ 4        \
        P                  ! \        4       4      ;_uu_ 4        V P                  R4      pRRR4       RRR4       R#   + '       g   i     K  ; i  + '       g   i     L.; i  + '       g   i     R# ; i)-ezS
            def f():
                print(x)
                return None
        ru  Nz	--unknownr  z--exact)rr  r  r|  rC   
SystemExitrv  redirect_stderrr   )r3   
base_flagsflagr(  s   &   r'   test_invocationCommandLineTest.test_invocation|  s    &
  	 D4((((. )(  z**++HJ77((5 8 +* )((
 87 +**s/   B9.'C C	'C 9C
CC  C1	c                0    R pRpV P                  W4       R# )a = 1a=  
            0,0-0,0:            ENCODING       'utf-8'
            1,0-1,1:            NAME           'a'
            1,2-1,3:            OP             '='
            1,4-1,5:            NUMBER         '1'
            1,5-1,6:            NEWLINE        ''
            2,0-2,0:            ENDMARKER      ''
        Nr  )r3   r  r  s   &  r'   test_without_flag!CommandLineTest.test_without_flag  s     	&)r)   c                D    R pRpR F  pV P                  WV4       K  	  R# )r  a=  
            0,0-0,0:            ENCODING       'utf-8'
            1,0-1,1:            NAME           'a'
            1,2-1,3:            EQUAL          '='
            1,4-1,5:            NUMBER         '1'
            1,5-1,6:            NEWLINE        ''
            2,0-2,0:            ENDMARKER      ''
        Nr  r  )r3   r  r  r  s   &   r'   test_exact_flagCommandLineTest.test_exact_flag  s*     &Dfd3 &r)   )rW   N)r  r  r  r  rj  staticmethodrn  rr  r|  r  r  r  r  r  r  r  s   @r'   rf  rf  ]  sB     9 3 36H6&*4 4r)   rf  c                   6   a  ] tR tRt o ]R 4       tR tRtV tR# )StringPrefixTesti  c                     \        4       p \        P                  ! \        P                  \        P
                  4       F1  p \        V R 24       V P                  VP                  4       4       K3  	  \        4       p\        P                  ! 4        F  p^ pR \        P                  ! W4       4        F  p\        P                  ! V4       F  p\        P                  ! V Uu. uF  qwVP                  4       3NK  	  up!   FF  pRP                  V4      p	V	R8X  d   K   \        V	 R 24       VP                  V	4       V^,          pKH  	  K  	  K  	  V^ 8X  g   K  Vu # 	  R#   \         d     EK7  i ; iu upi   \         d     K  i ; i)z""c              3   D   "   T F  pR P                  V4      x  K  	  R# 5i)ru  N)rA  )rD  ls   & r'   rE  <StringPrefixTest.determine_valid_prefixes.<locals>.<genexpr>  s!      SA 

Ss    ru  notN)set	itertoolschainr#  ascii_lowercaseascii_uppercaser/  addlowerrP  countcombinationspermutationsproductupperrA  )
single_char_valid_prefixeslettervalid_prefixeslengthnum_at_this_lengthprefixtcr  ps
             r'   determine_valid_prefixes)StringPrefixTest.determine_valid_prefixes  s^    &)U"  oof&<&<f>T>TUFxr]#*..v||~> V oo'F!""//0JS #//7A&..0KAQWWY0KLGGAJ: %	! A3bN
 +..q1.!3. M 8	* "Q&%%1 (   1L  + ! !s*   -E&$E9!(E>&E65E6>FFc                6   V P                  \        P                  ^ ,          R4       V P                  \        P                  R,          R4       \        \        P                  ^R P	                  R4      4      pV P                  WP                  4       4       R# )r   r   r  r7  Nr   )r0   r   StringPrefixr  r  r  )r3   defined_prefixess   & r'   test_prefixesStringPrefixTest.test_prefixes  sy     	..q137..r2C8 x44Qr:@@EF 	)+H+H+JKr)   r=   N)	r  r  r  r  r  r  r  r  r  r  s   @r'   r  r    s$     0& 0&fL Lr)   r  __main__).rv  r  r?  r  r#  rh  r#   r   unittestior   r   textwrapr   r   r   testr   test.supportr	   test.support.script_helperr
   r   r   test.support.numbersr   r   r(   r+   r  r)  r+  r;  r^  rn  r   r  r  rD  r  r  r^  rf  r  r  mainr=   r)   r'   <module>r     s<     	 	         #  " Y Y W	8 W	r$A A?:x :<#IH #IL<H <.|" |"~v28 v2r&GX &GR. U#H U#p2 28qEH qEh!'H-- 'K4h'' K4\ELx(( ELP zMMO r)   