YflX@s:dZdZdZddlmZddlmZmZddl Z ddl m Z ddl m Z ddlZddlZdd lTejd ejZejd ejZddlZejd d dddddgZ[eZd eed:e?d;e@d<eAd=eBd>eCd?eDd@eEdAeFi,ZGGdBdde jHddCZIdDdEZJdFdGZKdHdIZLdJZMdKZNeMeKdLeMeLeNZOdMZPdNZQdOZRdPZSdQZTeJeQeReSeTZUdRZVeJdSdTeLeVZWdUeVZXeJeWeXZYeJdVeYdWZZeJeZeYeUZ[dXZ\dYZ]dZZ^d[Z_d\Z`eJe\d]e\d^ZaeJe\d_e\d`ZbeJdadbdcd+dddedfd.ZcdgZdeJdhdidjZeeJecedeeZfeJe[efebePZgeOegZheJe\dkeJdldLe\dmeJdndLZieJdoeNeaZjeMeJeje[efeiePZkdpdqZldle]dne^d]e_d^e`dre_dse`dte_due`dve_dwe`dxe_dye`dze_d{e`d|e_d}e`d~e_de`de_de`de_de`de_de`de_de`de_de`de_de`de_de`ddddddddddddi&ZmiZnxdD]ZoeoeneozpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord)open)lookupBOM_UTF8N) TextIOWrapper)chain)*z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s^[ \t\f]*(?:[#\r\n]|$)COMMENTtokenizedetect_encodingNL untokenizeENCODING TokenInfo()[]:,;+-r/|&<>=.%{}z==z!=z<=z>=~^z<>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=@z@=c@s.eZdZddZeddZdS)rcCs.d|jt|jf}d|jd|S)Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)type)r(tok_name_replace)selfannotated_typer--/opt/alt/python35/lib64/python3.5/tokenize.py__repr__cszTokenInfo.__repr__cCs4|jtkr)|jtkr)t|jS|jSdS)N)r(OPstringEXACT_TOKEN_TYPES)r+r-r-r. exact_typehs zTokenInfo.exact_typeN)__name__ __module__ __qualname__r/propertyr3r-r-r-r.rbs  ztype string start end linecGsddj|dS)Nrrr)join)choicesr-r-r.grouposr:cGst|dS)Nr)r:)r9r-r-r.anypsr;cGst|dS)N?)r:)r9r-r-r.maybeqsr=z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX][0-9a-fA-F]+z 0[bB][01]+z 0[oO][0-7]+z(?:0+|[1-9][0-9]*)z[eE][-+]?[0-9]+z[0-9]+\.[0-9]*z\.[0-9]+z[0-9]+z [0-9]+[jJ]z[jJ]z(?:[bB][rR]?|[rR][bB]?|[uU])?z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'''"""z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z//=?z->z[+\-*/%&@|^=<>]=?z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*"z \\\r?\n|\ZcCstj|tjS)N)recompileUNICODE)exprr-r-r._compilesrFr'''r"""b'''b"""R'''R"""B'''B"""br'''br"""bR'''bR"""Br'''Br"""BR'''BR"""rb'''rb"""Rb'''Rb"""rB'''rB"""RB'''RB"""u'''u"""U'''U"""rRbBuUr'r"R'R"b'b"B'B"br'br"Br'Br"bR'bR"BR'BR"rb'rb"rB'rB"Rb'Rb"RB'RB"u'u"U'U"c@seZdZdS) TokenErrorN)r4r5r6r-r-r-r.rs rc@seZdZdS)StopTokenizingN)r4r5r6r-r-r-r.rs rc@s@eZdZddZddZddZddZd S) UntokenizercCs(g|_d|_d|_d|_dS)Nrr)tokensprev_rowprev_colencoding)r+r-r-r.__init__s   zUntokenizer.__init__cCs|\}}||jks9||jkr]||jkr]tdj|||j|j||j}|r|jjd|d|_||j}|r|jjd|dS)Nz+start ({},{}) precedes previous end ({},{})z\ r )rr ValueErrorformatrappend)r+startrowcol row_offset col_offsetr-r-r.add_whitespaces -    zUntokenizer.add_whitespacec Cst|}g}d}x}|D]u}t|dkrH|j||P|\}}}} } |tkru||_q|tkrP|tkr|j|qn|tkr|j | \|_ |_ qnl|t t fkrd}nQ|r;|r;|d} |dt| kr5|jj| t| |_ d}|j||jj|| \|_ |_ |t t fkr|j d7_ d|_ qWdj|jS)NFrTrr)iterlencompatr r ENDMARKERINDENTrDEDENTpoprrNEWLINEr rrr8) r+iterableitindents startlinettok_typetokenrendlineindentr-r-r.r sF              zUntokenizer.untokenizec CsJg}|jj}|dttfk}d}xt|g|D]}|dd\}} |tkru| |_qA|ttt t fkr| d7} |t kr|rd| } d}nd}|t kr|j| qAnW|t kr|jqAn;|ttfkrd}n |r8|r8||dd}|| qAWdS)NrFrrTrr)rrrr rr rNAMENUMBERASYNCAWAITSTRINGrrr) r+rrr toks_appendr prevstringtoktoknumtokvalr-r-r.rs8             zUntokenizer.compatN)r4r5r6rrr rr-r-r-r.rs   %rcCs=t}|j|}|jdk r9|j|j}|S)aTransform tokens back into Python source code. It returns a bytes object, encoded using the ENCODING token, which is the first token sequence output by tokenize. Each element returned by the iterable must be a token sequence with at least two elements, a token number and token value. If only two tokens are passed, the resulting output is poor. Round-trip invariant for full input: Untokenized source will match input source exactly Round-trip invariant for limited input: # Output bytes will tokenize back to the input t1 = [tok[:2] for tok in tokenize(f.readline)] newcode = untokenize(t1) readline = BytesIO(newcode).readline t2 = [tok[:2] for tok in tokenize(readline)] assert t1 == t2 N)rr rencode)rutoutr-r-r.r ?s  cCsd|ddjjdd}|dks=|jdrAdS|d ks\|jdr`dS|S)z(Imitates get_normal_name in tokenizer.c.N _rzutf-8zutf-8-latin-1 iso-8859-1 iso-latin-1latin-1- iso-8859-1- iso-latin-1-)rrr)rrr)lowerreplace startswith)orig_encencr-r-r._get_normal_nameZs" rc s1yjjWntk r*dYnXdd}d}fdd}fdd}|}|jtrd|d d}d }|s|gfS||}|r||gfStj|s||gfS|}|s||gfS||}|r!|||gfS|||gfS) a The detect_encoding() function is used to detect the encoding that should be used to decode a Python source file. It requires one argument, readline, in the same way as the tokenize() generator. It will call readline a maximum of twice, and return the encoding used (as a string) and a list of any lines (left as bytes) it has read in. It detects the encoding from the presence of a utf-8 bom or an encoding cookie as specified in pep-0263. If both a bom and a cookie are present, but disagree, a SyntaxError will be raised. If the encoding cookie is an invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, 'utf-8-sig' is returned. If no encoding is specified, then the default of 'utf-8' will be returned. NFzutf-8c s(y SWntk r#dSYnXdS)N) StopIterationr-)readliner-r. read_or_stop}s  z%detect_encoding..read_or_stopcs2y|jd}WnBtk rWd}dk rGdj|}t|YnXtj|}|sqdSt|jd}yt|}WnIt k rdkrd|}ndj|}t|YnXr.|dkr$dkr d}ndj}t||d 7}|S) Nzutf-8z'invalid or missing encoding declarationz {} for {!r}rzunknown encoding: zunknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig) decodeUnicodeDecodeErrorr SyntaxError cookie_rematchrr:r LookupError)r line_stringmsgrrcodec) bom_foundfilenamer-r. find_cookies6            z$detect_encoding..find_cookieTrz utf-8-sig)__self__nameAttributeErrorrrblank_rer)rrdefaultrrfirstsecondr-)rrrr.r es8  &        c Csst|d}yHt|j\}}|jdt||dd}d|_|SWn|jYnXdS)zXOpen a file in read only mode using the encoding detected by detect_encoding(). rbrline_bufferingTrcN) _builtin_openr rseekrmodeclose)rbufferrlinestextr-r-r.rs   rcCs_ddlm}m}t|\}}t|d}|d}t||||j|S)a The tokenize() generator requires one argument, readline, which must be a callable object which provides the same interface as the readline() method of built-in file objects. Each call to the function should return one line of input as bytes. Alternatively, readline can be a callable function terminating with StopIteration: readline = open(myfile, 'rb').__next__ # Example of alternate readline The generator produces 5-tuples with these members: the token type; the token string; a 2-tuple (srow, scol) of ints specifying the row and column where the token begins in the source; a 2-tuple (erow, ecol) of ints specifying the row and column where the token ends in the source; and the line on which the token was found. The line passed is the logical line; continuation lines are included. The first token sequence will always be an ENCODING token which tells you which encoding was used to decode the bytes stream. r)rrepeatr) itertoolsrrr r _tokenize__next__)rrrrconsumedrl_genemptyr-r-r.r s  c cs d}}}d}d!\}}d}dg} d} d} d} d} |dk r||dkred}tt|d"d#dVxy |}Wntk rd}YnX|dk r|j|}|d7}dt|}}|r|std ||j|}|rf|jd}}tt||d||||f||Vd$\}}d}q|r|d%dd kr|d&dd krtt ||||t|f|Vd}d}qq||}||}qn|dkr| r|sPd}xv||kr||dkr7|d7}n?||dkr\|t dt }n||dkrud}nP|d7}qW||krP||dkr||dkr=||dj d}|t|}tt |||f||t|f|Vtt ||d||f|t|f|Vqtt t f||dk||d||f|t|f|Vq|| d'kr| j|tt|d||df||f|Vx|| d(kri|| kr tdd|||f| dd)} | rC| | d*krCd} d} d} ttd||f||f|VqW| r| r| | d+krd} d} d} n!|std|dfd}x||krC ttj||}|r |jd\}}||f||f|}}}||kr-q|||||}}||ksx|dkr|dkr|dkrtt||||Vq@ |dkr| r| Vd} |dkrtt ||||Vq tt||||V| r d} q@ |dkrH|jd st| r.| Vd} tt ||||Vq@ |tkrtt|}|j||}|r|jd}|||}tt||||f|Vq ||f}||d}|}Pq@ |tks|dd tks|dd tkr|d,dkr||f}tt|p[t|dp[t|d }||dd}}|}Pq tt||||Vq@ |jr|d-kr| rt|dkrtnt||||Vqtt||||}|dkr| r|} q|dkr| r| j tkr| j!dkrd} | d.} tt| j!| j"| j| j#Vd} | r| Vd} |Vq@ |dkrd}q@ |dkr|d7}n|d kr|d8}| r| Vd} tt$||||Vqtt ||||f||df|V|d7}qWqW| rY | Vd} x;| ddD])}ttd|df|dfdVqj Wtt%d|df|dfdVdS)/Nr 0123456789rFz utf-8-sigzutf-8rrzEOF in multi-line stringrz\ rz\ r  z# #z z3unindent does not match any outer indentation levelz zEOF in multi-line statementr!z...T asyncawaitdef\z([{z)]})rr)rr)rr)rrrrrrrr)rrr)&rr rrrrrrr ERRORTOKENtabsizerstriprr rrIndentationErrorrrF PseudoTokenspanrrendswithAssertionError triple_quotedendpats single_quoted isidentifierrrrr(r1rrr0r) rrlnumparenlev continuednumcharscontstrneedcontcontlinerstashed async_defasync_def_indent async_def_nlrposmaxstrstartendprogendmatchrcolumn comment_tokennl_pos pseudomatchrsposeposrinitialrrr-r-r.rsj           2       $#'  - '    $                        'rcCs t|dS)N)r)rr-r-r.generate_tokenssrc sddl}ddddfdd}|jdd}|jdd d d d d dd|jdddddddd|j}y|jr|j}t|d}tt|j}WdQRXnd}t t j jd}x[|D]S}|j }|j r|j}d|j|j} td| t||jfqWWnbtk r} z?| jddd\} } || jd|| | fWYdd} ~ Xntk r} z5| jd\} } || jd|| | fWYdd} ~ Xntk r8} z|| |WYdd} ~ Xntk rf} z|| WYdd} ~ XnQtk rtdYn6tk r} zd| WYdd} ~ XnXdS)NrcSst|dtjdS)Nfile)printsysstderr)messager-r-r.perrorszmain..perrorcsg|r+|f||f}d|n+|rHd||fnd|tjddS)Nz%s:%d:%d: error: %sz %s: error: %sz error: %sr)rexit)r!rlocationargs)r"r-r.errorszmain..errorprogzpython -m tokenizedestrnargsr<metavarz filename.pyhelpz'the file to tokenize; defaults to stdinz-ez--exactexactaction store_truez(display token names using the exact typerzz %d,%d-%d,%d:z%-20s%-15s%-15rrrz interrupted zunexpected error: %s)argparseArgumentParser add_argument parse_argsrrlistr rrrstdinr(r,r3rrrr)r1rr%rrOSErrorKeyboardInterrupt Exception) r/r&parserr%rfrr token_type token_rangeerrrrr-)r"r.mainsN           // r=__main__)r>r?rGrHrKrLrIrJrMrNrOrPrSrTrQrRrUrVrWrXr[r\rYrZr]r^r_r`rarb)r@rArirjrkrlrmrnrorprqrrrsrtrurvrwrxryrzr{r|r}r~rrrrrr)~__doc__ __author__ __credits__builtinsrrcodecsrr collectionsiorrrrBrrrCASCIIrr__all__N_TOKENSrr)r r LPARRPARLSQBRSQBCOLONCOMMASEMIPLUSMINUSSTARSLASHVBARAMPERLESSGREATEREQUALDOTPERCENTLBRACERBRACEEQEQUALNOTEQUAL LESSEQUAL GREATEREQUALTILDE CIRCUMFLEX LEFTSHIFT RIGHTSHIFT DOUBLESTAR PLUSEQUALMINEQUAL STAREQUAL SLASHEQUAL PERCENTEQUAL AMPEREQUAL VBAREQUALCIRCUMFLEXEQUALLEFTSHIFTEQUALRIGHTSHIFTEQUALDOUBLESTAREQUAL DOUBLESLASHDOUBLESLASHEQUALATATEQUALr2 namedtuplerr:r;r= WhitespaceCommentIgnoreName Hexnumber Binnumber Octnumber Decnumber IntnumberExponent PointfloatExpfloat Floatnumber ImagnumberNumber StringPrefixSingleDoubleSingle3Double3TripleStringOperatorBracketSpecialFunny PlainTokenTokenContStr PseudoExtrasrrFrrrrrr7rrrr rr r rrr=r4r-r-r-r.sD            "                             _  ]    <