3
\8s � - @ s� d Z dZddlmZ ddlmZmZ ddlZddl m
Z
ddl m Z ddl Z
ddlZddlZddlT ejd ej�Zejd
ej�ZddlZejd d d
ddddg Z[eZd ee< ed Zdee< ed Zdee< ed7 Zee e e e e e!e"e#e$e%e&e'e(e)e*e+e,e-e.e/e0e1e2e3e4e5e6e7e8e9e:e;ee?e@eAeBeCeDeEeFd�,ZGG dd� dejHdd��ZIdd� ZJdd� ZKd d � ZLd ZMd ZNeMeKd eM � eLeN� ZOd!ZPd"ZQd#ZRd$ZSd%ZTeJeQeReSeT�ZUd&ZVeJd'd(�eLeV� ZWd)eV ZXeJeWeX�ZYeJd*eYd+ �ZZeJeZeYeU�Z[d,d-� Z\d.d/� Z]eJe\� � Z^d0Z_d1Z`d2Zad3ZbeJe^d4 e^d5 �ZceJe^d6 e^d7 �ZdeJd8d9d:d;dd?�Zed@ZfeJdAdBdC�ZgeJeeefeg�ZheJe[ehedeP�ZieOei ZjeJe^dD eJdEd � e^dF eJdGd � �ZkeJdHeNec�ZleMeJele[ehekeP� Zmi Znx@e\� D ]6Zoe_eneodE < e`eneodG < eaeneod4 < ebeneod5 < �q�W ep� Zqep� Zrx\e\� D ]RZsx$esdG esdE fD ]Zteqjuet� �q4W x$esd5 esd4 fD ]Zterjuet� �qZW �q W dIZvG dJdK� dKew�ZxG dLdM� dMew�ZyG dNdO� dO�ZzdPd� Z{dQdR� Z|dSd
� Z}dTdU� ZdVd � Z~dWdX� ZdYdZ� Z�d[d\� Z�e�d]k�r�e�� dS )^zKa-Ping Yee zpGvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro, Raymond Hettinger, Trent Nelson, Michael Foord� )�open)�lookup�BOM_UTF8N)�
TextIOWrapper)�chain)�*z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)�COMMENT�tokenize�detect_encoding�NL�
untokenize�ENCODING� TokenInfo� � � ),�(�)�[�]�:�,�;�+�-r �/�|�&�<�>�=�.�%�{�}z==z!=z<=z>=�~�^z<>z**z+=z-=z*=z/=z%=z&=z|=z^=z<<=z>>=z**=z//z//=�@z@=c @ s e Zd Zdd� Zedd� �ZdS )r c C s$ d| j t| j f }d| j|d� S )Nz%d (%s)z8TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r))�type)r( �tok_name�_replace)�self�annotated_type� r- � /usr/lib64/python3.6/tokenize.py�__repr__d s zTokenInfo.__repr__c C s( | j tkr | jtkr t| j S | j S d S )N)r( �OP�string�EXACT_TOKEN_TYPES)r+ r- r- r. �
exact_typei s
zTokenInfo.exact_typeN)�__name__�
__module__� __qualname__r/ �propertyr3 r- r- r- r. r c s ztype string start end linec G s ddj | � d S )Nr r r )�join)�choicesr- r- r. �groupp s r: c G s t | � d S )Nr )r: )r9 r- r- r. �anyq s r; c G s t | � d S )N�?)r: )r9 r- r- r. �mayber s r= z[ \f\t]*z #[^\r\n]*z\\\r?\nz\w+z0[xX](?:_?[0-9a-fA-F])+z0[bB](?:_?[01])+z0[oO](?:_?[0-7])+z (?:0(?:_?0)*|[1-9](?:_?[0-9])*)z[eE][-+]?[0-9](?:_?[0-9])*z)[0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?z\.[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*z[0-9](?:_?[0-9])*[jJ]z[jJ]c C sn ddddddg} t dg�}xN| D ]F}x@tj|�D ]2}x,tjdd � |D �� D ]}|jdj|�� qJW q0W q W |S )
N�b�r�u�f�br�fr� c S s g | ]}||j � f�qS r- )�upper)�.0�cr- r- r. �
� s z(_all_string_prefixes..)�set�
_itertools� permutations�product�addr8 )�_valid_string_prefixes�result�prefix�tr@ r- r- r. �_all_string_prefixes� s
rR c C s t j| t j�S )N)�re�compile�UNICODE)�exprr- r- r. �_compile� s rW z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z'''z"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z!=z//=?z->z[+\-*/%&@|^=<>]=?r% z[][(){}]z\r?\nz\.\.\.z[:;.,@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*�'z"[^\n"\\]*(?:\\.[^\n"\\]*)*�"z
\\\r?\n|\Z� c @ s e Zd ZdS )�
TokenErrorN)r4 r5 r6 r- r- r- r. r[ � s r[ c @ s e Zd ZdS )�StopTokenizingN)r4 r5 r6 r- r- r- r. r\ � s r\ c @ s, e Zd Zdd� Zdd� Zdd� Zdd� Zd S )
� Untokenizerc C s g | _ d| _d| _d | _d S )Nr r )�tokens�prev_row�prev_col�encoding)r+ r- r- r. �__init__� s zUntokenizer.__init__c C s� |\}}|| j k s&|| j kr>|| jk r>tdj||| j | j���|| j }|rb| jjd| � d| _|| j }|r�| jjd| � d S )Nz+start ({},{}) precedes previous end ({},{})z\
r � )r_ r` �
ValueError�formatr^ �append)r+ �start�row�col�
row_offset�
col_offsetr- r- r. �add_whitespace� s
zUntokenizer.add_whitespacec C s4 t |�}g }d}�x|D �]
}t|�dkr8| j||� P |\}}}} }
|tkrV|| _q|tkr`P |tkrv|j|� qnl|tkr�|j � | \| _
| _ qnL|t t
fkr�d}n:|r�|r�|d } |d t| �kr�| jj| � t| �| _ d}| j|� | jj|� | \| _
| _ |t t
fkr| j
d7 _
d| _ qW dj| j�S )NFr Tr r rD �����)�iter�len�compatr
ra � ENDMARKER�INDENTrf �DEDENT�popr_ r` �NEWLINEr r^ rl r8 ) r+ �iterable�it�indents� startlinerQ �tok_type�tokenrg �end�line�indentr- r- r. r � sF
zUntokenizer.untokenizec
C s� g }| j j}|d ttfk}d}x�t|g|�D ]�}|d d� \}} |tkrR| | _q.|ttt t
fkrj| d7 } |t kr�|r~d| } d}nd}|t kr�|j| � q.n>|t
kr�|j� q.n*|ttfkr�d}n|r�|r�||d � d}|| � q.W d S )Nr Fr rc Tr rm )r^ rf ru r r r
ra �NAME�NUMBER�ASYNC�AWAIT�STRINGrr rs rt )
r+ r{ rv rx � toks_appendry �
prevstring�tok�toknum�tokvalr- r- r. rp s8
zUntokenizer.compatN)r4 r5 r6 rb rl r rp r- r- r- r. r] � s
%r] c C s* t � }|j| �}|jd k r&|j|j�}|S )N)r] r ra �encode)rv �ut�outr- r- r. r = s
c C sH | d d� j � jdd�}|dks*|jd�r.dS |d ks@|jd
�rDdS | S )N� �_r zutf-8zutf-8-�latin-1�
iso-8859-1� iso-latin-1�latin-1-� iso-8859-1-� iso-latin-1-)r� r� r� )r� r� r� )�lower�replace�
startswith)�orig_enc�encr- r- r. �_get_normal_nameX s
r� c s� y �j j�W n tk
r$ d �Y nX d� d }d}�fdd�}� �fdd�}|� }|jt�rpd� |dd � }d }|s||g fS ||�}|r�||gfS tj|�s�||gfS |� }|s�||gfS ||�}|r�|||gfS |||gfS )
NFzutf-8c s y� � S t k
r dS X d S )N� )�
StopIterationr- )�readliner- r. � read_or_stop{ s z%detect_encoding..read_or_stopc s� y| j d�}W n4 tk
rB d}�d k r6dj|��}t|��Y nX tj|�}|sVd S t|jd��}y t|�}W n: t k
r� �d kr�d| }n dj�|�}t|��Y nX � r�|dkr؈d kr�d}n
dj��}t|��|d 7 }|S )
Nzutf-8z'invalid or missing encoding declarationz {} for {!r}r zunknown encoding: z unknown encoding for {!r}: {}zencoding problem: utf-8z encoding problem for {!r}: utf-8z-sig)
�decode�UnicodeDecodeErrorre � SyntaxError� cookie_re�matchr� r: r � LookupError)r} � line_string�msgr� ra �codec)� bom_found�filenamer- r. � find_cookie� s6
z$detect_encoding..find_cookieTr z utf-8-sig)�__self__�name�AttributeErrorr� r �blank_rer� )r� ra �defaultr� r� �first�secondr- )r� r� r� r. r
c s8
&
c C sV t | d�}y0t|j�\}}|jd� t||dd�}d|_|S |j� � Y nX d S )N�rbr T)�line_bufferingr? )�
_builtin_openr
r� �seekr �mode�close)r� �bufferra �lines�textr- r- r. r � s
r c C sB ddl m}m} t| �\}}t| d�}|d�}t||||�j|�S )Nr )r �repeatr� )� itertoolsr r� r
rn � _tokenize�__next__)r� r r� ra �consumed�rl_gen�emptyr- r- r. r � s
c ! c s d } }}d}d!\}}d }dg} d }
d} d} d}
|d k rX|dkrFd}t t|d"d#d�V d}d}�x
y|}| � }W n tk
r� d}Y nX |d k r�|j|�}|d7 }dt|� }}|�r�|s�td |��|j|�}|�r|jd� }}t t||d |� |||f|| �V d$\}}d }nf|�rn|d%d � d k�rn|d&d � d
k�rnt t || ||t|�f|�V d}d }qdn|| }|| }qd�n@|dk�r�| �r�|�s�P d}xf||k �r|| dk�r�|d7 }n6|| dk�r�|t
d t
}n|| dk�r�d}nP |d7 }�q�W ||k�rP || dk�r�|| dk�r�||d � j d�}|t|� }t t |||f||t|� f|�V t t
||d � ||f|t|�f|�V qdt t
t f|| dk ||d � ||f|t|�f|�V qd|| d' k�r | j|� t t|d |� |df||f|�V xv|| d( k �r�|| k�r8tdd|||f��| d d)� } | �rd| | d* k�rdd} d}
d} t td||f||f|�V �qW | �r�|
�r�| | d+ k�r�d} d}
d} n|�s�td|df��d}�x�||k �rhtt�j||�}|�r:|jd�\}}||f||f| }} }||k�r �q�|||� || } } | |k�sZ| dk�rp| dk�rp| dk�rpt t| || |�V �qd| dk�r�|
�r�|
V d }
|dk�r�t t
| || |�V n t t| || |�V | �r8d}
�qd| dk�r�|
�r�|
V d }
t t | || |�V �qd| tk�rltt| �}|j||�}|�rN|jd�}|||� } t t| |||f|�V n||f}||d � }|}P �qd| tk�s�| d d
� tk�s�| d d � tk�r| d, dk�r�||f}ttj| ��p�tj| d ��p�tj| d
��}||d � d }}|}P nt t| || |�V �qd| j� �r�| d-k�rL| �rLt | dk�r8t nt | || |�V �q�t t | || |�} | dk�rv|
�rv| }
�q�| d k�r�|
�r�|
j t k�r�|
j dk�r�d} | d. } t t |
j |
j!|
j|
j"�V d }
|
�r�|
V d }
| V nX| d k�r�d}nH| d k�r|d7 }n| d k�r|d8 }|
�r&|
V d }
t t#| || |�V n*t t || ||f||d f|�V |d7 }�q�W qdW |
�r~|
V d }
|�r�|d/ dk�r�t td|d t|�f|d t|�d fd�V x0| dd � D ] } t td|df|dfd�V �q�W t t$d|df|dfd�V d S )0Nr �
0123456789rD Fz utf-8-sigzutf-8r� r zEOF in multi-line stringr z\
r z\
rc � � z#
�#z
z3unindent does not match any outer indentation levelz
zEOF in multi-line statementr! z...T�
�async�await�def�\z([{z)]})rD r )r r )r r )rD r ����������rm rm rm rm rm rm )r� r� rm rm )%r r
r� r� ro r[ r� r| r� �
ERRORTOKEN�tabsize�rstripr r rf rr �IndentationErrorrs rW � PseudoToken�spanr� ru �
triple_quoted�endpats�
single_quoted�get� isidentifierr� r� r r( r1 rg r} r0 rq )!r� ra �lnum�parenlev� continued�numchars�contstr�needcont�contlinerx �stashed� async_def�async_def_indent� async_def_nl� last_liner} �pos�max�strstart�endprog�endmatchr| �column�
comment_token�nl_pos� pseudomatchrg �spos�eposr{ �initialr� r~ r- r- r. r� � sr
*
"
. r� c C s
t | d �S )N)r� )r� r- r- r. �generate_tokens� s r� c
s( dd l } dd� � d� fdd� }| jdd�}|jdd d
d d � |jd
ddddd� |j� }y�|jr�|j}t|d��}tt|j��}W d Q R X nd}t t
j jd �}xF|D ]>}|j }|j
r�|j}d|j|j } td| t| |jf � q�W W �n8 tk
�r: }
z2|
jd dd� \} } ||
jd || | f� W Y d d }
~
X n� tk
�r� }
z*|
jd \} } ||
jd || | f� W Y d d }
~
X n� tk
�r� }
z||
|� W Y d d }
~
X nx tk
�r� }
z||
� W Y d d }
~
X nN tk
�r� td� Y n2 tk
�r" }
z� d|
� � W Y d d }
~
X nX d S ) Nr c S s t | tjd� d S )N)�file)�print�sys�stderr)�messager- r- r. �perror� s zmain..perrorc sR |r"|f| | f }� d| � n"|r8� d|| f � n � d| � t jd� d S )Nz%s:%d:%d: error: %sz
%s: error: %sz error: %sr )r� �exit)r� r� �location�args)r� r- r. �error� s zmain..errorzpython -m tokenize)�progr� r< z filename.pyz'the file to tokenize; defaults to stdin)�dest�nargs�metavar�helpz-ez--exact�exact�
store_truez(display token names using the exact type)r� �actionr� r� zz %d,%d-%d,%d:z%-20s%-15s%-15rr r z interrupted
zunexpected error: %s)NN)�argparse�ArgumentParser� add_argument�
parse_argsr� r� �listr r� r� r� �stdinr( r� r3 rg r| r� r) r1 r� r� r[ r� �OSError�KeyboardInterrupt� Exception)
r r� �parserr� r� rA r^ r{ �
token_type� token_range�errr} r� r- )r� r. �main� sN
&& r �__main__)��
__author__� __credits__�builtinsr r� �codecsr r � collections�ior r� r rJ rS r� r{ rT �ASCIIr� r� �__all__�N_TOKENSr r) r r
�LPAR�RPAR�LSQB�RSQB�COLON�COMMA�SEMI�PLUS�MINUS�STAR�SLASH�VBAR�AMPER�LESS�GREATER�EQUAL�DOT�PERCENT�LBRACE�RBRACE�EQEQUAL�NOTEQUAL� LESSEQUAL� GREATEREQUAL�TILDE�
CIRCUMFLEX� LEFTSHIFT�
RIGHTSHIFT�
DOUBLESTAR� PLUSEQUAL�MINEQUAL� STAREQUAL�
SLASHEQUAL� PERCENTEQUAL�
AMPEREQUAL� VBAREQUAL�CIRCUMFLEXEQUAL�LEFTSHIFTEQUAL�RIGHTSHIFTEQUAL�DOUBLESTAREQUAL� DOUBLESLASH�DOUBLESLASHEQUAL�AT�ATEQUALr2 �
namedtupler r: r; r= �
Whitespace�Comment�Ignore�Name� Hexnumber� Binnumber� Octnumber� Decnumber� Intnumber�Exponent�
Pointfloat�Expfloat� Floatnumber�
Imagnumber�NumberrR rW � StringPrefix�Single�Double�Single3�Double3�Triple�String�Operator�Bracket�Special�Funny�
PlainToken�Token�ContStr� PseudoExtrasr� r� �_prefixrI r� r� rQ r@ rM r� r
r[ r\ r] r r� r
r r� r� r r4 r- r- r- r. � s
_ ] x<