U Zf8@s~ddlmZmZmZmZddlZddlZddlmZdZ e rDddl Z Gddde Z e Z Gddde Zdd Zd d ZdS) )absolute_importdivisionprint_functionunicode_literalsNFc@seZdZdZddZdS) MultiTokenzdObject used to monkeypatch ply.lex so that we can return multiple tokens from one lex operation.cCs|dj|_||_dS)Nr)typetokens)selfrr 8/tmp/pip-unpacked-wheel-vqb3mfeu/stone/frontend/lexer.py__init__s zMultiToken.__init__N)__name__ __module__ __qualname____doc__r r r r r r src@sxeZdZdZdZddZddZddZd d Zd Z e d 7Z e d 7Z e d7Z e d7Z e d7Z e d7Z e d7Z dZ dZ dZ dZ dZdZdZdZdZdZdddd d!d"d#d$d%d&d'd(d)d*d+d,d-gZd.d/d0d1d2d3d4d5d6d7d8d9d: Ze ee7Z d;d<Zd=d>Zd?d@ZdAdBZdCdDZdEdFZdGdHZdIdJZdKdLZ dMdNZ!dOdPZ"dQdRZ#dSdTZ$dUdVZ%dWdXZ&dYdZZ'd[Z(d\d]Z)d^S)_Lexerz' Lexer. Tokenizes stone files. ))WSIGNOREZ inclusivecCs.d|_d|_d|_td|_d|_g|_dS)Nzstone.stone.lexer)lex tokens_queue cur_indentlogging getLogger_logger last_tokenerrors)r r r r r "s  zLexer.__init__cKs6tjfd|i||_g|_d|_|j|ddS)z Required by ply.yacc for this to quack (duck typing) like a ply lexer. :param str file_data: Contents of the file to lex. moduler N)rrrinput)r Z file_datakwargsr r r r,sz Lexer.inputcCs|jr|jd|_n|j}t|trH|j|j|jd|_n|dkr|j dkr|jr|jj dkrt dd|jj |jj }|j||j }t dd|jj |jj }|j|g|d|_ |jd|_n||_|jS)zf Returns the next LexToken. Returns None when all tokens have been exhausted. rN)NEWLINEZLINErrDEDENT )rpoprrtoken isinstancerextendrrr _create_tokenlinenolexposappend)r r newline_tokenZ dedent_countZ dedent_tokenr r r r#9s:    z Lexer.tokencCs,|||}|sq(|jd|q dS)z;Logs all tokens for human inspection. Useful for debugging.zToken %rN)rr#rdebug)r datar#r r r testXs  z Lexer.test)ZIDKEYWORDPATHDOT)r INDENTr)COMMAZEQLPARRPAR)ZBOOLEANFLOATZINTEGERZNULLSTRING)LBRACKETRBRACKET)LBRACERBRACECOLON)Q)ATz\.z\[z\]=,z\?z\{z\}z\:@alias annotationannotation_typeattrsby deprecateddocZexampleerrorextendsimport namespacepatchroutestructunion union_closedZ ANNOTATIONZANNOTATION_TYPEATTRSZ DEPRECATEDZBYZEXTENDSZIMPORTPATCHZROUTEZSTRUCTZUNIONZ UNION_CLOSED) rCrDrErGrFrJrKrMrNrOrPrQcCs|jd|S)z\(r)lexerZ push_stater r#r r r t_LPARs z Lexer.t_LPARcCs|j|S)z\))rTZ pop_staterUr r r t_RPARs z Lexer.t_RPARcCs|jdk|_|S)z\btrue\b|\bfalse\btrue)valuerUr r r t_ANY_BOOLEANs zLexer.t_ANY_BOOLEANcCs t|_|S)z\bnull\b) NullTokenrYrUr r r t_ANY_NULLszLexer.t_ANY_NULLcCs>|j|jkr6|jdkr |jr |S|j|jd|_|S|SdS)z[a-zA-Z_][a-zA-Z0-9_-]*rDr/N)rYKEYWORDSrRESERVEDgetrrUr r r t_ANY_IDs   zLexer.t_ANY_IDcCs|S)z\/[/a-zA-Z0-9_-]*r rUr r r t_ANY_PATHszLexer.t_ANY_PATHcCst|j|_|S)z-?\d+(\.\d*(e-?\d+)?|e-?\d+))floatrYrUr r r t_ANY_FLOATs zLexer.t_ANY_FLOATcCst|j|_|S)z-?\d+)intrYrUr r r t_ANY_INTEGERs zLexer.t_ANY_INTEGERcsd}|jj|jd7_|jdd}d}tdt|D]N}||}|rt|dkrZd}n |dkrfd}||7}d}q<|d krd}q<||7}qsz&Lexer.t_ANY_STRING..) rTr'rYcountrangelen_indent_level_to_spaces_countr splitlinesjoin)r rjescapedsZnew_stricZlines_without_indentationr rpr t_ANY_STRINGs,   zLexer.t_ANY_STRINGcCs|jj|jd7_|jd}|dkr|jj|dk}| oN|jj|dk}|sX|rtdd|j|jt|jd}|j|_||}|r|S|r|r|j d||S|S|d8}q"dS) [#][^\n]*\n+rrfrrlrN) rTr'rYrrr(lexdatar&rt!_create_tokens_for_next_line_dentrinsert)r r#rzZis_full_line_commentZis_partial_line_commentr+ dent_tokensr r r t_INITIAL_comment%s0 zLexer.t_INITIAL_commentcCsN|jj|jd7_tdd|j|jt|jd}|j|_||dS)r}rrrfN)rTr'rYrrr&r(rt_check_for_indent)r r#r+r r r t_WSIGNORE_commentCszLexer.t_WSIGNORE_commentcCs@|jj|jd7_||}|r8|jd||S|SdS)\n+rrN)rTr'rYrrrrr)r r+rr r r t_INITIAL_NEWLINELs  zLexer.t_INITIAL_NEWLINEcCs&|jj|jd7_||dS)rrN)rTr'rYrrr)r r+r r r t_WSIGNORE_NEWLINEVszLexer.t_WSIGNORE_NEWLINEcCsr||}|dks|dkrdS|dkr*dnd}t|d|jd|jt|j}|gt|}|j|7_t|S)z Starting from a newline token that isn't followed by another newline token, returns any indent or dedent tokens that immediately follow. If indentation doesn't change, returns None. Nrr2r r!rf) _get_next_line_indent_deltar&r'r(rtrYabsrr)r r+ indent_deltaZ dent_typeZ dent_tokenrr r r r[s z'Lexer._create_tokens_for_next_line_dentcCs6||}|dks|dkrdS|jd|jjfdS)zw Checks that the line following a newline is indented, otherwise a parsing error is generated. Nrfz-Line continuation must increment indent by 1.)rrr)rTr')r r+rr r r rps zLexer._check_for_indentcCs|jdkstd|jt|j}|t|jjkr6dS|jj|dtj dd}|s\dS| }t|}|dkrxdS|ddkrdSt||}|ddkr|j d|jj fdS|t|j}|dS) aX Returns the change in indentation. The return units are in indentations rather than spaces/tabs. If the next line's indent isn't relevant (e.g. it's a comment), returns None. Since the return value might be 0, the caller should explicitly check the return type, rather than rely on truthiness. rz3Can only search for a dent starting from a newline.Nrfr#zIndent is not divisible by 4.)rAssertionErrorr(rtrYrTr~splitoslineseplstriprr)r'rur)r r+Z next_line_posroZlstripped_lineZlstripped_line_lengthindentrr r r rs.     z!Lexer._get_next_line_indent_deltaz cCsR|jd|jd|jj|jdt|jdd|jjf|j ddS)NzIllegal character %r at line %drzIllegal character %s.urf) rr,rYrTr'rr)reprrskiprUr r r t_ANY_errorszLexer.t_ANY_errorN)*r rrrZstatesr rr#r.rZt_DOTZ t_LBRACKETZ t_RBRACKETZt_EQZt_COMMAZt_QZt_LBRACEZt_RBRACEZt_COLONZt_ATr]r^tuplevaluesrVrWrZr\r`rarcrer|rrrrrrrZt_ignorerr r r r rs    $  &rcCs$t}||_||_||_||_|S)z Helper for creating ply.lex.LexToken objects. Unfortunately, LexToken does not have a constructor defined to make settings these values easy. )rZLexTokenrrYr'r() token_typerYr'r(r#r r r r&s r&cCs|dS)Nrr )rr r r rusru) __future__rrrrrrZply.lexrZ_MYPYtypingobjectrr[rr&rur r r r s