U d>e+Z,Gd?d@d@e&Z-GdAdBdBe&Z.dCdDZ/dS)E)absolute_importdivisionunicode_literals) text_type) http_clienturllibN)BytesIOStringIO)EOFspaceCharacters asciiLettersasciiUppercase)_ReparseException)_utilscCsg|]}|dqSasciiencode.0itemr9/tmp/pip-unpacked-wheel-jqcau075/html5lib/_inputstream.py srcCsg|]}|dqSrrrrrrrscCsg|]}|dqSrrrrrrrs>.)sumr!r#rrrr*XszBufferedStream._bufferedBytescCs<|j|}|j||jdd7<t||jd<|Sr%)r r3r!appendr"r&)r#r2datarrrr/[s   zBufferedStream._readStreamcCs|}g}|jd}|jd}|t|jkr|dkr|dks>t|j|}|t||krl|}|||g|_n"t||}|t|g|_|d7}|||||||8}d}q|r|||d|S)Nrr )r"r&r!r+r6r/join)r#r2remainingBytesrv bufferIndex bufferOffset bufferedData bytesToReadrrrr0bs&     zBufferedStream._readFromBufferN) __name__ __module__ __qualname____doc__r$r)r.r3r*r/r0rrrrr3s  rcKst|tjs(t|tjjr.t|jtjr.d}n&t|drJt|dt }n t|t }|rdd|D}|rvt d|t |f|St |f|SdS)NFr3rcSsg|]}|dr|qS) _encoding)endswith)rxrrrrs z#HTMLInputStream..z3Cannot set an encoding with a unicode input, set %r) isinstancer HTTPResponserresponseaddbasefphasattrr3r TypeErrorHTMLUnicodeInputStreamHTMLBinaryInputStream)sourcekwargs isUnicode encodingsrrrHTMLInputStream}s       rTc@speZdZdZdZddZddZddZd d Zd d Z d dZ dddZ ddZ ddZ dddZddZdS)rNProvides a unicode stream of characters to the HTMLTokenizer. This class takes care of character encoding and removing or replacing incorrect byte-sequences and also provides column and line tracking. i(cCsZtjsd|_ntddkr$|j|_n|j|_dg|_tddf|_| ||_ | dS)Initialises the HTMLInputStream. HTMLInputStream(source, [encoding]) -> Normalized stream from source for use by html5lib. source can be either a file-object, local filename or a string. The optional encoding parameter must be a string that indicates the encoding. If specified, that encoding will be used, regardless of any BOM or later declaration (such as in a meta element) Nu􏿿r rutf-8certain) rsupports_lone_surrogatesreportCharacterErrorsr&characterErrorsUCS4characterErrorsUCS2newLineslookupEncoding charEncoding openStream dataStreamreset)r#rPrrrr$s   zHTMLUnicodeInputStream.__init__cCs.d|_d|_d|_g|_d|_d|_d|_dS)Nr)r( chunkSize chunkOffseterrors prevNumLines prevNumCols_bufferedCharacterr5rrrrbszHTMLUnicodeInputStream.resetcCst|dr|}nt|}|SzvProduces a file object from source. source can be either a file object, local filename or a string. r3)rLr r#rPr rrrr`s z!HTMLUnicodeInputStream.openStreamcCsT|j}|dd|}|j|}|dd|}|dkr@|j|}n ||d}||fS)N rrr )r(countrgrfindrh)r#r,r(nLines positionLine lastLinePospositionColumnrrr _positions   z HTMLUnicodeInputStream._positioncCs||j\}}|d|fS)z:Returns (line, col) of the current position in the stream.r )rsre)r#linecolrrrr"szHTMLUnicodeInputStream.positioncCs6|j|jkr|stS|j}|j|}|d|_|S)zo Read one character from the stream or queue if available. Return EOF when EOF is reached. r )rerd readChunkr r()r#recharrrrrws   zHTMLUnicodeInputStream.charNcCs|dkr|j}||j\|_|_d|_d|_d|_|j|}|j rX|j |}d|_ n|s`dSt |dkrt |d}|dksd|krdkrnn|d|_ |dd}|j r| || d d }| d d }||_t ||_d S) NrcrFr r iz rl T)_defaultChunkSizersrdrgrhr(rerar3rir&ordrZreplace)r#rdr7lastvrrrrvs0           z HTMLUnicodeInputStream.readChunkcCs(ttt|D]}|jdqdS)Ninvalid-codepoint)ranger&invalid_unicode_refindallrfr6)r#r7_rrrr[sz*HTMLUnicodeInputStream.characterErrorsUCS4cCsd}t|D]}|rqt|}|}t|||drrt|||d}|tkrl|j dd}q|dkr|dkr|t |dkr|j dqd}|j dqdS)NFrTryir ) rfinditerr|groupstartrisSurrogatePairsurrogatePairToCodepointnon_bmp_invalid_codepointsrfr6r&)r#r7skipmatch codepointr'char_valrrrr\#s"  z*HTMLUnicodeInputStream.characterErrorsUCS2Fc Cszt||f}Wnhtk rx|D]}t|dks$tq$ddd|D}|sZd|}td|}t||f<YnXg}||j|j }|dkr|j |j krqn0| }||j kr| |j|j |||_ q| |j|j d| s~qq~d|} | S)z Returns a string of characters from the stream up to but not including any character in 'characters' or EOF. 'characters' must be a container that supports the 'in' method and iteration over its characters. rccSsg|]}dt|qS)z\x%02x)r|)rcrrrrHsz5HTMLUnicodeInputStream.charsUntil..z^%sz[%s]+N)charsUntilRegExKeyErrorr|r+r9recompilerr(rerdendr6rv) r# charactersoppositecharsrregexr;mrrrrr charsUntil:s0    z!HTMLUnicodeInputStream.charsUntilcCsT|tk rP|jdkr.||j|_|jd7_n"|jd8_|j|j|ksPtdSr%)r rer(rdr+)r#rwrrrungetis   zHTMLUnicodeInputStream.unget)N)F)r@rArBrCr{r$rbr`rsr"rwrvr[r\rrrrrrrNs   & /rNc@sLeZdZdZdddZddZd d Zdd d Zd dZddZ ddZ dS)rOrUN windows-1252TcCsn|||_t||jd|_d|_||_||_||_||_ ||_ | ||_ |j ddk sbt |dS)rVidrN)r` rawStreamrNr$ numBytesMetanumBytesChardetoverride_encodingtransport_encodingsame_origin_parent_encodinglikely_encodingdefault_encodingdetermineEncodingr_r+rb)r#rPrrrrr useChardetrrrr$s  zHTMLBinaryInputStream.__init__cCs&|jdj|jd|_t|dS)Nrr})r_ codec_info streamreaderrrarNrbr5rrrrbszHTMLBinaryInputStream.resetcCsLt|dr|}nt|}z||Wntk rFt|}YnX|Srj)rLrr.r) Exceptionrrkrrrr`s z HTMLBinaryInputStream.openStreamcCs|df}|ddk r|St|jdf}|ddk r:|St|jdf}|ddk rX|S|df}|ddk rt|St|jdf}|ddk r|djds|St|jdf}|ddk r|S|rpzddl m }Wnt k rYnXg}|}|j s<|j |j}t|tst|s&q<||||q|t|jd}|j d|dk rp|dfSt|jdf}|ddk r|StddfS)NrXr tentativezutf-16)UniversalDetectorencodingr) detectBOMr^rrdetectEncodingMetarname startswithrchardet.universaldetectorr ImportErrordonerr3rrGr2r+r6feedcloseresultr.r)r#chardetr_rbuffersdetectorr!rrrrrsR           z'HTMLBinaryInputStream.determineEncodingcCs|jddkstt|}|dkr&dS|jdkrFtd}|dk stnT||jdkrf|jddf|_n4|jd|df|_|td|jd|fdS)Nr rXutf-16beutf-16lerWrzEncoding changed from %s to %s)r_r+r^rrr.rbr)r# newEncodingrrrchangeEncodings   z$HTMLBinaryInputStream.changeEncodingc Cstjdtjdtjdtjdtjdi}|jd}t|t s"rOc@seZdZdZddZddZddZdd Zd d Zd d Z ddZ ddZ e e e Z ddZe eZefddZddZddZddZdS) EncodingByteszString-like object with an associated position and various extra methods If the position is ever greater than the string length then an exception is raisedcCst|tstt||SN)rGr2r+__new__lowerr#valuerrrrFszEncodingBytes.__new__cCs d|_dS)Nr)rsrrrrr$JszEncodingBytes.__init__cCs|Srrr5rrr__iter__NszEncodingBytes.__iter__cCs>|jd}|_|t|kr"tn |dkr.t|||dS)Nr rrsr& StopIterationrMr#prrr__next__Qs  zEncodingBytes.__next__cCs|Sr)rr5rrrnextYszEncodingBytes.nextcCsB|j}|t|krtn |dkr$t|d|_}|||dSr%rrrrrprevious]s zEncodingBytes.previouscCs|jt|krt||_dSrrsr&r)r#r"rrr setPositionfszEncodingBytes.setPositioncCs*|jt|krt|jdkr"|jSdSdS)Nrrr5rrr getPositionks  zEncodingBytes.getPositioncCs||j|jdSNr )r"r5rrrgetCurrentByteuszEncodingBytes.getCurrentBytecCsH|j}|t|kr>|||d}||kr4||_|S|d7}q||_dS)zSkip past a list of charactersr Nr"r&rsr#rrrrrrrzs  zEncodingBytes.skipcCsH|j}|t|kr>|||d}||kr4||_|S|d7}q||_dSrrrrrr skipUntils  zEncodingBytes.skipUntilcCs(|||j}|r$|jt|7_|S)zLook for a sequence of bytes at the start of a string. If the bytes are found return True and advance the position to the byte after the match. Otherwise return False and leave the position alone)rr"r&)r#r2r;rrr matchBytesszEncodingBytes.matchBytescCs>z |||jt|d|_Wntk r8tYnXdS)zLook for the next sequence of bytes matching a given sequence. If a match is found advance the position to the last byte of the matchr T)indexr"r&rs ValueErrorrr1rrrjumpTos   zEncodingBytes.jumpToN)r@rArBrCrr$rrrrrrpropertyr"r currentBytespaceCharactersBytesrrrrrrrrrBs      rc@sXeZdZdZddZddZddZdd Zd d Zd d Z ddZ ddZ ddZ dS)rz?Mini parser for detecting character encoding from meta elementscCst||_d|_dS)z3string - the data to work on for encoding detectionN)rr7rr#r7rrrr$s zEncodingParser.__init__c Csd|jkrdSd|jfd|jfd|jfd|jfd|jfd|jff}|jD]}d}z|jdWntk rzYqYnX|D]D\}}|j|rz|}WqWqtk rd}YqYqXq|sHqqH|j S) Nsr7rr5rrrrszEncodingParser.handleCommentcCs|jjtkrdSd}d}|}|dkr,dS|ddkr\|ddk}|r|dk r||_dSq|ddkr|d}t|}|dk r||_dSq|ddkrtt|d}|}|dk rt|}|dk r|r||_dS|}qdS) NTFrs http-equivr s content-typecharsetscontent) r7rr getAttributerr^ContentAttrParserrparse)r# hasPragmapendingEncodingattrtentativeEncodingcodec contentParserrrrrs8      zEncodingParser.handleMetacCs |dS)NF)handlePossibleTagr5rrrrsz%EncodingParser.handlePossibleStartTagcCst|j|dS)NT)rr7rr5rrrrs z#EncodingParser.handlePossibleEndTagcCsb|j}|jtkr(|r$||dS|t}|dkrD|n|}|dk r^|}qLdS)NTr)r7rasciiLettersBytesrrrspacesAngleBracketsr)r#endTagr7rrrrrrs    z EncodingParser.handlePossibleTagcCs |jdS)Nrrr5rrrrszEncodingParser.handleOthercCs|j}|ttdgB}|dks2t|dks2t|dkr>dSg}g}|dkrV|rVqnX|tkrj|}qnD|dkrd|dfS|tkr|| n|dkrdS||t |}qF|dkr| d|dfSt ||}|dkrJ|}t |}||kr"t |d|d|fS|tkr<|| q||qnJ|d krbd|dfS|tkr||| n|dkrdS||t |}|t krd|d|fS|tkr|| n|dkrdS||qdS) z_Return a name,value pair for the next attribute in the stream, if one is found, or None/Nr )rN=)rrr8)'"r) r7rr frozensetr&r+r9asciiUppercaseBytesr6rrrr)r#r7rattrName attrValue quoteCharrrrrsb             zEncodingParser.getAttributeN) r@rArBrCr$rrrrrrrrrrrrrs$rc@seZdZddZddZdS)rcCst|tst||_dSr)rGr2r+r7rrrrr$aszContentAttrParser.__init__cCsz|jd|jjd7_|j|jjdkss      JgIb='