U
fL @ sn d dl mZ d dlmZ d dlmZmZ ddlmZm Z m
Z
ddl m Z m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z G dd dZ G d d
d
e Z G d d d e Z G d
d de Z G dd de Z!G dd de Z"G dd de Z#G dd de Z$G dd de Z%G dd de Z&edd ee' ee' e(d d d Z)ed d d'e'e*e(e*d#d$d%Z+d&S )( ) lru_cache) getLogger)ListOptional ) COMMON_SAFE_ASCII_CHARACTERSTRACE UNICODE_SECONDARY_RANGE_KEYWORD)is_accentuated is_arabicis_arabic_isolated_formis_case_variableis_cjk is_emoticon is_hangul is_hiragana is_katakanais_latinis_punctuation is_separator is_symbolis_thaiis_unprintable
remove_accent
unicode_rangec @ sP e Zd ZdZeedddZeddddZddd d
Ze e
dd d Z dS )
MessDetectorPluginzy
Base abstract class used for mess detection plugins.
All detectors MUST extend and implement given methods.
characterreturnc C s t dS )z@
Determine if given character should be fed in.
NNotImplementedErrorselfr r# H/opt/alt/python38/lib64/python3.8/site-packages/charset_normalizer/md.pyeligible% s zMessDetectorPlugin.eligibleNc C s t dS )z
The main routine to be executed upon character.
Insert the logic in witch the text would be considered chaotic.
Nr r! r# r# r$ feed+ s zMessDetectorPlugin.feedr c C s t dS )zB
Permit to reset the plugin to the initial state.
Nr r" r# r# r$ reset2 s zMessDetectorPlugin.resetc C s t dS )z
Compute the chaos ratio based on what your feed() has seen.
Must NOT be lower than 0.; No restriction gt 0.
Nr r( r# r# r$ ratio8 s zMessDetectorPlugin.ratio) __name__
__module__ __qualname____doc__strboolr% r& r) propertyfloatr* r# r# r# r$ r s r c @ sZ e Zd ZddddZeedddZedddd Zddd
d Ze e
dd d
Z dS ) TooManySymbolOrPunctuationPluginNr' c C s" d| _ d| _d| _d | _d| _d S )Nr F)_punctuation_count
_symbol_count_character_count_last_printable_charZ_frenzy_symbol_in_wordr( r# r# r$ __init__B s
z)TooManySymbolOrPunctuationPlugin.__init__r c C s | S N isprintabler! r# r# r$ r% J s z)TooManySymbolOrPunctuationPlugin.eligiblec C sp | j d7 _ || jkrf|tkrft|r8| jd7 _n.| dkrft|rft|dkrf| jd7 _|| _d S )Nr F ) r6 r7 r r r4 isdigitr r r5 r! r# r# r$ r& M s
z%TooManySymbolOrPunctuationPlugin.feedc C s d| _ d| _d| _d S Nr )r4 r6 r5 r( r# r# r$ r) _ s z&TooManySymbolOrPunctuationPlugin.resetc C s0 | j dkrdS | j| j | j }|dkr,|S dS )Nr 333333?)r6 r4 r5 )r" Zratio_of_punctuationr# r# r$ r* d s
z&TooManySymbolOrPunctuationPlugin.ratio r+ r, r- r8 r/ r0 r% r& r) r1 r2 r* r# r# r# r$ r3 A s r3 c @ sZ e Zd ZddddZeedddZedddd Zddd
d Ze e
dd d
Z dS )TooManyAccentuatedPluginNr' c C s d| _ d| _d S r> r6 _accentuated_countr( r# r# r$ r8 q s z!TooManyAccentuatedPlugin.__init__r c C s | S r9 )isalphar! r# r# r$ r% u s z!TooManyAccentuatedPlugin.eligiblec C s( | j d7 _ t|r$| jd7 _d S Nr )r6 r
rD r! r# r# r$ r& x s z TooManyAccentuatedPlugin.feedc C s d| _ d| _d S r> rC r( r# r# r$ r) ~ s z TooManyAccentuatedPlugin.resetc C s* | j dk rdS | j| j }|dkr&|S dS )N r? gffffff?rC )r" Zratio_of_accentuationr# r# r$ r* s
z TooManyAccentuatedPlugin.ratiorA r# r# r# r$ rB p s rB c @ sZ e Zd ZddddZeedddZedddd Zddd
d Ze e
dd d
Z dS )UnprintablePluginNr' c C s d| _ d| _d S r> )_unprintable_countr6 r( r# r# r$ r8 s zUnprintablePlugin.__init__r c C s dS NTr# r! r# r# r$ r% s zUnprintablePlugin.eligiblec C s( t |r| jd7 _| jd7 _d S rF )r rI r6 r! r# r# r$ r& s zUnprintablePlugin.feedc C s
d| _ d S r> )rI r( r# r# r$ r) s zUnprintablePlugin.resetc C s | j dkrdS | jd | j S )Nr r? rG )r6 rI r( r# r# r$ r* s
zUnprintablePlugin.ratiorA r# r# r# r$ rH s rH c @ sZ e Zd ZddddZeedddZedddd Zddd
d Ze e
dd d
Z dS ) SuspiciousDuplicateAccentPluginNr' c C s d| _ d| _d | _d S r> _successive_countr6 _last_latin_characterr( r# r# r$ r8 s z(SuspiciousDuplicateAccentPlugin.__init__r c C s | ot|S r9 )rE r r! r# r# r$ r% s z(SuspiciousDuplicateAccentPlugin.eligiblec C st | j d7 _ | jd k rjt|rjt| jrj| rJ| j rJ| jd7 _t|t| jkrj| jd7 _|| _d S rF )r6 rN r
isupperrM r r! r# r# r$ r& s z$SuspiciousDuplicateAccentPlugin.feedc C s d| _ d| _d | _d S r> rL r( r# r# r$ r) s z%SuspiciousDuplicateAccentPlugin.resetc C s | j dkrdS | jd | j S )Nr r? r< )r6 rM r( r# r# r$ r* s
z%SuspiciousDuplicateAccentPlugin.ratiorA r# r# r# r$ rK s rK c @ sZ e Zd ZddddZeedddZedddd Zddd
d Ze e
dd d
Z dS )SuspiciousRangeNr' c C s d| _ d| _d | _d S r> )"_suspicious_successive_range_countr6 _last_printable_seenr( r# r# r$ r8 s zSuspiciousRange.__init__r c C s | S r9 r: r! r# r# r$ r% s zSuspiciousRange.eligiblec C sx | j d7 _ | s&t|s&|tkr0d | _d S | jd krD|| _d S t| j}t|}t||rn| jd7 _|| _d S rF )r6 isspacer r rR r is_suspiciously_successive_rangerQ )r" r unicode_range_aunicode_range_br# r# r$ r& s"
zSuspiciousRange.feedc C s d| _ d| _d | _d S r> )r6 rQ rR r( r# r# r$ r) s zSuspiciousRange.resetc C s" | j dkrdS | jd | j }|S )N r? r< )r6 rQ )r" Z ratio_of_suspicious_range_usager# r# r$ r* s
zSuspiciousRange.ratiorA r# r# r# r$ rP s rP c @ sZ e Zd ZddddZeedddZedddd Zddd
d Ze e
dd d
Z dS )SuperWeirdWordPluginNr' c C s: d| _ d| _d| _d| _d| _d| _d| _d| _d| _d S )Nr F ) _word_count_bad_word_count_foreign_long_count_is_current_word_bad_foreign_long_watchr6 _bad_character_count_buffer_buffer_accent_countr( r# r# r$ r8 s z SuperWeirdWordPlugin.__init__r c C s dS rJ r# r! r# r# r$ r% s z SuperWeirdWordPlugin.eligiblec C s6 | r| j|7 _t|r,| jd7 _| jdkrt|dksJt|rt|dkrt|dkrt|dkrt |dkrt
|dkrd| _d S | jsd S | st |st
|r| jr| jd7 _t| j}| j|7 _|dkrP| j| dkrd| _t| jd rP| jd rPtdd | jD dkrP| jd7 _d| _|d kr| jrd
d t| jtd |D }d}|rt|| d
krd}|s| jd7 _d| _| jr| jd7 _| jt| j7 _d| _d| _d| _d | _n6|dkr2| dkr2t|r2d| _| j|7 _d S )Nr FT g(\?c s s | ]}| V qd S r9 rO ).0_r# r# r$ - s z,SuperWeirdWordPlugin.feed..rW c S s g | ]\}}| r|qS r# rd )re cir# r# r$
2 s z-SuperWeirdWordPlugin.feed..r r@ rY > -rf <~>=|)rE r` r
ra r^ r r r r r r rS r r rZ lenr6 r] rO allr\ zipranger[ r_ r= r )r" r Z
buffer_lengthZcamel_case_dstZprobable_camel_casedr# r# r$ r& s
zSuperWeirdWordPlugin.feedc C s4 d| _ d| _d| _d| _d| _d| _d| _d| _d S )NrY Fr )r` r] r^ r[ rZ r6 r_ r\ r( r# r# r$ r) P s zSuperWeirdWordPlugin.resetc C s$ | j dkr| jdkrdS | j| j S )N
r r? )rZ r\ r_ r6 r( r# r# r$ r* Z s zSuperWeirdWordPlugin.ratiorA r# r# r# r$ rX s E
rX c @ s^ e Zd ZdZddddZeedddZeddd d
Zddd d Z e
e dd
dZ dS )CjkInvalidStopPluginu
GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and
can be easily detected. Searching for the overuse of '丅' and '丄'.
Nr' c C s d| _ d| _d S r> _wrong_stop_count_cjk_character_countr( r# r# r$ r8 h s z CjkInvalidStopPlugin.__init__r c C s dS rJ r# r! r# r# r$ r% l s z CjkInvalidStopPlugin.eligiblec C s4 |dkr| j d7 _ d S t|r0| jd7 _d S )N> 丄 丅r )rx r ry r! r# r# r$ r& o s
zCjkInvalidStopPlugin.feedc C s d| _ d| _d S r> rw r( r# r# r$ r) v s zCjkInvalidStopPlugin.resetc C s | j dk rdS | j| j S )N r? )ry rx r( r# r# r$ r* z s
zCjkInvalidStopPlugin.ratio)
r+ r, r- r. r8 r/ r0 r% r& r) r1 r2 r* r# r# r# r$ rv b s rv c @ sZ e Zd ZddddZeedddZedddd Zddd
d Ze e
dd d
Z dS )ArchaicUpperLowerPluginNr' c C s. d| _ d| _d| _d| _d| _d | _d| _d S )NFr T)_buf _character_count_since_last_sep _successive_upper_lower_count#_successive_upper_lower_count_finalr6 _last_alpha_seen_current_ascii_onlyr( r# r# r$ r8 s z ArchaicUpperLowerPlugin.__init__r c C s dS rJ r# r! r# r# r$ r% s z ArchaicUpperLowerPlugin.eligiblec C s$ | ot|}|dk}|r| jdkr| jdkrV| dkrV| jdkrV| j| j7 _d| _d| _d | _d| _| j d7 _ d| _d S | jdkr|
dkrd| _| jd k r| r| j s| r| j r| jdkr| jd7 _d| _qd| _nd| _| j d7 _ | jd7 _|| _d S )NFr @ r Tr< )
rE r
r r= r r r r r~ r6 isasciirO islower)r" r Z is_concernedZ chunk_sepr# r# r$ r& sF
z ArchaicUpperLowerPlugin.feedc C s. d| _ d| _d| _d| _d | _d| _d| _d S )Nr FT)r6 r r r r r~ r r( r# r# r$ r) s z ArchaicUpperLowerPlugin.resetc C s | j dkrdS | j| j S )Nr r? )r6 r r( r# r# r$ r* s
z ArchaicUpperLowerPlugin.ratiorA r# r# r# r$ r} s
* r} c @ sZ e Zd ZddddZddddZeeddd Zeddd
d Ze e
dd d
Z dS )ArabicIsolatedFormPluginNr' c C s d| _ d| _d S r> r6 _isolated_form_countr( r# r# r$ r8 s z!ArabicIsolatedFormPlugin.__init__c C s d| _ d| _d S r> r r( r# r# r$ r) s z ArabicIsolatedFormPlugin.resetr c C s t |S r9 )r r! r# r# r$ r% s z!ArabicIsolatedFormPlugin.eligiblec C s( | j d7 _ t|r$| jd7 _d S rF )r6 r r r! r# r# r$ r& s z ArabicIsolatedFormPlugin.feedc C s | j dk rdS | j| j }|S )NrG r? r )r" Zisolated_form_usager# r# r$ r* s
z ArabicIsolatedFormPlugin.ratio) r+ r, r- r8 r) r/ r0 r% r& r1 r2 r* r# r# r# r$ r s r )maxsize)rU rV r c C s | dks|dkrdS | |kr dS d| kr4d|kr4dS d| ksDd|krHdS d| ksXd|krld| kshd|krldS | d| d }}|D ] }|tkrq||kr dS q| dk|dk }}|s|rd | ksd |krdS |r|rdS d
| ksd
|kr d | ksd |krdS | d ks |d kr dS d | ksHd |ksH| d kr|d krd
| ks\d
|kr`dS d| kstd|krxdS | d ks|d krdS dS )za
Determine if two Unicode range seen next to each other can be considered as suspicious.
NTFZLatinZ EmoticonsZ Combining )HiraganaKatakanaCJKHangulz Basic Latin)r r PunctuationZForms)splitr )rU rV Zkeywords_range_aZkeywords_range_belZrange_a_jp_charsZrange_b_jp_charsr# r# r$ rT sl rT i 皙?F)decoded_sequencemaximum_thresholddebugr c C sX dd t D }t| d }d}|dk r0d}n|dkr>d}nd }t| d
t|D ]d\}}|D ]} | |r`| | q`|d kr|| d ks||d krTtd d
|D }||krT qqT|rNtd}
|
t
d| d| d| t| dkr(|
t
d| dd |
t
d| dd |D ] } |
t
| j d| j q,t
|dS )zw
Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
c S s g | ]
}| qS r# r# )re Zmd_classr# r# r$ rj : s z mess_ratio..r r? i r r
r c s s | ] }|j V qd S r9 )r* )re dtr# r# r$ rg Q s z mess_ratio..Zcharset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=r| zStarting with: Nz
Ending with: iz: )r __subclasses__rq rs rt r% r& sumr logr __class__r* round) r r r Z detectorslengthZmean_mess_ratioZ!intermediary_mean_mess_ratio_calcr indexdetectorloggerr r# r# r$
mess_ratio2 sF
r N)r F), functoolsr loggingr typingr r Zconstantr r r utilsr
r r r
r r r r r r r r r r r r r r r3 rB rH rK rP rX rv r} r r/ r0 rT r2 r r# r# r# r$ s8 L"/%1i L H