3
ܓid ( @ s d dl Z d dlZd dlZd dlZd dlmZmZmZ d dlZd dl Z d dl
Z
d dl Z d dl Z d dl
Z
d dlZd dlZd dlZd dlZd dlmZ d dlmZ yd dlmZ W n ek
r d dlmZ Y nX ejdZ dZ yd dl Z e j e d Z W nF ee!fk
r< d d
l"m#Z# d Z$e j%d e#e$d
dd d Y nX dZ&ddddddddddddd d d d d d!d"d#d$d%d&d'gZ'd(d) Z(dUd*d+Z)d,d' Z*dVd.dZ+d/d0 Z,d1d2 Z-d3d Z.dWd4dZ/d5d Z0dXd6dZ1e2d7d8d9Z3e&dddddddddddf d:d%Z4dYd;d&Z5dZdd$Z7d\d@dZ8e j9fdAdBZ:dCd Z;dDd ZdGd Z?e j@dHdId"ZAdJd ZBg fdKd#ZCG dLdM dMZDG dNdO dOejEZFG dPdQ dQejEZGG dRd dZHG dSd dZIG dTd deJZKdS )] N)PopenPIPESTDOUT)closing)deque)parse)
parse_versionZsosFT)fillzWARNING: Failed to load 'magic' module version >= 0.4.20 which sos aims to use for detecting binary files. A less effective method will be used. It is recommended to install proper python3-magic package with the module.
H )Zreplace_whitespacei, TIMEOUT_DEFAULTImporterHelperSoSTimeoutError TempFileUtilboldfile_is_binaryfileobjfindget_human_readablegrep
import_module
is_executablelistdirr path_exists
path_isdir path_isfile path_islink path_join recursive_dict_values_by_key shell_outsos_get_command_outputtac_logstailc C s t jd}yB|j| dd\}}}|s(| S t jdd|jd}| d| }|S tk
r } ztjd| d | | S d
}~X nX d
S ) z< Convert the version into a PEP440 compliant version scheme.z2^([0-9][0-9.]*(?:(?:a|b|rc|.post|.dev)[0-9]+)*)\+? )maxsplitz[+~]+.-+zUnable to format z to pep440 format: N)recompilesplitsubstrip Exceptionlogdebug)verZpublic_version_re_ZpublicZlocalZsanitized_localZpep440_versionerr r3 /usr/lib/python3.6/utilities.pyformat_version_to_pep440P s r5 c C s |rt | }t|S t| S )z6 Converts the version to PEP440 format before parsing )r5 r )r0 Zpep440Z
ver_pep440r3 r3 r4 sos_parse_versiona s r6 c
C s< t | d(}tj| j|kr*|j| d |j S Q R X dS )z,Returns the last number_of_bytes of filenamerb N)openosstatst_sizeseekread)filenameZnumber_of_bytesfr3 r3 r4 r" j s rc C sV t | trJyt| |ddS tk
rF tjd| d ttj S X nt| S dS )z@Returns a file-like object that can be used as a context managerzutf-8)encodingz fileobj: z could not be openedN)
isinstancestrr9 IOErrorr. r/ r ioStringIO)Z path_or_filemoder3 r3 r4 r r s
c C sH dd d
d d}x2|j D ]&\}}| |krt| | d| S qW | S ) zCConverts a number of bytes to a shorter, more human friendly formatr# (
)TGMKz.1fl i @i i )itemsfloat)Z num_bytesZsizesZsymbolsizer3 r3 r4
convert_bytes~ s
rT c
C sZ | d
d dkrVt | ddd.}|j }tjd |r8d
S tjd |rHd S W dQ R X dS dS )a Helper to determine if a given file is a certificate or not.
This is especially helpful for `sos clean`, which cannot obfuscate
certificate files content and instead, by default, will keep them as is.
:param fname: The full path of the file to check
:type fname: ``str``
:returns: The type of the certificate or ``None``
:rtype: ``string`` or ``None``
N.csr.crt.pemrA zutf-8)rB z-----BEGIN CERTIFICATE-----Z certificatez"-----BEGIN [A-Z]+ PRIVATE KEY-----Zcertificatekey)rV rW rX )r9 r> r( search)fnamer@ Zcontentr3 r3 r4 file_is_certificate s r\ c s t rNy4tj| ddg} jdko8t fdd|D S tk
rL Y nX t| ddd*}y|jd d
S tk
r d S X W d Q R X d S )
az Helper to determine if a given file contains binary content or not.
This is especially helpful for `sos clean`, which cannot obfuscate binary
data and instead, by default, will remove binary files.
:param fname: The full path of the file to check binaryness of
:type fname: ``str``
:returns: True if binary, else False
:rtype: ``bool``
ztext/zinode/binaryc 3 s | ]} j j|V qd S )N)Z mime_type
startswith).0Z_mt)_ftupr3 r4 s z!file_is_binary..Ztrzutf-8)rB r# FTN) magic_modmagicdetect_from_filenamerB anyr- r9 r> UnicodeDecodeError)r[ Z_mimesZtfiler3 )r` r4 r s
c c s |r"t jj|jt jj}||7 }xrt j|D ]d\}}}|rX|jt jj|krX|dd= |rltj|| rlq.x$tj|| D ]}t jj||V qzW q.W dS )zGenerator function to find files recursively.
Usage::
for filename in find("*.properties", "/var/log/foobar"):
print filename
N) r: pathdirnamecountsepwalkfnmatchfilterjoin) Z file_patternZtop_dirZ max_depthZ path_patternZ
base_depthrg ZdirlistZfilelistnamer3 r3 r4 r s
c sB g }x8|D ]0}t | }|j fdd|D W dQ R X q
W |S )zReturns lines matched in fnames, where fnames can either be pathnames to
files to grep through or open file objects to grep through line by linec 3 s | ]}t j |r|V qd S )N)r( match)r_ line)patternr3 r4 ra s zgrep..N)r extend)rr Zfiles_or_pathsZmatchesZfopZfor3 )rr r4 r s
&c s\ t jjddjt jj} g fdd|D }rJ|fdd|D 7 }tdd |D S )z6Returns if a command matches an executable on the PATHPATH c s g | ]}t jj| qS r3 )r: rg rn )r_ p)commandr3 r4
s z!is_executable..c s g | ]}t jj |jd qS )/)r: rg rn lstrip)r_ c)sysrootr3 r4 rx s c s s | ]}t j|t jV qd S )N)r: accessX_OK)r_ rg r3 r3 r4 ra s z is_executable..)r: environgetr* rg pathsepre )rw r| pathsZ
candidatesr3 )rw r| r4 r s )urlc
C s ddl m}m} y`|| }|js&|jrnd}|jr:||j7 }|jrP|d|j 7 }||j||j|j |j
|j fS | S t k
r | S X dS )zK
Replace username:password@ with ********@ in proxy URL if present
r )urlparse
urlunparsez ********@:N)
Z urllib.parser r ZusernameZpasswordZhostnameZportschemerg ZparamsZqueryZfragmentr- )r r r Z
parsed_urlZnetlocr3 r3 r4 scrub_url_credential s
r c
4 s fdd}
fdd}rdyt j}W n tk
rJ ddddS X |j|j|jd tjj }d |d
< |rx.|j D ]"\}}|r|||< q|j |d qW |rt
d rd
|rdnd d| d| } t j | }g }xT|D ]L}|j
dr*d|kr*tj|}|r |j| n
|j| q|j| qW |
rr|rJt}n&| rftjtjj|
d}n
t|
d}nt}zyt|d||rtntd |d|
dv}|
r|r| rtjtjj|
d}n
t|
d}t|j|||}n
t||}nt |j||}r"x|j r || q
W nzy|j |r2|nd W n` t k
r |j |
r| rt|
d}t!||d W d Q R X d|_ d|j" |j#dS X x|j$ d krqW |
r| rt|
d }t!|||j#p|j%dk W d Q R X |j%d kr d }n|j" }|j%||j#dS Q R X W nB t&k
rb } z$| j't'j(krNddddS | W Y d d } ~ X nX W d t)|d r||j* X d S ) zExecute a command and return a dictionary of status and output,
optionally changing root or current working directory before
executing command.
c sb rdkrt j rPt jtjj t jtjj t jtjj r^t j d S )Nry )
r: chrootsetgidpwdgetpwnamZpw_gidsetuidZpw_uidchdirpw_dirr3 )r r runasr3 r4 _child_prep_fn s
z.sos_get_command_output.._child_prep_fnc s, s| j dkr | j ttjd d S )N| g{Gz?)poll terminater timesleep)proc)pollerr3 r4
_check_poller s z-sos_get_command_output.._check_poller ru )ZstatusoutputZ truncated)HOMEZLOGNAMEZPWDZUSERzC.UTF-8LC_ALLNtimeoutztimeout z --foreground zs ry *)dirwbFr# T)shellstdoutstderrbufsizeenvZ close_fdsZ
preexec_fnr r ~ close)r r )+r r KeyErrorupdater r: r copyrQ popr shlexr* r^ globrs appendr tempfileZ
TemporaryFilerg rh r9 r r
HeadReaderr
FakeReader
TailReaderrunningwaitr- r r! get_contentsis_fullr
returncodeOSErrorerrnoENOENThasattrr ) rw r r r r r Z
foregroundr] sizelimitr Zto_fileZtacr r r Zpwd_userZcmd_envkeyvalueargsZ
expanded_argsargZ expanded_argZ_outputrv readerf_dstr er3 )r r r r r4 r s
c C s d}d}d}| j tj| j jdkr,dS tj| j dtjd}|j d }}|| |krdd}x^|dkr|j|d|}||d |kr|d
krfP qf|rd }n|j ||d |d |}qfW W dQ R X dS ) zPython implementation of the tac utility with support
for multiline logs (starting with space). It is intended
to reverse the output of 'journalctl --reverse'.
rL r N)r} r# TFr )
flushr: fstatfilenor< mmapZ ACCESS_READrS rfindwrite) f_srcr Z
drop_last_logZ NEWLINE_BZ NEWLINE_IZSPACE_IZmmZsep1Zsep2r3 r3 r4 r! s(
c s j dd }yt t t |g}W n> tk
rd } z"td d|jj |W Y dd}~X nX fddtj |tj
D }rfdd|D }|S )
a Imports the module module_fqname and returns a list of defined classes
from that module. If superclasses is defined then the classes returned will
be subclasses of the specified superclass or superclasses. If superclasses
is plural it must be a tuple of classes.r% r# z"Error while trying to load module z: Nc s g | ]\}}|j kr|qS r3 )
__module__)r_ ZcnameZclass_)
module_fqnamer3 r4 rx s z!import_module..c s g | ]}t | r|qS r3 )
issubclass)r_ m) superclassesr3 r4 rx s r )
rpartition
__import__globalslocals ImportErrorprint __class____name__inspectZ
getmembersZisclass)r r Z module_namemoduler modulesr3 )r r r4 r s
rJ c C s t | |||dd S )zeShell out to an external command and return the output or the empty
string in case of error.
)r r r r )r )cmdr r Zrunatr3 r3 r4 r s r8 c C sR dddddg}d}x$| dkr6|dk r6|d 7 }| d
} qW | d | d || S )
NBZKiBZMiBZGiBZTiBr i rU r# g @r% r@ r3 )rS Z precisionsuffixesZ suffixindexr3 r3 r4 r s c C s> |r,|t jkr,| j|s,t jj|| jd} t||}|| S )Nry )r: rj r^ rg rn rz getattr)rg r| methodr Z_methr3 r3 r4 _os_wrapper s
r c C s$ d| krt | |dtdS t | |dS )Nr r )r exists)r r )rg r| r3 r3 r4 r s c C s t | |dS )Nisdir)r )rg r| r3 r3 r4 r s c C s t | |dS )Nisfile)r )rg r| r3 r3 r4 r s c C s t | |dS )Nislink)r )rg r| r3 r3 r4 r s c C s t | |dtS )Nr )r r: )rg r| r3 r3 r4 r s )r| c G s8 |r&| j | r&tjj|| jtj} tjj| f| S )N)r^ r: rg rn rz rj )rg r| rv r3 r3 r4 r s c C s d| d S )a" Helper to make text bold in console output, without pulling in
dependencies to the project unneccessarily.
:param text: The text to make bold
:type text: ``str``
:returns: The text wrapped in the ASCII codes to display as bold
:rtype: ``str``
z[1mz[0mr3 )textr3 r3 r4 r
s
c s g }g |j | t| trx| j D ]\} j| | sF||krV|j t q&ttr&y$|j tfdd|D d W n tk
r Y nX j j q&W n
|j | fdd|D S )a Recursively compile all elements of a potentially nested dict by a set
of keys. If a given key is a dict within ``dobj``, then _all_ elements
within that dict, regardless of child keys, will be returned.
For example, if a Plugin searches the devices dict for the 'storage' key,
then all storage devices under the that dict (e.g. block, fibre, etc...)
will be returned. However, if the Plugin specifies 'block' via ``keys``,
then only the block devices within the devices['storage'] dict will be
returned.
Any elements passed here that are _not_ keys within the dict or any nested
dicts will also be returned.
:param dobj: The 'top-level' dict to intially search by
:type dobj: ``dict``
:param keys: Which keys to compile elements from within ``dobj``. If no
keys are given, all nested elements are returned
:param keys: ``list`` of ``str``
:returns: All elements within the dict and any nested dicts
:rtype: ``list``
c 3 s | ]}| kr | V qd S )Nr3 )r_ r )vr3 r4 ra B s z/recursive_dict_values_by_key..r c s g | ]}| kr|qS r3 r3 )r_ d)_filtr3 r4 rx M s z0recursive_dict_values_by_key..)rs rC dictrQ r r
IndexErrorkeys)Zdobjr Z_itemskr3 )r r r4 r s&
c @ s8 e Zd ZdZdd Zedd Zdd Zedd Zd
S ) r zUsed when we are writing directly to disk without sizelimits,
this allows us to keep more simplified flows for executing,
monitoring, and collecting command output.
c C s || _ || _d S )N)processr] )selfr r] r3 r3 r4 __init__V s zFakeReader.__init__c C s dS )NFr3 )r r3 r3 r4 r Z s zFakeReader.is_fullc C s | j s
dS dS )Nru r )r] )r r3 r3 r4 r ^ s zFakeReader.get_contentsc C s | j j d kS )N)r r )r r3 r3 r4 r a s zFakeReader.runningN) r r __qualname____doc__r propertyr r r r3 r3 r3 r4 r P s
r c s@ e Zd ZdZd Z fddZdd Zdd Zed d
Z Z
S ) r zzUsed to 'head' the command output (f_src) to a given size
without deadlocking sos. Takes a sizelimit value in MB.
i c s8 t j || _|| _|d | _|| _d| _| j d S )Ni T)superr r r remainingr] r start)r r r r r] )r r3 r4 r m s
zHeadReader.__init__c C s^ xH| j dkrH| jjt| j | j}|s(P | jj| | j t|8 _ qW | jj d| _ dS )zReads from the f_src (Popen stdout pipe) until we reach sizelimit.
once done, close f_src to signal the program that we are done.
r FN)
r r r> min COPY_BUFSIZEr r lenr r )r Zbufr3 r3 r4 runv s
zHeadReader.runc C s | j s
dS dS )Nru r )r] )r r3 r3 r4 r s zHeadReader.get_contentsc C s
| j dkS )Nr )r )r r3 r3 r4 r s zHeadReader.is_fulli ) r r r r r r r r r r
__classcell__r3 r3 )r r4 r f s
r c s< e Zd ZdZ fddZdd Zdd Zedd Z Z S )
r zUsed to tail the command output to a given size without deadlocking
sos.
Takes a sizelimit value in MB, and will compile stdout from Popen into a
string that is limited to the given sizelimit.
c s^ t j || _|| _d| _d | _|r>|d }t|| j | _t| jd| _d| _| j d S )Ni i )maxlenT)
r r chanr] chunksizeslotsintr r r )r Zchannelr r] )r r3 r4 r s
zTailReader.__init__c C sN y*x$| j j| j}|sP | jj| qW W n ttfk
rB Y nX d| _dS )al Reads from the channel (pipe) that is the output pipe for a
called Popen. As we are reading from the pipe, the output is added
to a deque. After the size of the deque exceeds the sizelimit
earlier (older) entries are removed.
This means the returned output is chunksize-sensitive, but is not
really byte-sensitive.
FN)r r> r r r
ValueErrorrE r )r rq r3 r3 r4 r s zTailReader.runc C sH x| j rtjd qW | js2djdd | jD S djdd | jD S )z-Returns the contents of the deque as a stringg{Gz?ru c s s | ]}|j d dV qdS )zutf-8ignoreN)decode)r_ lnr3 r3 r4 ra s z*TailReader.get_contents..r c s s | ]
}|V qd S )Nr3 )r_ r r3 r3 r4 ra s )r r r r] rn r )r r3 r3 r4 r s
zTailReader.get_contentsc C s | j s
dS t| j| j kS )z?Checks if the deque is full, implying that output was truncatedF)r r r )r r3 r3 r4 r s zTailReader.is_full)
r r r r r r r r r r r3 r3 )r r4 r s
r c @ s8 e Zd ZdZdd Zdd Zdd Zdd Zd
d Zd S )
r
zProvides a list of modules that can be imported in a package.
Importable modules are located along the module __path__ list and modules
are files that end in .py.
c C s
|| _ dS )zopackage is a package module
import my.package.module
helper = ImporterHelper(my.package.module)N)package)r r
r3 r3 r4 r s zImporterHelper.__init__c C s t jj|}t jj|\}}|S )z-Returns the plugin module name given the path)r: rg basenamesplitext)r rg basero r1 r3 r3 r4 _plugin_name s zImporterHelper._plugin_namec s fdd|D }|j |S )Nc s( g | ] }d |kr|j dr j|qS )r z.py)endswithr )r_ Zplugin)r r3 r4 rx s z9ImporterHelper._get_plugins_from_list..)sort)r Zlist_pluginsr3 )r r4 _get_plugins_from_list s
z%ImporterHelper._get_plugins_from_listc C s0 t jj|r,ttd|}| j|}|r,|S g S )Nz*.py)r: rg r listr r )r rg Zpy_filesZpnamesr3 r3 r4 _find_plugins_in_dir s
z#ImporterHelper._find_plugins_in_dirc C s6 g }x,| j jD ] }tjj|r|j| j| qW |S )zQReturns the list of importable modules in the configured python
package. )r
__path__r: rg r rs r )r r rg r3 r3 r4 get_modules s
zImporterHelper.get_modulesN) r r r r r r r r r r3 r3 r3 r4 r
s c @ s$ e Zd Zdd Zdd Zdd ZdS )r c C s || _ g | _d S )N)tmp_dirfiles)r r r3 r3 r4 r s zTempFileUtil.__init__c C s2 t j| jd\}}tj|d}| jj||f |S )N)r zw+)r Zmkstempr r: fdopenr r )r fdr[ Zfobjr3 r3 r4 new s zTempFileUtil.newc C sl x`| j D ]V\}}y|j |j W n tk
r8 Y nX ytj| W q tk
r\ Y qX qW g | _ d S )N)r r r r- r: unlink)r r[ r@ r3 r3 r4 clean s
zTempFileUtil.cleanN)r r r r r r r3 r3 r3 r4 r s c @ s e Zd ZdS )r N)r r r r3 r3 r3 r4 r s )T)rA )NN)N)F)N)rJ NN)r8 )Lr: r r( r
subprocessr r r Zloggingrl r r r r Z threadingr rF r
contextlibr collectionsr Zpackaging.versionr r r Z
pkg_resourcesZ getLoggerr. rb rc rd __file__AttributeErrortextwrapr msgZwarningr __all__r5 r6 r" r rT r\ r r r r rD r r r! r r r rg r r r r r r rj r r r r ZThreadr r r
r r r r3 r3 r3 r4 s
"
"
*
6%<,