9MjoBdZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z ddlmZmZddlmZddlmZddlmZdd lmZmZdd lmZmZd d lmZm Z dd l!m"Z#ddl$m%Z%m&Z&m'Z'd dl(m)Z)m*Z*m+Z+d dl,m-Z-m.Z.m/Z/d dl0m1Z1d dl2m3Z3d dl4m5Z5m6Z6d dl7m8Z8ddl9m:Z:GddZ;GddZ<Gdde;e<Z=Gdde;Z>Gdde;e<Z?Gd d!e;Z@Gd"d#e;ZAGd$d%e;ZBGd&d'e;ZCdS)(zA This module contains class implementing MongoDB API interaction N)partial)ListAnyIterable)SessionResponse) HTTPAdapter)RequestException)Retry)Schema SchemaError)is_feature_visibleis_feature_hidden_server_wide) supportedget_advice_instance)gettext)detailed_advice_schemauser_sites_info_schemaadvice_list_schema) api_serverprotoadviser_api_server) XRayAPIErrorXRayAPIEmptyResponseTaskNotFoundError)open_local_storage)Task)get_xray_exec_useruser_mode_verificationread_jwt_token)filter_by_non_existencec $eZdZdZdddedefdZdZdZed efd Z ed e fd Z d efd Z dde ded efdZdde ded e fdZ ddede dededed e f dZddededefdZd ded e fdZd!deded efdZd ded e fdZdZdS)"Clientz Base client class unavailabletracing_task_id system_idr(c||_||_tjd|_t dt ddgt gdd}t|}t|_ |j td|t|j j d |j _ dS) N api_clientGETPOST)iii)totalallowed_methodsstatus_forcelistbackoff_factor) max_retries:// )timeout)r)task_idlogging getLoggerloggerr frozensetr rsessionmountrrrequest)selfr)r( retry_confadapters P/opt/cloudlinux/venv/lib64/python3.11/site-packages/xray/apiclient/api_client.py__init__zClient.__init__0s"& ' 55 %ufo66&77    *555yy  e==='222&t|';RHHH c>|jjd|jd|jS)N::z::tracing_task_id=) __class____name__ main_endpointr7r?s rB__repr__zClient.__repr__@s*.)aaT-?aaSWS_aaarDc.|jjd|jS)NrF)rGrHr7rJs rB__str__zClient.__str__Cs.);;T\;;;rDreturnc:ttd)z Base endpoint z$instances are to set their endpoint!)NotImplementedError_rJs rBrIzClient.main_endpointFs "!$J"K"KLLLrDc"dtiS)zL Updated request headers :return: dict with headers zX-Authr!rJs rB_headerszClient._headersMs .**++rDc d|ji}|r||tj|S)ze Construct query string :return: string including system_id and given kwargs r))r)updateurllibparse urlencode)r?kwargsinitials rB _query_stringzClient._query_stringUs@ /  # NN6 " " "|%%g...rDTresponsewith_status_checkc|js_|jd|j|jd|j}t td|z|jd|ji|jd|j|j|j |jdkr| }ndd i}|r8|dd kr,t td t|z|S) z Perform preprocessing checks of received Response: - is it OK, e.g. 200 OK - was it successful, e.g. status == ok and extract JSON from the Response object :return: full JSON representation of response zServer responded: %s:z#Unable to connect to server with %s resp_data)errnoextraz[%s:%s] Response received %sstatusokz"Received unsuccessful response: %s) rer:debugtext status_codereasonrrQinfourljsonstr)r?r\r] request_dataresults rB_preprocess_responsezClient._preprocess_response_s{ r K  4hm D D D&2FFX_FFL788<G*"HM2  K  ;X=QS[Sbdldp q q q  3 & &]]__FF%F  Zh4''"1%I#J#JSQW[[#XYYY rDc|||} |d}|jd|j||t ||S#t $rt cYSwxYw)z Check received response :param response: a requests.Response object :return: 'result' dict from JSON representation of response r]roz[%s] Received response data %sN)ro)rpr:rjrhrKeyErrordict)r?r\r]rodatas rB_process_responsezClient._process_response}s **8GX*YY (#D K  =x?SUY Z Z Z|*&9999K   66MMM sO%PPPsA B B;&B66B;c |jd| |j||j}n8#t $r+}|||dtd|d}~wwxYw|||S)NzSending DELETE request to %sr|z DELETE failedz)Failed to DELETE data to X-Ray API serverrr) r:rjr<deleterSr rrQrv)r?rwryr]rrs rB_deletezClient._deletes 7BBB <&&x&GGDD   ++8_a0[.\.\  %%d>O%PPPs!? A4 &A//A4c 2||jd|}|jd| |j||j}n8#t$r+}|||dtd|d}~wwxYw|S)z GET request to endpoint or to main endpoint if no endpoint given :param endpoint: target URL :return: a requests Response object N?zSending GET request to %sr|z GET failedz(Failed to GET data from X-Ray API server) rIr[r:rjr<getrSr rrQr?rwrrs rB_raw_getzClient._raw_gets  ,EEt/A/A/C/CEEH 4h??? <##Hdm#DDDD   ++8\1-W+X+X   s!A B)&BBcZ||}|||S)z GET request to endpoint or to main endpoint if no endpoint given :param endpoint: target URL :return: full dict from JSON representation of response without any processing rr)rrp)r?rwr]rs rB _get_fullzClient._get_fulls/}}X&&((AR(SSSrDc||} ||S#t$r}t|j|d}~wwxYw)z GET request to endpoint or to main endpoint if no endpoint given :param endpoint: target URL :return: 'result' dict from JSON representation of response )r7N)rrvrrr7rs rB_getz Client._getsg }}X&& A))$// /# A A A#DL999q @ As, AA  Ac|jd||d|i |jdj}n#tt f$r|}YnwxYw|d|}t td|zS)z Process received exception :param exc: original exception :param api_endpoint: requested endpoint :param log_message: text for logging the error :param exc_message: text for internal exception z %s with %srw)rbrz: z%s. Please, try again later)r:errorargsri IndexErrorAttributeErrorrrQ)r?exc api_endpoint log_message exc_messageexc_infoexception_datas rBrzClient._give_xray_exceptions , S\@Z[[[ x{)HHN+   HHH (55855A;<<~MNNNs5A  A T)NTTT)TTN)NT)rH __module__ __qualname____doc__rmrCrKrMpropertyrIrtrSr[rboolrrprvrrrrrrrDrBr%r%+sBOIIISI3IIII bbb<<<MsMMMXM ,$,,,X,/////X$Z]<(tW[&"& Q Q Q Q Q  Q  Q  Q Q Q QDQQQtQtQQQQ"TT#TTQTTTTT A AS AD A A A AOOOOOrDr%c>eZdZdZedefdZdedefdZ dS) TaskMixinz1 A mixin class with Task related methods rNcdS)z( Limit processed fields ) rkrd client_ip tracing_by tracing_count starttime ini_location initial_count request_count auto_taskuserrrJs rB task_fieldszTaskMixin.task_fieldss   rD dict_viewcrfd|D}|d|d<tdi|S)z@ Turn dictionary structure into valid Task type c.i|]\}}|jv||Srr.0kvr?s rB z#TaskMixin._task..s,QQQda1@P;P;PQ;P;P;PrDr(r7r)itemsr)r?r task_views` rB_taskzTaskMixin._task sMRQQQioo&7&7QQQ ():; )  i   rDN) rHrrrrtuplerrtrrrrDrBrrsg U   X $!t!!!!!!!rDrc`eZdZdZedefdZdeffd Zdee ffd Z xZ S) TasksClientz! 'tasks' endpoint client rNc&tdtdS)z& Base endpoint: tasks r4z/api/xray/tasksrrrJs rBrIzTasksClient.main_endpoints 77J7777rDc HtdS)z Construct query string. Aimed to get auto tasks only :return: string including system_id and type=auto autotype)superr[r?rYrGs rBr[zTasksClient._query_strings ww$$&$111rDcbt}fd|DS)z# Get list of Tasks c:g|]}|Sr)r)ritemr?s rB z)TasksClient.get_tasks..,s%222T 4  222rD)rr)r?rurGs` rB get_taskszTasksClient.get_tasks's1ww||~~2222T2222rD) rHrrrrrmrIr[rrr __classcell__rGs@rBrrs8s888X8 222222234:3333333333rDrceZdZdZdddedeffdZdZedd Zd e dd fd Z de fdZ edefdZ de fdZde defdZdedd fdZdedd fdZddZddZxZS) DBMClientzD Client class using local dbm storage instead of remote API r&r'r)r(ct||d|_||_dS)N)r)r()rrC task_object_db_open_db)r?r)r(rGs rBrCzDBMClient.__init__4s; 9oNNN==??rDc8|jdSr)rcloserJs rB__del__zDBMClient.__del__9s rDrN dbm._Databasec,tjddS)zC Open dbm DB :return: corresponding object z/root/local_mongoc)dbmopenrrDrBrzDBMClient._db_open<s x+S111rD post_dataNcHtj||j|j<dS)z3 Update a DBM task with given data N)rldumpsrr7)r?rs rBrzDBMClient._postDs""&I!6!6rDc tj|j|jS#t tjf$r"}ttd|d}~wwxYw)z: Get saved DBM data :return: dict zFailed to load taskN) rlloadsrr7decodersJSONDecodeErrorrrQ)r?rs rBrzDBMClient._getJsq  @:dht|4;;==>> >$./ @ @ @q!67788a ? @s58A0A++A0c2tjjSr)uuiduuid1hexrrDrB_idz DBMClient._idTsz||rDcn|}|j|d<tdi||_|jS)1 Get saved task :return: r7r)rr7rr)r? saved_tasks rBget_taskzDBMClient.get_taskXs< YY[[ $ 9--*--rDtaskc||_|j|_d|_||||_|jS)a: Create new task and get unique ID url --> URL client_ip --> IP tracing_by --> time|request_qty tracing_count --> COUNT ini_location --> PATH status --> processing :param task: a Task instance :return: task ID hold)rr7rdras_dictrr?rs rBcreatezDBMClient.createbsMxxzz |   4<<>>"""|rDrc|j|d|j_||j_||jdS)z Update started|continued task status --> running starttime --> new timestamp :return: Nrunning)rrrdrrrr?rs rBrUzDBMClient.updateusW   # MMOOO"+%." 4#++--.....rDcountc|j|d|j_||j_||jdS)} Update stopped task status --> stopped tracing_count --> new value :return: Nstopped)rrrdrrrr?rs rBstopzDBMClient.stopsW   # MMOOO"+).& 4#++--.....rDc|j|d|j_||jdS)Y Complete tracing task status --> completed :return: N completed)rrrdrrrJs rBcompletezDBMClient.completesL   # MMOOO"- 4#++--.....rDc |j|j=dSz6 Delete tracing task :return: N)rr7rJs rBrzDBMClient.deletes HT\ " " "rD)rNrrNN)rHrrrrmrCr staticmethodrrtrrrrrrintrUrrrrrs@rBrr/sBO###S#3###### 222\27t77777 @d@@@@    \  $    4C& / / / / / / /# /$ / / / / / / / /########rDrceZdZdZedefdZdeffd ZdeddfdZ deddfdZ dd Z dd Z e defd Zd edefd ZdeddfdZdeddfdZdddededdfdZdeddfdZddZddZddZxZS) APIClientz% X-Ray task API client class rNc&tdtdS)z% Base endpoint: task r4z/api/xray/taskrrJs rBrIzAPIClient.main_endpoints 66J6666rDc |jdkr't|jStS)z Construct query string :return: string either including system_id and task_id or system_id only r&r')r7rr[rs rBr[zAPIClient._query_stringsE <= ( (77(((FF Fww$$&&&rDrNc|jd|}||d|D}|d|_dS)z POST request to "create a task" API endpoint with given data Obtains a task ID :param post_data: dict with POST data z/create?c&i|]\}}|dk ||S)rrrrrs rBrz*APIClient._post_create..s*-e-e-etq!TUYdTdTdaTdTdTdrDr(N)rIr[rrr7)r?rrw response_datas rB _post_createzAPIClient._post_createsc (HH$2D2D2F2FHH 8-e-ey?P?P-e-e-eff $%67 rDcn|jd|}|||dS)z| POST request to "update a task" API endpoint with given data :param post_data: dict with POST data z/update?NrIr[r)r?rrws rB _post_updatezAPIClient._post_updates@ (HH$2D2D2F2FHH 8Y'''''rDc|jdd}|d|}||dS)z< GET request to "share a task" API endpoint Nz/share-request?rIr[r)r?share_endpointrws rB_sharezAPIClient._sharesL+CRC0$KKT5G5G5I5IKK (rDcl|jd|}||dS)z> POST request to "delete a task" API endpoint z/delete?Nr)r?rws rBrzAPIClient._deletes>(HH$2D2D2F2FHH 8rDcP||S)r)rrrJs rBrzAPIClient.get_tasks zz$))++&&&rDrcd|_fd|DjS)aH Create new task and get unique ID url --> URL client_ip --> IP tracing_by --> time|request_qty tracing_count --> COUNT ini_location --> PATH status --> processing :param task: a Task instance :return: updated Task instance rc.i|]\}}|jv||Srrrs rBrz$APIClient.create..s,\\\DAqa4K[F[F[1aF[F[F[rD)rdr rrr7rs` rBrzAPIClient.createsQ  \\\\DLLNN,@,@,B,B\\\]]]|rDrc6|d|ddS)z Update started|continued task status --> running starttime --> new timestamp :param starttime: time of starting the Task r)rdrNrrs rBrUzAPIClient.updates& YYGGHHHHHrDrc4|d|idS)zw Update tracing_count only. No status updated tracing_count --> new value :return: rNrrs rBupdate_count_onlyzAPIClient.update_count_onlys# ?E233333rD)rrrcH|d|i}n||d}||dS)a Update tracing_count only. No status updated request_count --> new value tracing_count --> new value if given :param request_count: number of requests already traced :param tracing_count: number of requests left to trace Nr)rrr)r?rrrus rBupdate_counts_onlyzAPIClient.update_counts_onlys=  #]3DD%2]SSD $rDc6|d|ddS)rr)rdrNrrs rBrzAPIClient.stops& YGGHHHHHrDc4|ddidS)rrdrNrrJs rBrzAPIClient.completes# 8[122222rDc.|dS)z5 Share tracing task :return: N)rrJs rBsharezAPIClient.share!s rDc.|dSr)rrJs rBrzAPIClient.delete(s rDr)rHrrrrrmrIr[rtr rrrr rrrrrUrrrrr"rrrs@rBrrs7s777X7 '''''''8d8t8888(d(t(((('$''''4D IIIIII4s4t4444NR    3  s  VZ     I#I$IIII3333rDrc>eZdZdZedefdZdeddfdZdS) SendClientz) X-Ray requests API client class rNc&tdtdS)) Base endpoint: requests r4z/api/xray/requestsrrJs rBrIzSendClient.main_endpoint5s ::J::::rDruNcr|jd|}|||ddS)zS Send given data to ClickHouse :param data: dict with data rF)ryNrr?rurws rB__call__zSendClient.__call__<sD (AA4+=+=+?+?AA 8TE 22222rD) rHrrrrrmrIrtr*rrDrBr%r%0sg;s;;;X; 3T3d333333rDr%c eZdZdZedefdZdeffd ZdefdZ ddede d e defd Z dedefd Z ded e defd Z xZS) UIAPIClientz, X-Ray User plugin API client class rNc&tdtdS)r'r4z /api/xrayrrJs rBrIzUIAPIClient.main_endpointJs 11J1111rDc rd|D}tjdi|S)z Construct query string :return: string including system_id and given kwargs, filtered by non-empty values ci|] \}}||| Srrr s rBrz-UIAPIClient._query_string..WsIIIA1=1===rDr)rrr[)r?rY filter_emptyrGs rBr[zUIAPIClient._query_stringQs> JIIII $uww$44|444rDc|t}|jd|}||}|dD]}|ddkrt j|dd}t|5}|d d kr|d |j z |d <|j |d <dddn #1swxYwY|S)zd Get list of tasks and return not processed (full) response from API server )rz/tasks?rordrr(r5) digest_sizertimerrrN) r[rrIrhashlibblake2bencode hexdigestrprocessed_requests)r?qsrwr\rfake_idstorages rB get_task_listzUIAPIClient.get_task_listZsQ   %7%9%9  : :(5555>>(++X& C CDH~,,od+<&=&D&D&F&FTVWWWaaccG#G,, C %//,0,AGD^,^D)(/(B_% C C C C C C C C C C C C C C Cs**C  C$ 'C$ Nr7 time_fromtime_toct||||}|jd|}||S)zG Get list of requests collected for given tracing task )r(r=r>z /requests?r[rIr)r?r7r=r>r9rws rBget_request_listzUIAPIClient.get_request_listosG  9V]  ^ ^(88B88~~h'''rDcp||}|jd|}||S)zB Get hourly requests stats for given tracing task r'z/requests_stats_by_hour?r@)r?r7r9rws rBget_requests_stats_by_hourz&UIAPIClient.get_requests_stats_by_hourwsB    8 8(FF"FF~~h'''rD request_idcr|||}|jd|}||S)zU Get collected statistics for given request ID of given tracing task )r(rDz /request?r@)r?r7rDr9rws rBget_request_datazUIAPIClient.get_request_datasD  J  O O(77277~~h'''rD)NN)rHrrrrrmrIr[rtr<rrArCrFrrs@rBr,r,Es2s222X2 5555555t*((((S(\`(((((#($(((((((((((((((rDr,ceZdZdZfdZdededefdZede fdZ ede fdZ de fd Z dedd fd Zdd e d edefdZdedd fdZdeefdZddededefdZdefdZdedefdZdedefdZdedefdZxZS)SmartAdviceAPIClient( X-Ray Adviser API client class cLtddSN not_needed)r)rrCr?rGs rBrCzSmartAdviceAPIClient.__init__$ <00000rDruschemarNc ||S#t$rf}|jd||jdp |jd}t tdt|zd}~wwxYw)z&Validate given data using given schemaz#Failed to validate API response: %szMalformed API response: %sN) validater r:rerrorsautosrrQrm)r?rurPrmsgs rB _validatezSmartAdviceAPIClient._validates K??4(( ( K K K K  CT J J J(2,-!'"+Cq!=>>SIJJ J Ks BA!BBcdtdS)r'https://z/apirrJs rBrIz"SmartAdviceAPIClient.main_endpoint 3,2222rDcdS)z3 Limit fields available for update )rdsourcerirrJs rBfields_allowedz#SmartAdviceAPIClient.fields_alloweds .-rDc dtD}t}|r|d|f|d|Dt j|dS)za Construct query string :return: string including types and given kwargs cg|]}d|fSrr)r_ts rBrz6SmartAdviceAPIClient._query_string..sAAABFB<AAArDusernamec g|] \}}|||f Srrr s rBrz6SmartAdviceAPIClient._query_string..s%???41aQ?A???rD,)safe)supported_advice_typesrappendextendrrVrWrX)r?rYrZ user_contexts rBr[z"SmartAdviceAPIClient._query_strings BA*@AAA)++  7 NNJ 5 6 6 6??6<<>>???@@@|%%gC%888rDNcL|jd}|||dddS)z] Send given data to Adviser microservice :param data: dict with data z /requests/addFryr]NrIrr)s rBr*zSmartAdviceAPIClient.__call__s5 (777 8TEU KKKKKrDrwrxc |jd| |j|||j}n8#t $r+}|||dtd|d}~wwxYw||dS)z Perform PATCH request to given endpoint. Add payload as JSON. :param endpoint: target URL :param payload: dict with data to PATCH :return: full response zSending PATCH request to %s)rlr}z PATCH failedz/Failed to PATCH data to Smart Advice API serverNFrr) r:rjr<patchrSr rrQrp)r?rwrxrrs rB_patchzSmartAdviceAPIClient._patchs 6AAA <%%hWdm%TTDD   ++8^Q/`-a-a   (((GGGs"A A5 &A00A5cJ|jd}|||ddS)z9 Send statistics to Adviser microservice z/requests/metadataFrrNrlr)s rB send_statzSmartAdviceAPIClient.send_stats3(<<< 8TU ;;;;;rD advice_listcg}t|}|D]g}t|dd}t|j|ddr)t |js||h|S)z Loop over advices and remove those which have non-existing users and those which are invisible. :param advice_list: list of advices received from API advicermetadatarb)r#rr module_namerrg)r?rrvisible_advicesfilteredradvice_instances rB_filter_advice_listz(SmartAdviceAPIClient._filter_advice_lists*;77 - -D1$x.2HIIO!+T*-=j-I -3O4OPP - &&t,,,rDTFrxshow_allc|jd||}||d}||t}|r||}|S)z Get list of advice :param filtered: Automatically removes invisible advices and those which are inked to non-existing users. z /advice/list?)r{FrrrurP)rIr[rrWrrz)r?rxr{rwr\s rBrrz SmartAdviceAPIClient.advice_listsw (^^t7I7IS[7I7\7\^^>>(e>DD>>x8J>KK  ://99HrDc|jd|}||d}||tS)z> Get urls/advices information per user`s site z/advice/site_info/Frrr})rIrrWr)r?rbrwr\s rB site_infozSmartAdviceAPIClient.site_infosF(FFHFF>>(e>DD~~84J~KKKrD advice_idc|jd|d}||d}||tS)z= Get details of an advice by given advice_id z /v2/advice/z/detailsFrrr})rIrrWr)r?rrwr\s rBadvice_detailsz#SmartAdviceAPIClient.advice_detailssI(HHYHHH>>(e>DD~~84J~KKKrDc fd|D}jd|}||S)z Partial update of an advice by given advice_id. Fields allowed for update are limited by fields_allowed property c.i|]\}}|jv||Sr)r^rs rBrz6SmartAdviceAPIClient.update_advice.. s,LLLA18K3K3K13K3K3KrDz/advice/)rrIro)r?rrYrurws` rB update_advicez"SmartAdviceAPIClient.update_advicesQ MLLLLLL(==)=={{8T***rDcJ|jd}|||dddS)z: Sends analytics data to the microservice z/analytics/eventsF)r]rzryrlr)s rBreportzSmartAdviceAPIClient.reports2(;;;zz(DEu_dzeeerDr)TF)rHrrrrCrr rWrrmrIrr^r[rtr*rorqrrzrrrrrrrrrrs@rBrHrHs>11111KcK6KcKKKK3s333X3 ....X. 9 9 9 9 9LTLdLLLLHHsHTHSHHHH&>>zz(\:$>bgzhhhrDcl|jd||}||S)Nz /cdn/usage?)rrrs rB get_usagezAWPProvisionAPIClient.get_usageQs<(``T5G5GS]5G5^5^``yy"""rDr)rHrrrrCrmr[rrIrrrtrvrrrrrrrrs@rBrrsu11111.....3s333X3 XX(XtXW[XXXX  c  3         #  s  S      # s S    ix}iiii#C########rDr)Drrr4rlr8 urllib.parserVr functoolsrtypingrrrrequestsrrrequests.adaptersr requests.exceptionsr $requests.packages.urllib3.util.retryr rPr r !xray.internal.clwpos_safe_importsrradviser.advice_typesrrfrxrayrrQxray.apiclient.schemasrrrinternal.constantsrrrinternal.exceptionsrrrinternal.local_countersrinternal.typesrinternal.user_plugin_utilsrr internal.utilsr"xray.adviser.advice_helpersr#r%rrrrr%r,rHrrrDrBrs   &&&&&&&&&&&&&&&&&&))))))000000666666&&&&&&&& \[[[[[[[eeeeeeeeeeFFFFFFFFFFWWWWWWWWWW888888!!!!!!SSSSSSSS++++++??????DODODODODODODODON!!!!!!!!B33333&)3338p#p#p#p#p#p#p#p#fKKKKK KKK\33333333*@(@(@(@(@(&@(@(@(FMfMfMfMfMf6MfMfMf`;#;#;#;#;#F;#;#;#;#;#rD