HmdDjddlZddlZddlZddlZddlZddlmZddlmZm Z ddl Z ddl m Z n#e$rddl Z YnwxYwddlZddlZddlmZmZddlmZmZmZddlmZddlmZddlmZddlZd Zd Zd Zd ZdZ dZ!ej"Z#d Z$dZ%dZ&dZ'dZ(dZ)dZ*dZ+dZ,dZ-e.dkre-Z/eej01ej2de/j3e/j4dej01ej2de/j3e/j4dZ5ee#e5e/j6ej01ej2de/j3Z7ej01e7e/j4de j8e/j9zZ:e:dzZ;eej01e7e/j4de#d ZZ>e/j<Zdddn #1swxYwYn#eDeEeFf$r e/j>Z>YnwxYwed"e<e,e/j9e>e@5B5BCC   q6R<<  's1v66D&tT22D  - -dCF ; ; ; ;!    KK6: ; ; ; HHHH   s1AA11B)B$$B)ADE.EEc r|dddD]}|d}tj|d|dd|dd|}tj|r| t ||}t|t}| d |id |i#ttf$r'}t d |zYd }~d }~wwxYw |tj|j|t#j|}|D]T}t'|} | d r.|| ddU#ttf$rYwxYwd S)z For all the DATA/METADATA modifications GFID, If symlink, directly convert to Path using Readlink. If not symlink, try to get PGFIDs via xattr query and populate it to pgfid table, collect inodes in inodegfid table rMODIFYrrr .glusterfsrrgfidrNztrusted.pgfid..) gfidpath_getstriposr%joinislinkrr rgfidpath_updaterrrr inodegfid_addstatst_inoxattrlistr startswith pgfid_addsplit) r"r#r$r/pr%r& file_xattrsxx_strs r'populate_pgfid_and_inodegfidrDSs**R+J+JKK1v||~~ GLL d1Q3iacD I I 7>>!    +E488*466..06~????W%    :Q>???  ,,RWQZZ->EEE#jmm $GGA,Q//E''(899G&00S1A1A"1EFFF G W%    /s+ ?C  DC<<DBFF43F4c g}tj|d|dd|dd|}tj|s. t j|}|D]}t |}|drt j||}t |} | tj \} } t|| } tj| | } t| |} | | #ttf$r'}t d|zYd}~d}~wwxYwn#ttf$rYnwxYw|S)Nr,rr-r.ztrusted.gfid2path.r)r4r%r5isdirr;r<rr=getxattrr?seprr appendrrrr )r"r/r hardlinksr@rArBrCvv_strpgfidbnr%fullpathr&s r'enum_hard_links_using_gfid2pathrPwsI  UL$qs)T!A#YEEA 7==   *Q--K  ! !(++##$899 !q%00A,Q//E % BF 3 3IE2!3E5AA#%7<<b#9#9#6x#F#F!((2222#W-!!! $BQ$FGGG ! ! !!    D  s>BE-AD0/E-0E(E#E-#E((E--FFc.d}|dddD]y}|d}td|zt |||}d|}|d|id|izdS) Nrr*r+rzProcessing gfid %s,rr/)r2r3rdebugrPr5r7)r"r#rr%r$r/rJs r'!gfid_to_all_paths_using_gfid2pathrTs D**R+J+JKKHH1v||~~ )D01113E4FF xx ""&&GGGGHHc2 tddisdSfd}t  fd}fdtjddD}D]} t|d}ttj ||||d L#ttf$r'}td |zYd}~d}~wwxYwdS) z For all the pgfids collected, Converts to Path and does readdir on those directories and looks up inodegfid table for matching inode number. convertedrNc tj|}n#ttf$rd}YnwxYw|r#d|jir|jSdSNinoder4lstatrrinodegfid_existsr:r%str#s r' inode_filterz.gfid_to_path_using_pgfid..inode_filters $BB!   BBB   .117BI2FGG 9 t ..c|}|dzd}t|}||ddid|idS)NrWrZ)r3r append_path1inodegfid_update)r%rZrbrick_path_lenr#s r'output_callbackz1gfid_to_path_using_pgfid..output_callbacksrzz||N1$%%&"4..##D%000''a(87E:JKKKKKrUcPg|]"}tj|#Sr4r%r5.0dirnamer"s r' z,gfid_to_path_using_pgfid..?AAA7<<w//AAArUbrick_ignore_dirsrRF) callback_func filter_func ignore_dirs subdirs_crawlr)rDr]lenconfget_optr? pgfid_getrr r4r%r5rrrr ) r"r#rr`rhrtr$r%r&rgs ``` @r'gfid_to_path_using_pgfidrzs !777  * *K+; < <     ZZN L L L L L L LAAAA< 344::3??AAAK''))   's1v66D eT**.)($  & & & & & !    KK6: ; ; ; HHHH   sACD-DDcddisdSfd}tfd}fdtjddD}t |||dS) NrWrc tj|}n#ttf$rd}YnwxYw|r#d|jir|jSdSrYr[r^s r'r`z2gfid_to_path_using_batchfind..inode_filterrarbc|}|dzd}t|t}||dS)Nrd)r3r rre)r%rZrgr#s r'rhz5gfid_to_path_using_batchfind..output_callbacksRzz||N1$%%&"4..##D%00000rUcPg|]"}tj|#Srjrkrls r'roz0gfid_to_path_using_batchfind..rprUrqrR)rrrsrt)r]rvrwrxr?r )r"r#r`rhrtrgs`` @r'gfid_to_path_using_batchfindrs  * *K+; < <     ZZN111111AAAA< 344::3??AAAK  o! """"""rUctj|d5}tj|}|D]}|d}|ddkr!|ddvr|||W|ddvr|js| |||ddkr!|dd vr| |||ddkr#|dd kr| |||ddkr |dd vr| || d d d d S#1swxYwYd S) zF Parses a Changelog file and populates data in gfidpath table zutf-8)encoding rEr-)CREATEMKNODMKDIR)DM)LINKSYMLINKr)UNLINKRMDIRN) codecsopenr4r%basenamer3r?when_create_mknod_mkdironly_namespace_changeswhen_data_metawhen_link_symlink when_renamewhen_unlink_rmdir)r#filenamerf changelogfilelinedatas r'parse_changelog_to_dbrs X 0 0 0FA((22  F FD::<<%%c**DAw#~~$q'-I"I"I66}dKKKKaJ&&2G"11-FFFaCDG/B$B$B00EEEEaCDGx$7$7**=$????aCDG/B$B$B00EEE# FFFFFFFFFFFFFFFFFFFsD+EEEctjtjd|j}tj||jdtj|j z} t|5}t| }dddn #1swxYwYn$#ttt f$r |j}YnwxYw t%jt%j|||t*t,n7#t$j$r%} t1|d| t2Yd} ~ nd} ~ wwxYwt5|j|} tj|d} t%j| ||t:} n?#t$j$r-} t1|jd|d| t2Yd} ~ nd} ~ wwxYwt2d  g} t%j d krt%j!} | D]x}|"d |zr tG| ||t%j$|B#t $r*} t2%d |d| Yd} ~ qd} ~ wwxYw| &t%j d kn7#t$j$r%} t1|d | t2Yd} ~ nd} ~ wwxYwt2dt2dtO|| | &t2dt2dtQ|| || &t2dt2dtS|| | &t2d| S)z Makes use of libgfchangelog's history API to get changelogs containing changes from start and end time. Further collects the modified gfids from the changelogs and writes the list of gfid to 'gfid_list' file. session_dir %s.statusNz Changelog register failed: )rz.glusterfs/changelogsz: z& Historical Changelogs not available: z$[1/4] Starting changelog parsing ...rz.%szError parsing changelog file z Error during Changelog Crawl: z![1/4] Finished changelog parsing.z.[2/4] Starting 'pgfid to path' conversions ...z+[2/4] Finished 'pgfid to path' conversions.z*[3/4] Starting 'gfid2path' conversions ...z'[3/4] Finished 'gfid2path' conversions.z=[4/4] Starting 'gfid to path using batchfind' conversions ...z:[4/4] Finished 'gfid to path using batchfind' conversions.)*r4r%r5rwrxsessionvolumeurllib quote_plusr"rintreadr3 ValueErrorrrstartlibgfchangelogcl_init cl_registerCHANGELOG_LOG_LEVELCHANGELOG_CONN_RETRIESChangelogExceptionrrr outfilecl_history_changelogCHANGELOGAPI_NUM_WORKERSnodeinfocl_history_scancl_history_getchangesendswithrcl_history_doner commitr(rTr)r"hash_dirlog_filerendrr status_filerr&r#cl_path actual_endchangeschanges r' get_changesrs',,t|M::#|--K',,{DK 6#4TZ#@#@@BBK +   *!(())E * * * * * * * * * * * * * * *  ) M   "5(H#68N P P P P  ,MMM 555!! >> KK<===%e^TBBB KK9::: KK"### 777 KKLMMM s9C4C< CC  CC CC54C595D//E#>EE#F99G5#G00G5A K&%JK& J7 J2-K&2J77.K&&L5LLc|dr|dt|dz }tj|j}t j|}t| }tj ||}t|dttj tjd|j|jd|z}t|d|d |t'||||||S) zG Init function, prepares working dir and calls Changelog query /rrdT exit_on_errrlog_dirzchangelog.%s.logz! Started Changelog Crawl. Start: z, End: )rrvr4r%rnrhashlibsha1encodestr hexdigestr5rrrwrxrrrr)r"rrr working_dir brickhashrs r'changelog_crawlrvs ~~c&aE 1 n%'//$,//K U\\^^,,II''))**I',,{I66K ;D8888w||DL33 L K.:<rsq ))))))@@@@@@@@!!!!!!!MMMMM ........**********------%%%%%%''''''      !!!H!!!H6 H H H555p%"%"%"PFFF4[[[|GGG26 z 9;;D F27<<  Y//t{ K Kw||LDL33 L K+--HL4:...',,|t|M::DLIIK',,{DK #46#4TZ#@#@@BBK!F*O F27<< T[ 1 1t C  h k"" .aAFFHHNN,,-- . . . . . . . . . . . . . . .GW-   JEEE   byyc)$)++!>J ?% T/3 ' ' %1 GGCC OO $ $ $ % % % % % % % % % % % % % % % KK4:::;E:GHHH CHQKKKKKWsT/ ;;- I80H4( I4H88I;H8<III K77K;>K;