
    Hmde:                     T   d dl Z d dlZd dlZd dlZd dlZd dlmc mZ d dl	m
Z
 d dlmZmZ d dlZd dlmZ d dlZd dlZd dlmZ d dlmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ d dl%m&Z&m'Z' d dl(Z(d dl(m)Z)  e*ed	          rej+        ne,Z+d
 Z- G d de.          Z/d Z0d Z1ddZ2dS )    N)Lock)ECHILDESRCH)SSH)rconf)selectwaitpid
errno_wraplfgrabpidfileset_term_handlerGsyncdErrorThreadfinalizeVolinfoVolinfoFromGconfgf_eventEVENT_GEOREP_FAULTYget_up_nodesunshare_propagation_supported)GeorepStatusset_monitor_status)pipe
ParseErrorc                    |                                 }|                    ||          }|                    ||          }|                    ||          }t	          j        d|           |r|s| |                    |          z
  } |dk    r|n|}t          | dz   |z            }| dz   |z  }	|	dk    r|dz   }|r|rdt          |          z   S |r|sdt          |          z   S t          |          S )Nzprimary-distribution-countr      hot_cold_)	is_tierdisperse_countreplica_countdistribution_countgconf	setconfigget_hot_bricks_countintstr)
	brick_idxvolhottierr    r!   distribute_countsubvol_sizecntrems
             @/usr/lib/x86_64-linux-gnu/glusterfs/python/syncdaemon/monitor.pyget_subvol_numr1   %   s   ;;==D''c22N%%dC00M--dC88	O02BCCC ?S ? 8 8 > >>	$2Q$6$6..MK
y1}+
,
,Cq=K
'C
QwwAg  C  
 s S!!3xx    c                   Z    e Zd ZdZdZdZdZdZdZeeeegZ	d Z
ed             Zd	 Zd
 ZdS )Monitorz-class which spawns and manages gsyncd workerszInitializing...StartedActiveFaultyinconsistentc                 H    t                      | _        i | _        i | _        d S N)r   lockstatestatus)selfs    r0   __init__zMonitor.__init__H   s    FF	
r2   c                      t          d            t          t          j        t          j                     t
          j        gt          g           d S )Nc                      t                      S r:   )r   )as    r0   <lambda>z#Monitor.terminate.<locals>.<lambda>Q   s    $4$6$6 r2   )r   r
   oskillgetpidsignalSIGTERMr    r2   r0   	terminatezMonitor.terminateM   sA     	6677727bikk\6>:UGDDDDDr2   c	                    | j                             |d         d         d          sct          t          j        d          |d         d         |d         d         |d         d         ||d|          | j         |d         d         <   d}	d.d}
d	 }d
 }t          j        d          }|	dv r|d         d                             d          \  }}|d         d         }|}t          |t          j        d                    }||f|vr<t          |          dk    r)t          j        |          }|d|d         }|d         }| j	        
                                 | j         |d         d                                      | j                   t          j        t          d|d         d         |                     t!                      \  }}t#          j                    }|dk    rt#          j        |           |dt(          j        j        t(          j        j        dt1          |          d|d         d         d|d         d         d|d         d         d|dt1          |d                   d|d|gz   }t(          j        j        |dt(          j        j        gz  }|d         r|                    d           t(          j        j        r|                    d           t          j        d           }|r t#          j        t:          j        |           ntt?                      r3t          j        d!           g d"}||z   }t#          j         d#|           n3t          j        d$           t#          j        t:          j        |           |!                    |           t#          j        |           | j	        "                                 tG          j#                    }tI          |fd%d%|          d         }t#          j        |           |r |
|t"          j%                  }	|	1t          j        t          d&|d         d         '                     nt          j        d(|d         d         z             tG          j#                    ||z   k     rv |
|t"          j%                  }	|	0t          j        t          d)|d         d         '                     n.tG          j&        d           tG          j#                    ||z   k     vnht          j        t          d*|d         d         |+                     tO          t"          j(        |tR          j*        gtV          g            |
|          }	|	 |
|          }	 ||	          rd}	n ||	          }	|	dv rv| j         |d         d                                      | j,                   t[          t\          |j/        |d         d         |d         d         ||||d         d         ,           tG          j&        d-           |	dv | j         |d         d                                      | j0                   |	S )/aF  the monitor loop

        Basic logic is a blantantly simple blunt heuristics:
        if spawned client survives 60 secs, it's considered OK.
        This servers us pretty well as it's not vulneralbe to
        any kind of irregular behavior of the child...

        ... well, except for one: if children is hung up on
        waiting for some event, it can survive aeons, still
        will be defunct. So we tweak the above logic to
        expect the worker to send us a signal within 60 secs
        (in the form of closing its end of a pipe). The worker
        does this when it's done with the setup stage
        ready to enter the service loop (note it's the setup
        stage which is vulnerable to hangs -- the full
        blown worker blows up on EPIPE if the net goes down,
        due to the keep-alive thread)
        r   dirN
state-filehostuuidz::c                     	 t          | |          \  }}|sd S |S # t          $ r}|j        t          k    rY d }~dS  d }~ww xY w)N)r	   OSErrorerrnor   )pop2res        r0   nwaitzMonitor.monitor.<locals>.nwaits   sh    1A F    7f$$22222s     
A<<Ac                 n    t          j        |           o!t          j        |           t          j        k    S )z, child terminated due to receipt of SIGUSR1 )rD   WIFSIGNALEDWTERMSIGrG   SIGUSR1ss    r0   exit_signalledz'Monitor.monitor.<locals>.exit_signalled   s&    N1%%L2;q>>V^+KMr2   c                 V    t          j        |           rt          j        |           S dS )Nr   )rD   	WIFEXITEDWEXITSTATUSr^   s    r0   exit_statusz$Monitor.monitor.<locals>.exit_status   s'    |A )~a(((1r2   zconnection-timeout)r   r   r   @ssh-portzstarting gsyncd worker)bricksecondary_nodeworkerz--feedback-fdz--local-pathz--local-nodez--local-node-idz--secondary-idz--subvol-num   z--resource-remotez--resource-remote-idz-c   z--is-hottierz--debugzaccess-mountz#Worker would mount volume privately)unsharez-mz--propagationprivaterl   z/Mount is not private. It would be lazy umountedrI   z*worker died before establishing connection)rg   zworker(%s) connectedzworker died in startup phasea  Worker not confirmed after wait, aborting it. Gsyncd invocation on remote secondary via SSH or gluster primary mount might have hung. Please check the above logs for exact issue and check primary or secondary volume for errors. Restarting primary/secondary volume accordingly might help.)rg   timeout)primary_volumeprimary_nodeprimary_node_idsecondary_hostsecondary_volumecurrent_secondary_host
brick_path
   )r   )1r=   getr   r#   splitr   lenrandomchoicer;   acquireset_worker_statusST_INITlogginginfor   r   rD   forkcloser   argsprimary	secondaryr'   config_fileappenddebugexecvsys
executabler   execvpaddreleasetimer   WNOHANGsleepr
   rE   rG   SIGKILLr   	ST_FAULTYr   r   volumeST_INCON)r>   wargvcpidssecondary_volrr   r   suuidsecondarynodesretrY   r`   rd   conn_timeoutremote_userremote_host	remote_idrt   secondary_up_hosts
remote_newprpwcpidargs_to_workeraccess_mountunshare_cmdcmdt0sos                                r0   monitorzMonitor.monitorU   sj   ( {qtE{D11 	P'3EIl4K4K45aDL45aDK45aDL4;@N@M5O(P (PDK!U$ 	 	 	 		N 	N 	N	 	 	
 y!566Vmm'(tAw}}S'9'9$K!QI &1"!-nei
>S>S!T!T&	2:LLL)**Q..!'/A!B!BJ-8[[*Q--"HK *1I IK!U$66t|DDDL4"#A$u++68 8 8 9 9 9 VVFB799Dqyy!%J&J(#SWW"AaDK"AaDL%qtF|$e"C!II'*I) " :)5"tUZ-C&DDNQ4 :")).999:# 5")))444$y88 AHS^^<<<<466 	A&KLLL'2 '2 '2)N:	)S1111 '2 3 3 3@@@IIdOOOHRLLLIBr2|44Q7BHRLLL "eD"*--?L %1*+A$u+"7 "7 "7 8 8 8 8 M"81Q4;"FGGG)++\(999#eD"*55?#L,J23A$u+*? *? *? @ @ @!
1 )++\(999  J qT%[+- - -. . . 27T6>$:UGDDDeDkk{eDkk~c"" 5!k#&&&==K!U,>>t~NNN0,3N*+A$v,-.qT&\,:.;4J()!U5 5 5 5 JrNNNs Vmmt 	AaDK 224=AAA
r2   c           	      (    t           j                            t          j                  t          j        d         gt                      g }|D ]I} fd}	t          |	|g          }
|
                                 |	                    |
           J j
                                         t          t          j        d           j                    j
                                         |D ]}
|
                                 d S )Nr   c           
      V   	                     | 
          \  }}t          j        d           	j                                         D ]/}t          t          j        |t          j	        gt          g           0	j                                         t          d           d S )Nr   )exval)r   r   r   r;   r|   r
   rD   rE   rG   r   r   r   r   )r   r   _r   r   r   rr   r   r   r>   r   s      r0   wmonzMonitor.multiplex.<locals>.wmon  s    ,,q$}'5w~W Wa
1	!!###! I IDrwv~(>HHHH	!!###q!!!!!!r2   )targetr   rM   )rD   pathbasenamer   r   r   setr   startr   r;   r|   r   r#   rw   
ST_STARTEDr   join)r>   wspxr   r   rr   r   r   tawxr   tr   r   s   ` `````    @@r0   	multiplexzMonitor.multiplex  s'     00#(1+> 	 	B" " " " " " " " " " " " d"...AGGIIIIIaLLLL 		59\22DODDD	 	 	AFFHHHH	 	r2   N)__name__
__module____qualname____doc__r~   r   	ST_STABLEr   r   _ST_ORDr?   staticmethodrJ   r   r   rI   r2   r0   r4   r4   =   s        77GJIIH'9h7G  
 E E \Eq q qf         r2   r4   c           
         t           j        j        rt          | j        d          }nt          | j        | j        d          }t          j        dt          |j
                  z              g }d }d }t          j        d          gt          j        d                                          z   dt          t          j        d                    gz   |j        gz   }t          j        d|j        z              t           j        j        rt          |j        d	          }nt          |j        d
|d	          }|j
        }|j        }|j                            d          d         }|j        }t          j        dd           }	d|j        z   dz   |j        z   }
|	r|	|
k    rt          j        d|
           t          j        dt          |          z              t%          d |D                       }t'          j        |          fd|D             }g }t+          |j
                  D ]\  }}t           j        j        |d         k    rt|                    d                    |d         |d         g                    }|                    |||t5          |          z           t7          |||          |f           t          j        dt          |          z              ||||| |fS )NT)r   zprimary bricks: zssh-commandzssh-optionsz-prf   zsecondary SSH gateway: F	localhostre   rQ   zstime-xattr-prefixztrusted.glusterfs..zsecondary bricks: c              3   6   K   | ]}|d          |d         fV  dS )rN   rO   NrI   ).0bs     r0   	<genexpr>zdistribute.<locals>.<genexpr>P  s/      AAA!F)QvY/AAAAAAr2   c                 H    g | ]}d          dz   |d         z   |d         fS )userre   r   r   rI   )r   hraps     r0   
<listcomp>zdistribute.<locals>.<listcomp>R  s4    KKKq3v;$qt+QqT2KKKr2   rO   :rN   rL   zworker specs: )r   r   use_gconf_volinfor   r   r   rN   r   r   reprbricksr#   rw   rx   r'   remote_addrrO   r$   r   r   parse_ssh_address	enumeratelocal_node_idis_hotr   r   ry   r1   )r   r   mvolpreluderr   r   svolsbricksr   old_stime_xattr_prefixnew_stime_xattr_prefixr   
secondarys
workerspexidxrg   r   r   s                    @r0   
distributer   +  s   z# C===w~w|TBBBM$tDK'8'88999GNMy''(	-  &&(()	s59Z(())*+ 
	 G
 M+i.CCDDDz# N	 0%@@@y'guMMMkGIE*0055b9N$M #Y';TBB1DI=C!Y'! F!777,.DEEEM&g6777AAAAAAAN

	
*
*CKKKKNKKKJJ,, ( (
U:#uV}44[[5=%,*G!H!HIIFu)#J*?@-c4@@%' ( ( ( M"T*%5%55666um^WnTTr2   c                     t           j        j        r?t          t          j        t	          j                     t          j        gt          g           	  t                      j        t          | |           S r:   )r   r   pause_on_startr
   rD   rE   rF   rG   SIGSTOPr   r4   r   r   )localremotes     r0   r   r   `  sX     z  E27bikk\6>:UGDDD=799
5& 9 9::r2   Tc                 b   t          j        d          }t                      s3t          j                            d           t          j        d           dt          _        | sdS t                      \  }}t          j                    }|r't          j        |           t          j                     t          j        |           t          j                     t          j        t          j        t          j                  }t          j        t          j        t          j        fD ])}t          j        ||                                           *t          |dz             st+          d          t          j        |dz   |           t/          |fdd           t          j        |           dS )	z/set up logging, pidfile grabbing, daemonizationzpid-filezpidfile is taken, exiting.
rj   TNz.tmpzcannot grab temporary pidfilerI   )r#   rw   r   r   stderrwriteexitr   pid_file_ownedr   rD   r   r   setsidopendevnullO_RDWRstdinstdoutdup2filenor   renamer   )	go_daemonpid_filexyr   dnfs          r0   startupr   k  sV   y$$H== 
7888E 66DAq799D 



HQKKKIKKK	RY	'	'BiSZ0    
AHHJJx&()) ;9:::Ih*** A4RHQKKKKKr2   )T)3rD   r   r   rG   r   xml.etree.ElementTreeetreeElementTreeXET	threadingr   rS   r   r   rz   resourcer   gsyncdconfigr#   libgfchangelogr   
syncdutilsr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   gsyncdstatusr   r   py2py3r   hasattrr   SyntaxErrorr1   objectr4   r   r   r   rI   r2   r0   <module>r     s;   
			 



    # # # # # # # # #                                   7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7 7
 : 9 9 9 9 9 9 9       &wsL99JS^^{
  0k k k k kf k k k\2U 2U 2Uj; ; ;           r2   