
    #Sha9                        U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlm	Z	 ddl
m
Z
mZmZmZ ddlmZ dZi Zej        edf         ed<    G d	 d
e
j        e                   Z G d dej                  Z G d de
j                  Z	 	 	 	 	 	 d!de	dededededededej        e         ddfdZ G d d           Z dS )"    N)Path   )corejob_environmentloggerutils)R   subprocess.Popen['bytes']
_PROCESSESc                   ,    e Zd Z	 	 ddej        eef         dedej        e         dej	        d         ddf
 fd	Z
ddedefdZedefd            Zddedej        eef         fdZddeddfdZddZddZdej        fdZdej        ddfdZ xZS )LocalJobr   Nfolderjob_idtasksprocessr   returnc                     t                                          |||           d| _        | j        | _        || _        | j        D ]	}||_        
d S )NF)super__init___cancel_at_deletion	_sub_jobs_process)selfr   r   r   r   sjob	__class__s         Z/var/www/html/movieo_spanner_bot/venv/lib/python3.11/site-packages/submitit/local/local.pyr   zLocalJob.__init__   s]     	///#( 59^N 	$ 	$D#DMM	$ 	$    Fforce_checkc                 B    |                                  d         }|dk    S )z#Override to avoid using the watcherjobStateRUNNING)get_info)r   r    states      r   donezLocalJob.done.   s    
+	!!r   c                 v    	 |                                                      dd          S # t          $ r Y dS w xY w)zState of the jobr"   unknownUNKNOWN)r$   get	Exceptionr   s    r   r%   zLocalJob.state3   sG    	==??&&z9=== 	 	 	99	s   '* 
88forcemodec                     | j         &d}| j        j                                        rd}d|iS | j                                         }|d}n|dk     rd}nd}d|iS )z,Returns information about the job as a dict.NzNO PROCESS AND NO RESULTFINISHEDr"   r#   r   INTERRUPTED)r   pathsresult_pickleexistspoll)r   r.   r%   r5   s       r   r$   zLocalJob.get_info<   sw    = .Ez'..00 #"&&}!!##<EEAXX!EEEE""r   Tcheckc                 `    | j         &| j                             t          j                   d S d S N)r   send_signalsignalSIGINT)r   r6   s     r   cancelzLocalJob.cancelL   s0    =$M%%fm44444 %$r   c                 z    | j         3| j                             t                                                     dS dS )zBSends preemption / timeout signal to the job (for testing purpose)N)r   r9   LocalJobEnvironment_usr_sigr,   s    r   
_interruptzLocalJob._interruptP   s;    =$M%%&9&B&B&D&DEEEEE %$r   c                    | j         rA|                                                     d          dk    s|                     d           | j        j                                        r"t                              | j	        d            d S d S )Nr"   r0   F)r6   )
r   r$   r*   r<   r2   r3   r4   r   popr   r,   s    r   __del__zLocalJob.__del__U   s    # 	)==??&&z22j@@%(((:#**,, 	.NN4;-----	. 	.r   c                 n    t          | j                  }d |d<   | j        | j        t          | j        <   |S )Nr   )dict__dict__r   r   r   )r   outs     r   __getstate__zLocalJob.__getstate___   s5    4=!!J=$&*mJt{#
r   r%   c                     | j                             |           t                              | j        d           | _        d S r8   )rF   updater   r*   r   r   )r   r%   s     r   __setstate__zLocalJob.__setstate__f   s3    U###"t{D99r   )r   N)F)r-   )Tr   N)__name__
__module____qualname__tpUnionr   strSequenceintOptionalr   boolr&   propertyr%   Dictr$   r<   r@   rC   AnyrH   rK   __classcell__r   s   @r   r   r      s       
 #'<@$ $s#$ $ {3	$
 89$ 
$ $ $ $ $ $ " " " " " " "
 s    X# #S #rwsCx/@ # # # # 5 5D 5D 5 5 5 5F F F F
. . . .bf    :"& :T : : : : : : : :r   r   c                   0    e Zd ZdddddddZded	d
fdZd
S )r>   SUBMITIT_LOCAL_JOB_IDSUBMITIT_LOCAL_NTASKSSUBMITIT_LOCAL_JOB_NUM_NODESSUBMITIT_LOCAL_NODEIDSUBMITIT_LOCAL_GLOBALIDSUBMITIT_LOCAL_LOCALID)r   	num_tasks	num_nodesnodeglobal_rank
local_rank	countdownr   Nc                     | j         }t          j                                        d| d| d           t	          j        t                     d S )NzRequeued job z (z remaining timeouts))r   r   
get_loggerinfosysexitLOCAL_REQUEUE_RETURN_CODE)r   rh   jids      r   _requeuezLocalJobEnvironment._requeuew   sQ    k  !W!W!W	!W!W!WXXX*+++++r   )rM   rN   rO   _envrT   rp    r   r   r>   r>   m   sT        ),3'0. D,# ,$ , , , , , ,r   r>   c                   |    e Zd ZdZeZ	 	 	 ddej        ee	f         de
dedej        e         d	df
 fd
Zed	ej        e         fd            Zdej        d	df fdZded	ee         fdZed	efd            Zd	e
fdZdeded	efdZedej        eef         d	efd            Zde	d	ej        e         fdZ xZS )LocalExecutora  Local job executor
    This class is used to hold the parameters to run a job locally.
    In practice, it will create a bash file in the specified directory for each job,
    and pickle the task function and parameters. At completion, the job will also pickle
    the output. Logs are also dumped in the same directory.

    The submission file spawn several processes (one per task), with a timeout.


    Parameters
    ----------
    folder: Path/str
        folder for storing job submission/output and logs.

    Note
    ----
    - be aware that the log/output folder will be full of logs and pickled objects very fast,
      it may need cleaning.
    - use update_parameters to specify custom parameters (n_gpus etc...).
             ?Nr   max_num_timeoutmax_pickle_size_gbpythonr   c                    t                                          |||           |t          j        t          j                  n|| _        t          j        	                    | j
                  }|                    dd           d S )N)rx   rw   T)parentsexist_ok)r   r   shlexquoterl   
executablery   r   JobPathsget_first_id_independent_folderr   mkdir)r   r   rw   rx   ry   indep_folderr   s         r   r   zLocalExecutor.__init__   s     	1+ 	 	
 	
 	

 6<^ek#.111~EEdkRR4$77777r   c                     dhS )z4Parameters that can be set through update_parameterssetuprr   )clss    r   _valid_parameterszLocalExecutor._valid_parameters   s     yr   kwargsc                    |                     dd          dk    rt          d          |                     dd          }|                     dd          }t          |t          j                  st          d| d	          t          d
 |D                       st          d| d          t          |          dk    r(|t          |          k    rt          | d| d           t                      j        di | dS )ao  Update the parameters of the Executor.

        Valid parameters are:
        - timeout_min (float)
        - gpus_per_node (int)
        - visible_gpus (Sequence[int])
        - tasks_per_node (int)
        - nodes (int). Must be 1 if specified
        - signal_delay_s (int): signal (lately: USR2) delay before timeout

        Other parameters are ignored
        nodesr      z0LocalExecutor can use only one node. Use nodes=1gpus_per_nodevisible_gpusrr   zProvided visible_gpus=z  is not an instance of Sequence.c              3   @   K   | ]}t          |t                    V  d S r8   )
isinstancerT   ).0xs     r   	<genexpr>z<LocalExecutor._internal_update_parameters.<locals>.<genexpr>   s,      <<!:a%%<<<<<<r   z( contains an element that is not an int.z gpus requested, but only z were specified visible.N)	r*   
ValueErrorr   rP   rS   alllenr   _internal_update_parameters)r   r   gpus_requestedr   r   s       r   r   z)LocalExecutor._internal_update_parameters   s,    ::gq!!A%%OPPPOQ77zz."55,44 	fdldddeee<<|<<<<< 	nlllllmmm|q  ^c,6G6G%G%G!cc\ccc   	,+55f55555r   commandc                    | j                             dd          }| j                             dd          }| j                             dd          }|dk    rt          |          n	|d |         }t          | j        ||d                    d |D                       | j                             d	d
          | j                             dd          | j                             dd          | j                             dd                    }t          | j        t          |j                  |t          t          |                              }|S )Ntasks_per_noder   r   r   r   rr   ,c              3   4   K   | ]}t          |          V  d S r8   )rR   )r   ks     r   r   z0LocalExecutor._submit_command.<locals>.<genexpr>   s(      !7!7Q#a&&!7!7!7!7!7!7r   timeout_ming       @signal_delay_s   stderr_to_stdoutFr   )r   r   r   cuda_devicesr   r   r   r   )r   r   r   r   )

parametersr*   rangestart_controllerr   joinr   rR   pidlist)r   r   ntasksn_gpusr   gpusr   jobs           r   _submit_commandzLocalExecutor._submit_command   sD    $$%5q99$$_a88**>2>> , 2 2uV}}}WfW8M";!!7!7$!7!7!777++M3???../?DD!_001CUKK/%%gr22	
 	
 	
 $;s7;'7'7PTUZ[aUbUbPcPc
 
 
 
r   c           	          d                     | j        dt          j        t	          | j                            g          S )N z-u -m submitit.core._submit)r   ry   r}   r~   rR   r   r,   s    r   _submitit_command_strz#LocalExecutor._submitit_command_str   s5    xx&CU[QTUYU`QaQaEbEbcdddr   c                 F    d}| j                             dd          }||z  S )Nr   r   )r   r*   )r   r   r   s      r   
_num_taskszLocalExecutor._num_tasks   s*    "o112BAFF~%%r   uidc                     dS )N rr   )r   r   r   s      r   _make_submission_file_textz(LocalExecutor._make_submission_file_text   s    rr   stringc                     dS )N0rr   )r   s    r   #_get_job_id_from_submission_commandz1LocalExecutor._get_job_id_from_submission_command   s	     sr   submission_file_pathc                     g S r8   rr   )r   r   s     r   _make_submission_commandz&LocalExecutor._make_submission_command   s    	r   )ru   rv   N)rM   rN   rO   __doc__r   	job_classrP   rQ   rR   r   rT   floatrU   r   classmethodSetr   rY   r   r	   r   rW   r   r   r   staticmethodbytesr   Listr   rZ   r[   s   @r   rt   rt   }   s        * I  !$'#'	8 8hsDy)88 "8 C 	8
 
8 8 8 8 8 8  "&+    [6BF 6t 6 6 6 6 6 66s x{    , es e e e Xe&C & & & &
# C C     BHUCZ4H S    \T bgcl        r   rt   r   r         @r   Frr   r   r   r   r   r   r   r   r   r   c                    t          t          j                  }|                    t	          |          |t	          t          d|z                      t	          t          |                    dd|rdndd||rdnd
  
         t          j        ddt	          |           g}	t          |          }
|
r8d		                    t          |          t          j	        |	          gz             }	t          j        |	|
|
          }|S )zTStarts a job controller, which is expected to survive the end of the python session.<   r   1r   local)
r^   SUBMITIT_LOCAL_COMMANDSUBMITIT_LOCAL_TIMEOUT_SSUBMITIT_LOCAL_SIGNAL_DELAY_Sr`   r_   SUBMITIT_STDERR_TO_STDOUTSUBMITIT_EXECUTORCUDA_VISIBLE_DEVICESSUBMITIT_LOCAL_WITH_SHELLz-mzsubmitit.local._localz && )shellenv)rE   osenvironrJ   rR   rT   rl   r   rV   r   r   r}   
subprocessPopen)r   r   r   r   r   r   r   r   r   proc_cmd
need_shellr   s               r   r   r      s     rz

CJJ!.11&!$Sk)9%:%:!;!;&)#n*=*=&>&>!%()9"A##r!))."6##B     .Es6{{SHeJ E;;tE{{ej.B.B-CCDDxzsCCCGNr   c                       e Zd ZdZdefdZdej        dej	        ddfdZ
dd	Zdd
Zddedej        ej        e                  fdZddeddfdZdS )
ControllerzThis controls a job:
    - instantiate each of the tasks
    - sends timeout signal
    - stops all tasks if one of them finishes
    - cleans up the tasks/closes log files when deleted
    r   c                    t          t          j        d                   | _        t	          j        t          j        d                   | _        t          t          j        d                   | _        t          t          j        d                   | _        t          t          j        d                   | _
        g | _        g | _        g | _        t          t          j        d                   }t          |rt          j                    nt          j                              | _        t%          |          | _        t)          j        t(          j        | j                   d S )Nr^   r   r   r   r   r   )rT   r   r   r   r}   splitr   	timeout_sr   rV   r   r   stdoutsstderrsrR   getppidgetpidr   r   r   r:   SIGTERM_forward_signal)r   r   
with_shells      r   r   zController.__init__  s    "*%<=>>{2:.F#GHHRZ(BCDD!"*-L"MNN $RZ0K%L M M02
/1/1"*%@ABB
zBrz|||ry{{CC6llfnd&:;;;;;r   signumargsr   Nc                 f    | j         D ](}	 |                    |           # t          $ r Y %w xY wd S r8   )r   r9   r+   )r   r   r   tasks       r   r   zController._forward_signal,  sY    J 	 	D  ((((   	 	s   !
..c                 F     j                             d            fdt           j                  D             }d |D              _         j        r j        nd |D              _        t           j                  D ]}t          t          j	                  }|
                    t          |          t          |           j                    j                            t          j         j        d| j        |          j        |         d	                     d S )
NT)r|   c                 P    g | ]"}t          j        j        j        |          #S rr   )r   r   r   r   )r   r   r   s     r   
<listcomp>z*Controller.start_tasks.<locals>.<listcomp>5  s+    VVVaTXq99VVVr   c                 B    g | ]}|j                             d           S a)stdoutopenr   ps     r   r   z*Controller.start_tasks.<locals>.<listcomp>6  s&    :::qc**:::r   c                 B    g | ]}|j                             d           S r   )stderrr   r   s     r   r   z*Controller.start_tasks.<locals>.<listcomp>7  s(    BeBeBeZ[18==QTCUCUBeBeBer   )rb   ra   r]   Fzutf-8)r   r   r   r   encoding)r   r   r   r   r   r   r   rE   r   r   rJ   rR   r   r   appendr   r   r   )r   r2   r   r   s   `   r   start_taskszController.start_tasks3  s3   4(((VVVV5CUCUVVV::E:::'+'<et||BeBe_dBeBeBet{## 	 	Arz""CJJ'*1vvs1vveiem     J L<?<?$  	 	 	 		 	r   c                    t           j        t           j        fD ]}|                     |           |t           j        k    rdt	          d | j        D                       rFt          j        d           t	          d | j        D                       rt          j        d           g | _        | j        | j	        z   }g g c| _        | _	        |D ]}|
                                 d S )Nc              3   B   K   | ]}|                                 d u V  d S r8   r5   r   ts     r   r   z(Controller.kill_tasks.<locals>.<genexpr>M  s/      +Q+QAFFHH,<+Q+Q+Q+Q+Q+Qr   gMbP?c              3   B   K   | ]}|                                 d u V  d S r8   r   r   s     r   r   z(Controller.kill_tasks.<locals>.<genexpr>O  s/      <<Aqvvxx4'<<<<<<r   rv   )r:   r;   SIGKILLr   anyr   timesleepr   r   close)r   sigfilesfs       r   
kill_taskszController.kill_tasksH  s    M6>2 	$ 	$C  %%%fm##+Q+Qdj+Q+Q+Q(Q(Q#
5!!!<<<<<<< $JsOOO
t|+%'"dl 	 	AGGIIII	 	r      freqc                    | j         s
J d            || j        z  }|| j        | j        z
  z  }t          |          D ]y}d | j         D             }t	          d |D                       r|c S ||k    r,|                     t                                                     t          j	        d|z             zd | j         D             S )a  Waits for all tasks to finish or to time-out.

        Returns
        -------
        Sequence[Optional[int]]:
            Exit codes of each task.
            Some tasks might still have not exited, but they will have received the "timed-out" signal.
        zNothing to do!c                 6    g | ]}|                                 S rr   r   r   s     r   r   z#Controller.wait.<locals>.<listcomp>f  s     777q!&&((777r   c              3      K   | ]}|d uV  	d S r8   rr   r   es     r   r   z"Controller.wait.<locals>.<genexpr>g  s&      55Q1D=555555r   rv   c                 6    g | ]}|                                 S rr   r   r   s     r   r   z#Controller.wait.<locals>.<listcomp>n  s     ---Q---r   )
r   r   r   r   r   r   r>   r?   r  r  )r   r	  timeoutalmost_timeoutstep
exit_codess         r   waitzController.waitW  s     z+++++z'$2E!EF 'NN 	# 	#D77DJ777J55*55555 "!!!!~%%$$%8%A%A%C%CDDDJsTz""""--$*----r      	max_retryc                 >   t          |          D ]}	 |                                  |                                 }t          d |D                       }|s	 |                                   d S 	 |                                  t# |                                  w xY wd S )Nc              3   ,   K   | ]}|t           k    V  d S r8   )rn   r  s     r   r   z!Controller.run.<locals>.<genexpr>w  s(      QQa#<<QQQQQQr   )r   r   r  r   r  )r   r  _r  requeues        r   runzController.runp  s     y!! 	" 	"A"  """!YY[[
QQjQQQQQ !!!!!! !!!!!!!!	" 	"s   ABBrL   )r  )r  )rM   rN   rO   r   r   r   r:   SignalsrP   rY   r   r   r  rT   rS   rU   r  r  rr   r   r   r   r     s         <t < < < <fn RV        *   . . .bk"+c2B&C . . . .2" "S " " " " " " "r   r   )r   r   r   r   Frr   )!r   r}   r:   r   rl   r  typingrP   pathlibr   r   r   r   r   	core.corer	   rn   r   rX   rR   __annotations__Jobr   JobEnvironmentr>   PicklingExecutorrt   rT   r   rV   rS   r   r   rr   r   r   <module>r$     s    
				       



            7 7 7 7 7 7 7 7 7 7 7 7      
    9;
BGC445 : : :M: M: M: M: M:tx{ M: M: M:`, , , , ,/8 , , , q q q q qD) q q qn "    	
    ;s !   Dh" h" h" h" h" h" h" h" h" h"r   