
    #Sh"                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	Z	d dl
mZ ddlmZ ddlmZmZmZ ddlmZmZ d	ed
dfdZd	ed
dfdZd	ed
dfdZd	ed
dfdZd	ed
dfdZd	ed
dfdZd	ed
dfdZd	ed
dfdZe	j                            ej                            dd           d          d	ed
dfd            Z d	ed
dfdZ!d	ed
dfdZ"d	ed
dfdZ#d	ed
dfdZ$d#de%de%d
e%fdZ&d	ed
dfdZ'd	ed
dfd Z(d!ed
dfd"Z)dS )$    N)Path)AutoExecutor   )helpers)job_environment	test_coreutils   )local
test_debugtmp_pathreturnc                    dt           dt           fd}t          j        |           }|                    dd           |                    |d          }|                    d           |                                5  t          j        t          d	          5  |                    d           d d d            n# 1 swxY w Y   |                    |d
          }d d d            n# 1 swxY w Y   |	                                g dk    sJ |
                    d                                          dk    sJ |
                    d
                                          d
k    sJ |
                    d
                                          d
k    sJ |                                J |                                sJ t          j        t          d	          5  |
                    d                                           d d d            n# 1 swxY w Y   |	                                dgk    sJ |
                    d                                          dk    sJ |
                    d          |u sJ |                                sJ t          j        |          }t          j        |          }|	                                dgk    sJ |j        J ~t          j        |          }|j        
J d            d S )Npr   c                 <    t          j                    }| |j        z  S N)r   JobEnvironment
local_rank)r   job_envs     _/var/www/html/movieo_spanner_bot/venv/lib/python3.11/site-packages/submitit/local/test_local.pyfuncztest_local_job.<locals>.func   s    !0227%%%       r
   tasks_per_nodenodes)r   zwith executor.batch)matchr   )r   r
   r   zmust be between 0 and 2   r   z0garbage collection should I removed finished job)intr   LocalExecutorupdate_parameterssubmitbatchpytestraisesRuntimeErrorresultstaskresult	exceptiondone
ValueErrorpickledumpsloads_process)r   r   executorjob1job2bjob3s          r   test_local_jobr6      s   & & & & & & "8,,Haq999??4##Da000			 ( (]</DEEE 	9 	9&&a&888	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9 	9tQ''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( <<>>YYY&&&&99Q<<  A%%%%99Q<<  A%%%%99Q<<  A%%%%>>###99;;;	z)B	C	C	C  		!               <<>>aS    99Q<<  A%%%%99Q<<499;;;TA<??D<<>>aS    =$$$<??D=  "T     sH   >C*B=1C*=C	C*C	C**C.1C.(HHHc                    t          j        d          }t          j        |           }|                    ||j        |j                  }t          t          ||j        |j                            d |D             k    sJ d S )N   c                 6    g | ]}|                                 S  r)   .0js     r   
<listcomp>z(test_local_map_array.<locals>.<listcomp>E        -G-G-GQahhjj-G-G-Gr   )	r   CheckFunctionr   r    	map_arraydata1data2listmap)r   gr1   jobss       r   test_local_map_arrayrI   A   s{     ##A"8,,Ha!'22DAqw(())-G-G$-G-G-GGGGGGGr   c                 >   t          j        d          fdt          j        j                  D             }t          j        |           }|                    |          }t          t          j        j                            d |D             k    sJ d S )Nr8   c                 B    g | ]\  }}t          j        ||          S r:   )	functoolspartial)r=   xyrG   s      r   r?   z+test_local_submit_array.<locals>.<listcomp>J   s,    
H
H
H$!Q9Q1%%
H
H
Hr   c                 6    g | ]}|                                 S r:   r;   r<   s     r   r?   z+test_local_submit_array.<locals>.<listcomp>M   r@   r   )
r   rA   ziprC   rD   r   r    submit_arrayrE   rF   )r   fnsr1   rH   rG   s       @r   test_local_submit_arrayrT   H   s     ##A
H
H
H
H#agqw2G2G
H
H
HC"8,,H  %%DAqw(())-G-G$-G-G-GGGGGGGr   c                     dd}t          j        |           }|                    |          }|                                }t	          |t
          j                  sJ |j        d         }d|v sJ d|v sJ d S )Nr   c                       t          d          )NFailed on purpose)r&   r:   r   r   failing_jobz%test_local_error.<locals>.failing_jobQ   s    .///r   r   	TracebackrW   r   N)r   r    r"   r*   
isinstancer	   FailedJobErrorargs)r   rX   r1   jobr*   	tracebacks         r   test_local_errorr`   P   s    0 0 0 0 "8,,H
//+
&
&CIi!566666q!I)####)++++++r   c                      G d d          }t          j        |           }|                    |j                  }t	          |                                |          sJ d S )Nc                       e Zd ZdS )-test_pickle_output_from_main.<locals>.MyClassN)__name__
__module____qualname__r:   r   r   MyClassrc   ^   s        r   rg   )r   r    r"   __call__r[   r)   )r   rg   r1   r^   s       r   test_pickle_output_from_mainri   ]   st            "8,,H
//'*
+
+CcjjllG,,,,,,,r   c                     d	d}t          j        |           }|                    dd           |                    |          }|                                }t          |t          j                  sJ |j        d         }d|v sJ d|v sJ d S )
Nr   c                  p    t          j                    } | j        dk    rt          d| j                   d S )Nr   zFailed on purpose: )r   r   r   r&   r   s    r   flakyz(test_get_first_task_error.<locals>.flakyg   s@    !022!!IW5GIIJJJ "!r   r   r
   r   r   rY   zFailed on purpose: 1rZ   )	r   r    r!   r"   r*   r[   r	   r\   r]   )r   rm   r1   r^   r*   r_   s         r   test_get_first_task_errorrn   f   s    K K K K
 "8,,Haq999
//%
 
 CIi!566666q!I)####!Y......r   c                 F   d
d}t          j        |           }|                    dd           |                    |          }|                                 |                                }|J d|v sJ d|v sJ |                                }|J d|v sJ d	|v sJ d S )Nr   c                      t          j                    } t          d| j                   t          d| j        t          j                   d S )Nz
hello fromzbye from)file)r   r   printr   sysstderrrl   s    r   helloztest_stdout.<locals>.hellow   sC    !022lG.///j',3:>>>>>>r   r   r
   r   zhello from 0
zhello from 1
zbye from 0
zbye from 1
rZ   )r   r    r!   r"   waitstdoutrt   )r   ru   r1   r^   rw   rt   s         r   test_stdoutrx   v   s    ? ? ? ?
 "8,,Haq999
//%
 
 CHHJJJZZ\\Fv%%%%v%%%%ZZ\\FV####V######r   c                    dd}t          j        |           }|                    |          }|j        dk    sJ |j                            t          j                   t          j	        d           |j        dk    sJ d S )Nr   c                  H    t          j        d           t          d          )Nx   rW   )timesleepr&   r:   r   r   rX   z test_killed.<locals>.failing_job   s    
3.///r   RUNNINGr
   INTERRUPTEDrZ   )
r   r    r"   stater0   send_signalsignalSIGKILLr|   r}   )r   rX   r1   r^   s       r   test_killedr      s    0 0 0 0 "8,,H
//+
&
&C9	!!!!LV^,,,JqMMM9%%%%%%r   SUBMITIT_SLOW_TESTSFslow)reasonc           	          ddt           dt           dt           dt           fd}t          j        |           }|                    d           |                    |d	d
          }|                                dk    sJ d S )Nr{   rN   rO   r}   r   c                 4    t          j        |           | |z   S r   )r|   r}   )rN   rO   r}   s      r   fz test_long_running_job.<locals>.f   s    
51ur   r8   timeout_min(   r   *   )r{   )r   r   r    r!   r"   r)   )r   r   r1   r^   s       r   test_long_running_jobr      s     S S  s     "8,,H1---
//!R
#
#C::<<2r   c                     t          j        d          }t          d          D ]$}|                    t          j        |d           %t          j        | d          }|                    dd           |	                    |          }|
                                 |                                }|J t          j        d	|          }|sJ d
|             d|v sJ d|             d|v sJ d|             d S )NT)verbose   r
   )rN   r}   max_num_timeoutg?r   signal_delay_szStarting from [123]/20z'Should have resumed from a checkpoint:
ztimed-out too many timeszUnexpected stdout:
z(0 remaining timeouts))r   FunctionSequencerangeaddr   
do_nothingr   r    r!   r"   rv   rw   research)r   r   rN   r1   r^   rw   r   s          r   test_requeuingr      s%   #D111D2YY 5 5%!4444"8Q???H6!DDD
//$

CHHJJJZZ\\FI/88EEEEVEEEE5%///1P1P1P///#v---/Nf/N/N-----r   c                 0    G d dt           j                  }t          j        | d          }|                    dd           |                     |d                    }|                                 |                                }|sJ d|v sJ d S )	Nc                   *    e Zd ZddefdZddefdZdS )'test_custom_checkpoint.<locals>.SlackerTslackc                     |r4t          dd           t          j        d           t          d          t          dd           dS )NSlackingTflush
   zI really don't want to workzWorking hardzworked hard)rr   r|   r}   r&   selfr   s     r   rh   z0test_custom_checkpoint.<locals>.Slacker.__call__   sR     Bj----
2"#@AAA.---- =r   c                 T    |rt          dd           t          j        | d          S )Nz4Interrupted while slacking. I won't slack next time.Tr   F)r   )rr   r	   DelayedSubmissionr   s     r   __submitit_checkpoint__z?test_custom_checkpoint.<locals>.Slacker.__submitit_checkpoint__   s6     ZLTXYYYY*4u====r   N)T)rd   re   rf   boolrh   r   r:   r   r   Slackerr      sT        	! 	!$ 	! 	! 	! 	!	> 	> 	> 	> 	> 	> 	> 	>r   r   r
   r   g?r   TzI won't slack next time.)r   Checkpointabler   r    r!   r"   rv   rw   )r   r   r1   r^   rw   s        r   test_custom_checkpointr      s    > > > > >'( > > > "8Q???H6!DDD
//''$--
(
(CHHJJJZZ\\FMM6%//////r   c                    t          j        | dd          }t          j        | t	          |j                  d          }|j        }|                                 |j                                        }|j	                                        }|r%t          |                                          |k    sJ d| d            d S )Nz@python -c 'import os;print(os.environ["SUBMITIT_LOCAL_JOB_ID"])'r
   r   r   zPID link is broken (stderr: ))r   start_controllerr	   JobPathsstrpidrv   rw   	read_textrt   r   strip)r   processpathspgrw   rt   s         r   test_make_subprocessr      s    $Vde  G N8S%5%5q99E	BLLNNN\##%%F\##%%FYc&,,..))R///1YPV1Y1Y1Y/////r   c                    t          j        |           }|                    t          j        d          }|j        dk    sJ |                                 t          j        d           |j        dk    sJ |                    t          j        d          }|j        }|                                 |j        dk    sJ |	                                J ~t          j        d           |	                                dk    sJ d S )Nr   r~   g?r   )
r   r    r"   r|   r}   r   cancelr0   cancel_at_deletionpoll)r   r1   r^   r   s       r   test_cancelr      s    "8,,H
//$*b
)
)C9	!!!!JJLLLJsOOO9%%%%
//$*b
)
)ClG9	!!!!<<>>!!!JsOOO<<>>Rr   rN   rO   c                     dS )NB   r:   )rN   rO   s     r   f66r      s    2r   c                 $   t          | d          }| dz  }|                    d| g           |                    t          d          }t	          j        d           |                                dk    sJ |                                sJ d S )	Nr   )cluster
setup_doneztouch )local_setup   r
   r   )r   r!   r"   r   r|   r}   r)   exists)r   r1   
setup_filer^   s       r   
test_setupr      s    Hg666HL(J,AZ,A,A+BCCC
//#r
"
"CJqMMM::<<2r   c                 *   t          j        |           }|                    t          dd          }t          j        | |j                                                  }|j        t          u sJ |j        dk    sJ |j	        ddik    sJ |j
        J dS )z9Check we can load submission just from a path and job id.C   D   )rO   )r   rO   N)r   r    r"   r   LocalJobjob_id
submissionfunctionr]   kwargs_result)r   r1   r^   r   s       r   test_load_submissionr      s    "8,,H
//#rR/
(
(C#*55@@BBJ#%%%%?e####b	))))%%%%%r   weird_tmp_pathc                     t          j        | dz            }|                    t          dd                                           d S )Nz%jr   r   )r   r    r"   r   r)   )r   r1   s     r   test_weird_dirr     s?    ">D#899HOOCR  '')))))r   )r   )*rL   osr-   r   r   rs   r|   pathlibr   r$   submititr    r   corer   r   r	   r   r   r6   rI   rT   r`   ri   rn   rx   r   markskipifenvirongetr   r   r   r   r   r   r   r   r   r   r:   r   r   <module>r      sl       				  				  



         ! ! ! ! ! !       4 4 4 4 4 4 4 4 4 4        %UT %Ud %U %U %U %UPH4 HD H H H HHd Ht H H H H
,t 
, 
, 
, 
, 
,-4 -D - - - -/ / / / / / $$ $4 $ $ $ $,
&$ 
&4 
& 
& 
& 
& 
'<eDDDVTTD T    UTOT Od O O O O 0T 0d 0 0 0 00	Z4 	ZD 	Z 	Z 	Z 	Z $  4        & 3 3 s     $    &4 &D & & & &*4 *D * * * * * *r   