
    Whl                       d dl mZ d dlmZ d dlZd dlZd dlmZmZ d dl	Z	d dl
mZ d dl
mZ d dl
mZ d dl
mZ d d	l
mZ d d
l
mZ d dl
mZ d dl
mZ d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlZ  ej!        d ej"        dd          d          Z# ej$        d ej%        dd          d          Z&dZ' ej(        e)           ej*        Z* ej+        e,          Z-d\d Z.d! Z/d]d%Z0d^d*Z1d^d+Z2	 	 	 	 	 	 	 	 	 	 d_d`d<Z3ej4        dadA            Z5g Z6dbdDZ7	 dcdddKZ8dedNZ9dfdQZ:i e:_;        dgdSZ<i e<_=        dhdTZ>didUZ?djdWZ@dkd[ZAdS )l    )annotations)SequenceN)AnyCallable)	cache_key)compilation_cache)config)distributed)lib)
monitoring)path)profiler)traceback_util)mlir)version)
xla_client)irjax_disable_most_optimizationsJAX_DISABLE_MOST_OPTIMIZATIONSFzTry not to do much optimization work. This can be useful if the cost of optimization is greater than that of running a less-optimized program.%jax_compiler_detailed_logging_min_ops%JAX_COMPILER_DETAILED_LOGGING_MIN_OPS
   zHow big should a module be in MLIR operations before JAX enables detailed compiler logging? The intent of this flag is to suppress detailed logging for small/uninteresting computations.)helpbackend	xc.Clientreturnintc                    ~ dS )Nr    )r   s    W/var/www/html/movieo_spanner_bot/venv/lib/python3.11/site-packages/jax/_src/compiler.pyget_latest_profile_versionr"   G   s
    	    c                    |dz  }|dk     r|S | j         D ]*}|D ]%}|D ] }t          ||          }|dk     r|c c c S !&+|S )N   r   )regions_walk_operations)opkregionblockchild_ops        r!   r'   r'   L   s    q&!UUH
  f    (Xq))q55((((((( 
 
(r#   module	ir.Moduleboolc                L    t           j        }t          | j        |          dk     S )zBReturns 'true' if detailed logging should be enabled for 'module'.r   )"_COMPILER_DETAILED_LOGGING_MIN_OPSvaluer'   	operation)r-   bounds     r!   use_detailed_loggingr5   Y   s"    
,
2%	&*E	2	2Q	66r#   module_namestrr   Nonec                    t           j        j        rt          j        nt          j        }t                              |d| |           d S )Nz5Persistent compilation cache hit for '%s' with key %r)r	   log_compilesr2   loggingWARNINGDEBUGloggerlog)r6   r   hit_log_prioritys      r!   log_persistent_cache_hitrA   _   sK    )/)<)B )goo"= **V)% % % % %r#   c                    t           j        j        rt          j                    rt
          j        nt
          j        }t          	                    |d| |           d S )Nz6PERSISTENT COMPILATION CACHE MISS for '%s' with key %r)
r	   explain_cache_missesr2   r   is_persistent_cache_enabledr;   r<   r=   r>   r?   )r6   r   miss_log_prioritys      r!   log_persistent_cache_missrF   f   s`    !6<+-IKK+w %] 
 	** X)% % % % %r#   Tnum_replicasnum_partitionsuse_spmd_partitioninguse_shardy_partitioneruse_auto_spmd_partitioning!auto_spmd_partitioning_mesh_shapelist[int] | Noneauto_spmd_partitioning_mesh_idsenv_options_overridesdict[str, str] | Nonefdo_profilebytes | Nonedetailed_loggingxc.Client | Nonexc.CompileOptionsc                   t          j                    }| |_        ||_        |j        }||_        ||_        ||_        |	|	|_        |r|pg |_	        |pg |_
        |Jt                              d| ||           t          j        |          }|j        dk    r|dk    r|dddf         }| |j        d         k    r%d}t#          |                    ||                     ||j        d         k    r%d}t#          |                    ||                    |j        t(          k    r& t          j        d t,          g          |          }t           j                            |          }|                                | k    sJ |                                |k    sJ ||_        t8          j        j        |_        t8          j        j        |_        |fd	d
g}tA          |          }|D ]*}||v r$tC          |||"                    |                     +tG          |$                                          |_%        |j        j&        }tN          j(        tN          j(        |_)        tT          j        rd|_+        d|_,        d|_-        t8          j.        j        sd|_/        t8          j0        j        }|dk    r#||_1        t                              d|           nutd          |_1        |tg          j4        d           nRtk          |          }|dk    r#||_1        t                              d|           nt          6                    d           |
|_7        tq          j9                    rtt          dk    rtw          j<        t8          j=        j        pd          }t8          j>        j        pd}|dk    sd|v r;|dz  }t          |          |_@        d|_A        t                              d|           |dk    sd|v rv|dz  }t          |          |_B        t                              d|           t          jD        jE        dk    rt           jF        jG        |_H        nt           jF        jI        |_H        |S )a  Returns the compile options to use, as derived from flag values.

  Args:
    num_replicas: Number of replicas for which to compile.
    num_partitions: Number of partitions for which to compile.
    device_assignment: Optional ndarray of jax devices indicating the assignment
      of logical replicas to physical devices (default inherited from
      xla_client.CompileOptions). Must be consistent with `num_replicas` and
      `num_partitions`.
    use_spmd_partitioning: boolean indicating whether to enable SPMD or MPMD
      partitioning in XLA.
    use_shardy_partitioner: boolean indicating whether to use the Shardy
      partitioner in XLA. Shardy is a new open sourced propagation framework for
      MLIR. Currently Shardy is experimental in JAX. See
      www.github.com/openxla/shardy.
    use_auto_spmd_partitioning: boolean indicating whether to automatically
      generate XLA shardings for SPMD partitioner.
    auto_spmd_partitioning_mesh_shape: device mesh shape used to create
      auto_spmd_partitioning search space.
    auto_spmd_partitioning_mesh_ids: device ids used to create
      auto_spmd_partitioning search space.
    env_options_overrides: dict of additional options parsed by the compiler
    fdo_profile: Optional profile for feedback-directed optimization passed to
      XLA.
    detailed_logging: Is this an "interesting" computation about which XLA would
      be wise to log compilation information?
    backend: the client, if available.
  NzKget_compile_options: num_replicas=%s num_partitions=%s device_assignment=%sr%   r   z8device_assignment does not match num_replicas: {} vs {}.z:device_assignment does not match num_partitions: {} vs {}.c                    | j         S Nid)ds    r!   <lambda>z%get_compile_options.<locals>.<lambda>   s     r#   )otypesexec_time_optimization_effortmemory_fitting_effortTFrematerializationzSget_compile_options XLA-AutoFDO profile: using JAX XLA profile version %d from flagzGget_compile_options: no backend supplied; disabling XLA-AutoFDO profilezMget_compile_options XLA-AutoFDO profile: using XLA-AutoFDO profile version %dzaget_compile_options XLA-AutoFDO profile: XLA-AutoFDO profile version is 0; this should not happen)r      #    allxla_gpu_kernel_cache_filez!Enabling XLA kernel cache at '%s'%xla_gpu_per_fusion_autotune_cache_dirz%Enabling XLA autotuning cache at '%s')JxcCompileOptionsrG   rH   executable_build_optionsrI   rK   rJ   rQ   rL   rN   r>   debugnparrayndimshape
ValueErrorformatdtypeobject	vectorizer   DeviceAssignmentcreatereplica_countcomputation_countdevice_assignmentr	   r^   r2   r_   dictsetattrpoplistitemsenv_option_overridesdebug_optionsr   	cuda_pathxla_gpu_cuda_data_dir_DISABLE_MOST_OPTIMIZATIONSxla_backend_optimization_level!xla_llvm_disable_expensive_passesxla_test_all_input_layoutsenable_remat_opt_passxla_disable_hlo_passesjax_xla_profile_versionprofile_version_NO_PROFILE_DONT_RETRIEVEr;   infor"   errorxla_detailed_loggingr   rD   jaxlib_versionpathlibPathcompilation_cache_dir"persistent_cache_enable_xla_cachesr7   re   2xla_gpu_enable_llvm_module_compilation_parallelismrf   r
   global_state
process_idAutotuneCacheModeUPDATE(xla_gpu_experimental_autotune_cache_modeREAD)rG   rH   rx   rI   rJ   rK   rL   rN   rO   rQ   rS   r   compile_optionsbuild_optionsmsgoverrides_on_build_optionsnamer   r   fdo_profile_versionr   enabled_flagskernel_cache_pathautotune_cache_paths                           r!   get_compile_optionsr   p   s   T %''/!-/#1/ !:-(=-%-G-*)?-& +M Z6W6][]M34S4YWYM1"
LLUn&79 9 9 !233 	!##.A*=*=+AAAtG4(.q111Fcszz"3\BBCCC*0333Hcszz"3^DDEEE&((D",~~seDDD
 +223DEE**,,<<<<..00NBBBB(9O%060T0Z--(.(D(J-%& 	()@"B !677* F F	&	&	&t%:%>%>t%D%DEEE+/0E0K0K0M0M+N+NO(!:H-]*--M' & 534M06:M3/4M,		%	+ ?+>M( #:@q  &=O#
LL >(* * * * '@O#l 3 4 4 4 4 7w??		!	!*=' <(	* 	* 	* 	* 	 P 	Q 	Q 	Q (8-$ 244 [*9T9T <4:@bAAD=CIrM!<!M!M!<<034E0F0Fm-IMmFll68IJJJ!HM!Y!Y #JJ<?@S<T<Tm9ll:<OPPP 
	!	,	1	1ACAUA\>>ACAUAZ>	r#   optionshost_callbacksSequence[Any]xc.LoadedExecutablec                   t          | dd          rt          j        |          }n|}	 |r|                     |||          S |                     ||          S # t          j        $ r!}t          D ]} ||          }||||d }~ww xY w)Nneeds_str_irT)r   r   )r   )getattrr   module_to_bytecodecompilerg   XlaRuntimeError_XLA_RUNTIME_ERROR_HANDLERS)r   r-   r   r   built_ceerror_handlerhandler_results           r!   backend_compiler     s     Wnd++ %f--GGG  __
7>     ??7G?<<<		   4 $ $$}Q''n		#!# 
$
Gs   A A B*BB
handler_fn0Callable[[xc.XlaRuntimeError], Exception | None]c                :    t                               |            dS )a  Registers a custom exception handler for XLA runtime errors.

  Registering a custom handler allows re-raising a more informative exception
  after encountering an XLARuntimeError.

  Args:
    handler_fn: A function which returns a new exception to replace the original
      XLA runtime error, or None if the original error should be propagated.

  Returns:
    A new exception or None.
  N)r   append)r   s    r!   "register_xla_runtime_error_handlerr   ;  s     $$Z00000r#   computationdevices
np.ndarrayr   pgle_profilerprofiler.PGLEProfiler | Nonec                b   |j         j        d         }t          j        |          j        }t          j        |d          x}rt          j        d|           t          j
        |           }	t          d |                                D                       dk    }
t          |                                d           j        }t          j        j        ot          j        j        dk    }|	sT|
r@|r>t$          j        j        -t+          |||| t$          j        j        |          |j        _        t1          | |||          S t3          j        d	           	 t          j        j        rt8          j        j        }nt8          j        j        }t          j         |||| |
          }nN# tB          j"        j#        $ r7}tH          %                    d|           t1          | |||          cY d }~S d }~ww xY w|rtM          |||| ||
|||	  	        }tO          j(                    }tS          ||||           \  }}tO          j(                    |z
  }|W|J tU          ||           t3          j        d           t3          j+        d||z
             t3          j+        d|           |S t          j,        j        r[|
rYt$          j        j        Ht          |          dk    r5t[          ||           t]          | |||t$          j        j        |||          S t[          ||           t_          | |||||          S )Nsym_namer   zDumped the module to %s.c                    h | ]	}|j         
S r    )process_index).0devices     r!   	<setcomp>z(compile_or_get_cached.<locals>.<setcomp>^  s    
@
@
@F6
@
@
@r#   r%   c                    | j         S rX   rY   )r   s    r!   r\   z'compile_or_get_cached.<locals>.<lambda>a  s    FI r#   )keyr   z1/jax/compilation_cache/compile_requests_use_cache)ignore_callbackszKcompile_or_get_cached: unable to generate cache key, skipping the cache: %sz!/jax/compilation_cache/cache_hitsz-/jax/compilation_cache/compile_time_saved_secz//jax/compilation_cache/cache_retrieval_time_sec)0r3   
attributesr   
StringAttrr2   r   dump_module_to_filer;   r   r   is_cache_usedlenflattenminr   r	   enable_pglepgle_profiling_runsr
   r   client_share_fdo_profilesri   rQ   r   r   record_event-remove_custom_partitioning_ptr_from_cache_keycache_key_typeIgnoreCallbacksCUSTOM_PARTITIONINGNOget_cache_keyrg   _xlar   r>   r   _resolve_pgle_module_cache_keytime	monotonic_cache_readrA   record_event_duration_secsshare_binary_between_hostsrF   _compile_and_share_module_compile_and_write_cache)r   r   r   r   r   r   r   r6   	dumped_touse_compilation_cacheis_multi_processmin_device_process_idis_auto_pgle_usedr   r   excache_retrieval_startretrieved_executableretrieved_compile_timecache_retrieval_times                       r!   compile_or_get_cachedr   M  s    "-j9(h''-+*;	BBBY 8L+Y777+9'BB 

@
@goo.?.?
@
@
@AAAE  oo55    G6#=#Ca#G  
 + $+7 &-#  .: 7K)+ + + MNNN+;A ;'7K'7:!/)  II 
	  + + +
LL *+-/ / /7K)+ + + + + + + ++  .
 
I .**1<9ow28 28..)),AA%!---[)444?@@@)7!557 7 7 )9;OQ Q Q  '-
 
"
)
5 n


"
"k9555$ '	 	 	 k9555#  s   AF# #G.7,G)#G.)G.r   r   c	                    |j         j        }	d|j         _        t          j        | |||t          j        j                  }
|	|j         _        |}t          ||
          r|
}||                                 n|	t          |	          dk    r|
}|r?t          j        j        .t          | |||t          j        j        |          |j         _        n?|	|j         _        t                              d|t          |j         j                             |S )Ns   pgle profiledr   z1Compiling module %s with FDO profile of length %d)ri   rQ   r   r   r   r   ALL_is_executable_in_cachedisabler   r
   r   r   r   r>   rj   )r   r   r   r   r   r   r   r6   r   rQ   pgle_profiled_module_key
result_keys               r!   r   r     s,     8D+9I/*6.<$(  :E/*6*W&>?? )J 3{#3#3a#7#7)J K4;G
&-#  .:: >Io.:ll
=

o6B
C
C  
 
r#   global_client*lib.xla_extension.DistributedRuntimeClientc                   | j         j        d         }t          j        |          j        }|j        j        }|t          |          dk    r|S d|j        _        	 t          j	        | |||t          j        j                  dz   }	n># t          j        j        $ r'}
t                               d|
           |cY d }
~
S d }
~
ww xY w|	t$          j        v rt$          j        |	         S t(          j        j        }t,          j        j        |k    r3t                               d||           |                    |	|           n2t                               d||           |                    |	|          }|t$          j        |	<   |S )Nr   r   r#   	_fdo_synczYcompile_or_get_cached: unable to generate cache key, skipping the fdo profile sharing: %sz+Module %s. Sharing FDO profile. Process %d.zEModule %s. Waiting for FDO profile which should be set by process %d.)r3   r   r   r   r2   ri   rQ   r   r   r   r   r   r   rg   r   r   r>   r   r   modules_profilesr	   %share_binary_between_hosts_timeout_msr
   r   r   rj   key_value_set_bytesblocking_key_value_get_bytes)r   r   r   r   r   min_process_idr   r6   rQ   profile_keyr   share_timeouts               r!   r   r     s    "-j9(h''-+8D+C,,119</*6'*.	
 	
 	 K 
	    
LL	/
  
  '888/<<>D-(N::
LL5  
 %%k;????
LLO  
  <<] K 7B&{3	s   *B CB=7C=Cfirst_process_idc                H   t           j        j        }|t          j        v rt          j        |         S t
          j        j        |k    rpt          	                    d||           t          | |||||          }	|                     |	          }
t          j        |
          }
|                    ||
           n\t          	                    d||           |                    ||          }
t          j        |
          }
|                     |
|          }	|	t          j        |<   |	S )Nz+Process %d compiling and sharing module: %sz&Waiting for module: %s from process %d)r	   r   r2   r   modules_cacher
   r   r   r>   rj   r   serialize_executabler   compress_executabler   r   decompress_executabledeserialize_executable)r   r   r   r   r   r6   r   r   r   
executableserialized_executables              r!   r   r   S  sI    >D-+999$29==(,<<<
LL>!;0 0 0) J $88DD-A  %%i1FGGGG
LL9;!# # #)FF=  .C  // J 8B))4	r#   c                    t          j                    }t          | |||          }t          j                    |z
  }t          |||| ||           |S rX   )r   r   r   _cache_write)	r   r   r   r   r6   r   
start_timer  compile_times	            r!   r   r     sd     ~*{O^ * !!J.,{GZ   
r#   c           	         	 t          j        | |          S # t          $ rL}t          j        j        r t          j        d| dt          |          j	         d|            Y d}~dS d}~ww xY w)z<Checks if executable is presented in cache on a given key
  6Error reading persistent compilation cache entry for '': : NF)
r   is_executable_in_cache	Exceptionr	   raise_persistent_cache_errorsr2   warningswarntype__name__)r   r   r   s      r!   r   r     s    3GYGGG	   +1 M	4	4 	4r((+	4 	4/1	4 	45 5 5 55555s    
A-AA((A--tuple[xc.LoadedExecutable | None, int | None]c           	         	 t          j        |||          S # t          $ rL}t          j        j        r t          j        d|  dt          |          j	         d|            Y d}~dS d}~ww xY w)ziLooks up the `computation` and it's compilation time in the persistent
  compilation cache repository.
  r  r  r  N)NN)
r   get_executable_and_timer  r	   r  r2   r  r  r  r  )r6   r   r   r   r   s        r!   r   r     s    	4?G- - -	   +1 M	6	6 	6 HH-	6 	613	6 	67 7 7 :::::s    
A.AA))A.compile_time_secsfloatr  c           	        t           j        j        rt          j                    rt
          j        nt
          j        }t          j	        j
        dk    rt                              |d           dS |rt                              |d|           dS t           j        j        }||k     r t                              |d|||           dS t                              d|||           	 t          j        | |||t!          |                     dS # t"          $ rL}t           j        j        r t'          j        d| dt+          |          j         d	|            Y d}~dS d}~ww xY w)
zqWrites the `serialized_computation` and its compilation time to the
  persistent compilation cache repository.
  r   z8Not writing persistent cache entry since process_id != 0NztNot writing persistent cache entry for '%s' because it uses host callbacks (e.g. from jax.debug.print or breakpoint)z]Not writing persistent cache entry for '%s' because it took < %.2f seconds to compile (%.2fs)z2'%s' took at least %.2f seconds to compile (%.2fs)z6Error writing persistent compilation cache entry for 'r  r  )r	   rC   r2   r   rD   r;   r<   r=   r
   r   r   r>   r?   &persistent_cache_min_compile_time_secsrj   put_executable_and_timer   r  r  r  r  r  r  )	r   r  r6   r   r  r   log_prioritymin_compile_timer   s	            r!   r	  r	    s    06%'CEE%'// }  (A--
JJ|IK K K
F 
JJ	>?JL L L FBH)))
JJ	%&13C	  
 F
LL<%'8: : :7-;
GS9J5K5KM M M M M	 7 7 7+1 M	6	6 	6 HH-	6 	613	6 	67 7 7 7 7 7 7 7 77s   $%D 
E!AEE!)r   r   r   r   )r-   r.   r   r/   )r6   r7   r   r7   r   r8   )
NTFFNNNNTN)rG   r   rH   r   rI   r/   rJ   r/   rK   r/   rL   rM   rN   rM   rO   rP   rQ   rR   rS   r/   r   rT   r   rU   )
r   r   r-   r.   r   rU   r   r   r   r   )r   r   rX   )r   r   r   r.   r   r   r   rU   r   r   r   r   r   r   )r   r.   r   r   r   rU   r   r   r   r   r   r/   r   r7   r6   r7   r   r   r   r7   )r   r.   r   r   r   rU   r   r   r   r   r   rR   )r   r   r   r.   r   rU   r   r   r   r   r6   r7   r   r7   r   r   r   r   )r   r   r   r.   r   rU   r   r   r6   r7   r   r7   r   r   )r   r/   )
r6   r7   r   r7   r   rU   r   r   r   r  )r   r7   r  r  r6   r7   r   r   r  r   r   r   r   r8   )B
__future__r   collections.abcr   r;   r   typingr   r   r  jax._srcr   r   r   r	   r
   r   r   r   r   r   r   jax._src.interpretersr   jax._src.libr   r   r   rg   jax._src.lib.mlirr   numpyrk   	bool_flagbool_envr   int_flagint_envr1   r   register_exclusion__file__rh   	getLoggerr  r>   r"   r'   r5   rA   rF   r   annotate_functionr   r   r   r   r   r   r   r   r  r   r   r   r	  r    r#   r!   <module>r1     s  " # " " " " " $ $ $ $ $ $                    0 0 0 0 0 0 & & & & & & % % % % % %                         $ $ $ $ $ $       # # # # # # & & & & & & 2 2 2 2 2 2 ) ) ) ) ) )                 /f.$FO4e<<MN N  &5V_+FN:B??	A	& & & "   ! !( + + +"		8	$	$   

 
 
7 7 7 7% % % %% % % % "&#(',:>8<37 $! $a a a a aH 
       F ! 1 1 1 10 37~ ~ ~ ~ ~`2 2 2 2n8 8 8 8v (*  $- - - -^ +-  '   $      &/7 /7 /7 /7 /7 /7r#   