
    Wh`                      U d dl mZ d dlmZmZmZ d dlZd dlmZ d dl	Z	d dl
Zd dlmZ d dlZd dlmZmZmZ d dlZd dlZd dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlm Z  d dlm!Z! d dlm"Z" d dlm#Z# d dlm$Z$ d dlm%Z% d dlm&Z' d dlm(Z( d dlm)Z) d dlm*Z* d dlm+Z+ d dlm,Z, d dlm-Z- d dlm.Z. d dlm/Z/ d dl0m1Z1 d dl2m3Z3m4Z4m5Z5m6Z6m7Z7 d dl8m9Z9m:Z: d dl;m<Z<m=Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZF d dlGmHZH d dlImJZJ d d lKmLZL d d!lMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZU d d"lVmWZWmXZXmYZY d d#lZm[Z[ d d$lZm\Z\ d d%lZm]Z^ d d&lZm_Z_ d d'lZm`Z` d d(lambZbmcZcmdZdmeZemfZfmgZg d d)lhmiZimjZjmkZkmlZlmmZm d d*lnmoZompZp eZqe/jr        escZsZte/ju        evcZvZw e.jx        ey           eZzeZ{e.j|        d+ e}            fd)d7            Z~d*d9Zeeee{d:f         f         Zd+d?Z ej        d@dAdBg          Zd,dGZ G dH dI          Z e            Zd-dSZd.dWZd/d[Zd0d\Zd1d^Zd2daZdb Z edc          Zd3dfZeZeee1f         Z G dg dhe!j                  Z edi          Zd4dxZee^j        _        d5d|Zd6d}Zd6d~Zd6dZee!j        e!j        <   d6dZee!j        e!j        <   eee{         df         Zd Zee!j        e<   d7dZd8dZd9dZd:dZd;dZd Zd Z e\j        ee           d Zd<dZd<dZd=dZd Zd Zee!j        _        d Zd>dZd?dZd Zd@dZ G d de          Zd Z G d de          Zd ZdAdZd Zd Zd Z G d de!j                  Z G d de!j1                  Zd Zi Zded<   dBdZeee#j        <   d Zeee$j        <    e!j        d          Zd+e_        eΠ                    e>j        j                   eΠ                    e>j        j                    e\j        ee\j        e>j                             ee>j        ed           e[j        e<    ee>j        d          e[j        e<   d Z e`j        eeڦ           d Z e!j        d¦          Zd+e_        eݠ                    dÄ            eݠ                    dĄ             e\j        edń            dƄ Zee[j        e<    e`j        edǄ            i Zded<   dɄ Zdʄ Zd˄ Zi Zded<   d̈́ Zd΄ Zdτ ZdЄ Zdф Zd҄ Zdӄ ZdԄ ZdՄ Zdք Zdׄ Z ej        e<j                                        e?j                                        e@j                                        eDj                                        eAj                                        eBj                                        eCj                                        e(j                                        ej                                        e*j                                        eFj                                        e+j                                                  D ])Z eee!j                  r ee            ee           *eEj        j        eEj        j        eEj        j        eEj        j        eEj        j        fD ]Z ee            ee            ee>j                  d؄             Z ee>j                  dل             Z eeΦ          dڄ             Z eeΦ            eeݦ          dۄ             Z  eeݦ            ee>j                    ee>j                    ee>j                    ee>j                    ee>j                    ee>j                    ee>j                  d܄             Z ee>j                    ee)j	                  d݄             Z
 ee)j	                  dބ             Z eej                  d߄             Z eej                  d             Z ee!j                  d             Z ee#j                  d             Z ee#j                    eej                  d             Z eej                    eej                  d             Z eej                    ee$j                  d             Z ee$j                    ee`j                  d             Z ee`j                    eeEj        j                  d             Z eeEj        j                  d             Z eeEj        j                  d             Z eeEj        j                  d             Z eeEj        j                   d             Z! eeEj        j                     ee!j"                  d             Z# ee!j"                  d             Z$ ee"j%                  d             Z& ee"j'                  d             Z( ee"j'                  d             Z) eeEj*        j+                  d             Z, eeEj*        j+                    eeHj-                  d             Z. eeHj-                   ee<j/        =  ee<j/                  d             Z0 ee<j/                   dCdZ1e1e[j2        _        d Z3d Z4e4e`j5        _        d Z6e6e^j7        _        e'j8        d             Z9d Z:dDdZ;d Z<e<e`j=        e<   dEdZ>e>e^j?        e<   d Z@d	 ZAdFdGdZBdHdZCeCe^jD        e<   dFd d ddddddddZEe'jF        d             ZGe'j8        d             ZHd ZId ZJeZK G d de!j1                  ZL G d de!j                  ZMd ZNe'jO        d             ZPe'j8        d             ZQdId!ZRdJd#ZSe'jO        d$             ZTd% ZUd& ZVdKd(ZWdS (L      )annotations)CallableHashableSequenceN)partial)prod)AnyTypeVarUnion)NamedShardingPartitionSpec)ad_checkpoint)ad_util)callback)config)core)custom_derivatives)	debugging)dispatch)dtypes)linear_util)ops)pjit)prng)random)sharding_impls)source_info_util)traceback_util)util)Tracer)AbstractMeshMesh	AxisTypesset_abstract_meshget_abstract_mesh)_shared_code_pmap_prepare_pmap)
laxparallelslicingwindowed_reductionsconvolutionfftlinalgspecialcontrol_flowann)ffi)ir)sdy)HashableFunctionHashablePartialunzip2as_hashable_functionmemoizepartition_list
split_list
subs_list2)flatten_fun_nokwargsshaped_abstractifyargnums_partial)batching)mlir)partial_eval)pxla)ad)tree_maptree_flattentree_unflattentree_structuretree_leaveskeystr)broadcast_prefixprefix_errors	PyTreeDefgenerate_key_pathsKeyPath) host_local_array_to_global_array global_array_to_host_local_arrayTfr   meshMesh | AbstractMeshin_specsSpecs	out_specs	check_repboolautofrozenset[AxisName]c                *    t          | |||||          S )a  Map a function over shards of data.

  Note:
    ``shard_map`` is an experimental API, and still subject to change. For an
    introduction to sharded data, refer to :ref:`sharded-computation`. For a more
    in-depth look at using ``shard_map``, refer to `SPMD multi-device parallelism with shard_map`_.

  Args:
    f: callable to be mapped. Each application of ``f``, or "instance" of ``f``,
      takes as input a shard of the mapped-over arguments and produces a shard
      of the output.
    mesh: a ``jax.sharding.Mesh`` representing the array of devices over which
      to shard the data and on which to execute instances of ``f``. The names of
      the ``Mesh`` can be used in collective communication operations in ``f``.
      This is typically created by a utility function like
      :func:`jax.experimental.mesh_utils.create_device_mesh`.
    in_specs: a pytree with :class:`~jax.sharding.PartitionSpec` instances as leaves,
      with a tree structure that is a tree prefix of the args tuple to be mapped
      over. Similar to :class:`~jax.sharding.NamedSharding`, each ``PartitionSpec``
      represents how the corresponding argument (or subtree of arguments) should
      be sharded along the named axes of ``mesh``. In each ``PartitionSpec``,
      mentioning a ``mesh`` axis name at a position expresses sharding the
      corresponding argument array axis along that positional axis; not
      mentioning an axis name expresses replication. If an argument, or argument
      subtree, has a corresponding spec of None, that argument is not sharded.
    out_specs: a pytree with :class:`~jax.sharding.PartitionSpec` instances as leaves,
      with a tree structure that is a tree prefix of the output of ``f``. Each
      ``PartitionSpec`` represents how the corresponding output shards should be
      concatenated. In each ``PartitionSpec``, metioning a ``mesh`` axis name at
      a position expresses concatenation of that mesh axis's shards along the
      corresponding positional axis. Not mentioning a ``mesh`` axis name
      expresses a promise that the output values are equal along that mesh axis,
      and that rather than concatenating only a single value should be produced.
    check_rep: If True (default) enable additional validity checks and automatic
      differentiation optimizations. The validity checks concern whether any mesh
      axis names not mentioned in ``out_specs`` are consistent with how the outputs
      of ``f`` are replicated. Must be set False if using a Pallas kernel in ``f``.
    auto: (experimental) an optional set of axis names from ``mesh`` over which we
      do not shard the data or map the function, but rather we allow the
      compiler to control sharding. These names cannot be used in ``in_specs``,
      ``out_specs``, or in communication collectives in ``f``.

  Returns:
    A callable that applies the input function ``f`` across data sharded according to
    the ``mesh`` and ``in_specs``.

  Examples:
    For examples, refer to :ref:`sharded-computation` or `SPMD multi-device parallelism with shard_map`_.

  .. _SPMD multi-device parallelism with shard_map: https://jax.readthedocs.io/en/latest/notebooks/shard_map.html
  )
_shard_map)rR   rS   rU   rW   rX   rZ   s         `/var/www/html/movieo_spanner_bot/venv/lib/python3.11/site-packages/jax/experimental/shard_map.py	shard_mapr_   V   s    n 
AtXy)T	B	BB    Specs | Callable[[], Specs]c                R    t                     s#t          d  dt                      d          t          t          t
          f          s#t          d dt                     d                              j                  st          d dj                   t          t          j                   t                    st          t          j                   t          j                   t          j         fd                        }|S )Nz>shard_map requires a callable for its first argument, but got 	 of type .ztshard_map requires a `jax.sharding.Mesh` or a `jax.sharding.AbstractMesh` instance for its second argument, but got zshard_map requires auto=z# to be a subset of mesh.axis_names=c            
     L   t          j                  }t          |           \  }}t          ||          \  }t	          | d           }n,# t
          $ r t          |           ^}} |d          d w xY wt          d t          |          D                       \  }}t          |||d          \  }}t          ||||           t          t          t          |                    }t          fd            }	x}
rt          |||	          }	 t!          j        |g|R ||	|
d}n# t$          $ ru}|j        \  }t)                    sSt+          t,          j                     |          }t1          d |D                       r|d	z  }t          |          d Y d }~n]d }~wt2          $ rM}|j        \  }t)                    s+t5                       |          }t          |          d Y d }~nd }~ww xY wt7                       |          S )
Nc                
    | d u S N xs    r^   <lambda>z-_shard_map.<locals>.wrapped.<locals>.<lambda>   s
    AI r`   is_leafzshard_map in_specsc              3  (   K   | ]\  }}|||fV  d S rg   rh   ).0iss      r^   	<genexpr>z._shard_map.<locals>.wrapped.<locals>.<genexpr>   s7       (: (:41a+,= *+A+8===(: (:r`   Fc                    t                    r&             } t          t          j        |            n} t	                       t                      g             j        z            }t          | |          }n,# t          $ r t          | |          ^}} |d          d w xY wt          t          t          |                    S )Nzshard_map out_specs)callable_check_specsSpecErrorTypeoutrG   object
num_leavesrK   
ValueErrorrL   tuplemap_canonicalize_spec)
out_specs_dummyout_specs_flate_rZ   rW   out_trees        r^   out_names_thunkz4_shard_map.<locals>.wrapped.<locals>.out_names_thunk   s    	)		 Y[[
]&
D9999
XXZZ&((hhjj6K)KLLe,Z??>> 1 1 1j%00Aa%&&D01 3)>::;;;s   .A? ?)B(rS   in_namesr   rX   rewriterZ   c              3  8   K   | ]}|t           uo|j         V  d S rg   )no_failshape)ro   fails     r^   rr   z._shard_map.<locals>.wrapped.<locals>.<genexpr>   s0      HH$t7"54:~HHHHHHr`   z In particular, for rank 0 outputs which are not constant over the mesh, add at least one (singleton) axis to them so that they can be concatenated using out_specs.)lu	wrap_initrF   r=   rK   rz   rL   r7   	enumerater?   _check_specs_vs_argsr{   r|   r}   r9   _efficient_transpose_rewriteshard_map_pbind
_SpecErrorargsrt   _spec_rank_errorrv   rw   any	_RepError_inout_rep_errorrG   )r   fun	args_flatin_treein_specs_flatr   r   dyn_argnumsin_names_flatr   r   out_flatfailsmsgr   rZ   rX   rR   rU   rS   rW   s                 @r^   wrappedz_shard_map.<locals>.wrapped   s    ,q//C%d++Iw(g66MC)(D2E2EG G G . . .Hd++ea!A"##-. "( (: (:9];S;S (: (: (: ": ":K$S+y%HHNCD'8[-QZ[[[#0-@@AAM< < < < < < W< w T(dM?SSc(!
  $})Y  hh  ( ( (vfei   (}0!XXZZERRHH%HHHHH 	D
 C D# oo4'( ( ( ( (  ( ( (vfei   (q$

IuEEoo4'( ( ( ( ((
 ((**h///s1   A )A9D7 7
HA+F11H>AHH)rt   	TypeErrortype
isinstancer"   r!   issubset
axis_namesrz   ru   rv   inputrw   r   wrapsr   api_boundary)rR   rS   rU   rW   rX   rZ   r   s   `````` r^   r]   r]      s    
! 7
 6 6 6+/776 6 6 7 7 7	D4.	/	/ N
 M04M M?CDzzM M M N N N 
t	'	' ;
 : : :(,: : ; ; ;}"Hd333	)		 5"It444:a==30 30 30 30 30 30 30 30 30  =30h 
.r`   .specr   return	AxisNamesc                b    t          | t                    rd t          |           D             S | S )Nc                N    i | ]"\  }}||t          |t                    r|n|f#S rg   )r   r{   )ro   rp   namess      r^   
<dictcomp>z&_canonicalize_spec.<locals>.<dictcomp>   sE     B B B5u/@ 
5%00>uuuh/@/@/@r`   )r   r   r   )r   s    r^   r}   r}      sH    m$$ B B%dOOB B B B Kr`   rv   r   rw   
error_typespecsr	   Nonec                  	 | t           j        k    r|t          d          fdt          fdt	          |          D                       rd S | t           j        k    rdnd		fdt          |          D             }|st          |          D ]c\  }}|D ][}t          |t                    s|f}|D ]>}|v r8|                    d	 dt          |           d	t          |                      ?\dt          d
	 d dd                    |          z   d	 dz             t          d
	 dd                    |          z   d	 dz             )Nzshard_map in_specs argument must be a pytree of `jax.sharding.PartitionSpec` instances, but it was None.
Instead of `in_specs=None`, did you mean `in_specs=P()`, where `P = jax.sharding.PartitionSpec`?c                    t          | t                    sdS | D ]'}t          |t                    s|f}|D ]
}|v r  dS (dS )NFT)r   r   r{   )pr   namerZ   s      r^   
check_specz _check_specs.<locals>.check_spec   st    a'' U  u%%   $4<<  4r`   c              3  .   K   | ]} |          V  d S rg   rh   )ro   r   r   s     r^   rr   z_check_specs.<locals>.<genexpr>   s+      331A333333r`   inrw   c                    g | ]H\  }}t          |t                    d  dt          |           d| dt          |          j         d	IS )  _specs is rc   , )r   PrJ   r   __name__)ro   keyrj   prefixs      r^   
<listcomp>z _check_specs.<locals>.<listcomp>   su     
K 
K 
Kc1Aq9I9I
K
Nv
N
NVC[[
N
Na
N
N$q'':J
N
N
N 
K 
K 
Kr`   r   r   z refers to z
shard_map z5_specs argument cannot refer to an axis marked auto (z	), but:



z

Check the z"_specs values passed to shard_map.zR_specs argument must be a pytree of `jax.sharding.PartitionSpec` instances, but:

)rv   r   r   allrI   rN   r   r{   appendrJ   reprrz   join)
r   r   rZ   msgsr   r   r   r   r   r   s
     `     @@r^   ru   ru      s<   =&&&5=
	23 3 3
	 	 	 	 	 	3333E 2 233333;VV!44444%&
K 
K 
K 
K*511
K 
K 
K$	 A$U++ Q QQ Q Q%%'' 	(% 	Q 	QDT\\KKOVOO6#;;OO4::OOPPP	QQ 	*V 	* 	*	* 	* 	*
++d

	@@ @ @	@A A A
 	:6 : : :D>> > >>	? 	? ?r`   c                      e Zd ZdS )NoFailNr   
__module____qualname__rh   r`   r^   r   r               r`   r   r"   r   rM   r   Sequence[int]r   Sequence[P]xsr   c                &   t          t          |          }d t          ||          D             }t          d |D                       r=t	          |||          }t          t          j        | |||          }	t          |	          t          t          t          |                    }
fdt          ||
          D             }t          d |D                       r3t	          |||          }t          | |||          }	t          |	          d S )Nc                V    g | ]&\  }}t          |          |j        k    s|nt          'S rh   )lenndimr   )ro   r   as      r^   r   z(_check_specs_vs_args.<locals>.<listcomp>  sB     
4 
4 
4a q66QV##!! 
4 
4 
4r`   c              3  (   K   | ]}|t           uV  d S rg   r   ro   rR   s     r^   rr   z'_check_specs_vs_args.<locals>.<genexpr>  '      ((a'	((((((r`   c                    g | ]<\  }t          fd |                                D                       rnt          =S )c              3  p   K   | ]0\  }}j         |         t          fd |D                       z  V  1dS )c              3  2   K   | ]}j         |         V  d S rg   r   ro   nrS   s     r^   rr   z<_check_specs_vs_args.<locals>.<listcomp>.<genexpr>.<genexpr>  s)      $?$?qTZ]$?$?$?$?$?$?r`   N)r   r   )ro   dnsr   rS   s      r^   rr   z2_check_specs_vs_args.<locals>.<listcomp>.<genexpr>  sc       / /q" 71:$?$?$?$?B$?$?$? ? ?? / / / / / /r`   )r   itemsr   )ro   r   r   rS   s     @r^   r   z(_check_specs_vs_args.<locals>.<listcomp>  sw     
8 
8 
8a  / / / / / %/ / / / / <!!4;
8 
8 
8r`   c              3  (   K   | ]}|t           uV  d S rg   r   r   s     r^   rr   z'_check_specs_vs_args.<locals>.<genexpr>  r   r`   )r|   r>   zipr   _expand_failr   rv   r   rz   r{   r}   _spec_divisibility_error)rR   rS   r   rU   r   r   r   in_avalsr   r   r   s    `         r^   r   r     s8    #R(((
4 
4-22
4 
4 
4$((4((((( d33D
=.7Hd
K
KC
S//.>>??-
8 
8 
8 
8h66
8 
8 
8$ 	((4((((( d33D
"1dGXt
D
DC
S// r`   r   #Sequence[core.ShapedArray | NoFail]list[core.ShapedArray | NoFail]c                \    t           g| j        z  }t          ||          D ]
\  }}|||<   |S rg   )r   ry   r   )r   r   r   fail_rp   rR   s         r^   r   r   $  s@     -49w7I+I%+t$$  daE!HH	,r`   treer   strc                   t          |dt          |                    }| t          j        k    rd\  }}t	          ||          }nd| d}}g }	t          |||          D ]\  \  }
}\  }}d}| t          j        k    r||^}}|j        t          |j                  k     r@t          |j        
                                          |j                 }d| | d| d| d	}nt          |j        j                                                  d
         }|j        t          j        j        k    sJ d| | d|j        t          |j        j                  z
  dz    d| d|j         d	
}|	                    d| dt)          |
           d| dt          |           d| t)          |           | d|                                 d|j         d|j         dt          |           d           |	sJ t          |	          dk    r|	d         dd          g}	d| d| d| dd                    |	          z   dz   d | d!| d"| d#| d$	z   }t1          d% t          |||          D                       r	|d&| d'z  }|S )(Nr   )r   r   rw   z(*args) z, where  is bound to 's parameter '', is the index     component of 's varargs parameter 'z* r   r   z which has length z, but z has shape z, which has rank z (and z < )r      #shard_map applied to the function 'z' was given an zG_specs entry which is too long to be compatible with the corresponding zput value from the function:

r   zEntries in zQ_specs must be of length no greater than the number of axes in the corresponding z=put value.

Either revise the spec to be shorter, or modify 'z' so that its zputs have sufficient rank.c              3  2   K   | ]\  }\  }}|j          V  d S rg   )r   )ro   r   avals      r^   rr   z#_spec_rank_error.<locals>.<genexpr>S  s.      KK<1iq$TYKKKKKKr`   z0

For scalar values (rank 0), consider using an z>_specs entry of `P()`, where `P = jax.sharding.PartitionSpec`.)getattrr   rv   r   _try_infer_args_iter_pathsidxr   	argumentslistkeys	signature
parametersvalueskindinspect	ParameterVAR_POSITIONALr   r   rJ   	str_shortr   r   r   )r   rR   r   r   r   fun_namer   basebar   spec_keyr   fail_keyr   extraarg_keyr   
param_nameparamr   s                       r^   r   r   ,  s    Q
CFF++(=&&&LFD	D	!	!BBX...DF	$,7eU,K,K G G(x(4E](((R^kg	s2<((	(	(",++--..w{;
.D .' . . . .). . . R\,335566r:zW.=====FD F' F FK#bl&=">">>BF FF F7<zF F F 	KK	FV 	F 	F6(++ 	F 	F 	F 	Ft99	F 	F	F!!	F#(	F 	F59^^5E5E	F 	F )	F 	F ,09	F 	F :=T	F 	F 	FG G G G
 
++YY!^^T!WQRR[MT
Cx 
C 
C
C 
C 
C 
C 
C ;;t
  &
&
8v 
8 
806
8 
8=E
8 
8 
8 
8 
8	
8# 	KK;tUE+J+JKKKKK G F F F F GC	*r`   c                   t          | |          }t          | dt          |                     }g }t          |||          D ]\  \  }}	\  }
}d}||
^}}|j        t          |j                  k     r>t          |j                                                  |j                 }d| d| d| d}nt          |j	        j
                                                  d         }|j        t          j        j        k    sJ d| d|j        t          |j	        j
                  z
  d	z    d
| d|j         d	}t#          |	          }|                                D ]\  }}|j        |         t)          fd|D                       z  rt          |          d	k    rd| nd|d          d}t          |          d	k    rdnd}t)          fd|D                       }|                    dt-          |
           d|                                 | dt-          |           d|	 d| d|j        |          d| d| d| d| d|j        |                     |sJ t          |          d	k    r|d         dd          g}d| d t1          j                                                   d!j         d"d#                    |          z   d#z   d$| d%z   }|S )&Nr   r   z, where argsr   r   r   r   r   r   r   r   c              3  2   K   | ]}j         |         V  d S rg   r   r   s     r^   rr   z+_spec_divisibility_error.<locals>.<genexpr>n  s)      88djm888888r`   zaxes zaxis 'r   'ztotal c              3  2   K   | ]}j         |         V  d S rg   r   r   s     r^   rr   z+_spec_divisibility_error.<locals>.<genexpr>q  s)      ,,A$*Q-,,,,,,r`   z* argsz
 of shape z corresponds to in_specsz
 of value z, which maps array axis z
 (of size z
) to mesh z (of zsize z), but z does not evenly divide r   r   z' was given argument arrays with axis sizes that are not evenly divisible by the corresponding mesh axis sizes:

The mesh given has shape  with corresponding axis names .

r   a  Array arguments' axis sizes must be evenly divisible by the mesh axis or axes indicated by the corresponding elements of the argument's in_specs entry. Consider checking that in_specs are correct, and if so consider changing the mesh axis sizes or else padding the input and adapting 'z' appropriately.)r  r  r   r  r  r   r  r  r  r  r	  r
  r  r  r  r  r   r}   r   r   r   r   rJ   r  r{   r   r   )rR   rS   r   r   r   r  r  r   r  r   r  r   r  r  r   r  r  r   r   r   axistotalszr   s    `                      r^   r   r   X  s    q$"Q
CFF++(	$,7eU,K,K    (x(4E	~kg	s2<((	(	(",++--..w{;
. . .h . .). . . R\,335566r:zW.=====F F FK#bl&=">">>BF FF F7<zF F F t$$E 
  
 2	A8888R88888	8 	 "2ww{{|r|||0AA0A0A0AGGaKKR,,,,,,,,,VH%%  1A1A 5  &,X&6&6 BF %& 26*Q-       ')  24  z!}	 	  	  	 
  
++YY!^^T!WQRR[MT
<x 
< 
< &+4:+<+<+>+>%?%?
< 
< &*_	
< 
< 
<
 ;;t

  &
&
G -5	
G 
G 
G

G# 
*r`   list[set | NoFail]c                   t          | dt          |                     }g }t          |||          D ]$\  \  }}\  }	t          |          }
t	          ||
          }t          |          dk    rd                    t          t          |                    }d                    t          t                              }d                    t          t          fd|D                                 }|                    dt          |           d| d| d| d	| 
           |\  }|                    dt          |           d| d
| d           &|sJ t          |          dk    r|d         dd          g}d| dt          |j                                                   d|j         dd                    |          z   dz   dz   }|S )Nr   r   ,c                    g | ]}|v|	S rh   rh   )ro   r   reps     r^   r   z$_inout_rep_error.<locals>.<listcomp>  s    GGGa!3,,,,,r`   z* out_specsr   zS which implies that the corresponding output value is replicated across mesh axes {z*}, but could only infer replication over {z&}, which is missing the required axes zS which implies that the corresponding output value is replicated across mesh axis 'z0', but could not infer replication over any axesr   r   r   z}' was given out_specs which require replication which can't be statically inferred given the mesh:

The mesh given has shape r  r  r   zCheck if these output values are meant to be replicated over those mesh axes. If not, consider revising the corresponding out_specs entries. If so, consider disabling the check by passing the check_rep=False argument to shard_map.)r  r   r  r}   _unmentionedr   r   r|   r   rJ   r{   r   r
  r   )rR   rS   r   r   r   r  r   r  r   r  dstunmentionedneed_repgot_repdiff	need_rep_r   r&  s                    @r^   r   r     sX   Q
CFF++(	$+6tUE+J+J K K'x3
T
"
"CtS))K
;!#c;//00hS#''gXXc#GGGG;GGGHHIId
kk7x(( 7 7d 7 77 7EL7 7 157 78 8 8 8 ji
kkJx(( J Jd J JJ J JK K K K 
++YY!^^T!WQRR[MT
<x 
< 
< &+4:+<+<+>+>%?%?
< 
< &*_	
< 
< 
<
 ;;t

  &
&
2	
2# 
*r`   r   list[AxisName]c                f    d |                                 D             fd| j        D             S )Nc                    h | ]	}|D ]}|
S rh   rh   ro   r   r   s      r^   	<setcomp>z_unmentioned.<locals>.<setcomp>  %    555B"55Qa5555r`   c                    g | ]}|v|	S rh   rh   ro   r   name_sets     r^   r   z _unmentioned.<locals>.<listcomp>  s#    	:	:	:(9(9!(9(9(9r`   )r
  r   )rS   r   r6  s     @r^   r'  r'    s:    55%,,..555(	:	:	:	:T_	:	:	::r`   c                    t          |dg|j        z            }	  t          j        |           j        | S # t
          t          f$ r Y d S w xY wNF)rG   ry   r  r  r   r   rz   )rR   r   
dummy_argss      r^   r  r    sa    dUGdo$=>>*$7Q$j11
Z	    44s   7 AATlist[T | NoFail]1list[tuple[tuple[KeyPath, P], tuple[KeyPath, T]]]c                    t          | |          }t          |          }t          t          |          t          |                    }d }t          |||          }d t	          ||          D             S )Nc                    | d u pDt          |           t          u o.t          |           dk    ot          | d                   t          u S )Nr   r   )r   r{   r   r   ri   s    r^   rk   z_iter_paths.<locals>.<lambda>  s=    19TQ5 0 TSVVq[ TT!A$ZZST_ r`   rl   c                <    g | ]\  }\  }}|
|t           u|||ffS rg   r   )ro   rq   r  	fail_datas       r^   r   z_iter_paths.<locals>.<listcomp>  sD     
9 
9 
9)A,AXy]y77 x#
$777r`   )rG   rN   rH   rK   r   )r   r   r   failuresfailures_augspecs_leaf	specs_augs           r^   r  r    s    D%(((#H--,.//1CE1J1JKK&	T	T$vx>>>)
9 
9L))
9 
9 
9 9r`   c                      e Zd ZdZd Zd ZdS )ShardMapPrimitiveTc                4    |^}} |j         t          ||fi |S rg   )process_shard_mapr   )selftracefun_and_argsparamsr   r   s         r^   bind_with_tracez!ShardMapPrimitive.bind_with_trace  s*    JC$"5";TDDVDDDr`   c                   t          |          }|                    d          }t          j        t          j        t
          j                  |d          }|                    d          t          fd          |d<   |g|fS )Njaxprrh   	out_namesc                      S rg   rh   )axess   r^   rk   z3ShardMapPrimitive.get_bind_params.<locals>.<lambda>  s    T r`   closurer   )dictpopr   hashable_partialr   r   
eval_jaxprr5   )rJ  rM  
new_paramsrP  subfunrS  s        @r^   get_bind_paramsz!ShardMapPrimitive.get_bind_params  s{    fJNN7##E do!>!>rJJF>>+&&D$4\\\\4$P$P$PJ !8Zr`   N)r   r   r   multiple_resultsrN  r\  rh   r`   r^   rG  rG    s;        E E E         r`   rG  r_   rK  pe.DynamicJaxprTraceprimcore.Primitivelu.WrappedFun
in_tracersSequence[Any]r   tuple[AxisNames, ...]r   #Callable[[], tuple[AxisNames, ...]]r   	frozensetSequence[pe.DynamicJaxprTracer]c          	         t           j        |          }d |D             }
t          t          t                    ||
          }t	          j        t          j                                                            5  t          t          j        |                    5  t          j        ||          \  }}}\   d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t           |            |           |rMt          t          t                    |          }t!          ||          }t#           |            |           t          t$          |          }fdt'           |            |          D             }t)          j                     fd|D             }t           j        |          }t           j        t           j        |                    }t           j        |          }i ft1          |          z  t3          |          z   }t	          j        t          j                                                            5  t          j        |          }d d d            n# 1 swxY w Y   t7          |t3           |                      ||||	          }t	          j        |j        j                  }t          j        g ||||||          } j         !                    |           |S )Nc                    g | ]	}|j         
S rh   r   ro   ts     r^   r   z&_shard_map_staging.<locals>.<listcomp>  s    )))af)))r`   c           	     R    g | ]#\  }}t          t          ||                    $S rh   )_check_shapedarray_unshard_aval)ro   r   r   rS   s      r^   r   z&_shard_map_staging.<locals>.<listcomp>  sG     E E E5$ "-eT"B"BCC E E Er`   c                <    g | ]}t          j        |          S rh   )peDynamicJaxprTracer)ro   r   source_inforK  s     r^   r   z&_shard_map_staging.<locals>.<listcomp>  s(    QQQ!&ua==QQQr`   rS   r   rQ  rP  rX   r   rZ   )"r|   to_jaxpr_tracerr   _shard_avalr   extend_axis_env_ndr  r   r   r$   r   get_abstract_mesh_from_avalsrq  trace_to_jaxpr_dynamic_check_names_in_names_to_rep
_check_rep_check_repsrn  r   r   currentgetvarmakevarr   r{   convert_constvars_jaxprrV  filter_named_axis_effectseffectsr   new_jaxpr_eqnframeadd_eqn)rK  r_  rR   rb  rS   r   r   rX   r   rZ   r   	in_avals_rP  
out_avals_constsin_repout_rep	out_avalsout_tracersinvars	constvarsoutvarsin_names_stagedrM  effseqnrs  s   `   `                     @r^   _shard_map_stagingr    s    5(*55*))j)))('+t,,hAA)TZ%5%5%7%7 8 899 L L$;IFFGGL L$&$=a$K$K!E:vrL L L L L L L L L L L L L L L L L L L L L L L L L L L L L L   *--- 2)400(;;Fuf--Goo''111$j11)E E E E"%oo&7&7"C"CE E E) (**+QQQQQyQQQ+u|Z((&%,E$96 B BCC){++'ECKK'%//9/tDJ$4$4$6$67788 . .&u--E. . . . . . . . . . . . . . .TO 1 122%#W4A A A& 
	't	G	G$
.9.v.v{	, 	,#+c	sH   "C"#C?C"C	C"C	C""C&)C&I==JJr   core.AbstractValuecore.ShapedArrayc                >    t          | t          j                  sJ | S rg   )r   r   ShapedArrayrj  s    r^   rn  rn    s     	D$*	+	+++	+	+r`   c                    t          |          t          j        v r)t          j        t          |                   | ||          S t          dt          |                     NzUnsupported aval type: )r   r   shard_aval_handlersNotImplementedErrorrS   r   r   s      r^   rv  rv    sS    	$ZZ4+++#DJJ/eTBBBBd4jjBBCCCr`   c                    t          |          t          j        v r)t          j        t          |                   | ||          S t          dt          |                     r  )r   r   unshard_aval_handlersr  r  s      r^   ro  ro    sS    	$ZZ4---%d4jj1$tDDD
DT

DD
E
EEr`   c                ~    t          |t          j                  sJ t           fdt	          |j                  D                       }t          j        j        rHt           j
        t          j         j        i          }t          |t          d g|j        z             }nd }|                    ||          S )Nc           	   3     K   | ]9\  }}|t          fd                     |d          D                       z  V  :dS )c              3  2   K   | ]}j         |         V  d S rg   r   r   s     r^   rr   z0_shard_shaped_array.<locals>.<genexpr>.<genexpr>  s)      GGtz!}GGGGGGr`   rh   Nr   getro   rp   r!  rS   r   s      r^   rr   z&_shard_shaped_array.<locals>.<genexpr>  sk       8 82 $GGGGeii26F6FGGGGGG 8 8 8 8 8 8r`   )
axis_typesr   sharding)r   r   r  r{   r   r   r   sharding_in_typesvaluer!   shape_tupler#   
Collectiver   r   r   r   update)rS   r   r   	new_shapenew_meshnew_shardings   ``    r^   _shard_shaped_arrayr    s    	D$*	+	+++	+ 8 8 8 8 8!*4:!6!68 8 8 8 8)# i&:DO%LN N NH 1tfty.@+ABBLLL	9|	<	<<r`   c                v    t          |t          j                  sJ t           fdt	          |j                  D                       }t          j        j        rDt                    
                    |j                  }t          t                      |          }nd }|                    ||          S )Nc           	   3     K   | ]9\  }}|t          fd                     |d          D                       z  V  :dS )c              3  2   K   | ]}j         |         V  d S rg   r   r   s     r^   rr   z2_unshard_shaped_array.<locals>.<genexpr>.<genexpr>&  s)      FFdjmFFFFFFr`   rh   Nr  r  s      r^   rr   z(_unshard_shaped_array.<locals>.<genexpr>&  sk       8 82 FFFFUYYq"5E5EFFFFFF 8 8 8 8 8 8r`   r  )r   r   r  r{   r   r   r   r  r  _names_to_pspec_normalized_specr   r   r%   r  )rS   r   r   r  r   r  s   ``    r^   _unshard_shaped_arrayr  #  s    	D$*	+	+++	+ 8 8 8 8 8!*4:!6!68 8 8 8 8)# 5!!2249==D !2!4!4d;;LLL	9|	<	<<r`   c          	        ~t          |j        ||          D ]H\  }	}
}t          j        |	j        t          |||
j                            st          j        d          It          j        t          |j	        
                                                    5  t          j        |           d d d            n# 1 swxY w Y   |rot          t          t          |          |          }t          |||          }t          ||          D ]*\  }}t!          |||          st          j        d          +d |j        D             }t          t          t$          |          ||          }t          j        |j        |j                  }||fS )NzLshard_map argument avals not compatible with jaxpr binder avals and in_namesz7shard_map can't prove output is sufficiently replicatedc                    g | ]	}|j         
S rh   rj  ro   rj   s     r^   r   z(_shard_map_typecheck.<locals>.<listcomp>D  s    555!qv555r`   )r   r  r   
typecompatr   rv  JaxprTypeErrorrw  r{   r   r   check_jaxprr|   r   r{  r|  _valid_repeatsr  ro  r  r  r   )r   rP  rS   r   rQ  rX   r   rZ   in_atomsvrj   in_namer  r  r&  r(  out_avals_shardedr  r  s                      r^   _shard_map_typecheckr  4  s   
5<8<< C CmaG?16;tWaf#E#EFF C !B C C CC uTZ%5%5%7%78899  U               =)400(;;Fuf--G++ = =SD#s++ =! #< = = 	== 65u}555'-..	;LMM)		't	G	G$	Ds   B::B>B>set[AxisName]c                h    t          | j                  d |                                D             z
  S )Nc                    h | ]	}|D ]}|
S rh   rh   r1  s      r^   r2  z#_in_names_to_rep.<locals>.<setcomp>K  s%     H H HrR H H H H H Hr`   )setr   r
  )rS   r   s     r^   r{  r{  J  s.    	T_		 H HU\\^^ H H H	HHr`   rP  
core.Jaxprr  Sequence[RepType]c                  	 i 	d	fd}d	fd
}t          ||j        t          | j                  gt	          |j                  z             t          ||j        |           t          j        |          }|j        D ]}t          
                    |j        t          t          |j                            } || gt          ||j                  R i |j        }|j        j        rGt!          |          t          u r|gt	          |j                  z  n|}t          ||j        |           n ||j        d         |           t          j        |	|           t          ||j                  S )Nrj   	core.Atomr   RepTypec                N    t          |           t          j        u r|          nd S rg   r   r   Var)rj   envs    r^   readz_check_rep.<locals>.readQ  s#    !WW((3q66d2r`   r  core.Varvalr   c                    || <   d S rg   rh   )r  r  r  s     r^   writez_check_rep.<locals>.writeT  s    CFFFr`   r   )rj   r  r   r  )r  r  r  r  r   r   )r|   r  r  r   r   r  r   	last_usedeqns_check_rulesr  	primitiver   _rule_missingrM  r]  r   r  clean_up_dead_vars)
rS   rP  r  r  r  r  r   ruler  r  s
            @r^   r|  r|  M  s   !##3 3 3 3 3 3      eU_s4?334s5?7K7KKLLLeU\6"""nU##): / /aAK)L)LMMDd4:#dAH--:::::G{# #.27mms.B.B	C	NN**g	%G$$$$eAIaL'"""AsI....	T5=	!	!!r`   r&  r  r(  c                j    |d u p/t          t          | |                                        |          S rg   )r  r'  r   )rS   r&  r(  s      r^   r  r  e  s1    		BLs3344==cBBBr`   c                (    t          d|  d          )NzNo replication rule for z. As a workaround, pass the `check_rep=False` argument to `shard_map`. To get this fixed, open an issue at https://github.com/jax-ml/jax/issuesr  )r_  r   __s      r^   r  r  h  s*    6 6 6 6	7 	7 7r`   ctxmlir.LoweringRuleContextir.Attributec                |   d |                                 D             }t          | ||          }t          j        |j        t          j                  r)t          j        ||          }t          j	        |          }|
                    |j                  }|r|j        D ]	}d|_        
|                                S )Nc                $    i | ]\  }}|D ]}||S rh   rh   ro   rp   r   r   s       r^   r   z._shardy_shard_map_sharding.<locals>.<dictcomp>s  +    	<	<	<ea	<	<$	<	<	<	<r`   F)r   _make_scoped_manual_shardingr   
issubdtypedtypeextendedr   physical_shardingr   physical_aval_to_sdy_shardingr   dimension_shardings	is_closedbuild)	r  rS   rZ   r   aval_inrS  r   sdy_shardingdim_shardings	            r^   _shardy_shard_map_shardingr  p  s     
=	<u{{}}	<	<	<$#Ct44"w}fo66 *		)'2	6	6B ))G$$W\22,	 %$8 % %$l					r`   c                \   d |j         D             }t          | j        j        t          j                  r?t          |j                  | j        j        j        z
  }t	          j        |||z
            }	n*t	          j        |t          |j                  |z
            }	| j        	                    |	          }
g | j
        |R }|
j        j        }|j        t          j        fd|D                       }|dk    rt          j        t!          |j                                                            5  t%          j        |
|| j        t%          j                    dg|R d| j
        i\  }}d d d            n# 1 swxY w Y   |S t,          j                            t3          t5          t6          | ||          || j                            }t,          j                            t3          t5          t6          | ||          || j                            }t3          t$          j        | j                  }t-          j        ||||t,          j                             tB          j"                            d |D                                           }tB          j#        $                    |j%        t3          t$          j        |                    }tC          j&        |          5  t          j        t!          |j                                                            5  t%          j        |
|| j        t%          j                    dg|j'        R d| j
        i\  }}t-          j(        d |D                        d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   |j)        S )	Nc                    g | ]	}|j         
S rh   rj  ro   r  s     r^   r   z._shard_map_lowering_shardy.<locals>.<listcomp>      ,,,!qv,,,r`   axis_contextc                     g | ]
}|         S rh   rh   )ro   r   
mesh_shapes     r^   r   z._shard_map_lowering_shardy.<locals>.<listcomp>  s    AAAjmAAAr`   r   rh   dim_var_valuesc                L    g | ]!}t           j                            |          "S rh   )r3   
StringAttrr  ro   rp   s     r^   r   z._shard_map_lowering_shardy.<locals>.<listcomp>  s(    FFFQBM--a00FFFr`   c                6    g | ]}t          j        |          S rh   )r3   Valuer  s     r^   r   z._shard_map_lowering_shardy.<locals>.<listcomp>  s     222!"(1++222r`   )*r  r   module_contextr  r   SPMDAxisContextrf  r   manual_axesreplacer  r   npr   r   rw  r{   r   rA   jaxpr_subcomp
name_stackTokenSetr4   TensorShardingPerValueAttrr  r|   r   r  avals_in	avals_outaval_to_ir_typeManualComputationOpManualAxesAttrr3   	ArrayAttrBlockcreate_at_startbodyInsertionPointr  ReturnOpresults)r  in_nodesrP  rS   r   rQ  rZ   r  	free_axesnew_axis_contextsub_ctxr   r  manual_axes_size	out_nodesr   in_shardingsout_shardingsoutput_typesmanual_computation_opblock
out_nodes_r  s                         @r^   _shard_map_lowering_shardyr%    sC   ,,u|,,,)"/1OPP 1 $/**S-?-L-XXI%5dI<LMM%5i((4/1 1&&4D&EE'	)3	)	)	)$$0+z*WAAAA[AAABB		 tz'7'7'9'9!:!:	;	; - -'
5#.$-//2-@D- - -+- -li- - - - - - - - - - - - - - - /33C(#tT::5 5  , 044S(#tT::6  6  ! !- T)3=99,1D,	
,

FF+FFF
G
GI IJ J (
"
" #d&:I"F"FH H%	 4 4!8DJ"! "! 4 4&+>Co+ + +)+ +MJ L22z2223334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 4 
	&&sI   88E<<F F %9NAN9NN			NN		NN #N c               N   ~~t           j        j        rt          | ||||||          S d |j        D             }	d |j        D             }
t          t          t          | ||          || j	        |	|          }t          j        |t          |j                  |z
            }| j                            |          }t!          j        t%          |j                                                            5  t+          j        d| j        |d ||	|
| j        g|R | j        t          t4          ||	          t          t4          ||
          d\  }}d d d            n# 1 swxY w Y   |                     |           t          t          t8          | ||          ||
| j        |          S )Nc                    g | ]	}|j         
S rh   rj  r  s     r^   r   z'_shard_map_lowering.<locals>.<listcomp>  r  r`   c                    g | ]	}|j         
S rh   rj  r  s     r^   r   z'_shard_map_lowering.<locals>.<listcomp>  s    ...1...r`   r  
shmap_body)r  	arg_namesresult_names)r   use_shardy_partitionerr  r%  r  r  r|   r   
_xla_shardr  r   r  rf  r   r  r  r   rw  r{   r   r   rA   call_loweringr
  	tokens_inr  _pspec_mhlo_attrsset_tokens_out_xla_unshardr  )r  rP  rS   r   rQ  rX   r   rZ   r  r  r  	in_nodes_r  r  r$  
tokens_outs                   r^   _shard_map_loweringr5    s   "( ?%XudHi? ? ?,,u|,,,).....*'*c466#,X' ')#3
Ido&&-  &&4D&EE'uTZ%5%5%7%78899 D D!/cneT7ICMD$-D D>A>P'9==*IzBB	D D DJ
D D D D D D D D D D D D D D D Z   	W\3d33Y
]J
( 
( (s   5AEE!Ec                    | j         j        }t          |t          j                  r|j        }nt          i           }t          |t          j        |          |          S )N)_manual_axes)	r  r  r   r   r  r  rf  r   array_mapping_to_axis_resources)r  rS   rS  axis_ctxr  s        r^   r  r    sa    ,(.899  &KKB--K	
N:4@@
  
  
   r`   c                   t          fd|j                                        D                       dk    r|S t          j        |t          |j                  z
  |          }d |                                D             }t          | ||          }	t          j	        |j
        t          j                  r)t          j        ||	          }	t          j        |          }|	                    |j                                                  }
r!t'          t)          |j                            nt'                      }t+          j        | |||
|          }t+          j        | ||||          S )Nc                "    g | ]\  }}|v	|S rh   rh   ro   r   sizerZ   s      r^   r   z_xla_shard.<locals>.<listcomp>  "    
C
C
CGAtQd]]4]]]r`   r   c                $    i | ]\  }}|D ]}||S rh   rh   r  s       r^   r   z_xla_shard.<locals>.<dictcomp>  r  r`   unspecified_dims)r   r   r   rC   manual_protorf  r   r  r   r  r  r  r   r  r   r  _to_xla_hlo_shardingr   to_protor  rangerA   wrap_with_sharding_opwrap_with_full_to_shard_op)r  rS   rZ   r   r  aval_outrj   rB  rS  r   shard_protounspecifiedsxs     `          r^   r-  r-    sL   	
C
C
C
Ctz//11
C
C
CDDIIH"7Ido,F,F,MtTT,	<	<u{{}}	<	<	<$#Ct44"w}fo66 *		)'2	6	6B ))G''55>>@@+,0;E',''(((cee+!#q';3>@ @ @"		(b(L+	V	VVr`   c                   t          fd|j                                        D                       dk    r|S d |                                D             }t          | ||          }t	          j        |j        t          j                  r)t          j	        ||          }t          j        |          }r!t          t          |j                            nt                      }	t          j        |t#          |j                  z
  |          }
t'          j        | |||
|	          }|                    |j                                                  }t'          j        | ||||	          S )Nc                "    g | ]\  }}|v	|S rh   rh   r<  s      r^   r   z _xla_unshard.<locals>.<listcomp>  r>  r`   r   c                $    i | ]\  }}|D ]}||S rh   rh   r  s       r^   r   z _xla_unshard.<locals>.<dictcomp>  r  r`   r@  )r   r   r   r  r   r  r  r  r   r  r   r  r  rE  r   rC   rB  rf  r   rA   rF  rC  rD  wrap_with_shard_to_full_op)r  rS   rZ   r   r  rH  rj   rS  r   rJ  rB  rK  rI  s     `          r^   r2  r2    sJ   	
C
C
C
Ctz//11
C
C
CDDIIH	<	<u{{}}	<	<	<$#Ct44"x~v77 ,		)(B	7	7B!(++H-1<E(-(()))suu+"7Ido,F,F,MtTT,!#q'<R]^^^"''66??AA+		(b(K)4
6 
6 6r`   c                    t          |t          j                  r4t          t	          | j        t          |j                                      S dS )Nr   )r   r   r  r   r|   r  rE  r   )r   r   s     r^   r0  r0    s@    d&'' 1s59eDI..//000	r`   c           
        | D ]}t          |d          rt          |j        t                    rs|j        j        j        |j        k    rMt          |          }t          d|j        j        j         d|j         d|                                           |j        j        }t          |t                    rt          d          t          |t                    sJ |S )Nr  zMesh shape of the input z4 does not match the mesh shape passed to shard_map  z for shape zyPlease pass `jax.Array`s with a `NamedSharding` as input to `shard_map` when passing `AbstractMesh` to the mesh argument.)hasattrr   r  r   rS   r  r>   rz   r  r!   r"   )r   rS   r   r   s       r^   get_mesh_from_argsrS    s
     aq* *QZ"G"G 	
	$(8	8	8!!$$@qz'B @ @ @ @-1^^-=-=@ @A A 	A Z_dl## J
	IJ J J 
D$				+r`   c                  |	rt           ~~	t          |t                    rt          ||          }t	          t          t          |          ||          }t	          t          t          |          |          }
t          ||||
|          \  }}d |D             }t           |            |           |rt          | |            |           t	          t           |                      }t	          t          t          ||          ||          S )Nc           	     t    g | ]5}t          j        |j        d          d t          j        |                    6S r   )r   mapped_avalr   get_avalr  s     r^   r   z#_shard_map_impl.<locals>.<listcomp>  s7    OOOQt
At}Q/?/?@@OOOr`   )r  r   r!   rS  r|   r   _unmatch_specr{  
_run_shmaprz  r}  r  _match_spec)rK  r_  r   r   rS   r   r   rX   r   rZ   r  outsr  r  pspecss                  r^   _shard_map_implr^    s   	$$$
Dl## *dD))D	W]D))8T	:	:$w'..99&S$fi@@-$OO$OOO)  ),,, 2oo''111 1 122&	W[$	22FD	A	AAr`   c                   t          ||          }t          t          t          |          ||          }t	          j        |          5  t	          j        |j                                                  5   | j	        | }t          t          |j        |                    \  }}	d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   ||	fS rg   )ShardMapTracer|   r   ShardMapTracerr   set_current_tracerw  r   r   call_wrappedr7   to_val_rep_pair)
rR   rS   r   repsrX   rK  rb  ansr\  r  s
             r^   rZ  rZ    s>   
i
(
(%7>5114>>*e$$ > >		 !1!1!3!3	4	4 > >ANJ'cS!6<<==mdG> > > > > > > > > > > > > > >> > > > > > > > > > > > > > > 
ws6   	,C50B1%C1B5	5C8B5	9CCCc                ~      rt                     dz   nd}d t           fdt          |          D              S )Nr   r   c                @    | t          |           dk    r| d         n| S )Nr   r   )r   )rl  s    r^   rk   z!_names_to_pspec.<locals>.<lambda>'  s     Q]s1vv{{QqTT r`   c              3  T   K   | ]"}                      |                    V  #d S rg   )r  )ro   rp   r   unpacks     r^   rr   z"_names_to_pspec.<locals>.<genexpr>(  s7      DD!		!--DDDDDDr`   )maxr   rE  )r   ndminrj  s   ` @r^   r  r  %  sM    !
(#e**q..q%AA&	DDDDDuU||DDD	EEr`   srcrj   JaxTypec                n   t          j                    5  t          j        d          5   t          j        t          t          | t          |                                                              |          cd d d            cd d d            S # 1 swxY w Y   d d d            d S # 1 swxY w Y   d S r8  )	r   eval_contextjaxdisable_jitjitr6   _unmatchr{   r   )rS   rm  rj   s      r^   rY  rY  *  sa    K KCOE22 K KG37?8T53E3EFFGGJJK K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K K Ks5   B*AB9B*B	B*B	B**B.1B.c                    t          t          |                    }t          | j                  } t	          t
          | |f|d          |          S NF)rX   )r  rV  r   r   r_   _add_singleton)rS   src_tuprj   rm  r(  s        r^   rt  rt  .  sI    W&&#	$/#	F>4#	F	F	Fq	I	IIr`   Sequence[AxisNames]avalsSequence[core.ShapedArray]c                    d t          | |          D             }t          d |D                       rt          |          d S )Nc                Z    g | ](\  }}|rt          |          |j        k     s|nt          )S rh   )rk  r   r   )ro   r   r   s      r^   r   z _check_names.<locals>.<listcomp>5  sE     
) 
) 
)a 
5Q!&!!g 
) 
) 
)r`   c              3  (   K   | ]}|t           uV  d S rg   r   r   s     r^   rr   z_check_names.<locals>.<genexpr>7  r   r`   )r   r   r   )r   rz  r   s      r^   rz  rz  3  sZ    
) 
)%''
) 
) 
)$((4(((((@
40@0@*@@@r`   c                      e Zd ZdS )r   Nr   rh   r`   r^   r   r   8  r   r`   r   c                      fdt          ||          D             }t          d |D                       rt          |          d S )Nc                J    g | ]\  }}t          ||          s|nt           S rh   )r  r   )ro   r   rrS   s      r^   r   z_check_reps.<locals>.<listcomp>;  sB     
( 
( 
(a "$1--
:!!7 
( 
( 
(r`   c              3  (   K   | ]}|t           uV  d S rg   r   r   s     r^   rr   z_check_reps.<locals>.<genexpr>=  r   r`   r   r   r   )rS   r   re  r   s   `   r^   r}  r}  :  sc    
( 
( 
( 
(%&&
( 
( 
($((4(((((?	$*???r`   c                      e Zd ZdS )r   Nr   rh   r`   r^   r   r   >  r   r`   r   c                    d t          ||          D             }t          d |D                       rt          |          d S )Nc                P    g | ]#\  }}|                     |          s|nt          $S rh   )r   r   )ro   r(  rm  s      r^   r   z _check_reps2.<locals>.<listcomp>A  sA     
0 
0 
0c3 \\#&&
3##G 
0 
0 
0r`   c              3  (   K   | ]}|t           uV  d S rg   r   r   s     r^   rr   z_check_reps2.<locals>.<genexpr>C  r   r`   r  )rS   	reps_destre  r   s       r^   _check_reps2r  @  sX    
0 
0i..
0 
0 
0$((4(((((?	$*???r`   pspecc           
     V   t          t          | ||          }t          j                    5  t	          j        d          5   t	          j        |t          | |                    |          cd d d            cd d d            S # 1 swxY w Y   d d d            d S # 1 swxY w Y   d S )NF)r   )r6   _matchr   rp  rq  rr  rs  r   )rS   rX   r  rj   fns        r^   r[  r[  E  sg   vtY66" D DCOE22 D D@372]4%?%?@@@CCD D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D D Ds5   B -B-BB
	
BB
	BB"%B"c                p    t          | j                  } t          t          | |f|d          |          S rv  )r   r   r_   _rem_singleton)rS   rX   r  rj   rm  s        r^   r  r  K  s6    	$/#	H>4#%	H	H	H	K	KKr`   c                F    |                      | j        dd                    S Nr   reshaper   ri   s    r^   r  r  P  s    aii444r`   c                (     | j         dg| j        R  S r  r  ri   s    r^   rw  rw  Q  s    iai4AG4444r`   c                  N    e Zd ZU ded<   ded<   d Zd Zd Zd Zd	 Zd
 Z	d Z
dS )r`  r"   rS   rY   checkc                "    || _         || _        d S rg   )rS   r  )rJ  rS   r  s      r^   __init__zShardMapTrace.__init__W  s    DIDJJJr`   c                    t          |t                    r|j        |j        fS t          |t                    rt          d          t          | j        i |          }|d fS )Nz(Shouldn't have any non-shard_map tracers)r   ra  r  r&  r    	ExceptionrY  rS   )rJ  r  val_s      r^   rd  zShardMapTrace.to_val_rep_pair[  sa    #~&& Wcg	C	 	  @AAA49b#..d4Zr`   c                p   t          t          | j        |                    \  }}t                              |          }|r || j        g|R i |}nt          t          |t          |	                                          | j                  }t          j                    5  t          j        d          5   t          j        |          | }d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   t                              |t!          t"          |                    }	| j        r |	| j        g|R i |nt'                      }
|j        rOt+          |
          t&          u r|
gt-          |          z  n|
}
t          t!          t.          |           |
|          S t/          | |
|          S r8  )r7   r|   rd  eager_rulesr  rS   r6   _prim_applierr{   r   r   rp  rq  rr  rs  r  r   r  r  r  r]  r   r   ra  )rJ  r_  tracersrM  in_valsr  
eager_ruleout_valsrR   rep_ruler  s              r^   process_primitivezShardMapTrace.process_primitived  s   S!5w??@@OGV&&J (DI::::6::hh
-uV\\^^/D/Ddi
P
Pa ( ( 6 6 ( (371::w'( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (gmT&B&BCCH8<
Mhhty46444V444G C-1']]c-A-A	CMM))wg..BBB$222s6   "C27CC2C	C2"C	#C22C69C6c                (    t          d| d          )NzEager evaluation of `z` inside a `shard_map` isn't yet supported. Put a `jax.jit` around the `shard_map`-decorated function, and open a feature request at https://github.com/jax-ml/jax/issues !r  )rJ  call_primitiver   r  rM  s        r^   process_callzShardMapTrace.process_callt  s*    
	1 	1 	1 	12 2 2r`   c                     t          d          )NzEager evaluation of `pmap` inside a `shard_map` isn't yet supported.Put a `jax.jit` around the `shard_map`-decorated function, and open a feature request at https://github.com/jax-ml/jax/issues !r  )rJ  map_primitiver   r  rM  s        r^   process_mapzShardMapTrace.process_map{  s    
	FG G Gr`   c                   |rd}t          |          ~~~t          t          | j        |                    \  }}t	          || j        ||| j                  \  }	}
t          t          t          |           |
|	          S )Nzcustom_jvp symbolic_zeros support with shard_map is not implemented; please open an issue at https://github.com/jax-ml/jax/issues	r  r7   r|   rd  rZ  rS   r  r   ra  )rJ  r_  r   jvpr  symbolic_zerosr   r  r  r  r  s              r^   process_custom_jvp_callz%ShardMapTrace.process_custom_jvp_call  s     %4c  $$$c>S!5w??@@OGV"3	7FDJOOHgw~t,,gx@@@r`   c                   |rd}t          |          ~~~~~t          t          | j        |                    \  }	}
t	          || j        |	|
| j                  \  }}t          t          t          |           ||          S )Nzcustom_vjp symbolic_zeros support with shard_map is not implemented; please open an issue at https://github.com/jax-ml/jax/issuesr  )rJ  r_  r   fwdbwdr  	out_treesr  r   r  r  r  r  s                r^   process_custom_vjp_callz%ShardMapTrace.process_custom_vjp_call  s     %4c  $$$c3	>S!5w??@@OGV"3	7FDJOOHgw~t,,gx@@@r`   N)r   r   r   __annotations__r  rd  r  r  r  r  r  rh   r`   r^   r`  r`  S  s         ***+++    3 3 3 2 2 2G G G
A 
A 
A
A 
A 
A 
A 
Ar`   r`  c                  R    e Zd ZU ded<   ded<   d Zed             Zd Zdd
ZeZ	dS )ra  r  r&  rn  r  c                0    || _         || _        || _        d S rg   _tracer&  r  rJ  rK  r&  r  s       r^   r  zShardMapTracer.__init__      DKDHDHHHr`   c                ~    t          j        | j                  }t          j        | j        j        j        d|          S Nr   )r   rX  r  rW  r  rS   r=  )rJ  r   s     r^   r   zShardMapTracer.aval  s0    =""DDK,11d;;;r`   c                    | j         t          | j        j        j                  k    rLt          j                    5  t          j        | j        d                   cd d d            S # 1 swxY w Y   d S d S r  )	r&  r  r  rS   r   r   rp  to_concrete_valuer  rJ  s    r^   r  z ShardMapTracer.to_concrete_value  s    x3t{'23333 3 3%dhqk223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 Ts   A''A+.A+r   r   c                   t          j                    5  t          | j                  }d d d            n# 1 swxY w Y   | j        j        }dd                    t          t          |j	                             dd                    fdt          t          j        |j                  |          D                       S )N(r   z,)
c           
   3  B   K   | ]\  \  }}}d | d d| d| d	V  dS )zOn z at mesh coordinates z = z:
r  Nrh   )ro   r  devicer#  r   s       r^   rr   z)ShardMapTracer.__str__.<locals>.<genexpr>  sk       O O MS&5 	LfKK:KK#KK%KKKO O O O O Or`   )r   rp  r  r  r  rS   r   r|   r   r   r   r  ndenumeratedevices)rJ  blocksrS   r   s      @r^   __str__zShardMapTracer.__str__  s   				  DH~~f              ;D=TYYs38899===J99 O O O O$'t|(D(Df$M$MO O O O O Os   6::Nr   r   
r   r   r   r  r  propertyr   r  r  __repr__rh   r`   r^   ra  ra    sz         ,,,,,,  
 < < 8<  O O O O (((r`   ra  c                f      fd}t          |j                  } t          ||||d          | S )Nc                      j         t          t          |           i t                    }t	          t
          |          S rg   )r   r|   r  rV  rE   rw  )r   r\  
params_tupr_  s     r^   applyz_prim_applier.<locals>.apply  s;    49c.$//D4
3C3CDDDND)))r`   F)r   r   r_   )r_  r  rS   r   r  r   s   ``    r^   r  r    sP    * * * * * * 
4?		$	25$dE	2	2D	99r`   zdict[core.Primitive, Callable]r  r   Callable[..., Any]effectdebugging.DebugEffectc                   ~t          j                    5  t          t          t          |           }d d d            n# 1 swxY w Y   t          t          j        | j                  |          D ]\  \  }}} ||  g S rg   )r   rp  r   r|   r  r  r  r  )rS   r   r  r   
all_blocksr  r  r  s           r^   _debug_callback_eager_ruler    s     ' 'c$oo&J' ' ' ' ' ' ' ' ' ' ' ' ' ' '"2>$,#?#?LL  msFVHf	)s   >AAc               >    ~ ~~|D ]}|t          d|           |S )Nzadevice_put with explicit device not allowed within shard_map-decorated functions, but got device )rz   )rS   srcsr  copy_semanticsr   r  s         r^   _device_put_eager_ruler    sV    
D. R Rf QHNQ Q R R R  
)r`   psum2c                    || z  S rg   rh   )r  	axis_sizes     r^   rk   rk     s
    y1} r`   rS  c               (    ~t          j        | ||dS NrS  axis_index_groups)pbroadcast_pr   )ctsrS  r  r   s       r^   _psum2_transpose_ruler    s    
		Cd>O	P	P	PPr`   c                    t          |t                    s|fn|}|s| S t          |           \  }}t          j        ||d d}t          ||          S r  )r   r{   rF   r  r   rG   )rj   	axis_namerS  r   treedefyss         r^   
pbroadcastr    s\    '	599	H)y$	1HQ+"g"44@@@"		$	$$r`   r  c                    |S rg   rh   rS  r  r   s      r^   rk   rk     s    T r`   c                    |S rg   rh   r  s      r^   rk   rk     s    d r`   c                   |S rg   rh   )r  rS  r  rj   s       r^   rk   rk     s    a r`   c               n    t          d |D                       rt          t          j        | ||d}||fS )Nc              3  B   K   | ]}t          |          t          u V  d S rg   )r   int)ro   r  s     r^   rr   z&_pbroadcast_batcher.<locals>.<genexpr>  s-      ,,tds	,,,,,,r`   r  )r   r  r  r   )vals_indims_inrS  r  vals_outs        r^   _pbroadcast_batcherr    sQ    ,,t,,,,,G4G.Gd1BD D D(	7	r`   c               &    t          j        | ||dS r  )psum2_pr   )r  rS  r  r   s       r^   rk   rk     s    lCd>OPPP r`   _rewrite_rulesc                      fdS )Nc                :    t                               |           S rg   )r  
setdefault)r  r_  s    r^   rk   z<lambda>.<locals>.<lambda>  s    .*C*CD!*L*L r`   rh   r_  s   `r^   rk   rk     s     L L L L r`   c                ^    t                               | t          t          |                     S rg   )r  r  r   _standard_rewrite_ruler  s    r^   rk   rk     s%    dG,BD$I$IJJ r`   c           	     v    t                               | t          t          | t          |                              S rg   )r  r  r   _no_rewriter  )r   s    r^   rk   rk     s*    aaa!I!IJJ r`   r  c                      fdS )Nc                :    t                               |           S rg   )r  r  )r  r_  s    r^   rk   z<lambda>.<locals>.<lambda>  s    <+B+B4+N+N r`   rh   r  s   `r^   rk   rk     s    NNNN r`   c                ^    t                               | t          t          |                     S rg   )r  r  r   _standard_checkr  s    r^   rk   rk   
  s     ((w/M/MNN r`   c                     | j         |i |} ||g|R i |}| j        r,t          |          t          u r|n|gt	          |          z  }n|g|g}}||fS rg   )r   r]  r   r  r   )	r_  r  rS   r  r   rM  r  r  out_rep_s	            r^   r  r    s    TY&v&&(D))))&))'	 /w--4//wwgYX5NHH"gYhH	8	r`   c                   	 |rt          j        | nt          |j                  		fdt          ||          D             } | j        |i |}| j        r	gt          |          z  n	g}| j        s|gn|}||fS )Nc           	     v    g | ]5\  }}|z
  r)t          |t          fd |D                                 n|6S )c              3  $   K   | ]
}|v|V  d S rg   rh   )ro   r   r  s     r^   rr   z4_standard_rewrite_rule.<locals>.<listcomp>.<genexpr>  s-      CCQ(1B1Bq1B1B1B1BCCr`   r  r{   )ro   rj   rm  r  s      r^   r   z*_standard_rewrite_rule.<locals>.<listcomp>  sh     E E E(.3cHn$:aCCCC#CCCCCDDD"#E E Er`   )r  intersectionr   r   r   r]  r   )
r_  rS   r  r   rM  args_	out_vals_r  r  r  s
            @r^   r  r    s    *0JSv&&c$/6J6J(E E E E25dF2C2CE E E%di)&)))+/+@PXJY''xj' $ 5Di[[9(	7	r`   c                    d |D             }|r,|d d         |dd          k    st          d|  d| d          |r|d         nd S )Nc                    g | ]}||S rg   rh   )ro   r  s     r^   r   z#_standard_check.<locals>.<listcomp>"  s    0001!-Q---r`   r   r   z
Primitive z7 requires argument replication types to match, but got . Please open an issue at https://github.com/jax-ml/jax/issues and as a temporary workaround pass the check_rep=False argument to shard_mapr   r  )r_  rS   r  r  in_rep_s        r^   r  r    s     10000' QWSbS\WQRR[00
 P P P)/P P P Q Q Q 	(D(r`   c                     t          |           t          t          |                       t          |           t          t          |                      d S rg   )register_checkr   _standard_collective_checkregister_rewrite_standard_collective_rewriter  s    r^   register_standard_collectiver  *  sO    .w94@@AAA4!=tDDEEEEEr`   c                     t          |           t          t          |                       t          |           t          t          |                      d S rg   )r  r   _reduction_collective_checkr  _reduction_collective_rewriter  s    r^   register_reduction_collectiver  .  sO    .w:DAABBB4!>EEFFFFFr`   c          	     H    ~~|||v rt          d|  d| d| d          |S )NCollective ? must be applied to a device-varying replication type, but got & for collective acting over axis name r  r  )r_  rS   x_repr  rM  s        r^   r  r  2  sj    
F
]i5((
 P$ P P16P P&/P P P Q Q Q
 
,r`   c                    t          |t                    s|fn|}|\  }t          |          }||z  x}rt          |t          |                    } | j        |fd|i|}	|	g||z
  gfS )Nr  r   r{   r  r  r   )
r_  rS   r  rj   r  rM  r"  axis_name_setpbroadcast_axis_nameout_vals
             r^   r  r  =  s    ",Y">">MyllI)&%i..-*U22 31e01122ADIa779777'
U]*+	++r`   c          	         ~~t          |t                    s|fn|}t          fd|D                       rt          d|  d d| d          t	          |          z  S )Nc              3      K   | ]}|v V  	d S rg   rh   )ro   r   r"  s     r^   rr   z._reduction_collective_check.<locals>.<genexpr>K  s'      33!u*333333r`   r  r   r!  r  )r   r{   r   r  r  )r_  rS   r"  rS  rM  s     `  r^   r  r  G  s    
F"4//	9$T$
]c3333d33333]
 P$ P P16P P&*P P P Q Q Q
 
T	r`   c                    t          |t                    s|fn|}|\  }t          |          }||z  x}rt          |t          |                    } | j        |fd|i|\  }	|	g||z  gfS )NrS  r$  )
r_  rS   r  rj   rS  rM  r"  axes_setpbroadcast_axesr'  s
             r^   r  r  S  s    "4//	9$T$&%YY( 5((_ .1eO,,--ATYq..t.v..('
UX%&	&&r`   c                   J rg   rh   )r   rS  r  r  s       r^   _psum_checkr.  p  s    r`   c                    |t           t          |t                    s|fn|}t          |          fd|D             } fdt	          ||          D             }t          j        |||d}||fS )Nc                    g | ]}|z  S rh   rh   )ro   r  axes_s     r^   r   z!_psum_rewrite.<locals>.<listcomp>z  s    '''1QY'''r`   c           	     v    g | ]4\  }t          |t          fd j        D                                 5S )c              3  *   K   | ]}|z  v 	|V  d S rg   rh   )ro   r   r1  rm  s     r^   rr   z+_psum_rewrite.<locals>.<listcomp>.<genexpr>{  s1      NNQQ%#+=M=Mq=M=M=M=MNNr`   )r  r{   r   )ro   rj   rm  r1  rS   s     @r^   r   z!_psum_rewrite.<locals>.<listcomp>{  sZ     , , ,q# aNNNNN$/NNNNNOO , , ,r`   r  )r  r   r{   r  r   r  r   )	rS   r  rS  r  r   r  r  r'  r1  s	   `       @r^   _psum_rewriter4  t  s     "*=$="4//	9$T$
d))%'''''''', , , , ,T6**, , ,%L%d>OPPP'	'	r`   c                    t                    t          u sJ t          fd|D                       rt          d| d d          t           fd|D                       }fd|D             S )Nc              3  @   K   | ]}|t                    |z  V  d S rg   r  ro   r  rS  s     r^   rr   z_psum2_check.<locals>.<genexpr>  s-      881!-TQ----88r`   zNCollective psum must be applied to a device-varying replication type, but got r!  . Please open an issue at https://github.com/jax-ml/jax/issues, and as a temporary workaround pass the check_rep=False argument to shard_mapc              3  H   K   | ]}|t          j                  n|V  d S rg   r  r   ro   r  rS   s     r^   rr   z_psum2_check.<locals>.<genexpr>  6      JJaT_%%%JJJJJJr`   c                4    g | ]}|t                    z  S rh   r7  r8  s     r^   r   z _psum2_check.<locals>.<listcomp>  #    	(	(	(A!c$ii-	(	(	(r`   )r   r{   r   r  rS   rS  r  r  s   ``  r^   _psum2_checkrA    s    	du				888888888 Q
 P17P P&*P P P Q Q Q
 JJJJ6JJJJJ&	(	(	(	(	(	(	((r`   c                    t                    t          u sJ t          fd|D                       st          d| d d          t           fd|D                       }fd|D             S )Nc              3  D   K   | ]}|d u pt                    |z  V  d S rg   r7  r8  s     r^   rr   z$_pbroadcast_check.<locals>.<genexpr>  s6      88AQ$Y'#d))a-888888r`   zXCollective pbroadcast must be applied to a non-device-varying replication type, but got r!  r9  c              3  H   K   | ]}|t          j                  n|V  d S rg   r;  r<  s     r^   rr   z$_pbroadcast_check.<locals>.<genexpr>  r=  r`   c                4    g | ]}|t                    z
  S rh   r7  r8  s     r^   r   z%_pbroadcast_check.<locals>.<listcomp>  r?  r`   )r   r{   r   r  r@  s   ``  r^   _pbroadcast_checkrF    s    	du					8888888	8	8 Q
 P17P P '+P P P Q Q Q JJJJ6JJJJJ&	(	(	(	(	(	(	((r`   c                   t          |          t          ur|fn|}t          | j                  t          |          z
  S rg   )r   r{   r  r   )rS   r  s     r^   _axis_index_checkrH    s6    "&y//U":":yll	)	TZ3y>>	))r`   c               b    t          | ||          \  }}t          j        j        |d|i|}||fS NrP  )_replication_rewrite_nomatchr   pjit_pr   rS   r  rP  r   kwargsjaxpr_r  r  s           r^   _pjit_rewriterP    s>    0ufEE/&'[t<6<V<<(	7	r`   c               .    t          | |j        |          S rg   r|  rP  rS   rP  r  rN  s       r^   _pjit_checkrT    s    	D%+v	.	..r`   c                   t          j        |          }t          | ||          \  }}|j        |j        c}\   t          j        j        |d|i|}||fS rJ  )rq  close_jaxprrK  rP  r  r   remat_pr   rM  s           r^   _remat_rewriterX    s]    >%  &0vvFF/&'lFM)%"'EUEfEE(	7	r`   c               $    t          | ||          S rg   r|  rS  s       r^   _remat_checkr[    s    	D%	(	((r`   c               $    t          | ||          S rg   rZ  rS   
call_jaxprr  rN  s       r^   _core_call_checkr_    s    	D*f	-	--r`   c                    g S rg   rh   )rS   r  r   s      r^   _debug_callback_rulera    s    	)r`   c               @    t                      gt          |          z  S rg   r  r   rS   result_avalsr   r  s       r^   _pure_callback_rulerf        
%%3|$$	$$r`   c               @    t                      gt          |          z  S rg   rc  rd  s       r^   _io_callback_ruleri    rg  r`   c                     t          |          S rg   )r  )rS   r   r   s      r^   _device_put_rulerk    s    	b/r`   c               @    t                      gt          |          z  S rg   rc  )rS   r  r   r  s       r^   _custom_lin_rulerm    s    
%%3y>>	!!r`   c                   t          |||g          \  }}}t          | |j        |          }t          ||g          \  }}||k    st          d| d| d          |S )Nz=Scan carry input and output got mismatched replication types z and r9  )r;   r|  rP  r  )	rS   rP  
num_consts	num_carryr  r   carry_rep_inr  carry_rep_outs	            r^   _scan_checkrs    s    !&:y*ABB!\1tU[&11')55-		&	&
  )   0=      ! ! !
 
.r`   c                  t          |||g          \  }}}	t          d|z             D ]W}
g |||	}t          | ||          \  }
}t          ||g          \  }}t          t          j        ||          }||k    r n|}XJ d            d t          |||          D             }g ||}t          | |||          }t          j	        j
        j        ||||d|}||fS )Nr   FzFixpoint not reachedc           	     x    g | ]6\  }}|z
  r)t          |t          fd |D                                 n|7S )c              3  $   K   | ]
}|v|V  d S rg   rh   ro   r   r(  s     r^   rr   z+_scan_rewrite.<locals>.<listcomp>.<genexpr>  '      ==Aa==r`   r  ro   rj   rm  r(  s      @r^   r   z!_scan_rewrite.<locals>.<listcomp>  sj     
M 
M 
M"-!S#S3Y*Q=========
>
>
>
M 
M 
Mr`   )rP  ro  rp  )r;   rE  rK  r|   opand_r   _replication_rewrite_matchr0   loopsscan_pr   )rS   r  rP  ro  rp  r   rM  	const_reprq  xs_repr   r  r  rr  ys_reprO  r  s                    r^   _scan_rewriter    s=   $.v
I7N$O$O!)\6Y 
) 
)a2	2L262G-dE7CCJAw&w<<M6}==M}$$e"ll((((5
M 
M14T671K1K
M 
M 
M$%m%f%'%dE7GDD&&+6jIQ QIOQ Q(	7	r`   c                   |^}}t          | |d         j        |          }|dd          D ]-}t          | |j        |          }||k    st          d          .|S )Nr   r   zThe branches of cond produced mismatched replication types. Please open an issue at https://github.com/jax-ml/jax/issues, and as a temporary workaround pass the check_rep=False argument to shard_map)r|  rP  r  )rS   branchesr  r   args_repr  branchr  s           r^   
_cond_ruler    s{    ,!htXa[.99' & &f$h77Hw % & & &  
.r`   c                  
 |^}
t           |d         
          \  }|dd          D ]6}t           |
          \  }}rt          t          j        |          4|7t          t	          t          j        |                    t          
 fd|D                       }t          j        j        j	        |d|i}	|	fS )Nr   r   c              3  <   K   | ]}t          |          V  d S rg   )r|  )ro   r  r  rS   r  s     r^   rr   z _cond_rewrite.<locals>.<genexpr>&  sE       , , /tVXwOO , , , , , ,r`   r  )
rK  r|   rz  r{  r   r{   r0   conditionalscond_pr   )rS   r  r  r   pred_repr   r  r  	branches_r  r  r  s   `         @@r^   _cond_rewriter    s    (X+D(1+xHH*!W  f.tVXFFKAx BGWh//gggg**G44' , , , , , ,"*, , , , ,)&-2DM9MM(	7	r`   c                *    t          | j                  S rg   r;  )rS   r   r  s      r^   _platform_index_ruler  +  s    	T_		r`   c               b    t          | ||          \  }}t          j        j        |d|i|}||fS rJ  )rK  r   closed_call_pr   )rS   r  r^  r   rN  	new_jaxprr  r  s           r^   _closed_call_rewriter  0  s@    3D*fMM)W$dF)FvFF(	7	r`   c               .    t          | |j        |          S rg   rR  r]  s       r^   _closed_call_checkr  6  s    	D**F	3	33r`   c               .    t          | |j        |          S rg   rR  )rS   r^  jvp_jaxpr_thunkro  r  r  s         r^   _custom_jvp_call_checkr  ;  s     
D**F	3	33r`   c          
     8    |rd}	t          |	          t           ||          \  }
}t          ||g          \  }g t          j         fd            }t          | fd          }t          j        j        ||
|||||d}rd         n|}||fS )NPlease open an issue at https://github.com/jax-ml/jax/issues and as a temporary workaround pass the check_rep=False argument to shard_mapc                     t          j         |   }t          |          \  }}                    |           |j        |j        fS rg   )r   ClosedJaxprrK  r   rP  r  )zeros	fwd_jaxpr
fwd_jaxpr_r  fwd_jaxpr_thunkr  rS   out_rep2s       r^   fwd_jaxpr_thunk_z8_custom_vjp_call_jaxpr_rewrite.<locals>.fwd_jaxpr_thunk_N  sP     //5"9:I6tYPPJOOGZ...r`   c                      d         S r  rh   )r  s   r^   rk   z0_custom_vjp_call_jaxpr_rewrite.<locals>.<lambda>U  s    ! r`   )	fun_jaxprr  r  ro  r  r  r   )	r  rK  r;   rq  _memoize_rewrite_bwdr   custom_vjp_call_jaxpr_pr   )rS   r  r  r  r  ro  r  r  r   r   
fun_jaxpr_r  r   r  bwd_r\  r  r  s   `  `            @@r^   _custom_vjp_call_jaxpr_rewriter  @  s      #C c
"
""4T9fMM*g&:,//*!W(;/ / / / / / / ;/ 
c4!4!4!4!4g	>	>$		3	8z3Cy
Q 
Q 
Q$ $0HQKK'	wr`   c               .    t          | |j        |          S rg   rR  )rS   r  r  r   s       r^   _custom_vjp_call_jaxpr_checkr  ]  s    	D)/6	2	22r`   c               v    t          t          j        j        | g|R  }|gt	          |j        j                  z  S rg   )r  r0   solveslinear_solve_pr   solver  )rS   jaxprsr  r   r  s        r^   _linear_solve_checkr  a  s9    L/>NvNNN'
S/00	00r`   c               X    t          t          j        | g|R  }|gt          |          z  S rg   )r  r2   
ffi_call_pr   )rS   re  r  r   r  s        r^   _ffi_call_checkr  g  s0    CND:6:::'
S&&	&&r`   c                    |S rg   rh   )rS   r"  y_reps      r^   
_tie_checkr  o  s    	,r`   batching.BatchTracer   Sequence[batching.BatchTracer]c
                   t          t          | j        |                    \  }
}t          d |D                       rt          d t          ||          D             }| j        j        d |D             }t          j	        j
        s!t                    |z  rt          d          fdt          ||          D             }| j        j        t          fdD                       z  }t          j        | j        j        || j        j                  }n| j        }t          j        || j        |t)          |                    \  }t+                    fd            }t-          |||||		          }t/          j        | j                  5   |j        |g|
R i |}d d d            n# 1 swxY w Y   t7          t          j        | t;          j                    
          }t          ||                       S )Nc              3  J   K   | ]}t          |t          j                  V  d S rg   )r   r@   
RaggedAxis)ro   r   s     r^   rr   z#_shard_map_batch.<locals>.<genexpr>  s/      ==Ax*	+	+======r`   c                6    g | ]\  fd D             S )c                L    i | ] }|t           j        uo|k    z   |         !S rh   r@   
not_mappedro   axr   r   s     r^   r   z/_shard_map_batch.<locals>.<listcomp>.<dictcomp>  sE     $ $ $ ("55A!r'BE"I $ $ $r`   rh   ro   r   r   s    @@r^   r   z$_shard_map_batch.<locals>.<listcomp>  sZ     L L L(0q$ $ $ $ $"$ $ $ L L Lr`   c                J    h | ] }|                                 D ]	}|D ]}|
!S rh   r
  ro   r   r   r   s       r^   r2  z#_shard_map_batch.<locals>.<setcomp>  s:    III%ellnnIIbIIAIIIIIr`   z7vmap spmd_axis_name cannot appear in shard_map in_specsc                F    g | ]\  }}|t           j        uri ||in|S rh   r  ro   r   r   spmd_axis_names      r^   r   z$_shard_map_batch.<locals>.<listcomp>  sV     E E E %A 129L0L0L,r,1^,,,E E Er`   c              3  2   K   | ]}j         |         V  d S rg   r   r   s     r^   rr   z#_shard_map_batch.<locals>.<genexpr>  s)      +R+RaDJqM+R+R+R+R+R+Rr`   rT  c                 F    t                                               S rg   )_batch_out_names)out_dimsr   r  s   r^   new_out_names_thunkz-_shard_map_batch.<locals>.new_out_names_thunk  s#    NHHJJ8I8IJJJr`   r   )rs  )r7   r|   to_batch_infor   r  r   	axis_data	spmd_namer   disable_vmap_shmap_errorr  r  rz   r=  r   r@   AxisDatar   batch_subtracetagr{   r8   rV  r   rb  parent_tracer   r   BatchTracerr   r~  )rK  r_  r   rb  rS   r   r   rX   r   rZ   r  in_dimsnew_in_namesusednew_sizenew_axis_datar  rZ  r  make_tracerr  r  s       ` `             @@r^   _shard_map_batchr  w  s    C 3Z@@AA'7==W===== 
L L47'4J4JL L L,?,.II8IIID*0 RS5H5H45O RPQQQE E E E),\7)C)CE E EL#t+R+R+R+R>+R+R+R'R'RRH%eo&:HeoF_``MMOM)#uy-wXX-#x000K K K K K K 10K $79#$0 0 0* e011 6 6ty5w555*55H6 6 6 6 6 6 6 6 6 6 6 6 6 6 6,e$4$<$>$>@ @ @+	[(HHJJ	/	//s   F;;F?F?c                     d t          ||          D             } Zd |D             }t          j        j        s!t	                     |z  rt          d           fdt          ||          D             }|S )Nc                6    g | ]\  fd D             S )c                L    i | ] }|t           j        uo|k    z   |         !S rh   r  r  s     r^   r   z/_batch_out_names.<locals>.<listcomp>.<dictcomp>  sE     # # # q 33?R@%) # # #r`   rh   r  s    @@r^   r   z$_batch_out_names.<locals>.<listcomp>  sZ     I I I'/ua# # # # #!# # # I I Ir`   c                J    h | ] }|                                 D ]	}|D ]}|
!S rh   r  r  s       r^   r2  z#_batch_out_names.<locals>.<setcomp>  s:    JJJ%u||~~JJrJJ!AJJJJJr`   z8vmap spmd_axis_name cannot appear in shard_map out_specsc                F    g | ]\  }}|t           j        uri ||in|S rh   r  r  s      r^   r   z$_batch_out_names.<locals>.<listcomp>  sP     > > >#b! /0x7J.J.J*R*>***> > >r`   )r   r   r  r  r  rz   )r  dimsrQ  
out_names_r  s   `    r^   r  r    s    I I36y$3G3GI I I*JJ9JJJD*0 SS5H5H45O SQRRR> > > >'*:t'<'<> > >J	r`   c
                    t          t           j        |                    \  }
}d |D             }d |D             }t          |
|f          \  }}t	          j        | j                  }t	          j        |          \  }d t          ||          D             }t                    fd            }t          |g ||R ||||	          }t	          j        ||          \  }}|                     j        |ft          |          z   |          }t           |            |          \  }}d t          ||          D             } fdt          ||          D             S )	Nc                D    g | ]}t          |          t          j        uS rh   r   rD   Zerork  s     r^   r   z"_shard_map_jvp.<locals>.<listcomp>  s%    FFF477"')FFFr`   c                L    g | ]!}t          |          t          j        ur|nd "S rg   r  rk  s     r^   r   z"_shard_map_jvp.<locals>.<listcomp>  s.    FFF477"'))aatFFFr`   c                    g | ]	\  }}||
S rh   rh   ro   r  nzs      r^   r   z"_shard_map_jvp.<locals>.<listcomp>  s!    EEEVR"EbEEEr`   rT  c                 f                 } g | d t          |                        D             R S )Nc              3  $   K   | ]\  }}||V  d S rg   rh   r  s      r^   rr   z>_shard_map_jvp.<locals>.new_out_names_thunk.<locals>.<genexpr>  s+      JJfb"rJrJJJJJJr`   )r   )out_axr   which_nz_outs    r^   r  z+_shard_map_jvp.<locals>.new_out_names_thunk  sB    _FKVKJJFLLNN(C(CJJJKKKr`   r   c                    g | ]A\  }}|8t          j        t          j        |                                                    n|BS rg   )rD   r  r   rX  to_tangent_aval)ro   r   rl  s      r^   r   z"_shard_map_jvp.<locals>.<listcomp>  sW     ; ; ;Q AB	q))99;;<<<q ; ; ;r`   c                B    g | ]\  }}t          j        ||          S rh   )rD   	JVPTracer)ro   r   rl  rK  s      r^   r   z"_shard_map_jvp.<locals>.<listcomp>  s+    	M	M	M1",ua
#
#	M	M	Mr`   )r7   r|   to_primal_tangent_pairrF   rD   jvp_subtracer  nonzero_tangent_outputsr   r8   rV  	traceablerN  r  r{   rG   )rK  r   rR   r  rS   r   r   rX   r   rZ   primalstangentswhich_nzr   r   f_jvptangent_in_namesr  rM  r   result
primal_outtangent_outr  s   `     `                @r^   _shard_map_jvpr    s   S!=wGGHH'8FFXFFF(FFXFFF(233-$
/!UY
'
'%2599%EEs8X'>'>EEE000L L L L L 10L T$Bh$B1A$B$B 3yd, , ,& L00/%&&u'9E8eDkk;QSYZZ&*88::v>>*k; ; [99; ; ;+	M	M	M	MJ0L0L	M	M	MMr`   c
           	        *+,- t           j        |          }d |D             }
t          j        |
          \  }}}t          j        ||          \  },t          |           *t          t          t          |          ||          }t          j        | j	        d          }t          |          }t          j        |g |R g |R           \  }+t                    *+fd            }t          |g ,R ||||	          }|                     j        |g|R |          } +            \  }}}}}}t!          d t#          ||          D                       }t%          |t'          |          |z
  g          \  }}|j        rJ t          j        |                       \  }} |            -t+          |||||          }*,-fdt#          ||          D             } g | R i ft'          |          z  z   g |R z   }t           j        |          }!t           j        |          }"d |D             }#t          ||||d||		          }$t          t          t.          |          ||          }% fd
|%D             }&t1          j        |j        |j                  }'t          j        g |!|"|#R |&||$|'t;          j                              }(|&D ]	})|(|)_        
t          j         ||&|          S )Nc                    g | ]	}|j         
S rh   )pvalrk  s     r^   r   z+_shard_map_partial_eval.<locals>.<listcomp>  s    &&&af&&&r`   FrT  c                                  \  } }}}}}t          j        | 	                      \  }}t          d t          | |          D                       }g |dif|z  R S )Nc              3  ,   K   | ]\  }}|d u o|d u V  d S rg   rh   ro   f1f2s      r^   rr   zC_shard_map_partial_eval.<locals>.known_out_names.<locals>.<genexpr>  s3      OOB"*+tOOOOOOr`   r   )rq  r:   sumr   )
in_fwdout_fwd
out_knownsr   rP  out_known_namesnum_res	all_namesauxr   s
          r^   known_out_namesz0_shard_map_partial_eval.<locals>.known_out_names  s    /2suu,FGZE1*:7H7HIIAOO#fg:N:NOOOOOG;_;I07:;;;r`   r   c              3  ,   K   | ]\  }}|d u o|d u V  d S rg   rh   r
  s      r^   rr   z*_shard_map_partial_eval.<locals>.<genexpr>  s3      MMfb"d
)rTzMMMMMMr`   c                F    g | ]\  }}||         n||         ndiS r  rh   )ro   r  r  r  known_in_namesknown_out_names_s      r^   r   z+_shard_map_partial_eval.<locals>.<listcomp>  sV     B B B"("b &(^~b!!')~##9~B B Br`   c                :    g | ]}|                                 |S rh   )is_knownrk  s     r^   r   z+_shard_map_partial_eval.<locals>.<listcomp>  s%    <<<1qzz||<Q<<<r`   rt  c                v    g | ]5}t          j        t           j                            |          d           6S rg   )rq  JaxprTracer
PartialValunknown)ro   r   rK  s     r^   r   z+_shard_map_partial_eval.<locals>.<listcomp>  sG     % % % r}'<'<Q'?'?FF % % %r`   )!r|   ru  rq  partition_pvalsr:   _all_mesh_names_except_spmdr   rv  trace_to_subjaxpr_nounits_fwd2r  _promote_scalar_residualspartial_eval_wrapper_nounitsr8   rV  rN  r  r  r   r;   r   r  r<   new_instantiated_constro  r   r  r  r   new_eqn_reciper   r~  recipemerge_lists).rK  r   rR   r  rS   r   r   rX   r   rZ   in_pvals	in_knownsr   	in_constsunk_in_namesin_avals_shardedf_knownr  known_paramsrw   r  r  r  r  rP  r  r  
out_constsnon_fwd_resunk_out_namesr   res	res_namesconst_tracersenv_tracersunk_arg_tracers
unk_paramsr  r  r  r  rl  r  r  r  r  s.   `     `                                   @@@@r^   _shard_map_partial_evalr9    s   %w//'&&g&&&(#%#5h#?#? )Xy!#!29h!G!G,)$66)d33\8LL'59e<<!""!09,*,,. .,'3 000< < < < < < 10< 4*<N*<*<&5%D2 2 2, 	##E$68M98M8M|\\#?Bsuu<&':0%MMFG8L8LMMMMM'&sSXX-?,@AA*k_	&z??3D3DEE-$_&&67Iz;GG#B B B B B B,/,@,@B B B) 9"S!114Dl4D4DD,e2C88-E)3//+<<<<</+5E#$0 0 0* '-..?PQQ)% % % %#% % %+		't	G	G$
JMJKJ/JJ%{J 0 8 : :	< 	<# &&a3	
K	<	<<r`   c                     | |i |\  }\  }}}}}d t          |||j                  D             }	t          ||	          }d |D             }||||||ffS )Nc                @    g | ]\  }}}|d u o|d u o|j         j         S rg   r   r   )ro   r  r  r  s       r^   r   z-_promote_scalar_residuals.<locals>.<listcomp>  sM     E E Er2q :9"*9QV\)9 E E Er`   c                t    g | ]5}t          |d d          s t          j                            |d          n|6S )r   rh   )r   )r  rq  r(   	broadcastr  s     r^   r   z-_promote_scalar_residuals.<locals>.<listcomp>  sP     % % % 3:!Wb2I2IP!!!T***q % % %r`   )r   r  _promote_scalar_residuals_jaxpr)
rR   r   rN  rP  in_fwdsout_fwds	out_pvalsr0  r  whichs
             r^   r#  r#    s    ;<1d;Mf;M;M8%	8'8Y
CE E5?CCE E E%
)%
7
7%% %#% % %*	(Iz3?	??r`   c                     t           j         fd            }d t           j                  D             }g |d  j        D             }t          j        ||          \   }}\    S )Nc                     t          | t          j                  g          \  }}d t          |          D             }t	          j        |g|R  S )Nc                :    g | ]\  }}|rt          |          n|S rh   r  ro   rj   ws      r^   r   z@_promote_scalar_residuals_jaxpr.<locals>.fun.<locals>.<listcomp>  s.    
E
E
ETQ(>!q
E
E
Er`   )r;   r   r  r   r   rY  )res_and_argsr3  r   rP  rC  s      r^   r   z,_promote_scalar_residuals_jaxpr.<locals>.fun  sX    <#eo*>*>)?@@IC
E
ESe__
E
E
EC?5#-----r`   c                ^    g | ]*\  }}|rt          j        d dd|j                  n|j        +S r   Nr   r   unmapped_avalr   )ro   r  rI  s      r^   r   z3_promote_scalar_residuals_jaxpr.<locals>.<listcomp>	  sK     8 8 81a :;Ft!!T1af555 8 8 8r`   c                    g | ]	}|j         
S rh   rj  r  s     r^   r   z3_promote_scalar_residuals_jaxpr.<locals>.<listcomp>  s    999qAF999r`   )r   r   r   r  r  rq  ry  )rP  rC  r   	res_avalsr   r   s   ``    r^   r?  r?    s    <. . . . . <.8 8u668 8 8):y:99EL999:(-c8<</%Ar	,r`   c                |    d |                                 D             |z  fdt          |           D             S )Nc                    h | ]	}|D ]}|
S rh   rh   r1  s      r^   r2  z _unmentioned2.<locals>.<setcomp>  r3  r`   c                    g | ]}|v|	S rh   rh   r5  s     r^   r   z!_unmentioned2.<locals>.<listcomp>  s#    	L	L	L!8:K:K!:K:K:Kr`   )r
  r!  )rS   r   rZ   r6  s      @r^   _unmentioned2rT    sH     65%,,..555<(	L	L	L	L066	L	L	LLr`   c          
     2   d fdt          ||           D             } fdt          |          D             }t          | |f          \  }	}
t          j        fd            }t	          j        |          \  }t          ||
          \  }}d t          ||           D             d t          |          D             z   }fd}t          j        |g|	R t          |          ||d}t           |            |          S )	Nc                    |dk    r| |z  n| S r  rh   )rj   ys     r^   rk   z&_shard_map_transpose.<locals>.<lambda>  s    aAQ r`   c                h   g | ]\  }}t          |          t          j        u r(t          j        t          ||j                            nes"t          j        |          t
          j        k    r|n? |t          t          j
        j        t          |                                        S rh   )r   rD   r  rv  r   r   r  float0r   r|   r   r  rT  )ro   r   rj   rZ   mb_divrS   r   s      r^   r   z(_shard_map_transpose.<locals>.<listcomp>  s     , , , "a 8<Aww"'7I7IRW[r1622333 O6<??fm;;116!T#djnmD"d.K.KLLMMNN, , ,r`   c           	         g | ]J\  }}t          |          t          j        ur|n't          j        t          ||j                            KS rh   )r   rD   UndefinedPrimalrv  r   )ro   r   rj   rS   s      r^   r   z(_shard_map_transpose.<locals>.<listcomp>  s^     
, 
, 
,b! Awwb000!!

[r16::
;
;
, 
, 
,r`   c                   t          t          t          j        |          |          \  }}t	          j        t	          j                  t          t          j        |          d          \  }}}} t          j        |          | }t          j	        |j
        ddg ||R |           }	fdt          
|          D             }|S )NFrh   c                   g | ]\  }}t          |          t          j        u r(t          j        t          ||j                            n?r|n;t
          j                            |t          t          |                              S rh   )
r   rD   r  ro  r   rq  r(   psumr{   rT  )ro   r   rj   rZ   rS   r   s      r^   r   z;_shard_map_transpose.<locals>.fun_trans.<locals>.<listcomp>-  s     ) ) ) B 8<Aww"'7I7I27=r1622333 CQQW\\!U=r4#@#@AABB) ) )r`   )r:   r|   rD   is_undefined_primalrq  partial_eval_jaxpr_nounitsrV  r   jaxpr_as_funbackward_passrP  r   )out_ctsr   r3  undefsjaxpr_knownjaxpr_unknownr   res_reshapedrw   rZ   r   rP  rS   r   s            r^   	fun_transz'_shard_map_transpose.<locals>.fun_trans$  s     R%;T!B!BDIIKC')'D
us2#94@@%(I (I$K114$[1137L

UB(@,(@(@(@' C) ) ) ) ) ) 3'') ) )C Jr`   c                N    g | ]"\  }}t          |          t          j        u |#S rh   r  ro   r   rj   s      r^   r   z(_shard_map_transpose.<locals>.<listcomp>7  s.    GGGTQQrw0F0Fq0F0F0Fr`   c                N    g | ]"\  }}t          |          t          j        u |#S rh   )r   rD   r\  rk  s      r^   r   z(_shard_map_transpose.<locals>.<listcomp>8  s/    NNNTQDGG2;M,M,Mq,M,M,Mr`   c                 b    t          d t                                  D                       S )Nc              3  $   K   | ]\  }}||V  d S rg   rh   )ro   r   r  s      r^   rr   zD_shard_map_transpose.<locals>.new_out_names_thunk.<locals>.<genexpr>;  s+      JJ95"rJJJJJJJr`   )r{   r   )r   
nz_arg_ctss   r^   r  z1_shard_map_transpose.<locals>.new_out_names_thunk:  s0    JJHjjll(C(CJJJJJJr`   r   )r   rF   r   r   rD   nonzero_outputsr=   r   r   r{   rG   )rd  rP  rS   r   rQ  rX   r   rZ   r   all_argsr   ri  fun_trans_flatr   r  r  r   rZ  ro  s    ```  ``         @@r^   _shard_map_transposers    s   ..&, , , , , , , y'**, , ,'
, 
, 
, 
,8T**
, 
, 
,$ #GT?33(G<        < ,Y77)Z1)WEE.( HGSG,,GGGNNS4((NNNO K K K K K K  &*U<5H5H)Y  ( 


H	-	--r`   saveableCallable[..., pe.RematCases_]unks_inSequence[bool]inst_inr  core.JaxprEqnStuple[core.JaxprEqn, core.JaxprEqn, Sequence[bool], Sequence[bool], list[core.Var]]c                ,   |j         d         |j         d         c}t          j        j                                                  5  t          j        |||dd|           \  }}}}}	d d d            n# 1 swxY w Y   t          |j                  |	z
  }
t          j	        |          |
d          }t          |j        |
g          \  }}d t          |          D             fd|D             }d t          ||          D             }t          j        |j         d         j                                                  5  t          j        |dg|
z  |z             }t          |||          \  }}d d d            n# 1 swxY w Y   t          j        |j                  }t          j        |j                  }t#          ||j                  \  }}t#          ||j                  \  }t#          ||j                  \  }}t#          ||j                  \  }}t          j                    t)          ||t+          t,          j        |          ||||t1          |j         |          t1          |j         |          	  	        \  }}fd	t          |j        d |	         |          D             }t          j        |g ||j        ||j        |j                  }t;          ||||          }t          j        g ||||j        ||j        |j                  }t          |j                  t          |j                  k    sJ d
 t          |j        |          D             }|fdd |D             D             z  }||||||z   fS )NrP  rS   Fc                4    i | ]\  }}t          |          |S rh   )id)ro   rp   r  s      r^   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<dictcomp>R  s$    666$!QRUUA666r`   c                T    g | ]$}                     t          |                    %S rh   )r  r}  )ro   r  idx_maps     r^   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<listcomp>S  s+    222AW[[A222r`   c                $    g | ]\  }}|d u o|d u S rg   rh   r
  s      r^   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<listcomp>T  s)    
H
H
HR2:$"*
H
H
Hr`   T)rP  c           	     \    g | ](\  }}| t          d i|j                            )S rV  )ro  r   )ro   varrI  r  rS   newvars      r^   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<listcomp>b  sT     M M M3!MvvmD1i.#(CCDD M M Mr`   c                R    g | ]$\  }}t          |          t          j        u  |"|%S rh   r  )ro   rj   insts      r^   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<listcomp>l  s=     3 3 3GAtaDH$$T$ $$$r`   c                     g | ]
}|         S rh   rh   )ro   rR   out_binders_knowns     r^   r   z3_partial_eval_jaxpr_custom_rule.<locals>.<listcomp>n  s    SSS #SSSr`   c                    h | ]}||S rg   rh   r  s     r^   r2  z2_partial_eval_jaxpr_custom_rule.<locals>.<setcomp>n  s    -R-R-RAAMaMMMr`   )rM  r   rw  r   r   rq  partial_eval_jaxpr_customr   r  _jaxpr_forwardingr;   r   r   prune_jaxpr_outputs_add_reshapesremove_named_axis_effectsr   r:   r  gensym_pe_custom_paramsr|   rz  not_rV  r  r  r  rs  r<   ) rt  rv  rx  r  rP  rf  jaxpr_stagedunks_outinst_outr  num_out_primalsr  out_varsres_varsr  rC  	ins_knownr   
ins_stagedout_binders_stagedparams_knownparams_staged	residuals	eqn_knownfull_res
eqn_stagednew_instr  r  rS   r  r  s                               @@@@@r^   _partial_eval_jaxpr_custom_ruler  F  sw   
 
7#SZ%7+%tz//1122 V V
$UGWeUHUU ;Kx7V V V V V V V V V V V V V V V +,,w6/,,_-=-=>&!+"57HII(H66)H"5"5666'2222222'
H
H3vw3G3G
H
H
H%sz&17==??@@ P P(tf6NQV6VWWK -e[, O OKP P P P P P P P P P P P P P P .{DOLL+/doNN,44,)Q'#+>>Q #*55-!Z(3;??!;==&+<wBGX..&'5
3:[)))4
,+O+O+O,Q ,Q(,yM M M M M M !4XgX!>FFM M M)y*J,=*J	*J"}lK<O"0 0) 4EyQQ( 8( 8Z 8:L #} , 4coG G* 
Z		3|':#;#;	;	;	;	;3 3s3:w77 3 3 3(
SSSS-R-R-R-R-RSSSS(	J(Hy4H	HHs$    A88A<?A<1FF
Fc           	     p    d t           j        d t                                        D             t                    sfS j        sj        rJ t
          j         fd            }d j        D             }t          j        ||          \  }\   \   t
          j         fd            }d t          j        d t                                        D             }g |d j        t                     d          D             }t          j        ||          \  }\   \   fS )Nc                2    g | ]\  }}|o|j         j         S rh   r<  ro   rI  r  s      r^   r   z!_add_reshapes.<locals>.<listcomp>u  s>     F F F1 "afl" F F Fr`   c                     t          j        dg| R  }t          |t          |          t	                    z
  g          \  }}d |D             }g ||S )Nrh   c                >    g | ]}|j         st          |          n|S rh   )r   rw  r  s     r^   r   z0_add_reshapes.<locals>.known.<locals>.<listcomp>~  s,    
@
@
@qAG2>!
@
@
@r`   )r   rY  r;   r   r  )r   rw   	out_knownr3  rf  rC  s       r^   knownz_add_reshapes.<locals>.knownz  sf    
/+r
1D
1
1
1Cc#hhU&;%<==NIs
@
@C
@
@
@CYr`   c                    g | ]	}|j         
S rh   rj  r  s     r^   r   z!_add_reshapes.<locals>.<listcomp>  s    111af111r`   c                     t          | t                    g          \  }}d t          |          D             }t          j        dg||R  S )Nc                :    g | ]\  }}|rt          |          n|S rh   rG  rH  s      r^   r   z1_add_reshapes.<locals>.staged.<locals>.<listcomp>  s.    
G
G
GTQ(>!q
G
G
Gr`   rh   )r;   r   r   r   rY  )r   res_insr3  r  rC  which_s       r^   stagedz_add_reshapes.<locals>.staged  sZ    4#e**..ID#
G
GSv5F5F
G
G
GC?<8c8C8888r`   c                ^    g | ]*\  }}|rt          j        d dd|j                  n|j        +S rL  rM  r  s      r^   r   z!_add_reshapes.<locals>.<listcomp>  sQ     J J J1a :;Ft!!T1af555 J J Jr`   c                    g | ]	}|j         
S rh   rj  r  s     r^   r   z!_add_reshapes.<locals>.<listcomp>  s    MMMqAFMMMr`   )	r   r  r   r   r  r   r   rq  ry  )	rC  rf  r  r  r  r   r  rP  r  s	   ```     @r^   r  r  s  s   F FE<#6{E

{#CDDF F F&	V6[,66"A<+AAA	A<     <
 21k0111(4UHEE+q"b<9 9 9 9 9 9 <9J Jv|':;CJJ;'GHHJ J J)NyNMML,?E

,LMMMN(5fhGG,2r	l	""r`   c	                0   |d         }	t          |	          t          | |d                   \  }
t          ||d                   \  }
digt          |          z  z   t          |t	                    t	                              }t          ||d                   \  }
}fdt          ||          D             }||z   }t          ||d                   \  }
}t          |t	          |          t	          |          d          }||fS )	NrS   r   rQ  r   )r   rQ  c                F    g | ]\  }}||         n||         ndiS r  rh   )ro   r  r  r  in_names_knownout_names_knowns      r^   r   z%_pe_custom_params.<locals>.<listcomp>  sU     B B B"("b &(^~b!!&(nr""9~B B Br`   F)r   rQ  rX   )r!  r:   r  rV  r{   r   )rv  rx  kept_outs_knownkept_outs_stagedr  r  rC  r  r  rS   r   new_params_knownr  r4  out_names_stagednew_params_stagedr  r  r  s                   @@@r^   r  r    s_    
f	$)$//)$Wl:.FGG.!%o|K7PQQ!_#9~&6U&CC/,~1F1F$)/$:$:< < < &g}Z/HII!_B B B B B B,/,@,@B B B) //&'7{9STT!=53I3I%*+;%<%<O O O	,i	77r`   tuple[AxisName, ...]c                t    t          j                    j        t          fd| j        D                       S )Nc              3  $   K   | ]
}|v|V  d S rg   rh   )ro   r   
spmd_namess     r^   rr   z._all_mesh_names_except_spmd.<locals>.<genexpr>  s-      JJ4z3I3It3I3I3I3IJJr`   )r   get_axis_envspmd_axis_namesr{   r   )rS   rK  r  s     @r^   r!  r!    s9     ""2*	JJJJJJJ	J	JJr`   used_outputs
list[bool]'tuple[list[bool], core.JaxprEqn | None]c                   t          |           s.t          j        |          sdgt          |j                  z  d fS |j        d         }t          j        |j        	                                          5  t          j
        |j        d         |           \  }}d d d            n# 1 swxY w Y   t          |          st          |           s|j        s|d fS t          ||j        d                   \  }}t          | |j        d                   \  }}t          |j        |t          |          t          |                    }t          j        |j        |j                  }	t          j        d t%          |j        |          D             d t%          |j        |           D             |j        ||	|j                  }
||
fS )	NFrS   rP  r   rQ  )rP  r   rQ  c                    g | ]	\  }}||
S rh   rh   )ro   r  r  s      r^   r   z"_shard_map_dce.<locals>.<listcomp>  s!    ???wq$$????r`   c                    g | ]	\  }}||
S rh   rh   )ro   rj   r  s      r^   r   z"_shard_map_dce.<locals>.<listcomp>  s!    AAAwq$DAAAAr`   )r   rq  has_effectsr   r  rM  r   rw  r   r   	dce_jaxprr  r:   rV  r{   r  r   r  r   r  r  rs  )r  r  rS   rP  used_inputsr   r   rQ  rZ  r  new_eqns              r^   _shard_map_dcer    s   	\		 +2>##6#6 +7S__$d**	F	$tz//1122 I Icj&9<HHE;I I I I I I I I I I I I I I I	[		  #l"3"3  EM   cj.DEEKAx!,
;0GHHLAycjh %i 0 02 2 2J)%-IID??#cj+66???AA#ck<88AAAz4: :G s   6$B&&B*-B*rh   )in_axesout_axesstatic_broadcasted_argnumsr  backendr  donate_argnumsglobal_arg_shapesc          	          t                    nt           ||          \   f	dfd}
fd}||
_        |
S )Nc                   	 t          | |
  
        }|j        D ]}t          j        |           t	          t          |          
f          }t          |j        ||j        |j	        
          \  }}}t          |j        |t          |                    }t          j        |d t          |j                  D                       }	|	||j        ||fS )Nc              3  $   K   | ]\  }}||V  d S rg   rh   )ro   rp   r  s      r^   rr   z-pmap.<locals>.infer_params.<locals>.<genexpr>  s+      KKfasKKKKKKKr`   )r  )r'   	flat_argsr   	check_argr"   _get_devices_cached_shard_mapflat_funin_axes_flatout_axes_thunkrP   r  rq  rs  r   donated_invarsr   )r   rN  r   argrS   _pmappedrU   rW   flat_global_argsjitted_fr  r  r  r  donate_tuplerR   r  r  static_broadcasted_tuples             r^   infer_paramszpmap.<locals>.infer_params  s   a(,D"GWiv	O 	OA{  Q((9,77D$5	
D!.!*:I%G %G!Hh	7	T4>>+ +wKK	!2B(C(CKKKM M MH %qz4BBr`   c                      | i |\  }}}}} || }t          || |                      }t           |            |          S rg   )rQ   rG   )	r   rN  r  r  r   rS   rW   r\  r  s	           r^   r   zpmap.<locals>.wrapped  s`    ,///X48%&D+D$		DDD((**d+++r`   c                 :     | i |\  }}}}} |j         | i |S rg   )lower)r   rN  r  r   r  s       r^   r  zpmap.<locals>.lower  s9    '<888HaAq8>4*6***r`   )r{   r&   r  )rR   r  r  r  r  r  r  r  r  r  r   r  r  r  r  s   ```` ```    @@@r^   pmapr    s     &1E'NNNw'6G.7S 7S3)%|C C C C C C C C C C C C C, , , , ,+ + + + + '-	.r`   c           	         t          t          t          t                    |                    }fd}t	          | |          }t          |j        |||dt                                ||fS )Nc                 Z    t          t          t                                            S rg   )r|   r   _axis_to_spec)r  r  s   r^   rk   z#_cached_shard_map.<locals>.<lambda>  s#    c'-;;^^=M=MNN r`   F)rX   rZ   )r{   r|   r   r  _handle_reshapesr]   rc  rf  )r  rS   r  r  r  rU   rW   r   s      ``   r^   r  r    s|    3w}i88,GGHH(NNNNN)<@@#
S%tXy$9;;8 8 8
I
 r`   c                    t          d t          |          t          |                    } | | }t          d t          |          t           |                                S )Nc                6    || nt          j        | |          S N)r  )jnpsqueezerj   r  s     r^   rk   z"_handle_reshapes.<locals>.<lambda>  s    RZS[5L5L5L r`   c                6    || nt          j        | |          S r  )r  expand_dimsr  s     r^   rk   z"_handle_reshapes.<locals>.<lambda>  s    RZS_QR5P5P5P r`   )rE   r  )rR   r  r  r   rN  rw   s         r^   r  r    sf    	LLt**d7mm
- 
-$	4#	PPs))T.."2"233
5 
5 5r`   c                    t          |t                    rd g|z  | gz   }t          | S |t                      S t          |          rg   )r   r  r   r   )r  r  r   s      r^   r  r    sJ    C FRK9+%Ee9	z33J
B--r`   c                    || j         t          j         |          }n!| j         t          j                     n| j         }t          j                    dk    r|d | j                 S |d | j                 S )N)r  r   )r  rq  process_countglobal_axis_sizelocal_axis_size)r   r  devss      r^   r  r    sw    QY.;w'''DDI-3;===19D1###$$	 q  	!!r`   c                  T    e Zd ZU ded<   ded<   d Zedd            Zd	 ZddZeZ	dS )RewriteTracerr  r&  Valr  c                0    || _         || _        || _        d S rg   r  r  s       r^   r  zRewriteTracer.__init__  r  r`   r   r  c                4    t          j        | j                  S rg   )r   rX  r  r  s    r^   r   zRewriteTracer.aval  s    ="""r`   c                4    t          j        | j                  S rg   )r   r  r  r  s    r^   r  zRewriteTracer.to_concrete_value  s    !$(+++r`   r   c                *    t          | j                  S rg   )r   r  r  s    r^   r  zRewriteTracer.__str__"  s    tx==r`   N)r   r  r  r  rh   r`   r^   r  r    s|         
(((  
 # # # 8#, , ,   (((r`   r  c                  R    e Zd ZU ded<   ded<   ded<   d Zd Zd	 Zd
 Zd Zd Z	dS )RewriteTracez
core.Tracer  zcore.TraceTagr  r"   rS   c                0    || _         || _        || _        d S rg   r  r  rS   )rJ  r  r  rS   s       r^   r  zRewriteTrace.__init__+  s    $DDHDIIIr`   c                    t          |t                    r!|j        j        | j        u r|j        |j        fS |t          | j        j                  fS rg   )	r   r  r  r  r  r&  r  rS   r   )rJ  r  s     r^   rd  zRewriteTrace.to_val_rep_pair0  sK    #}%% ,#*.DH*D*DWcg#di*++++r`   c                   t                               |t          t          |                    }t	          t          | j        |                    \  }}t          j        | j	                  5   || j
        |g|R i |\  }}d d d            n# 1 swxY w Y   t          t          t          |           ||          }	|j        r|	n|	d         S r  )r  r  r   r  r7   r|   rd  r   rb  r  rS   r  r]  )
rJ  r_  rb  rM  r  r  in_repsr  out_repsr  s
             r^   r  zRewriteTrace.process_primitive7  s   dGM4$@$@AADc$"6
CCDDGW		 1	2	2 H H4	7GWGGGGGhH H H H H H H H H H H H H H HgmT22HhGGK/C;;[^Cs   -BBBc                   t          t          | j        |                    \  }}t          || j        | j        t          |                    \  }}t          j        | j	                  5   |j
        |g|R i |}d d d            n# 1 swxY w Y   t          t          t          |            |            |          S rg   )r7   r|   rd  _rewrite_subtracer  rS   r{   r   rb  r  r   r   r  )	rJ  r  rR   rb  rM  r  r  r  r  s	            r^   r  zRewriteTrace.process_call?  s    c$"6
CCDDGW#AtxE'NNKKKAx		 1	2	2 < <$$Q;;;;F;;h< < < < < < < < < < < < < < <w}d++XXZZBBBs   +B		BBc               p   |rd}t          |          t          t          | j        |                    \  }}t	          || j        | j        |          \  }}	t	          || j        | j        |dz            \  }}
t          j        | j	                  5   |j
        ||g|R d|i}d d d            n# 1 swxY w Y   t          j        |	|
          \  }}|s=||d t          |          dz           dz  k    sJ |d t          |          dz           }t          t          t          |           ||          S )Nr  r   r  )r  r7   r|   rd  r
  r  rS   r   rb  r  r   r   merge_linear_auxr   r   r  )rJ  r_  r   r  r  r  r   r  r  	out_reps1	out_reps2r  fstr  s                 r^   r  z$RewriteTrace.process_custom_jvp_callF  s    %c  $$$c$"6@@AAGW&sDHdiIINC&sDHdi1MMNC		 1	2	2 N N3MgMMMnMMhN N N N N N N N N N N N N N N'	9==MC /"53x==A#5"56:::::-3x==A--.hw}d++Xx@@@s   B22B69B6c                    |rd}t          |          t          t           j        |                    \  }	}
t	          | j         j        |
          \  }} fd|
D             }t	          | j         j        |          \  }}t          | j        ||
          }t          j	         j
                  5   |j        |||g|	R ||d}d d d            n# 1 swxY w Y   t          j        ||          \  }}|s& |            \  }}t          ||j        g          \  }}t          t!          t"                     ||          S )Nr  c                P    g | ]"}|t          j        j                  fD ]}|#S rh   )r  rS   r   )ro   r  r_rJ  s      r^   r   z8RewriteTrace.process_custom_vjp_call.<locals>.<listcomp>`  s8    PPP!!S9M5N5N1OPP22PPPPr`   )r  r  )r  r7   r|   rd  r
  r  rS   r  r   rb  r  r   r   r  r;   ry   r   r  )rJ  r_  r   r  r  r  r  r  r   r  r  r  fwd_in_repsr  r  r  r  r   res_trees   `                  r^   r  z$RewriteTrace.process_custom_vjp_callW  s    %c  $$$c$"6@@AAGW&sDHdiIINCPPPPwPPPK&sDHdiMMNC
sDIy'
:
:C		 1	2	2 9 93S 97 9 9i)79 9 9h9 9 9 9 9 9 9 9 9 9 9 9 9 9 9 '	9==MC @IKKkax(*=)>??kaw}d++Xx@@@s   6CCCN)
r   r   r   r  r  rd  r  r  r  r  rh   r`   r^   r  r  &  s         ***  
, , ,D D DC C CA A A"A A A A Ar`   r  c                    t          t          t                    |          }fd}t          | |          \  } }t	          | ||          S )Nc                 0     fd             D             S )Nc                J    g | ]}t          t          |                     S rh   )r  r'  r   s     r^   r   zB_efficient_transpose_rewrite.<locals>.<lambda>.<locals>.<listcomp>n  s+    PPP#l43344PPPr`   rh   )rS   r   s   r^   rk   z._efficient_transpose_rewrite.<locals>.<lambda>n  s$    PPPPoo>O>OPPP r`   )r|   r   r{  $_efficient_transpose_rewrite_nomatch
_match_rep)r   rS   r   r   r  out_reps_dstout_reps_srcs    ` `   r^   r   r   l  sY    ($//::'PPPPP,:3gNN#|	C|\	:	::r`   c                   t          j                    5 }t          j                    }t          |||          }t	          t          t          |          ||          }t          j        |          5   | | }	d d d            n# 1 swxY w Y   t          t	          |j	        |	                    \  }
}~~~	d d d            n# 1 swxY w Y   |
                    |           |
S )Nr  )r   take_current_traceTraceTagr  r|   r   r  rb  r7   rd  store)rR   r  rS   r  r   parentr  rl  rb  rf  r  r  s               r^   r  r  r  sM      F
-//CF#DAAAAW]A..>>J			"	"  AzNc              A$5s ; ;<<Hh	:s               ++h	/s6   AC2B8CB	CB	+CC
Cc                     | | }t          |          r
 |            n|}t          |          r
 |            n|}t          |||           d t          |||          D             }|S )Nc           	     x    g | ]6\  }}|z
  r)t          |t          fd |D                                 n|7S )c              3  $   K   | ]
}|v|V  d S rg   rh   rw  s     r^   rr   z(_match_rep.<locals>.<listcomp>.<genexpr>  rx  r`   r  ry  s      @r^   r   z_match_rep.<locals>.<listcomp>  so     
K 
K 
K QS CF) *Q=========
>
>
>
K 
K 
Kr`   )rt   r  r   )rR   rS   out_reps_src_out_reps_dst_r   r\  r  r  s           r^   r  r    s    	
D$$,]$;$;N,$,]$;$;N,t\<000
K 
K$'lL$I$I
K 
K 
K$	+r`   core.ClosedJaxprSequence[set[AxisName]]out_rep_dstc                &   t          j        t          t          j        |j        |j                            }t          || |          \  }}t          || ||          }t          j
        ||j                  \  }}}\   t          j        ||          S rg   )r   r   r   r   rY  rP  r  r  r  rq  ry  r   r  )	rS   rP  r  r(  rR   r  rO  r   r  s	            r^   r|  r|    s}     	l74?EKFFGG!3AtVDD*!WD';//!3Au~FF&!VR		&&	)	))r`   ,tuple[core.ClosedJaxpr, list[set[AxisName]]]c                   t          j        t          t          j        |j        |j                            }t          || |          \  }}t          j	        ||j
                  \  }}}\   t          j        ||           |            fS rg   )r   r   r   r   rY  rP  r  r  rq  ry  r   r  )rS   rP  r  rR   r  rO  r   r  s           r^   rK  rK    su    
 	l74?EKFFGG!3AtVDD*!W3Au~FF&!VR		&&	)	)7799	44r`   c                ,   t          j                    5 }t          |          t          |          k    s&J t          |          t          |          f            t          |||          }t	          t          t          |          ||          }t          j        |          5   | | }	d d d            n# 1 swxY w Y   t          t	          |j	        |	                    \  }
}|
                    |           |
cd d d            S # 1 swxY w Y   d S rg   )r   r  r   r  r|   r   r  rb  r7   rd  r  )rR   r  r  rS   r  r  r  rl  rb  r\  r  r  s               r^   r
  r
    sy      Lw<<3w<<'''#g,,G)E'''\3--AW]A..AAJ			"	"  Q
^d              A$5t < <==Hh	KK                 s6   BD	$B6*D	6B:	:D	=B:	>>D		DDc                      fd}|S )Nc                     t          j                    }t          t          j                  |                       \  }} |j        |  }t          t           |            |          S rg   )r   r  r
  r   r   rc  r|   _match_replication)	r   r  r  
reps_thunkrw   r  r  rS   reps_dsts	        r^   new_bwdz_rewrite_bwd.<locals>.new_bwd  s`    
-//C(c):):CwwyyQQD*
$
T
"C!::<<3???r`   rh   )r  rS   r  r1  r2  s   ```` r^   r  r    s=    @ @ @ @ @ @ @ @
 
.r`   c                      z
  r8t                               |t           fdD                       d           \  } z
  r)t          |t          fd D                                 }|S )Nc              3  $   K   | ]
}|v|V  d S rg   rh   )ro   r   rm  s     r^   rr   z%_match_replication.<locals>.<genexpr>  s'      #C#C!asllAllll#C#Cr`   r  c              3  $   K   | ]
}|v|V  d S rg   rh   rw  s     r^   rr   z%_match_replication.<locals>.<genexpr>  s'      ;;!asllAllll;;r`   )r  r   r{   r  )rm  r(  rj   s   `` r^   r/  r/    s    3Y .	ae#C#C#C#Cs#C#C#CCC(, 
 
. 
.BA3Y =1e;;;;s;;;;;<<A	
(r`   	jax.Arrayc                    t          | t                    r| j        S t          | t          j                  rt          | j                  S t          dt          t          |                     z            )zEFor a jax.Array, return what axes it is known to be replicated along.z!get_replication not defined on %s)
r   r  r&  r@   r  get_replicationr  rz   r   r   ri   s    r^   r8  r8    sb     =!! 5L8'(( "15!!!6d1ggFGGGr`   )rR   r   rS   rT   rU   rV   rW   rV   rX   rY   rZ   r[   )rR   r   rS   rT   rU   rV   rW   ra   rX   rY   rZ   r[   )r   r   r   r   )r   rv   r   r	   r   r   )rR   r   rS   r"   r   rM   rU   rV   r   r   r   r   r   r   r   r   )r   rM   r   r   r   r   r   r   )r   rv   rR   r   r   rM   r   rV   r   r   r   r   )rR   r   rS   r"   r   rM   r   rV   r   r   r   r   )rR   r   rS   r"   r   rM   r   rV   r   r"  r   r   )rS   r"   r   r   r   r.  )r   rM   r   rV   r   r;  r   r<  )rK  r^  r_  r`  rR   ra  rb  rc  rS   r"   r   rd  r   re  rX   rY   r   rY   rZ   rf  r   rg  )r   r  r   r  )rS   r"   r   r   r   r  r   r  )rS   r"   r   r   r   r  )rS   r"   rP  r  r  r  r   r  )rS   r"   r&  r  r(  r   r   rY   )r_  r`  )r  r  r   r  )r  r  )r   r   r   r  r   r   )r   r   r   r   )rS   r"   rm  r   rj   rn  r   rn  )r   ry  rz  r{  r   r   )
rS   r"   rX   rY   r  r   rj   rn  r   rn  )r   r  r  r  )rK  r  r_  r`  r   ra  rb  r  rS   r"   r   rd  r   re  rX   rY   r   rY   rZ   rf  r   r  )rS   r"   r   r   rZ   r[   r   r.  )
rt  ru  rv  rw  rx  rw  r  ry  r   rz  rg   )rS   r"   r   r  )r  r  r  ry  r   r  )
rS   r"   rP  r&  r  r'  r(  r'  r   r&  )rS   r"   rP  r&  r  r'  r   r*  )rj   r6  r   r  (X  
__future__r   collections.abcr   r   r   enum	functoolsr   r  	itertoolsitmathr   operatorrz  typingr	   r
   r   numpyr  rq  	jax.numpyr  jax.shardingr   r   jax._srcr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   jax._src.corer    jax._src.meshr!   r"   r#   r$   r%   jax._src.apir&   r'   jax._src.laxr(   r)   lax_parallelr*   r+   r,   r-   r.   r/   r0   r1   jax._src.extendr2   jax._src.lib.mlirr3   jax._src.lib.mlir.dialectsr4   jax._src.utilr5   r6   r7   r8   r9   r:   r;   r<   jax.api_utilr=   r>   r?   jax._src.interpretersr@   rA   rB   rq  rC   rD   jax.tree_utilrE   rF   rG   rH   rI   rJ   jax._src.tree_utilrK   rL   rM   rN   rO    jax.experimental.multihost_utilsrP   rQ   r   safe_mapr|   
unsafe_mapsafe_zipr   
unsafe_zipregister_exclusion__file__rV   AxisNamer   rf  r_   r]   rV  r  r{   r   r}   Enumrv   ru   r   r   r   r   r   r   r   r'  r  r:  r  rn  MaybeTracer	PrimitiverG  r   r  DynamicJaxprTracerI  rn  rv  ro  r  r  r  r  r  r  r  r  custom_typechecksr{  r|  r  r  r  r%  r5  register_loweringr  r-  r2  r0  rS  r^  	EvalTracerZ  r  rY  rt  rz  r  r   r}  r   r  r[  r  r  rw  Tracer`  ra  r  r  r  r  debug_callback_pr  device_put_pr  r]  def_implpsum_pimpldef_effectful_abstract_evalabstract_eval
_lowerings_batched_reduction_collectivefancy_primitive_batchers_names_in_paramskippable_batchersr  
deflinear2r  r  def_abstract_evalr  primitive_batchersr  r  register_standard_rewriteregister_norewriter  r  register_standard_checkr  r  r  r  r  r  r  r  r  chain__dict__r
  or   r}  cumsum_pcumlogsumexp_p	cumprod_pcummax_pcummin_pr   r.  r4  rA  rF  all_gather_pall_to_all_p
ppermute_preduce_scatter_ppmin_ppmax_paxis_index_prH  rL  rP  rT  rW  rX  r[  call_pr_  ra  pure_callback_prf  io_callback_pri  rk  custom_lin_prm  r~  rs  r  r  r  r  r  platform_index_pr  r  r  r  custom_jvp_call_pr  r  r  r  r  r  r  r  r  tie_pr  r  
BatchTracer  r  JVPTracer9  
JaxprTracetransformation2r#  r?  rT  rs  primitive_transposesr  partial_eval_jaxpr_custom_rulesr  r  r!  r  	dce_rulesr  cacher  r  r  r  r  r  r  r   transformation_with_aux2r  r  r|  rK  r
  r  r/  r8  rh   r`   r^   <module>r     s"   # " " " " " " 8 8 8 8 8 8 8 8 8 8                       & & & & & & & & & &     



       5 5 5 5 5 5 5 5 " " " " " "                         ' ' ' ' ' '                   & & & & & &                         # # # # # # % % % % % % # # # # # #                  . . . . . . . . . . . . . . 9 9 9 9 9 9 9 96 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6 6                    * * * * * *3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 S R R R R R R R R R * * * * * * & & & & & & 4 4 4 4 4 4 & & & & & & $ $ $ $ $ $@ @ @ @ @ @ @ @ @ @ @ @ @ @ @ @= = = = = = = = = = = = = =P P P P P P P P -Z-Z ! !( + + + 	 26*3)++6C 6C 6C 6C 6CpG G G GT eHcM**+	    	/GU+;<<&? &? &? &?P        
&((   (   * * * *X- - - -^" " " "H; ; ; ;
   GCLL	9 	9 	9 	9 GVO$                 ,,& & & &N *<  &   D D D DF F F F= = = = .A ) *
= 
= 
= 
= 0E 4+ , Ht#
$  ( '; { #I I I I" " " "0C C C C7 7 7 7   .' .' .'b( ( (.  {$7 8 8 8     W W W W 6 6 6 6      "B B B $3    F F F F
K K K KJ J J
A A A A
 " ! ! ! ! ! ! !@ @ @ !        	      @ @ @
D D D DL L L
 5 4 4 4 4 4DA DA DA DA DADJ DA DA DAN    T[   >: : : /1 0 0 0 0    +EI& '   &<H! "
 $.
!
!    $) * * *  # #L$7$E F F F  w0C D E E E	',
4g
,
,. . 	 !' * (/w|/KV'T'T G $Q Q Q g, - - -% % % t~l++ $    AA B B B   JJ K K K  |%O%O P P P  
 -@ L ) lQ QR R R 24 3 3 3 3LL K K K K  02 1 1 1 1NNNN     	) 	) 	)F F FG G G	 	 	, , ,
 
 
' ' ' 
#,%%'')9)@)@)B)B%.5577"))++[-A-H-H-J-J,%%'')?)?)A)A,%%'')9)@)@)B)B-&&((#,*=*=*?*?/((**
, 
, 	! 	!A Z4>"" !Aa   


%|'9'H


&(:(C


%
'  A !A #$$  %$ ,%&&	 	 '&	 	) 	) 	)  7    
) 
) 
)  <        \6 7 7 7  \6 7 7 7  \4 5 5 5  \: ; ; ;  l1 2 2 2  l1 2 2 2 )*** * +**  <, - - - $+  
 / / / -'((  )( %&&) ) '&) . . . 	*++  ,+  9- . . . ())% % *)%  8+ , , , &''% % ('%  8) * * * %&&  '&  8( ) ) )   " " ! "  2? # # # ")**
 
 +*
 ,$+,,  -,. )011  21 ,+233  43 ):;;  <;  <,= > > >$$%%  &%
 "##4 4 $#4 "4554 4 654 $<==  >=8 ":;;3 3 <;3 #2331 1 431  ,-< = = =' '  '  #. ) ) )	    39   
$0 $0 $0 $0J )9  %	 	 	N N N. !/ 1= 1= 1=d #: @ @ @
 
 
M M M M). ). ).T (<  $)I )I )I )IV $  "; /# # #88 8 8.K K K K K       ( +[ "q1$&dd" " " " "J   
 5 5 5  " " " 
    DK   (DA DA DA DA DA4: DA DA DAL; ; ; 
 
 
   
* 
* 
* 
*5 5 5 5 	 	 	    H H H H H Hr`   