
    Wh                   \   d Z ddlmZ ddlmZ ddlZddlmZ ddlZddlZddl	m
Z
mZmZ ddlZddlZddlZddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZmZmZ ddlm Z m!Z! ddl"m#Z# ddl$mZ% ddl&m'Z'm(Z(m)Z) ddl*m+Z+ ddl*m,Z, ddl*m-Z- ddl*m.Z. ddl"m/Z/ ddl"m0Z0 ddl"m1Z2 ddl3m4Z4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z= ddl>m?Z? ddl@mAZAmBZBmCZC ddlDmEZE ejF        ejG        ejH        ejI        gZJejK        gZLddZMdd"ZNddd$ZO G d% d&e          ZP G d' d(e          ZQdd-ZRdd.ZS e-jT        d/          ZUdd1ZVdd4ZWeUjX        d5             ZYeUjZ        d6             Z[d7 Z\d8 Z]d9 Z^ e/j_        eUe\d           e]e/j`        eU<   e^e0ja        eU<    e#jb        eU e#jc        eYd:;                      e-jT        d<          Zdd=ed_e        d>Zfddde%jK        d?ddDZgdde%jK        dEddGZhedjX        dH             ZiedjZ        dI             ZjdJ ZkdK ZldL Zmeke/jn        ed<   ele/j`        ed<   eme0ja        ed<    e#jb        ed e#jc        eid=;                      e-jT        dM          ZoddNddTZpd=dNddUZqeojX        dV             ZreojZ        dW             ZsdX ZtdY ZudZ Zv e/j_        eodet           eue/j`        eo<   eve0ja        eo<    e#jb        eo e#jc        erd:;                      e-jT        d[          Zwd=ew_e        dd]Zxdd^Zyd_ ZzewjX        dd`            Z{ewjZ        dda            Z|ddbZ}ddcZ~dddZe}e/jn        ew<   e~e/j`        ew<   ee0ja        ew<    e#jb        ew e#jc        e{d=;                      e-jT        de          ZddddfddoZddtZddxZddyZejZ        ddz            Z e#jc        ed:;          Zd{ Zd| Z e#jc        ed:;          Zdd}Zdd~ZddZddZ e/j_        eede           ee/j`        e<   ee0ja        e<    e#jb        ee            e.j        e           e=j        r e#jb        eed           e=j        r e#jb        eed            e-jT        d          ZddZd Zd Zd ZejX        d             ZejZ        d             Zd Zd Zd Zd Z e/j_        eeed           ee/j`        e<   ee0ja        e<    e#jb        e e#jc        ed:;                      e-jT        d          Zd=e_e        ddZd ZejX        dd            ZejZ        dd            ZddZd Zee0ja        e<   ee/jn        e<    e#jb        e e#jc        ed=;                      e-jT        d          Zd=e_e        ddZejX        d             Zd ZejZ        d             Zd Zd Z e#jc        ed=;          Zee/jn        e<   ee0ja        e<    e#jb        ee            e-jT        d          Zd=e_e        dddZddZejX        d             Zd Zd:d:d:ddZd:d:d:ddZd ZejZ        d             Zd Zd Z e#jc        ed=;          Zee/jn        e<   ee0ja        e<    e#jb        ee           ddddddZddddZddZd dZdddddZd ZddZddddĄZddǄZdȄ ZddʄZdd̄Zdd̈́ZÐdd΄Zdτ ZŐd	d҄ZƐd
dӄZd:d:dddԜdd܄Zddddddddޜdd߄Zd ZʐddZej         G d de                      Zd Zd Z e0j        eeeeed           dS (  zHBCOO (Bached coordinate format) matrix object and associated primitives.    )annotations)SequenceN)partial)Any
NamedTupleProtocol)lax)	tree_utilvmap)	JAXSparse)
nfold_vmap_count_stored_elements_dot_general_validated_shapeCuSparseEfficiencyWarningSparseEfficiencyErrorSparseEfficiencyWarningShape
SparseInfo)
coo_spmv_p
coo_spmm_p)mlir)safe_zipunzip2
split_list)api_util)config)core)dispatch)ad)batching)partial_eval)_constranges_like	remaining_dot_general_batch_dim_numsDotDimensionNumbers)GatherDimensionNumbersGatherScatterMode)
gpu_sparse)_unique)Array	ArrayLike	DTypeLike)canonicalize_axisc                   | \  }}|\  }}|j         dz
  t          |d u           z   t          fd|D                       st          d|dd          d ||f||ffD             \  }}	|)t	          d t          ||f|          D                       }t          |g|j        R |j        |j	                  }
||	|
fS )	N   c              3  @   K   | ]}|d u pd|cxk    ok     nc V  d S Nr    ).0bn_batchs     b/var/www/html/movieo_spanner_bot/venv/lib/python3.11/site-packages/jax/experimental/sparse/bcoo.py	<genexpr>z,_bcoo_batch_dims_to_front.<locals>.<genexpr>B   sH      ??qQ$Y*!q****7****??????    zEbatch_dims must be None or satisfy 0 < dim < n_batch. Got batch_dims=z for n_batch=.c              3  x   K   | ]5\  }}|t          j        |dg          nt          j        ||d          V  6d S r3   )r	   expand_dimsjnpmoveaxisr5   argbdims      r8   r9   z,_bcoo_batch_dims_to_front.<locals>.<genexpr>E   sg       #E #E
#t $(<cocA3S\#tQ5O5O#E #E #E #E #E #Er:   c              3  :   K   | ]\  }}||j         |         V  d S Nshaper5   rA   dims      r8   r9   z,_bcoo_batch_dims_to_front.<locals>.<genexpr>I   s2      hhSX[XgSYs^XgXgXgXghhr:   indices_sortedunique_indices)
ndimboolallNotImplementedErrormaxzipr   rF   rJ   rK   )batched_args
batch_dimsspinfo
batch_sizedataindices	data_bdimindices_bdimbatched_databatched_indicesbatched_spinfor7   s              @r8   _bcoo_batch_dims_to_frontr]   >   sC   -$&)\L1tLD$8999'	????J???	?	? D
 C%/C C7>C C C D D D#E #Ey)G\+BC#E #E #E, hhCw4T4ThhhhhJz9FL99-3-B-3-BD D D. 
	66r:   matBCOOnseintreturnc                \   t          j        |          }|dk    sJ | j        |k    r| S || j        k    rQ| j        g d t	          | j                  D             t          |          R          }| j        dd|ddf         }nt          j	        | j        g | j        j
        d| j                 || j        j
        | j        dz   d         R           }|j        g d t	          | j                  D             t          | j                  R                              | j                  }t          j	        | j        g | j        j
        dd         || j        j
        d	         R           }|j        dd| j        ddf                             | j                  }|j        d| j        dddf                             t          j        | j
        | j        | j        | j        z            |j        
                    }t!          ||f| j
        | j        | j                  S )zsReturn a copy of `mat` with the specified nse.
  Note that if nse < mat.nse, this will potentially discard data.
  r   c              3  4   K   | ]}t          d           V  d S rD   slicer5   is     r8   r9   z _bcoo_set_nse.<locals>.<genexpr>\   s(      ??quT{{??????r:   .N   rE   c              3  4   K   | ]}t          d           V  d S rD   re   rg   s     r8   r9   z _bcoo_set_nse.<locals>.<genexpr>`   s(      >>aeDkk>>>>>>r:   dtyperF   rJ   rK   )operatorindexr`   rV   ranger7   rf   rW   r>   
zeros_likerF   atsetarrayn_sparsern   r_   rJ   rK   r^   r`   rV   rW   s       r8   _bcoo_set_nsery   S   s2    	s#	W^^JCG^^8L??E#+,>,>???LsLLMDk#ttQQQ,'GG>#(*qCHN<CK<,H*q#*qPSPXP^_b_jmn_n_o_oPp*q*qrrrD7O>>5+=+=>>>OcgOOPTTUXU]^^DnS[0e#+2CCRC2H0e#0es{O`acOd0e0efffGjhswh)*..s{;;Gjcghh)*..sy3;s{]`]iOiCi9j?F}0N 0N 0N O OG	tWoSY / /
1 
1 
1 1r:   
int | Nonec                f  	 | j         | j        | j        }}}t          |||          }|dk                        t          t          |j        dz   |j                                      }t          d t          |j        d |j                           D                       }|                    |d          }t          j        ||j        |j        |j        z            |j                  		fd} t          ||j                  ||          }t          t!          ||f|          |	          S )
Nr   ri   c              3  ,   K   | ]\  }}|d k    |V  dS ri   Nr4   r5   rh   ss      r8   r9   z'bcoo_eliminate_zeros.<locals>.<genexpr>p   s2      \\AUVZ[U[U[1U[U[U[U[\\r:   Tkeepdimsrm   c                X    t          j        |d d d f         d d d f         |           S rD   )r>   where)rh   m
fill_values     r8   <lambda>z&bcoo_eliminate_zeros.<locals>.<lambda>s   s.    39Qqqq$wZD!!!G)<a@@ r:   rE   r`   )rV   rW   rF   _validate_bcoorN   tuplerr   r7   rL   	enumerater>   rv   rw   rn   r   bcoo_sum_duplicatesr_   )
r^   r`   rV   rW   rF   propsmaskdims_to_contractfr   s
            @r8   bcoo_eliminate_zerosr   l   s   3;	$
w
.
.%
!)uU]Q%6	BBCC	D	D$\\7=%-3P)Q)Q\\\\\	"T	2	2$yu}U]U^-KKLT[Tabbb*@@@@!(Jq%-(($77'	T4/???S	I	I	IIr:   c                  8    e Zd ZU ded<   ded<   ded<   ded<   dS )BCOOPropertiesra   r7   rw   n_denser`   N)__name__
__module____qualname____annotations__r4   r:   r8   r   r   x   s4         ,,,---,,,
(((((r:   r   c                  >    e Zd Zedd            Zedd            ZdS )	Bufferrb   r   c                    d S rD   r4   selfs    r8   rF   zBuffer.shape   s    Cr:   r   c                    d S rD   r4   r   s    r8   rn   zBuffer.dtype   s    #r:   N)rb   r   )rb   r   )r   r   r   propertyrF   rn   r4   r:   r8   r   r   ~   s<         8 8r:   r   rV   rW   rF   Sequence[int]c           
        t          ||          }|\  }}}}t          |          }t          d t          | j        d |         |d |                   D                       rt          d| j        d|          | j        |d          |f|||z   d          z   k    r t          d| j        d|d|d|          |S )Nc              3  (   K   | ]\  }}|d |fvV  dS r}   r4   r5   s1s2s      r8   r9   z!_validate_bcoo.<locals>.<genexpr>   s/      YYvr2Ar7	YYYYYYr:   z4data batch dimensions not compatible for data.shape=, shape=zInvalid data.shape=	 for nse=
, n_batch=
, n_dense=)_validate_bcoo_indicesr   anyr   rF   
ValueError)rV   rW   rF   r   r7   rw   r   r`   s           r8   r   r      s    
 %
0
0%$)!'8Wc
,,%YY$*XgX2Fhwh)X)XYYYYY Z
XXXPUXX
Y
YY	ZcVeGh,>,?,?&@@@@
P
PP#PP'PPgPP
Q
QQ	,r:   c           
     4   t          j        | j        t           j                  sJ t	          |          }| j        dd          \  }}t          | j                  dz
  }t          |          |z
  |z
  }|dk    sJ t          d t          | j        d |         |d |                   D                       rt          d| j        d|          | j        |d          ||fk    r t          d| j        d|d	|d
|          t          ||||          S )Nrk   r1   r   c              3  (   K   | ]\  }}|d |fvV  dS r}   r4   r   s      r8   r9   z)_validate_bcoo_indices.<locals>.<genexpr>   s/      \\vr2Ar7	\\\\\\r:   z:indices batch dimensions not compatible for indices.shape=r   zInvalid =indices.shape=r   r   r   )r7   rw   r   r`   )r>   
issubdtypern   integerr   rF   lenr   r   r   r   )rW   rF   r`   rw   r7   r   s         r8   r   r      s8   	s{	3	333	3
,,%-$-#x"'JJ 8+'	A\\'-2I5QYRYQY?)[)[\\\\\ `
^GM^^V[^^
_
__]788h//
TTTcTTgTT'TT
U
UU	(GQT	U	U	UUr:   bcoo_todenser,   c                D    t          | j        | j        | j                  S )zConvert batched sparse matrix to a dense matrix.

  Args:
    mat: BCOO matrix.

  Returns:
    mat_dense: dense version of ``mat``.
  rT   )_bcoo_todenserV   rW   _info)r^   s    r8   r   r      s     
sxSY	?	?	??r:   rT   r   c                   t                               t          j        |           t          j        |          |          S )a  Convert batched sparse matrix to a dense matrix.

  Args:
    data : array of shape ``batch_dims + (nse,) + block_dims``.
    indices : array of shape ``batch_dims + (n_sparse, nse)``
    spinfo : SparseInfo. In particular, this includes the shape
      of the matrix, which is equal to ``batch_dims + sparse_dims + block_dims``
      where ``len(sparse_dims) == n_sparse``

  Returns:
    mat : array with specified shape and dtype matching ``data``
  r   )bcoo_todense_pbindr>   asarrayrV   rW   rT   s      r8   r   r      s3     
		S[..G0D0DV		T	TTr:   c          	        |j         }t          | |          \  }}}}t          d t          |d |         j         d |                   D                       }t          t	          j        |ddd          t          fdt          |          D                       }t          d |d |         D                       }	t	          j        g |	t	          j        d          R dddt                    d d         }
|s*|                     |t          |
          | j
        	          } t          j        || j
                  j        |
|z                                |           S )
Nc              3     K   | ];\  }}|d k    rt          j        |t                    nt          j        |          V  <dS r}   npzerosra   aranger5   r   i_ss      r8   r9   z%_bcoo_todense_impl.<locals>.<genexpr>   sc       R RC *-RXa%%%ry|| R R R R R Rr:   ijTindexingsparsec              3  L   K   | ]}t          d           |fz            V  d S rD   re   r5   rh   gridrW   s     r8   r9   z%_bcoo_todense_impl.<locals>.<genexpr>   s8      OO!WTU4[[!$445OOOOOOr:   c              3  >   K   | ]}t          j        |          V  d S rD   r   r   r5   r   s     r8   r9   z%_bcoo_todense_impl.<locals>.<genexpr>   s*      ==ry||======r:   ri   rl   )r   rn   )rF   r   r   rQ   r   meshgridrr   r   sumrM   rn   r>   r   rt   add)rV   rW   rT   rF   r7   rw   _
ind_slices
sparse_indbatch_slices	batch_indr   s    `         @r8   _bcoo_todense_implr      sx   
,%*4%@@'8Q R R#&uXgXhwh8O#P#PR R R R R*	r{JdCCC	D	D$OOOOOuXOOOOO*==U8G8_=====,		Ml	MBIaLL	M	M4	M	M	M$Dkk#2#)	 I88Gd9ooTZ8HHD	5$*	%	%	(Z)?	@	D	DT	J	JJr:   c               f    |j         }t          | ||           t          j        || j                  S rD   )rF   r   r   ShapedArrayrn   )rV   rW   rT   rF   s       r8   _bcoo_todense_abstract_evalr      s0    
,%w&&&		%	,	,,r:   c               &    t          | ||          S )Nr   )r   )data_dotrV   rW   rT   s       r8   _bcoo_todense_jvpr      s    	x	8	8	88r:   c                   |j         }t          j        |          sJ t          j        |          rt          d          | j         |k    sJ | j        |j        j        k    sJ t          ||           |fS )N/Cannot transpose with respect to sparse indices)rF   r    is_undefined_primalr   rn   aval_bcoo_extract)ctrV   rW   rT   rF   s        r8   _bcoo_todense_transposer      s    
,%			%	%%%	%G$$ H
F
G
GG	U					TY_	$	$	$	$	w	#	#W	,,r:   c               T    t          | ||          \  }}}t          |||          dfS )Nr   r   )r]   r   )rR   rS   rT   rV   rW   s        r8   _bcoo_todense_batching_ruler      s4    3L*fUU$	tWV	4	4	4a	77r:   F)multiple_resultsbcoo_fromdenseTz
The error arose for the nse argument of bcoo_fromdense. In order for
BCOO.fromdense() to be used in traced/compiled code, you must pass a concrete
value to the nse (number of stored elements) argument.
r`   r7   r   index_dtyper7   r   r   r.   c          	         t          j        |           } |}|t          | ||          }t          j        t
          j        |t                    }t          t          | ||||          | j
        dd          S )a  Create BCOO-format sparse matrix from a dense matrix.

  Args:
    mat : array to be converted to BCOO.
    nse : number of specified elements in each batch
    n_batch : number of batch dimensions (default: 0)
    n_dense : number of block_dimensions (default: 0)
    index_dtype : dtype of sparse indices (default: int32)

  Returns:
    mat_bcoo: BCOO representation of the matrix.
  Nr   Tro   )r>   r   r   r   concrete_or_errorrp   rq   _TRACED_NSE_ERRORr_   _bcoo_fromdenserF   )r^   r`   r7   r   r   nse_arrnse_ints          r8   r   r      s     	C# #'_$S'7;;G"8>7<MNN'	ocw*57 7 7Id4
I 
I 
I Ir:   r7   r   r   tuple[Array, Array]c                   t          j        |           } t          j        t          j        |t                    }t                              | ||||          S )aA  Create BCOO-format sparse matrix from a dense matrix.

  Args:
    mat : array to be converted to BCOO, with ``ndim = n_batch + n_sparse + n_dense``.
    nse : number of specified elements in each batch
    n_batch : number of batch dimensions (default: 0)
    n_dense : number of block_dimensions (default: 0)
    index_dtype : dtype of sparse indices (default: int32)

  Returns:
    data : array of shape ``mat.shape[:n_batch] + (nse,) + mat.shape[mat.ndim - n_dense:]``
      and dtype ``mat.dtype``
    indices : array of shape ``mat.shape[:n_batch] + (n_sparse, nse)``
  r   )	r>   r   r   r   rp   rq   r   bcoo_fromdense_pr   )r^   r`   r7   r   r   s        r8   r   r     sR      	C#x~s4EFF#			sWg+6 
 
8 
8 8r:   c                  t          j        |           } | j        |z
  |z
  | dk    }|dk    r,|                    d t	          |          D                       }t          t          |d          fd            } ||          }|s(t          j        |j        d |         dfz   |          }n,t          j	        t          j
        ||          d|dz             }t          ||           }|                    t          t	          ||j                                      d         }	t          j        |	j        d|z  fz   |          |	k     }
|
|dz   t#          d           fz  |d	z  z            }
t          j        |
|d          }||fS )
Nr   c                    g | ]}|d z    	S ri   r4   rg   s     r8   
<listcomp>z(_bcoo_fromdense_impl.<locals>.<listcomp>*  s    666!q1uX666r:   F)Nbroadcastedc                ^    | j         r$t          j        | | j        d                    S dS )Nsizer   r4   )rL   r>   nonzerorF   )arw   r`   s    r8   _nonzeroz&_bcoo_fromdense_impl.<locals>._nonzero+  s5    v E[(1CDDDD2r:   ri   .Nr   rD   )r>   r   rL   r   rr   r   r   r   rF   r?   rv   r   r   listr	   broadcasted_iotarn   rf   r   )r^   r`   r7   r   r   r   r   rW   rV   true_nsetrue_nonzerosrw   s    `         @r8   _bcoo_fromdense_implr  $  s   C#X')(
($q[[8866uW~~66677D
:e444     54 HTNN'	 Li
8G8,Qx7EEGGl39Wk::Aw{KKG	w	$	$$XXd5$)445566yA(&x~tg~7NPWXX[cc-1t >7AR RS-	=$	*	*$	wr:   c                   | j         |z
  |z
  }| j        d |         |fz   | j        ||z   d          z   }| j        d |         ||fz   }t          j        || j                  t          j        ||          fS rD   )rL   rF   r   r   rn   )r^   r`   r7   r   r   rw   
data_shapeindex_shapes           r8   _bcoo_fromdense_abstract_evalr  >  s    X')(y'"cV+ci(8J8K8K.LL*	(7(#sHo5+		*ci	0	0$2B;P[2\2\	\\r:   c               $   | \  }|\  }t          |||||          }|\  }	}
t          |          t          j        u r t          j                            |	          }nt          |
|          }|t          j                            |
          f}||fS )Nr   )r   typer    Zerofrom_primal_valuer   )primalstangentsr`   r7   r   r   MMdotprimals_outrV   rW   r   tangents_outs                r8   _bcoo_fromdense_jvpr  E  s    "!
%$sGWZefff+-$	$ZZ27w((..HHWd++HBG55g>>?,	l	""r:   c                  | \  }}|j         |z
  |z
  }|j        |j        d |         |fz   |j        ||z   d          z   k    sJ |j        |j        d |         ||fz   k    sJ |j        |k    sJ t          |t          j                  rt          d          t	          j        |          sJ t          ||t          |j
        j                            S )Nr   r   )rL   rF   rn   
isinstancer    r  r   r   r   r   r   )	r   r  r`   r7   r   r   rV   rW   rw   s	            r8   _bcoo_fromdense_transposer  U  s    -$Vg'(	qwxx(C61AGGh<N<O<O4PP	P	P	P	P	!'(7(+xo=	=	=	=	=	+	%	%	%	%!! H
F
G
GG			"	"""	"	tWZ-E-E	F	F	FFr:   c                   | \  }|\  }d|cxk    r|k    sn t          d|d|          t          |||dz   ||          ||ffS )Nr   z'Expected 0 < bdim <= n_batch; got bdim=r   ri   r   )r   r   )rR   rS   r`   r7   r   r   r  rB   s           r8   _bcoo_fromdense_batching_ruler  `  sx    "!
%$
t



w




M$MM7MM
N
NN	Wq['Wb	c	c	cfjlpeq	qqr:   bcoo_extractassume_uniquesparrarrr-   r  bool | Nonec                  t          | t                    st          dt          |                     t	          j        |          }|j        | j        k    rt          d| j        d|j                  || j        }t          | j
        ||          }t          || j
        ffi | j                                        S )a   Extract values from a dense array according to the sparse array's indices.

  Args:
    sparr : BCOO array whose indices will be used for the output.
    arr : ArrayLike with shape equal to self.shape
    assume_unique : bool, defaults to sparr.unique_indices
      If True, extract values for every index, even if index contains duplicates.
      If False, duplicate indices will have their values summed and returned in
      the position of the first index.

  Returns:
    extracted : a BCOO array with the same sparsity pattern as self.
  zGFirst argument to bcoo_extract should be a BCOO array. Got type(sparr)=zshape mismatch: sparr.shape=z	 a.shape=Nr  )r  r_   	TypeErrorr
  r>   r   rF   r   rK   r   rW   r   _asdict)r  r  r  r   rV   s        r8   r  r  s  s     
E4	 	  b
`RVW\R]R]``
a
aa	k#!W
AAAqwAA
B
BB(M	u}a}	E	E	E$	tU]#	=	=u{':':'<'<	=	==r:   c               <    t                               | ||          S )a  Extract BCOO data values from a dense array at given BCOO indices.

  Args:
    indices: An ndarray; see BCOO indices.
    arr: A dense array.
    assume_unique: bool, default=True
      If True, then indices will be assumed unique and a value will be extracted
      from arr for each index. Otherwise, extra work will be done to de-duplicate
      indices to zero-out duplicate extracted values.

  Returns:
    An ndarray; see BCOO data.
  r  )bcoo_extract_pr   )rW   r  r  s      r8   r   r     s     
		Wc		G	GGr:   c          	         t          j        |          }t           |j                  }|s1t	           |j        d          \   }|}t           |j                  }t          d t          |j        d |j                  j        d |j                           D                       }t          t          j	        |ddd          t           fdt          |j                  D                       }t          d |j        d |j                 D                       }t          j	        g |t          j        d          R dddt                    d d	         }	||	z   s	|d          }
n%|j        |	|z                                d
d          }
|j        dk    r|j        dk    r|rGt!          j        |
t%          |
j        |j        |j                  t          |
j                            }
n}t%          |
j        |j        |j                  }|j        t)          d           fz  t)          d          fz   }t          j        |
|          j        |                             |
          }
|sW|j        g|
j        |j        dz   d          R fd}t          |j                  D ]}t/          |          } ||
|          }
|
S )NT)rF   return_indexc              3     K   | ];\  }}|d k    rt          j        |t                    nt          j        |          V  <dS r}   r   r   s      r8   r9   z%_bcoo_extract_impl.<locals>.<genexpr>  sc       b bC *-RXa%%%ry|| b b b b b br:   r   r   c              3  L   K   | ]}t          d           |fz            V  d S rD   re   r   s     r8   r9   z%_bcoo_extract_impl.<locals>.<genexpr>  s8      UU!WTU4[[!$445UUUUUUr:   c              3  >   K   | ]}t          j        |          V  d S rD   r   r   s     r8   r9   z%_bcoo_extract_impl.<locals>.<genexpr>  s*      GGry||GGGGGGr:   ri   rl   fillr   moder   rE   c                l    t          j        |           j        |                             |           S NrE   )r>   rs   rt   r   )rrh   unbatched_out_shapes     r8   r   z_bcoo_extract_impl.<locals>.f  s/    ^A%8999<Q?CCAFFFr:   )r>   r   r   rF   _unique_indicesr   rQ   r7   r   r   rr   rw   r   rt   getr`   r	   broadcast_in_dim_tuple_replacerL   rf   rs   ru   r   )rW   r  r  r   sort_indoriginal_propsr   r   r   r   result	out_shapeindr   r   r   r/  s   `              @@r8   _bcoo_extract_implr9    s   C#
 #)
4
4%	 7'sytTTTGXN"7CI66E b b#&sy%-'@'-P^QVQ^P^B_#`#`b b b b b*	r{JdCCC	D	D$UUUUUuU^?T?TUUUUU*GGSY~~-FGGGGG,		Ml	MBIaLL	M	M4	M	M	M$Dkk#2#)	i	 KYFFVI
*+//V/JJF
^qUY!^^ K#v|U]EIFFfkHZHZ\ \ff !u}n>PQQiMU4[[N*eAhh[8c~fI6669#>BB6JJf	 !)-QU]Q=N=O=O0PQQG G G G G5=!!  
q''aaQvx  F	-r:   c                   t          |          }t          | |j                  \  }}}}|j        d |         |fz   |j        |j        |z
  d          z   }t	          j        ||j                  S rD   )rM   r   rF   rL   r   r   rn   )rW   r  r  r   r7   r   r`   r7  s           r8   _bcoo_extract_abstract_evalr;    sn    
=!3GSYGG'1gsi!SF*SYsx'7I7J7J-KK)		)SY	/	//r:   c               J    | j         |j         k    sJ t          || |          S )Nr  )rF   r   )arr_dotrW   r  r  s       r8   _bcoo_extract_jvpr>    s,    	#)	#	#	#	#	w}	E	E	EEr:   c                  |st          d          t          j        |          sJ t          j        |          rt          d          | j        |j        j        k    sJ |t          | |t          |j        j                            fS )Nz2transpose of bcoo_extract with assume_unique=Falser   r   )	rO   r    r   r   rn   r   r   r   rF   )r   rW   r  r  s       r8   _bcoo_extract_transposer@    s    	 T
R
S
SS			$	$$$	$G$$ H
F
G
GG	SX^	#	#	#	#	-GJsx~4N4NOOO	OOr:   c                  | \  }}t          d |D                       sJ |d         |d         }t          j        ||f          }n|d         U|d         }t          |j                  }|                    ||j        |                    t          j        |||f          }n)|d         |d         k    rt          d          |d         }|j        dz
  }||k    rt          d|d|          t          |||          |fS )	Nc              3     K   | ]}|d uV  	d S rD   r4   )r5   r6   s     r8   r9   z._bcoo_extract_batching_rule.<locals>.<genexpr>  s&      //qQd]//////r:   r   ri   z+bcoo_extract with unequal batch dimensions.r1   zbatch_dims=z' out of range for indices with n_batch=r  )r   r	   r=   r   rF   insertr2  rO   rL   r   r   )rR   rS   r  rW   r  rB   result_shaper7   s           r8   _bcoo_extract_batching_rulerE    s$   ,'3	//J///	/	///	/]a=Dogw//GG!}a=D	??LgmD1222

sL4'
:
:CC!}
1%% MNNNa=DL1'	W__
N
NNGNN
O
OO	w=	A	A	A4	GGr:   bcoo_transposepermutationc                    t           j         j        | j                  }t	           fd|D                       }t          || j                  S )u-  Transpose a BCOO-format array.

  Args:
    mat: A BCOO-format array.
    permutation:  A tuple or list or ndarray which contains a permutation of
      [0,1,..,N-1] where N is the number of axes of ``mat`` in the order of
      batch, sparse, and dense dimensions. The i’th axis of the returned array
      corresponds to the axis numbered permutation[i] of ``mat``. Transpose
      permutation currently does not support permuting batch axes with non-batch
      axes nor permuting dense axes with non-dense axes.

  Returns:
    A BCOO-format array.
  rG  rT   c              3  2   K   | ]}j         |         V  d S rD   rE   r5   pr^   s     r8   r9   z!bcoo_transpose.<locals>.<genexpr>	  s)      66QCIaL666666r:   )rF   rK   )_bcoo_transposerV   rW   r   r   r_   rK   )r^   rG  buffersr7  s   `   r8   rF  rF    s]     CHck{SVS\]]]'6666+66666)	gYs7I	J	J	JJr:   c                   t          |          }|t          t          t          |j                                      k    r| |fS t                              | |||          S NrI  )r   rr   r   rF   bcoo_transpose_pr   )rV   rW   rG  rT   s       r8   rM  rM    sd    k""+E%FL 1 1223333=  wK(. ! 0 0 0r:   c                F  	
 t          |t          t          t          j        f          s t          dt          |           d          t          t          |                    t          t          t          |                              k    rt          d| d| d          t          | ||          \  	
}}|d 	         }	fd|		
z            D             }	
fd|	
z   d          D             }	rPt          t          |                    t          t          	                    k    rt          d| d	d          |rPt          t          |                    t          t          |                    k    rt          d	| d
|d          |||fS )Nz8transpose permutation must be a tuple/list/ndarray, got r;   zQtranspose permutation isn't a permutation of operand dimensions, got permutation z for shape c                    g | ]}|z
  S r4   r4   )r5   rL  r7   s     r8   r   z)_validate_permutation.<locals>.<listcomp>  s    OOOWOOOr:   c                     g | ]
}|z
  z
  S r4   r4   )r5   rL  r7   rw   s     r8   r   z)_validate_permutation.<locals>.<listcomp>  s"    QQQ1Hw&QQQr:   zUtranspose permutation cannot permute batch axes with non-batch axes; got permutation z, with n_batch=zUtranspose permutation cannot permute dense axes with non-dense axes; got permutation z, with n_dense=)r  r   r   r   ndarrayr   r
  sortedrr   r   r   rO   )rV   rW   rG  rF   r   r   
batch_permsparse_perm
dense_permr7   rw   s            @@r8   _validate_permutationrZ    s   	K%rz!:	;	; e
ctT_O`O`ccc
d
dd
6+5s5zz):):#;#;;;
 H'2H H?DH H H I I I"0w"F"F'8Wa8G8$*OOOOk'7X;M2M&NOOO+QQQQQGh<N<O<O0PQQQ* Rvj))**eE'NN.C.CCC
 Q1<Q QELQ Q Q R R R Rvj))**eE'NN.C.CCC
 Q1<Q QELQ Q Q R R R	[*	,,r:   c                   t          | |||j                  \  }}}t          |           |d|f         j        g |dz   R  } | j        g |fd|D             R  } | |fS )N.ri   c              3  (   K   | ]}|z   d z   V  dS r}   r4   r5   dr7   s     r8   r9   z'_bcoo_transpose_impl.<locals>.<genexpr>,  s+      /T/TAGa/T/T/T/T/T/Tr:   )rZ  rF   r   	transpose)rV   rW   rG  rT   rW  rX  rY  r7   s          @r8   _bcoo_transpose_implr`  '  s    (=dG[Z`Zf(g(g%*k:
OO'/GC$%/RRWRgPQkRRR'		U	UW	U/T/T/T/T/T/T/T	U	U	U$	wr:   c               r  	 t          | |||j                  \  }}}t          |          	t          j        |j                  g |		dz            }t          j        | j                  g |		fd|D                      }t          j        || j                  t          j        ||j                  fS )Nri   c              3  (   K   | ]}|z   d z   V  dS r}   r4   r]  s     r8   r9   z0_bcoo_transpose_abstract_eval.<locals>.<genexpr>4  s,      <a<aQRQ[1_<a<a<a<a<a<ar:   )rZ  rF   r   r   rv   r   r   rn   )
rV   rW   rG  rT   rW  r   rY  indices_shaper  r7   s
            @r8   _bcoo_transpose_abstract_evalrd  /  s    3D';PVP\]]*a
OO'(7=))*MJ*M*M1*MN-x
##$bj$b'$b<a<a<a<aV`<a<a<a$bc*		*dj	1	143CMSZS`3a3a	aar:   c                   | \  }}|\  }}t          ||||          }t          ||||          \  }	}||	t          j                            |          ffS rP  )rM  r    r  r  )
r  r  rG  rT   rV   rW   r   r   r  data_dot_outs
             r8   _bcoo_transpose_jvprg  7  se    -$+(Ag;vVVV+#Hg;W]^^^/,	|RW%>%>w%G%GH	HHr:   c               T   | \  }}t          |t          j                  sJ t          j        |          rt	          d          |j        |j        j        k    sJ t          t          fd|D                                 }t          t          t          t          j        |                              }t          j        d t!          |j        dz
            D             t          |j        dd                    z   t                    }	t'          ||	||          \  }
}|
|fS )Nr   c              3  2   K   | ]}j         |         V  d S rD   rE   )r5   rL  rT   s     r8   r9   z,_bcoo_transpose_transpose.<locals>.<genexpr>D  s)      DD1v|ADDDDDDr:   c                    g | ]}d S r   r4   rg   s     r8   r   z-_bcoo_transpose_transpose.<locals>.<listcomp>G  s    @@@1Q@@@r:   r1   rk   rm   rI  )r  r    r  r   r   rn   r   r   r   r   mapra   r   argsortr>   r   rr   rL   rF   rM  )r   rV   rW   rG  rT   data_ct
indices_ct	ct_spinforev_permutationdummy_indices
data_transr   s       `       r8   _bcoo_transpose_transposers  >  s   ':	J	(	(((	(G$$ H
F
G
GG	$)/	)	)	)	)DDDDDDDDDEE)S"*["9"9::;;/)@@glQ.>(?(?@@@4VXVYVYHZC[C[[cfggg-!'=o^ghhh-*a	Z	r:   c                   t          | ||          \  }}}dgd |D             R }t          ||||          \  }}d |D             }d t          ||f|          D             }||fS )Nr   c              3      K   | ]	}|d z   V  
dS r}   r4   )r5   rL  s     r8   r9   z-_bcoo_transpose_batch_rule.<locals>.<genexpr>M  s&      99a!e999999r:   rI  c                    g | ]}|d nd	S r3   r4   )r5   rB   s     r8   r   z._bcoo_transpose_batch_rule.<locals>.<listcomp>O  s     GGGDDLDDaGGGr:   c                H    g | ]\  }}|t          j        |dg          n| S r3   )r	   squeezer@   s      r8   r   z._bcoo_transpose_batch_rule.<locals>.<listcomp>P  sI     E E E#t (,|ck#s### E E Er:   )r]   rM  rQ   )	rR   rS   rG  rT   rV   rW   batched_permutationbatch_dims_outargs_outs	            r8   _bcoo_transpose_batch_ruler|  K  s    3L*fUU$:99[999::!$=PY_```-$GGJGGG.E E"D'?NCCE E E(	>	!!r:   bcoo_dot_general)	precisionpreferred_element_typeout_typelhsBCOO | Arrayrhsdimension_numbersr'   r~  Noner  c          
     0   ~~t          | t                    rvt          |t                    rat          | j        |j        |          }t	          | j        | j        |j        |j        | j        |j        ||          }t          ||          S t          | t                    r$t          | j        | j        |||| j                  S t          |t                    r$t          | |j        |j        |||j                  S t          j        | |||          S )a<  A general contraction operation.

  Args:
    lhs: An ndarray or BCOO-format sparse array.
    rhs: An ndarray or BCOO-format sparse array..
    dimension_numbers: a tuple of tuples of the form
      `((lhs_contracting_dims, rhs_contracting_dims),
      (lhs_batch_dims, rhs_batch_dims))`.
    precision: unused
    preferred_element_type: unused

  Returns:
    An ndarray or BCOO-format sparse array containing the result. If both inputs
    are sparse, the result will be sparse, of type BCOO. If either input is dense,
    the result will be dense, of type ndarray.
  
lhs_spinfo
rhs_spinfor  r  rE   r  r  r  )r  r  r  r  r  )r  r_   r   rF   _bcoo_spdot_generalrV   rW   r   _bcoo_dot_general_bcoo_rdot_generalr	   dot_general)r  r  r  r~  r  r  rF   bufss           r8   r}  r}  a  s<   , T Jz#t44 J(CI):< <Esxch*-)	1B6LN N ND E""""#t 
JSXs{CK\4J(+	3 3 3 3 #t Jc38S[L]5K),4 4 4 4 ?37H2HJ J J Jr:   lhs_datalhs_indicesr   r  c                  |\  \  }}\  }}	t          j        |          t          j        |          f}
t          j        |          t          j        |	          f}|t          j        |          }t                              t          j        |           t          j        |          t          j        |          |
|f||          S Nr  )r   _ensure_index_tupler   rn   bcoo_dot_general_pr   r>   r   )r  r  r  r  r  r  lhs_contractrhs_contract	lhs_batch	rhs_batchcdimsbdimss               r8   r  r    s     :K6< 6I'55'557%'	22'	224%'X&<==		 	 X!6!6K8P8PRUR]^aRbRb495>8N,6 
! 
8 
8 8r:   rhs_datarhs_indicesr  c               B   t          d |D                       }t          ||| |||          }d |D             \  }}	t          |j                  |z
  }
g t	          |	          t	          |
|j                  t	          |	|
          R }t          j        ||          S )Nc              3  ,   K   | ]}|d d d         V  d S Nrl   r4   r5   r^  s     r8   r9   z%_bcoo_rdot_general.<locals>.<genexpr>  s,      9]9]a!DDbD'9]9]9]9]9]9]r:   )r  r  r  c              3  @   K   | ]}t          |d                    V  dS r   Nr   r  s     r8   r9   z%_bcoo_rdot_general.<locals>.<genexpr>  s,      >>qQqT>>>>>>r:   )r   r  r   rF   rr   rL   r	   r_  )r  r  r  r  r  r  dimension_numbers_reversedr6  
n_contractr7   n_swaprG  s               r8   r  r    s     5:9]9]K\9]9]9]4]4]X{CJ/I4JL L L& ?>,=>>>*gz  :-&W%..W5#=#=Wgv@V@VWW+	v{	+	++r:   c          
        t          j        |           } t          j        |          }t          j        |          }t          | j        |j        |j        ||          }|j        d         |j        dz
  |\  \  }}\  }	}
t          fdt          ||          D                       \  }t          fdt          ||          D                       \  }|	sr|g |	t          t                    |	          }| 
                    g |t          | j                            } |
                    g |t          |j                            }rXt          fdD                       t          j        g t          t                                        }|d|f         }g |
||t          t          |j                  |
|          }|
                    |          }fd}t          |t                    z
            }t          j        |t          t          |
          t                    z
                      }t          j        |j        |j                  } ||| ||          S )	Nr  rl   r1   c                *    g | ]\  }}|k     ||fS r4   r4   r5   lr.  r7   s      r8   r   z*_bcoo_dot_general_impl.<locals>.<listcomp>  s0     1R 1R 1Rq!Q[[QF[[[r:   c                *    g | ]\  }}|k    ||fS r4   r4   r  s      r8   r   z*_bcoo_dot_general_impl.<locals>.<listcomp>  s0     1S 1S 1Sq!Q'\\QF\\\r:   c              3  "   K   | ]	}|z
  V  
d S rD   r4   r]  s     r8   r9   z)_bcoo_dot_general_impl.<locals>.<genexpr>  s'      EEaa'kEEEEEEr:   .c                   t          fdt                    D                       }|d t                             }|t                    d          }|rJj        dk    r?t	          j        d j        d d         D             ddid j        dz
           }g ||R }t          t          t          
          t                    z                       }t          j
        ||j        |                             dd	          g g f||ff
          }	|r | j        |                             |	          S |	                    t          t          |	j        | j        z
                      | j                  S )Nc              3  ,   K   | ]}d |f         V  dS r   r4   )r5   rh   r  s     r8   r9   z9_bcoo_dot_general_impl.<locals>.result.<locals>.<genexpr>  s,      ==CF#======r:   r1   c              3  >   K   | ]}t          j        |          V  d S rD   )r>   r   )r5   ns     r8   r9   z9_bcoo_dot_general_impl.<locals>.result.<locals>.<genexpr>  s*      ::aCJqMM::::::r:   rl   r   r   r)  r   r*  )r  rm   )r   rr   r   rL   r>   r   rF   r   rM   r	   r  rt   r1  r   r   rn   )	out_arrayr  r  r  idx	idx_rightidx_out	idx_batchrS   prodlhs_contracting_blhs_contracting_srw   r  s     `       r8   r6  z&_bcoo_dot_general_impl.<locals>.result  s   
====U8__===
=
=C+S*+++,I#'(())*G +[%)),::;#4SbS#9::: .+*Q..0i +I*	**ieC 122T:K5L5LLMMNNJ?8SVI%6%:%:ST%:%U%UHz:&>?2HJ J JD  W\'"&&t,,,XXeE$)in"<==>>ioXVVVr:   )r>   r   _bcoo_dot_general_abstract_evalr   rF   rL   r   r   r%   rr   r_  r   rv   r   r   r	   r=   r   rn   )r  r  r  r  r  r  out_avallhs_contractingrhs_contractingr  r  rhs_contracting_brhs_contracting_srW  rX  rhs_permr6  r  r  r  r7   rw   s       `             @@@@r8   _bcoo_dot_general_implr    s"   [""(K((+C#,X]K<Lch?PDZ8BD D D( r"(q '?P<$?O&<y))/ 1R 1R 1R 1RAA1R 1R 1R *S *S&&)/ 1S 1S 1S 1SAA1S 1S 1S *T *T&&  Z# Zk9kywDUVVkYjkJ!!"OJ"Ow1N1N"OPPH''(X*(XuWkFV7W7W(XYYK  0EEEE3DEEEEE)`/`)E(OOM^2_2_`aaKc;./KGy G, G/@ Gsx)_EEG(h#W W W W W W W W" fg,=(>(>>??&U3y>>7SAR=S=S3STTUU#i77)		8[#	6	66r:   c                 
 t          j        t          j        d                              t          j        |j        | j                  t          j        |j        |j                  ||          }|\  \  }}\  }	}t          | ||j                  \  
}}|	r(t          |	          
k    rt          d|	d
          t          
fd|D                       rt          d          t          j        |j        |j                  S )Nr  static_argnameszqbcoo_dot_general batch dimensions must be among the batch dimensions in the sparse representation.
got lhs_batch=r   c              3  *   K   | ]}|z   k    V  d S rD   r4   )r5   r^  r7   rw   s     r8   r9   z2_bcoo_dot_general_abstract_eval.<locals>.<genexpr>  s-      ::Qg 	 ::::::r:   z4bcoo_dot_general: contracting over dense dimensions.)jaxjitr	   r  
eval_shapeShapeDtypeStructrF   rn   r   rP   rO   r   r   r   )r  r  r  r  r  r  r  r  r   r  r7   rw   s             @@r8   r  r    s.    WS_6efffqq

z/
@
@

sy#)
4
4-!7	 r 9 9( *;&?AA*8[*BRSS'8Q '3y>>W,,
&& &"& &' ' '
 	:::::/::::: V
T
U
UU		(.(.	9	99r:   c                   | j         t          vr%t          j        d| j         dt                     dS |j         t
          vr%t          j        d|j         dt                     dS |j        st          j        dt                     dS dS )NzJbcoo_dot_general cusparse/hipsparse lowering not available for data.dtype=z). Falling back to default implementation.TzMbcoo_dot_general cusparse/hipsparse lowering not available for indices.dtype=zbcoo_dot_general GPU lowering requires matrices with sorted indices. To sort the rows in your matrix, use e.g. mat = mat.sort_indices(). Falling back to the default implementation.F)rn   CUSPARSE_DATA_DTYPESwarningswarnr   CUSPARSE_INDEX_DTYPESrJ   r   s      r8   _bcoo_dot_general_fallbackr    s    	Z+++M PP P P+- - - 4}111M S S S S+- - - 4  M $ &?@ @ @ 45r:   c                  t           j        j        st          | |||||          S |\  \  }}\  }}	t	          | ||j                  \  }
}}}	|j        dk    rt          nt          }t          | |||||          }|j
        t          vrt          | |||||          S |                     |j
                  } |                    |j
                  }t          |          dk    rt          |          dk    r|j        dv r|
||fdk    rt          | ||          st          j        |j        d         |j
                  |                                }}d}dg|j        R }t%          ||||          \  }}}|                    | |||d         dk    r|j        n|||          }|d         S t          |          dk    rt          |          dk    r|j        dv r|
||fdk    rt          | ||          sy|d d df         |d d df         }}|d         dk    }|j        }t%          ||||          \  }}}|                    | |||d         dk    r|j        n|||          }|d d	         S t          | |||||
          S )Nr  ri   r   )ri   r1   )r   ri   r   F)r_  rF   )r   r1   r   rl   )r  r  r  )r   bcoo_cusparse_loweringvaluer  r   rF   rL   r   r   r  rn   r  astyper   r  r>   r   ravel!_coo_correct_out_of_bound_indicesr   T)r  r  r  r  r  r  r  r  r  r   r7   rw   r   coo_matmul_pr  rowcolr_  rF   outs                       r8   _bcoo_dot_general_gpu_implr    s    
	&	, !(K+5   
 2C.<A"0Z-#/ #/'8Wa"x1}}*,,k3'1	  ( ^///!(K+5   
 __X^,,(

8>""# 
,1Y1!4!4V9K9KHg
&)
3
3(;
KK 4y*1-{/@AA;CTCTCVCVCI"!""E7S%SSOCe


Hc3%1!_%9%9CEEs&/u  > >C q6MLQ3y>>Q#6#638v;M;Mh(I55*8[*MM 6111a4 +aaad"3CaA%IE7S%SSOCe


Hc3%1!_%9%9CEEs&/u  > >C ss8O!(K+
57 7 7 7r:   c               ,    t          | |||||          S r  r  )lhs_data_dotr  r  r  r  r  r  s          r8   _bcoo_dot_general_jvp_lhsr  M  s,    	<cM^2HU_
a 
a 
a ar:   c               ,    t          ||| |||          S r  r  )rhs_dotr  r  r  r  r  r  s          r8   _bcoo_dot_general_jvp_rhsr  R  s,    	8['M^2HU_
a 
a 
a ar:   c               h    t          j        |          rJ |\  \  }}\  }	}
t          |j                  }t          j        |          r|j        j        n|j        }t          t          |          ||	          }t          t          |          ||
          }t          t          t          |	||                    \  }}}t          j        |          r||f||
ff}t          t          j        |t          j        |                              }t          |	          |z   |z   }t          t          t          t          j        |                              }t          j        |j        dz
  dz  |j        d         fz             }t#          |j        d d                   |j        d         dz  z    	 t%          |||            d}n# t&          $ r d}Y nw xY w|rut)          |||t+                               \  }}t#           fd|D                       }t-          | |||	          }t)          |||t+          |                    \  }}n<t/          j        | ||	          }t/          j        ||          }t5          ||          }|||fS ||f|	|ff}t          t          j        |t          j        |                              }t          t          j        t          |
          |z   |z                       }t7          ||| |||
          }||t/          j        ||          fS )Nr1   r   rk   rl   TFrI  c              3  (   K   | ]}|         V  d S rD   r4   )r5   rh   placeholder_shapes     r8   r9   z._bcoo_dot_general_transpose.<locals>.<genexpr>w  s)      GGa.q1GGGGGGr:   r  r  r  r  )r    r   r   rF   r   rL   r%   rr   rk  r   r$   r   takerl  ra   r>   emptyr   rZ  rO   rM  r   bcoo_dot_general_sampledr	   r  r_  r   r  )!r   r  r  r  r  r  r  r  r  r  r  lhs_ndimrhs_ndimlhs_keptrhs_kept	ans_batchans_lhsans_rhsdimslhs_contract_sorted_by_rhsrG  out_axesplaceholder_dataindices_can_be_untransposedr   lhs_indices_Tresult_T_shaperesult_Tr6  out_dense_T	out_denserhs_contract_sorted_by_lhsr  s!                                   @r8   _bcoo_dot_general_transposer  W  s_   #K0000	09J6< 6I!""( 4S99GSX]]sx(uXi@@(uXi@@( #D+i8*T*T U U)WgH%% (B")8!4y)6L MD!%bglBJ|<T<T&U&U!V!Vy//H,/IIKCRZ445566H y+"2Q"6$!>+BSTVBWAY!YZZk/4558I"8MPT8TT),k;HYZZZ %)!!  * * *$)!!!* # 5()9;T_0:;L0M0MO O OaGGGG;GGGGGn)"c=TXYYYh!(Mx)3N)C)CE E Eifaa OBtDDDk-X66i[)44f;##w)Y!78D!%bglBJ|<T<T&U&U!V!VBJtI1KKhVWWXXHxbZ6L157 7 7F [#-"A"AAAs   8G GGc                  | \  }}}|\  }}}t          | d d         |d d         ||d n|j        |                   \  }}	}
t          t          |j                  |j        fd|f|          \  }}t          ||	||
||          }||fS )Nr1   rU   r   r  )r]   rF   r&   r   rL   r  )rR   rS   r  r  r  r   r  rhs_bdimnew_lhs_datanew_lhs_indicesnew_lhs_spinfonew_dimension_numbersresult_batch_dimbatched_outs                 r8   _bcoo_dot_general_batch_ruler    s    )!Q.!Q2K!j!nj'ttSYx-@3B 3B 3B/, -H
:ch'!X8I-K -K))!,Q_9O4IK K K+ 
&	&&r:   cuda)platformrocmr  ABc                   |\  \  }}\  }}t          j        |          t          j        |          f}t          j        |          t          j        |          f}	t                              | ||||	f          S )ah  A contraction operation with output computed at given sparse indices.

  Args:
    lhs: An ndarray.
    rhs: An ndarray.
    indices: BCOO indices.
    dimension_numbers: a tuple of tuples of the form
      `((lhs_contracting_dims, rhs_contracting_dims),
      (lhs_batch_dims, rhs_batch_dims))`.

  Returns:
    BCOO data, an ndarray containing the result.
  r  )r   r  bcoo_dot_general_sampled_pr   )
r  r  rW   r  r  r  r  r  r  r  s
             r8   r  r    s     :K6< 6I'55'557%'	22'	224%	#	(	(Aw<A5> 
) 
K 
K Kr:   c          	     N    t          |t          j        | |||                    S )Nr  r~  )r   r	   r  )r  r  rW   r  r~  s        r8   _bcoo_dot_general_sampled_slowr    s'    	w1HYen o o o	p	ppr:   c                  ~|\  \  }}\  }}|s|s|s|rJ | j         |j         cxk    rdk    sn J |j         dz
  }	|j        d         }
|j        d         }|	|
z   dk    sJ |	dk    rY| j        |d d df                                      dd          |j        |d d df                                      dd          z  S |	dk    r5| d d d f         |j        |d                                      dd          z  S |	dk    rN| d d d d f         |d d d d f         z  }t	          j        |t          |           t          |          |fd	          S t          d
          )Nri   r1   rl   rk   r   r)  r*  .r   r   ri   r1   too many batch dimensions.)rL   rF   rt   r1  r	   r2  r   r   r  r  rW   r  r~  r  r  r  r  r7   rw   r`   r  s                r8    _bcoo_dot_general_sampled_simpler    s    9J6< 6IElEiE9EE	E	
16				Q						L1']2(b#	8	q	 	 	 	 \\DA##A#>>d7111a4=!%%6a%@@A B!||QQQW:WV_-11v!1LLLL!||
AAAtTM
QtQQQ}-
-Cc!ffc!ffc%:IFFF
1
2
22r:   c               >   |\  \  }}\  }}|s|rJ t          |          t          |          cxk    rdk    sn J | j        |j        cxk    rdk    sn J |j        dz
  }	|j        d         }
|j        d         }|	|
z   dk    sJ |	dk    r|d         dk    rdgndg}|d         dk    rdgndg}| j        t	          t          d           t          d           f|d         |d d df                                                dd          } |j        t	          t          d           t          d           f|d         |d d df                                                dd          }t          j        | |||f||ff|          S |	dk    r|d         dk    rdgndg}|d         dk    rdgndg}|j        t	          t          d           t          d           f|d         |d	                                                dd          }|d         dk    rdg}t          j        | |||f||ff|          S |	dk    rQt          j        | |||f||ff|          }t          j	        t          j
        |d
          g |j        |R d          S t          d          )Nri   r1   rl   rk   r   r)  r*  r  r  )r1   r  r  )r   rL   rF   rt   r3  rf   r1  r	   r  r2  r=   r   r  s                r8   !_bcoo_dot_general_sampled_simple2r    s    :K6< 6I%9%%	%	\		c,//	4	4	4	41	4	4	4	4	4	4	
16				Q						L1']2(b#	8	q	 	 	 	 \\#A!++!I#A!++!I	^U4[[%++6	!gaaaQRdmTTUYY_ersYttA	^U4[[%++6	!gaaaQRdmTTUYY_ersYttA?1a\<4PS\^gRh3i%.0 0 0 0\\#A!++!I#A!++!I	^U4[[%++6	!gfoVVW[[agtu[vvAA!Sl?1a\<4PS\^gRh3i%.0 0 0 0\\
/!QL,3OR[]fQg2h$-/ / /CT : :<Mci<M<M<MyYYY
1
2
22r:   c                  t          j        |           } t          j        |          }t          j        |          }|\  \  }}\  }}|j        dz
  }|j        d         }	t          j        j        }
|s=|s;|s9|s7| j        |j        cxk    rdk    r n n|	|z   dk    rt          | ||||
          S t          |          dk    r9|s7| j        |j        cxk    rdk    r n n|	|z   dk    rt          | ||||
          S t          | ||||
          S )Nr1   rl   ri   r  )r>   r   rL   rF   r	   	PrecisionHIGHESTr  r   r  r  )r  r  rW   r  r  r  r  r  r7   rw   r~  s              r8   _bcoo_dot_general_sampled_implr    sZ   	k!nn!	k!nn!K  '9J6< 6IL1']2(m#)  u< u9 u	 u
&AF



a




Hw$6!$;$;+Aq'M^jstttt!I!&AF2G2G2G2Ga2G2G2G2G2GHW^L^bcLcLc,Q7N_ktuuuu 
(1gIZfo	p	p	ppr:   c               p    t          j        fd| |          \  }t          j        d ||          \  }|S )Nc                 (    t          j        | digS )Nr  )r	   r  )argsr  s    r8   r   z9_bcoo_dot_general_sampled_abstract_eval.<locals>.<lambda>  s    cot6q_p6q6q5r r:   c                     t          |  gS rD   )r   )r!  s    r8   r   z9_bcoo_dot_general_sampled_abstract_eval.<locals>.<lambda>  s    }d7K6L r:   )peabstract_eval_fun)r  r  rW   r  dense_resultsparse_results      `  r8   '_bcoo_dot_general_sampled_abstract_evalr'    sF    &'r'r'r'rtuwxyy-,'(L(LgWcdd.-	r:   c                  t          |d          r|j        j        n|j        }t          |d          r|j        j        n|j        }t          |||          }t	          j        t          j        || j                            }t          | ||d          \  }} |d d d d}	 t	          j
        t          j                  | ||fi |	\  }}|||fS )Nr   Tr  )r  r~  r  r  )hasattrr   rF   r   r    UndefinedPrimalr   r   rn   r@  get_primitive_transposer	   dot_general_p)
r   r  r  rW   r  A_shapeB_shape	mat_shaper^   kwdss
             r8   #_bcoo_dot_general_sampled_transposer1    s    #Av..;AFLLAG'#Av..;AFLLAG'*7G=NOO)
4+Irx@@AA#'GSMMM+'20$(
 
$ 
7	#C$5	6	6r1a	H	H4	H	H$!Q	
Awr:   c               (    t          | |||          S Nr  r  )A_dotr  r  rW   r  s        r8   _bcoo_dot_general_sampled_jvp_Ar6  &  s    	!%GGX	Y	Y	YYr:   c               (    t          || ||          S r3  r4  )B_dotr  r  rW   r  s        r8   _bcoo_dot_general_sampled_jvp_Br9  )  s    	!!UGGX	Y	Y	YYr:   c               <    fd} t          ||d          |  dfS )Nc                *    t          | ||          S r3  )r  )r  r  rW   r  s      r8   implz2_bcoo_dot_general_sampled_batch_rule.<locals>.impl-  s    )!QK\]]]]r:   r   )in_axesr  r   )rR   rS   r  r<  s     ` r8   $_bcoo_dot_general_sampled_batch_ruler>  ,  sB    ^ ^ ^ ^ ^	3dJ	3	3	3\	BA	EEr:   bcoo_spdot_generalc          
         |\  \  }}	\  }
}t          j        |          t          j        |	          f}t          j        |
          t          j        |          f}t                              | |||||||f|          S )Nr  )r   r  bcoo_spdot_general_pr   )r  r  r  r  r  r  r  r  r  r  r  r  r  r  s                 r8   r  r  A  s     :K6< 6I'55'557%'	22'	224%		"	"8[(K.8Z6;U^:P 
# 
R 
R Rr:   c          	        |j         |j         t          | |          }	t          ||          }
|	j        |
j        cxk    rdk    sn J |	j        |
j        cxk    rdk    sn J fdD             fdD             k    sJ t	          d          |	j        k     sJ t	          d          |
j        k     sJ g fdt                    D             fdt                    D             R }|d d t          j        t                    f         }|d d t          j        t                    f         }|d d t          j        t          t          |	j                            t                    f         }|d d t          j        t          t          |
j                            t                    f         }|d d d f         |d d d f         k                        d          }t          j        t          j        fd	D             |j                  t          |j        d
z
                      }t          j        t          j        fdD             |j                  t          |j        d
z
                      }||k                         d          }||k                         d          }t          j        ||d d d f         z  |d d d f         z  | d d d f         |d d d f         z  d                                          }t          j        |	j        |
j        |j         d         |j         d         z   gt          j        ||                    }|j        d d d d d |j         d         f                             |d d d f                   }|j        d d d d |j         d         d f                             |d d d f                   }|                    t1          |          |j         d                   }t3          ||t5          |          |          S )Nr   c                     g | ]
}|         S r4   r4   r5   r^  	lhs_shapes     r8   r   z1_bcoo_spdot_general_unbatched.<locals>.<listcomp>W  s    	0	0	01)A,	0	0	0r:   c                     g | ]
}|         S r4   r4   r5   r^  	rhs_shapes     r8   r   z1_bcoo_spdot_general_unbatched.<locals>.<listcomp>W  s    4[4[4[aYq\4[4[4[r:   rl   defaultc              3  *   K   | ]\  }}|v	|V  d S rD   r4   )r5   rh   r   r  s      r8   r9   z0_bcoo_spdot_general_unbatched.<locals>.<genexpr>\  1      EEDAqA_,D,Da,D,D,D,DEEr:   c              3  *   K   | ]\  }}|v	|V  d S rD   r4   )r5   rh   r   r  s      r8   r9   z0_bcoo_spdot_general_unbatched.<locals>.<genexpr>]  rL  r:   rm   c                     g | ]
}|         S r4   r4   rD  s     r8   r   z1_bcoo_spdot_general_unbatched.<locals>.<listcomp>i      555y|555r:   ri   c                     g | ]
}|         S r4   r4   rG  s     r8   r   z1_bcoo_spdot_general_unbatched.<locals>.<listcomp>l  rO  r:   rE   rT   r`   )rF   r   r7   r   rP   rw   r   r>   rv   ra   r%   rr   rN   r=   rn   rL   r   r  r  r`   result_typert   ru   reshaper   _bcoo_sum_duplicatesr   )r  r  r  r  r  r  r  r  out_nser  r  r7  lhs_irhs_ilhs_jrhs_joverlaplhs_fill_valuerhs_fill_value	lhs_valid	rhs_validout_dataout_indicesrE  rH  s         ``               @@r8   _bcoo_spdot_general_unbatchedra  N  s   ))xi88#xi88#		(	(	(	(q	(	(	(	(	(	(		(	(	(	(q	(	(	(	(	(	(	0	0	0	0	0	0	04[4[4[4[?4[4[4[	[	[	[	[	_b	)	)	)CL	8	8	8	8	_b	)	)	)CL	8	8	8	8GEEEEIi((EEEGEEEEIi((EEEG G) aaa?#>>>>
?%
aaa?#>>>>
?%
aaa9U3<-@-@/#R#RZ]^^^^
_%
aaa9U3<-@-@/#R#RZ]^^^^
_%
 111d7^uT111W~-22266'?I5555_555U[III	%*q. . ?I5555_555U[III	%*q. . ~%**2..)~%**2..)Yw111d7!33iaaa6HH4(8D!!!G+<<aA AAF  	37CGU[_u{2-NO #[ I IK K K+qqq!!!%5ek"o%556::5D>JJ+qqq!!!U[_%5%556::5qqq>JJ+##CMM;3DR3HII+ 
hJY<W<W<W]d	e	e	eer:   c          
        |j         }|j         }	t          | ||          t          |||	          j        j        cxk    rdk    sn J t          | j        |j        |j        |j        ||||          \  }
}|
j         d         }|\  \  }}\  }}g |t          t          j                  |          }g |t          t          j                  |          }|                     g |t          j        | j	                            } |                    g |t          j        |j	                            }|                    g |t          j        |j	                            }|                    g |t          j        |j	                            }t          j        t          t          |j        d                    t          |	j        d                    fd|D             fd|D             |          }t          |j        t          |          z
  d          }t          |j        t          |          z
  d	          }t          |t          |                    } || |||          S )
Nr   r  rl   c                $    g | ]}|j         z
  S r4   r7   )r5   r^  r  s     r8   r   z,_bcoo_spdot_general_impl.<locals>.<listcomp>      @@@1q3;@@@r:   c                $    g | ]}|j         z
  S r4   rd  )r5   r^  r  s     r8   r   z,_bcoo_spdot_general_impl.<locals>.<listcomp>  re  r:   )r  r  r  r  rU  )NNr   r   r=  )r   r   NN)rF   r   r   !_bcoo_spdot_general_abstract_evalr   r%   rr   r7   r_  rL   	functoolsr   ra  r   r   r   )r  r  r  r  r  r  r  r  rE  rH  	data_avalr   rU  r  r  r  r  lhs_batch_permrhs_batch_permfuncr  r  s                       @@r8   _bcoo_spdot_general_implrn  |  s    ))xi88#xi88#		(	(	(	(q	(	(	(	(	(	(2M;#X]K4DjDU13 3 3,)Q OB'?P<$?O&<y) KYJ5+=+=y!I!IJ.JYJ5+=+=y!I!IJ. U. U5hm3T3T UVV( U. U5hm3T3T UVV(%%&^&^s{KL\9]9]&^__+%%&^&^s{KL\9]9]&^__+ 
	8Ickll344Ickll344@@@@@@@@@@@@@@
 
 
$ 
D#+I6@R	S	S	S$	D#+I6@R	S	S	S$	D#i..	)	)$	hX{	;	;;r:   c          	         |j         |j         }t          j        t          j        d                              t          j         j                  t          j        |j                  ||          }	t                     }
t          |          }|\  \  }}\  |
j	        s|j	        rt          d          rt                    |
j        k    sr't                    |j        k    rt          d          |rGt          |          |
j        k     s t          |          |
j        |
j        z   k    rt          d          |rGt          |          |j        k     s t          |          |j        |j        z   k    rt          d          |j        t                    k    r'|
j        t          |          k    rt!          d          |
j        t          |          k    r|
j        nd|j        t          |          k    r|j        ndz  }|
j        |j        z   t                    z
  }t          |t%          j        |	j         |d                              }t)          j        t-           fdt/          |
j                  D                       t-          fd	t/          |
j                  D                                 }t)          j        t-          fd
t/          |j                  D                       t-          fdt/          |j                  D                                 }g fdD             |||R }g fdD             ||||
j        |j        z   dt          |          z  z
  R }t1          j        ||	j                  }t1          j        |j                  }t          |||	j                    ||fS )Nr  r  z)bcoo_spdot_general with dense dimensions.z`bcoo_spdot_general: batch_dims must correspond to batch dimensions of the sparse representation.z?bcoo_spdot_general only supports contraction of sparse indices.zXbcoo_spdot_general: cannot have unused batch dims on rhs with unused sparse dims on lhs.ri   c              3  :   K   | ]}|vj         |         V  d S rD   rE   )r5   rH   r  r  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  4      	T	T#s)?S?S(.
?S?S?S?S	T	Tr:   c              3  :   K   | ]}|vj         |         V  d S rD   rE   )r5   rH   r  r  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  5      	W	WS#YBVBV+
C
 BVBVBVBV	W	Wr:   c              3  :   K   | ]}|vj         |         V  d S rD   rE   )r5   rH   r  r  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  rq  r:   c              3  :   K   | ]}|vj         |         V  d S rD   rE   )r5   rH   r  r  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  rs  r:   c              3  (   K   | ]}|         V  d S rD   r4   r5   rH   rE  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  '      **in******r:   c              3  (   K   | ]}|         V  d S rD   r4   rw  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  rx  r:   r1   )rF   r  r  r	   r  r  r  rn   r   r   rO   rP   r7   minrw   r   r   r`   mathr  r   broadcast_shapesr   rr   r   r   )r  r  r  r  r  r  r  r  rH  r  r  r  r  r  rU  out_n_batchlhs_batch_shaperhs_batch_shaper  rc  rj  indices_avalr  rE  r  s   ````                  @@@r8   rh  rh    sF    ))WS_6efffqq	9hn55	9hn55)3	 r 5 5( 	xi88#xi88#?P<$?O&<y)[ KCK K
I
J
JJ BC	NNck11y1S^^WZWbEbEb
  A  B  B  B a#o..<<O@T@TX[XcfifrXr@r@r
_
`
`` a#o..<<O@T@TX[XcfifrXr@r@r
_
`
``[3y>>!!clS5I5I&I&I
o
p
pp s?3333SWWs?3333SWW< 
 ck)C	NN:+8>+,,#?@@AA''		T	T	T	T	Ts{););	T	T	TTT		W	W	W	W	WE#+,>,>	W	W	WWW / '		T	T	T	T	Ts{););	T	T	TTT		W	W	W	W	WE#+,>,>	W	W	WWW /
****	***  	 *
E****	***EE E 	E \CL(1s?/C/C+CC	E E- z8>::)!-1BCC,L(.999	L	  r:   c          
        t          |j                  }t          |j                  }t          d t          | |          D                       }t	          | d d         |d d         ||          \  }	}
}t	          | dd          |dd          ||          \  }}}t          ||fd|          \  }}t          |	|
||||||          }|||ffS )Nc              3  :   K   | ]\  }}||j         |         V  d S rD   rE   rG   s      r8   r9   z1_bcoo_spdot_general_batch_rule.<locals>.<genexpr>  s2      cchc3SVSb39S>SbSbSbSbccr:   r1   r   r   r   )r  r  r  r  )r   rF   rP   rQ   r]   r&   r  )rR   rS   r  r  r  r  r  r  rU   r  r  r  r  r  r  s                  r8   _bcoo_spdot_general_batch_ruler    s   !""(!""(cc#lJ2O2Occccc*&?!j!njZ'I 'I 'I#(K&?jnjZ'I 'I 'I#(K(CF$5)7 )7%%#Hk8[6G/9j;QS S S+ 
')9:	::r:   c                   | \  }}}}|\  }}}	}
t          | i |}t          |          t          j        u sJ t          |
          t          j        u sJ d}t          |          t          j        ur|t          ||||fi |d         z  }t          |	          t          j        ur|t          |||	|fi |d         z  }||t          j                            |d                   gfS )Nr   ri   )r  r
  r    r  r  )r  r  r0  r  r  r  r  r  lhs_indices_dotrhs_data_dotrhs_indices_dotr  rf  s                r8   _bcoo_spdot_general_jvpr    s	   18.(K;AI>,#W555+	o		"'	)	)	)	)	o		"'	)	)	)	),	,rw&&'k8[aa\`aabcddL	,rw&&'+|[aa\`aabcddL	|RW%>%>{1~%N%NO	OOr:   bcoo_sort_indicesc                |    t          j        | j        d| j        i\  }}t	          ||f| j        d| j                  S )zxSort indices of a BCOO array.

  Args:
    mat : BCOO array

  Returns:
    mat_out : BCOO array with sorted indices.
  rT   Tro   )bcoo_sort_indices_pr   _bufsr   r_   rF   rK   )r^   rV   rW   s      r8   r  r    sJ     &*CIHciHH-$	tWoSYt /
1 
1 
1 1r:   c                   t          | ||j                  }|j        dk    r| |fS t          t          |j        d          } ||          \  }}t          d |j                  } || |          } | |fS )Nr   Fr   c                    | |         S rD   r4   r^  rL  s     r8   r   z)_bcoo_sort_indices_impl.<locals>.<lambda>'  
    AaD r:   )r   rF   rw   r   _bcoo_sort_indices_unbatchedr7   )rV   rW   rT   r   r   permpermutes          r8   _bcoo_sort_indices_implr     s    
w
5
5%
^q=-u}%PPP!!G**-'4((%-88'	t		$	wr:   c                      j         \  }} fdt          |          D             }t          j        g |t          j         j        |          R |          ^  }t          j                   |fS )Nc              3  0   K   | ]}d d |f         V  d S rD   r4   r5   rh   rW   s     r8   r9   z/_bcoo_sort_indices_unbatched.<locals>.<genexpr>.  s/      ..gaaadm......r:   )num_keys)rF   rr   r	   sortiotarn   r>   column_stack)rW   r`   r   idx_colsr  s   `    r8   r  r  +  sx    =&#q....U1XX...(8EhE(D(DEEPQRRR.7D		'	"	"D	((r:   c          
     H   t          | ||j                  }|j        dk    r| |fS t          j        g t          t          |j        d |j                 | j        d |j                           |j        | j        |j        dz   d          R | j	        | j
                  }||fS )Nr   ri   	weak_type)r   rF   rw   r   r   rk  rP   r7   r`   rn   r  )rV   rW   rT   r   data_outs        r8    _bcoo_sort_indices_abstract_evalr  2  s    
w
5
5%
^q=1c#w}^em^,dj%-.HII 1
Y1EMA-../1 126*X X X( 
7	r:   c                   t          | ||          \  }}}t                              |||          \  }}d}|d         
|d         }d}||f|fS )Nr   r  ri   r   r  )r]   r  r   )rR   rS   rT   rV   rW   r  indices_outr  s           r8    _bcoo_sort_indices_batching_ruler  <  sh    3L*fUU$-2242PP(K( ]a.KH
K	 (	**r:   c                  t          g | |j        R  }|j        dk    r| |fS | \  }}|\  }}t          t          |j                  } ||          \  }	}
t          d |j                  } |||
          }t          j                            |          }t          |          t          j        u rt          j                            |          n |||
          }||	f||ffS )Nr   c                    | |         S rD   r4   r  s     r8   r   z(_bcoo_sort_indices_jvp.<locals>.<lambda>P  r  r:   )
r   rF   rw   r   r  r7   r    r  r  r
  )r  r  rT   r   rV   rW   r   r   r   r  r  r  r  indices_dot_outrf  s                  r8   _bcoo_sort_indices_jvpr  G  s    

0'
06<
0
0
0%
^qH-$+(A-u}==!ajj+t((%-88'WT4  (G--g66/8<X"'8Q8Q**8444W^W^_gimWnWn,
K	 <"A	AAr:   r   c                    t          | j        | j        | j        |          \  }}t	          ||f| j        dd          S )a  Sums duplicate indices within a BCOO array, returning an array with sorted indices.

  Args:
    mat : BCOO array
    nse : integer (optional). The number of specified elements in the output matrix. This must
      be specified for bcoo_sum_duplicates to be compatible with JIT and other JAX transformations.
      If not specified, the optimal nse will be computed based on the contents of the data and
      index arrays. If specified nse is larger than necessary, data and index arrays will be padded
      with standard fill values. If smaller than necessary, data elements will be dropped from the
      output matrix.

  Returns:
    mat_out : BCOO array with sorted indices and no duplicate indices.
  rQ  Tro   )rT  rV   rW   r   r_   rF   rx   s       r8   r   r   f  sK     'sxSYTWXXX-$	tWoSYt!
# 
# 
# #r:   c                   | t          j        t          j        |d          }t                              | |||          S )N$nse argument of bcoo_sum_duplicates.rQ  )r   r   rp   rq   bcoo_sum_duplicates_pr   )rV   rW   rT   r`   s       r8   rT  rT  y  s;    _

 6\
]
]C		#	#D'&c	#	J	JJr:   c          
     X   t          | ||j                  }t          ||j        dd          \  }}}|!|j        dk    rdn|                                }t          |||j                  }|j        dk    r|                     |j        d          } t          j	        g t          t          |j        d |j                 | j        d |j                           || j        |j        dz   d          R | j                  }d }	t          |	|j                  }	 |	|||           }||fS )	NTrF   return_inversereturn_true_sizer   ri   r`   rF   r   rm   c                F    | j         |                             |d          S Ndrop)r+  rt   r   )d_outr   r^  s      r8   r   z+_bcoo_sum_duplicates_impl.<locals>.<lambda>  s     ? ? r:   )r   rF   r0  rw   rP   _adjust_indices_nser   r7   r>   r  rk  rn   r   )
rV   rW   rT   r`   r   r  mappingnse_batchedr  r  s
             r8   _bcoo_sum_duplicates_implr  ~  sL   
w
5
5%&56<t'M 'M 'M#+w[~""!!(9(9C#KSMMM+
^q88EMD811DY >S'-"?NU]NA[\\ >>"j):););<> >EIZQ Q Q(??'w..'WXw--(	;	r:   c                  t          | |          }||j        k    r| dd |d d f         } nt          j        t	          j        ||j        |j        |j        z            | j                  g | j	        d d         ||j        z
  | j	        d         R | j
        dz
  f          }t          j        | |g| j
        dz
            } | S )	N.rm   rk   rl   ri   )operandrF   broadcast_dimensionsr1   	dimension)r   r`   r	   r2  r>   rv   r7   rw   rn   rF   rL   concatenate)rW   r`   rF   r   r)  s        r8   r  r    s    
 %
0
0%EIc4C4l#GGiemEMEN,JJKSZS`aaaEgmCRC E#	/E7=3DEE#L1,.  D
 owo9IJJJG	.r:   )r  r%  r  c                   t          | |          }t          t          ||j        d          |||          }t	          ||j        d          } ||           S )N)rF   r  r%  r  Fr  )r   r   _unique_indices_unbatchedr7   r   )rW   rF   r  r%  r  r   r   s          r8   r0  r0    sc    
 %
0
0%'uU]^^/D+,/1 1 1! EMu555!	
7r:   c          	     T   t          | |          }|j        dk    r}d}t          j        | |df          }|f}|rg |t          j        |d          R }|rg |t          j        |d          R }|rg ||R }t          |          dk    r|d         n|S t          j        t          j        |d |j                 | j                  d          }	| |	k    	                    dd	          }
t          j
        |
|	|           } t          | d||||j        |	
          }|r:|rdnd}g |d |         ||                                         ||dz   d          R }|rJ|d         }|| |	k    	                                                    |j                  z
  }g |d d         |R }|S )Nr   ri   rE   int32rm   )r   rl   Tr   )axisr  r%  r  r   r   r1   )r   rw   r>   rs   r   r   r=   rv   rn   r   r   r+   r`   r  r  )rW   rF   r  r%  r  r   r`   r  r  r   out_of_boundsr  s               r8   r  r    s   
 %
0
0%
^q
C.a999K.C 21c139S00011c 21c139S00011c cK3KKcXX]]3q66+sy)?w}UUUW[\\*j(--b4-@@-ImZ99'a\!1	j	Z 	Z 	Z# 9
"!!C
8CI
8s3x~~''
8#cAghh-
8
8C 
b'C
J&++--44SY??
?C
CH
c

C	*r:   c                   | j         |j         k    sJ | j          d|j                      t          |          | j         dz   k    s$J t          |           d| j         dz                | j         dk    rCt          t          || j         d          |          } t	          || j                   | |          S | |d         k    ||d         k    z  }|rFt          j        |d|           } t          j        ||d         |          }|d         |d         dz   f}nEt          j        ||d         |           } t          j        |d|          }|d         dz   |d         f}| ||fS )Nz != ri   )rF   r_  r   )rL   r   r   r  r   r>   r   )r  r  rF   r_  r   r   s         r8   r  r    sr    
SX			#(::::				Usx!|	#	#	#E

%F%F1%F%F	#	#	#X\\1CHII&)	= 	= 	=A":a""3,,,
q/cU1Xo	.$ %
)D!S
!
!C
)D%(C
(
(C1XuQx!|$EE
)D%(C
(
(C
)D!S
!
!C1X\58$E	c5r:   c          
        |t          d          t          | ||j                  }t          j        g |j        d |j                 ||j        R |j        |j                  }t          j        g t          t          |j        d |j                 | j        d |j                           || j        |j        dz   d          R | j        | j                  }||fS )Nbcoo_sum_duplicates: nse must be specified when using the function within jit, vmap, and other transformations requiring abstract evaluation.)rn   r  ri   r  )r   r   rF   r   r   r7   rw   rn   r  rk  rP   )rV   rW   rT   r`   r   r  r  s          r8   "_bcoo_sum_duplicates_abstract_evalr    s   [
 [ \ \ \
w
5
5% !V7=%-#@!V#!Vu~!V!V(/ART T T++c#w}^em^,dj%-.HII ++:ema'(()+ +,0J$.R R R( 
;	r:   c                   t          | ||          \  }}}t                              ||||          \  }}|d         t          j        |dg          }d}	nd}	||f|	fS )NrQ  ri   r   r  r  )r]   r  r   r	   rx  )
rR   rS   rT   r`   rV   rW   
new_spinfor  r  r  s
             r8   "_bcoo_sum_duplicates_batching_ruler    sx    7jRXYY$/44T7:[^4__(K ]+kA3//KHHH
K	 (	**r:   c          
        t          g | |j        R  }| \  }}|\  }}t          ||j        dd          \  }	}
}|t          j        |          }	 t          j        t          j        |d          }n"# t
          j	        $ r t          d          w xY wt          |	||j                  }	|j        dk    r8|                    |j        d          }|                    |j        d          }t          j        g t          t           |j        d |j                 |j        d |j                           ||j        |j        dz   d          R |j        	          }|}|j        rd
 }nd }t'          ||j                  } |||
|          }t(          j                            |	          }t/          |          t(          j        u rt(          j                            |          n |||
|          }||	f||ffS )NTr  r  r  r  r   r   ri   rm   c                F    | j         |                             |d          S r  r  xrh   ys      r8   r   z*_bcoo_sum_duplicates_jvp.<locals>.<lambda>  s    ad1gkk!&k99 r:   c                    | S rD   r4   r  s      r8   r   z*_bcoo_sum_duplicates_jvp.<locals>.<lambda>
  s    a r:   )r   rF   r0  r>   r   r   r   rp   rq   ConcretizationTypeErrorr   r  rw   r7   r  rk  rP   rn   r   r   r    r  r  r
  )r  r  rT   r`   r   rV   rW   r   r   r  r  r  r  rf  r  r  s                   r8   _bcoo_sum_duplicates_jvpr    sC   

0'
06<
0
0
0%-$+(A&56<t'M 'M 'M#+w[
'+

C\

 6\
]
]CC		% \ \ \
 [ \ \ \\ $KSMMM+
^q88EMD811D||EMD|99HY >S'-"?NU]NA[\\ >>"j):););<> >EIZQ Q Q(, ]  99GGGw..'WXw--(G--k::/8<X"'8Q8Q**8444W^W^_kmtv~WW,
K	 <"A	AAs    A2 2Berrorr7   r   on_inefficientr  
str | Nonec          	       
 | j         nt          j                  | j        nt          j                  f| j         | j        fk    r| S | j        z
  z
  }dvrt          d          dk     rt          d           dk     rt          d           |dk     rt          ddd	| j         d
          fd}| j        k    rSt          d | j         | j        | j        z
           D                       r |d| j         d| j         d d           | j         k    rJt          d | j        | j                  D                       r |d| j         d| j          d d           | j        | j	        }}| j        | j         
| j        }|k     r|z
  t          t          
dz             fd            }	 |	||          \  }} |j        g |j        d
         t          j        |j        

dz                      |j        
dz   d         R  } |j        g |j        d
         t          j        |j        

dz                      |j        
dz   d         R  }}
k     r7
z
  t          t                    fd            }	 |	||          \  }}
|k    r<|z
  t          t          
dz             fd            }	 |	||          \  }}}
k    r;
z
  t          t          
          
fd            }	 |	||          \  }}
t          ||f          S )a  Update the storage layout (i.e. n_batch & n_dense) of a BCOO matrix.

  In many cases this can be done without introducing undue storage overhead. However,
  increasing ``mat.n_batch`` or ``mat.n_dense`` will lead to very inefficient storage,
  with many explicitly-stored zeros, unless the new batch or dense dimensions have size
  0 or 1. In such cases, ``bcoo_update_layout`` will raise a :class:`SparseEfficiencyError`.
  This can be silenced by specifying the ``on_inefficient`` argument.

  Args:
    mat : BCOO array
    n_batch : optional(int) the number of batch dimensions in the output matrix. If None,
      then n_batch = mat.n_batch.
    n_dense : optional(int) the number of dense dimensions in the output matrix. If None,
      then n_dense = mat.n_dense.
    on_inefficient : optional(string), one of ``['error', 'warn', None]``. Specify the
      behavior in case of an inefficient reconfiguration. This is defined as a reconfiguration
      where the size of the resulting representation is much larger than the size of the
      input representation.

  Returns:
    mat_out : BCOO array
      A BCOO array representing the same sparse array as the input, with the specified
      layout. ``mat_out.todense()`` will match ``mat.todense()`` up to appropriate precision.
  N)r  r  NzJon_inefficent={on_inefficient!r}; expected one of ['error', 'warn', None].r   z"n_batch must be non-negative; got z"n_dense must be non-negative; got zsum of n_batch=z and n_dense=z  cannot be larger than mat.ndim=r;   c                    dk    r| dz  } t          |           dk    r"| dz  } t          j        | t                     d S d S )Nr  z` To disable this error, set the on_inefficient argument of bcoo_update_layout to 'warn' or None.r  zX To disable this warning, set the on_inefficient argument of bcoo_update_layout to None.)category)r   r  r  r   )msgr  s    r8   _maybe_err_or_warnz.bcoo_update_layout.<locals>._maybe_err_or_warnN  si      	 9 :c!#&&&	6	!	!	 / 0cmC"9:::::: 
"	!r:   c              3  "   K   | ]
}|d k    V  dS r}   r4   r  s     r8   r9   z%bcoo_update_layout.<locals>.<genexpr>Z  s&      	D	DA!a%	D	D	D	D	D	Dr:   zFor matrix of shape z, increasing n_dense from z to z  results in inefficient storage.c              3  "   K   | ]
}|d k    V  dS r}   r4   r  s     r8   r9   z%bcoo_update_layout.<locals>.<genexpr>]  s&      "Q"QQ1q5"Q"Q"Q"Q"Q"Qr:   z, increasing n_batch from ri   r   c                ~    | j         t          j        | j        d                    g| j        d          R  }t	          j        fd| j        d          D             ddi}t	          j        t	          j        |j        d         j        f          gt          t          j
        |                    }||fS )Nc              3  N   K   | ]}t          j        |j                   V   dS rm   Nr>   r   rn   )r5   r   rh   s     r8   r9   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>k  s4      PPqcj!':::PPPPPPr:   r   r   r   )rS  r{  r  rF   r>   r   r  broadcast_tor   rk  r  )r^  rh   new_dmeshesnew_ir  s    `   r8   _updatez#bcoo_update_layout.<locals>._updateh  s    ai	!'"1"+..====e|PPPPAGBQBKPPP +%)+ +f 0U[^QV4L M M  9!$SY!7!7 9 : :eE\r:   r1   c                   j         d         } | j        t          j        | j         d dz                      g| j         dz   d          R  }t	          j        fdg j         d          |R D             ddi} j        t          j        j         d dz                      gj         dz   d          R  }t	          j        g d |d d         D             |R           }||fS )Nrk   ri   c              3  N   K   | ]}t          j        |j                   V   dS r  r  )r5   r^  rh   s     r8   r9   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>  s4      XXqcj!':::XXXXXXr:   r   r   c              3  >   K   | ]}|                                 V  d S rD   )r  )r5   r   s     r8   r9   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>  s*      !A!A!''))!A!A!A!A!A!Ar:   rl   )rF   rS  r{  r  r>   r   r  )r^  rh   r`   r  r  r  r  s    `    r8   r  z#bcoo_update_layout.<locals>._update{  s   GBKcai	!'&1q5&/22EQWQUVV_EEEe|XXXXDWagbqbkDWSVDWDWXXX +%)+ +fai	!'&1q5&/22EQWQUVV_EEEeI!A!AVCRC[!A!A!AI5IIJJeE\r:   c                    t          j        |  d                    j        t          | d                                                 |           }|d           }||fS r-  )r>   rs   rt   r   ru   )r^  rh   r  r  r  r   rF   s       r8   r  z#bcoo_update_layout.<locals>._update  sa    nQeWHII&6777:5A233==IMMaPPe!feE\r:   c                   j         d         }t          fdt          	          D                       t          j        |          fz   }g 
         |j         d         	z
  R }t          j        d d 	d f         |          }g 
         || j         | j        z
  d          R }t          j        | |          j        |         	                    |           }||fS )Nrk   c              3  0   K   | ]}d d |f         V  d S rD   r4   )r5   jrh   s     r8   r9   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>  s/      ,,a!AAAqD',,,,,,r:   rl   rE   )
rF   r   rr   r>   r   r  rL   rs   rt   ru   )r^  rh   r`   r  new_i_shaper  new_d_shaper  current_n_batchr  r7   r   rF   s    `      r8   r  z#bcoo_update_layout.<locals>._update  s    GBKc,,,,588,,,,,
3/AAcKeOG34KcK172;?KKkqABBx55eWeOG34WcWAGAFWDTDUDU<VWWknQk2225c:>>qAAeE\r:   rE   )r7   rp   rq   r   rL   r   r   rF   rV   rW   r   r   rS  r{  r  r_   )r^   r7   r   r  rw   r  new_datanew_indicescurrent_n_denser  r  r  rF   s    ```      @@@r8   bcoo_update_layoutr    s5   B #?CKKw0G0G'"?CKKw0G0G'wCK555JX')(222
a
b
bbq[[
C'CC
D
DDq[[
C'CC
D
DD\\
 Cw C Cg C C7:xC C C D D D; ; ; ; ; 		D	DG8CHs{,B#BC	D	D	DDD  Uci U U+U U+2U U U V V Vs{s"Q"Q#)CK<O2P"Q"Q"QQQ Uci U U+U U+2U U U V V V (CKK(
)%K/K/'!AZ?Q.///    0/ $GHk::Hkx G0@0@!A G $	(.[\I\9\*] ^ ^G!)!0C0D0D!EG G GH &+% P{'89I/9I'J P&*i0A/SbefSfBf0g&h&hP'2'819L9M9M'NP P PK O'!AZ7###    $# $GHk::HkO/!AZ?Q.///      0/ $GHk::HkO/!AZ?+++        ,+ $GHk::HkO	x%U	3	3	33r:   )shardingr   r  c          	     f    t          t          | j        | j        | j        ||          |          S )a  Expand the size and rank of a BCOO array by duplicating the data.

  A BCOO equivalence to jax.lax.broadcast_in_dim.

  Args:
    mat: A BCOO-format array.
    shape: The shape of the target array.
    broadcast_dimensions: The dimension in the shape of the target array which
      each dimension of the operand (``mat``) shape corresponds to.

  Returns:
    A BCOO-format array containing the target array.
  )rT   rF   r  rE   )r_   _bcoo_broadcast_in_dimrV   rW   r   )r^   rF   r  r  s       r8   bcoo_broadcast_in_dimr    sG     
$SXs{39+0:NP P P 
 
 
 r:   c                  t          |j                  t          |          k    rt          d|j        d|d          t          | ||j                  }t	          ||j        |j        g          \  }}}t          |d          t          |t          |                    k    rt          d          t          |d          t          |t          |                    k    rt          d          t          ||j        d         t          |                    }	|j	        o,t          |          t          ||j	         d                   z
  }
t          |          |	z
  |
z
  t          j        |j        |j        |j        |j        z                      t          j        ||	|	z                      k    rt          d	          | |}}|	|
f|j        |j	        fk    rt          j        |g |d|	         |j        ||	z   d         R g ||	fd
|D             R           }t          j        |g |d|	         |j        |j        R g ||	|	dz   R           }|j        k    rig |d|	         |j        R }t!          j        |t$                    |	z
  }t!          j        ||          j        d|f                             |          }||fS )z'BCOO equivalent of lax.broadcast_in_dimzspinfo.shape=z and broadcast_dimensions=z must have the same lengthr   rI  z>Cannot mix batch and sparse dimensions during broadcast_in_dimz>Cannot mix sparse and dense dimensions during broadcast_in_dimNz*Adding sparse dimensions with lengths != 1c              3  (   K   | ]}|d z   z
  V  dS r}   r4   )r5   r6   new_n_sparses     r8   r9   z)_bcoo_broadcast_in_dim.<locals>.<genexpr>  s-      9c9cST!a%,:N9c9c9c9c9c9cr:   )rF   r  ri   rE   .)r   rF   r   r   r   r7   rw   rP   rz  r   r{  r  rO   r	   r2  r`   r>   rv   ra   rs   rt   ru   )rV   rW   rT   rF   r  r   rS   sparse_dims
dense_dimsnew_n_batchnew_n_denser  r  r8  r  s                 @r8   r  r    sC    	#23333
]]],@]]]
^
^^
w
5
5%(23G%-Y^YgIh(i(i%*k:Q#k3u::"F"F"FFF
U
V
VVa   3z3u::#F#F#FFF
U
V
VV (8#e**MMM+Y#e**s3G3X/Y/Y"Y+Uk)K7,	Yv|EM5=5>+IIJKKtyY^_jkv  zF  lF  `F  ZG  PH  PH  H  H
J
K
KKK( ;EM5=#AAA#HTl{l#TUYT{\7Q7R7R1STTdzd;d9c9c9c9cXb9c9c9cddf f fH &{?l{l#?UY???HzH;HaHHJ J JK
 U^##;eL[L!;59;l;;E
)K
%
%
3C>+U;;;>sCxHLL[YYK	;	r:   operandsSequence[BCOO]r  c                  t          j                  t          d | D                       st          d|            t	          j        t          j        d                              d | D                       }t          d | D                       dk    rt          d          d	 | D             }t          |          dk    rjt          |          dk    rWt          d
 | D                       rd | D             } n%t          d | D                       rd | D             } d | D             }t          |          dk    rt          d          | d         j        | d         j        c}fd| D             }fd| D             }k     rfd|D             }fd|D             }t          t          |                    t          t          |                    cxk    rdk    sn t          d          k     r}d | D             }t          |          dk    rt          |          fd| D             } t          j        d | D                       }t          j        d | D                       }	n|z   k     rt          j        dgfd| dd         D             z   | d         j        j                  }
t          j        d | D                       }	t          j        fdt'          | |
          D                       }nt          d           t)          |	|f|j        !          S )"aP  Sparse implementation of :func:`jax.lax.concatenate`

  Args:
    operands : Sequence of BCOO arrays to concatenate. The arrays must have equal
      shapes, except in the `dimension` axis. Additionally, the arrays must have
      have equivalent batch, sparse, and dense dimensions.
    dimension : Positive integer specifying the dimension along which to concatenate
      the arrays. The dimension must be among batch or sparse dimensions of the input;
      concatenation along dense dimensions is not supported.

  Returns:
    A BCOO array containing the concatenation of the inputs.
  c              3  @   K   | ]}t          |t                    V  d S rD   )r  r_   r5   ops     r8   r9   z#bcoo_concatenate.<locals>.<genexpr>  s,      55bZD!!555555r:   zIbcoo_concatenate: expected operands to be a sequence of BCOO arrays. Got r  r  c                L    g | ]!}t          j        |j        |j                  "S r4   )r   r   rF   rn   r
  s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>  s)    
C
C
CB4BHbh//
C
C
Cr:   c                    h | ]	}|j         
S r4   )r   r
  s     r8   	<setcomp>z#bcoo_concatenate.<locals>.<setcomp>  s    	(	(	("*	(	(	(r:   ri   zAbcoo_concatenate requires inputs to have matching nse dimensions.c                    h | ]	}|j         
S r4   rd  r
  s     r8   r  z#bcoo_concatenate.<locals>.<setcomp>  s    ---brz---r:   c              3  N   K   | ] }|j         d k    |j        d          dk    V  !dS )r   ri   Nr7   rF   r
  s     r8   r9   z#bcoo_concatenate.<locals>.<genexpr>  s3      
B
B"*//28A;!////
B
Br:   c                J    g | ] }|j         d k    rt          |d          n|!S )r   ri   rd  r7   r  r
  s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>  6    bbbSUrzQ$R3333Bbbbr:   c              3  N   K   | ] }|j         d k    |j        d         d k    V  !dS ri   r   Nr  r
  s     r8   r9   z#bcoo_concatenate.<locals>.<genexpr>  s3      DD"BJ!OORXa[AOOOODDr:   c                J    g | ] }|j         d k    rt          |d          n|!S )ri   r   rd  r  r
  s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>  r  r:   c                    h | ]	}|j         
S r4   rd  r
  s     r8   r  z#bcoo_concatenate.<locals>.<setcomp>  s    //////r:   zCbcoo_concatenate requires inputs to have matching batch dimensions.r   c                8    g | ]}|j         j        d          S rD   rW   rF   r5   r  r7   s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>  s'    AAA"2:#HWH-AAAr:   c                8    g | ]}|j         j        d          S rD   )rV   rF   r  s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>  s&    ===b"'-)===r:   c                @    g | ]}|d          |dz   d          z   S Nri   r4   r5   r   r  s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>	  s3    NNN1Qz	z]Qy1}~~%66NNNr:   c                @    g | ]}|d          |dz   d          z   S r  r4   r  s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>
  s3    LLL!AjyjMAi!mnn$55LLLr:   z6concatenation of arrays with broadcasted batch indicesc                    h | ]	}|j         
S r4   r   r
  s     r8   r  z#bcoo_concatenate.<locals>.<setcomp>  s    &&&rBF&&&r:   c                0    g | ]}t          |          S r4   )ry   )r5   r  r`   s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>  s#    <<<R-C((<<<r:   c                    g | ]	}|j         
S r4   )rW   r
  s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>  s    "A"A"A"2:"A"A"Ar:   c                    g | ]	}|j         
S r4   rV   r
  s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>      ;;;B;;;r:   c                *    g | ]}|j                  S r4   rE   )r5   r  r  s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>  s     KKKrrx	2KKKr:   Nrl   rm   c                    g | ]	}|j         
S r4   r%  r
  s     r8   r   z$bcoo_concatenate.<locals>.<listcomp>  r&  r:   c                j    g | ]/\  }}|j         j        d z
  f                             |          0S ).)rW   rt   r   )r5   r  offsetr  r7   s      r8   r   z$bcoo_concatenate.<locals>.<listcomp>  sT     #R #R #R'1r6 $&:=i'6I1I#J#N#Nv#V#V #R #R #Rr:   z%Concatenation along dense dimensions.rE   )rp   rq   rN   r   r  r  r	   r  r  r   rP   r7   rw   ru   rO   r   cumsumrW   rn   r   r_   rF   )r  r  r  	n_batchesrw   index_batchesdata_batchesnsesr  r  offsetsr7   r`   s    `         @@r8   bcoo_concatenater1    s    nY'')	55H555	5	5 (
 '$' ' ( ( ( WS_nEEEPP
C
C(
C
C
C Q  ( 		(	(x	(	(	())A--
X
Y
YY--H---) 	^^qS^^q00

B
Bh
B
B
BBB cbbYabbbhh	DDDDD	D	D cbbYabbbh//h///I^^q
Z
[
[[qk)8A;+?'8AAAAAAA-====H===,NNNNNNNMLLLL|LLLL
c-  
!
!S\):):%;%;
@
@
@
@q
@
@
@
@
V
W
WW&&X&&&D
4yyA~~IIc<<<<8<<<h/"A"A"A"A"AYWWWK;;(;;;yQQQHH7X%%%iKKKKXcrc]KKKK&qk179 9 9G;;(;;;wOOOH/ #R #R #R #R #R5=h5P5P#R #R #R,35 5 5KK E
F
FF	x%X^	<	<	<<r:   )
dimensionsr  	new_sizesr2  Sequence[int] | Nonec          	     d     j         j        d j                  j        j        d j                 cxk    r j        d j                 k    rn nt	          d          t           j         j         j        g          \  }}}t           j         j         |p t          t           j
                             j                  \  }}}	t          j        |          }
t          j        |          }t          j        |          }|
dk    r&|
|vr"t          d j         d| d j                   |dk    r)|
|z  |vr"t          d j         d| d j                   |                    |
d	
          }|                    |
|z  d	
          }t          |t%          |          t%          |          g          \  }}}t'          j         j        g | j        |R g | j         fd|	D             R           }t'          j         j         g | j         j        R g | j         j        dz   R           |s$t-          j        g | j        dR           n3|r0t          fd|D                       } fd|D             }t-          j        |t          |          d          }t3          j        d          5  t-          j         j         j         j         j        z            j                  k                        dd          }ddd           n# 1 swxY w Y   t-          j        ||          }t-          j        d |D             d          t-          j         |t-          j        |j                            tC          |f|          S )a  Sparse implementation of {func}`jax.lax.reshape`.

  Args:
    operand: BCOO array to be reshaped.
    new_sizes: sequence of integers specifying the resulting shape. The size
      of the final array must match the size of the input. This must be specified
      such that batch, sparse, and dense dimensions do not mix.
    dimensions: optional sequence of integers specifying the permutation order of
      the input shape. If specified, the length must match ``operand.shape``.
      Additionally, dimensions must only permute among like dimensions of mat:
      batch, sparse, and dense dimensions cannot be permuted.

  Returns:
    out: reshaped array.
  Nz4reshape of arrays with broadcasted batch dimensions.ri   zJbcoo_reshape: new shape cannot mix batch and sparse dimensions; got shape=z new_shape=z with n_batch=zJbcoo_reshape: new shape cannot mix sparse and dense dimensions; got shape=z with n_dense=right)sidec              3  2   K   | ]}|j         z   d z   V  dS r}   rd  rK  s     r8   r9   zbcoo_reshape.<locals>.<genexpr>P  s.      +T+TAAOa,?+T+T+T+T+T+Tr:   r3  r2  r   rE   c              3  ,   K   | ]}d |f         V  dS r   r4   r  s     r8   r9   zbcoo_reshape.<locals>.<genexpr>Y  s+      <<1wsAv<<<<<<r:   c                T    g | ]$}t          j        j        |z                      %S r4   )ra   rF   r7   )r5   rh   r^   s     r8   r   z bcoo_reshape.<locals>.<listcomp>Z  s.    IIIC	#+/233IIIr:   clip)r  r+  allowrm   rl   Tr   c                    g | ]
}|d          S )r   r4   )r5   r  s     r8   r   z bcoo_reshape.<locals>.<listcomp>`  s    HHH#s9~HHHr:   )r  )"rW   rF   r7   rV   rO   r   rw   rZ  r   rr   rL   r{  r  r   cumprodr   r   searchsortedra   r	   rS  r`   r>   
empty_likeravel_multi_indexr  numpy_rank_promotionrv   rn   r   unravel_indexr  r   r_   )r^   r3  r2  r  batch_shapesparse_shapedense_shaperW  rX  rY  rU   sparse_size
cuml_shapei1i2new_batch_shapenew_sparse_shapenew_dense_shaperV   
index_colsflat_indicesoob_indicesnew_index_colsrW   s   `                      @r8   bcoo_reshaperS  %  s   $ 
k%)E````S_TWT_S_I``````
T
U
UU+5ci#+s|A\+]+](+|[(=Hck:?uSX)?)?)L )L%*k:y%%*	,''+z)$$*1__:55
 _"%)_ _8A_ _QTQ\_ _ ` ` `A*{2*DD
 _"%)_ _8A_ _QTQ\_ _ ` ` ` z88"zK7gFF"7A)cRTggWZ[]W^W^M_7`7`4/#_ 
HE/E37E_EEUUS[U+T+T+T+T+T+T+TUU
W 
W 
W$ KKD_DcgDs|DD::S[:#+/::< < <'
 
 `nW,Jo,Jsw,J,J,JKKKGG 	`<<<<<<<<<JIIII[IIIL(%:M:MTZ[[[L		!'	*	* W W	#)CKs|A[4[*\07!? !? !? ?@CBQU@V@V W W W W W W W W W W W W W W W &|5EFFNoHHHHHrRRRGiSY/?w}%U%U%UW^__G	tWoY	/	/	//s   AL00L47L4c                P    t          j        t          j        d                              t          j         j         j                  |          } fd|D             } fd|D             } fd|D             } j         j	        }}|rt          j        ||          }|s|r%t          j        || fd|D             z             }|rt          j         j         j         j         j        z            |j                  }t          j         fd|D                       }	|j        d	|	f                             d
          }|j        d	|	f                             ||	         dz
            }t          j        |dk     ||          }t%          ||f j                  S )z,Sparse implementation of {func}`jax.lax.rev`r2  r  c                *    g | ]}|j         k     |S r4   rd  r5   r^  r  s     r8   r   zbcoo_rev.<locals>.<listcomp>l  s%    ===aW_)<)<)<)<)<r:   c                X    g | ]&}j         |cxk    rj         j        z   k     !n n|'S r4   r7   rw   rW  s     r8   r   zbcoo_rev.<locals>.<listcomp>m  sG    dddq'/Q*c*c*c*cSZScAc*c*c*c*c*c*c*c*cr:   c                :    g | ]}|j         j        z   k    |S r4   rY  rW  s     r8   r   zbcoo_rev.<locals>.<listcomp>n  s-    QQQago@P.P)P)P)P)P)Pr:   c                *    g | ]}|d z   j         z
  S r   )rw   rW  s     r8   r   zbcoo_rev.<locals>.<listcomp>u  s%    1_1_1_q!a%':J2J1_1_1_r:   rm   c                $    g | ]}|j         z
  S r4   rd  rW  s     r8   r   zbcoo_rev.<locals>.<listcomp>z  s     AAAGO+AAAr:   .rl   ri   r   rE   )r  r  r	   revr  r  rF   rn   rV   rW   r>   rv   r7   rw   rt   mulr   r   r_   )
r  r2  r   rS   r  r  rV   rW   rF  spdimss
   `         r8   bcoo_revr`  f  s    
gcg777BB

w}gm
<
< C ! !! >===:===*ddddJddd+QQQQ:QQQ*,$ 6gg*555G a: a74J1_1_1_1_T^1_1_1_$_```D <9W]7?GOgN^<^+^_#*=2 2 2LYAAAA[AAABBFjf%))"--Gjf%)),v*>*BCCGi!\7;;G	tWoW]	3	3	33r:   c                   t           fdD                       t           fdD                       rt          d j         d          t           fdD                       }t	          j         fdt           j                  D             t                    }t           fdD                       }t          j
         j        ||z             }t          j
         j        d	|f         |          }t          fd
t           j                  D                       }t          ||f| j         j                  S )a  Sparse implementation of {func}`jax.lax.squeeze`.

  Squeeze any number of size 1 dimensions from an array.

  Args:
    arr: BCOO array to be reshaped.
    dimensions: sequence of integers specifying dimensions to squeeze.

  Returns:
    out: reshaped array.
  c              3  B   K   | ]}t          |j                  V  d S rD   )r/   rL   r5   rH   r  s     r8   r9   zbcoo_squeeze.<locals>.<genexpr>  s0      LL#&sCH55LLLLLLr:   c              3  :   K   | ]}j         |         d k    V  dS r}   rE   rc  s     r8   r9   zbcoo_squeeze.<locals>.<genexpr>  s.      3331	333333r:   zPcannot select an axis to squeeze out which has size not equal to one, got shape=z and dimensions=c              3  2   K   | ]}|j         k     |V  d S rD   rd  r5   r^  r  s     r8   r9   zbcoo_squeeze.<locals>.<genexpr>  s)      >>1a#+ooQoooo>>r:   c                ,    g | ]}|j         z   v|S r4   rd  )r5   rh   r  r2  s     r8   r   z bcoo_squeeze.<locals>.<listcomp>  s6     @ @ @_J>> >>>r:   rm   c              3  X   K   | ]$}|j         j        z   k    |j        z
  d z   V  %dS r}   rY  rf  s     r8   r9   zbcoo_squeeze.<locals>.<genexpr>  sN       9 9aS[3<777 %)77779 9r:   .c              3  *   K   | ]\  }}|v	|V  d S rD   r4   )r5   rh   r   r2  s      r8   r9   zbcoo_squeeze.<locals>.<genexpr>  s1      NN$!Q!::M:MA:M:M:M:MNNr:   ro   )r   r   r   rF   r   rv   rr   rw   ra   r	   rx  rV   rW   r   r_   rJ   rK   )r  r2  rS   r  r  r  r  r7  s   ``      r8   bcoo_squeezerj    s    LLLLLLLLL*3333
33333 A
 @"%)@ @2<@ @ A A A>>>>
>>>>>* @ @ @ @ @U3<%8%8 @ @ @GJL L L+ 9 9 9 9: 9 9 9 9 9*[:
#:;;(CK[(89:FF+NNNN)CI"6"6NNNNN)	x%Y /@R
T 
T 
T Tr:   )stridesstart_indiceslimit_indicesrk  c               >   t          | t                    st          dt          |                      d |D             }d |D             }|d |D             }ndg| j        z  }t          |          t          |          cxk    rt          |          cxk    r| j        k    rn nt          d| j                   t          |          | j        k    r't          dt          |           d	| j                   t          d
 |D                       rt          d|           t          d t          ||| j
                  D                       st          d|d|d| j
                   t          || j        | j        g          \  }}}t          || j        | j        g          \  }}}	t          || j        | j        g          \  }
}}g }g }t          t          |||
                    D ]\  }\  }}}|                    | j        j
        |         | j
        |         k    rt%          d          nt%          |||                     |                    | j        j
        |         | j
        |         k    rt%          d          nt%          |||                     |                    t%          d                     |                    t%          d          t%          d          g           t          t          ||	|                    D ]-\  }\  }}}|                    t%          |||                     .| j        t+          |                   }| j        t+          |                   }t+          d t          |||          D                       }t          || j        | j        g          \  }}}| j        rt-          j        t-          j        ||j                  t5          | j        dz                       }t-          j        t-          j        ||j                  t5          | j        dz                       }t-          j        t-          j        ||j                  t5          | j        dz                       }t-          j        ||k    ||k     z  ||z
  |z  dk    z  dd          }t-          j        |||z
  |z   dz
  |z  ||z
  |z   dz
  |z            }t9          j        |d         t5          | j        dz   | j        dz   | j        z                       }t-          j        ||d          }t=          j        |          }| j         |k    r$tC          ||tE          |          |          \  }}t          ||f|          S )a  Sparse implementation of {func}`jax.lax.slice`.

  Args:
    mat: BCOO array to be reshaped.
    start_indices: sequence of integers of length `mat.ndim` specifying the starting
      indices of each slice.
    limit_indices: sequence of integers of length `mat.ndim` specifying the ending
      indices of each slice
    strides: (not implemented) sequence of integers of length `mat.ndim` specifying
      the stride for each slice

  Returns:
    out: BCOO array containing the slice.
  6bcoo_slice: input should be BCOO array, got type(mat)=c                6    g | ]}t          j        |          S r4   rp   rq   rg   s     r8   r   zbcoo_slice.<locals>.<listcomp>  "    <<<8>!$$<<<r:   c                6    g | ]}t          j        |          S r4   rq  rg   s     r8   r   zbcoo_slice.<locals>.<listcomp>  rr  r:   Nc                6    g | ]}t          j        |          S r4   rq  rg   s     r8   r   zbcoo_slice.<locals>.<listcomp>  s"    222Qx~a  222r:   ri   z,bcoo_slice: indices must have size mat.ndim=zlen(strides) = z; expected c              3  "   K   | ]
}|d k    V  dS r  r4   r   s     r8   r9   zbcoo_slice.<locals>.<genexpr>  s&      !!Aa!!!!!!r:   z5strides must be a sequence of positive integers; got c              3  L   K   | ]\  }}}d |cxk    o|cxk    o|k    nc V   dS r  r4   )r5   startendr   s       r8   r9   zbcoo_slice.<locals>.<genexpr>  sn       X X!T %&&&&3&&&&$&&&& X X X X X Xr:   z/bcoo_slice: invalid indices. Got start_indices=z, limit_indices=z and shape=c              3  :   K   | ]\  }}}||z
  |z   d z
  |z  V  dS r}   r4   )r5   rw  rx  strides       r8   r9   zbcoo_slice.<locals>.<genexpr>  sW       O OsF 	5[6A&(O O O O O Or:   rm   r   rl   T)r  r   r  rE   rQ  )#r  r_   r   r
  rL   r   r   r   rN   r   rF   r   r7   rw   r   rQ   appendrV   rf   rW   extendr   r>   r=   rv   rn   rr   r   r	   r   r{  r  r`   rT  r   )r^   rl  rm  rk  start_batchstart_sparsestart_dense	end_batch
end_sparse	end_densestride_batchstride_sparsestride_densedata_slicesindex_slicesrh   rw  rx  rz  r  r  	new_shaper   new_shape_sparsestartsendsstrides_keep	keep_datanew_nses                                 r8   
bcoo_slicer    s     
C		 Z
XTRUYYXX
Y
YY<<m<<<-<<m<<<-22'222GGcCHnG3}--IIIIWIIIIIIIII
NCHNN
O
OO\\SX
Js7||JJJJ
K
KK!!!!!!! X
VWVV
W
WW	 X X%-m]CI%V%VX X X 
X 
X @
 ? ? ?%? ?369? ? @ @ @ ,6mckSVS_E`+a+a(+|[%/S\?Z%[%[")Z.83;PSP\B].^.^+,|+,!*3{I|+T+T!U!U l la	%fchnQ&739Q<&G&GuT{{{USXZ]_eMfMfgggs{'8';sy|'K'KdQVW\^aciQjQjkkkkU4[[!!!uT{{E$KK0111!*3{I|+T+T!U!U 2 2a	%fuUC001111XeK(()(E,//0+ O O&}mWMMO O O O O) &i#+s|1LMM!q\ P_SY|;;LMMMuUXU`cdUdOeOeffF?39Z{7HIII5QTQ\_`Q`KaKabbDsyk>OPPPRWX[XcfgXgRhRhiiH7K6)kD.@A 6)X5:<T+ + +D )D;#7(#BQ#F8"S!F]X59hFH HK VeCK!OS[ST_WZWbEb.c.cddIyHa00Hi())G
w2+jy&A&A&AwP P Ph 
x%Y	7	7	77r:   Sequence[Any]slice_sizesc                L   t          d |D                       }t          j        t          j        d                              t          j        | j        | j                  ||           t          | t                    st          dt          |                      t          d |D                       }t          d |D                       sJ t          d |D                       sJ t          |          t          |          cxk    r| j        k    rn nt!          d| j                   t          d	 t#          || j                  D                       st          d
| d| j                   t%          || j        | j        g          \  }}}t%          || j        | j        g          \  }}}g }	g }
g }g }t+          |r|d         nt,          j        d          }t1          t#          ||                    D ]\  }\  }}| j        j        |         | j        |         k    }| j        j        |         | j        |         k    }|	                    |r|n|           |
                    |rdn|           |                    |r|n|           |                    |rdn|           |	                    |           |
                    | j                   |                    ||g           |                    | j        | j        g           |	                    |           |
                    |           t          j        | j        |	|
          }t          j        | j        ||          }|}| j        rt=          j        ||j                  }t=          j        ||j                  }t=          j        | j        | j        | j        | j        z            |j                  }t=          j         |dk     ||z   |          }t=          j!        |d||z
            }t=          j"        |tG          | j        dz                       }t=          j"        |tG          | j        dz                       }t=          j"        |tG          | j        dz                       }t=          j        ||k    |||z   k     z  dd          }t=          j         |||z
  |          }t          j"        |d         tG          | j        dz   | j        dz   | j$        z                       }t=          j         ||d          }| j        tK          j&        |          k    r8tK          j&        |          }tO          ||tQ          |          |          \  }}t          ||f|          S )a  Sparse implementation of {func}`jax.lax.dynamic_slice`.

  Args:
    mat: BCOO array to slice.
    start_indices: a list of scalar indices, one per dimension. These values
      may be dynamic.
    slice_sizes: the size of the slice. Must be a sequence of non-negative
      integers with length equal to `ndim(operand)`. Inside a JIT compiled
      function, only static values are supported (all JAX arrays inside JIT
      must have statically known size).

  Returns:
    out: BCOO array containing the slice.
  c              3  >   K   | ]}t          j        |          V  d S rD   rq  rg   s     r8   r9   z%bcoo_dynamic_slice.<locals>.<genexpr>  s,      ==AhnQ''======r:   r  r  ro  c              3  >   K   | ]}t          j        |          V  d S rD   )r>   r   rg   s     r8   r9   z%bcoo_dynamic_slice.<locals>.<genexpr>  s*      >>1A>>>>>>r:   c              3  ^   K   | ](}t          j        |j        t          j                  V  )d S rD   )r>   r   rn   r   r   rg   s     r8   r9   z%bcoo_dynamic_slice.<locals>.<genexpr>  s2      HHQS^AGRZ00HHHHHHr:   c              3  ,   K   | ]}|j         d k    V  dS )r4   NrE   rg   s     r8   r9   z%bcoo_dynamic_slice.<locals>.<genexpr>  s(      22qQW]222222r:   z4bcoo_dynamic_slice: indices must have size mat.ndim=c              3  <   K   | ]\  }}d |cxk    o|k    nc V  dS r  r4   )r5   
slice_size	axis_sizes      r8   r9   z%bcoo_dynamic_slice.<locals>.<genexpr>  sC      cc.Cj)Q*))))	))))ccccccr:   zIslice_sizes must be less than or equal to operand shape, got slice_sizes z for operand shape r   ri   rm   rl   Tr   r  rE   rQ  ))r   r  r  r	   dynamic_slicer  r  rF   rn   r  r_   r   r
  rN   r   rL   r   rQ   r   r7   rw   r#   r   r  r   rV   rW   r{  r`   r|  r>   rv   r   r<  r=   rr   r   r{  r  rT  r   )r^   rl  r  r}  r~  r  
size_batchsize_sparse
size_dense
data_start
data_sizesindices_startindices_sizeszerorh   rw  r   data_is_broadcastindices_is_broadcastr  r  r  r  sizesrF  r  r  r  s                               r8   bcoo_dynamic_slicer    s    =======+'#
-=>>>II

sy#)
4
4m! J # # # 
C		 Z
XTRUYYXX
Y
YY>>>>>>>-	HH-HHH	H	HHH	H	22M222	2	222	23{++7777sx77777
VCHVV
W
WW	ccs;X[XaGbGbccc	c	c T
 S'2S SGJyS S T T T ,6mckSVS_E`+a+a(+|[(2;cl@[(\(\%*k:**--	M?a  rx	C	C$#CZ$@$@AA > >a%q)SYq\9;,Q/39Q<?/:ddU;;;,6aa$777!5@5AAA2<====DCGd|$$$.///K   JsxZ@@(!#+}mLL+)\ PY|;+<===FIk):;;;E9SYs{CK#,4N'NOWbWhiiiLYvz6L#8&AAFXfa!566F_VU3;?%;%;<<FOE5q#9#9::E?<s{Q1G1GHHL7K6)kFUN.JKRZ^___D)D+"6>>KVeCK!OS[ST_WZWbEb.c.cddIyHa00H
w;''''	+&&g2+jy&A&A&AwP P Ph 
x%Y	7	7	77r:   c                X    t          fdt          |           D                       S )Nc              3  2   K   | ]\  }}|k    rn|V  d S rD   r4   )r5   rh   tr8  vals      r8   r9   z!_tuple_replace.<locals>.<genexpr>>  s3      AA$!Qa3hhssAAAAAAAr:   )r   r   )tupr8  r  s    ``r8   r3  r3  =  s0    	AAAAA)C..AAA	A	AAr:   axesc               t    t          | j        | j        | j        |          \  }}}t	          ||f|          S )a  Sum array element over given axes.

  Args:
    mat: A BCOO-format array.
    shape: The shape of the target array.
    axes:  A tuple or list or ndarray which contains axes of ``mat`` over which
      sum is performed.

  Returns:
    A BCOO-format array containing the result.
  )rT   r  rE   )_bcoo_reduce_sumrV   rW   r   r_   )r^   r  r_  r`  r7  s        r8   bcoo_reduce_sumr  @  sF     &6	hCID&: &: &:"(K	x%Y	7	7	77r:   tuple[Array, Array, Shape]c          
         |j         t          fdD                       sJ t           |          \  }}t          t	                              t          fdD                       }                     |           rt          j        t          j	        z            |j
                  t          |j        dz
                      }t          j        ||k     d          } j        |j        k    r:t          j        |t          t          |j         j                                      }t          j        | d           fdt                    D             }	|	s2t          j        t!          |j         dz   d          |j
                  }n|dt#          j	        |	          f         }fd	D             }
|
D ]} j         |         dk    rm|j         |         dk    r |         z   nvt          j         t!           j         ||                   t          t           j                                       n&|j         |         dk    r                     |            j         |         |j         |         k    sJ t          t          t	          t                              |
z
                      }t           fd
|D                       }|t'          j         fd|
D                       z  }t          j         g || j         dz   d          R g ||
t           j                  R            t          j        |g |||j         dz   d          R g ||
t          |j                  R           }t          fdt          t-                              D                       } ||fS )Nc              3  R   K   | ]!}d |cxk    ot                    k     nc V  "dS r  r  )r5   r   rF   s     r8   r9   z#_bcoo_reduce_sum.<locals>.<genexpr>R  sE      //QQ!    c%jj    //////r:   c              3  :   K   | ]}|z   k    |z
  d z   V  dS r}   r4   )r5   axr7   rw   s     r8   r9   z#_bcoo_reduce_sum.<locals>.<genexpr>W  s;      RR2w?Q9Q9QR(]Q&9Q9Q9Q9QRRr:   rm   ri   rl   r   c                "    g | ]}|z   v	|S r4   r4   )r5   rh   r  r7   s     r8   r   z$_bcoo_reduce_sum.<locals>.<listcomp>d  s'    FFFaa'k.E.E.E.E.Er:   .c                     h | ]
}|k     |S r4   r4   )r5   r  r7   s     r8   r  z#_bcoo_reduce_sum.<locals>.<setcomp>k  s    222rR'\\\\\r:   c              3  2   K   | ]}j         |         V  d S rD   rE   r5   rh   rV   s     r8   r9   z#_bcoo_reduce_sum.<locals>.<genexpr>z  s)      @@A$*Q-@@@@@@r:   c                *    g | ]}j         |         S r4   rE   r  s     r8   r   z$_bcoo_reduce_sum.<locals>.<listcomp>{  s    ???qTZ]???r:   c              3  0   K   | ]}|v|         V  d S rD   r4   )r5   rh   r  rF   s     r8   r9   z#_bcoo_reduce_sum.<locals>.<genexpr>  s+      II1D==E!H====IIr:   )rF   rN   r   rV  ru   r   r   r>   r=   rv   rn   rr   rL   r	   r   r   r3  r   r2  r{  r  rS  r   )rV   rW   rT   r  r   r`   
dense_axesr   r   
sparse_idx
batch_axesr  new_batch_dimsrL  r  r7  r7   rw   rF   s   `  `            @@@r8   r  r  P  s   
,%	////$///	/	///	/,T7EBB'8Q	D				$ RRRRRRRRRR*	*		$ $	igw112'-HHHGL1 J 77Z',,Dy49_T5ty$))D)D#E#EFFd9T4##D GFFFF5??FFF*	 1iw}gk1EEw}UUGGc28J///0G 3222T222*  	/ 	/bz"~	r	a		eBi#D.Rr*S*SUZ[`aeaj[k[kUlUlmm	r	a		xx||:b>W]2......E'NN 3 3j @AABB.@@@@@@@@@/$)????J???@@@'	TKKK$*Wq[\\2JKKO~O
OU7DI5N5NOO
Q 
Q$ KQ/Q7QW]7Q;<<5PQQU.U:Ugw|8T8TUUW W' IIIIIeCJJ&7&7IIIII)	w		!!r:   c                    t          | j        | j        |j        |j        | j        |j                  \  }}}t	          ||f|          S )zAn element-wise multiplication of two sparse arrays.

  Args:
    lhs: A BCOO-format array.
    rhs: A BCOO-format array.

  Returns:
    An BCOO-format array containing the result.
  )r  r  rE   )_bcoo_multiply_sparserV   rW   r   r_   )r  r  r_  r`  r7  s        r8   bcoo_multiply_sparser    sS     &;	hSXs{sy& & &"(K 
x%Y	7	7	77r:   c                  |j         }|j         }t          | ||          }t          |||          }	t          |          t          |          k    rt          d| d|           |j        |	j        k    rt          d| d|	           t          |j        |	j                  }
t          j	        t          ||
d          ||
d                    }t          ||
          } || |||          \  }}||t          j        ||          fS )NzFbcoo_multiply_sparse: arrays must have same number of dimensions, got z, zIbcoo_multiply_sparse: arrays with differing numbers of dense dimensions: )rE  rH  )rF   r   r   r   r   rO   rz  r7   ri  r   _bcoo_multiply_sparse_unbatchedr   r>   r|  )r  r  r  r  r  r  rE  rH  r  r  r7   _mulrV   rW   s                 r8   r  r    sM   ))xi88#xi88#^^s9~~%%
 4$4 4(14 4 5 5 5[CK
 @36@ @:=@ @ A A AS[))'		:%.wxx%8%.wxx%8
: 
: 
:$ 
D'	"	"$$xhDD-$	w,Y	BB	BBr:   c               t   t          | ||          }t          |||          }|j        dk    s|j        dk    sJ |j        r<t          t          | |f|          d          j        \  } }t          | ||          }nB|j        r;t          t          ||f|          d          j        \  }}t          |||          }t          j        d t          t          |d |j	                 |d |j	                                     D             t                    }|j        |j        z  }	t          j        |d d d |f         |d d d |f         k    d          }
t          j        |
|	|j        |j        f          \  }}| j        |                             dd	          |j        |                             dd	          z  }t          j        |j        |                             dt#          |d
          	          |j        |                             dt#          |d
          	                    }||fS )Nr   rE   rd  c                6    g | ]\  }\  }}|d k    |d k    |S r   r4   )r5   rh   r   r   s       r8   r   z3_bcoo_multiply_sparse_unbatched.<locals>.<listcomp>  s8     , , ,+!Xb"Qww277 #*77r:   rm   rl   r   r)  r*  rI  )r   r7   r  r_   r  r>   rv   r   r   rw   ra   r`   rN   r   rt   r1  maximumrP   )r  r  r  r  rE  rH  r  r  r  r`   r   i_lhsi_rhsrV   rW   s                  r8   r  r    sW   xi88#xi88#
+

q 0 0 0	1 	[ ;.tX{4KS\/]/]/]ghiiioHk
;	
:
:CC
{ ;.tX{4KS\/]/]/]ghiiioHk
;	
:
:C	 , ,8Ims|m<TV_`madam`mVn3o3o)p)p , , ,36
8 
8 
8$
 	#'# 
QQQd]+{4D=/II2	N	N$T#'8JKKK,%
+e

 
 f
 
;
;
+e

 
 f
 
;
;<$KnUVIq8Q8Q8QRRnUVIq8Q8Q8QRRT T' 
wr:   sp_matvc                F    t          | j        | j        || j                  S )zAn element-wise multiplication between a sparse and a dense array.

  Args:
    lhs: A BCOO-format array.
    rhs: An ndarray.

  Returns:
    An ndarray containing the result.
  r   )_bcoo_multiply_denserV   rW   r   )r  r  s     r8   bcoo_multiply_denser    s!     
fk6>1V\	R	R	RRr:   c                  |j         }|j        dk    rt          j        | |          S ||j         k    r#t          j        | t	          ||                    S t          j        |j         |          |k    rt          d|d|j                   t          j        |t          t          |          |j        z
                      }t          | ||          t          t          j                  fd            } || ||          S )zNBroadcasted elementwise multiplication between a BCOO array and a dense array.r   zmultiplication between sparse and dense is only implemented for cases where the output shape matches the sparse matrix shape. Got shape=z
, v.shape=r  c                   j         d         |j        j        z
  k    sJ t          fdt	          j         d                   D                       }t          d t          ||j                   D                       }| ||         z  S )Nri   c              3  0   K   | ]}d d |f         V  d S rD   r4   r  s     r8   r9   z5_bcoo_multiply_dense.<locals>._mul.<locals>.<genexpr>  s/      ??!1??????r:   c              3  0   K   | ]\  }}|d k    r|ndV  dS r  r4   r~   s      r8   r9   z5_bcoo_multiply_dense.<locals>._mul.<locals>.<genexpr>  s2      AAtq!Q!VVAAAAAAr:   )rF   rL   r   r   rr   rQ   )rV   rW   r  r8  r   s    `  r8   r  z"_bcoo_multiply_dense.<locals>._mul  s    =qv55555
????uW]1-='>'>???
?
?C
AAs3/@/@AAA
A
AC!C&=r:   )rF   rL   r	   r^  r   r|  rO   r=   rr   r   r   r   r   r7   )rV   rW   r  rT   rF   r  r   s         @r8   r  r    s!    ,%Vq[[74
ag74w22333!'5))U22
	 7    
oas5zzAF23344!
w
.
.%
:'''    ('
 
dGQ		r:   )rK   indices_are_sortedr+  r   r  r(   rK   rM   r  r+  str | GatherScatterMode | Nonec          	         t           j         j         j                   |t          j        }t	          j        |          }|t          j        k    rt          d|d          t          ||||||          }	d}
 t          j
        t          j        |
          j        t          j         j         j                  t          j        |j        |j                  fi |	}|j        |j        |j        }t'          |d          g j        z  }d t+           j                  D             }t-           j                  t/          |          D ]4\  }}|d|f                                         ||<   ||         |<   d||<   5 fd	} t3          ||f
          |          }t5          |g |j        dd         |j        dd         R t7          t+          |j                                      }|j        r	t7          fdt+          t9          |j                            D                       }t;          j        |j        t>                    }t;          j         |j                  |t;          j!        |z             <   tE          |dt9          |                             tE          t+          t9          |                              k    rtG          |d          }tI          |t7          |                    }|%                    |j                  &                    |j                  S )zBCOO version of lax.gather.Nzbcoo_gather: mode=z not yet supported.)r  r  rK   r  r+  r   r  r   c                    g | ]}d S rD   r4   rg   s     r8   r   zbcoo_gather.<locals>.<listcomp>	  s    AAAtAAAr:   .c                J    t          |           }t          |          S )Nr  rU  )r  rj  )rW   slccollapsed_slice_dimsfull_slice_sizesr  s     r8   
slice_funczbcoo_gather.<locals>.slice_func	  s,    
Wg;K
L
L
LC(<====r:   rg  rl   ri   r9  c              3  $   K   | ]
}|v|V  d S rD   r4   )r5   rH   offset_dimss     r8   r9   zbcoo_gather.<locals>.<genexpr>$	  s9       1 1sK// ////1 1r:   rm   rd  rG  )'r   rV   rW   rF   r)   PROMISE_IN_BOUNDSfrom_anyrO   dictr  r  r	   gatherr  r  rn   r  r  start_index_mapr#   rL   rr   r   r   r  r   rS  r   r   r   r   ra   r   rv   ru   r  rF  rS  r  )r  rl  r  r  rK   r  r+  r   parsed_moder0  r  r  r  full_start_indicesr=  rh   r  r  r6  rS   rG  r  r  r  s   `                    @@@r8   bcoo_gatherr    sZ    w>>> 
\.D!*400+%777
HdHHH
I
II	 1{+@RJ
0 
0 
0$
7/LSWSZAAAL

w}gm
<
<

}2M4G
H
H   (
 "-+*?%5/ *0q)A)A(BW\(QAAU7<-@-@AAA''-((((  da)#q&17799q%a.QGAJJ> > > > > > > 04
WJ///0BCC&<#CRC(<6<+;<<U6;''((* * *&
 \ 	D 1 1 1 1eC,?,?&@&@ 1 1 1 1 1J(6;c222K68i6L6LKk1223
;'J'())Ss:1G1G-H-HHH "&!444fFk0B0BCCCF		'	'	.	.x~	>	>>r:   ri   )lhs_dilationrhs_dilationr  feature_group_countbatch_group_countr~  r  c       	           t          j        t          j        ||||||||	|

  
        } t	          j        |          t	          j        | j        | j                  t	          j        |j        |j                            }t          |t          j                  rt          |j                  dk    sJ |j        d         j        }|d         d| j        dz
  z  k    rt!          d          |d         d|j        dz
  z  k    rt!          d          |d	         d| j        dz
  z  k    rt!          d
          |d         |d         cxk    rdk    rn nt!          d          | j        d d         |j        d d         cxk    rdk    rn nt!          d          t#          | d          r| j        j        nt#          |d          r|j        j        nd}|d         \  }t'          t)          | |          t)          ||          |          S )N)	window_stridespaddingr  r  r  r  r  r~  r  ri   r   r  r   r1   z#bcoo convolution with lhs_dilation.r  r  z.bcoo convolution with non-unit window_strides.r  r  z,bcoo convolution with non-unit group counts.)ri   ri   z:bcoo convolution with leading dimensions other than (1, 1)rW   r  r  )r  )ri  r   r	   conv_general_dilatedr  
make_jaxprr  rF   rn   r  r   ClosedJaxprr   eqnsparamsrL   rO   r)  rW   _bcoo_conv_1d_convert_to_1d_for_conv)r  r  r  r  r  r  r  r  r  r~  r  rm  jaxprr  r   s                  r8   bcoo_conv_general_dilatedr  0	  s?   
 
		#WlN_-AR2H
J 
J 
J$ #.

s3CIsyII"3CIsyIIK K%	E4+	,	,EUZA1E1E1E	E:a=&N1 555
C
D
DDNtsx!|444
C
D
DDA!666
N
O
OO F+@$AFFFFQFFFFF
L
M
MMYrr]cim----v-----
Z
[
[[&-c9&=&= ""+23	+B+B ck''  I('	.sK@@.sK@@&
( 
( 
( (r:   c                v   t          | t          j        t          j        f          r;t          j        | d          }t          j        |t          |          dfd          }nt          | t                    r| 
                    dd          } t          j        | j        d          }t          j        | j        d          }t          j        t          j        |d          | j        d         k     |d          }n t!          dt#          |            d	          t          ||f| j        dd          
          S )Nr   ri   ri   r   r1   )r7   r   r   rl   z)bcoo_conv_general_dilated: input of type z not recognized.rE   )r  r  r,   r   rU  r	   rx  r  r   r_   update_layoutrV   rW   r>   r   rF   r   r
  )r^   r   rV   rW   s       r8   r  r  U	  s   ci,-- 
];sF##D";TABBGG#t ]


Aq

1
1C;sx((Dk#+v..G9S[$//#)B-?qIIDD
[S		[[[
\
\\	tWoSYqrr]	3	3	33r:   r  c                D   | j         | j        cxk    r|j         cxk    r|j        cxk    rdk    sn J | j        |j        k    sJ t          t	          t
          |                    }t          |          dk    sJ | j        d d d f         |j        d d d f         z                                  }|d         |j	        z
  }| j	        d d d f         |d d d f         z                                   }|dk     }t          j        |d|          }t          j        |d|          }t          d| j        d         |d         z   |d         z   |j        d         z
  dz             }t          j        |d          }t          j        |d          }t!          ||fdd|f          S )Nri   r1   r   r  )r   ri      rE   )rL   rw   rn   r   rk  ra   r   rV   r  rW   r>   r   rP   rF   r	   r=   r_   )r  r  r  r  r*  r  r   dimsizes           r8   r  r  c	  s   	S\	B	B	B	BSX	B	B	B	B	B	B	B	B	B	B	B	B	B	B	ci				#c7##$$'	W				hqqq$w#(47"33::<<(1:#&QQQW%tQQQw7>>@@+
/$	$;//+YtQ))(39Q<'!*,wqz9CIaLH1LMM'_Xv..(Y77+	x%aG_	=	=	==r:   c                      e Zd ZU dZded<   ded<   ded<    ed           Z ed           Z ed	           Z ed
           Z	 ed           Z
ded<   ded<    ed           Z ed           ZddddC fdZd ZdDdZdDdZdDdZedej        ddddEd'            Zedddd(dFd*            Zedd+dddd,dGd.            Zedd+ddd/dHd3            Zddd4d5dId8ZdJdKd;ZdDd<ZdLd=ZdMdNd@ZdA ZedB             Z xZ S )Or_   ar  Experimental batched COO matrix implemented in JAX

  Args:
    (data, indices) : data and indices in batched COO format.
    shape : shape of sparse array.

  Attributes:
    data : ndarray of shape ``[*batch_dims, nse, *dense_dims]`` containing the
      explicitly stored data within the sparse matrix.
    indices : ndarray of shape ``[*batch_dims, nse, n_sparse]`` containing the
      indices of the explicitly stored data. Duplicate entries will be summed.

  Examples:
    Create a sparse array from a dense array:

    >>> M = jnp.array([[0., 2., 0.], [1., 0., 4.]])
    >>> M_sp = BCOO.fromdense(M)
    >>> M_sp
    BCOO(float32[2, 3], nse=3)

    Examine the internal representation:

    >>> M_sp.data
    Array([2., 1., 4.], dtype=float32)
    >>> M_sp.indices
    Array([[0, 1],
           [1, 0],
           [1, 2]], dtype=int32)

    Create a dense array from a sparse array:

    >>> M_sp.todense()
    Array([[0., 2., 0.],
           [1., 0., 4.]], dtype=float32)

    Create a sparse array from COO data & indices:

    >>> data = jnp.array([1., 3., 5.])
    >>> indices = jnp.array([[0, 0],
    ...                      [1, 1],
    ...                      [2, 2]])
    >>> mat = BCOO((data, indices), shape=(3, 3))
    >>> mat
    BCOO(float32[3, 3], nse=3)
    >>> mat.todense()
    Array([[1., 0., 0.],
           [0., 3., 0.],
           [0., 0., 5.]], dtype=float32)
  r,   rV   rW   r   rF   c                &    | j         j        d         S )Nrk   r  r   s    r8   r   zBCOO.<lambda>	  s    dl04 r:   c                    | j         j        S rD   )rV   rn   r   s    r8   r   zBCOO.<lambda>	  s
    	 r:   c                     | j         j        dz
  S )Nr1   )rW   rL   r   s    r8   r   zBCOO.<lambda>	  s    $,"3a"7 r:   c                &    | j         j        d         S r  r  r   s    r8   r   zBCOO.<lambda>	  s    4<#5b#9 r:   c                0    | j         j        dz
  | j        z
  S r  )rV   rL   r7   r   s    r8   r   zBCOO.<lambda>	  s    $).1"4t|"C r:   rM   rJ   rK   c                B    t          | j        | j        | j                  S rD   )r   rF   rJ   rK   r   s    r8   r   zBCOO.<lambda>	  s!    
4:t7J+/+>!@ !@ r:   c                    | j         | j        fS rD   )rV   rW   r   s    r8   r   zBCOO.<lambda>	  s    DL 9 r:   FrI   r!  r   r   c                  t          t          j        |          \  | _        | _        || _        || _        t                                          |t          |                     t          | j        | j        | j                   d S r-  )rk  r>   r   rV   rW   rJ   rK   super__init__r   r   rF   )r   r!  rF   rJ   rK   	__class__s        r8   r	  zBCOO.__init__	  sn    !#+t44DIt|(D(D	GGTu...49dlDJ77777r:   c                `   | j         j        }	 | j        }| j        }| j        }| j        }t          | j                  }d|}|r|d|z  }|r|d|z  }| d| | | d}n#  | d}Y nxY wt          | j	        t          j                  rt          | j	                  j         d| d}|S )	N, nse=r   r   ()z(<invalid>)[])r
  r   r`   r7   r   rn   r   rF   r  rV   r   Tracerr
  )	r   namer`   r7   r   rn   rF   extrarepr_s	            r8   __repr__zBCOO.__repr__	  s    >"D/Hcggje4:e 3kke	*%???*%	*%???*%...u.e...ee#"""eee $)T[)) 5di)44E444eLs   0A$ $A-rb   c                     t          d          )NzBCOO.reshaperO   r   r!  kwargss      r8   rS  zBCOO.reshape	  s    
n
-
--r:   c                     t          d          )NzBCOO.astyper  r  s      r8   r  zBCOO.astype	  s    
m
,
,,r:   c                     t          d          )NzBCOO.sumr  r   s    r8   r   zBCOO.sum	  s    
j
)
))r:   Nr   r`   r   r   r7   r^   r`   rz   r   r.   r   ra   r7   c               *    t          |||||          S )z7Create a BCOO array from a (dense) :class:`~jax.Array`.r  )r   )clsr^   r`   r   r   r7   s         r8   	fromdensezBCOO.fromdense	  s*     	sWgO O O Or:   )r   r   r7   DTypeLike | Nonec               <   |dk    s|dk    rt          d          |                                }t          j        |j                  }t          j        |j        |j        f                              |pt          j	                  } | ||f|j
        dd          S )z5Create a BCOO array from a :mod:`scipy.sparse` array.r   z+BCOO.fromscipy with nonzero n_dense/n_batchFro   )rO   tocoor>   r   rV   r  r  r  r  r  rF   )r  r^   r   r   r7   rV   rW   s          r8   from_scipy_sparsezBCOO.from_scipy_sparse	  s     !||w!|| MNNN
))++C;sx  D12299 sy" "G 3gci#% % % %r:   r  )rn   r   r   r7   r`   rn   c          
        t          |          }t          |          |z
  |z
  }|dk     s|dk     s|dk     s|dk     rt          d|d|d|d|          t          |||g          \  }}	}
t	          j        g |||
R |          }t	          j        g |||R t	          j        |	          |          } | ||f|dd          S )z?Create an empty BCOO instance. Public method is sparse.empty().r   Invalid inputs: shape=r   r   r  Tro   )r   r   r   r   r>   r   fullrv   )r  rF   rn   r   r   r7   r`   rw   rE  rF  rG  rV   rW   s                r8   _emptyzBCOO._empty	  s	    %LLE5zzG#g-H!||w{{gkkS1WWR%RRGRRRRCRRSSS-7?R-S-S*K{96{6C6+66>>Dh44c4844ci6M6M{[[G3geD"$ $ $ $r:   rn   r   r7   r   r   r  kc                  d|z
  |z
  }|dk     s|dk     s|dk     rt          d||f d|d|          dk    rt          ||z
            }	nt          |z   |          }	|	dk    r|                     ||f||||          S |dk    s|dk    r/|                     t	          j        |||          |||	          S |dk    rt	          j        |	|          }
t	          j        |	|          }t          |d          t          |          t	          j	        t          j        |t          j        dk    fd
fd                    t          j        |t          j        dk    fdfd                    g          }n%t	          j        ||          }
t	          j        ||          }|t          |          z   }dk     r_|
j        d t                                                 d          }
|j        d t                                                 |          }nhdk    rb|
j        |t                    z
  d                              d          }
|j        |t                    z
                               |          }|
d d d f         }
|d d d d f         } | |
|f||fdd          S )Nr1   r   r%  r   r   r(  ri   rm   r   c                      S rD   r4   r  s   r8   r   zBCOO._eye.<locals>.<lambda>!
      d r:   c                      S rD   r4   r)  s   r8   r   zBCOO._eye.<locals>.<lambda>!
      A r:   c                      S rD   r4   r,  s   r8   r   zBCOO._eye.<locals>.<lambda>"
  r-  r:   c                      S rD   r4   r/  s   r8   r   zBCOO._eye.<locals>.<lambda>"
  r0  r:   Tro   )r   rz  r'  r  r>   eyeonesr   r#   r  r	   subcondr   rt   absru   )r  r   r  r)  rn   r   r7   r   rw   	diag_sizerV   r  rW   r  s      `         @r8   _eyez	BCOO._eye
  s    7{W$H!||w{{gkkPAPP'PPgPPQQQ1uuaQ--iia!eQ--iA~~ZZAe '  : : : {{gkk]]371a%888#*G'2  4 4 4 !||Xiu---dJy444cC^^d
a..a SXa1fllllIIII>>??SXa1fllllIIII>>??"A B Bgg Xau%%%d
1K000g&!,,,g	
QwwAw##A&&*Wc!ffW%))!,,q55wq3q66z{{#''***QQZ(,,Q//!!!T']d4&g3gq!fT"$ $ $ $r:   r  r  r  strc               (    t          | |||          S )a  Update the storage layout (i.e. n_batch & n_dense) of a BCOO matrix.

    In many cases this can be done without introducing undue storage overhead. However,
    increasing ``mat.n_batch`` or ``mat.n_dense`` will lead to very inefficient storage,
    with many explicitly-stored zeros, unless the new batch or dense dimensions have size
    0 or 1. In such cases, ``update_layout`` will raise a :class:`SparseEfficiencyError`.
    This can be silenced by specifying the ``on_inefficient`` argument.

    Args:
      n_batch : optional(int) the number of batch dimensions in the output matrix. If None,
        then n_batch = mat.n_batch.
      n_dense : optional(int) the number of dense dimensions in the output matrix. If None,
        then n_dense = mat.n_dense.
      on_inefficient : optional(string), one of ``['error', 'warn', None]``. Specify the
        behavior in case of an inefficient reconfiguration. This is defined as a reconfiguration
        where the size of the resulting representation is much larger than the size of the
        input representation.

    Returns:
      mat_out : BCOO array
        A BCOO array representing the same sparse array as the input, with the specified
        layout. ``mat_out.todense()`` will match ``mat.todense()`` up to appropriate precision.
    r  )r  )r   r7   r   r  s       r8   r  zBCOO.update_layout2
  s    2 dGWUcddddr:   Tremove_zerosc                J    |rt          | |          S t          | |          S )aq  Return a copy of the array with duplicate indices summed.

    Additionally, this operation will result in explicit zero entries removed, and
    indices being sorted in lexicographic order.

    Because the size of the resulting representation depends on the values in the
    arrays, this operation is not compatible with JIT or other transforms. To use
    ``sum_duplicates`` in such cases, you may pass a value to `nse` to specify the
    desired size of the output representation.

    Args:
      nse : integer (optional), if specified, gives the number of specified elements in
        the output sparse representation; if it is larger than the number required, data
        will be padded with zeros and indices will be padded with out-of-bounds values.
        If it is smaller than the number required, data will be silently discarded.
      remove_zeros : bool (default=True). If True, remove explicit zeros from the data
        as part of summing duplicates. If False, then explicit zeros at unique indices
        will remain among the specified elements. Note: remove_zeros=True is incompatible
        with autodiff.
    r   )r   r   )r   r`   r<  s      r8   sum_duplicateszBCOO.sum_duplicatesM
  s2    *  0!$C0000 3////r:   c                     t          |           S )z0Return a copy of the matrix with indices sorted.)r  r   s    r8   sort_indiceszBCOO.sort_indicesg
  s    T"""r:   c                     t          |           S )z$Create a dense version of the array.)r   r   s    r8   todensezBCOO.todensek
  s    r:   r  r4  c                    t          |t           j                  ddd         n|          }t           |          }t	           fd|D                       } fd| j         j         j        z            D             }t	          |          t	          t           j                            k    r j        }nd}t          |j	        |j
        f|| j                  S )z,Create a new array containing the transpose.Nrl   r  c              3  2   K   | ]}j         |         V  d S rD   rE   )r5   rh   r   s     r8   r9   z!BCOO.transpose.<locals>.<genexpr>s
  s)      00aDJqM000000r:   c                $    g | ]}|j         z
  S r4   rd  )r5   rL  r   s     r8   r   z"BCOO.transpose.<locals>.<listcomp>t
  s6     N N N t|# N N Nr:   Fro   )r   rr   rL   rF  r   r7   rw   rJ   r_   rV   rW   rK   )r   r  r  mat_Tshape_TrX  	is_sorteds   `      r8   r_  zBCOO.transposeo
  s   T\5++DDbD11tLLD4T222E0000400000GN N N N t|dm/K!KLN N NK[U5#7#78888%ii iU]+7(9LN N N Nr:   c                P    | j         | j        f| j                                        fS rD   )rV   rW   r   r!  r   s    r8   tree_flattenzBCOO.tree_flatten
  s$    It|$dj&8&8&:&:::r:   c                    t                               |           }|\  |_        |_        |                                h dk    rt          d|           |j        j        di | |S )N>   rF   rJ   rK   z&BCOO.tree_unflatten: invalid aux_data=r4   )object__new__rV   rW   keysr   __dict__update)r  aux_datachildrenobjs       r8   tree_unflattenzBCOO.tree_unflatten
  sr    
..

C$CHck}}GGGGGBxBBCCCCL##(###Jr:   )r!  r   rF   r   rJ   rM   rK   rM   rb   r_   )r^   r,   r`   rz   r   r.   r   ra   r7   ra   rb   r_   )r   r   r   ra   r7   ra   rb   r_   )rF   r   rn   r   r   r.   r   ra   r7   ra   r`   ra   rb   r_   )r   ra   r  ra   r)  ra   rn   r   r   r.   r7   ra   r   ra   rb   r_   )r7   rz   r   rz   r  r:  rb   r_   )NT)r`   rz   r<  rM   rb   r_   )rb   r,   rD   )r  r4  rb   r_   )!r   r   r   __doc__r   r   r`   rn   r7   rw   r   r   r  r	  r  rS  r  r   classmethodr   r  r  r#  r'  r9  r  r>  r@  rB  r_  rJ  rT  __classcell__)r
  s   @r8   r_   r_   x	  s        0 0f +++...,,,4455#
(//
0
0%H7788'X99::(HCCDD'
( @ @ A A%
(99
:
:% ',E8 8 8 8 8 8 8 8  (. . . .- - - -* * * * 6:UWU] O O O O O ;O CG'(% % % % % ;% =A\caA$ $ $ $ $ ;$ EI$+Aa+$ +$ +$ +$ +$ ;+$Z 48t*1e e e e e e60 0 0 0 04# # # #   N N N N N ; ; ;   ;    r:   c                   ||S ||j         k    rt          d| d|j          d          t           | |j        |           | |j        |          f|j        d |         |j        |dz   d          z   |j        |j                  S )NzCannot map in_axis= for BCOO array with n_batch=zK. in_axes for batched BCOO operations must correspond to a batch dimension.ri   ro   r7   r   r_   rV   rW   rF   rJ   rK   )contr   r  r  s       r8   _bcoo_to_eltr]  
  s    	\J	S[
 a4 a ack a a a b b b	ttCHd##TT#+t%<%<=Iete$sy';; /@R
T 
T 
T Tr:   c                   ||S ||j         k    rt          d| d|j          d          t           | ||j        |           | ||j        |          f|j        d |         |fz   |j        |d          z   |j        |j                  S )NzBCOO: cannot add out_axis=rZ  zC. BCOO batch axes must be a contiguous block of leading dimensions.ro   r[  )r\  r  eltr  s       r8   _bcoo_from_eltr`  
  s    	\J	CK
 Y$ Y YUXU` Y Y Y Z Z Z	ttIsx..YT0R0RSIete$	|3ci6FF /@R
T 
T 
T Tr:   rD   )r^   r_   r`   ra   rb   r_   )r^   r_   r`   rz   rb   r_   )rV   r   rW   r   rF   r   rb   r   )rW   r   rF   r   rb   r   )r^   r_   rb   r,   )rV   r,   rW   r,   rT   r   rb   r,   )r^   r,   r`   rz   r7   ra   r   ra   r   r.   rb   r_   )r^   r,   r`   ra   r7   ra   r   ra   r   r.   rb   r   )r  r_   r  r-   r  r  rb   r_   )rW   r,   r  r,   rb   r,   )r^   r_   rG  r   rb   r_   )
rV   r,   rW   r,   rG  r   rT   r   rb   r   )rG  r   rT   r   )r  r  r  r  r  r'   r~  r  r  r  rb   r  )r  r,   r  r,   r  r,   r  r'   r  r   r  r   rb   r,   )r  r,   r  r,   r  r,   r  r'   r  r   r  r   rb   r,   )r  r   )
r  r,   r  r,   rW   r,   r  r'   rb   r,   )r  r,   r  r,   r  r,   r  r,   r  r   r  r   r  r'   r  r   rb   r   )r  r   r  r   )r^   r_   rb   r_   )
rV   r,   rW   r,   rT   r   r`   rz   rb   r   )
r^   r_   r7   rz   r   rz   r  r  rb   r_   )r^   r_   rF   r   r  r   rb   r_   )rV   r,   rW   r,   rT   r   rF   r   r  r   rb   r   )r  r  r  ra   rb   r_   )r^   r_   r3  r   r2  r4  rb   r_   )r  r_   r2  r   rb   r_   )
r^   r_   rl  r   rm  r   rk  r4  rb   r_   )r^   r_   rl  r  r  r   rb   r_   )r^   r_   r  r   rb   r_   )
rV   r,   rW   r,   rT   r   r  r   rb   r  )r  r_   r  r_   rb   r_   )r  r,   r  r,   r  r,   r  r,   r  r   r  r   rb   r  )r  r_   r  r,   rb   r,   )
rV   r,   rW   r,   r  r,   rT   r   rb   r,   )r  r_   rl  r,   r  r(   r  r   rK   rM   r  rM   r+  r  rb   r_   rU  )r  r_   r  r_   r  r   rb   r_   )rV  
__future__r   collections.abcr   ri  r   r{  rp   typingr   r   r   r  numpyr   r  r	   r
   r   jax.experimental.sparse._baser   jax.experimental.sparse.utilr   r   r   r   r   r   r   r   "jax.experimental.sparse._loweringsr   r   jax._src.interpretersr   	jax.numpyr>   jax.utilr   r   r   jax._srcr   r   r   r   r    r!   r"   r#  jax._src.lax.laxr#   r$   r%   r&   r'   jax._src.lax.slicingr(   r)   jax._src.libr*   jax._src.numpy.setopsr+   jax._src.typingr,   r-   r.   jax._src.utilr/   float32float64	complex64
complex128r  r  r  r]   ry   r   r   r   r   r   	Primitiver   r   r   def_implr   def_abstract_evalr   r   r   r   defjvpprimitive_transposesprimitive_batchersregister_lowering	lower_funr   r   r   r   r   r  r  r  r  r  primitive_jvpsr#  r  r   r9  r;  r>  r@  rE  rQ  rF  rM  rZ  r`  rd  rg  rs  r|  r  r}  r  r  r  r  "_bcoo_dot_general_default_loweringr  r  _bcoo_dot_general_gpu_loweringr  r  r  r  simple_implcuda_is_supportedrocm_is_supportedr  r  r  r  r  r  r'  r1  r6  r9  r>  rA  r  ra  rn  rh  r  r  r  r  r  r  r  r  r  _bcoo_sort_indices_hlor  r   rT  r  r  r0  r  r  r  r  r  _bcoo_sum_duplicates_hlor  r  r  r1  rS  r`  rj  r  r  r3  r  r  r  r  r  r  r  r  r  r  r  register_pytree_node_classr_   r]  r`  register_vmappablera   r4   r:   r8   <module>r     sw   O N " " " " " " $ $ $ $ $ $             , , , , , , , , , ,      



                   3 3 3 3 3 3                   
 F E E E E E E E & & & & & &       1 1 1 1 1 1 1 1 1 1                         $ $ $ $ $ $ * * * * * * 4 4 4 4 4 4T T T T T T T T T T T T T T J J J J J J J J # # # # # # ) ) ) ) ) ) 7 7 7 7 7 7 7 7 7 7 + + + + + + 
BJbmL 
 7 7 7 7*1 1 1 12	J 	J 	J 	J 	J    Z       X      V V V V"  //	@ 	@ 	@ 	@U U U U  K K K" !- - "!-
9 9 9- - -8 8 8 
	.+T 2 2 2*A  '.I N +  ~~t~(0 (0 (0 1 1 1 "4>"233 $(  !  59"#ciI I I I I I. =>a-0Y8 8 8 8 8 8*   2 #] ] $#]# # # 	G 	G 	Gr r r ': " #,E ( )0M , -  '4*1 *1 *1 2 2 2  // OS > > > > > >2 @D H H H H H H  $ $ $L !0 0 "!0F F FP P PH H H* 
	.$ 1 2 2 2*A  '.I N +  ~~t~(0 (0 (0 1 1 1 "4>"233 $(  !K K K K&0 0 0 0- - -$     #b b b $#bI I I I       " " " " ': " #,E ( )0J , -  '4*1 *1 *1 2 2 2 $T^$677  (,48"	)J )J )J )J )J )JV8 8 8 8 , , , ,87 87 87 87t %: : : &%:* &4T^U&4 &4 &4 "  (77 77 77r "0"8 "8 "8 a a a a
a a a a
1B 1B 1B 1Bf' ' ' ' 
	
7?X Y Y Y.I * +2N . /  )+M N N N  ' ( ( ( K$86K K K K K$86K K K K ,T^,FGG K K K K,q q q3 3 3,3 3 3D $q q %$q( -  .-
  Z Z ZZ Z ZF F F
 
	
$&E
)41 1 16Y 2 3:^ 6 7  DN1EJJJL L L &t~&:;; (,  %R R R R,f ,f ,f\ #< #< #< #<J '@! @! @! ('@!D; ; ; ;$P P P 5S 0 1*A & '  +^T^t.5 .5 .5 6 6 6 %dn%899 '+  $1 1 1 1   ) ) ) &  '&	+ 	+ 	+B B B  (d4 4 4  *@ % &3S / 0  *,B C C C ''<== )-  &# # # # #&K K K K
        7<!&     AF+05    :  * (
 
 )(

+ 
+ 
+B B BB *4>6 6 6  ,D ' (5W 1 2  ,.F G G G
 <@W[4;D4 D4 D4 D4 D4 D4P $(     (& & & &PD= D= D= D=P 59>0 >0 >0 >0 >0 >0B4 4 48T T T T: 04K8 K8 K8 K8 K8 K8ZO8 O8 O8 O8dB B B8 8 8 8 5" 5" 5" 5"n8 8 8 8C C C C,  <
S 
S 
S 
S       < (-+07;!=? =? =? =? =? =?@ ,0dVZ23qTX59#( #( #( #( #( #(J4 4 4> > > >* %P P P P P9 P P &%PhT T TT T T  D#sL.$ O O O O Or:   