
    vh                      d Z ddlmZ ddlmZ ddlZddlmZ ddlZddlZddl	m
Z
mZmZ ddlZddlZddlZddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZmZmZmZ ddlm Z m!Z! ddl"m#Z# ddl$mZ% ddl&m'Z'm(Z(m)Z) ddl*m+Z+ ddl*m,Z, ddl*m-Z- ddl*m.Z. ddl"m/Z/ ddl"m0Z0 ddl"m1Z2 ddl3m4Z4m5Z5m6Z6m7Z7m8Z8 ddl9m:Z:m;Z; ddl<m=Z= ddl>m?Z? ddl@mAZAmBZBmCZC ddl&mDZD ej                  ej                  ej                  ej                  gZIej                  gZKddZLddZMdddZN G d  d!e      ZO G d" d#e      ZPdd$ZQdd%ZR e-j                  d&      ZTdd'ZU	 	 dd(ZVeTj                  d)        ZXeTj                  d*        ZZd+ Z[d, Z\d- Z] e/j                  eTe[d       e\e/j                  eT<   e]e0j                  eT<    e#j                  eT e#j                  eXd./              e-j                  d0      Zcd1ec_d        d2Zeddde%j                  d3	 	 	 	 	 dd4Zfdde%j                  d5	 	 	 dd6Zgecj                  d7        Zhecj                  d8        Zid9 Zjd: Zkd; Zleje/j                  ec<   eke/j                  ec<   ele0j                  ec<    e#j                  ec e#j                  ehd1/              e-j                  d<      Zndd=dd>Zod1d=dd?Zpenj                  d@        Zqenj                  dA        ZrdB ZsdC ZtdD Zu e/j                  endes       ete/j                  en<   eue0j                  en<    e#j                  en e#j                  eqd./              e-j                  dE      Zvd1ev_d        ddFZw	 	 	 	 	 	 ddGZxdH Zyevj                  ddI       Zzevj                  ddJ       Z{ddKZ|ddLZ}ddMZ~e|e/j                  ev<   e}e/j                  ev<   e~e0j                  ev<    e#j                  ev e#j                  ezd1/              e-j                  dN      ZddddO	 	 	 	 	 	 	 ddPZ	 	 	 	 	 	 	 	 ddQZ	 	 	 	 	 	 	 	 ddRZ	 	 ddSZej                  	 	 ddT       Z e#j                  ed./      ZdU ZdV Z e#j                  ed./      Z	 	 ddWZ	 	 ddXZ	 	 ddYZ	 	 ddZZ e/j                  eede       ee/j                  e<   ee0j                  e<    e#j                  ee        e.j                  e       e=j                  r e#j                  eed[\       e=j                  r e#j                  eed]\        e-j                  d^      Zdd_Zd` Zda Zdb Zej                  dc        Zej                  dd        Zde Zdf Zdg Zdh Z e/j                  eeed       ee/j                  e<   ee0j                  e<    e#j                  e e#j                  ed./              e-j                  di      Zd1e_d        	 	 	 	 	 	 	 	 	 	 ddjZdk Zej                  ddl       Zej                  ddm       ZddnZdo Zee0j                  e<   ee/j                  e<    e#j                  e e#j                  ed1/              e-j                  dp      Zd1e_d        ddqZej                  dr        Zds Zej                  dt        Zdu Zdv Z e#j                  ed1/      Zee/j                  e<   ee0j                  e<    e#j                  ee        e-j                  dw      Zd1e_d        dddxZddyZej                  dz        Zd{ Zd.d.d.d|d}Zd.d.d.d|d~Zd Zej                  d        Zd Zd Z e#j                  ed1/      Zee/j                  e<   ee0j                  e<    e#j                  ee       dddd	 	 	 ddZdd	 ddZ	 	 	 	 ddZddZddd	 	 	 ddZd ZddZdd	 	 	 ddZddZd ZddZddZddZ	 	 	 	 	 	 ddZd ZddZddZd.d.ddd	 	 	 	 	 	 	 	 	 	 	 ddZdddddddd	 ddZd ZddZej                   G d de             Zd Zd Z e0j                  eeeeed       y)zHBCOO (Bached coordinate format) matrix object and associated primitives.    )annotations)SequenceN)partial)Any
NamedTupleProtocol)lax)	tree_utilvmap)	JAXSparse)
nfold_vmap_count_stored_elements_dot_general_validated_shapeCuSparseEfficiencyWarningSparseEfficiencyErrorSparseEfficiencyWarningShape
SparseInfo)
coo_spmv_p
coo_spmm_p)mlir)safe_zipunzip2
split_list)api_util)config)core)dispatch)ad)batching)partial_eval)_constranges_like	remaining_dot_general_batch_dim_numsDotDimensionNumbers)GatherDimensionNumbersGatherScatterMode)
gpu_sparse)_unique)Array	ArrayLike	DTypeLike)canonicalize_axisc                l   | \  }}|\  }}|j                   dz
  t        |d u       z   t        fd|D              st        d|dd      d ||f||ffD        \  }}	|t	        d t        ||f|      D              }t        |g|j                  |j                  |j                        }
||	|
fS )	N   c              3  J   K   | ]  }|d u xs d|cxk  xr k  nc   y wNr    ).0bn_batchs     W/opt/face_recognition/venv/lib/python3.12/site-packages/jax/experimental/sparse/bcoo.py	<genexpr>z,_bcoo_batch_dims_to_front.<locals>.<genexpr>B   s'     ?qQ$Y*!q*7**?    #zEbatch_dims must be None or satisfy 0 < dim < n_batch. Got batch_dims=z for n_batch=.c              3     K   | ]7  \  }}|t        j                  |dg      nt        j                  ||d       9 y wr3   )r	   expand_dimsjnpmoveaxis)r5   argbdims      r8   r9   z,_bcoo_batch_dims_to_front.<locals>.<genexpr>E   sA      #E
#t $(<coocA3S\\#tQ5OO#Es   =?c              3  F   K   | ]  \  }}|	|j                   |     y wNshaper5   r@   dims      r8   r9   z,_bcoo_batch_dims_to_front.<locals>.<genexpr>I   s"     hSX[XgSYYs^h   
!!indices_sortedunique_indices)
ndimboolallNotImplementedErrormaxzipr   rE   rJ   rK   )batched_args
batch_dimsspinfo
batch_sizedataindices	data_bdimindices_bdimbatched_databatched_indicesbatched_spinfor7   s              @r8   _bcoo_batch_dims_to_frontr]   >   s    -$&)\LL1tLD$899'	?J?	?
 1%/MwjC D D#Ey)G\+BC#E, hCw4ThhJz9FLL9-3-B-B-3-B-BD. 
	66    BCOOc                   t        j                  |      }|dk\  sJ | j                  |k(  r| S || j                  k  rO| j                  g d t	        | j
                        D        t        |         }| j                  dd|ddf   }nt        j                  | j                  g | j                  j                  d| j
                   || j                  j                  | j
                  dz   d       }|j                  g d t	        | j
                        D        t        | j                           j                  | j                        }t        j                  | j                  g | j                  j                  dd || j                  j                  d	         }|j                  dd| j                  ddf   j                  | j                        }|j                  d| j                  dddf   j                  t        j                  | j                  | j
                  | j
                  | j                  z    |j                  
            }t!        ||f| j                  | j"                  | j$                        S )zsReturn a copy of `mat` with the specified nse.
  Note that if nse < mat.nse, this will potentially discard data.
  r   c              3  2   K   | ]  }t        d         y wrC   slicer5   is     r8   r9   z _bcoo_set_nse.<locals>.<genexpr>\   s     ?quT{?   .N   rD   c              3  2   K   | ]  }t        d         y wrC   rb   rd   s     r8   r9   z _bcoo_set_nse.<locals>.<genexpr>`   s     >aeDk>rf   dtyperE   rJ   rK   )operatorindexnserV   ranger7   rc   rW   r>   
zeros_likerE   atsetarrayn_sparserl   r_   rJ   rK   matrp   rV   rW   s       r8   _bcoo_set_nsery   S   s    	s#	/WW^JCGG^88L?E#++,>?LsLMDkk#ttQ,'G>>#((*qCHHNN<CKK,H*q#*qPSPXPXP^P^_b_j_jmn_n_oPp*qrD77O>5+=>OcggOPTTUXU]U]^DnnS[[0e#++2C2CCR2H0e#0es{{O`O`acOd0efGjjhswwh)*..s{{;Gjjcggh)*..syy3;;s{{]`]i]iOi9j?F}}0N OG	tWoSYY // //
1 1r^   c                n  	 | j                   | j                  | j                  }}}t        |||      }|dk(  j	                  t        t        |j                  dz   |j                                    }t        d t        |j                  d |j                         D              }|j	                  |d      }t        j                  ||j                  |j                  |j                  z    |j                        		fd} t        ||j                        ||      }t        t!        ||f|      |	      S )
Nr   rg   c              3  2   K   | ]  \  }}|d k(  s|  ywrg   Nr4   r5   re   ss      r8   r9   z'bcoo_eliminate_zeros.<locals>.<genexpr>p   s     \AUVZ[U[1\s   Tkeepdimsrk   c                N    t        j                  |d d d f   d d d f   |       S rC   )r>   where)re   m
fill_values     r8   <lambda>z&bcoo_eliminate_zeros.<locals>.<lambda>s   s%    399Qq$wZD!G)<a@ r^   rD   rp   )rV   rW   rE   _validate_bcoorN   tuplerq   r7   rL   	enumerater>   ru   rv   rl   r   bcoo_sum_duplicatesr_   )
rx   rp   rV   rW   rE   propsmaskdims_to_contractfr   s
            @r8   bcoo_eliminate_zerosr   l   s    3;;		$
w
.%
!)uU]]Q%6		BC	D$\7==%--3P)Q\\	"T	2$yyu}}U]]U^^-KLT[TaTab*@!(Jq%--($7'	T4/?S	IIr^   c                  6    e Zd ZU ded<   ded<   ded<   ded<   y)BCOOPropertiesintr7   rv   n_denserp   N)__name__
__module____qualname____annotations__r4   r^   r8   r   r   x   s    ,-,
(r^   r   c                  0    e Zd Zedd       Zedd       Zy)Bufferc                     y rC   r4   selfs    r8   rE   zBuffer.shape   s    r^   c                     y rC   r4   r   s    r8   rl   zBuffer.dtype   s    r^   N)returnr   )r   r   )r   r   r   propertyrE   rl   r4   r^   r8   r   r   ~   s      r^   r   c           
     J   t        ||      }|\  }}}}t        |      }t        d t        | j                  d | |d |       D              rt        d| j                  d|      | j                  |d  |f|||z   d  z   k7  r!t        d| j                  d|d|d|      |S )Nc              3  .   K   | ]  \  }}|d |fv  ywr|   r4   r5   s1s2s      r8   r9   z!_validate_bcoo.<locals>.<genexpr>   s     Yvr2Ar7	Y   z4data batch dimensions not compatible for data.shape=, shape=zInvalid data.shape=	 for nse=
, n_batch=
, n_dense=)_validate_bcoo_indicesr   anyr   rE   
ValueError)rV   rW   rE   r   r7   rv   r   rp   s           r8   r   r      s    
 %
0%$)!'8Wc
,%Y$**Xg2Fhw)XYY
LYPUxX
YY	ZZcVeGh,>,?&@@@
+

}J#{';gZP
QQ	,r^   c           
        t        j                  | j                  t         j                        sJ t	        |      }| j
                  dd  \  }}t        | j
                        dz
  }t        |      |z
  |z
  }|dk\  sJ t        d t        | j
                  d | |d |       D              rt        d| j
                  d|      | j
                  |d  ||fk7  r!t        d| j
                  d|d	|d
|      t        ||||      S )Nri   r1   r   c              3  .   K   | ]  \  }}|d |fv  ywr|   r4   r   s      r8   r9   z)_validate_bcoo_indices.<locals>.<genexpr>   s     \vr2Ar7	\r   z:indices batch dimensions not compatible for indices.shape=r   zInvalid =indices.shape=r   r   r   )r7   rv   r   rp   )r>   
issubdtyperl   integerr   rE   lenr   r   r   r   )rW   rE   rp   rv   r7   r   s         r8   r   r      s	   	s{{	33	3
,%--$-#x"'J 8+'	A\'--2I5QYRY?)[\\
RGMMCSS\V[U]^
__]]78h/
/ 0
cV;gZ{'T
UU	(GQT	UUr^   bcoo_todensec                Z    t        | j                  | j                  | j                        S )zConvert batched sparse matrix to a dense matrix.

  Args:
    mat: BCOO matrix.

  Returns:
    mat_dense: dense version of ``mat``.
  rT   )_bcoo_todenserV   rW   _info)rx   s    r8   r   r      s     
sxxSYY	??r^   c               ~    t         j                  t        j                  |       t        j                  |      |      S )a  Convert batched sparse matrix to a dense matrix.

  Args:
    data : array of shape ``batch_dims + (nse,) + block_dims``.
    indices : array of shape ``batch_dims + (n_sparse, nse)``
    spinfo : SparseInfo. In particular, this includes the shape
      of the matrix, which is equal to ``batch_dims + sparse_dims + block_dims``
      where ``len(sparse_dims) == n_sparse``

  Returns:
    mat : array with specified shape and dtype matching ``data``
  r   )bcoo_todense_pbindr>   asarrayrV   rW   rT   s      r8   r   r      s.     
		S[[.G0DV		TTr^   c          	        |j                   }t        | |      \  }}}}t        d t        |d | j                   d |       D              }t        t	        j
                  |ddd      t        fdt        |      D              }t        d |d | D              }	t	        j
                  g |	t	        j                  d      dddt              d d }
|s'| j                  |t        |
      | j                  	      } t        j                  || j                        j                  |
|z      j                  |       S )
Nc              3     K   | ];  \  }}|d k(  rt        j                  |t              nt        j                  |       = ywr|   npzerosr   aranger5   r~   i_ss      r8   r9   z%_bcoo_todense_impl.<locals>.<genexpr>   s<      RC *-RXXa%ryy|C R   AAijTindexingsparsec              3  D   K   | ]  }t        d       |fz        y wrC   rb   r5   re   gridrW   s     r8   r9   z%_bcoo_todense_impl.<locals>.<genexpr>   s$     O!WTU4[!$445O    c              3  F   K   | ]  }t        j                  |        y wrC   r   r   r5   r~   s     r8   r9   z%_bcoo_todense_impl.<locals>.<genexpr>   s     =ryy|=   !rg   rj   )r   rl   )rE   r   r   rQ   r   meshgridrq   r   sumrM   rl   r>   r   rs   add)rV   rW   rT   rE   r7   rv   _
ind_slices
sparse_indbatch_slices	batch_indr   s    `         @r8   _bcoo_todense_implr      s   
,,%*4%@'8Q R#&uXghw8O#PR R*	r{{JdC	D$OuXOO*=U8G_==,		Ml	MBIIaL	M4	M$Dk#2)	88Gd9oTZZ8HD	5$**	%	(	(Z)?	@	D	DT	JJr^   c               t    |j                   }t        | ||       t        j                  || j                        S rC   )rE   r   r   ShapedArrayrl   )rV   rW   rT   rE   s       r8   _bcoo_todense_abstract_evalr      s.    
,,%w&			%	,,r^   c                   t        | ||      S )Nr   )r   )data_dotrV   rW   rT   s       r8   _bcoo_todense_jvpr      s    	x	88r^   c                  |j                   }t        j                  |      sJ t        j                  |      rt        d      | j                   |k(  sJ | j                  |j
                  j                  k(  sJ t        ||       |fS )N/Cannot transpose with respect to sparse indices)rE   r    is_undefined_primalr   rl   aval_bcoo_extract)ctrV   rW   rT   rE   s        r8   _bcoo_todense_transposer      sw    
,,%				%%	%G$
F
GG	U			TYY__	$$	$	w	#W	,,r^   c               D    t        | ||      \  }}}t        |||      dfS )Nr   r   )r]   r   )rR   rS   rT   rV   rW   s        r8   _bcoo_todense_batching_ruler      s+    3L*fU$	tWV	4a	77r^   F)multiple_resultsbcoo_fromdenseTz
The error arose for the nse argument of bcoo_fromdense. In order for
BCOO.fromdense() to be used in traced/compiled code, you must pass a concrete
value to the nse (number of stored elements) argument.
rp   r7   r   index_dtypec          	         t        j                  |       } |}|t        | ||      }t        j                  t
        j                  |t              }t        t        | ||||      | j                  dd      S )a  Create BCOO-format sparse matrix from a dense matrix.

  Args:
    mat : array to be converted to BCOO.
    nse : number of specified elements in each batch
    n_batch : number of batch dimensions (default: 0)
    n_dense : number of block_dimensions (default: 0)
    index_dtype : dtype of sparse indices (default: int32)

  Returns:
    mat_bcoo: BCOO representation of the matrix.
  r   Trm   )r>   r   r   r   concrete_or_errorrn   ro   _TRACED_NSE_ERRORr_   _bcoo_fromdenserE   )rx   rp   r7   r   r   nse_arrnse_ints          r8   r   r      so     	C# #'_$S'7;G""8>>7<MN'	ocw*57IId4
I Ir^   r7   r   r   c                   t        j                  |       } t        j                  t        j
                  |t              }t        j                  | ||||      S )aA  Create BCOO-format sparse matrix from a dense matrix.

  Args:
    mat : array to be converted to BCOO, with ``ndim = n_batch + n_sparse + n_dense``.
    nse : number of specified elements in each batch
    n_batch : number of batch dimensions (default: 0)
    n_dense : number of block_dimensions (default: 0)
    index_dtype : dtype of sparse indices (default: int32)

  Returns:
    data : array of shape ``mat.shape[:n_batch] + (nse,) + mat.shape[mat.ndim - n_dense:]``
      and dtype ``mat.dtype``
    indices : array of shape ``mat.shape[:n_batch] + (n_sparse, nse)``
  r   )	r>   r   r   r   rn   ro   r   bcoo_fromdense_pr   )rx   rp   r7   r   r   s        r8   r   r     sN      	C#x~~s4EF#			sWg+6 
 
8 8r^   c                  t        j                  |       } | j                  |z
  |z
  | dk7  }|dkD  r,|j                  t	        |      D cg c]  }|dz    
 c}      }t        t        |d      fd       } ||      }|s)t        j                  |j                  d | dfz   |      }n.t        j                  t        j                  ||      d|dz         }t        ||       }	|j                  t        t	        ||j                                    d   }
t        j                  |
j                   d|z  fz   |      |
k  }||dz   t#        d       fz  |dz  z      }t        j$                  ||	d      }	|	|fS c c}w )	Nr   rg   F)Nbroadcastedc                h    | j                   r%t        j                  | | j                  d        S y)Nsizer   r4   )rL   r>   nonzerorE   )arv   rp   s    r8   _nonzeroz&_bcoo_fromdense_impl.<locals>._nonzero+  s+    vv[[(1CDDr^   .Nrg   rC   )r>   r   rL   r   rq   r   r   r   rE   r?   ru   r   r   listr	   broadcasted_iotarl   rc   r   )rx   rp   r7   r   r   r   re   r   rW   rV   true_nsetrue_nonzerosrv   s    `          @r8   _bcoo_fromdense_implr  $  sb   C#XX')(
($q[88uW~6!q1uX67D
:e4 5 TN'	ii

8G,Qx7EGll399Wk:Aw{KG	w	$$XXd5$))456yA(&&x~~tg~7NPWX[cc-1t >7AR RS-	=$	*$	w% 7s   E:c                  | j                   |z
  |z
  }| j                  d | |fz   | j                  ||z   d  z   }| j                  d | ||fz   }t        j                  || j                        t        j                  ||      fS rC   )rL   rE   r   r   rl   )rx   rp   r7   r   r   rv   
data_shapeindex_shapes           r8   _bcoo_fromdense_abstract_evalr	  >  s    XX')(yy'"cV+cii(8J8K.LL*		(7#sHo5+			*cii	0$2B2B;P[2\	\\r^   c                  | \  }|\  }t        |||||      }|\  }	}
t        |      t        j                  u r t        j                  j	                  |	      }nt        |
|      }|t        j                  j	                  |
      f}||fS )Nr   )r   typer    Zerofrom_primal_valuer   )primalstangentsrp   r7   r   r   MMdotprimals_outrV   rW   r   tangents_outs                r8   _bcoo_fromdense_jvpr  E  s    "!
%$sGWZef+-$	$Z277ww((.HWd+HBGG55g>?,	l	""r^   c                  | \  }}|j                   |z
  |z
  }|j                  |j                  d | |fz   |j                  ||z   d  z   k(  sJ |j                  |j                  d | ||fz   k(  sJ |j                  |k(  sJ t        |t        j
                        rt        d      t	        j                  |      sJ t        ||t        |j                  j                              S )Nr   r   )rL   rE   rl   
isinstancer    r  r   r   r   r   r   )	r   r  rp   r7   r   r   rV   rW   rv   s	            r8   _bcoo_fromdense_transposer  U  s    -$VVg'(	qwwx(C61AGGGh<N<O4PP	PP	P	!''(7+xo=	==	=	+	%%	%!
F
GG				""	"	tWZ-E	FFr^   c               |    | \  }|\  }d|cxk  r|k  sn t        d|d|      t        |||dz   ||      ||ffS )Nr   z'Expected 0 < bdim <= n_batch; got bdim=r   rg   r   )r   r   )rR   rS   rp   r7   r   r   r  rA   s           r8   _bcoo_fromdense_batching_ruler  `  sZ    "!
%$
t
w

?$7*M
NN	Wq['Wb	cfjlpeq	qqr^   bcoo_extractassume_uniquec                  t        | t              st        dt        |             t	        j
                  |      }|j                  | j                  k7  r%t        d| j                  d|j                        || j                  }t        | j                  ||      }t        || j                  ffi | j                  j                         S )a   Extract values from a dense array according to the sparse array's indices.

  Args:
    sparr : BCOO array whose indices will be used for the output.
    arr : ArrayLike with shape equal to self.shape
    assume_unique : bool, defaults to sparr.unique_indices
      If True, extract values for every index, even if index contains duplicates.
      If False, duplicate indices will have their values summed and returned in
      the position of the first index.

  Returns:
    extracted : a BCOO array with the same sparsity pattern as self.
  zGFirst argument to bcoo_extract should be a BCOO array. Got type(sparr)=zshape mismatch: sparr.shape=z	 a.shape=r  )r  r_   	TypeErrorr  r>   r   rE   r   rK   r   rW   r   _asdict)sparrarrr  r   rV   s        r8   r  r  s  s     
E4	 
^RVW\R]Q_`
aa	kk#!WW
4~ZqwwjA
BB((M	u}}a}	E$	tU]]#	=u{{':':'<	==r^   c               2    t         j                  | ||      S )a  Extract BCOO data values from a dense array at given BCOO indices.

  Args:
    indices: An ndarray; see BCOO indices.
    arr: A dense array.
    assume_unique: bool, default=True
      If True, then indices will be assumed unique and a value will be extracted
      from arr for each index. Otherwise, extra work will be done to de-duplicate
      indices to zero-out duplicate extracted values.

  Returns:
    An ndarray; see BCOO data.
  r  )bcoo_extract_pr   )rW   r!  r  s      r8   r   r     s     
		Wc		GGr^   c          	         t        j                  |      }t         |j                        }|s3t	         |j                  d      \   }|}t         |j                        }t        d t        |j                  d |j                    j                  d |j                         D              }t        t        j                  |ddd      t         fdt        |j                        D              }t        d |j                  d |j                   D              }t        j                  g |t        j                  d      dddt              d d	 }	||	z   s|d    }
n#|j                  |	|z      j                  d
d      }
|j                  dk(  r|j                  dk7  r|rTt!        j"                  |
t%        |
j                  |j                  |j                        t        |
j&                              }
nt%        |
j                  |j                  j                        }|j                  t)        d       fz  t)        d      fz   }t        j*                  |
|      j                  |   j-                  |
      }
|s]j                  g|
j                  |j                  dz   d  fd}t        |j                        D ]  }t/        |      }  ||
      }
|
S )NT)rE   return_indexc              3     K   | ];  \  }}|d k(  rt        j                  |t              nt        j                  |       = ywr|   r   r   s      r8   r9   z%_bcoo_extract_impl.<locals>.<genexpr>  s<      bC *-RXXa%ryy|C br   r   r   c              3  D   K   | ]  }t        d       |fz        y wrC   rb   r   s     r8   r9   z%_bcoo_extract_impl.<locals>.<genexpr>  s$     U!WTU4[!$445Ur   c              3  F   K   | ]  }t        j                  |        y wrC   r   r   s     r8   r9   z%_bcoo_extract_impl.<locals>.<genexpr>  s     Gryy|Gr   rg   rj   fillr   moder   rD   c                j    t        j                  |       j                  |   j                  |       S NrD   )r>   rr   rs   r   )rre   unbatched_out_shapes     r8   r   z_bcoo_extract_impl.<locals>.f  s+    ^^A%89<<Q?CCAFFr^   )r>   r   r   rE   _unique_indicesr   rQ   r7   r   r   rq   rv   r   rs   getrp   r	   broadcast_in_dim_tuple_replacerL   rc   rr   rt   r   )rW   r!  r  r   sort_indoriginal_propsr   r   r   r   result	out_shapeindr   r   r   r/  s   `              @@r8   _bcoo_extract_implr9    sa   C#
 #))
4%	'syytTGXN"7CII6E b#&syy%--'@'--P^QVQ^Q^B_#`b b*	r{{JdC	D$UuU^^?TUU*GSYY~-FGG,		Ml	MBIIaL	M4	M$Dk#2)	i	YFVVI
*+//V/JF
^^qUYY!^##v||U]]EIIFfkkHZ\f !u}}n>P>PQiMMU4[N*eAh[8c~~fI699#>BB6Jf	)--QU]]Q=N=O0PQG5==! 
q'avx F	-r^   c                   t        |      }t        | |j                        \  }}}}|j                  d | |fz   |j                  |j                  |z
  d  z   }t	        j
                  ||j                        S rC   )rM   r   rE   rL   r   r   rl   )rW   r!  r  r   r7   r   rp   r7  s           r8   _bcoo_extract_abstract_evalr;    sm    
=!3GSYYG'1gsii!SF*SYYsxx'7I7J-KK)			)SYY	//r^   c               T    | j                   |j                   k(  sJ t        || |      S )Nr  )rE   r   )arr_dotrW   r!  r  s       r8   _bcoo_extract_jvpr>    s'    	#))	##	#	w}	EEr^   c               .   |st        d      t        j                  |      sJ t        j                  |      rt        d      | j                  |j
                  j                  k(  sJ |t        | |t        |j
                  j                              fS )Nz2transpose of bcoo_extract with assume_unique=Falser   r   )	rO   r    r   r   rl   r   r   r   rE   )r   rW   r!  r  s       r8   _bcoo_extract_transposer@    sz    	
R
SS				$$	$G$
F
GG	SXX^^	##	#	-GJsxx~~4NO	OOr^   c                  | \  }}t        d |D              sJ |d   |d   }t        j                  ||f      }nr|d   R|d   }t        |j                        }|j                  ||j                  |          t        j                  |||f      }n|d   |d   k7  rt        d      |d   }|j                  dz
  }||k\  rt        d|d|      t        |||      |fS )	Nc              3  $   K   | ]  }|d u 
 y wrC   r4   )r5   r6   s     r8   r9   z._bcoo_extract_batching_rule.<locals>.<genexpr>  s     /qQd]/s   r   rg   z+bcoo_extract with unequal batch dimensions.r1   zbatch_dims=z' out of range for indices with n_batch=r  )r   r	   r=   r  rE   insertr2  rO   rL   r   r   )rR   rS   r  rW   r!  rA   result_shaper7   s           r8   _bcoo_extract_batching_rulerE    s    ,'3	/J/	//	/]a=Doogw/G!}a=D		?LgmmD12


sL4'
:C!}
1% MNNa=DLL1'	W_

}$LG:N
OO	w=	A4	GGr^   bcoo_transposec                    t         j                   j                  | j                        }t	         fd|D              }t        || j                        S )u-  Transpose a BCOO-format array.

  Args:
    mat: A BCOO-format array.
    permutation:  A tuple or list or ndarray which contains a permutation of
      [0,1,..,N-1] where N is the number of axes of ``mat`` in the order of
      batch, sparse, and dense dimensions. The i’th axis of the returned array
      corresponds to the axis numbered permutation[i] of ``mat``. Transpose
      permutation currently does not support permuting batch axes with non-batch
      axes nor permuting dense axes with non-dense axes.

  Returns:
    A BCOO-format array.
  permutationrT   c              3  <   K   | ]  }j                   |     y wrC   rD   r5   prx   s     r8   r9   z!bcoo_transpose.<locals>.<genexpr>	  s     6QCIIaL6   )rE   rK   )_bcoo_transposerV   rW   r   r   r_   rK   )rx   rI  buffersr7  s   `   r8   rF  rF    sI     CHHckk{SVS\S\]'6+66)	gYs7I7I	JJr^   c                   t        |      }|t        t        t        |j                                    k(  r| |fS t        j                  | |||      S NrH  )r   rq   r   rE   bcoo_transpose_pr   )rV   rW   rI  rT   s       r8   rN  rN    sR    k"+E%FLL 1233=  wK(. ! 0 0r^   c                   t        |t        t        t        j                  f      st        dt        |       d      t        t        |            t        t        t        |                  k7  rt        d| d| d      t        | ||      \  }}}}|d | }||||z    D 	cg c]  }	|	|z
  	 }
}	|||z   d  D 	cg c]
  }	|	|z
  |z
   }}	|r;t        t        |            t        t        |            k7  rt        d| d|d      |r;t        t        |            t        t        |            k7  rt        d| d|d      ||
|fS c c}	w c c}	w )	Nz8transpose permutation must be a tuple/list/ndarray, got r;   zQtranspose permutation isn't a permutation of operand dimensions, got permutation z for shape zUtranspose permutation cannot permute batch axes with non-batch axes; got permutation z, with n_batch=zUtranspose permutation cannot permute dense axes with non-dense axes; got permutation z, with n_dense=)r  r   r  r   ndarrayr  r  sortedrq   r   r   rO   )rV   rW   rI  rE   r7   rv   r   r   
batch_permrL  sparse_perm
dense_perms               r8   _validate_permutationrY    s   	K%rzz!:	;
NtT_O`Naabc
dd
6+5s5z):#;;
 ''2m;ugQH I I"0w"F'8Wa8G$*&1'7X;M&NOWO+O0;Gh<N<O0PQ1Hw&Q*Qvj)*eE'N.CC
 11<=MWJaQ R Rvj)*eE'N.CC
 11<=MWJaQ R R	[*	,, PQs   #E;Ec                   t        | |||j                        \  }}}t        |       |d|f   j                  g |dz    } | j                  g |fd|D         } | |fS )N.rg   c              3  .   K   | ]  }|z   d z     ywr|   r4   r5   dr7   s     r8   r9   z'_bcoo_transpose_impl.<locals>.<genexpr>,  s     /TAGa/T   )rY  rE   r   	transpose)rV   rW   rI  rT   rV  rW  rX  r7   s          @r8   _bcoo_transpose_implr`  '  s    (=dG[Z`ZfZf(g%*k:
O'/GC$%//RRWRgPQkR'		U	UW	U/T/T	U$	wr^   c                 	 t        | |||j                        \  }}}t        |      	t        j                  |j                        g |		dz      }t        j                  | j                        g |		fd|D           }t        j                  || j                        t        j                  ||j                        fS )Nrg   c              3  .   K   | ]  }|z   d z     ywr|   r4   r\  s     r8   r9   z0_bcoo_transpose_abstract_eval.<locals>.<genexpr>4  s     <aQRQ[1_<ar^  )rY  rE   r   r   ru   r   r   rl   )
rV   rW   rI  rT   rV  r   rX  indices_shaper  r7   s
            @r8   _bcoo_transpose_abstract_evalrd  /  s    3D';PVP\P\]*a
O'((7==)*MJ*M*M1*MN-xx

#$bj$b'$b<aV`<a$bc*			*djj	143C3CMSZS`S`3a	aar^   c                   | \  }}|\  }}t        ||||      }t        ||||      \  }	}||	t        j                  j                  |      ffS rQ  )rN  r    r  r  )
r  r  rI  rT   rV   rW   r   r   r  data_dot_outs
             r8   _bcoo_transpose_jvprg  7  sX    -$+(Ag;vV+#Hg;W]^/,	|RWW%>%>w%GH	HHr^   c               H   | \  }}t        |t        j                        sJ t        j                  |      rt	        d      |j
                  |j                  j
                  k(  sJ t        t        fd|D                    }t        t        t        t        j                  |                  }t        j                  t!        |j"                  dz
        D 	cg c]  }	d c}	t        |j$                  dd        z   t              }
t'        ||
||      \  }}||fS c c}	w )Nr   c              3  <   K   | ]  }j                   |     y wrC   rD   )r5   rL  rT   s     r8   r9   z,_bcoo_transpose_transpose.<locals>.<genexpr>D  s     D1v||ADrM  r1   rg   ri   rk   rH  )r  r    r  r   r   rl   r   r   r   r  mapr   r   argsortr>   r   rq   rL   rE   rN  )r   rV   rW   rI  rT   data_ct
indices_ct	ct_spinforev_permutationre   dummy_indices
data_transr   s       `        r8   _bcoo_transpose_transposerr  >  s    ':	J	((	(G$
F
GG	$))//	))	)DDDE)S"**["9:;/))gllQ.>(?@1Q@4VXVYHZC[[cfg-!'=o^gh-*a	Z	 As   	Dc                  t        | ||      \  }}}dgd |D        }t        ||||      \  }}|D cg c]  }|d nd
 }}t        ||f|      D 	cg c]   \  }	}|t        j                  |	dg      n|	" }
}	}|
|fS c c}w c c}}	w )Nr   c              3  &   K   | ]	  }|d z     ywr|   r4   )r5   rL  s     r8   r9   z-_bcoo_transpose_batch_rule.<locals>.<genexpr>M  s     9a!e9   rH  )r]   rN  rQ   r	   squeeze)rR   rS   rI  rT   rV   rW   batched_permutationrA   batch_dims_outr@   args_outs              r8   _bcoo_transpose_batch_rulerz  K  s    3L*fU$:9[9:!$=PY_`-$<FGDDLDa/G.G"D'?NCE#t (,|ckk#s#< E( E	>	!! HEs   B%Bbcoo_dot_general)	precisionpreferred_element_typeout_shardingc          
     n   ~~t        | t              rt        |t              r}t        | j                  |j                  |      }t	        | j
                  | j                  |j
                  |j                  | j                  |j                  ||      }t        ||      S t        | t              r/t        | j
                  | j                  |||| j                        S t        |t              r/t        | |j
                  |j                  |||j                        S t        j                  | |||      S )a<  A general contraction operation.

  Args:
    lhs: An ndarray or BCOO-format sparse array.
    rhs: An ndarray or BCOO-format sparse array..
    dimension_numbers: a tuple of tuples of the form
      `((lhs_contracting_dims, rhs_contracting_dims),
      (lhs_batch_dims, rhs_batch_dims))`.
    precision: unused
    preferred_element_type: unused

  Returns:
    An ndarray or BCOO-format sparse array containing the result. If both inputs
    are sparse, the result will be sparse, of type BCOO. If either input is dense,
    the result will be dense, of type ndarray.
  
lhs_spinfo
rhs_spinfodimension_numbersr}  rD   r  r}  r  )r  r}  r  r  r}  )r  r_   r   rE   _bcoo_spdot_generalrV   rW   r   _bcoo_dot_general_bcoo_rdot_generalr	   dot_general)lhsrhsr  r|  r}  r~  rE   bufss           r8   r{  r{  a  s    , Tz#t4(CII):<Esxxchh*-))		1B6LND E""#tSXXs{{CK\4J(+		3 3 #tc388S[[L]5K),4 4 ??37H2HJ Jr^   c                  |\  \  }}\  }}	t        j                  |      t        j                  |      f}
t        j                  |      t        j                  |	      f}|t        j                  |      }t        j                  t        j                  |       t        j                  |      t        j                  |      |
|f||      S Nr  )r   _ensure_index_tupler   rl   bcoo_dot_general_pr   r>   r   )lhs_datalhs_indicesr  r  r}  r  lhs_contractrhs_contract	lhs_batch	rhs_batchcdimsbdimss               r8   r  r    s     :K6< 6I''5''57%''	2''	24%'XX&<=		 	 X!6K8PRUR]R]^aRb495>8N,6 
! 
8 8r^   c                  t        d |D              }t        ||| |||      }d |D        \  }}	t        |j                        |z
  }
g t	        |	      t	        |
|j
                        t	        |	|
      }t        j                  ||      S )Nc              3  ,   K   | ]  }|d d d     y wNrj   r4   r5   r]  s     r8   r9   z%_bcoo_rdot_general.<locals>.<genexpr>  s     9]a!DbD'9]s   )r  r  r}  c              3  8   K   | ]  }t        |d            ywr   Nr   r  s     r8   r9   z%_bcoo_rdot_general.<locals>.<genexpr>  s     >qQqT>s   )r   r  r   rE   rq   rL   r	   r_  )r  rhs_datarhs_indicesr  r}  r  dimension_numbers_reversedr6  
n_contractr7   n_swaprI  s               r8   r  r    s     5:9]K\9]4]X{CJ/I4JL& ?,=>*gz :-&W%.W5#=Wgv@VW+	v{	++r^   c          
        t        j                  |       } t        j                  |      }t        j                  |      }t        | j                  |j                  |j                  ||      }|j                  d   |j
                  dz
  |\  \  }}\  }	}
t        t        ||      D cg c]  \  }}|k  s||f c}}      \  }t        t        ||      D cg c]  \  }}|k\  s||f c}}      \  }|	srng |	t        t              |	      }| j                  g |t        | j
                              } |j                  g |t        |j
                              }rGt        fdD              t        j                  g t        t                          }|d|f   }g |
||t        t        |j
                        |
|      }|j                  |      }fd}t        |t              z
        }t        j                   |t        t        |
      t              z
              }t        j"                  |j                  |j$                        } ||| ||      S c c}}w c c}}w )Nr  rj   r1   c              3  (   K   | ]	  }|z
    y wrC   r4   r\  s     r8   r9   z)_bcoo_dot_general_impl.<locals>.<genexpr>  s     Eaa'kE   .c                   t        fdt              D              }|d t               }|t              d  }|rOj                  dkD  r@t	        j
                  d j                  d d D        ddid j                  dz
   }g ||}t        t        t        
      t              z               }t        j                  ||j                  |   j                  dd	      g g f||ff
      }	|r| j                  |   j                  |	      S |	j                  t        t        |	j                  | j                  z
              | j                        S )Nc              3  ,   K   | ]  }d |f     ywr   r4   )r5   re   r  s     r8   r9   z9_bcoo_dot_general_impl.<locals>.result.<locals>.<genexpr>  s     =CF#=   r1   c              3  F   K   | ]  }t        j                  |        y wrC   )r>   r   )r5   ns     r8   r9   z9_bcoo_dot_general_impl.<locals>.result.<locals>.<genexpr>  s     :aCJJqM:r   rj   r   r   r)  r   r*  )r}  rk   )r   rq   r   rL   r>   r   rE   r  rM   r	   r  rs   r1  r   r   rl   )	out_arrayr  r  r  idx	idx_rightidx_out	idx_batchrS   prodlhs_contracting_blhs_contracting_srv   r}  s     `       r8   r6  z&_bcoo_dot_general_impl.<locals>.result  sL   
=U8_=
=C+S*+,I#'()*G[%%),,:;#4#4Sb#9:.+**Q.0i +I*	*ieC 12T:K5LLMNJ??8SVVI%6%:%:ST%:%UHz:&>?2HJD \\'"&&t,,XXeE$))inn"<=>iooXVVr^   )r>   r   _bcoo_dot_general_abstract_evalr   rE   rL   r   r   r%   rq   r_  r   ru   r   r   r	   r=   r   rl   )r  r  r  r  r}  r  out_avallhs_contractingrhs_contractingr  r  lr.  rhs_contracting_brhs_contracting_srV  rW  rhs_permr6  r  r  r  r7   rv   s       `               @@@@r8   _bcoo_dot_general_implr    s   [["(K(+C#,X]]K<L<Lchh?PDZ8BD( r"(q '?P<$?O&<y))/A1Rq!Q[QF1R *S&&)/A1Sq!Q'\QF1S *T&& #k9kywDUVkYjkJ!!"OJ"Ow1N"OPH''(X*(XuWkFVFV7W(XYK E3DEE))`/`)E(OM^2_`aKc;./KGy G, G/@ Gsxx)_EG(h#W" fg,=(>>?&U3y>7SAR=S3STU#ii7)		8[#	66U1R1Ss   1J
?J
$J
2J
c               :  
 t        j                  t        j                  d      j	                  t        j
                  |j                  | j                        t        j
                  |j                  |j                        ||      }|\  \  }}\  }	}t        | ||j                        \  
}}|	rt        |	      
k\  rt        d|	d
      t        
fd|D              rt        d      t        j                  |j                  |j                        S )Nr  static_argnameszqbcoo_dot_general batch dimensions must be among the batch dimensions in the sparse representation.
got lhs_batch=r   c              3  .   K   | ]  }|z   k\    y wrC   r4   )r5   r]  r7   rv   s     r8   r9   z2_bcoo_dot_general_abstract_eval.<locals>.<genexpr>  s     :Qg 	 :r^  z4bcoo_dot_general: contracting over dense dimensions.)jaxjitr	   r  
eval_shapeShapeDtypeStructrE   rl   r   rP   rO   r   r   r   )r  r  r  r  r}  r  r  r  r   r  r7   rv   s             @@r8   r  r    s     WWS__6efqq


z//
@


syy#))
4-!7	 r 9( *;&?AA*8[*BRBRS'8Q3y>W,
L7*&' '
 	:/::
T
UU			(..(..	99r^   c                >   | j                   t        vr)t        j                  d| j                   dt               y|j                   t
        vr)t        j                  d|j                   dt               y|j                  st        j                  dt               yy)NzJbcoo_dot_general cusparse/hipsparse lowering not available for data.dtype=z). Falling back to default implementation.TzMbcoo_dot_general cusparse/hipsparse lowering not available for indices.dtype=zbcoo_dot_general GPU lowering requires matrices with sorted indices. To sort the rows in your matrix, use e.g. mat = mat.sort_indices(). Falling back to the default implementation.F)rl   CUSPARSE_DATA_DTYPESwarningswarnr   CUSPARSE_INDEX_DTYPESrJ   r   s      r8   _bcoo_dot_general_fallbackr    s    	ZZ++MM %%NP+- }}11MM ( ((QS+-   MM $ &?@ r^   c                  t         j                  j                  st        | |||||      S |\  \  }}\  }}	t	        | ||j
                        \  }
}}}	|j                  dk(  rt        nt        }t        | |||||      }|j                  t        vrt        | |||||      S | j                  |j                        } |j                  |j                        }t        |      dk(  rt        |      dk(  r|j                  dv r|
||fdk(  rt        | ||      st        j                   |j
                  d   |j                        |j#                         }}d}dg|j
                  }t%        ||||      \  }}}|j'                  | |||d   dk(  r|j(                  n|||      }|d   S t        |      dk(  rt        |      dk(  r|j                  dv r}|
||fdk(  rut        | ||      sh|d d df   |d d df   }}|d   dk(  }|j
                  }t%        ||||      \  }}}|j'                  | |||d   dk(  r|j(                  n|||      }|d d	 S t        | |||||
      S )Nr  rg   r   )rg   r1   )r   rg   r   F)r_  rE   )r   r1   r   rj   )r  r  r}  )r   bcoo_cusparse_loweringvaluer  r   rE   rL   r   r   r  rl   r  astyper   r  r>   r   ravel!_coo_correct_out_of_bound_indicesr   T)r  r  r  r  r}  r  r  r  r  r   r7   rv   r   coo_matmul_pr  rowcolr_  rE   outs                       r8   _bcoo_dot_general_gpu_implr    s    
	&	&	,	,!(K+5 
 2C.<A"0Z--#/'8Wa"xx1}*,,k3'1	( ^^//!(K+5 
 __X^^,(

8>>"# 
,1Y1!4V9KHg
&)
3(;
Kyy**1-{/@/@A;CTCTCVCI"!!"E7S%SOCe


Hc3%1!_%9CEEs&/u  >C q6MLQ3y>Q#6388v;Mh(I5*8[*M1a4 +ad"3CaA%IE7S%SOCe


Hc3%1!_%9CEEs&/u  >C s8O!(K+
57 7r^   c               $    t        | |||||      S r  r  )lhs_data_dotr  r  r  r  r}  r  s          r8   _bcoo_dot_general_jvp_lhsr  M  s     	<cM^2HU_
a ar^   c               $    t        ||| |||      S r  r  )rhs_dotr  r  r  r  r}  r  s          r8   _bcoo_dot_general_jvp_rhsr  R  s     	8['M^2HU_
a ar^   c                   t        j                  |      rJ |\  \  }}\  }	}
t        |j                        }t        j                  |      r|j                  j
                  n|j
                  }t        t        |      ||	      }t        t        |      ||
      }t        t        t        |	||            \  }}}t        j                  |      r||f||
ff}t        t        j                  |t        j                  |                  }t        |	      |z   |z   }t        t        t        t        j                  |                  }t        j                   |j
                  dz
  dz  |j                  d   fz         }t#        |j                  d d       |j                  d   dz  z    	 t%        |||        d}|rZt)        |||t+                     \  }}t#         fd|D              }t-        | |||	      }t)        |||t+        |            \  }}n:t/        j0                  | ||	      }t/        j2                  ||      }t5        ||      }|||fS ||f|	|ff}t        t        j                  |t        j                  |                  }t        t        j                  t        |
      |z   |z               }t7        ||| |||
      }||t/        j2                  ||      fS # t&        $ r d}Y :w xY w)Nr1   r   ri   rj   TFrH  c              3  (   K   | ]	  }|     y wrC   r4   )r5   re   placeholder_shapes     r8   r9   z._bcoo_dot_general_transpose.<locals>.<genexpr>w  s     Ga.q1Gr  r  r  r}  r  )r    r   r   rE   r   rL   r%   rq   rj  r  r$   r   takerk  r   r>   emptyr   rY  rO   rN  r   bcoo_dot_general_sampledr	   r  r_  r   r  )!r   r  r  r  r  r}  r  r  r  r  r  lhs_ndimrhs_ndimlhs_keptrhs_kept	ans_batchans_lhsans_rhsdimslhs_contract_sorted_by_rhsrI  out_axesplaceholder_dataindices_can_be_untransposedr   lhs_indices_Tresult_T_shaperesult_Tr6  out_dense_T	out_denserhs_contract_sorted_by_lhsr  s!                                   @r8   _bcoo_dot_general_transposer  W  s   ##K00	09J6< 6I!!"( 44S9SXX]]sxx(uXi@(uXi@( #D+i8*T U)WgH%")8!4y)6L MD!%bgglBJJ|<T&U!Vy/H,/IIKCRZZ456H yy+"2"2Q"6$!>+BSBSTVBWAY!YZk//458I8I"8MPT8TT),k;HYZ %)!
 #()9;T_0:;L0MOaG;GGn)"c=TXYh!(Mx)3N)CEifa OOBtDk--X6i[)4f;##w)Y!78D!%bgglBJJ|<T&U!VBJJtI1KKhVWXHxbZ6L157F [#--"AAA;  *$)!*s   *K& &K54K5c                   | \  }}}|\  }}}t        | d d |d d ||d n|j                  |         \  }}	}
t        t        |j                        |j                  fd|f|      \  }}t        ||	||
||      }||fS )Nr1   rU   r   r  )r]   rE   r&   r   rL   r  )rR   rS   r  r}  r  r   r  rhs_bdimnew_lhs_datanew_lhs_indicesnew_lhs_spinfonew_dimension_numbersresult_batch_dimbatched_outs                 r8   _bcoo_dot_general_batch_ruler    s    )!Q.!Q2K!j!nj'tSYYx-@3B/, -H
:chh'!X8I-K))!,Q_9O4IK+ 
&	&&r^   cuda)platformrocmr  c                   |\  \  }}\  }}t        j                  |      t        j                  |      f}t        j                  |      t        j                  |      f}	t        j                  | ||||	f      S )ah  A contraction operation with output computed at given sparse indices.

  Args:
    lhs: An ndarray.
    rhs: An ndarray.
    indices: BCOO indices.
    dimension_numbers: a tuple of tuples of the form
      `((lhs_contracting_dims, rhs_contracting_dims),
      (lhs_batch_dims, rhs_batch_dims))`.

  Returns:
    BCOO data, an ndarray containing the result.
  r  )r   r  bcoo_dot_general_sampled_pr   )
ABrW   r  r  r  r  r  r  r  s
             r8   r  r    s     :K6< 6I''5''57%''	2''	24%	#	(	(Aw<A5> 
) 
K Kr^   c          	     H    t        |t        j                  | |||            S )Nr  r|  )r   r	   r  )r	  r
  rW   r  r|  s        r8   _bcoo_dot_general_sampled_slowr    s     	w1HYen o	ppr^   c                  ~|\  \  }}\  }}|s|s|s|rJ | j                   |j                   cxk(  rdk(  sJ  J |j                   dz
  }	|j                  d   }
|j                  d   }|	|
z   dk(  sJ |	dk(  rO| j                  |d d df      j                  dd      |j                  |d d df      j                  dd      z  S |	dk(  r-| d d d f   |j                  |d      j                  dd      z  S |	dk(  rA| d d d d f   |d d d d f   z  }t	        j
                  |t        |       t        |      |fd	      S t        d
      )Nrg   r1   rj   ri   r   r)  r*  .r   r   rg   r1   too many batch dimensions.)rL   rE   rs   r1  r	   r2  r   r   r	  r
  rW   r  r|  r  r  r  r  r7   rv   rp   r  s                r8    _bcoo_dot_general_sampled_simpler    sq    9J6< 6Ili9E	E	
166	Q			LL1']]2(b#	8	q	  	 \DDA##A#>dd71a4=!%%6a%@A B!|QW:WV_-11v!1LLL!|
AtTM
QtQ}-
-Cc!fc!fc%:IFF
1
22r^   c                  |\  \  }}\  }}|s|rJ t        |      t        |      cxk(  rdk(  sJ  J | j                  |j                  cxk(  rdk(  sJ  J |j                  dz
  }	|j                  d   }
|j                  d   }|	|
z   dk(  sJ |	dk(  r|d   dk(  rdgndg}|d   dk(  rdgndg}| j                  t	        t        d       t        d       f|d   |d d df            j                  dd      } |j                  t	        t        d       t        d       f|d   |d d df            j                  dd      }t        j                  | |||f||ff|      S |	dk(  r|d   dk(  rdgndg}|d   dk(  rdgndg}|j                  t	        t        d       t        d       f|d   |d	            j                  dd      }|d   dk(  rdg}t        j                  | |||f||ff|      S |	dk(  rYt        j                  | |||f||ff|      }t        j                  t        j                  |d
      g |j                  |d      S t        d      )Nrg   r1   rj   ri   r   r)  r*  r  r  )r1   r  r  )r   rL   rE   rs   r3  rc   r1  r	   r  r2  r=   r   r  s                r8   !_bcoo_dot_general_sampled_simple2r    s    :K6< 6I9%	%	\	c,/	41	44	44	4	
166	Q			LL1']]2(b#	8	q	  	 \#A!+!I#A!+!I	^U4[%+6	!gaQRdmTUYY_ersYtA	^U4[%+6	!gaQRdmTUYY_ersYtA??1a\<4PS\^gRh3i%.0 0\#A!+!I#A!+!I	^U4[%+6	!gfoVW[[agtu[vAA!Sl??1a\<4PS\^gRh3i%.0 0\
//!QL,3OR[]fQg2h$-/CT :<Mcii<M<MyYY
1
22r^   c               >   t        j                  |       } t        j                  |      }t        j                  |      }|\  \  }}\  }}|j                  dz
  }|j                  d   }	t        j
                  j                  }
|s@|s>|s<|s:| j                  |j                  cxk(  rdk(  rn n|	|z   dk(  rt        | ||||
      S t        |      dk(  r<|s:| j                  |j                  cxk(  rdk(  rn n|	|z   dk(  rt        | ||||
      S t        | ||||
      S )Nr1   rj   rg   r  )r>   r   rL   rE   r	   	PrecisionHIGHESTr  r   r  r  )r	  r
  rW   r  r  r  r  r  r7   rv   r|  s              r8   _bcoo_dot_general_sampled_implr    s   	kk!n!	kk!n!KK '9J6< 6ILL1']]2(mm##) <9	
&&AFF
a
Hw$6!$;+Aq'M^jstt!I!&&AFF2Ga2GHW^L^bcLc,Q7N_ktuu 
(1gIZfo	ppr^   c                  t        j                  dt        j                  | |ft	                    }t        j                  fd| ||      \  }t        j                  dt        ||fi       }t        j                  d |||      \  }|S )N&bcoo_dot_general_sampled_abstract_evalr  c                 0    t        j                  | digS )Nr  )r	   r  )argsr  s    r8   r   z9_bcoo_dot_general_sampled_abstract_eval.<locals>.<lambda>  s    coot6q_p6q5r r^   )
debug_infoc                     t        |  gS rC   )r   )r  s    r8   r   z9_bcoo_dot_general_sampled_abstract_eval.<locals>.<lambda>  s    }d7K6L r^   )r   r  r	   r  dictpeabstract_eval_funr   )r	  r
  rW   r  dbgdense_resultsparse_results      `   r8   '_bcoo_dot_general_sampled_abstract_evalr&    s    DOOaVTL]5^	`#&&'rtuwx257-,D)G\+BB	H#''(LgWc368.-	r^   c                  t        |d      r|j                  j                  n|j                  }t        |d      r|j                  j                  n|j                  }t        |||      }t	        j
                  t        j                  || j                              }t        | ||d      \  }} |d d d d}	 t	        j                  t        j                        | ||fi |	\  }}|||fS )Nr   Tr  )r  r|  r}  r~  )hasattrr   rE   r   r    UndefinedPrimalr   r   rl   r@  get_primitive_transposer	   dot_general_p)
r   r	  r
  rW   r  A_shapeB_shape	mat_shaperx   kwdss
             r8   #_bcoo_dot_general_sampled_transposer0    s    #Av.AFFLLAGG'#Av.AFFLLAGG'*7G=NO)
4++Irxx@A#'GSM+'20$(
 $ 
7	#	#C$5$5	6r1a	H4	H$!Q	
Awr^   c                    t        | |||      S Nr  r  )A_dotr	  r
  rW   r  s        r8   _bcoo_dot_general_sampled_jvp_Ar5  ,  s    	!%GGX	YYr^   c                    t        || ||      S r2  r3  )B_dotr	  r
  rW   r  s        r8   _bcoo_dot_general_sampled_jvp_Br8  /  s    	!!UGGX	YYr^   c               4    fd} t        ||d      |  dfS )Nc                "    t        | ||      S r2  )r  )r	  r
  rW   r  s      r8   implz2_bcoo_dot_general_sampled_batch_rule.<locals>.impl3  s    )!QK\]]r^   r   )in_axesr  r   )rR   rS   r  r;  s     ` r8   $_bcoo_dot_general_sampled_batch_ruler=  2  s$    ^	3dJ	3\	BA	EEr^   bcoo_spdot_generalc          
         |\  \  }}	\  }
}t        j                  |      t        j                  |	      f}t        j                  |
      t        j                  |      f}t        j                  | |||||||f|      S )Nr  )r   r  bcoo_spdot_general_pr   )r  r  r  r  r  r  r  r}  r  r  r  r  r  r  s                 r8   r  r  G  s     :K6< 6I''5''57%''	2''	24%		"	"8[(K.8Z6;U^:P 
# 
R Rr^   c          	        |j                   }	|j                   }
t        | ||	      }t        |||
      }|j                  |j                  cxk(  rdk(  sJ  J |j                  |j                  cxk(  rdk(  sJ  J D cg c]  }|	|   	 c}D cg c]  }|
|   	 c}k(  sJ t	        d      |j
                  k  sJ t	        d      |j
                  k  sJ g fdt        |	      D        fdt        |
      D        }|d d t        j                  t              f   }|d d t        j                  t              f   }|d d t        j                  t        t        |j
                              t              f   }|d d t        j                  t        t        |j
                              t              f   }|d d d f   |d d d f   k(  j                  d      }t        j                  t        j                  D cg c]  }|	|   	 c}|j                        t        |j                  dz
              }t        j                  t        j                  D cg c]  }|
|   	 c}|j                        t        |j                  dz
              }||k  j                  d      }||k  j                  d      }t        j                   ||d d d f   z  |d d d f   z  | d d d f   |d d d f   z  d      j#                         }t        j$                  |j&                  |j&                  |j                   d   |j                   d   z   gt        j(                  ||            }|j*                  d d d d d |j                   d   f   j-                  |d d d f         }|j*                  d d d d |j                   d   d f   j-                  |d d d f         }|j/                  t1        |      |j                   d         }t3        ||t5        |      |	      S c c}w c c}w c c}w c c}w )
Nr   rj   defaultc              3  2   K   | ]  \  }}|vs|  y wrC   r4   )r5   re   r~   r  s      r8   r9   z0_bcoo_spdot_general_unbatched.<locals>.<genexpr>b       EDAqA_,DaE   c              3  2   K   | ]  \  }}|vs|  y wrC   r4   )r5   re   r~   r  s      r8   r9   z0_bcoo_spdot_general_unbatched.<locals>.<genexpr>c  rE  rF  rk   rg   rD   rT   rp   )rE   r   r7   r   rP   rv   r   r>   ru   r   r%   rq   rN   r=   rl   rL   r   r  r  rp   result_typers   rt   reshaper   _bcoo_sum_duplicatesr   )r  r  r  r  r  r  r  r  out_nse	lhs_shape	rhs_shaper  r  r]  r7  lhs_irhs_ilhs_jrhs_joverlaplhs_fill_valuerhs_fill_value	lhs_valid	rhs_validout_dataout_indicess         ``                  r8   _bcoo_spdot_general_unbatchedrZ  T  s   ))xi8#xi8#		(q	((	((	(		(q	((	((	( /	01)A,	0?4[aYq\4[	[[	[	_b	)CLL	88	8	_b	)CLL	88	8GEIi(EGEIi(EG) a?#>>
?%
a?#>>
?%
a9U3<<-@/#RZ]^^
_%
a9U3<<-@/#RZ]^^
_%
 1d7^uT1W~-2226'??II_5y|5U[[I	%**q.. ??II_5y|5U[[I	%**q.. ~%**2.)~%**2.)YYw1d7!33ia6HH4(8D!G+<<aAAF  		377CGGU[[_u{{2-NO #[ IK+q!%5ekk"o%556::5D>J+q!U[[_%556::5q>J+##CM;3D3DR3HI+ 
hJY<W]d	eeG 
14[$ 6 6s   P-P2P7+P<c               h   |j                   }|j                   }	t        | ||      }
t        |||	      }|
j                  |j                  cxk(  rdk(  sJ  J t        | j                  |j                  |j                  |j                  ||||      \  }}|j                   d   }|\  \  }}\  }}g |t        t        |
j                        |      }g |t        t        |j                        |      }| j                  g |t        |
j                  | j                              } |j                  g |t        |j                  |j                              }|j                  g |t        |
j                  |j                              }|j                  g |t        |j                  |j                              }t        j                  t        t        ||
j                  d        t        |	|j                  d        |D cg c]  }||
j                  z
   c}|D cg c]  }||j                  z
   c}|      }t        ||j                  t        |      z
  d      }t        ||
j                  t        |      z
  d      }t        |t        |            } || |||      S c c}w c c}w )Nr   r  rj   )r  r  r  r  rL  )NNr   r   r<  )r   r   NN)rE   r   r   !_bcoo_spdot_general_abstract_evalr   r%   rq   r7   r_  rL   	functoolsr   rZ  r   r   r   )r  r  r  r  r  r  r  r}  rM  rN  r  r  	data_avalr   rL  r  r  r  r  lhs_batch_permrhs_batch_permr]  funcs                          r8   _bcoo_spdot_general_implrc    sl    ))xi8#xi8#		(q	((	((	(2MM;##X]]K4D4DjDU13,)Q OOB'?P<$?O&<y) KYJ5+=y!IJ.JYJ5+=y!IJ. U. U5hmm3T UV( U. U5hmm3T UV(%%&^&^s{{KL\L\9]&^_+%%&^&^s{{KL\L\9]&^_+ 
		8Ickkl34Ickkl340?@1q3;;@0?@1q3;;@
$ 
D#++I6@R	S$	D#++I6@R	S$	D#i.	)$	hX{	;; A@s   J*#J/c          	         |j                   |j                   }t        j                  t        j                  d      j                  t        j                   j                        t        j                  |j                        ||      }	t               }
t        |      }|\  \  }}\  |
j                  s|j                  rt        d      rt              |
j                  k\  sr#t              |j                  k\  rt        d      |rHt        |      |
j                  k  s%t        |      |
j                  |
j                  z   k\  rt        d      |rHt        |      |j                  k  s%t        |      |j                  |j                  z   k\  rt        d      |j                  t              kD  r#|
j                  t        |      kD  rt!        d      |
j                  t        |      kD  r|
j"                  nd|j                  t        |      kD  r|j"                  ndz  }|
j                  |j                  z   t              z
  }t        |t%        j&                  |	j                   |d              }t)        j*                  t-         fdt/        |
j                        D              t-        fd	t/        |
j                        D                    }t)        j*                  t-        fd
t/        |j                        D              t-        fdt/        |j                        D                    }g fdD        |||}g fdD        ||||
j                  |j                  z   dt        |      z  z
  }t1        j2                  ||	j                        }t1        j2                  |j                        }t        |||	j                          ||fS )Nr  r  z)bcoo_spdot_general with dense dimensions.z`bcoo_spdot_general: batch_dims must correspond to batch dimensions of the sparse representation.z?bcoo_spdot_general only supports contraction of sparse indices.zXbcoo_spdot_general: cannot have unused batch dims on rhs with unused sparse dims on lhs.rg   c              3  F   K   | ]  }|vsj                   |     y wrC   rD   )r5   rG   r  r  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  !     	T#s)?S(..
	T   	!!c              3  F   K   | ]  }|vsj                   |     y wrC   rD   )r5   rG   r  r  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  #     	WS#YBV+

C
 	Wrg  c              3  F   K   | ]  }|vsj                   |     y wrC   rD   )r5   rG   r  r  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  rf  rg  c              3  F   K   | ]  }|vsj                   |     y wrC   rD   )r5   rG   r  r  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  ri  rg  c              3  (   K   | ]	  }|     y wrC   r4   r5   rG   rM  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>       *in*r  c              3  (   K   | ]	  }|     y wrC   r4   rm  s     r8   r9   z4_bcoo_spdot_general_abstract_eval.<locals>.<genexpr>  rn  r  r1   )rE   r  r  r	   r  r  r  rl   r   r   rO   rP   r7   minrv   r   r   rp   mathr  r   broadcast_shapesr   rq   r   r   )r  r  r  r  r  r  r  r}  rN  r  r  r  r  r  rL  out_n_batchlhs_batch_shaperhs_batch_shaper  rc  r_  indices_avalr  rM  r  s   ````                  @@@r8   r]  r]    sm    ))WWS__6efqq	9hnn5	9hnn5)3	 r 5( 	xi8#xi8#?P<$?O&<y)[[CKK
I
JJC	Nckk1yS^WZWbWbEb
  A  B  B#o.<O@TX[XcXcfifrfrXr@r
_
``#o.<O@TX[XcXcfifrfrXr@r
_
``[[3y>!cllS5I&I
o
pp s?33SWWs?33SWW< 
 ckk)C	N:+8>>+,#?@A'''		Ts{{);	TT		WE#++,>	WW/ ''		Ts{{);	TT		WE#++,>	WW/
*	*  	*
E*	*EE E 	E \\CLL(1s?/C+CC	E- z8>>:)!!-1B1BC,L(..9	L	  r^   c          
     H   t        |j                        }t        |j                        }t        d t        | |      D              }t	        | d d |d d ||      \  }	}
}t	        | dd  |dd  ||      \  }}}t        ||fd|      \  }}t        |	|
||||||      }|||ffS )Nc              3  F   K   | ]  \  }}|	|j                   |     y wrC   rD   rF   s      r8   r9   z1_bcoo_spdot_general_batch_rule.<locals>.<genexpr>  s"     chc3SVSb399S>crH   r1   r  r   r   )r  r  r  r}  )r   rE   rP   rQ   r]   r&   r  )rR   rS   r  r  r}  r  r  r  rU   r  r  r  r  r  r  s                  r8   _bcoo_spdot_general_batch_rulerz    s    !!"(!!"(c#lJ2Occ*&?!j!njZ'I#(K&?jnjZ'I#(K(CF$5)7%%#Hk8[6G/9j;QS+ 
')9:	::r^   c                   | \  }}}}|\  }}}	}
t        | i |}t        |      t        j                  u sJ t        |
      t        j                  u sJ d}t        |      t        j                  ur|t        ||||fi |d   z  }t        |	      t        j                  ur|t        |||	|fi |d   z  }||t        j                  j	                  |d         gfS )Nr   rg   )r  r  r    r  r  )r  r  r/  r  r  r  r  r  lhs_indices_dotrhs_data_dotrhs_indices_dotr  rf  s                r8   _bcoo_spdot_general_jvpr    s    18.(K;AI>,#W55+	o	"''	))	)	o	"''	))	),	,rww&'k8[a\`abcddL	,rww&'+|[a\`abcddL	|RWW%>%>{1~%NO	OOr^   bcoo_sort_indicesc                    t        j                  | j                  d| j                  i\  }}t	        ||f| j
                  d| j                        S )zxSort indices of a BCOO array.

  Args:
    mat : BCOO array

  Returns:
    mat_out : BCOO array with sorted indices.
  rT   Trm   )bcoo_sort_indices_pr   _bufsr   r_   rE   rK   )rx   rV   rW   s      r8   r  r    sI     &**CIIHciiH-$	tWoSYYt //
1 1r^   c                   t        | ||j                        }|j                  dk(  r| |fS t        t        |j
                  d      } ||      \  }}t        d |j
                        } || |      } | |fS )Nr   Fr   c                    | |   S rC   r4   r]  rL  s     r8   r   z)_bcoo_sort_indices_impl.<locals>.<lambda>-  
    AaD r^   )r   rE   rv   r   _bcoo_sort_indices_unbatchedr7   )rV   rW   rT   r   r   permpermutes          r8   _bcoo_sort_indices_implr  &  sq    
w
5%
^^q=-u}}%P!G*-'4(%--8'	t	$	wr^   c                      j                   \  }} fdt        |      D        }t        j                  g |t        j                   j
                  |      |      ^  }t        j                         |fS )Nc              3  0   K   | ]  }d d |f     y wrC   r4   r5   re   rW   s     r8   r9   z/_bcoo_sort_indices_unbatched.<locals>.<genexpr>4  s     .gadm.   )num_keys)rE   rq   r	   sortiotarl   r>   column_stack)rW   rp   r   idx_colsr  s   `    r8   r  r  1  sc    ==&#q.U1X.(88EhE(DEPQR.7D			'	"D	((r^   c          
        t        | ||j                        }|j                  dk(  r| |fS t        j                  g t        t        |j                  d |j                   | j                  d |j                         |j                  | j                  |j                  dz   d  | j                  | j                        }||fS )Nr   rg   	weak_type)r   rE   rv   r   r   rj  rP   r7   rp   rl   r  )rV   rW   rT   r   data_outs        r8    _bcoo_sort_indices_abstract_evalr  8  s    
w
5%
^^q=1c#w}}^emm,djj%--.HI 1
YY1EMMA-./126**X( 
7	r^   c                   t        | ||      \  }}}t        j                  |||      \  }}d}|d   |d   }d}||f|fS )Nr   ry  rg   r   r  )r]   r  r   )rR   rS   rT   rV   rW   r  indices_outr  s           r8    _bcoo_sort_indices_batching_ruler  B  sb    3L*fU$-2242P(K( ]a.KH
K	 (	**r^   c                  t        g | |j                   }|j                  dk(  r| |fS | \  }}|\  }}t        t        |j
                        } ||      \  }	}
t        d |j
                        } |||
      }t        j                  j                  |      }t        |      t        j                  u rt        j                  j                  |      n |||
      }||	f||ffS )Nr   c                    | |   S rC   r4   r  s     r8   r   z(_bcoo_sort_indices_jvp.<locals>.<lambda>V  r  r^   )
r   rE   rv   r   r  r7   r    r  r  r  )r  r  rT   r   rV   rW   r   r   r   r  r  r  r  indices_dot_outrf  s                  r8   _bcoo_sort_indices_jvpr  M  s    

0'
06<<
0%
^^qH-$+(A-u}}=!j+t(%--8'T4 (GG--g6/8<X"''8Q**84W^_gimWn,
K	 <"A	AAr^   r   c                    t        | j                  | j                  | j                  |      \  }}t	        ||f| j
                  dd      S )a  Sums duplicate indices within a BCOO array, returning an array with sorted indices.

  Args:
    mat : BCOO array
    nse : integer (optional). The number of specified elements in the output matrix. This must
      be specified for bcoo_sum_duplicates to be compatible with JIT and other JAX transformations.
      If not specified, the optimal nse will be computed based on the contents of the data and
      index arrays. If specified nse is larger than necessary, data and index arrays will be padded
      with standard fill values. If smaller than necessary, data elements will be dropped from the
      output matrix.

  Returns:
    mat_out : BCOO array with sorted indices and no duplicate indices.
  rH  Trm   )rK  rV   rW   r   r_   rE   rw   s       r8   r   r   l  sC     'sxxSYYTWX-$	tWoSYYt!
# #r^   c                   |%t        j                  t        j                  |d      }t        j                  | |||      S )N$nse argument of bcoo_sum_duplicates.rH  )r   r   rn   ro   bcoo_sum_duplicates_pr   )rV   rW   rT   rp   s       r8   rK  rK    s:    _

 
 6\
]C		#	#D'&c	#	JJr^   c          
        t        | ||j                        }t        ||j                  dd      \  }}}|!|j                  dk(  rdn|j	                         }t        |||j                        }|j                  dk(  r| j                  |j                  d      } t        j                  g t        t        |j                  d |j                   | j                  d |j                         || j                  |j                  dz   d  | j                        }d }	t        |	|j                        }	 |	|||       }||fS )	NTrE   return_inversereturn_true_sizer   rg   rp   rE   r   rk   c                B    | j                   |   j                  |d      S Ndrop)r+  rs   r   )d_outr   r]  s      r8   r   z+_bcoo_sum_duplicates_impl.<locals>.<lambda>  s     ? r^   )r   rE   r0  rv   rP   _adjust_indices_nser   r7   r>   r  rj  rl   r   )
rV   rW   rT   rp   r   r  mappingnse_batchedr  r  s
             r8   _bcoo_sum_duplicates_implr    s!   
w
5%&56<<t'M#+w[~~"!(9C#KSM+
^^q88EMMD81DYY >S'--"?NU]]A[\ >>"jj):);<>EIZZQ(?'w.'Xw-(	;	r^   c                  t        | |      }||j                  k  r| dd |d d f   } | S t        j                  t	        j
                  ||j                  |j                  |j                  z    | j                        g | j                  d d ||j                  z
  | j                  d   | j                  dz
  f      }t        j                  | |g| j                  dz
        } | S )	N.rk   ri   rj   rg   )operandrE   broadcast_dimensionsr1   	dimension)r   rp   r	   r2  r>   ru   r7   rv   rl   rE   rL   concatenate)rW   rp   rE   r   r)  s        r8   r  r    s    
 %
0%EIIc4C4l#G 
. iiemmEMMENN,JKSZS`S`aEgmmCR E#		/E7==3DE#LL1,.D
 oowo9IJG	.r^   )r  r%  r  c                   t        | |      }t        t        ||j                  d  |||      }t	        ||j                  d      } ||       S )N)rE   r  r%  r  Fr  )r   r   _unique_indices_unbatchedr7   r   )rW   rE   r  r%  r  r   r   s          r8   r0  r0    sN    
 %
0%'uU]]^/D+,/1! EMMu5!	
7r^   c          	        t        | |      }|j                  dk(  rxd}t        j                  | |df      }|f}|rg |t        j                  |d      }|rg |t        j                  |d      }|rg ||}t        |      dk(  r|d   S |S t        j                  t        j                  |d |j                   | j                        d      }	| |	k\  j                  dd	      }
t        j                  |
|	|       } t        | d||||j                  |	
      }|r)|rdnd}g |d | ||   j                         ||dz   d  }|r>|d   }|| |	k(  j                         j                  |j                        z
  }g |d d |}|S )Nr   rg   rD   int32rk   )r   rj   Tr   )axisr  r%  r  r   r   r1   )r   rv   r>   rr   r   r   r=   ru   rl   r   r   r+   rp   r  r  )rW   rE   r  r%  r  r   rp   r  r  r   out_of_boundsr  s               r8   r  r    s   
 %
0%
^^q
C..a9K.C1c1399S01c1c1399S01ccK3KcX]3q6++syy)?w}}UW[\*j(--b4-@-IImZ9'a\!1		j	Z#!C
8CI
8s3x~~'
8#cAgh-
8C
b'C
J&++-44SYY?
?C
CH
c
C	*r^   c                   | j                   |j                   k(  s J | j                    d|j                           t        |      | j                   dz   k(  s"J t        |       d| j                   dz           | j                   dkD  r<t        t        || j                   d  |      } t	        || j                         | |      S | |d   k\  ||d   k\  z  }|r?t        j                  |d|       } t        j                  ||d   |      }|d   |d   dz   f}n>t        j                  ||d   |       } t        j                  |d|      }|d   dz   |d   f}| ||fS )Nz != rg   )rE   r_  r   )rL   r   r   r  r   r>   r   )r  r  rE   r_  r   r   s         r8   r  r    sO    
SXX	:#((4z::		Usxx!|	#FE
|41~%FF	#XX\1CHHI&)	=A":a"3,,
q/cU1Xo	.$
))D!S
!C
))D%(C
(C1XuQx!|$E
))D%(C
(C
))D!S
!C1X\58$E	c5r^   c          
        |t        d      t        | ||j                        }t        j                  g |j                  d |j
                   ||j                  |j                  |j                        }t        j                  g t        t        |j                  d |j
                   | j                  d |j
                         || j                  |j
                  dz   d  | j                  | j                        }||fS )Nbcoo_sum_duplicates: nse must be specified when using the function within jit, vmap, and other transformations requiring abstract evaluation.)rl   r  rg   r  )r   r   rE   r   r   r7   rv   rl   r  rj  rP   )rV   rW   rT   rp   r   r  r  s          r8   "_bcoo_sum_duplicates_abstract_evalr    s    [
 [ \ \
w
5%  !V7==%--#@!V#!Vu~~!V(/ARART++c#w}}^emm,djj%--.HI ++::emma'()+,0JJ$..R( 
;	r^   c                   t        | ||      \  }}}t        j                  ||||      \  }}|d   t        j                  |dg      }d}	nd}	||f|	fS )NrH  rg   r   r  ry  )r]   r  r   r	   rv  )
rR   rS   rT   rp   rV   rW   
new_spinfor  r  r  s
             r8   "_bcoo_sum_duplicates_batching_ruler    so    7jRXY$/44T7:[^4_(K ]++kA3/KHH
K	 (	**r^   c          
     <   t        g | |j                   }| \  }}|\  }}t        ||j                  dd      \  }	}
}|t        j                  |      }	 t        j                  t        j                  |d      }t        |	||j                        }	|j                  dk(  r:|j	                  |j                  d      }|j	                  |j                  d      }t        j                  g t        t         |j                  d |j                   |j                  d |j                         ||j                  |j                  dz   d  |j"                  	      }|}|j$                  rd
 }nd }t'        ||j                        } |||
|      }t(        j*                  j-                  |	      }t/        |      t(        j*                  u rt(        j*                  j-                  |      n	 |||
|      }||	f||ffS # t
        j                  $ r t        d      w xY w)NTr  r  r  r  r   r   rg   rk   c                B    | j                   |   j                  |d      S r  r  xre   ys      r8   r   z*_bcoo_sum_duplicates_jvp.<locals>.<lambda>  s    add1gkk!&k9 r^   c                    | S rC   r4   r  s      r8   r   z*_bcoo_sum_duplicates_jvp.<locals>.<lambda>  s    a r^   )r   rE   r0  r>   r   r   r   rn   ro   ConcretizationTypeErrorr   r  rv   r7   r  rj  rP   rl   r   r   r    r  r  r  )r  r  rT   rp   r   rV   rW   r   r   r  r  r  r  rf  r  r  s                   r8   _bcoo_sum_duplicates_jvpr    s   

0'
06<<
0%-$+(A&56<<t'M#+w[
''+
C\

 
 6\
]C $KSM+
^^q88EMMD81D||EMMD|9HYY >S'--"?NU]]A[\ >>"jj):);<>EIZZQ(, ]]9GGw.'Xw-(GG--k:/8<X"''8Q**84W^_kmtv~W,
K	 <"A	AA) 
	%	% \
 [ \ \\s   %G< <Herrorr7   r   on_inefficientc          	       
 | j                   nt        j                        | j                  nt        j                        f| j                   | j                  fk(  r| S | j                  z
  z
  }dvrt        d      dk  rt        d       dk  rt        d       |dk  rt        ddd	| j                   d
      fd}| j                  kD  r]t        d | j                   | j                  | j                  z
   D              r& |d| j                   d| j                   d d       | j                   kD  rOt        d | j                  | j                    D              r& |d| j                   d| j                    d d       | j                  | j                  }}| j                  | j                   
| j                  }|k  r|z
  t        t        
dz         fd       }	 |	||      \  }} |j                  g |j                  d
 t        j                  |j                  

dz          |j                  
dz   d  } |j                  g |j                  d
 t        j                  |j                  

dz          |j                  
dz   d  }}
k  r,
z
  t        t              fd       }	 |	||      \  }}
|kD  r1|z
  t        t        
dz         fd       }	 |	||      \  }}}
kD  r0
z
  t        t        
      
fd       }	 |	||      \  }}
t        ||f      S )a  Update the storage layout (i.e. n_batch & n_dense) of a BCOO matrix.

  In many cases this can be done without introducing undue storage overhead. However,
  increasing ``mat.n_batch`` or ``mat.n_dense`` will lead to very inefficient storage,
  with many explicitly-stored zeros, unless the new batch or dense dimensions have size
  0 or 1. In such cases, ``bcoo_update_layout`` will raise a :class:`SparseEfficiencyError`.
  This can be silenced by specifying the ``on_inefficient`` argument.

  Args:
    mat : BCOO array
    n_batch : optional(int) the number of batch dimensions in the output matrix. If None,
      then n_batch = mat.n_batch.
    n_dense : optional(int) the number of dense dimensions in the output matrix. If None,
      then n_dense = mat.n_dense.
    on_inefficient : optional(string), one of ``['error', 'warn', None]``. Specify the
      behavior in case of an inefficient reconfiguration. This is defined as a reconfiguration
      where the size of the resulting representation is much larger than the size of the
      input representation.

  Returns:
    mat_out : BCOO array
      A BCOO array representing the same sparse array as the input, with the specified
      layout. ``mat_out.todense()`` will match ``mat.todense()`` up to appropriate precision.
  N)r  r  NzJon_inefficent={on_inefficient!r}; expected one of ['error', 'warn', None].r   z"n_batch must be non-negative; got z"n_dense must be non-negative; got zsum of n_batch=z and n_dense=z  cannot be larger than mat.ndim=r;   c                |    dk(  r| dz  } t        |       dk(  r!| dz  } t        j                  | t               y y )Nr  z` To disable this error, set the on_inefficient argument of bcoo_update_layout to 'warn' or None.r  zX To disable this warning, set the on_inefficient argument of bcoo_update_layout to None.)category)r   r  r  r   )msgr  s    r8   _maybe_err_or_warnz.bcoo_update_layout.<locals>._maybe_err_or_warnT  sP     	 9 :c!#&&	6	!	 / 0cmmC"9: 
"r^   c              3  &   K   | ]	  }|d kD    ywr|   r4   r  s     r8   r9   z%bcoo_update_layout.<locals>.<genexpr>`  s     	DA!a%	Dru  zFor matrix of shape z, increasing n_dense from z to z  results in inefficient storage.c              3  &   K   | ]	  }|d kD    ywr|   r4   r  s     r8   r9   z%bcoo_update_layout.<locals>.<genexpr>c  s     "QQ1q5"Qru  z, increasing n_batch from rg   r   c                    | j                   t        j                  | j                  d        g| j                  d   }t	        j
                  fd| j                  d  D        ddi}t	        j                  t	        j                  |j                  d   j                  f      gt        t        j                  |            }||fS )Nc              3  `   K   | ]%  }t        j                  |j                          ' ywrk   Nr>   r   rl   )r5   r~   re   s     r8   r9   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>q  s#     Pqcjj!''::P   +.r   r   r   )rJ  rq  r  rE   r>   r   r  broadcast_tor   rj  r  )r]  re   new_dmeshesnew_ir  s    `   r8   _updatez#bcoo_update_layout.<locals>._updaten  s    aii		!''"1+.==e||PAGGBQKP +%)+f 0 0U[[^QVV4L M  9!$SYY!7 9 :eE\r^   r1   c                   j                   d   } | j                  t        j                  | j                   d dz          g| j                   dz   d   }t	        j
                  fdg j                   d  |D        ddi} j                  t        j                  j                   d dz          gj                   dz   d   }t	        j                  g d |d d D        |      }||fS )Nri   rg   c              3  `   K   | ]%  }t        j                  |j                          ' ywr  r  )r5   r]  re   s     r8   r9   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>  s#     Xqcjj!''::Xr  r   r   c              3  <   K   | ]  }|j                           y wrC   )r  )r5   r   s     r8   r9   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>  s     !A!'')!A   rj   )rE   rJ  rq  r  r>   r   r  )r]  re   rp   r  r  r  r  s    `    r8   r  z#bcoo_update_layout.<locals>._update  s    GGBKcaii		!''&1q5/2EQWWQUV_Ee||XDWaggbqkDWSVDWX +%)+faii		!''&1q5/2EQWWQUV_EeI!AVCR[!AI5IJeE\r^   c                    t        j                  |  d        j                  t        | d           j	                  |       }|d   }||fS r-  )r>   rr   rs   r   rt   )r]  re   r  r  r  r   rE   s       r8   r  z#bcoo_update_layout.<locals>._update  sU    nnQeWHI&67::5A23=IMMaPe!feE\r^   c                   j                   d   }t        fdt        	      D              t        j                  |      fz   }g 
 |j                   d   	z
  }t        j
                  d d 	d f   |      }g 
 || j                   | j                  z
  d  }t        j                  | |      j                  |   j                  |       }||fS )Nri   c              3  0   K   | ]  }d d |f     y wrC   r4   )r5   jre   s     r8   r9   z6bcoo_update_layout.<locals>._update.<locals>.<genexpr>  s     ,a!AqD',r  rj   rD   )
rE   r   rq   r>   r   r  rL   rr   rs   rt   )r]  re   rp   r  new_i_shaper  new_d_shaper  current_n_batchr  r7   r   rE   s    `      r8   r  z#bcoo_update_layout.<locals>._update  s    GGBKc,58,,

3/AAcKeOG4KcK1772;?KkqABx5eWeOG4WcWAGGAFFWDTDU<VWknnQk255c:>>qAeE\r^   rD   )r7   rn   ro   r   rL   r   r   rE   rV   rW   r   r   rJ  rq  r  r_   )rx   r7   r   r  rv   r  new_datanew_indicescurrent_n_denser  r  r  rE   s    ```      @@@r8   bcoo_update_layoutr  !  s   B #?CKKw0G'"?CKKw0G'wCKK55JXX')(22
a
bbq[
9'C
DDq[
9'C
DD\
'wjgZ 877:xxjC D D; 		DG8CHHs{{,BC	DD-cii[8R++d7)3SU Vs{{s"Q#))CKK2P"QQ-cii[8R++d7)3SU V ((CKKK(
))%KK/KK/'!AZ?Q./ 0 $Hk:Hkx G0@!A G $		(..[\I\*] ^G!)!0C0D!EGH &+%% P{'8'89I/'J P&*ii0A0A/SbefSf0g&hP'2'8'819L9M'NPK O'!AZ7# $ $Hk:HkO/!AZ?Q./ 0 $Hk:HkO/!AZ?+ , $Hk:HkO	x%U	33r^   )shardingc          	     t    t        t        | j                  | j                  | j                  ||      |      S )a  Expand the size and rank of a BCOO array by duplicating the data.

  A BCOO equivalence to jax.lax.broadcast_in_dim.

  Args:
    mat: A BCOO-format array.
    shape: The shape of the target array.
    broadcast_dimensions: The dimension in the shape of the target array which
      each dimension of the operand (``mat``) shape corresponds to.

  Returns:
    A BCOO-format array containing the target array.
  )rT   rE   r  rD   )r_   _bcoo_broadcast_in_dimrV   rW   r   )rx   rE   r  r  s       r8   bcoo_broadcast_in_dimr    s7     
$SXXs{{399+0:NP 
 r^   c                  t        |j                        t        |      k7  rt        d|j                  d|d      t        | ||j                        }t	        ||j
                  |j                  g      \  }}}t        |d      t        |t        |            kD  rt        d      t        |d      t        |t        |            kD  rt        d      t        ||j
                  d t        |            }	|j                  xr% t        |      t        ||j                   d       z
  }
t        |      |	z
  |
z
  t        j                  |j                  |j
                  |j
                  |j                  z          t        j                  ||	|	z          k7  rt        d	      | |}}|	|
f|j
                  |j                  fk7  rt        j                  |g |d|	 |j                  ||	z   d g ||	fd
|D              }t        j                  |g |d|	 |j                  |j                  g ||	|	dz         }|j                  k7  rhg |d|	 |j                  }t!        j"                  |t$              |	z
  }t!        j&                  ||      j(                  d|f   j+                  |      }||fS )z'BCOO equivalent of lax.broadcast_in_dimzspinfo.shape=z and broadcast_dimensions=z must have the same lengthr   rB  z>Cannot mix batch and sparse dimensions during broadcast_in_dimz>Cannot mix sparse and dense dimensions during broadcast_in_dimNz*Adding sparse dimensions with lengths != 1c              3  .   K   | ]  }|d z   z
    ywr|   r4   )r5   r6   new_n_sparses     r8   r9   z)_bcoo_broadcast_in_dim.<locals>.<genexpr>  s     9cST!a%,:N9cr^  )rE   r  rg   rD   .)r   rE   r   r   r   r7   rv   rP   rp  r   rq  r  rO   r	   r2  rp   r>   ru   r   rr   rs   rt   )rV   rW   rT   rE   r  r   rS   sparse_dims
dense_dimsnew_n_batchnew_n_denser  r  r8  r  s                 @r8   r  r    s    	#233
&A,@+BB\]
^^
w
5%(23G%--Y^YgYgIh(i%*k:Q#k3u:"FF
U
VVa 3z3u:#FF
U
VV (8#e*M+Y#e*s3G3X/Y"Y+Uk)K7,	YYv||EMM5==5>>+IJKtyyY^_jkv  zF  lF  ZG  PH  H
J
KKK( ;EMM5==#AA##HTl{#TUYYT{\7Q7R1STdzd;d9cXb9cdfH &&{?l{#?UYY??HzH;HaHJK
 U^^#;eL[!;599;l;E
))K
%
3C>>+U;>>sCxHLL[YK	;	r^   c          
     `	   t        j                  |      }t        d | D              st        d|        t	        j
                  t        j                  d      j                  | D cg c],  }t        j                  |j                  |j                        . c}|      }t        | D ch c]  }|j                   c}      dkD  rt        d      | D ch c]  }|j                   }}t        |      dk7  rt!        |      dk(  rt        d | D              r,| D cg c]   }|j                  dk(  rt#        |d	      n|" } }n=t        d
 | D              r+| D cg c]   }|j                  dk(  rt#        |d	      n|" } }| D ch c]  }|j                   }}t        |      dk7  rt        d      | d   j                  | d   j$                  }}| D cg c]  }|j&                  j                  d|  }}| D cg c]  }|j(                  j                  d|  }}||k  r6|D 	cg c]  }	|	d| |	|dz   d z    }}	|D 	cg c]  }	|	d| |	|dz   d z    }}	t        t+        |            t        t+        |            cxk(  rdk(  st-        d       t-        d      ||k  r| D ch c]  }|j.                   }
}t        |
      dk7  r$t!        |
      }| D cg c]  }t1        ||       } }t        j                  | D cg c]  }|j&                   c}|      }t        j                  | D cg c]  }|j(                   c}|      }n|||z   k  rt3        j4                  dg| dd D cg c]  }|j                  |    c}z   | d   j&                  j                        }t        j                  | D cg c]  }|j(                   c}|      }t        j                  t7        | |      D cg c]2  \  }}|j&                  j8                  d||z
  f   j;                  |      4 c}}|      }nt-        d      t=        ||f|j                        S c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}	w c c}	w c c}w c c}w c c}w c c}w c c}w c c}w c c}}w )aP  Sparse implementation of :func:`jax.lax.concatenate`

  Args:
    operands : Sequence of BCOO arrays to concatenate. The arrays must have equal
      shapes, except in the `dimension` axis. Additionally, the arrays must have
      have equivalent batch, sparse, and dense dimensions.
    dimension : Positive integer specifying the dimension along which to concatenate
      the arrays. The dimension must be among batch or sparse dimensions of the input;
      concatenation along dense dimensions is not supported.

  Returns:
    A BCOO array containing the concatenation of the inputs.
  c              3  <   K   | ]  }t        |t                y wrC   )r  r_   r5   ops     r8   r9   z#bcoo_concatenate.<locals>.<genexpr>  s     5bZD!5r  zIbcoo_concatenate: expected operands to be a sequence of BCOO arrays. Got r  r  rg   zAbcoo_concatenate requires inputs to have matching nse dimensions.c              3  `   K   | ]&  }|j                   d k(  s|j                  d    dk(   ( yw)r   rg   Nr7   rE   r   s     r8   r9   z#bcoo_concatenate.<locals>.<genexpr>  s'     
B"**/288A;!
B   ..r   r7   c              3  `   K   | ]&  }|j                   d k(  s|j                  d   d k(   ( ywrg   r   Nr  r   s     r8   r9   z#bcoo_concatenate.<locals>.<genexpr>  s'     D"BJJ!ORXXa[ADr  zCbcoo_concatenate requires inputs to have matching batch dimensions.Nz6concatenation of arrays with broadcasted batch indicesrj   rk   .z%Concatenation along dense dimensions.rD   )rn   ro   rN   r   r  r  r	   r  r  r   r   rE   rl   r   r   r7   rP   r  rv   rW   rV   rt   rO   rp   ry   r   cumsumr   rs   r   r_   )operandsr  r  r  	n_batchesr7   rv   index_batchesdata_batchesr~   nsesrp   r  r  offsetsoffsets                   r8   bcoo_concatenater    s;    nnY')	5H5	5
 $:' ( ( WWS__nEPP:B
CB4BHHbhh/
C Q ( 	x	("**	()A-
X
YY$,-brzz-)- 	^qS^q0

Bh
BBYabSUrzzQ$R3BNbhb	DD	DYabSUrzzQ$R3BNbhb&.//I/^q
Z
[[qk))8A;+?+?8'8@A"2::##HW-A-A4<=b"''--)=,=@MN1Qz	]Qy1}~%66NMN?KL!AjyMAi!mn$55LLL
c- 
!S\):%;
@q
@
V
WW A
V
WW%&rBFF&D&
4yA~Ic3;<R-C(<h<//"A"2::"AYWK(;B;yQH7X%%iiXcr]Krrxx	2KK&qk11779G(;B;wOH//5=h5P#R'1r6 $&::==i'6I1I#J#N#Nv#V #R,35K E
FF	x%X^^	<<a D	( .
 cb/ B=NL ' ="A;K;#Rsf   (1Q+QQ$%Q)%Q.Q3 Q8= Q=(RR)RRR2R/R 
:R%47R*
)
dimensionsr  c          	     x     j                   j                  d j                    j                  j                  d j                   cxk7  r& j                  d j                   k7  rt	        d       t         j                   j                   j                  g      \  }}}t         j                   j                   |xs t        t         j                               j                        \  }}}	t        j                  |      }
t        j                  |      }t        j                  |      }|
dk7  r,|
|vr(t        d j                   d| d j                         |dk7  r/|
|z  |vr(t        d j                   d| d j                          |j#                  |
d	
      }|j#                  |
|z  d	
      }t        |t%        |      t%        |      g      \  }}}t'        j(                   j                  g | j*                  |g | j                   fd|	D              }t'        j(                   j                   g | j*                   j                  g | j                   j                  dz         |s*t-        j.                  g | j*                  d      n_|r\t        fd|D              }|D cg c]'  }t%         j                   j                  |z            ) }}t-        j0                  |t        |      d      }t3        j4                  d      5  t-        j6                   j                   j                   j                   j                  z    j8                        k\  j;                  dd      }ddd       t-        j<                  ||      }t-        j>                  |D cg c]  }|d   	 c}d      t-        j@                  t-        j6                  |j8                              tC        |f|      S c c}w # 1 sw Y   xY wc c}w )a  Sparse implementation of {func}`jax.lax.reshape`.

  Args:
    operand: BCOO array to be reshaped.
    new_sizes: sequence of integers specifying the resulting shape. The size
      of the final array must match the size of the input. This must be specified
      such that batch, sparse, and dense dimensions do not mix.
    dimensions: optional sequence of integers specifying the permutation order of
      the input shape. If specified, the length must match ``operand.shape``.
      Additionally, dimensions must only permute among like dimensions of mat:
      batch, sparse, and dense dimensions cannot be permuted.

  Returns:
    out: reshaped array.
  Nz4reshape of arrays with broadcasted batch dimensions.rg   zJbcoo_reshape: new shape cannot mix batch and sparse dimensions; got shape=z new_shape=z with n_batch=zJbcoo_reshape: new shape cannot mix sparse and dense dimensions; got shape=z with n_dense=right)sidec              3  B   K   | ]  }|j                   z   d z     ywr|   r  rK  s     r8   r9   zbcoo_reshape.<locals>.<genexpr>V  s     +TAAOa,?+T   	new_sizesr  r   rD   c              3  ,   K   | ]  }d |f     ywr   r4   r  s     r8   r9   zbcoo_reshape.<locals>.<genexpr>_  s     <1wsAv<r  clip)r  r+  allowrk   rj   Tr   r   )r  )"rW   rE   r7   rV   rO   r   rv   rY  r   rq   rL   rq  r  r   cumprodr   r   searchsortedr   r	   rJ  rp   r>   
empty_likeravel_multi_indexr  numpy_rank_promotionru   rl   r   unravel_indexr  r   r_   )rx   r  r  r  batch_shapesparse_shapedense_shaperV  rW  rX  rU   sparse_size
cuml_shapei1i2new_batch_shapenew_sparse_shapenew_dense_shaperV   
index_colsre   flat_indicesoob_indicesnew_index_colsr  rW   s   `                        @r8   bcoo_reshaper0  +  s   $ 
kk%)E`S_TWT_T_I``
T
UU a ,6cii#++s||A\+](+|[(=HHckk:?uSXX)?)L%*k:yy%*		,'+zz)$*1_:5
 ""%))K	{.QTQ\Q\P]_ ` `A*{2*D
 ""%))K	{.QTQ\Q\P]_ ` ` z8"zK7gF"7A)cRTgWZ[]W^M_7`4/#_ 
HHE/E377E_EUUS[[U+T+TU
W$ KKKKD_DcggDs||D::S[[:#++/:<'
 
nnW,Jo,Jsww,J,JKG<<<J=HIC		#++/23ILI((%:MTZ[L		!	!'	* W		#))CKKs||A[*\07!? ?@CBQU@V W &&|5EFNooH#s9~HrRGiiSYY/?w}}%UW^_G	tWoY	// JW W Is   ,P&7A$P+P7+P4c           	        t        j                  t        j                  d      j	                  t        j
                  | j                  | j                        |      }|D cg c]  }|| j                  k  s| }}|D cg c]4  }| j                  |cxk  r| j                  | j                  z   k  s0n n|6 }}|D cg c]!  }|| j                  | j                  z   k\  s |# }}| j                  | j                  }}|rt        j                  ||      }|s|r8t        j                  |||D cg c]  }|dz   | j                  z
   c}z         }|rt        j                  | j                  | j                  | j                  | j                  z    |j                        }	t        j                  |D cg c]  }|| j                  z
   c}      }
|j                  d|
f   j                  d      }|j                  d|
f   j!                  |	|
   dz
        }t        j"                  |dk  |	|      }t%        ||f| j                        S c c}w c c}w c c}w c c}w c c}w )	z,Sparse implementation of {func}`jax.lax.rev`r  r  rg   rk   .rj   r   rD   )r  r  r	   revr  r  rE   rl   r7   rv   rV   rW   r>   ru   rs   mulr   r   r_   )r  r  r   r]  rS   r  r  rV   rW   r#  spdimss              r8   bcoo_revr6  l  s    
ggcgg7BB


w}}gmm
< C !! &=aW__)<=*=&dq'//Q*cSZScScAc*cd+d%Qagoo@P@P.P)PQ*Q,,$ggg*5G:774JT^1_q!a%':J:J2J1_$_`D99W]]7??GOOgN^N^<^_#*==2LYY[AGOO+ABFjjf%))"-Gjjf%)),v*>*BCGii!\7;G	tWoW]]	33' >dQ 2`
 Bs0   #I8I1I4I!I#I3I:I$c                   t         fdD              t         fdD              rt        d j                   d      t         fdD              }t	        j
                  t         j                        D cg c]  }| j                  z   vr| c}t              }t         fdD              }t        j                   j                  ||z         }t        j                   j                  d|f   |      }t        fd	t         j                        D              }t        ||f| j                    j"                  
      S c c}w )a  Sparse implementation of {func}`jax.lax.squeeze`.

  Squeeze any number of size 1 dimensions from an array.

  Args:
    arr: BCOO array to be reshaped.
    dimensions: sequence of integers specifying dimensions to squeeze.

  Returns:
    out: reshaped array.
  c              3  J   K   | ]  }t        |j                          y wrC   )r/   rL   r5   rG   r!  s     r8   r9   zbcoo_squeeze.<locals>.<genexpr>  s     L#&sCHH5Lr:   c              3  B   K   | ]  }j                   |   d k7    ywr|   rD   r9  s     r8   r9   zbcoo_squeeze.<locals>.<genexpr>  s     331	3r  zPcannot select an axis to squeeze out which has size not equal to one, got shape=z and dimensions=c              3  B   K   | ]  }|j                   k  s|  y wrC   r  r5   r]  r!  s     r8   r9   zbcoo_squeeze.<locals>.<genexpr>  s     >1a#++oQ>s   rk   c              3  z   K   | ]2  }|j                   j                  z   k\  r|j                  z
  d z    4 ywr|   )r7   rv   r<  s     r8   r9   zbcoo_squeeze.<locals>.<genexpr>  s;      9aS[[3<<77 %) 9s   8;.c              3  2   K   | ]  \  }}|vs|  y wrC   r4   )r5   re   r~   r  s      r8   r9   zbcoo_squeeze.<locals>.<genexpr>  s     N$!Q!::MANrF  rm   )r   r   r   rE   r   ru   rq   rv   r7   r   r	   rv  rV   rW   r   r_   rJ   rK   )	r!  r  rS   re   r  r  r  r  r7  s	   ``       r8   bcoo_squeezer?    s3    LLL*3
33
 ""%)),=*@ A A>
>>*U3<<%8 @_J>  @GJL+ 9: 9 9*[[:
#:;(CKK[(89:F+N)CII"6NN)	x%Y //@R@R
T T@s    E)stridesc                  t        | t              st        dt        |              |D cg c]  }t	        j
                  |       }}|D cg c]  }t	        j
                  |       }}|#|D cg c]  }t	        j
                  |       }}ndg| j                  z  }t        |      t        |      cxk7  rt        |      cxk7  r| j                  k7  rn nt        d| j                         t        |      | j                  k7  r$t        dt        |       d| j                         t        d |D              rt        d|       t        d	 t        ||| j                        D              st        d
|d|d| j                         t        || j                  | j                  g      \  }}}t        || j                  | j                  g      \  }}	}
t        || j                  | j                  g      \  }}}g }g }t!        t#        |||            D ]  \  }\  }}}|j%                  | j&                  j                  |   | j                  |   k7  rt)        d      nt)        |||             |j%                  | j*                  j                  |   | j                  |   k7  rt)        d      nt)        |||              |j%                  t)        d             |j-                  t)        d      t)        d      g       t!        t#        ||
|            D ]%  \  }\  }}}|j%                  t)        |||             ' | j&                  t/        |         }| j*                  t/        |         }t/        d t        |||      D              }t        || j                  | j                  g      \  }}}| j                  rt1        j2                  t1        j4                  ||j6                        t9        | j                  dz               }t1        j2                  t1        j4                  |	|j6                        t9        | j                  dz               }t1        j2                  t1        j4                  ||j6                        t9        | j                  dz               }t1        j                  ||k\  ||k  z  ||z
  |z  dk(  z  dd      }t1        j:                  |||z
  |z   dz
  |z  ||z
  |z   dz
  |z        }t=        j2                  |d   t9        | j                  dz   | j                  dz   | j>                  z               }t1        j:                  ||d      }tA        jB                  |      }| jD                  |kD  rtG        ||tI        |      |      \  }}t        ||f|      S c c}w c c}w c c}w )a  Sparse implementation of {func}`jax.lax.slice`.

  Args:
    mat: BCOO array to be reshaped.
    start_indices: sequence of integers of length `mat.ndim` specifying the starting
      indices of each slice.
    limit_indices: sequence of integers of length `mat.ndim` specifying the ending
      indices of each slice
    strides: (not implemented) sequence of integers of length `mat.ndim` specifying
      the stride for each slice

  Returns:
    out: BCOO array containing the slice.
  6bcoo_slice: input should be BCOO array, got type(mat)=Nrg   z,bcoo_slice: indices must have size mat.ndim=zlen(strides) = z; expected c              3  &   K   | ]	  }|d k    ywr  r4   r   s     r8   r9   zbcoo_slice.<locals>.<genexpr>  s     !Aa!ru  z5strides must be a sequence of positive integers; got c              3  T   K   | ]   \  }}}d |cxk  xr |cxk  xr |k  nc  " ywr  r4   )r5   startendr   s       r8   r9   zbcoo_slice.<locals>.<genexpr>  s2      X!T %&3&$&& Xs   &(z/bcoo_slice: invalid indices. Got start_indices=z, limit_indices=z and shape=c              3  @   K   | ]  \  }}}||z
  |z   d z
  |z    ywr|   r4   )r5   rE  rF  strides       r8   r9   zbcoo_slice.<locals>.<genexpr>  s3      OsF 	5[6A&(Os   rk   r   rj   T)r  r   r  rD   rH  )%r  r_   r  r  rn   ro   rL   r   r   r   rN   r   rE   r   r7   rv   r   rQ   appendrV   rc   rW   extendr   r>   r=   ru   rl   rq   r   r	   r   rq  r  rp   rK  r   )rx   start_indiceslimit_indicesr@  re   start_batchstart_sparsestart_dense	end_batch
end_sparse	end_densestride_batchstride_sparsestride_densedata_slicesindex_slicesrE  rF  rH  r  r  	new_shaper   new_shape_sparsestartsendsstrides_keep	keep_datanew_nses                                 r8   
bcoo_slicer`    s%     
C	
LTRUYKX
YY.;<8>>!$<-<.;<8>>!$<-<*12Qx~~a 2G2cCHHnG3}-IWII
CCHH:N
OO\SXX
s7|nKzJ
KK!!!
LWIV
WW	 X%-m]CII%VX 
X
G8H I'%'{399+? @ @ ,6mckkSVS_S_E`+a(+|[%/S\\?Z%[")Z.83;;PSP\P\B].^+,|+,!*3{I|+T!U la	%fchhnnQ&7399Q<&GuT{USXZ]_eMfgs{{'8'8';syy|'KdQVW\^aciQjkl U4[!uT{E$K01!*3{I|+T!U 2a	%fuUC012XXeK()(E,/0+ O&}mWMO O) &i#++s||1LM!q\\__SYY|;;L;LMuUXU`U`cdUdOefF??399Z{7H7HI5QTQ\Q\_`Q`KabDsyyk>O>OPRWX[XcXcfgXgRhiH77K6)kD.@A 6)X5:<T+D ))D;#7(#BQ#F8"S!F]X59hFHK VeCKK!OS[[ST_WZWbWbEb.cdIyyHa0Hii()G
ww2+jy&AwPh 
x%Y	77s =<2s   V5V:2V?c                n   t        d |D              }t        j                  t        j                  d      j                  t        j                  | j                  | j                        ||       t        | t              st        dt        |              t        d |D              }t        d |D              sJ t        d |D              sJ t        |      t        |      cxk7  r| j                  k7  rn nt!        d| j                         t        d	 t#        || j                        D              st        d
| d| j                         t%        || j&                  | j(                  g      \  }}}t%        || j&                  | j(                  g      \  }}}g }	g }
g }g }t+        |r|d   nt,        j.                  d      }t1        t#        ||            D ]  \  }\  }}| j2                  j                  |   | j                  |   k7  }| j4                  j                  |   | j                  |   k7  }|	j7                  |r|n|       |
j7                  |rdn|       |j7                  |r|n|       |j7                  |rdn|        |	j7                  |       |
j7                  | j8                         |j;                  ||g       |j;                  | j8                  | j(                  g       |	j;                  |       |
j;                  |       t        j                  | j2                  |	|
      }t        j                  | j4                  ||      }|}| j(                  r>t=        j>                  ||j                        }t=        j>                  ||j                        }t=        j>                  | j                  | j&                  | j&                  | j(                  z    |j                        }t=        j@                  |dk  ||z   |      }t=        jB                  |d||z
        }t=        jD                  |tG        | j&                  dz               }t=        jD                  |tG        | j&                  dz               }t=        jD                  |tG        | j&                  dz               }t=        j                  ||k\  |||z   k  z  dd      }t=        j@                  |||z
  |      }t        jD                  |d   tG        | j&                  dz   | j&                  dz   | jH                  z               }t=        j@                  ||d      }| j8                  tK        jL                  |      kD  r1tK        jL                  |      }tO        ||tQ        |      |      \  }}t        ||f|      S )a  Sparse implementation of {func}`jax.lax.dynamic_slice`.

  Args:
    mat: BCOO array to slice.
    start_indices: a list of scalar indices, one per dimension. These values
      may be dynamic.
    slice_sizes: the size of the slice. Must be a sequence of non-negative
      integers with length equal to `ndim(operand)`. Inside a JIT compiled
      function, only static values are supported (all JAX arrays inside JIT
      must have statically known size).

  Returns:
    out: BCOO array containing the slice.
  c              3  F   K   | ]  }t        j                  |        y wrC   )rn   ro   rd   s     r8   r9   z%bcoo_dynamic_slice.<locals>.<genexpr>   s     =AhnnQ'=r   slice_sizesr  rB  c              3  F   K   | ]  }t        j                  |        y wrC   )r>   r   rd   s     r8   r9   z%bcoo_dynamic_slice.<locals>.<genexpr>  s     >1A>r   c              3  x   K   | ]2  }t        j                  |j                  t        j                         4 y wrC   )r>   r   rl   r   r   rd   s     r8   r9   z%bcoo_dynamic_slice.<locals>.<genexpr>  s#     HQS^^AGGRZZ0Hs   8:c              3  :   K   | ]  }|j                   d k(    yw)r4   NrD   rd   s     r8   r9   z%bcoo_dynamic_slice.<locals>.<genexpr>	  s     2qQWW]2s   z4bcoo_dynamic_slice: indices must have size mat.ndim=c              3  B   K   | ]  \  }}d |cxk  xr |k  nc   ywr  r4   )r5   
slice_size	axis_sizes      r8   r9   z%bcoo_dynamic_slice.<locals>.<genexpr>  s"     c.Cj)Q*)	))cs   zIslice_sizes must be less than or equal to operand shape, got slice_sizes z for operand shape r   rg   rk   rj   Tr   r  rD   rH  ))r   r  r  r	   dynamic_slicer  r  rE   rl   r  r_   r  r  rN   r   rL   r   rQ   r   r7   rv   r#   r   r  r   rV   rW   rI  rp   rJ  r>   ru   r   r  r=   rq   r   rq  r  rK  r   )rx   rK  rd  rM  rN  rO  
size_batchsize_sparse
size_dense
data_start
data_sizesindices_startindices_sizeszerore   rE  r   data_is_broadcastindices_is_broadcastr  r  rX  rZ  sizesr#  r]  r^  r_  s                               r8   bcoo_dynamic_slicerw    s    ===+''#

-=>II


syy#))
4m! J # 
C	
LTRUYKX
YY>>>-	H-H	HH	H	2M2	22	23{+7sxx7
KCHH:V
WW	cs;X[XaXaGbc	c
 ''2m3FsyykS T T ,6mckkSVS_S_E`+a(+|[(2;cll@[(\%*k:**--	Ma rxx	C$#CZ$@A >a%q)SYYq\9;;,,Q/399Q<?/dU;,a$7!55A2=> DCGGd|$./K JsxxZ@(!!#++}mL+)\\YY|;+<+<=FIIk):):;E99SYYs{{CKK#,,4NOWbWhWhiLYYvz6L#8&AFXXfa!56F__VU3;;?%;<FOOE5q#9:E??<s{{Q1GHL77K6)kFUN.JKRZ^_D))D+"6>KVeCKK!OS[[ST_WZWbWbEb.cdIyyHa0H
ww;''		+&g2+jy&AwPh 
x%Y	77r^   c                B    t        fdt        |       D              S )Nc              3  6   K   | ]  \  }}|k(  rn|  y wrC   r4   )r5   re   tr8  vals      r8   r9   z!_tuple_replace.<locals>.<genexpr>D  s      A$!Qa3hsA%As   )r   r   )tupr8  r{  s    ``r8   r3  r3  C  s    	A)C.A	AAr^   c                   t        | j                  | j                  | j                  |      \  }}}t	        ||f|      S )a  Sum array element over given axes.

  Args:
    mat: A BCOO-format array.
    shape: The shape of the target array.
    axes:  A tuple or list or ndarray which contains axes of ``mat`` over which
      sum is performed.

  Returns:
    A BCOO-format array containing the result.
  )rT   axesrD   )_bcoo_reduce_sumrV   rW   r   r_   )rx   r~  rX  rY  r7  s        r8   bcoo_reduce_sumr  F  s<     &6	hhCIID&:"(K	x%Y	77r^   c          
         |j                   t        fdD              sJ t         |      \  }}t        t	                    t        fdD              } j                  |       rt        j                  t        j                  z    |j                        t        |j                  dz
              }t        j                  ||k  d      } j                  |j                  kD  r=t        j                  |t        t        |j                   j                                    }t        j                  | d       t              D 	cg c]  }	|	z   vs|	 }
}	|
s9t        j                  t!        |j                   dz   d      |j                        }n|dt#        j                  |
      f   }D ch c]
  }|k  s	| }}|D ]  } j                   |   dk(  rg|j                   |   dk(  r	 |   z   not        j$                   t!         j                   ||         t        t         j                                     n#|j                   |   dk(  r j                  |        j                   |   |j                   |   k(  rJ  t        t        t	        t                    |z
              }t         fd|D              }|t'        j(                  |D 	cg c]  }	 j                   |	    c}	      z  }t        j*                   g || j                   dz   d  g ||t         j                               t        j*                  |g |||j                   dz   d  g ||t        |j                              }t        fd	t        t-                    D              } ||fS c c}	w c c}w c c}	w )
Nc              3  P   K   | ]  }d |cxk  xr t              k  nc   ywr  r  )r5   r   rE   s     r8   r9   z#_bcoo_reduce_sum.<locals>.<genexpr>X  s"     /QQ! c%j  /s   #&c              3  @   K   | ]  }|z   k\  s|z
  d z     ywr|   r4   )r5   axr7   rv   s     r8   r9   z#_bcoo_reduce_sum.<locals>.<genexpr>]  s&     R2w?Q9QR(]Q&Rs   rk   rg   rj   r   .c              3  <   K   | ]  }j                   |     y wrC   rD   )r5   re   rV   s     r8   r9   z#_bcoo_reduce_sum.<locals>.<genexpr>  s     @A$**Q-@rM  c              3  2   K   | ]  }|vs|     y wrC   r4   )r5   re   r~  rE   s     r8   r9   z#_bcoo_reduce_sum.<locals>.<genexpr>  s     I1D=E!HIs   	
)rE   rN   r   rU  rt   r   r   r>   r=   ru   rl   rq   rL   r	   r   r   r3  r   r2  rq  r  rJ  r   )rV   rW   rT   r~  r   rp   
dense_axesr   r   re   
sparse_idxr  
batch_axesnew_batch_dimsr)  r_  r7  r7   rv   rE   s   `  `             @@@r8   r  r  V  se   
,,%	/$/	//	/,T7EB'8Q	D		$ RRR*	*	$	iigw12'--HGLL1J 777Z',Dyy499__T5tyy$)))D#EFd99T4#D !?Faa'k.EF*F	iiw}}gk1Ew}}UGc288J//0G "2rR'\2*2  	/bzz"~	r	a	eBi##D.Rr*SUZ[`aeajaj[kUlm	r	a	xx|::b>W]]2....	/ E'N 3j @AB.@@@/$))J?qTZZ]?@@'	TKKK$**Wq[\2JKO~O
OU7DII5NO
Q$ KKQ/Q7QW]]7Q;<5PQU.U:Ugw||8TUW' IeCJ&7II)	w		!!C G 3  @s   O	'O	
OO,O
c                    t        | j                  | j                  |j                  |j                  | j                  |j                        \  }}}t	        ||f|      S )zAn element-wise multiplication of two sparse arrays.

  Args:
    lhs: A BCOO-format array.
    rhs: A BCOO-format array.

  Returns:
    An BCOO-format array containing the result.
  )r  r  rD   )_bcoo_multiply_sparserV   rW   r   r_   )r  r  rX  rY  r7  s        r8   bcoo_multiply_sparser    sO     &;	hhSXXs{{syy&"(K 
x%Y	77r^   c                  |j                   }|j                   }t        | ||      }t        |||      }	t        |      t        |      k7  rt        d| d|       |j                  |	j                  k7  rt        d| d|	       t        |j                  |	j                        }
t        j                  t        ||
d  ||
d        }t        ||
      } || |||      \  }}||t        j                  ||      fS )NzFbcoo_multiply_sparse: arrays must have same number of dimensions, got z, zIbcoo_multiply_sparse: arrays with differing numbers of dense dimensions: )rM  rN  )rE   r   r   r  r   rO   rp  r7   r^  r   _bcoo_multiply_sparse_unbatchedr   r>   rr  )r  r  r  r  r  r  rM  rN  r  r  r7   _mulrV   rW   s                 r8   r  r    s   ))xi8#xi8#^s9~%
 $+R	{4 5 5[[CKK
 336%r#@ A AS[[)'			:%.wx%8%.wx%8
:$ 
D'	"$xhD-$	w,,Y	B	BBr^   c          
        t        | ||      }t        |||      }|j                  dk(  s|j                  dk(  sJ |j                  r5t        t        | |f|      d      j                  \  } }t        | ||      }n@|j                  r4t        t        ||f|      d      j                  \  }}t        |||      }t        j                  t        t        |d |j                   |d |j                               D 	
cg c]  \  }\  }	}
|	dk7  r|
dk7  r| c}
}	}t              }|j                  |j                  z  }t        j                  |d d d |f   |d d d |f   k(  d      }t        j                  |||j                  |j                  f      \  }}| j                  |   j                  dd	      |j                  |   j                  dd	      z  }t        j                   |j                  |   j                  dt#        |d
      	      |j                  |   j                  dt#        |d
      	            }||fS c c}
}	}w )Nr   rD   r  rg   rk   rj   r   r)  r*  rB  )r   r7   r  r_   r  r>   ru   r   r   rv   r   rp   rN   r   rs   r1  maximumrP   )r  r  r  r  rM  rN  r  r  re   r   r   r  rp   r   i_lhsi_rhsrV   rW   s                     r8   r  r    s    xi8#xi8#
++
q 01	1 	[[.tX{4KS\/]ghiooHk
;	
:C
{{.tX{4KS\/]ghiooHk
;	
:C	8Ims||<TV_`madamamVn3o)p , ,+!Xb"Qw27  ,36
8$
 	#''# 
Qd]+{4D=/II2	N$T#''8JK,%
++e

 
 f
 
;
++e

 
 f
 
;<$KKnnUVIq8QRnnUVIq8QRT' 
w!,s   >Ic                \    t        | j                  | j                  || j                        S )zAn element-wise multiplication between a sparse and a dense array.

  Args:
    lhs: A BCOO-format array.
    rhs: An ndarray.

  Returns:
    An ndarray containing the result.
  r   )_bcoo_multiply_denserV   rW   r   )sp_matvs     r8   bcoo_multiply_denser    s!     
fkk6>>1V\\	RRr^   c                  |j                   }|j                  dk(  rt        j                  | |      S ||j                   k(  r t        j                  | t	        ||            S t        j
                  |j                   |      |k7  rt        d|d|j                         t        j                  |t        t        |      |j                  z
              }t        | ||      t        t        j                        fd       } || ||      S )zNBroadcasted elementwise multiplication between a BCOO array and a dense array.r   zmultiplication between sparse and dense is only implemented for cases where the output shape matches the sparse matrix shape. Got shape=z
, v.shape=r  c                   j                   d   |j                  j                  z
  k(  sJ t        fdt	        j                   d         D              }t        d t        ||j                         D              }| ||   z  S )Nrg   c              3  0   K   | ]  }d d |f     y wrC   r4   r  s     r8   r9   z5_bcoo_multiply_dense.<locals>._mul.<locals>.<genexpr>  s     ?!1?r  c              3  4   K   | ]  \  }}|d k7  r|nd  ywr  r4   r}   s      r8   r9   z5_bcoo_multiply_dense.<locals>._mul.<locals>.<genexpr>  s     Atq!Q!V"As   )rE   rL   r   r   rq   rQ   )rV   rW   r  r8  r   s    `  r8   r  z"_bcoo_multiply_dense.<locals>._mul  sn    ==qvv5555
?uW]]1-='>?
?C
As3/@A
AC!C&=r^   )rE   rL   r	   r4  r   rr  rO   r=   rq   r   r   r   r   r7   )rV   rW   r  rT   rE   r  r   s         @r8   r  r    s     ,,%VVq[774
agg774w233!''5)U2
	177*  
ooas5zAFF234!
w
.%
:' (
 
dGQ	r^   )rK   indices_are_sortedr+  r   c          	         t         j                   j                   j                         |t        j
                  }t	        j                  |      }|t        j
                  k7  rt        d|d      t        ||||||      }	d}
 t        j                  t        j                  |
      j                  t        j                   j                   j                        t        j                  |j                  |j                        fi |	}|j                   |j"                  |j$                  }t'        |d      g j(                  z  }t+         j(                        D cg c]  }d }}t-         j                        t/        |      D ]*  \  }}|d|f   j1                         ||<   ||   |<   d||<   ,  fd} t3        ||f	      |      }t5        |g |j                  dd
 |j                  dd t7        t+        |j(                                    }|j                  rt7        fdt+        t9        |j                              D              }t;        j<                  |j(                  t>              }t;        j@                  |j(                        |t;        jB                  |z         <   tE        |dt9        |             tE        t+        t9        |                  k7  rtG        |d      }tI        |t7        |            }|jK                  |j                        jM                  |j                        S c c}w )zBCOO version of lax.gather.Nzbcoo_gather: mode=z not yet supported.)r  rd  rK   r  r+  r   r  r   .c                :    t        |       }t        |      S )Nrc  r2  )rw  r?  )rW   slccollapsed_slice_dimsfull_slice_sizesr  s     r8   
slice_funczbcoo_gather.<locals>.slice_func 	  s     
Wg;K
LC(<==r^   r\  rj   rg   r  c              3  *   K   | ]
  }|vr|  y wrC   r4   )r5   rG   offset_dimss     r8   r9   zbcoo_gather.<locals>.<genexpr>*	  s       1sK/  1s   rk   r  rI  )'r   rV   rW   rE   r)   PROMISE_IN_BOUNDSfrom_anyrO   r   r  r  r	   gatherr  r  rl   r  r  start_index_mapr#   rL   rq   r  r   r  r   r0  r   r   r   r   r   r   ru   rt   r  rF  rJ  r  )r  rK  r  rd  rK   r  r+  r   parsed_moder/  r  r  r  full_start_indicesre   r<  r  r  r6  rS   rI  r  r  r  s   `                    @@@r8   bcoo_gatherr    s    w> 
\..D!**40+%777
 3dW4GH
II	 1{+@RJ
0$
7/LSWWSZZALL


w}}gmm
<


}22M4G4G
H (
 "--+*??%55/ *0q)A(BW\\(Q-27<<-@AtA'A'--(( da)#q&1779q%a.QGAJ> 04
WJ/0BC&<##CR(<6<<+;<U6;;'(*&
 \\ 1eC,?&@ 1 1J((6;;c2K68ii6LKk123
;'J()Ss:1G-HH "&!4fFk0BCF		'	.	.x~~	>>5 Bs   (	M	rg   )lhs_dilationrhs_dilationr  feature_group_countbatch_group_countr|  r}  c       	           t        j                  t        j                  ||||||||	|

      } t	        j
                  |      t	        j                  | j                  | j                        t	        j                  |j                  |j                              }t        |t        j                        rt        |j                        dk(  sJ |j                  d   j                  }|d   d| j                  dz
  z  k7  rt!        d      |d   d|j                  dz
  z  k7  rt!        d      |d	   d| j                  dz
  z  k7  rt!        d
      |d   |d   cxk7  rdk7  rt!        d       | j                  d d |j                  d d cxk7  rdk7  rt!        d       t#        | d      r| j$                  j                  n#t#        |d      r|j$                  j                  nd}|d   \  }t'        t)        | |      t)        ||      |      S )N)	window_stridespaddingr  r  r  r  r  r|  r}  rg   r   r  r   r1   z#bcoo convolution with lhs_dilation.r  r  z.bcoo convolution with non-unit window_strides.r  r  z,bcoo convolution with non-unit group counts.)rg   rg   z:bcoo convolution with leading dimensions other than (1, 1)rW   r  r  )r  )r^  r   r	   conv_general_dilatedr  
make_jaxprr  rE   rl   r  r   ClosedJaxprr   eqnsparamsrL   rO   r(  rW   _bcoo_conv_1d_convert_to_1d_for_conv)r  r  r  r  r  r  r  r  r  r|  r}  rb  jaxprr  r   s                  r8   bcoo_conv_general_dilatedr  6	  s   
 
			#WlN_-AR2H
J$ #..
s33CIIsyyI"33CIIsyyIK%	E4++	,UZZA1EE	E::a=&N1 55
C
DDNtsxx!|44
C
DDA!66
N
OO F+@$AFQF
L
MM G 	YYr]ciim-v-
Z
[[ . '.c9&=""+23	+Bckk''  I('	.sK@.sK@&
( (r^   c                n   t        | t        j                  t        j                  f      r9t        j                  | d      }t        j                  |t        |      dfd      }nt        | t              r| j                  dd      } t        j                  | j                  d      }t        j                  | j                  d      }t        j                  t        j                  |d      | j                  d   k  |d      }nt!        dt#        |        d	      t        ||f| j                  dd  
      S )Nr   rg   rg   r   r1   )r7   r   r   rj   z)bcoo_conv_general_dilated: input of type z not recognized.rD   )r  r  r,   r   rT  r	   rv  r  r   r_   update_layoutrV   rW   r>   r   rE   r  r  )rx   r   rV   rW   s       r8   r  r  [	  s    cii,-;;sF#D"";TABG#t


Aq

1C;;sxx(Dkk#++v.G99S[[$/#))B-?qID
?S	{JZ[
\\	tWoSYYqr]	33r^   c                H   | j                   | j                  cxk(  r&|j                   cxk(  r|j                  cxk(  rdk(  sJ  J | j                  |j                  k(  sJ t        t	        t
        |            }t        |      dk(  sJ | j                  d d d f   |j                  d d d f   z  j                         }|d   |j                  z
  }| j                  d d d f   |d d d f   z   j                         }|dk  }t        j                  |d|      }t        j                  |d|      }t        d| j                  d   |d   z   |d   z   |j                  d   z
  dz         }t        j                  |d      }t        j                  |d      }t!        ||fdd|f      S )Nrg   r1   r   r  )r   rg      rD   )rL   rv   rl   r   rj  r   r   rV   r  rW   r>   r   rP   rE   r	   r=   r_   )r  r  r  r  r  r  r   dimsizes           r8   r  r  i	  sr   	S\\	BSXX	B	B	BB	BB	B	cii		#c7#$'	W		hhq$w#((47"33::<(1:#&QW%tQw7>>@+
/$		$;/+YYtQ)(399Q<'!*,wqz9CIIaLH1LM'__Xv.(Y7+	x%aG_	==r^   c                      e Zd ZU dZded<   ded<   ded<    ed       Z ed       Z ed	       Z ed
       Z	 ed       Z
ded<   ded<    ed       Z ed       Zddd	 	 	 d, fdZd Zd-dZd-dZd-dZedej(                  ddd	 	 	 	 	 d.d       Zedddd	 	 	 	 	 d/d       Zedddddd	 	 	 	 	 	 	 d0d        Zeddddd!	 	 	 	 	 	 	 d1d"       Zddd#d$	 	 	 d2d%Zd3d4d&Zd-d'Zd5d(Zd6d7d)Zd* Zed+        Z xZ S )8r_   ar  Experimental batched COO matrix implemented in JAX

  Args:
    (data, indices) : data and indices in batched COO format.
    shape : shape of sparse array.

  Attributes:
    data : ndarray of shape ``[*batch_dims, nse, *dense_dims]`` containing the
      explicitly stored data within the sparse matrix.
    indices : ndarray of shape ``[*batch_dims, nse, n_sparse]`` containing the
      indices of the explicitly stored data. Duplicate entries will be summed.

  Examples:
    Create a sparse array from a dense array:

    >>> M = jnp.array([[0., 2., 0.], [1., 0., 4.]])
    >>> M_sp = BCOO.fromdense(M)
    >>> M_sp
    BCOO(float32[2, 3], nse=3)

    Examine the internal representation:

    >>> M_sp.data
    Array([2., 1., 4.], dtype=float32)
    >>> M_sp.indices
    Array([[0, 1],
           [1, 0],
           [1, 2]], dtype=int32)

    Create a dense array from a sparse array:

    >>> M_sp.todense()
    Array([[0., 2., 0.],
           [1., 0., 4.]], dtype=float32)

    Create a sparse array from COO data & indices:

    >>> data = jnp.array([1., 3., 5.])
    >>> indices = jnp.array([[0, 0],
    ...                      [1, 1],
    ...                      [2, 2]])
    >>> mat = BCOO((data, indices), shape=(3, 3))
    >>> mat
    BCOO(float32[3, 3], nse=3)
    >>> mat.todense()
    Array([[1., 0., 0.],
           [0., 3., 0.],
           [0., 0., 5.]], dtype=float32)
  r,   rV   rW   r   rE   c                4    | j                   j                  d   S )Nri   rW   rE   r   s    r8   r   zBCOO.<lambda>	  s    dll004 r^   c                .    | j                   j                  S rC   )rV   rl   r   s    r8   r   zBCOO.<lambda>	  s    		 r^   c                4    | j                   j                  dz
  S )Nr1   )rW   rL   r   s    r8   r   zBCOO.<lambda>	  s    $,,"3"3a"7 r^   c                4    | j                   j                  d   S r  r  r   s    r8   r   zBCOO.<lambda>	  s    4<<#5#5b#9 r^   c                N    | j                   j                  dz
  | j                  z
  S )Nrg   )rV   rL   r7   r   s    r8   r   zBCOO.<lambda>	  s    $))..1"4t||"C r^   rM   rJ   rK   c                X    t        | j                  | j                  | j                        S rC   )r   rE   rJ   rK   r   s    r8   r   zBCOO.<lambda>	  s#    
4::t7J7J+/+>+>!@ r^   c                2    | j                   | j                  fS rC   )rV   rW   r   s    r8   r   zBCOO.<lambda>	  s    DLL 9 r^   FrI   c                   t        t        j                  |      \  | _        | _        || _        || _        t        | !  |t        |             t        | j                  | j                  | j                         y r-  )rj  r>   r   rV   rW   rJ   rK   super__init__r   r   rE   )r   r  rE   rJ   rK   	__class__s        r8   r  zBCOO.__init__	  sZ    !#++t4DIt|(D(D	GTu.499dllDJJ7r^   c                   | j                   j                  }	 | j                  }| j                  }| j                  }| j
                  }t        | j                        }d|}|r|d|z  }|r|d|z  }| d| | | d}t        | j                  t        j                        r%t        | j                        j                   d| d}|S #  | d}Y TxY w)	N, nse=r   r   ()z(<invalid>)[])r  r   rp   r7   r   rl   r  rE   r  rV   r   Tracerr  )	r   namerp   r7   r   rl   rE   extrarepr_s	            r8   __repr__zBCOO.__repr__	  s    >>""D/HHcggjje4::e 3&ke	%[z?*%	%[z?*%awugeWA.e$))T[[)dii))*!E7!4eL#k"es   AC Cc                    t        d      )NzBCOO.reshaperO   r   r  kwargss      r8   rJ  zBCOO.reshape	  s    
n
--r^   c                    t        d      )NzBCOO.astyper  r  s      r8   r  zBCOO.astype	  s    
m
,,r^   c                    t        d      )NzBCOO.sumr  r   s    r8   r   zBCOO.sum	  s    
j
))r^   Nr   rp   r   r   r7   c               "    t        |||||      S )z7Create a BCOO array from a (dense) :class:`~jax.Array`.r  )r   )clsrx   rp   r   r   r7   s         r8   	fromdensezBCOO.fromdense	  s     	sWgO Or^   )r   r   r7   c               R   |dk7  s|dk7  rt        d      |j                         }t        j                  |j                        }t        j
                  |j                  |j                  f      j                  |xs t        j                        } | ||f|j                  dd      S )z5Create a BCOO array from a :mod:`scipy.sparse` array.r   z+BCOO.fromscipy with nonzero n_dense/n_batchFrm   )rO   tocoor>   r   rV   r  r  r  r  r  rE   )r  rx   r   r   r7   rV   rW   s          r8   from_scipy_sparsezBCOO.from_scipy_sparse	  s     !|w!| MNN
))+C;;sxx D1299 syy"G gcii#% %r^   r  )rl   r   r   r7   rp   c          
     l   t        |      }t        |      |z
  |z
  }|dk  s|dk  s
|dk  s|dk  rt        d|d|d|d|      t        |||g      \  }}	}
t	        j
                  g |||
|      }t	        j                  g |||t	        j                  |	      |      } | ||f|dd      S )z?Create an empty BCOO instance. Public method is sparse.empty().r   Invalid inputs: shape=r   r   r  Trm   )r   r   r   r   r>   r   fullru   )r  rE   rl   r   r   r7   rp   rv   r"  r#  r$  rV   rW   s                r8   _emptyzBCOO._empty	  s     %LE5zG#g-H!|w{gkS1W0%G:[zC6RSS-7?R-S*K{996{6C6+6>Dhh44c484cii6M{[GgeD"$ $r^   rl   r   r7   r   c                  d|z
  |z
  }|dk  s
|dk  s|dk  rt        d||f d|d|      dkD  rt        ||z
        }	nt        |z   |      }	|	dk  r| j                  ||f||||      S |dkD  s|dkD  r,| j                  t	        j
                  |||      |||	      S |dk(  rt	        j                  |	|      }
t	        j                  |	|      }t        |d      t        |      t	        j                  t        j                  |t        j                  dk\  fd
fd            t        j                  |t        j                  dk  fdfd            g      }nt	        j                  ||      }
t	        j                  ||      }|t        |      z   }dk  rO|
j                  d t               j!                  d      }
|j                  d t               j!                  |      }nYdkD  rT|
j                  |t              z
  d  j!                  d      }
|j                  |t              z
     j!                  |      }|
d d d f   }
|d d d d f   } | |
|f||fdd      S )Nr1   r   r  r   r   r  rg   rk   r   c                      S rC   r4   rs  s   r8   r   zBCOO._eye.<locals>.<lambda>'
      d r^   c                      S rC   r4   ks   r8   r   zBCOO._eye.<locals>.<lambda>'
      A r^   c                      S rC   r4   r  s   r8   r   zBCOO._eye.<locals>.<lambda>(
  r  r^   c                      S rC   r4   r  s   r8   r   zBCOO._eye.<locals>.<lambda>(
  r  r^   Trm   )r   rp  r  r  r>   eyeonesr   r#   r  r	   subcondr   rs   absrt   )r  r   r  r  rl   r   r7   r   rv   	diag_sizerV   r  rW   rs  s      `         @r8   _eyez	BCOO._eye

  sj    7{W$H!|w{gk/Ax{';gZPQQ1uaQ-ia!eQ-iA~ZZAe '  : : {gk]]3771a%8#*G'2  4 4 !|XXiu-dJJy4cC^d
a.a  SXXa1flI>?SXXa1flI>?"A Bg XXau%d

1K0g&!,,g	
QwwwA##A&**Wc!f%))!,q5wwq3q6z{#''***QQZ(,,Q/!T']d4&ggq!fT"$ $r^   r  r  c                    t        | |||      S )a  Update the storage layout (i.e. n_batch & n_dense) of a BCOO matrix.

    In many cases this can be done without introducing undue storage overhead. However,
    increasing ``mat.n_batch`` or ``mat.n_dense`` will lead to very inefficient storage,
    with many explicitly-stored zeros, unless the new batch or dense dimensions have size
    0 or 1. In such cases, ``update_layout`` will raise a :class:`SparseEfficiencyError`.
    This can be silenced by specifying the ``on_inefficient`` argument.

    Args:
      n_batch : optional(int) the number of batch dimensions in the output matrix. If None,
        then n_batch = mat.n_batch.
      n_dense : optional(int) the number of dense dimensions in the output matrix. If None,
        then n_dense = mat.n_dense.
      on_inefficient : optional(string), one of ``['error', 'warn', None]``. Specify the
        behavior in case of an inefficient reconfiguration. This is defined as a reconfiguration
        where the size of the resulting representation is much larger than the size of the
        input representation.

    Returns:
      mat_out : BCOO array
        A BCOO array representing the same sparse array as the input, with the specified
        layout. ``mat_out.todense()`` will match ``mat.todense()`` up to appropriate precision.
    r  )r  )r   r7   r   r  s       r8   r  zBCOO.update_layout8
  s    2 dGWUcddr^   c                :    |rt        | |      S t        | |      S )aq  Return a copy of the array with duplicate indices summed.

    Additionally, this operation will result in explicit zero entries removed, and
    indices being sorted in lexicographic order.

    Because the size of the resulting representation depends on the values in the
    arrays, this operation is not compatible with JIT or other transforms. To use
    ``sum_duplicates`` in such cases, you may pass a value to `nse` to specify the
    desired size of the output representation.

    Args:
      nse : integer (optional), if specified, gives the number of specified elements in
        the output sparse representation; if it is larger than the number required, data
        will be padded with zeros and indices will be padded with out-of-bounds values.
        If it is smaller than the number required, data will be silently discarded.
      remove_zeros : bool (default=True). If True, remove explicit zeros from the data
        as part of summing duplicates. If False, then explicit zeros at unique indices
        will remain among the specified elements. Note: remove_zeros=True is incompatible
        with autodiff.
    r   )r   r   )r   rp   remove_zeross      r8   sum_duplicateszBCOO.sum_duplicatesS
  s!    * !$C00 3//r^   c                    t        |       S )z0Return a copy of the matrix with indices sorted.)r  r   s    r8   sort_indiceszBCOO.sort_indicesm
  s    T""r^   c                    t        |       S )z$Create a dense version of the array.)r   r   s    r8   todensezBCOO.todenseq
  s    r^   c                    t        |t         j                        ddd   n|      }t         |      }t	         fd|D              }| j
                   j
                   j                  z    D cg c]  }| j
                  z
   }}t	        |      t	        t         j                              k(  r j                  }nd}t        |j                  |j                  f|| j                        S c c}w )z,Create a new array containing the transpose.Nrj   r  c              3  <   K   | ]  }j                   |     y wrC   rD   )r5   re   r   s     r8   r9   z!BCOO.transpose.<locals>.<genexpr>y
  s     0aDJJqM0rM  Frm   )r  rq   rL   rF  r   r7   rv   rJ   r_   rV   rW   rK   )r   r~  r  mat_Tshape_TrL  rW  	is_sorteds   `       r8   r_  zBCOO.transposeu
  s    T\5+DbD1tLD4T2E0400G t||dmm/KLN t||# NK N[U5#788%%i iU]]+7(9L9LN NNs   3C3c                f    | j                   | j                  f| j                  j                         fS rC   )rV   rW   r   r  r   s    r8   tree_flattenzBCOO.tree_flatten
  s'    IIt||$djj&8&8&:::r^   c                    t         j                  |       }|\  |_        |_        |j	                         h dk7  rt        d|       |j                  j                  di | |S )N>   rE   rJ   rK   z&BCOO.tree_unflatten: invalid aux_data=r4   )object__new__rV   rW   keysr   __dict__update)r  aux_datachildrenobjs       r8   tree_unflattenzBCOO.tree_unflatten
  s\    
..
C$CHck}}GG@xkBCCCLL#(#Jr^   )r  tuple[Array, Array]rE   Sequence[int]rJ   rM   rK   rM   r   r_   )rx   r,   rp   
int | Noner   r.   r   r   r7   r   r   r_   )r   DTypeLike | Noner   r   r7   r   r   r_   )rE   r   rl   r  r   r.   r   r   r7   r   rp   r   r   r_   )r   r   r  r   r  r   rl   r  r   r.   r7   r   r   r   r   r_   )r7   r  r   r  r  strr   r_   )NT)rp   r  r  rM   r   r_   )r   r,   rC   )r~  Sequence[int] | Noner   r_   )!r   r   r   __doc__r   r   rp   rl   r7   rv   r   r   r  r  r  rJ  r  r   classmethodr   r  r  r  r  r  r  r  r  r  r_  r  r  __classcell__)r  s   @r8   r_   r_   ~	  s   0f 	+.	,45#
/
0%78'9:(CD'
 @ A%
9
:% ',E8#8=A8(.-* 6:UWU]U] OO+.O7;O O CG'(%!$%36%?C% % =A\caA$$(+$69$BF$ $ EI$+Aa+$!+$69+$HK+$TX+$ +$Z 48t*1e$'e6:e604#N ;  r^   c                .   ||S ||j                   k\  rt        d| d|j                    d      t         | |j                  |       | |j                  |      f|j
                  d | |j
                  |dz   d  z   |j                  |j                        S )NzCannot map in_axis= for BCOO array with n_batch=zK. in_axes for batched BCOO operations must correspond to a batch dimension.rg   rm   r7   r   r_   rV   rW   rE   rJ   rK   )contr   r{  r  s       r8   _bcoo_to_eltr'  
  s    	\J	S[[
*4&0Mckk] [a a b b	tCHHd#T#++t%<=IIet$syy';; //@R@R
T Tr^   c                4   ||S ||j                   kD  rt        d| d|j                    d      t         | ||j                  |       | ||j                  |      f|j
                  d | |fz   |j
                  |d  z   |j                  |j                        S )NzBCOO: cannot add out_axis=r$  zC. BCOO batch axes must be a contiguous block of leading dimensions.rm   r%  )r&  rj  eltr  s       r8   _bcoo_from_eltr*  
  s    	\J	CKK
1$7TUXU`U`Ta bY Y Z Z	tIsxx.YT0RSIIet$	|3cii6FF //@R@R
T Tr^   rC   )rx   r_   rp   r   r   r_   )rx   r_   rp   r  r   r_   )rV   r   rW   r   rE   r  r   r   )rW   r   rE   r  r   r   )rx   r_   r   r,   )rV   r,   rW   r,   rT   r   r   r,   )rx   r,   rp   r  r7   r   r   r   r   r.   r   r_   )rx   r,   rp   r   r7   r   r   r   r   r.   r   r  )r   r_   r!  r-   r  zbool | Noner   r_   )rW   r,   r!  r,   r   r,   )rx   r_   rI  r  r   r_   )
rV   r,   rW   r,   rI  r  rT   r   r   r  )rI  r  rT   r   )r  BCOO | Arrayr  r+  r  r'   r|  Noner}  r,  r   r+  )r  r,   r  r,   r  r,   r  r'   r}  r   r  r   r   r,   )r  r,   r  r,   r  r,   r  r'   r}  r   r  r   r   r,   )r  r   )
r	  r,   r
  r,   rW   r,   r  r'   r   r,   )r  r,   r  r,   r  r,   r  r,   r  r   r  r   r  r'   r}  r   r   r  )r  r   r  r   )rx   r_   r   r_   )
rV   r,   rW   r,   rT   r   rp   r  r   r  )
rx   r_   r7   r  r   r  r  z
str | Noner   r_   )rx   r_   rE   r   r  r  r   r_   )rV   r,   rW   r,   rT   r   rE   r   r  r  r   r  )r	  zSequence[BCOO]r  r   r   r_   )rx   r_   r  r  r  r  r   r_   )r!  r_   r  r  r   r_   )
rx   r_   rK  r  rL  r  r@  r  r   r_   )rx   r_   rK  zSequence[Any]rd  r  r   r_   )rx   r_   r~  r  r   r_   )
rV   r,   rW   r,   rT   r   r~  r  r   tuple[Array, Array, Shape])r  r_   r  r_   r   r_   )r  r,   r  r,   r  r,   r  r,   r  r   r  r   r   r-  )r  r_   r  r,   r   r,   )
rV   r,   rW   r,   r  r,   rT   r   r   r,   )r  r_   rK  r,   r  r(   rd  r   rK   rM   r  rM   r+  zstr | GatherScatterMode | Noner   r_   r  )r  r_   r  r_   r  r  r   r_   )r   
__future__r   collections.abcr   r^  r   rq  rn   typingr   r   r   r  numpyr   r  r	   r
   r   jax.experimental.sparse._baser   jax.experimental.sparse.utilr   r   r   r   r   r   r   r   "jax.experimental.sparse._loweringsr   r   jax._src.interpretersr   	jax.numpyr>   jax._src.utilr   r   r   jax._srcr   r   r   r   r    r!   r"   r!  jax._src.lax.laxr#   r$   r%   r&   r'   jax._src.lax.slicingr(   r)   jax._src.libr*   jax._src.numpy.setopsr+   jax._src.typingr,   r-   r.   r/   float32float64	complex64
complex128r  r  r  r]   ry   r   r   r   r   r   	Primitiver   r   r   def_implr   def_abstract_evalr   r   r   r   defjvpprimitive_transposesprimitive_batchersregister_lowering	lower_funr   r   r   r   r   r  r	  r  r  r  primitive_jvpsr#  r  r   r9  r;  r>  r@  rE  rR  rF  rN  rY  r`  rd  rg  rr  rz  r  r{  r  r  r  r  "_bcoo_dot_general_default_loweringr  r  _bcoo_dot_general_gpu_loweringr  r  r  r  simple_implcuda_is_supportedrocm_is_supportedr  r  r  r  r  r  r&  r0  r5  r8  r=  r@  r  rZ  rc  r]  rz  r  r  r  r  r  r  r  r  _bcoo_sort_indices_hlor  r   rK  r  r  r0  r  r  r  r  r  _bcoo_sum_duplicates_hlor  r  r  r  r0  r6  r?  r`  rw  r3  r  r  r  r  r  r  r  r  r  r  r  register_pytree_node_classr_   r'  r*  register_vmappabler   r4   r^   r8   <module>rT     sk   O " $     , ,   
    3  
 F &  6 6     $ * 4T T J # ) 7 7 + 

BJJbmmL 
 7*12	JZ X V"  /	@UU  K K" !!- "-
9-8 
		.+T 2*A   '.I  N +   ~~t~~(0 1 "4>>"23 $(  !  59"#ciiII2;ILPI. =>a-0YY8!*8;N8*  2 ##] $]# 	Gr ':  " #,E  ( )0M  , -   '4*1 2  / OS >2 @D H  $ $L !!0 "0FPH* 
		.$ 1 2*A   '.I  N +   ~~t~~(0 1 "4>>"23 $(  !K&0!.08B0GZ0-$   ##b $bI " ':  " #,E  ( )0J  , -   '4*1 2 $T^^$67  (,48"&	)J(;)J $)J .2)J ,8	)JV8)<8.18 #-8 278 ,*=,/2,@J,OT,87?I87t %%:HR: &:* &4T^^U&4 "(77r "0"8 aBLa
aBLa
1BDN1Bf'EO' 
		
7?X Y.I  * +2N  . /   )+M N   ' ($86K$86K ,T^^,FG K,q3,3D $$q %q( --	 .	ZZF
 
		
$&E
)416Y  2 3:^  6 7   DNN1EJL &t~~&:; (,  %R$.R<FR[nR03R8KR,f\ #< #<J ''@! (@!D;$P 5S  0 1*A  & '   +^T^^t.5 6 %dnn%89 '+  $1  ) && '	+B  (d4  *@  % &3S  / 0   *,B C ''<= )-  &#&K
     7<!& AF+05:* ((
 )

+BB *4>>6  ,D  ' (5W  1 2   ,.F G
 <@W[4;D4'1D4@DD4P $(,0(&1>&CV&PD=P 59>01>0#'>0B48T: 04K8,K88<K8ZO8dB8 5"n8C&0C>HCMgC,<
S < (-+07;!=?#9=?"=? !%=? %)	=?
 5=? '+=?@ ,0dVZ23qTX59#( ?C#(J4>* %%P9 P &PhTT   D#sL.$ Or^   