
    vh                       U d dl mZ d dlZd dlmZmZ d dlZd dlmZ d dl	m
Z
mZ d dlZd dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZmZmZmZ d dl m!Z!m"Z"m#Z#m$Z$ d dl%m&Z' d dl(m)Z)m*Z*m+Z+m,Z, d dl-m.Z. d dl/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z: e1e;cZ;Z<e2e=cZ=Z> ej~                  d       G d d             Z@ ej~                  d       G d d             ZA ej~                  d       G d d             ZB G d d      ZC eC       ZDd ZEd|dZFd  ZG e+eBeEeG       d! ZH ej~                  d       G d" d             ZId}d#ZJd$ ZK	 	 	 	 	 	 	 	 d~d%ZL	 	 	 	 	 	 dd&ZM	 	 	 	 	 	 dd'ZN	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd(ZOe
ZPe
ZQe
ZRe
ZSe
ZTeg e"f   ZUeeeUePeRgeQf   ZVeeeSeQeRgePf   ZWeeSge.f   ZXdd)ZYi ZZd*e[d+<   	 	 	 	 	 	 	 	 dd-Z\i Z]d.e[d/<   dd0Z^i Z_d1e[d2<   	 	 	 	 	 	 dd3Z`i Zad4e[d5<   eChabd6e[d7<   dd8Zcdd9Zdej                  	 	 	 	 dd:       Zfeege
   ege
   ege
   gehege
   ege
   f   f   Zii Zjd;e[d<<   d= Zkd> Zld? Zmd@ Zn eod      ZpdZq G dA dBe"      Zr ej~                  d       G dC dD             ZsdE Zt G dF d,e!      Zu	 	 ddGZvej                  dH        Zxej                  ddI       Zy	 	 	 	 	 	 	 	 ddJZzej                  dK        Z{dL Z|dM Z}dN Z~dO ZdP Z	 	 	 	 	 	 ddQZe9	 	 	 	 	 	 ddR       Z	 	 	 	 ddSZdT ZdU ZdV Ze9	 	 	 	 	 	 ddW       Zej                  dX        Zej                  dY        Zej                  dZ        Zd[ Z G d\ d]      Z e       Zej                  d^        Z	 	 	 	 	 	 	 	 dd_Zej                  d`        ZddaZedbehe
eedeheedf   dbf   f   f   f   Zi Zdce[dd<   i Zdee[df<    G dg dh      Z e       Zi Zdee[di<   dj Zdk Zdl Zdm Zdn Zdo Zdp Zdq ZddrZ	 	 	 	 	 	 ddsZdt ZdduZddvZ G dw dxe      ZddyZdz Zeee<   d{ ZeeejV                  <    eejX                         y)    )annotationsN)CallableSequence)partial)AnyUnion)config)core)source_info_util)linear_util)PartitionSpec)NamedSharding)mesh)ZeroSymbolicZeroadd_jaxvalsadd_jaxvals_p)TraceTracerTraceTagAxisName)partial_eval)tree_unflattentree_flattenregister_pytree_node	PyTreeDef)Array)unzip2safe_mapsafe_zip
split_listcanonicalize_axismoveaxisas_hashable_functioncurrymemoizeweakref_lru_cachetuple_insertT)frozenc                  L    e Zd ZU ded<   ded<   ded<   d	dZej                  Zy)
JumbleTycore.Varbinderzint | Tracer | core.Varlengthzcore.DShapedArrayelt_tyc                f    dt        | j                         d| j                   d| j                   S )NVar:z => )idr-   r.   r/   selfs    Y/opt/face_recognition/venv/lib/python3.12/site-packages/jax/_src/interpreters/batching.py__repr__zJumbleTy.__repr__6   s,    DKK!4;;-tDKK=AA    Nreturnstr__name__
__module____qualname____annotations__r7   dataclassesreplace r8   r6   r+   r+   1   s%    

!!
B'r8   r+   c                  B    e Zd ZU ded<   ded<   ddZej                  Zy)IndexedAxisSizer,   idxzArray | core.Var | Tracerlengthsc                J    | j                    dt        | j                         S )Nz.Var)rG   r3   rF   r4   s    r6   r7   zIndexedAxisSize.__repr__?   s    ll^4488~..r8   Nr9   r<   rC   r8   r6   rE   rE   ;   s    -$$/'r8   rE   c                  "    e Zd ZU ded<   ded<   y)Jumbler+   avalr   dataNr=   r>   r?   r@   rC   r8   r6   rJ   rJ   E   s    .+r8   rJ   c                      e Zd Zy)
JumbleAxisNr=   r>   r?   rC   r8   r6   rO   rO   K       r8   rO   c                   g }| j                   j                  j                  D cg c]M  }t        |      t        u r8|j                  |j                        xs |j                  t        |            n|O }}| j                   j                  j                  t        |            }| j                   j                  |      }|| j                  f|fS c c}w )NrG   shaper/   )rK   r/   rU   typerE   appendrG   rB   lenupdatetuplerL   )jumblerG   d	new_shaper/   rK   s         r6   _jumble_flattenr_   S   s    ' **002 q'_, ~~aii(KAIIc'lI,K234 2) 2 ;;$$5+;$<&			F		+$
6;;		%%2s   AC
RaggedAxisc                    | j                   }| j                  }t        |      dk7  rt        d      |d   d   }|d   d   }|||fS )N   z)Multiple ragged axes not yet implemented.r   )stacked_axisragged_axesrY   
ValueError)dimrc   rd   ragged_axis_dimragged_axis_lengths        r6   _ragged_axis_partsri   ]   sZ    !!,+
@
AAN1%/"1~a(	(:	::r8   c                H   |\  }}| j                   j                  D cg c]7  }t        |      t        u r"|j	                  ||j
                  dz
           n|9 }}| j                   j                  t        |            }| j	                  |      } t        | |      S c c}w )Nrb   rS   rT   rV   )	r/   rU   rW   rE   rB   rG   rZ   r[   rJ   )rK   xrG   rL   r]   r^   r/   s          r6   _jumble_unflattenrl   g   s    -'4 ))+ q'_, yyQ!7y8234 +) + ;;E)$45&	V	$$	d	+s   <Bc                   t        j                  t        j                  dt        j                  d                  }|dk7  rt
        t        |j                        }|d= |D ]  \  }}t        ||      ||dz
  <    t        j                  t        |      |j                  |j                        }t        t        || |      |      S )NrC   int32r   rb   )r
   r1   ShapedArraynpdtypeNotImplementedErrorlistrU   rE   DShapedArrayr[   	weak_typerJ   r+   )		axis_sizerc   rd   rk   r-   rU   ragged_axissegment_lensr/   s	            r6   _jumble_resultry   q   s    88D$$R'):;<&Q

qww-%Ah#. Ak<*6<@E+a-AU5\177AKK@&	F3Q	77r8   c                  :    e Zd ZU ded<   ded<   ed        ZddZy)	r`   intrc   ztuple[tuple[int, Any], ...]rd   c                8    t        | j                  d   d         S )Nr   rb   )rY   rd   r4   s    r6   sizezRaggedAxis.size   s     t"1%&&r8   c                h      fdt        fd j                  D              }t        |      S )Nc                l    j                   | kD  r
| k\  r| dz   S j                   | k  r
| k  r| dz
  S | S Nrb   )rc   )axdstr5   s    r6   	move_axisz/RaggedAxis.move_stacked_axis.<locals>.move_axis   sB    			R	B#IAv			R	B#IAvir8   c              3  8   K   | ]  \  }} |      |f  y wNrC   ).0r   sizesr   s      r6   	<genexpr>z/RaggedAxis.move_stacked_axis.<locals>.<genexpr>   s     N	EimU+N   )r[   rd   r`   )r5   r   new_axesr   s   `` @r6   move_stacked_axiszRaggedAxis.move_stacked_axis   s-     NT=M=MNNHc8$$r8   N)r5   r`   r   r{   r:   r`   )r=   r>   r?   r@   propertyr}   r   rC   r8   r6   r`   r`   }   s'     +*' '
	%r8   c                    g }t        |      D ]3  \  }}| j                  D ]  \  }}||k(  s|j                  ||f        3 5 t        | j                  |      S r   )	enumeraterd   rX   _sorted_ragged_axisrc   )rf   permnew_ragged_axesrF   old_idxr   r}   s          r6   transpose_ragged_axesr      sf    /o lc7OO D	BT{+
 
S--	??r8   c           	     D    t        | t        t        |d                   S )Nc                    | d   S Nr   rC   )ps    r6   <lambda>z%_sorted_ragged_axis.<locals>.<lambda>   s
    !A$ r8   )key)r`   r[   sorted)rc   rd   s     r6   r   r      s    	L%{(O"P	QQr8   c                    |r6|D cg c]  \  }}t        ||       |f }}}t        t        ||       |      S t        ||       S c c}}w r   )r"   r   )ndimrc   rd   r   sz	canonicals         r6   make_batch_axisr      sS    
 ALMvr2#B-r2MIM0tDiPP\400 Ns   Ac                   t        | t              rut        |      }t        j                  t        j
                  dt        j                  d                  }| j                  D ]  \  }}t        ||      ||<    t        |      S |S )NrC   rn   )
isinstancer`   rs   r
   r1   ro   rp   rq   rd   rE   r[   )bdim
data_shaperesultr-   rw   rx   s         r6   bdim_as_shaper      sw    j!*FXXd&&r288G+<=>F%)%5%5 B!\+FLAf[B=r8   c                    t        |      D cg c]#  \  }}t        |t              r||j                  f% }}}t	        t        |      | |      S c c}}w r   )r   r   rE   rG   r   rY   )rc   r   ir}   rd   s        r6   shape_as_bdimr      sT     3<J2G 7wq$t_5 T\\" 7+ 7	Z,	DD7s   (Ac           
     (   || S t        |      t        d |D              k(  sJ  G d d      }|D cg c]  \  }} ||       c}}|D 	cg c]R  \  }}	|	rKt        |      t        j                  u r.|j                  t        fd|j                  D                    n|T }
}}	|D cg c]  }t        j                  |       }}t        |t              r ||j                        n|}t        |
|      D ]A  \  }}t        |t              rt        |j                  t        j                   |||             C |D ch c]>  }t        |      t        j                  u s|j                  D ]  }t        |      |u r| @ }}}t#        t%        ||            }||z
  }|D cg c]  }|j&                  df }}t)        g ||      D ci c]  \  }}|t+        j,                  |       c}}|D cg c]O  }t        |      t        j                  u r.|j                  t        fd|j                  D                    n|dfQ }}t/        j0                  | g ||      S c c}}w c c}	}w c c}w c c}}w c c}w c c}}w c c}w )	Nc              3  &   K   | ]	  \  }}|  y wr   rC   )r   _explicits      r6   r   z%_update_annotation.<locals>.<genexpr>   s     %L;1hh%Ls   c                      e Zd Zd Zy) _update_annotation.<locals>.Namec                    || _         y r   )a)r5   r   s     r6   __init__z)_update_annotation.<locals>.Name.__init__   s    A46r8   Nr=   r>   r?   r   rC   r8   r6   Namer      s    %r8   r   c              3  v   K   | ]0  }t        |      t        j                  u r|j                     n| 2 y wr   )rW   peDBIdxval)r   r]   namess     r6   r   z%_update_annotation.<locals>.<genexpr>   s5       2$% 15Q2880Caee J  2s   69rT   Fc              3  B   K   | ]  }j                  ||        y wr   )get)r   r]   name_maps     r6   r   z%_update_annotation.<locals>.<genexpr>   s     $IAX\\!Q%7$Is   T)rY   sumrW   r
   rt   rZ   r[   rU   get_avalr   r   rK   zipr`   rr   rX   unmapped_avalsetmapr   r   r   r   luannotate)f	orig_typerv   	axis_nameexplicit_in_dimsrx   r   r   r   eavalss	new_avalsr   r]   	mentioned
expl_names
impl_namesn	impl_partr   	expl_partr   r   s                         @@r6   _update_annotationr      s}    q		#%L)%L"L	LL	L& & )
*tq!47
*% ?HN6:a1d1g*** 88%  2)* 2 28 3012 N% N *66At}}Q6)6))V<tINN)"%)* 5da!Z t))"a34	5 $ 4QtAw$2C2C'C447d?  4q 4) 43tY'(*:%*%/0U|0)0)23MZ3M*3M)NOAa!nO(GPRBC47d/// u$I$IIJ56> R) R	Q0)0i0	11+ +N 74 1ORs1   I-AI3/I9 I>'"I>*J J	<AJc                "   t         j                  t        |            }|r |t        t        | |      |||      S t        |      t
        u r|t        urt        d      d t        |j                  j                  j                        D        \  \  }}t        |j                  j                  d|dz   |j                  fg      }t!        | |j                  |      S t#        |t$              s|P|xr( t'        |t)        t+        j                  |                  }| t!        | ||t-        j.                               S |S t#        | t0              r0t#        |t2              r t!        | |dt-        j.                               S J dt        |              )Nz3jumble input without using jumble_axis in_axes specc              3  L   K   | ]  \  }}t        |      t        u r||f  y wr   )rW   rE   )r   r   r   s      r6   r   zto_elt.<locals>.<genexpr>  s,      1QH/ R 1s   "$r   rb   zUnexpected type in ELT? )to_elt_handlersr   rW   r   to_eltrJ   jumble_axis	TypeErrorr   rK   r/   rU   r   rL   r   rG   BatchTracerr   r{   r"   rY   rp   r   current
BatchTracerO   )traceget_idxrk   spechandlerr]   ias
batch_axiss           r6   r   r      sZ   Q('765'2GQEEAw&;KLL1'(;(;<1IHQ !a1Q32D1EFJuaffj11$=%dC,<=D q$(8(@(@(BC (%&( %$D*)E
 1&6&>&>&@AA 7,T!WI665r8   zdict[type, ToEltHandler]r   r   c                    t         j                  t        |            }|r fd} |||||      S  j                  |      \  }}	t        |	      t        u r5|t
        urt        d      t        ||	j                  |	j                  |      S 	 t         j                  j                  ||	||      S # t        $ r t        |j                  |      d w xY w)Nc                $    t        | ||      S r   )from_elt)rv   eltaxisr   	mesh_axisr   s      r6   _contzfrom_elt.<locals>._cont  s    eY	1c4@@r8   z5ragged output without using jumble_axis out_axes spec)from_elt_handlersr   rW   to_batch_infor`   r   r   ry   rc   rd   	matchaxis	axis_datanameSpecMatchError	batch_dim)
r   rv   r   r   rk   r   r   r   r   r   s
   ` ``      r6   r   r     s    !!$q'*'A5)Q--!!!$)#t	$Z:;MNN)T%6%68H8H#NN;u++Y	T3( ( ;1akk40d:;s   #B5 5"Czdict[type, FromEltHandler]r   c                    ddl m} t        j                  t	        |             }|r ||       S |j                  dt        |             S )Nr   laxrn   )jaxr   make_iota_handlersr   rW   iotar{   )rv   r   r   s      r6   	make_iotar   .  s>     ""4	?3'988GS^,,r8   zdict[type, MakeIotaHandler]r   c                    ||ft         | <   t        j                  |       |t        | <   |t        | <   |r
|t
        |<   y y r   )
vmappables
spec_typesaddr   r   r   )	data_type	spec_typeaxis_size_typer   r   r   s         r6   register_vmappabler   9  sC     %n5*Y..%/)!)IY">2Yr8   zdict[type, tuple[type, type]]r   z	set[type]r   c                    t         j                  |       \  }}t        | = t        | = |t        v rt        |= t
        ht         j                         D ch c]  \  }}|	 c}}z  ay c c}}w r   )r   popr   r   r   rO   valuesr   )r   r   r   r   s       r6   unregister_vmappabler  D  sb     nnY/!^i 	"))>* lJ4E4E4GHLIqiHH Hs   A&c                J    t        |       t        u xs t        |       t        v S r   )rW   rJ   r   )rk   s    r6   is_vmappabler  O  s    	aF		3d1g33r8   c                ~    t        ||      \  }} | |i |}t        |t              \  }}|j                  |       |S )N)is_leaf)r   r   r  store)r   r  in_tree	args_flatpy_args	py_kwargsansout_trees           r6   flatten_fun_for_vmapr  R  sE     &gy9'9	7 i #sL9-#x++h	*r8   z'dict[core.Primitive, RaggedMaskingRule]ragged_prop_rulesc                p    |d   }|dd  D ]  }||k7  s	t        | d|        |gt        |      z  }||fS )Nr   rb   z != )re   rY   )
eqn_paramsinvar_raggednessoutvarsfirst_invar_raggednessother_invar_raggednessoutvar_raggednesss         r6   ragged_mask_elementwise_ruler  e  si    +A. 0 4 P!77016L5MNOOP ..W=	,	,,r8   c                V    t        |      rt        d|       |d gt        |      z  fS )Nzunexpected invar_raggedness: )anyre   rY   r  r  r  s      r6   ragged_mask_assert_no_op_ruler  p  s6    	
45E4FG
HH	D6CL0	00r8   c                $    |d gt        |      z  fS r   rY   r  s      r6   ragged_mask_no_op_ruler  v  s    	D6CL0	00r8   c                8    t        |      dk(  sJ |       |}||fS r   r  )r  r  r  s      r6   ragged_mask_transfer_identityr   z  s0     
	!	#5%55	#&	,	,,r8   c                  J    e Zd Zg dZ	 d		 d
dZed        Zd Zd Zd Z	d Z
y)r   )r   r   source_infoNc                >   t         j                  j                  rgt        |      t        t
        t        fv sJ t        |      t
        u r8t        j                  |      }d|cxk  rt        |j                        k  sJ  J || _        || _        || _        || _        y r   )r	   enable_checksvaluerW   	NotMappedr{   r`   r
   r   rY   rU   _tracer   r   r"  )r5   r   r   r   r"  rK   s         r6   r   zBatchTracer.__init__  s    !!)_C <<<<	iC	}}S!I/DJJ/////DKDHDN"Dr8   c                "   t        j                  | j                        }| j                  j                  j
                  `t        j                  j                  rF|j                  |j                  t        | j                  j                  j
                        z
        }| j                  t        u r|S t        | j                        t        u r8t        j                   |j"                  | j                     | j                  |      S t        | j                        t$        u rt        j                   |j"                  | j                  j&                     | j                  j&                  |      }t)        |j"                        }| j                  j*                  D ]?  \  }}t-        | j                  |d      }| j                  j&                  |k  r|dz  }|||<   A t        j.                  t1        |      |j2                  |j4                        S y )Nvmar   rb   )rU   rq   ru   )r
   r   r   r'  r   	spmd_namer	   
_check_vmar%  rZ   r*  	frozensetr   
not_mappedrW   r{   mapped_avalrU   r`   rc   rs   rd   r   rt   r[   rq   ru   )r5   rK   new_avalrU   rw   segment_lengthssize_tracers          r6   rK   zBatchTracer.aval  s   =="D{{&&2				 	 {{9T[[%:%:%D%DEE  G~~#k	dnn		$djj8$..$OO	dnn		+!!

4>>../1L1LdTh8>>"e*...*D*D )
&+!$++B>>&&4

+(k	)
 U5\)-9 9 
,r8   c                h    | j                   t        u rt        j                  | j                        S | S r   )r   r.  r
   
full_lowerr   r4   s    r6   r4  zBatchTracer.full_lower  s&    ~~#__TXX&&kr8   c                x    | j                   ydt        |        dt        j                  | j                          S )N z!
This BatchTracer with object id z was created on line:
  )r"  r3   r   	summarizer4   s    r6   _origin_msgzBatchTracer._origin_msg  sD    0D
 ;#--d.>.>?@B Cr8   c                :    d| j                   fd| j                  fgS )Nr   r   )r   r   r4   s    r6   	_contentszBatchTracer._contents  s    DHHT^^<==r8   c                    | j                   t        | j                         t        u rt        j                  | j
                        S | S r   )r   rW   r{   r
   get_referentr   r4   s    r6   r<  zBatchTracer.get_referent  s6    ~~dnn!5!<txx((kr8   r   )r   zNotMapped | int | RaggedAxisr"  z"source_info_util.SourceInfo | None)r=   r>   r?   	__slots__r   r   rK   r4  r8  r:  r<  rC   r8   r6   r   r     sD    1) BF
#>
# 9 9,C>r8   r   c                  6    e Zd ZU ded<   ded<   ded<   ded<   y)AxisDatar   r   r}   r+  explicit_mesh_axisNrM   rC   r8   r6   r?  r?    s    	*	*/r8   r?  c                    | j                   }t        t        |j                  ||       }t	        |j
                  |      S r   )r@  Pr(   r   r   r   )r   orig_shardingr   r   new_specs        r6   get_sharding_for_vmaprE    s;    $$# ]//s;<(	}))8	44r8   c                  D     e Zd Z fdZd Zd Zd ZddZd Zd Z	 xZ
S )	r   c                p    t         |           || _        t        |t              sJ || _        || _        y r   )superr   parent_tracer   r?  r   tag)r5   rI  rJ  r   	__class__s       r6   r   zBatchTrace.__init__  s4    	G$Di***DNDHr8   c                    t        |t              r:|j                  j                  | j                  u r|j                  |j
                  fS |t        fS r   )r   r   r'  rJ  r   r   r.  )r5   r   s     r6   r   zBatchTrace.to_batch_info  s<    #{#

$(((BWWcmm##*_r8   c                    t         j                  j                  r* |j                  t	        t
        j                  |      i | t        t	         j                  |            \  }}t        d |D              }|t        v r|rE|t        v r=t         fdt        |   |      D              s|j                   j                  ||      S t        j                   j                        5  t        |    j                   ||fi |\  }}d d d        n}|r|j                   j                  ||      S |t"        v r<t        j                   j                        5  t#        |   ||fi |\  }}d d d        nt%        dj'                  |            t)        j*                         }	|j,                  r^t        j                   j                        5  t/              D 
cg c]  \  }
}|t0        urt3         |
||	      n|
 c}}
cd d d        S t0        urt3         ||	      S S # 1 sw Y   xY w# 1 sw Y   xY wc c}}
w # 1 sw Y   y xY w)Nc              3  ,   K   | ]  }|t         u   y wr   r.  )r   r   s     r6   r   z/BatchTrace.process_primitive.<locals>.<genexpr>  s     A$*,As   c              3  P   K   | ]  }j                   j                  |k(    y wr   )r   r   )r   r   r5   s     r6   r   z/BatchTrace.process_primitive.<locals>.<genexpr>  s*      F# nn))Y6 Fs   #&z&Batching rule for '{}' not implemented)r	   dynamic_shapesr%  abstract_evalr   r
   r   r   r   allfancy_primitive_batchersskippable_batchersr  bind_with_tracerI  set_current_tracer   primitive_batchersrr   formatr   r   multiple_resultsr   r.  r   )r5   r   tracersparamsvals_indims_inargs_not_mappedval_outdim_outsrcrk   r]   s   `           r6   process_primitivezBatchTrace.process_primitive  s>   ""aooDMM73??c$"4"4g>?GWAAAO$$
%% F'9!'<V'DF F   !2!2GVDD##D$5$56 	:5a8nngw:28:
'7	: 	: 
t00'6BB	
 	 !!$"3"34 M-a0'LVLM M   H O OPQ RSS

"
"
$C!!$"3"34 313Aq 120CD!Q,J 33 3
 
* $#6 9079#	: 	:M M33 3s6   (H#H/I"H;>I#H,/H8;II
c           
        |j                   sJ t        ||j                  d|j                              }t	        t        | j                  |            \  }}t        |      \  }}t        || j                  | j                  t        |            \  }}	t        ||j                  | j                  j                  | j                  j                  ||      }t!        j"                  | j$                        5   |j&                  |g||i |}
d d d        t)        
 |	             \  }
}	t+        j,                         }t/        |
|	      D cg c]  \  }}t1        | |||       c}}S # 1 sw Y   \xY wc c}}w )Nr   )r   )rZ  dictr   r=   r   r   r   indirectify_ragged_axesbatch_subtracerJ  r   r[   r   in_typer}   r   r
   rW  rI  bindresolve_ragged_axesr   r   r   r   )r5   call_primitiver   r[  r\  valsdimsrx   f_dims_outvals_outrb  vr]   s                 r6   process_callzBatchTrace.process_call  sC   ****&vzz&!**=>FD..89JD$06L$!!TXXt~~uT{KLB	
AIIt~~**DNN,?,?|
UB 
		 1 1	2 I$$$RH,HHHhI,XxzBHh

"
"
$C5885LMTQKaC(MM	I I Ns   5E)E5)E2c           
        t        t        | j                  |            \  }}d t        fdt	        ||d         D              }t        fdt	        ||d         D              }t        || j                  | j                  |      \  }|d   t              fd       }	t        |||	      }
t        j                  | j                        5   |j                  |g|i |
}d d d        t	                              D cg c]  \  }} ||      r
||k  r|d	z   n| }}}t        j                         }t	        |      D cg c]  \  }}t!        | |||       c}}S # 1 sw Y   xY wc c}}w c c}}w )
Nc                    | d uxr |t         uS r   rO  )in_out_axisr]   s     r6   both_mappedz+BatchTrace.process_map.<locals>.both_mapped!  s    $<*)<<r8   c              3  N   K   | ]  \  }} ||      r
||k  r|d z   n|  ywrb   NrC   r   r]   in_axisrv  s      r6   r   z)BatchTrace.process_map.<locals>.<genexpr>#  s5      6
!W !!,ggk7J6   "%in_axesc              3  N   K   | ]  \  }} ||      r
||k  r|d z
  n|  ywrx  rC   ry  s      r6   r   z)BatchTrace.process_map.<locals>.<genexpr>&  s5      6
!W 7A&7Q;a!eA=6r{  out_axes_thunk)closurec                 T    t         fdt                              D              S )Nc              3  N   K   | ]  \  }} ||      r
||k  r|d z   n|  ywrx  rC   )r   out_axisr]   rv  s      r6   r   zEBatchTrace.process_map.<locals>.new_out_axes_thunk.<locals>.<genexpr>0  s8      J#! $/x#;H8a<RZZ Jr{  )r[   r   )rv  ro  r~  s   r6   new_out_axes_thunkz2BatchTrace.process_map.<locals>.new_out_axes_thunk.  s-     J'*>+;XZ'HJ J Jr8   )r|  r~  rb   )r   r   r   r[   r   rg  rJ  r   r$   re  r
   rW  rI  ri  r   r   r   )r5   map_primitiver   r[  r\  rl  rm  new_in_axesnew_dimsr  
new_paramsrp  r]   r  	dims_out_rb  rq  rv  ro  r~  s                    @@@r6   process_mapzBatchTrace.process_map  s   D..89JD$= 6D&"346 6K  6D&"346 6H !DHHdnnhGKAx,-N .1J 2J fkBTUJ			 1 1	2 <###A;;
;h< &)^5E%FH!a &h2x1}Q!K HI H

"
"
$C5895MNTQKaC(NN< <H Os   E1E=F1E:c          
        t        t        | j                  |            \  }}t        || j                  | j
                  |      \  }}t        || j                  | j
                  |      \  }}	|j                  | j                  ||g|t        |            }
t        j                  ||	      \  }}t        j                         }t        |
|      D cg c]  \  }}t        | |||       c}}S c c}}w )N)symbolic_zeros)r   r   r   rg  rJ  r   batch_custom_jvp_subtracerV  rI  re  r   merge_linear_auxr   r   r   r   )r5   primfunjvpr[  r  in_valsin_dims	out_dims1	out_dims2out_valsfstout_dimsrb  rq  r]   s                   r6   process_custom_jvp_callz"BatchTrace.process_custom_jvp_call:  s    c$"4"4g>?GW#C4>>7KNC.sDHHdnngVNC##D$5$5S7K77K$($GIH''	9=MC

"
"
$C5885LMTQKaC(MMMs   C1c          
        t        t        | j                  |            \  }D 	
cg c]  }	|	t        fD ]  }
|
  }}	}
t	        || j
                  | j                        \  }}t	        || j
                  | j                  |      \  }fd}t        || j
                  | j                  |      }|j                  | j                  |||ft        |      z   t        |            }t        j                  |      \  }}|s:        \  }}}|j                  t        d |D              z
  }t!        ||g      \  }}t#        j$                         }t'        ||      D 
cg c]  \  }}
t)        | ||
|       c}
}S c c}
}	w c c}
}w )Nc                             \  } } }t                      }|D cg c]  }|t        |      n|    }}g ||S c c}w r   )iternext)r   
input_fwdspruned_dimsr   	full_dimsr  r  	out_treess        r6   bwd_in_dimsz7BatchTrace.process_custom_vjp_call.<locals>.bwd_in_dimsL  sX    "aJ%kKUVa	4$wqzAViV'y';'' Ws   A)r  r  c              3  $   K   | ]  }|d u 
 y wr   rC   )r   r   s     r6   r   z5BatchTrace.process_custom_vjp_call.<locals>.<genexpr>Y  s     )LA!4-)Ls   )r   r   r   r.  rg  rJ  r   batch_custom_vjp_bwdrV  rI  r[   re  r   r  
num_leavesr   r!   r   r   r   r   )r5   r  r  fwdbwdr[  r  r  r  in_dimr]   fwd_in_dimsr  r  r  r  r  r   res_treer  num_resrb  rq  r  r  s         `                @@r6   process_custom_vjp_callz"BatchTrace.process_custom_vjp_callD  sh   c$"4"4g>?GW#*IVZ4HIq1I1IKI#C4>>7KNC#C4>>;ONC( sDHHdnnk7
SC##D$5$5%(#sOeGn$D$(9^$\^H ''	9=MC )a:##c)L)L&LLgx'3ka

"
"
$C5885LMTQKaC(MM+ J* Ns   F'F	)r   lu.WrappedFun)r=   r>   r?   r   r   rc  rr  r  r  r  __classcell__)rK  s   @r6   r   r     s,    9BN$OLNNr8   c                6    t        | ||      }t        |||      S r   )_batch_inner_batch_outer)r  r   r  out_dim_destsr   s        r6   batchr  `  s!     3	=1!	aG	,,r8   c                    t               }t        j                  d      5   | ||g| \  }}d d d        t        j                        5  ~d d d        S # 1 sw Y   *xY w# 1 sw Y   S xY w)Nvmap)r   r   transform_name_stackr
   ensure_no_leaks)r   r   r  r  rJ  outsr   s          r6   r  r  f  se    
#,,V4 ,C+7+KD%,E"--	+, ,-	+s   AA#A #A-c                J   t        |      r |       n|}t        j                         5 }t        ||      t	        fd      }t        j
                  |      5  t        t        t        |      ||      }d d d        t        j
                        5  t        j                  j                  j                  fg      5  t        j                  j                        5   |  }	t        |      r |       n|}t        t        t        j                  j                        t!        t#        |	            |	|      }
d d d        d d d        d d d        d d d        
fS # 1 sw Y   xY w# 1 sw Y   1xY w# 1 sw Y   5xY w# 1 sw Y   9xY w# 1 sw Y   
fS xY w)Nc                 j    t        t         j                        dt        j                               S r   )r   r   r}   r   r   )r   r   s   r6   r   z_batch_inner.<locals>.<lambda>s  s(    +eYy~~-F&6&>&>&@B r8   )callabler
   take_current_tracer   r&   rW  r   r   r   extend_axis_env_ndr   r}   add_spmd_axis_namesr+  r   r@  rangerY   )r   r   r  rJ  r  r  rI  rF   
in_tracersr  r  r   s    `         @r6   r  r  n  sq   !'*GI'  <L|S)4E
 B CC				- Fwvuc2GWEjF

 
 
' <

!
!INNINN#C"D
E<

"
"9#6#6
7< 
^d)1-)@mommWXuinni>Z>Z[3t9%t]<h< < << 
5F F< < < < < << 
5sl   2FE&:F-F
 E>$AE2	E>
F
F&E/	+F2E;7E>>FF

F	FF"c                    t         dd       ddt        j                  fd       }t        |d d       } |t	        | |            S )Nc                v    || S t        | j                        }|||   |z  g|||dz    | j                  |      S r   rs   rU   reshape)argr   	tile_sizerU   s       r6   	tile_axiszvtile.<locals>.tile_axis  sF    |jOE#U4[I%=>E$tAv;;ur8   c                    || S t        | j                        }||   ||dz      z  g|||dz    | j                  |      S )Nrb      r  )outr   rU   s      r6   untile_axiszvtile.<locals>.untile_axis  sL    |jOE+d1f56E$tAv;;ur8   c                    d t        |      D        }xs t        |d       }|J d        | t         |      |       }t        	|      S )Nc              3  F   K   | ]  \  }}|	|j                   |     y wr   rT   )r   rk   r   s      r6   r   z.vtile.<locals>._map_to_tile.<locals>.<genexpr>  s      WDAqQWWQZWs   
!!zNo mapped arguments?)r  )r    r  r   )
r   r	  r   
tile_size_outputs_flatin_axes_flatout_axes_flatr  r  r  s
        r6   _map_to_tilezvtile.<locals>._map_to_tile  s^    WHY$EWE/d5$/J!9#99!c)j99lSTL{L-88r8   )r   
int | None)r%   r   transformation2r?  r  )	f_flatr  r  r  r   r  r   r  r  s	    ```   @@r6   vtiler    s^    
 	 	 9 9 y)T48)	eFI|]K	LLr8   c                .   t        j                         5 }t        |||      }t        j                  |      5  t	        |      r |       n|}t        ||      \  }}t        ||      D 	cg c])  \  }}	|	 t        |||	t        j                               n|+ }
}}	 | |
 }d d d        t        t        |j                              \  }}t        |      \  }}d d d        |j                         g S c c}	}w # 1 sw Y   _xY w# 1 sw Y   3xY wr   )r
   r  r   rW  r  rj  r   r   r   r   r   r   r   rf  r  )r   r  rJ  r   r  r  rI  r   rk   rf   r  r  r  r  rx   s                  r6   rg  rg    s     	?L|S)4E				& %g.	Gg,Wg>gw<?<QS28!Ss  q#/?/G/G/IJ,-. Sj S
^d  E$7$7 >?Hh4X>L(	? ++h	#<	#(	##S 	? 	?s4   #D3C?+.C9
C?!8D9C??D	DDc                   t        d | D              sg | fS t        j                         dfd}| D cg c]  }t        |t              r ||      n| }}j                         D cg c]  \  }}|	 }}}||fS c c}w c c}}w )Nc              3  >   K   | ]  }t        |      t        u   y wr   )rW   r`   )r   r]   s     r6   r   z*indirectify_ragged_axes.<locals>.<genexpr>  s     1qT!W
"1   c           
     2   g }| j                   D ]g  \  }}j                  t        t        j                  |            |t        j                  t                    f      \  }}|j                  ||f       i t        | j                  t        |            S r   )rd   
setdefaultr3   r
   r<  r   r   rY   rX   r`   rc   r[   )r]   r   rw   r1  r   dbidxaxis_maps         r6   canonicalize_segment_lengthsz=indirectify_ragged_axes.<locals>.canonicalize_segment_lengths  s    O() 3$_$$
T/
0BHHS]3
46ha k512	3
 anneO&<==r8   r]   r`   r:   r`   )r  collectionsOrderedDictr   r`   r  )rm  r  r]   r  r   r   rx   r  s          @r6   rf  rf    s    	1D1	1t8O1<1H1H1J(> <@A67Az* +1-012 A( A ( 121!2,2	x	A2s   !A?,Bc                p    dfd}| D cg c]  }t        |t              r ||      n| }}|S c c}w )Nc                    g }| j                   D ]C  \  }}t        t        j                  |            }t	        |      }|j                  ||f       E t        | j                  t        |            S r   )	rd   r3   r
   r<  _locate_valuerX   r`   rc   r[   )r]   r   rw   r1  r   r%  r  r  s         r6   r  zTindirectify_ragged_axes_against_inputs_outputs.<locals>.canonicalize_segment_lengths  sn    O() 3$_t  12cC(3ek5123 anneO&<==r8   r  )r   r`   )rm  r  r  r  r]   r  s    ``   r6   .indirectify_ragged_axes_against_inputs_outputsr    sL    > <@A67Az* +1-012 A( A	/As   !3c                    t        |      D ]*  \  }}| t        |      k(  st        j                  |      c S  t        |      D ]*  \  }}| t        |      k(  st        j                  |      c S  J d       )NzCould not find segment lengths)r   r3   r   InDBIdxOutDBIdx)r   r  r  ix	candidates        r6   r  r    su     ) mb)
bmZZ^ !* mb)
bm[[_ 100r8   c           
         |D ch c]3  }t        |t              s|j                  D ]  \  }}|j                   5 }}}}|D cg c]F  }t        |t              r2t        |j                  t         fd|j                  D                    n|H }}t               D cg c]  \  }}||vs| c}}  |fS c c}}}w c c}w c c}}w )Nc              3  F   K   | ]  \  }}||j                      f  y wr   )r   )r   rw   lengths_idxrl  s      r6   r   z&resolve_ragged_axes.<locals>.<genexpr>  s-      J7{K (koo)>? Js   !)r   r`   rd   r   rc   r[   r   )rl  rm  r]   r   r  idxsr   rk   s   `       r6   rj  rj    s    #' 
1 
1a:a+D"#--
1q+ // 
1/ 
1$ 
1
 8<
= 34Z:& Q^^ J;<==J JK -.. 
=$ 
= "$	911D=!	9$	t
1
= 
:s   B<"B<AC#C0Cc           
           fd|D cg c]F  }t        |t              r2t        |j                  t        fd|j                  D                    n|H }}|S c c}w )Nc                    t        | t        j                        r| j                     S t        | t        j                        sJ | j                     S r   )r   r   r  r   r  )rF   r  r  s    r6   fetchz9resolve_ragged_axes_against_inputs_outputs.<locals>.fetch  sC    #rzz"SWWR[[)))cggr8   c              3  8   K   | ]  \  }}| |      f  y wr   rC   )r   rw   r  r  s      r6   r   z=resolve_ragged_axes_against_inputs_outputs.<locals>.<genexpr>  s)      J7{K ({);< Jr   )r   r`   rc   r[   rd   )r  r  rm  r]   r  s   ``  @r6   *resolve_ragged_axes_against_inputs_outputsr    sp     8<
= 34Z:& Q^^ J;<==J JK -.. 
=$ 
= 
+	
=s   AAc                .    t        | |t        |            S r   )_batch_jaxpr2r[   )closed_jaxprr   r|  s      r6   batch_jaxpr2r    s     
|Yg	??r8   c                z   t        j                  t        j                  |       | j                  j
                        }t        ||      \  }}t        |||      }t        t        || j                        D cg c]0  \  }}t        |t              rt        | j                  ||      n||f2 c}}      \  }}g }	t        ||      D ]  \  }}
|
t        u r|	j!                  |        t        j"                  |j$                  |
||j&                        }|j(                  Lt*        j,                  j.                  r2|j1                  |j2                  t5        |j(                        z        }|	j!                  |        t7        j8                  ||	      \  }}}\   t        j:                  ||       |       fS c c}}w )N
debug_infor)  )r   	wrap_initr
   jaxpr_as_funjaxprr  _batch_jaxpr_inner_batch_jaxpr_outerr   r   in_avalsr   r`   handle_ragged
unsafe_zipr.  rX   r   r}   r@  r+  r	   r,  r%  rZ   r*  r-  r   trace_to_jaxpr_dynamicClosedJaxpr)r  r   r|  r   out_axesrf   rK   in_axes2avals_in	avals_in2b	jaxpr_outr   constss                 r6   r  r     s    	ll4$$\2*00;;=!"1i0+!XIw/! 7L$9$9:< #t 
C	$ L))35+.+6< =(H )Hh/ 	gdAJt
..!T9#?#?Ad				(""Ii6I6I,J!JK$t	  66q)D)Q			)V	,hj	88!<s   95F7
c                   t        |j                        }|j                  D ];  \  }}| |j                     j                  j
                  |||j                  |k  z
  <   = |j                  t        |            }|j                  |fS )NrT   )	rs   rU   rd   r   rq   boundrc   rZ   r[   )r   rf   rK   r^   r   dbir0  s          r6   r  r    s~    4::) Jfa,4SWW,=,C,C,I,IIa3##a'()J[[uY/[0(			8	##r8   c                j    t        |t              rt        |      n|}t        | |t        |      |      S r   )r   rs   r[   _batch_jaxpr)r  r   
in_batchedinstantiateinsts        r6   batch_jaxprr  $  s-    )+t<{	+$	lIuZ/@$	GGr8   c                T   t        |t              s*t        |t        t        f      rt	        d |D              sJ t        |t              r|gt        | j                        z  }|D cg c]  }|rdnt         }}|D cg c]  }|rdnt         }}t        | |||      S c c}w c c}w )Nc              3  <   K   | ]  }t        |t                y wr   )r   bool)r   r	  s     r6   r   z_batch_jaxpr.<locals>.<genexpr>+  s     7ajD!7s   r   )
r   r  rs   r[   rS  rY   	out_avalsr.  zero_if_mappedbatch_jaxpr_axes)r  r   r  r  r	  r|  r  out_axes_dests           r6   r  r  (  s    
[$
'
[4-
0
7;7
79 8 T"-#l&<&<"==K-78!Q#8'8=HIT1.0I-I	,	7M	JJ 9Is   (B ?B%c                B    t        | |t        |      t        |            S r   )_batch_jaxpr_axesr[   )r  r   r|  r  s       r6   r  r  2  s    	<E'NE-DX	YYr8   c           
        t        j                  t        j                  |       | j                  j
                        }t        ||      \  }}t        ||||      \  }}t        |||      }t        | j                  |      D cg c];  \  }}|t        ur,t        j                  |j                  |||j                        n|= }	}}t        j                   ||	      \  }
}}\   t        j"                  |
|       |       fS c c}}w )Nr  )r   r  r
   r  r  r  r  _match_axes_jaxprr  r  r   r.  r   r}   r@  r   r  r  )r  r   r|  r  r   r  out_batchedrK   r	  r  r
  r   r  s                r6   r  r  5  s    	ll4$$\2*00;;=!"1i0+!X$Q	=(K.![Iw/! (2,2G2G'QS $D! *$   D!*!=!=?  S( S  66q(C)Q			)V	,km	;;Ss   A Dc           
        t        j                         5 }t        |||      }t        ||      \  }}t	        ||      D 	
cg c]  \  }	}
|
t        ||	|
      n|	 }}	}
t        j                  |      5  t        j                  |j                  |j                  fg      5  t        j                  |j                        5   | | }d d d        d d d        d d d        t        t        |j                              \  }}t        |||      }d d d        |j!                         S c c}
}	w # 1 sw Y   ixY w# 1 sw Y   mxY w# 1 sw Y   qxY w# 1 sw Y   FxY wr   )r
   r  r   rj  r   r   rW  r  r   r}   r  r+  r   r   r   r  r  )r   r  r   rJ  r|  r  rI  r   r   r   rf   r  r  r  r  new_out_axess                   r6   r  r  E  sN     %L|S)4E$Wg6JAw"%gw"79c3 36/+eS#.sJ 9J 9

 
 
' 

!
!INNINN#C"D
E

"
"9#6#6
7 
^d    E$7$7 >?HhA'8%L% ++l	/9     % %se   ,ED)E4-E! D;D/	D;E7E)E/D84D;;E EE	EEc                    | ||g| } |       }t        ||      D 	
cg c]  \  }	}
|
t        u r|	t        u rd ndn|
 }}	}
t        |      t        |      k7  r|\  }|gt        |      z  }t	        t        t        |j                  |j                  |j                        |||      }|D 
cg c]  }
|
d u }}
|j                  |       |S c c}
}	w c c}
w r   )r  r  r.  rY   r   r   r   r   r}   r@  r  )r   r  r   r  r  r   r|  r  r  rb  r   out_axis_destr   s                r6   r  r  V  s     ug(((Z( $.h#FHsC ^+  :-D1145 H- H 	3x=("NM"Oc(m3MINNINN"557=(4( -::SD:+:++k	/H ;s    C.Cc           
         t        |      r |       n|}t        ||      D cg c]6  \  }}t        |t              rt	        |t        j                  |            n|8 }}}t               } | ||g| S c c}}w r   )r  r  r   r{   r"   rp   r   r   )r   r   r  r  rk   r   rJ  s          r6   r  r  h  s}    !'*GI'",Wg">@B 3=R2Er2771:. @' @
#	
3	"'	""@s   ;A6c                <    | |k(  r| S | t         u r|S |t         u r| S | S r   rO  )rk   ys     r6   _merge_bdimsr(  p  s+    !VHJHJHHr8   c                      e Zd Zy)ZeroIfMappedNrP   rC   r8   r6   r*  r*  z  rQ   r8   r*  c                   |j                   }|j                  }t        j                         5 }t	        |||      }	t        ||dz        D 
cg c]Q  \  }
}||
nGt        |
      t        u r*t        t        j                  |||
j                              nt        |	|
|      S }}
}t        j                  |	      5   | | }d d d        d d d        t        t        	j                              \  }}t        |t!        |      dz  g      \  }}t        |t!        |      dz  g      \  }}t        t"        ||      }t        t%        t&        |	j(                  j*                  ||      |||      }t        t%        t,        |	j(                  j*                  ||      |||      }|j/                  |       ||z   S c c}}
w # 1 sw Y   xY w# 1 sw Y   xY w)Nr  )r}   r@  r
   r  r   r   rW   r   r/  rK   r   rW  r   r   r   r!   rY   r(  r   r   r   r   _matchaxis_symzerosr  )r   r  rJ  r   r  r  r}   r   rI  r   r   rf   r  out_tracersr  r  out_primalsout_tangentsout_primal_bdsout_tangent_bdss                       r6   r  r    s   	$**)  EL|S)4E #&gw{";= c3 ##Y,. t//c388DE4?sC4PQ =J = 
			& E67nkEE c%"5"5{CD(H(CMQ4F3GH+|$.x#h-1:L9M$N!./~?(WY(<(<dIN#X=+W0%//2F2FiX$h>,++h	|	##=E EE Es6    GAF6#G;F<G6G<G	GGc                     j                   j                  j                   fd}t        j                  | j
                        S )Nc                    t              r        n}t        | |      D cg c]B  \  }}t        |      t        u r*t        t	        j
                  	||j                              n|D } }}t        | |      D cg c]  \  }}t        |      t        u rd n| }}}t        
|      \  }}t        |	|      } |j                  |  S c c}}w c c}}w r   )
r  r   rW   r   r
   r/  rK   rg  _match_axes_and_sumcall_wrapped)argsin_dims_rk   rf   r]   bwd_out_dims_thunkr   r   rv   r  r  r   r  rJ  s          r6   new_bwdz%batch_custom_vjp_bwd.<locals>.new_bwd  s    $W-wy7H dH-/3 Aw,& )))S!&&AB,-. /D /  h/1Aq Q</Q6 1H 1)#sIxHD.tY	9-}>D4d##/1s   AC>Cr  )r}   r   r@  r   r  r  )	r  rJ  r   r  r  r:  r   rv   r   s	   ````` @@@r6   r  r    sC     nn)nn)**)
$ 
$ 
g#..	99r8   c           	     V     | | }t        t        t        |||d       |       ||      S )NT	sum_match)r   r   r,  )r   rv   r   r   r9  r  r  r  s           r6   r4  r4    s7     [(	W()Y	#%}h
8 8r8   c           	     L   t        |t        t        f      r||k(  r|S t        |      t        |      cxk(  r	t        k(  rKn nHt        j                  |||j                        } t        |      t        j                  ||||            S |t        u r9|t        ur1 t        |      t        j                  |||j                  |            S |t        u r2|r0 t        |      t        j                  |||j                              S t        | |||f      t        | ||||||      S )Nr<  )r   r   r   rW   r{   r
   r/  rK   r   r.  re   r   )r   r   r   rb  r   rk   r=  rK   s           r6   r,  r,    s     D,'(
czh	cd3i	&3	&b#qvv.dT!WT''CyABB	
	s*4T!WT''CCDD	
	yT!WT%%b#qvv677	1c3/00YIsCiPPr8   .z"dict[core.Primitive, BatchingRule]rX  zdict[core.Primitive, Callable]rT  c                      e Zd Zd Zy)AxisPrimitiveBatchersProxyc                "    fd}|t         |<   y )Nc                F     | j                   | j                  d ||fi |S r   )r}   r   )r   rl  rm  r\  batchers       r6   wrappedz7AxisPrimitiveBatchersProxy.__setitem__.<locals>.wrapped  s#    Y^^Y^^T4PPPr8   )rT  )r5   r  rC  rD  s     ` r6   __setitem__z&AxisPrimitiveBatchersProxy.__setitem__  s    Q%,T"r8   N)r=   r>   r?   rE  rC   r8   r6   r@  r@    s    -r8   r@  rU  c                2    t        t        |       t        | <   y r   )r   vectorized_batcherrX  r  s    r6   defvectorizedrI    s    $%7>Tr8   c                n    t        fddd  D              sJ         | j                  |i |d   fS )Nc              3  .   K   | ]  }d    |k(    yw)r   NrC   )r   bd
batch_dimss     r6   r   z%vectorized_batcher.<locals>.<genexpr>  s     :RZ]b :s   rb   r   )rS  ri  )r  batched_argsrM  r\  s     ` r6   rG  rG    sA    	::ab>:	:FJF	:	L	+F	+Z]	::r8   c                2    t        t        |       t        | <   y r   )r   broadcast_batcherrX  rH  s    r6   defbroadcastingrQ    s    $%6=Tr8   c           	     z  	 t        |      dkD  sJ t        d t        ||      D              \  	t        	fdt        ||      D              r3 | j                  |i |}| j
                  r|ft        |      z  fS |fS t        ||      D cg c])  \  }}t        j                  |      rt        ||d      n|+ }}}t        d |D              }t        ||      D cg c]  \  }}t        |||       }}} | j                  |i |}| j
                  r|dt        |      z  fS |dfS c c}}w c c}}w )a[  Process a primitive with built-in broadcasting.

  Args:
    args: the possibly-batched arguments
    dims: list or tuple of the same length as `args`, where each
      entry indicates the batching state of the corresponding entry to `args`:
      either an int indicating the batch dimension, or else `not_mapped`
      indicating no batching.
  rb   c              3  N   K   | ]  \  }}|t         ur|j                  |f  y wr   )r.  rU   )r   rk   r]   s      r6   r   z$broadcast_batcher.<locals>.<genexpr>  s+      ,TQ
* WWaL ,s   #%c              3     K   | ]C  \  }}t        j                  |      r)t        j                  |j                        xr |k(   E y wr   )rp   r   r
   definitely_equal_shaperU   )r   rk   r]   rf   rU   s      r6   r   z$broadcast_batcher.<locals>.<genexpr>  sD      	4Q
 
	$	$UAGG	4	Ac	A 	4s   A	Ac              3  F   K   | ]  }t        j                  |        y wr   )rp   r   )r   rk   s     r6   r   z$broadcast_batcher.<locals>.<genexpr>  s     (arwwqz(s   !r   r   )rY   r  r   rS  ri  rZ  rp   r   bdim_at_frontmax_handle_scalar_broadcasting)
r  r6  rm  r\  r  rk   r]   r   rf   rU   s
           @@r6   rP  rP    sJ    
TQ ,St_ , ,*% 	4t_	4 4 $))T
$V
$C'+'<'<C##c("#L3*L D$)1 ')ggajM!Q"a7 )D )(4((D@CD$P1'a3PDP
$))T
$V
$C%)%:%:CC!HaH) Qs   .D1%D7c           
         ddl m} |t        u s| t        j                  |      k(  r|S |j                  |t        t        t        j                  |      |                   S )Nr   r   )r   r   r.  rp   r   expand_dimsr[   r  )ndrk   r]   r   s       r6   rZ  rZ    sG     *_bggaj(H??1eE"''!*b$9:;;r8   c                4    t        t        | |      t        | <   y r   )r   reducer_batcherrX  )r  idents     r6   
defreducerra    s    $_dEBTr8   c           
        fd}|\  |\  }t        |t              rt        t        j                  t        j
                  ||      |t        j                  |d                  } |||      }d|v rt        |j                        } | j                  fd|i||fS t        |t              r|J d       t        t        j                  t        j
                  ||j                        |t        j                  |d                  } |||j                        }g }	g }
|j                  D ]7  \  }}||v r|	j                  ||f       |
j                   |||      |f       9 t        |t        |j                  t        |	                   | j                  fd|i|}|t        j                   ||
      fS J )Nc           	         t        t        t        j                  t        j                  j
                        |             j                  |            S r   )r{   rs   rp   deletearanger   index)axesr   operands     r6   r  z!reducer_batcher.<locals>.out_axis  s6    tBIIbii5t<=CCDIJJr8   rb   input_shape)ri  rg  z5TODO Ragged batching a reduction requires an identity)r   r{   r[   rp   wherelessr   re  rU   ri  r`   rc   rd   rX   mask_ragged_axesr   r   )r  r`  rN  rM  rg  r\  r  r   bdim_outaxes_to_maskragged_axes_outrw   r1  r   rh  s                 @r6   r_  r_    s   K('
%$c"''$-tRVVD!_EFDd#HF6f499W24262H<<$
#UUU"''$(9(9:D"&&q/RSDd//0H LO(,(8(8 O$_		[/:;{ ;_MN	O
 
4#4#4eL6IJLGTYYw4T4V4F?7<<?KKK5r8   c           	     >   t        ||      D ch c]  \  }}|t        us|j                  |    c}}\  }t        ||      D cg c]  \  }}t        |||       }}} | j                  |i |}| j
                  r|dt        |      z  fS |dfS c c}}w c c}}w )zdA batching rule for primitives that support matching leading batch
  dimensions in all arguments.
  rW  r   )r   r.  rU   rX  ri  rZ  rY   )r  r6  rm  r\  rk   rL  r}   r  s           r6   expand_dims_batcherrq  1  s     %(dO
L51br7K1772;
L%$25dD/	BB-2t
$	B$	B		4"6"##'#8#8#tc#h	FsAhF M	Bs   BBBc                z    |j                   D ]+  \  }}t        |j                  ||ff      }t        | ||      } - | S r   )rd   r`   rc   _mask_one_ragged_axis)rh  r`  	axis_specrw   r1  this_axis_specs         r6   rl  rl  :  sS     '0&;&; D"k?+!? ACN#GUNCGD 
.r8   c                   ddl m} t        |j                        dk(  sJ d       |j                  d   \  }} || j                        }|j                  d| j                  |      }|j                  |d      }|j                  || j                  |j                  g      }	||	k  }
|j                  |
| |j                  || j                              S )Nr   r   rb   z#Mask just one ragged axis at a timern   )r   r   rY   rd   rq   broadcasted_iotarU   convert_element_typebroadcast_in_dimrc   select	broadcast)rh  r`  rt  r   rw   r1  r%  	positionsrG   limitsmasks              r6   rs  rs  C  s     	Y""	#q	(O*OO	(!*!6!6q!9+

%""7GMM;G) $$_g>'w}}y5568&	V	$	D'3==#F	GGr8   c                    t        || j                        }t        |t              rt	        | ||      |fS t        |t
              r*t	        | |j                  |      }||j                  |      fS t        d|       )Nz"Unrecognized batch dimension type )	r"   r   r   r{   r#   r`   rc   r   r   )rh  r   r   r   s       r6   r   r   T  sw    #w||,#cGT3',,$
#gt00#6F4))#...
8?
@@r8   c           	     J   ddl m} t        t        j                  |             }|j                  ||       t        t        j                  t        j                  t        |            |            }t        j                  |       }|j                  j                  j                  rd }t        t!        |j                  j"                  ||       }|j                  j%                  |      }	t'        j(                  |	j                        5  |j+                  | |||	      } t,        j.                  j0                  rSt        j2                         j4                  }
t        |
      dkD  rt6        |
rt        j8                  | t        |
            } | cd d d        S # 1 sw Y   y xY w)Nr   r   )r   )out_shardingrb   )r   r   rs   rp   rU   insertr[   rd  re  rY   r
   r   shardingr   emptyrB  r(   r   rZ   mesh_libuse_abstract_meshry  r	   r,  r%  get_axis_envspmd_axis_namesrr   pvary)rk   r   r   r   r   rU   broadcast_dimsx_avalrD  r  
spmd_namess              r6   r{  r{  `  s3   
rxx{
%,,tR299SZ#8$?@.==&__IV__114CD(__###2( !!(--0 	Q~HMA$$&66j	Z1	!!	JJq%
+,	 	 	s   BFF"c           
     |   |t         k(  rt        |||      }|j                  j                  |j                  dd        }t        t        j                  t        j                  dt        j                  d                  |j                  d   |      }t        ||      S 	 t        j                  |      }	||k(  r|S t        |      t        |      cxk(  r	t        k(  rn nt!        |||      S |t"        u r6|t"        ur.t%        ||t'        |t        j(                  |      dz         |      S |t"        u r|r|j+                  |      S t-        | t        j.                        s#| t        j0                  urt3        d	| d
|       t5        d d d       # t        $ r!}
t        d|dt        |       d      |
d }
~
ww xY w)Nrb   rT   rC   rn   r   zOutput from batched function z with type z is not a valid JAX typez"vmap has mapped output (axis_name=z) but out_axes is )r   rX  rK   rZ   rU   r+   r
   r1   ro   rp   rq   rJ   r   r   rW   r{   r#   r.  r{  r"   r   r   r   _TempAxisNameno_axis_namere   r   )r   r   r   rb  r   rk   r=  r/   rK   r   r   s              r6   r   r   x  s   Kab!AVV]]]-FDHHT--b"((72CDEGGAJ(D$?AaA 	CZHCyDI$$AsC  
jS
2Q-c2771:>BINN
jY55:y$"4"45***<)=OPSuUVV4t,,! 
 A
3A5Awi79 :?@AAs   !F 	F;F66F;c                      e Zd Zd Zy)r   c                .    || _         || _        || _        y r   )leaf_idxrb  r   )r5   r  rb  r   s       r6   r   zSpecMatchError.__init__  s    DMDHDHr8   Nr   rC   r8   r6   r   r     s    r8   r   c                J    |t         u rt        | |d|      S t        | |d      S )Nr   )r   )r.  r{  r#   )rk   r   r}   r   s       r6   rX  rX    s*    	ZQa955AtQr8   c                2   |\  }}| \  }}||k(  rt        ||      |fS |t        u r(t        ||j                  |   |      }t        ||      |fS |t        u r(t        ||j                  |   |      }t        ||      |fS t	        |||      }t        ||      |fS r   )r   r.  r{  rU   r#   )rN  rM  bdxbdyrk   r'  s         r6   add_batchedr    s    (#s	$!QCZq!c!!
j!QWWS\3'Aq!c!!
j!QWWS\3'Aq!c!!CAq!c!!r8   c               z    t        d |D              rt        t        j                  j                  | ||d}||fS )Nc              3  >   K   | ]  }t        |      t        u   y wr   )rW   r{   )r   r   s     r6   r   z!_pvary_batcher.<locals>.<genexpr>  s     ,tds	,r  )rg  axis_index_groups)r  rr   r
   pvary_pri  )r]  r^  rg  r  rp  s        r6   _pvary_batcherr    s<    ,t,,
\\d,=?(	7	r8   )rf   r`   r:   ztuple[int, int, int])rf   r`   r   ztuple[int, ...]r:   r`   )r   r{   rc   r{   rd   z"list[tuple[int, Array | core.Var]]r:   int | RaggedAxis)r   r  r   
core.Shaper:   r  )rc   r{   r   r  r:   r  )r   r  r   zcore.InputType | Nonerv   zcore.AxisSizer   r   r   z!Sequence[int | RaggedAxis | None]rx   zSequence[Array]r:   r  )
r   r   r   GetIdxrk   	Vmappabler   MapSpecr:   Elt)r   r   rv   AxisSizer   MeshAxisr   r{   rk   r  r   r  r:   r  )rv   r  r:   r   )r   rW   r   rW   r   rW   r   r   r   r   r   zCallable | None)r   rW   r:   None)rk   r   r:   r  )r   r   r  zlu.Storer  r   )r  r  r:   r  )r   r   )
r  r  r  tuple[int | None, ...]r  r  r  r  r   r   )r  core.ClosedJaxprr|  (tuple[int | NotMapped | RaggedAxis, ...]r:   zAtuple[core.ClosedJaxpr, tuple[int | NotMapped | RaggedAxis, ...]])r  r  r|  r  r:   z4tuple[core.ClosedJaxpr, tuple[int | NotMapped, ...]])r   zlist[core.AbstractValue]rf   r`   rK   zcore.ShapedArrayr:   ztuple[int, core.ShapedArray])r  r  r   r?  r|  Sequence[int]r  r  )r  r  rJ  zcore.TraceTagr   r?  r  z"Callable[[], Sequence[int | None]]r  zSequence[int | None]r:   r  )F)rh  r   rt  r`   r:   r   r   )
__future__r   r  collections.abcr   r   rA   	functoolsr   typingr   r   numpyrp   jax._srcr	   r
   r   r   r   jax._src.partition_specr   rB  jax._src.sharding_implsr   r   r  jax._src.ad_utilr   r   r   r   jax._src.corer   r   r   r   jax._src.interpretersr   r   jax._src.tree_utilr   r   r   r   jax._src.typingr   jax._src.utilr   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r   
unsafe_mapr   r  	dataclassr+   rE   rJ   rO   r   r_   ri   rl   ry   r`   r   r   r   r   r   r   r  r  r  r  r  r  ToEltHandlerFromEltHandlerMakeIotaHandlerr   r   r@   r   r   r   r   r   r   r   r  r  transformation_with_aux2r  rs   r[   RaggedMaskingRuler  r  r  r  r   rW   r&  r.  r   r?  rE  r   r  r  r  r  r  rg  rf  r  r  rj  r  r  r  r  r  r  r  r  r  r  r  r(  r*  r  r  r  r4  r,  r{   BatchingRulerX  rT  r@  axis_primitive_batchersrU  rI  rG  rQ  rP  rZ  ra  r_  rq  rl  rs  r   r{  r   	Exceptionr   rX  r  r  r  mutable_array_prC   r8   r6   <module>r     s   #  .       % & 6 1 % K K ; ; 4A A !L L L L
 CZCZ d#    $  d#    $  d#  $
  l&; V_.? @	8 d#% % $%2@R	1
	1	1 4	1 		1	
	(2	7A	EE#-E2BE+2+2!6+2+2)1+2 8+2 "	+2
 +2^ 		
	"f*	69g>CD8XsG<iGHH:u,-7: -/) .;;;#*;/8;& 13 - 2- 35 / 4?'?3;?"1? -/
) .#
I $	4  (3<  	#YS	49%uT#YS	-A'BB  >@ : ?-11- J	
9& 9v d#  $5GN GNV-%2-    $M.M/M  M 	MB $ $ "
1&
@"
@ 6
@ I	
@ 9"9 69 <	9 96$($-I$HKZ <!)<,<=J< <     " # #   $ $,:$,:"D: )=: BO:( 8 8Q* 	#uS$eCI&6&; <<=
=>? ;= 7 <;= 8 =- - 56  79 3 8?;>I:<C>GHH&0H5:H"A0-6Y  " %0 = ! $2 4<<   d"" #r8   