
    vhD             
      
   U d dl mZ d dlmZmZmZmZ d dlZd dlm	Z	 d dl
Z
d dlmZ d dlZd dlmZmZmZ d dlZd dlZd dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlm Z  d dlm!Z! d dlm"Z" d dlm#Z$ d dlm%Z% d dlm&Z& d dlm'Z' d dlm(Z( d dl)m*Z* d dl)m+Z+m,Z, d dl-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 d dl4m5Z5m6Z6 d dl7m8Z8 d dl9m:Z:m;Z; d dl<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZG d dlHmIZI d dlHmJZJ d dlHmKZL d dlHmMZM d dlHmNZN d d lOmPZPmQZQmRZRmSZSmTZTmUZU d d!lVmWZWmXZXmYZYmZZZm[Z[ d d"l\m]Z]m^Z^ eZ_e(j                  eacZaZbe(j                  edcZdZe e'j                  eg       eZheZid ej       ddd#d$	 	 	 	 	 dd%Zkd& Zl G d' d(      Zm em       Znd) Zoend*dd+Zpd,d,d-	 	 	 	 	 	 	 	 	 	 	 	 	 dd.Zqd/ Zrdd0Zs ej                  d1d2d3g      Zudd4Zv G d5 d6      Zw ew       Zx	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd7Zy	 	 	 	 dd8Zz	 	 	 	 	 	 	 	 	 	 	 	 dd9Z{	 	 	 	 	 	 	 	 	 	 	 	 dd:Z|	 	 	 	 	 	 dd;Z}dd<Z~d= Z ed>      Z	 	 dd?Ze$j                  d@        ZeZeee+f   Z G dA dBej                        Z edC      Z e(j                  dDd#E      ddF       ZdG Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddHZeeLj                  _        ddIZddJZ	 	 	 	 ddKZ	 	 	 	 ddLZ	 	 	 	 ddMZeej(                  ej*                  <   	 	 ddNZeej.                  ej*                  <   dO Zeej2                  e<   ddPZ	 	 	 	 ddQZ	 	 	 	 ddRZdS ZdT ZdU Z eJj@                  ee       dV ZddWZddXZddYZdZ Zd[ Zd\ Zd] Zd^ ZeejT                  _        d_ Z	 	 dd`Zda ZddbZ G dc dde      Zde Z G df dge      Z	 	 	 	 	 	 ddhZdi Zdj Zdk Zdl Z G dm dnejp                        Z G do dpejV                        Zdq Zi Zdreds<   	 	 	 	 	 	 ddtZeee j~                  <   du Zeee!j                  <   dv Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddwZeeIj                  _        dx Zdy ZeeNj                  _        	 	 ddzZeeLj                  _        dd{ZeeNj                  _        e$j                  d|        Ze$j                  dd}       Zdd~Z	 	 ddZ	 	 ddZeeNj                  e<   	 	 	 	 	 	 	 	 	 	 ddZeeLj                  e<   	 	 	 	 	 	 ddZd Z	 	 	 	 	 	 ddZ	 	 	 	 	 	 ddZ	 	 ddZeeLj                  e<   dd d ddddddddZe$j                  d        Ze$j                  d        Zd Zd Zy)    )annotations)CallableHashableSequenceSetN)partial)prod)AnyTypeVarUnion)NamedShardingPartitionSpec)ad_util)api_util)config)core)	debugging)dispatch)dtypes)linear_util)sharding_impls)source_info_util)traceback_util)util)pvary)Tracertypeof)AbstractMeshMeshAxisTypeuse_abstract_meshget_abstract_meshget_concrete_mesh)_shared_code_pmap_prepare_pmap)ir)hlosdy)
HashableFunctionHashablePartialunzip2as_hashable_functionmemoizepartition_listmerge_lists
split_list
subs_list2fun_name)batching)mlir)partial_eval)pxla)ad)tree_maptree_flattentree_unflattentree_structuretree_leaveskeystr)broadcast_prefixprefix_errors	PyTreeDefgenerate_key_pathsKeyPath) host_local_array_to_global_array global_array_to_host_local_arrayT)
axis_namesin_specsmesh	check_vmac              J    t        |||||      | fdS t        | fi S )a
  Map a function over shards of data using a mesh of devices.

  See the docs at https://docs.jax.dev/en/latest/notebooks/shard_map.html.

  Args:
    f: callable to be mapped. Each application of ``f``, or "instance" of ``f``,
      takes as input a shard of the mapped-over arguments and produces a shard
      of the output.
    mesh: (optional, default None) a ``jax.sharding.Mesh`` representing the
      array of devices over which to shard the data and on which to execute
      instances of ``f``. The names of the ``Mesh`` can be used in collective
      communication operations in ``f``. If mesh is None, it will be inferred
      from the context which can be set via `jax.sharding.use_mesh` context
      manager.
    in_specs: (optional, default None) a pytree with
      ``jax.sharding.PartitionSpec`` instances as leaves, with a tree structure
      that is a tree prefix of the args tuple to be mapped over. Similar to
      ``jax.sharding.NamedSharding``, each ``PartitionSpec`` represents how the
      corresponding argument (or subtree of arguments) should be sharded along
      the named axes of ``mesh``. In each ``PartitionSpec``, mentioning a
      ``mesh`` axis name at a position expresses sharding the corresponding
      argument array axis along that positional axis; not mentioning an axis
      name expresses replication. If ``None``, all mesh axes must be of type
      `Explicit`, in which case the in_specs are inferred from the argument types.
    out_specs: a pytree with ``PartitionSpec`` instances as leaves, with a tree
      structure that is a tree prefix of the output of ``f``. Each
      ``PartitionSpec`` represents how the corresponding output shards should be
      concatenated. In each ``PartitionSpec``, mentioning a ``mesh`` axis name
      at a position expresses concatenation of that mesh axis's shards along the
      corresponding positional axis; not mentioning a ``mesh`` axis name
      expresses a promise that the output values are equal along that mesh axis,
      and that rather than concatenating only a single value should be produced.
    axis_names: (optional, default set()) set of axis names from ``mesh`` over
      which the function ``f`` is manual. If empty, ``f``, is manual
      over all mesh axes.
    check_vma: (optional) boolean (default True) representing whether to enable
      additional validity checks and automatic differentiation optimizations.
      The validity checks concern whether any mesh axis names not mentioned in
      ``out_specs`` are consistent with how the outputs of ``f`` are replicated.

  Returns:
    A callable representing a mapped version of ``f``, which accepts positional
    arguments corresponding to those of ``f`` and produces output corresponding
    to that of ``f``.
  )rG   rF   	out_specsrE   rH   c                    t        | fi S N)
_shard_map)gkwargss    M/opt/face_recognition/venv/lib/python3.12/site-packages/jax/_src/shard_map.py<lambda>zshard_map.<locals>.<lambda>   s    Z,V,     )dictrM   )frJ   rE   rF   rG   rH   rO   s         @rP   	shard_maprU   L   s6    ` TH	%<&Y,,	A	 	  rR   c                :    |
t               S t        d g|z  | gz    S rL   )P)	axis_nameaxiss     rP   _axes_to_pspecrZ      s&    	\3J	
TFTMYK'	((rR   c                      e Zd Zd Zd Zy)InferFromArgsc                     y)Nzjax.sharding.Infer selfs    rP   __repr__zInferFromArgs.__repr__   s    rR   c                    t         dfS Nr^   )_get_default_inferr_   s    rP   
__reduce__zInferFromArgs.__reduce__   s    ##rR   N)__name__
__module____qualname__ra   re   r^   rR   rP   r\   r\      s     $rR   r\   c                     t         S rL   )Inferr^   rR   rP   rd   rd      s    	,rR   )in_axesc         	        t        |t        t        f      rt        d|       |-|t        ur%t        |t
        t        f      st        d| d      |t        ur*t        d t        |      D              st        d| d      t        d t        |      D              st        d| d      |t        u rd nt        t        t        |      |d 	      }t        t        t        |      |d
 	      }t        | d |||hdd      S )Nz:smap axis_name should be a `str` or a `Hashable`, but got zsmap in_axes must be an int, None, jax.sharding.Infer, or a tuple of entries corresponding to the positional arguments passed to the function, but got .c              3  <   K   | ]  }t        |t                y wrL   
isinstanceint.0ls     rP   	<genexpr>zsmap.<locals>.<genexpr>   s     ?QjC ?   zqsmap in_axes must be an int, None, jax.sharding.Infer, or (nested) container with those types as leaves, but got c              3  <   K   | ]  }t        |t                y wrL   ro   rr   s     rP   ru   zsmap.<locals>.<genexpr>   s     ?AZ3?rv   z^smap out_axes must be an int, None, or (nested) container with those types as leaves, but got c                
    | d u S rL   r^   xs    rP   rQ   zsmap.<locals>.<lambda>   
    d rR   is_leafc                
    | d u S rL   r^   ry   s    rP   rQ   zsmap.<locals>.<lambda>   r{   rR   T)rG   rF   rJ   rE   rH   _smap)rp   listtuple	TypeErrorrj   rq   allr<   r8   r   rZ   rM   )rT   rk   out_axesrX   rF   rJ   s         rP   smapr      s7   	D%=)

DYKPR RgU2
WsEl
+
	%Ya	)* * U
?+g*>?
?
	::A!	EF F 
?X)>?	?
 ;;C*AG H H &dw~y9724  w~y9824)	AD8y ){d$
H HrR   F)_skip_mesh_checkr   c          
          t               st        d  dt                d      t        j                         t
        j                   fd              }|S )Nz>shard_map requires a callable for its first argument, but got 	 of type rm   c            	        t              \  t        j                  t        j                  d| i             }t        |       \  }}t        j                  ||      \  }	 t        | d       }rt        fdD              r^|D cg c]  }t        |      j                   }}t        d |D              sJ |       |D 	cg c]  }	t        |	j                         }}	t        d t!        |      D              \  }
}t        j"                  ||
|d	      \  }}t%        ||
||       t&        fd
       }rt)        ||      }	 t+        j,                  |g|||d}tA                      S # t        $ r t        |       ^}} |d      d w xY wc c}w c c}	w # t.        $ rh}|j0                  \  }t3              sEt5        t6        j8                          |      }t;        d |D              r|dz  }t        |      d Y d }~d }~wt<        $ rC}|j0                  \  }t3              s t?                |      }t        |      d Y d }~d }~ww xY w)NrU   
debug_infoc                
    | d u S rL   r^   ry   s    rP   rQ   z-_shard_map.<locals>.wrapped.<locals>.<lambda>   s
    AI rR   r|   zshard_map in_specsc              3  ^   K   | ]$  }j                   |   t        j                  k(   & y wrL   _name_to_typer    Explicitrs   arG   s     rP   ru   z._shard_map.<locals>.wrapped.<locals>.<genexpr>   s'     K1Dq!X%6%66K   *-c              3  $   K   | ]  }|d u  
 y wrL   r^   rs   is     rP   ru   z._shard_map.<locals>.wrapped.<locals>.<genexpr>   s     2qd2s   c              3  .   K   | ]  \  }}|||f  y wrL   r^   )rs   r   ss      rP   ru   z._shard_map.<locals>.wrapped.<locals>.<genexpr>   s$      (:41a+,= *+A (:s   Fc                 4   t              r#        } t        t        j                  |        n} t	                t               g        j                  z        }	 t        | |      }t        |      S # t        $ r t        | |      ^}} |d      d w xY w)Nzshard_map out_specs)callable_check_specsSpecErrorTypeoutr:   object
num_leavesr>   
ValueErrorr?   r   )
out_specs_dummyout_specs_flate_rE   rJ   out_trees        rP   out_specs_thunkz4_shard_map.<locals>.wrapped.<locals>.out_specs_thunk   s    	)	[
]&&
J?
XZ&(hj6K6K)KLe1)*e< >""  1j%0A%&D01s   A6 6!BrG   rF   r   rH   manual_axesc              3  J   K   | ]  }|t         uxr |j                     y wrL   )no_failshape)rs   fails     rP   ru   z._shard_map.<locals>.wrapped.<locals>.<genexpr>   s#     H$t7"54::~5Hs   !#z In particular, for rank 0 outputs which are not constant over the mesh, add at least one (singleton) axis to them so that they can be concatenated using out_specs.)!_shmap_checkslu	wrap_initr   r   r9   flatten_fun_nokwargsr>   r   r?   r   r   sharding_manual_specspecr+   	enumerateargnums_partial_check_specs_vs_argsr-   _implicit_pvary_on_outputshard_map_pbind
_SpecErrorargsr   _spec_rank_errorr   r   any	_RepError_inout_vma_errorr:   )r   fun	args_flatin_treein_specs_flatr   r   r   arg_sr   dyn_argnumsr   out_flatfailsmsgr   r   r   rE   rH   rT   rF   rG   rJ   s                  @rP   wrappedz_shard_map.<locals>.wrapped   s    %T:x%5u>D*
,,	h))+q$CEC%d+Iw11#w?MC.&
D"57m 	K
KK+45avay!!5e52M22AMA2AFGA|J7GmG!' (:9];S (: ":K--c;	5QNCD'8[-"$ # # %c?;c(!!
"" $})Y "h$ (*h//k  .Hd+ea!"#-. 6G>  (vvfei }00!XZERH%HH
 C D# o4' !  (vvfei q$
IuEo4' !(s=   +F F)F.F3 !F&3	I.<AHI.+9I))I.)r   r   typer   wrapsr   api_boundary)	rT   rG   rF   rJ   rE   rH   r   r   r   s	   ```````` rP   rM   rM      sn     
!
  c47)16 7 7 ::a=A0 A0  A0D 
.rR   c                     !t                 j                  rCt        d      t               }|s,|j                  s  j                  |k7  rt        d| d        t	         t
        t        f      st        d  dt                d      t	        |t        t        f      st        dt        |             t	        |t              rt        |      }|st         j                        }|j                   j                        st        d| d	 j                         |Vt         fd
|D              sBdj                   fd|D              }|rd| d| }t        |      d|d| }t        |      |t        t         j"                  ||       t%        |      st        t         j&                  ||        |fS )Nz`The context mesh cannot be empty. Use `jax.sharding.use_mesh(mesh)` to enter into a mesh contextzThe context mesh z+ should match the mesh passed to shard_map ztshard_map requires a `jax.sharding.Mesh` or a `jax.sharding.AbstractMesh` instance for its second argument, but got r   rm   zU`axis_names` argument of shard_map should be of type `frozenset` or `set`. Got type: z"jax.shard_map requires axis_names=z# to be a subset of mesh.axis_names=c              3  ^   K   | ]$  }j                   |   t        j                  k(   & y wrL   r   r   s     rP   ru   z _shmap_checks.<locals>.<genexpr>$  s'     MQd  #x'8'88Mr   , c              3  N   K   | ]  }t        j                  |           y wrL   )strr   r   s     rP   ru   z _shmap_checks.<locals>.<genexpr>%  s!     J!3t11!45Js   "%z)in_axes was not specified when axis_name=z was of type zzshard_map in_specs argument must be a pytree of `jax.sharding.PartitionSpec` instances, but it was `None` when axis_names=z are of type )r"   emptyr   abstract_meshrp   r   r   r   r   	frozensetsetrE   issubsetr   joinr   r   inputr   r   )	rG   rE   rF   rJ   r   r   ctx_mesh
axis_typesr   s	   `        rP   r   r     s   	\DzzHI I !"HX^^h&hZ (v    
D4.	/
 004vYtDzl!M N N 
JC 0	1
	!*-.	01 1 
C :&J	4??+J			T__	-

,ZL 9??+	-. . 
M*M
MJzJJJ8 E \#c C.]:,8c C.$$h
;	)	""Iz:	z	rR   c                   g }|D ]  }||j                  |       t        |t              rq|D cg c]
  }|| v r|nd  }}|r|d   |j                          |r|d   d |v rt	        d|       |j                  t        |      dk(  rd n
t        |             |j                  || v r|nd         t        | S c c}w )NzInvalid spec: r   )appendrp   r   popr   lenrW   )r   r   r   r   ptemps         rP   r   r   5  s    
# 2ay	jjm	Au	789!1#a-9d9T"X%
 T"X%	>$011	jjTaU4[9	jja;&D12 
C. :s   B>r   r   r   c                  
 | t         j                  k(  r|t        d      fd
t        
fdt	        |      D              ry | t         j                  k(  rdnd}t        |      D cg c]@  \  }}t        |t              s+d| dt        |       d| d	t        |      j                   d
	B }}}|st        |      D ]\  \  }}|D ]R  }t        |t              s|fn|}|D ]6  }	|	|	vs|j                  d| dt        |       dt        |	              8 T ^ t        d| d ddj                  |      z   d| dz         t        d| ddj                  |      z   d| dz         c c}}w )Nzshard_map in_specs argument must be a pytree of `jax.sharding.PartitionSpec` instances, but it was None.
Instead of `in_specs=None`, did you mean `in_specs=P()`, where `P = jax.sharding.PartitionSpec`?c                    t        | t              sy| D ](  }t        |t              s|fn|}|D ]  }||vs  y * y)NFT)rp   r   r   )r   namesnamer   s      rP   
check_specz _check_specs.<locals>.check_specR  sU    a' (6uhEe $K 7
 rR   c              3  .   K   | ]  } |        y wrL   r^   )rs   r   r   s     rP   ru   z_check_specs.<locals>.<genexpr>\  s     31A3s   inr   z  _specs is r   r   z refers to z
shard_map z8_specs argument must refer to an axis marked as manual (z	), but:



z

Check the z"_specs values passed to shard_map.zR_specs argument must be a pytree of `jax.sharding.PartitionSpec` instances, but:

)r   r   r   r   r<   rA   rp   rW   r=   r   rf   r   r   reprr   r   )
error_typespecsr   prefixkeyrz   msgsr   r   r   r   s     `       @rP   r   r   J  s   =&&&5=
	23 3 	3E 233
!4!444%&*51
Kc1Aq9I vhfVC[MaS	$q':J:J9K2
N 
K$ 
K	$U+ QQ Q% *5% 8e 	QD$k"9KK"VHF6#;-{4:,OP	QQQ 
VH (M	6
++d
	H>@	@A A
 	6( 9 :D(<>>	? ?
Ks   +AE2c                      e Zd Zy)NoFailNrf   rg   rh   r^   rR   rP   r   r   s  s    rR   r   c           	       	 t        t        j                  |      }t        ||      D 	cg c]%  \  }}	t	        |      |	j
                  k  s|	nt        ' }
}}	t        d |
D              r5t        |||
      }
t        t        j                  | |||
      }t        |      t        t        t        |            }t        ||      D 		cg c]0  \  	}t        	fd|j                         D              r	nt        2 }
}	}t        d |
D              r't        |||
      }
t!        | |||
      }t        |      y c c}	}w c c}}	w )Nc              3  ,   K   | ]  }|t         u  y wrL   r   rs   rT   s     rP   ru   z'_check_specs_vs_args.<locals>.<genexpr>}       (a'	(   c              3  l   K   | ]+  \  }}j                   |   t        fd |D              z   - yw)c              3  <   K   | ]  }j                   |     y wrL   r   rs   nrG   s     rP   ru   z1_check_specs_vs_args.<locals>.<genexpr>.<genexpr>  s     $?qTZZ]$?   N)r   r	   )rs   dnsr   rG   s      rP   ru   z'_check_specs_vs_args.<locals>.<genexpr>  s5      /q" 771:$?B$? ?? /s   14c              3  ,   K   | ]  }|t         u  y wrL   r   r   s     rP   ru   z'_check_specs_vs_args.<locals>.<genexpr>  r   r   )mapr   shaped_abstractifyzipr   ndimr   r   _expand_failr   r   r   r   r   _spec_to_namesitems_spec_divisibility_error)rT   rG   r   rF   r   r   xsin_avalsr   r   r   r   in_names_flatr   s    `       `    rP   r   r   v  s<    (("-(-2
4a q6QVV#!
0 
4$ 
4(4((d3D
=..7Hd
KC
S/NM:;- h6
8 
8a  / %/ /!4;< 
8$ 
8 	(4((d3D
"1dGXt
DC
S/ )
4
8s   *D;	5Ec                `    t         g| j                  z  }t        ||      D ]
  \  }}|||<    |S rL   )r   r   r  )r   r   r   fail_r   rT   s         rP   r	  r	    s>     -49w7I7I+I%+t$ daE!H	,rR   c                   t        |      }| t        j                  k(  rd\  }}t        ||      }nd| d}}g }	t	        |||      D ]  \  \  }
}\  }}d}| t        j                  k(  r	|^}}|j
                  t        |j                        k  r?t        |j                  j                               |j
                     }d| | d| d| d}nt        |j                  j                  j                               d	   }|j                  t        j                  j                   k(  sJ d| | d
|j
                  t        |j                  j                        z
  dz    d| d|j"                   d
}|	j%                  d| dt'        |
       d| dt        |       d| t'        |       | d|j)                          d|j*                   d|j*                   dt        |       d        |	sJ t        |	      dk(  r	|	d   dd  g}	d| d| d| ddj-                  |	      z   dz   d| d | d!| d"| d#	z   }t/        d$ t	        |||      D              r	|d%| d&z  }|S )'N)r   r   r   z(*args) z, where  is bound to 's parameter '',r    is the index     component of 's varargs parameter 'z* r   r   z which has length z, but z has shape z, which has rank z (and z < )r      #shard_map applied to the function 'z' was given an zG_specs entry which is too long to be compatible with the corresponding zput value from the function:

r   zEntries in zQ_specs must be of length no greater than the number of axes in the corresponding z=put value.

Either revise the spec to be shorter, or modify 'z' so that its zputs have sufficient rank.c              3  B   K   | ]  \  }\  }}|j                      y wrL   )r  )rs   r   avals      rP   ru   z#_spec_rank_error.<locals>.<genexpr>  s     K<1iq$TYYKs   z0

For scalar values (rank 0), consider using an z>_specs entry of `P()`, where `P = jax.sharding.PartitionSpec`.)util_fun_namer   r   _try_infer_args_iter_pathsidxr   	argumentsr   keys	signature
parametersvalueskindinspect	ParameterVAR_POSITIONALr   r   r=   	str_shortr  r   r   )r   rT   treer   r   r2   r   basebar   spec_keyr   fail_keyr  extraarg_keyr   
param_nameparamr   s                       rP   r   r     s    1(=&&&LFD	D	!BXJg.DF	$,7eU,K G(x(4E](((R^kg	s2<<(	(",,++-.w{{;
D6'-z B)l". R\\,,3356r:zzW..=====D6'.KK#bll&=&=">>BC Dj 6uzzl"F 	KK
VHF6(+,D6Ht9+V&!"5'T^^5E4F G))F499+ST1	FGG& 
+Y!^T!WQR[MT.xj8  !A
C ;;t
  &
& vh 006x 8==EJ GH6
8	
8# 	K;tUE+JKK@ IF F GC	*rR   c                d   t        | |      }t        | dt        |             }g }t        |||      D ]  \  \  }}	\  }
}d}||
^}}|j                  t        |j                        k  r=t        |j                  j                               |j                     }d| d| d| d}nt        |j                  j                  j                               d   }|j                  t        j                  j                  k(  sJ d| d|j                  t        |j                  j                        z
  d	z    d
| d|j                    d	}t#        |	      }|j%                         D ]  \  }}|j&                  |   t)        fd|D              z  s+t        |      d	kD  rd| nd|d    d}t        |      d	kD  rdnd}t)        fd|D              }|j+                  dt-        |
       d|j/                          | dt-        |       d|	 d| d|j&                  |    d| d| d| d| d|j&                  |             |sJ t        |      d	k(  r	|d   dd  g}d| d t1        j&                  j                                d!j2                   d"d#j5                  |      z   d#z   d$| d%z   }|S )&Nrf   r  z, where argsr  r  r  r   r  r  r  r  c              3  <   K   | ]  }j                   |     y wrL   r   r   s     rP   ru   z+_spec_divisibility_error.<locals>.<genexpr>  s     8djjm8r  zaxes zaxis 'r   'ztotal c              3  <   K   | ]  }j                   |     y wrL   r   r   s     rP   ru   z+_spec_divisibility_error.<locals>.<genexpr>  s     ,A$**Q-,r  z* argsz
 of shape z corresponds to in_specsz
 of value z, which maps array axis z
 (of size z
) to mesh z (of zsize z), but z does not evenly divide r  r  z' was given argument arrays with axis sizes that are not evenly divisible by the corresponding mesh axis sizes:

The mesh given has shape  with corresponding axis names .

r   a  Array arguments' axis sizes must be evenly divisible by the mesh axis or axes indicated by the corresponding elements of the argument's in_specs entry. Consider checking that in_specs are correct, and if so consider changing the mesh axis sizes or else padding the input and adapting 'z' appropriately.)r!  getattrr   r"  r#  r   r$  r   r%  r&  r'  r(  r)  r*  r+  r,  r   r
  r  r   r	   r   r=   r-  r   rE   r   )rT   rG   r.  r   r   r0  r2   r   r1  r   r2  r  r3  r4  r   r5  r6  r   r  r  rY   totalszr   s    `                      rP   r  r    s    q$"Q
CF+(	$,7eU,K  (x(4E	~kg	s2<<(	(",,++-.w{{;
yhZ @)l". R\\,,3356r:zzW..=====yKK#bll&=&=">>BC Dj 6uzzl"F 4 E 
 2	A8R88	8"2w{rd|&Aq0AGaKR,,,VH%&j1A0B5' J&&,X&6%7z$ H%%&Cz$**Q-
fE%bT4Lzz!}o		 
  4 
+Y!^T!WQR[MT.xj 9% &+4::+<+<+>%?$@ A%%)__$5U	
<
 ;;t

  &
&, -5:5E	
G

G# 
*rR   c                V   t        | dt        |             }g }t        |||      D ]  \  \  }}\  }	}
t        ||      }t	        |      dkD  rdj                  t        t        t        |t        |                        }dj                  t        t        t        ||
                  }dj                  t        t        t        ||D cg c]	  }||
v s| c}                  }|j                  dt        |       d| d| d| d| d	       |\  }|j                  dt        |       d| d
| d        |sJ t	        |      dk(  r	|d   dd  g}d| dt        |j                  j                                d|j                   ddj                  |      z   dz   dz   }|S c c}w )Nrf   r  ,z* out_specsr   zU which implies that the corresponding output value is only varying across mesh axes {z} and not {z4}, but it was inferred to be possibly varying over {}zS which implies that the corresponding output value is replicated across mesh axis 'z0', but could not infer replication over any axesr   r  r  z}' was given out_specs which require replication which can't be statically inferred given the mesh:

The mesh given has shape r;  r<  r   zCheck if these output values are meant to be replicated over those mesh axes. If not, consider revising the corresponding out_specs entries. If so, consider disabling the check by passing the check_vma=False argument to `jax.shard_map`.)r=  r   r"  _unmentionedr   r   r  order_wrt_mesh_spec_to_vmar   r=   r   r   r(  rE   )rT   rG   r.  r   r   r2   r   r1  r   r2  vmaunmentionedneed_vmagot_vmar   diff	need_rep_r   s                     rP   r   r     s   Q
CF+(	$+6tUE+J K'x3tT*K
;!#c>$T8J#KLMhS.s";<=gXXc#~
K4q184 6 7 8d
kkx()dV 4jdV ,%%,IR12 ji
kkx()dV 4[HJKK$ 
+Y!^T!WQR[MT.xj 9% &+4::+<+<+>%?$@ A%%)__$5U	
<
 ;;t

  &
&
8	
8# 
*1 5s   	F&F&c                `    t        |      }| j                  D cg c]	  }||vs| c}S c c}w rL   )rE  rE   )rG   r   vma_setr   s       rP   rC  rC    s+    '__	9(8!	99	9s   	++c                    t        |dg|j                  z        }	  t        j                  |       j                  | S # t
        t        f$ r Y y w xY wNF)r:   r   r*  r&  r   r   r   )rT   r.  
dummy_argss      rP   r!  r!    sS    dUGdoo$=>*$7Q$$j11
Z	  s   !> AATc                    t        | |      }t        |      }t        t        |      t        |            }d }t        |||      }t	        ||      D 	
cg c]  \  }\  }	}
||
t
        ur||	|
ff c}
}	}S c c}
}	}w )Nc                |    | d u xs7 t        |       t        u xr$ t        |       dk(  xr t        | d         t        u S )Nr  r  )r   r   r   rW   ry   s    rP   rQ   z_iter_paths.<locals>.<lambda>#  s7    19TQ5 0 TSVq[ TT!A$ZST_ rR   r|   )r:   rA   r;   r>   r  r   )r.  r   r   failuresfailures_augspecs_leaf	specs_augr   r2  	fail_datas              rP   r"  r"    s    D%((#H-,./1CE1JK&	T$vx>)L)
9 
9)A,AXy]y7 x#
$ 
9 9 
9s   A9c                     | |i |}t        | |             D cg c]9  \  }}t        |t        t        |      t	        |      j
                  z
              ; c}}S c c}}w rL   )r  r   r   rE  r   rF  )rT   r   r   rO   r   osps          rP   r   r   +  s]    (8_%67
9a 5b)F1IMM9:
; 
9 9 
9s   >Ac                  "    e Zd ZdZd Zd Zd Zy)ShardMapPrimitiveTc                &     | j                   |i |S rL   )
_true_bind)r`   r   paramss      rP   r   zShardMapPrimitive.bind7  s    4??D+F++rR   c                <    |^}} |j                   t        ||fi |S rL   )process_shard_mapr   )r`   tracefun_and_argsra  r   r   s         rP   bind_with_tracez!ShardMapPrimitive.bind_with_trace:  s'    JC$"5"";TDVDDrR   c                   t        |      }|j                  d      }t        j                  t        j                  t
        j                  |j                        |d      }|j                  d      t        fd      |d<   |g|fS )Njaxprr   r^   rJ   c                      S rL   r^   )axess   rP   rQ   z3ShardMapPrimitive.get_bind_params.<locals>.<lambda>F  s    T rR   closurer   )	rS   r   r   hashable_partialr   r   
eval_jaxprr   r)   )r`   ra  
new_paramsrh  subfunrj  s        @rP   get_bind_paramsz!ShardMapPrimitive.get_bind_params?  sy    fJ"w/E  doo9>9I9I"K!&,F >>+&D$4\4$PJ !8ZrR   N)rf   rg   rh   multiple_resultsr   rf  rq  r^   rR   rP   r^  r^  4  s    ,E
 rR   r^  rU      )max_sizetrace_context_in_keyc                   t        | j                        |z
  }t               }|j                  r| }t               t               }}|D ]{  }|j                  |   t
        j                  k(  r|j                  |       5|j                  |   t
        j                  k(  sJ ||j                  |   f       |j                  |       } g }| j                  D ]o  }||v r |j                  t
        j                         '||v r |j                  t
        j                         K||v sJ |j                  t
        j                         q t        | j                  | j                  t        |            S )N)r   )r   rE   r"   r   r   r    Autoaddr   r   Manualr   
axis_sizesr   )	rG   r   
not_manualcur_meshexplicit_axes	auto_axesr   new_axis_typesr   s	            rP   _as_manual_meshr  M  sC   4??#k1* (^^H UCE- aa HMM1mmA##A&(*;*;; (
X##A&>( (; .?? /aKHOO,	
iHMM*-H--./ 
doot!&~!6
8 8rR   c                    t        j                  | j                  j                         D cg c]  \  }}||v r||f c}}      S c c}}w rL   )r   extend_axis_env_ndr   r  )rG   r   kvs       rP   _extend_axis_envr  i  sK    		 	 TZZ5E5E5G "7TQ%&+%5 $%a& "7 
8 8 "7s   A
c                  t        j                         }	t        | j                  |	      }
t	        |
|      }t        ||t        |j                        z        }|D cg c]  }|j                   }}t	        t        t        |||      ||      }t        ||      5  t        |      5  t        j                  |      5  t        j                  ||      \  }}}\   d d d        d d d        d d d        t!         |              |r?j"                  D cg c]  }|j                  j$                   }}t'        | |       |       t	        t(        |      }t+         |       |      D cg c]  \  }}t)        t-        ||||             }}}|D cg c]  }t        j.                  | ||	       }}t	        | j0                  |      }t	        | j0                  t	        |
            }t	        | j2                  |      }t5               ft7        |      z  t9        |      z   }t        ||      5  t        |      5  t        j                  |      5  t        j:                        }d d d        d d d        d d d        t=        ||t9         |             ||      }t?        j@                  |jB                  |jD                        }t        jF                  g |||||||	      }| jH                  jK                  |       |S c c}w # 1 sw Y   ,xY w# 1 sw Y   1xY w# 1 sw Y   6xY wc c}w c c}}w c c}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)Nsource_inforG   rF   rJ   rh  rH   r   )&r   currentr   to_jaxpr_tracerr  r  r   r   r  _shard_avalr  r!   r   
_check_vmapetrace_to_jaxpr_dynamic_check_namesoutvarsrF  _check_vmas_check_shapedarrayr  _unshard_avalDynamicJaxprTracergetvarmakevarrW   r   r   convert_constvars_jaxprrS   r   filter_named_axis_effectseffectsrE   new_jaxpr_eqnframeadd_eqn) rd  primrT   
in_tracersrG   rF   r   rH   r   r  r  
inner_meshtr  	in_avals_rh  
out_avals_constsr  out_vma	out_avalsr   r  r   out_tracersinvars	constvarsr  in_specs_stagedra  effseqns                                    rP   _shard_map_stagingr  m  s   
 !((*+E11{K/?J/*t[3t7G7G3H%HI*()aff)()'+t[)Dh){+ L->z-J L)$L$&$=$=a$K!E:vrL L L  *-#(==1aqvvzz1G1o'1$j1)!$_%6	!BD4 "-it"LM D) DGPQ!&&ua=Q+Qu||Z(&%,,OV <=){+'SFS[(5?:/{+ .->z-J .)$.&&u-E. . . TO 12%#>& 
	'	't	G$
.9.v.v{	,#++c	9 *L L L L L L
 2DQ
. . . . . .s   L L(L>LL#LL,)!L1L7,M8ML<$M,ML
LL	LL)<MMM	MMc           	     v    t        |       D ci c]  \  }}||t        |t              r|n|f c}}S c c}}w rL   )r   rp   r   )r   r   r   s      rP   r
  r
    sE    #D/
@aU-> j.UUH
< 
@ @ 
@s   "5c                >    t        | t        j                        sJ | S rL   )rp   r   ShapedArray)r  s    rP   r  r    s    	D$**	++	+	+rR   c                    t        |      t        j                  v r%t        j                  t        |         | ||||      S t        dt        |             NzUnsupported aval type: )r   r   shard_aval_handlersNotImplementedError)rG   r   rH   r   r  s        rP   r  r    sR    	$Z4+++##DJ/k904d< <5d4j\BCCrR   c                    t        |      t        j                  v r$t        j                  t        |         | |||      S t        dt        |             r  )r   r   unshard_aval_handlersr  )rG   rH   r   r  s       rP   r  r    sK    	$Z4---%%d4j1$	4NN
 7T
|D
EErR   c                   	 t        |t        j                        sJ t        |      	t	         	fdt        |j                        D              }t         |t         j                        z        }t        ||j                  j                        }|rt        |      n	t               }||j                  z  }|j!                  |||      S )Nc           	   3  r   K   | ].  \  }}|t        fd j                  |d      D              z   0 yw)c              3  <   K   | ]  }j                   |     y wrL   r   r   s     rP   ru   z0_shard_shaped_array.<locals>.<genexpr>.<genexpr>  s     Gtzz!}Gr  r^   Nr	   getrs   r   r?  rG   r   s      rP   ru   z&_shard_shaped_array.<locals>.<genexpr>  s8      82 $Geii26FGGG 8   47r   r   rF  )rp   r   r  r
  r   r   r   r  r   r   r   r   r   rE  r   rF  update)
rG   r   rH   r   r  	new_shapemanual_meshnew_shardingrF  r   s
   `        @rP   _shard_shaped_arrayr    s    	D$**	++	+

% 8!*4::!68 8)kC8H8H4I&IJ+{DMM,>,>?,'TY[#dhh#	9|	EErR   c                    t        |t        j                        sJ t        |      t	         fdt        |j                        D              }|j                  |j                        }|j                  dk(  rt               }ng }t        ||j                  j                        D ]  \  }}|r|s|j                  |       |r|s|j                  |       2|s|s|j                  d        H|r|sJ t        |t              r|n|f}t        |t              r|n|f}|j                  ||z           t        | }t               j                   r j"                  n	t               }	t%        |	|      }
t'        |	j(                        |rt+        fd|j,                  D              n	t+               }|j/                  ||
|      S )Nc           	   3  r   K   | ].  \  }}|t        fd j                  |d      D              z   0 yw)c              3  <   K   | ]  }j                   |     y wrL   r   r   s     rP   ru   z2_unshard_shaped_array.<locals>.<genexpr>.<genexpr>  s     FdjjmFr  r^   Nr  r  s      rP   ru   z(_unshard_shaped_array.<locals>.<genexpr>  s8      82 FUYYq"5EFFF 8r  r   c              3  ,   K   | ]  }|v s|  y wrL   r^   )rs   r  r   s     rP   ru   z(_unshard_shaped_array.<locals>.<genexpr>  s     ;!{*:1;   	r  )rp   r   r  r
  r   r   r   _normalized_spec_for_avalr  rW   r  r   r   r   r   r"   r   r   r   r   r   r   rF  r  )rG   rH   r   r  r  
names_specout_specname_saval_snew_meshr  rF  r   r   s   `           @@rP   _unshard_shaped_arrayr    s   	D$**	++	+

% 8!*4::!68 8)--dii8*	YY!^sHHj$--*<*<= )	&f&  %fe46)%fe46)() h'H$5$7$=$=d  ! x2,H(()+ 
;dhh;	;$; 	9|	EErR   c               R   t        |j                  ||      D ]R  \  }}	}
t        j                  |j                  t        ||||
|	j                              r?t        j                  d       t        ||      5  t        j                  |      5  t        j                  |       d d d        d d d        |rc|j                  D cg c]  }|j                  j                   }}t        ||      D ]'  \  }}t        |||      rt        j                  d       |j                  D 	cg c]  }	|	j                   }}	t        t        t         ||      ||      }t        j"                  |j$                  |j&                        }||fS # 1 sw Y   xY w# 1 sw Y   xY wc c}w c c}	w )NzLshard_map argument avals not compatible with jaxpr binder avals and in_specsz7shard_map can't prove output is sufficiently replicated)r  r  r   
typecompatr  r  JaxprTypeErrorr  r   r  check_jaxprr  rF  _valid_repeatsr  r   r  r  r  rE   )r   rh  rG   rF   rJ   rH   r   in_atomsr  rz   in_specr  rF  r  out_avals_shardedr  r  s                    rP   _shard_map_typecheckr    s    5<<8< CmaG??166;k9gqvv$7 8 !B C CC
 k* F,=,=i,H U #(==1aqvvzz1G1Wi0 GXD#x0!!EG 	GG (-}}5!qvv55'-y99#%)		'	't	G$	D    2
 6s0   9FF%FF(F$F	FFc                    t        t        | |            t        | j                        z
  }t        fd|D              ryy)Nc              3  &   K   | ]  }|v  
 y wrL   r^   )rs   urF  s     rP   ru   z!_valid_repeats.<locals>.<genexpr>  s     acs   FT)r   rC  r   r   )rG   rF  r   ums    `  rP   r  r    s8    
<d#$s4+;+;'<<"2	rR   c                z   t        | ||      }t        j                  |j                  t        j                        r+t        j                  ||      }t        j                  |      }|j                  |j                        }t        |      t        |j                        k  r|j                  D ]	  }d|_         |S )NT)_make_scoped_manual_shardingr   
issubdtypedtypeextendedr   physical_shardingr   physical_aval_to_sdy_shardingr  r   rE   dim_shardingsis_open)ctxrG   r   r   aval_inr  sdy_shardingdim_shardings           rP   _shardy_shard_map_shardingr    s     $Ct4"w}}foo6		)	)'2	6B  )G$$W\\2,DOO,,$22 "!l"	rR   c                N    t        | |t                     }|j                  d      S Nr   )r  rW   r  )r  rG   r  s      rP    _shardy_shard_map_token_shardingr    s%     $Cqs3"			Q	rR   c                F    t        | t              rt               }||S | S | S rL   )rp   r   r#   )rG   concrete_meshs     rP   _get_spmdaxis_ctx_meshr    s)    l#%'M)5=?4?	+rR   c                   | j                   j                  }|j                  D 	cg c]  }	|	j                   }
}	t	        |t
        j                        r#t        |j                        |j                  z
  }n|}t        j                  t        |      |      }| j                   j                  |      }| j                  j                         D cg c]  }| j                  j                  |       }}t        |      }t!        ||      }t#        j$                  |D cg c]  }|j&                  |    c}      dk(  rt)        ||      5  t+        j,                  |      5  t/        j0                  ||| j2                  t/        j4                  t7        | j                  j                         |            dg|d| j8                  i\  }}| j;                  |       d d d        d d d        S t=        t?        tA        tB        | ||      || jD                              }t        | j8                        }tG        | |      g||z   z  |z   }t        jH                  |      jK                         }t=        t?        tA        tB        | ||      || jL                              }tG        | |      g|z  |z   }t        jH                  |      jK                         }tN        jP                  j                         g|z  t=        t?        t.        jR                  | jL                              z   }g | j8                  ||}tU        jV                  |t/        jX                  |      ||tT        jZ                  j                  t\        j^                  j                  |D cg c]!  }t\        j`                  j                  |      # c}                  }t\        jb                  je                  |jf                  g d | j8                  D        tN        jP                  j                         g|z  t?        t.        jR                  |
            }t]        jh                  |      5  t)        ||      5  t+        j,                  |      5  t/        j0                  ||| j2                  t/        j4                  t7        | j                  j                         |jj                  d |             dg|jj                  ||z   d  d| j8                  i\  }}tU        jl                  g |jo                         D 	cg c]  \  }}	|		 c}	}|D cg c]  }t]        jp                  |       c}       t        |j                               }|js                  t/        j4                  t7        | j                  j                         |jt                  d |                   }| j;                  |       d d d        d d d        d d d        |jt                  |d  S c c}	w c c}w c c}w # 1 sw Y   4xY w# 1 sw Y   S xY wc c}w c c}	}w c c}w # 1 sw Y   axY w# 1 sw Y   exY w# 1 sw Y   ixY w)Naxis_contextr  r^   dim_var_valuesc              3  l   K   | ],  }t        |t        j                        r|n|j                   . y wrL   )rp   r&   Typer   r   s     rP   ru   z-_shard_map_lowering_shardy.<locals>.<genexpr>P  s&     MQjBGG$!&&	0Ms   24);module_contextr  r  r  rp   r   SPMDAxisContextr   rE   r   r  replace	tokens_inr  r  r   rD  npr	   r   r  r   r  r4   jaxpr_subcomp
name_stackTokenSetr  r  set_tokens_outr   r  r   r  avals_inr  SdyArrayListbuild	avals_outr'   	TokenTypeaval_to_ir_typer(   ManualComputationOpflatten_ir_valuesManualAxesAttrr&   	ArrayAttr
StringAttrBlockcreate_at_startbodyInsertionPointr$  ReturnOpr  Valueupdate_tokensresults)r  in_nodesrh  rG   rF   rJ   r   rH   axis_ctxr  r  shardy_manual_axesnew_axis_contextsub_ctxefftokens
num_tokensr   	out_nodes
tokens_outin_shardingsnum_dim_varsout_shardingsoutput_typesr   r   manual_computation_opblock
out_nodes_r   rz   s                                  rP   _shard_map_lowering_shardyr%    s   ,,($||,!qvv,),.889 #4??3h6J6JJ$#33T"K1&&4D&E'.1mm.C.C.EFsCMMc"F&F6{*t%78+WW[1djjm12a7	$	, %f.?.?	.J %"00
5#..
--CMM113V<
=
- - ++	-i
 
$% % 	',c4
E
CLL"#, S''(,3C>?,.02>?,,,\:@@B,	',c4
E
S]]$%- 'sD135?@BOP- --m<BBD-==$$&'*4s4//?@A, 
33	2	2	2$11
T"M	
,,

+FQBMM--a0F
GI	J ((
"
"  .M#:L:LM .
--


		+.D  ),./%
 	%  #"24"E #)$#!//cMM!!#U__[j%AC 	D
	+ __Z456	+
 ))+J
 LL (5
8H8H8J)K1!)K (5)3(5 6!"((1+ 6 7Z'')*J))$--!6!>!>{
!K9M +N OJz"# # # 
	&	&z{	33M - G 2% % % 4 G *L 6# # # # # #s   W>"XX
X A;XX&X'Y!Y7B*X7!X,-
X77X2BX7YYX	XX$,X77Y <YY	YYc                  t         j                  j                  rt        | |||||||      S |j                  D cg c]  }|j
                   }	}|j                  D 
cg c]  }
|
j
                   }}
t        t        t        | ||      || j                  |	|      }t        j                  t        |      |      }| j                  j                  |      }t!        ||      5  t        j"                  |      5  t%        j&                  d| j(                  |d ||	|| j*                  g|| j,                  t        t.        ||	      t        t.        ||      d\  }}d d d        d d d        | j1                         t        t        t2        | ||      ||| j4                        S c c}w c c}
w # 1 sw Y   UxY w# 1 sw Y   YxY w)Nr  
shmap_body)r  	arg_namesresult_names)r   use_shardy_partitionervaluer%  r  r  r  r  r   
_xla_shardr  r   r  r  r  r  r  r  r4   call_loweringr  r  r  _pspec_mhlo_attrsr   _xla_unshardr  )r  rh  rG   rF   rJ   rH   r   r  r  r  rz   r  	in_nodes_r  r  r$  r  s                    rP   _shard_map_loweringr1  e  s   ""((%XudHiiQ Q  %||,!qvv,), %.1.*.'*c4=x,,	85)#33T"K1&&4D&E'k* DF,=,=i,H D!//cnneT7ICMMD$-D ))'9=*IzBDJ
D D Z 	W\3k:I

4 4 -.D D D Ds0   F%F*#F;9A"F/F;/F8	4F;;Gc                   | j                   j                  }|j                  }t        |t        j
                        r8|j                  |j                  D ci c]  }|t        j                   c}      }t        ||      S c c}w rL   )r  r  r   rp   r   r  update_axis_typesr   r    ry  r   )r  rG   r   r  r   s        rP   r  r  ~  sm    ,,(			$.889!!%-%9%9:HOO	:<D	tT	"" 	;s   Bc                   t        |j                  j                         D cg c]  \  }}||v s| c}}      dk(  r|S t        | ||      }	t	        j
                  |j                  t        j                        r+t        j                  ||	      }	t        j                  |      }|	j                  |j                        j                         }
t        |      t        |j                         k  rt#        t%        |j                              n	t#               }t'        j(                  | |||
|      }t+        j,                  ||t#        |j.                        z  |      }t'        j0                  | ||||      S c c}}w Nr  )unspecified_dims)r	   r   r  r  r   r  r  r  r   r  r   r  _to_xla_hlo_shardingr  to_protor   rE   r   ranger4   wrap_with_sharding_opr6   manual_protor   wrap_with_full_to_shard_op)r  rG   r   r   r  aval_outrz   r   sizer  shard_protounspecifiedsxr;  s                 rP   r,  r,    s6   	tzz//1
FGAtQ+5E4
FG1LH#Ct4"w}}foo6		)	)'2	6B  )G''5>>@+$s4??';; U7<<()AD !!#q';3>@"""{S!1!122D:,		(	(b(L)4
6 6 Gs
   E9
E9
c                   t        |j                  j                         D cg c]  \  }}||v s| c}}      dk(  r|S t        | ||      }	t	        j
                  |j                  t        j                        r+t        j                  ||	      }	t        j                  |      }t        |      t        |j                        k  rt        t        |j                               n	t               }
t	        j
                  |j                  t        j                        rt        j                  |      }t#        j$                  ||t        |j&                        z  |      }t)        j*                  | ||||
      }|	j-                  |j                         j/                         }t)        j0                  | ||||
      S c c}}w r5  )r	   r   r  r  r   r  r  r  r   r  r   r  r   rE   r   r9  r  r6   r;  r   r4   r:  r7  r8  wrap_with_shard_to_full_op)r  rG   r   r   r  r=  rz   r   r>  r  r@  r;  rA  r?  s                 rP   r/  r/    s^   	tzz//1
FGAtQ+5E4
FG1LH#Ct4"x~~v7		)	)(B	7B!!(+H$s4??';; U7<<()AD w}}foo6  )G""{S!1!122D:,!!#q'<3>@"''6??A+		(	(b(K)4
6 6 Gs
   F<
F<
c                    t        |t        j                        r=t        |       }t	        t        |j                  t        |j                                    S y)Nr  )	rp   r   r  r
  r   r  r  r9  r  )r   r  r   s      rP   r.  r.    s>    d&&'4 Es599eDII./00	rR   c           
        | D ]  }t        |d      st        |j                  t              s+|j                  j                  j
                  |j
                  k7  r_t        j                  |      }t        d|j                  j                  j
                   d|j
                   d|j                                |j                  j                  } t        |t              rt        d      t        |t              sJ |S )Nr   zMesh shape of the input z4 does not match the mesh shape passed to shard_map  z for shape zyPlease pass `jax.Array`s with a `NamedSharding` as input to `shard_map` when passing `AbstractMesh` to the mesh argument.)hasattrrp   r   r   rG   shape_tupler   r  r   r-  r   r   )r   rG   r   r  s       rP   get_mesh_from_argsrH    s     aq**QZZ"G	
	$	$(8(8	8&&q)&qzz'B'B&C D  !T^^-=,>@A 	A ZZ__d l#
	IJ J 
D$			+rR   c                ,    t        t        | |            S rL   )rW   rD  )rG   rF  s     rP   _vma_to_specrJ    s    	
>$$	%%rR   c                &    t        d | D              S )Nc              3  Z   K   | ]#  }|t        |t              r|n|fD ]  }|  % y wrL   )rp   r   )rs   r   r   s      rP   ru   z_spec_to_vma.<locals>.<genexpr>  s>      Cam",Q"6QQDC  C1 Cs   +!+)r   )r   s    rP   rE  rE    s    	 Cd C 
C CrR   c                @    t        fd| j                  D              S )Nc              3  ,   K   | ]  }|v s|  y wrL   r^   )rs   r   rz   s     rP   ru   z!order_wrt_mesh.<locals>.<genexpr>  s     4QQ!Vq4r  )r   rE   )rG   rz   s    `rP   rD  rD    s    	4$//4	44rR   c          
        t        |      t        j                        k  rt        ~t        t              rt               }	|	|	nt        |      t               }
t        t        t        ||
      ||      }t        t        |      }t        ||||||
      \  }}|D cg c]9  }t        j                  |j                  d   dt        j                   |            ; }}t#         |       |       |r't%         |       |       t'        fd|D              }nt'        fd|D              } |       }t        t        t(        |      |||      S c c}w )N)context_meshr   c              3  6   K   | ]  }t        |        y wrL   )rJ  )rs   rrG   s     rP   ru   z"_shard_map_impl.<locals>.<genexpr>  s     >|D!,>s   c              3  H   K   | ]  }t        j                          y wrL   )rW   rE   )rs   r   rG   s     rP   ru   z"_shard_map_impl.<locals>.<genexpr>  s     ;aq);   ")r   rE   r  rp   r   r#   rH  r"   r  r   _unmatch_specrE  
_run_shmapr   mapped_avalr   get_avalr  r  r   _match_spec)rd  r  r   r   rG   rF   r   rH   r   r  r|  in_vmaoutsr  rz   r  
src_pspecs
dst_pspecss       `             rP   _shard_map_implr^    s8   DOO,,

l#%'M)5=4DdD)D (	W]D)(Kt
$|X&&S$T69%'-$JNOQt
At}}Q/?@O)O ),o'1>g>>J;7;;J *	W[$	2J

  Ps   &>Ec           
     2   t        ||||      }t        t        t        |      ||      }t	        ||t        |j                        z        }	t        j                  |      5  t        ||      5  t        |	      5  t        j                  |      5   | j                  | }
t        t        |j                  |
            \  }}d d d        d d d        d d d        d d d        fS # 1 sw Y   %xY w# 1 sw Y   )xY w# 1 sw Y   -xY w# 1 sw Y   fS xY wrL   )ShardMapTracer  r   ShardMapTracerr  r   r   r   set_current_tracer  r!   r   r  call_wrappedr+   to_val_vma_pair)rT   rG   r   r   vmasrH   rP  rd  r  r  ansr[  r  s                rP   rV  rV    s   
k9l
C%7>514>*t[3t7G7G3H%HI*u% <'7k'J <*%<'-'8'8'C<
!..*
%C3u44c:;MD'< < < < 
w	< < < < < < < 
wsT   !D
.C>:C22C&	C2
C>D
&C/+C22C;7C>>D	D

Dc                h   t        j                         5  t        j                  d      5  t	        |      5   t        j
                  t        t        | ||            |      cd d d        cd d d        cd d d        S # 1 sw Y   nxY wd d d        n# 1 sw Y   nxY wd d d        y # 1 sw Y   y xY wrO  )r   eval_contextjaxdisable_jitr!   jitr*   _unmatch)rG   rH   r  rz   rP  s        rP   rU  rU    s     KS__U3 K,'KG377?8T9gFGJK K K K K K K K K K Ks?   B(B+A>"	B+	B(>BB
	B(B	B((B1c                    |r!t        |      }t        t        | |            }nt        | j                        }d} t	        t
        | |f||      |      S )NFrG   rF   rJ   rH   )rE  rW   rD  rE   rU   _add_singleton)rG   rH   r  rz   	used_axesdsts         rP   rl  rl    sY    W%I
N4+
,C
DOO
CI
7>z I
778
: :rR   c                    t        | |      D cg c]'  \  }}|rt        |      |j                  kD  r|nt        ) }}}t	        d |D              rt        |      y c c}}w )Nc              3  ,   K   | ]  }|t         u  y wrL   r   r   s     rP   ru   z_check_names.<locals>.<genexpr>  r   r   )r  r   r  r   r   r   )r   avalsr\  r   r   s        rP   r  r    sc    5%(
*b! c"g&!G
3 
*$ 
*(4((
T
 )
*s   ,Ac                      e Zd Zy)r   Nr   r^   rR   rP   r   r         rR   r   c           	         t        ||      D cg c]  \  }}t        | ||      s|nt         }}}t        d |D              rt	        |      y c c}}w )Nc              3  ,   K   | ]  }|t         u  y wrL   r   r   s     rP   ru   z_check_vmas.<locals>.<genexpr>  r   r   )r  r  r   r   r   )rG   r   re  r\  rF  r   s         rP   r  r    s\    UD)
+b# $D#r2#
? 
+$ 
+(4((
D/ )
+s   Ac                      e Zd Zy)r   Nr   r^   rR   rP   r   r     rv  rR   r   c           
     2   t        t        | |||      }t        j                         5  t	        j
                  d      5   t	        j                  |t        | |            |      cd d d        cd d d        S # 1 sw Y   nxY wd d d        y # 1 sw Y   y xY w)NF)r   )r*   _matchr   rh  ri  rj  rk  r   )rG   rH   	src_pspec	dst_pspecrz   fns         rP   rY  rY    s    vtY	9E" HCOOE2 HD3772]4%CDQGH H H H H H Hs"   B'A8%	B8B	=BBc                6     t        t        | |||      |      S )Nrn  )rU   _rem_singleton)rG   rH   r|  r}  rz   s        rP   r{  r{    s(    
=>y&)
==>
@ @rR   c                0    t        j                  | d      S Nr   rY   jnpsqueezery   s    rP   r  r  !  s    ckk!!44rR   c                0    t        j                  | d      S r  r  expand_dimsry   s    rP   ro  ro  "  s    cooaa88rR   c           	     ~   t         j                  j                  st         j                  j                  sy t	        |       D cg c]   }t        |dg       D ]  }|j                   " }}}	 t        j                  d|       y c c}}w # t        j                  $ r}t        d|j                   d      d d }~ww xY w)Naddressable_shardsrU   zInvalid value (z%) encountered in sharded computation.)r   
debug_nansr+  
debug_infsr<   r=  datar   check_specialr   InternalFloatingPointErrorFloatingPointErrorty)r[  rW  r   bufsr   s        rP   _maybe_check_specialr  $  s    				 	 ):):)@)@&(. 
<T4!5r:
< && 
<& 
<$ 
<f;-
< 
	,	, f
qttf4YZ
[aeefs   %B-B
 
B<B77B<c                  p     e Zd ZU dZded<   ded<   ded<   ded	<    fd
Zd Zd Zd Zd Z	d Z
d Z xZS )r`  )rG   r   checkrP  r   rG   frozenset[AxisName]r   boolr  r   rP  c                Z    t         |           || _        || _        || _        || _        y rL   )super__init__rG   r   r  rP  )r`   rG   r   r  rP  	__class__s        rP   r  zShardMapTrace.__init__5  s,    	GDI"DDJ$DrR   c                   t        |t              r|j                  |j                  fS t        |t              rt        d|       t        | j                  | j                  t               || j                        }|t               fS )Nz*Shouldn't have any non-shard_map tracers: )rp   ra  valrF  r   	ExceptionrU  rG   r  rW   rP  r   )r`   r  val_s      rP   rd  zShardMapTrace.to_val_vma_pair<  sk    #~&WWcgg	C	 B3%HII499djj!#sD<M<MNd9;rR   c           
     `   t        t        | j                  |            \  }}| j                  r |j                  d |D        i |\  }}t        |      t        u rt        |      n|}t        d |      }t        t        t        t        | j                        |            }	t        t        t        | j                        |      }
n+t               }t        | j                  j                        x}	}
t        j!                  |      }|r || j                  g|i |}nt#        t$        || j                  t        |j'                               | j                  |	|
      }t)        j*                         5  t-        j.                  d      5  t-        j0                  d      5  t-        j2                  d      5  t5        | j6                        5   t-        j8                  |      | }d d d        d d d        d d d        d d d        d d d        t;               |j<                  rBt?        |t        t        f      r|n|gtA        |      z  }t        t        tB        |       ||      S tC        | ||      S # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w)Nc              3  2   K   | ]  }t        |        y wrL   )r   rs   r  s     rP   ru   z2ShardMapTrace.process_primitive.<locals>.<genexpr>H  s     )E&))Es   c                    | j                   S rL   )rF  )r   s    rP   rQ   z1ShardMapTrace.process_primitive.<locals>.<lambda>J  s
    155 rR   F)"r+   r  rd  r  abstract_evalr   r   r   r8   r   rJ  rG   r   rW   rE   eager_rulesr  r*   _prim_applierr  r   rh  ri  rj  r  r  r!   rP  rk  r  rr  rp   r   ra  )r`   r  tracersra  in_valsrZ  r  r   r  rF   rJ   
eager_ruleout_valsrT   s                 rP   process_primitivezShardMapTrace.process_primitiveE  s'   S!5!5w?@OGVzz'T'')EW)EPPli&*9o&=%	"9i)4gGL$))<fEFh7<;WEigtyy3344h&JDII::6:h

tzz5+@$))
Ia  (!7 (9N (NN5!(#4T5F5F#G(3771:w'( ( ( ( ( 8$&wu>i#h-/ .BB$22( ( ( ( ( ( ( ( ( (sl   5J$J!J7J 	I4&J 	.J6J>J$4I=9J 	 J	JJJJ!	J$$J-c                     t        d| d      )NzEager evaluation of `z` inside a `shard_map` isn't yet supported. Put a `jax.jit` around the `shard_map`-decorated function, and open a feature request at https://github.com/jax-ml/jax/issues !r  )r`   call_primitiver   r  ra  s        rP   process_callzShardMapTrace.process_callb  s!    

/ 01 	12 2rR   c                    t        d      )NzEager evaluation of `pmap` inside a `shard_map` isn't yet supported.Put a `jax.jit` around the `shard_map`-decorated function, and open a feature request at https://github.com/jax-ml/jax/issues !r  )r`   map_primitiver   r  ra  s        rP   process_mapzShardMapTrace.process_mapi  s    
	FG GrR   c          	         ~~~t        t        | j                  |            \  }}t        || j                  | j
                  ||| j                  | j                        \  }}	t        t        t        |       |	|      S rL   )
r+   r  rd  rV  rG   r   r  rP  r   ra  )
r`   r  r   jvpr  symbolic_zerosr  rZ  r  r  s
             rP   process_custom_jvp_callz%ShardMapTrace.process_custom_jvp_callo  sm    c>S!5!5w?@OGV"3		43C3CW#)4::t7H7HJHgw~t,gx@@rR   c           	        |rd}t        |      ~~~~~t        t        | j                  |            \  }	}
t	        || j
                  | j                  |	|
| j                  | j                        \  }}t        t        t        |       ||      S )Nzcustom_vjp symbolic_zeros support with shard_map is not implemented; please open an issue at https://github.com/jax-ml/jax/issues)r  r+   r  rd  rV  rG   r   r  rP  r   ra  )r`   r  r   fwdbwdr  	out_treesr  r   r  rZ  r  r  s                rP   process_custom_vjp_callz%ShardMapTrace.process_custom_vjp_callw  s    4c  $$c3	>S!5!5w?@OGV"3		43C3CW#)4::t7H7HJHgw~t,gx@@rR   )rf   rg   rh   	__slots____annotations__r  rd  r  r  r  r  r  __classcell__)r  s   @rP   r`  r`  -  sD    >)*""	+%3:2GAArR   r`  c                  J    e Zd ZU ded<   ded<   d Zed        Zd Zd
dZeZ	y	)ra  r  rF  JaxTyper  c                    || _         t        |t              rt        |      }t        |t              sJ || _        || _        y rL   )_tracerp   r   r   rF  r  )r`   rd  rF  r  s       rP   r  zShardMapTracer.__init__  s:    DK#scNcc9%%%DHDHrR   c                   t        j                  | j                        }t        j                  | j                  j
                  j                  d|      }t        t        | j                  j
                  | j                  j                        |j                  j                        }t        j                  j                  r| j                  n	t!               }|j#                  ||      S )Nr   )r   rF  )r   rX  r  rW  r  rG   r>  r   r  r   r   r   r   r  r+  rF  r   r  )r`   r  r   r  rF  s        rP   r  zShardMapTracer.aval  s    =="D


4;;++00!T
:C (($++*A*ABL ''--$((9;C::|:55rR   c                F   | j                   t               k(  rit        j                         5  t	        | j
                  j                        5  t        j                  | j                  d         cd d d        cd d d        S y # 1 sw Y   nxY wd d d        y # 1 sw Y   y xY wr  )	rF  r   r   rh  r!   r  rP  to_concrete_valuer  r_   s    rP   r  z ShardMapTracer.to_concrete_value  s}    xx9; 3 1$++2J2J K 3%%dhhqk23 3 3 3 3 3 3 3s#    B"B.	BB	BB c                   t        | j                  j                  j                        | j                  z
  }t        | t        |            } t        j                         5  t        | j                  j                        5  t        | j                        }d d d        d d d        | j                  j                  }ddj                  t        t        |j                               ddj                  fdt!        t#        j$                  |j&                              D              S # 1 sw Y   xY w# 1 sw Y   xY w)N(r   z,)
c           
   3  H   K   | ]  \  \  }}}d | d d| d| d	  yw)zOn z at mesh coordinates z = z:
r  Nr^   )rs   r#  devicer#  rE   s       rP   ru   z)ShardMapTracer.__str__.<locals>.<genexpr>  s?      O MS&5 fX*:,c#c%KOrT  )r   r  rG   rE   rF  r   r   r   rh  r!   rP  r   r  r   r  r   r  r  ndenumeratedevices)r`   pb_namesblocksrG   rE   s       @rP   __str__zShardMapTracer.__str__  s    4;;##../$((:HuX'D				 /0H0HI DHH~f ;;DTYYs389:"=J99 O$'t||(Df$MO O O	   s$   ! D7D+D7+D4	0D77E N)returnr   )
rf   rg   rh   r  r  propertyr  r  r  ra   r^   rR   rP   ra  ra    s8    , 6 6	O (rR   ra  c                t      fd}t        |      t        u rt        |      n|} t        |||||      | S )Nc                 v     j                   t        t        |       i t              }t	        t
        |      S rL   )r   r  r  rS   r8   ro  )r   r[  
params_tupr  s     rP   applyz_prim_applier.<locals>.apply  s1    499c.$/D4
3CDDND))rR   rn  )r   r   r   rU   )r  rH   r  rG   rF   rJ   r   r  s   ` `     rP   r  r    sD    * "&iE!9d9oy)
(5th)&
()-
/ /rR   zdict[core.Primitive, Callable]r  c                   ~t        j                         5  t        t        t        |       }d d d        t        t        j                  | j                              D ]  \  \  }}} ||   g S # 1 sw Y   DxY wrL   )r   rh  r  r  r   r  r  r  )	rG   callbackeffectpartitionedr   
all_blocksr#  r  r  s	            rP   _debug_callback_eager_ruler    sq      'c$o&J'"2>>$,,#?L msFVf	)	' 's   A00A9c               :    ~ ~~|D ]  }|t        d|        |S )Nzadevice_put with explicit device not allowed within shard_map-decorated functions, but got device )r   )rG   srcsr  copy_semanticsr  r  s         rP   _device_put_eager_ruler    sG    
D. Rf HHNxQ R RR 
)rR   c           	        t        ||      D cg c]0  \  }}|t        j                  u r|nt        j                  |||      2 }}}| j
                  j                  t        fd|D              z  }t        j                  | j
                  j                  || j
                  j                  | j
                  j                        }	||	fS c c}}w )Nc              3  <   K   | ]  }j                   |     y wrL   r   r   s     rP   ru   z*_modify_specs_axis_data.<locals>.<genexpr>  s     )FA$**Q-)Fr  )r  r3   
not_mappedr6   
batch_spec	axis_datar>  r	   AxisDatar   	spmd_nameexplicit_mesh_axis)
rd  r   rG   rF   in_dimsr\  r  new_in_specsnew_sizenew_axis_datas
     `       rP   _modify_specs_axis_datar    s    "8W57b! X000"doob!T6RR 7, 7__!!T)F)F%FF(##ooHeoo&?&?oo((*- 
}	$$7s   5Cc	           	        t        t        | j                  |            \  }	}
t        d |
D              rt        | j
                  j                  | j
                  j                  g|D ch c]  }t        |      D ]  }|  }}}t        j                  j                  st              |z  rt        d      t        | |||
      \  }}nM|D ch c]  }t        |      D ]  }|  }}}t              |z  rt        d      t        | |||
      \  }}nSt        ||
      D cg c]0  \  }}|t         j"                  u r|nt%        j&                  ||d       2 }}}| j
                  }t!        j(                  || j*                  |t-        |
            \  }t/              fd       }t1        |||||      }t3        j4                  | j6                        5   |j8                  |g|	i |}d d d        t;        t         j<                  | t?        j@                               }t        |              S c c}}w c c}}w c c}}w # 1 sw Y   [xY w)Nc              3  P   K   | ]  }t        |t        j                           y wrL   )rp   r3   
RaggedAxis)rs   r  s     rP   ru   z#_shard_map_batch.<locals>.<genexpr>  s     =Ax**	+=s   $&z7vmap spmd_axis_name cannot appear in shard_map in_specszCvmapped away explicit mesh axis cannot appear in shard_map in_specsrk  c                 4    t                               S rL   )_batch_out_specs)r  out_dimsr   spmd_axis_names   rP   new_out_specs_thunkz-_shard_map_batch.<locals>.new_out_specs_thunk  s    N,>
+-/ /rR   r   r  )!r+   r  to_batch_infor   r  r  r  r  rE  r   disable_vmap_shmap_errorr+  r   r   r  r  r3   r  r6   r  batch_subtracetagr   r,   rS   r   rb  parent_tracer   r   BatchTracerr   r  )rd  r  r   r  rG   rF   r   rH   r   r  r  r   r   usedr  r  r\  r  r  ro  r  make_tracerr  r  r  s         `               @@@rP   _shard_map_batchr    sO   
 C 3 3Z@A'7=W==
??,,.99"?$L,>?qA?A?D?**00S5H45OPQQ"9~tXw#8L-%"?$L,>?qA?A?D?
% , - -"9!47#<L- "%Xw!79A x222BAt8TT 9L 9OOM))#uyy-wX-#x0/ 1/ $79 +-* e001 6tyy5w5*5H6,,e$4$<$<$>@+	[(HJ	//= @ @96 6s   2I	I25I*II$c           	        | |D ch c]  }t        |      D ]  }|  }}}t        j                  j                  st	        |       |z  rt        d      t        ||      D cg c]0  \  }}|t        j                  u r|nt        j                  |||       2 c}}S ||D ch c]  }t        |      D ]  }|  }}}t	        |      |z  rt        d      t        ||      D cg c]0  \  }}|t        j                  u r|nt        j                  |||      2 c}}S t        ||      D cg c]0  \  }}|t        j                  u r|nt        j                  ||d       2 c}}S c c}}w c c}}w c c}}w c c}}w c c}}w )Nz8vmap spmd_axis_name cannot appear in shard_map out_specszDvmapped away explicit mesh axis cannot appear in shard_map out_specs)rE  r   r  r+  r   r   r  r3   r  r6   r  )	r  r  dimsrJ   r   r   r  r\  r  s	            rP   r  r    s   #@$\$-?@A@A@D@**00S^d5JQRRY-/A x***BAy0QQ / /%#@$\$-?@A@A@D@
% - . . Y-/A x***BOOB#567 / /
 Y-/A x***BAt0LL / / A/ A//s   E+&5E1&E7*5E=25Fc	           	        t        t        | j                  |            \  }	}
|
D cg c]  }t        |      t        j
                  u }}|
D cg c]!  }t        |      t        j
                  ur|nd # }
}t        |	|
f      \  }}t	        j                  || j                        }t	        j                  |      \  }t        ||      D cg c]
  \  }}|s	| }}}t              fd       }t        |g |||||      }t	        j                  ||      \  }}|j                  | j                  |ft!        |      z   |      }t#         |       |      \  }}t        ||      D cg c]?  \  }}|6t	        j
                  t%        j&                  |      j)                               n|A }}}t        ||      D cg c]  \  }}t	        j*                  | ||       c}}S c c}w c c}w c c}}w c c}}w c c}}w )Nrk  c                 L            } g | d t        |               D        S )Nc              3  ,   K   | ]  \  }}|s	|  y wrL   r^   rs   axnzs      rP   ru   z>_shard_map_jvp.<locals>.new_out_specs_thunk.<locals>.<genexpr>0  s     Jfb"rrJ   
)r  )out_axr   which_nz_outs    rP   r  z+_shard_map_jvp.<locals>.new_out_specs_thunk-  s*    FKVKJFLN(CJKKrR   r   )r+   r  to_primal_tangent_pairr   r7   Zeror9   jvp_subtracer  nonzero_tangent_outputsr  r,   rS   	traceablerf  r  r   r:   r   rX  to_tangent_aval	JVPTracer)rd  r   rT   r  rG   rF   r   rH   r   primalstangentsr  which_nzr   r   f_jvpr\  r  tangent_in_specsr  ra  r   result
primal_outtangent_outr   r	  s         `                   @rP   _shard_map_jvpr  #  s   S!=!=wGH'8=EF47"'')F(F=EF47"'')at3F(F23-$
//!UYY
'%2259%'*8X'>EVR"bEE0L 1L T$Bh$B1A$B 3y')& LL0/%&&u'9'9E8eDk;QSYZ&*8:v>*k [9;Q AB	q)99;<qP ;+ ;03J0L	M1",,ua
#	MM' GF F;	Ms$   "G-&G2
G7G72AG=!Hc	           
       23 t        | j                  |      }|D 	cg c]  }	|	j                   }
}	t        j                  |
      \  }}}t        j
                  ||      \  }}t        t        t        |      ||      }t        j                  || j                  |j                  d      }t        |      }t        j                  |g |g |      \  }3t        |      2t              23fd       }t        g |||      }|j!                  | j"                  |g||      } 3       \  }}}}}}t%        d t'        ||      D              }t)        |t+        |      |z
  g      \  }}|j,                  rJ t        j
                  |              \  }} |       } t/        |||||      }!t1        |      }"g }#t'        ||      D ]  \  }$}%|$|#j3                  ||$          |%|#j3                  | |%          4r:t5        |"      j6                  }&|#j3                  t9        t;        |&                   p|#j3                  t9        2              g |#t9               ft+        |      z  z   g |z   }t        | j<                  |!      }'t        | j                  |      }(|D 	cg c]  }	|	j?                         r|	 })}	|j@                  D *cg c]  }*|*jB                   }+}*t        ||||      },t        t        tD              ||+      }-|-D .cg c]6  }.t        jF                  | t        jH                  jK                  |.      d       8 }/}.tM        jN                  |jP                  jR                        }0t        jT                  | g |'|(|)|/||,|0tW        jX                               }1|/D ]	  }	|1|	_-         t]        ||/|      S c c}	w c c}	w c c}*w c c}.w )NFrk  c            	                \  } } }}} } t        j                  | 
             \  } }r-|D cg c]!  }t        t        	|j                              # }}nt              gt        |      z  }g ||S c c}w rL   )r  r.   rW   rD  rF  r   )r   
out_knowns	res_avalsout_known_specsr   	res_specs	all_namesauxrH   rG   r   s         rP   known_out_specsz0_shard_map_partial_eval.<locals>.known_out_specsK  s    (+%Aq*iA**:7HIA;DEa1^D!%%01EiEY<.3y>1i)_)y)) Fs   &A<r   c              3  6   K   | ]  \  }}|d u xr |d u   y wrL   r^   rs   f1f2s      rP   ru   z*_shard_map_partial_eval.<locals>.<genexpr>[  s$     Mfb"d
)rTz)M   r  )/r  r  pvalr  partition_pvalsr.   r   r  trace_to_subjaxpr_nounits_fwd2r  r   _promote_scalar_residualspartial_eval_wrapper_nounits2_all_newly_manual_mesh_namesr,   rS   rf  r  sumr  r0   r   r  r1   iterr   nextrF  rW   rD  new_instantiated_constis_knownr  r  r  JaxprTracer
PartialValunknownr   r  r  rE   new_eqn_reciper   r  reciper/   )4rd  r   rT   r  rG   rF   r   rH   r   r  in_pvals	in_knownsr  	in_constsunk_in_specsknown_in_specsin_avals_shardedf_knownr"  known_paramsr   in_fwdout_fwdr  r  rh  envnum_res
out_constsnon_fwd_resunk_out_specsr   known_out_specs_resres_avals_iterr  r%  r&  res_vmaconst_tracersenv_tracersunk_arg_tracersr  r  
unk_paramsr  r   r  r  r  r   r!  s4       ` ``                                          @@rP   _shard_map_partial_evalrO  <  s    %%w/'%&aff&(&#%#5#5h#? )Xy!#!2!29h!G,dKK%x1''599allEJ!"!119,*,.,'3*4=)0* 1* 4*<N*<&5"-/, 	##E$6$68M98M$0	2#7:u4&':y%MFG8LMM'&sSX-?,@A*k__	&&z?3DE-$&67Iz;G#	?.)FG$ 
'fb"	~~b)*	'+,	~&**>$89:9&
' 9!##c(!225E|5EE,e22C8-E))3/+ '<1qzz|Q</<',}}5!qvv55+5'[B* '-y9=#%) $% r}}'<'<Q'?F %+ %		'	't	G$
%!Q=!Q;!Q!Q%{J 0 8 8 :	<# &a3&	Zj	99A 'd =5%s   O3O	OO##;O(c	           
     P  ./01 t        t        | j                  |            \  }	}
t        d |
D              }t	        j
                  || j                  ||j                        \  }0t        |0      }t        |      .t        0      .0fd       }t        |||      }|j                  | j                  |g|	|      } 0       \  }}/}}}t        d t        ||      D              }|d | }||d  }t!        |||	||      }t        |||      D cg c]"  \  }}}t#        |dd      dk(  xr
 |d u xr |d u $ }}}}t%        |      5  t'        t)        |t+        j,                        z              5  t/        j0                        5  t3        /|      /d d d        d d d        d d d                }t        |||      D cg c]  \  }}}||| }}}}t5        |      } g }!t        ||      D ]  \  }}||!j7                  ||          ||!j7                  ||          4r:t9        |       j:                  }"|!j7                  t=        t?        |"                   p|!j7                  t=        .              g |!t=               ftA        |      z  d t        ||      D        }#t        d	 t                |      D              1t        1      1fd
       }$t        |#|$|      }%/fd}&t        |
|      D '(cg c]
  \  }'}(|(s	|' })}'}(|j                  | jB                  tE        jF                  |&/j                        g|||)|%      }*t5        |*      }+t        ||      D (,cg c]1  \  }(},|(rt9        |+      nt        jH                  jK                  |,      3 }-}(},t        tM        t        jN                  |       |||-      S c c}}}w # 1 sw Y   JxY w# 1 sw Y   OxY w# 1 sw Y   TxY wc c}}}w c c}(}'w c c},}(w )Nc              3  R   K   | ]  }t        |      t        j                  u ! y wrL   )r   r7   r  r  s     rP   ru   z'_shard_map_linearize.<locals>.<genexpr>  s     :Aa':s   %'rk  c            	     .           \  } }}}}}t        | ||      D cg c]  \  }}}||| } }}}        }r-| D cg c]!  }t        t        |j                              # }	}nt        
      gt	        |       z  }	g |	|S c c}}}w c c}w rL   )r  rW   rD  rF  r   )r  r   r@  rA  rR  r%  r&  rJ   r   r  r   rH   linearize_outs_thunkrG   r   s             rP   fwd_out_specs_thunkz1_shard_map_linearize.<locals>.fwd_out_specs_thunk  s    *>*@'Iq!Q$'	67$C / /yq"bJ2:  /I /!I;DEa1^D!%%01EiEY<.3y>1i#Y###/ Fs   B&Br   c              3  6   K   | ]  \  }}|d u xr |d u   y wrL   r^   r$  s      rP   ru   z'_shard_map_linearize.<locals>.<genexpr>  $     Q&"bB$J-2:-Qr'  r   r^   c              3  ,   K   | ]  \  }}|s	|  y wrL   r^   r  s      rP   ru   z'_shard_map_linearize.<locals>.<genexpr>  s     A62rbBAr  c              3  *   K   | ]  \  }}|r|  y wrL   r^   r  s      rP   ru   z'_shard_map_linearize.<locals>.<genexpr>  s      #62r!  #s   c                      S rL   r^   )tangent_out_specss   rP   tangent_out_specs_thunkz5_shard_map_linearize.<locals>.tangent_out_specs_thunk  s	    rR   c                 2    t        j                  dg|  S rc   )r   rn  )r   	lin_jaxprs    rP   	f_tangentz'_shard_map_linearize.<locals>.f_tangent  s    ??9b0400rR   r   )(r+   r  r
  r   r7   linearize_subtracer  r   _promote_scalar_residuals_linr-  r,   rS   rf  r  r.  r  r1   r=  r  r!   r  r   r   r   r  _promote_scalar_residuals_jaxprr/  r   r0  rF  rW   rD  r   tangent_tracer   r   r  from_primal_valuer   maybe_linearize_tracer)2rd  r   rT   r  rG   rF   r   rH   r   r  r  nzs_inf_primalrT  
fwd_paramsall_fwd_resultsr  nzs_outrB  r@  rA  num_res_outrE  primals_out	residualsr  r%  r&  args_to_promoterJ   rR  
res_avals2rI  r  rJ  r  r[  tangent_paramsr^  r  r  nz_tangents_innz_tangents_outnz_tangents_out_iterprimaltangents_outr   r]  rS  rZ  s2       ` ``                                      @@@@rP   _shard_map_linearizeru    sa    S!=!=wGH'8:::&#%#8#8EIIvq||#\ ( *85IJ(*4=) 45	$ 6	$ ()Y*  //8.g.
</8L8N5)WifgQC<PQQ+-+-+';L)),Y)HJ J%r2 T7B/25S"*StS J/ J{+ L/$c$BRBR>S0STUL)$L 0	?KIL L L )#&y&'#B . .iaR:"*  .* .
#.)FG$ 
'fb"	~x|$	y}%	~&**>$89:9&
'C9 CvC0 CAS6%:AC, #S1BG-L # # 12 3,8O{4.
1 &)6%:A'1bbAA.A//	||I)*>*>? ***(*+9;/ o.$'$=? b& 13$+,8Q8QRX8YY ?, ?	WR..6Wl	[[[JL L L L L L
.< B?sZ    'O',PO;O.&O;.PP
P#P6P".O83O;;P	 PPc                
    | |i |} |       \  }}}}}}t        d t        ||      D              }|d | }	||d  }
|	D cg c]1  }t        |dd      s t        j                  j                  |d      n|3 }	}g |	|
S c c}w )Nc              3  6   K   | ]  \  }}|d u xr |d u   y wrL   r^   r$  s      rP   ru   z0_promote_scalar_residuals_lin.<locals>.<genexpr>  rV  r'  r   r^   r  )r.  r  r=  ri  lax	broadcast)rT   rS  r   rO   rf  r   r@  rA  rj  rl  r  rz   s               rP   r`  r`    s    	46# 4 6!Q1fgQC<PQQ+,;)'!# 29GR1Hsww  D)aO #) #	)	g	#s   6B c                x    | |i |\  }\  }}}}}t        |||j                        D 	
cg c])  \  }	}
}|	d u xr |
d u xr |j                  j                   + }}
}	}t	        ||      }|D cg c]1  }t        |dd      s t        j                  j                  |d      n|3 }}||||||ffS c c}}
}	w c c}w )Nr   r^   rx  )	r  r  r  r   ra  r=  ri  ry  rz  )rT   r   rO   rh  in_fwdsout_fwds	out_pvalsrD  rB  r%  r&  r  whichrz   s                 rP   r+  r+    s    ;<d;Mf;M8%	8'8Y
C5??CE Er2q :9"*9QVV\\)99 E% E
)%
7%#% 3:!Wb2I!!!T*qP %* %	(Iz3?	??E%s   .B0/6B7c           	          fd}t         j                        D cg c]4  \  }}|r!t        j                  dd|j                        n|j                  6 }}}g | j
                  D cg c]  }|j                   c}}t        j                  t        j                  | j                        |      \   }}\    S c c}}w c c}w )Nc                     t        | t        j                        g      \  }}t        |      D cg c]  \  }}|rt	        |      n| }}}t        j                  |g| S c c}}w rL   )r0   r   r  r  r  r   rn  )res_and_argsrH  r   rz   wrh  r  s        rP   r   z,_promote_scalar_residuals_jaxpr.<locals>.fun  sd    <#eoo*>)?@IC58e_
ETQ>!q(
EC
E??5#--- Fs   A'r  r   r   )r  r  r   unmapped_avalr  r  r  r  r   r   r   )rh  r  r   r  r  r  r  r   s   ``      rP   ra  ra    s    .
 u681a 45t!!!Q/!&&@ 8) 8:y:ELL9qAFF9:(--ll35#3#34h@/%Ar	,89s   9C/C	c                ^    t        |      }t        | |      D cg c]  }||vr|
 c}S c c}w rL   )rE  _all_mesh_names_except_spmd)rG   r   r   name_setr   s        rP   _unmentioned2r    s<     $(0{C 
 h  
    
 s   *c                  d }t        ||       D 	
cg c]  \  }	}
t        |
      t        j                  u r,t        j                  t	        |	|
j
                              nds&t        j                  |
      t        j                  k(  r|
n: ||
t        t        j                  j                  t        |	                         } }	}
t        fdt        |      D              }t        | |f      \  }}fd}t!        j"                  |j$                        }t        j&                  |      \  }t)        j*                  ||      \  }}t        ||       D 
cg c]#  \  }}
t        |
      t        j                  us"|% c}
}t        |      D 
cg c]#  \  }}
t        |
      t        j,                  us"|% c}
}z   }fd}	 t/        j0                  |g|t        |      |d}t?         |             S c c}
}	w c c}
}w c c}
}w # t2        t4        f$ r}t7        d       	 t9        j:                  d      5  t/        j0                  |g|t        |      |d}d d d        n# 1 sw Y   nxY wt)        j<                  |       n# t2        t4        f$ r}|d d }~ww xY wY d }~d }~ww xY w)	Nc                    |dk7  r| |z  S | S )Nr  r^   )rz   ys     rP   rQ   z&_shard_map_transpose.<locals>.<lambda>   s    aA Q rR   c           
   3     K   | ]P  \  }}t        |      t        j                  ur|n+t        j                  t        ||j                               R y wrL   )r   r7   UndefinedPrimalr  r  )rs   r\  rz   rH   r   rG   s      rP   ru   z'_shard_map_transpose.<locals>.<genexpr>  sY      1 2q G2#5#55q!!t[)RHJJ 1s   AAc                   t        t        j                  |      }t        ||      \  }}t	        j
                  t	        j                        |d      \  }}}} t        j                  |      | }t        j                  |j                  ddg |||       t        |      d  }	t        |      \  }}
t        |
|	      D cg c]  \  }}t        |      t        j                  u r+t        j                  t        ||j                               n7r|n3t"        j$                  j'                  |t)        t+        |                   }	}}|D cg c]  }t-        j.                  |       }}t1        |||	      S c c}}w c c}w )NFr^   )r  r7   is_undefined_primalr.   r  partial_eval_jaxpr_nounitsclose_jaxprr   jaxpr_as_funbackward_passrh  r   r  r   r  r  r  ri  ry  psumr   r  r   zero_from_primalr/   )out_ctsr   in_undefrH  undefsjaxpr_knownjaxpr_unknownr   res_reshapedin_ctsin_ct_specsr\  rz   rR  	res_zerosrH   rF   rh  r   rG   s                  rP   fun_trans_callablez0_shard_map_transpose.<locals>.fun_trans_callable  sm   2))40H 40KC')'D'D
ux(0$K114$$[137LUB(@,(@(@'	,F $Hh7NA{ {F35 "a aBGG# ggmD)R@A.777<<5tR)M#NOP 5F 5 7::))!,:I:xF335 ;s   BE>Fr   c                 F    t        d t                       D              S )Nc              3  ,   K   | ]  \  }}|s	|  y wrL   r^   )rs   r\  r  s      rP   ru   zD_shard_map_transpose.<locals>.new_out_specs_thunk.<locals>.<genexpr>(  s     DBDr  )r   r  )rF   
nz_arg_ctss   rP   r  z1_shard_map_transpose.<locals>.new_out_specs_thunk'  s    D#h
"=DDDrR   r   zsInvalid nan value encountered in the backward pass of a shard_map function. Calling the de-optimized backward pass.T) r  r   r7   r  r  r  r   r  float0r	   r  r   r  r  r   r9   r   r   r   nonzero_outputsr   r   r  r   r   r  ZeroDivisionErrorprintri  rj  _raise_no_nan_in_deoptimizedr:   )r  rh  rG   rF   rJ   rH   r   r   mb_divr\  rz   all_argsr   r  	fun_transfun_trans_flatr   r   r  r  r   r   r   e2r  s    ``` ``                 @rP   _shard_map_transposer    s    /&
 y'*	 "a 
aBGG	 	ggk$YAFFCD&/6<<?fmm3S!T#djjnnmD"k.RSTUV'  
 1  $/1 
1$ #GT?3(G4 4$ ll-%:J:JK),,Y7)Z%::9gN.( G,GTQQrww0FqG4(NTQDG2;M;M,MqNO E/!!!(,u\7J+y!H& 

H	--{F HN /	0 /	 > ?/ ??4  %%%%,05;N/9#%% % % ++A.  12 D //sr   B6H,#HH&#H
H&H$ $K
3K?J&'J;	J&J		J&K&J=5J88J==KK
c                   |j                   d   |j                   d   }}|j                   d   |j                   d   }}t        ||      5  t        j                  |      5  t	        t        ||t        |j                        z              5  t        j                  |||dd|       \  }}	}
}}d d d        d d d        d d d        t        j                        z
  }t        j                  |      |d  }t        |j                  |g      \  }}t        |      D ci c]  \  }}t        |      | }}}|D cg c]  }|j!                  t        |             }}t#        ||      D cg c]  \  }}|d u xr |d u  }}}|j                   d   }t        ||      5  t	        t        ||t        |j                        z              5  t        j                  |      5  t        j$                  |dg|z  |z         }t'        ||	      \  }}	d d d        d d d        d d d        t)        j*                  ||j,                        }t)        j*                  	|j,                        }	t/        ||j0                        \  }}t/        
|j                        \  }}t/        ||j0                        \  }}t/        |j                        \  }}t)        j2                         }g g }}t#        |	j0                  d | |      D ]  \  } }!|!s	|r)t5        t7        || j8                  j:                              nt5        t=        ||            }"|j?                   |tA        |||"| j8                                     |j?                  |"        |rHt#        ||      D  #cg c]0  \  } }#|#)t5        t7        || j8                  j:                              2 }$} }#n"t5        t=        ||            gtC        |      z  }$tE        ||tG        tH        jJ                  |
      ||||$|tM        |j                   |      tM        |j                   |	      
      \  }%}&t        jN                  |g |||jP                  |%|jR                  |jT                  |jV                        }'tY        |||||      }(t        jN                  g |(|||jP                  |&|	jR                  |jT                  |jV                        })t        |)j0                        t        |	j0                        k(  sJ t#        |j0                  |      D *+cg c]$  \  }*}+t[        |*      t(        j\                  u r|+s|*& },}*}+|,|D ch c]  }||	 c}D -cg c]  }-||-   	 c}-z  },|'|)|
||,|z   fS # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY wc c}}w c c}w c c}}w # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY wc c}#} w c c}+}*w c c}w c c}-w )Nrh  rG   rH   r   FT)rh  )/ra  r  r   r  r!   r  r   r   r  partial_eval_jaxpr_customr   r  _jaxpr_forwardingr0   r   idr  r  prune_jaxpr_outputs_add_reshapesr   remove_named_axis_effectsrE   r.   r  gensymrW   rD  r  rF  r-  r   r  r.  _pe_custom_paramsr  opnot_rS   r  	primitiver  r  r  r1   r   Var).saveableunks_ininst_inr  rh  rG   rH   r   r  jaxpr_stagedunks_outinst_outrC  num_out_primalsr@  out_varsres_varsr   r  idx_maprA  r%  r&  r  	ins_knownr   out_binders_known
ins_stagedout_binders_stagednewvarrl  staged_in_res_specsvarr  rnr[  out_res_specs_knownparams_knownparams_staged	eqn_knownfull_res
eqn_stagedrz   instnew_instrT   s.                                                 rP   _partial_eval_jaxpr_custom_ruler  C  s   
 

7#SZZ%7%::k2CJJ}4M[){+ VV->->y-I V/$c$BRBR>S0STUV 	$$UGWeUHU ;Kx7V V V ++,w6/,_-=>&!+"5"57HI(H"+H"56$!QRUAX6'6)12AW[[A2'236vw3G
HR2:$"*$
H%
H	F	${+ P/$c$BRBR>S0STUP)$P ((tf6NQV6VWK -e[, OK	P P P
 ..{DOOL+//dooN,4,)Q'#++>Q #**5-!Z(3;;?!;;=&#%r )L''159 %fc1 nT388<<01 <T; OP 	vmD)RJKL  $% ),Xw)?N%sA19 ^D#((,,?@ N N 	

&t[
9:<>A%jI 1wBGGX.&'.
3::[)4

,+O!Q, y*J,=*J	*J"}}lK<O<O"9) 4EyQ( 8( 8Z 8:L #} , 4 4cooswwP* 
Z	3|':':#;	;;	;"3::w7 3GAtaDHH$T  3( 3
-RAAMa-RS #SS(	J(Hy4H	HHgV V V V V V 72
HP P P P P P(N"3-RSs   	V",V!V,V4V"V/9!V5+V:,W	W.W WW5W'+)W-W3$W3-W8VVV	V""V, W
WW	WW$c           	        
 t         j                  d t                      D cg c]   \  }}|xr |j                  j                   " c}}
t        
      sfS j                  sj                  rJ  fd}j                  D cg c]  }|j                   }}t        j                  t        j                  |j                        |      \  }\   \    
fd}t        
j                  d t                      D cg c]4  \  }}|r!t        j                  dd|j                        n|j                  6 }	}}g |	j                  t               d  D cg c]  }|j                   c}}t        j                  t        j                  |j                        |      \  }\   \   fS c c}}w c c}w c c}}w c c}w )Nc                     t        j                  dg|  }t        |t        |      t	              z
  g      \  }}|D cg c]  }|j
                  st        |      n| }}g ||S c c}w rc   )r   rn  r0   r   r.  r   ro  )r   r   	out_knownrH  rz   r  r  s        rP   knownz_add_reshapes.<locals>.known  sq    
//+r
1D
1Cc#hU&;%<=NIs<?
@qAGG>!2
@C
@Y As    A*r   c                     t        | t              g      \  }}t        |      D cg c]  \  }}|rt        |      n| }}}t	        j
                  dg|| S c c}}w rc   )r0   r   r  r  r   rn  )	r   res_insrz   r  rH  r  r  which_s	         rP   stagedz_add_reshapes.<locals>.staged  sd    4#e*.ID#58v5F
GTQ>!q(
GC
G??<8c8C88 Hs   Ar  r   )r  r  r   r  r   r   r  r  r  r   r   r   r   r  )r  r  r  r  r  r  r  r   r  r  r  s   ```       @rP   r  r    s   
 E<#6#6{E
#CDF1 "affll"" F&	V[,66""<+A+AA	A
 *001aff1(144ll5[%;%;<hH+q"b9
 v|':':;CJ'GHJ1a 45t!!!Q/!&&@ J) JNyNL,?,?E
,LMqAFFMN(55ll6l&=&=>J,2r 
l	""1F 2JMs   %F=G9G&Gc
                   t        | |d         \  }
}t        ||d         \  }}||z   }t        |      t        |d   j                        k(  sJ t        |t	        |
      t	        |            }t        ||	d         \  }}t        |      }t        ||      D cg c]  \  }}||
|   n|||   n
t        |        }}}||z   }t        ||	d         \  }}t        |	t	        |      t	        |            }||fS c c}}w )NrF   rJ   rh  )rF   rJ   )r.   r   r  rS   r   r/  r  r0  )r  r  kept_outs_knownkept_outs_stagedr@  rA  r  r  r  r  in_specs_knownr   out_specs_knownnew_params_knownr  iter_stagedr%  r&  r  out_specs_stagednew_params_stageds                        rP   r  r    s?    %Wl:.FG.!%o|K7PQ!_#&99/	_	\'%:%B%B!C	CC	C,~1F$)/$:< &g}Z/HI!_()+ 0367/CE%+R &(^~b!&(nr"K ! E) E //&'7{9ST!=53I%*+;%<>	,	,,Es   #C=c                    t        j                         }|j                  t        fd| j                  D              S )Nc              3  2   K   | ]  }|vr|v r|  y wrL   r^   )rs   r   r   
spmd_namess     rP   ru   z._all_mesh_names_except_spmd.<locals>.<genexpr>  s)      Bj(T[-@  Bs   )r   get_axis_envspmd_axis_namesr   rE   )rG   r   axis_envr  s    ` @rP   r  r    s=     (''*	 B B 
B BrR   c                    t        j                         }t        |j                        t	               x}j
                  s|} t        |j                        nt        |j                        t        fd| j                  D              S )Nc              3  8   K   | ]  }|z  vr|v r|  y wrL   r^   )rs   r   already_manual_namesr   vmap_spmd_namess     rP   ru   z/_all_newly_manual_mesh_names.<locals>.<genexpr>  s.      )1EEE;&  )s   )
r   r  r   r  r"   r   r   rz  r   rE   )rG   r   r  r   r  r  s    `  @@rP   r-  r-    sx     (001/')
)(	0	0Dx334 x223	 ) ) 
) )rR   c                   t        |       s0t        j                  |      sdgt        |j                        z  d fS |j
                  d   }|j
                  d   }|j
                  d   }t        ||      5  t        j                  |      5  t        t        ||t        |j                        z              5  t        j                  |j
                  d   |       \  }}d d d        d d d        d d d        t              st        |       sj                  s|d fS t        ||j
                  d         \  }}t        | |j
                  d         \  }}	t!        |j
                  t#        |      t#        |	            }
t%        j&                  |j                  |j(                        }t        j*                  t-        |j                  |      D cg c]
  \  }}|s	| c}}t-        |j.                  |       D cg c]
  \  }}|s	| c}}|j0                  |
||j2                  |j4                        }||fS # 1 sw Y   ^xY w# 1 sw Y   cxY w# 1 sw Y   hxY wc c}}w c c}}w )	NFrG   r   rH   rh  rF   rJ   )rh  rF   rJ   )r   r  has_effectsr   r  ra  r  r   r  r!   r  r   r   	dce_jaxprr  r.   rS   r   r   r  rE   r  r  r  r  r  r  )used_outputsr  rG   r   rH   rh  used_inputsr   rF   rJ   ro  r  r  r  rz   new_eqns                   rP   _shard_map_dcer    s   	\	2>>##67S_$d**	F	$

=)+jj%){+ IV->->y-I I/$c$BRBR>S0STUIcjj&9<HE;I I I 
[	#l"3EMM cjj.DEKAx!,

;0GHLAycjjh %i 02J))%--IDcjj+6?wq$$?ckk<8Awq$DAz4#''CG I I I I I I 	@AsT   5I,I7'H:I&I
I!
I!
<
I'I':I?II	IIr^   )rk   r   static_broadcasted_argnumsr  backend	axis_sizedonate_argnumsglobal_arg_shapesc          	          t              nt         ||      \   f	dfd}
fd}||
_        |
S )Nc                   	 t        | |
      }|j                  D ]  }t        j                  |        t	        t        |      f      }t        |j                  ||j                  |j                        \  }}}t        |j                  |t        |            }t        j                  |t        |j                        D 	
cg c]
  \  }	}
|
s	|	 c}
}	      }|||j                   ||fS c c}
}	w )N)r  )r%   	flat_argsr   	check_argr   _get_devices_cached_shard_mapflat_funin_axes_flatout_axes_thunkrC   r   ri  rk  r   donated_invarsr   )r   rO   r   argrG   _pmappedrF   rJ   flat_global_argsr   r  jitted_frX   r  r  r  donate_tuplerT   rk   r   static_broadcasted_tuples               rP   infer_paramszpmap.<locals>.infer_params  s    a(,D"GWiv	OA{{ Q(9,7D$5	

D!..!*:*:I%G!Hh	7	T4>+ww(1!2B2B(CKfasKMH %qzz4BB Ls   
C5C5c                 p     | i |\  }}}}} || }t        || |             }t         |       |      S rL   )rD   r:   )	r   rO   r  r
  r   rG   rJ   r[  r  s	           rP   r   zpmap.<locals>.wrapped  sK    //X4%&D+D$	DD(*d++rR   c                 D     | i |\  }}}}} |j                   | i |S rL   )lower)r   rO   r  r   r  s       rP   r  zpmap.<locals>.lower	  s3    '88HaAq8>>4*6**rR   )r   r$   r  )rT   rX   rk   r   r  r  r  r  r  r  r   r  r  r  r  s   ```` ```    @@@rP   pmapr    s[     &1E'Nw'6G.7S3)%|C C,+ '-	.rR   c           
         t        t        t        t              |            }fd}t	        | |      }t        |j                  |||dt        |j                              ||fS )Nc                 B    t        t        t                             S rL   )r  r   _axis_to_spec)rX   r  s   rP   rQ   z#_cached_shard_map.<locals>.<lambda>  s    c'-;^=MN rR   F)rG   rF   rJ   rH   rE   )	r   r  r   r  _handle_reshapesrM   rc  r   rE   )r  rG   r  r  rX   rF   rJ   r   s      ``   rP   r  r    sd    3w}i8,GH(N)<@#
S%%D8(E #DOO 46 I
 rR   c                    t        d t        |      t        |            } | | }t        d t        |      t         |                   S )Nc                8    || S t        j                  | |      S Nr  r  rz   r  s     rP   rQ   z"_handle_reshapes.<locals>.<lambda>  s    RZ S[[5L rR   c                8    || S t        j                  | |      S r  r  r  s     rP   rQ   z"_handle_reshapes.<locals>.<lambda>   s    RZ S__QR5P rR   )r8   r   )rT   rk   r  r   rO   r   s         rP   r  r    sF    	Lt*d7m
-$	4#	Ps)T."23
5 5rR   c                t    t        |t              rd g|z  | gz   }t        | S |
t               S t        |      rL   )rp   rq   rW   r   )rX   r  r   s      rP   r  r  #  s>    CFRK9+%Ee9	z3J
B-rR   c                   |#| j                   t        j                   |      }n,| j                   t        j                          n| j                   }t        j                         dkD  r|d | j                   S |d | j                   S )N)r  r  )r  ri  process_countglobal_axis_sizelocal_axis_size)r   r  devss      rP   r  r  ,  sr    QYY.;;w'DII-3;;=199D1###$$	 q  	!!rR   rL   )
rJ   SpecsrE   Set[AxisName]rF   zSpecs | NonerG   Mesh | AbstractMesh | NonerH   r  )rX   AxisName)rT   r   rG   r$  rF   r"  rJ   zSpecs | Callable[[], Specs]rE   r#  rH   r  r   r  r   r  r  r   )r   rW   r  rW   )r   r   r   r
   r  None)rT   r   rG   Mesh | AbstractMeshr   r@   rF   r"  r   Sequence[int]r   zSequence[P]r  r   r  r&  )r   r@   r   r(  r   z#Sequence[core.ShapedArray | NoFail]r  list[core.ShapedArray | NoFail])r   r   rT   r   r.  r@   r   r"  r   r)  r  r   )rT   r   rG   r'  r.  r@   r   r"  r   r)  r  r   )rT   r   rG   r'  r.  r@   r   r"  r   zlist[set | NoFail]r  r   )rG   r'  r  list[AxisName])r.  r@   r   r"  r   zlist[T | NoFail]r  z1list[tuple[tuple[KeyPath, P], tuple[KeyPath, T]]])r   r   )rd  zpe.DynamicJaxprTracer  core.PrimitiverT   lu.WrappedFunr  zSequence[Any]rG   r   rH   r  r   r   r  zSequence[pe.DynamicJaxprTracer])r   r   )r  core.AbstractValuer  zcore.ShapedArray)rG   r   r  r-  r  r-  )rG   r   r   r   r  r-  r  r-  )rG   r   rF  r#  r  r  )r  mlir.LoweringRuleContextr  zsharding_impls.SdyArray)r  r.  r  zir.Attribute)r  r.  )r  r-  r  r   )rG   r   rz   r  r  r  )rt  zSequence[core.ShapedArray]r  r&  )
rG   r   r|  r   r}  r   rz   r  r  r  )r  zCallable[..., Any]r  zdebugging.DebugEffectr  r  )rd  zbatching.BatchTracer  r+  r   r,  r  Sequence[batching.BatchTracer]rG   r   rH   r  r   r   r  r/  )rd  zpe.JaxprTracerT   r,  )rT   r,  )rT   r   )rh  
core.Jaxprr  Sequence[bool])rG   r   r   r  r  r*  )rh  r0  )
r  zCallable[..., pe.RematCases_]r  r1  r  r1  r  core.JaxprEqnr  zStuple[core.JaxprEqn, core.JaxprEqn, Sequence[bool], Sequence[bool], list[core.Var]])r  r1  r  r0  r  r0  r  ztuple[core.Jaxpr, core.Jaxpr])rG   r   r   r  r  ztuple[AxisName, ...])r  z
list[bool]r  r2  r  z'tuple[list[bool], core.JaxprEqn | None])
__future__r   collections.abcr   r   r   r   enum	functoolsr   r*  mathr	   operatorr  typingr
   r   r   numpyr  ri  	jax.numpyr  jax.shardingr   r   jax._srcr   r   r   r   r   r   r   r   r   r   r   r   r   jax._src.corer   r   r   jax._src.meshr   r   r    r!   r"   r#   jax._src.apir$   r%   jax._src.lib.mlirr&   jax._src.lib.mlir.dialectsr'   r(   jax._src.utilr)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r   jax._src.interpretersr3   r4   r5   r  r6   r7   jax.tree_utilr8   r9   r:   r;   r<   r=   jax._src.tree_utilr>   r?   r@   rA   rB    jax.experimental.multihost_utilsrC   rD   rW   safe_mapr  
unsafe_mapsafe_zipr  
unsafe_zipregister_exclusion__file__r"  r%  r   rU   rZ   r\   rj   rd   r   rM   r   r   Enumr   r   r   r   r   r	  r   r  r   rC  r!  rQ  r"  transformation2r   r  MaybeTracer	Primitiver^  r   cacher  r  r  DynamicJaxprTracerc  r
  r  r  r  r  r  r  r  r  r  custom_typechecksr  r  r  r  r%  r1  register_loweringr  r,  r/  r.  rH  rJ  rE  rD  r^  	EvalTracerV  rU  rl  r  r  r   r  r   rY  r{  r  ro  r  Tracer`  ra  r  r  r  r  debug_callback_pr  device_put_pr  r  
BatchTracer  r  JVPTracerO  
JaxprTraceru  LinearizeTracer`  r+  ra  r  r  primitive_transposesr  partial_eval_jaxpr_custom_rulesr  r  r  r-  r  	dce_rulesr  r  r  r  r  r^   rR   rP   <module>ra     s   # = =      & &  
  5        & # % #   (A A 9   /6 6 6 + & 4 & $@ @= =P --Z--Z ! ! !( + 	4!#%'+154!$4!.4!BF4!l)
$ $ 	
   H> ).ULL+FL(L59L "&L 7;L HPL^2h& 		/GU+;<'?R  
(*5>JO/: 	 (:5**"**2;*DI***/2*X--*-2;-DI-*-/2-^"!"*<"AD"H:
 CL	9F	9 9 9
 GVO$   *  , St48 5868%%'5%:G%%(,% +/% >G% '	%L *<   &@D(D-?DF*F/AF
F$6
F;M
F .A  )) *F1F> 0E  4++ ,* ';  { #	! 	!  I4X4.   {$7 8#6$6("&C52 $3  KK: 	 H(H-4H9@H@ 5 8fUADJJ UAp(T[[ (T/ /1+ 0 ! "	
  +EI&& ' &<H!! "
%)0)0&4)0;H)0.)06:)0 +/)0 >G)0 
(	)0T )9   %/,N. !/ D:,D:J #: L\Z ';   #  @ @
 % A. *A.D (<   $:I+:I6D:I:I"/:I:Ix $  " "; /#)# *#/L#<-2B
B0B5IB)
)0)5I)$ ? . +[ "q1$&dd"J  
 5 5"rR   