
    vh?                      U d dl mZ d dlmZ d dlmZmZmZ d dlZd dl	Z	d dl
mZ d dlZd dlZd dlZd dlZd dlmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlm Z! d dlm"Z# d dlm$Z$ d dlm%Z% d dlm&Z& d dlm'Z' d dlm(Z( d dlm)Z) d dlm*Z* d dlm+Z+ d dlm,Z- d dl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z: d dl;m<Z= d dl>m?Z? d dl;m@Z@ d dl;mAZA d dl;mBZB d dl;mCZC d dl;mDZD d d lEmFZF d d!lGmHZI d d"lJmKZK d d#lJmLZM d d$lNmOZO d d%lPmQZQ d d&lRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^ d d'l_m`Z`maZambZb d d(lcmdZd d d)lemfZf d d*lgmhZhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZtmuZumvZv d d+lwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZmZmZmZmZ d d,lmZmZmZmZmZmZ eyecZZezecZZ e)j                  e       eeTeYeWf   ZeeTeWf   ZeeSeYeWf   ZeeSeWf   Z ej*                  e      Z G d- d.e      Zdd/Zd0 Z	 	 dd1ZeMj8                  j;                  d23      ZeMj8                  j;                  d23      Zdd4Zdd5Zefdd6       Zefd7        Zefd8        Zd9 Zd: Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd;Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd<Z G d= d>e      Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd?Z G d@ dA      Ze+j                  	 	 	 	 	 	 	 	 	 	 	 	 ddB       Z	 	 	 	 	 	 	 	 	 	 ddCZ	 	 	 	 	 	 	 	 	 	 ddDZ	 	 ddEZ	 	 	 	 	 	 ddFZ	 	 ddGZ G dH dIe(jd                        ZeXeXdddddJdddJddf	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddKZdL ZdM ZdN ZdO Z G dP dQ      Z e+jt                  dRdJS      	 	 ddT       Z e       ZdUedV<   	 	 	 	 	 	 	 	 ddWZ	 	 	 	 	 	 	 	 	 	 ddXZ ee!jt                  eY      	 	 	 	 	 	 	 	 	 	 dd[       Z e+jt                  dRdJS      	 	 dd\       Zeeeereej                     f   Z ej                         Z	 	 	 	 	 	 dd]Z	 	 	 	 	 	 dd^Zd_ Z e	j                  d`a       G db dZ             Z	 	 	 	 	 	 ddcZ	 	 	 	 dddZ ej                  de      Zd`e_        d`e_        ddfZee_        dg Zee_        dh Zdi Zdj Zdk Zdl Z	 	 ddmZdn Z ej                         Zdo Zedp        Zdq ZeӐj                  e       dr Ze	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddt       Zdu Zee=j                  e<   dv Zdw Zee=j                  e<   ddxZdy Zeej                  e<   dz ZeӐj                  e       	 	 	 	 dd{Z	 	 dd|Z eCj                  ee       	 	 	 	 dd}ZeeBj                  e<   eBj                  eBj                  e<   	 	 	 	 	 	 	 	 dd~Zd ZeeAj                  e<   d ZeeAj                  e<   	 	 ddZee=j                  e<   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddZ  ee=j                  dse       e=j                  e<   e!jt                  	 	 dd       Z	 	 ddZeeAj
                  e<   e	 	 	 	 	 	 dd       Z	 	 ddZee=j                  e<   	 	 	 	 	 	 ddZ	e	ej                  e<   d Zd Zd Zd Z ej                  d      Zej                  e       ej!                  d         eAj"                  ed        d Z eCj                  ee       d ZeeBj                  e<   d eBj(                  e<   d Z ej                  d      Zd`e_        d Zej!                  e       d Zej                  e       d Z eAj"                  ee       d Z eCj                  ee       d ZeeBj                  e<   d eBj(                  e<   d Z ej                  d      Zd Zej!                  e       d Zej                  e       d Z  eAj"                  ee        d Z! eCj                  ee!       d Z"e"eBj                  e<   d eBj(                  e<   	 	 d	 	 	 ddZ#dddddZ$ejJ                  d        Z&dddddZ'ejJ                  d        Z(d Z) ej                  d      Z*e*j!                  d         eAj"                  e*d        d Z+e*j                  e+       d Z, eCj                  e*e,       d Z-e-eBj                  e*<   d eBj(                  e*<   ddZ.d Z/d Z0d Z1e*jd                  e	j                   G d d                    Z3e*jd                  e	j                   G d d                    Z4d Z5e!jl                  d        Z7y)    )annotations)defaultdict)CallableSequenceIterableN)partial)
NamedTupleAnyUnioncast)api)api_util)config)core)dispatch)dtypes)linear_util)mesh)op_shardings)profiler)sharding_impls)source_info_util)stages)traceback_util)	tree_util)util)
xla_bridge)argnums_partial_exceptflatten_axesflatten_funflatten_fun_nokwargsdonation_vectorcheck_callableresolve_argnumsargnames_partial_except
debug_infohoist_obj_attrs_check_no_aliased_ref_args"_check_no_aliased_closed_over_refs)partial_eval)PartitionSpec)xla)ad)batching)mlir)pxla)ir)func)jax_jit)
xla_client)AbstractMesh)Sharding)NamedShardingGSPMDShardingSingleDeviceShardingPmapShardingAUTOUNSPECIFIEDUnspecifiedValueprepare_axis_resourcesparse_flatten_op_shardingcanonicalize_shardingflatten_spec_internal_use_concrete_mesh)FormatDeviceLocalLayout
AutoLayout)	RefEffect)api_boundary)tree_flattentree_unflattentreedef_is_leaftree_structuretree_leavestreedef_childrenbroadcast_prefix
all_leavesprefix_errorskeystr	PyTreeDefnone_leaf_registrytree_maptree_flatten_with_path)HashableFunctionsafe_mapsafe_zipwrapsdistributed_debug_log
split_listsplit_list_checkedweakref_lru_cachemerge_lists	subs_listfun_namefun_qual_name)BoxListdne_sentineljax_setattrjax_getattrjax_extendattrc                      e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ded<   d
ed<   ded<   d
ed<   ded<   d
ed<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded<   ded <   d!ed"<   d# Zd$ Zy%)&PjitInfozThings that we know about a jit instance before it is called.

  In other words, this structure contains arguments to jit()/pjit(),
  preprocessed and validated.
  strfun_sourceinfozinspect.Signature | Nonefun_signaturebooluser_specified_in_shardingsrR   in_shardings_treedeftuple[Any, ...]in_shardings_leavesout_shardings_treedefout_shardings_leavesin_layouts_treedefin_layouts_leavesout_layouts_treedefout_layouts_leavestuple[int, ...]static_argnumstuple[str, ...]static_argnamesdonate_argnumsdonate_argnamesxc.Device | Nonedevice
str | Nonebackendkeep_unusedinline
Any | Noneabstracted_axesuse_resource_envtuple[tuple[str, Any], ...]compiler_options_kvsc                    t        |       S Nidselfs    H/opt/face_recognition/venv/lib/python3.12/site-packages/jax/_src/pjit.py__hash__zPjitInfo.__hash__   s    d8O    c                
    | |u S r    r   others     r   __eq__zPjitInfo.__eq__   s    5=r   N)__name__
__module____qualname____doc____annotations__r   r   r   r   r   ri   ri   c   s    
 ))  $#!!&&""''$$  %%!!""!!""

,33r   ri   c                4	   t        | |||      \  }}|D ]  }t        j                  |        |j                  rt	        |j                        }g ||}	 t        j                         rt        j                  j                  sl|j                  d   j                  j                  sIt        t
        j                  |      }t        j                  |       t!        |i |j                  \  }}	}
n#t#        j$                  |i |j                  }d }	d }
|jX                  r|jZ                  j]                         \  }}t_        |j`                  |j`                  g      \  }}tc        ||      }te        ||f      }tA        |jX                  |      D ]  \  \  }}}|tf        jh                  u r3||   jk                  tc        |jl                  |jn                               N|tf        jp                  u r:tc        |jl                  |jn                        D ]  }||   js                  |        J  n|jZ                  }|j                  rBtu        d |j                  D              }tw        |g      \  }}ty        |j                  |       tc        |      }|||||j                  d   |j                  |jX                  	
f	S # t&        j(                  $ rc}|j*                  \  }t-        | dt-        | dt/        |                   }t'        j0                  |||d|j2                        }t5        |      d d }~wt6        j8                  $ r}|j2                  dgt;        |      z  n|j2                  }|j                  d   j<                  rt?        |j*                  d         |tA        |||jB                        D ]d  \  }}}	 t7        jD                  |       # t6        j8                  $ r2}t?        d| d|jG                          d	tI        |       d
      |d }~ww xY w tK        d      |d }~wtL        jN                  $ rV}t-        | dd      r&tQ        d|jR                   d| jT                         d tM        jV                  || ||       Y d }~d }~ww xY w)Njaxprr   r   jit r   z
Argument 'z' of shape z	 of type z is not a valid JAX type.Unreachable_apply_primitiveFzinvalid value (z) encountered in c              3  <   K   | ]  \  }}}|j                     y wr   
num_leaves).0_end_trees      r   	<genexpr>z&_python_pjit_helper.<locals>.<genexpr>   s     SHa,,S   )=_infer_paramsr   	check_argattrs_tracked_get_statesr   trace_state_cleanr   debug_key_reusevalueparamsr   is_highmap
full_lowercheck_eval_args_pjit_call_impl_pythonpjit_pbindr   DeviceAssignmentMismatchErrorargsgetattrrj   !_device_assignment_mismatch_error	arg_names
ValueErrorr,   InvalidInputExceptionlenconsts	TypeErrorzipin_avalscanonicalize_dtype	str_shorttypeAssertionErrorr   InternalFloatingPointErrorFloatingPointErrortyr   maybe_recursive_nan_checkbox_dataout_treechildrenr\   r   rI   rL   peBoxAttrsettreedefleavesListAttrappendsumr[   _set_states)funjit_infor   kwargsp	args_flatarginit_statesout_flatcompiledr   efailsr`   msgr   nameavalr   box_treedefr   box_flatbox_outr   ikindbitemnum_states_outfinal_statesoutss                                  r   _python_pjit_helperr      s   sHdF;,!Y cs __aoo.K*+*	*I"= )?)?)E)E!''//dooy1i
9%%;Y%S!((%S"h(i41884hhh6 ZZJJ//1K+H{7M7MxObOb6cdHh[(3G$(FAJJ0 	D1		q	nQYY9:2;;"199ahh7 	!D
)

4
 	! 	u zzH__S1??SSN'>2BCL(.	(	+$
(Iqxx/@
//1::x
; ;a 
	-	- $VVFEsNGCSX,NOH

2
2%E1;;8C
S/t#		"	" 1)*)<s9~%!++IxxaffQi a' IqzzB ?/#tT	?

 
 
%(( 	?4&DNN,<+= >s)578=>?	?? =)q0		,	, =s&.6GHXHXGYZ[aee&&q#tV<<=sX   B=K RAL22RA7P+ OP+P)-PPP++RARRc                T    | d uxr# | j                         xr | j                          S r   )
is_enabledis_fdo_consumed)pgle_profilers    r   _need_to_rebuild_with_fdor      s3    
t
# 2(@(@(B 2//113r   c
           
        t        j                  ||      \  }
}| d uxr
 t        | t         j                        xr t        | j                  t         j
                        xr | j                  j                   xr | j                  j                   xr | j                  j                   xr} t        d |
D              xri |d u xrc | xr^ | xrY t        d |D               xrD t        j                  j                  xr t        d g |||D               xr t        |	       }|r#|
D cg c]  }|j                   }}|
D cg c]  }|j                    }}t#        t%        |            D cg c]  }|| j&                  v  }}t)        | j*                  | j,                        D cg c]]  \  }}|t.        j0                  urDt3        j4                  |j6                  t2        j8                        rt;        j<                  ||      n|_ }}}t        j>                  | j@                  ||| jB                  |||| jD                        }|S d }|S c c}w c c}w c c}w c c}}w )Nc              3  P   K   | ]  }t        |t        j                           y wr   )
isinstancexc	ArrayImplr   xs     r   r   z%_get_fastpath_data.<locals>.<genexpr>   s     CajBLL)C   $&c              3  <   K   | ]  }t        |t                y wr   )r   rF   )r   r   s     r   r   z%_get_fastpath_data.<locals>.<genexpr>   s     <q*Q	*<r   c              3     K   | ]@  }t        |d       xr. t        j                  |j                  t        j                         B yw)dtypeN)hasattrr   
issubdtyper  prng_key)r   r   s     r   r   z%_get_fastpath_data.<locals>.<genexpr>   s<      45 	WO&"3"3CIIv"OO45s   AA)#r0   reflatten_outputs_for_dispatchr   MeshExecutableunsafe_callExecuteReplicatedordered_effectshas_unordered_effectshas_host_callbacksallanyr   r   r   r   r   
_committedranger   _kept_var_idxr   _in_shardingsr   r   abstract_tokenr   r  r  extendedr   physical_shardingMeshExecutableFastpathDataxla_executable_out_shardings_dispatch_in_layouts)
executabler   r   r   r   r   effectsr   r   r   out_reflatteneduse_fastpatho	out_avalsout_committedr   kept_var_bitvecsain_shardingsfastpath_datas                        r   _get_fastpath_datar%     s    #AA(HU/8  7
Z!4!4
57
Z++T-C-C
D7 $$44
4	7
 $$::
:7 $$77
77 C?C
C7 T
!7 
7 ,7 <G<<
<7 %%++ 5 453Y33F345 15 67$ (6
6' , !01A1I1+:;aQ\\;M; %c)n 57 J444 7O 7 
00*2E2EF	 Aq D'''F,=,=aggv,W 	((A.	L  33!!8\!!9m_'')M 
 M	! 2;7s   2II6I0A"I$i    )capacityc                    | rt         S t        S r   )#_cpp_pjit_cache_explicit_attributes_cpp_pjit_cache_fun_only)contains_explicit_attributess    r   _get_cpp_global_cacher+    s    !..##r   c                    t          fd       }t        j                  j                  j                  j
                  j                  j                  j                  j                  j                  j                  j                  j                  j                  j                        }t         j"                  j%                  t'                |j(                  j*                  |t,        j.                  t        j0                  t3        |j4                        	      } t7               |      } |_        |_        t=        |      }t>        |_         tB        |_"        tF        |_$        tJ        |_&        |S )Nc                    t         j                  j                  rt        dj                   d      t        g| i |\	  }}}}}}}}	}
t        |	||||||j                  |j                  j                  |

      }||t        |
      fS )Nre-tracing function # for `jit`, but 'no_tracing' is set)r   
no_tracingr   RuntimeErrorrk   r   r%  r  r   r   r   )r   r   r   r   r   r   r   r   r   r  r   maybe_fastpath_datar   r   s               r   
cache_missz_cpp_pjit.<locals>.cache_miss  s    /0G0G/H I: : ; ; #6c8"Ud"Uf"U T8Xy%,Hi=(u||X%=%=}N $&?&NNNr   )r|   r}   r   r   ro   rq   rr   rs   rt   ru   rv   rw   r   )'rG   r0   JitGlobalCppCacheKeysr|   r}   r   r   ro   rq   rr   rs   rt   ru   rv   rw   r   r   _xlapjitr`   ry   r{   r   dispatch_registrycc_shard_argr+  r*  rY   _fun	_jit_infor   jit_evict_fnclear_cache	jit_lowerlower	jit_tracetracejit_eval_shape
eval_shape)r   r   r3  	cache_key
cpp_pjit_fcpp_pjitted_fcpp_jitted_f_classs   ``     r   	_cpp_pjitrG    sH   O O ((,,..__h&6&6#88"66$::#88!44 22"66!44#88:) ww||smS*h&=&=	9+F+F
IBBC	E* %*Z(--$- M*#/ &&"0 
r   c           
        t        | j                  | j                  ||      \  }}t        d t	        |j
                        D              }t        j                  |j                  |j                  |      }t        t        |fi |j                  dd i}t        j                  |j                  d   ||j                  d   |j                  |||j                  |j                         S )Nc              3  ,   K   | ]  \  }}|s	|  y wr   r   r   r   ds      r   r   zjit_trace.<locals>.<genexpr>Q  s     Htq!aH   
r   r   r   )r   r9  r:  tuple	enumeratedonated_invarsr   make_args_infoin_treer   r   _resolve_and_lowerr   Tracedr   r   
num_consts)jit_funcr   r   r   r   r|   	args_infolower_callables           r   r?  r?  N  s    x}}h.@.@$O,!YHy1A1A'BHH.##AIIqzz>J)-y /AHH /)-/.	hhwAHHV$4ajjiall
< <r   c                <    t        | g|i |j                         S r   )r?  r>  )rU  r   r   s      r   r=  r=  Z  s     	8	-d	-f	-	3	3	55r   c           	     *   t        | j                  | j                  ||      \  }}|j                  d   D cg c]  }t	        |t
              rd n| }}g }t        |j                  d   j                  |      D ]~  \  }}	|	/|j                  j                  j                  r|j                  n|	}n|	}|j                  t        j                  |j                  |j                  ||j                                 t#        |j$                  |      S c c}w )Nout_shardingsr   )sharding	weak_type)r   r9  r:  r   r   r=   r   r  r[  r   _are_all_axes_explicitr   r   ShapeDtypeStructshaper  r\  rI   r   )
rU  r   r   r   r   r!  rZ  outr"  out_ss
             r   rA  rA  ^  s    	x}}h&8&8$	G$!QHH_57 &a)9:4A 7- 7
#ahhw'11=A <ha}

>>!**Ea
aJJs##AGGQWWq./kk; << 


C	((7s   Dc                    | j                          t        j                  | j                         t        j                          y r   )_clear_cache_create_pjit_jaxprevict_functionr9  _infer_params_cachedcache_clearr   s    r   r;  r;  n  s,    ##DII.""$r   c                   t        | d       \  }}g g }}|D ]  }t        |t              r7|j                  |j                         |j                  |j
                         Jt        |t        t        f      rt        d|       |j                  d        |j                  |        t        |      t        |      k(  sJ t        ||      t        ||      fS )Nc                
    | d u S r   r   r   s    r   <lambda>z,_split_layout_and_sharding.<locals>.<lambda>u  s
    !t) r   is_leafz`jax.jit` does not accept device-local layouts directly. Create a `Format` instance wrapping this device-local layout and pass that to `jit` instead. Got )rH   r   rC   r   device_local_layoutr[  rD   rE   r   r   rI   )entriesentries_flatr   layouts	shardingsr   s         r   _split_layout_and_shardingrs  t  s    &w8KL,29' a!VnnQ**+qzz"	A):6	7(()s,- -
 nnTq 
WY	''	'		)>'9+M	MMr   c               f   |r%t         j                  j                  st        d      t	        |        |	|it        j                  dt               ||	t        d|d|	      |t        |t              st        d      |t        |t              st        d      t        |t              rt        |      }t        |      \  }}t        |      \  }}t        |d      }t        |d	d
      }|duxr t        |t               }t        j                  |      \  }}t        j                  |      \  }}t        j                  |      \  }}t        j                  |      \  }}t!        j"                  |       }t!        j$                  |       }t'        | |||||      \  }}}}|dnt        |j)                               }t+        di d|d|d|d|dt        |      d|dt        |      d|dt        |      d|dt        |      d|d|d|d|d|d|	d|d|
d |d!|d"|S )#zParses the arguments to jit/pjit.

  Performs any preprocessing and validation of the arguments that we can do
  ahead of time before the jit()-ed function is invoked.
  z6abstracted_axes must be used with --jax_dynamic_shapesNzbackend and device argument on jit is deprecated. You can use `jax.device_put(..., jax.local_devices(backend="cpu")[0])` on the inputs to the jitted function to get the same behavior.z>can't specify both a device and a backend for jit, got device=z and backend=zTIf backend or device is specified on jit, then in_shardings should not be specified.zUIf backend or device is specified on jit, then out_shardings should not be specified.r#  rZ  Tallow_unconstrained_dimsr   rk   rl   rn   ro   rq   rr   rs   rt   ru   rv   rw   ry   r{   r|   r}   r   r   r   r   r   r   r   )r   dynamic_shapesr   r   r#   warningswarnDeprecationWarningr   r=   listrM  rs  r>   none_lrflattenr   rk   rl   r$   itemsri   )r   r#  rZ  ry   r{   r|   r}   r   r   r   r   r   compiler_optionsr   
in_layoutsout_layoutsrn   rq   ro   rs   rr   ru   rt   rw   rv   rk   rl   r   s                               r   _parse_jit_argumentsr    s     V2288
M
NNF.MM	C 		 g1 &$YnG:7 8 8
<AQ(R ? @ @ MCS)T @ A A d# &L7E*l9-H+}'nE,(BFH- ".T!9 "P%/>N%O!O  /6ool.K++070N--*1//**E'',3OOK,H))**3/.((--ET	=./>FB./>? !1 8" 0 6 6 89 	 
3%
3#
3 %@
3 2	
3
 ""56
3 4
3 ##78
3 .
3   12
3 0
3 !!34
3 &
3 (
3 9G
3 (
3 17
3 AH
3  
3 )/
3  (!
3" *#
3$ 2%
3 3r   c               L    t        | |||||||||	|
|||      }t        | |      S )z8jit() and pjit() are thin wrappers around this function.r#  rZ  ry   r{   r|   r}   r   r   r   r   r   r  r   )r  rG  )r   r#  rZ  ry   r{   r|   r}   r   r   r   r   r   r  r   r   s                  r   make_jitr    s@      ",m%%':J)+( 
3	!!r   c                  r    e Zd ZU ded<   ded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   ded<   ded<   y)
PjitParamsz	list[Any]r   dict[str, Any]r   tuple[core.AbstractValue, ...]r   rR   rQ  r   tuple[bool, ...]rO  rz   r   intrT  z7list[tuple[PyTreeDef, PyTreeDef, tuple[Any, str, Any]]]r   r{  r   N)r   r   r   r   r   r   r   r  r    s<    

**""/HH.r   r  c                f
   t        j                  d|        t        |      }|r|j                  rt	        d      /|j
                  s|j                  rj                  st	        d      t        |j                  g|i |}t        j                  | |      }	t        |	|j                  |d      \  }	}
~t        |	|j                  |      \  }	}~t!        ||
|      \  }
}}|rt#        |	|      }	t%        |
|f      \  }}t'        |	|      \  }}t)        ||      \  }}|j*                  s|j,                  r<t.        j0                  j2                  s"t5        |j*                  |j,                  |      }ndt7        |      z  }t        |j
                  xs |j                        }|r@t9        |j                  |j
                        }t%        |      \  }}t;        |      x}}|x}}nTt;        fd|j<                  D              }|j>                  }t;        fd	|j@                  D              }|jB                  }d |vsJ d |vsJ t.        jD                  j2                  r-|J tG        jH                  ||      }t;        d
 |D              }n5|rt;        d |D              x}}n|}t;        d tK        ||      D              }|J tM        |||jN                  |jP                  |||jR                  ||	      \  }}tU        ||      }tW        |||tY        |jZ                              \  }}} }!t.        j\                  j2                  rt_        |g |j`                  ||       tc        ||||!       te        |||jf                  |jh                  tk        |d      t;        |       |jl                  jn                  |      \  }"}#t7        |      t7        |      cxk(  rt7        |      k(  sJ  J t.        jD                  j2                  r%tq        ts        tt        jv                  |      |      }$ng }$g |$|}%ty        d |!D              }&t7        |$      |&z   t7        |      z   }'tz        f|'z  |z   }d|'z  |z   }d|'z  |z   }t7        |      t7        |      cxk(  r-t7        |      cxk(  r|&t7        |      z   t7        |%      z   k(  sJ  J t}        |||"||#|t        |      |j                  |jZ                  |j                        }(t        ||(|| |       ||j                  t7        |      |!|
      |%fS )Nzpjit._infer_params_implz<pjit does not support kwargs when in_shardings is specified.zpMesh context manager should not be used with jit when backend or device is also specified as an argument to jit.r&   T)allow_invalidFc              3  :   K   | ]  }t        |d d        yw)r#  r   N_create_sharding_for_arrayr   r   ctx_meshs     r   r   z%_infer_params_impl.<locals>.<genexpr>>  s$       ) 	#8QF )   c              3  :   K   | ]  }t        |d d        yw)rZ  r   Nr  r  s     r   r   z%_infer_params_impl.<locals>.<genexpr>B  s$      !* 	#8QG!*r  c              3  ,   K   | ]  \  }}|s	|  y wr   r   )r   r"  r   s      r   r   z%_infer_params_impl.<locals>.<genexpr>N  s     041aaQ0rL  c              3  F   K   | ]  }t        j                  |        y wr   r   shaped_abstractifyr   s     r   r   z%_infer_params_impl.<locals>.<genexpr>P  s     Qat66q9Q   !c              3     K   | ]>  \  }}|j                   r)t        j                  |t        j                  |            n| @ y wr   )has_qddr   AvalQDDcur_qdd)r   r"  r   s      r   r   z%_infer_params_impl.<locals>.<genexpr>S  s?      D#q! 9:		DLLDLLO4 Ds   AAr   closurec              3     K   | ]:  \  }}\  }}}|t         j                  t         j                  fv r|j                   < y wr   )r   	ReadWriter   r   )r   	init_treer   r   s       r   r   z%_infer_params_impl.<locals>.<genexpr>s  sA      M.HiLQ4)-",,

1K)K )) Ms   A Ar   r   r#  rZ  r  r  rO  r  r   r   r   r   )Dr   
test_eventrm   rn   r   r   r   empty_flat_axes_specsr   lu	wrap_initr   ry   r%   r{   _flatten_boxes_handle_boxesrH   r    r'   r|   r}   r   
debug_nansr   r"   r   $_create_sharding_with_device_backendrM  rq   ro   rs   rr   rw  r   infer_lambda_input_typer   _process_in_axis_resourcesrt   ru   r&   _attr_cache_indexrd  	IgnoreKeyr   mutable_array_checksr)   r   _attr_cachedata_update%_check_and_canonicalize_out_shardingsrv   rw   rV   r   _debug_info_extract_implicit_argsr   r   	InputTyper   r<   dictra   r   r   r  r   ))r   jir  dbgr   r   r   have_kwargs
axes_specsfdyn_args
dyn_kwargsr   explicit_argsrQ  flat_funr   rO  device_or_backend_setr[  r   r   rq   rs   ro   rr   in_typein_shardings_flatin_layouts_flat
attr_tokenr   r   r  r   out_shardings_flatout_layouts_flatimplicit_argsr   num_attrs_innum_extra_argsr   s)     `                                      r   _infer_params_implr  
  sT    //+S1V+R33
FH H 


bii<= =   2 2DTDVD*ll33'!&q"*;*;TQUV+!X
)!R-?-?H-!Z#1#x#L (JaA':(>?-"1g.(H+HmD(M2--v7H7H7N7N$R%6%68J8JGTNM 22N
 rzz6RYY73BIIrzzJH"8,OFG16v>.3::0  )'' ) ) 22  !*((!* * 44	(	((	(	)	))	)   ((]CG0700HQ=QQQGhG D'*7M'BD DG			'A/R11,,.C[(R$_
 !73*,>Yryy%9-;)%M   &&&s,Dell,DV,DmT7JF)N123I3I-hCI//1F*H&&
 
]	s#45	M_9M	MM	MM	M  *T^^W%}6M M...) M%M M,}%4s6{B."n~58IIn,>/n,~=.

 C$8 M
n
M!-F!;c)n!LM N M N M $& "#"..YY22& 
FFHgxz"CMM3v;!8
-.7
8 8r   c                  (    e Zd ZU dZdgZded<   d Zy)InferParamsCacheEntryz.Mutable value object for _infer_params_cached.pjit_paramszPjitParams | Nonec                    d | _         y r   )r  r   s    r   __init__zInferParamsCacheEntry.__init__  s
    Dr   N)r   r   r   r   	__slots__r   r  r   r   r   r  r    s    6o)  r   r  c                    t               S r   )r  )r   r   	signaturer   r  s        r   rf  rf    s     
	  r   c                X   |j                   rpt        j                  j                  j                  }t        |      5  t        j                  |j                        5  t        | |||      cd d d        cd d d        S t        | |||      S # 1 sw Y   nxY wd d d        ## 1 sw Y   ,xY wr   )	r   mesh_libthread_resourcesenvphysical_meshrB   use_abstract_meshabstract_mesh_infer_params_internal)r   r  r   r   	phys_meshs        r   r   r     s     ))--;;I
%i
0 ;

$
$Y%<%<
=;#CT6:; ; ; 
 Rv	66; ; ; ; ;s#    B B*	B B	B  B)c           
        t        j                         }t        d| |||j                  |j                  |j
                  |j                        }t        d t        ||f      D              }t        j                  j                  s|r&t        | |||||d       \  }}||j                  |z   fS t        j                  |t!        |j#                               t!        |j%                               |j                  |j                  t&        j(                        \  }	}
t+        | ||
      }t-        | ||	||      }|j.                  ht        | ||||||      \  }}|j0                  s/|j2                  s#|j4                  d   j6                  j8                  r||j                  |z   fS ||_        |j.                  |j.                  j                  |
z   fS )Nr   )ry   r{   
sourceinfor  c              3  H   K   | ]  }t        |t        t        f        y wr   r   rb   rc   r   s     r   r   z)_infer_params_internal.<locals>.<genexpr>  s     R*Qd,Rs    ")r   r   )r  get_concrete_meshr&   ry   r{   rk   rl   r  rL   r   rw  r   r  r   r3   parse_argumentsrM  valueskeysr   default_registry_infer_input_typerf  r  r   r   r   r   r   )r   r  r   r   r  r  	any_boxesr   r   r  dynargsavalsentrys                r   r  r    s    '')(S$r/@/@((R5F5F  	"#
 Rk4.6QRR)  I%c2x&*FTCLAyahh"""..
E&--/"E&++-$8":K:K)446)W Cg
.%
sB	5(
C%
%R3fu>LAy!**(9(?(?(G(G9$$$E			E--44w>	>>r   c           
        g }	 t        |      D ])  \  }}|j                  t        j                  |             + 	 t        j                  j                  rt        |||       t        |      S # t        $ r# d|j
                      }t	        d| d      d t        $ r2 d|j
                      }t        d|  dt               d| d      d w xY w)	Nzargument path is zUAn overflow was encountered while parsing an argument to a jitted computation, whose .zpath zError interpreting argument to z8 as an abstract array. The problematic value is of type z# and was passed to the function at z.
This typically means that a jit-wrapped function was called with a non-array argument, and this argument was not marked as static using the static_argnums or static_argnames parameters of jax.jit.)rN  r   r   r  OverflowErrorr   r   r   r   r  r   r(   rM  )r   r  r  r  r   r   arg_patharg_descriptions           r   r  r    s
   
%-( /1ll4**1-./$   &&sE=9	u% 
 "3==#3"45H
$:Q(  
 	cmmA./0O
'u -++/7) 4)* +BB 	s   7A. .A%Cc                   t        |      }| D cg c]  \  }}|rt        |      nd }}}t        |d      J ~~t        |       D ]  \  }\  }}|rt        |t        j
                        s&||   }	|	J t        |j                  |	j                  j                        D ]c  \  }
}t        |
t        j                        s!||
j                     |||
j                  <   t	        j                  ||
j                     |      rcJ   t        d |D              sJ t        ||       D cg c]  \  }\  }}|r| c}}}S c c}}w c c}}}w )z
  Given an input type and explicitly-passed arguments (per the user-facing API
  calling convention), extract implicit axis size arguments from shapes of
  explicit arguments (for the trace-time / jaxpr-level calling convention).
  Nc              3  $   K   | ]  }|d u 
 y wr   r   r   s     r   r   z)_extract_implicit_args.<locals>.<genexpr>  s     )qQd])   )iternextrN  r   r   DShapedArrayr   r_  r   DBIdxvalsame_referentr  )r  r  explicit_args_r   explr   r   r   explicitr   d1d2r   r   s                 r   r  r    sD    &.@G	HWQ$$~
D
0	H$	H	nd	#	++	+^ 'w/ 	4a	$:dD,=,=>
q'C??djj#((..1 4B	B

	#<$rvv,!!$rvv,333	4	4 
)D)	))	) w/	9	9	6Aqq!	99! 
I  
:s   E-E
;E
c                8    | y |rt         d }t        | ||      S )Nc                    t        | t              xr t        | j                               xs t        | t              xr t        | d       S )Nc                
    | d u S r   r   rj  s    r   rk  z3_flat_axes_specs.<locals>.ax_leaf.<locals>.<lambda>  s
    Q$Y r   )r   r  rO   r  rM  )ls    r   ax_leafz!_flat_axes_specs.<locals>.ax_leaf	  s?    q$:Jqxxz$: Hq% GZ3F%GIr   )NotImplementedErrorrN   )r   r   r   r  s       r   r  r    s*    T&&I 
/4	99r   c                      e Zd Zd ZddZy)
JitWrappedc                    t         )zSee ``jax.eval_shape``.r  r   r   r   s      r   rB  zJitWrapped.eval_shape  s    
r   c                    t         r   r	  r
  s      r   r@  zJitWrapped.trace  s    
r   Nreturnzstages.Traced)r   r   r   rB  r@  r   r   r   r  r    s    r   r  Fc                4    t        | |||||||||	|
||d      S )u&  Makes ``fun`` compiled and automatically partitioned across multiple devices.

  NOTE: This function is now equivalent to jax.jit please use that instead.
  The returned function has semantics equivalent to those of ``fun``, but is
  compiled to an XLA computation that runs across multiple devices
  (e.g. multiple GPUs or multiple TPU cores). This can be useful if the jitted
  version of ``fun`` would not fit in a single device's memory, or to speed up
  ``fun`` by running each operation in parallel across multiple devices.

  The partitioning over devices happens automatically based on the
  propagation of the input partitioning specified in ``in_shardings`` and
  the output partitioning specified in ``out_shardings``. The resources
  specified in those two arguments must refer to mesh axes, as defined by
  the :py:func:`jax.sharding.Mesh` context manager. Note that the mesh
  definition at :func:`~pjit` application time is ignored, and the returned function
  will use the mesh definition available at each call site.

  Inputs to a :func:`~pjit`'d function will be automatically partitioned across devices
  if they're not already correctly partitioned based on ``in_shardings``.
  In some scenarios, ensuring that the inputs are already correctly pre-partitioned
  can increase performance. For example, if passing the output of one
  :func:`~pjit`'d function to another :func:`~pjit`’d function (or the same
  :func:`~pjit`’d function in a loop), make sure the relevant
  ``out_shardings`` match the corresponding ``in_shardings``.

  .. note::
    **Multi-process platforms:** On multi-process platforms such as TPU pods,
    :func:`~pjit` can be used to run computations across all available devices across
    processes. To achieve this, :func:`~pjit` is designed to be used in SPMD Python
    programs, where every process is running the same Python code such that all
    processes run the same :func:`~pjit`'d function in the same order.

    When running in this configuration, the mesh should contain devices across
    all processes. All inputs arguments must be globally shaped.
    ``fun`` will still be executed across *all* devices in the mesh,
    including those from other processes, and will be given a global view of the
    data spread across multiple processes as a single array.

    The SPMD model also requires that the same multi-process :func:`~pjit`'d
    functions must be run in the same order on all processes, but they can be
    interspersed with arbitrary operations running in a single process.

  Args:
    fun: Function to be compiled. Should be a pure function, as side-effects may
      only be executed once. Its arguments and return value should be arrays,
      scalars, or (nested) standard Python containers (tuple/list/dict) thereof.
      Positional arguments indicated by ``static_argnums`` can be anything at
      all, provided they are hashable and have an equality operation defined.
      Static arguments are included as part of a compilation cache key, which is
      why hash and equality operators must be defined.
    in_shardings: Pytree of structure matching that of arguments to ``fun``,
      with all actual arguments replaced by resource assignment specifications.
      It is also valid to specify a pytree prefix (e.g. one value in place of a
      whole subtree), in which case the leaves get broadcast to all values in
      that subtree.

      The ``in_shardings`` argument is optional. JAX will infer the shardings
      from the input :py:class:`jax.Array`'s, and defaults to replicating the input
      if the sharding cannot be inferred.

      The valid resource assignment specifications are:

      - :py:class:`Sharding`, which will decide how the value
        will be partitioned. With this, using a mesh context manager is not
        required.
      - :py:obj:`None` is a special case whose semantics are:
          - if the mesh context manager is *not* provided, JAX has the freedom to
            choose whatever sharding it wants.
            For in_shardings, JAX will mark is as replicated but this behavior
            can change in the future.
            For out_shardings, we will rely on the XLA GSPMD partitioner to
            determine the output shardings.
          - If the mesh context manager is provided, None will imply that the
            value will be replicated on all devices of the mesh.
      - For backwards compatibility, in_shardings still supports ingesting
        :py:class:`PartitionSpec`. This option can *only* be used with the
        mesh context manager.

        - :py:class:`PartitionSpec`, a tuple of length at most equal to the rank
          of the partitioned value. Each element can be a :py:obj:`None`, a mesh
          axis or a tuple of mesh axes, and specifies the set of resources assigned
          to partition the value's dimension matching its position in the spec.

      The size of every dimension has to be a multiple of the total number of
      resources assigned to it.
    out_shardings: Like ``in_shardings``, but specifies resource
      assignment for function outputs.
      The ``out_shardings`` argument is optional. If not specified, :py:func:`jax.jit`
      will use GSPMD's sharding propagation to determine how to shard the outputs.
    static_argnums: An optional int or collection of ints that specify which
      positional arguments to treat as static (compile-time constant).
      Operations that only depend on static arguments will be constant-folded in
      Python (during tracing), and so the corresponding argument values can be
      any Python object.

      Static arguments should be hashable, meaning both ``__hash__`` and
      ``__eq__`` are implemented, and immutable. Calling the jitted function
      with different values for these constants will trigger recompilation.
      Arguments that are not arrays or containers thereof must be marked as
      static.

      If ``static_argnums`` is not provided, no arguments are treated as static.
    static_argnames: An optional string or collection of strings specifying
      which named arguments to treat as static (compile-time constant). See the
      comment on ``static_argnums`` for details. If not
      provided but ``static_argnums`` is set, the default is based on calling
      ``inspect.signature(fun)`` to find corresponding named arguments.
    donate_argnums: Specify which positional argument buffers are "donated" to
      the computation. It is safe to donate argument buffers if you no longer
      need them once the computation has finished. In some cases XLA can make
      use of donated buffers to reduce the amount of memory needed to perform a
      computation, for example recycling one of your input buffers to store a
      result. You should not reuse buffers that you donate to a computation, JAX
      will raise an error if you try to. By default, no argument buffers are
      donated.

      If neither ``donate_argnums`` nor ``donate_argnames`` is provided, no
      arguments are donated. If ``donate_argnums`` is not provided but
      ``donate_argnames`` is, or vice versa, JAX uses
      :code:`inspect.signature(fun)` to find any positional arguments that
      correspond to ``donate_argnames``
      (or vice versa). If both ``donate_argnums`` and ``donate_argnames`` are
      provided, ``inspect.signature`` is not used, and only actual
      parameters listed in either ``donate_argnums`` or ``donate_argnames`` will
      be donated.

      For more details on buffer donation see the
      `FAQ <https://docs.jax.dev/en/latest/faq.html#buffer-donation>`_.
    donate_argnames: An optional string or collection of strings specifying
      which named arguments are donated to the computation. See the
      comment on ``donate_argnums`` for details. If not
      provided but ``donate_argnums`` is set, the default is based on calling
      ``inspect.signature(fun)`` to find corresponding named arguments.
    keep_unused: If `False` (the default), arguments that JAX determines to be
      unused by `fun` *may* be dropped from resulting compiled XLA executables.
      Such arguments will not be transferred to the device nor provided to the
      underlying executable. If `True`, unused arguments will not be pruned.
    device: This argument is deprecated. Please put your arguments on the
      device you want before passing them to jit.
      Optional, the Device the jitted function will run on. (Available devices
      can be retrieved via :py:func:`jax.devices`.) The default is inherited
      from XLA's DeviceAssignment logic and is usually to use
      ``jax.devices()[0]``.
    backend: This argument is deprecated. Please put your arguments on the
      backend you want before passing them to jit.
      Optional, a string representing the XLA backend: ``'cpu'``, ``'gpu'``, or
      ``'tpu'``.

  Returns:
    A wrapped version of ``fun``, set up for just-in-time compilation and
    automatically partitioned by the mesh available at each call site.

  For example, a convolution operator can be automatically partitioned over
  an arbitrary set of devices by a single :func:`~pjit` application:

  >>> import jax
  >>> import jax.numpy as jnp
  >>> import numpy as np
  >>> from jax.sharding import Mesh, PartitionSpec
  >>> from jax.experimental.pjit import pjit
  >>>
  >>> x = jnp.arange(8, dtype=jnp.float32)
  >>> f = pjit(lambda x: jax.numpy.convolve(x, jnp.asarray([0.5, 1.0, 0.5]), 'same'),
  ...         in_shardings=None, out_shardings=PartitionSpec('devices'))
  >>> with Mesh(np.array(jax.devices()), ('devices',)):
  ...   print(f(x))  # doctest: +SKIP
  [ 0.5  2.   4.   6.   8.  10.  12.  10. ]
  Tr  )r  )r   r#  rZ  ry   r{   r|   r}   r   r   r   r   r   r  s                r   r6  r6    s5    n 
	M#_#_fgf%8H
 r   c                ^    t        |       \  t              t        fdf      S )Nc                     t               S r   )rI   )r   valss   r   rk  z!hashable_pytree.<locals>.<lambda>  s    .$"? r   r  )rH   rM  rV   )pytreer   r  s    @@r   hashable_pytreer    s1    v&-$	t$	?#*D/
3 3r   c                z   || | j                   rt        S t        |t        t        t
        f      r|S | 8d| d}|dk(  r|d| dz  }t        |      |dk(  r	|d| dz  }t        |      | j                   rt        | d| d	| d
      |t        |t              sJ |       t        j                  | |      S )Nz2jax.jit only supports `Sharding`s being passed to z_. Looks like you are passing either `PartitionSpec` or `None` which is not allowed in jax.jit.
r#  z
Note that z argument is optional. JAX will infer the shardings from the input jax.Array's and will default to replicating the input if the sharding cannot be inferred.rZ  z is optional. If not specified, jax.jit will use GSPMD's sharding propagation to figure out what the sharding of the output(s) should be.zL requires a non-empty mesh if you are passing `PartitionSpec`s or `None` to zL! Is a mesh defined at the call site? Alternatively, provide `Sharding`s to z3 and then the mesh context manager is not required.)
r  r<   r   r;   r=   r6   r1  r+   r   create_mesh_pspec_sharding)r   r   r   api_namer   s        r   r  r    s   YDLDJJD*H56H	\f 12C ~	
4& !; ; <c s
	 
	 	
4& !- - .c s
	ZZ
* **. 088<v >:	:; ; 
jM25A5	2		2	24	;;r   c                    | |J t        |       }nA|4| J t        t        j                  |      j                         d         }nt	        d      d|_        |S )Nr   zUnreachable!T)r9   xbget_backendlocal_devicesr   _device_backend)r   r   r`  s      r   r  r    se    ??
v
&C>>
r~~g6DDFqI
JC

((#	*r   c           
     F   	 t        t        | |||            S # t        $ r Y nw xY wt        t	        |            rJ |rt        |      t         us%t        |      t        |j                               k7  r|  d|  d}t        |      t         ur|d|  dt        |       dz  }nTt        |      t        |j                               k7  r/|d|  dt        |       dt        |j                                d	z  }t        |j                               d
k(  r;t        | ||f       |d|  dz  }t        |      # t        $ r Y t        |      w xY wt        |      |}t        |t               g|j                  z        }t        ||      }|r|d   } ||       J d       )Ntupled_argsz specification must be a tree prefix of the positional arguments tuple passed to the `pjit`-decorated function. In particular, zj must either be a None, a PartitionSpec, or a tuple of length equal to the number of positional arguments.z But z is not a tuple: got z	 instead.z4 is the wrong length: got a tuple or list of length z for an args tuple of length r     z> Given the corresponding argument being passed, it looks like z/ might need to be wrapped in a singleton tuple.r   zPlease open a bug report!)rM  r   r   rJ   rK   r   r   r   rI   
PytreeLeafr   rP   )	whattreerr  r  r   	axis_tree
dummy_treeerrorsr   s	            r   flatten_axis_resourcesr&    s   	dD)MNN	 		
 ^I677	7 d9oU2)nDMMO(<< V   LMC Ie#	uTF/Y/@	JJc	Y3t}}/	/	dVOY  =T]]_%&a) *c 4==?q dYL1 	 ))- /%& 	' S/  S/  S/) dZ\NT__$DE*J/&q	A
D'M ,++s    	&&D9 9	EEc                      e Zd Zd Zy)r   c                     y)Nzpytree leafr   r   s    r   __repr__zPytreeLeaf.__repr__C  s    ]r   N)r   r   r   r)  r   r   r   r   r   B  s    *r   r   i   )max_sizetrace_context_in_keyc	           	        |st        |      \  }}	t        | |      }
t        |
t              r|
ft	        |      z  }nt        d||
d      }t        ||      }||ft	        |      z  }nt        d||d      }t	        |j                        t	        |      k7  }t        j                  j                  sR|sPt        || |j                  t	        |            dd       t        || |j                  t	        |            d       ||fS )	Nzpjit in_shardingsTr  zpjit in_layoutszpjit argumentsFallow_uneven_shardingzjit arguments)rM   rI   r   r=   r   r&  r   r   rw  r   pjit_check_aval_shardingsafe_arg_namescheck_aval_layout_compatibility)ro   rq   rt   ru   r   rQ  r&   r  kwsr   orig_in_shardingsr  r  r  r   s                  r   r  r  F  s   
 
!'*JGQ$%9;NO !#34*,s8}<.W&7TK 02CD*!mc(m3O,7JDBO j**+s8}<-				$	$].6Z66s8}E-UL $!
!!#h-0/C 
O	++r   zset[str]!callsites_with_tracing_cache_missc                   ! | \  }}}\  }}}}	|\  }
}}\  }}}}g  d# fd}d$d!dd% !fd}d% !fd}d& fd}	 	 d' !fd}||
k7  rt        |      t        |
      k7  rd |d	||
       nt        t        ||
            D ]  \  }\  }}|d
   j                  }||k(  r|dk(  r$|d
   |d
   k7  r |d| d||       >|dk(  r ||d   d
   |d   d
          Y|dk(  r ||d   |d          n|dk(  r ||d   |d           |d| d|dd |dd         r D cg c]  }d|d
   vs| c} ||k7  r
 |d||       ||k7  r
 |d||       ||k7  rs	 |||       ||k7  r
 |d||       ||k7  r
 |d||       |	|k7  rt        |	      t        |      k(  sJ t        t        |	|            D cg c]  \  }\  }}||k7  rd| d| d|  }}}} j	                  ddj                  |      z   dz   t        |      f        s
 |d | |       t        j                  t         d! "            }|d
   t        |d         fS c c}w c c}}}w )(a0  Explanations of differences between the cache keys, along with diff sizes.

  Result: a pair of a list of explanations for differences, and the total size
    of the differences. The sizes are used to pick the old key with the smallest
    different size for the explanation that is shown to the user.
  c           	     @    j                  d|  d| d| ddf       y )Nz
different z:
    now: z
 != before: zQ.
explanation unavailable! please open an issue at https://github.com/jax-ml/jax.
   )r   )	key_fieldwhat_kwhat_okdiffss      r   unavailablez,diff_tracing_cache_keys.<locals>.unavailabley  s9    	LLi[VHN7) LB 
B 
	r   c           
         t        t        |       t        |            }t        t        |       t        |            |z
  }|t        d t	        | d | |d |       D              z  }|S )Nc              3  ,   K   | ]  \  }}||k7    y wr   r   )r   e1e2s      r   r   zBdiff_tracing_cache_keys.<locals>.list_diff_size.<locals>.<genexpr>  s      ?&"bR2X ?s   )minr   maxr   r   )s1s2min_len	diff_sizes       r   list_diff_sizez/diff_tracing_cache_keys.<locals>.list_diff_size  sh    #b'3r7#GCGSW%/I ?C8G028G-> ? ? ?Ir   Fc           	         | \  }}|\  }}||k7  rj                  d| d| df       ||k7  rMj                  ddj                  d |D               ddj                  d |D                ||      f       y y )Nz0different static_argnums:
  dynamic argnums now  and before r  z&different value of static args:
  now , c              3  F   K   | ]  }t        |j                          y wr   reprr  r   r"  s     r   r   zUdiff_tracing_cache_keys.<locals>.explain_transform_argnums_partial.<locals>.<genexpr>  s     Aad155kAr  c              3  F   K   | ]  }t        |j                          y wr   rL  rN  s     r   r   zUdiff_tracing_cache_keys.<locals>.explain_transform_argnums_partial.<locals>.<genexpr>  s     #HADK#Hr  )r   join)param_kparam_okdyn_argnums_kstatic_args_kdyn_argnums_okstatic_args_okr;  rG  s         r   !explain_transform_argnums_partialzBdiff_tracing_cache_keys.<locals>.explain_transform_argnums_partial  s    #* M=%-"NN&ll$$1?,~>NP
 &llIIA=AAB$))#H#HHIK 
79: 'r   c           	        | \  }|\  }t        |j                  j                               D cg c]  \  }}||j                  f }}}t        |j                  j                               D cg c]  \  }}||j                  f }}}||k7  rNj                  ddj	                  d |D               ddj	                  d |D               d ||      f       y y c c}}w c c}}w )Nz)different value of static kwargs:
  now {rJ  c              3  B   K   | ]  \  }}| d t        |         yw: NrM  r   kvs      r   r   zVdiff_tracing_cache_keys.<locals>.explain_transform_argnames_partial.<locals>.<genexpr>  s#     Nda1#RQy 1N   z} and before {c              3  B   K   | ]  \  }}| d t        |         ywrZ  r\  r]  s      r   r   zVdiff_tracing_cache_keys.<locals>.explain_transform_argnames_partial.<locals>.<genexpr>  s#     %UDAq2d1gY&7%Ur`  })sortedr  r~  r   rP  )rQ  rR  static_kwargs_kstatic_kwargs_okr^  r_  r;  rG  s         r   "explain_transform_argnames_partialzCdiff_tracing_cache_keys.<locals>.explain_transform_argnames_partial  s    O o11779:<da155z <O < /3399;<>tq!AEE
 > >**lliiNoNNO P II%UDT%UUVVXZ *:
;=> +	<>s   C 1C&c                   | j                   |j                   k7  st        |       \  }}t        t        |            }t        |      \  }}t        t        |            }t        |      d t        ||       t        |      d t        ||       k(  rR|j	                         d   }|j	                         d   }	j                  d| d| d| d|	 t        ||z
        f       y t        |       }
t        |
      dk  r|
n|
d d dz   }
t        |      }t        |      dk  r|n|d d dz   }d|
 d| g}t        t        j                  | |            }|D ]M  \  }}}}|^}}d	d
g|j                     }|j                  d| t        t        |             d| d| d| 	       O j                  dj                  |      t        |      f       y )Nr  zBdifferent number of args and kwargs, but same total number.
  now z args and kwargs with keys z

  before I   ...zdifferent input pytree:
  now: z
  before: r   r   z  * at , now rI  z, so 
)r   rM   r   rA  	node_datar   absrj   r{  r   equality_errors_pytreedefidxrQ   rM  rP  )	in_tree_k
in_tree_okargs_tree_kkwargs_tree_k	nr_args_kargs_tree_okkwargs_tree_ok
nr_args_okkeys_kkeys_okin_tree_k_strin_tree_ok_strdifferrspaththing1thing2explanationfstbasedifferent_leaf_countr;  s                       r   explain_in_tree_diffz5diff_tracing_cache_keys.<locals>.explain_in_tree_diff  s(   %00J4I4II $4I#> k=&{34i%5j%A"lN'45j
;
'(CY
)C
D
<
()D#i*D
EF((*1- **,Q/ k "!( #$ &")	%
 i'(*	+ 		NM&)-&82&=]',u4 _N(+N(;b(@n("-5 .}o >'(* +D 	33IzJKD-1  )ffkjcDh(d
kkD6&t-.fVHL Q   
LL$))D/3t9-.r   c           	        d} j                   t        |             }d dj                  fdt        ||       D              }t        |      dk  r|n|d d dz   }d| g}d}t        || |      D ]8  \  }}}	||	k(  rd}
t	        |      t	        |	      cxk(  rt
        j                  k(  rn n |       |	      }}|
 |j                  |	j                        z  }
|j                  |	j                  k7  r.|d	|j                   d
z  }|d	|	j                   d
z  }d}|
dz  }
n[|j                  |	j                  k7  rB|j                  dd      }|	j                  dd      }|
dz  }
nt        |      t        |	      }}|t        d|
      z  }|j                  d| d| d|        ; |r|j                  d       j                  dj                  |      |f       y )Nr   c                T    t        | d      r| j                  d      S t        |       S )Nr   T)short_dtypes)r  r   rj   )ats    r   arg_type_to_strzPdiff_tracing_cache_keys.<locals>.explain_args_type_diff.<locals>.arg_type_to_str  s&    	[	!|||..2wr   rJ  c              3  >   K   | ]  \  }}| d  |         ywrZ  r   )r   anr  r  s      r   r   zJdiff_tracing_cache_keys.<locals>.explain_args_type_diff.<locals>.<genexpr>  s0      A%r2 !TOB$7#89 As   rh  ri  z$different input types:
  types now: Fz{weak_type=rb  Tr  )r  mesh_axis_typesz	    * at rj  rI  zwhere weak_type=True often means a Python builtin numeric value, and 
weak_type=False means a jax.Array.
See https://docs.jax.dev/en/latest/type_promotion.html#weak-types.rk  )r0  r   rP  r   r   r   ShapedArrayr_  r\  r[  r   rj   rB  r   )args_kargs_okrF  r   
args_k_strr|  add_weak_type_hintr   arg_t_karg_t_okthis_arg_diff_sizerC  rD  r  r&   r;  rG  s                @r   explain_args_type_diffz7diff_tracing_cache_keys.<locals>.explain_args_type_diff  s   I)
))#f+6I
  A),Y)?A AJ":3CR59PJ3J<@AD#&y&'#B @gx	H	h	g$x.	<D,<,<	< )?8+DBnW]]HNNKK 2 22
,w0014
4"
,x112"5
5"#


!
!2!22  dD I"!!tT!J"

!
Ws8}B3q,--i
kkIdV6"\">?'@* 
kkOP 
LL$))D/9-.r   Tzfun_transforms lengthr   r  zfun_transforms[z] transformr    r  _argnums_partial_argnames_partialzfun_transforms.z paramsNzfun_transforms._argnums_partial
fun_paramsfun_in_typearg_attr_data
arg_inlinez  [z]: now rI  zadifferent tracing context, e.g. due to config or context manager.
found differences at positions
z, and
zK
compare to tuple returned by config.trace_context() in jax/_src/config.py.z"something (unexpected empty diffs)c                    | d   S Nr  r   )rK  s    r   rk  z)diff_tracing_cache_keys.<locals>.<lambda>1  s
    AaD r   key)r8  rj   )rC  r   rD  r   r  r  )rQ  rM  rR  rM  )rp  rR   rq  rR   )r  tuple[core.AbstractValue]r  r  )
r   rN  r   r   r   rP  r   unzip2rc  r   )"r^  oldkr&   fun_transforms_kfun_params_kfun_in_type_karg_in_type_karg_attr_data_karg_inline_kctx_kfun_transforms_okfun_params_okfun_in_type_okarg_in_type_okarg_attr_data_okarg_inline_okctx_okr<  rW  rf  r  r  r   tott_namerK  c_kc_okidxsdiffs_and_sizesr  r;  rG  s"     `                            @@@r   diff_tracing_cache_keysr  k  s    =>:\=1M?L5@D>m^4N$mf!#% :>'/R)/&?)/V **
$5 66!)"$57 "#&68I"JK *!Wa17H _$hQ42a5=
s+62
>
]"
qtAw1a
1
''
+AaD"Q%
8
((
,QqT2a5
9
ofXW5qubfE)0 O1!B!A$!NQOE]"lM:n$|];n$-A=.9((2BC]"lM:
f_u:V$$$"+Cv,>"?P P;C3$; !GC5TF3 PD P	LL
,		

9
9
 
T	 
4a>KKu. AB/
!
c/!"45	663 PPs   'I4I>Ic                   t         j                  j                  ry |d   d   j                  ry | j                  }|j
                  }|rt        j                  |      sy g j                  fd}t        j                  t        j                               } d| d|dz  dd       d	}	|r|	d
| z  }	|j                  x}
r|	d|
 z  }	|j                  }|s|sV d| dt        | j                         |	        |t        v r d        |       S t        j!                  |        |       S  d| |	        |j#                         D cg c]  }||k7  rt%        |||       }}|sJ d       t'        |d       }|D cg c]  }|d   |d   k(  s|d    }}dddfd}t)        |      dk(  r d        ||d          n d       |D ]
  } ||         |        y c c}w c c}w )N      c                 j    t         j                  t        j                  dj	                               S )Nrk  )loggerlogloggingWARNINGrP  )r   s   r   rk  z,explain_tracing_cache_miss.<locals>.<lambda>B  s    GOOTYYs^< r   zTRACING CACHE MISS at z	 costing g     @@z.3fz ms because:r   z defined at :z  never seen function:
    z id=z~  but seen another function defined on the same line; maybe the function is
  being re-defined repeatedly, preventing caching?z  for z:we must find some diffs if key differs from all cache keysc                    | d   S r  r   )r_  s    r   rk  z,explain_tracing_cache_miss.<locals>.<lambda>\  s
    ad r   r  r  r   c                2    |j                  ddd| z  z         S )Nrk   )replace)indentr   s     r   indent_subsequent_linesz;explain_tracing_cache_miss.<locals>.indent_subsequent_lines_  s    ;;tTC&L011r   c                8    | D ]  } d d|      z           y )Nz  * key with    r   )r|  rK  r  r   s     r   
p_one_diffz.explain_tracing_cache_miss.<locals>.p_one_diffa  s'     91!Q7789r   zE  all previously seen cache keys are different. Closest previous key:zR  all previously seen cache keys are different. Several previous keys are closest:)r  r  r   rj   r  rj   )r|  Sequence[str])r   check_tracer_leaksr   r  r&   func_filenamer   is_user_filenamer   	summarizecurrentfunc_lineno	func_namer   r  r4  addr  r  rA  r   )r   unseen_fcacher  elapsed_secr&   r  donecallsitesrc_infor  r  okr;  min_diffrK  smallest_diffsr  r  r   r   s                     @@@r   explain_tracing_cache_missr  5  s,    $$fVAY]]F~~***-+<<]K	#	jj!	<$''(8(@(@(BC(XJic0A#/FlST (,}o..H***[*!K=!!H"")U$YKtBsuuI;xjIJ44 	= > 6M (++H56MfYKz"# ZZ\0SBY #3J7 0% 0	LLLN+("'?Q1Q48A;+>AaD?.?29 	AMN~a ! + , m &+0
 @s   G2G7G7)explainr  c           
        t        j                  d       ~t        j                  j                  rt        d| j                   d      t        j                  d| j                  t        j                        5  t        j                  j                  rHt        j                  t        j                  | t!        t"        j$                  |                  \  }}}g }nt        j&                  | |      \  }}}}d d d        t        j(                  j                  rddlm}  |       t/        d D              r+t        j0                  t        j2                              }	|}
nt#        j4                  |      }	g }
|	|
fS # 1 sw Y   xY w)	Ncreate_pjit_jaxprr.  r/  zMFinished tracing + transforming {fun_name} for pjit in {elapsed_time:.9f} sec)r`   eventr   )check_key_reuse_jaxprc              3     K   | ]?  }t        |t        j                        xs t        j                  |      j                   A y wr   )r   r   Tracertypeofr  r   cs     r   r   z%_create_pjit_jaxpr.<locals>.<genexpr>  s1     N!At{{	#	=t{{1~'='=	=Ns   AA)r   r  r   r0  r   r1  r  r   log_elapsed_timer   JAXPR_TRACE_EVENTrw  r   trace_to_jaxpr_dynamic2r  annotater   r   r  trace_to_jaxpr_dynamicr    jax.experimental.key_reuse._corer  r  close_jaxprconvert_constvars_jaxprClosedJaxpr)r   r  	attr_dataignored_inliner   global_out_avalsr   r   r  closed_jaxprfinal_constss              r   rd  rd  q  sT    //%&
-ceeW 5- - . .  U||8#=#=? _ ""(*(B(B
++c48
9);%evm797P7PQTV]7^4ev}_ !!F%  	NvNN>>""<"<U"CDLL##E62LL	|%5}	DD/_ _s   :A>FFc           	        t        | |      }t        |t        t        f      r|ft	        |      z  }	nt        d |       |d      }	t        ||      }
|
|
ft	        |      z  }nt        d |       |
d      }t        j                  j                  sPt        |	| |j                  t	        |            dd       t        || |j                  t	        |            d       |	|fS )Nzpjit out_shardingsFr  zpjit out_layoutszpjit outputsr-  zjit outputs)rI   r   r=   r6   r   r&  r   rw  r   r/  safe_result_pathsr1  )rr   rs   rv   rw   r   r  r&   r  orig_out_shardingsr  r  r  s               r   r  r    s     &&;=QR"%5x$@A,.Y?/hj*< 24FG+#~I6-HJH 
			$	$I$
$$S^4e5 $)$
$$S^4 
-	--r   c                    t         j                  | j                  t        t                    }| j
                  | j
                  |k(  sJ || j                  | j                  |f   S r   )_seen_attrs
setdefaultr  r   r{  r  
transformsr   )r   r  r  s      r   seen_attrs_getr    sT     
 
 D(9
:%		w 66	6	

G4	55r   c                F   t        | |      }t        |      D ]}  \  }}|D ]p  \  }}}}}	|t        j                  t        j                  fv s,t        ||t              }
t        |
      \  }}t        t        j                  |      }||k7  s|	|k7  sp z |c S  t        |      S r   )r  rN  r   r  r   r   rd   rH   r   r   r  r   )r   r  casesr   recordsobjattrr   r   r  r  r  treedef_avals_s                 r   r  r    s     g
&%e$ ja+2 'T4%	",,

+	+c4.%c*hT,,d3h%6/5 h 
Ur   c                    d }|D 	cg c]0  \  }}\  }}}	|||	|t        t        j                   |||            f2 }
}}}}}	t        | |      }|t	        |      k(  r|j                  |
       y y c c}	}}}}w )Nc                6    t        t        | |t                    S r   )rL   r   rd   )r  r  s     r   rk  z(_attr_cachedata_update.<locals>.<lambda>  s    [dL)IJ r   )r   r   r  r  r   r   )r   r  r   r   r   r  r   r  r  r   r  r   s               r   r  r    s    J&4AC C0A0T4 4y#dkk6#t;L*MN C' C
g
&%#e*_	LL Cs   5A4T)frozenc                  $    e Zd ZU ded<   d Zd Zy)r  r
   r  c                ,    t        | j                        S r   )hash	__class__r   s    r   r   zIgnoreKey.__hash__  s    r   c                "    t        |t              S r   )r   r  r   s     r   r   zIgnoreKey.__eq__  s    eY''r   N)r   r   r   r   r   r   r   r   r   r  r    s    
( (r   c                <   t        || |      D ]  \  }}}t        |t        t        f      r|rd| nd}|j                  }		 t        |d      r|j                  |	       n|j                  t        |	             |j                  t        |	            }|J t        j                  |      \  }}t        |      D ]6  \  }}|r	|	|   |z  dk7  st        d| | d| d	| d
| d|	|    d|	 d        y # t        $ r}
t        d| | d| d|
       d }
~
ww xY w)N with pytree key path r   check_compatible_avalOne of z. is incompatible with its sharding annotation r[  r   z was given the sharding of z6, which implies that the global size of its dimension z should be divisible by z, but it is equal to z (full shape: ))r   r   r=   r;   r_  r  r  _to_xla_hlo_shardingr   r   r   get_num_ways_dim_shardedrN  )rr  
flat_avalsnames	what_avalr.  r   r!  r   name_strr_  r   hlo_shardingnum_ways_dim_shardedr   r   sizes                   r   r/  r/    sv    :y%8 3mdAt!&-.26'v.BHJJE
" 
+	,	&	s5z* ))#e*5L###*CCLQ!12 34"uQx$!';79+hZ 8 c "==>C @))-.CE!H: N)).q	2 3 	33-3  "I;xj )s"QC!" ""s   8C77	D DDc                    t        || |      D ]>  \  }}}|t        |t              r|rd| nd}	 |j                  |j                         @ y # t
        $ r}t        d| | d| d|       d }~ww xY w)Nr  r   r  z, is incompatible with its layout annotation r[  )r   r   rE   r  r_  r   )	rq  r  r  r  r   r  r   r  r   s	            r   r1  r1    s    :w6 	"mdAtyJq*-26'v.BH"djj)	"  "I;xj )s"QC!" ""s   A	A4A//A4r6  c                .    | j                   j                  S r   )r   r   )r   r   s     r   _is_highr    s    			r   c                t   t        j                  |       \  } }g ||}t        t        |      fi |}| j                  D cg c]  }t        |j                                }}| j                  D cg c]  }t        |j                                }}t        d | j                  D              }t        |||fi |}t        | j                  |      D 	
cg c]:  \  }}	|j                  r|j                  |	      n|j                  |	      D ]  }
|
 < }}	}}
t        j                  |       }t        j                   |d|i|}t#        ||g      \  }}t%        |      }t'        | j(                  j*                        D ci c]  \  }}||
 }}}| j(                  j*                  D ]w  }|j,                  |j,                  }t/        j0                  |t        |j2                  j5                  |                  } |j2                  j6                  ||||      g|  y t9        |d       J t%        |      }| j                  D cg c]<  } |j:                  t/        j0                  |t        |j                                      > }}t9        |d       J |S c c}w c c}w c c}
}	}w c c}}w c c}w )Nc              3  h   K   | ]*  }|j                   st        |j                                , y wr   )r  r   lo_ty)r   r   s     r   r   z_to_lojax.<locals>.<genexpr>*  s!     X$4<<C

%Xs   22r   )r   convert_const_himutables_converted_mutables_add_paramsr   in_aval_qddsr!  r  r   final_aval_qdds_lojax_expand_paramsr   r  
read_loval	lower_vallower_jaxprr   r   r[   r  rN  r   invars	final_qdditislicer   	lo_ty_qddupdate_from_lovalr  	raise_val)r   hi_argsr   closed_over_himutablesr   
lo_nums_inr  lo_nums_outlo_muts_outr   lo_vallo_argslo_jaxprall_outsout_mutlo_outsout_mut_r   r_  in_idxqddlo_valslo_outs_hi_outss                           r   	_to_lojaxrB     s   "$"="=e"D%	/$/w/')#.D*EPP& /4.@.@AdDJJL!A*A).9AQWWY9+9X%2G2GXX+
KOO& $'u'9'97#C 5 5a26,,DOOA.!%!25  5V 5' 5
 ^^E"([['<<V<(;-8'7 '](&u{{'9'9:;TQAqD;&;;; Ba{{KKc		(C(8(8(=$>?gaffsGF1I$6AA	B
 
h		%%	% '](oo' Q[["))Hc!'')n=> '' '	h		%%	%	.? B9
5 <'s    J2 J#?J(J/AJ5c               Z    d| z  |z   }t         f| z  |z   }d| z  |z   }t        ||||      S )Nr  r   )rO  r#  r  r<   r  )nrO  r#  r  r   s        r   r#  r#  J  sE    a<.0.!#l2,{Z'*	f^,#
% %r   c                   d }	 |	| |      } |	| |      } |	| |      } |	||      } |	||      }t         f|z  |z   }d|z  |z   }t        ||||||      }
|
S )Nc                :    t        d t        | |      D              S )Nc              3  <   K   | ]  \  }}|f|z  D ]  }|   y wr   r   )r   rE  r   ys       r   r   z7_lojax_expand_params.<locals>.expand.<locals>.<genexpr>Y  s&     =tq!QD1H=q==r   rM  r   )nsxss     r   expandz$_lojax_expand_params.<locals>.expandX  s    =s2r{===r   r   )rO  r#  r  rZ  r  rD  )nums_innums_outmuts_outrO  r#  r  rZ  r  r   rM  
new_paramss              r   r&  r&  S  s    
>'N3.'L3,'J3*(M3-(K3+ .8+m;-("[0+F>!-*"/[J* 
r   c                P   t        j                  |      rdt        |      z  S g }t        | |||      D ]e  \  }}}}t	        |dd      }	t        |d      r2|j                  j                  }
t        j                  |
||      rd n|
}nd\  }
}t        |t              xs t        t	        |dd       t              }|:|	r&|r|j                  d        |j                  |       |j                  d        |	rf|sd|
bt        j                  ||
      sLd}t        |t              rd}t        d	| d
|
 dt!        j"                  |      j%                          d|       t        |t&              rt        j                  |||      rd n|}|j                  |       h t)        |      S )Nr   r  Tformat)NNr[  r   a   The layout given to `jax.jit` is `DeviceLocalLayout.AUTO` but the corresponding argument passed is a `jax.Array` with a concrete layout. Consider passing a `jax.ShapeDtypeStruct` instead of `jax.Array` as an argument to the jitted function  when using `DeviceLocalLayout.AUTO`.zWLayout passed to jit does not match the layout on the respective arg. Got pjit layout: z,
arg layout:  for arg shape: r  )r0   !check_device_backend_on_shardingsr   rX   r   r  rS  rn  is_default_layoutr   r=   r:   r   is_user_xla_layout_equalrE   r   r   r  r   rD   rM  )r   jit_in_layoutsresolved_in_shardingsr   resolved_in_layoutsr   jit_in_lrsr   	committed
arg_layoutdispatch_arg_layoutis_pmap_sharding	extra_msgs                 r   _resolve_in_layoutsrb  j  s   
 
++,ABS(((!)
N18"= 1+c8R\40I
 sH::11j%)%;%;JD%QT",  )3%j%"2'78 Q"73
D#A<P 	

$
$T
*

$
$%8
9""4(
 "$//*E	h
+6   ..6Z 8))3 5((,(?(?(D(N(N(P'QQR&K) * 	* %X/@A((2t< BJ   *c1+d 
"	##r   c                   g }t        | ||      D ]d  \  }}}||j                  d        t        |t              r)t	        j
                  |||      r|j                  d        T|j                  |       f t        |      S r   )rX   r   r   rD   r0   rV  rM  )r  rZ  r  new_out_layoutsout_lra  out_avals          r   _resolve_out_layoutsrg    s    / (mY O $eUH}T"
U-
.

 
 x
8T"U#$ 
	r   c                    t        | t              r| S |rt        | t              rt        S | S t        | t              sJ t        j                  |       rt        S t        d      )NzFHaving uncommitted Array sharded on multiple devices is not supported.)r   r=   r:   r<   r6   r   is_single_device_shardingr  )arg_sr]  s     r   finalize_arg_shardingrk    sb    '(L&ul;[FFx(((		+	+E	2 !E F Fr   c                V   t        j                  |      r|S g }| D ]]  }t        |dd       }|t        |t              r$t        |dd      s2|j                  |t        j                  j                  d f       _ g }t        | |      D ]  \  }}t        |d      r%|j                  |j                  t        |dd      fnt        df\  }}t        |t              r|j                  j                  rt        d}}t        |t               r|j                  t#        ||             t        |t$        j&                        r>|j(                  s2t+        j,                         dkD  rt/        d|j0                   d|       t        |t               s|j2                  r|j4                  |j4                  k7  rIt/        d|j4                   d	|j4                   d
t7        j8                  |      j;                                |rt        |t              s}t=        j>                  |jA                  |jB                        |jA                  |jB                              s5t/        d| d| d
t7        j8                  |      j;                                |j                  |        tE        |      S )Nr[  r  TFr  a  Passing non-trivial shardings for numpy inputs is not allowed. To fix this error, either specify a replicated sharding explicitly or use `jax.make_array_from_process_local_data(...)` to convert your host local numpy inputs to a jax.Array which you can pass to jit. If the numpy input is the same on each process, then you can use `jax.make_array_from_callback(...) to create a `jax.Array` which you can pass to jit. Got arg shape: z, arg value: zgMemory kinds passed to jax.jit does not match memory kind on the respective arg. Got pjit memory kind: z, arg memory kind: rT  z^Sharding passed to pjit does not match the sharding on the respective arg. Got pjit sharding: z,
arg sharding: )#r0   rU  r   r   r:   r   r   MismatchTypeARG_SHARDINGr   r  r[  r<   r7   r   r  r=   rk  npndarrayis_fully_replicatedr  process_countr   r_  _is_concretememory_kindr   r  r   r   are_op_shardings_equalr  ndimrM  )	r   pjit_in_shardingscommitted_arg_shardingsr"  rj  rY  r   	pjit_in_sr]  s	            r   _resolve_in_shardingsrz    s    
++,=> VaAz4(E } %&q,%$$eV-@-@-M-Mt%TUV /1D"34 -.nc9 #3
38P wsL$'GH)51 E9 %'EJJ,<,<$eYe)-.""#8	#JK
S"**
%++



q
 	 "ii[cU	<
= 
	= /0U5G5G   E$5$5588A8M8M7N O""'"3"3!4 5 33C8BBDEGH H
 5,/33..sxx8**38846  11: <,,17 3))-)@)@)E)O)O)Q(R	T U U
 ""9-[-.^ 
$	%%r   c                    t        | |      }t        | |||j                        }t        |||j                        }t        |||||||||	|
||||      S )N)lowering_platformslowering_parametersr   )rz  rb  r   rg  r  _pjit_lower)r   r   r#  rZ  r  r  rO  r  r   r   r   r|  r}  r   r   s                  r   rR  rR    sh    
 't\:,"4\#(>>3*$[-Q+	\=*khk6;O+-!
# #r   c                Z   i d }}t         j                  j                  rt         j                  j                  dkD  r| }t        j                  |      }|Ot        j                  t         j                  j                  t         j                  j                        }|t        |<   |j                         }|||d<   |
t        |j                               z   }
t        || |||||||||	d t        j                         ||
      j                         }|j                   ret         j"                  j                  rKt%        j&                  ||j(                  |j*                  | j,                  j.                  |j0                         t         j2                  j                  rd }t5        |j7                         d      r|j7                         j8                  }||j;                         }t=        d|fd|fd|fd|fd	|fd
t?        t@        jB                  |      fd|f        |jD                  | ||fS )Nr   fdo_profile)r   r#  rZ  r  r  rO  r  r   r   r   r|  r}  r   r   fingerprintzRunning pjit'd functionr#  rZ  r  r  zabstract args)#r   enable_pgler   pgle_profiling_runs_pgle_profiler_dictgetr   PGLEProfilerpgle_aggregation_percentileconsume_fdo_profilerM  r~  rR  r/   LoweringParameterscompile_auto_spmd_loweringenable_checksr0   %check_array_xla_sharding_layout_matchr  _in_layoutsr   r  r  distributed_debugr  runtime_executabler  hexrZ   r   r   abstractifyr  )r   r#  rZ  r  r  rO  r  r   r   r   r   r   pgle_compile_optionsr   compilation_target_keyr  r   r  s                     r   r   r     s    )+D&"<"<"B"BQ"F"'++,BCM++

$
$
*
*

,
,
2
24m 5B01
  335K,7=)-6J6P6P6R0SS  
%l!jnd113!/	 GI  !!f&:&:&@&@..h$$h&:&:!7!79 ## Kx**,m<//1==kOO%k4d;)<8*M:'4(+6*C0@0@$,GH(+68 
		t	$h	==r   c                8     t        j                           fdS )Nc                 >     t        j                                |  S r   )r   jaxpr_as_fun)r   r   s    r   rk  z#_get_jaxpr_as_fun.<locals>.<lambda>b  s    1t((148 r   )weakrefrefr  s   `          r   _get_jaxpr_as_funr  W  s     ++e
%	88r   c                   	
 
	 fd}t         	
      }t        d t              D              }t        j                  |d d d d d d d       }t        j                  j                  ||g g |t        j                  t        j                  t        |j                        	       S )Nc                     t        
	d\  }}}t        |t        |      |g g j                  j                  d |
      }||t        |      fS Nr  )r   r%  rK   r  r   r   )args_kwargs_r   r   r   r$  r   r   r  rO  r  r#  r   r   r   r   r  rZ  s         r   call_impl_cache_missz-_pjit_call_impl.<locals>.call_impl_cache_missi  sw    (>	U#
+,@)B%Hh '.*D(Bu||T=:M ]$=m$LLLr   c              3  ,   K   | ]  \  }}|s	|  y wr   r   rJ  s      r   r   z"_pjit_call_impl.<locals>.<genexpr>y  s     G1Q!GrL  )r|   r}   r   r   ro   rq   rr   rs   rt   ru   rv   rw   )r  rM  rN  r0   r4  r   r5  r6  r   r7  r8  r+  r*  )r   r#  rZ  r  r  rO  r  r   r   r   r   r   r  r  donated_argnumsrC  s   ````````````    r   _pjit_call_implr  e  s    
M 
M \=*khk6! G	.(AGG/(($d4\ };@) 

A#RY!!4#4#4IBBC
E GK
L Lr   c                 B    t        j                  d       t        | i |S )N
pjit_lower)r   r  _pjit_lower_cached)r   r   s     r   r~  r~    s    //,	T	,V	,,r   r   c               Z    t        j                  | d|||||t        |      |||
|||      S )Nr   )r   context_meshr   r|  r}  r   )r0   lower_sharding_computationrM  )r   r#  rZ  r  r  rO  r  r   r   r   r   r|  r}  r   s                 r   r  r    s?    " 
	(	(UD,+u^4H/+-!
# #r   c                   |d   rt        d |d   D              rt        d |d   D              rt        d |d   D              rt        d |d	   D              r|d
   }t        j                  j                  rMt	        j
                  |       5  t	        j                  |j                  |j                  g|ddi}d d d        n-t        j                  | ||j                  |j                  g| }D cg c]  }| j                  ||       c}S |d
   }t        j                  j                  rt        ||d   |d	         \  }}}}	t        ||||	      }g }
t        |      D ]  }t        |      t        j                   u r|j"                  D cg c]X  }t        |      t        j$                  u r||j&                     n+t        |      t        j(                  u r|
|j&                     n|Z }}|j+                  t-        d |D                    }|
j/                  t        j0                  | ||              t	        j2                  t5        | j6                  |      t5        | j8                  |
      t:        ||j<                  |      }| j>                  jA                  |       tC        |
      }|D cg c]#  }t        |      tD        u r||   n
tG        |      % }
}tG        |d       J |
S tI        d |j                  D              rtK        jL                  |      \  }}|D cg c]  }| jO                  ||       }}g |d   tP        ftS        |      z  z   }g |d   dtS        |      z  z   }g |d   dtS        |      z  z   }t        |||||      }| jU                  t:        g ||||      }
|
S | jU                  t:        |||      }
|
S # 1 sw Y   xY wc c}w c c}w c c}w c c}w )Nr   c              3  <   K   | ]  }t        |t                y wr   r   r=   r   r   s     r   r   z$pjit_staging_rule.<locals>.<genexpr>  s     	Ja*Q(
)	Jr   r#  c              3  <   K   | ]  }t        |t                y wr   r  r   r  s     r   r   z$pjit_staging_rule.<locals>.<genexpr>  s     	Ka*Q(
)	Kr   rZ  c              3  $   K   | ]  }|d u  
 y wr   r   r  s     r   r   z$pjit_staging_rule.<locals>.<genexpr>  s     	2!t)	2r  r  c              3  $   K   | ]  }|d u  
 y wr   r   r  s     r   r   z$pjit_staging_rule.<locals>.<genexpr>  s     	3!t)	3r  r  r   propagate_source_infoFr   rZ  r  c              3  F   K   | ]  }t        j                  |        y wr   )r   get_referent)r   rK  s     r   r   z$pjit_staging_rule.<locals>.<genexpr>  s     &Kt'8'8';&Kr  r_  c              3  P   K   | ]  }t        |t        j                           y wr   )r   r   MutableArrayr  s     r   r   z$pjit_staging_rule.<locals>.<genexpr>  s     
B:a**+
Br   r   rO  r  )r   r#  r  rO  )source_info)+r  r   rw  r   r   set_current_trace
eval_jaxprr   r   r   inline_jaxpr_into_traceto_jaxpr_tracer_pjit_forwardingr  	_out_typer   r  r_  InDBIdxr  OutDBIdxupdaterM  r   DynamicJaxprTracernew_jaxpr_eqnr   getvarmakevarr   r  frameadd_eqnr  r  r  r  r0   _move_mutable_consts	new_constr<   r   default_process_primitive)r@  r  r   r   r   r`  r   in_fwdrZ  r  out_tracersr   rK  r_  eqnout_tracers_r  r   r  r#  r  rO  rQ  s                          r   pjit_staging_ruler    s    X		J6.3I	JJ		K6/3J	KK		2VL1	22		3VM2	337OE"" !!%( ;ooekk5<< ;$ ;49;; ; &&
ekk5<<@:>@c;>?aE!!![1??
/%  0@vo&}(=1?-E6=+&])+FK%  J	dt((	( !JJ( !%Q4<< 7aee'+Aw$--'?QUU# ( ( {{&KU&K!K{L..udKHIJ 

	%,,s5==+>mm["C 
KK$L"$ #1gn47$|2DD $K $d#+++ 
 
BU\\
BB--e4ME67=>!eooa->F>-VN+-V0LLL)6,')Gc&k,AAJ1v./1Hs6{4JJNfE!+NLJ11 $  *+ 2 GK 
 11f+ 2 7K 
Y; ; @($
 ?s%   /O8O&AO+7(O0)O5O#c           	        t        j                  | j                        }t        |||      D cg c]  \  }}}t	        |t
              r||nd  }}}}|D cg c]  }|d u  }}t        j                  | |      } t        d t        ||      D              }t        d t        ||      D              }| |||fS c c}}}w c c}w )Nc              3  ,   K   | ]  \  }}|s	|  y wr   r   r   r  r^  s      r   r   z#_pjit_forwarding.<locals>.<genexpr>       Dda!DrL  c              3  ,   K   | ]  \  }}|s	|  y wr   r   r  s      r   r   z#_pjit_forwarding.<locals>.<genexpr>  r  rL  )r   _jaxpr_forwardingr   r   r   r=   prune_closed_jaxpr_outputsrM  )	r   rZ  r  r  fwdosolr  keeps	            r   r  r    s    11%++>&"6=+FH HR b"23
CL H& H#	$!t)	$$	$

'
't
4%Dc-&>DD-Dc+&>DD+	{	22H	$s   !B;Cc                   t         j                  j                  sd gt        | j                        z  | fS t        | j                  d   | j                  d   | j                  d         \  }}}}t        | j                  |      D cg c]
  \  }}|	| }}}t        | j                  |||      }| j                  ||      }	||	fS c c}}w )Nr   rZ  r  r  )r   outvars)
r   rw  r   r   r  r  r   r   r  r  )
r  r   r  rZ  r  r_  r  new_outvarsrQ  new_eqns
             r   pjit_forwarding_ruler    s    				$	$6C$$c)).>	jj3::o6

=8Q/S+%"3;;7Etq!19E+ECJJe= +-*KKz;K?'		 Fs   
C
C
c                   g }t        | j                  j                        D ci c]  \  }}||
 }}}t        | j                  j                        D ci c]#  \  }}t        |      t        j
                  u r||% }}}| j                  j                  D ]  }|j                  }t        |      t        j                  u rv|j                  j                  D cg c]<  }||v rt	        j                  ||         n||v rt	        j                  ||         n|> }	}|j                  t        |	            }|j                  |        |S c c}}w c c}}w c c}w )Nr  )rN  r   r*  r   r   Varr  r   r  r_  r  r  r  rM  r   )
r   r`  r   r_  r=  r   out_idxr   rK  r_  s
             r   r  r     s8   
#&u{{'9'9:;TQAqD;&;'(:(:; %daQ488# T %' %;; a66DDzT&&& &&,,( +,v+t||F1I&,-Lt}}WQZ( (e ( [[uU|[,dJJt 
* <%
(s   E(EAEc          	     d    t        j                  | t        |t        ||j                              S )N)
call_jaxpr)r   _check_callr   r  r   )ctx_factoryr   in_atomsr   s       r   _pjit_typecheckr    s+    			+vxv%++>
@ @r   c                2    | j                   | j                  fS r   )r  r  )r   rZ  r   r   s       r   _pjit_abstract_evalr    s    	%--	''r   c	                T   | j                   }	| j                   j                  }
d }t        |
t        j                        r|
j
                  }n0t        |
t        j                        r|
j                  j                  }t        ||||t        j                  ||j                        t        j                  ||j                        |||f
}|	j                  j                  |d       }||D cg c]  }t        |t               rd n| }}|D cg c]  }t        |t               rd n| }}t#        |	j$                        }t'        j(                  |	|||| j*                  ||d|||      }t#        |	j$                        |kD  }|rd|	j,                  vr||	j                  |<   |S c c}w c c}w )NF)arg_shardingsresult_shardingsuse_sharding_annotationsr  arg_layoutsresult_layoutstpu)module_contextaxis_contextr   r   ShardingContextnum_devicesSPMDAxisContextr   r  r   r0   SemanticallyEqualShardingsr   r  cached_primitive_loweringsr  r=   r   host_callbacksr/   lower_jaxpr_to_fun
name_stack	platforms)ctxr   r   r  r#  rZ  r  r  r  mod_ctxaxis_ctxr  r  r2   r   r  r  r  num_callbackshas_callbackss                       r   _pjit_cached_lower_jaxpr_to_funr    s   
 ',,(+.889&&K(N::;--$$K	ug{		(	(u~~	F		(	(	H	[(	,#
 
	+	+	/	/T	:$	\MYZZ+;<T!CZMZP]^1
1.> ?QF^^ ../M""ugs~~#6F!&{	<D ../-?ME):)::04g((-	+% [^s   *F F%c                  t        | j                  j                               }t        t        j
                  | j                        }t	        j                         gt        |      z  |z   }t	        j                  |      }t        | ||t        |      ||||d	      }|D cg c]  }| j                  j                  |       }}g | j                  ||}t        j                  |t         j"                  j                  |j$                  j&                        t	        j(                  |            }t	        j*                  | |       t	        j,                  |j.                  |      }t1        |t        |      g      \  }}| j                  j3                  t	        j4                  t7        ||                  }| j9                  |       |S c c}w )Nr   )r  )r{  	tokens_inr  r   r/   aval_to_ir_type	avals_out
token_typer   flatten_ir_typesr  rM  r  dim_var_valuesfunc_dialectCallOpr1   FlatSymbolRefAttrr   r   flatten_ir_valueswrap_compute_type_in_placeunflatten_ir_values_like_typesresultsr[   update_tokensTokenSetr   set_tokens_out)r  r   r   r#  rZ  r  r  rO  r  r   r   r   r   r  output_typesflat_output_typesr2   effr  call	out_nodestokens
tokens_outs                          r   _pjit_loweringr  C  sy    &&()'T))3==9,//#$s7|3lB,++L9	(	4gZ
$
 299#s}}  %9)9	13	1		1D	1$			.1155diiooF!33D9
;$ !!#t,11$,,M) S\N;&)}}**4==Wf9M+NO*Z 	 :s   ""Gc                   	 t        j                  |      \  }}t        j                  | |      \  }}t         	fdt	        |||j
                        D              }t         	fdt	        |||j                        D              }t        d |D              rt        d |D              st        d      t        j                  |||||||	|
|||d}t        j                  |||      }||fS )Nc              3  t   K   | ]/  \  }}}|#t        ||j                  |j                        n| 1 y wr   _pjit_batcher_for_sharding	spmd_namerv  )r   axis_inr   r   	axis_datar  s       r   r   z _pjit_batcher.<locals>.<genexpr>j  sM      N '1d 
	 !GY-@-@(!%,"#$N   58c              3  t   K   | ]/  \  }}}|#t        ||j                  |j                        n| 1 y wr   r  )r   axis_outr  r   r#  r  s       r   r   z _pjit_batcher.<locals>.<genexpr>o  sM      R (At 
	 !Hi.A.A8!%,#$%Rr$  c              3  $   K   | ]  }|d u  
 y wr   r   r   r  s     r   r   z _pjit_batcher.<locals>.<genexpr>u  s     ,Aa4i,r  c              3  $   K   | ]  }|d u  
 y wr   r   r(  s     r   r   z _pjit_batcher.<locals>.<genexpr>v  s     -Aa4i-r  z1Concrete layouts are not supported for vmap(jit).r  )r.   indirectify_ragged_axesbatch_jaxpr2rM  r   r   r  r  r  r   r   *resolve_ragged_axes_against_inputs_outputs)r#  vals_indims_inr   r#  rZ  r  r  rO  r  r   r   r   r   segment_lens	new_jaxpraxes_outvals_outresolved_axes_outs   `        `         r   _pjit_batcherr4  `  s    #::7C, --eYH)X  N "'<9K9KL	N N,
  R  #8]I<O<OP	R R- ,,
,
--
-
;= = [[
!	-/( IIx#	$	$$r   c           	        t        | t              r| S | j                  |      }|t        j                  |      r| S t        | t
              r]t        | j                  t              rCt        | j                  t        j                  | j                  |t        j                              S |j                         j                         }t        |j                         }|j#                  |d       ||_        t%        | j&                  |t)        | dd             }t        j*                  |gd g| d       d   S t        | t
              rOt        | j                  t              r5t        | j                  t        j                  | j                  ||            S t        | t
              r| j                  }||j,                  rt/        d|       t1        ||      d   }	t        |t        j                  |	||            S )Nr  _internal_device_list)_device_listr   aB  If you are using spmd_axis_name parameter of jax.vmap, please make sure to run your jitted function inside the mesh context manager. Only `jax.lax.with_sharding_constraint` with `jax.sharding.NamedSharding` as an input can be transformed with spmd_axis_name batching rules outside of an explicit mesh context manager scope)r   r=   r  r   is_op_sharding_replicatedr7   r   r5   r0   
batch_specspecr+   UNCONSTRAINEDto_protocloner{  tile_assignment_dimensionsinsertr8   _device_assignmentr   $_get_out_sharding_from_orig_shardingr  r   r?   )
r!  dimspmd_axis_namer   rv  hlo_snew_optadnew_gsr:  s
             r   r   r     s    #$H

 
 
&%//6h!]#
166<(H
&&$//!&&#}/J/JKM M^^##%F
v00
1CJJsA(+F%	fQ 7>@F 44fXvq$OPQRR!]#
166<(H
&&$//!&&#~>@ @!]#VVd|tzz
 %!" " %UD1!4DdoodC8: :r   c                   |D cg c]  }t        |      t        j                  u }}t        j                  ||d      \  }}d }t	        ||      }t	        ||      }t        j                  g |  ||      |g | ||      g | ||      g | ||      g | ||      g | ||      ||	|
||d}t        |t        |j                  j                        g      \  }}t        |      t        |j                  j                        k(  sJ t        |      }|t        ||j                        D cg c]'  \  }}|rt        |      nt        j                  |      ) c}}fS c c}w c c}}w )NFinstantiatec                (    d t        | |      D        S )Nc              3  ,   K   | ]  \  }}|s	|  y wr   r   r   nzr   s      r   r   z3_pjit_jvp.<locals>._filter_zeros.<locals>.<genexpr>  s     1%"abA1rL  )r   is_nz_lr  s     r   _filter_zerosz _pjit_jvp.<locals>._filter_zeros  s    13w?11r   r  )r   r-   Zero	jvp_jaxprr   r   r   r[   r   r   r  r  r   r  r  )
primals_intangents_inr   r#  rZ  r  r  rO  r  r   r   r   r   r  is_nz_tangents_in	jaxpr_jvpis_nz_tangents_outrQ  _filter_zeros_in_filter_zeros_outoutputsprimals_outtangents_outtangents_out_itrN  r   s                             r   	_pjit_jvpr_    s    8CC!tAwbgg-CC"$,,E#3)2],=>m-?@KK 11$[11C\C$4\$BCGmG&7&FG=:= 0 <=AKA"3K"@AI~I(8(HI/1' )3u{{7J7J3K2LM+|	[	S!4!45	55	5&/	'*+=u'OQ#2t 13tO,E Q 
Q Q3 D2Qs   "E0?,E5c               ~   	
)*+,-./ t        j                         \  }}+/t        f|z  .d|z  -d|z  ,.t              z   }-t              z   }d }t	        j
                  |j                        }t        ||g      \  }}|t        |      D cg c]  \  }}}t        |t              r||nd  c}}}z   }~~|D cg c]  }|d u  }}t	        j                  ||      } |||      } |||      }t        ||g      \  }}t        |      }~~t        |j                  j                        |z
  }t        |j                  j                  |g      \  }}t        |      D  !ci c]  \  } }!t!        |!      |  c}!} *t        *fd|D              }"*j#                         D #!ci c]  \  }#}!|#|!|"z    c}!}#*|D !cg c]  }!*j%                  t!        |!             c}!d g|z  z   }$|$D cg c]  }|d u  }}t	        j                  ||      } |||      } |||      }~)
	 +,-./fd}%d )t'        j(                  ||||	
d}&t+        |$|&|&      }&t+        |||&      }&t        |&|g      \  }'}(|(+|'|%fS c c}}}w c c}w c c}!} w c c}!}#w c c}!w c c}w )Nr   r  c                :    t        d t        | |      D              S )Nc              3  ,   K   | ]  \  }}|s	|  y wr   r   r   r   r  s      r   r   z:_pjit_linearization.<locals>.keep_where.<locals>.<genexpr>       >wq$>rL  rJ  r  should_keeps     r   
keep_wherez'_pjit_linearization.<locals>.keep_where      >#a"5>>>r   c              3  8   K   | ]  }t        |      v  y wr   r   )r   r_  idx_maps     r   r   z&_pjit_linearization.<locals>.<genexpr>  s     6r!uG#6s   c                    
|      }t        j                  g ||  
      z    
       
      z    
       
      z   d}j                  j                  D cg c]  }|j                  j                          }}t        |      }t        |      D cg c]'  \  }}|rt        |      nt        j                  |      ) }	}}|	S c c}w c c}}w r  )r   r   r   r  r   to_tangent_avalr  r   r  r-   rR  )consts_tangentstangents_nznz_tangents_outr_  tangent_avals_outnz_tangents_out_r   rN  r]  rQ  r   r  rO  r  r#  r   r   r   r   nzsnzs_outr  rZ  res_donatedres_layoutsres_shardingstangent_jaxprs             r   tangent_funz(_pjit_linearization.<locals>.tangent_fun	  s   X.Kkk 3; 3 3"35E#G]; j1K?!';7$S.9KG13O <A;;;N;NOa//1OOO,%():G%DF!b /1D)*bggdmC FL F	 PFs   3!C!0,C&c                :    t        d t        | |      D              S )Nc              3  ,   K   | ]  \  }}|s	|  y wr   r   rM  s      r   r   z=_pjit_linearization.<locals>._filter_zeros.<locals>.<genexpr>	  s     6ur126rL  rJ  rO  s     r   rQ  z*_pjit_linearization.<locals>._filter_zeros	  s    6GQ666r   r  )r-   linearize_jaxprr<   rM  r   r  r   r[   r   r   r=   r  r   r   r  rN  r   r~  r  r   r   r_   )0rs  r   r#  rZ  r  r  rO  r  r   r   r   r   rT  primal_jaxprnum_residualsprimal_out_shardingsprimal_out_layoutsrg  r  
in_fwd_resin_fwd_primalr  r  r  r  r  kept_resr   num_kept_residualsnum_out_primalsres_varsout_varsr   r_  offsetr^  out_fwdry  ansresiduals_ans
primal_ansrQ  rj  rt  ru  rv  rw  rx  s0   ````````````                             @@@@@@@r   _pjit_linearizationr    s    9;8J8J5RU8V5,w.=0--'+=(+&})=="U;%77?  2 23&(-A*m]KG 
"b# ,-"*c$F & -#	$!t)	$$	$..|TB,#$8$?!"4d;4-1+(A8}
H **2236HH/!,"4"4"<"<?Q>RS(H"+H"56$!QRUAX6'6X66&'.}}7tq!QF
]7')12AW[[A2dVo5MM'$	%!t)	%$	%..|TB,#$8$?!"4d;
  (7 	Z|!-"6) 2#1% +!)=
	?# 	'3$#&*c*#(}o>-	Wm[	88A
 
% 772	%s$   #!J
J$7J)8J/!J5?J:c               
   |D cg c]  }|j                    }}t        d |D              }t        d |D              }t        j                  ||d      \  }}}}}t        |      }t        d |D              }d } |||      t        ft        |      z  z   } |||      dt        |      z  z   }t        |j                        t        |      z
  }t        j                  |j                        }t        ||g      \  }}t        d |D              sJ t         |||       |||      |      D  cg c]  \  }}} t        |t              r|| nd  c} }}|z   }~~|D !cg c]  }!|!d u  }"}!t        j                  ||"      } |||"      } |||"      }t        |"|g      \  }#}$t        |#      }~"~#~$t        |j                  j                   |g      \  }%}&t#        |%      D '(ci c]  \  }'}(t%        |(      |' })}'}(d g|z  |&D (cg c]  }(|)j'                  t%        |(             c}(z   }*|*D !cg c]  }!|!d u  }"}!t        j                  ||"      } |||"      } |||"      }~"t)        | |||      | |||      | |||      |||	|
|	      }+t        |+d
         t        |+d   j                        k(  sJ t        |+d         t        |+d   j                        k(  sJ |D ,cg c]#  },|,j+                         s|,j-                         % }-},t/        j0                  |-i |+}.t3        |*|.|.      }.t3        ||-|.      }.t        |.t        |.      t        |      z
  g      \  }/}0t5        |0      }1|D !cg c]"  }!|!t7        |1      ng |j8                  |-|!   $ }0}!t7        |1d       J t;        | j<                  |0      }2t        j>                  |dgt        |0      z  dgt        |      z  z         } |||      t        ft        |2      z  z   }3 |||      dt        |2      z  z   }4 |||      dt        |2      z  z   }5t)        ||3|4 |||       |||      |5|||	|
|      }6|D cg c]  }|j                   j+                         r|! }7}|j                  }8|8D 9cg c]6  }9t        j@                  | t        jB                  jE                  |9      d       8 }:}9t        jF                  | g |7|2|:t.        |6|jH                  tK        jL                               };|:D ]	  }|;|_'         tQ        ||/|:      S c c}w c c} }}w c c}!w c c}(}'w c c}(w c c}!w c c},w c c}!w c c}w c c}9w )Nc              3  <   K   | ]  }|j                           y wr   )is_known)r   pvs     r   r   z%_pjit_partial_eval.<locals>.<genexpr>7	  s     5bBKKM5r   c              3  "   K   | ]  }|  	 y wr   r   )r   r^  s     r   r   z%_pjit_partial_eval.<locals>.<genexpr>8	  s     /!e/   FrI  c              3  "   K   | ]  }|  	 y wr   r   )r   uks     r   r   z%_pjit_partial_eval.<locals>.<genexpr><	  s     3V3r  c                :    t        d t        | |      D              S )Nc              3  ,   K   | ]  \  }}|s	|  y wr   r   rc  s      r   r   z9_pjit_partial_eval.<locals>.keep_where.<locals>.<genexpr>@	  rd  rL  rJ  re  s     r   rg  z&_pjit_partial_eval.<locals>.keep_where?	  rh  r   r   c              3  $   K   | ]  }|d u  
 y wr   r   )r   r  s     r   r   z%_pjit_partial_eval.<locals>.<genexpr>K	  s     ,1Q$Y,r  r  rZ  r   r  Tr  )r   r#  r  rZ  r  rO  r  r   r   r   r   ))pvalrM  r   partial_eval_jaxpr_nounits_fwdr<   r   r  r  r   r[   r  r   r   r=   r  r   r  rN  r   r  r  r  	get_knownr   r   r_   r  r  r   r   new_instantiated_constmove_binders_to_backJaxprTracer
PartialValunknownnew_eqn_reciper  r   r  reciper^   )<r@  r   r#  rZ  r  r  rO  r  r   r   r   r   
in_tracersr  in_pvals	known_insunknown_insknown_jaxprunknown_jaxprunknown_outsres_out_avalsr  
known_outsrg  known_out_shardingsknown_out_layoutsr  r  r  in_fwd_res_r  r  r  r  r  kept_primalsr  r  r  r   r_  rj  r  known_paramsr  known_inputsall_known_outsknown_out_valsresidual_valsresidual_vals_residual_tracersunknown_in_shardingsunknown_in_layoutsunknown_donated_invarsunknown_paramsunknown_tracers_inunknown_out_avalsr   unknown_tracers_outr  s<                                                               r   _pjit_partial_evalr  0	  s   
 ))aff)()5H55)/Y//+''{N F+}lM:|$,3l33*? $M:>'>C,>>?!+z: 3}#556 --.]1CC/11+2C2CD&)&?2CD-	,,	,,	, 
]J
/
[*
-}> 
"b# ,-"*c$F
 & [#	$!t)	$$	$--k4@+"#6= !2D9%d_,=>,%/
L( "+"3"3";";o=NO(H"+H"56$!QRUAX6'6F_$H'MqBqE(:'MM'$	%!t)	%$	%--k4@+"#6= !2D9
jy&I'J	2#	:[/1, 
\/*	+s<3H3R3R/S	SS	S	\-(	)Sg1F1P1P-Q	QQ	Q ,4ERr{{}",,.E,E;;==.WnnE.V\>B. #n"5M8J"J!KL  .-&.CMO>? ,-94'6%,,66q9: O- O	nd	#	++	+55}E ))dVc-00E7S=M3MMO- %\;?(NS1A-BBC":{;!C(8$99:&~{C&-=)>>?'#}l;[,7+/1. $.GaQVV__5FGG#-- $
 	nnUBMM11$7>  	%!I#5!I8H!I- ('//*224	6# .a3.	\>3F	GGm *. 
% 7'M	%& FO> HsG   T;4!U #U(U
!U4U+UU8'U!-U&U&#;U+c           
        t        j                  | |d         \  }}	t        j                  | |d         \  }
}	t        j                  ||d         \  }	}t        j                  | |d         \  }}	t        j                  ||d         \  }	}t        |t        |
      g |t        g|z  t        |      g |d g|z  t        |            }t        |d         t        |d   j                        k(  sJ t        |d         t        |d   j                        k(  sJ t        |d         t        |d   j                        k(  sJ t        |d         t        |d   j                        k(  sJ t        j                  ||d         \  }	}dg|z  |z   }t        j                  ||d         \  }	}g t        g|z  |}t        j                  ||d         \  }	}t        j                  ||d         \  }	}g d g|z  |}t        j                  ||d         \  }	}t        |t        |      t        |      t        |      t        |      t        |            }t        |d         t        |d   j                        k(  sJ t        |d         t        |d   j                        k(  sJ t        |d         t        |d   j                        k(  sJ t        |d         t        |d   j                        k(  sJ ||fS )	NrO  r#  rZ  r  r  )r#  rZ  r  r  rO  r   F)r   partition_listr  rM  r<   r   r   r  )unks_ininst_inkept_outs_knownkept_outs_stagednum_res_out
num_res_inparams_knownparams_stageddonated_invars_knownr   in_shardings_knownout_shardings_knownin_layouts_knownout_layouts_knownnew_params_knowndonated_invars_stagedin_shardings_stagedout_shardings_stagedin_layouts_stagedout_layouts_stagednew_params_stageds                        r   (_pjit_partial_eval_custom_params_updaterr  	  s_    --g|DT7UV++G\.5QRa,,_l?>[\!))'<3MNA**?L<WX!,',-?'@)G*= )G+6-+*E)G%*+;%<&Q(9&QTF[<P&Q)./C)DF 
n-	.#l76K6T6T2U	UU	U	o.	/3|G7L7V7V3W	WW	W	l+	,L4I4R4R0S	SS	S	m,	-\'5J5T5T1U	UU	U  ..wFV8WX! 'J.1FF,,WmN6ST!K;-*4K7JK--.>o@^_!**7M,4OP!@+@.?@++,<mM>Z[!=(-.A(B)./C)D&+,=&>',-?'@*/0E*FH 
~.	/3}W7M7V7V3W	WW	W	/	0Cg8N8X8X4Y	YY	Y	|,	-]75K5T5T1U	UU	U	}-	.#mG6L6V6V2W	WW	W	,	,,r   c                l    t        j                  | |      \  }}}}t        j                  ||      }||fS r   )r   r  r   r  )r   r   transpose_jaxprr   r   r   s         r   _pjit_transpose_tracer  	  s?     /1.G.G	8/+/1fm$$_f=/	-	''r   c               8   d }t        j                  t        j                  |j                  j
                        }t        j                  ||d      }t        || f      \  }}t        ||      \  }}g  |t        j                  ||       |t        j                  ||       }g  |t        j                  ||       |t        j                  ||       }t        d |D              }t        ||      \  }} |       } |t        j                  |t        |t               g|j                  z              } |t        j                  |t        |t               g|j                  z              }|r_t!        |      }t#        d |D              }g ||}t$        ft'        |      z  |z   }t$        f|z  |z   }dt'        |      z  |z   }d|z  |z   }	 t)        j*                  ||||||dt'        |      z  |||	|
|d}|rt9        g      \  } }t;        ||        t        |      S # t,        j.                  $ r^}t1        d	       	 t        j                  |d ||       }t-        j2                  |       n# t4        t6        f$ r}|d d }~ww xY wY d }~d }~ww xY w)
Nc                @     t         fdt        ||      D              S )Nc              3  D   K   | ]  \  }}t        |      us|  y wr   )r   )r   r   mzr   s      r   r   z6_pjit_transpose.<locals>.prune_type.<locals>.<genexpr>	  s      Kuq"R8JKs     rJ  )r   rL  maybe_zeross   `  r   
prune_typez#_pjit_transpose.<locals>.prune_type	  s    KB 4KKKr   r  Fc              3  F   K   | ]  }t        j                  |        y wr   )r   get_aval)r   cts     r   r   z"_pjit_transpose.<locals>.<genexpr>
  s     PBdmmB/Pr  c              3  <   K   | ]  \  }}}|j                     y wr   r   )r   r   
final_trees      r   r   z"_pjit_transpose.<locals>.<genexpr>
  s     T2B!Z
--Tr   r   r  r  zqInvalid nan value encountered in the backward pass of a jax.jit function. Calling the de-optimized backward pass.)r  r  r-   closed_backward_passr   r  hashable_partialrH   r!   UndefinedPrimalrR  rM  r  rI   objectr   r   r   r<   r   r   r   r   r   print_raise_no_nan_in_deoptimizedr   ZeroDivisionErrorr[   r   )!cts_inr   r#  rZ  r  r  rO  r  r   r   r   r   rT  r  bodyprimals_and_nz_cts_in
in_treedefcts_out_treedef_thunktranspose_in_shardingstranspose_in_layoutsglobal_cts_in_avalsr  r   cts_out_treedeftranspose_out_shardingstranspose_out_layoutsr   num_attr_outs
nz_cts_outr   r   r@  r   s!                                    r   _pjit_transposer  	  s   
L 
b--%++:Q:Q	R$			T5%	0$&2J3G&H# 4T: F$""L*=/""J
;f- P:OPP#8
$! /=)+/&gg_vxj?3M3M&MNP %gg_vxj?3M3M&MNP
 }-KTmTTMBkB,AB)^c+.>>AWW*n}<?VV"S%558LL#m36KK/	+-')#&;"<<13J2 )*}oFL*|,		44! 
	,	, 
/	 > ?/

!
!%z6
Ba ++A.  12 D
 /
/s<   +H( (J;JI5J5JJJJJc                    t        j                  | j                  |      \  }}t        j                  || j
                        |fS r   )r   	dce_jaxprr   r   r  r   )r   used_outputsr0  used_inputss       r   _dce_jaxpr_pjitr  :
  s9     <<\B)[			)U\\	2K	??r   c                   t        |       s0t        j                  |      sdgt        |j                        z  d fS t        |j                  d   t        |             \  }}d }|j                  }t        || ||d   |       ||d   |        ||d   |       ||d   |        ||d   |      	      }t        |      st        |       s|j                  s|d fS t        j                  t        |j                  |      D cg c]
  \  }}|s	| c}}t        |j                  |       D cg c]
  \  }}|s	| c}}|j                  ||j                  |j                  |j                         }	||	fS c c}}w c c}}w )
NFr   c                :    t        d t        | |      D              S )Nc              3  ,   K   | ]  \  }}|s	|  y wr   r   rc  s      r   r   z:dce_jaxpr_pjit_rule.<locals>.keep_where.<locals>.<genexpr>L
  s     9wq$D9rL  rJ  )rL  keepss     r   rg  z'dce_jaxpr_pjit_rule.<locals>.keep_whereK
  s    9#b%.999r   r#  rZ  r  r  rO  )r   r#  rZ  r  r  rO  )r  r   has_effectsr   r*  r  r   rM  r  r  r   r  r   r  	primitiver  r  )
r  r  
dced_jaxprr  rg  
eqn_paramsrQ  r_  usedr  s
             r   dce_jaxpr_pjit_ruler  B
  se    
\	2>>##67S_$d**+	jj5.0*k: zz*j8+Fz/:LIJ|4kBZ6E
+; <kJ* 
[	#l"3J<N<N  cjj+6?wq$$?ckk<8Awq$DAz:#5#5sQG  	@As   
E5
E5
-
E;8E;c           	        t        | j                        }|d= t        |d         s|d= t        d |d   D              r|d= t        d |d   D              r|d= t        d |d   D              r|d= t        d	 |d
   D              r|d
= |d   s|d= |d   |d   j                  r|d= |d   s|d= |d   j
                  |j                  vrf|j                  |d   j
                  j                  | j                         |j                  |d   j
                  j                  | j                         |d= t        j                  | ||dgt        |      z         S )Nr   rO  c              3  <   K   | ]  }t        |t                y wr   r  r   r!  s     r   r   z _pjit_pp_rule.<locals>.<genexpr>k
  s     IQA'	(Ir   r#  c              3  <   K   | ]  }t        |t                y wr   r  r
  s     r   r   z _pjit_pp_rule.<locals>.<genexpr>m
  s     JQA'	(Jr   rZ  c              3  $   K   | ]  }|d u  
 y wr   r   r(  s     r   r   z _pjit_pp_rule.<locals>.<genexpr>o
  s     1qd1r  r  c              3  $   K   | ]  }|d u  
 y wr   r   r(  s     r   r   z _pjit_pp_rule.<locals>.<genexpr>q
  s     2qd2r  r  r   r  r   r   r   )r   )r  r   r  r  r  r   shared_jaxprssuggest_same_var_namesr*  r  r   _pp_eqnrc  )r  contextsettingsr   s       r   _pjit_pp_ruler  d
  sh    

&X	V$%	& I&2HII~J&2IJJ1F<011|2F=122}		}J6*#5#;#;z	&	'%&G_'"7"77""6'?#8#8#?#?L""6'?#8#8#@#@#++N Vn	c7HfXv5N	OOr   c                ,   | D ]c  }t        |t              s|j                  j                  t	        fd|j
                  D              rIt        d|j
                   d d       t        j                         }|j                  rt        d      y )Nc              3     K   | ]\  }|t         j                  urH|Ft        |t              r|n|fD ].  }j                  |   t
        j                  j                  k(   0 ^ y wr   )r+   r;  r   rM  _name_to_typer  AxisTypeAuto)r   axesr   r   s      r   r   z+check_shardings_are_auto.<locals>.<genexpr>
  sn      Hm999d>N!+D%!8tgH  !!!$(9(9(>(>> H> Hs   A"A%zoThe spec of NamedSharding passed to with_sharding_constraint can only refer to Auto axes of the mesh. Got spec=z
 and mesh=z*. You probably meant to use `reshard` API?zwwith_sharding_constraint cannot be used when all axes of the mesh are of type `Explicit`. Please use the `reshard` API.)
r   r7   r   r  r  r:  r   r  get_abstract_meshr]  )shardings_flatr!  cur_meshr   s      @r   check_shardings_are_autor  
  s     Eaa'66D H66H H <<=FF8 D&BDE EE '')($$
	=> > %r   c                   t        |       \  }}|D  cg c]  } t        j                  |        }} t        |      \  }}t	        |dd      }~t        t        d||            }~t        t        d||            }~t        j                         t        j                         n#t        j                  j                  j                  }	|D 
cg c]  }
t        |	|
dd       }}
t        ||      D ]*  \  }}t        |t         t"        f      st%        d| d       ~|D cg c]'  }t        |t&              rt)        |      n	t+               ) }}t-        ||d	t/        |      z  d
d       t1        |       t3        ||d	t/        |      z  d
       t        ||||      D cg c]!  \  }}}}t4        j7                  ||||	|      # }}}}}t9        ||      S c c} w c c}
w c c}w c c}}}}w )a  Mechanism to constrain the sharding of an Array inside a jitted computation

  This is a strict constraint for the GSPMD partitioner and not a hint. For examples
  of how to use this function, see `Distributed arrays and automatic parallelization`_.

  Inside of a jitted computation, with_sharding_constraint makes it possible to
  constrain intermediate values to an uneven sharding. However, if such an
  unevenly sharded value is output by the jitted computation, it will come out
  as fully replicated, no matter the sharding annotation given.

  Args:
    x: PyTree of jax.Arrays which will have their shardings constrained
    shardings: PyTree of sharding specifications. Valid values are the same as for
      the ``in_shardings`` argument of :func:`jax.experimental.pjit`.
  Returns:
    x_with_shardings: PyTree of jax.Arrays with specified sharding constraints.

  .. _Distributed arrays and automatic parallelization: https://docs.jax.dev/en/latest/notebooks/Distributed_arrays_and_automatic_parallelization.html
  rr  Tru  z"with_sharding_constraint shardingsz with_sharding_constraint layoutswith_sharding_constraintz7One of with_sharding_constraint arguments got sharding zJ which is not allowed. Please only pass `jax.sharding.Sharding` instances.r   z"with_sharding_constraint argumentsr-  r[  layoutr  unconstrained_dims)rH   r   r  rs  r>   rM  r   r  r  r  r  r  r  r  r   r   r=   r;   r   r7   get_unconstrained_dims	frozensetr/  r   r  r1  sharding_constraint_pr   rI   )r   rr  x_flatr"  x_avals_flatrq  user_shardingsuser_shardings_flatuser_layouts_flatr  r"  r  r!  ur#  xfr  udr   s                      r   r  r  
  sH   ( a,&$6<=$))!,=,=1)<'9)t=.7~NP5tWEG '/&@&@&B&Nh  "$$((66  12 /|Q/IK 2. 2 ."56 Oda!&-.CA3 GN NO OO
  "01 &a7 /q1=F[I 1 1 lEC,?$?* " >*!"3\"'#.?*@"@"FH "&.:K"46
7 
7 b!Q  
$
$R!A2>8: % < 
7$ 
7
 
d	##c >&21
7s   GG!,G$#&G)
c                    | S r   r   rj  s    r   _identity_fnr0  
  s    AXr   c           	     <   t        |t              rt        |j                  t              r|j                  sLt        |t              r<t        | d      s0t        j                         }|J t        ||j                        }nJt        j                  |       }t        | d      s(t        dt        |        d|j                                t        | j                  t              s2t        dt        | j                         d|j                                | j                  j                  j                   |j                  j                   k7  rTt        d| j                  j                  j                    d|j                  j                    d|j                                t        | j                  j                  |j                        }|Ut        | d      r(| j                  j#                  || j$                        r| S  t'        j(                  t*        |      |       S t        | d	      rA| j,                  j.                  |k(  r(| j                  j#                  || j$                        r| S  t'        j(                  t*        t1        ||            |       S )
Nr[  ztTarget sharding contains a `jax.sharding.AbstractMesh` which requires the input passed should be a `jax.Array`. Got z with shape z}The sharding on the input must be a `NamedSharding` since the target sharding has an `AbstractMesh` in it. Got sharding type  for shape Mesh shape of the input 6 does not match the mesh shape of the target sharding rZ  rS  )r   r7   r   r5   r  r  r  r  r:  r   r  r   r   r   r[  r   shape_tupleis_equivalent_torv  r   r   r0  rS  rn  rC   )r   r[  r"  r  r#  concrete_meshr   s          r   _sharding_constraint_implr9  
  s   =)-:lL#IAz"002m&&&}hmm<h$$Q'dQ
#QyT^^%5$689 	9 

M2QZZ !T^^-=,>@A 	A 
	$	$(A(A	A&qzz'B'B&C D))*+dnn6F5GIJ 	J qzz>h^q*!**"="=h"Oh8377<x8;;8!=!=!G	

##Haff5hH377<vfh/GHKKr   sharding_constraintc                    | S r   r   r   r   s     r   rk  rk    s    q r   c                0    t        j                  | fi |fS r   )r&  r   r  r   r   s      r   rk  rk    s    '<'A'A"'O'O&Q r   c                  | j                   \  }| j                  \  }| j                  j                  }t	        j
                  |j                  t        j                        rt        j                  |      }t        |t        j                        r-|j                  r!t        j                  |||j                         }t"        j$                  j&                  r|j)                  |j                         }n)|j+                  |j                         j-                         }t        j.                  | ||||      }	|t        j0                  | |	|||      }	|	gS )N)unspecified_dims)avals_inr  r  r  r   r  r  r  r   physical_avalr   r   r  manual_axesr/   add_manual_axesrv  r   use_shardy_partitionerr   _to_sdy_shardingr  r<  wrap_with_sharding_opwrap_with_layout_op)
r  x_noder[  r"  r  r#  in_avalrf  r   r`  s
             r   !_sharding_constraint_hlo_loweringrK    s   \\('mm)(,,(w}}foo6  )G>99:##HhEH""((((6H,,W\\:CCEH""	68X8J	L#

"
"3Xvw
GC
,r   c           	     d   | j                   t        |t              rr|j                  D ch c]  }t        |t              r|n|fD ]  }|   }	}}t        | j                         |	z  r%t        d| j                    d|j                         |\  }
|\  }|D ch c]
  }|||k  z    }}| j                   |j                  |       t        ||| j                   ||
j                        }|rt        |t              rrt        |j                        d g|
j                  t        |j                        z
  z  z   }|D ]  }t        j                  ||<    t        |j                  t        |       }|t        d|       t         j#                  |
|||t%        |            }||fS c c}}w c c}w )Nzvmap spmd_axis_name z> cannot appear in with_sharding_constraint spec, but got spec zPConcrete layout is not supported for vmap(with_sharding_constraint). Got layout r!  )r!  r   r7   r:  rM  r   r   r  r   rv  r{  r   r+   r;  r   r  r&  r   r%  )r#  r-  r.  r[  r"  r  r#  rK  rE  r  r   rK  r.  vmapped_shardingnew_specr,  rI  s                    r   _sharding_constraint_batcherrO  +  s    $Hm)L  ?"&r51bu?  ?A ?D ?
9$&-i.A.A-B CF"--* + + "!"!1CD2a2gDD 1/9&&aff>J'7G$))*dVqvvDTDYDY@Z7Z-[[H 0!//hqk0$}h79 
	X	      "#56 ! 8! 
A+C? Es   #F'F-c                     yNr   r   r   s    r   rk  rk  R      r   c                    t        |       \  }}t        d||      }t        ||      D cg c](  \  }}t        j	                  |t        |dd            * }}}t        ||      S c c}}w )Nzmesh_cast shardings	mesh_castF)check_mesh_consistencydst_sharding)rH   r   rX   mesh_cast_pr   r@   rI   )rL  rZ  r'  r   r  r   r!  r   s           r   rU  rU  W  s     $/&' 5wN.
 6>2	 !Q 
/U<  =(  
	**s   -A'rU  c           	     x   | j                   }||k(  r| S |j                  j                  s|j                  j                  r| j                  |      S |j                  j                  |j                  j                  k7  rJt        d|j                  j                   d|j                  j                   d| j                                |j                  j                  |j                  j                  k(  rS|j                  |j                  k7  r:t        d|j                  j                   d|j                  j                   d      |j                  j                  rJ|j                  j                  r3t        t        |j                        t        |j                              D ]  \  }}|||m|k|j                  j                  |   t        j                  j                  k(  r4|j                  j                  |   t        j                  j                   k(  sJ z|m|k|j                  j                  |   t        j                  j                   k(  r4|j                  j                  |   t        j                  j                  k(  sJ ||k7  st        d| d	|        | j                  |      S )
Nr[  r3  r4  r2  zsmesh_cast should only be used when AxisType changes between the input mesh and the target mesh. Got src axis_types=z and dst axis_types=zG. To reshard between the same mesh, use `jax.sharding.reshard` instead?zBExplicit data movement in mesh_cast is not allowed. Got src spec: z and dst spec: )r[  r   r  r  r6  r   r   _axis_types_dictr:  _any_axis_explicitrX   rA   r  r  r  r  Explicit)r   rX  src_shardingr!  rK  s        r   _mesh_cast_abstract_evalr`  d  s   ,\!K 1 1 7 7;;;--""l&7&7&C&CC

"<#4#4#@#@"A B))*+dnn6F5G	IJ J ((L,=,=,N,NN


|00
0
	#((99: ;#((99: ;>	>? ? ))l.?.?.R.Rl&7&78%l&7&78: '1	
qy	
q}!!//2h6G6G6L6LL %%33A6(:K:K:T:TT	V U	
19!!//2h6G6G6P6PP %%33A6(:K:K:P:PP	R Q	
as/!&' 	''  
l	++r   c                :    t        j                  t        | |      S NrW  )r   apply_primitiverY  r   rX  s     r   _mesh_cast_implre    s    		!	!+q|	LLr   c                Z    t         j                  | |j                  j                        gS rb  )rY  r   r   r[  r  r   rX  s      r   _mesh_cast_transpose_rulerh    s#    


2AFFOO

<	==r   c               (   | j                   \  }| j                  \  }t        j                  j                  r|j                  |j                        n(|j                  |j                        j                         }t        j                  | |||      gS r   rA  r  r   rE  r   rF  rv  r  r<  r/   lower_with_sharding_in_typesr  rI  rX  r   aval_outprotos         r   _mesh_cast_hlo_loweringro    w    
,,%$mm)(++11 ((3,,TYY7@@B  
+
+C5
I	JJr   c                v    |\  }|\  }t        j                  | ||      }t        j                  ||      }||fS rb  )r.   get_sharding_for_vmaprY  r   r#  r-  r.  rX  r   rK  vmapped_dst_shardingrI  s           r   _mesh_cast_batcherru    sG    "!"!!77q"q';<!	
A+r   c                     yrQ  r   rR  s    r   rk  rk    rS  r   c                   t        |       \  }}t        d||      }|D cg c]  }t        j                  |       }}g }t	        |||      D ]~  \  }}}	t        |	d      }
|
t        d|	 d      |
j                  |
j                  j                  |j                              }
|j                  t        j                  ||
              t        ||      S c c}w )Nzreshard shardingsreshardziReshard should only be used with out_shardings which are non-None and have a nonempty mesh. Got sharding r  )r:  rW  )rH   r   r   r  rX   r@   r   r  r:  _normalized_spec_for_avalrv  r   	reshard_pr   rI   )rL  rZ  r'  r   r  r   r(  r   x_avalr!  dss              r   rx  rx    s     $/&' 3WmL.6<=$))!,=,=(v|^D 8la	q)	,B	z4453a9  
99&++F	GBOOINN12N678 
	** >s   Crx  c           	     X   | j                   }|j                  j                  sw|j                  j                  |j                  j                  k7  rJt	        d|j                  j                   d|j                  j                   d| j                                | j                  |      S )NzMesh of the input z0 does not equal the mesh of the target sharding r2  r[  )r[  r   r  r  r   r   r  )r   rX  r_  s      r   _reshard_abstract_evalr~    s    ,



!
!%%):):)H)HH

\..<<= >++,K8H7I	KL L 
l	++r   c                :    t        j                  t        | |      S rb  )r   rc  rz  rd  s     r   _reshard_implr    s    		!	!)Q\	JJr   c                v    t         j                  | |j                  j                         j                        gS rb  )rz  r   r   to_cotangent_avalr[  rg  s      r   _reshard_transpose_ruler    s+    
..!&&*B*B*D*M*M.
N	OOr   c               (   | j                   \  }| j                  \  }t        j                  j                  r|j                  |j                        n(|j                  |j                        j                         }t        j                  | |||      gS r   rj  rl  s         r   _reshard_hlo_loweringr    rp  r   c                    | j                   J |\  }|\  }t        j                  | ||      }t        j	                  ||      }||fS rb  )r!  r.   rr  rz  r   rs  s           r   _reshard_batcherr    sV    				$$	$"!"!!77q"nnQ%9n:!	
A+r   c                     yrQ  r   rR  s    r   rk  rk    rS  r   c           	        t        j                         }t        |      \  }}t         j                  }|D ]b  }	t	        |	t
              s|j                  s-||	j                  k7  rt        d| d| d|	j                         |	j                  j                  }d |j                  r|j                  rt        d| d| d      |j                  s&|j                  s||k7  rt        d| d| d      |}
n7|j                  r|j                  s|}
n|j                  s|j                  sJ |}
| |
j                  } t	        | t              s| f} | D ]q  }|s|
j                  |   t         j                  j                  k(  s1|t         j                  j                  t         j                  j                   hv sht#        d	       |
j%                  | D ci c]  }|| c}      S c c}w )
NzShardings passed to z) should have the same mesh. Got one mesh z and another zContext mesh zb cannot be empty. Please use `jax.sharding.use_mesh` API to enter into a mesh context when using `z` API.z) must match the mesh passed to shardings zI. Recommended approach is to use `jax.sharding.use_mesh` context manager.zGoing from `Manual` AxisType to `Auto` or `Explicit` AxisType is not allowed. Please file a bug at https://github.com/jax-ml/jax/issues with your use case)r  r  rH   empty_abstract_meshr   r7   r  r   r   r  
axis_namesrM  r  r  Manualr  r^  r  update_axis_types)r  	axis_typer   rr   error_on_manual_to_auto_explicitr  flat_shardingsr   sharding_meshr   mesh_to_user"  s               r   _get_new_meshr    s    '')("9-.!..- +a!]#  ]aff%<"4& )"O=:; 	; ff**m+ X^^

z "F&	  
		X^^ ( $_ 667 7 K8>>K""x~~55K	\!!D	D%	 7D !a(!!!$(9(9(@(@@h'',,h.?.?.H.HII ! !	! 
	&	&d'C9'C	DD'Cs   
G$)r  out_shardingc                     fd}|S )Nc                 R   !d|v r|j                  d      }nt        d      }t        t        j                  j
                  d|d      t        j                        5  t        fd|       }t        | |      }  | i |}d d d        t        |      S # 1 sw Y   xY w)Nr  z1Missing required keyword argument: 'out_sharding'	auto_axesT)rr  r  c                ~    t        j                  t        j                  |       j                  j                        S r   )r   modify_spec_for_auto_manualr  r[  r:  )r"  new_meshs    r   rk  z.auto_axes.<locals>.decorator.<locals>.<lambda>  s,    D$D$D
--

#
#
(
((%4 r   )	popr   r  r  r  r  r  rT   rU  )	r   r   _out_shardingin_specsr`  r  r  r   r  s	        @r   	decoratorzauto_axes.<locals>.decorator  s    	6	!

>2KLL"mh$$k])-/H 
	#	#H	- ! 459;htX&d  c	!
 S-((! !s   %$BB&r   )r   r  r  r  s   ``` r   r  r    s    )" 
r   c               '     K   t        | t        j                  j                  d      }t        j                  |      5  d  d d d        y # 1 sw Y   y xY ww)Nuse_auto_axes)r  r  r  r  r  r  r  s     r   r  r  &  sE     4!2!2!7!7I(!!(+ 
	
 
 
   ;AA	AAA)r  in_shardingc                     fd}|S )Nc                 J   !d|v r|j                  d      }nt        d      }t        t        j                  j
                  dd      }t        j                  |      5  t        | |      }  | i |}d d d        t        d       }t        ||      S # 1 sw Y   "xY w)Nr  z0Missing required keyword argument: 'in_sharding'explicit_axesT)r  c                    t        j                  t        j                  |       j                  j                  t        j                               S r   )r   r  r  r[  r:  r  r  )r  s    r   rk  z2explicit_axes.<locals>.decorator.<locals>.<lambda><  s5    4#C#Ca!!&&(B(B(D$F r   )	r  r   r  r  r  r^  r  rU  rT   )	r   r   _in_shardingr  r`  	out_specsr  r   r  s	         r   r  z explicit_axes.<locals>.decorator/  s    	&	 zz-0JKK lT8#4#4#=#=>BDH		#	#H	- !t\*d  c!  FGJLIS)$$! !s   #BB"r   )r   r  r  r  s   ``` r   r  r  -  s    %  
r   c               '     K   t        | t        j                  j                  d      }t        j                  |      5  d  d d d        y # 1 sw Y   y xY ww)Nuse_explicit_axes)r  r  r  r^  r  r  s     r   r  r  A  sH     4!2!2!;!;.0(!!(+ 
	
 
 
r  c           	        t        |       \  }}|D  cg c]  } t        j                  |        }} t        t	        d||            }t        d |D              r%t        d|D cg c]  }t        |       c}       t        ||dt        |      z  d       t        ||      D cg c]  \  }}t        j                  ||       }}}t        ||      S c c} w c c}w c c}}w )Nzwith_layout_constraint layoutsc              3  >   K   | ]  }t        |t                 y wr   )r   rD   r(  s     r   r   z)with_layout_constraint.<locals>.<genexpr>O  s     D!Z,-	-Ds   zTlayouts passed to `with_layout_constraint` must be of type `DeviceLocalLayout`. Got r   z with_layout_constraint arguments)r"  )rH   r   r  rM  r   r  r   r   r1  r   r   layout_constraint_pr   rI   )	r   rq  r'  r"  r(  layouts_flatr  r-  r   s	            r   with_layout_constraintr  J  s    a,&$6<=$))!,=,=|$Dd$+- .,D|DD
	%7C%D!d1g%D$E	GH H "L%#l*;";(* 6<0
2b! 
"
"2a
"
0 
2$ 
2	d	## > &E
2s   C#C
'!C!layout_constraintc                    | S r   r   r<  s     r   rk  rk  [  s    Q r   c                0    t        j                  | fi |fS r   )r  r   r>  s      r   rk  rk  ]  s    ':'?'?'Mf'M&O r   c                  t        | t        j                        st        dt	        |              | j
                  j                  |k(  r| S  t        j                  t        t        || j                              |       S )NzLwith_layout_constraint in eager mode can only be applied to jax.Arrays. Got r5  )r   r   r   r   r   rS  rn  r   r   r0  rC   r[  )r   r"  s     r   _layout_constraint_implr  _  sl    	Ar||	$
	 G9	&' ' XX!!V+H	HVFAJJ-G	H	KKr   c               n    | j                   \  }| j                  \  }t        j                  | ||||      gS r   )rA  r  r/   rH  )r  rI  r"  r   rf  s        r   _layout_constraint_hlo_loweringr  i  s4    
,,%$mm)(

"
"3&$
G	HHr   c                    t         r   r	  )r#  r-  r.  r"  s       r   _layout_constraint_batcherr  p  s    r   c                     yrQ  r   rR  s    r   rk  rk  s  rS  r   c                h    | j                   J t        d t        | j                         D              S )Nc              3  J   K   | ]  \  }}|t         j                  u r|  y wr   )r+   r;  )r   r   r  s      r   r   z)get_unconstrained_dims.<locals>.<genexpr>y  s)      ;Dm999  ;   !#)r:  r%  rN  r[  s    r   r$  r$  w  s4    		""	"	 ;Ihmm$< ; 
; ;r   c           
        t        || d d D cg c]  \  }}}|j                   c}}      }t        | |      D ]  \  \  }}\  }}}}	|t        j                  u rt        ||	      }
t        |||
       :|t        j                  u r~|	\  }
t        |||
       _|t        j                  u rt        ||	      }
|j                  |
       |t        j                  u r#t        ||	      D ]  }|j                  |        J  y c c}}w )N)r[   r   r   r   r  rI   re   Appendrg   r   r   r   r   )r   r  r   tdvalssr   r  r  r   r   r  r   s               r   r   r   ~  s    
T-:LMhaQBMMM
N%36}e3L /&7%S$r||7F+c#tS!		
dcS$$		7F+c	ggcl		 &1 $

4 U Ns   Dc                   g }| D ]  \  }}\  }}}|t         j                  u rEt        ||      rt        ||      nt        }t        |      \  }}	||	k(  sJ |j                  |       b|t         j                  u ru|t         j                  u r7|j                         }t        |      \  }}	||	k(  sJ |j                  |       |t         j                  u rJ  |S r   )r   r  r  rf   rd   rH   extendr  r   r  r   )
r   r  r   r   r  r  r   r"  r   r  s
             r   r   r     s    	$'4 #gq#3dr||'.sD'9[d#|d%d+fh   
kk&		
		WWYd%d+fh   
kk&		
U!" 
+r   c                 B    t        j                  t        d            S )NT)static)metadata)dataclassesfieldr  r   r   r   r  r    s    			D$5	66r   c                  0    e Zd ZU ded<    e       Zded<   y)BoxTreer{  r   rR   r   Nr   r   r   r   r  r   r   r   r   r  r    s     ,x'9r   r  c                  0    e Zd ZU ded<    e       Zded<   y)ListTreer{  r   zPyTreeDef | Noner   Nr  r   r   r   r  r    s     ,$h'&r   r  c                     t        d t        ||f      D              r||g fS g i t        j                          fd}t	        |||f      \  }}||fS )Nc              3  J   K   | ]  }t        |t        t        f         y wr   r  r   s     r   r   z!_flatten_boxes.<locals>.<genexpr>  s     MAZC;'	'Mr  c                   t        
      }t        | t        t        f      r	j	                  t        |       |      x}|k7  rnt        |       j                  }t        rFd| dj                   dj                   d|  dj                  |    dj                  |          d| d|       t        |       t        u rWt        | j                        \  }}t        d	 |D              }j                  |t         j"                  f       t%        ||      S t        |       t        u r-j                  |t         j&                  f       t)        g d       S | S )
Na zJ instance can't be passed as an argument more than once, but when tracing  for z, the object z appeared at both arguments z and zat both flat index z and flat index c              3  F   K   | ]  }t        j                  |        y wr   r  r(  s     r   r   z0_flatten_boxes.<locals>.visit.<locals>.<genexpr>  s     <((+<r  )r  r   rb   rc   r  r   r   r   r   func_src_info
traced_forr   rH   _valrM  r   r   r   r  r   r  )r   r   dup_idx	type_namer   r   r   r   r  id_first_occurrencesr  s          r   visitz_flatten_boxes.<locals>.visit  sO   T
A1sDk"(33BqE1=	=!Cq'""i
 	 yk $$'$5$5#6eCNN;K Ls6]]7#$E#--*:);=> >  y(8
<> > Aw#~$QVV,ofg<V<<booq"**o&VW%%	aDooq"++&'b$hr   )r  rL   r,  countrT   )r  r   r   r  r   r  r  s   `   @@@r   r  r    se    MdF^1LMM(	$, %$0,$	vx	r   c                ~   | |}}g }g fd}t        |||fd       \  }} ||i |}t        |      d   D ]g  \  }	}
t        |
t        t        f      st        |
      j                  }t        d| d|j                   d|j                   d|
 d	t        |	       
       s|S g }D ]  }t        |      t        u r4t        |j                        \  }}|j                  t        ||             Ht        |      t        u r4t        |j                        \  }}|j                  t        ||             J  ||fS )
Nc                   t        |       t        u r<t        t        | j                  | j
                              }j                  |       |S t        |       t        u r$t               }d|_	        j                  |       |S | S )NT)
r   r  rb   rI   r   r   r   r  rc   _is_arg)r   boxlstarg_mutabless      r   r  z_handle_boxes.<locals>.visit  sm    Aw'qyy!((34c#j	aH	Fcck#jhr   c                .    t        | t        t        f      S r   )r   r  r  rj  s    r   rk  z_handle_boxes.<locals>.<lambda>  s    Jq7H:M,N r   rl  r   r  zJ instance can't be returned from a transformed function, but when tracing r  z the object z appeared at result)rT   rU   r   rb   rc   r   r   r   r  r  rQ   rH   r  r   r  r  )__f__dbgr   r   r  r  new_argsr  r`  r~  leafr  
extra_outsmutabler   r   r  s                   @r   r  r    se   S!(, %$"NP,$	46#*3/2 AjdD$d$t*%%iyk ((+(9(9':%?O Pv0@A AA 
J* gG}$W\\2ofg01	g$	$W\\2ofg12U 
Sr   )r   r   r   ri   )r  z&pxla.MeshExecutableFastpathData | None)r*  rm   r  )r   r   r#  r
   rZ  r
   ry   int | Sequence[int] | Noner{   str | Iterable[str] | Noner|   r  r}   r  r   rm   r   r~   r   r   r   rm   r   r   r  dict[str, Any] | Noner   rm   r  ri   )r   r   r#  r
   rZ  r
   ry   r  r{   r  r|   r  r}   r  r   rm   r   r~   r   r   r   rm   r   r   r  r  r   rm   r  r
   )r   r   r  ri   r  mesh_lib.Mesh | Noner  core.DebugInfor   rp   r   r  r   z%tuple[core.AbstractValue, ...] | Noner  tuple[PjitParams, list[Any]])r   r   r   ri   r  zjax_jit.ArgumentSignaturer   r  r  r  r  r  )
r   r   r  ri   r   rp   r   r  r  r  )r   r   r  r  r  r  )r  z)Sequence[tuple[core.AbstractValue, bool]]r  zSequence[Any]r  zSequence[core.Tracer])r  z"list[pe.AbstractedAxesSpec] | None)r   r   r#  r
   rZ  r
   ry   r  r{   r  r|   r  r}   r  r   rm   r   r~   r   r   r   rm   r   r   r  r  r  r  )r&   r  )r^  rM  r  rM  r&   zlu.DebugInfor  ztuple[Sequence[str], int])
r   lu.WrappedFunr  rm   r  r  r  rM  r  float)
r   r  r  z-core.InputType | Sequence[core.AbstractValue]r  r  r  r  r  zutuple[core.ClosedJaxpr, list[Any], list[core.AbstractValue], list[tuple[PyTreeDef, PyTreeDef, tuple[Any, str, Any]]]])r   r  r  /core.InputType | tuple[core.AbstractValue, ...]r  r{  )r   r  r  r  r  r  )r  r  r  rj   r.  rm   )r  r  r  rj   )r  rm   )rw  Sequence[PjitSharding]r  r  )r   core.ClosedJaxprr  pxla.MaybeLayoutr  r  r   rj   r   rm   r   rm   r   r   r|  tuple[str, ...] | Noner}  zmlir.LoweringParametersr   zprofiler.PGLEProfiler | None)r   r  r  zlist[core.AbstractValue])r  zmlir.LoweringRuleContextr   rj   r   r  )r.  rx   r   r  )r!  zSharding | UnspecifiedValuerB  zint | batching.RaggedAxisrC  r  rv  r  )r@  zpe.JaxprTracer   r  )r  Sequence[bool]r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  ztuple[dict, dict])r   r  r   zSequence[core.AbstractValue])r   r  )r   r  r  r  r  z#tuple[core.ClosedJaxpr, list[bool]])r  z
list[bool]r  core.JaxprEqnr  z'tuple[list[bool], core.JaxprEqn | None])r  r  r  zcore.JaxprPpContextr  zcore.JaxprPpSettingsr  zcore.pp.Doc)NF)r  str | tuple[str, ...] | Noner  zmesh_lib.AxisTyper   rj   )r  r  )r[  r7   (8  
__future__r   collectionsr   collections.abcr   r   r   
contextlibr  	functoolsr   inspect	itertoolsr,  r  r  typingr	   r
   r   r   rx  numpyro  jax._srcr   r   r   r   r   r   r   r  r   r  r   r   r   r   r   r   r   r   r   r  jax._src.api_utilr   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   jax._src.interpretersr*   r   jax._src.partition_specr+   r,   r-   r.   r/   r0   jax._src.lib.mlirr1   jax._src.lib.mlir.dialectsr2   r  jax._src.libr3   r4   r   jax._src.meshr5   jax._src.shardingr6   jax._src.sharding_implsr7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   jax._src.layoutrC   rD   rE   jax._src.state.typesrF   jax._src.traceback_utilrG   jax._src.tree_utilrH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   r|  rT   rU   jax._src.utilrV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   jax._src.attrsrb   rc   rd   re   rf   rg   r   
unsafe_mapr   
unsafe_zipregister_exclusion__file__PjitShardingPjitShardingMinusUnspecifiedMeshShardingMeshShardingMinusUnspecified	getLoggerr   r  ri   r   r   r%  r5  PjitFunctionCacher)  r(  r+  rG  r?  r=  rA  r;  rs  r  r  r  r  r  rf  r   r  r  r  r  Wrappedr  r6  r  r  r  r&  r   r  r  r   r4  r   r  r  rd  r  rM  r  rj   r{  AbstractValue
AttrRecordWeakKeyDictionaryr  r  r  r  	dataclassr  r/  r1  	Primitiver   multiple_resultsskip_canonicalizationr  r   rB  to_lojaxr#  r&  rb  rg  rk  rz  rR  r  r   r  r  def_implr~  r  r  custom_staging_rulesr  r  forwarding_rulesr  r  custom_typechecksr  def_effectful_abstract_evalr  r  register_loweringr4  fancy_primitive_batchersragged_mask_no_op_ruleragged_prop_rulesr   r_  primitive_jvpsr  primitive_linearizationsr  custom_partial_eval_rulesr  $closed_call_partial_eval_custom_rulepartial_eval_jaxpr_custom_rulesr  r  primitive_transposesr  r  	dce_rulesr  pp_eqn_rulesr  r  r0  r9  r&  def_abstract_eval
deflinear2rK  rO  skippable_batchersrU  rY  r`  re  rh  ro  ru  rx  rz  r~  r  r  r  r  r  r  contextmanagerr  r  r  r  r  r  r  r  r$  r   r   r  register_dataclassr  r  r  transformation2r  r   r   r   <module>r=     s   # # 8 8        / /         & % !  # %  #   %& & & & 5 1 % $ * & &   ;   ) & &/ / / /
 B A * 0P P P P5 5 5 5, , CZCZ ! ! !( +]$4d:;$]D%89 ]$4d:;$]D%89 			8	$$z $NE;P3- ,-h 7744d4C  ')gg&?&?&?&N #$1f < < 6 6 ) )%N*X3(+X3)CX3 +EX3 *D	X3
 +EX3 '+X3 5EX3 #-X3 7;X3 +5X3 ,AX3 ,0X3 5=X3t""  " 8	"
 9" 8" 9" " &" !" " )"  5"  $" ),"4
 
A8	A8A8 #A8 
	A8
 A8 A8 4A8 "A8H  !	!! )! -	!
 #! ! !7	77'67@N7#7?	??'6?@N?#?:(F4:4:: :<:<:  $$15261526#"&.2}	}} } /	}
 0} /} 0} } } } }  } ,} }@3<:
4,l+ + T6 ,>L , 7 ,D /2e !8 3G7G7G7'3G78QG7T9		9	"&9	/39		9	"9	x 	56#E	#E:#E #E 	#E
D#E 7#EL T6. . 7.@ 63	40B0B+CCD
'g'')6	6<6 
6	< 	  d#( ( $(3"/33+/3D" -":="  
	 # 'P %.;$z
 FH&5H&V#  0g//1 7>r 
9 
9LB   -
 ## !	#
 "# # # # 6# /# 1# 0# #49t #4   3	 3  F 
 @ "1  v (  " "#6 7%*-%6F%P*4   v~ .*%**%)*%X -: ! !& )%-%D%D  6 "%:"%:	"%:4J%: %:PQ> &  & Q9f ':  F #{H.{Hz (:  V $,-,-&4,-#,-7E,- ,- #&,- 6:,- KO,- 	,-^ B33W46  " "6 *
 ($@( 
(J5+J5V #2    @@+;@(@ @ D > +V P.P0P5@P< *  & 
>*F$P %LP ''<=    8 9  ' '(8 9 #QS&   ,8:%L <X ! !"7 85A  1 2
	+ dnn[)$( !$,J   6 7M   _ %> k4 5K   {$; < 2D ! !+ .+7  K (+  DNN9%	, 
  2 3K 	  = !P i0 1K   y"7 8 0@ ! !) ,)5  I &
 FJ38,E.,E69,E\ <@* 
 
 @D"( 
 
$  %dnn%89   % %&6 7 !OQL   4 5I   *689S ! !"5 63?  / 0;&*7        ' '  ' F & &r   