
    &Vf                         d dl Z d dlZd dlmZ d dlZd dlZd dlmZ d dlm	Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ  G d dej                  Zd Z G d de          ZdS )    N)partial)backend)	callbacks)
optimizers)tree)distribution_lib)trainer)array_slicing)data_adapter_utils)EpochIterator)traceback_utilsc                   N    e Zd Z fdZ	 	 ddZd Zd Zd ZddZdd	Z	dd
Z
ej        	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd            Zej        	 	 	 	 	 	 	 	 dd            Zej        	 dd            Z	 	 	 	 d dZ	 	 	 d!dZd Zd Zd Z	 	 	 	 d"dZ	 	 	 	 d#dZ	 	 	 	 	 d$dZ xZS )%
JAXTrainerc                     t                                                       d | _        d | _        d | _        d| _        d S )NT)super__init__train_functiontest_functionpredict_function_jax_state_synced)self	__class__s    Z/var/www/html/software/conda/lib/python3.11/site-packages/keras/src/backend/jax/trainer.pyr   zJAXTrainer.__init__   s?    "! $!%    FNc	           	         i }	| j         r||	d<    | j        |||fddi|	\  }
}}|r | j                                         || _        |                     ||||||
|          \  }}|r| j                                         |\  }}}|}|ru| j        nt          t          | j        j        |                    }t          j
        |          5  | j                            |          }ddd           n# 1 swxY w Y   |||
||ffS )z?This method is stateless and is intended for use with jax.grad.trainingreturn_lossesT)xyy_predsample_weightNstate_mapping)_call_has_training_argstateless_call_losses_overrideclearstateless_compute_loss	optimizerlistzip	variablesr   StatelessScope
scale_loss)r   trainable_variablesnon_trainable_variablesmetrics_variablesr   r   r!   r   optimizer_variableskwargsr    losseslossr,   unscaled_lossmappings                   r   compute_loss_and_updatesz#JAXTrainer.compute_loss_and_updates   s    & 	*!)F: 3F$2E#3
 3
 	3

 3
 3
/'  	+!'')))$*D!55#' 6 
 
i  	*!''))) 	J	57H
  	723t~79LMMNNG'g>>> 7 7~00667 7 7 7 7 7 7 7 7 7 7 7 7 7 7#	
 
 	
s   C55C9<C9c           
         |\  }}}}t          j        |          \  }}}	t          j        | j        d          }
 |
||||||	d|          \  \  }}}|\  }}}}| j                            |||          \  }}t          j        d t          | j
        |          D                       5 }| j                            |t          j        |          d         j        d                    |                     ||||	          }d d d            n# 1 swxY w Y   g }| j
        D ]5}|                    |          }||j        }|                    |           6|}|                     ||||          }||fS )NT)has_aux)r   r2   c                     g | ]	\  }}||f
S  r<   .0ref_vvs      r   
<listcomp>z)JAXTrainer.train_step.<locals>.<listcomp>x   0       E1 
  r   r"   r   r!   )r   unpack_x_y_sample_weightjaxvalue_and_gradr8   r)   stateless_applyr   r-   r+   r1   _loss_trackerupdate_stater   flattenshapecompute_metricsget_current_valuevalueappend_enforce_jax_state_sharding)r   statedatar/   r0   r2   r1   r   r   r!   grad_fnr5   auxgradsr6   r    scopelogsnew_metrics_variablesr?   new_vs                        r   
train_stepzJAXTrainer.train_stepW   sG    	
#0I$OO1m$)4
 
 
 %W# 3	
 	
 	
sU  	L 79J N**(;
 
	

 #  #D$:<M N N  
 
 
 		E
 ++T\!__Q-?-Ea-H ,    ''1fmDDD		E 		E 		E 		E 		E 		E 		E 		E 		E 		E 		E 		E 		E 		E 		E !#+ 	0 	0E++E22E}!((////100#	
 
 U{s   ,ADDDc           	         |\  }}}t          j        |          \  }}}|                     ||||||d          \  }	}
|
\  }}}}t          j        d t          | j        |          D                       5 }| j                            |t          j
        |          d         j        d                    |                     ||||          }d d d            n# 1 swxY w Y   g }| j        D ]5}|                    |          }||j        }|                    |           6|}|                     ||d |          \  }}}}|||f}||fS )NF)r   c                     g | ]	\  }}||f
S r<   r<   r=   s      r   rA   z(JAXTrainer.test_step.<locals>.<listcomp>   rB   r   r"   r   rC   r/   r0   r2   r1   )r   rD   r8   r   r-   r+   r1   rH   rI   r   rJ   rK   rL   rM   rN   rO   rP   )r   rQ   rR   r/   r0   r1   r   r   r!   r5   rT   r6   r    rV   rW   rX   r?   rY   _s                      r   	test_stepzJAXTrainer.test_step   s   
 		
#0I$OO1m11# 2 
 
	c  	L 79J #  #D$:<M N N  
 
 
 		E
 ++T\!__Q-?-Ea-H ,    ''1fmDDD		E 		E 		E 		E 		E 		E 		E 		E 		E 		E 		E 		E 		E 		E 		E !#+ 	0 	0E++E22E}!((////1 ,, 3$; $/	 - 
 
	
#  #

 U{s   7ACC"Cc                     |\  }}i }| j         rd|d<   t          j        |          \  }}} | j        |||fi |\  }}|                     d |d d           \  }}}}||fS )NFr   r]   )r$   r   rD   r%   rP   )	r   rQ   rR   r/   r0   r3   r   r^   outputss	            r   predict_stepzJAXTrainer.predict_step   s    7<44& 	'!&F:$=dCC1a+>4+>!8!,
 ,
?E,
 ,
(( ,, $$; $"	 - 
 
	
# ///r   c                       j         |sd S  fdfd} j        dk    r|n j        s6 j        r/t	          t
          j        d          fd            }| _         d S  _         d S )Nc                 @    |d         }                     | |          S Nr   )rZ   rQ   rR   r   s     r   one_train_stepz6JAXTrainer.make_train_function.<locals>.one_train_step   s     7D??5$///r   c                 6    |D ]} | |g          \  }} || fS Nr<   )rQ   rR   single_step_datarW   rg   s       r   multi_train_stepsz9JAXTrainer.make_train_function.<locals>.multi_train_steps   s:    $( H H ,nU5E4FGGee;r      rQ   donate_argnamesc                      | |          S ri   r<   )rQ   rR   rZ   s     r   compiled_train_stepz;JAXTrainer.make_train_function.<locals>.compiled_train_step   s    !z%...r   )r   steps_per_executionrun_eagerlyjit_compiler   rE   jit)r   forcerk   rp   rg   rZ   s   `   @@r   make_train_functionzJAXTrainer.make_train_function   s    *5*F	0 	0 	0 	0 	0	 	 	 	 	
 #a''*JJ'J 	-D$4 	-
 SWg666/ / / / 76/ #6D #-Dr   c                       j         |sd S  fdfd} j        dk    r|n j        s6 j        r/t	          t
          j        d          fd            }| _         d S  _         d S )Nc                 @    |d         }                     | |          S re   )r_   rf   s     r   one_test_stepz4JAXTrainer.make_test_function.<locals>.one_test_step  s     7D>>%...r   c                 6    |D ]} | |g          \  }} || fS ri   r<   )rQ   rR   rj   rW   ry   s       r   multi_test_stepsz7JAXTrainer.make_test_function.<locals>.multi_test_steps
  s:    $( G G +mE4D3EFFee;r   rl   rQ   rm   c                      | |          S ri   r<   )rQ   rR   r_   s     r   compiled_test_stepz9JAXTrainer.make_test_function.<locals>.compiled_test_step  s     y---r   )r   rq   rr   rs   r   rE   rt   )r   ru   r{   r}   ry   r_   s   `   @@r   make_test_functionzJAXTrainer.make_test_function  s    )%)F	/ 	/ 	/ 	/ 	/	 	 	 	 	
 #a''(II%I 	+D$4 	+
 SWg666. . . . 76. "4D "+Dr   c                       j         	|s j         S  fdfd} j        dk    r|n j        s' j        r t          j        fd            }| _         d S  _         d S )Nc                 @    |d         }                     | |          S re   )rb   rf   s     r   one_predict_stepz:JAXTrainer.make_predict_function.<locals>.one_predict_step&  s"    7D$$UD111r   c                      | |d d                   \  }}|dd          D ])} | |g          \  }}t          j        d ||          }*||fS )Nrl   c                 D    t           j                            | |g          S ri   )rE   numpyconcatenate)t1t2s     r   <lambda>zOJAXTrainer.make_predict_function.<locals>.multi_predict_steps.<locals>.<lambda>3  s    39#8#8"b#B#B r   )r   map_structure)rQ   rR   ra   r/   rj   step_outputsr   s         r   multi_predict_stepsz=JAXTrainer.make_predict_function.<locals>.multi_predict_steps*  s    +;+;E48+L+L(G($(H 	 	 4D4D%&5 511 ,BB  
 ///r   rl   c                      | |          S ri   r<   )rQ   rR   rb   s     r   compiled_predict_stepz?JAXTrainer.make_predict_function.<locals>.compiled_predict_step@  s    #|E4000r   )r   rq   rr   rs   rE   rt   )r   ru   r   r   r   rb   s   `   @@r   make_predict_functionz JAXTrainer.make_predict_function"  s     ,U,((	2 	2 	2 	2 	2	0 	0 	0 	0 	0 #a''.LL+L 		1D$4 		1W1 1 1 1 W1 %:D!!! %1D!!!r   rl   auto        Tr   c                 \   |                      d           d | _        |r"| t          j        |||f|          \  \  }}}}|t	          j        |          \  }}}t          ||||||	|
| j                  }|                     |           t          |t          j                  s$t          j        |d|dk    |||j        |           }|                                  |                                  d| _        |                                 | j        p|}t%          ||          D ]
}|                                  |                    |           d| _        |                                D ]\  }}|                    |           | j        r!|                     ddddd	          }d| _        |                     ||          \  }}|\  }}}}||||d
| _        |                     |          }|                    ||           | j        r n|                                  t=          j        d          5  tA          | !                    |                    }d d d            n# 1 swxY w Y   || "                    ||          rtG          | dd           "t          ||||p|| j        |d          | _        | $                    ||||p|||dd          }d |%                                D             }|&                    |           |'                    ||           |} | j        r nt          | j(        tR          j*                  r%|dk    r| j(        +                    | j,                   tG          | dd           | `|-                    |            d | _        | j.        S )Nfit)validation_split)r   r   r!   
batch_sizesteps_per_epochshuffleclass_weightrq   iteratorTr   add_historyadd_progbarverboseepochsstepsmodelFr/   r0   r2   r1   purge_model_variablesr]   	allow_all_eval_epoch_iterator)r   r   r!   r   rq   r   r   )r   r   r!   r   r   r   return_dict_use_cached_eval_datasetc                      i | ]\  }}d |z   |S )val_r<   )r>   namevals      r   
<dictcomp>z"JAXTrainer.fit.<locals>.<dictcomp>  s/       +44FTM3  r   )rW   )/_assert_compile_calledr   r
   train_validation_splitr   rD   JAXEpochIteratorrq   _symbolic_build
isinstancecallbacks_moduleCallbackListnum_batches$_record_training_state_sharding_specrv   stop_trainingon_train_begin_initial_epochrangereset_metricson_epoch_beginr   enumerate_epochon_train_batch_begin_get_jax_stater   
_jax_state_pythonify_logson_train_batch_endjax_state_syncrE   	spmd_modedict_get_metrics_result_or_logs_should_evalgetattrevaluateitemsupdateon_epoch_endr)   optimizers_module	Optimizerfinalize_variable_valuestrainable_weightson_train_endhistory)!r   r   r   r   r   r   r   r   validation_datar   r   r!   initial_epochr   validation_stepsvalidation_batch_sizevalidation_freqval_xval_yval_sample_weightepoch_iteratorepochsteprR   rQ   rW   r/   r0   r2   r1   
epoch_logsval_logstraining_logss!                                    r   r   zJAXTrainer.fitI  s   ( 	##E***$(! 		 7 "/!EA}%8H" " "	  &
 #;OLL	! *'!+% $ 8	
 	
 	
 	n555 )%5%BCC 		(5 #qL$0  I 	11333  """"  """+<}=&11 Y	 Y	E   $$U+++%)D",<<>> % %
d..t444 ) 	3 //,004,0*..2 0  E .3D*"11%>>e '+'% ,?/F+>):	# # ++D11,,T4888% E !!! {++ J J!$"B"B4"H"HII
J J J J J J J J J J J J J J J *t/@/@0 0* 4!7>>F0@&7#8#FJ,0,D(8 %1 1 1D-  =="34B
*' $-1 ) 	 	 8@8H8H   !!(+++""5*555&M!  t~'8'BCC	L

N33D4JKKK 4/66B)M222|s   9#I((I,	/I,	c	           	         |                      d           |	                    dd          }
|	rt          d|	           |
r| j        }nt	          |||||d| j                  }|                     |           t          |t          j	                  s$t          j	        |d|dk    |d	|j
        | 
          }|                                  |                                  d| _        |                                 d }|                                  d| _        |                                D ]\  }}|                    |           | j        r |                     dddd          }d| _        |                     ||          \  }}|\  }}}|||d| _        |                     |          }|                    ||           | j        r n|                                  t3          j        d          5  |                     |          }d d d            n# 1 swxY w Y   |                    |           d | _        |r|S |                     |          S )Nr   r   FzArguments not recognized: )r   r   r!   r   r   r   rq   r   Tr   rl   r   r/   r0   r1   r   r/   r0   r1   r   )r   pop
ValueErrorr   r   rq   r   r   r   r   r   r   r~   stop_evaluatingon_test_beginr   r   r   on_test_batch_beginr   r   r   r   on_test_batch_endr   rE   r   r   on_test_end_flatten_metrics_in_order)r   r   r   r   r   r!   r   r   r   r3   use_cached_eval_datasetr   rW   r   rR   rQ   r/   r0   r1   s                      r   r   zJAXTrainer.evaluate  s    	##J///"(**-G"O"O 	DB&BBCCC" 	!6NN .+% %$($<  N 	n555 )%5%BCC 		(5 #qL$0  I 	11333!!!$!!!!%(88::  	  	JD$))$///% /++(,,0&**.	 ,   */&,,UD99KD%
 	#'! (;+B%6 DO ''--D''d333#  	
 ];'' 	: 	:33D99D	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	: 	:d### 	K--d333s   5HHHc           	         t          |||d| j                  }t          d |                                 D                       sn|                                D ]Y\  }}t          j        |d                   \  }}}t          j                    5   | |           d d d            n# 1 swxY w Y    t          |t          j                  s$t          j        |d|dk    |d|j        |           }|                                  |                                  d| _        |                                 d }	d| _        d }
d }|                                D ]\  }}|                    |           | j        r5|                     dd	          }|                     d
           d| _        n
|d         |f}|                     ||          \  }} |	||
          }
|                    |d|i           | j        r nd|i| _        |                                  |                                 d | _        t5          j        |t8          j        |
          S )NF)r   r   r   r   rq   c              3   $   K   | ]}|j         V  d S ri   builtr>   layers     r   	<genexpr>z%JAXTrainer.predict.<locals>.<genexpr>m  $      CC55;CCCCCCr   r   Trl   r   c                 h    |t          j        d |           }nt          j        | d ||            |S )Nc                     | gS ri   r<   )batch_outputs    r   r   z?JAXTrainer.predict.<locals>.append_to_outputs.<locals>.<lambda>  s    , r   c                 ,    |                      |          S ri   )rO   )outputr   s     r   r   z?JAXTrainer.predict.<locals>.append_to_outputs.<locals>.<lambda>  s    |1L1L r   )r   r   map_structure_up_to)batch_outputsra   s     r   append_to_outputsz-JAXTrainer.predict.<locals>.append_to_outputs  sQ    ,77! 
 (!LL!	   Nr   )r/   r0   )r0   ra   r0   )r   rq   all_flatten_layersr   r   rD   r   r-   r   r   r   r   r   r   stop_predictingon_predict_beginr   on_predict_batch_beginr   _purge_model_variablesr   on_predict_batch_endr   r   on_predict_endr   r   npr   )r   r   r   r   r   r   r   r^   rR   r   ra   r0   r   rQ   r   s                  r   predictzJAXTrainer.predict`  s   
 *!! $ 8
 
 
 CCD,@,@,B,BCCCCC 	)99;;  4,Ed1gNN1a+--  DGGG               )%5%BCC 		(5 #qL$0  I 	11333""$$$$""$$$	 	 	 "&"&%5577 	 	GD!,,T222% 	<++(,,0 ,   ++D+III).&&q#:;595J5Jq6 62M2 ('w??G**4)]1KLLL#  &'>

 	  """'r~wOOOs   B''B+	.B+	c                 N   |                      d           |,|t          d| d|           t          j        ||          }|||f}t	          |          }|                     |           |                                  |                                  |                     ddddd          }d| _	        | 
                    ||g          \  }}|\  }	}
}}|	|
||d| _        |                                  t          j        d	 |          }|r|S |                     |          S )
Ntrain_on_batchzkArguments `sample_weight` and `class_weight` cannot be specified at the same time. Received: sample_weight=z, class_weight=
data_batchTFr   r]   c                 *    t          j        |           S ri   r  arrayr   s    r   r   z+JAXTrainer.train_on_batch.<locals>.<lambda>      BHQKK r   )r   r   r   class_weight_to_sample_weights_distribute_datar   r   rv   r   r   r   r   r   r   r   r   )r   r   r   r!   r   r   rR   rQ   rW   r/   r0   r2   r1   s                r   r  zJAXTrainer.train_on_batch  s    	##$4555#( 3/<3 3 %13 3   /M< M 1m$%% 	---11333  """ ## $$( $""' $ 
 
 "'))%$88e 	
# $7'>#6!2	
 
 	 !"7"7>> 	K--d333r   c                    |                      d           |||f}t          |          }|                     |           |                                  |                                  |                     dddd          }d| _        |                     ||g          \  }}|\  }}	}
||	|
d| _        | 	                                 t          j        d |          }|r|S |                     |          S )Ntest_on_batchr  TFr   r   c                 *    t          j        |           S ri   r
  r  s    r   r   z*JAXTrainer.test_on_batch.<locals>.<lambda>  r  r   )r   r  r   r   r~   r   r   r   r   r   r   r   r   )r   r   r   r!   r   rR   rQ   rW   r/   r0   r1   s              r   r  zJAXTrainer.test_on_batch  s0    	##O4441m$%%---11333!!! ## $$(""'	 $ 
 
 "'((77e KPG46G#6'>!2
 

 	 !"7"7>> 	K--d333r   c                    t          d |                                 D                       s6t          j                    5   | |           d d d            n# 1 swxY w Y   |                                  |                                  |                     dddd          }d| _        |                     ||fg          \  }}d|i| _	        | 
                                 t          j        d |          }|S )Nc              3   $   K   | ]}|j         V  d S ri   r   r   s     r   r   z.JAXTrainer.predict_on_batch.<locals>.<genexpr>  r   r   TFr   r0   c                 *    t          j        |           S ri   r
  r  s    r   r   z-JAXTrainer.predict_on_batch.<locals>.<lambda>2  s    RXa[[ r   )r   r   r   r-   r   r   r   r   r   r   r   r   r   )r   r   rQ   r   r0   s        r   predict_on_batchzJAXTrainer.predict_on_batch  sW   CCD,@,@,B,BCCCCC 	'))  Q              11333""$$$## $$(#"'	 $ 
 
 "'151F1FQD62
 2
.. &'>
 	*+@+@-PPs   AAAc                    t          | dd           r| j        rd S | j                            dd           }| j                            dd           }| j                            dd           }| j                            dd           }|r0t	          | j        |          D ]\  }}|                    |           |r0t	          | j        |          D ]\  }}|                    |           |r5t	          | j        j	        |          D ]\  }}|                    |           |r0t	          | j
        |          D ]\  }}|                    |           d| _        d S )Nr   r/   r0   r2   r1   T)r   r   r   getr+   r/   assignr0   r)   r,   r1   )r   r/   r0   r2   r1   r?   r@   s          r   r   zJAXTrainer.jax_state_sync5  s   t\400 	D4J 	F"o112GNN"&/"5"5%t#
 #
 #o112GNN O//0CTJJ 	  8:MNN    qQ" 	 ,.E     q Q 	  8:MNN    qQ 	  68IJJ    qQ!%r   c                     d | j         D             | _        d | j        D             | _        t	          | d          r#| j        d | j        j        D             | _        ng | _        d | j        D             | _	        d S )Nc                 &    g | ]}|j         j        S r<   rN   shardingr>   r@   s     r   rA   zCJAXTrainer._record_training_state_sharding_spec.<locals>.<listcomp>P  s*     .
 .
 .
!"AG.
 .
 .
r   c                 &    g | ]}|j         j        S r<   r  r  s     r   rA   zCJAXTrainer._record_training_state_sharding_spec.<locals>.<listcomp>S  s*     2
 2
 2
!"AG2
 2
 2
r   r)   c                 &    g | ]}|j         j        S r<   r  r  s     r   rA   zCJAXTrainer._record_training_state_sharding_spec.<locals>.<listcomp>W  s*     2 2 2%& 2 2 2r   c                 &    g | ]}|j         j        S r<   r  r  s     r   rA   zCJAXTrainer._record_training_state_sharding_spec.<locals>.<listcomp>\  s*     ,
 ,
 ,
!"AG,
 ,
 ,
r   )
r/   _trainable_variable_shardingsr0   !_non_trainable_variable_shardingshasattrr)   r,   _optimizer_variable_shardingsr1   _metrics_variable_shardingsr   s    r   r   z/JAXTrainer._record_training_state_sharding_specO  s    .
 .
&*&>.
 .
 .
*2
 2
&*&B2
 2
 2
. 4%% 	4$.*D2 2*..*B2 2 2D.. 24D.,
 ,
&*&<,
 ,
 ,
(((r   c                    |pg }|pg }|pg }|pg }t          t          |                    D ]6}t          j                            ||         | j        |                   ||<   7t          t          |                    D ]6}t          j                            ||         | j        |                   ||<   7t          t          |                    D ]6}t          j                            ||         | j        |                   ||<   7t          t          |                    D ]6}t          j                            ||         | j        |                   ||<   7||||fS )a  Enforce the sharding spec constraint for all the training state.

        Since the output of the train/eval step will be used as inputs to next
        step, we need to ensure that they have the same sharding spec, so that
        jax.jit won't have to recompile the train/eval function.

        Note that this function will also rely on the recorded sharding spec
        for each of states.

        This function is expected to be called within the jitted train/eval
        function, especially around the end of the function.
        )	r   lenrE   laxwith_sharding_constraintr"  r#  r%  r&  )r   r/   r0   r2   r1   is         r   rP   z&JAXTrainer._enforce_jax_state_sharding`  s   & 27R"9"?R17R-3s.//00 	 	A%(W%E%E#A&(J1(M& &"" s23344 	 	A),)I)I'*6q9* *#A&& s.//00 	 	A%(W%E%E#A&(J1(M& &"" s,--.. 	 	A#&7#C#C!!$d&Fq&I$ $a    #	
 	
r   c                     |r| j         D ]	}d|_        
|r| j        D ]	}d|_        
|r| j        j        D ]	}d|_        
|r| j        D ]}d|_        
dS dS )a  Remove all the model variable for memory saving.

        During JAX training, since the training function are stateless, we have
        to pass in and get the model weights over and over, during which the
        copy of the weights that attached to the KerasVariable are still and
        occupying extra memory. We remove those variable to save memory (for
        better memory utilization) at the beginning of the epoch, and reattach
        the value back to variables at the end of the epoch, via
        `jax_state_sync()`.
        N)r/   _valuer0   r)   r,   r1   )r   r/   r0   r2   r1   r@   s         r   r   z!JAXTrainer._purge_model_variables  s    "  	 -    " 	 1     	 ^-     	 +    	  	    r   c                    g }|r$|                     d | j        D                        |r$|                     d | j        D                        |r)|                     d | j        j        D                        |r$|                     d | j        D                        |r|                     ||||           t          |          S )Nc                     g | ]	}|j         
S r<   rN   r  s     r   rA   z-JAXTrainer._get_jax_state.<locals>.<listcomp>      DDDa!'DDDr   c                     g | ]	}|j         
S r<   r1  r  s     r   rA   z-JAXTrainer._get_jax_state.<locals>.<listcomp>  s    HHHa!'HHHr   c                     g | ]	}|j         
S r<   r1  r  s     r   rA   z-JAXTrainer._get_jax_state.<locals>.<listcomp>  r2  r   c                     g | ]	}|j         
S r<   r1  r  s     r   rA   z-JAXTrainer._get_jax_state.<locals>.<listcomp>  s    BBBa!'BBBr   r]   )rO   r/   r0   r)   r,   r1   r   tuple)r   r/   r0   r2   r1   r   rQ   s          r   r   zJAXTrainer._get_jax_state  s      	FLLDD4+CDDDEEE" 	JLLHH4+GHHHIII 	FLLDD4>+CDDDEEE 	DLLBB4+ABBBCCC  	''$7(?$7"3	 (    U||r   )FN)F)NNNrl   r   Nr   NTNNr   NNNrl   )NNNr   NNNF)Nr   NN)NNNF)NNF)NNNN)FFFF)FFFFF)__name__
__module____qualname__r   r8   rZ   r_   rb   rv   r~   r   r   filter_tracebackr   r   r  r  r  r  r   r   rP   r   r   __classcell__)r   s   @r   r   r      sl       & & & & &  8
 8
 8
 8
t9 9 9v7 7 7r0 0 0.- - - -@+ + + +@%1 %1 %1 %1N % 
"#m m m &%m^ % 
d4 d4 d4 &%d4L %HLTP TP TP &%TPr :4 :4 :4 :4~ '4 '4 '4 '4R  2& & &4
 
 
& ! $ .
 .
 .
 .
d " %!       @ " %!#       r   r   c                     t          j                    fd}t          j        ||           S t          j        t          j        |           S )Nc                 b                         | j                  }t          j        | |          S ri   )get_data_layoutrK   jax_distribution_libdistribute_data_input)dlayoutdistributions     r   distribute_single_valuez1_distribute_data.<locals>.distribute_single_value  s+    !11!'::F'=aHHHr   )r   rC  r   r   rE   
device_put)rR   rD  rC  s     @r   r  r    s`    #022L	I 	I 	I 	I 	I !"94@@@!#.$777r   c                       e Zd Zd Zd ZdS )r   c                 Z    |                      | j                                                  S ri   )_prefetch_numpy_iteratordata_adapterget_jax_iteratorr'  s    r   _get_iteratorzJAXEpochIterator._get_iterator  s,    ,,..00
 
 	
r   c              #      K   t          j                    dfd	} |d           r%                                V   |d           #dS dS )a  Shard and prefetch batches on device.

        Most of the implementation has been borrowed from
        `flax.jax_utils.prefetch_to_device`

        This utility takes an iterator and returns a new iterator which fills an
        on device prefetch buffer. Eager prefetching can improve the performance
        of training loops significantly by overlapping compute and data
        transfer.
           c                 |    t          j        |           D ]$}                    t          |                     %d S ri   )	itertoolsislicerO   r  )nrR   numpy_iteratorqueues     r   enqueuez:JAXEpochIterator._prefetch_numpy_iterator.<locals>.enqueue  sG    !(;; 5 5-d3344445 5r   )rQ  rl   N)rM  )collectionsdequepopleft)r   rR  rT  rS  s    ` @r   rH  z)JAXEpochIterator._prefetch_numpy_iterator  s       !##
	5 	5 	5 	5 	5 	5 	5 	! 	--//!!!GAJJJ  	 	 	 	 	r   N)r7  r8  r9  rK  rH  r<   r   r   r   r     s2        
 
 

    r   r   ) rU  rO  	functoolsr   rE   r   r  	keras.srcr   r   r   r   r   r   keras.src.backendr   r?  keras.src.distributionkeras.src.trainersr	   base_trainer keras.src.trainers.data_adaptersr
   r   !keras.src.trainers.epoch_iteratorr   keras.src.utilsr   Trainerr   r  r   r<   r   r   <module>rb     s                 



           3 3 3 3 3 3 5 5 5 5 5 5       F F F F F F 3 3 3 3 3 3 6 6 6 6 6 6 : : : : : : ? ? ? ? ? ? ; ; ; ; ; ; + + + + + +q q q q q% q q qh
8 
8 
8    }     r   