
    _nd!B                    
   d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
Z
ddlZddlmZmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZ ddlmZ ddlmZm Z  ddlm!Z! ddlm"Z" ddlm#Z# ddlm$Z$ ddlm%Z% ddlm&Z& ddlm'Z' ddlm(Z( ddlm)Z) ddlm*Z* ddlm+Z+ ddlm,Z, ddl-m.Z. ddl-m/Z/ ddl-m0Z0 ddl1m2Z2 ddl1m3Z3 dd l1m4Z4 dd!l1m5Z5 dd"l6m7Z7 dd#l6m8Z8 dd$l6m9Z9 dd%l6m:Z: dd&l6m;Z; dd'l6m<Z< dd(l6m=Z= dd)l6m>Z> dd*l6m?Z? dd+l@mAZAmBZBmCZC dd,l@mDZDmEZE dd-lFmGZG dd.lHmIZI dd/lJmKZKmLZL dd0lMmNZN dd1lOmPZP dd2lQmRZR dd3lSmTZT dd4lUmVZV dd5lWmXZX dd6lYmZZZ dd7lYm[Z[ dd8l\m]Z] dd9l^m_Z_ dd:l1m`Z` dd;l1maZa dd<lbmcZc dd=lmdZd 	 ee n# ef$ r dZeY nw xY w G d> d?eZ          Zg G d@ dAeg          Zh G dB dCeZ          Zi G dD dEei          Zj G dF dG          Zk ejl        dH          Zm eem          Zn ejo        g dI          Zp ejo        g dJ          Zq e ejr        dK                    ZsdL ZtdM ZudN ZvdO ZwdP ZxdQ ZydR ZzdS Z{e
j|        }                    dT          dU             Z~dV ZdW ZdX ZdY ZdZ Zd[ Zd\ Zd] Zd^ Zd_ Zd` Zda Zdb Zdc Zdd Zde Zdf Ze
j|        }                    dT          dg             Zdh Zdi Zdj Zdk Zdl Zdm Zdn Zdo Zdp Zdq Zdr Zds Zdt Zdu Zdv Zdw Zdx Zdy Zdz Zd{ Zd| Zd} Zd~ Zd Zd Zd Zd Zd Zd Zd Zd Z G d deG          Zd Zd Zd Zd Zd Zd Ze
j|        }                    dT          d             Zd Zd Z G d deZ          Ze
j|                            dej        dg          d             Ze
j|                            dej        dg          d             Zd Ze
j|        }                    d          e
j|                            dej        ddg          d                         Ze
j|        }                    d          e
j|                            dej        ddg          e
j|                            dddg          e
j|                            dddg          d                                                 Zd Ze
j|                            ddeddddfdeedddddfdeedddddfg          d             Zd Zd Zd ZdS )zTest the validation module    N)partial)sleep)
coo_matrix
csr_matrix)FitFailedWarning)FailingClassifier)assert_almost_equal)assert_array_almost_equal)assert_array_equal)assert_allclose)CheckingClassifierMockDataFrame)_num_samples)cross_val_scoreShuffleSplit)cross_val_predict)cross_validate)permutation_test_score)KFold)StratifiedKFold)LeaveOneOut)LeaveOneGroupOut)LeavePGroupsOut)
GroupKFold)GroupShuffleSplit)learning_curve)validation_curve)_check_is_permutation)_fit_and_score)_score)make_regression)load_diabetes)	load_iris)load_digits)explained_variance_score)make_scorer)accuracy_score)confusion_matrix)precision_recall_fscore_support)precision_score)r2_score)mean_squared_error)check_scoring)RidgeLogisticRegressionSGDClassifier)PassiveAggressiveClassifierRidgeClassifier)RandomForestClassifier)KNeighborsClassifier)SVC	LinearSVC)KMeans)MLPRegressor)SimpleImputer)LabelEncoder)Pipeline)StringIO)BaseEstimator)clone)OneVsRestClassifier)shuffle)make_classification)make_multilabel_classification)OneTimeSplitter)GridSearchCVc                   4    e Zd ZdZd ZddZd Zd	dZd ZdS )
MockImprovingEstimatorz+Dummy classifier to test the learning curvec                 0    || _         d| _        d | _        d S Nr   )n_max_train_sizestrain_sizesX_subset)selfrI   s     Mlib/python3.11/site-packages/sklearn/model_selection/tests/test_validation.py__init__zMockImprovingEstimator.__init__Z   s    !2    Nc                 8    || _         |j        d         | _        | S rH   rK   shaperJ   rL   rK   y_subsets      rM   fitzMockImprovingEstimator.fit_        #>!,rO   c                     t           NNotImplementedErrorrL   Xs     rM   predictzMockImprovingEstimator.predictd       !!rO   c                     |                      |          rdt          | j                  | j        z  z
  S t          | j                  | j        z  S )Ng       @)_is_training_datafloatrJ   rI   rL   r\   Ys      rM   scorezMockImprovingEstimator.scoreg   sP    !!!$$ 	Dt/0043IIII)**T-CCCrO   c                     || j         u S rX   rK   r[   s     rM   r`   z(MockImprovingEstimator._is_training_datan       DM!!rO   rX   NN	__name__
__module____qualname____doc__rN   rU   r]   rd   r`    rO   rM   rF   rF   W   ss        55  
   
" " "D D D D" " " " "rO   rF   c                   2     e Zd ZdZd fd	Zd ZddZ xZS )!MockIncrementalImprovingEstimatorz*Dummy classifier that provides partial_fitNc                 f    t                                          |           d | _        || _        d S rX   )superrN   xexpected_fit_params)rL   rI   rt   	__class__s      rM   rN   z*MockIncrementalImprovingEstimator.__init__u   s1    *+++#6   rO   c                     | j         |v S rX   rs   r[   s     rM   r`   z3MockIncrementalImprovingEstimator._is_training_dataz   s    v{rO   c                    | xj         |j        d         z  c_         |d         | _        | j        rt	          | j                  t	          |          z
  }|r t          dt          |           d          |                                D ]c\  }}|| j        v rSt          |          t          |          k    r3t          d| dt          |           dt          |           d          bd S d S )Nr   zExpected fit parameter(s) z
 not seen.zFit parameter z has length z; expected .)	rJ   rR   rs   rt   setAssertionErrorlistitemsr   )rL   r\   yparamsmissingkeyvalues          rM   partial_fitz-MockIncrementalImprovingEstimator.partial_fit}   s.   AGAJ&1# 	$233c&kkAG $JgJJJ   %llnn  
U$222|8 8!!__8% 8% )9 9 9,u:M:M 9 9&21oo9 9 9  	 	 rO   rX   )rj   rk   rl   rm   rN   r`   r   __classcell__ru   s   @rM   rp   rp   r   sf        447 7 7 7 7 7
         rO   rp   c                   4    e Zd ZdZd	dZd Zd Zd
dZd ZdS )MockEstimatorWithParameterz-Dummy classifier to test the validation curve      ?c                 "    d | _         || _        d S rX   )rK   param)rL   r   s     rM   rN   z#MockEstimatorWithParameter.__init__   s    


rO   c                 8    || _         |j        d         | _        | S rH   rQ   rS   s      rM   rU   zMockEstimatorWithParameter.fit   rV   rO   c                     t           rX   rY   r[   s     rM   r]   z"MockEstimatorWithParameter.predict   r^   rO   Nc                 N    |                      |          r| j        n	d| j        z
  S )N   )r`   r   )rL   r\   r~   s      rM   rd   z MockEstimatorWithParameter.score   s'    !33A66JtzzA
NJrO   c                     || j         u S rX   rf   r[   s     rM   r`   z,MockEstimatorWithParameter._is_training_data   rg   rO   )r   rh   ri   rn   rO   rM   r   r      ss        77     
" " "K K K K" " " " "rO   r   c                   (     e Zd ZdZ fdZd Z xZS )%MockEstimatorWithSingleFitCallAllowedz<Dummy classifier that disallows repeated calls of fit methodc                     t          | d          r
J d            d| _        t                                          ||          S )Nfit_called_zfit is called the second timeT)hasattrr   rr   rU   )rL   rK   rT   ru   s      rM   rU   z)MockEstimatorWithSingleFitCallAllowed.fit   sB    4//PP1PPPPww{{8X...rO   c                     t           rX   rY   r[   s     rM   r]   z-MockEstimatorWithSingleFitCallAllowed.predict   r^   rO   )rj   rk   rl   rm   rU   r]   r   r   s   @rM   r   r      sM        FF/ / / / /
" " " " " " "rO   r   c                   P    e Zd ZdZddZ	 	 	 	 	 	 	 	 	 ddZd Zd Zdd	Zdd
Z	dS )MockClassifierz-Dummy classifier to test the cross-validationr   Fc                 "    || _         || _        d S rX   aallow_nd)rL   r   r   s      rM   rN   zMockClassifier.__init__   s     rO   Nc           	         || _         || _        |	| _        |
 |
|            | j        r#|                    t          |          d          }|j        dk    r| j        st          d          |P|j        d         |j        d         k    s4J d	                    |j        d         |j        d                               ||j        d         t          t          j        t                              k    sMJ d	                    |j        d         t          t          j        t                                                    |Rd}|j        d         |j        d         k    s4J |	                    |j        d         |j        d                               |md	}|j        t          j        k    sVJ |	                    |j        d         |j        d
         t          j        d         t          j        d
                               | S )zThe dummy arguments are to test that this fit function can
        accept non-array arguments through cross-validation, such as:
            - int
            - str (this is actually array-like)
            - object
            - function
        N   zX cannot be dr   zKMockClassifier extra fit_param sample_weight.shape[0] is {0}, should be {1}zIMockClassifier extra fit_param class_prior.shape[0] is {0}, should be {1}zRMockClassifier extra fit_param sparse_sample_weight.shape[0] is {0}, should be {1}zUMockClassifier extra fit_param sparse_param.shape is ({0}, {1}), should be ({2}, {3})r   )	dummy_int	dummy_str	dummy_objr   reshapelenndim
ValueErrorrR   formatnpuniquer~   P_sparse)rL   r\   rc   sample_weightclass_priorsparse_sample_weightsparse_paramr   r   r   callbackfmts               rM   rU   zMockClassifier.fit   s   ( #""HTNNN= 	&		#a&&"%%A6Q;;t};_---$ &q)QWQZ777??Ev!'*AGAJ@ @ 877 "$Q'3ry||+<+<<<<))/0A!0Dc")TU,,FWFW)X)X =<<  +2  (-a0AGAJ>>>

$*1-qwqzA A>>> #6   %777"1%"1%q!q!	: :777 rO   c                 n    | j         r#|                    t          |          d          }|d d df         S )Nr   r   )r   r   r   rL   Ts     rM   r]   zMockClassifier.predict   s6    = 	&		#a&&"%%AAwrO   c                     |S rX   rn   r   s     rM   predict_probazMockClassifier.predict_proba   s    rO   c                 @    ddt          j        | j                  z   z  S )N      ?r   )r   absr   rb   s      rM   rd   zMockClassifier.score   s    a"&..())rO   c                      | j         | j        dS )Nr   r   )rL   deeps     rM   
get_paramszMockClassifier.get_params  s    V777rO   )r   F)	NNNNNNNNNrh   )F)
rj   rk   rl   rm   rN   rU   r]   r   rd   r   rn   rO   rM   r   r      s        77! ! ! ! != = = =~  
  * * * *8 8 8 8 8 8rO   r   )
      )
r   r   r   r   r   r   r   r      r   )
r   r   r   r   r   r   r   r   r   r      c                     t                      } t          dd          D ]C}|| _        t          | t          t
                    }t          ||                     t          t
                               t          j	        t
          t
          d d d         g          }t          | t          |          }t          ||                     t          |                     t          | t          t
                    }t          ||                     t          t
                               t          | t          |          }t          ||                     t          |                     Ed }t          |          } t          | t                                          t
                                          d          }t          |          } t          | t          t
                                          d          }t          j        t                    5  t          | t          t
          d	
           d d d            n# 1 swxY w Y   t          d d d d t          j        f         }t          d          } t          | |t
                    }t          d          } t          j        t                    5  t          | |t
          d           d d d            d S # 1 swxY w Y   d S )Nir   r   c                 ,    t          | t                    S rX   
isinstancer|   rw   s    rM   <lambda>z&test_cross_val_score.<locals>.<lambda>(      :a.. rO   check_Xr   cvcheck_ysklearnscoringT)r   Fraiseerror_score)r   ranger   r   r\   y2r   rd   r   column_stackX_sparser   tolistpytestraisesr   newaxis)clfr   scoresmultioutput_y
list_checkX_3ds         rM   test_cross_val_scorer     s   


C3^^ G G a,,6399Q#3#3444 R"X77 h>>6399X}#E#EFFF h336399Xr#:#:;;; !h>>6399X}#E#EFFFF /.J
Z
0
0
0CS!((**biikka@@@F
Z
0
0
0CS!RYY[[Q777F	z	"	" 7 7QI66667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 QQQ2:D
$
'
'
'CS$++F
%
(
(
(C	z	"	" < <T27;;;;< < < < < < < < < < < < < < < < < <s$   H::H>H>2KKKc                      t          d          \  } }t          d          }t          |dddgi          }t          || |d	
           d S )NT
return_X_yauto)gammaCr   r   )
param_gridr   )n_jobs)r#   r5   rD   r   )r\   r~   r   grids       rM   test_cross_validate_many_jobsr   <  s_     %%%DAq
F


Cq"g777D4Aa((((((rO   c            
         t          d          \  } }t                      }d}t          j        t          |          5  t          || |t          t                    t          t                    f           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || |t          t                    f           d d d            n# 1 swxY w Y   t          j        t          |dz             5  t          || |d           d d d            n# 1 swxY w Y   t          j        t          |dz             5  t          || |d	           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || |t          t                    gg           d d d            n# 1 swxY w Y   d
}t          j        t          d          5  t          || |t                                 d d d            n# 1 swxY w Y   t          j        t          |          5  t          || |d           d d d            n# 1 swxY w Y   t          t                    }dt          j         d}t          j        t          |          5  t          || ||           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || |d|i           d d d            n# 1 swxY w Y   t          j        t          d          5  t          t                      | |d           d d d            d S # 1 swxY w Y   d S )Nr   random_statez.*must be unique strings.*matchr   zEmpty list.*rn   zDuplicate.*)f1_micror   zB.*scoring is invalid.*Refer to the scoring glossary for details:.*zAn empty dictr   [Scoring failed. The score on this train-test partition for these parameters will be set to z. Details: 
fooz#'mse' is not a valid scoring value.mse)rA   r   r   r   r   r   r&   r*   r'   dictr)   r   nanwarnsUserWarningr5   )r\   r~   	estimatorerror_message_regexpmulticlass_scorerwarning_messages         rM   )test_cross_validate_invalid_scoring_paramr   F  s@   A...DAq  I 8 
z)=	>	>	> 
 
 11;~3N3NO		
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
z)=	>	>	> Q Qy!Q_1M1M0OPPPPQ Q Q Q Q Q Q Q Q Q Q Q Q Q Q 
z)=)N	O	O	O 4 4y!Q33334 4 4 4 4 4 4 4 4 4 4 4 4 4 4 
z)=)M	N	N	N J Jy!Q0HIIIIJ J J J J J J J J J J J J J J 
z)=	>	>	> R Ry!Q+o2N2N1O0PQQQQR R R R R R R R R R R R R R R 	M 
 
z	9	9	9 : :y!Q9999: : : : : : : : : : : : : : : 
z)=	>	>	> 3 3y!Q22223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 $$CDD
	9;	 	 	  
k	9	9	9 C Cy!Q0ABBBBC C C C C C C C C C C C C C C 
k	9	9	9 L Ly!Q8I0JKKKKL L L L L L L L L L L L L L L 
z)N	O	O	O 3 3suuaE22223 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3s   :BB	B	+'CC"%C"D''D+.D+E00E47E4(G

GG2 HH"%H"I$$I(+I(.KKK4LLL< M))M-0M-c                      t          d          \  } }t          dt                      fdt                      fg          }t	          || |d          }|d         }t          |t                    sJ t          d |D                       sJ d S )NTr   imputer
classifier)return_estimatorr   c              3   @   K   | ]}t          |t                    V  d S rX   )r   r;   ).0r   s     rM   	<genexpr>z7test_cross_validate_nested_estimator.<locals>.<genexpr>  s,      KK9z)X..KKKKKKrO   )r#   r;   r9   r   r   r   r|   all)r\   r~   pipelineresults
estimatorss        rM   $test_cross_validate_nested_estimatorr    s     $'''FQ(>++,	
 H Xq!dCCCG%Jj$'''''KK
KKKKKKKKKKrO   c            
      p   t                      } t          dd          \  }}t          d          }t          dd          \  }}t	          dd          }|||f|||ffD ]\  }}}	t          |	d          }
t          |	d	          }g }g }g }g }g }|                     ||          D ]\  }}t          |                              ||         ||                   }	|	                     |
|	||         ||                              |	                     ||	||         ||                              |	                     |
|	||         ||                              |	                     ||	||         ||                              |	                    |	           t          j        |          }t          j        |          }t          j        |          }t          j        |          }t          j        |          }|||||f}t          |	|||           t          |	|||           d S )
N   r   	n_samplesr   r   linearkernelr   neg_mean_squared_errorr   r2)r   r!   r.   rA   r5   r-   splitr>   rU   appendr   array"check_cross_validate_single_metric!check_cross_validate_multi_metric)r   X_regy_regregX_clfy_clfr   r\   r~   est
mse_scorer	r2_scorertrain_mse_scorestest_mse_scorestrain_r2_scorestest_r2_scoresfitted_estimatorstraintestr   s                       rM   test_cross_validater$    sc   	B #Ra@@@LE5
Q


C '!DDDLE5
XA
.
.
.CeS)E5#+>?  =  =	1c"30HIII
!#t444	88Aq>> 	* 	*KE4**..51U844C##JJsAeHah$G$GHHH""99S!E(AeH#E#EFFF""::c1T7AdG#D#DEEE!!))C4!D'"B"BCCC$$S))))8$455(?33(?33.11H%677 
 	+31f===)#q!V<<<<A =  =rO   c                    |\  }}}}}dD ]\  }	}
|	r+t          | ||dd          }t          |d         |           nt          | ||dd          }t          |t                    sJ t	          |          |
k    sJ t          |d         |           |	r-t          | ||dgd          }t          |d	         |d           nt          | ||dgd          }t          |t                    sJ t	          |          |
k    sJ t          |d
         |           t          | ||dd          }t          |d                   D ]E\  }}t          |j        ||         j                   t          |j        ||         j                   Fd S )N))Tr   )Fr   r  Tr   return_train_scoretrain_scoreF
test_scorer  train_r2test_r2)r   r   r   )	r   r
   r   r   r   	enumerater	   coef_
intercept_)r   r\   r~   r   r  r  r  r   r!  r'  dict_lenmse_scores_dictr2_scores_dictkr  s                  rM   r  r    s    	 )@ M M$H 	,Q#;PT  O &om&DFVWWWW,Q#;PU  O /400000?##x////!/,"?QQQ  		+QD6d  N &nZ&@/SWXXXX+QD6e  N .$/////>""h....!.";^LLLL %Q3d  O OK899 M M3CI'8';'ABBBCN,=a,@,KLLLLM MrO   c           	      T   |\  }}}}}d }	dt          t                    dd|	f}
h d}|                    ddh          }dD ]}|
D ]}|rAt          | |||d	          }t	          |d         |           t	          |d         |           nt          | |||d
	          }t          |t                    sJ t          |                                          |r|n|k    sJ t	          |d         |           t	          |d         |           t          |d                   t          j        k    sJ t          |d                   t          j        k    sJ t          |d                   t          j        k    sJ t          |d                   t          j        k    sJ t          j        |d         dk              sJ t          j        |d         dk               sJ t          j        |d         dk              sJ t          j        |d         dk               sJ ސd S )Nc                 p    |                      |          }t          ||          t          ||           dS )Nr  r  )r]   r+   r,   )r   r\   r~   y_preds       rM   custom_scorerz8check_cross_validate_multi_metric.<locals>.custom_scorer  s=    Q1f%%'9!V'D'D&D
 
 	
rO   r5  r  >   r+  fit_time
score_timetest_neg_mean_squared_errorr*  train_neg_mean_squared_error)TFTr&  Fr+  r:  r8  r9  r   r   )r&   r+   unionr   r
   r   r   rz   keystyper   ndarrayr  )r   r\   r~   r   r  r  r  r   r!  r7  all_scoringkeys_sans_trainkeys_with_trainr'  r   
cv_resultss                   rM   r  r    s    	
 
 
 	)h''&>	
 	
 	K  O &++	34 O , "9 "9" !	9 !	9G! +Aw4  
 **Z*@/RRR)=>@P    ,Aw5  
 j$/////z(())#5J?    &j&;^LLL%89?  
 
9-.."*<<<<
#@ABBbjPPPP
:.//2:====
<011RZ???? 6*Z0A5666666*Z025666666*\2a7888886*\2R7888888C!	9"9 "9rO   c                     t          ddd          \  } }t          d          }t                      t          d          t	                      t                      g}d}|D ]}t          j        t          |          5  t          || ||	           d d d            n# 1 swxY w Y   t          j        t          |          5  t          || ||	           d d d            n# 1 swxY w Y   d S )
N   r   r   )r
  	n_classesr   r  r  z*The 'groups' parameter should not be None.r   )r   r\   r~   r   )rA   r5   r   r   r   r   r   r   r   r   r   )r\   r~   r   	group_cvserror_messager   s         rM   #test_cross_val_score_predict_groupsrJ  @  s}    qqIIIDAq
X


C 		I AM > >]:];;; 	< 	<cQ!;;;;	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	< 	<]:];;; 	> 	>qA"====	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	> 	>> >s$   >BB"	%B"	C$$C(	+C(	z(ignore: Using or importing the ABCs fromc                  6  	 t           t           fg} 	 ddlm}m} |                     ||f           n# t
          $ r Y nw xY w| D ]S\  	 t                     	t                    }}fd}	fd}t          ||          }t          |||d           Td S )Nr   Series	DataFramec                 $    t          |           S rX   r   rs   InputFeatureTypes    rM   r   z-test_cross_val_score_pandas.<locals>.<lambda>d      Z+;<< rO   c                 $    t          |           S rX   rP  rs   
TargetTypes    rM   r   z-test_cross_val_score_pandas.<locals>.<lambda>e      Az!:!: rO   r   r   r   r   )
r   pandasrM  rN  r  ImportErrorr\   r   r   r   
typesrM  rN  X_dfy_sercheck_dfcheck_seriesr   rR  rV  s
           @@rM   test_cross_val_score_pandasra  V  s     ]+,E,,,,,,,,fi())))   (- 0 0$
$ '&q))::b>>e<<<<:::: <HHHT5Q/////0 0   3 
A A c                  $   t          d          } t                      }|j        |j        }}t	          d          }t          | |||          }t	          d          }g }|                    ||          D ]v\  }}t          j        t          |          t                    }	t          j        t          |          t                    }
d|	|<   d|
|<   |                    ||f           wt          | |||          }t          ||           d S )Nr  rG  r   r   dtyper   )r5   r#   datatargetr   r   r  r   zerosr   boolr  r   )svmirisr\   r~   kfoldscores_indicescv_masksr"  r#  
mask_train	mask_testscores_maskss               rM   test_cross_val_score_maskrr  j  s   
X


C;;D9dkqA!HHE$S!Q5999N!HHEH{{1a(( ' 'tXc!ffD111
HSVV4000	
5	$&&&&"31:::L~|44444rO   c                     t          d          } t                      }|j        |j        }}t	          j        ||j                  }t          | ||          }t          d          } t          | ||          }t          ||           t          d           } t          | ||          }t          ||           t          d          } t          j
        t                    5  t          | ||           d d d            n# 1 swxY w Y   t          j
        t                    5  t          | |                                |           d d d            d S # 1 swxY w Y   d S )NprecomputedrG  r  c                 6    t          j        | |j                  S rX   )r   dotr   )rs   r~   s     rM   r   z2test_cross_val_score_precomputed.<locals>.<lambda>  s    "&AC.. rO   )r5   r#   rf  rg  r   rv  r   r   r
   r   r   r   r   )rj  rk  r\   r~   linear_kernelscore_precomputedscore_linearscore_callables           rM    test_cross_val_score_precomputedr{  }  s   
]
#
#
#C;;D9dkqAF1acNNM']A>>
X


C"31--L/>>> 00
1
1
1C$S!Q//N/@@@ ]
#
#
#C	z	"	" # #Q"""# # # # # # # # # # # # # # #
 
z	"	" 8 8]1133Q7778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8s$   $DD	D&$EEEc            	      H  	 t                      } t          j        d         }t          t	          j        t                              }t          t	          j        dg          t	          j        dg          t	          j        dg          ffd          }t          t	          j	        d                    }dd	t                      	fd}t	          j        |          t	          j        |d	|z            ||	|d
}t          | t          t          |           d S )Nr   r   )r   r   rR   r   *   42c                 V    | j         k    sJ | j        k    sJ | j        k    sJ d S rX   )r   r   r   )r   	DUMMY_INT	DUMMY_OBJ	DUMMY_STRs    rM   assert_fit_paramsz:test_cross_val_score_fit_params.<locals>.assert_fit_params  sD     }	))))}	))))}	))))))rO   r   )r   r   r   r   r   r   r   r   
fit_params)r   r\   rR   r   r   r   r~   r   r  eyeobjectonesfullr   )
r   r
  rF  W_sparser   r  r  r  r  r  s
          @@@rM   test_cross_val_score_fit_paramsr    s   


C
IBIaLL!!I	1#1#!67w  H "&))$$HIII* * * * * * * ++wy#	/:: ( %	 	J CA*555555rO   c                  4   t                      } g fd}t          j        d          5  t          |          }t	          | t
          t          |d          }d d d            n# 1 swxY w Y   t          |g d           t                    dk    sJ d S )Nc                 6                         | |f           dS )Nr   )r  )y_test	y_predict_score_func_argss     rM   
score_funcz3test_cross_val_score_score_func.<locals>.score_func  s!     3444srO   Trecordr   )r   r   )r   r   r   )	r   warningscatch_warningsr&   r   r\   r~   r   r   )r   r  r   rd   r  s       @rM   test_cross_val_score_score_funcr    s   


C     
	 	-	-	- B Bj))Q7qAAAB B B B B B B B B B B B B B B uooo...  A%%%%%%s   .A&&A*-A*c                       G d d          } t          j        t                    5  t           |             t                     d d d            d S # 1 swxY w Y   d S )Nc                       e Zd ZdS )4test_cross_val_score_errors.<locals>.BrokenEstimatorN)rj   rk   rl   rn   rO   rM   BrokenEstimatorr    s        rO   r  )r   r   	TypeErrorr   r\   )r  s    rM   test_cross_val_score_errorsr    s            
y	!	! . .))1---. . . . . . . . . . . . . . . . . .s   AAAc                  ^   t                      } t          d          }t          || j        | j                  }t          |g dd           t          || j        | j        d          }t          |g dd           t          || j        | j        d          }t          |g dd           d S )Nr  rG  )
ףp=
?r   r  r  r   r   accuracyr   f1_weighted)r#   r5   r   rf  rg  r
   )rk  r   r   	zo_scores	f1_scoress        rM   3test_cross_val_score_with_score_func_classificationr    s    ;;D
X


C S$)T[99Ff&B&B&BAFFF  TYZPPPIi)E)E)EqIII  TY]SSSIi)E)E)EqIIIIIrO   c                     t          dddd          \  } }t                      }t          || |          }t          |g dd           t          || |d	          }t          |g dd           t          || |d
	          }t	          j        g d          }t          ||d           t          t                    }t          || ||	          }t          |g dd           d S )Nr  rE  r   r   )r
  
n_featuresn_informativer   )gGz?r  r  gGz?gq=
ףp?r   r  r   r  )g(\؇gzGIgGz&qg\(qg)\G)r!   r.   r   r
   r   r  r&   r%   )	r\   r~   r  r   	r2_scoresneg_mse_scoresexpected_neg_mser   	ev_scoress	            rM   /test_cross_val_score_with_score_func_regressionr    s   RBaVWXXXDAq
''C S!Q''Ff&D&D&DaHHH  Q4888Ii)G)G)GKKK %S!Q8PQQQNx N N NOOn.>BBB 233GQ7;;;Ii)G)G)GKKKKKrO   c                     t                      } | j        }t          |          }| j        }t	          d          }t          d          }t          |||d|d          \  }}}|dk    sJ t          |dd	           t          |||d|dt          j	        |j
                  d
          \  }	}
}|	|k    sJ ||k    sJ t	          d          }t          d          }t          |||d|dt          j	        |j
                  d
          \  }	}
}|	|k    sJ ||k    sJ d }t          |          }t          |||d||d
          \  }}
}t          |dd           t          |dd           t          j        t          j        t          |                    d          }t          |||d|d          \  }}}|dk     sJ |dk    sJ d S )Nr  rG  r   r  r  )n_permutationsr   r   g?        r   r   )r  r   r   groupsr   c                     | |k                                     | |k                                     z
  | j        d         z  S rH   )sumrR   )y_truer6  s     rM   custom_scorez,test_permutation_score.<locals>.custom_score+  s;    6!&&((Ff,<+A+A+C+CCv|TUVVrO   d   )r  r   r   r   g(\?g{Gz?r   r   皙?)r#   rf  r   rg  r5   r   r   r	   r   r  sizer&   modaranger   )rk  r\   r   r~   rj  r   rd   r   pvaluescore_group_pvalue_group
svm_sparse	cv_sparser  scorers                   rM   test_permutation_scorer    sG   ;;D	A!}}HA
X


C			B2Q"Z  E66 3;;;;Q'''#9		wqv	$ 	$ 	$ KL %6!!!! H%%%J""I#9	wqv	$ 	$ 	$ KL %6!!!!W W W &&F-Q#v"1  E1f tQ'''a((( 	ryQ  !$$A2Q"Z  E66 3;;;;C<<<<<<rO   c                  t   t          j        dt           j                                      dd          } t           j        | dd d f<   t          j        ddg| j        d         dz            }t          dt          d	t           j        
          fdt                      fg          }t          || |           d S N   rd  r   r   r   r   r   r   mean)strategymissing_valuesr   )r   r  float64r   r   repeatrR   r;   r9   r   r   r\   r~   ps      rM   &test_permutation_test_score_allow_nansr  @  s    
	#RZ(((00R88AfAadG
	1a&!'!*q.))AvbfMMMN>++,	
	 	A 1a#####rO   c            
      F   t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          d          }d}t          j        t          |	          5  t          || |           d d d            n# 1 swxY w Y   d
}t          j        t          |	          5  t          || |dt          j
        d          i           d d d            n# 1 swxY w Y   t          || |dt          j
        d          i           d S )Nr  r   r   r   r   Texpected_sample_weight#Expected sample_weight to be passedr   /sample_weight.shape == \(1,\), expected \(8,\)!r   r  )r   r  r   r  r   r   r   r{   r   r   r  r\   r~   r   err_msgs       rM   &test_permutation_test_score_fit_paramsr  N  s   
	#r2&&A
!qA37"##A
D
9
9
9C4G	~W	5	5	5 * *sAq)))* * * * * * * * * * * * * * * AG	z	1	1	1 T TsAqorwqzz5RSSSST T T T T T T T T T T T T T T31/272;;1OPPPPPPs$   5BBB;(C//C36C3c                  t   t          j        dt           j                                      dd          } t           j        | dd d f<   t          j        ddg| j        d         dz            }t          dt          d	t           j        
          fdt                      fg          }t          || |           d S r  )r   r  r  r   r   r  rR   r;   r9   r   r   r  s      rM   test_cross_val_score_allow_nansr  ]  s    
	#RZ(((00R88AfAadG
	1a&!'!*q.))AvbfMMMN>++,	
	 	A Aq!rO   c                  P   t          j        ddgddgddgddgddgddgddgddgddgddgg
          } t          j        ddgddgddgddgddgddgddgddgddgddgg
          }t          d	          }t          t          d
          }t          t          d          }t          t          d          }t          || ||          }t          || ||          }t          || ||          }t          |g d           t          |g d           t          |g d           d S )Nr   r   r   r   r   r   )n_neighborsmicro)averagemacrosamplesr   )r   r         ?r   gUUUUUU?)r   r   r  r         ?)r   r  r4   r&   r*   r   r	   )	r\   r~   r   scoring_microscoring_macroscoring_samplesscore_microscore_macroscore_sampless	            rM   test_cross_val_score_multilabelr  k  s   
GFFFGGFHHG	
	 	A 	
Q!Q!Q!Q!Q!Q!Q!Q!QRSUVQWX	 	A 1
-
-
-CAAAMAAAM!/9EEEO!#q!]CCCK!#q!]CCCK#CAGGGM%D%D%DEEE%D%D%DEEE'F'F'FGGGGGrO   c            
         t          d          \  } }t                      }t                      }t          j        |          }|                    | |          D ]E\  }}|                    | |         ||                    |                    | |                   ||<   Ft          || ||          }t          ||           t          || |          }t          |          t          |          k    sJ t                      }t          || ||          }t          |          t          |          k    sJ |                                 }||t          j        |          k    z  }t          |          }t          |||          }t          t          |          t          |                     t          t          d          |           }t          |          t          |          k    sJ  G d d          }	t!          j        t$                    5  t          || | |	                       d d d            n# 1 swxY w Y   t'          d          \  } }d}
t!          j        t*          |
	          5  t          t-          d
          | |dt          d                     d d d            d S # 1 swxY w Y   d S )NTr   r   r   )n_initc                       e Zd ZddZdS )%test_cross_val_predict.<locals>.BadCVNc              3      K   t          d          D ]0}t          j        g d          t          j        g d          fV  1d S )Nr   )r   r   r   r   )r   r            )r   r   r  )rL   r\   r~   r  is        rM   r  z+test_cross_val_predict.<locals>.BadCV.split  sY      1XX H Hh|||,,bh.G.GGGGGGH HrO   rh   )rj   rk   rl   r  rn   rO   rM   BadCVr    s.        	H 	H 	H 	H 	H 	HrO   r  zNumber of classes in training fold \(2\) does not match total number of classes \(3\). Results may not be appropriate for your use case.r   	liblinearsolverr   r   methodr   )r"   r   r.   r   
zeros_liker  rU   r]   r   r
   r   r   copymedianr   r7   r   r   r   r#   r   RuntimeWarningr/   )r\   r~   r   r  preds2r"  r#  predsXspr  r   s              rM   test_cross_val_predictr    s8   D)))DAq	B
''C ]1Fxx1~~ , ,t%!E(###{{1T7++tc1aB///EeV,,,c1a((Eu::Q	Bc1aB///Eu::Q
&&((C33C
S//Cc3**Ec%jj#a&&111fF333Q77Eu::QH H H H H H H H
 
z	"	" 1 1#q!00001 1 1 1 1 1 1 1 1 1 1 1 1 1 1 %%%DAq	< 
 
nO	<	<	< 
 
k222"Qxx	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
s$   I  II;0J88J<?J<c            
      z   t          ddd          \  } }t          t          d          | |d          }|j        d	k    sJ t	          d
          \  } }t          t          d          | |d          }|j        dk    sJ | d d         } |d d         }d}t          j        t          |          5  t          t                      | |dt          d                     d d d            n# 1 swxY w Y   t          d
          \  } }t          dd          }t          || |d          }|j        dk    sJ t          j        |          }| |         ||         }} d}t          j        t          |          5  t          || |t          d          d           d d d            d S # 1 swxY w Y   d S )Nr   2   r   rF  r
  r   r  r  decision_functionr  )r   Tr      r   r  zOnly 1 class/es in training fold, but 2 in overall dataset. This is not supported for decision_function with imbalanced folds. To fix this, use a cross-validation technique resulting in properly stratified foldsr   r  r  ovo)r  decision_function_shape)i  -   zOutput shape \(599L?, 21L?\) of decision_function does not match number of classes \(7\) in fold. Irregular decision_function .*r   n_splitsr   r  )rA   r   r/   rR   r#   r   r   r   r2   r   r$   r5   r   argsort)r\   r~   r  rI  r  indr   s          rM   .test_cross_val_predict_decision_function_shaper    s   bqIIIDAq+...1=P  E ;%%%%DAq+...1=P  E ;("""" 	
$3$A	$3$A	1  
z	7	7	7 
 
q!,?E!HH	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 

 $'''DAq
Xu
=
=
=Cc1a0CDDDE;*$$$$
*Q--CS61S6qA	  
z)=	>	>	> W W#q!q(9(9(9BUVVVVW W W W W W W W W W W W W W W W W Ws$   6.C00C47C4 #F00F47F4c                     t          ddd          \  } }t          t          d          | |d          }|j        d	k    sJ t	          d
          \  } }t          t          d          | |d          }|j        dk    sJ d S )Nr   r   r   r  r  r  r   r  r   r   Tr   r  rA   r   r/   rR   r#   r\   r~   r  s      rM   *test_cross_val_predict_predict_proba_shaper    s    bqIIIDAq+...1_  E ;'!!!!%%%DAq+...1_  E ;(""""""rO   c                     t          ddd          \  } }t          t          d          | |d          }|j        d	k    sJ t	          d
          \  } }t          t          d          | |d          }|j        dk    sJ d S )Nr   r   r   r  r  r  predict_log_probar  r  Tr   r  r  r  s      rM   .test_cross_val_predict_predict_log_proba_shaper    s    bqIIIDAq+...1=P  E ;'!!!!%%%DAq+...1=P  E ;(""""""rO   c                  V   t                      } | j        | j        }}t          |          }t	          j        ||d d d         g          }t          dd          }t          |||          }|j        dk    sJ t          |||          }|j        dk    sJ t          |||          }t          |j        d           t          |||          }t          |j        d           d }t          |          }t          ||                                |                                          }t          |	          }t          |||                                          }t          t          d
          |                                |                                d          }t          t          d
          ||                                d          }|d d d d t          j        f         }d }	t          |	          }t          |||          }t          |j        d           d S )Nr   Fr   )fit_interceptr   )r  )r  r   c                 ,    t          | t                    S rX   r   rw   s    rM   r   z4test_cross_val_predict_input_types.<locals>.<lambda>-  r   rO   r   r   r  r  r  r  c                     | j         dk    S )Nr   )r   rw   s    rM   r   z4test_cross_val_predict_input_types.<locals>.<lambda>D  s    1 rO   )r#   rf  rg  r   r   r   r.   r   rR   r   r   r   r/   r   )
rk  r\   r~   r   r   r   predictionsr   r   check_3ds
             rM   "test_cross_val_predict_input_typesr    s   ;;D9dkqA!}}HOQ$$B$L11M
e!
4
4
4C $CA..K&&&& $C=AAK((((#C155K{(&111 $C=AAK{((333 /.J
Z
0
0
0C#CQXXZZ@@K
Z
0
0
0C#CAHHJJ77K $+...	

	

"	  K $+...		

"	  K QQQ2:D$$H
X
.
.
.C#Cq11K{(&11111rO   c                  6  	 t           t           fg} 	 ddlm}m} |                     ||f           n# t
          $ r Y nw xY w| D ]S\  	 t                     	t                    }}fd}	fd}t          ||          }t          |||d           Td S )Nr   rL  c                 $    t          |           S rX   rP  rQ  s    rM   r   z/test_cross_val_predict_pandas.<locals>.<lambda>X  rS  rO   c                 $    t          |           S rX   rP  rU  s    rM   r   z/test_cross_val_predict_pandas.<locals>.<lambda>Y  rW  rO   rX  r   r   )
r   rY  rM  rN  r  rZ  r\   r   r   r   r[  s
           @@rM   test_cross_val_predict_pandasr!  J  s     ]+,E,,,,,,,,fi())))   (- 2 2$
$&&q))::b>>e<<<<:::: <HHH#tUq111112 2rb  c                     t          dddddd          \  } }d|d<   t          dd          }t          d          }t          |                    | |                    \  }}t          || ||d	
          }||d                  d         dk    sJ t          j        ||d                  d d df         dk              sJ t          j        ||d                  d d ddf         dk              sJ t          j        ||d                  dk              sJ t          |	                    d          t          j
        |j                  d           d S )Nr  r   r   r   )r
  r  n_redundantr  n_clusters_per_classr   r  r   r  r	  r   r  axis   )decimal)rA   r/   r   r|   r  r   r   r  r
   r  r  rR   )r\   r~   r   r   r"  r#  
yhat_probas          rM   !test_cross_val_predict_unbalancedr+  ^  sk     DAq AaD
!K
@
@
@C	!	$	$	$Brxx1~~&&KE4"31OLLLJT!W:a=A6*T!W%aaad+q0111116*T!W%aaa1f-1222226*T!W%)*****jnn!n44bgag6F6FPRSSSSSSrO   c                  0   t                      } t          j                            d          }|                    dd          }t          | |d dd          }t          |d d df         |           t          | |d dd          }t          ||           d S )	Nr~  r  r   r   r]   )r~   r   r  r   r   )r   r   randomRandomStaterandr   r   )mock_classifierrngr\   y_haty_hat_probas        rM   test_cross_val_predict_y_noner4  t  s    $&&O
)


#
#CbAoqDQyQQQEAaaadGU####dq  K A{#####rO   c                  "   t                      } | j        | j        }}t                      }dt	          t          j        |j        d                             i}t          ||||d          }t          |t          j
        d                     d S )Nr   r   r   )r  r   )r#   rf  rg  r   r   r   r  rR   r   r   r  )rk  r\   r~   r   r  r   s         rM   &test_cross_val_score_sparse_fit_paramsr6    s{    ;;D9dkqA


C(*RVAGAJ5G5G*H*HIJQjQ???Aq"'!**%%%%%rO   c                     d} d}t          | dddddd          \  }}t          | |dz
  |z  z            }dD ]9}t          j        d	          5 }t	          |||t          |
          t          j        ddd          |d          \  }}}	}
}d d d            n# 1 swxY w Y   t          |          dk    rt          d|d         j
        z            |j        dk    sJ |	j        dk    sJ |
j        dk    sJ |j        dk    sJ t          |t          j        ddd                     t          |                    d          t          j        ddd                     t          |	                    d          t          j        ddd                     |
j        dk    sJ |j        dk    sJ t          j        d	          5 }t	          |||t!          ||           t          j        ddd          |          \  }}}d d d            n# 1 swxY w Y   t          |          dk    rt          d|d         j
        z            t          ||           t          ||	           ;d S )Nr  r   r   r   r   r
  r  r  r#  rF  r$  r   FTTr  r	  皙?r   r   )r   rJ   r@   return_timesUnexpected warning: %r)r   r   rE  r&  ffffff?r  r
  r
  )r   rJ   r@   )rA   rF   r  r  r   r   r   linspacer   RuntimeErrormessagerR   r   r
   r  re  rC   )r
  r
  r\   r~   r   shuffle_trainwrJ   train_scorestest_scores	fit_timesscore_timestrain_sizes2train_scores2test_scores2s                  rM   test_learning_curverK    sK   IH  DAq 'yX\X4M'NOOI& -= -=$D111 	Q (+++KS"55%!  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	  q66A::7!A$,FGGG!W,,,, G++++')))) G++++;Ar2(>(>???!,"3"3"3";";R[cSU=V=VWWW!+"2"2"2":":BKSRT<U<UVVV )++++ I---- $D111 	Q8F"H	JJJKS"55%9 9 95L-	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 q66A::7!A$,FGGG!->>>!,<<<<[-= -=s$   ?BB	"B	=HH	H	c                     t          ddddddd          \  } }t          d          }t          || d dt          j        dd	d
                    \  }}}t          |t          j        ddd
                     t          |                    d          t          j        dd	d
                     t          |                    d          t          j        dd	d
                     d S )Nr  r   r   r   r8  rE  r   r:  r   r   )r~   r   rJ   r&  r=  )rA   rF   r   r   r?  r   r
   r  r\   r  r   rJ   rD  rE  s         rM    test_learning_curve_unsupervisedrN    s      DAq 'r**I-;1Cb0I0I. . .*K{ {BK2r$:$:;;;l//Q/77S#r9R9RSSSk..A.66Cb8Q8QRRRRRrO   c            	         t          ddddddd          \  } }t          d          }t          j        }t	                      t          _        	 t          || |dd          \  }}}t          j                                        }t          j                                         |t          _        nM# t          j                                        }t          j                                         |t          _        w xY wd	|v sJ d S )
Nr  r   r   r   r8  rE  r   )r   verbosez[learning_curve])rA   rF   sysstdoutr<   r   getvalueclose)r\   r~   r   
old_stdoutrJ   rD  rE  outs           rM   test_learning_curve_verboserW    s     DAq 'r**IJCJ 1?q!12
 2
 2
.\; j!!##


 j!!##

$$$$$$s   B/ /A
C9c            	          t          ddddddd          \  } }t          d          }t          j        t                    5  t          || |d           d d d            d S # 1 swxY w Y   d S )Nr   r   r   r8  T)exploit_incremental_learningrA   rF   r   r   r   r   r\   r~   r   s      rM   5test_learning_curve_incremental_learning_not_possibler\    s      DAq 'q))I	z	"	" K Ky!QTJJJJK K K K K K K K K K K K K K K K K Ks   A##A'*A'c                     t          ddddddd          \  } }t          d          }dD ]}t          || |dd	t          j        d
dd          |          \  }}}t          |t          j        ddd                     t          |                    d          t          j        ddd                     t          |                    d          t          j        d
dd                     d S )Nr  r   r   r   r8  rE  r9  r   Tr:  r   r   )r   rY  rJ   r@   r&  r=  rA   rp   r   r   r?  r   r
   r  )r\   r~   r   rB  rJ   rD  rE  s          rM   (test_learning_curve_incremental_learningr_    s     DAq 2"55I& W W1?)-Cb11!2
 2
 2
.\; 	;Ar2(>(>???!,"3"3"3";";R[cSU=V=VWWW!+"2"2"2":":BKSRT<U<UVVVVW WrO   c                     t          ddddddd          \  } }t          d          }t          || d ddt          j        d	d
d                    \  }}}t          |t          j        ddd                     t          |                    d          t          j        dd
d                     t          |                    d          t          j        d	d
d                     d S )Nr  r   r   r   r8  rE  r   Tr:  r   r   )r~   r   rY  rJ   r&  r=  r^  rM  s         rM   5test_learning_curve_incremental_learning_unsupervisedra    s      DAq 2"55I-;	
%)KS"--. . .*K{ {BK2r$:$:;;;l//Q/77S#r9R9RSSSk..A.66Cb8Q8QRRRRRrO   c            	         t          ddddddd          \  } }t          j        ddd          }t          dd d	
          }t	          || ||dd          \  }}}t	          || |d|d	          \  }}}	t          ||           t          |                    d          |                    d                     t          |                    d          |	                    d                     d S )Nr  r   r   r   r8  r  r   r   Fmax_itertolr@   r   T)rJ   r   rY  )r   rJ   rY  r&  )rA   r   r?  r1   r   r   r
   r  )
r\   r~   rJ   r   train_sizes_inctrain_scores_inctest_scores_inctrain_sizes_batchtrain_scores_batchtest_scores_batchs
             rM   <test_learning_curve_batch_and_incremental_learning_are_equalrl  6  sP     DAq +c3**K+QD%PPPI9G		%): : :6O% @N		%*@ @ @<)+< (9:::1%%'9'>'>A'>'F'F   !$$&7&<&<!&<&D&D    rO   c            	         t          ddddddd          \  } }t          d          }t          j        t                    5  t          || |dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dd	d
g           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dddg           d d d            n# 1 swxY w Y   t          j        t                    5  t          || |dddg           d d d            d S # 1 swxY w Y   d S )Nr  r   r   r   r8  rE  r   r   rJ   r  r   r:  g?   rZ  r[  s      rM   0test_learning_curve_n_sample_range_out_of_boundsrp  ]  s/     DAq 'r**I	z	"	" B By!Q11a&AAAAB B B B B B B B B B B B B B B	z	"	" F Fy!Q13*EEEEF F F F F F F F F F F F F F F	z	"	" F Fy!Q13*EEEEF F F F F F F F F F F F F F F	z	"	" C Cy!Q11b'BBBBC C C C C C C C C C C C C C C	z	"	" C Cy!Q11b'BBBBC C C C C C C C C C C C C C C C C CsZ   A%%A),A)	B,,B03B0C33C7:C7D::D>D>FF	Fc                  <   t          ddddddd          \  } }t          d          }d}t          j        t          |          5  t          || |dt          j        dd	d          
          \  }}}d d d            n# 1 swxY w Y   t          |ddg           d S )Nr   r   r   r   r8  zzRemoved duplicate entries from 'train_sizes'. Number of ticks will be less than the size of 'train_sizes': 2 instead of 3.r   gQ?r   rn  )	rA   rF   r   r   r  r   r   r?  r   )r\   r~   r   r   rJ   r  s         rM   1test_learning_curve_remove_duplicate_sample_sizesrr  t  s     DAq 'q))I	G  
nO	<	<	< 
 
*q!r{4a/H/H
 
 
Q
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 {QF+++++s   -A??BBc                     t          ddddddd          \  } }t          d          }t          d          }t          || ||t	          j        d	d
d                    \  }}}t          |t	          j        ddd                     t          |                    d          t	          j        dd
d                     t          |                    d          t	          j        d	d
d                     d S )Nr  r   r   r   r8  rE  r   r	  r:  r   r   rn  r&  r=  )	rA   rF   r   r   r   r?  r   r
   r  )r\   r~   r   r   rJ   rD  rE  s          rM   (test_learning_curve_with_boolean_indicesrt    s     DAq 'r**I				B-;1aBBKS",E,E. . .*K{ {BK2r$:$:;;;l//Q/77S#r9R9RSSSk..A.66Cb8Q8QRRRRRrO   c                  ^   t          j        ddgddgddgddgd	d
gddgddgddgddgddgddgd	d
gddgddgddgg          } t          j        g d          }t          j        g d          }t          dd d          }t          d          }t	          || ||dt          j        ddd          |dd	  	        \  }}}t          |                    d          t          j        g d                     t          |                    d          t          j        g d                      t          j	        t                    5  t	          || ||dt          j        ddd          |d!"           d d d            n# 1 swxY w Y   t	          || ||dt          j        ddd          |ddd#
  
        \  }}	}
t          |	                    d          |                    d                     t          |
                    d          |                    d                     d S )$Nr   r   r   r   r   r  r  r     r(                       rE  	   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Frc  r	  333333?r   T)r   r   rJ   r  r@   r   r&  )r  r  Iq?)r  r  r  r   )r   r   rJ   r  r   )r   r   rJ   r  r@   r   rY  )r   r  r1   r   r   r?  r
   r  r   r   r   )r\   r~   r  r   r   ri  rj  rk  rf  rg  rh  s              rM    test_learning_curve_with_shuffler    s    	FFFFHHHHHFGHHHH	
	 	A& 	>>>??AXCCCDDF ,QD%PPPI	Q			B?M		KS!,,
@ 
@ 
@<)+< Q''2I2I2I)J)J   A&&1I1I1I(J(J   
z	"	" 

 

Ca00		
 		
 		
 		


 

 

 

 

 

 

 

 

 

 

 

 

 

 

 :H		KS!,,%): : :6O% 1%%'9'>'>A'>'F'F   !$$&7&<&<!&<&D&D    s   	,FFFc                  N   t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          d          }d}t          j        t          |	          5  t          || |d
           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          || |d
dt          j
        d          i           d d d            n# 1 swxY w Y   t          || |d
dt          j
        d          i           d S )Nr  r   r   r   r   Tr  r  r   r   r   z/sample_weight.shape == \(1,\), expected \(2,\)!r   )r   r  )r   r  r   r  r   r   r   r{   r   r   r  r  s       rM   test_learning_curve_fit_paramsr    s   
	#r2&&A
!qA37"##A
D
9
9
9C4G	~W	5	5	5 7 7sAqg66667 7 7 7 7 7 7 7 7 7 7 7 7 7 7 AG	z	1	1	1 
 
A7QR7T	
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 QwORWR[[3Q     s$   5BBB=)C22C69C6c                  ~   t          ddddddd          \  } }t          ddg          }d}t          j        t          |	          5  t          || |d
dt          j        ddd          d           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          || |d
dt          j        ddd          ddt          j        d
          i           d d d            n# 1 swxY w Y   t          || |d
dt          j        ddd          ddt          j        d          i           d S )Nr  r   r   r   r8  rE  r   z9Expected fit parameter\(s\) \['sample_weight'\] not seen.r   r   Tr:  r   r   r   )r   rY  rJ   r   z2Fit parameter sample_weight has length 3; expected)r   rY  rJ   r   r  )	rA   rp   r   r   r{   r   r   r?  r  )r\   r~   r   r  s       rM   3test_learning_curve_incremental_learning_fit_paramsr    s     DAq 2"6GHHIJG	~W	5	5	5 	
 	
)-Cb11	
 	
 	
 	
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 CG	~W	5	5	5 

 

)-Cb11'4		
 		
 		
 		


 

 

 

 

 

 

 

 

 

 

 

 

 

 

 		%)KS"--#RWQZZ0	 	 	 	 	 	s%   +A??BB'A C33C7:C7c            	         t          ddddddd          \  } }t          j        ddd          }t          j        d          5 }t          t                      | |d|d	          \  }}d d d            n# 1 swxY w Y   t          |          dk    rt          d
|d         j	        z            t          |                    d          |           t          |                    d          d|z
             d S )Nr   r   r   r8  r   Tr  r   
param_nameparam_ranger   r<  r&  )rA   r   r?  r  r  r   r   r   r@  rA  r
   r  )r\   r~   r  rC  rD  rE  s         rM   test_validation_curver  /  sR     DAq +aB''K		 	-	-	- 
$4&((#%
 %
 %
!k
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 1vvzz3adlBCCCl//Q/77EEEk..A.66KHHHHHs   %A66A:=A:c            	          t          ddddddd          \  } }t          j        ddd          }t          t	                      | |d|d          \  }}d S )Nr   r   r   r8  r   r   r  )rA   r   r?  r   r   )r\   r~   r  r  s       rM   %test_validation_curve_clone_estimatorr  J  sx      DAq +aB''K-//		  DAqqqrO   c                     d} d}t          dd          \  }}t          t          dd          ||dg dt          || 	          
          }t	          t          j        t          j        |          dd d f         d            t          t          dd          ||dg dt          |d          
          }t	          t          j        t          j        |          dd d f         d            t          t          dd          ||dg dt          |          
          }t	          t          j	        |          t          j	        |                     d S )Nr  r   r   r	  r  r  r   )r:  r:  r  r  r>  r  )r   r   r   r   r   Tr
  r@   r	  )
rA   r   r5   rC   r
   r   vsplithstackr   r  )r
  r
  r\   r~   scores1scores2scores3s          rM   +test_validation_curve_cv_splits_consistencyr  `  s~   IH1===DAq8!,,,		(((H	BBB  G ry7););L!!!O)LaPPQQ8!,,,		((((D111  G ry7););L!!!O)LaPPQQ8!,,,		((((###  G bhw//'1B1BCCCCCrO   c                  f   t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          d          }d}t          j        t          |	          5  t          || |d
g dd           d d d            n# 1 swxY w Y   d}t          j        t          |	          5  t          || |d
g dddt          j
        d          i           d d d            n# 1 swxY w Y   t          || |d
g dddt          j
        d          i           d S )Nr  r   r   r   r   Tr  r  r   	foo_param)r   r   r   r   )r  r  r   r  r   )r  r  r   r  )r   r  r   r  r   r   r   r{   r   r   r  r  s       rM    test_validation_curve_fit_paramsr    s   
	#r2&&A
!qA37"##A
D
9
9
9C4G	~W	5	5	5 
 
"!			
 	
 	
 	

 
 
 
 
 
 
 
 
 
 
 
 
 
 
 AG	z	1	1	1 	
 	
"!		'4	
 	
 	
 	
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 		II#RWR[[1     s$   5BB B-C::C>C>c                  ~   t           j                            d          } t          j        d          }|                     |           t          |d          sJ t          t          j        |d          d          rJ d|d<   t          |d          rJ t          t          j        |df          d          rJ d S )Nr   r     )r   r-  r.  r  r@   r   deleter  )r1  r  s     rM   test_check_is_permutationr    s    
)


"
"C
	#AKKNNN C((((($RYq"%5%5s;;;;;AaD$Q,,,,, %RY1v%6%6<<<<<<<rO   c                  >   t          ddddd          \  } }t          |           }t          |          }t          t          d                    }t	          || |d	          }t	          |||d	          }|                                }t          ||           d S )
Nr   r   FT)rF  n_labelsallow_unlabeledreturn_indicatorr   r  rG  r   r   )rB   r   r?   r5   r   toarrayr
   )r\   r~   r   y_sparseclassifr  preds_sparses          rM   (test_cross_val_predict_sparse_predictionr    s    )  DAq !}}H!}}H!#X"6"6"677Ggq!333E$WhRHHHL''))LlE22222rO   c           
      &   t          dd          }|j        dk    r(|dk    rt          |          fnt          |          df}n|j        }t	          j        |          }|                    ||          D ]V\  }}t          |                               ||         ||                   }  t          | |          ||                   ||<   W||dz   |dz
  |
                    d          fD ]$}	t          t          | ||	||          |           %d	S )
z@Helper for tests of cross_val_predict with binary classificationr   Fr  r   r  r   strr  N)r   r   r   rR   r   rh  r  r>   rU   getattrastyper   r   )
r  r\   r~   r  r   	exp_shapeexpected_predictionsr"  r#  tgs
             rM   check_cross_val_predict_binaryr    s.   	5	)	)	)B 	v{{!'+>!>!>SVVIISVVQK		G	8I..xx1~~ C CtCjjnnQuXqx00%9WS&%9%9!D'%B%BT"" !a%Q0 
 
c1bB???AU	
 	
 	
 	

 
rO   c           
         t          dd          }t          j        t          j                  j        }||dd}t          j        t          |          t          t          |                    f||         t          j                  }t          j        |d          \  }}	|	                    ||	          D ]\  }
}t          |                               ||
         |	|
                   }  t          | |          ||                   }t          j        |	|
                   }||t          j        ||          <   ||d	z   |d
z
  |                    d          fD ]$}t          t!          | ||||          |           %dS )zDHelper for tests of cross_val_predict with multiclass classificationr   Fr  r   r  r  r   rd  Treturn_inverser   r   r  r  N)r   r   finfor  minr  r   rz   r   r  r>   rU   r  ix_r  r   r   )r  r\   r~   r  r   	float_mindefault_valuesr  r  y_encr"  r#  
fold_preds
i_cols_fitr  s                  rM   "check_cross_val_predict_multiclassr    s   	5	)	)	)B $$(I&& N
 7	QSVV~f5RZ   y4000HAuxx5)) D DtCjjnnQuXuU|44)WS&))!D'22
YuU|,,
9CRVD*5566 !a%Q0 
 
c1bB???AU	
 	
 	
 	

 
rO   c                 J   t          dd          }t          j        t          j                  j        }||dd}j        d         }g }t          |          D ]}	t          t          dd|	f                             }
|
dk    r|d	k    rt          |          f}nt          |          |
f}|	                    t          j
        |||         t          j        
                     fdt          j        d                   D             }t          j        |d          }|                    ||          D ]\  }}t          |                               ||         ||                   }  t          | |          ||                   }t          |          D ]o}	t          j        ||         dd|	f                   }||	         j        dk    r||	         ||	         |<   It          j        ||          }||	         ||	         |<   pӉdz   dz
                      d          fD ]s}t)          | ||||          }t          |          t          |          k    sJ t          t          |                    D ]}t+          ||         ||                    tdS )zCheck the output of cross_val_predict for 2D targets using
    Estimators which provide a predictions as a list with one
    element per class.
    r   Fr  r   r  r   Nr   r  rd  c                     g | ]<}t          j        d d |f         d          d         d d t           j        f         =S )NTr  r   )r   r   r   )r   r  r~   s     rM   
<listcomp>z6check_cross_val_predict_multilabel.<locals>.<listcomp>   sX        		!AAAqD'$///2111bj=A  rO   r&  r  r  )r   r   r  r  r  rR   r   r   rz   r  r  concatenater  r>   rU   r  r   r   r  r  r   r   )r  r\   r~   r  r   r  r  	n_targetsexpected_predsi_coln_classes_in_labelr  
y_enc_colsr  r"  r#  r  	fold_colsidxr  cv_predict_outputr  s     `                   rM   "check_cross_val_predict_multilabelr    s   
 
5	)	)	)B $$(I&& N
 
INy!! 
 
 Qqqq%x[!1!122""v1D'D'DQ	IIQ!34IGI~f5RZHHH	
 	
 	
 	

   qwqz""  J N:A...Exx5)) 
? 
?tCjjnnQuXuU|44)WS&))!D'22
9%% 	? 	?E	%,qqq%x"899Ie$)Q...8.?u%d++fT9---7->u%c**	? !a%Q0 E E-c1bBOOO$%%^)<)<<<<<s,--.. 	E 	EA-a0.2CDDDD	EE ErO   c                 \    t          dd          \  }}dD ]}t          | |||           d S )Nr   r   )rF  r   r  r   r  )rA   r  )r  r\   r~   r  s       rM   *check_cross_val_predict_with_method_binaryr  9  sJ     ;;;DAqM : :&sAq&9999: :rO   c                     t                      }|j        |j        }}t          ||d          \  }}dD ]}t	          | |||           d S )Nr   r   r  )r#   rf  rg  r@   r  )r  rk  r\   r~   r  s        rM   .check_cross_val_predict_with_method_multiclassr  A  s`    ;;D9dkqA1aa(((DAqM > >*31f====> >rO   c                  z    t          t          d                     t          t          d                     d S )Nr  r  )r  r/   r  rn   rO   rM   "test_cross_val_predict_with_methodr  I  sF    ./A/U/U/UVVV2+...    rO   c                      t                      } | j        | j        }}t          ||d          \  }}dD ]%}t	          dd          }t          ||||           &d S )Nr   r   r  log_lossr   )lossr   )r#   rf  rg  r@   r0   r  )rk  r\   r~   r  r  s        rM   &test_cross_val_predict_method_checkingr  P  st     ;;D9dkqA1aa(((DAqM > >!<<<*31f====> >rO   c                      t                      } | j        | j        }}t          ||d          \  }}t	          t          dd          dddgid	
          }dD ]}t          ||||           d S )Nr   r   r~  r  r%  r   r:  r   r   r   r  )r#   rf  rg  r@   rD   r/   r  )rk  r\   r~   r  r  s        rM   /test_gridsearchcv_cross_val_predict_with_methodr  [  s    ;;D9dkqA1aa(((DAq
;???#QxUV  C N > >*31f====> >rO   c                      d} d}t          | d|dd          \  }}t          t          dd	                    }d
D ]}t          ||||           d S )Nr  r   r   r   r~  r
  r  rF  r  r   r  r   )r  r   )r   r  r  )rB   r?   r/   r  )n_samprF  r\   r~   r  r  s         rM   1test_cross_val_predict_with_method_multilabel_ovrr  f  s     FI)1	aVX  DAq 0RSTTT
U
UC8 A A&sAq@@@@@A ArO   c                       e Zd Zd ZdS )RFWithDecisionFunctionc                     |                      |          }d}t          |t                    s
J |            d |D             }|S )Nz?This helper should only be used on multioutput-multiclass tasksc                 L    g | ]!}|j         d          dk    r|dddf         n|"S )r   r   Nr   r}  )r   r  s     rM   r  z<RFWithDecisionFunction.decision_function.<locals>.<listcomp>|  s6    CCCQWQZ1__111b5!CCCrO   )r   r   r|   )rL   r\   probsmsgs       rM   r  z(RFWithDecisionFunction.decision_functionx  sP    ""1%%O%&&+++++CCUCCCrO   N)rj   rk   rl   r  rn   rO   rM   r  r  t  s#            rO   r  c                  D   d} t          dd| dd          \  }}|d d dfxx         |d d df         z  cc<   d	D ]e}t          dd
          }t          j                    5  t          j        d           t          ||||           d d d            n# 1 swxY w Y   fd S )Nr   r  r   r   r~  r  r   r   r   r  r  n_estimatorsr   ignorer  )rB   r  r  r  simplefilterr  )rF  r\   r~   r  r  s        rM   0test_cross_val_predict_with_method_multilabel_rfr    s&    I)Y1SU  DAq aaadGGGqAwGGGM I I$!!DDD$&& 	I 	I!(+++.sAqHHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	II Is    (BB	B	c                  l   t           j                            d          } |                     ddd          }t          j        g d          }t          d          }dD ]S}t          j                    5  t          j        d	           t          ||||           d d d            n# 1 swxY w Y   Td S )
Nr   r   )rx  r   r  )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  )
r   r-  r.  normalr  r/   r  r  r  r  )r1  r\   r~   r  r  s        rM   -test_cross_val_predict_with_method_rare_classr    s    )


"
"C

1ah
''A
;;;<<A
K
0
0
0CM B B$&& 	B 	B!(+++.sAq&AAA	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	B 	BB Bs   5'B((B,	/B,	c                     t           j                            d          } |                     ddd          }t          j        ddgddgddgddgddgg          }dD ]e}t          dd          }t          j                    5  t          j        d	           t          ||||
           d d d            n# 1 swxY w Y   fd S )Nr   r   )r   r   r  r   )r   r  r   r  r  r  )
r   r-  r.  r  r  r  r  r  r  r  )r1  r\   r~   r  r  s        rM   ;test_cross_val_predict_with_method_multilabel_rf_rare_classr    s+    )


"
"C

1ag
&&A
1a&1a&1a&1a&1a&9::A8 I I$!!DDD$&& 	I 	I!(+++.sAqHHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	II Is   (B77B;	>B;	c                    t          j        t          |          |g          }t          ||          }|                    | |          D ]\  }}	|                    | |         ||                     || |	                   }
|dk    r$t          j        t          |	          |f          }n@t          j        t          |	          |ft          j        |j                  j	                  }|
|d d |j
        f<   |||	<   |S )Nr   )r   rh  r   r  r  rU   r  r  re  r  classes_)r\   r~   r   classesr  r  r  funcr"  r#  expected_predictions_exp_pred_tests               rM   get_expected_predictionsr    s    8SVVW$5663Dxx1~~ 3 3t%!E(### $QtW_$$Hc$ii%9::MMGTG$bh/C/I&J&J&N M *?aaao&%2T""rO   c            	         t          j        d                              dd          } t          j        d t	          d          D                       }d}t          d          }t          d          }t                      }g d	}|D ]}t          d
          }t          || |||          }	t          | |||||          }
t          |
|	           t          || |||          }	t          | |||||          }
t          |
|	           t          t          j        t	          d          d          d          }t          || |||          }	|                    |          }t          | |||||          }
t          |
|	           d S )Nr  r  r   c                     g | ]}|d z  S )r   rn   )r   rs   s     rM   r  z7test_cross_val_predict_class_subset.<locals>.<listcomp>  s    ...a!r'...rO   r   r   r	  r   r  r  r  r  r   r   )r   r  r   r  r   r   r:   r/   r   r  r
   r@   r  fit_transform)r\   r~   r  kfold3kfold4lemethodsr  r  r  r  s              rM   #test_cross_val_predict_class_subsetr    s   
	#sA&&A
..5::...//AGAFAF	BIIIG E E 444 (Q&VLLL  8q&'3 
  
 	""6DDD (Q&VLLL7q&'3 
  
 	""6DDD BIeBii,,1==='Q&VLLLQ7q&'3 
  
 	""6DDDD5E ErO   c                     t                      } | j        | j        }}t                      }t	          j        dd          }|                    d           |                                 t          j	        |j
        t          j                  t          j	        |j
        ddt          j                  	 t          |||fd	
           t          j        t                    5  t          |||fd
           d d d            n# 1 swxY w Y   d\  t!          d          D ]=}	 t#          j        |j
                    d S # t&          $ r t)          d           Y :w xY wd S # d\  t!          d          D ]<}	 t#          j        |j
                    w # t&          $ r t)          d           Y 9w xY ww xY w)NwbF)moder  s   Hello world!!!!!rd  rn   r)rR   r  re  c                     S rX   rn   )r  r\   r~   rd   s      rM   r   z#test_score_memmap.<locals>.<lambda>  s    U rO   r   c                     S rX   rn   )r  r\   r~   r   s      rM   r   z#test_score_memmap.<locals>.<lambda>  s     rO   rh   r   r   )r#   rf  rg  r   tempfileNamedTemporaryFilewriterT  r   memmapnamer  r   r   r   r   r   osunlinkWindowsErrorr   )rk  r\   r~   r   tfr  rd   r   s         @@rM   test_score_memmapr	    s,   ;;D9dkqA


C		$$u	=	=	=BHH !!!HHJJJYrwbj111FIbgRcDDDEQ+B+B+B+BCCCC]:&& 	I 	ICA/G/G/G/GHHHH	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I 	I
 #q 	 	A	"'"""   c




		 	 #q 	 	A	"'"""   c




		sf   9/E* (D?E* DE* DE* -E		E%$E%*F=FF=F9	6F=8F9	9F=c                  V   t           t           fg} 	 ddlm}m} |                     ||f           n# t
          $ r Y nw xY w| D ]c\  t                      }|j        |j        }} |           |          }}fd}fd}	t          ||	          }
t          |
||           dd S )Nr   rL  c                 $    t          |           S rX   rP  rQ  s    rM   r   z4test_permutation_test_score_pandas.<locals>.<lambda>  rS  rO   c                 $    t          |           S rX   rP  rU  s    rM   r   z4test_permutation_test_score_pandas.<locals>.<lambda>  rW  rO   rX  )r   rY  rM  rN  r  rZ  r#   rf  rg  r   r   )r\  rM  rN  rk  r\   r~   r]  r^  r_  r`  r   rR  rV  s              @@rM   "test_permutation_test_score_pandasr    s     ]+,E,,,,,,,,fi())))   (- 1 1$
${{y$+1&&q))::a==e<<<<:::: <HHHsD%00001 1rb  c            
         t          t           j                  } t          j        dd          }t          j        d          }| |d t                      d d dd d g	}ddi}t          j        t          d          5  t          |i | d d d            n# 1 swxY w Y   t          j        d	          }t          j        t          |          5  t          | |d
d           d d d            n# 1 swxY w Y   t          j        t          |          5  t          | |d
d           d d d            n# 1 swxY w Y   t          j        t          |          5  t          | ||d
d           d d d            n# 1 swxY w Y   t          j        t          |          5  t          | ||dt           j        gd
d           d d d            n# 1 swxY w Y   |                                 dk    sJ d S )Nr   r   r~  r   r   r   z%Failing classifier failed as requiredr   zerror_score must be the string 'raise' or a numeric value. (Hint: if using 'raise', please make sure that it has been spelled correctly.)r   zunvalid-stringr   r   	parameter)r  r  r   r   r  )r   FAILING_PARAMETERr   r  r  r   r   r   r   r   reescaper   r   r   r   rd   )failing_clfr\   r~   fit_and_score_argsfit_and_score_kwargsrI  s         rM   test_fit_and_score_failingr    s   #$5$GHHK
	!RA


A%q$dAtTR)73	z)P	Q	Q	Q D D*C.BCCCD D D D D D D D D D D D D D D I	O M 
z	7	7	7 K K{A!9IJJJJK K K K K K K K K K K K K K K 
z	7	7	7 L LQ1:JKKKKL L L L L L L L L L L L L L L 
z	7	7	7 N N{AqQ<LMMMMN N N N N N N N N N N N N N N 
z	7	7	7 	
 	
"*<=(	
 	
 	
 	
	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 	
 #%%%%%%sZ   9BBBC,,C03C0D22D69D6E99E= E="GGGc                  .   t          dd          \  } }t          dd          }t          t                                          |                     \  }}|| |t                      ||dg}dddd d	d
}t          |i |}|d         |d         k    sJ d S )Nr  r   r	  r  r  r  r:  )rd  re  T)
parametersr  return_parametersr  )rA   r5   nextr   r  r   r   )r\   r~   r   r"  r#  r  r  results           rM   test_fit_and_score_workingr  F  s    !<<<DAq
XA
.
.
.C|~~++A..//KE4q!TVVUD!<#&s33! 
 /H3GHHF,#7#EEEEEEErO   c                   &    e Zd ZddZddZddZdS )DataDependentFailingClassifierNc                     || _         d S rX   max_x_value)rL   r"  s     rM   rN   z'DataDependentFailingClassifier.__init__V  s    &rO   c                 j    || j         k                                    }|rt          d| d          d S )NzClassifier fit failed with z values too high)r"  r  r   )rL   r\   r~   num_values_too_highs       rM   rU   z"DataDependentFailingClassifier.fitY  sO     4#3388:: 	S.ASSS  	 	rO   c                     dS )Nr  rn   rb   s      rM   rd   z$DataDependentFailingClassifier.score`  s    srO   rX   rh   )rj   rk   rl   rN   rU   rd   rn   rO   rM   r  r  U  sP        ' ' ' '        rO   r  r   c                 h   t          d          }t          j        dd          }t          j        d          }|||g}d| d}d}t	          j        d	|d
          d| t          j                  }t          j        t          |          5  t          |i | d d d            d S # 1 swxY w Y   d S )Nr  r!  r   r   r~  r   r  z8ValueError: Classifier fit failed with 1 values too highzh2 fits failed.+total of 3.+The score on these train-test partitions for these parameters will be set to r   z.+flagsr   )r  r   r  r  r  compileDOTALLr   r   r   r   )r   r  r\   r~   cross_validate_argscross_validate_kwargsindividual_fit_error_messager   s           rM   -test_cross_validate_some_failing_fits_warningr.  d  sE    1Q???K
	!RA


A&1-#$[AA 	C ! j	S!-0	S 	S4P	S 	S i	  O 
&o	>	>	> F F+E/DEEEF F F F F F F F F F F F F F F F F F   B''B+.B+c                 h   t          t           j                  }t          j        dd          }t          j        d          }|||g}d| d}d}t          j        d| t
          j                  }t          j	        t          |	          5  t          |i | d d d            d S # 1 swxY w Y   d S )
Nr   r   r~  r  r  z1ValueError: Failing classifier failed as requiredz4All the 7 fits failed.+your model is misconfigured.+r'  r   )r   r  r   r  r  r  r)  r*  r   r   r   r   )r   r  r\   r~   r+  r,  r-  rI  s           rM   *test_cross_validate_all_failing_fits_errorr1  ~  s%    $$5$GHHK
	!RA


A&1-#$[AA#V J	*'	* 	*i  M 
z	7	7	7 F F+E/DEEEF F F F F F F F F F F F F F F F F Fr/  c                      t          |          rX   )r   )r   r\   r~   	error_msgs       rM   _failing_scorerr4    s    
Y

rO   zignore:lbfgs failed to converger   c           	         t          d          \  }}t          d                              ||          }d}t          t          |          }| dk    rKt          j        t          |          5  t          |||d	|| 
           d d d            d S # 1 swxY w Y   d S d|  }t          j	        t          |          5  t          |||d	|| 
          }t          ||            d d d            d S # 1 swxY w Y   d S )NTr   r   rd  "This scorer is supposed to fail!!!r3  r   r   r   )r   r   r   r   )r#   r/   rU   r   r4  r   r   r   r   r   r   r   )r   r\   r~   r   r3  failing_scorerwarning_msgr   s           rM   #test_cross_val_score_failing_scorerr;    s   
 %%%DAq
a
(
(
(
,
,Q
2
2C4I_	BBBNg]:Y777 	 	Qa[   	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	=/:= = 	 \+[999 	1 	1$Qa[  F FK000		1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1 	1s$   1BBB&C44C8;C8r'  TFwith_multimetricc           
         t          d          \  }}t          d                              ||          }d}t          t          |          }|rt          t                    }|||d}	n|}	| dk    rLt          j        t          |	          5  t          |||d
|	||            d d d            d S # 1 swxY w Y   d S d|  }
t          j        t          |
	          5  t          |||d
|	||           }|D ]C}d|v r=d|v r#||         D ]}t          |t                    sJ -t          ||         |            D	 d d d            d S # 1 swxY w Y   d S )NTr   r   r6  r7  r8  )score_1score_2score_3r   r   r   )r   r   r'  r   r   r    _score_2)r#   r/   rU   r   r4  r&   r,   r   r   r   r   r   r   r   ra   r   )r   r'  r<  r\   r~   r   r3  r9  non_failing_scorerr   r:  r  r   r  s                 rM   "test_cross_validate_failing_scorerrC    sz    %%%DAq
a
(
(
(
,
,Q
2
2C4I_	BBBN !();<<%)%
 
 !g]:Y777 		 		#5'   		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		 		=/:= = 	 \+[999 	C 	C$#5'  G  
C 
Cs??!S(( ") 8 8A#-a#7#777778
 (kBBB
C	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	C 	Cs%   B44B8;B8!AEEEc                     dS )Ng;pΈ^@rn   )r  jr2  s      rM   three_params_scorerrF    s    6rO   z:train_score, scorer, verbose, split_prg, cdt_prg, expectedr   )r   r   r   r   zS\[CV\] END .................................................... total time=   0.\ds)sc1sc2r   ze\[CV 2/3\] END  sc1: \(train=3.421, test=3.421\) sc2: \(train=3.421, test=3.421\) total time=   0.\dsr   zW\[CV 2/3; 1/1\] END ....... sc1: \(test=3.421\) sc2: \(test=3.421\) total time=   0.\dsc           	         t          dd          \  }}t          dd          }	t          t                                          |                    \  }
}|	||||
||d d g	}|||d}t          |i | |                                 \  }}|                    d          }t          |          dk    rt          j	        ||d	                   sJ d S t          j	        ||d                   sJ d S )
Nr  r   r	  r  r  )r'  split_progresscandidate_progress
r   r   )
rA   r5   r  r   r  r   
readouterrr   r  r   )capsysr(  r  rP  	split_prgcdt_prgexpectedr\   r~   r   r"  r#  r  r  rV  r  outliness                    rM   test_fit_and_score_verbosityrT    s   F !<<<DAq
XA
.
.
.C|~~++A..//KE4 q!VUD'4N)#% 
 &?*>???  FCyyH
8}}qx(1+.......x(1+.......rO   c                      d} d }d d d |g}t          j        t          |           5  t          |dt          j        i d d d            d S # 1 swxY w Y   d S )Nz&scoring must return a number, got Nonec                     d S rX   rn   )r   X_tests     rM   two_params_scorerz%test_score.<locals>.two_params_scorer3	  s    trO   r   r   )r   r   r   r    r   r   )rI  rX  r  s      rM   r)  r)  0	  s    <M   d,=>	z	7	7	7 8 8"77778 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8 8s   AAAc                      d } t          ddd          \  }}t          d          }|                    ||           t          |||d|           }g d	}|D ]}d
                    |          |v sJ d S )Nc                     |                      |          }t          ||          }|d         |d         |d         |d         dS )N)r   r   rG  )r   r   )r   r   tnfpfntp)r]   r(   )r   r\   r~   r6  cms        rM   r7  zPtest_callable_multimetric_confusion_matrix_cross_validate.<locals>.custom_scorer<	  sB    Qa((hbhbhbhOOOrO   (   r   r~  )r
  r  r   r   r   )r   r   r[  ztest_{})rA   r6   rU   r   r   )r7  r\   r~   r  rC  score_namesr  s          rM   9test_callable_multimetric_confusion_matrix_cross_validaterc  ;	  s    P P P
 KKKDAq

$
$
$CGGAqMMMQaGGGJ***K 4 4%%333334 4rO   c                  l    t          d          \  } }t          t                      | |dd           dS )z^Check that regressors with partial_fit is supported.

    Non-regression test for #22981.
    r~  r   Tr   )rY  r   N)r!   r   r8   )r\   r~   s     rM   *test_learning_curve_partial_fit_regressorsre  K	  s=    
 +++DAq <>>1adqQQQQQQrO   )rm   r  r  rQ  r   r  	functoolsr   timer   r   numpyr   scipy.sparser   r   sklearn.exceptionsr   )sklearn.model_selection.tests.test_searchr   sklearn.utils._testingr	   r
   r   r   sklearn.utils._mockingr   r   sklearn.utils.validationr   sklearn.model_selectionr   r   r   r   r   r   r   r   r   r   r   r   r   r   #sklearn.model_selection._validationr   r   r    sklearn.datasetsr!   r"   r#   r$   sklearn.metricsr%   r&   r'   r(   r)   r*   r+   r,   r-   sklearn.linear_modelr.   r/   r0   r1   r2   sklearn.ensembler3   sklearn.neighborsr4   sklearn.svmr5   r6   sklearn.clusterr7   sklearn.neural_networkr8   sklearn.imputer9   sklearn.preprocessingr:   sklearn.pipeliner;   ior<   sklearn.baser=   r>   sklearn.multiclassr?   sklearn.utilsr@   rA   rB   $sklearn.model_selection.tests.commonrC   rD   r  	NameErrorrF   rp   r   r   r   r  r\   r   r  r~   r   r  r   r   r   r   r  r$  r  r  rJ  markfilterwarningsra  rr  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r!  r+  r4  r6  rK  rN  rW  r\  r_  ra  rl  rp  rr  rt  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r	  r  r  r  r  parametrizer   r.  r1  r4  r;  rC  rF  rT  r)  rc  re  rn   rO   rM   <module>r     sd       				 				 



                    / / / / / / / / / / / / / / G G G G G G 6 6 6 6 6 6 < < < < < < 5 5 5 5 5 5 2 2 2 2 2 2 D D D D D D D D 1 1 1 1 1 1 A A A A A A A A 5 5 5 5 5 5 2 2 2 2 2 2 : : : : : : ) ) ) ) ) ) 3 3 3 3 3 3 / / / / / / 4 4 4 4 4 4 3 3 3 3 3 3 . . . . . . 5 5 5 5 5 5 2 2 2 2 2 2 4 4 4 4 4 4 E E E E E E > > > > > > 6 6 6 6 6 6 , , , , , , * * * * * * & & & & & & ( ( ( ( ( ( 4 4 4 4 4 4 ' ' ' ' ' ' * * * * * * , , , , , , ; ; ; ; ; ; + + + + + + $ $ $ $ $ $ . . . . . . ) ) ) ) ) ) I I I I I I I I I I M M M M M M M M 3 3 3 3 3 3 2 2 2 2 2 2 & & & & & & & & " " " " " " / / / / / / ( ( ( ( ( ( . . . . . . % % % % % %       & & & & & &       2 2 2 2 2 2 ! ! ! ! ! ! 0 0 0 0 0 0 ; ; ; ; ; ; @ @ @ @ @ @ 0 0 0 0 0 0LL   LLL" " " " "] " " "6    (>   <" " " " " " " ",	" 	" 	" 	" 	",F 	" 	" 	"R8 R8 R8 R8 R8 R8 R8 R8n BGG:a==BH+++,, RX,,,--:fbfQii  '< '< '<T) ) )<3 <3 <3~L L L&,= ,= ,=^-M -M -M`F9 F9 F9R> > >, FGG0 0 HG0&5 5 5&8 8 86 6  6  6F& & & . . .J J J&L L L0? ? ?D$ $ $Q Q Q  H H H<5
 5
 5
p1W 1W 1Wh# # # # # # 22 22 22j FGG2 2 HG2$T T T,
$ 
$ 
$& & &:= := :=zS S S&% % %4K K K W W W4S S S0$ $ $NC C C., , ,,S S S(K K K\  &- - -`I I I6  ,+D +D +D\# # #L= = =3 3 3$
 
 
*
 
 
81E 1E 1Eh: : :> > >  > > >> > >A A A	 	 	 	 	3 	 	 	I I I"B B BI I I"     *&E &E &ER  4 FGG1 1 HG1(&& && &&RF F F    ]    44F F 54F2 44F F 54F*      =>>G(<==1 1 >= ?>14 =>>G(<==-e}==+eT];;:C :C <; >= >= ?>:Cz   @ $	
 '0CDD?	
 '0CDD$	
' @/ /A @/.8 8 84 4 4 R R R R Rs   F FF