
    _nd];             
          d Z ddlmZmZ ddlmZ ddlmZmZ ddl	m
Z
 ddlZddlZddlmZ ddlZddlZddlmZ ddlZddlmZmZmZmZmZmZmZmZ dd	lm Z m!Z! dd
l"m#Z#m$Z$m%Z% ddl&m'Z'm(Z( ddl&m)Z) ddl*m+Z+ ddl*m,Z, ddl*m-Z- ddl.m/Z/ ddl.m0Z0 ddl.m1Z1 ddl.m2Z2 ddl.m3Z3 ddl.m4Z4 ddl.m5Z5 ddl.m6Z6 ddl.m7Z7 ddl.m8Z8 ddl.m9Z9 ddl.m:Z: ddl;m<Z< ddl=m>Z> ddl?m@Z@mAZA ddlBmCZC dd lBmDZD dd!lEmFZF dd"lGmHZH dd#lGmIZI dd$lGmJZJ dd%lKmLZL dd&lKmMZM dd'lKmNZN dd(lKmOZO dd)lKmPZP dd*lKmQZQ dd+lKmRZR dd,lSmTZT dd-lUmVZV dd.lWmXZX dd/lYmZZZm[Z[m\Z\ dd0l]m^Z^ dd1l_m`Z`  G d2 d3          Za G d4 d5e@          Zb ejc        d6d6gd7d6gd8d8gd9d8gg          Zd ejc        g d:          Zed; Zfejg        h                    d<e9 e
e:d=>          g          ejg        h                    d?deid@fdAdgidgeidBfdAdieidCfg          dD                         ZjdE ZkdF ZldG Zmejg        h                    dHe7e8g          dI             ZnedJ             ZodK ZpdL ZqdM ZrdN ZsdO ZtdP ZudQ ZvdR ZwdS ZxdT ZydU ZzdV Z{dW Z| G dX dYe'          Z}edZ             Z~d[ Zd\ Zejg        h                    d]d6d9g          ejg        h                    d^e8e7g          d_                         Zd` Zda Zdb Zdc Zedd             Zde Zdf Zdg Zdh Zdi Zdj Zdk Zejg        h                    dle7dmdnd8d=giife8dnd8d=gid9dofg          dp             Zdq Zdr Zds Zdt Zejg        h                    d^ e8 eD            dudvd=giw           e7 eD            dudvd=gix          g          dy             Zejg        h                    d^ e8 eIdz{          d|dvd=gid}~           e7 eIdz{          d|dvd=gid}          g          d             Zd Zd Z e            d             Zd Zd Zd Zd Zd Z G d de'          Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zejg        h                    dddzg          ejg        h                    dle7dmdug diife8dug diddofg          d                         Zd Zd Zd Zd Zd Zd Zd Zejg        h                    ddzdg          d             Zd Zd Zejg        h                    de7dddgife8d e%d8d          ifg          d             Zejg        h                    de7dddgife8d e%dd          ifg          d             Zejg                            d          ejg        h                    dHe7e8g          ejg        h                    deeg          d                                     Zejg        h                    ddzdg          d             ZdS )zTest the search module    )IterableSized)StringIO)chainproduct)partialN)GeneratorType)assert_array_equalassert_array_almost_equalassert_allcloseassert_almost_equalignore_warningsMinimalClassifierMinimalRegressorMinimalTransformer)CheckingClassifierMockDataFrame)	bernoulliexponuniform)BaseEstimatorClassifierMixin)is_classifier)make_classification)
make_blobs)make_multilabel_classification)train_test_split)KFold)StratifiedKFold)StratifiedShuffleSplit)LeaveOneGroupOut)LeavePGroupsOut)
GroupKFold)GroupShuffleSplit)GridSearchCV)RandomizedSearchCV)ParameterGrid)ParameterSampler)BaseSearchCV)FitFailedWarning)	LinearSVCSVC)DecisionTreeRegressor)DecisionTreeClassifier)KMeans)KernelDensity)LocalOutlierFactor)KNeighborsClassifierf1_score)recall_score)accuracy_score)make_scorer)roc_auc_score)confusion_matrix)r2_score)euclidean_distances)SimpleImputer)Pipeline)RidgeSGDClassifierLinearRegression)HistGradientBoostingClassifier)OneTimeSplitterc                   T    e Zd ZdZddZd Zd Zd Zd ZeZ	eZ
eZdd	ZddZd ZdS )MockClassifierz8Dummy classifier to test the parameter search algorithmsr   c                     || _         d S N	foo_param)selfrH   s     Ilib/python3.11/site-packages/sklearn/model_selection/tests/test_search.py__init__zMockClassifier.__init__P       "    c                 |    t          |          t          |          k    sJ t          j        |          | _        | S rF   )lennpuniqueclasses_rI   XYs      rJ   fitzMockClassifier.fitS   s3    1vvQ	!rM   c                     |j         d         S Nr   shape)rI   Ts     rJ   predictzMockClassifier.predictX   s    wqzrM   c                     || j         z   S rF   rG   rI   rT   s     rJ   	transformzMockClassifier.transform[       4>!!rM   c                     || j         z
  S rF   rG   r^   s     rJ   inverse_transformz MockClassifier.inverse_transform^   r`   rM   Nc                 &    | j         dk    rd}nd}|S )N         ?        rG   )rI   rT   rU   scores       rJ   rg   zMockClassifier.scoree   s     >AEEErM   Fc                     d| j         iS NrH   rG   )rI   deeps     rJ   
get_paramszMockClassifier.get_paramsl   s    T^,,rM   c                      |d         | _         | S ri   rG   )rI   paramss     rJ   
set_paramszMockClassifier.set_paramso   s    ,rM   )r   NN)F)__name__
__module____qualname____doc__rK   rV   r\   r_   rb   predict_probapredict_log_probadecision_functionrg   rk   rn    rM   rJ   rD   rD   M   s        BB# # # #  
  " " "" " " M   - - - -    rM   rD   c                   (    e Zd ZdZed             ZdS )LinearSVCNoScorez0A LinearSVC classifier that has no score method.c                     t           rF   )AttributeErrorrI   s    rJ   rg   zLinearSVCNoScore.scorew   s    rM   N)rp   rq   rr   rs   propertyrg   rw   rM   rJ   ry   ry   t   s3        ::  X  rM   ry   rd      )rd   rd   r   r   c                 ~     t                      fdt          t                               D             k    sJ d S )Nc                      g | ]
}|         S rw   rw   ).0igrids     rJ   
<listcomp>z3assert_grid_iter_equals_getitem.<locals>.<listcomp>   s    <<<a$q'<<<rM   )listrangerO   )r   s   `rJ   assert_grid_iter_equals_getitemr      sC    ::<<<<5T+;+;<<<<<<<<<rM   klass
   n_iterz input, error_type, error_messagez1Parameter .* a dict or a list, got: 0 of type intfooz Parameter .* is not a dict \(0\)z}Parameter (grid|distribution) for parameter 'foo' (is not|needs to be) (a list or a numpy array|iterable or a distribution).*c                 z    t          j        ||          5   | |           d d d            d S # 1 swxY w Y   d S )Nmatch)pytestraises)r   input
error_typeerror_messages       rJ   test_validate_parameter_inputr      s     
z	7	7	7  e                 s   044c            
         dg di} t          |           }t          |t                    sJ t          |t                    sJ t	          |          dk    sJ t          |           ddgg dd}t          |          }t	          |          dk    sJ t          d          D ]T}t          d	 |D                       }|t          d
 t          |d         |d                   D                       k    sJ Ut          |           t          i           }t	          |          dk    sJ t          |          i gk    sJ t          |           t          j        t                    5  |d          d d d            n# 1 swxY w Y   t          dddgii ddgig          }t	          |          dk    sJ t          |          ddiddii ddigk    sJ t          |           d S )Nr   rd   r      r      r   )hamspameggs)r   bar   c           	   3      K   | ]9}t          t          t          |                                                     V  :d S rF   )tupler   sorteditemsr   ps     rJ   	<genexpr>z&test_parameter_grid.<locals>.<genexpr>   s?      GGAU56!''))#4#4677GGGGGGrM   c              3   (   K   | ]\  }}d |d|fV  dS )r   r   Nrw   )r   xys      rJ   r   z&test_parameter_grid.<locals>.<genexpr>   s@       
 
%)QUAua 
 
 
 
 
 
rM   r   rd   Cr         ?)r'   
isinstancer   r   rO   r   r   setr   r   r   r   
IndexError)params1grid1params2grid2r   pointsempty	has_emptys           rJ   test_parameter_gridr      so    iii G'""EeX&&&&&eU#####u::????#E***!f%<%<%<==G'""Eu::???? 1XX 
 
GGGGGGG 
 
-4WU^WU^-T-T
 
 
 
 
 
 
 
 
 
 $E*** "Eu::????;;2$#E***	z	"	"  a               aW~rC#<@AAIy>>Q	??Qx#rBc
CCCCC#I.....s   *	E??FFc                     t                      } t          | dg didd          }t          j        }t	                      t          _        |                    t          t                     |t          _        |j        j	        dk    sJ t          |j        d         j        g d           |                    t          t                     |                    t                     |                    t                     |                    t                     d|_        t%          j        t(                    5  |                    t          t                     d d d            d S # 1 swxY w Y   d S )NrH   r   r   cvverboser   param_foo_paramsklearn)rD   r%   sysstdoutr   rV   rT   r   best_estimator_rH   r
   cv_results_datarg   rt   rv   r_   scoringr   r   
ValueError)clfgrid_search
old_stdouts      rJ   test_grid_searchr      sv   


Cs[)))$<ANNNKJCJOOAqCJ&0A5555{./@AF			RRR aa   !!!$$$! $K	z	"	"  1                 s   8!E&&E*-E*c                     t          dt                      fg          } dt                      t                      gi}t          | |d          }|                    t
          t                     |j        d         }t          |d         t                    sJ t          |d         t                    sJ t          |d         d          rJ t          |d         d          rJ |d         |j
        usJ |d         |j
        usJ t          |d         d         d          rJ t          |d         d         d          rJ d S )N	regressorr   r   param_regressorr   rd   coef_)r=   r@   r>   r%   rV   rT   r   r   r   hasattrr   )pipe
param_gridr   regressor_resultss       rJ   test_grid_search_pipeline_stepsr      sX   k#3#5#56788D 0 2 2EGG<=JtZA666KOOAq#/0AB'*,<====='*E22222(+W55555(+W55555Q{'BBBBBQ{'BBBBBz+.q17;;;;;z+.q17;;;;;;;rM   SearchCVc           	         t          j        d                              dd          }t          j        dgdz  dgdz  z             }t	          ddg          } | |d	g d
idd          }d}t          j        t          |          5  |                    ||t          j	        d                     d d d            n# 1 swxY w Y   d}t          j        t          |          5  |                    ||t          j	        d          t          j
        d                     d d d            n# 1 swxY w Y   |                    ||t          j	        d          t          j
        d                     d S )Nd   r   r      rd   r   r   )expected_fit_paramsrH   r   r   raiser   error_scorez0Expected fit parameter\(s\) \['eggs'\] not seen.r   )r   z)Fit parameter spam has length 1; expected)r   r   )rP   arangereshapearrayr   r   r   AssertionErrorrV   oneszeros)r   rT   r   r   searchererr_msgs         rJ   test_SearchCV_with_fit_paramsr      s   
	#r2&&A
!qA37"##A
&&1A
B
B
BCxk9995!QQQH BG	~W	5	5	5 - -Q,,,- - - - - - - - - - - - - - - :G	~W	5	5	5 ? ?Q

">>>? ? ? ? ? ? ? ? ? ? ? ? ? ? ?LLABGBKKbhrllL;;;;;s$   
+CCC)>D33D7:D7c                  \   t          d          } t          dd          \  }}g d}t          d          }t          | d|id          }|                    ||           t          |d|id          }|                    ||           |j        |j        k    sJ |                    ||          |                    ||          k    sJ t          |d|i          }t          j        t          d	
          5  |                    dgg           d d d            d S # 1 swxY w Y   d S )Nr   random_stater   r   centers皙?rd   r   r   accuracyr   z
no scoringr   rd   )
r+   r   ry   r%   rV   best_params_rg   r   r   	TypeError)r   rT   r   Csclf_no_scorer   grid_search_no_scores          rJ   test_grid_search_no_scorer      s    
#
#
#C1a000DAq	B#333LsS"IzBBBKOOAq'sBiTTTQ"""  ,0HHHHHQ""&:&@&@A&F&FFFFF (sBi@@	y	5	5	5 ( (  1#'''( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (s   <D!!D%(D%c                     t          dddd          \  } }t          d          }ddgi}t          ||d 	                              | |          }t          ||d
	                              | |          }t          t	                      |d	                              | |          }t          ||d	                              | |          }|                    | |          }|                    | |          }	|                    | |          }
|                    | |          }|dk     sJ |	dk     sJ ||	k    sJ t          |	|           t          ||
           d S )Nr   r   皙?r   	n_samples	n_classesflip_yr   r   r   r   r   r   roc_aucre   )r   r+   r%   rV   ry   rg   r   )rT   r   r   r   search_no_scoringsearch_accuracysearch_no_score_method_auc
search_aucscore_no_scoringscore_accuracyscore_no_score_auc	score_aucs               rJ   test_grid_search_score_methodr    s   #TUVVVDAq

#
#
#C#<D$S$===AA!QGG"3jAAAEEaKKO!-D)" " "	c!Qii  c4;;;??1EEJ )..q!44$**1a00N399!Q??  A&&I s????C&&&&(8999	#566666rM   c                     t           j                            d          } t          ddd          \  }}|                     ddd          }t          d          }ddgi}t                      t          d          t          d	          t                      g}d
}|D ]u}t          |||          }	t          j        t          |          5  |	                    ||           d d d            n# 1 swxY w Y   |	                    |||           vt                      t!                      g}
|
D ]*}t          |||          }	|	                    ||           +d S )Nr      r   )r   r   r   r   r   r   rd   n_splitsz*The 'groups' parameter should not be None.r   r   )groups)rP   randomRandomStater   randintr+   r!   r"   r#   r$   r%   r   r   r   rV   r   r    )rngrT   r   r  r   r   	group_cvs	error_msgr   gsnon_group_cvss              rJ   test_grid_search_groupsr  /  s    )


"
"CqqIIIDAq[[Ar""F

#
#
#C!:D 	A	I =I $ $#t+++]:Y777 	 	FF1aLLL	 	 	 	 	 	 	 	 	 	 	 	 	 	 	
q!F####$&&(>(@(@AM  #t+++
q! s   C00C4	7C4	c                     t          j        d                              dd          } t          j        dgdz  dgdz  z             }g d}t	          t          d          d|i          }|                    | |           t          |j        j	        |j	                   t	          t                      d	d
dgi          }|                    | |           t          |d          rJ t	          t          d          d|i          }t          |d          rJ t	          t          d          d|id          }|                    | |           t          |d          rJ d S )Nr   r   r   r   rd   r   r   r   alphare   g       @rR   F)refit)rP   r   r   r   r%   r+   rV   r
   r   rR   r>   r   )rT   r   r   r   s       rJ   test_classes__propertyr  N  sf   
	#r2&&A
!qA37"##A	Bya8883)DDKOOAq{2;[=QRRR uww3*(=>>KOOAq{J///// ya8883)DDK{J///// ya8883)5QQQKOOAq{J///////rM   c                  @   t                      } t          | ddgid          }|                    t          t                     t          |d          sJ t          | ddgidd          }|                    t          t                     t          |d          sJ d S )NrH   rd   r   r   r   r   )r   r   )rD   r%   rV   rT   r   r   r&   )r   r   random_searchs      rJ   test_trivial_cv_results_attrr  g  s    


Cs[1#$61===KOOAq;.....&s[1#,>qQOOOMa;.......rM   c            
         t                      } d ddgfD ]}t          | dg didd          }|                    t          t                     t          |d          s t          |d	          rt          |d
          sJ dD ]Y}d| d}t          j        t          |          5   t          ||          t                     d d d            n# 1 swxY w Y   Zd}dD ]i}t          j        t          |          5  t          | i |ddd                              t          t                     d d d            n# 1 swxY w Y   jd S )Nr   	precisionrH   r   Fr   r  r   r   best_index_r   )r\   rt   ru   r_   rb   z`refit=False`. z9 is available only after refitting on the best parametersr   zIFor multi-metric scoring, the parameter refit must be set to a scorer key) r   Trecallr   )accprec)r  r   )rD   r%   rV   rT   r   r   r   r   r{   getattrr   )r   r   r   fn_namer  r  s         rJ   test_no_refitr"  s  s   


C:{34 1 1"3iii(@RSTTT1%677	
]33	
 ^44	
 	
 	

 	1 	1G3' 3 3 3  ~Y??? 1 1-W--a0001 1 1 1 1 1 1 1 1 1 1 1 1 1 1	1  	T  5  ]:Y777 	 	Ruj+.V.V  c!Qiii	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 s$   %CCC>5D??E	E	c                     t          ddd          \  } }t                      }t          |dddgi          }t          j        t
                    5  |                    | d d         |           d d d            d S # 1 swxY w Y   d S )	N   r   r   r   
n_featuresr   r   r   re      )r   r+   r%   r   r   r   rV   )X_y_r   r   s       rJ   test_grid_search_errorr*    s     33QOOOFB
++C	cC#s,	-	-B	z	"	"  
r$3$x                 s   A==BBc                  <   t          ddd          \  } }dgdgdgd}t          d	
          }t          ||          }|                    | |           t          ddd          }|                    | |           t	          |j        |j        j                   d S )Nr$  r   r   r%  re   rbfr   )r   kernelgammaautor.  )r   r,   r%   rV   r
   
dual_coef_r   )r(  r)  
param_dictr   r   s        rJ   test_grid_search_one_grid_pointr3    s     33QOOOFBu3%@@J
F


C	c:	&	&BFF2rNNN
E
-
-
-CGGBOOOs~r'9'DEEEEErM   c                      t                      } d }t          | dt          dd          id          }|                    t          t
                     |j        j        dk    sJ d S )NrH   rd   r   r   r   r   )rD   r%   r   rV   rT   r   r   rH   )r   r   s     rJ   /test_grid_search_when_param_grid_includes_ranger5    sd    


CKs[%1++$>1EEEKOOAq&0A555555rM   c                  *   t          ddd          \  } }ddi}t          d          }t          j        d	          }t	          ||          }t          j        t          |
          5  |                    | |           d d d            n# 1 swxY w Y   dg i}t                      }t          j        d          }t	          ||          }t          j        t          |
          5  |                    | |           d d d            n# 1 swxY w Y   ddi}t          d          }t          j        d          }t	          ||          }t          j        t          |
          5  |                    | |           d d d            n# 1 swxY w Y   dt          j        d          i}t                      }t	          ||          }t          j        t                    5  |                    | |           d d d            d S # 1 swxY w Y   d S )Nr   r   r   r%  r   rd   r/  r0  zParameter grid for parameter 'C' needs to be a list or a numpy array, but got 1 (of type int) instead. Single values need to be wrapped in a list with one element.r   zIParameter grid for parameter 'C' need to be a non-empty sequence, got: []z1,2,3zParameter grid for parameter 'C' needs to be a list or a numpy array, but got '1,2,3' (of type str) instead. Single values need to be wrapped in a list with one element.)r   r   )r   r,   reescaper%   r   r   r   rV   r   rP   r   )rT   r   r2  r   r  searchs         rJ   test_grid_search_bad_param_gridr:    s   JJJDAqqJ
F


C		@ I
 #z**F	y		2	2	2  

1a               rJ
%%C	S I #z**F	z	3	3	3  

1a               wJ
F


C		. I
 #z**F	y		2	2	2  

1a               rwv'J
%%C#z**F	z	"	"  

1a                 sH   )BBB(DDD)FFF$HHHc                     t          ddd          \  } }t                      }t          |dddgi          }|                    | d d         |d d                    |                    | dd                    }|j        j        }t          j        |           } t                      }t          |dddgi          }|                    | d d         	                                |d d                    |                    | dd                    }|j        j        }t          j        ||k              d	k    sJ ||k    sJ d S )
Nr$  r   r   r%  r   r   re   r'  g?)r   r+   r%   rV   r\   r   r   sp
csr_matrixtocoorP   mean)r(  r)  r   r   y_predr   y_pred2C2s           rJ   test_grid_search_sparserC    s6    33QOOOFB
++C	cC#s,	-	-BFF2dsd8RXZZ344!!F
A	r		B
++C	cC#s,	-	-BFF2dsd8>>RX&&&jjCDD""G			B76W$%%,,,,777777rM   c                  v   t          ddd          \  } }t                      }t          |dddgid	          }|                    | d d
         |d d
                    |                    | d
d                    }|j        j        }t          j        |           } t                      }t          |dddgid	          }|                    | d d
         |d d
                    |                    | d
d                    }|j        j        }t          ||           ||k    sJ d }t          |d          }	t          |dddgi|		          }|                    | d d
         |d d
                    |                    | d
d                    }
|j        j        }||k    sJ t          ||
           d S )Nr$  r   r   r%  r   r   re   f1r   r'  c                 $    t          | |           S rF   r3   )y_true_y_pred_s     rJ   f1_lossz0test_grid_search_sparse_scoring.<locals>.f1_loss  s    '****rM   F)greater_is_better)r   r+   r%   rV   r\   r   r   r<  r=  r
   r7   )r(  r)  r   r   r@  r   rA  rB  rI  F1Lossy_pred3C3s               rJ   test_grid_search_sparse_scoringrN    s    33QOOOFB
++C	cC#s,d	;	;	;BFF2dsd8RXZZ344!!F
A	r		B
++C	cC#s,d	;	;	;BFF2dsd8RXjjCDD""G			Bvw'''7777+ + + E:::F	cC#s,f	=	=	=BFF2dsd8RXjjCDD""G			B7777vw'''''rM   c                     t          ddd          \  } }t          j        | d d         | d d         j                  }|d d         }t	          d          }t          |dd	d
gi          }|                    ||           |j        dk    sJ t          j        | dd          | d d         j                  }|dd          }|                    |          }t          j	        ||k              dk    sJ t          j        t                    5  |                    |                                |           d d d            d S # 1 swxY w Y   d S )Nr$  r   r   r%  r'  precomputedr-  r   r   re   )r   rP   dotr[   r,   r%   rV   best_score_r\   r?  r   r   r   tolist)	r(  r)  K_trainy_trainr   r   K_testy_testr@  s	            rJ   #test_grid_search_precomputed_kernelrY    s    !33QOOOFB fRXr$3$xz**G#hG
]
#
#
#C	cC#s,	-	-BFF7G>Q VBsttHb#hj))FXFZZF76V#$$)))) 
z	"	" * *
w~~)))* * * * * * * * * * * * * * * * * *s   )EEEc                  .   t          j        d          } t          j        d          }t          d          }t	          |dddgi          }t          j        t                    5  |                    | |           d d d            d S # 1 swxY w Y   d S )N)r      )r   rP  rQ  r   r   re   )	rP   r   r   r,   r%   r   r   r   rV   )rU  rV  r   r   s       rJ   3test_grid_search_precomputed_kernel_error_nonsquarer\  9  s     hx  GgennG
]
#
#
#C	cC#s,	-	-B	z	"	" ! !
w   ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !s   &B

BBc                   &    e Zd ZdZddZd Zd ZdS )BrokenClassifierz*Broken classifier that cannot be fit twiceNc                     || _         d S rF   	parameterrI   ra  s     rJ   rK   zBrokenClassifier.__init__G  rL   rM   c                 8    t          | d          rJ d| _        d S )Nhas_been_fit_T)r   rd  rI   rT   r   s      rJ   rV   zBrokenClassifier.fitJ  s&    411111!rM   c                 @    t          j        |j        d                   S rX   rP   r   rZ   r^   s     rJ   r\   zBrokenClassifier.predictN      x
###rM   rF   )rp   rq   rr   rs   rK   rV   r\   rw   rM   rJ   r^  r^  D  sL        44# # # #" " "$ $ $ $ $rM   r^  c                     t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          t                      dddgigdd	          }|                    | |           d S )
Nr   r   r   r   rd   ra  r  Tr   r  )rP   r   r   r   r%   r^  rV   )rT   r   r   s      rJ   
test_refitrk  R  s    
 		#r2&&A
!qA37"##A
kAq623[PT  C GGAqMMMMMrM   c                      d } t          ddd          \  }}t          t          d          dg did	| 
          }|                    ||           |j        dk    sJ t          |d          rJ dS )z^
    Test refit=callable, which adds flexibility in identifying the
    "best" estimator.
    c                 4   t          ddd          \  }}t          t          d          dg didd	
          }|                    ||           |j        dk    sJ |j                                        D ]}|| v sJ 	| d                                         S )z
        A dummy function tests `refit=callable` interface.
        Return the index of a model that has the least
        `mean_test_score`.
        r   r   *   r%  r   r   {Gz?r   rd   r  Trj  r   mean_test_score)r   r%   r+   rV   r  r   keysargmin)
cv_resultsrT   r   r   keys        rJ   refit_callablez+test_refit_callable.<locals>.refit_callablef  s     #SQRPPP12&&&...!	
 
 
 	1!#### ?'')) 	% 	%C*$$$$$+,33555rM   r   r   rn  r%  r   r   ro  r  rj  r   rS  N)r   r%   r+   rV   r  r   )rv  rT   r   r   s       rJ   test_refit_callablerw  `  s    6 6 62 LLLDAq
r"""	nnn	  C GGAqMMM?asM*******rM   c                     d } t          ddd          \  }}t          t          d          ddd	gid
|           }t          j        t
          d          5  |                    ||           ddd           dS # 1 swxY w Y   dS )zb
    Test implementation catches the errors when 'best_index_' returns an
    invalid result.
    c                     dS )zT
        A dummy function tests when returned 'best_index_' is not integer.
        Nrw   rt  s    rJ   refit_callable_invalid_typezEtest_refit_callable_invalid_type.<locals>.refit_callable_invalid_type  s	     trM   r   r   rn  r%  r   r   r   rd   r  rj  z&best_index_ returned is not an integerr   N)r   r%   r+   r   r   r   rV   )r{  rT   r   r   s       rJ    test_refit_callable_invalid_typer|    s       LLLDAq
r"""	sAh)	  C 
y(P	Q	Q	Q  1                 s   A==BBout_bound_value	search_cvc                      fd}t          ddd          \  }} |t          d          ddd	gid
|          }t          j        t          d          5  |                    ||           ddd           dS # 1 swxY w Y   dS )zg
    Test implementation catches the errors when 'best_index_' returns an
    out of bound result.
    c                     S )zV
        A dummy function tests when returned 'best_index_' is out of bounds.
        rw   )rt  r}  s    rJ   refit_callable_out_boundz?test_refit_callable_out_bound.<locals>.refit_callable_out_bound  s     rM   r   r   rn  r%  r   r   r   rd   r  rj  zbest_index_ index out of ranger   N)r   r+   r   r   r   rV   )r}  r~  r  rT   r   r   s   `     rJ   test_refit_callable_out_boundr    s         LLLDAq
)r"""	sAh&	  C 
z)I	J	J	J  1                 s   A<<B B c                     d } t          ddd          \  }}t          t                    dd}t          t	          d          d	g d
i||           }|                    ||           |j        dk    sJ t          |d          rJ dS )zC
    Test refit=callable in multiple metric evaluation setting
    c                 B    d| v sJ | d                                          S )z
        A dummy function tests `refit=callable` interface.
        Return the index of a model that has the least
        `mean_test_prec`.
        mean_test_prec)rs  rz  s    rJ   rv  z8test_refit_callable_multi_metric.<locals>.refit_callable  s-      :----*+22444rM   r   r   rn  r%  r  )Accuracyr  r   r   ro  rj  r   rS  N)r   r7   r6   r%   r+   rV   r  r   )rv  rT   r   r   r   s        rJ    test_refit_callable_multi_metricr    s    
5 5 5 LLLDAq&~66LLG
r"""	nnn	  C GGAqMMM?asM*******rM   c                     t          j        d                              dddd          } t          j        d                              ddd          }d	 }d
 }t          ||dg          }t	          |dg di          }|                    | |                              t          t                     t          |d          sJ d S )N,  r   r   r   r   i        c                 (    | j         dd          dk    S )Nrd   )r   r   r   rY   r   s    rJ   check_Xz#test_gridsearch_nd.<locals>.check_X  s    wqrr{i''rM   c                 (    | j         dd          dk    S )Nrd   )r  r  rY   r  s    rJ   check_yz#test_gridsearch_nd.<locals>.check_y  s    wqrr{g%%rM   rV   )r  r  methods_to_checkrH   r   r   )
rP   r   r   r   r%   rV   rg   rT   r   r   )X_4dy_3dr  r  r   r   s         rJ   test_gridsearch_ndr    s    9^$$,,RAq99D9[!!))"a44D( ( (& & &   C
 s[)))$<==KOOD$%%a+++;.......rM   c                     t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          d dg          }t          d	
          }t          |dg di|          }|                    |                                 |          	                    | |           t          |d          sJ d S )Nr   r   r   r   rd   c                 ,    t          | t                    S rF   r   r   r  s    rJ   <lambda>z test_X_as_list.<locals>.<lambda>      *Q-- rM   rV   )r  r  r   r  rH   r   r   r   rP   r   r   r   r   r   r%   rV   rT  rg   r   rT   r   r   r   r   s        rJ   test_X_as_listr    s    
	#r2&&A
!qA37"##A
--  C 
			Bs[)))$<DDDKOOAHHJJ""((A...;.......rM   c                     t          j        d                              dd          } t          j        dgdz  dgdz  z             }t	          d dg          }t          d	
          }t          |dg di|          }|                    | |                                          	                    | |           t          |d          sJ d S )Nr   r   r   r   rd   c                 ,    t          | t                    S rF   r  r  s    rJ   r  z test_y_as_list.<locals>.<lambda>  r  rM   rV   )r  r  r   r  rH   r   r   r   r  r  s        rJ   test_y_as_listr     s    
	#r2&&A
!qA37"##A
--  C 
			Bs[)))$<DDDKOOAqxxzz""((A...;.......rM   c                  T   t           t           fg} 	 ddlm}m} |                     ||f           n# t
          $ r Y nw xY wt          j        d                              dd          }t          j	        dgdz  dgdz  z             }| D ]\   |           |          }}fd}fd}t          ||	          }	t          |	d
g di          }
|
                    ||                              ||           |
                    |           t          |
d          sJ d S )Nr   )Series	DataFramer   r   r   rd   c                 $    t          |           S rF   r   )r   InputFeatureTypes    rJ   check_dfz#test_pandas_input.<locals>.check_df!  s    a!1222rM   c                 $    t          |           S rF   r  )r   
TargetTypes    rJ   check_seriesz'test_pandas_input.<locals>.check_series$  s    a,,,rM   )r  r  rH   r   r   )r   pandasr  r  appendImportErrorrP   r   r   r   r   r%   rV   rg   r\   r   )typesr  r  rT   r   X_dfy_serr  r  r   r   r  r  s              @@rJ   test_pandas_inputr    s    ]+,E,,,,,,,,i())))    		#r2&&A
!qA37"##A(- 3 3$*&&q))::a==e	3 	3 	3 	3 	3	- 	- 	- 	- 	- !<HHH"3iii(@AAe$$**4777D!!!{M2222223 3s   3 
A A c                  $   t          dd          \  } }t          ddd          }ddg}d	D ]N}t          |t          g d
          ||          }|                    | |           |j        d         dk    sJ Ot          |t          g d
          d          }|                    | |           |j        d         dk    sJ t          |t          g d
                    }|                    |            |j        d         dk    sJ d S )N2   r   r   r   r  rd   )r   initn_initadjusted_rand_scorefowlkes_mallows_score)r  r  )r   r   r   )
n_clusters)r   r   r  r  r   r   r   )r   r   )r   r/   r%   dictrV   r   )rT   r   kmr   r  r   s         rJ   test_unsupervised_grid_searchr  /  sV   333DAq	QXa	8	8	8B %&=>GA ; ;"4999555we
 
 
 	1'5::::: 
tyyy111;R  K OOAq#L1Q6666 rdiii.H.H.HIIIKOOA#L1Q666666rM   c                     d } t          ddddgddgddgg          \  }}t          t                      t          g d          |           }|                    |           |j        d	         dk    sJ |j        d
k    sJ d S )Nc                      | j         dk    rdndS )Nr   rn  r   	bandwidth)	estimatorrT   s     rJ   custom_scoringz2test_gridsearch_no_predict.<locals>.custom_scoringN  s    (C//rrQ6rM   r   rd   r   )cluster_stdr   r   ro  r  r  r  rn  )r   r%   r0   r  rV   r   rS  )r  rT   _r9  s       rJ   test_gridsearch_no_predictr  K  s    7 7 7 #AAAQRTUPV?WXXXDAq...111  F
 JJqMMM{+s2222######rM   c                     ddgt          dd          d} t          | dd          }d |D             }t          |          dk    sJ |D ]%}|d	         d
v sJ d|d         cxk    rdk    sn J &dg di} t          | dd          }d |D             d |D             k    sJ dt          dd          i} t          | dd          }d |D             d |D             k    sJ d S )Nr,  linearr   rd   )r-  r   r   )param_distributionsr   r   c                     g | ]}|S rw   rw   r   r   s     rJ   r   z&test_param_sampler.<locals>.<listcomp>b  s    """Qq"""rM   r-  )r,  r  r   )r   rd   r   r   r   r   r   r     	   r   r   c                     g | ]}|S rw   rw   r  s     rJ   r   z&test_param_sampler.<locals>.<listcomp>m      !ArM   c                     g | ]}|S rw   rw   r  s     rJ   r   z&test_param_sampler.<locals>.<listcomp>m      #7#7#7!A#7#7#7rM   c                     g | ]}|S rw   rw   r  s     rJ   r   z&test_param_sampler.<locals>.<listcomp>s  r  rM   c                     g | ]}|S rw   rw   r  s     rJ   r   z&test_param_sampler.<locals>.<listcomp>s  r  rM   )r   r(   rO   )r  samplersamplessamples       rJ   test_param_samplerr  \  sy   &+X%6WQ]]KK/  G #"'"""Gw<<2 % %h#44444F3K$$$$1$$$$$$$  B B BC/  G  w#7#7w#7#7#777771./  G  w#7#7w#7#7#7777777rM   c                    | j         t          fd|D                       sJ t          fd|D                       sJ t          fd|D                       rJ t          fd|D                       sJ | j        r| j                                        ndg}|D ]"}d|z           j        t          j        k    sJ #d S )Nc              3   b   K   | ])}t          |         t          j        j                  V  *d S rF   r   rP   maMaskedArray)r   paramrt  s     rJ   r   z/check_cv_results_array_types.<locals>.<genexpr>y  s7      XXEz*U+RU->??XXXXXXrM   c              3   D   K   | ]}|         j         t          k    V  d S rF   )dtypeobjectr   ru  rt  s     rJ   r   z/check_cv_results_array_types.<locals>.<genexpr>z  s/      EE3z#$.EEEEEErM   c              3   b   K   | ])}t          |         t          j        j                  V  *d S rF   r  r  s     rJ   r   z/check_cv_results_array_types.<locals>.<genexpr>{  s6      XXc:joru/@AAXXXXXXrM   c              3   x   K   | ]4}|                     d           |         j        t          j        k    V  5dS )rankN)
startswithr  rP   float64r  s     rJ   r   z/check_cv_results_array_types.<locals>.<genexpr>|  sW        ~~f%%3+     rM   rg   zrank_test_%s)	r   allanymultimetric_scorer_rr  r  rP   int32)r9  
param_keys
score_keysscorer_keysru  rt  s        @rJ   check_cv_results_array_typesr  v  s,   #JXXXXZXXXXXXXXEEEE*EEEEEEEEXXXXZXXXXXXXX            ,2+>M&.%%'''WIK B B.3./5AAAAAB BrM   c                      t          t                                                     t          ||z   dz                        t           fd||z   D                       sJ d S )N)rm   c              3   <   K   | ]}|         j         fk    V  d S rF   rY   )r   ru  rt  n_cands     rJ   r   z(check_cv_results_keys.<locals>.<genexpr>  s1      UUcz#$	1UUUUUUrM   )r
   r   rr  r  )rt  r  r  r  s   `  `rJ   check_cv_results_keysr    sx    z  !!6*z*AK*O#P#P   UUUUUZ*=TUUUUUUUUUUrM   c                    	 t          ddd          \  } }d}d}t          dgdd	gd
dg          t          dgddg          g}d}d}|}t          t                      ||d          }|                    | |           |j        	t          	d         dk              sJ 	fd|D             sJ 	fd|D             sJ t          |||           t          	|||           |j        	t          |j        d                   }t          	fdt          |          D                       sJ t          	fdt          |          D                       sJ d S )Nr  r   rn  r%  r   r   r,  rd   r   r   r-  r   r.  polyr   r-  degreeparam_Cparam_degreeparam_gammaparam_kernelrq  mean_train_scorerank_test_scoresplit0_test_scoresplit1_test_scoresplit2_test_scoresplit0_train_scoresplit1_train_scoresplit2_train_scorestd_test_scorestd_train_scoremean_fit_timestd_fit_timemean_score_timestd_score_timeT)r   r   return_train_scorer  c              3   V   K   | ]#}|d k    t          |         dk              V  $dS )r  r   Nr  r   krt  s     rJ   r   z.test_grid_search_cv_results.<locals>.<genexpr>  s?      RR1@Q;Q;QC
1"##;Q;Q;Q;QRRrM   c              3   ^   K   | ]'}d |v|dk    t          |         dk              V  (dS )timer  rd   Nr  r  s     rJ   r   z.test_grid_search_cv_results.<locals>.<genexpr>  sT        ??q$555 	JqMQ5555 rM   rm   c              3      K   | ]P}d          |         dk    d         j         |         o&d         j         |         od         j         |          V  QdS r  r  r  r  r  Nmaskr   r   rt  s     rJ   r   z.test_grid_search_cv_results.<locals>.<genexpr>           n%a(H44 y!&q) 7=).q17~.3A66 5444 rM   c              3      K   | ]Q}d          |         dk    d         j         |          o&d         j         |          od         j         |         V  RdS r  r,  r  r  r  Nr  r  s     rJ   r   z.test_grid_search_cv_results.<locals>.<genexpr>           n%a(E11 9%*1-- 3}-21553>*/2 2111 rM   )r   r  r%   r,   rV   r   r  r  r  rO   r   )
rT   r   r  n_grid_pointsrm   r  r  n_candidatesr9  rt  s
            @rJ   test_grid_search_cv_resultsr    s-   KKKDAqHM "g(	
 	
 	
 	 q6		
 	
 	
F  LJJ" !L(v$  F JJq!#Jz+,122222RRRRZRRRRRR         !Z@@@*j*lKKK#Jv)(344L     |$$             |$$         rM   c                  8  
 t          ddd          \  } }d}d}dgt          d	          t          d
	          ddgddgdg}d}d}|}t          t                      |||d          }|                    | |           |j        
t          |||           t          
|||           t          |j        d                   }	t          
fdt          |	          D                       sJ t          
fdt          |	          D                       sJ d S )Nr  r   rn  r%  r      r,  r   )scaler   r  r  r   r  r  r  T)r   r   r  r  rm   c              3      K   | ]P}d          |         dk    d         j         |         o&d         j         |         od         j         |          V  QdS r  r  r  s     rJ   r   z0test_random_search_cv_results.<locals>.<genexpr>  r  rM   c              3      K   | ]Q}d          |         dk    d         j         |          o&d         j         |          od         j         |         V  RdS r  r  r  s     rJ   r   z0test_random_search_cv_results.<locals>.<genexpr>  r  rM   )r   r   r&   r,   rV   r   r  r  rO   r  r   )rT   r   r  n_search_iterrm   r  r  r  r9  r  rt  s             @rJ   test_random_search_cv_resultsr%    s   KKKDAqHM 7R5s;K;K;KLL81v..F LJJ" F"  F JJq!#J Z@@@*j*fEEEv)(344L     |$$             |$$         rM   zSearchCV, specialized_paramsr   r   )r  r   c                    t          ddgddgddgddggdddd          \  }}t          j        |j        d         t                    }d|t          j        |dk              d         d d d         <   d|t          j        |dk              d         d d d         <   || g| |gg}t                      |d	d
} | di ||                    ||           t          j        fdt          j
                  D                       }j        d         d         }j        d         d         }	t          j        fdt          j
                  D                       }
j        d         d         }j        d         d         }j        d         d         dk    sJ t          |ddg           t          |
ddg           |t          j        t          j        |                    k    sJ |	t          j        t          j        |                    k    sJ |t          j        d          k    sJ |t          j        d          k    sJ d S )Nr   rd   r   FP   )r   r   r  shuffler   r  r   T)r  r   r  c                 <    g | ]}j         d |z           d         S )split%d_test_scorer   r   r   sr9  s     rJ   r   z+test_search_default_iid.<locals>.<listcomp>?  s<     	
 	
 	
 3a78;	
 	
 	
rM   rq  r  c                 <    g | ]}j         d |z           d         S )zsplit%d_train_scorer   r,  r-  s     rJ   r   z+test_search_default_iid.<locals>.<listcomp>H  s<     	
 	
 	
 4q89!<	
 	
 	
rM   r  r  r  gUUUUUU?rw   )r   rP   r   rZ   boolwherer,   rV   r   r   	n_splits_r   r   r   approxr?  std)r   specialized_paramsrT   r   r  r   common_paramstest_cv_scores	test_meantest_stdtrain_cv_scores
train_mean	train_stdr9  s                @rJ   test_search_default_iidr=    s    Q!Q!Q!Q0  DAq 7171:T***D%&D!q&		!	SSqS	!"%&D!q&		!	SSqS	!" $-4%	'B"%%%rNNMX<<<);<<F
JJq!X	
 	
 	
 	
6+,,	
 	
 	
 N "#45a8I!"23A6Hh	
 	
 	
 	
6+,,	
 	
 	
 O #$67:J"#45a8Ii(+q0000NQ	N333OaV,,,bgn&=&=>>>>>>v}RVN%;%;<<<<<< q))))))a((((((((rM   c                     t          ddd          \  } }d}t          dgddgd	dg
          t          dgddg          g}g }t          t                    t          t                    dddfD ]M}t          t                      |||d          }|                    | |           |                    |           Nt          |  d S )Nr  r   rn  r%  r   r,  rd   r   r   r  r  r   r  r   r  r   r  F)r   r   r   r  )
r   r  r7   r6   r5   r%   r,   rV   r  *compare_cv_results_multimetric_with_single)rT   r   r  rm   grid_searchesr   r   s          rJ   'test_grid_search_cv_results_multimetricrB  ^  s   KKKDAqH "g(	
 	
 	
 	 q6		
 	
 	
F  M 00K<U<UVV 	* 	*
 #EEh67%
 
 
 	1[)))).>>>>rM   c                     t          ddd          \  } }d}d}t          t          j        ddd          t          j        d	d
dd                    }dD ]}g }dD ]t}|rd}t	          |t
                    rdn|}nd}t          |d          }	t          |	|||||d
          }
|
                    | |           |	                    |
           ut          |  t          |d
         |d         |           d S )Nr  r   rn  r%  r   r   rd   r   r   )base)r   r.  TF)r?  r   r  Tr   F)probabilityr   )r   r   r  r   r  r   )r   r  rP   logspacer   r   r,   r&   rV   r  r@  )compare_refit_methods_when_refit_with_acc)rT   r   r  r$  rm   r  random_searchesr   rH  r   r  s              rJ   )test_random_search_cv_results_multimetricrL    sO   KKKDAqHM BKAq))RAC1P1P1PQQQF 
 
E 	2 	2G $"&0%&@&@K

e#+B???C.$$*  M a###""=11112ODD1A 2E	
 	
 	
 	
/
 
rM   c                    | j         sJ t          t          | j                  d           | j        d |j                                        D             }|                    d |j                                        D                        t          fddD                       sJ t          j	        
                    d                                 D             d |                                D                        dS )	zCompare multi-metric cv_results with the ensemble of multiple
    single metric cv_results from single metric grid/random searchr?  c                 B    i | ]\  }}t          j        d d|          |S )_score$	_accuracyr7  subr   r  vs      rJ   
<dictcomp>z>compare_cv_results_multimetric_with_single.<locals>.<dictcomp>  s9       15Ay+q))1  rM   c                 B    i | ]\  }}t          j        d d|          |S )rO  _recallrQ  rS  s      rJ   rU  z>compare_cv_results_multimetric_with_single.<locals>.<dictcomp>  s,    WWW1	9a	(	(!WWWrM   c              3   T   K   | ]"}t          j        |         d k              V  #dS )rd   N)rP   r  )r   r  cv_results_multis     rJ   r   z=compare_cv_results_multimetric_with_single.<locals>.<genexpr>  sL       	
 	
 F#A&!+,,	
 	
 	
 	
 	
 	
rM   )r
  r  r  r	  c                 D    i | ]\  }}|                     d           ||S _timeendswithrS  s      rJ   rU  z>compare_cv_results_multimetric_with_single.<locals>.<dictcomp>  s/    NNN$!Q!**W:M:MNANNNrM   c                 D    i | ]\  }}|                     d           ||S r[  r]  rS  s      rJ   rU  z>compare_cv_results_multimetric_with_single.<locals>.<dictcomp>  s/    PPP$!QAJJw<O<OPAPPPrM   N)r  r
   r   r  r   r   updater  rP   testingassert_equal)search_multi
search_acc
search_reccv_results_acc_recrY  s       @rJ   r@  r@    sI    $$$$vl2335KLLL#/ 9C9O9U9U9W9W   WW
8N8T8T8V8VWWW   	
 	
 	
 	
	
 	
 	

 
 
 
 
 JNN*0022NNNPP,2244PPP    rM   c           	         |j         |k    sJ |r| j         dk    sJ n| j         rJ dS t          ddd          \  }}dD ]@}t           t          | |          |           t          ||          |                     At          |                     ||          |                    ||                     dD ]&}t          | |          t          ||          k    sJ 'dS )	zDCompare refit multi-metric search methods with single metric methodsr   Nr   r   rn  r%  )r\   rt   ru   )r  rS  r   )r  r   r   r   rg   )rc  rd  r  rT   r   methodru  s          rJ   rJ  rJ    s,   u$$$$ !Z/////%%%%CCCDAqC 
 
)GL&))!,,.Igj&.I.I!.L.L	
 	
 	
 	
 **1a00*2B2B1a2H2HIII= F F|S))WZ-E-EEEEEEF FrM   	max_depthr   )r  r  )r  r   c                     t          ddd          \  }}|                     ||           d}t          j        t          |          5  |                     |           d d d            d S # 1 swxY w Y   d S )Nr   r   rn  r%  z@'DecisionTreeClassifier' object has no attribute 'score_samples'r   )r   rV   r   r   r{   score_samples)r~  rT   r   r   s       rJ   "test_search_cv_score_samples_errorrl    s     CCCDAqMM!Q QG	~W	5	5	5 # #"""# # # # # # # # # # # # # # # # # #s   	A,,A03A0T)noveltyn_neighborsr  )r  r  r   )r  r   r   c           	         t           j                            d          }d}d}t          ||z            }||z
  }t	          |dddgddggdd          d         }t          j        ||                    dd	|df
          gd          }t          j        dg|z            }d|| d <   |                     ||           t          | 
                    |          | j        
                    |                     d S )Nrn  r  g333333?r   r   r   )r   r&  r   r  r   ir   )lowhighsize)axisrd   r~   )rP   r  r	  intr   concatenater   r   rV   r   rk  r   )r~  r  r   outliers_fraction
n_outliers	n_inliersrT   y_trues           rJ   #test_search_cv_score_samples_methodrz    s.   " )


#
#CI&233JJ&I 	Q!Q 	 	 	 		A 	3;;2AZO;LLMTUVVVA XqcIo&&FFJ;<< MM!V ""I$=$K$KA$N$N    rM   c                     t          dd          \  } }dg di}t          t                      |d          }t          t                      d|d	          }||fD ]}|                    | |           |j        }t          |d
         d         |d
         d                    t          |d         d         |d         d                    t          j        |d
         d         |d
         d                   rJ t          j        |d         d         |d         d                   rJ t          |j        d         g d           d S )Nr  rn  r  r   )rd   gjt?MbP?T)r   r  r   )r   r  r  rq  r   rd   r  r   r  )rd   rd   r   )	r   r%   r,   r&   rV   r   r   rP   allclose)rT   r   r   r   r  r9  rt  s          rJ   (test_search_cv_results_rank_tie_breakingr~  &  s   444DAq ((()JsuuPTUUUK&aZD  M . N N

1a'
 	()!,j9J.KA.N	
 	
 	
 	)*1-z:L/Ma/P	
 	
 	
 ;()!,j9J.KA.N
 
 	
 	
 	
 ;)*1-z:L/Ma/P
 
 	
 	
 	
 	F./@A999MMMM'N NrM   c                     dgdgdgdgdggg d}} t                      t                      f}ddd gi}t                      }|D ]E}t          |||	                              | |          }t          |j        d
         dd g           Fd S )Nrd   r   r   r   r   )r   r   r   r   rd   r   r   r   param_random_state)r-   r.   r   r%   rV   r
   r   )rT   r   
estimatorsest_parametersr   estr   s          rJ   !test_search_cv_results_none_paramr  H  s    C!qcA3$oooqA'))+A+C+CDJ$q$i0N	B U U"
 
 
 #a))	 	
 	;23GH1d)TTTTU UrM   c                     t          d          } dgdgdgdgg}g d}t          | dddgidd	          }t          | dddgiddd
          }||fD ]}|                    ||           dD ]L}t	          j        |j        |         dk              sJ t	          j        |j        |         dk               sJ MdD ]Y}|j        |         d         dk    sJ |j        |         d         dk    sJ t	          j        |j        |         dk               sJ Zt          |d          sJ t          |j	        t                    sJ |j	        dk    sJ d S )Nr   r   rd   r   r   r   )r   rd   rd   r   r   r   )r   r   r   )r  r	  )r
  r  rf   refit_time_)r+   r%   r&   rV   rP   r  r   r   r   r  float)svcrT   r   r  rsr9  ru  s          rJ   test_search_cv_timingr  W  s   

#
#
#C 	
 	
 	
 	
	A 	A	cC!Q=QA	>	>	>B	C#1v1!A	N	N	NBr( ' '

1a4 	7 	7C 6&,S1Q6777776&,S1A56666668 	7 	7C%c*1-2222%c*1-44446&,S1A5666666v}-----&,e44444!Q&&&&&' 'rM   c            
      .   d} t          d          }t          dd          \  }}g d}dD ]}t          |d|i|| 	                              ||          j        }t          |                                          }d
t          d t          |           D                       z   }t          t          j        ||                    sJ t          |           }	j        } t          |          D ]\  }
|                    |
           t          j        fdt          |           D                       }t          |	                    ||                    D ]\  }\  }}|                    ||         ||                    |dk    r0t%          ||         |                    ||                             }n7|dk    r1|                    ||                   }t+          ||         |          }t-          |||                    d S )Nr   r   r   r   r   r   )rE  r   r   )r   r   )rq  r  c              3       K   | ]	}d |z  V  
dS )r+  Nrw   )r   cv_is     rJ   r   z9test_grid_search_correct_score_results.<locals>.<genexpr>  sA       G
 G
,0 4'G
 G
 G
 G
 G
 G
rM   r  )r   c                 <    g | ]}j         d |z                    S r+  r,  )r   r.  candidate_ir   s     rJ   r   z:test_grid_search_correct_score_results.<locals>.<listcomp>  s<         +,@1,DEkR  rM   rE  r   )r+   r   r%   rV   r   r   rr  r   r   r  rP   in1dr   r2  	enumeratern   r   splitr4   r\   rv   r8   r   )r  r   rT   r   r   rg   rt  result_keysexpected_keysr   r   	cv_scoresr   traintestcorrect_scoredecr  r   s                    @@rJ   &test_grid_search_correct_score_resultsr    sZ   H

#
#
#C1a000DAq	B" A A"3b	5XNNN __Q**6
 :??,,--> G
 G
49(OOG
 G
 G
 B
 B
 
 27=+6677777h///('mm 	A 	ANKNNQN    "8__   I %.bhhq!nn$=$= A A =E4%!E(+++D==$,QtWckk!D'6J6J$K$KMMi''//$88C$1!D'3$?$?M#M9Q<@@@@A	AA ArM   c                     t                      } t          | dg didd          }|                    t          t                     t          j        t          j        |                    }t          |	                    t                    |	                    t                               t          | dg diddd          }|                    t          t                     t          j        t          j        |                    }t          |	                    t                    |	                    t                               d S )NrH   r   Tr   r  )r  r   r   )rD   r%   rV   rT   r   pickleloadsdumpsr   r\   r&   )r   r   grid_search_pickledr  random_search_pickleds        rJ   test_pickler    s    


Cs[)))$<DQOOOKOOAq ,v|K'@'@AAk11!446I6Q6QRS6T6TUUU&k999%T!  M a"Lm)D)DEEa  "7"?"?"B"B    rM   c            	         t          dd          \  } }dg di}t                      }t          d          t          d          g}|D ]}t	          |||          }|                    | |           |j        d         }t          t          |                    D ]} |j	        di ||          t          |                    | |                    D ]p\  }	\  }
}|                    | |
         ||
                    |                    | |         ||                   }t          ||j        d	|	z           |                    q|D ]}t          |||d
          }|                    | |           |j        d         }t          t          |                    D ]} |j	        di ||          t          |                    | |                    D ]p\  }	\  }
}|                    | |
         ||
                    |                    | |         ||                   }t          ||j        d	|	z           |                    qd S )NTr   )return_indicatorr   ri  )rd   r   r   r   r   r   rm   r+  r   )r   r   rw   )r   r   r-   r.   r%   rV   r   r   rO   rn   r  r  rg   r   r&   )rT   r   r  r   r  r  r   
res_paramscand_ir   r  r  r  r  s                 rJ   &test_grid_search_with_multioutput_datar    s    *4aPPPDAq!<<<0N	B 	1---A...J   "32>>>1 ,X6
C
OO,, 		 		FCN00Z/000$-bhhq!nn$=$=   =E4%!E(+++ #		!D'1T7 ; ;#!+,@1,DEfM   		   *32aPPP!Q".x8
C
OO,, 		 		FCN00Z/000$-bhhq!nn$=$=   =E4%!E(+++ #		!D'1T7 ; ;#!!-.BQ.FGO   			 rM   c                      t          j        d                              dd          } g d}t          d          }t	          |i d                              | |          }t          |d	          rJ d S )
Nr[  r   r~   r   r   rd   rd   rd   F)rH  r   r   rt   )rP   r   r   r,   r%   rV   r   rT   r   r   r  s       rJ   test_predict_proba_disabledr    sx    
	"a$$AA
%
 
 
 C	c2!	$	$	$	(	(A	.	.Br?+++++++rM   c                  d   t          j        dt           j                                      dd          } t           j        | dd d f<   g d}t          dt          dt           j        	          fd
t                      fg          }t          |dg did          	                    | |           d S )Nr[  r)  r   r~   r   r  imputerr?  )strategymissing_values
classifierclassifier__foo_paramr   r   )
rP   r   r  r   nanr=   r<   rD   r%   rV   )rT   r   r   s      rJ   test_grid_search_allows_nansr    s    
	"BJ'''//266AfAadGAvbfMMMN>++,	
	 	A ,iii8Q???CCAqIIIIIrM   c                   4    e Zd ZdZdZddZddZd Zd	dZdS )
FailingClassifierz,Classifier that raises a ValueError on fit()r   Nc                     || _         d S rF   r`  rb  s     rJ   rK   zFailingClassifier.__init__  rL   rM   c                 N    | j         t          j        k    rt          d          d S )Nz%Failing classifier failed as required)ra  r  FAILING_PARAMETERr   re  s      rJ   rV   zFailingClassifier.fit  s+    >.@@@DEEE A@rM   c                 @    t          j        |j        d                   S rX   rg  r^   s     rJ   r\   zFailingClassifier.predict  rh  rM   c                     dS )Nrf   rw   rS   s      rJ   rg   zFailingClassifier.score  s    srM   rF   ro   )	rp   rq   rr   rs   r  rK   rV   r\   rg   rw   rM   rJ   r  r    so        66# # # #F F F F$ $ $     rM   r  c            	      (   t          ddd          \  } }t                      }t          |dg digddd	
          t          j        dt          j                  }t          j        t          |          5  	                    | |           d d d            n# 1 swxY w Y   t          j        d                   }fdt          fdt          |          D                       sJ t          |dg digddt          d          
          t          j        dt          j                  }t          j        t          |          5  	                    | |           d d d            n# 1 swxY w Y   t          j        d                   }t          fdt          |          D                       sJ j        d         }|d         dk    r|d         dk    sJ ||j                 dk    sJ j        |j        k    sJ d S )Nr[  r   r   r%  ra  r   rd   r   r   Frf   r   r  r   z5 fits failed.+total of 15.+The score on these train-test partitions for these parameters will be set to 0\.0.+5 fits failed with the following error.+ValueError.+Failing classifier failed as requiredflagsr   rm   c                 l     t          j         fdt          j                  D                       S )Nc                 <    g | ]}j         d |z                    S r  r,  )r   r.  r  r   s     rJ   r   zPtest_grid_search_failing_classifier.<locals>.get_cand_scores.<locals>.<listcomp>3  s+    VVVQR^0145a8VVVrM   )rP   r   r   r2  )r   r  s   `rJ   get_cand_scoresz<test_grid_search_failing_classifier.<locals>.get_cand_scores1  s<    xVVVVV%BUBUVVV
 
 	
rM   c              3      K   | ]F}j         d          |         t          j        k    #t          j         |          dk              V  GdS )param_parameterrf   N)r   r  r  rP   r  r   r  r  r  s     rJ   r   z6test_grid_search_failing_classifier.<locals>.<genexpr>7  sp       	
 	
~/08 23 3 F??6**c1223 3 3 3	
 	
rM   r  z5 fits failed.+total of 15.+The score on these train-test partitions for these parameters will be set to nan.+5 fits failed with the following error.+ValueError.+Failing classifier failed as requiredc              3      K   | ]T}j         d          |         t          j        k    #t          j        t          j         |                              V  UdS )r  N)r   r  r  rP   r  isnanr  s     rJ   r   z6test_grid_search_failing_classifier.<locals>.<genexpr>P  su        >+,V4./ / 	rx//0011/ / / / rM   r  r   rd   r   )r   r  r%   r7  compileDOTALLr   warnsr*   rV   rO   r   r  r   r  r  r  )rT   r   r   warning_messager  ranksr  r  s         @@rJ   #test_grid_search_failing_classifierr    s    KKKDAq


C 

yyy	!"
 
 
B j	 i  O 
&o	>	>	>  
q!              r~h/00L
 
 
 
 

 	
 	
 	
 	
 	
--	
 	
 	
     

yyy	!"%LL
 
 
B j	 i  O 
&o	>	>	>  
q!              r~h/00L     L))        N,-E 8q==U1X]]]]&'1,,,,>S2222222s$   :BB!$B!E66E:=E:c                  f   t          ddd          \  } }t                      }t          |dt          j        gdz  igd          }t	          j        d	t          j        
          }t          j        t          |          5  |
                    | |           d d d            d S # 1 swxY w Y   d S )Nr[  r   r   r%  ra  r   rf   r   zrAll the 15 fits failed.+15 fits failed with the following error.+ValueError.+Failing classifier failed as requiredr  r   r   r  r%   r  r7  r  r  r   r   r   rV   )rT   r   r   r  r  s        rJ   )test_grid_search_classifier_all_fits_failr  `  s   KKKDAq


C	
);<q@	AB
 
 
B j	 i	  O 
z	9	9	9  
q!                 s   B&&B*-B*c                     t          ddd          \  } }t                      }t          |dg digddd	
          }t          j        t
                    5  |                    | |           d d d            d S # 1 swxY w Y   d S )Nr[  r   r   r%  ra  r  r   Fr   r  )r   r  r%   r   r   r   rV   r  s       rJ   )test_grid_search_failing_classifier_raiser  u  s     KKKDAq


C 

yyy	!"
 
 
B 
z	"	"  
q!                 s   A::A>A>c                  x   ddgg dddddgig} t          | d	          }d}d
}d|||fz  }t          j        t          |          5  t	          |           d d d            n# 1 swxY w Y   t          | d
	          }t	          |          }t          |          d
k    sJ t          |           D ]}||v sJ 	t          t          | d	                    d
k    sJ t          d          t          d          t          d          d} t          | dd          }t	          |          }t          |          dk    sJ d |D             }t          t          |                    dk    sJ t          d          g dd}t          |d	          }t	          |          }t          |          dk    sJ d S )Nr   rd   )abc)firstsecondthirdtwovaluesr  r   r  z}The total space of parameters %d is smaller than n_iter=%d. Running %d iterations. For exhaustive searches, use GridSearchCV.r   i  r   c   rn  )r   r   c                 B    g | ]}d |d         |d         |d         fz  S )z	a%db%dc%dr  r  r  rw   r   s     rJ   r   z7test_parameters_sampler_replacement.<locals>.<listcomp>  s0    PPP1qvqvqv&>>PPPrM   r   r  )
r(   r   r  UserWarningr   rO   r'   r   r   r   )	rm   r  r   	grid_sizeexpected_warningr  r  hashable_samplesparams_distributions	            rJ   #test_parameters_sampler_replacementr    sG    a&OOO44	5(#$F va000GFI	14=vy3Q	R 
 
k)9	:	:	:  W               va000G7mmGw<<1'' ! !     t44455:::: 99599599==FvbrBBBG7mmGw<<2PPPPPs#$$%%++++ %.cNNoooNN21===G7mmGw<<1s   A$$A(+A(c                  
   ddgi} t          j        d                              dd          }g d}t          t	          d          | d	
          }t          |d          rJ |                    ||           |                    |           |                    |           ddgi} t          t	          d          | d	
          }t          |d          rJ |                    ||           t          |d          rJ d S )Nlosslog_loss   r   r~   r   r   r   rd   rd   rd   hinge)r  r   )r  r   r   rt   )	rP   r   r   r%   r?   r   rV   rt   ru   )r   rT   r   r   s       rJ   #test_stochastic_gradient_loss_paramr    s0    	J 		"a$$AA
W---*  C sO,,,,,GGAqMMMa!
 		J W---*  C sO,,,,,GGAqMMMsO,,,,,,,rM   c                      t          j        d                              dd          } g d}t          d          }t	          |dddgid	
          }|                    | |           d S )Nr   r~   r  r   r   r   r   r   r   )r   r   )rP   r   r   r+   r%   rV   r  s       rJ   %test_search_train_scores_set_to_falser    sl    
	!Q##AA

#
#
#C	csS#J&7A	>	>	>BFF1aLLLLLrM   c            	        	
 d} d}t          | d          \  }}t          t          d          dg dit          ||           d	
          		                    ||           t          t          d          dg dit          |          d	
          }|                    ||           t          t          |d	d                              ||          t                    sJ t          t          d          dg dit          |d	d                              ||          d	
          }|                    ||           t          t          d          dg dit          |d	d          d	
          }|                    ||           d }t          j
                             ||j                   ||j                             t          j
                            d 	j                                        D             d |j                                        D                        t          t          d          dg dit          |d	          d	
          		                    ||           dD ]ni }t          d          D ]"
	
fdt          d          D             |
<   #t          |d         |d                    t          |d         |d                    od S )Nr   r   r   r  r   r   )r   r   g333333?)r  r   T)r   r   r  r  )r  r(  r   c                 :    dD ]}|                      |           | S )N)r  r	  r
  r  )pop)rt  ru  s     rJ   _pop_time_keysz>test_grid_search_cv_splits_consistency.<locals>._pop_time_keys  s0    
 	  	 C NN3rM   c                 D    i | ]\  }}|                     d           ||S r[  r]  rS  s      rJ   rU  z:test_grid_search_cv_splits_consistency.<locals>.<dictcomp>  s/    LLL$!Q

78K8KLALLLrM   c                 D    i | ]\  }}|                     d           ||S r[  r]  rS  s      rJ   rU  z:test_grid_search_cv_splits_consistency.<locals>.<dictcomp>  s/    MMM$!QG9L9LMAMMMrM   )r   r   r   r   )r  r(  )r  r  r   c                 @    g | ]}j         d |fz                    S )zsplit%d_%s_scorer,  )r   r.  r  param_i
score_types     rJ   r   z:test_grid_search_cv_splits_consistency.<locals>.<listcomp>1  s?     ) ) ) 1Q
OCDWM) ) )rM   rd   r   r   )r   r%   r+   rB   rV   r   r   r  r	   rP   ra  rb  r   r   r   r   )r   r  rT   r   gs2gs3gs4r  per_param_scoresr  r  r  s            @@@rJ   &test_grid_search_cv_splits_consistencyr    s   IHCCCDAq	q!!!)H	BBB	
 
 
B FF1aLLL
q!!!)(###	  C GGAqMMM xA>>>DDQJJ     q!!!)(DqAAAGG1MM	  C GGAqMMM
q!!!)(DqAAA	  C GGAqMMM   Js'')H)H   JLL"...00LLLMM#///11MMM   
q!!!---.(D111	
 
 
B FF1aLLL ( 	L 	L
Qxx 	 	G) ) ) ) ) )q) ) )W%%
 	""21"57G7JKKK!"21"57G7JKKKK	L 	LrM   c                     t                      } t          | dg didd          }|                    t          t                     |                    |                    t                              }t          t          |           d S )NrH   r   r   r   )rD   r%   rV   rT   r   rb   r_   r
   )r   r   X_round_trips      rJ   +test_transform_inverse_transform_round_tripr  :  sw    


Cs[)))$<ANNNKOOAq001F1Fq1I1IJJLq,'''''rM   c                    	 d 	fd G fddt                     } t          d          t          ddd	          \  	 | d
                              	          } dddgidddgig          }|j        } ||           t          |          D ]]}|d                                         rA|dd          dk    r3|dvr/t          ||          t          ||          k    sJ d|z              ^d S )Nc                    |j         }t          |                                           t          |          k    sJ | D ]}|                    d          svt	          j        | |                   | |<   | |         j        j        dk    r"t          ||         | |         d|z              lt          ||         | |         d|z              d S )Nr\  Oz	Checking )r   )
r   r   rr  r^  rP   
asanyarrayr  kindr
   r   )resultsgscvexp_resultsr  s       rJ   check_resultsz-test_custom_run_search.<locals>.check_resultsD  s    &gllnn%%)<)<<<<< 		Y 		YA::g&& Y]71:66
1:#(C//&#A
K!O     $KNGAJVWXXXX		Y 		YrM   c                 P    t          | d                                        S )NTr  )r%   rV   )r   rT   r   r   s    rJ   fit_gridz(test_custom_run_search.<locals>.fit_gridR  s(    CEEEII!QOOOrM   c                   ,     e Zd Z fdZfdZ xZS ).test_custom_run_search.<locals>.CustomSearchCVc                 <     t                      j        |fi | d S rF   superrK   rI   r  kwargs	__class__s      rJ   rK   z7test_custom_run_search.<locals>.CustomSearchCV.__init__V  (    EGGY11&11111rM   c           	           |ddiddig          } | dddgi                      |ddiddig          } | dddgidddgig                     d S )Nri  rd   r   min_samples_splitr   r   rw   )rI   evaluater  r  r
  s      rJ   _run_searchz:test_custom_run_search.<locals>.CustomSearchCV._run_searchY  s    ha 0;2BCDDGM'88[1a&,A#B#BCCCh!4a 8;NPR:STUUGM;A/2E2w1OPQQ    rM   )rp   rq   rr   rK   r  __classcell__)r  r  r
  s   @rJ   CustomSearchCVr  U  sW        	2 	2 	2 	2 	2	 	 	 	 	 	 	 	 	 	rM   r  r   r   r   r   )r   n_informativer   Tr	  ri  rd   r   r  r   r   r~   r  >   rR   r   r  r   zAttribute %s not equal)r)   r-   r   rV   r   dirislowerr   )
r  mycvr  r  attrrT   r  r   r
  r   s
        @@@@@rJ   test_custom_run_searchr  C  s   Y Y YP P P P P P P            Q
/
/
/CAANNNDAq>#$777;;AqAAD8kAq6*-@1b',JKLLDGM'4   D		 	 	GOO	RSS	S  PQ Q 4&&'$*=*====(4/ >==	 	rM   c                      G d dt                     }  | t                                                    t          t                      G d dt                     }t          j        t          d          5   |t                                                    t          t                     d d d            d S # 1 swxY w Y   d S )Nc                   &     e Zd Z fdZddZ xZS );test__custom_fit_no_run_search.<locals>.NoRunSearchSearchCVc                 <     t                      j        |fi | d S rF   r  r  s      rJ   rK   zDtest__custom_fit_no_run_search.<locals>.NoRunSearchSearchCV.__init__x  r  rM   Nc                     | S rF   rw   )rI   rT   r   r  
fit_paramss        rJ   rV   z?test__custom_fit_no_run_search.<locals>.NoRunSearchSearchCV.fit{  s    KrM   ro   )rp   rq   rr   rK   rV   r  r  s   @rJ   NoRunSearchSearchCVr"  w  sL        	2 	2 	2 	2 	2	 	 	 	 	 	 	 	rM   r'  c                        e Zd Z fdZ xZS )3test__custom_fit_no_run_search.<locals>.BadSearchCVc                 <     t                      j        |fi | d S rF   r  r  s      rJ   rK   z<test__custom_fit_no_run_search.<locals>.BadSearchCV.__init__  r  rM   )rp   rq   rr   rK   r  r&  s   @rJ   BadSearchCVr)    s8        	2 	2 	2 	2 	2 	2 	2 	2 	2rM   r+  z_run_search not implemented.r   )r)   r,   rV   rT   r   r   r   NotImplementedError)r'  r+  s     rJ   test__custom_fit_no_run_searchr-  v  s       l    ""1a(((2 2 2 2 2l 2 2 2 
*2P	Q	Q	Q % %CEEq!$$$% % % % % % % % % % % % % % % % % %s   76B::B>B>c                     t          d                              t                    } d | D              d}t          t	                      dg di| d          }t          j        t          d	
          5  |                    t          d |         t          d |                    d d d            d S # 1 swxY w Y   d S )Nr   r  c                     g | ]}|S rw   rw   )r   us     rJ   r   z0test_empty_cv_iterator_error.<locals>.<listcomp>  s    OOO1QOOOrM   r   r  r|  rp  r   r   r   n_jobszNNo fits were performed. Was the CV iterator empty\? Were there no candidates\?r   )
r   r  rT   r&   r>   r   r   r   rV   r   )r   
train_sizeridges      rJ   test_empty_cv_iterator_errorr6    s    
				 	 	#	#B OOOOO Juww2D2D2D(E"UVWWWE 
*
 
 
 2 2 			!KZK.!KZK.1112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s   51B33B7:B7c                  V    G d dt                     }  | d          }d}t          t                      dg di|d	          }t          j        t
          d
          5  |                    t          d |         t          d |                    d d d            d S # 1 swxY w Y   d S )Nc                       e Zd Zd ZdS ).test_random_search_bad_cv.<locals>.BrokenKFoldc                     dS )Nrd   rw   )rI   argskws      rJ   get_n_splitsz;test_random_search_bad_cv.<locals>.BrokenKFold.get_n_splits  s    1rM   N)rp   rq   rr   r=  rw   rM   rJ   BrokenKFoldr9    s#        	 	 	 	 	rM   r>  r   r  r   r  r1  r   r2  zXcv.split and cv.get_n_splits returned inconsistent results. Expected \d+ splits, got \d+r   )	r   r&   r>   r   r   r   rV   rT   r   )r>  r   r4  r5  s       rJ   test_random_search_bad_cvr?    s       e   
 
a	 	 	 BJuww2D2D2D(E"UVWWWE 

 
 
 2 2 			!KZK.!KZK.1112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2s    1BB"%B"r  F)r   r   r   r  r   c                    t          dd          \  }} G d d          } | t                      f |            d|d|}t          j        t                    5 }|                    ||           d d d            n# 1 swxY w Y   |rdd	gndg}t          |          t          |          k    sJ t          ||          D ]!\  }	}
d
|
 dt          |	j	                  v sJ "|j
        d                                         }t          j        |j
        d                   }t          |j
        d         |         |           t          j        |j
        d         |          |k               sJ d S )Nr   r   )r   r   c                       e Zd ZdZd Zd ZdS )Htest_searchcv_raise_warning_with_non_finite_score.<locals>.FailingScorerz1Scorer that will fail for some split but not all.c                     d| _         d S rX   )n_countsr|   s    rJ   rK   zQtest_searchcv_raise_warning_with_non_finite_score.<locals>.FailingScorer.__init__  s    DMMMrM   c                 Z    | xj         dz  c_         | j         dz  dk    rt          j        S dS )Nrd   r   r   )rD  rP   r  )rI   r  rT   r   s       rJ   __call__zQtest_searchcv_raise_warning_with_non_finite_score.<locals>.FailingScorer.__call__  s0    MMQMM}q A%%v1rM   N)rp   rq   rr   rs   rK   rF  rw   rM   rJ   FailingScorerrB    s8        ??	 	 		 	 	 	 	rM   rG  r   )r   r   r  r  r  zOne or more of the z scores are non-finiter  rq  )r   r.   r   r  r  rV   rO   zipstrmessager   maxrP   r  r
   r  )r   r5  r  rT   r   rG  r   warn_msgset_with_warningmsgdataset	last_ranknon_finite_masks                rJ   1test_searchcv_raise_warning_with_non_finite_scorerR    s   $ ;;;DAq
 
 
 
 
 
 
 
 8  -	 
  D 
k	"	" hA               -?L((VHx==C 0111111H&677 Y YWDWDDDCKHXHXXXXXX  !237799Iht/0ABCCOt'(9:?KYWWW6$"#456FG)STTTTTTTs   A??BBc                     d } t          ddd          \  }}t          d          }t          |ddd	gi| d
          }|                    ||           g d}|D ] }d                    |          |j        v sJ !|                    |          }t          ||          }|                    ||          t          j
        |d                   k    sJ d S )Nc                     |                      |          }t          ||          }|d         |d         |d         |d         dS )N)r   r   r   rd   )rd   r   )rd   rd   tnfpfntp)r\   r9   )r   rT   r   r@  cms        rJ   custom_scorerzAtest_callable_multimetric_confusion_matrix.<locals>.custom_scorer  sB    Qa((hbhbhbhOOOrM   (   r   rn  r%  r   r   r   rd   rX  rj  rV  zmean_test_{}rU  )r   r+   r%   rV   formatr   r\   r9   rg   r   r3  )	r\  rT   r   r  r9  score_namesnamer@  r[  s	            rJ   *test_callable_multimetric_confusion_matrixra    s   P P P
 KKKDAq

$
$
$C#c1XTRRRF
JJq!***K A A$$T**f.@@@@@@^^AF	!V	$	$B<<1r$x!8!8888888rM   c                     d } t          ddd          \  }}t          d          }t          |ddd	gi| d
          }t          |ddd	gid
dgd
          }|                    ||           |                    ||           |j        t          j        |j                  k    sJ |j        |j        k    sJ |                    ||          t          j        |                    ||                    k    sJ d S )Nc                 n    |                      |          }t          ||          t          ||          dS )N)r  r   )r\   r5   r6   r  rT   r   r@  s       rJ   r\  zHtest_callable_multimetric_same_as_list_of_strings.<locals>.custom_scorer  s:    Q"1f--&q&11
 
 	
rM   r]  r   rn  r%  r   r   r   rd   r  rj  r   	r   r+   r%   rV   rS  r   r3  r  rg   )r\  rT   r   r  search_callable
search_strs         rJ   1test_callable_multimetric_same_as_list_of_stringsrh    s1   
 
 
 KKKDAq

$
$
$C"cC8_m8  O cC8_x&<H  J 1NN1a&&-
8N*O*OOOOO&**@@@@@  A&&&-
8H8HA8N8N*O*OOOOOOOrM   c                  F   d } t          ddd          \  }}t          d          }t          |ddd	gi| d
          }t          |ddd	gidd          }t          |ddd	gidgd          }|                    ||           |                    ||           |                    ||           |j        t          j        |j                  k    sJ |j        |j        k    sJ |                    ||          t          j        |                    ||                    k    sJ |j        t          j        |j                  k    sJ |j        |j        k    sJ |                    ||          t          j        |                    ||                    k    sJ d S )Nc                 L    |                      |          }t          ||          S rF   )r\   r5   rd  s       rJ   r\  zHtest_callable_single_metric_same_as_single_string.<locals>.custom_scorer'  s!    QAv&&&rM   r]  r   rn  r%  r   r   r   rd   Trj  r  re  )r\  rT   r   r  rf  rg  search_list_strs          rJ   1test_callable_single_metric_same_as_single_stringrl  %  s   ' ' ' KKKDAq

$
$
$C"cC8_m4  O cC#q?HHUUUJ"cC8_xj  O 1NN1a1&&-
8N*O*OOOOO&**@@@@@  A&&&-
8H8HA8N8N*O*OOOOO&&-
8N*O*OOOOO&**@@@@@  A&&&-
8H8HA8N8N*O*OOOOOOOrM   c                     d } t          ddd          \  }}t          t          d          ddd	gi| d
          }d}t          j        t
          |          5  |                    ||           d d d            d S # 1 swxY w Y   d S )Nc                 
    ddiS )Nbad_namerd   rw   r  rT   r   s      rJ   
bad_scorerzBtest_callable_multimetric_error_on_invalid_key.<locals>.bad_scorerC  s    ArM   r]  r   rn  r%  r   r   r   rd   	good_namerj  z`For multi-metric scoring, the parameter refit must be set to a scorer key or a callable to refitr   )r   r%   r+   r   r   r   rV   )rq  rT   r   r   rN  s        rJ   .test_callable_multimetric_error_on_invalid_keyrs  A  s       KKKDAq
r"""	sAh	  C	,  
z	-	-	-  1                 s   A??BBc                     d } t          ddd          \  }}t                      }t          |dg dig| dd	
          }t          j        dt          j                  }t          j        t          |          5  |	                    ||           d d d            n# 1 swxY w Y   t          |j        d         g d           d S )Nc                 
    ddiS Nr  rd   rw   rp  s      rJ   r\  zBtest_callable_multimetric_error_failing_clf.<locals>.custom_scorerY      qzrM   r[  r   r   r%  ra  r  Fr   r  zm5 fits failed.+total of 15.+The score on these train-test partitions for these parameters will be set to 0\.1r  r   mean_test_acc)rd   rd   r   )r   r  r%   r7  r  r  r   r  r*   rV   r   r   )r\  rT   r   r   r  r  s         rJ   +test_callable_multimetric_error_failing_clfry  V  s)      KKKDAq


C	
yyy	!"
 
 
B j	Ki  O
 
&o	>	>	>  
q!               BN?3[[[AAAAAs   ;BB"%B"c                  z   d } t          ddd          \  }}t                      }t          |dt          j        gdz  ig| dd	
          }d}t	          j        d| t          j                  }t          j        t          |          5  |
                    ||           d d d            d S # 1 swxY w Y   d S )Nc                 
    ddiS rv  rw   rp  s      rJ   r\  zBtest_callable_multimetric_clf_all_fits_fail.<locals>.custom_scorert  rw  rM   r[  r   r   r%  ra  r   Fr   r  z1ValueError: Failing classifier failed as requiredz5All the 15 fits failed.+your model is misconfigured.+r  r   r  )r\  rT   r   r   r  individual_fit_error_messager   s          rJ   +test_callable_multimetric_clf_all_fits_failr}  r  s/      KKKDAq


C	
);<q@	AB
 
 
B $W J	*'	* 	*i  M 
z	7	7	7  
q!                 s   B00B47B4c                  p   d} t          |           \  }}t                      }dddgi}t          ||          }t          ||d          }t	          |d          rJ t	          |d          rJ |                    ||           |                    ||           |j        | k    sJ |j        | k    sJ d S )Nr   )r&  max_iterr   rd   r   n_features_in_)r   rA   r%   r&   r   rV   r  )r&  rT   r   gbdtr   r  r  s          rJ   test_n_features_inr    s     J*555DAq)++Dq!f%J	dJ	'	'B	D*Q	7	7	7Br+,,,,,r+,,,,,FF1aLLLFF1aLLL
****
******rM   pairwisec                       G  fddt                     } |            }d}t          |ddgi          } |                                d         k    s
J |            dS )z
    Test implementation of BaseSearchCV has the pairwise tag
    which matches the pairwise tag of its estimator.
    This test make sure pairwise tag is delegated to the base estimator.

    Non-regression test for issue #13920.
    c                       e Zd Z fdZdS )Stest_search_cv_pairwise_property_delegated_to_base_estimator.<locals>.TestEstimatorc                     diS Nr  rw   rI   r  s    rJ   
_more_tagsz^test_search_cv_pairwise_property_delegated_to_base_estimator.<locals>.TestEstimator._more_tags  s    ))rM   N)rp   rq   rr   r  r  s   rJ   TestEstimatorr    s.        	* 	* 	* 	* 	* 	* 	*rM   r  z.BaseSearchCV pairwise tag must match estimatorrn  r   r  N)r   r%   	_get_tags)r  r  r  attr_messager   s   `    rJ   <test_search_cv_pairwise_property_delegated_to_base_estimatorr    s    * * * * * * * * * * -//CCL	cMB40	1	1Br||~~j1111<11111rM   c                       G d dt                     }  |             }d}dD ]S}|                    |           t          |ddgi          }||                                d         k    s
J |            Td	S )
z
    Test implementation of BaseSearchCV has the pairwise property
    which matches the pairwise tag of its estimator.
    This test make sure pairwise tag is delegated to the base estimator.

    Non-regression test for issue #13920.
    c                       e Zd ZddZd ZdS )Xtest_search_cv__pairwise_property_delegated_to_base_estimator.<locals>.EstimatorPairwiseTc                     || _         d S rF   r  r  s     rJ   rK   zatest_search_cv__pairwise_property_delegated_to_base_estimator.<locals>.EstimatorPairwise.__init__  s    $DMMMrM   c                     d| j         iS r  r  r|   s    rJ   r  zctest_search_cv__pairwise_property_delegated_to_base_estimator.<locals>.EstimatorPairwise._more_tags  s    ..rM   N)T)rp   rq   rr   rK   r  rw   rM   rJ   EstimatorPairwiser    s7        	% 	% 	% 	%	/ 	/ 	/ 	/ 	/rM   r  z4BaseSearchCV _pairwise property must match estimatorrG  r  rn  r   r  N)r   rn   r%   r  )r  r  r  _pairwise_settingr   s        rJ   =test_search_cv__pairwise_property_delegated_to_base_estimatorr    s    / / / / /M / / / 


CIL* M M 1222#t455 BLLNN:$>>>>>>>>M MrM   c                     d} d}t          | d          \  }}ddgi}t                      }t          |||          }|                    ||           |                    |          }t          |          }t          d	          }t          |||          }|                    ||           |                    |          }	d
}
||	k                                    s
J |
            dS )z
    Test implementation of BaseSearchCV has the pairwise tag
    which matches the pairwise tag of its estimator.
    This test ensures the equivalence of 'precomputed'.

    Non-regression test for issue #13920.
    r  r   r   r  rn  r   r   rP  )metricz2GridSearchCV not identical with precomputed metricN)r   r2   r%   rV   r\   r;   r  )r   r  rT   r   grid_paramsr   r   preds_originalX_precomputedpreds_precomputedr  s              rJ   ;test_search_cv_pairwise_property_equivalence_of_precomputedr    s     IHCCCDAq 2$'K 
 
 C	c;8	4	4	4BFF1aLLLZZ]]N (**M
m
4
4
4C	c;8	4	4	4BFF=!

=11GL//4466DDDDDDDrM   zSearchCV, param_searchr  r   rp  r   c                      G d dt           t                    } |  |            |          }t          d          \  }}|                    ||d           |j        j        dk    sJ d S )Nc                   $    e Zd ZddZddZd ZdS ),test_scalar_fit_param.<locals>.TestEstimatorNc                     || _         d S rF   )r  )rI   r  s     rJ   rK   z5test_scalar_fit_param.<locals>.TestEstimator.__init__  s    DFFFrM   c                     || _         d S rF   )r_)rI   rT   r   rs       rJ   rV   z0test_scalar_fit_param.<locals>.TestEstimator.fit  s    DGGGrM   c                 F    t          j        t          |                    S )NrY   )rP   r   rO   r^   s     rJ   r\   z4test_scalar_fit_param.<locals>.TestEstimator.predict  s    83q66++++rM   rF   )rp   rq   rr   rK   rV   r\   rw   rM   rJ   r  r    sK        	 	 	 		 	 	 		, 	, 	, 	, 	,rM   r  rn  r   )r  )r   r   r   rV   r   r  )r   param_searchr  modelrT   r   s         rJ   test_scalar_fit_paramr    s    , , , , , , , , H]]__l33EB///DAq	IIabI #r))))))rM   r  c                     t          t          d          ddi\  }}}} G d dt                    }d } |  |            |          }||f|dd}	 |j        ||fi |	 d S )Nrn  r   r   c                   (     e Zd Z	 	 	 	 d fd	Z xZS )9test_scalar_fit_param_compat.<locals>._FitParamClassifierNc                     t                                          |||           |dk    sJ t          |          sJ t          |t                    sJ |d         j        dk    sJ |d         j        dk    sJ | S )N)sample_weightr   r   rd   )r  rV   callabler   r   ndim)rI   rT   r   r  tuple_of_arraysscalar_paramcallable_paramr  s          rJ   rV   z=test_scalar_fit_param_compat.<locals>._FitParamClassifier.fit	  s     GGKK1MK:::!####N+++++ ou55555"1%*a////"1%*a////KrM   )NNNN)rp   rq   rr   rV   r  r&  s   @rJ   _FitParamClassifierr  	  sK        
  	 	 	 	 	 	 	 	 	 	rM   r  c                      d S rF   rw   rw   rM   rJ   _fit_param_callablez9test_scalar_fit_param_compat.<locals>._fit_param_callable(	  s    rM   )r  r  r  )r   r   r?   rV   )
r   r  X_trainX_validrV  y_validr  r  r  r%  s
             rJ   test_scalar_fit_param_compatr  	  s     *:	"	-	-	-*<>* *&GWgw    m   (   H((**L99E $W-- J
 EIgw--*-----rM   z)ignore:The total space of parameters 4 is	Predictorc                 \   t           j                            d          }|                    dd          t          j        dgdz  dgdz  z             }}t          dt                      fd |            fg          }dd	gdd	gd
} | ||d          }|                    ||           |j        	                                |	                                k    sJ |
                    |          }t          |          rNt          |d           |                    ||          t          j        t!          ||                    k    sJ d S t#          ||                                           |                    ||          t          j        t'          ||                    k    sJ d S )Nr      r   r   rd   r[  transformer	predictorr   )transformer__parampredictor__paramar   r  )rP   r  r	  randnr   r=   r   rV   r   rr  r\   r   r
   rg   r   r3  r6   r   r?  r:   )	r   r  r  rT   r   r  rm   r9  r@  s	            rJ   1test_search_cv_using_minimal_compatible_estimatorr  <	  s    )


"
"C99RRXqcAgb&899qA
+--	.iikk0JK E
  !"gW F XeV999F
JJq!##%%6666^^AFV H61%%%||Aq!!V]>!V3L3L%M%MMMMMMM)))||Aq!!V]8Av3F3F%G%GGGGGGGrM   c                 d   t          dddd          \  }}t          d          }ddgi}t          ||d	d
d
|                              ||           |                                 j        }|rt          j        d|          }nt          j        d|          }t          |          d
k    sJ dS )zkCheck that search cv with verbose>2 shows the score for single
    metrics. non-regression test for #19658.r   r   r   r   r   r   r   r   r   r   )r   r   r   r  z$score=\(train=[\d\.]+, test=[\d.]+\)zscore=[\d\.]+N)	r   r+   r%   rV   
readouterroutr7  findallrO   )capsysr  rT   r   r   r   capturedr   s           rJ   test_search_cv_verbose_3r  [	  s     #TUVVVDAq

#
#
#C#<D-   
c!Qiii  ""&H 7
BHMM
+X66u::??????rM   )rs   collections.abcr   r   ior   	itertoolsr   r   	functoolsr   r  r   r  r	   r7  numpyrP   scipy.sparsesparser<  r   sklearn.utils._testingr
   r   r   r   r   r   r   r   sklearn.utils._mockingr   r   scipy.statsr   r   r   sklearn.baser   r   r   sklearn.datasetsr   r   r   sklearn.model_selectionr   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   sklearn.model_selection._searchr)   #sklearn.model_selection._validationr*   sklearn.svmr+   r,   sklearn.treer-   r.   sklearn.clusterr/   sklearn.neighborsr0   r1   r2   sklearn.metricsr4   r5   r6   r7   r8   r9   r:   sklearn.metrics.pairwiser;   sklearn.imputer<   sklearn.pipeliner=   sklearn.linear_modelr>   r?   r@   sklearn.ensemblerA   $sklearn.model_selection.tests.commonrB   rD   ry   r   rT   r   r   markparametrizer   r   r   r   r   r   r   r  r  r  r  r"  r*  r3  r5  r:  rC  rN  rY  r\  r^  rk  rw  r|  r  r  r  r  r  r  r  r  r  r  r  r  r%  r=  rB  rL  r@  rJ  rl  rz  r~  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r-  r6  r?  rR  ra  rh  rl  rs  ry  r}  r  r  r  r  r  r  filterwarningsr  r  rw   rM   rJ   <module>r     sr     + + + + + + + +       $ $ $ $ $ $ $ $        



       				           	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 E D D D D D D D 1 1 1 1 1 1 1 1 1 1 7 7 7 7 7 7 7 7 & & & & & & 0 0 0 0 0 0 ' ' ' ' ' ' ; ; ; ; ; ; 4 4 4 4 4 4 ) ) ) ) ) ) 3 3 3 3 3 3 : : : : : : 4 4 4 4 4 4 3 3 3 3 3 3 . . . . . . 5 5 5 5 5 5 0 0 0 0 0 0 6 6 6 6 6 6 1 1 1 1 1 1 4 4 4 4 4 4 8 8 8 8 8 8 @ @ @ @ @ @ & & & & & & & & . . . . . . / / / / / / " " " " " " + + + + + + 0 0 0 0 0 0 2 2 2 2 2 2 $ $ $ $ $ $ ( ( ( ( ( ( * * * * * * ' ' ' ' ' ' ) ) ) ) ) ) , , , , , , $ $ $ $ $ $ 8 8 8 8 8 8 ( ( ( ( ( ( % % % % % % G G G G G G G G G G ; ; ; ; ; ; @ @ @ @ @ @
$ $ $ $ $ $ $ $N    y    BHr2hR1a&1a&122BH\\\= = = ='':JSU2V2V2V"WXX&	
IKL1#,	I'JKAJF	
	    YX
"/ "/ "/J  2< < <$ l4F%GHH< < IH<$ ( ( (07 7 78  >0 0 02	/ 	/ 	/$ $ $N  F F F6 6 6% % %P  *!( !( !(H* * *:! ! !$ $ $ $ $} $ $ $ 
 
 
*+ *+ *+Z  0 *RG44'9<&HII  JI 540+ + +:/ / /*/ / // / / 3 3 3>7 7 78$ $ $"8 8 84B B B$V V VM M M`< < <~ "	sQGn56	cAr7^qQQR 5) 5) 5)p ?  ?  ?F!
 !
 !
H# # #LF F F& ,,..!,q"g 6	
 	
 	
 	,,..K!R;Q	
 	
 	
 	# 	# 	# ((666!.B 8	
 	
 	

 	((666%2w/	
 	
 	
   BN N NDU U U %' %' %'P"A "A "AJ  $+ + +\, , ,J J J       &N3 N3 N3b  *  *% % %P- - -@  \L \L \L~( ( (0 0 0f% % %(2 2 222 2 22 -t}=="	{LLL&ABC%0,,,$?1MM	
	 	*U *U	 	 >=*UZ9 9 9.P P P4P P P8  *B B B8  8+ + +" dE]33@ @ 43@&M M M2E E E< S3+&'*<sGGAqMM>R)ST * *	 *( 	#t-.	gwwtS'9'9:; .. .. ..f GHHl4F%GHH'79J&KLLH H ML IH IHH8 -e}==  >=  rM   