
    c4                         d Z ddlZddlZddlZddlmZ ddlZddl	m
Z
 ddlmZmZ ddlmZ ddlmZmZmZmZ  G d d	ej                  Zed
k    r( ej        dej                    ej                     dS dS )zN
Automated tests for checking transformation algorithms (the models package).
    N)partial)argsort)CoherenceModelBOOLEAN_DOCUMENT_BASED)LdaModel)get_tmpfilecommon_textscommon_dictionarycommon_corpusc                       e Zd ZeZeZeZd Z	d Z
d Zd Zd Zd Zd Zd Zd	 Zd
 Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Z d Z!dS )TestCoherenceModelc                 >    g dg dg _         g dg dg _        g dg dg _        g dg dg _        g dg d	g _        g  _         j         D ](} j                             fd
|D                        )t           j         j	        ddd           _
        d S )N)humancomputersystem	interface)graphminorstreeseps)userr   r   r   )timer   surveyr   )tokenr   r   r   )znot a tokenz	not an idztests usingz	this list)zshould raisezan errorzto pass	correctly)aaaaabbbbbccccceeeee)dddddfffffgggghhhhhhc                 4    g | ]}j         j        |         S  )
dictionarytoken2id).0r   selfs     ?lib/python3.11/site-packages/gensim/test/test_coherencemodel.py
<listcomp>z,TestCoherenceModel.setUp.<locals>.<listcomp><   s#    "V"V"Vu4?#;E#B"V"V"V       r   )corpusid2word
num_topicspasses
iterations)topics1topics2topics3topics4topics5	topicIds1appendr   r.   r&   ldamodel)r)   topics   ` r*   setUpzTestCoherenceModel.setUp   s
    988///

 211111

 988///
 EDD@@@
 100000
 \ 	X 	XEN!!"V"V"V"VPU"V"V"VWWWW ;A
 
 
r,   c                     |t           v rt           j         j        |          nt           j         j        |          t          dd j        i}t          dd j        i}t          dd j        i}t          dd j	        i}t          dd j        d         gi} 
                    t           fd            
                    t           fd                                |                                |                                                                |                                d         |                                                                |                                t           j                                        |                                |                                           dS )	z8Check provided topic coherence algorithm on given topicsr.   r&   	coherence)textsr&   r?   topicsr   c                  *    t          ddj        i S NrA   r%   )r   r6   kwargsr)   s   r*   <lambda>z<TestCoherenceModel.check_coherence_measure.<locals>.<lambda>R       n.[.[DL.[TZ.[.[ r,   c                  *    t          ddj        i S rC   )r   r7   rD   s   r*   rF   z<TestCoherenceModel.check_coherence_measure.<locals>.<lambda>S   rG   r,   Nr%   )r   dictr.   r&   r@   r   r3   r4   r5   r8   assertRaises
ValueErrorassertEqualget_coherenceget_coherence_per_topicassertIsInstancenpdoubleassertGreater)r)   r?   cm1cm2cm3cm4cm5rE   s   `      @r*   check_coherence_measurez*TestCoherenceModel.check_coherence_measureC   s   .. 	]T]^^^FF
tR[\\\F;;DL;F;;;;DL;F;;;;DL;F;;==DN=f== @@T\!_$5@@@*&[&[&[&[&[\\\*&[&[&[&[&[\\\**,,c.?.?.A.ABBB4466q93;L;L;N;NOOOc//1129===3,,..0A0A0C0CDDDDDr,   c                 0    |                      d           dS )z5Test U_Mass topic coherence algorithm on given topicsu_massNrX   r)   s    r*   	testUMasszTestCoherenceModel.testUMassY       $$X.....r,   c                 0    |                      d           dS )z2Test C_v topic coherence algorithm on given topicsc_vNr[   r\   s    r*   testCvzTestCoherenceModel.testCv]   s    $$U+++++r,   c                 0    |                      d           dS )z4Test C_uci topic coherence algorithm on given topicsc_uciNr[   r\   s    r*   testCucizTestCoherenceModel.testCucia   s    $$W-----r,   c                 0    |                      d           dS )z5Test C_npmi topic coherence algorithm on given topicsc_npmiNr[   r\   s    r*   	testCnpmizTestCoherenceModel.testCnpmie   r^   r,   c                 >    t          | j        | j        d           dS )zDPerform sanity check to see if u_mass coherence works with LDA ModelrZ   )modelr.   r?   N)r   r:   r.   r\   s    r*   testUMassLdaModelz$TestCoherenceModel.testUMassLdaModeli   s#    
 	T]4;(SSSSSSr,   c                 >    t          | j        | j        d           dS )zAPerform sanity check to see if c_v coherence works with LDA Modelr`   ri   r@   r?   Nr   r:   r@   r\   s    r*   testCvLdaModelz!TestCoherenceModel.testCvLdaModelp   s!    T]$*NNNNNNr,   c                 >    t          | j        | j        d           dS )zCPerform sanity check to see if c_w2v coherence works with LDAModel.c_w2vrl   Nrm   r\   s    r*   testCw2vLdaModelz#TestCoherenceModel.testCw2vLdaModelt   !    T]$*PPPPPPr,   c                 >    t          | j        | j        d           dS )zCPerform sanity check to see if c_uci coherence works with LDA Modelrc   rl   Nrm   r\   s    r*   testCuciLdaModelz#TestCoherenceModel.testCuciLdaModelx   rr   r,   c                 >    t          | j        | j        d           dS )zDPerform sanity check to see if c_npmi coherence works with LDA Modelrf   rl   Nrm   r\   s    r*   testCnpmiLdaModelz$TestCoherenceModel.testCnpmiLdaModel|   s!    T]$*QQQQQQr,   c                 &   |                      t          t          | j        | j        d           |                      t          t          | j        | j        | j        d           |                      t          t          | j        | j        d           dS )z&Test if errors are raised on bad inputrZ   )rA   r.   r?   r`   rA   r.   r&   r?   )rA   r&   r?   N)rJ   rK   r   r3   r.   r&   r\   s    r*   
testErrorszTestCoherenceModel.testErrors   s     	t|DK 	 	
 	
 	

 	t|DK% 	 	
 	
 	

 	t| 	 	
 	
 	
 	
 	
r,   c                    t          t          | j        | j        | j        d          } |            t          j                    dz
  }}|                     |j        |           t          dd          D ]'}|                      ||          j        |           (t          dd          D ]'}|                      ||          j        |           (d S )NrZ   rx      )	processes   )
r   r   r3   r.   r&   mp	cpu_countrL   r}   range)r)   	get_modelri   	used_cpusps        r*   testProcessesz TestCoherenceModel.testProcesses   s    N<[c
 
 
	 %9;;(:y)444r1 	J 	JAYY333=yIIIIq! 	B 	BAYY333=qAAAA	B 	Br,   c                 4   t          d          }t          | j        | j        | j        d          }|                    |           t          j        |          }|                     |                                |                                k               d S )Nzgensim_models_coherence.tstrZ   rx   	r   r   r3   r.   r&   saveload
assertTruerM   r)   fnameri   model2s       r*   testPersistencez"TestCoherenceModel.testPersistence   s    9::<[c
 
 
 	

5$U++++--1E1E1G1GGHHHHHr,   c                 4   t          d          }t          | j        | j        | j        d          }|                    |           t          j        |          }|                     |                                |                                k               d S )Nzgensim_models_coherence.tst.gzrZ   rx   r   r   s       r*   testPersistenceCompressedz,TestCoherenceModel.testPersistenceCompressed   s    <==<[c
 
 
 	

5$U++++--1E1E1G1GGHHHHHr,   c                    t          d          }t          | j        | j        | j        d          }|                                 |                    |           t          j        |          }|                     |j	                   | 
                    |                                |                                k               d S )Ngensim_similarities.tst.pklrZ   rx   )r   r   r3   r.   r&   estimate_probabilitiesr   r   assertIsNotNone_accumulatorr   rM   r   s       r*   4testPersistenceAfterProbabilityEstimationUsingCorpuszGTestCoherenceModel.testPersistenceAfterProbabilityEstimationUsingCorpus   s    9::<[c
 
 
 	$$&&&

5$U++V0111++--1E1E1G1GGHHHHHr,   c                    t          d          }t          | j        | j        | j        d          }|                                 |                    |           t          j        |          }|                     |j	                   | 
                    |                                |                                k               d S )Nr   r`   )rA   r@   r&   r?   )r   r   r3   r@   r&   r   r   r   r   r   r   rM   r   s       r*   3testPersistenceAfterProbabilityEstimationUsingTextszFTestCoherenceModel.testPersistenceAfterProbabilityEstimationUsingTexts   s    9::<tzdoY^
 
 
 	$$&&&

5$U++V0111++--1E1E1G1GGHHHHHr,   c                 `   t          | j        | j        d          }t          dd| j        i|}|                                 |j        }|                     |           | j        |_        | 	                    ||j                   | j
        |_        | 	                    d |j                   d S )NrZ   r>   rA   r%   )rI   r.   r&   r   r3   r   r   r   rA   rL   r4   r)   rE   rS   accumulators       r*   $testAccumulatorCachingSameSizeTopicsz7TestCoherenceModel.testAccumulatorCachingSameSizeTopics   s    T[T_PXYYY;;DL;F;;""$$$&[)))\
c&6777\
s/00000r,   c                 t   t          | j        | j        d          }t          dd| j        i|}|                                 |j        }|                     |           d | j        D             |_        | 	                    ||j                   | j        |_        | 	                    ||j                   d S )NrZ   r>   rA   c                 "    g | ]}|d d         S )Nr-   r%   )r(   ts     r*   r+   zITestCoherenceModel.testAccumulatorCachingTopicSubsets.<locals>.<listcomp>   s     222ae222r,   r%   )
rI   r.   r&   r   r3   r   r   r   rA   rL   r   s       r*   "testAccumulatorCachingTopicSubsetsz5TestCoherenceModel.testAccumulatorCachingTopicSubsets   s    T[T_PXYYY;;DL;F;;""$$$&[)))22T\222
c&6777\
c&677777r,   c                    t          | j        | j        d          }t          dd| j        i|}|                                 |                     |j                   | j        |_	        g }| j        j
                                        D ].}t          ||j        d          }|                    |           /|                     t!          j        ||j                             |                     |j                   d S )NrZ   r>   rA   T)topnreverser%   )rI   r.   r&   r   r3   r   r   r   r:   ri   state
get_lambdar   r   r9   r   rP   array_equalrA   assertIsNone)r)   rE   rS   rA   r;   bestns         r*   &testAccumulatorCachingWithModelSettingz9TestCoherenceModel.testAccumulatorCachingWithModelSetting   s    T[T_PXYYY;;DL;F;;""$$$S-...M	](3355 	! 	!EE$???EMM%    vsz::;;;#*+++++r,   c                    t          | j        | j        dd          }t          d	d| j        i|}|                                 |                     |j                   |j        }|j        }d|_	        | 
                    ||j                   | 
                    dt          |j        d                              | 
                    ||j                   d|_	        | 
                    ||j                   | 
                    dt          |j        d                              | 
                    ||j                   |                     t                    5  d|_	        d d d            d S # 1 swxY w Y   d S )
N   rZ   r.   r&   r   r?   rA      r   r~      r%   )rI   r.   r&   r   r3   r   r   r   _topicsr   rL   lenrA   rJ   rK   r)   rE   rS   r   topics_befores        r*   0testAccumulatorCachingWithTopnSettingGivenTopicszCTestCoherenceModel.testAccumulatorCachingWithTopnSettingGivenTopics   s   T[T_1X`aaa;;DL;F;;""$$$S-...&c&6777C
1..///444 c&6777C
1..///444z** 	 	CH	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s   E22E69E6c                    t          | j        | j        dd          }t          dd| j        i|}|                                 |                     |j                   |j        }|j        }d|_	        | 
                    ||j                   | 
                    dt          |j        d                              | 
                    ||j                   d|_	        | 
                    dt          |j        d                              d S )	Nr   rZ   r   ri   r   r   r   r%   )rI   r.   r&   r   r:   r   r   r   r   r   rL   r   rA   r   s        r*   /testAccumulatorCachingWithTopnSettingGivenModelzBTestCoherenceModel.testAccumulatorCachingWithTopnSettingGivenModel   s    T[T_1X`aaa;;4=;F;;""$$$S-...&c&6777C
1..///444C
1../////r,   c                    | j         | j        g}t          j        || j        | j        d          }|                     |j                   |D ]#}||_        |                     |j                   $|	                    |          \  \  }}\  }}| 
                    t          j        |          |d           | 
                    t          j        |          |d           |                     ||           d S )Nr`   r&   r@   r?   r~   )r3   r4   r   
for_topicsr&   r@   r   r   rA   compare_model_topicsassertAlmostEqualrP   meanrR   )r)   rA   cm
topic_listcoherence_topics1
coherence1coherence_topics2
coherence2s           r*   testCompareCoherenceForTopicsz0TestCoherenceModel.testCompareCoherenceForTopics  s   ,-&tdjES S SR_--- ! 	2 	2J"BI  1111 ##F++ 	I'	J)H*;Z 	rw'899:qIIIrw'899:qIII:z22222r,   c                    | j         | j         g}t          j        || j        | j        d          }|                     |j                   |D ]#}||_        |                     |j                   $|                    |          \  \  }}\  }}| 	                    t          j        |          |d           | 	                    t          j        |          |d           | 	                    ||d           d S )Nr`   r   r~   )places)r:   r   
for_modelsr&   r@   r   r   ri   compare_modelsr   rP   r   )r)   modelsr   ri   r   r   r   r   s           r*   testCompareCoherenceForModelsz0TestCoherenceModel.testCompareCoherenceForModels"  s   -/&tdjES S SR_---  	2 	2EBH  1111 f%% 	I'	J)H*;Z 	rw'899:qIIIrw'899:qIIIz:a@@@@@r,   c                 t    | j         g gz   }t          | j        |dd          }|                                 dS )z9Test if CoherenceModel works with document without tokensr`   r{   )ri   r@   r?   r}   N)r@   r   r:   rM   )r)   r@   r   s      r*   testEmptyListz TestCoherenceModel.testEmptyList4  sA    
bT!$-uYZ[[[
r,   N)"__name__
__module____qualname__r	   r@   r
   r&   r   r.   r<   rX   r]   ra   rd   rg   rj   rn   rq   rt   rv   ry   r   r   r   r   r   r   r   r   r   r   r   r   r   r%   r,   r*   r   r      s        E"JF#
 #
 #
JE E E,/ / /, , ,. . ./ / /T T TO O OQ Q QQ Q QR R R
 
 
$B B BI I II I I	I 	I 	I	I 	I 	I	1 	1 	1	8 	8 	8, , ,  ,0 0 0 3 3 3$A A A$    r,   r   __main__z)%(asctime)s : %(levelname)s : %(message)s)formatlevel)__doc__loggingunittestmultiprocessingr   	functoolsr   numpyrP   gensim.matutilsr   gensim.models.coherencemodelr   r   gensim.models.ldamodelr   gensim.test.utilsr   r	   r
   r   TestCaser   r   basicConfigDEBUGmainr%   r,   r*   <module>r      s*                    # # # # # # O O O O O O O O + + + + + + Y Y Y Y Y Y Y Y Y Y Y Ya a a a a* a a aH	 z GJRYR_````HMOOOOO r,   