c z dZddlmZmZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddlZ ddlmZn #e$rdZYnwxYwdZdgZddgZgdZd Zd Zd Zd ZGd dZedkrKejdejejddejej ejdZ!e"ejdkr4e#e$de%zej&dejdZ'e'(dr)ej)*ej+e'Z,nej)*e'Z,e"ejdkre-ejdZ.ne,j/Z.e"ejdkre-ejdZ0ne,j1Z0ejde.e0ee,e.e0Z2ej34e0Z5ejde j6e0e0fe j7Z8ej39e2dD]hZ:e;de:DZe:ee2e0*ZPeePeH\ZQZRZSejeOz ZO[P[SeQj@Be j?eRBe j?dzcZTZR[Qe#d+eHeOeRdeRd!fzed,e8eTeReMeN[TeD]ZUeD]ZVejd-eHeUeVejZOeUejWjX_YejWZe2e5eHeVeU.Z[ejeOz ZOe[j\jTBe j?e[j\jRBe j?dzcZTZR[[e#d/eHeUeVeOeRdeRd!fzed0e8eTeReMeN[Tejd1eHeUejZOejWZe2e5eHd2d eU3Z[ejeOz ZOe[j\jTBe j?e[j\jRBe j?dzcZTZR[[e#d4eHeUeOeRdeRd!fzed5e8eTeReMeN[T^ejd6e!dSdS)7aUSAGE: %(program)s MATRIX.mm [CLIP_DOCS] [CLIP_TERMS] Check truncated SVD error for the algo in gensim, using a given corpus. This script runs the decomposition with several internal parameters (number of requested factors, iterative chunk size) and reports error for each parameter combination. The number of input documents is clipped to the first CLIP_DOCS. Similarly, only the first CLIP_TERMS are considered (features with id >= CLIP_TERMS are ignored, effectively restricting the vocabulary size). If you don't specify them, the entire matrix will be used. Example: ./svd_error.py ~/gensim/results/wiki_en_v10k.mm.bz2 100000 10000 )print_functionwith_statementN) sparsesvdi,i'i)rFctrXtjdt|jt j|Stj S)z3Spectral norm ("norm 2") of a symmetric matrix `a`.z&computing spectral norm of a %s matrix) COMPUTE_NORM2logginginfostrshapescipylinalgeigvalshmaxnpnan)as 5lib/python3.11/site-packages/gensim/test/svd_error.pynorm2r3sN =s17||LLL|$$Q''++---v ctjdtj||z|jz S)Ng?)rsqrtmultiplysumsize)diffs rrmser <s6 73T40044666B C CCrc ~tj|tjtj||j }||z }tj|t |}}td||||z |||z t|fztj dS)NzF%s error: norm_frobenius=%f (/ideal=%g), norm2=%f (/ideal=%g), RMSE=%g) rdotdiagTrnormrprintr sysstdoutflush) nameaatusideal_nfideal_n2errnfn2s r print_errorr3@s 6!RVBGAJJ,, - - -C3JC Y^^C %**B P r2="b8mT#YY? @JrceZdZdZdZdS) ClippedCorpusc2||_||c|_|_dSN)corpusmax_docs max_terms)selfr8r9r:s r__init__zClippedCorpus.__init__Ls (0)% t~~~rc#pKtjjjD]}fd|DVdS)Nc4g|]\}}|jk||fS)r:).0fwr;s r z*ClippedCorpus.__iter__..Rs-@@@daQ-?@Aq6@@@r) itertoolsislicer8r9)r;docs` r__iter__zClippedCorpus.__iter__PsX#DK?? A AC@@@@c@@@ @ @ @ @ A ArN)__name__ __module__ __qualname__r<rGr?rrr5r5Ks7<<<AAAAArr5__main__z)%(asctime)s : %(levelname)s : %(message)s)formatlevelz running %s r__doc__rbz2z"using %i documents and %i featureszcomputing corpus * corpus^T)dtypei) chunksizec#4K|]}t|VdSr7)len)r@rFs r rVus(003c#hh000000r)num_nnz num_termsnum_docsrRz1computing full decomposition of corpus * corpus^tz .spectrum.npyz(****************************************z4%i factors, ideal error norm_frobenius=%f, norm_2=%fz******************************)endbaselinez$computing SVDLIBC SVD for %i factors)rXz8SVDLIBC SVD for %i factors took %s s (spectrum %f .. %f)SVDLIBCzKcomputing incremental SVD for %i factors, %i power iterations, chunksize %i)id2word num_topicsrS power_itersz_incremental SVD for %i factors, %i power iterations, chunksize %i took %s s (spectrum %f .. %f)zincremental SVDz;computing multipass SVD for %i factors, %i power iterationsi)r_r`rSonepassrazOmultipass SVD for %i factors, %i power iterations took %s s (spectrum %f .. %f)z multipass SVDzfinished running %s)]rO __future__rrr osr'timerPrDnumpyr scipy.linalgrgensimr ImportErrorFACTORS CHUNKSIZE POWER_ITERSr rr r3r5rH basicConfigINFOr joinargvpathbasenameprogramrUr&globalslocalsexitfnameendswithcorporaMmCorpusBZ2FilemmintnrYmrXr8utilsFakeDictr_zerosfloat64r+grouperchunkrrWmatutils corpus2cscfloat32r$toarrayastypereigh spectrum_s spectrum_usavefactorsr"r#r0r% ideal_fror/taken corpus_ramutr-vtr,rarSmodelslsimodelP2_EXTRA_ITERSLsiModelmodel projectionr?rrrs  65555555  #######III  % DM oo  DDDAAAAAAAA zm1GJRYR^____GLsxx11222gsx{++G s38}}q ggii "VVXX-... HQKE ~~e, ^ $ $[S[%7%7 8 8 ^ $ $U + + s38}}q C   K s38}}q C   LGL5q!<<< ]2q! $ $Fl##A&&GGL./// "(Aq6 , , ,C%%f%==#00%00000**5'QY\Y\]bYcYckmku*vv  u  EGLDEEE **RZ C"\..s33J DDbD!Jddd#%J BGEO #Z000==rvjHWH-vrvgbgj'>R6S6SU_`a`a`ackdkck`kUlUn/o/oppp s INN3'' gk* hNRY[dfnQooppp hB JBHa\**HBHg,?,?H V V V   GL? I I IDIKKE33Fa3HHJ! *g66IB2DIKK%'E4;;rz**AHHRZ,@,@!,CDAq ELeQqT1R512 3 3 3 K 31i B B B&% % K&    a[)" 8C &5 ..G'[/" e+')00<N>P>W>WXZXb>c>cef>f1Ak9eQqT1R5IJ  -sAq)XNNNA GLVX_al m m mDIKKEM**Gt;+EDIKK%'E#%,,RZ88%:J:L:S:STVT^:_:_ab:bDAq ED+uadAbE:;    KaIx H H HK% NGL&00000[m1m1s7AA