U Be7@sddlZddlZddlmZmZmZmZmZmZddl m Z ddZ Gddde Z Gdd d e d ZGd d d e d ZGd dde d ZGddde d ZGddde d ZGddde d ZGddde d ZGdddZdS)N) VocabularyMLELidstoneLaplaceWittenBellInterpolatedKneserNeyInterpolatedpadded_everygramscs8tdddddddgdd fd d td td fDfS)Nabcdz)Z unk_cutoffcsg|]}t|qSr).0sentZ ngram_orderry/var/www/nmhs-web.org.in/public_html/infoladakh/backend/venv/lib/python3.8/site-packages/nltk/test/unit/lm/test_models.py sz&_prepare_test_data..abcdZegadbe)rlistrrrr_prepare_test_datas  rc sXeZdZdZdddddddd d g Zfd d Zejd eddZ e ddZ Z S)ParametrizedTestsz,Metaclass for generating parametrized tests.r r r)r r )e)r)wc sN|dg}t|D]&\}\}}}|||||d|<qt||||S)N score_testsz test_score_{})get enumerateadd_score_testformatsuper__new__) clsnamebasesdctZscoresiwordcontextexpected_score __class__rrr+/s zParametrizedTests.__new__r2csBd}tfddjjD}t|ddks>t|dS)NzThe context is {}c3s|]}j|VqdSN)modelscore)rr$r2selfrr :sz2ParametrizedTests.test_sum_to_1..gHz>g?)r)sumr7vocabpytestapproxAssertionError)r:r2messagesrr9r test_sum_to_17s zParametrizedTests.test_sum_to_1cs"dfdd}|S)Nzword='{}', context={}cs*|j}t|dks&tdS)N-C6?)r7r8r>r?r@)r:r8r2r3rAr1rr test_methodAsz5ParametrizedTests.add_score_test..test_method)r))r:r1r2r3rFrrErr(=s z ParametrizedTests.add_score_test) __name__ __module__ __qualname____doc__Zcontextsr+r>markZ parametrizerC classmethodr( __classcell__rrr4rr!s    rc@sZeZdZdZddgdfddgdfddd gZed d Zd d ZddZddZ ddZ dS) TestMleBigramzUnit tests for MLE ngram model.r r rr"rrNr)r Ng$I$I?)yNg۶m۶m?cCs*td\}}td|d|_|j|dSNZ vocabularyrrr7fitr:r=Z training_textrrr setup_methodXs zTestMleBigram.setup_methodcCs"|jddg}t|stdS)Nr r")r7logscoremathisinfr@)r:rXrrrtest_logscore_zero_score^sz&TestMleBigram.test_logscore_zero_scorecCsTddddddg}d}d}t|j|d |ks4tt|j|d |ksPtdS) Nrr )r r )r r )r r )r r r rg(\?g_vO@rDr>r?r7entropyr@ perplexity)r:ZtrainedHr`rrrtest_entropy_perplexity_seencsz*TestMleBigram.test_entropy_perplexity_seencCs<ddddg}t|j|s"tt|j|s8tdS)Nr\r r )r r r])rYrZr7r_r@r`)r:Z untrainedrrrtest_entropy_perplexity_unseen{s z,TestMleBigram.test_entropy_perplexity_unseencCsVd}d}dddddddg}t|j|d |ks6tt|j|d |ksRtdS) Ng~jt@gs @rrr)-r!)rrDr^)r:rar`textrrr test_entropy_perplexity_unigramss z.TestMleBigram.test_entropy_perplexity_unigramsN) rGrHrIrJr%rLrWr[rbrdrgrrrrrNHs    rN) metaclassc@s2eZdZdZdddgdfdddgZed d Zd S) TestMleTrigramzMLE trigram model tests)r )r r rr r r)r Ngqq?rOrPNgUUUUUU?cCs*td\}}td|d|_|j|dSNrSrTrVrrrrWs zTestMleTrigram.setup_methodN)rGrHrIrJr%rLrWrrrrris ric@s@eZdZdZddgdfdddgZedd Zd d Zd d ZdS)TestLidstoneBigramzUnit tests for Lidstone classr r 88?)r Ng"u)?)rNgк{?)rPNgL?cCs,td\}}tdd|d|_|j|dS)NrR皙?rSrrr7rUrVrrrrWs zTestLidstoneBigram.setup_methodcCsd|jjkstdS)Nror7gammar@r:rrr test_gammaszTestLidstoneBigram.test_gammacCsTddddddg}d}d}t|j|d |ks4tt|j|d |ksPtdS) Nr\rcr r r r r r r rg=,Ԛ]@g鷯 1@rDr^r:rfrar`rrrtest_entropy_perplexitysz*TestLidstoneBigram.test_entropy_perplexityN rGrHrIrJr%rLrWrtrzrrrrrms  rmc@sHeZdZddgdfddgdfdddgdfdddgdfgZeddZd S) TestLidstoneTrigramr r rnr"gqq?r cCs,td\}}tdd|d|_|j|dS)NrlrorSrprVrrrrWs z TestLidstoneTrigram.setup_methodN)rGrHrIr%rLrWrrrrr|s     r|c@s@eZdZdZddgdfdddgZedd Zd d Zd d ZdS)TestLaplaceBigramzUnit tests for Laplace classr r gqq?)r NgtE]t?)rNgF]tE?)rPNgF]tE?cCs*td\}}td|d|_|j|dSrQ)rrr7rUrVrrrrWs zTestLaplaceBigram.setup_methodcCsd|jjkstdS)NrrqrsrrrrtszTestLaplaceBigram.test_gammacCsTddddddg}d}d}t|j|d |ks4tt|j|d |ksPtdS) Nr\rcrurvrwrxgQ @gݓz!@rDr^ryrrrrz sz)TestLaplaceBigram.test_entropy_perplexityNr{rrrrr}s  r}c@sDeZdZeddZdddddgdfdd dgd fdd dgdfgZd S) !TestWittenBellInterpolatedTrigramcCs*td\}}td|d|_|j|dSrk)rrr7rUrVrrrrW:s z.TestWittenBellInterpolatedTrigram.setup_method)r Ngqq?)rNgrjr r grq?r gqq?rNrGrHrIrLrWr%rrrrr~9s    r~c@sDeZdZeddZdddddgdfdd dgd fdd dgdfgZd S) TestKneserNeyInterpolatedTrigramcCs*td\}}td|d|_|j|dSrk)rrr7rUrVrrrrW^s z-TestKneserNeyInterpolatedTrigram.setup_method)r N?)rNr)rPNrr r g?r gHzG?rNrrrrrr]s    rc@sTeZdZdZeddZddZddZdd Zd d Z d d Z ddZ ddZ dS)TestNgramModelTextGenerationz$Using MLE model, generate some text.cCs*td\}}td|d|_|j|dSrkrTrVrrrrWs z)TestNgramModelTextGeneration.setup_methodcCs|jjdddkstdS)Nrl random_seedr r7generater@rsrrrtest_generate_one_no_contextsz9TestNgramModelTextGeneration.test_generate_one_no_contextcCsP|jjdgddkst|jjddgddks2t|jjddgddksLtdS)Nr  text_seedr r r rrsrrr"test_generate_one_limiting_contextsz?TestNgramModelTextGeneration.test_generate_one_limiting_contextcCs|jjddddkstdS)N)r rrRrrr rrsrrr test_generate_one_varied_contextsz=TestNgramModelTextGeneration.test_generate_one_varied_contextcCsLt|jjtdg}|j||jjdddddddddddgksHtdS) NZbdbdbd)r r rr r r)r r7orderrrUrr@)r:Zmore_training_textrrrtest_generate_cycles  z0TestNgramModelTextGeneration.test_generate_cyclecCs(|jjdddddddddgks$tdS) Nr)rr"rlrr r r r rrsrrrtest_generate_with_text_seeds z9TestNgramModelTextGeneration.test_generate_with_text_seedcCs(|jjddd|jjdddks$tdS)N)Zaliensrlrrrrsrrrtest_generate_oov_text_seedsz8TestNgramModelTextGeneration.test_generate_oov_text_seedc CsJtt|jjddW5QRX|jjddd|jjddksFtdS)Nr6rrlrr)r>Zraises TypeErrorr7rr@rsrrrtest_generate_None_text_seeds   z9TestNgramModelTextGeneration.test_generate_None_text_seedN) rGrHrIrJrLrWrrrrrrrrrrrr}s  r)rYr>Znltk.lmrrrrrrZnltk.lm.preprocessingr rtyperrNrirmr|r}r~rrrrrrs   'N;=$