o h8Z@sjdZddlZddlZddlZddlmZddlZddlmZddl m Z ddl Tej j Zgdgdgd gd gd gd gd gdgdgdg gZddddddddddddddddd dd!d"d#d$d%dd&d'dd(d)d*d+d,d-d.d/dd0d1d1d2d3dd4d5dd6d7d8d9d:d;d;dd?d@d*d*d+dAd=d.dBdCdDdDd2dEd-dFdGddHdIdIdJddKdLdMg gZdNdOZdPdQZdRdSZdTZdUdVgZdWdXgZdYdZZd[d\Zd]Zd^d_Zd`daZdbdcZdddeZdfdgZdhdiZ djZ!dkdlZ"dmZ#dndoZ$dpZ%dqdrZ&dsZ'dtZ(dudvZ)dwdxZ*dydzZ+d{Z,d|d}Z-d~Z.ddZ/d0Z1ddZ2d0Z3ddZ4dS)z$ Basic tests of the data conversion N)ZipFile)CoNLL)Document)*) 1NousilPRON_!Number=Plur|Person=1|PronType=Prs3nsubjr start_char=0|end_char=4) 2avonsavoirAUXr 5Mood=Ind|Number=Plur|Person=1|Tense=Pres|VerbForm=Finr aux:tenser start_char=5|end_char=10) r atteint atteindreVERBr 0Gender=Masc|Number=Sing|Tense=Past|VerbForm=Part0rootr start_char=11|end_char=18) 4laleDETr 0Definite=Def|Gender=Fem|Number=Sing|PronType=Art5detr start_char=19|end_char=21) r"finr%NOUNr Gender=Fem|Number=Singr objr start_char=22|end_char=25) z6-7dur r r r r r r start_char=26|end_char=28) 6der-ADPr r 8caser r ) 7rrr r 1Definite=Def|Gender=Masc|Number=Sing|PronType=Artr/r#r r ) r/sentierr3r&r Gender=Masc|Number=Singr"nmodr start_char=29|end_char=36) 9.r8PUNCTr r r punctr start_char=36|end_char=37)rrr r r r)idtextlemmauposfeatsheaddeprelmisc)rrrrrr)r=rrrrrr)rrr r!r#r$)rHr%r&r'r(r))r*r+)r>r?rE)rIr-r.r0)r>r?r@rArCrD)rJr2)r>r?r@rArBrCrD)rKr3r4r5r6) r8r9r:r;)r>r?r@rArCrDrEcCsHtt\}}|tks Jt|t|ksJtdd|Ds"JdS)Ncss|] }t|dkVqdS)rN)len).0xrPc/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/tests/common/test_data_conversion.py -sz%test_conll_to_dict..)r convert_conllCONLLDICTrMall)dictsemptyrPrPrQtest_conll_to_dict)s rYcCs2tt}ddd|dD}|tksJdS)NcS g|] }dd|dDqS)cSg|]}|dqS splitrNsentencerPrPrQ 2z1test_dict_to_conll... r^rNdocrPrPrQrb2 z&test_dict_to_conll..{:c} )rrUformatr_rT)documentconllrPrPrQtest_dict_to_conll/srmcCsBtt}|}t|}ddd|dD}|tksJdS)aX Test the conversion from raw dict to Document and back This code path will first turn start_char|end_char into start_char & end_char fields in the Document That version to a dict will have separate fields for each of those Finally, the conversion from that dict to a list of conll entries should convert that back to misc cSrZ)cSr[r\r^r`rPrPrQrb@rcz?test_dict_to_doc_and_doc_to_dict...rdr^rerPrPrQrb@rgz4test_dict_to_doc_and_doc_to_dict..rhriN)rrUto_dictrjr_rT)rkrWrlrPrPrQ test_dict_to_doc_and_doc_to_dict5s rouj # sent_id = yandex.reviews-f-8xh5zqnmwak3t6p68y4rhwd4e0-1969-9253 # genre = review # text = Как- то слишком мало цветов получают актёры после спектакля. 1 Как как-то ADV _ Degree=Pos|PronType=Ind 7 advmod _ SpaceAfter=No 2 - - PUNCT _ _ 3 punct _ _ 3 то то PART _ _ 1 list _ deprel=list:goeswith 4 слишком слишком ADV _ Degree=Pos 5 advmod _ _ 5 мало мало ADV _ Degree=Pos 6 advmod _ _ 6 цветов цветок NOUN _ Animacy=Inan|Case=Gen|Gender=Masc|Number=Plur 7 obj _ _ 7 получают получать VERB _ Aspect=Imp|Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act 0 root _ _ 8 актёры актер NOUN _ Animacy=Anim|Case=Nom|Gender=Masc|Number=Plur 7 nsubj _ _ 9 после после ADP _ _ 10 case _ _ 10 спектакля спектакль NOUN _ Animacy=Inan|Case=Gen|Gender=Masc|Number=Sing 7 obl _ SpaceAfter=No 11 . . PUNCT _ _ 7 punct _ _ # sent_id = 4 # genre = social # text = В женщине важна верность, а не красота. 1 В в ADP _ _ 2 case _ _ 2 женщине женщина NOUN _ Animacy=Anim|Case=Loc|Gender=Fem|Number=Sing 3 obl _ _ 3 важна важный ADJ _ Degree=Pos|Gender=Fem|Number=Sing|Variant=Short 0 root _ _ 4 верность верность NOUN _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Sing 3 nsubj _ SpaceAfter=No 5 , , PUNCT _ _ 8 punct _ _ 6 а а CCONJ _ _ 8 cc _ _ 7 не не PART _ Polarity=Neg 8 advmod _ _ 8 красота красота NOUN _ Animacy=Inan|Case=Nom|Gender=Fem|Number=Sing 4 conj _ SpaceAfter=No 9 . . PUNCT _ _ 3 punct _ _ unКак- то слишком мало цветов получают актёры после спектакля.uFВ женщине важна верность, а не красота.z5yandex.reviews-f-8xh5zqnmwak3t6p68y4rhwd4e0-1969-9253rcCstd}t|jdksJ|d|jdjdksJ|d|jdjdks*J|d|jdjdks8Jtttt|jD]+\}\}}}||j ksOJ||j ksVJ||j ks]Jt|jdksfJ| rlJqAd |}|d}t|dksJ|dd}t|dksJ|d|dksJ|d|dksJ|d|dksJ|jdjdjdksJ|jdjdjd ksJd S) z_ Refactored the test for the Russian doc so we can use it to test various file methods rdrFrr<r={:C}riz list:goeswithN)RUSSIAN_SAMPLEr_rM sentencescomments enumeratezip RUSSIAN_TEXT RUSSIAN_IDSr?sent_idindexhas_enhanced_dependenciesrjwordsrCrD)rflinessent_idx expected_text expected_idrarsrPrPrQcheck_russian_doces*    rcCs|d}tjtd}t|t||t|dd }|}Wdn1s)wY|ds5J|}|| dd}tt dd}||ksQJt|}t|dS)z6 Specifically test the write_doc2conll method russian.conll input_strutf-8encodingNriz # sent_id = 4) r conll2docrrrwrite_doc2conllopenreadendswithstripfind)tmp_pathfilenamerfr%r?sampledoc2rPrPrQtest_write_russian_docs      ra{ # newdoc # sent_id = 1 # text = It is hers. # previous = Which person owns this? # comment = copular subject 1 It it PRON PRP Number=Sing|Person=3|PronType=Prs 3 nsubj _ _ 2 is be AUX VBZ Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin 3 cop _ _ 3 hers hers PRON PRP Gender=Fem|Number=Sing|Person=3|Poss=Yes|PronType=Prs 0 root _ SpaceAfter=No 4 . . PUNCT . _ 3 punct _ _ cCsJtjtd}t}t|||}|dsJ|tks#JdS)Nrri) rrENGLISH_SAMPLEioStringIOrgetvaluerr)rfoutput output_valuerPrPrQtest_write_to_ios  rcCstjtd}|d}t||tj||ddt| }|}Wdn1s+wYtdtd}||ks>JdS)Nrz english.conlla)moderi)rrrrrr)rrfrr%r?expectedrPrPrQtest_write_doc2conll_appends    rcCstjtd}t|dS)zI Test that a doc with comments gets converted back with comments rN)rrrrrrfrPrPrQtest_doc_with_commentss  rcCstjtd}d|d}t|dksJ|dd}t|dks%J|D]&}|d}t|d ks.N)rrrrrjr_rMrV)rfrsrawordpiecesrPrPrQtest_unusual_miscs   rc Cst6}tj|d}t|ddd }|tWdn1s#wYtj |d}t |WddS1s=wYdS)z( Test loading a doc from a file rwrrN) input_file) tempfileTemporaryDirectoryospathjoinrwriterrrrr)tempdirrfoutrfrPrPrQ test_files    "rc CstO}tj|d}d}t|d%}||d}|t Wdn1s,wYWdn1s;wYt j ||d}t |WddS1sVwYdS)z, Test loading a doc from a zip file z russian.ziprrN)rzip_file) rrrrrrrrrrencoderrr)rrrzoutrrfrPrPrQ test_zip_files   "ra| # text = Teferi's best friend is Karn # sent_id = 0 1 Teferi _ _ _ _ 0 _ _ start_char=0|end_char=6|ner=S-PERSON 2 's _ _ _ _ 1 _ _ start_char=6|end_char=8|ner=O 3 best _ _ _ _ 2 _ _ start_char=9|end_char=13|ner=O 4 friend _ _ _ _ 3 _ _ start_char=14|end_char=20|ner=O 5 is _ _ _ _ 4 _ _ start_char=21|end_char=23|ner=O 6 Karn _ _ _ _ 5 _ _ start_char=24|end_char=28|ner=S-PERSON cCstjtd}t|jdksJ|jd}t|jdksJgd}t|j|D]!\}}|j|ks2J|jr7Jt|j dks@J|j djrHJq'd |}|tksTJdS)z= Test that tokens get properly created with NER tags rr<rrI)S-PERSONOrrrrrpN) rr SIMPLE_NERrMrstokensrvnerrEr|rj)rfra EXPECTED_NERtokenrrlrPrPrQtest_simple_ner_conversions    ra| # text = This makes John's headache worse # sent_id = 0 1 This _ _ _ _ 0 _ _ start_char=0|end_char=4|ner=O 2 makes _ _ _ _ 1 _ _ start_char=5|end_char=10|ner=O 3-4 John's _ _ _ _ _ _ _ start_char=11|end_char=17|ner=S-PERSON 3 John _ _ _ _ 2 _ _ _ 4 's _ _ _ _ 3 _ _ _ 5 headache _ _ _ _ 4 _ _ start_char=18|end_char=26|ner=O 6 worse _ _ _ _ 5 _ _ start_char=27|end_char=32|ner=O cCstjtd}t|jdksJ|jd}t|jdksJ|r#Jgd}gd}t|j||D]"\}}}|j|ks>J|j rCJt|j |ksLJ|j dj rTJq2d |}|tks`JdS) z Test that tokens including MWT get properly created with NER tags Note that this kind of thing happens with the EWT tokenizer for English, for example rr<rrH)rrrrr)r<r<rFr<r<rpN) rrMWT_NERrMrsrr{rvrrEr|rj)rfrarEXPECTED_WORDSrrexpected_wordsrlrPrPrQtest_mwt_ner_conversions     ru # newpar # sent_id = aia_foorum_37 # text = Sestpeale ei mõistagi neid, kes koduaias sortidega tegelevad. 1 Sestpeale sest_peale ADV D _ 3 advmod 3:advmod _ 2 ei ei AUX V Polarity=Neg 3 aux 3:aux _ 3 mõistagi mõistma VERB V Connegative=Yes|Mood=Ind|Tense=Pres|VerbForm=Fin|Voice=Act 0 root 0:root _ 4 neid tema PRON P Case=Par|Number=Plur|Person=3|PronType=Prs 3 obj 3:obj|9:nsubj SpaceAfter=No 5 , , PUNCT Z _ 9 punct 9:punct _ 6 kes kes PRON P Case=Nom|Number=Plur|PronType=Int,Rel 9 nsubj 4:ref _ 7 koduaias kodu_aed NOUN S Case=Ine|Number=Sing 9 obl 9:obl _ 8 sortidega sort NOUN S Case=Com|Number=Plur 9 obl 9:obl _ 9 tegelevad tegelema VERB V Mood=Ind|Number=Plur|Person=3|Tense=Pres|VerbForm=Fin|Voice=Act 4 acl:relcl 4:acl SpaceAfter=No 10 . . PUNCT Z _ 3 punct 3:punct _ cCs~tjtd}t|jdksJ|jd}t|jdksJ|s#J|jdjd}|jdks2Jd |}|tks=JdS)Nrr<rrr=z 3:obj|9:nsubjrp) rr ESTONIAN_DEPSrMrsrr{r|depsrj)rfrarrlrPrPrQtest_deps_conversionGs    ruG # sent_id = ewtb2_000035_15 # text = Ja paari aasta pärast rôômalt maasikatele ... 1 Ja ja CCONJ J _ 3 cc 5.1:cc _ 2 paari paar NUM N Case=Gen|Number=Sing|NumForm=Word|NumType=Card 3 nummod 3:nummod _ 3 aasta aasta NOUN S Case=Gen|Number=Sing 0 root 5.1:obl _ 4 pärast pärast ADP K AdpType=Post 3 case 3:case _ 5 rôômalt rõõmsalt ADV D Typo=Yes 3 advmod 5.1:advmod Orphan=Yes|CorrectForm=rõõmsalt 5.1 panna panema VERB V VerbForm=Inf _ _ 0:root Empty=5.1 6 maasikatele maasikas NOUN S Case=All|Number=Plur 3 obl 5.1:obl Orphan=Yes 7 ... ... PUNCT Z _ 3 punct 5.1:punct _ u # sent_id = ewtb2_000035_15 # text = Ja paari aasta pärast rôômalt maasikatele ... 1 Ja ja CCONJ J _ 3 cc 5.1:cc _ 2 paari paar NUM N Case=Gen|Number=Sing|NumForm=Word|NumType=Card 3 nummod 3:nummod _ 3 aasta aasta NOUN S Case=Gen|Number=Sing 0 root 5.1:obl _ 4 pärast pärast ADP K AdpType=Post 3 case 3:case _ 5 rôômalt rõõmsalt ADV D Typo=Yes 3 advmod 5.1:advmod Orphan=Yes|CorrectForm=rõõmsalt 5.1 panna panema VERB V VerbForm=Inf _ _ 0:root Empty=5.1 cCttddS)zS Check that we can read and then output a sentence with empty dependencies rJN)check_empty_deps_conversionESTONIAN_EMPTY_DEPSrPrPrPrQtest_empty_deps_conversionlrcCr)zS The empty deps conversion should also work if the empty dep is at the end rHN)rESTONIAN_EMPTY_END_DEPSrPrPrPrQ!test_empty_deps_at_end_conversionrrrcCstj|dd}t|jdksJt|jdj|ksJt|jdj|ks(Jt|jdjdks4J|jd}d|}||ksDJ|jd}t||dksUJ|dddks_Jt|jdkshJt|jdj|kstJt|jdj|ksJt|jdjdksJdS) NFrignore_gappingr<rrprHr>)rHr<) rrrMrsrr| empty_wordsrjrn)rrrfrarl sentence_dictrPrPrQrxs   ru` # doc_id = this_is_a_doc # sent_id = ewtb2_000035_15 # text = Ja paari aasta pärast rôômalt maasikatele ... 1 Ja ja CCONJ J _ 3 cc 5.1:cc _ 2 paari paar NUM N Case=Gen|Number=Sing|NumForm=Word|NumType=Card 3 nummod 3:nummod _ 3 aasta aasta NOUN S Case=Gen|Number=Sing 0 root 5.1:obl _ 4 pärast pärast ADP K AdpType=Post 3 case 3:case _ 5 rôômalt rõõmsalt ADV D Typo=Yes 3 advmod 5.1:advmod Orphan=Yes|CorrectForm=rõõmsalt 5.1 panna panema VERB V VerbForm=Inf _ _ 0:root Empty=5.1 6 maasikatele maasikas NOUN S Case=All|Number=Plur 3 obl 5.1:obl Orphan=Yes 7 ... ... PUNCT Z _ 3 punct 5.1:punct _ cCs8tjtdd}d|tksJ|jdjdksJdS)NFrrpr this_is_a_doc)rrESTONIAN_DOC_IDrjrsdoc_idrrPrPrQtest_read_doc_idsra # text = Teferi's best friend is Karn # sent_id = 0 # notes = this sentence has a dependency index outside the sentence. it should throw an IndexError 1 Teferi _ _ _ _ 0 root _ start_char=0|end_char=6|ner=S-PERSON 2 's _ _ _ _ 1 dep _ start_char=6|end_char=8|ner=O 3 best _ _ _ _ 2 dep _ start_char=9|end_char=13|ner=O 4 friend _ _ _ _ 3 dep _ start_char=14|end_char=20|ner=O 5 is _ _ _ _ 4 dep _ start_char=21|end_char=23|ner=O 6 Karn _ _ _ _ 8 dep _ start_char=24|end_char=28|ner=S-PERSON cCs<tttjtd}WddS1swYdS)Nr)pytestraises IndexErrorrrSIMPLE_DEPENDENCY_INDEX_ERRORrrPrPrQtest_read_dependency_errorss "rau # doc_id = doc_1 # sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0020 # text = His mother was also killed in the attack. 1 His his PRON PRP$ Case=Gen|Gender=Masc|Number=Sing|Person=3|Poss=Yes|PronType=Prs 2 nmod:poss 2:nmod:poss _ 2 mother mother NOUN NN Number=Sing 5 nsubj:pass 5:nsubj:pass _ 3 was be AUX VBD Mood=Ind|Number=Sing|Person=3|Tense=Past|VerbForm=Fin 5 aux:pass 5:aux:pass _ 4 also also ADV RB _ 5 advmod 5:advmod _ 5 killed kill VERB VBN Tense=Past|VerbForm=Part|Voice=Pass 0 root 0:root _ 6 in in ADP IN _ 8 case 8:case _ 7 the the DET DT Definite=Def|PronType=Art 8 det 8:det _ 8 attack attack NOUN NN Number=Sing 5 obl 5:obl:in SpaceAfter=No 9 . . PUNCT . _ 5 punct 5:punct _ # doc_id = doc_1 # sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0028 # text = This item is a small one and easily missed. 1 This this DET DT Number=Sing|PronType=Dem 2 det 2:det _ 2 item item NOUN NN Number=Sing 6 nsubj 6:nsubj|9:nsubj:pass _ 3 is be AUX VBZ Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin 6 cop 6:cop _ 4 a a DET DT Definite=Ind|PronType=Art 6 det 6:det _ 5 small small ADJ JJ Degree=Pos 6 amod 6:amod _ 6 one one NOUN NN Number=Sing 0 root 0:root _ 7 and and CCONJ CC _ 9 cc 9:cc _ 8 easily easily ADV RB _ 9 advmod 9:advmod _ 9 missed miss VERB VBN Tense=Past|VerbForm=Part|Voice=Pass 6 conj 6:conj:and SpaceAfter=No 10 . . PUNCT . _ 6 punct 6:punct _ # doc_id = doc_2 # sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0029 # text = But in my view it is highly significant. 1 But but CCONJ CC _ 8 cc 8:cc _ 2 in in ADP IN _ 4 case 4:case _ 3 my my PRON PRP$ Case=Gen|Number=Sing|Person=1|Poss=Yes|PronType=Prs 4 nmod:poss 4:nmod:poss _ 4 view view NOUN NN Number=Sing 8 obl 8:obl:in _ 5 it it PRON PRP Case=Nom|Gender=Neut|Number=Sing|Person=3|PronType=Prs 8 nsubj 8:nsubj _ 6 is be AUX VBZ Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin 8 cop 8:cop _ 7 highly highly ADV RB _ 8 advmod 8:advmod _ 8 significant significant ADJ JJ Degree=Pos 0 root 0:root SpaceAfter=No 9 . . PUNCT . _ 8 punct 8:punct _ # sent_id = weblog-juancole.com_juancole_20051126063000_ENG_20051126_063000-0040 # text = The trial begins again Nov.28. 1 The the DET DT Definite=Def|PronType=Art 2 det 2:det _ 2 trial trial NOUN NN Number=Sing 3 nsubj 3:nsubj _ 3 begins begin VERB VBZ Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin 0 root 0:root _ 4 again again ADV RB _ 3 advmod 3:advmod _ 5 Nov. November PROPN NNP Abbr=Yes|Number=Sing 3 obl:tmod 3:obl:tmod SpaceAfter=No 6 28 28 NUM CD NumForm=Digit|NumType=Card 5 nummod 5:nummod SpaceAfter=No 7 . . PUNCT . _ 3 punct 3:punct _ cCstjtd}t|dksJt|djdksJt|djdks$Jdtddd}tj|d}t|dks>Jt|djdksIJt|djdksTJt|djdks_JdS)NrrFrr<rdr=)rconll2multi_docsMULTIPLE_DOC_IDSrMrsrr_)docsr?rPrPrQtest_read_multiple_doc_idss  rat # text = This is a test # sent_id = 0 1 This this PRON DT Number=Sing|PronType=Dem 4 nsubj _ start_char=0|end_char=4 2 is be AUX VBZ Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Fin 4 cop _ start_char=5|end_char=7 3 a a DET DT Definite=Ind|PronType=Art 4 det _ start_char=8|end_char=9 4 test test NOUN NN Number=Sing 0 root _ start_char=10|end_char=14|SpaceAfter=No cCsHtjtd}t|}gdgdgdgdgg}||ks"JdS)Nr) rThisthisr DTzNumber=Sing|PronType=Demrr r r) risberVBZz5Mood=Ind|Number=Sing|Person=3|Tense=Pres|VerbForm=Finrcopr zstart_char=5|end_char=7) r rrr rzDefinite=Ind|PronType=Artrr#r zstart_char=8|end_char=9) rtestrr&NNz Number=Singrrr z'SpaceAfter=No|start_char=10|end_char=14)rrENGLISH_TEST_SENTENCE convert_dictrn)rf convertedrrPrPrQtest_convert_dicts r)5__doc__rrrzipfilerstanzastanza.utils.conllrstanza.models.common.docr stanza.testsmarkpipeline pytestmarkrTrUrYrmrorrrrwrxrrrrrrrrrrrrrrrrrrrrrrrrlstriprrrrrPrPrPrQs                      35