o hh=@sdZddlmZddlZddlZddlZddlmZddlm m m m Z dZ ddZdd Zd e e fd d Zd dZddZd e e fddZddZd e e fddZd,ddZddZddZd d!Zd"d#Zd$d%Zd&d'Zd(d)Zd*d+ZdS)-z Utils for the processing of NER datasets These can be invoked from either the specific dataset scripts or the entire prepare_ner_dataset.py script ) defaultdictN)Document)traindevtestcCsxg}d}|D]3}|dkr||d}q|r,|ds |dr,|d|ddq|d|ddd}q|S)NFOB-S-I-T)append startswith)tagsnew_tags in_entitytagrZ/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/utils/datasets/ner/utils.py bioes_to_bios rc CstD]E}tj|d||f}tj|d||f}t|ddd}g}|D]}dd|D} t| } ddt|| D}||q%t||qd S) z Convert BIOES files back to BIO (not BIO2) Useful for preparing datasets for CoreNLP, which doesn't do great with the more highly split classes z %s.%s.bioesz %s.%s.bior) text_columnannotation_columncSsg|]}|dqS)rr.0xrrr /z(convert_bioes_to_bio..cSsg|] \}}|d|fqS)rr)rryrrrr1N) SHARDSospathjoinread_tsvrzipr write_sentences) base_input_pathbase_output_path short_nameshardinput_filenameoutput_filenameinput_sentences new_sentencessentencerrrrconvert_bioes_to_bio"s  r/bioc Cst||D]M\}}tj|d|||f}tj|s9tj|d||f} tj| r/| }n td|||| ftj|d||f} td|| ft|| qdS)a Convert BIO files to json It can often be convenient to put the intermediate BIO files in the same directory as the output files, in which case you can pass in same path for both base_input_path and base_output_path. This also will rewrite a BIOES as json %s.%s.%sz%s.%sz*Cannot find %s component of %s in %s or %s %s.%s.jsonzConverting %s to %sN) r$r r!r"existsFileNotFoundErrorprintprepare_ner_fileprocess_dataset) r&r'r(suffix shard_namesshards input_shard output_shardr* alt_filenamer+rrrconvert_bio_to_json6s   r>cCs6t}|D]}|D]}|D] \}}||q q q|S)z~ return the set of tags used in these datasets datasets is expected to be train, dev, test but could be any list )setadd)datasetsrdatasetr.wordrrrrget_tagsLs  rDc Cstjtj|dddt|dddE}t|D]7\}}t|D])\}}t|dkr1|dd}z |d |Wq!tyJtd |||fw|d qWddS1s\wYdS) z8 Write exactly one output file worth of dataset rT)exist_okwutf-8encodingr Nz%s %s z0Unable to process sentence %d word %d of file %s ) r makedirsr!splitopen enumeratelenwrite TypeError)r+rBfoutsent_idxr.word_idxrCrrrr%Ys    "r%c CsNt||D]\}}tj|d|||f}t||qt||||||ddS)a write all three pieces of a dataset to output_dir datasets should be 3 lists: train, dev, test each list should be a list of sentences each sentence is a list of pairs: word, tag after writing to .bio files, the files will be converted to .json r1)r9r:N)r$r r!r"r%r> rA output_dirr(r8r9r:r)rBr+rrr write_datasetis  rWcCsg}|D]}g}|D]}|d|d|dd}||q ||qt|ddd}tj||ddWddS1s?wYdS) Nrrr )textner multi_nerrFrGrHindent)r rMjsondump)r+rB json_datasetr. json_sentencerCrRrrrwrite_multitag_jsonzs  "rac Csnt||D]\}}tj|d|||f}t||qt||D]\}}tj|d||f}t||q dS)Nr1r2)r$r r!r"r%rarUrrrwrite_multitag_datasets  rbTF c Csft|dd }|} Wdn1swYdd| D} g} g} t| D]\} } | s9| r8| | g} q)|rA| drAq)| |}z||}Wntyb}z td|| | f|d}~ww|dkrhq)z||}Wn!ty}z|r{d}n td || | f|WYd}~nd}~ww|r||}|r|||<| |q)| ||fq)| r| | | S) z Read sentences from a TSV file Returns a list of list of (word, tag) If keep_broken_tags==True, then None is returned for a missing. Otherwise, an IndexError is thrown rGrHNcSsg|]}|qSr)striprrrrrrzread_tsv..#z,Could not find word index %d at line %d |%s|–z+Could not find tag index %d at line %d |%s|)rM readlinesrNr r rL IndexError)filenamerrremap_fn skip_commentskeep_broken_tagskeep_all_columns separatorfinlines sentencescurrent_sentenceline_idxlinepiecesrCerrrrr#sR        r#cCs$t|}t|}t||||dS)N)r listdirsortedrandom_shuffle_files) input_dirrVr( input_filesrrrrandom_shuffle_directorys r|c Csvi}|D]}|dd}||vrtd|||f|||<qt|t|ks*Jg}g}g} |D]0} | dd}|d}t|dt} | dkrS|| q2| dkr]|| q2| | q2tdt|t|t| ft|t|t| t|ksJ||| g} g} | D]}g}|D]} |tt j || dd q| |qt | ||t|t|t| fS) aU Shuffle the files into different chunks based on their filename The first piece of the filename, split by ".", is used as a random seed. This will make it so that adding new files or using a different annotation scheme (assuming that's encoding in pieces of the filename) won't change the distibution of the files .rz.Multiple files with the same prefix: %s and %sz.txt.4class.tsvr gffffff?g?z.Train files: %d Dev files: %d Test files: %dr) rL ValueErrorrOrandomseedr r5extendr#r r!r"rW)rzr{rVr( input_keysfr train_files dev_files test_filesrilocation file_listsrAfilesrBrrrrys>      $   rycst|}t|}tt}|D]&}|D]}||D] }||r$nqqntd||||qd} d} d} |D]-}t t dt |||ft ||||d|f\} } }| | 7} | | 7} | |7} q@t t d| | | ffdd|D}t |||dS)Nz=Could not assign %s to any of the divisions in the prefix_maprzProcessing %d files from %s%s-%sz?After shuffling: Train files: %d Dev files: %d Test files: %dcsg|]}d|fqS)rr)rdivisionr(rrrsz.random_shuffle_by_prefixes..) r rwrxrlistkeysr r~r r5rOrycombine_dataset)rzrVr( prefix_mapr{file_divisionsrirprefixnum_train_files num_dev_filesnum_test_filesd_traind_devd_testdataset_divisionsrrrrandom_shuffle_by_prefixess6        rc Csg}tD]B}g}|D]6}d||f}tj||} t| dd} t| } dd| D} || Wdn1s;wYq ||qt |||dS)Nr2rGrHcSsg|] }dd|DqS)cSsg|] }|d|dfqS)rXrYr)rrCrrrr'rz.combine_dataset...r)rr.rrrr'sz#combine_dataset..) rr r!r"rMr]loadrr rW) rzrVinput_datasetsoutput_datasetrAr) full_dataset input_datasetr* input_pathrorB convertedrrrrs    rcCs d}t}g}i}t|ddm}|D]I}|}|drq|s!q|dr?|dur.|||<g}|dddd}q|sGtd |||||vrVtd || |q|rl|rt|||<Wd|SWd|SWd|S1swY|S) z Read a prefix file such as the one for the Worldwide dataset the format should be africa: af_ ... asia: cn_ ... NrGrHre: _zBFound a prefix before the first label was assigned when reading %szFound the same prefix twice! %s) r?rMrdr endswithlowerreplace RuntimeErrorr r@)destination_file destinationknown_prefixesprefixesrrortrrrread_prefix_file,sD          rcCsBt|}tt|}t|WdS1swYdS)zj Read entities from a file, return a list of (text, label) Should work on both BIOES and BIO N)rMrr]rlist_doc_entities)rirodocrrrread_json_entitiesXs $rcCsg}|jD]}g}d}|jD]}|jdks|jdr;|jdr'||j|r:|dus/J|||fg}d}n||jdrq|durj|dkrj||jddkrj|rj|dusZJ|||fg}|jdd}||jnF|jds}|jdr|r|dusJ|||fg}d}||j|jdd}|jdr|dusJ||g}d}ntd|jdd}q|r|dusJ|||fqd d |D}|S) zO Return a list of (text, label) Should work on both BIOES and BIO NrzE-r r rr z)Expected BIO(ES) format in the json file!cSs g|] }t|d|dfqS)rr)tuplerrrrrs z%list_doc_entities..)rqtokensrYr r rXr)rentitiesr.current_entityprevious_labeltokenrrrrcsX      "       rc Gsg}|D] }t|}t|}||Wdn1swYqt|d}tj||ddWddS1s>wYdS)z; Combine multiple NER json files into one NER file NrFr r[)rMr]rrr^)r+input_filenamesrrironew_docrRrrr combine_filess    "r)NTFFrc)__doc__ collectionsrr]r rstanza.models.common.docr*stanza.utils.datasets.ner.prepare_ner_fileutilsrArYr6rrr/r>rDr%rWrarbr#r|ryrrrrrrrrrrs2    52", 1