o hV/@s2dZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z m Z ddl mZmZddlmZddlmZddlZedZGd d d e ZGd d d Zd dZedkredd ZedWdn1sqwYeddZeej ej!dddZ"ee"eddZeej dSdS)z Supports for pretrained data. N) BaseVocab VOCAB_PREFIXUNK_ID)open_read_binaryopen_read_text)DEFAULT_MODEL_DIR)UnpicklingErrorstanzacs$eZdZddZfddZZS)PretrainedWordVocabcCs&t|j|_ddt|jD|_dS)NcSsi|]\}}||qSr ).0iwr r X/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/models/common/pretrain.py sz3PretrainedWordVocab.build_vocab..)rdata_id2unit enumerate_unit2idselfr r r build_vocabs zPretrainedWordVocab.build_vocabcs t|}|r|dd}|S)N  )supernormalize_unitreplace)runit __class__r rrs  z"PretrainedWordVocab.normalize_unit)__name__ __module__ __qualname__rr __classcell__r r rrr sr c@sneZdZdZdddZeddZed d Zd d Zd dZ dddZ ddZ e ddZ e dddZdS)Pretrainz/ A loader and saver for pretrained embeddings. NTcCs"||_||_||_||_||_dSN)filename _vec_filename _csv_filename _max_vocab _save_to_file)rr( vec_filename max_vocab save_to_file csv_filenamer r r__init__%s  zPretrain.__init__cCt|ds ||jS)N_vocab)hasattrloadr3rr r rvocab, zPretrain.vocabcCr2)N_emb)r4r5r8rr r remb2r7z Pretrain.embc Cs|jdurtj|jrzkz tj|jdddd}Wnty3tj|jdddd}tdYnwt d |jt |t sJtd |jd |vsRd |vrZtd |jt|d |_|d |_t |jtjrut|j|_WdSttfyty}z|js|jst d |j||\}}WYd}~n&d}~ww|js|jstd |j|jdurt d|j|\}}||_||_|jr|jdusJd| |jdSdS)NcS|Sr'r storagelocr r r=zPretrain.load..T weights_onlycSr:r'r r;r r rr>?r?Fa+The saved pretrain has an old format using numpy.ndarray instead of torch to store weights. This version of Stanza can support reading both the new and the old formats. Future versions will only allow loading with weights_only=True. Please resave the pretrained embedding using this version ASAP.zLoaded pretrain from {}zFile {} exists but is not a stanza pretrain file. It is not a dict, whereas a Stanza pretrain should have a dict with 'emb' and 'vocab'r9r6zFile {} exists but is not a stanza pretrain file. A Stanza pretrain file should have 'emb' and 'vocab' fields in its state dictzXPretrained file exists but cannot be loaded from {}, due to the following exception: {}zCPretrained file {} does not exist, and no text/xz file was providedz]Pretrained filename %s specified, but file does not exist. Attempting to load from text filez|sz'Pretrain.write_text.. N) openr9shapewritelenr6rtojoin)rr(headerfoutword_dimword_idxwordrowr r r write_textps    "zPretrain.write_textcCs|jdur||j|j\}}}n|jdur||j\}}ntdt|ttt|kr3td|jttkrW|jt|ttkrW|d|jtt}|d|j}t|}||fS)NzVector file is not provided.z8Loaded number of vectors does not match number of words.) r)read_from_filer+r* read_from_csvrMrkrr )rwordsr9failedr6r r rrVs  $zPretrain.read_pretrainc Cstd|t|}t|}|D]}dd|D}Wdn1s&wYt|}t|dd}tj|tt|ftj d}t |D]\}}tj dd|| dDtj d||tt<qId d|D} | |fS) zD Read vectors from CSV Skips the first row z/Reading pretrained vectors from csv file %s ...cSsg|]}|qSr r r liner r rrfsz*Pretrain.read_from_csv..NrrdtypecSg|]}t|qSr floatrdr r rrfcSsg|] }|dddqS)rrr)rryr r rrfs) rHrXrcsvreaderrkrEzerosrfloat32rtensor) r(fin csv_readerrzlinesrowscolsr9rrwr r rrvs   2zPretrain.read_from_csvc std|td}d}dg}d}d}t|k}t|D]^\}} z| } Wn ty5|d7}Yqw| } | s=q| | } |rSd}t | dkrSt | dq| dd krk|durct d nt d | }q|rw|dkswt ||kr||| qWdn1swYdurtd d |Ddt |} tj| t tftjd} |durtjdd| dDtjd| t<t|D]\}} tjdd| dDtjd| |t t<qɇfdd|D} |dkrtd|| | |fS)zR Open a vector file using the provided function and read from it. z&Reading pretrained vectors from %s ...z[ \t]+TNrrFzzFMore than one line in the pretrain! Keeping the most recent onez,Found an unk line while reading the pretraincss|]}t|VqdSr')rkrdr r r sz*Pretrain.read_from_file..r{cSr}r r~rdr r rrfrz+Pretrain.read_from_file..cSr}r r~rdr r rrfrcs g|] }d|d qS)rN)rmryrr rrfs z&Failed to read %d lines from embedding)rHrXrecompilerrdecodeUnicodeDecodeErrorrstripr]rkinterrorrIappendminrErrrrr)r(r.tab_space_patternfirstrrxunk_linefrrzpiecesrr9rwr rrrus\            (2  zPretrain.read_from_file)NNr&TN)Fr')r!r"r#__doc__r1propertyr6r9r5rYrtrV staticmethodrvrur r r rr%"s   )  r%c Cs|r|Stj|d|}tj|rtd||Std|tj|d|}tj|r=td||Std||ddkrstjt|dd| dd d }tj|rltd||Std||S) a` When training a model, look in a few different places for a .pt file If a specific argument was passsed in, prefer that location Otherwise, check in a few places: saved_models/{model}/{shorthand}.pretrain.pt saved_models/{model}/{shorthand}_pretrain.pt ~/stanza_resources/{language}/pretrain/{shorthand}_pretrain.pt z{}.pretrain.ptzFound existing .pt file in %sz$Cannot find pretrained vectors in %sz{}_pretrain.ptrarpretrainz{}.ptr) rBrCrmrJrDrHrIfindrr])wordvec_pretrain_filesave_dir shorthandlangdefault_pretrain_file pretrain_filer r rfind_pretrain_files&   $ r__main__ztest.txtrz3 2 a 1 1 b -1 -1 c 0 0 ztest.ptTr@)#rrrBrlzmaloggingnumpyrOrEr6rrrstanza.models.common.utilsrrstanza.resources.commonrpickler rF getLoggerrHr r%rr!rhrorjrprintr9r5rer r r rs:    K)