o hz@svdZddlZddlmZddlmZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlmZddlmmmZddlmZddlmmZddlm Z e!dZ"dld d Z#edld d Z$edmddZ%eddZ&ddZ'ddZ(dlddZ)dnddZ*doddZ+dpd d!Z,d"d#Z-d$d%Z.d&d'Z/dqd)d*Z0dqd+d,Z1dqd-d.Z2d/d0Z3d1d2Z4d3d4Z5d5d6Z6d7d8Z7d9d:Z8dqd;d<Z9d=d>Z:drd?d@Z;dAdBZdGdHZ?dIdJZ@dKdLZAdMdNZBdOdPZCdQdRZDdsdTdUZEdVdWZFdXdYZGdZd[ZHd\d]ZId^d_ZJd`daZKdbdcZLdddeZMdfdgZNdhdiZOdjdkZPdS)tz Utility functions. N)Counter)contextmanager) lcode2lang)TRANSFORMER_NICKNAMES)UDErrorstanzac Cs|dd\}}t|}tj|d|}tj|d|}d}|dur9tj|||}tj|s8td||ntj|rB|}ntj|rK|}n td|||tj|d|} tj| d rl| d } | Stj| d rx| d } | S) z] Lookup the name of the word vectors file, given a directory and the language shorthand. _word2vecfasttextNzBWord vector type {} was specified, but directory {} does not existzICannot locate word vector directory for language: {} Looked in {} and {}z {}.vectors.xzz.txt)splitrospathjoinexistsFileNotFoundErrorformat) wordvec_dir shorthand wordvec_typelcodetcodelang word2vec_dir fasttext_dirlang_dirfilenamerU/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/models/common/utils.pyget_wordvec_files,   r ccsP|dur tjVdSt|ddd }|VWddS1s!wYdS)z Yields the given file if a file is given, or returns sys.stdout if filename is None Opens the file in a context manager so it closes nicely Nwutf-8encoding)sysstdoutopen)rfoutrrr output_stream:s  "r)r"ccs|dr$tj|d|d }|VWddS1swYdS|drGtj|d|d }|VWddS1s@wYdSt||d }|VWddS1s\wYdS)z Opens a file as an .xz file or .gz if it ends with .xz or .gz, or regular text otherwise. Use as a context eg: with open_read_text(filename) as fin: do stuff File will be closed once the context exits r rt)moder$N.gzr#)endswithlzmar'gzip)rr$finrrropen_read_textHs " ""r1c csp|dr#tj|dd }|VWddS1swYdS|drEtj|dd }|VWddS1s>wYdS|drt|C}|}t|dkr^tdt|d krhtd ||d }|VWdn1s}wYWddSWddS1swYdSt|dd }|VWddS1swYdS) aF Opens a file as an .xz file or .gz if it ends with .xz or .gz, or regular binary file otherwise. If a .zip file is given, it can be read if there is a single file in there Use as a context eg: with open_read_binary(filename) as fin: do stuff File will be closed once the context exits r rb)r+Nr,z.ziprzEmpty zip archiver z(zip file %s has more than one file in it) r-r.r'r/zipfileZipFilenamelistlen ValueError)rr0zin input_namesrrropen_read_binary_s2 " "    " "r:cCs ||kr|St||}||S)z Adjust the evaluation interval adaptively. If cur_dev_size <= thres_dev_size, return base_interval; else, linearly increase the interval (round to integer times of base interval). )round) cur_dev_sizethres_dev_size base_intervalalpharrrget_adaptive_eval_intervals r@c Cs|zt|}Wnty}ztd||d}~wwzt|}Wnty5}ztd||d}~wwt||}|S)NzCould not read %s)ud_evalload_conllu_filerevaluate)gold_conllu_filesystem_conllu_filegold_ude system_ud evaluationrrr ud_scoress rJcCstdd|Dr dS|dus#t|t|ks#Jdt|t||dur4t|tdd|DSt|tddt||DS)NcSsg|]}|dkqSrr.0xrrr z!harmonic_mean..rzEWeights has length {} which is different from that of the array ({}).cSsg|]}d|qS)r rrLrrrrOrPcss|] \}}||VqdSNr)rMrNr!rrr z harmonic_mean..)anyr6rsumzip)aweightsrrr harmonic_means 0 rYc Ksd}t|dkrdddd|D}|dkr2|d||||tjj|fd|||d |S|d krN|d ||||tjj|fd|||d |S|d krg|d |||tjj|f||d|S|dkr~|d||tjj |fd|i|S|dkr|d||||tjj|f|||d|S|dkr|d||||tjj|f|||d|S|dkr|d|tjj |fi|S|dkr|d||tjj |fd|i|S|dkrzddl m } Wnty} ztd| d} ~ ww|d|||| |f||ddd|S|d krEzddl} Wnty0} ztd!| d} ~ ww|d"|||| j|f||d|S|d#krwzddl} Wntyb} ztd$| d} ~ ww|d%|||| j|f||d|Std&|)'Nrz, css |] \}}d||fVqdS)z%s=%sNr)rMrNyrrrrRsz%dispatch_optimizer..amsgradz7Building Adam w/ amsgrad with lr=%f, betas=%s, eps=%f%sT)r\lrbetasepsamsgradwz8Building AdamW w/ amsgrad with lr=%f, betas=%s, eps=%f%ssgdz&Building SGD with lr=%f, momentum=%f%s)r]momentumadagradzBuilding Adagrad with lr=%f%sr]adamz,Building Adam with lr=%f, betas=%s, eps=%f%s)r]r^r_adamwz-Building AdamW with lr=%f, betas=%s, eps=%f%sadamaxzBuilding Adamax%sadadeltazBuilding Adadelta with lr=%f%s adabelief) AdaBeliefz]Could not create adabelief optimizer. Perhaps the adabelief-pytorch package is not installedz'Building AdaBelief with lr=%f, eps=%f%s)r]r_weight_decouplerectifymadgradzQCould not create madgrad optimizer. Perhaps the madgrad package is not installedz*Building MADGRAD with lr=%f, momentum=%f%smirror_madgradzXCould not create mirror_madgrad optimizer. Perhaps the madgrad package is not installedz0Building MirrorMADGRAD with lr=%f, momentum=%f%szUnsupported optimizer: {})r6ritemsdebugtorchoptimAdamAdamWSGDAdagradAdamaxAdadeltaadabelief_pytorchriModuleNotFoundErrorrlMADGRAD MirrorMADGRADr7r) name parameters opt_loggerr]r^r_rb extra_args extra_loggingrirGrlrrrdispatch_optimizersn          rg?g+?:0yE>Fc s| dur| nt} dd|D} d| dg}dd|D}t|dkr5| dkr5|d||| d| sd d|D}t|dkrk| durk|jjj}|| }g}t||D]|fd d|DqZt|dkr|dkr| d t|||||d |||d|dur||d d<n|dkr|d |j ||d|dur||d d<i}|dur||d<t ||f| ||||d|S)NcS8g|]\}}|jr|ds|ds|ds|qS bert_model.charmodel_forward.charmodel_backward. requires_grad startswithrMnprrrrO  z!get_optimizer..baseparam_group_nameparamscS.g|]\}}|jr|ds|dr|qSrrrrrrrrO  rcharlmrrr]cS$g|]\}}|jr|dr|qSrrrrrrrO$c0g|]\}}|jr|drd|vr|qSrz layer.%d.rrMr|param layer_numrrrO  z2Finetuning %d bert parameters with LR %s and WD %sbert weight_decayr~r]r^r_rb) loggernamed_parametersr6append bert_modelconfignum_hidden_layersrangeextendror}r)r|modelr]r^r_rbrbert_learning_ratebert_weight_decaycharlm_learning_rateis_peftbert_finetune_layersr~base_parametersr}charlm_parametersbert_parameters num_layers start_layerrrrr get_optimizers8    rc  s~dd|D} d| dg} dd|D}t|dkr-| dkr-| d||| dd }| sud d|D}t|dkre| d ure|jjj}|| }g}t||D]|fd d|DqTt|dkrtd |||dg}n d |j||dg}i}|d ur||d <dt || ft ||||d|i}|d ur|dkr|d ur||d <t ||ft ||||d||d<|S)zTSame as `get_optimizer`, but splits the optimizer for Bert into a seperate optimizercSrrrrrrrrO rz'get_split_optimizer..rrcSrrrrrrrrOrrrrNcSrrrrrrrrOrcrrrrrrrrOrrrgeneral_optimizerrrbert_optimizer) rr6rrrrrrr}rr)r|rr]r^r_rbrrrrrrrr}rrtrainable_parametersrrr optimizersrrrget_split_optimizer s8   "rcCs|jD]}||d<qdS)Nr]) param_groups) optimizernew_lr param_grouprrr change_lr6s  rcCs:g}t|D]\}}t|D] }||||qq|SrQ) enumeraterr)seq_lenswidthflatiljrrrflatten_indices:s  rcCs(||dks J|j|d|S)z+ Keep only the topk rows of grads. rN)sizedatazero_)gradtopkrrrkeep_partial_gradAsrTcCs6tj|s|rtd|tj|dddSdS)Nz(Directory {} does not exist; creating...T)exist_ok)rrrrinformakedirs)dverboserrr ensure_dirJs rcCsPt|d}tj||ddWdn1swY|r&td||S)Nr!)indentzConfig saved to file {})r'jsondumpprintr)rrroutfilerrr save_configQs rcCsHt| }t|}Wdn1swY|r"td||S)NzConfig loaded from file {})r'rloadrr)rrfrrrr load_configXs  rcCs@d}|D]\}}|d|t|7}qtd|ddS)Nz$Running with the following configs: z {} : {}  )rnrstrrr)rrkvrrr print_config_srcCs td|S)NNFD) unicodedata normalize)textrrrnormalize_textes rcCsfg}t||D])\}}g}|D]}|dkr||j|q| d}|||q||g7}q|S)zQ Unmap a list of list of indices, by optionally copying from src_tokens. rr )rVrid2word)indices src_tokensvocabresultindtokenswordsidxrrrunmap_with_copyhs  rcCsFg}|D]}tj|vr|tj}||d|g7}q||g7}q|S)z2 Prune decoded sequences after EOS token. N)constantEOSindex EOS_TOKEN)seqsoutsrrrrprune_decoded_seqsxs   rcCs&tj|vr|tj}|d|S|S)z$ Prune a decoded hypothesis N)rEOS_IDr)hyprrrr prune_hyps   rcCsBt|t|ks Jg}t||D] \}}||d|q|SrQ)r6rVr) data_listlensnlrrrrrprunes rcCslt|ts t|trt|tsJ|gtt|gt|}ddttt||dD}t|ddS)zt Sort a series of packed list, according to a ref list. Also return the original index before the sort. cSg|]}t|qSrlistrMtrrrrOrPzsort..)reverser N) isinstancetupler rr6rVsorted)packedrefr  sorted_packedrrrsorts"rcCsLt|t|ks Jdt|dkrgSddttt||D\}}|S)z: Unsort a sorted list, based on the original idx. 9Number of list elements must match with original indices.rcSrrrr rrrrOrPzunsort..)r6rVr) sorted_listoidxrunsortedrrrunsorts   rcs^|sggfSrtt|fdd|d}n tt|dd|d}tt|}|d|dfS)z Sort data and return both the data and the original indices. One useful application is to sort by length, which can be done with key=len Returns the data as a sorted list, then the indices of the original list. cs |dSNr rrNkeyrrs z#sort_with_indices..)rr cS|dSrrrrrrrr r)rrrrV)rrr orderedrrrrsort_with_indicess r!cCsg}d}d}t|D]A\}}t||kr/|dkr|||f|||df|d}d}q t|||krE|||f|}t|}q |t|}q |dkrY||t|f|S)a  Returns a list of intervals so that each interval is either <= batch_size or one element long. Long elements are not dropped from the intervals. data is a list of lists batch_size is how long to make each batch return value is a list of pairs, start_idx end_idx rr )rr6r)r batch_size intervalsinterval_start interval_sizerlinerrrsplit_into_batchess$   r'cCs@|dt|ks Jdddtt|dddD}||S)zR Unsort a sorted tensor on its 0-th dimension, based on the original idx. rrcSsg|]}|dqSrKrrLrrrrOrPz!tensor_unsort..cSrrrrrrrrrztensor_unsort..r)rr6rr) sorted_tensorrbackidxrrr tensor_unsortsr*cCsd|dur tdd}t|tj|t|t|tjr0tj|tj||S)zr Set a random seed on all of the things which might need it. torch, np, python random, and torch.cuda Nriʚ;) randomrandintrp manual_seednpseedcuda is_availablemanual_seed_all)r/rrrset_random_seeds        r3csrttrtdtrtddDt|tr,t|dtr,ttdd|D}tfdd|D}|S)Nrcs|] }|D]}|VqqdSrQrrMr[rNrrrrRz$find_missing_tags..csr4rQrr5rrrrRr6c3s|] }|vr|VqdSrQrrL known_tagsrrrRrS)r r setr)r8 test_tags missing_tagsrr7rfind_missing_tagss r<cCs0t||}t|dkrtd||dSdS)z Print a warning if any tags present in the second list are not in the first list. Can also handle a list of lists. rz6Found tags in {} missing from the expected tag set: {}TF)r<r6rwarningr)r8r: test_set_namer;rrrwarn_missing_tagss  r?cCsv|rtj|d}||kr|Stj||Stj|d}||kr*tj||}|dr7|dddS|dS)z Will return a recommended checkpoint name for the given dir, save_name, optional checkpoint_name For example, can pass in args['save_dir'], args['save_name'], args['checkpoint_save_name'] rz.ptNz_checkpoint.pt _checkpoint)rrr rr-)save_dir save_namecheckpoint_name model_dirrrrrD s rDcCstjrdSdS)zH Pick a default device based on what's available on this system r0cpu)rpr0r1rrrrdefault_devices rGcCs@|jdttdd|jdddddd |jd ddd d d d S)z? Add args which specify cpu, cuda, or arbitrary device z--devicez7Which device to run on - use a torch device string name)typedefaulthelpz--cudadevice store_constr0z Run on CUDA)destactionconstrJz--cpurFzIgnore CUDA and run on CPUN) add_argumentrrG)parserrrradd_device_args&srRcCs$ddl}td|||}|S)NrzLoading elmo: %s)elmoformanylangsrrEmbedder) elmo_modelrSrrr load_elmo.s rVtrainingcsNttjr tt}fdd|D}|d|d|dS)zF For record keeping purposes, log the arguments when training csg|] }d||fqS)z%s: %sr)rMrargsrrrO>sz%log_training_args..zARGS USED AT %s TIME: %s rN) r argparse Namespacevarsrkeysrupperr)rY args_loggerr|r] log_linesrrXrlog_training_args7s  racCsld}|ddur|ddurd}|ddr |ds|d r d}|d r4|d tvr2t|d }|Sd }|S) a Return the generic name of the biggest embedding used by a model. Used by POS and depparse, for example. TODO: Probably will make the transformer names a bit more informative, such as electra, roberta, etc. Maybe even phobert for VI, for example nocharlmwordvec_pretrain_fileN wordvec_file nopretrainrTcharlm_forward_filecharlm_backward_filer transformer)getr)rY embeddingrrrembedding_nameAs   rkc Ks:t|}d}d}|ddrd}d|vrd|d}d}|ddr+|ddr+d }d}|ddr>|ddr.rZzSpaceAfter=YeszSpaceAfter=No~z SpaceAfter=z SpacesAfter=rr r)r rTrrrrrrmisc_to_space_afters   rc Csdg}g}|D]\}}|jr#||dt|d|fq tdd|D}tdd|D}dt|dt|d }|D] }|||qFt d |dS) NzNORMS FOR MODEL PARAMTERSz%.6gz%dcs|] }t|dVqdS)rNr6rLrrrrRrSzlog_norms..csr)r NrrLrrrrRrSz %-zs %zs %sr) rrrrpnormitemnumelmaxrrrr) rlinesrr|rname_lennorm_len line_formatr&rrr log_normss&rcCsl|r|d||jn |r||_n|dur+|d||D]\}}d|_q"nd|_|d|dS)NrFbert_tokenizer)add_unsaved_modulertrainrr)rrrroforce_bert_savedr parameterrrrattach_bert_models   rcCsDz|dW|Sty!tjt}|dd|d}Y|Sw)z If the given name doesn't have %d in it, add %4d at the end of the filename This way, there's something to count how many models have been saved r rz_%04d) TypeErrorrrsplitextmodel_save_each_file) base_filenamerrrrbuild_save_each_filename$s  rrQ)r")NNNN) rrrNrNrFNN) rrrNrNrFN)T)NF)rW)Q__doc__rZ collectionsr contextlibrr/rloggingr.rr+rvr%rr3rpnumpyr.stanza.models.common.constantr%stanza.models.common.seq2seq_constantmodelscommonseq2seq_constantr!stanza.resources.default_packagesrstanza.utils.conll18_ud_evalutilsconll18_ud_evalrAr getLoggerrr r)r1r:r@rJrYrrrrrrrrrrrrrrrrrr!r'r*r3r<r?rDrGrRrVrarkr|rrrrrrrrrrrrrs          "  7 +-          9