o –h£7ã@s<dZddlZddlZddlZddlZddlZddlmZddlZddlZ ddl Z ddl Z ddl m Z m Z ddlmZddlmZddlmZddlmZmZdd lmZddlmmmZdd lTdd lm Z dd l!m"Z"e #d ¡Z$dd„Z%ddd„Z&ddd„Z'dd„Z(dd„Z)dd„Z*dd„Z+e,dkrœe'ƒdSdS)a, Entry point for training and evaluating a lemmatizer. This lemmatizer combines a neural sequence-to-sequence architecture with an `edit` classifier and two dictionaries to produce robust lemmas from word forms. For details please refer to paper: https://nlp.stanford.edu/pubs/qi2018universal.pdf. éN)Údatetime)ÚnnÚoptim)Ú DataLoader)ÚVocab)ÚTrainer)ÚscorerÚedit)Úutils)Ú*)ÚCoNLL)Ú_training_loggingÚstanzacCst ¡}|jdtddd|jdtddd|jdtddd|jdtdd d|jd tdd d|jd d d d gd|jdtdd|jddddd|jdddd|jdtdd|jdtdd|jd td!d|jd"td#d|jd$td#d|jd%tdd|jd&td!d|jd'd(gd)¢d*d+|jd,tdd|jd-td#d|jd.d/dd0d|jd1tttjƒd|jd2td3d|jd4d5dd6d|jd7d8dd9d|jd:dd;d|jdtdd?d|jd@tddAd|jdBtd3dCd|jdDtdEdFd|jdGtdHdId|jdJtdKd|jdLtdMdNd|jdOtdPd|jdQtdd|jdRtdSdTd|jdUtdVdWd|jdXtdYdZd|jd[td\d]d|jd^d_dd`da|jdbtdcdt   |¡|jdddded|jdfddgdh|S)iNz --data_dirz data/lemmazDirectory for all lemma data.)ÚtypeÚdefaultÚhelpz --train_filezInput file for data loader.z --eval_filez --output_filezOutput CoNLL-U file.z --gold_filez--modeÚtrainÚpredict)rÚchoicesz --shorthandz/Shorthand for the dataset to use. lang_dataset)rrz --no_dictÚ ensemble_dictÚ store_falsezADo not ensemble dictionary with seq2seq. By default use ensemble.)ÚdestÚactionrz --dict_onlyÚ store_truez)Only train a dictionary-based lemmatizer.)rrz --hidden_diméÈ)rrz --emb_dimé2z --num_layerséz --emb_dropoutgà?z --dropoutz --max_dec_lenz --beam_sizez --attn_typeÚsoft)rÚmlpÚlinearÚdeepzAttention type)rrrz --pos_dimz --pos_dropoutz --no_editr zODo not use edit classifier in lemmatization. By default use an edit classifier.z --num_editz--alphagð?z--no_posÚposz:Do not use UPOS in lemmatization. By default UPOS is used.z --no_copyÚcopyzhDo not use copy mechanism in lemmatization. By default copy mechanism is used to improve generalization.z--charlmzVTurn on contextualized char embedding using pretrained character-level language model.z--charlm_shorthandz=Shorthand for character-level language model training corpus.z--charlm_forward_filez$Exact path to use for forward charlmz--charlm_backward_filez%Exact path to use for backward charlmz--sample_trainzSubsample training data.z--optimÚadamzsgd, adagrad, adam or adamax.z--lrgü©ñÒMbP?z Learning ratez --lr_decaygÍÌÌÌÌÌì?z --decay_epochéz&Decay the lr starting from this epoch.z --num_epoché<z --batch_sizez--max_grad_normg@zGradient clipping.z --log_stepézPrint log every k steps.z --save_dirzsaved_models/lemmazRoot dir for saving models.z --save_namez%{shorthand}_{embedding}_lemmatizer.ptzFile name to save the modelz --caselessFznLowercase everything first before processing. This will happen automatically if 100%% of the data is caseless)rrrz--seediÒz--wandbzStart a wandb session and write the results of training. Only applies to training. Use --wandb_name instead to specify a namez --wandb_namezWName of a wandb session to start when training. Will default to the dataset short name)rr) ÚargparseÚArgumentParserÚ add_argumentÚstrÚintÚfloatÚlenr Ú EDIT_TO_IDr Úadd_device_args)Úparser©r1úS/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/models/lemmatizer.pyÚbuild_argparse!s\ r3cCsPtƒ}|j|d}|jrd|_t|ƒ}|dr |d d¡dnd}||d<|S)N©ÚargsTÚ shorthandÚ_rÚÚlang)r3Ú parse_argsÚ wandb_nameÚwandbÚvarsÚsplit)r5r0r9r1r1r2r:Zs r:cCsPt|d}t |d¡t d |d¡¡|ddkr"t|ƒdSt|ƒdS)Nr4ÚseedzRunning lemmatizer in {} modeÚmoder)r:r Úset_random_seedÚloggerÚinfoÚformatrÚevaluater4r1r1r2Úmaings    rFcCs4|jD]}|jD]}|j ¡|jkrdSqqdS)NFT)Ú sentencesÚwordsÚtextÚlower)ÚdocÚsentenceÚwordr1r1r2Ú all_lowercasess  ÿÿrNcCsbd}|dr |dr d}|dj|d|d}tj |¡d}| |d¡s/tj |d|¡}|S) NÚnocharlmÚcharlmÚcharlm_forward_fileÚ save_namer6)r6Ú embeddingrÚsave_dir)rDÚosÚpathr>Ú startswithÚjoin)r5rSÚ model_fileÚ model_dirr1r1r2Úbuild_model_filenamezsÿr[c#CsØt d |d¡¡tj|dd}t||d|dd}|j}|dj|d<|d j|d <tj|d d}t||d||d d }t  |d¡t |ƒ}t d|¡|d}|d}t  |¡t |ƒdksjt |ƒdkrqt  d¡dS|ds‚t|ƒr‚t d¡d |d<t d|¡t|||dd} t d¡|  | ¡¡t d¡|  |j ttg¡¡} |j tg| ¡t |j|¡t ||¡\} } } t d | d¡¡| dd¡rÜ|  |¡dS|dr ddl} |drí|dnd |d!}| j||d"| jj d#d$d%| jj d&d'd%t d(¡d}t |ƒ|d)}g}g}|d*}t! !¡}d+}t"d,|d)d,ƒD]}d}t#|ƒD]=\}}t! !¡}|d,7}| j$|dd-}||7}||d.dkrut! !¡|}t | t% &¡ 'd/¡||||d)|||¡¡q9t d¡g} g}t#|ƒD]\}}|  (||d0¡\}}| |7} |durŸ||7}q„| j)|j tg¡| |d1} | d2d¡rÆt d3¡|  *|j ttg¡| ¡} |j tg| ¡t |j|¡t ||¡\} } } ||j+|d}t d4 ||| ¡¡|drþ|  ,|| d5œ¡|d,ks | t-|ƒkr|  |¡t d6¡| }||d7kr6| |d8kr6|d9d:vr6||d;9}|  .|¡|| g7}t d<¡q0t d= |¡¡|drS|  /¡t-|ƒdt0 1|¡d,}!}"t d> |!|"¡¡dS)?Nz$[Loading data with batch size {}...]Ú batch_sizeÚ train_file©Ú input_fileF)Ú evaluationÚcharÚ vocab_sizer!Úpos_vocab_sizeÚ eval_fileT©Úvocabr`rTzUsing full savename: %sÚ output_fileÚ gold_filerz5[Skip training because no training data available...]ÚcaselesszBBuilding a caseless model, as all of the training data is caselesszBuilding lemmatizer in %sÚdevice)r5rfrjz)[Training dictionary-based lemmatizer...]zEvaluating on dev set...zDev F1 = {:.2f}édÚ dict_onlyr<r;z %s_lemmatizerr6)ÚnameÚconfigÚ train_lossÚmin)ÚsummaryÚ dev_scoreÚmaxz&[Training seq2seq-based lemmatizer...]Ú num_epochÚlrzJ{}: step {}/{} (epoch {}/{}), loss = {:.6f} ({:.3f} sec/batch), lr: {:.6f}r)ÚevalÚlog_stepz%Y-%m-%d %H:%M:%SÚ beam_size©Úeditsrz'[Ensembling dict with seq2seq model...]z1epoch {}: train_loss = {:.6f}, dev_score = {:.4f})rorrznew best model saved.Ú decay_epochéÿÿÿÿr)ÚsgdÚadagradÚlr_decayr8zTraining ended with {} epochs.z#Best dev F1 = {:.2f}, at epoch = {})2rBrCrDr Ú conll2docrrfÚsizer Ú ensure_dirr[Ú print_configr-ÚwarningrNrÚ train_dictÚraw_dataÚ predict_dictrKÚgetÚTEXTÚUPOSÚsetÚLEMMAÚwrite_doc2conllrÚscoreÚsaver<ÚinitÚrunÚ define_metricÚtimeÚrangeÚ enumerateÚupdaterÚnowÚstrftimerÚ postprocessÚensembleÚ num_examplesÚlogrsÚ update_lrÚfinishÚnpÚargmax)#r5Ú train_docÚ train_batchrfÚdev_docÚ dev_batchrYÚsystem_pred_filerhÚtrainerÚ dev_predsr7Údev_fr<r;Ú global_stepÚ max_stepsÚdev_score_historyÚbest_dev_predsÚ current_lrÚglobal_start_timeÚ format_strÚepochroÚiÚbatchÚ start_timeÚlossÚdurationÚ dev_editsÚpredsrzrrÚbest_fÚ best_epochr1r1r2r…s¼           ÿ€  €        rcCsø|d}|d}t|ƒ}t||d|d}|j|j}}|D]}| d¡s.| d¡s.|dvr4||||<qt d |d ¡¡tj |d d }t ||d ||d d } t | ƒdkrdt  d |d¡¡dS|  | j ttg¡¡} | dd¡rx| } nWt d¡g} g} t| ƒD]\} }| ||d¡\}}| |7} |durŸ| |7} q…|j| j tg¡| | d} | dd¡rÄt d¡| | j ttg¡| ¡} | ¡rÏ| | j| ¡} | j tg| ¡t | j|¡|durút ||¡\}}}t d |d|d¡¡dSdS)Nrgrhrj)rYrjr5Ú_dirÚ_file)r6z"Loading data with batch size {}...r\rdr^TrerzDSkip evaluation because no dev data is available... Lemma score: {} r6rlFzRunning the seq2seq model...rxryrz,[Ensembling dict with seq2seq lemmatizer...]z*Finished evaluation Lemma score: {} {:.2f}rk)r[rr5rfÚendswithrBrCrDr r€rr-r„r‡rKrˆr‰rŠr•rr™ršÚhas_contextual_lemmatizersÚupdate_contextual_predsr‹rŒrrrŽ)r5r¥rhrYr¦Ú loaded_argsrfÚkrKr²Ú dict_predsr·rzr±ÚbÚpsÚesr7rŽr1r1r2rEsN €   €  ýrEÚ__main__)N)-Ú__doc__ÚloggingÚsysrUÚshutilr“rr'ÚnumpyrŸÚrandomÚtorchrrÚstanza.models.lemma.datarÚstanza.models.lemma.vocabrÚstanza.models.lemma.trainerrÚstanza.models.lemmarr Ústanza.models.commonr Ú%stanza.models.common.seq2seq_constantÚmodelsÚcommonÚseq2seq_constantÚconstantÚstanza.models.common.docÚstanza.utils.conllr Ú stanza.modelsr Ú getLoggerrBr3r:rFrNr[rrEÚ__name__r1r1r1r2Ús@         9   }6 ÿ