o h#@sdZddlZddlZddlZddlmZddlmZddlm Z m Z ddl m Z m Z mZddlmZmZddlmZdd lmZed Zd d ZGd ddeZdS)z; A trainer class to handle training and testing of models. N)nn)Trainer)utilsloss) load_bertload_bert_with_peftNoTransformerFoundationCache)build_peft_wrapperload_peft_wrapper)Tagger) MultiVocabstanzacsRfdd|ddD}|d}|d}|d}|d}|d}||||||fS) z& Unpack a batch from the data loader. cs"g|] }|dur |ndqSN)to).0bdeviceT/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/models/pos/trainer.py s"z unpack_batch..N r)batchrinputsorig_idx word_orig_idxsentlenswordlenstextrrr unpack_batchsr#c@sBeZdZdZdddZdddZdd d Zdd d Zdd dZdS)rz A trainer for training models. Nc CsD|dur|j||||dn@||_||_t|jd\}}d} |jdr4d|jd<d} t||jt| d}t|||dur>|jnd|d||||jd| d |_|j ||_t j |jd |j|jd d |jd fd|j dd|j dd|j ddd|_ i|_|j ddrddl} | |j dd|jd} | |jd<dSdS)N)argsfoundation_cache bert_modeluse_peftT bert_finetunepos adapter_name share_hid emb_matrixr,r%r&bert_tokenizerforce_bert_saved peft_nameoptimlrg?beta2gư>initial_weight_decaybert_learning_ratepeftF)betaseps weight_decayr6is_peftrbert_optimizer max_stepsbert_scheduler)loadr$vocabrr loggerr embmodelrrget_split_optimizerget optimizers schedulers transformersget_linear_schedule_with_warmup) selfr$rApretrain model_filerr%r&r/r1rIwarmup_schedulerrrr__init__s,  4R zTrainer.__init__FcCst|jj}t||\}}}}}} |\} } } } }}}}|r%|jn|j|jD]}| q/|| | | | |||||||| \}}|dkrN|S|j }|rW|S| t jj|j|jd|jD]}|qn|jD]}|qz|S)Nr7 max_grad_norm)nextrD parametersrr#evaltrainrGvalues zero_graddataitembackwardtorchrrclip_grad_norm_r$steprH)rKrrSrrrrr r!r"word word_mask wordcharswordchars_maskuposxposufeats pretrained optimizerr_loss_val schedulerrrrupdate?s*   $   zTrainer.updateTcstjj}t||\}}}}}|\} } } } } }}}j| d}| | | | | |||||| \}}fdd|dDfdd|dDfdd|dDfddt|D}|rvt ||}|S) Nrcg|] }jd|qS)rarAunmaprsentrKrrrcz#Trainer.predict..crj)rbrkrmrorrrdrpcrj)featsrkrmrorrrerpcs,g|]fddtDqS)cs.g|]}|||gqSrr)rj) feats_seqsi upos_seqs xpos_seqsrrrgs.z.Trainer.predict...)range)r)rur rwrx)rvrrgs,) rQrDrRrr#rSsizetolistryrunsort)rKrr|rrrrr!r"r]r^r_r`rarbrcrd batch_sizerfpreds pred_tokensr)rurKr rwrxrpredict[s  $ zTrainer.predictc sj}|rfdd|D}|D]}||=q|jjd}jddr=ddlm}|jjjj d|d <zt j ||dd t d |WdSttfyZtyu}zt d |d WYd}~dSd}~ww)Ncs&g|]}|ddjjvr|qS).r)splitrDunsaved_modules)rkrorrrps&z Trainer.save..)rDrAconfigr'Fr)get_peft_model_state_dictr* bert_lora)_use_new_zipfile_serializationzModel saved to {}zSaving failed... z continuing anyway.)rD state_dictkeysrAr$rFr8rr&r1rZsaverBinfoformatKeyboardInterrupt SystemExit Exceptionwarning) rKfilename skip_modules model_stateskippedrparamsrerrorrls*   z Trainer.savec Csz tj|dddd}Wntytd|w|d|_|dur,|j|d|jvr6d|jd<|d }|rGt d d|jd <d }d}|jd d rpd}t |jdd |\} } }t | ||jt|} t d|n$t dd|d Drt d|t|}d}t|jd|\} } t|d|_d} |jdr|dur|j} t dd|d Drt d|t|}t|j|j| |jd|| | ||d |_|jj|dd ddS)z Load a model from file, with preloaded pretrain embeddings. Here we allow the pretrain to be None or a dummy input, and the actual use of pretrain embeddings will depend on the boolean config "pretrain" in the loaded args. cSs|Srr)storagelocrrrszTrainer.load..T) weights_onlyzCannot load model from {}rNr&rz2Found peft weights for POS; loading a peft adapterr'Fr)zLoaded peft with name %scs|]}|dVqdSz bert_model.N startswithrxrrr zTrainer.load..rDzModel %s has a finetuned transformer. Not using transformer cache to make sure the finetuned version of the transformer isn't accidentally used elsewhererArLcsrrrrrrrrrr,r-)strict)rZr@ BaseExceptionrBerrorrr$rirFdebugrr anyrrrr load_state_dictrArCr rD) rKrrLr$r% checkpoint lora_weightsr0r1r&r/r.rrrr@sF         &z Trainer.load)NNNNNN)F)T)NN) __name__ __module__ __qualname____doc__rOrirrr@rrrrrs   r)rsysloggingrZrstanza.models.common.trainerr BaseTrainerstanza.models.commonrr%stanza.models.common.foundation_cacherrr stanza.models.common.peft_configr r stanza.models.pos.modelr stanza.models.pos.vocabr getLoggerrBr#rrrrs