o
    h                     @   s&  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlT ejjejj gZ!e"dZ#e#$ej% dZ&e&dddZ'G dd dZ(dS )    )defaultdictN)nn)optim)Pipeline)constituency_parser)pretrain)	load_bertload_tokenizer)FoundationCache)set_random_seed)
lstm_model)
Transition)parser_training)trainer)tree_reader)*zstanza.constituency.trainera=  
( (S
    (VP (VBG Enjoying)
      (NP (PRP$  my) (JJ favorite) (NN Friday) (NN tradition)))
    (. .)))

( (NP
    (VP (VBG Sitting)
      (PP (IN in)
        (NP (DT a) (RB stifling) (JJ hot) (NNP South) (NNP Station)))
      (VP (VBG waiting)
        (PP (IN for)
          (NP (PRP$  my) (JJ delayed) (NNP @MBTA) (NN train)))))
    (. .)))

( (S
    (NP (PRP I))
    (VP
      (ADVP (RB really))
      (VBP hate)
      (NP (DT the) (NNP @MBTA)))))

( (S
    (S (VP (VB Seek)))
    (CC and)
    (S (NP (PRP ye))
      (VP (MD shall)
        (VP (VB find))))
    (. .)))
)treebankc          
      G   st   t |}|dd  }g }d| gt| }t|}t }|d }t||||||\}}	}	}	t|j	t
js8J |S )N--wordvec_pretrain_file	load_name)r   
read_treeslistr   
parse_argsr
   r   build_trainer
isinstancemodelr   	LSTMModel)
wordvec_pretrain_filer   argstrain_trees	dev_treessilver_treesfoundation_cachemodel_load_namer   _ r%   a/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/tests/constituency/test_trainer.pyr   =   s   

r   c                   @   sD  e Zd Zejdddd Zejdddd Zejdddd Zd	d
 Zdd Z	dd Z
dd Zdd ZdJddZdd Zdd Zdd Zdd ZdKd d!Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0d1 Zd2d3 Zd4d5 Zd6d7 Zd8d9 Zd:d; Zd<d= Z d>d? Z!d@dA Z"dBdC Z#dDdE Z$dFdG Z%dHdI Z&dS )LTestTrainerclass)scopec                 C   s
   t  dS )Nz/in/tiny_emb.pt)TEST_WORKING_DIR)selfr%   r%   r&   r   O   s   
z!TestTrainer.wordvec_pretrain_filec                 C   6   d}t |\}}t|d}|| || |S )a  
        Download the tiny-random-xlnet model and make a concrete copy of it

        The issue here is that the "random" nature of the original
        makes it difficult or impossible to test that the values in
        the transformer don't change during certain operations.
        Saving a concrete instantiation of those random numbers makes
        it so we can test there is no difference when training only a
        subset of the layers, for example
        z%hf-internal-testing/tiny-random-xlnetztiny-random-xlnetr   strmktempsave_pretrained)r+   tmp_path_factory
xlnet_namexlnet_modelxlnet_tokenizerpathr%   r%   r&   tiny_random_xlnetS   s   

zTestTrainer.tiny_random_xlnetc                 C   r,   )z
        Download the tiny-random-bart model and make a concrete copy of it

        Issue is the same as with tiny_random_xlnet
        z$hf-internal-testing/tiny-random-bartztiny-random-bartr-   )r+   r1   	bart_name
bart_modelbart_tokenizerr5   r%   r%   r&   tiny_random_bartf   s   

zTestTrainer.tiny_random_bartc                 C   s   d|g}t | dS )zJ
        does nothing, just tests that the construction went okay
        r   N)r   )r+   r   r   r%   r%   r&   test_initial_modelt   s   zTestTrainer.test_initial_modelc                 C   s   t jtd?}t|}|jj}tj|d}|	| tj
|s#J ||}|jj}||ks2J tdd |D s=J W d   dS 1 sHw   Y  dS )z
        Just tests that saving and loading works without crashs.

        Currently no test of the values themselves
        (checks some fields to make sure they are regenerated correctly)
        dir	parser.ptc                 s   s    | ]}t |tV  qd S N)r   r   .0xr%   r%   r&   	<genexpr>       z3TestTrainer.test_save_load_model.<locals>.<genexpr>N)tempfileTemporaryDirectoryr*   r   r   transitionsosr5   joinsaveexistsloadall)r+   r   
tmpdirnametrrG   filenametr2trans2r%   r%   r&   test_save_load_model|   s   

"z TestTrainer.test_save_load_modelc                 C   s  t jtdt}td g d}t|g|R  }tj|d}|| td dddd	d
ddddd|g}t|g|R  }t	
|jjj|jjjsJJ t	
|jjd j|jjd js\J t	
t	j|jjjt	j|jjjsrJ W d   dS 1 s}w   Y  dS )z\
        Test that starting a trainer with --relearn_structure copies the old model
        r<   i  --pattn_num_layers0--lattn_d_projrV   --hidden_size20--delta_embedding_dim10r>   i  rU   1rW   rV   rX   rY   rZ   r[   z--relearn_structurez--load_namer   N)rE   rF   r*   r   r   rH   r5   rI   rJ   torchallcloser   delta_embeddingweightoutput_layerslinalgnorm	word_lstmweight_ih_l0)r+   r   rN   r   rO   rP   rQ   r%   r%   r&   test_relearn_structure   s   
$."z"TestTrainer.test_relearn_structurec                 C   s   t j|d}t|ddd}|t |t W d    n1 s#w   Y  t j|d}t|ddd}|t W d    ||fS 1 sIw   Y  ||fS )Nz	train.mrgwzutf-8)encodingzeval.mrg)rH   r5   rI   openwriteTREEBANK)r+   rN   train_treebank_filefouteval_treebank_filer%   r%   r&   write_treebanks   s   

zTestTrainer.write_treebanksc                 G   s   g dddddddddd	d
d|d|d|dddddt j|dd|d|dddddd}|t| }t|}d |d< |S )NrU   rV   z--pattn_d_model128rW   z--use_lattnrX   rY   rZ   r[   r   z
--data_dirz
--save_dirz--save_nameztest.ptz--save_each_startz--save_each_nameeach_%02d.ptz--train_filez--eval_filez--epoch_size6z--train_batch_size3z--shorthanden_testwandb)rH   r5   rI   r   r   r   )r+   r   rN   rl   rn   additional_argsr   r%   r%   r&   training_args   sb   .
zTestTrainer.training_args   NFc                 C   s*  |du rg }|dd| g7 }|  |\}}	|r|dt|	g7 }| j||||	g|R  }
|
d }|s;tj|
d r;J tddd	t|d
}t	|
d|g}|j
 D ]}|jrm|
d durft|jdkseJ qQ|jdu smJ qQtj|
d sxJ |j
j}tjj|
d d	|j|j
jd}|jdusJ |jdusJ |jdksJ |j
 D ]}|jr|jdu sJ qtjj|
d d	|j|j
jd}|jdusJ |jdusJ |j|ksJ td|d D ]3}|| }tj|sJ tjj|d	|j|j
jd}|j|ksJ |j|rd| nd| ksJ q|
|fS )z
        Runs a test of the trainer for a few iterations.

        Checks some basic properties of the saved model, but doesn't
        check for the accuracy of the results
        Nz--epochsz%dz--silver_filesave_each_name	save_nameenztokenize, posT)lang
processorstokenize_pretokenizedr=   r"   grad_clipping   )load_optimizerr"   	peft_namecheckpoint_save_name      )ro   r.   rw   rH   r5   rK   r   TEST_MODELS_DIRr   trainr   
parametersrequires_gradlen_backward_hooksr   r   TrainerrL   r"   	optimizer	schedulerepochs_trainedrangebatches_trained)r+   r   rN   
num_epochs
extra_args
use_silver	exists_okr"   rl   rn   r   	each_nameretag_pipelinetrained_modelpr   rO   i
model_namer%   r%   r&   run_train_test   sN   $zTestTrainer.run_train_testc                 C   s>   t jtd}| || W d   dS 1 sw   Y  dS )L
        Test the whole thing for a few iterations on the fake data
        r<   NrE   rF   r*   r   r+   r   rN   r%   r%   r&   
test_train   s   "zTestTrainer.test_trainc           	      C   s,  t jtd=}ddg}| j||d|d\}}|j}dd | D }t|dks,J d	|D ]\}}|jd
ks;J dq.W d   n1 sFw   Y  t jtd;}ddg}| j||d|d\}}|j}dd | D }t|dkswJ d	tdd |D rt	dW d   dS 1 sw   Y  dS )r   r<   z--early_dropoutrs      r   r   c                 S   $   g | ]\}}t |tjr||fqS r%   r   r   DropoutrA   namemoduler%   r%   r&   
<listcomp>	     $ z2TestTrainer.test_early_dropout.<locals>.<listcomp>r   z&Didn't find any dropouts in the model!        z5Dropout module %s was not set to 0 with early_dropoutNz-1c                 S   r   r%   r   r   r%   r%   r&   r     r   c                 s   s    | ]
\}}|j d kV  qdS )r   N)r   )rA   r$   r   r%   r%   r&   rC     s    z1TestTrainer.test_early_dropout.<locals>.<genexpr>zJAll dropouts were 0 after training even though early_dropout was set to -1)
rE   rF   r*   r   r   named_childrenr   r   rM   AssertionError)	r+   r   rN   r   r$   r   dropoutsr   r   r%   r%   r&   test_early_dropout  s(   	"zTestTrainer.test_early_dropoutc                 C   B   t jtd}| j||dd W d   dS 1 sw   Y  dS )z
        Test the whole thing for a few iterations on the fake data

        This tests that it works if you give it a silver file
        The check for the use of the silver data is that the
        number of batches trained should go up
        r<   Tr   Nr   r   r%   r%   r&   test_train_silver  s   "zTestTrainer.test_train_silverc                 C   s   t jtd]}| j||dd\}}|d d }|d d }tj|s%J tj|r-J t|	 }| j||dddd tj|sFJ tj|sNJ t|	 j
|j
ks[J W d	   d	S 1 sfw   Y  d	S )
ag  
        Test the whole thing for a few iterations, then restart

        This tests that the 5th iteration save file is not rewritten
        and that the iterations continue to 10

        TODO: could make it more robust by verifying that only 5 more
        epochs are trained.  Perhaps a "most recent epochs" could be
        saved in the trainer
        r<   Fr   ry   rx   
   T)r   r   r   N)rE   rF   r*   r   rH   r5   rK   pathlibPathstatst_mtime)r+   r   rN   r   r$   save_5save_10save_5_statr%   r%   r&   test_train_checkpoint#  s   "z!TestTrainer.test_train_checkpointc                 C   sv  |  |\}}g d}|r|ddg7 }|r||7 }| j||d|d\}}tj|d d}	tt}
tdd	D ]&}|	| }tj|sDJ t	j
j|d
d}|j|ksSJ |
|jj | q6|rt|
dksgJ t|
 }|
|d  g dksyJ |
|d  ddgksJ |
|d  g dksJ d S t|
dksJ t|
 }|
|d  g dksJ |
|d  g dksJ d S )N)z--multistagerU   r\   rW   16   r   save_dirrq   r   	   Tr      r   )r   r   r   r   rx   r   )r      r   )r   rx   r   r   r   )ro   r   rH   r5   rI   r   r   r   rK   r   r   rL   r   r   word_input_sizeappendr   sortedkeys)r+   r   rN   	use_lattnr   rl   rn   r   r$   r   word_input_sizesr   r   rO   word_input_keysr%   r%   r&   run_multistage_tests=  s2   z TestTrainer.run_multistage_testsc                 C   r   )
        Test a multistage training for a few iterations on the fake data

        This should start with no pattn or lattn, have pattn in the middle, then lattn at the end
        r<   Tr   NrE   rF   r*   r   r   r%   r%   r&   test_multistage_lattn\     "z!TestTrainer.test_multistage_lattnc                 C   r   )r   r<   Fr   Nr   r   r%   r%   r&   test_multistage_no_lattne  r   z$TestTrainer.test_multistage_no_lattnc                 C   s   t jtdn}ddg}| j||d|d tj|d}tddD ]'}|| }tj	j
|d	d
}|j|ks5J t|jtjs>J t|jtjrGJ q tddD ]}|| }tj	j
|d	d
}|j|ksbJ t|jtjskJ qMW d   dS 1 sww   Y  dS )z^
        Test that the correct optimizers are built for a multistage training process
        r<   --optimadamwF)r   r   rq   r   r   Tr   r   r   N)rE   rF   r*   r   rH   r5   rI   r   r   r   rL   r   r   r   r   AdadeltaAdamW)r+   r   rN   r   r   r   r   rO   r%   r%   r&   test_multistage_optimizern  s"   "z%TestTrainer.test_multistage_optimizerc                 C   sJ   t jtd}ddg}| j|||d W d   dS 1 sw   Y  dS )zi
        Verify that grad clipping is not saved with the model, but is attached at training time
        r<   z--grad_clipping25r   Nr   )r+   r   rN   r   r%   r%   r&   test_grad_clip_hooks  s   "z TestTrainer.test_grad_clip_hooksc                 C   s  d}t |}t|dksJ g d}t|g|R  }|j|}t|dks)J t|d jdks4J |d jd j|d ksBJ |d jd usKJ t	|d jj
tjsWJ |d jj
jtg kseJ t|d jdkspJ |d jd j|d ks~J |d jd j|d jd ksJ t|d jdksJ |d jd j|d ksJ |d jd usJ t	|d jj
tjsJ |d jj
jtg ksJ t|d jd	ksJ |d jd j|d ksJ |d jd j|d jd ksJ d S )
Nz(ROOT (S (NP (PRP I)) (VP (VBP wan) (S (VP (TO na) (VP (VB lick) (NP (NP (NNP Sh'reyan) (POS 's)) (NNS antennae))))))))  (ROOT (S (NP (DT This) (NN interface)) (VP (VBZ sucks))))r   rT   r   r   r   r   r   )r   r   r   r   r   analyze_treespredictionstreestater   scorer]   TensorshapeSizeconstituentsvaluechildren)r+   r   test_str	test_treer   rO   resultsr%   r%   r&   test_analyze_trees  s.   
"&zTestTrainer.test_analyze_treesc                 C   sD   |  D ]\}}d| }|j|}t| | s dS qdS )zL
        Return True if all bert weights are close, False otherwise
        bert_model.FT)named_parametersr   get_parameterr]   r^   cpu)r+   
bert_modelparser_modelr   	parameter
other_nameother_parameterr%   r%   r&   bert_weights_allclose  s   z!TestTrainer.bert_weights_allclosec                 C   s   t jtdj}t }d|g}| j||||d\}}||\}}| ||s(J tj|d dd dd}	|	d	 }
t	d
d |
d 
 D sGJ tdd |
d 
 D sVJ tjj|d |d}|jj|u shJ W d    d S 1 ssw   Y  d S )Nr<   --bert_model)r   r"   rz   c                 S   s   | S r?   r%   )storagelocr%   r%   r&   <lambda>  s    z5TestTrainer.frozen_transformer_test.<locals>.<lambda>T)weights_onlyparamsc                 s   s    | ]	}| d  V  qdS )r   N
startswithr@   r%   r%   r&   rC     s    z6TestTrainer.frozen_transformer_test.<locals>.<genexpr>r   c                 s   s    | ]}| d V  qdS )zoutput_layers.Nr   r@   r%   r%   r&   rC     rD   )r"   )rE   rF   r*   r
   r   r   r   r]   rL   rM   r   anyr   r   r   r   )r+   r   transformer_namerN   r"   r   r   r   bert_tokenizer
checkpointr   r%   r%   r&   frozen_transformer_test  s   "z#TestTrainer.frozen_transformer_testc                 C      |  |d dS )zf
        Check that the parameters of the bert model don't change when training a basic model
        hf-internal-testing/tiny-bertNr   r+   r   r%   r%   r&   test_bert_frozen     zTestTrainer.test_bert_frozenc                 C      |  || dS zf
        Check that the parameters of an xlnet model don't change when training a basic model
        Nr  r+   r   r6   r%   r%   r&   test_xlnet_frozen  r  zTestTrainer.test_xlnet_frozenc                 C   r  r  r  )r+   r   r:   r%   r%   r&   test_bart_frozen  r  zTestTrainer.test_bart_frozenc                 C   sR  t jtd}d}d|dddg}| j||d|d\}}t }||\}}| ||r-J |d	 }	tj	|	s9J | 
||d
d
ddd|}
tjj|	|
|d}|jj|usVJ | ||r^J | |jj|shJ tj|d}tj	|rwJ |j|dd tjj||
|d}|jj|usJ |jdrJ W d
   d
S 1 sw   Y  d
S )z`
        Check that the parameters the bert model DO change over a single training step
        r<   r  r   --bert_finetuner   adadeltar   r   rz   N--no_bert_finetune--no_stage1_bert_finetuner   r"   ztest_resave_bert.ptF)save_optimizerr   )rE   rF   r*   r   r
   r   r   rH   r5   rK   rw   r   r   rL   r   r   rI   rJ   is_unsaved_module)r+   r   rN   r   r   r   r"   r   r   r   no_finetune_argsrO   new_save_namerQ   r%   r%   r&   test_bert_finetune_one_epoch  s*   "z(TestTrainer.test_bert_finetune_one_epochc           
      C   s   t jtd[}d|dddg}| j|||d\}}t }||\}}| ||r*J | ||dddd	d|}	tj	j
|d
 |	|d}|jjd rIJ |jjd rQJ |jj|usYJ W d   dS 1 sdw   Y  dS )za
        Check that the parameters of the transformer DO change when using bert_finetune
        r<   r   r  r   r   r   Nr  r  rz   r  bert_finetunestage1_bert_finetune)rE   rF   r*   r   r
   r   r   rw   r   r   rL   r   r   r   )
r+   r   r   rN   r   r   r"   r   r   r  r%   r%   r&   finetune_transformer_test   s   "z%TestTrainer.finetune_transformer_testc                 C   r   )z^
        Check that the parameters of a bert model DO change when using bert_finetune
        r  Nr  r  r%   r%   r&   test_bert_finetune  r  zTestTrainer.test_bert_finetunec                 C   r  )z`
        Check that the parameters of an xlnet model DO change when using bert_finetune
        Nr  r  r%   r%   r&   test_xlnet_finetune  r  zTestTrainer.test_xlnet_finetunec                 C   sz  t jtd}d}d|dddg}| j||d|d\}}t }||\}}| ||r-J | ||d	d	d
dd|dd
}	|jj	d }
t
j|d}td|
d D ]7}|| }t
j|s]J tjj||	|d}|jj|usnJ | ||rvJ ||
d kr| |jj|sJ qO|d }|d }tjj||	|d}tjj||	|d}| |jj|rJ W d	   d	S 1 sw   Y  d	S )z
        Check that the parameters the bert model DO change when using stage1_bert_finetune, but only for the first couple steps
        r<   r  r   --stage1_bert_finetuner   r   r   r   Nr  r  epochsrq   r   r  r   )rE   rF   r*   r   r
   r   r   rw   r   r   rH   r5   rI   r   rK   r   r   rL   r   )r+   r   rN   bert_model_namer   r   r"   r   r   r  r   r   r   r   rO   model_name_1model_name_2tr_1tr_2r%   r%   r&   test_stage1_bert_finetune   s2   "z%TestTrainer.test_stage1_bert_finetunec              
   C   s  t jtdq}d|dddddddg	}| j|||d\}}t }||\}}|jjd	ks.J d
|jjd	  }	| D ]4\}
}d|
 }|j	
|}|	|
v rbd|
v sTd|
v rUq:t| | raJ q:t| | snJ q:W d   dS 1 szw   Y  dS )z]
        Check that the parameters the bert model DO change when using bert_finetune
        r<   r   r  z--bert_finetune_layersr\   r   r   r   r   z	layer.%d.r   zrel_attn.seg_embedzrel_attn.r_s_biasN)rE   rF   r*   r   r
   r   confignum_hidden_layersr   r   r   r]   r^   r   )r+   r   r   rN   r   r   r"   r   r   
layer_namer   r   r   r   r%   r%   r&   #one_layer_finetune_transformer_testF  s"   "z/TestTrainer.one_layer_finetune_transformer_testc                 C   s   |  |d d S )Nr  r&  r  r%   r%   r&   test_bert_finetune_one_layer`     z(TestTrainer.test_bert_finetune_one_layerc                 C   s   |  || d S r?   r'  r  r%   r%   r&   test_xlnet_finetune_one_layerc  r)  z)TestTrainer.test_xlnet_finetune_one_layerc                 C   s0   d}d|ddddg}| j |t||d\}}d S )Nr  r   r  r   r   
--use_peftr   )r   r.   )r+   tmp_pathr   r   r   r   r%   r%   r&   test_peft_finetunef  s   zTestTrainer.test_peft_finetunec                    s  t jtdw}d}d}d|ddddg}| j||||d	\}}t|D ]R}|d
 | }|d
 |d  }	tjj||ddtjj|	|dd dd jj	
 D }
|dk rbt fdd|
D raJ q"|dkrtt fdd|
D stJ q"W d    d S 1 sw   Y  d S )Nr<   r   r  r   r  r   r   r+  r   ry   r   F)r   r   c                 S   s"   g | ]\}}| d dkr|qS )lorar   )find)rA   r   r$   r%   r%   r&   r   w  s   " z;TestTrainer.test_peft_twostage_finetune.<locals>.<listcomp>r   c                 3   8    | ]}t jj|  jj| V  qd S r?   r]   r^   r   r   r   r   rA   r   trainer_nexttrainer_prevr%   r&   rC   y      z:TestTrainer.test_peft_twostage_finetune.<locals>.<genexpr>c                 3   r0  r?   r1  r2  r3  r%   r&   rC   }  r6  )rE   rF   r*   r   r   r   r   rL   r   r   r   r   rM   )r+   r   rN   r   r   r   r   epochfilename_prevfilename_next
lora_namesr%   r3  r&   test_peft_twostage_finetunek  s,   "z'TestTrainer.test_peft_twostage_finetune)rx   NFFNr?   )'__name__
__module____qualname__pytestfixturer   r6   r:   r;   rS   rf   ro   rw   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r	  r
  r  r  r  r  r"  r&  r(  r*  r-  r;  r%   r%   r%   r&   r'   N   sL    






6
		 &r'   ))collectionsr   loggingr   rE   r?  r]   r   r   stanzar   stanza.modelsr   stanza.models.commonr   #stanza.models.common.bert_embeddingr   r	   %stanza.models.common.foundation_cacher
   stanza.models.common.utilsr   stanza.models.constituencyr   ,stanza.models.constituency.parse_transitionsr   r   r   r   stanza.testsmarkpipelinetravis
pytestmark	getLoggerloggersetLevelWARNINGrk   r   r'   r%   r%   r%   r&   <module>   s4    
