o
    –hBm  ã                   @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlm
  mZ d dlm  m  mZ d dlmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZmZ d dl m!Z! d d	l"m#Z#m$Z$ 	 e %d
¡Z&e %d¡Z'G dd„ deƒZ(dS )é    N)ÚBaseClassifier)Ú	CNNConfig)ÚSentimentDatum)ÚExtraVectorsÚ	ModelTypeÚbuild_output_layers)Úextract_bert_embeddings)Úget_long_tensorÚsort_all)Úattach_bert_model)ÚPAD_IDÚUNK_IDÚstanzazstanza.classifiers.trainerc                       sn   e Zd Z‡ fdd„Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zdd„ Z	dd„ Z
ddd„Zdd„ Zdd„ Z‡  ZS )ÚCNNClassifierc              	      sø  t t| ƒ ¡  || _|j}|j}|	p|}	t d||	¡ |
| _t	dFi d|j
“d|j“d|j“d|j“dt|ƒ“d|j“d|j“d	|j“d
|j“d|j“d|j“d|du“d|du“d|j“d|j“d|j“d|“d|j“d|	“d|“d|j“d|j“d|j“d|j“d|j“d|j“d|j“d|j “dt!j"“Ž| _#|j| _g | _$|j%}|  &d t'j(j)|d!d"¡ |  &d#|¡ |j*d$ | _+|j*d% | _,|  &d&|¡ |durÛt- d' .| /¡ ¡¡ |j0sÛt1d(ƒ‚|  &d)|¡ |duröt- d* .| /¡ ¡¡ |j0röt1d+ƒ‚t2| ||| j#j|	ƒ t' 3t4 5| j,¡t6 7| j,¡ d, ¡| _8d-d.„ t9|j:ƒD ƒ| _;| j#jt<j=ur|s1t1d/ .| j#j¡ƒ‚|js:| j,| j#_| j#jt<j>u rS| j#j| j,krSt1d0 .| j#j¡ƒ‚t?|ƒ| _@d1d.„ t9| j@ƒD ƒ| _At'j(t|ƒ| j#j| j#jd$d2| _Bt- d3 .| jBjCj*¡¡ nd| _@d| _Ad$| j#_d| _B| j#jt<j=u r˜| j,}n%| j#jt<j>u r¤| j,}n| j#jt<jDu r´| j,| j#j }n	t1d4 .| j#j¡ƒ‚|durß|jrÖt' E| /¡ |j¡| _F||j7 }n	d| _F|| /¡ 7 }|dur|jrøt' E| /¡ |j¡| _G||j7 }n	d| _G|| /¡ 7 }| j#jr>|du rt1d5ƒ‚| Hd6gg¡d$ j*d% }t'jEd7d%d8d9| _I| j#jr:t'jE|| j#jd:| _|| j#j }n|| }|dur„| j#jrm| j#j|j#jJkrY|j#jJd% | j#_t'jE| j#jd%d8d;| _Kt'jL M| jKjC¡ nd| _K|du ryt1d<ƒ‚| jj#jN| _O|| jO7 }| j#jrž| j#jd= }t'jPd!|| j#jd=d!d>d?| _n|}d| _d$| _Qt' R¡ | _Sd$| _Tt9| j#jƒD ]ª\}}tU|tVƒrýtW| jT|ƒ| _TtU| j#j
tVƒrÒ| j#j
}n| j#j
| }|| j#j  }t- d@|||¡ |  jQ|7  _Q| jS Xt'jYd%|||fdA¡ q´tU|tZƒr[t|ƒd=kr[|\}}tW| jT|ƒ| _TtU| j#j
tVƒr)tWd%| j#j
||  ƒ}n| j#j
| }|||  | j#j  }t- dB|||¡ |  jQ|7  _Q| jS Xt'jYd%|d%|f||fdC¡ q´t1dDƒ‚t- dE| jQ¡ t[| jQ| j#j| j#j\ƒ| _]t' ^| j#j¡| _dS )Ga]  
        pretrain is a pretrained word embedding.  should have .emb and .vocab

        extra_vocab is a collection of words in the training data to
        be used for the delta word embedding, if used.  can be set to
        None if delta word embedding is not used.

        labels is the list of labels we expect in the training data.
        Used to derive the number of classes.  Saving it in the model
        will let us check that test data has the same labels

        args is either the complete arguments when training, or the
        subset of arguments stored in the model save file
        z&bert_finetune %s / force_bert_saved %sÚfilter_channelsÚfilter_sizesÚ	fc_shapesÚdropoutÚnum_classesÚwordvec_typeÚextra_wordvec_methodÚextra_wordvec_dimÚextra_wordvec_max_normÚchar_lowercaseÚcharlm_projectionÚhas_charlm_forwardNÚhas_charlm_backwardÚuse_elmoÚelmo_projectionÚ
bert_modelÚbert_finetuneÚbert_hidden_layersÚforce_bert_savedÚuse_peftÚ	lora_rankÚ
lora_alphaÚlora_dropoutÚlora_modules_to_saveÚlora_target_modulesÚbilstmÚbilstm_hidden_dimÚmaxpool_widthÚ
model_typeÚ	embeddingT)ÚfreezeÚ
elmo_modelr   é   Úforward_charlmz&Got forward char model of dimension {}z*Got a backward charlm as a forward charlm!Úbackward_charlmz'Got backward char model of dimension {}z*Got a forward charlm as a backward charlm!g      $@c                 S   s   i | ]\}}|  d d¡|“qS )õ   Â ú )Úreplace©Ú.0ÚiÚword© r:   úc/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/models/classifiers/cnn_classifier.pyÚ
<dictcomp>’   ó    z*CNNClassifier.__init__.<locals>.<dictcomp>z;Should have had extra_vocab set for extra_wordvec_method {}z1extra_wordvec_dim must equal embedding_dim for {}c                 S   s   i | ]\}}||“qS r:   r:   r6   r:   r:   r;   r<   ž   s    )Únum_embeddingsÚembedding_dimÚmax_normÚpadding_idxzExtra embedding size: {}úunable to handle {}z1Model requires elmo, but elmo_model not passed inÚTesté   F)Úin_featuresÚout_featuresÚbias)rE   rF   )rG   z,Cannot have a bert model without a tokenizeré   gš™™™™™É?)Úbatch_firstÚ
input_sizeÚhidden_sizeÚ
num_layersÚbidirectionalr   z7Adding full width filter %d.  Output channels: %d -> %d)Úin_channelsÚout_channelsÚkernel_sizez,Adding filter %s.  Output channels: %d -> %d)rN   rO   ÚstriderP   z&Expected int or 2d tuple for conv sizezInput dim to FC layers: %dr:   )_Úsuperr   Ú__init__Úlabelsr    r#   ÚloggerÚdebugÚ	peft_namer   r   r   r   r   Úlenr   r   r   r   r   r   r   r   r   r!   r$   r%   r&   r'   r(   r)   r*   r+   r   ÚCNNÚconfigÚunsaved_modulesÚembÚadd_unsaved_moduleÚnnÚ	EmbeddingÚfrom_pretrainedÚshapeÚ
vocab_sizer?   ÚtloggerÚformatÚ
hidden_dimÚis_forward_lmÚ
ValueErrorr   Ú	ParameterÚtorchÚrandnÚnpÚsqrtÚunkÚ	enumerateÚvocabÚ	vocab_mapr   ÚNONEÚSUMÚlistÚextra_vocabÚextra_vocab_mapÚextra_embeddingÚweightÚCONCATÚLinearÚcharmodel_forward_projectionÚcharmodel_backward_projectionÚ
sents2elmoÚelmo_combine_layersÚnum_hidden_layersÚbert_layer_mixÚinitÚzeros_rK   Úbert_dimÚLSTMÚfc_input_sizeÚ
ModuleListÚconv_layersÚ
max_windowÚ
isinstanceÚintÚmaxÚappendÚConv2dÚtupler   r   Ú	fc_layersÚDropout)ÚselfÚpretrainrt   rT   Úcharmodel_forwardÚcharmodel_backwardr/   r   Úbert_tokenizerr"   rW   Úargsr    r#   Ú
emb_matrixÚtotal_embedding_dimÚelmo_dimÚconv_input_dimÚ
filter_idxÚfilter_sizer   Úfc_deltaÚfilter_heightÚfilter_width©Ú	__class__r:   r;   rS   8   sx  ÿþýüûúùø	÷
öõôóòñðïîìëêéèçåäãâ $


ý










û

þýzCNNClassifier.__init__c                 C   sd   |  j |g7  _ t| ||ƒ |d ur*|dv s|dkr,| jjs.| ¡ D ]\}}d|_q"d S d S d S d S )N©r1   r2   r   F)r[   ÚsetattrrZ   r#   Únamed_parametersÚrequires_grad)r   ÚnameÚmoduleÚ_Ú	parameterr:   r:   r;   r]     s   ûz CNNClassifier.add_unsaved_modulec                 C   s   |  d¡d | jv S )NÚ.r   )Úsplitr[   )r   r¥   r:   r:   r;   Úis_unsaved_module&  s   zCNNClassifier.is_unsaved_modulec                 C   sF   t  dt| jjƒ ¡ t  dt| jjƒ ¡ t  dt| jjƒ ¡ dS )ze
        Log some essential information about the model configuration to the training logger
        zFilter sizes: %szFilter channels: %szIntermediate layers: %sN)rc   ÚinfoÚstrrZ   r   r   r   ©r   r:   r:   r;   Úlog_configuration)  s   zCNNClassifier.log_configurationc                 C   s`   dg}|   ¡ D ]\}}|jr%| d¡d dvr%| d|t |¡ ¡ f ¡ qt d 	|¡¡ d S )NzNORMS FOR MODEL PARAMTERSr©   r   r¡   z%s %.6gÚ
)
r£   r¤   rª   r‹   ri   ÚnormÚitemrU   r¬   Újoin)r   Úlinesr¥   Úparamr:   r:   r;   Ú	log_norms1  s   €zCNNClassifier.log_normsc                    sˆ   |  |¡}ˆ d ur‡ fdd„|D ƒ}tjt|ƒ||d jd f|d}t|ƒD ]\}	}
||	 }||
jd  }|
||	||…d d …f< q'|S )Nc                    ó   g | ]}ˆ |ƒ‘qS r:   r:   ©r7   Úx©Ú
projectionr:   r;   Ú
<listcomp>;  ó    z1CNNClassifier.build_char_reps.<locals>.<listcomp>r   éÿÿÿÿ©Údevice)Úbuild_char_representationri   ÚzerosrX   ra   rn   )r   ÚinputsÚmax_phrase_lenÚcharlmr»   Úbegin_paddingsrÀ   Ú	char_repsÚchar_inputsÚidxÚrepÚstartÚendr:   rº   r;   Úbuild_char_reps8  s   
"zCNNClassifier.build_char_repsc                    s¼   t ˆ jjˆ jˆ j||dˆ jd urˆ jjnd ˆ jj ˆ jd	}ˆ jd ur,‡ fdd„|D ƒ}tj	t
|ƒ||d jd f|d}t|ƒD ]\}}|| }	|	|jd  }
||||	|
…d d …f< qA|S )NF)Úkeep_endpointsrL   ÚdetachrW   c                    s0   g | ]}ˆ   |¡ d ¡|jd dˆ j j  ‘qS )rH   )Úaxis)r   ÚsqueezeÚsumrE   )r7   Úfeaturer®   r:   r;   r¼   M  s   0 z9CNNClassifier.extract_bert_embeddings.<locals>.<listcomp>r   r¾   r¿   )r   rZ   r   r”   r   rE   r    rW   ri   rÂ   rX   ra   rn   )r   rÃ   rÄ   rÆ   rÀ   Úbert_embeddingsÚbert_inputsrÉ   rÊ   rË   rÌ   r:   r®   r;   r   C  s   ü
"z%CNNClassifier.extract_bert_embeddingsc           &         s’  t ˆ ¡ ƒj‰ ˆj‰‡fdd„‰dd„ |D ƒ}tdd„ |D ƒƒ}ˆj|kr(ˆj}g }g }g }g }g }g }|D ]¬}	ˆjrFt d|t	|	ƒ ¡}
nd}
||
 t	|	ƒ }| 
|
¡ | 
|¡ tg|
 }| ‡fdd„|	D ƒ¡ | tg| ¡ d	d„ t|ƒD ƒ}| 
|¡ | 
|¡ ˆjrÂtg|
 }|	D ]$}|ˆjv r¯ˆjr¦t ¡ d
k r¦| 
t¡ q| 
ˆj| ¡ q| 
t¡ q| tg| ¡ | 
|¡ ˆjjrâdg|
 }|	D ]}| 
|¡ qÍ| dg| ¡ | 
|¡ q6tj|dˆ d}ˆ |¡}t|ƒD ]\}}ˆj|| |< qôˆjr5tj|dˆ d}ˆ |¡}ˆjjtju r||g}nˆjjtju r,|| g}ntd ˆjj¡ƒ‚|g}ˆjd urOˆ ||ˆjˆj |ˆ ¡}| 
|¡ ˆj!d urfˆ ||ˆj!ˆj"|ˆ ¡}| 
|¡ ˆjjr¨ˆj#j$|dd}‡ fdd„|D ƒ}t %|¡}t &|dd¡}t &|dd¡}ˆ '|¡}| (d¡}ˆjj)r£ˆ )|¡}| 
|¡ ˆj*d ur»ˆ +|||ˆ ¡}| 
|¡ tj,|dd}ˆjj-rÑˆ -ˆ .|¡¡\}}| /d¡}g }t0ˆj1ˆjj2ƒD ]8\}}t3|t4ƒrþˆ .t5 6||ƒ (d¡¡¡} | 
| ¡ qà||ƒ &dd¡ 7dd¡} ˆ .t5 6| ¡¡} | 
| ¡ qà‡fdd„|D ƒ}!tj,|!dd}"|"}#ˆj8d d… D ]}$ˆ .t5 6|$|#ƒ¡¡}#q2ˆj8d |#ƒ}%|%S )Nc                    sT   ˆ   | d ¡}|d ur|S | d dkr"ˆ   | d d… d ¡}|d ur"|S ˆ   |  ¡ t¡S )Nr¾   ú')ÚgetÚlowerr   )r9   rÉ   )rp   r:   r;   Úmap_wordZ  s   z'CNNClassifier.forward.<locals>.map_wordc                 S   s    g | ]}t |tƒr|jn|‘qS r:   )rˆ   r   Útextr¸   r:   r:   r;   r¼   d  s     z)CNNClassifier.forward.<locals>.<listcomp>c                 s   s    | ]}t |ƒV  qd S )N)rX   r¸   r:   r:   r;   Ú	<genexpr>g  s   € z(CNNClassifier.forward.<locals>.<genexpr>r   c                    r·   r:   r:   r¸   )rÙ   r:   r;   r¼   ‚  r½   c                 S   s   g | ]
\}}|t kr|‘qS r:   )r   )r7   rÉ   r9   r:   r:   r;   r¼   ˆ  ó    g{®Gáz„?Ú F)r¤   rÀ   rB   éþÿÿÿ)Úoutput_layerc                    s   g | ]}t  |¡jˆ d ‘qS )r¿   )ri   ÚtensorÚtor¸   r¿   r:   r;   r¼   Ô  r=   r0   rD   rH   )Údimc                    s,   g | ]}t  |ˆ jj|jd  f¡ d ¡‘qS )rH   )ÚFÚ
max_pool2drZ   r+   ra   rÑ   )r7   Úoutr®   r:   r;   r¼   ù  s   , r¾   )9ÚnextÚ
parametersrÀ   rp   rŠ   r‡   ÚtrainingÚrandomÚrandintrX   r‹   r   Úextendrn   rt   ru   r   rZ   r   ri   rà   r-   rm   rv   r   r   rx   rr   rg   rd   r1   rÍ   rz   r2   r{   r/   r|   ÚstackÚ	transposer}   rÑ   r   r   r   Úcatr)   r   Ú	unsqueezeÚzipr†   r   rˆ   r‰   rã   ÚreluÚflattenrŽ   )&r   rÃ   rÄ   Úbatch_indicesÚbatch_unknownsÚextra_batch_indicesrÆ   Úend_paddingsÚelmo_batch_wordsÚphraseÚbegin_pad_widthÚend_pad_widthÚsentence_indicesÚsentence_unknownsÚextra_sentence_indicesr9   Úelmo_phrase_wordsÚinput_vectorsÚ
phrase_numÚextra_input_vectorsÚ
all_inputsÚchar_reps_forwardÚchar_reps_backwardÚelmo_arraysÚelmo_tensorsÚelmo_tensorrÔ   r§   r¹   Ú	conv_outsÚconvr›   Úconv_outÚ	pool_outsÚpooledÚprevious_layerÚfcrå   r:   )rÀ   rÙ   r   rp   r;   ÚforwardU  s¾   












€














zCNNClassifier.forwardTc                    s¦   ˆ   ¡ }|r‡ fdd„| ¡ D ƒ}|D ]}||= qt ˆ j¡}|d j|d< |d j|d< |d j|d< ||ˆ jˆ jdœ}ˆ jjrQddl	m
} |ˆ jˆ jd	|d
< |S )Nc                    s   g | ]	}ˆ   |¡r|‘qS r:   )r«   )r7   Úkr®   r:   r;   r¼     s    z,CNNClassifier.get_params.<locals>.<listcomp>r   r   r,   )ÚmodelrZ   rT   rt   r   )Úget_peft_model_state_dict)Úadapter_nameÚ	bert_lora)Ú
state_dictÚkeysÚdataclassesÚasdictrZ   r¥   rT   rt   r#   Úpeftr  r   rW   )r   Úskip_modulesÚmodel_stateÚskippedr  rZ   Úparamsr  r:   r®   r;   Ú
get_params  s$   üzCNNClassifier.get_paramsc                    s   ‡ fdd„|D ƒ}|S )Nc                    s   g | ]
}t  |ˆ jj¡‘qS r:   )ÚdataÚupdate_textrZ   r   )r7   Úsr®   r:   r;   r¼     rÜ   z1CNNClassifier.preprocess_data.<locals>.<listcomp>r:   )r   Ú	sentencesr:   r®   r;   Úpreprocess_data  s   zCNNClassifier.preprocess_datac                 C   s   dd„ |j D ƒS )Nc                 S   s   g | ]
}d d„ |j D ƒ‘qS )c                 S   s   g | ]}|j ‘qS r:   )rÚ   )r7   Útokenr:   r:   r;   r¼   #  s    z>CNNClassifier.extract_sentences.<locals>.<listcomp>.<listcomp>)Útokens)r7   Úsentencer:   r:   r;   r¼   #  rÜ   z3CNNClassifier.extract_sentences.<locals>.<listcomp>)r"  )r   Údocr:   r:   r;   Úextract_sentences!  s   zCNNClassifier.extract_sentences)T)Ú__name__Ú
__module__Ú__qualname__rS   r]   r«   r¯   r¶   rÍ   r   r  r  r#  r(  Ú__classcell__r:   r:   rŸ   r;   r   7   s     d 
0r   ))r  ÚloggingÚmathÚosré   ÚreÚnumpyrk   ri   Útorch.nnr^   Útorch.nn.functionalÚ
functionalrã   Ústanza.models.classifiers.dataÚmodelsÚclassifiersr  Ú)stanza.models.classifiers.base_classifierr   Ú stanza.models.classifiers.configr   r   Ústanza.models.classifiers.utilsr   r   r   Ú#stanza.models.common.bert_embeddingr   Ústanza.models.common.datar	   r
   Ústanza.models.common.utilsr   Ústanza.models.common.vocabr   r   Ú	getLoggerrU   rc   r   r:   r:   r:   r;   Ú<module>   s.    

