o
    h                     @   s2   d dl Z d dlZd dlmZ G dd dejZdS )    Nc                       s\   e Zd ZdZ		d fdd	Zdd Zd	d
 Zdd Zdd Zdd Z	e
dddZ  ZS )LangIDBiLSTMa  
    Multi-layer BiLSTM model for language detecting. A recreation of "A reproduction of Apple's bi-directional LSTM models
    for language identification in short strings." (Toftrup et al 2021)

    Arxiv: https://arxiv.org/abs/2102.06282
    GitHub: https://github.com/AU-DIS/LSTM_langid

    This class is similar to https://github.com/AU-DIS/LSTM_langid/blob/main/src/LSTMLID.py
    @   N        c
           
         s   t t|   || _|| _|| _|| _t|| _|| _	dd t
dd | j	 D D | _|	| _|d | _t|| _|| _tj|d| _|| _tj| j| j| jd| _tj| j| j| jddd| _t| jd	 | j| _tj| jd
| _d S )Nc                 S   s   g | ]}|d  qS )    ).0ir   r   U/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/models/langid/model.py
<listcomp>   s    z)LangIDBiLSTM.__init__.<locals>.<listcomp>c                 S   s   g | ]\}}||fqS r   r   )r   kvr   r   r	   r
      s    z<PAD>)weight)num_embeddingsembedding_dimpadding_idxT)
num_layersbidirectionalbatch_first   )p)superr   __init__r   r   
hidden_dimchar_to_idxlen
vocab_size
tag_to_idxsorteditems
idx_to_taglang_subsetr   tagset_size
batch_sizennCrossEntropyLoss
loss_traindropout_prob	Embeddingchar_embedsLSTMlstmLinearhidden_to_tagDropoutdropout)
selfr   r   r   r   r   r"   weightsr.   r    	__class__r   r	   r      s>   
$

	zLangIDBiLSTM.__init__c                    sP    j r fdd jD }tj||tjd _dS tjt j|tjd _dS )z
        Build language mask if a lang subset is specified (e.g. ["en", "fr"])

        The mask will be added to the results to set the prediction scores of illegal languages to -inf
        c                    s$   g | ]}| j v rd ntd qS )r   inf)r    float)r   langr/   r   r	   r
   C   s   $ z0LangIDBiLSTM.build_lang_mask.<locals>.<listcomp>)devicedtypeN)r    r   torchtensorr4   	lang_maskzerosr   )r/   r7   lang_mask_listr   r6   r	   build_lang_mask<   s   zLangIDBiLSTM.build_lang_maskc                 C   s   |  ||S )N)r%   )r/   Y_hatYr   r   r	   lossH   s   zLangIDBiLSTM.lossc                 C   s4   |  |}| |\}}| |}tj|dd}|S )Nr   dim)r(   r*   r,   r9   sum)r/   x_r   r   r	   forwardK   s
   

zLangIDBiLSTM.forwardc                    sL    |} j r| d }t fddt|D }|| }tj|ddS )Nr   c                    s   g | ]} j qS r   )r;   )r   rF   r6   r   r	   r
   ^   s    z2LangIDBiLSTM.prediction_scores.<locals>.<listcomp>r   rB   )r    sizer9   stackrangeargmax)r/   rE   prediction_probsprediction_batch_size
batch_maskr   r6   r	   prediction_scoresZ   s   zLangIDBiLSTM.prediction_scoresc                 C   s0   | j | j| j| j| j|  d}t|| dS )z Save a model at path )r   r   r   r   r   model_state_dictN)r   r   r   r   r   
state_dictr9   save)r/   path
checkpointr   r   r	   rR   b   s   zLangIDBiLSTM.savec              
   C   s   |du rt dtj|st d| tj|tddd}|d d }| |d	 |d
 |d |d |d |||d}||d  ||}|	| |S )z) Load a serialized model located at path NzETrying to load langid model, but path not specified!  Try --load_namez>Trying to load langid model from path which does not exist: %scpuT)map_locationweights_onlyrP   zloss_train.weightr   r   r   r   r   )r"   r0   r    )
FileNotFoundErrorosrS   existsr9   loadr7   load_state_dicttor>   )clsrS   r7   r"   r    rT   r0   modelr   r   r	   r[   n   s   

zLangIDBiLSTM.load)r   Nr   N)Nr   N)__name__
__module____qualname____doc__r   r>   rA   rG   rO   rR   classmethodr[   __classcell__r   r   r1   r	   r      s    
*r   )rY   r9   torch.nnr#   Moduler   r   r   r   r	   <module>   s    