o h@sdZddlmZddlmZddlZddlZddlmZddl m Z ddl m Z e dZed gd ZGd d d ZGd ddeZdddZdddZdddZdddZdS)z> Keeps BERT, charlm, word embedings in a cache to save memory ) namedtuple)deepcopyN)bert_embedding)CharacterLanguageModel)Pretrainstanza BertRecord)model tokenizerpeft_idsc@s:eZdZd ddZdddZdddZd d Zd d ZdS)FoundationCacheNFcCsP|duri|_i|_i|_t|_n|j|_|j|_|j|_|j|_||_dSN)bertcharlms pretrains threadingLocklocklocal_files_only)selfotherrr`/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/models/common/foundation_cache.py__init__s  zFoundationCache.__init__cCs|j|d|d\}}}||fSNr)load_bert_with_peft)rtransformer_namermt_rrr load_bert"szFoundationCache.load_bertcCs|durdS|jg||jvr)|dur|j}tj||d\}}t||i|j|<ntd||j|}|sD|j|j dfWdS||j vrOd|j |<n |j |d|j |<d||j |f}|j|j |fWdS1sswYdS)zU Load a transformer only once Uses a lock for thread safety N)NNNrzReusing bert %srz%s_%d) rrrrr!rloggerdebugr r r )rr peft_namerr r bert_recordrrrr&s&      $z#FoundationCache.load_bert_with_peftcCst|sdS|j)||jvrtd|tj|dd|j|<ntd||j|WdS1s3wYdS)NLoading charlm from %sFfinetunezReusing charlm from %s)rrr#r$rloadrfilenamerrr load_charlmAs   $zFoundationCache.load_charlmcCsr|durdS|j&||jvrtd|t||j|<ntd||j|WdS1s2wYdS)zc Load a pretrained word embedding only once Uses a lock for thread safety NzLoading pretrain %szReusing pretrain %s)rrr#r$rr+rrr load_pretrainNs   $zFoundationCache.load_pretrainNFr )__name__ __module__ __qualname__rr!rr-r.rrrrr s     r c@s$eZdZdZdddZdddZdS)NoTransformerFoundationCachea3 Uses the underlying FoundationCache, but hiding the transformer. Useful for when loading a downstream model such as POS which has a finetuned transformer, and we don't want the transformer reused since it will then have the finetuned weights for other models which don't want them NcCst||dur |jdS|dSr)r!r)rrrrrrr!hsz&NoTransformerFoundationCache.load_bertcCs t|||dur |jdS|dSr)rr)rrr%rrrrrks z0NoTransformerFoundationCache.load_bert_with_peftr )r0r1r2__doc__r!rrrrrr3_s r3cCs$|dur tj||dS|j||dS)zT Load a bert, possibly using a foundation cache, ignoring the cache if None Nr)rr!) model_namefoundation_cacherrrrr!nsr!cCs4|durtj||d\}}|||fS|j|||dSr)rr!r)r5r%r6rrrrrrrws rFcCsF|sdS|r tj|ddS|dur||Std|tj|ddS)NTr(r'F)rr*r-r#r$) charlm_filer6r)rrrr-}s  r-cCs.|sdS|dur ||Std|t|S)NzLoading pretrain from %s)r.r#r$r)r,r6rrrr.s   r.)NNr/r )r4 collectionsrcopyrloggingrstanza.models.commonrstanza.models.common.char_modelrstanza.models.common.pretrainr getLoggerr#rr r3r!rr-r.rrrrs       M