o hX@sdZddlmZmZddlmZddlZddlZddlm Z ddl m Z ddl m Z mZddlmZdd lmZdd lmZdd lmZmZd Zed ZGdddeZGdddeZdS)a  The BaseModel is passed to the transitions so that the transitions can operate on a parsing state without knowing the exact representation used in the model. For example, a SimpleModel simply looks at the top of the various stacks in the state. A model with LSTM representations for the different transitions may attach the hidden and output states of the LSTM to the word / constituent / transition stacks. Reminder: the parsing state is a list of words to parse, the transitions used to build a (possibly incomplete) parse, and the constituent(s) built so far by those transitions. Each of these components are represented using stacks to improve the efficiency of operations such as "combine the most recent 4 constituents" or "turn the next input word into a constituent" )ABCabstractmethod) defaultdictN)utils)transition_sequence)TransitionSchemeCloseConstituentTree)State TreeStack) ParseResult ScoredTreezstanza.constituency.trainercs^eZdZdZfddZeddZeddZedd Zed d Z ed d Z eddZ eddZ eddZ eddZeddZeddZeddZddZddZd d!Zed"d#Zed$d%ZdEd'd(Zd)d*ZdEd+d,Zd-d.Zd/d0ZdFd2d3Zd4d5Zd6d7Zd8d9Z dGd;d<Z!dGd=d>Z"dHd?d@Z#dAdBZ$dIdCdDZ%Z&S)J BaseModelas This base class defines abstract methods for manipulating a State. Applying transitions may change important metadata about a State such as the vectors associated with LSTM hidden states, for example. The constructor forwards all unused arguments to other classes in the constructor sequence, so put this before other classes such as nn.Module cs\tj|i|||_||_||_tt||_|jtj up*|jtj up*|jtj u|_ dSN) super__init___transition_scheme _unary_limit_reverse_sentencesortedlist _root_labelsrTOP_DOWNTOP_DOWN_UNARYTOP_DOWN_COMPOUND _is_top_down)selftransition_scheme unary_limitreverse_sentence root_labelsargskwargs __class__`/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/models/constituency/base_model.pyr1s   zBaseModel.__init__cCdS)z For each list of tagged words, builds a TreeStack of word nodes The word lists should be backwards so that the first word is the last word put on the stack (LIFO) Nr()rtagged_word_listsr(r(r)initial_word_queues=zBaseModel.initial_word_queuescCr*)zh Builds an initial transition stack with whatever values need to go into first position Nr(rr(r(r)initial_transitionsEr-zBaseModel.initial_transitionscCr*)zi Builds an initial constituent stack with whatever values need to go into first position Nr(r.r(r(r)initial_constituentsKr-zBaseModel.initial_constituentscCr*)zO Get the word corresponding to this position in the word queue Nr(r word_noder(r(r)get_wordQr-zBaseModel.get_wordcCr*)zj Transform the top node of word_queue to something that can push on the constituent stack Nr(rstater(r(r)transform_word_to_constituentWr-z'BaseModel.transform_word_to_constituentcCr*)zg When using a dummy node as a sentinel, transform it to something usable by this model Nr(rdummyr(r(r)dummy_constituent]r-zBaseModel.dummy_constituentcCr*)zj Build multiple constituents at once. This gives the opportunity for batching operations Nr()rlabelschildren_listsr(r(r)build_constituentscr-zBaseModel.build_constituentscCr*)z Add a multiple constituents to multiple constituent_stacks Useful to factor this out in case batching will help Nr(rconstituent_stacks constituentsr(r(r)push_constituentsir-zBaseModel.push_constituentscCr*)z Get the first constituent from the constituent stack For example, a model might want to remove embeddings and LSTM state vectors Nr(rr?r(r(r)get_top_constituentqr-zBaseModel.get_top_constituentcCr*)z Add a multiple transitions to multiple transition_stacks Useful to factor this out in case batching will help Nr(rtransition_stacks transitionsr(r(r)push_transitionsyr-zBaseModel.push_transitionscCr*)z Get the first transition from the transition stack For example, a model might want to remove transition embeddings before returning the transition Nr(rrEr(r(r)get_top_transitionr-zBaseModel.get_top_transitioncC|jS)zt Return ROOT labels for this model. Probably ROOT, TOP, or both (Danish uses 's', though) )rr.r(r(r)r#szBaseModel.root_labelscCrI)zF Limit on the number of consecutive unary transitions )rr.r(r(r)r!zBaseModel.unary_limitcCrI)z@ Transition scheme used - see parse_transitions )rr.r(r(r)r rJzBaseModel.transition_schemecCs |jtjuS)z^ Whether or not this model uses unary transitions, based on transition_scheme )rrrr.r(r(r)has_unary_transitionss zBaseModel.has_unary_transitionscCrI)z7 Whether or not this model is TOP_DOWN )rr.r(r(r) is_top_downzBaseModel.is_top_downcCrI)zG Whether or not this model is built to parse backwards )rr.r(r(r)r"rMzBaseModel.reverse_sentenceTcCtd)Nz-LSTMModel can predict, but SimpleModel cannotNotImplementedError)rstatesis_legalr(r(r)predictzBaseModel.predictcCrN)Nz5LSTMModel can weighted_choice, but SimpleModel cannotrO)rrQr(r(r)weighted_choicerTzBaseModel.weighted_choicecCsVdd|D}|r&t||D]\}}|||s%td|j||j|jqd|dfS)zK For each State, return the next item in the gold_sequence cSsg|]}|j|jqSr() gold_sequencenum_transitions).0yr(r(r) z*BaseModel.predict_gold..zZTransition {}:{} was not legal in a transition sequence: Original tree: {} Transitions: {}N)ziprR RuntimeErrorformatrW gold_treerV)rrQrRrEtransr5r(r(r) predict_golds  zBaseModel.predict_goldcsf||}||fddt|D}|r%ddt||D}|r1ddt||D}|S)zL what is passed in should be a list of list of preterminals c s2g|]\}}tt|dd|ddddd qS)rNg) sentence_length num_opens word_queuer_rVrEr? word_positionscore)r len)rXidxwqr?rEr(r)rZs  z=BaseModel.initial_state_from_preterminals..cSg|] \}}|j|dqS)r__replace)rXr_r5r(r(r)rZcSrl)rVrn)rXrVr5r(r(r)rZrp)r,r/r0 enumerater\)rpreterminal_lists gold_treesgold_sequences word_queuesrQr(rkr)initial_state_from_preterminalss    z)BaseModel.initial_state_from_preterminalscCsdd|D}|j|dddS)NcSsg|] }dd|DqS)cSsg|] \}}t|t|qSr(r )rXwordtagr(r(r)rZszABaseModel.initial_state_from_words...r()rXwordsr(r(r)rZsz6BaseModel.initial_state_from_words..rtrurw)r word_listsrsr(r(r)initial_state_from_wordssz"BaseModel.initial_state_from_wordsNcCsdd|D}|j|||dS)NcSsg|] }dd|DqS)cSs$g|]}t|jt|jdjqSr)r labelchildren)rXptr(r(r)rZszFBaseModel.initial_state_from_gold_trees...)yield_preterminalsrXtreer(r(r)rZs  z;BaseModel.initial_state_from_gold_trees..r{r|)rtreesrursr(r(r)initial_state_from_gold_treessz'BaseModel.initial_state_from_gold_treescCLg}t|D]}t|d}|durn||qt|dkr$||}|S)zd Read from the data_iterator batch_size trees and turn them into new parsing states Nr)rangenextappendrhr)r batch_size data_iterator state_batch_r_r(r(r)build_batch_from_treess     z BaseModel.build_batch_from_treescCsR|||}t|dkr|Stdd|D||j}ddt||D}|S)z Same as build_batch_from_trees, but use the model parameters to turn the trees into gold sequences and include the sequence rcSg|]}|jqSr(rm)rXr5r(r(r)rZzGBaseModel.build_batch_from_trees_with_gold_sequence..cSsg|] \}}|j|dqSrqrn)rXr5sequencer(r(r)rZrp)rrhrbuild_treebankr r"r\)rrrrrur(r(r))build_batch_from_trees_with_gold_sequences  z3BaseModel.build_batch_from_trees_with_gold_sequencecCr)z Read from the data_iterator batch_size tagged sentences and turn them into new parsing states Expects a list of list of (word, tag) Nr)rrrrhr~)rrrrrsentencer(r(r)build_batch_from_tagged_wordss     z'BaseModel.build_batch_from_tagged_wordsFc s g}g} |||} ttt| } tg} |rtt} t| dkr|| \}}}|r8|dur8ddt| |D} || |} |r\t|D]\}}t|t r[| | | | |j j j qDt t| D]?\}}||r||}|jrx|}|j}| t|t||jg|r|nd|r| | |nd| | ||qctdkrfddt| D} fddt| D} t|t| D]/}t| d}|s|||}t|dkrnt|} t| d}| || t|t| qt| dks!t|| }|S)ag Repeat transitions to build a list of trees from the input batches. The data_iterator should be anything which returns the data for a parse task via next() build_batch_fn is a function that turns that data into State objects This will be called to generate batches of size batch_size until the data is exhausted The return is a list of tuples: (gold_tree, [(predicted, score) ...]) gold_tree will be left blank if the data did not include gold trees if keep_scores is true, the score will be the sum of the values returned by the model for each transition transition_choice: which method of the model to use for choosing the next transition predict for predicting the transition based on the model predict_gold to just extract the gold transition from the sequence rNcSs"g|] \}}|j|j|dqS))rg)rorg)rXr5rgr(r(r)rZ3s"z-BaseModel.parse_sentences..cg|] \}}|vr|qSr(r()rXrir5remover(r)rZJrpcrr(r()rXri batch_idxrr(r)rZKrp)rrrhiterrr\ bulk_applyrr isinstancerrr?valuesetfinishedget_treer"reverser_rrrgaddrrunsort)rrbuild_batch_fnrtransition_choice keep_statekeep_constituents keep_scorestreebanktreebank_indicesr batch_indiceshorizon_iteratorr? pred_scoresrEscorest_idx transitionrir5predicted_treer_r horizon_state horizon_batchr(rr)parse_sentencessX       6        )zBaseModel.parse_sentencesc CsBt||||||||WdS1swYdS)z Given an iterator over the data and a method for building batches, returns a list of parse trees. no_grad() is so that gradients aren't kept, which makes the model run faster and use less memory at inference time N)torchno_gradr)rrrrrrrrr(r(r)parse_sentences_no_grad\s $z!BaseModel.parse_sentences_no_gradc Cs:|dur |jd}t|}|j||j||j|||d}|S)ai Return a ParseResult for each tree in the trees list The transitions run will be the transitions represented by the tree The output layers will be available in result.state for each result keep_state=True as a default here as a method which keeps the grad is likely to want to keep the resulting state as well Neval_batch_size)r)r$rrrra)rrrrrr tree_iteratorrr(r(r) analyze_treesfs  zBaseModel.analyze_treescCsLtdt||t|}|j||j||jddd}dd|D}|S)a} This parses tagged words and returns a list of trees. `parse_tagged_words` is useful at Pipeline time - it takes words & tags and processes that into trees. The tagged words should be represented: one list per sentence each sentence is a list of (word, tag) The return value is a list of ParseTree objects zProcessing %d sentencesF)rrcSsg|]}|jdjqSr) predictionsr)rXtr(r(r)rZr[z0BaseModel.parse_tagged_words..)loggerdebugrhevalrrrrS)rrzrsentence_iteratorrresultsr(r(r)parse_tagged_wordsws  zBaseModel.parse_tagged_wordscstg}g}gtt}tt||D]z\}\}} | s7d|j||} |r,t| t |  |q|j t |jdkrh|jrOd|j||} nd||} |r]t| t |  |q| ||\} } } }|| || | |r||t dq|D]!\}}fdd|D}|||}t||D]\}}||<qqt dkrчfd dt|D}fd dt|D}t |dkr|S|d d|D|}||d dt||||D}|S) ai Apply the given list of Transitions to the given list of States, using the model as a reference model: SimpleModel, LSTMModel, or any other form of model state_batch: list of States transitions: list of transitions, one per state fail: throw an exception on a failed transition, as opposed to skipping the tree z^Got stuck and couldn't find a legal transition on the following gold tree: {} Final state: {}z=Went infinite on the following gold tree: {} Final state: {}zWent infinite!: Final state: {}csg|]}|qSr(r()rXx)new_constituentsr(r)rZsz(BaseModel.bulk_apply..rcrr(r()rXrirrr(r)rZrpcrr(r()rXrir`rr(r)rZrpcSrr()rErr(r(r)rZrcSs2g|]\}}}}}|j|j||||dqS))rdrfrEr?)rord delta_opens)rXr5rrftransition_stackr?r(r(r)rZs  )rrrrrr\r^r_ to_string ValueErrorrerrorrrWrhre update_stateritemsr<rFr@)rrrEfailword_positionsr? callbacksrirrrrjcnccallbackkeyidxsdatacallback_constituents constituentnew_transitionsr()rrr)rs\             zBaseModel.bulk_apply)Tr)FFF)NTTT)F)'__name__ __module__ __qualname____doc__rrr,r/r0r3r6r9r<r@rBrFrHpropertyr#r!r rKrLr"rSrUrarwr~rrrrrrrrr __classcell__r(r(r&r)r's`                     H rcseZdZdZejeddffdd ZddZdd Z d d Z d d Z ddZ ddZ ddZddZddZddZddZZS) SimpleModelab This model allows pushing and popping with no extra data This class is primarily used for testing various operations which don't need the NN's weights Also, for rebuilding trees from transitions when verifying the transitions in situations where the NN state is not relevant, as this class will be faster than using the NN F)ROOTcstj||||ddS)N)r r!r"r#)rr)rr r!r"r#r&r(r)rszSimpleModel.__init__cCsLg}|D]}dg}|dd|D7}|d|jr|||q|S)NcSsg|]}|qSr(r()rXtag_noder(r(r)rZsz3SimpleModel.initial_word_queues..)rr"r)rr+rv tagged_wordsrer(r(r)r,s  zSimpleModel.initial_word_queuescCtddddSNr)rparentlengthr r.r(r(r)r/zSimpleModel.initial_transitionscCrrr r.r(r(r)r0rz SimpleModel.initial_constituentscC|Srr(r1r(r(r)r3zSimpleModel.get_wordcCs ||jSr)r3rfr4r(r(r)r6s z)SimpleModel.transform_word_to_constituentcCrrr(r7r(r(r)r9rzSimpleModel.dummy_constituentcCsPg}t||D]\}}t|tr|f}t|D]}t||d}q||q|S)N)rr)r\rstrreversedr r)rr:r;r?rrrr(r(r)r<s   zSimpleModel.build_constituentscCddt||DS)NcSg|] \}}||qSr(push)rXstackrr(r(r)rZ z1SimpleModel.push_constituents..r\r=r(r(r)r@ zSimpleModel.push_constituentscC|jSrrrAr(r(r)rB zSimpleModel.get_top_constituentcCr)NcSrr(r)rXrrr(r(r)rZrz0SimpleModel.push_transitions..rrCr(r(r)rFrzSimpleModel.push_transitionscCrrrrGr(r(r)rHrzSimpleModel.get_top_transition)rrrrrr UNARY_LIMITrr,r/r0r3r6r9r<r@rBrFrHrr(r(r&r)rs   r)rabcrr collectionsrloggingrstanza.models.commonrstanza.models.constituencyr,stanza.models.constituency.parse_transitionsrr%stanza.models.constituency.parse_treer stanza.models.constituency.stater %stanza.models.constituency.tree_stackr stanza.server.parser_evalrrr getLoggerrrrr(r(r(r)s&       4