o
    hL                     @   sZ   d Z ddlZddlmZmZ ddlmZmZ dd Zdd Z	d	d
 Z
edkr+e
  dS dS )zInvokes the Java tokensregex on a document

This operates tokensregex on docs processed with stanza models.

https://nlp.stanford.edu/software/tokensregex.html

A minimal example is the main method of this module.
    N)TokensRegexRequestTokensRegexResponse)send_requestadd_sentencec                 C   s   t | tdS )Nz;edu.stanford.nlp.ling.tokensregex.ProcessTokensRegexRequest)r   r   )request r   T/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/server/tokensregex.pysend_tokensregex_request   s   r	   c                 G   sh   t  }|D ]}|j| q|j}| j|_d}| jD ]}t|j|| |tdd |j	D  }qt
|S )Nr   c                 s   s    | ]}t |jV  qd S )N)lenwords).0tokenr   r   r   	<genexpr>   s    zprocess_doc.<locals>.<genexpr>)r   patternappenddoctext	sentencesr   sentencesumtokensr	   )r   patternsr   r   request_doc
num_tokensr   r   r   r   process_doc   s   
r   c                  C   s*   t jddd} | d}tt|dd d S )Nentokenize)
processorsz0Uro ruined modern.  Fortunately, Wotc banned himhimruined)stanzaPipelineprintr   )nlpr   r   r   r   main!   s
   r$   __main__)__doc__r    stanza.protobufr   r   $stanza.server.java_protobuf_requestsr   r   r	   r   r$   __name__r   r   r   r   <module>   s    	

