o he @sldZddlmZmZddlZddlZddlZddlZddlZddl m Z ddl Z ddl Z ddl Z ddlZddlmZddlmZddlmZmZmZmZmZmZmZddlmZmZdd lmZdd l m!Z!e"d Z#e$e %Z&d Z'd Z(e)de(dZ*e)de!Z+e)ddZ,e)dej-.e&dZ/dZ0Gddde1Z2Gddde3Z4Gddde3Z5edgdZ6ddZ7d d!Z8d"d#Z9d$d%Z:d&d'Z;dMd(d)Zd0d1Z?d2d3Z@dPd4d5ZAd6d7ZBd8d9ZCd:d;ZDdd?ZFe/dfd@dAZGdBdCZHe/e*de+dfdDdEZIdFdGZJe/e+e,dd-fdHdIZKdJe/didde*de+e,dd-f dKdLZLdS)Qz( Common utilities for Stanza resources. ) defaultdict namedtupleN)Path)tqdm) make_table)TOKENIZEMWTPOSLEMMADEPPARSENER SENTIMENT)PIPELINE_NAMESPROCESSOR_VARIANTS)PACKAGES)__resources_version__stanzaz:https://nlp.stanford.edu/software/stanza/stanza-resources/z?https://raw.githubusercontent.com/stanfordnlp/stanza-resources/STANZA_RESOURCES_URLmainSTANZA_RESOURCES_VERSIONSTANZA_MODEL_URLdefaultSTANZA_RESOURCES_DIRstanza_resources)pretrainforward_charlmbackward_charlmceZdZfddZZS)ResourcesFileNotFoundErrorcstd|d||_dS)NzResources file not found at: z" Try to download the model again.)super__init__resources_filepath)selfr! __class__R/var/www/html/env_mimamsha/lib/python3.10/site-packages/stanza/resources/common.pyr -s z#ResourcesFileNotFoundError.__init____name__ __module__ __qualname__r __classcell__r%r%r#r&r,rcr)UnknownLanguageErrorctd|||_dS)NzUnknown language requested: )rr unknown_languager"unknownr#r%r&r 2 zUnknownLanguageError.__init__r'r%r%r#r&r-1r,r-cr)UnknownProcessorErrorcr.)Nz"Unknown processor type requested: )rr unknown_processorr0r#r%r&r 7r2zUnknownProcessorError.__init__r'r%r%r#r&r36r,r3ModelSpecification processorpackage dependenciescCst|jddddS)z/ Create dir in case it does not exist. T)parentsexist_okN)rmkdir)pathr%r%r& ensure_dir=sr>c Cslzt|d }|}Wdn1swYWnty.}z|js)||_d}~wwt|S)z& Get the MD5 value of a path. rbN)openreadOSErrorfilenamehashlibmd5 hexdigest)r=findataer%r%r&get_md5Cs  rJcCs\td|d|dttj||}||WddS1s'wYdS)zD Fully unzip a file `filename` that's in a directory `dir`. zUnzip: /z...N)loggerdebugzipfileZipFileosr=join extractall)r=rCfr%r%r&unzipPs "rTcCs>t|d}t|jdksJd|dtj|jdjS)z: Get the root directory from a archived zip file. rrz Zip file at fz( seems to be corrupted. Please check it.)rNrOlenfilelistrPr=dirnamerC)rCzfr%r%r&get_root_from_zipfileXs  rZcCstj|o t||kS)zN Check if the file at `path` exists and match the provided md5 value. )rPr=existsrJ)r=rEr%r%r& file_existsar\cCsdtj|s ttjd||r.t|}||kr0||kr%td|||dSt d|||fdSdS)NzCannot find expected filez?Found a possibly older version of file %s, md5 %s instead of %szmd5 for %s is %s, expected %s) rPr=r[FileNotFoundErrorerrnoENOENTrJrLrM ValueError)r=rE alternate_md5file_md5r%r%r&assert_file_existsgs rdFc Cstjdv}tj|d|d}|r|t|dV}t|jd}d}d|} t|dd| | d #} |j |d D]} | rL| | | | t | q8Wd n1sWwYWd |jSWd |jS1sqwY|jS) z< Download a URL into a file as specified by `path`. )r T)streamproxieswbzcontent-lengthiz Downloading B)totalunit unit_scaledisabledesc) chunk_sizeN)rLlevelrequestsgetraise_for_statusr@intheadersr iter_contentwriteflushupdaterV status_code) urlr=rhrtverboserUrS file_sizedefault_chunk_sizeropbarchunkr%r%r& download_filers6        rTc Cst|j}t|t||r$|rtd|dStd|dStj|d"}t j |t j |d} t || ||t | |Wdn1sOwYt||||rftd|dStd|dS)a A complete wrapper over download_file() that also make sure the directory of `path` exists, and that a file matching the md5 value does not exist. alternate_md5 allows for an alternate md5 that is acceptable (such as if an older version of a file is okay) z File exists: N)dirzDownloaded file to )rparentr>r\rLinforMtempfileTemporaryDirectoryrPr=rQsplitrreplacerd) r|r=rhrErtlog_inforbbasedirtemptemppathr%r%r& request_files"   rcCsdg}tD]}|D] }|d|kr||qq|D]}|D] }|d|dkr)nq||q|S)Nr)rappend)processor_list sorted_listr7itemr%r%r&sort_processorss   rcCs|t}|||tvr#t||t|vr#td||||t<dS|||tvrFt||vrH|||tvrJtd||||t<dSdSdSdS)aAdd mwt if tokenize is passed without mwt. If tokenize is in the list, but mwt is not, and there is a corresponding tokenize and mwt pair in the resources file, mwt is added so no missing mwt errors are raised. TODO: how does this handle EWT in English? z8Language %s package %s expects mwt, which has been addedN)rrrrLwarning) processors resourceslangvaluer%r%r&add_mwts $ ,  rc Cstt}|rZtd|rt|vrt|vrt||||D]8\}}t|t s.t dt|t tt fs:t dt|t rB|g}|t vrP|rL|t vrPt||D]} |||vrw| |||vrwtd|d| d||| qR| ||tvr|||t| vrtd|d||t| |d||||t| |qR| ||tvrd||t| vr|||t| dvrtd|d||t| d|d||||t| d|qR| t|vrtd|d| d| d |d ||| qR|tkr2| d kr2td|d| d ||| qR|||vrLt|d| d ||| qRtd|d| dqRq |rtdt||vr|||tvr||t|D]"\}} |dkr||vrtd|d| d||| q|nUd} t D]D}|||vrq||||vrd} ||vrtd|d|d|||qt|d|d|d||dq| std|ddd|D}t|}|S)a Given a parsed resources file, language, and possible package and/or processors, expands the package to the list of processors Returns a list of processors Each item in the list of processors is a pair: name, then a list of ModelSpecification so, for example: [['pos', [ModelSpecification(processor='pos', package='gsd', dependencies=None)]], ['depparse', [ModelSpecification(processor='depparse', package='gsd', dependencies=None)]]] z$Processing parameter "processors"...zProcessor names must be stringsz Processor values must be stringszFound : .optionalz. Using external z variant for the z processor.identityz. Using identity lemmatizer.z: is not officially supported by Stanza, loading it anyway.z Can not find z' from official model list. Ignoring it.z!Processing parameter "package"...FTz is overwritten by zCan not find package: cs&g|]\}fdd|DgqS)csg|] }t|ddqS)Nr6)r5).0rkeyr%r& &sz6maintain_processor_list...r%)rplistr%rr&r&s&z+maintain_processor_list..)rlistrLrMrrritems isinstancestrratuplerPRETRAIN_NAMESr3rrrr rr) rrr8rallow_pretrain maybe_add_mwtrrrrflagr%r%r&maintain_processor_lists       $  <$$,    rc Cs||}|D]Q}|\}}g}|D]@}t|jt|v|tko!|jdkgsK||i|jidg} dd| D} |jt| d}td| ||j| |qt||d<q|S)a Expand the processor_list as given in maintain_processor_list to have the dependencies Still a list of model types to ModelSpecifications the dependencies are tuples: name and package for example: [['pos', (ModelSpecification(processor='pos', package='gsd', dependencies=(('pretrain', 'gsd'),)),)], ['depparse', (ModelSpecification(processor='depparse', package='gsd', dependencies=(('pretrain', 'gsd'),)),)]] rr9cSsg|] }|d|dfqS)modelr8r%r dependencyr%r%r&r?sz$add_dependencies..)r9z/Found dependencies %s for processor %s model %s) anyr8rr rs_replacerrLrMr) rrrlang_resourcesrr7 model_specsnew_model_specs model_specr9r%r%r&add_dependencies*s    rc Csg}g}|D]#}|\}}|D]}|j}|j}|||g|r(|dd|D7}qqddt|D}|D]\}}td|d|dq5||7}|S)z The flattened processor list is just a list of types & packages For example: [['pos', 'gsd'], ['depparse', 'gsd'], ['pretrain', 'gsd']] cSg|]}t|qSr%)rrr%r%r&rVz*flatten_processor_list..cSrr%)r)rrr%r%r&rWrzFind dependency rr)r8r9rsetrLrM) rflattened_processor_listdependencies_listrr7rrr8r9r%r%r&flatten_processor_listFs" rcCs|dkrd}n|dkr d}|durtjdkrtdtjS|}gd}||vr8td|dd |d t|tjS) NFERRORTINFOr)DEBUGrWARNINGWARNrCRITICALFATALz)Unrecognized logging level for pipeline: z. Must be one of , r)rLrqsetLevelupperrarQ) logging_levelr} all_levelsr%r%r&set_logging_level]s(   rcst|tr |}n|durtdt|jdt|tr%|}n|dur4tdt|jdt|tttfrdurGt ddn(ttrWt fddntt rdt ddn tdtjdt|tr~d d | d D}fd d |D}dnt|t rdd | D}n|durtdt|jdttrndurtdtjd|||fS)Nz,The parameter 'lang' should be str, but got z instead.z1The parameter 'model_dir' should be str, but got cSdSNrr%r%r%r%r&z-process_pipeline_parameters..csS)Nr%r%)rr%r&rrcSrrr%r%r%r%r&rrz>The parameter 'package' should be None, str, or dict, but got cSg|]}|qSr%striplower)rxr%r%r&rz/process_pipeline_parameters..,csi|]}||qSr%r%)rr7)r8r%r& s z/process_pipeline_parameters..cSsBi|]\}}|t|ttfrdd|Dn|qS)cSrr%r)rv_ir%r%r&rrz:process_pipeline_parameters...)rrrrr)rkvr%r%r&rs0z:The parameter 'processors' should be dict or str, but got zSThe parameter 'package' should be str, or a dict if 'processors' is a str, but got ) rrrr TypeErrortyper(rrrdictrr)r model_dirr8rr%)rr8r&process_pipeline_parameterswsl          rcCsn|tkr |dur t|}|dvrt}|d|d}td||dur-tj|d}t |||dddS) z= Downloads resources.json to obtain latest packages. N)stanford stanfordnlpz /resources_z.jsonz!Downloading resource file from %sresources.jsonT)rt) DEFAULT_RESOURCES_URLSTANZA_RESOURCES_GITHUBrSTANFORDNLP_RESOURCES_URLrLrMrPr=rQr)r resources_urlresources_branchresources_versionr!rhr%r%r&download_resources_jsons    rcCsf|dur tj|d}tj|st|t|dd}t|}Wd|S1s,wY|S)zA Unpack the resources json file from the given model_dir Nrzutf-8)encoding)rPr=rQr[rr@jsonload)rr!rGrr%r%r&load_resources_jsons   rcCs8||vrdS||}d|vr|d}||}d|vs|S)zi Get the resources for a lang from an already loaded resources json, following 'alias' if needed Naliasr%)rrrr%r%r&get_language_resourcessrcs:t||||d|dt|fddD}t|}|S)z< List the non-alias languages in the resources file Nr!rhcs*g|]}t|tsd|vr|qS)r)rr)rrrr%r&rs   z,list_available_languages..)rrsorted)rrrrrh languagesr%rr&list_available_languagess rcCs|dkr |dS|S)za Returns the url in the resources dict if model_url is default, or returns the model_url rr|)r)r model_urlr%r%r&expand_model_urlr]rc Cs||id|}tddg|} |rtj} ntj} | d|d|d| t||} |D]P\} } z4t| j||| d| dd tj ||| | d|||| | d |||| | d dd Wq.t y~}z t d | d| d|d}~wwdS)N lang_name ProcessorPackagez4Downloading these customized packages for language:  (z)... rKz.ptrrrCrErb)rErrbz@Cannot find the following processor and model name combination: rz;. Please check if you have provided the correct model name.) rsrrLrrMrrformatrPr=rQKeyErrorra) download_listrrrrrrhrrdownload_tablelog_msgr|rrrIr%r%r&download_modelssL   renc  Cst||t||||\}}}}| stjtj|ds.| s$tdt||||d| dt |} || vr:t |d| |vrVt d|d| |dd| |d}| |i d|} t | | }|dkr|dusst|d krt d |d | d t|j||d dtj||d | | |ddttj||d nt| |||dd}t| ||}t|}t|| |||| | ddt d|dS)NrzZAsked to skip downloading resources.json, but the file does not exist. Downloading anywayrr"z" is an alias for "rrrz+Downloading default packages for language: rz) ...z default.zipr default_md5)rET)r)rrrrrrrhrz)Finished downloading models and saved to )rrrPr=r[rQrLrrrr-rrsrrVrrrTrrrr)rrr8rrr}rrrrrh download_jsonrrr|rr%r%r&download+sP         r)NN)F)NNFTN)FT)M__doc__ collectionsrrr_rDrloggingrPpathlibrrrshutilrrN tqdm.autorstanza.utils.helper_funcrstanza.pipeline._constantsrrr r r r r stanza.pipeline.registryrr!stanza.resources.default_packagesrstanza._versionr getLoggerrLrhomeHOME_DIRrrgetenvrDEFAULT_RESOURCES_VERSIONDEFAULT_MODEL_URLr=rQDEFAULT_MODEL_DIRrr^rrar-r3r5r>rJrTrZr\rdrrrrrrrrrrrrrrrrr%r%r%r&s   $          bA     (