o hp@sdZddlZddlmZddlZddlZddlmZddlZddl Z ddl Z ddl m Z ddl mZmZmZddlmZmZmZddlTdd lmZdd lmZeZd d Zd dgZdddddddddddddd ZddeDZe e!Z"dd Z#d!d"Z$d#d$Z%d%d&Z&d'd(Z'd)d*Z(d+d,Z)d-d.Z*d/d0Z+d1d2Z,d3d4Z-d5d6Z.d7d8Z/d9d:Z0d;d<Z1d=d>Z2d?d@Z3dAdBZ4dCdDZ5dEdFZ6dGdHZ7dIdJZ8dKdLZ9dMdNZ:dOdPZ;dQdRZdWdXZ?dYdZZ@d[d\ZAd]d^ZBd_d`ZCdadbZDeEdckr eDdSdS)da3 Converts a directory of models organized by type into a directory organized by language. Also produces the resources.json file. For example, on the cluster, you can do this: python3 -m stanza.resources.prepare_resources --input_dir /u/nlp/software/stanza/models/current-models-1.5.0 --output_dir /u/nlp/software/stanza/models/1.5.0 > resources.out 2>&1 nlprun -a stanza-1.2 -q john "python3 -m stanza.resources.prepare_resources --input_dir /u/nlp/software/stanza/models/current-models-1.5.0 --output_dir /u/nlp/software/stanza/models/1.5.0" -o resources.out N) defaultdict)Path)__resources_version__) lcode2langtwo_to_three_lettersthree_to_two_letters)PACKAGES TRANSFORMERSTRANSFORMER_NICKNAMES)*)DATASET_MAPPING)get_tqdmcCst}|jdtdtdd|jdtdtdd|jdd d d d |jd tddd|}tj|j |_ tj|j |_ |j durPd |j |_ |S)Nz --input_dirz//u/nlp/software/stanza/models/current-models-%szRInput dir for various models. Defaults to the recommended home on the nlp cluster)typedefaulthelpz --output_dirz /u/nlp/software/stanza/models/%szOutput dir for various models.z--packages_only store_trueFzFsr=cCst|jddddS)NT)parentsexist_ok)rmkdir)dirr$r$r% ensure_dirIsrBcCstt|jt||dSN)rBrparentshutilcopy2)srcdstr$r$r% copy_fileMsrIcCst|d}t|S)Nrb)openreadhashlibmd5 hexdigest)rdatar$r$r%get_md5RsrQcCs|dddd}ttdddD]}||r,|dt|d }t|}nqtd ||dd\}}|||fS) zc Split model names by _ Takes into account packages with _ and processor types with _ N._cSs t| SrC)len)xr$r$r%_s z"split_model_name..)keyz#Could not find a processor type in )replacesortedending_to_processorkeysendswithrUAssertionErrorr!)model processorrpackager$r$r%split_model_nameWs  rccCs|dr |dd}|dr|dd}|ddfS|dr+|dd}|ddfS|d r;|dd }|ddfS|d }|d krZ||d d}|tvrZ|d|ddfS|ddfS)N _finetunedi _nopretrainiF _nocharlmiT_charlmirTrrY)r^rfindknown_nicknames)rb underscorenicknamer$r$r% split_packageis"              rlcCsft|\}}}|r |tvrdS|dur#||vr#|||vr#|||S||vr+||Std||f)Nz-pretrain not specified for lang %s package %s)rlno_pretrain_languages RuntimeError)rrbmodel_pretrainsdefault_pretrains uses_pretrainrTr$r$r%get_pretrain_packages  rrcCsJt|\}}}|s dS|dur||vr|||vr|||S||dSrC)rlget)rrb model_charlmsdefault_charlmsrT uses_charlmr$r$r%get_charlm_packages   rwcCsRt||dt}d|dg}t|d}|dur'|d|d|d|d|SNr,r`rbr.r/rrrprursappendrrbpretrain_package dependenciescharlm_packager$r$r%get_con_dependenciess  rcCt||ttSrC)rw pos_charlmsrurrbr$r$r%get_pos_charlm_packagercC`g}t||tt}|dur|d|dt||}|dur.|d|d|d|d|Srx)rr pos_pretrainsrpr{rrrbr~r}rr$r$r%get_pos_dependencies rcCs@t|\}}}|s dS|sdSd||ftvrdSt||itS)Nz%s_%s)rlLEMMA_CLASSIFIER_DATASETSrrrp)rrbrqrvr$r$r%get_lemma_pretrain_packagesrcCrrC)rw lemma_charlmsrurr$r$r%get_lemma_charlm_packagerrcC\g}t||}|dur|d|dt||}|dur,|d|d|d|d|Srx)rr{rrr$r$r%get_lemma_dependencies  rcCrrC)rwdepparse_charlmsrurr$r$r%get_depparse_charlm_packagerrcCrrx)rrdepparse_pretrainsrpr{rrr$r$r%get_depparse_dependenciesrrcCrrC)rw ner_charlmsrurr$r$r%get_ner_charlm_packagerrcCrrC)rr ner_pretrainsrprr$r$r%get_ner_pretrain_packagerrcCrrx)rr{rrr$r$r%get_ner_dependenciesrrcCsRt||dt}d|dg}t|d}|dur'|d|d|d|d|S)a Return a list of dependencies for the sentiment model Generally this will be pretrain, forward & backward charlm So far, this invariant is true: sentiment models use the default pretrain for the language also, they all use the default charlm for a language Nr,ryr.r/rzr|r$r$r%get_sentiment_dependenciess  rcCsp|dkr t||S|dkrt||S|dkrt||S|dkr$t||S|dkr-t||S|dkr6t||SiS)z Get the dependencies for a particular lang/package based on the package name The package can include descriptors such as _nopretrain, _nocharlm, _charlm which inform whether or not this particular model uses charlm or pretrain r8r6r7r9r0r1)rrrrrr)rarrbr$r$r%get_dependencies s      rc Cstt|j}i}|jr'tttj |j d}|j dD]}i||<q |D]}t d|tttj |j|}t |D]l}|dsKqCt|\}}}|jr^||j dvr^qCtj |j||} tj |j |d||d} t| | t| } t|||} ||vri||<|||vri|||<| r| | d||||<qCd| i||||<qCq)t dtj|ttj |j dd d d dS) Nresources.jsonrzProcessing models in .ptmodels)rNr~rNzHProcessed initial model directories. Writing preliminary resources.jsonwindent)r[rlistdirrrjsonloadrKrrrr!printtqdmr^rcrIrQrdump) r#dirs resourcesr model_dirrr`rbra input_path output_pathrNr~r$r$r% process_dirs"s6     &rcC2t||}|dur |dS|tvr|dS|dSNrgrerf)rrmr ud_packagerr$r$r%get_default_pos_packageI rcCrr)rrmrr$r$r%get_default_depparse_packageQrrc Csftttj|jd}|D] }|dkrqd||vrqtdd||Dr,q|t vr7t |d|j rC||j dvrCqt d|tt}||td }|D]/\}}|d krg|d krgqZ|d krlqZ|||t|||}|D] }||d |dq{qZg} tD]C}||vrt||D]6}tj|j|d||d} tj| rt d||| | | ||fqtd|d|d|d| qttj|j|dddtj} | D]\} }}| j| tj||ddqWdn 1swYttj|j|dd} | ||d<qt dtj|ttj|jdddddS)Nrurlaliascs|]}|dvVqdS)r/r.r, lang_nameNr$r:kr$r$r% az'process_default_zips.. not in default treebanks!!!rz&Preparing default models for language rr6identityoptionalr`rbrrz Model {} package {}: file {}z Processor z package z needed for z but cannot be found at z default.zipr)filenamearcname default_md5z5Processed default model zips. Writing resources.jsonrr) rrrKrrrrallr]default_treebanksr_rr!rrsetritemsaddr PROCESSORSr[existsformatr{FileNotFoundErrorzipfileZipFile ZIP_DEFLATEDwriterQr) r#rr models_neededpackagesrarbr~ dependency model_filesrzipfrr$r$r%process_default_zipsYsX     "&rcCs|dkrddiSt|}i}|tvrt||d<n||d<d||vr1|d||dvr1||d<d||vrH|d}|||dvrG||d<n|tvrPd|d<d ||vrpt|||d <|d ||d vrotd |d n |tvrztd |d ||vrt|||d <|d ||d vrtd |d n |tvrtd||tvrt||d<|tvrt||d<|tvrt||d<t ||}|r||d<|S)z Build a default package for this language Will add each of pos, lemma, depparse, etc if those are available Uses the existing models scraped from the language directories into resources.json, as relevant multilingualr3udr4r5r6rfrr7z'Expected POS model not in resources: %sz+Expected to find POS models for language %sr8z,Expected depparse model not in resources: %sz0Expected to find depparse models for language %sr9r0r1r) rdefault_tokenizerallowed_empty_languagesrr_r default_nersdefault_sentimentdefault_constituencyget_default_optional_processors)rrdefault_packagedefault_processorsexpected_lemmarr$r$r%get_default_processorssP          rcCs0i}|tvr t||d<|tvrt||d<|S)Nr1r2)optional_constituencyoptional_coref)rroptional_processorsr$r$r%rs   rcCs^||vrdS||dd|dd|}||||vr$|||<dStd|||fdS)NrgrTrfzKWARNING: wanted to use %s for %s accurate %s, but that model does not exist)rZr)rrcurrent_processorsra transformer new_modelr$r$r% update_processor_add_transformers   rcCst||}d|vr7|ddkr7|d}|dd}t||}|dur7|||dvr/||d<ntd||ftt|dd}|durldD] }t|||||qGd|vrl|ddsd|ddrlt|||d|t ||}|rw||d <|S) z] A package that, if available, uses charlm and transformer models for each processor r6rrfrgNzVWARNING: wanted to use %s for %s default_accurate lemma, but that model does not existr7r8r1r0r9r) rrZrrr rsr rr^get_optional_accurate)rrr lemma_modelrrrarr$r$r%get_default_accurates&    $ rcCsXt||}tt|dd}|dur dD] }t|||||q|tvr*t||d<|S)Nrr2)rr rsr rr)rrrrrar$r$r%rs  rcCsrt||}d|vr|d|D]$\}}d|vr6|dd}||||vr2td|||fq|||<q|S)a Build a packages entry which only has the nocharlm models Will make it easy for people to use the lower tier of models We do this by building the same default package as normal, then switching everything out for the lower tier model when possible. We also remove constituency, as it is super slow. Note that in the case of a language which doesn't have a charlm, that means we wind up building the same for default and default_nocharlm r1rgrfzYWARNING: wanted to use %s for %s default_fast processor %s, but that model does not exist)rpoprrZr)rrrrar`nocharlmr$r$r%get_default_fasts   rc Cstttj|jd}|D]}|dkrqd||vrqtdd||Dr,q|t vr7t |d|j rC||j dvrCqt ||}|||d<i||t<|||td <|tvr~|d kr~t||}|||td <t||}|||td <d ||vr,||d D]}d |i}d||vr|||dvr||d<d||vr|d||dvr|d|d<n|d||dvr|d|d<d||vrd|vr|d}|||dvr||d<d||vr#d|vr#d} |d||dvr|d} n|d||dvr|d} | dur#d|vrd|d<| |d<|||t|<qqtdtj|ttj|jdddddS)zz Build a package for a language's default processors and all of the treebanks specifically used for that language rrrcsrrr$rr$r$r%r-rz#process_packages..rrrrr default_fastdefault_accurater4r5r7rgrfr6r8Nrz+Processed packages. Writing resources.jsonrrr)rrrKrrrrrr]rr_rr!rrrmrrrr) r#rrrrrrb processors lemma_packagedepparse_packager$r$r%process_packages!sb            &rcCstttj|jd}i}|d|d<|D]\}|dkrqd||vr%q|tvr0t|dqt|}|||d<|| || <d| i|| <| t vrad| i|t | <q| t vrsd| i|t | <qtdtj |ttj|jdddd dS) Nrrrz not found in lcode2lang!rz0Processed lcode aliases. Writing resources.jsonrrr) rrrKrrrrrrlowerrrr)r#r resources_newrrr$r$r% process_lcodems,      &rcCshtttj|jd}ddi|d<ddi|d<d|d<td tj|ttj|jdd d d dS) Nrrnbnozzh-hanszhz_https://huggingface.co/stanfordnlp/stanza-{lang}/resolve/v{resources_version}/models/{filename}rz2Finalized misc attributes. Writing resources.jsonrrr) rrrKrrrrrr)r#rr$r$r% process_miscs   &rcCsVt}td|j|jf|jst|t||js)t|t|t |dSdS)NzConverting models from %s to %s) rrrr packages_onlyrrrrr)r#r$r$r%mains r__main__)F__doc__r collectionsrrrpathlibrrMrErstanzarstanza.models.common.constantrrr!stanza.resources.default_packagesrr r .stanza.utils.datasets.prepare_lemma_classifierr rstanza.utils.get_tqdmr rrrprocessor_to_endingrr\listr]rrBrIrQrcrlrrrwrrrrrrrrrrrrrrrrrrrrrrrrrrr__name__r$r$r$r%s          '49  L