Index

A

AutoJapaneseTokenizer (class in sumire.tokenizer.auto)

B

BaseTokenizer (class in sumire.tokenizer.base)

BaseVectorizer (class in sumire.vectorizer.base.common)

C

CountVectorizer (class in sumire.vectorizer.count)

D

decode() (sumire.tokenizer.spm.SentencePieceTokenizer method) (sumire.vectorizer.base.sklearn_vectorizer_base.SkLearnVectorizerBase method) description (sumire.vectorizer.swem.ModelCard attribute) (sumire.vectorizer.transformer_emb.ModelCard attribute)

dict_type (sumire.tokenizer.sudachi.SudachiTokenizer attribute) download_file() (in module sumire.utils.download_file)

E

encode() (sumire.tokenizer.spm.SentencePieceTokenizer method)

F

fit() (sumire.tokenizer.base.BaseTokenizer method) (sumire.tokenizer.spm.SentencePieceTokenizer method) (sumire.vectorizer.base.common.BaseVectorizer method) (sumire.vectorizer.base.sklearn_vectorizer_base.SkLearnVectorizerBase method) (sumire.vectorizer.base.transformer_vectorizer_base.TransformersVectorizerBase method) (sumire.vectorizer.swem.W2VSWEMVectorizer method) fit_transform() (sumire.vectorizer.base.common.BaseVectorizer method) from_pretrained() (sumire.tokenizer.auto.AutoJapaneseTokenizer class method) (sumire.tokenizer.base.BaseTokenizer class method) (sumire.tokenizer.jumanpp.JumanppTokenizer class method) (sumire.tokenizer.mecab.MecabTokenizer class method) (sumire.tokenizer.spacy_ginza.SpacyGinzaTokenizer class method) (sumire.tokenizer.spm.SentencePieceTokenizer class method) (sumire.tokenizer.sudachi.SudachiTokenizer class method) (sumire.vectorizer.base.common.BaseVectorizer class method) (sumire.vectorizer.base.sklearn_vectorizer_base.SkLearnVectorizerBase class method) (sumire.vectorizer.count.CountVectorizer class method) (sumire.vectorizer.swem.W2VSWEMVectorizer class method) (sumire.vectorizer.tfidf.TfidfVectorizer class method) (sumire.vectorizer.transformer_emb.TransformerEmbeddingVectorizer class method)

G

get_token_vectors() (sumire.vectorizer.base.common.GetTokenVectorsMixIn method) (sumire.vectorizer.swem.W2VSWEMVectorizer method) (sumire.vectorizer.transformer_emb.TransformerEmbeddingVectorizer method)

GetTokenVectorsMixIn (class in sumire.vectorizer.base.common)

J

JumanppTokenizer (class in sumire.tokenizer.jumanpp)

L

load_init_args() (sumire.vectorizer.base.transformer_vectorizer_base.TransformersVectorizerBase class method)

M

MecabTokenizer (class in sumire.tokenizer.mecab) model_cache_dir (sumire.tokenizer.spm.SentencePieceTokenizer attribute) model_computed_fields (sumire.vectorizer.swem.ModelCard attribute) (sumire.vectorizer.transformer_emb.ModelCard attribute) model_config (sumire.vectorizer.swem.ModelCard attribute) (sumire.vectorizer.transformer_emb.ModelCard attribute) model_fields (sumire.vectorizer.swem.ModelCard attribute) (sumire.vectorizer.transformer_emb.ModelCard attribute) model_name (sumire.vectorizer.transformer_emb.ModelCard attribute) ModelCard (class in sumire.vectorizer.swem) (class in sumire.vectorizer.transformer_emb) module scripts scripts.controllers scripts.controllers.evaluations scripts.data sumire sumire.tokenizer sumire.tokenizer.auto sumire.tokenizer.base sumire.tokenizer.jumanpp sumire.tokenizer.mecab sumire.tokenizer.spacy_ginza sumire.tokenizer.spm sumire.tokenizer.sudachi sumire.utils sumire.utils.download_file sumire.utils.run_sh sumire.utils.unbz_file sumire.utils.untar_file sumire.utils.unzip_file sumire.vectorizer sumire.vectorizer.base sumire.vectorizer.base.common sumire.vectorizer.base.sklearn_vectorizer_base sumire.vectorizer.base.transformer_vectorizer_base sumire.vectorizer.count sumire.vectorizer.swem sumire.vectorizer.tfidf sumire.vectorizer.transformer_emb tests tests.tokenizer tests.tokenizer.test_tokenizers tests.vectorizer tests.vectorizer.test_count_vectorizer tests.vectorizer.test_swem_vectorizer tests.vectorizer.test_tfidf_vectorizer tests.vectorizer.test_transformer_emb_vectorizer

N

name (sumire.vectorizer.swem.ModelCard attribute)

normalize (sumire.tokenizer.sudachi.SudachiTokenizer attribute)

R

run_sh() (in module sumire.utils.run_sh)

S

save_pretrained() (sumire.tokenizer.auto.AutoJapaneseTokenizer method) (sumire.tokenizer.base.BaseTokenizer method) (sumire.tokenizer.jumanpp.JumanppTokenizer method) (sumire.tokenizer.mecab.MecabTokenizer method) (sumire.tokenizer.spacy_ginza.SpacyGinzaTokenizer method) (sumire.tokenizer.spm.SentencePieceTokenizer method) (sumire.tokenizer.sudachi.SudachiTokenizer method) (sumire.vectorizer.base.common.BaseVectorizer method) (sumire.vectorizer.base.sklearn_vectorizer_base.SkLearnVectorizerBase method) (sumire.vectorizer.base.transformer_vectorizer_base.TransformersVectorizerBase method) (sumire.vectorizer.swem.W2VSWEMVectorizer method) scripts module scripts.controllers module scripts.controllers.evaluations module scripts.data module SentencePieceTokenizer (class in sumire.tokenizer.spm) setup_tagger() (sumire.tokenizer.mecab.MecabTokenizer method) setup_w2v_if_not_installed() (sumire.vectorizer.swem.W2VSWEMVectorizer method) SkLearnVectorizerBase (class in sumire.vectorizer.base.sklearn_vectorizer_base) SpacyGinzaTokenizer (class in sumire.tokenizer.spacy_ginza) SudachiTokenizer (class in sumire.tokenizer.sudachi) sumire module sumire.tokenizer module sumire.tokenizer.auto module sumire.tokenizer.base module sumire.tokenizer.jumanpp module sumire.tokenizer.mecab module

sumire.tokenizer.spacy_ginza module sumire.tokenizer.spm module sumire.tokenizer.sudachi module sumire.utils module sumire.utils.download_file module sumire.utils.run_sh module sumire.utils.unbz_file module sumire.utils.untar_file module sumire.utils.unzip_file module sumire.vectorizer module sumire.vectorizer.base module sumire.vectorizer.base.common module sumire.vectorizer.base.sklearn_vectorizer_base module sumire.vectorizer.base.transformer_vectorizer_base module sumire.vectorizer.count module sumire.vectorizer.swem module sumire.vectorizer.tfidf module sumire.vectorizer.transformer_emb module

T

test_common() (in module tests.vectorizer.test_transformer_emb_vectorizer) test_common_parameter() (in module tests.vectorizer.test_swem_vectorizer) test_count_vectorizer() (in module tests.vectorizer.test_count_vectorizer) test_jumanpp_tokenizer() (in module tests.tokenizer.test_tokenizers) test_mecab_ipadic_tokenizer() (in module tests.tokenizer.test_tokenizers) test_mecab_unidic_lite_tokenizer() (in module tests.tokenizer.test_tokenizers) test_mecab_unidic_tokenizer() (in module tests.tokenizer.test_tokenizers) test_spacy_ginza_tokenizer() (in module tests.tokenizer.test_tokenizers) test_spm_tokenizer() (in module tests.tokenizer.test_tokenizers) test_sudachi_tokenizer() (in module tests.tokenizer.test_tokenizers) test_tfidf_vectorizer() (in module tests.vectorizer.test_tfidf_vectorizer) test_transformer_model_type() (in module tests.vectorizer.test_transformer_emb_vectorizer) tests module tests.tokenizer module tests.tokenizer.test_tokenizers module tests.vectorizer module tests.vectorizer.test_count_vectorizer module tests.vectorizer.test_swem_vectorizer module

tests.vectorizer.test_tfidf_vectorizer module tests.vectorizer.test_transformer_emb_vectorizer module TfidfVectorizer (class in sumire.vectorizer.tfidf) tokenize() (sumire.tokenizer.auto.AutoJapaneseTokenizer method) (sumire.tokenizer.base.BaseTokenizer method) (sumire.tokenizer.jumanpp.JumanppTokenizer method) (sumire.tokenizer.mecab.MecabTokenizer method) (sumire.tokenizer.spacy_ginza.SpacyGinzaTokenizer method) (sumire.tokenizer.spm.SentencePieceTokenizer method) (sumire.tokenizer.sudachi.SudachiTokenizer method) (sumire.vectorizer.base.common.BaseVectorizer method) tokenizer (sumire.tokenizer.sudachi.SudachiTokenizer attribute) tokenizer_config_file (sumire.tokenizer.base.BaseTokenizer attribute) tokenizer_dict() (in module sumire.tokenizer.auto) tokenizer_file_prefix (sumire.tokenizer.spm.SentencePieceTokenizer attribute) tokenizer_name (sumire.vectorizer.swem.ModelCard attribute) transform() (sumire.vectorizer.base.common.BaseVectorizer method) (sumire.vectorizer.base.sklearn_vectorizer_base.SkLearnVectorizerBase method) (sumire.vectorizer.swem.W2VSWEMVectorizer method) (sumire.vectorizer.transformer_emb.TransformerEmbeddingVectorizer method) TransformerEmbeddingVectorizer (class in sumire.vectorizer.transformer_emb) TransformersVectorizerBase (class in sumire.vectorizer.base.transformer_vectorizer_base)

U

unbz_file() (in module sumire.utils.unbz_file) untar_file() (in module sumire.utils.untar_file)

unzip_file() (in module sumire.utils.unzip_file) url (sumire.vectorizer.swem.ModelCard attribute)

V

vectorizer_config_file (sumire.vectorizer.base.common.BaseVectorizer attribute)

W

w2v_dir (sumire.vectorizer.swem.W2VSWEMVectorizer attribute), [1]

W2VSWEMVectorizer (class in sumire.vectorizer.swem)