sumire package

Subpackages

• sumire.tokenizer package
  • Submodules
  • sumire.tokenizer.auto module
    • AutoJapaneseTokenizer
      • AutoJapaneseTokenizer.from_pretrained()
      • AutoJapaneseTokenizer.save_pretrained()
      • AutoJapaneseTokenizer.tokenize()
    • tokenizer_dict()
  • sumire.tokenizer.base module
    • BaseTokenizer
      • BaseTokenizer.fit()
      • BaseTokenizer.from_pretrained()
      • BaseTokenizer.save_pretrained()
      • BaseTokenizer.tokenize()
      • BaseTokenizer.tokenizer_config_file
  • sumire.tokenizer.jumanpp module
    • JumanppTokenizer
      • JumanppTokenizer.from_pretrained()
      • JumanppTokenizer.save_pretrained()
      • JumanppTokenizer.tokenize()
  • sumire.tokenizer.mecab module
    • MecabTokenizer
      • MecabTokenizer.from_pretrained()
      • MecabTokenizer.save_pretrained()
      • MecabTokenizer.setup_tagger()
      • MecabTokenizer.tokenize()
  • sumire.tokenizer.spacy_ginza module
    • SpacyGinzaTokenizer
      • SpacyGinzaTokenizer.from_pretrained()
      • SpacyGinzaTokenizer.save_pretrained()
      • SpacyGinzaTokenizer.tokenize()
  • sumire.tokenizer.spm module
    • SentencePieceTokenizer
      • SentencePieceTokenizer.decode()
      • SentencePieceTokenizer.encode()
      • SentencePieceTokenizer.fit()
      • SentencePieceTokenizer.from_pretrained()
      • SentencePieceTokenizer.model_cache_dir
      • SentencePieceTokenizer.save_pretrained()
      • SentencePieceTokenizer.tokenize()
      • SentencePieceTokenizer.tokenizer_file_prefix
  • sumire.tokenizer.sudachi module
    • SudachiTokenizer
      • SudachiTokenizer.dict_type
      • SudachiTokenizer.normalize
      • SudachiTokenizer.tokenizer
      • SudachiTokenizer.from_pretrained()
      • SudachiTokenizer.save_pretrained()
      • SudachiTokenizer.tokenize()
  • Module contents
• sumire.utils package
  • Submodules
  • sumire.utils.download_file module
    • download_file()
  • sumire.utils.run_sh module
    • run_sh()
  • sumire.utils.unbz_file module
    • unbz_file()
  • sumire.utils.untar_file module
    • untar_file()
  • sumire.utils.unzip_file module
    • unzip_file()
  • Module contents
• sumire.vectorizer package
  • Subpackages
    • sumire.vectorizer.base package
      • Submodules
      • sumire.vectorizer.base.common module
      • sumire.vectorizer.base.sklearn_vectorizer_base module
      • sumire.vectorizer.base.transformer_vectorizer_base module
      • Module contents
  • Submodules
  • sumire.vectorizer.count module
    • CountVectorizer
      • CountVectorizer.from_pretrained()
  • sumire.vectorizer.swem module
    • ModelCard
      • ModelCard.description
      • ModelCard.model_computed_fields
      • ModelCard.model_config
      • ModelCard.model_fields
      • ModelCard.name
      • ModelCard.tokenizer_name
      • ModelCard.url
    • W2VSWEMVectorizer
      • W2VSWEMVectorizer.w2v_dir
      • W2VSWEMVectorizer.fit()
      • W2VSWEMVectorizer.from_pretrained()
      • W2VSWEMVectorizer.get_token_vectors()
      • W2VSWEMVectorizer.save_pretrained()
      • W2VSWEMVectorizer.setup_w2v_if_not_installed()
      • W2VSWEMVectorizer.transform()
      • W2VSWEMVectorizer.w2v_dir
  • sumire.vectorizer.tfidf module
    • TfidfVectorizer
      • TfidfVectorizer.from_pretrained()
  • sumire.vectorizer.transformer_emb module
    • ModelCard
      • ModelCard.description
      • ModelCard.model_computed_fields
      • ModelCard.model_config
      • ModelCard.model_fields
      • ModelCard.model_name
    • TransformerEmbeddingVectorizer
      • TransformerEmbeddingVectorizer.from_pretrained()
      • TransformerEmbeddingVectorizer.get_token_vectors()
      • TransformerEmbeddingVectorizer.transform()
  • Module contents

Module contents