class

CrystalMoji::Ipadic::Tokenizer::Builder

Inherits CrystalMoji::TokenizerBase::Builder < Reference < Object

Constructors

new

Creates a default builder

Source

Class methods

default_kanji_length_threshold
Source
default_kanji_length_threshold=(default_kanji_length_threshold : Int32)
Source
default_kanji_penalty
Source
default_kanji_penalty=(default_kanji_penalty : Int32)
Source
default_other_length_threshold
Source
default_other_length_threshold=(default_other_length_threshold : Int32)
Source
default_other_penalty
Source
default_other_penalty=(default_other_penalty : Int32)
Source

Instance methods

build

Creates the custom tokenizer instance

Source
kanji_penalty(length_threshold : Int32, penalty : Int32) : Builder

Sets a custom kanji penalty

Source
kanji_penalty
Source
kanji_penalty=(kanji_penalty : Int32)
Source
kanji_penalty_length_threshold
Source
kanji_penalty_length_threshold=(kanji_penalty_length_threshold : Int32)
Source
mode(mode : TokenizerBase::Mode) : Builder

Sets the tokenization mode

Source
nakaguro_split
Source
nakaguro_split=(nakaguro_split : Bool)
Source
other_penalty(length_threshold : Int32, penalty : Int32) : Builder

Sets a custom non-kanji penalty

Source
other_penalty
Source
other_penalty=(other_penalty : Int32)
Source
other_penalty_length_threshold
Source
other_penalty_length_threshold=(other_penalty_length_threshold : Int32)
Source
split_on_nakaguro(split : Bool) : Builder

Predicate that splits unknown words on the middle dot character (U+30FB KATAKANA MIDDLE DOT)

Source

Nested types