class

CrystalMoji::TokenizerBase(T)

Inherits Reference < Object

Instance methods

debug_lattice(output_stream : IO, text : String)
Source
debug_tokenize(output_stream : IO, text : String)
Source
multi_tokenize(text : String, max_count : Int32, cost_slack : Int32) : Array(Array(T))
Source
multi_tokenize_by_slack(text : String, cost_slack : Int32) : Array(Array(T))
Source
multi_tokenize_n_best(text : String, n : Int32) : Array(Array(T))
Source
tokenize(text : String) : Array(TokenBase)
Source

Nested types