class

CrystalMoji::Compile::TokenInfoDictionaryCompilerBase(T)

Inherits CrystalMoji::Compile::Compiler < Reference < Object

Constructors

new(encoding : String)

Instance methods

add_mapping(source_id : Int32, word_id : Int32)

analyze_token_info(input : IO)

buffer_entries

buffer_entries=(buffer_entries : Array(CrystalMoji::Buffer::BufferEntry))

combined_sequential_file_input_stream(dir : String) : IO

compile

dictionary_entries

dictionary_entries=(dictionary_entries : Array(CrystalMoji::Dict::GenericDictionaryEntry) | Nil)

get_csv_files(dir : String) : Array(String)

make_generic_dictionary_entry(entry : T) : CrystalMoji::Dict::GenericDictionaryEntry

other_info

other_info=(other_info : CrystalMoji::Buffer::FeatureInfoMap)

parse(line : String) : T

pos_info

pos_info=(pos_info : CrystalMoji::Buffer::FeatureInfoMap)

read_token_info(input : IO)

surfaces

surfaces=(surfaces : Array(String))

word_ids_compiler

word_ids_compiler=(word_ids_compiler : CrystalMoji::Compile::WordIdMapCompiler)

write(directory_name : String)