BlingFire

SOURCE_URL = "https://github.com/kojix2/blingfire-crystal"

VERSION = {{ (`shards version /tmp/tmp.AiMABp/src/src/blingfire`).chomp.stringify }}

free_model(model)

ids_to_text(model, ids, skip_special_tokens = true, output_buffer_size = nil)

load_model(model_path)

normalize_spaces(text)

text_to_ids(model, text, max_len = nil, unk_id = 0)

text_to_ids_with_offsets(model, text, max_len = nil, unk_id = 0)

text_to_sentences(text)

text_to_sentences_with_model(model, text)

text_to_sentences_with_offsets(text)

text_to_sentences_with_offsets_with_model(model, text)

text_to_words(text)

text_to_words_with_model(model, text)

text_to_words_with_offsets(text)

text_to_words_with_offsets_with_model(model, text)