module

NLP::Tokenizer

Constants

NUMBER_PATTERN = /\d+(\.\d+)?/
PUNCTUATION_PATTERN = /[.!?,:;]/
WHITESPACE_PATTERN = /\s+/
WORD_PATTERN = /[a-zA-Z]+/

Basic tokenization patterns

Class methods

get_token_stats(tokens : Array(String)) : Hash(String, Int32)

Get token statistics

Source
get_token_type(token : String) : String

Get token type classification

Source
is_number?(token : String) : Bool

Check if a token is a number

Source
is_punctuation?(token : String) : Bool

Check if a token is punctuation

Source
is_word?(token : String) : Bool

Check if a token is a word (contains only letters)

Source
split_sentences(text : String) : Array(String)

Split text into sentences based on punctuation

Source
tokenize(text : String) : Array(String)

Tokenize text into an array of tokens

Source
tokenize_sentences(text : String) : Array(Array(String))

Advanced tokenization that preserves sentence structure

Source
tokenize_with_features(text : String) : Array(NamedTuple(token: String, type: String, position: Int32))

Advanced tokenization with linguistic features

Source