module

NLP::Tokenizer

Constants

NUMBER_PATTERN = /\d+(\.\d+)?/
PUNCTUATION_PATTERN = /[.!?,:;]/
WHITESPACE_PATTERN = /\s+/
WORD_PATTERN = /[a-zA-Z]+/

Basic tokenization patterns

Class methods

get_token_stats(tokens : Array(String)) : Hash(String, Int32)

Get token statistics

get_token_type(token : String) : String

Get token type classification

is_number?(token : String) : Bool

Check if a token is a number

is_punctuation?(token : String) : Bool

Check if a token is punctuation

is_word?(token : String) : Bool

Check if a token is a word (contains only letters)

split_sentences(text : String) : Array(String)

Split text into sentences based on punctuation

tokenize(text : String) : Array(String)

Tokenize text into an array of tokens

tokenize_sentences(text : String) : Array(Array(String))

Advanced tokenization that preserves sentence structure

tokenize_with_features(text : String) : Array(NamedTuple(token: String, type: String, position: Int32))

Advanced tokenization with linguistic features