module

NLP::TextProcessor

Class methods

calculate_term_frequency(tokens : Array(String)) : Hash(String, Float64)

Calculate term frequency

extract_keywords(text : String, max_keywords : Int32 = 10) : Array(String)

Extract keywords using simple frequency analysis

extract_ngrams(tokens : Array(String), n : Int32) : Array(Array(String))

Extract n-grams from tokens

get_stop_words

Get common English stop words

get_text_stats(text : String) : Hash(String, Int32 | Float64)

Calculate basic text statistics

normalize_text(text : String) : String

Normalize text for processing

remove_stop_words(tokens : Array(String)) : Array(String)

Remove common stop words

simple_stem(word : String) : String

Simple stemming (remove common suffixes)

stem_tokens(tokens : Array(String)) : Array(String)

Apply stemming to all tokens