class

ML::LLM::Generator

Inherits Reference / Object

High-level text generation interface

Constants

GPT_OSS_STOPS = ["<|end|>", "<|im_end|>"]

Default stop strings for gpt-oss models

Constructors

new(model : Model, n_ctx : Int32 = 0, n_batch : Int32 = 512, n_ubatch : Int32 = 0, n_threads : Int32 = 0, flash_attn : Bool = true, prompt_mode : PromptMode = PromptMode::Raw, sampler_seed : UInt32 | Nil = nil)
Source

Instance methods

ask(question : String, system : String | Nil = nil, max_tokens : Int32 = 256, temperature : Float32 = 0.7_f32) : String

Ask a question and get response (formats prompt automatically)

Source
ask_stream(question : String, system : String | Nil = nil, max_tokens : Int32 = 256, temperature : Float32 = 0.7_f32, &block : String -> ) : Int32

Stream response to a question (filtered - only final answer for gpt-oss)

Source
ask_stream_raw(question : String, system : String | Nil = nil, max_tokens : Int32 = 256, temperature : Float32 = 0.7_f32, &block : String -> ) : Int32

Stream raw response (no filtering)

Source
calculate_tps(tokens : Int32, elapsed : Time::Span) : Float64

Calculate tokens per second from wall clock time

Source
context
Source
finalize
Source
format_messages(messages : Array(ChatMessage)) : String

Format with explicit messages

Source
format_prompt(user_message : String, system_message : String | Nil = nil) : String

Format prompt based on current mode

Source
free
Source
generate(prompt : String, max_tokens : Int32 = 256, stop_on_eos : Bool = true, stop_strings : Array(String) | Nil = nil, temperature : Float32 = 0.8_f32, top_k : Int32 = 40, top_p : Float32 = 0.95_f32) : String

Generate text completion

Source
mean_logprob
Source
min_logprob
Source
model
Source
prompt_mode
Source
prompt_mode=(prompt_mode : PromptMode)
Source
stream(prompt : String, max_tokens : Int32 = 256, stop_on_eos : Bool = true, stop_strings : Array(String) | Nil = nil, &block : String -> ) : Int32

Stream generation with callback

Source
token_logprobs

Per-token log-probabilities from last generation (log-softmax of sampled token)

Source
tokens_per_second

Get tokens per second from last generation Note: llama.cpp perf stats may not populate correctly, use wall clock time instead

Source