class

Tiktoken::Encoding

Inherits Reference < Object

Class methods

cl100k_base

for_model(model_name : String)

o200k_base

o200k_harmony

p50k_base

p50k_edit

r50k_base

count(text : String, allowed_special : Set(String) = Set(String).new) : Int32

count_ordinary(text : String) : Int32

count_with_special_tokens(text : String) : Int32

decode(tokens) : String

decode_bytes(tokens) : Bytes

encode(text : String, allowed_special : Set(String) = Set(String).new)

encode_ordinary(text : String)

encode_with_special_tokens(text : String)

finalize