class

MicroGPT::MultiHeadAttention

Inherits MicroGPT::MathUtils < MicroGPT::Layer < Reference < Object

Constructors

new(d_model : Int32, n_heads : Int32, max_seq_len : Int32 = 128)

Uniform heads

Source
new(d_model : Int32, head_dims : Array(Int32), max_seq_len : Int32 = 128)

Heterogeneous heads — dimensions must sum to d_model

Source

Instance methods

backward(grad : Mat) : Mat
Source
backward_accumulate(grad : Mat) : Mat
Source
forward(x : Mat, mask : Mat | Nil = nil) : Mat
Source
head_dims
Source
heads
Source
ropes
Source
update(lr : Float64)
Source