class

MicroGPT::AttentionHead

Constructors

new(head_dim : Int32)

attn_weights

attn_weights=(attn_weights : Mat | Nil)

backward(grad : Mat) : Tuple(Mat, Mat, Mat)

Backward: returns {dq, dk, dv} for the fused projection backward

forward(q : Mat, k : Mat, v : Mat, mask : Mat | Nil = nil) : Mat

Forward: compute attention from pre-projected Q, K, V If mask is provided, it's added to scores (pre-computed -inf pattern)

head_dim

k=(k : Mat | Nil)

q=(q : Mat | Nil)

v=(v : Mat | Nil)