MicroGPT

SCHEMA = "{\n \"type\": \"object\",\n \"positional\": [\"file\"],\n \"properties\": {\n \"file\": {\n \"type\": \"string\",\n \"description\": \"Input text file for training/vocab\",\n \"short\": \"f\"\n },\n \"steps\": {\n \"type\": \"integer\",\n \"description\": \"Training steps (0 = inference only)\",\n \"default\": 1000,\n \"short\": \"s\"\n },\n \"heads\": {\n \"type\": \"string\",\n \"description\": \"Head type\",\n \"enum\": [\"uniform\", \"exponential\", \"prime\", \"pyramid\", \"ones\"],\n \"default\": \"uniform\"\n },\n \"backend\": {\n \"type\": \"string\",\n \"description\": \"Compute backend\",\n \"enum\": [\"crystal\", \"openblas\", \"cublas\"],\n \"default\": \"crystal\",\n \"short\": \"b\"\n },\n \"seq-len\": {\n \"type\": \"integer\",\n \"description\": \"Sequence length (context window)\",\n \"default\": 128\n },\n \"lookahead\": {\n \"type\": \"integer\",\n \"description\": \"Lookahead heads (0=none, -1=future model)\",\n \"default\": 0\n },\n \"d-model\": {\n \"type\": \"integer\",\n \"description\": \"Model embedding dimension\",\n \"default\": 64,\n \"short\": \"d\"\n },\n \"n-layers\": {\n \"type\": \"integer\",\n \"description\": \"Number of transformer layers\",\n \"default\": 2,\n \"short\": \"n\"\n },\n \"eval\": {\n \"type\": \"string\",\n \"description\": \"Eval prompts file (one per line)\",\n \"short\": \"e\"\n },\n \"model\": {\n \"type\": \"string\",\n \"description\": \"Model checkpoint path (default: derived from input file)\",\n \"short\": \"m\"\n },\n \"no-save\": {\n \"type\": \"boolean\",\n \"description\": \"Skip saving model after training\",\n \"default\": false\n },\n \"lr\": {\n \"type\": \"number\",\n \"description\": \"Learning rate\",\n \"default\": 0.0003\n },\n \"compare\": {\n \"type\": \"string\",\n \"description\": \"Train multiple models: d64n2,d128n2,d128n3\",\n \"short\": \"c\"\n },\n \"cooperative\": {\n \"type\": \"string\",\n \"description\": \"Cooperative ensemble: expert specs (e.g. d16n1,d32n2,d32n2)\",\n \"short\": \"C\"\n },\n \"stream-dim\": {\n \"type\": \"integer\",\n \"description\": \"Shared stream dimension for cooperative mode\",\n \"default\": 64\n },\n \"no-counter\": {\n \"type\": \"boolean\",\n \"description\": \"Disable counter expert (expert 0 uses tokens normally)\",\n \"default\": false\n },\n \"active-dims\": {\n \"type\": \"integer\",\n \"description\": \"Active stream dimensions (0=full, masks unused dims)\",\n \"default\": 0\n },\n \"config\": {\n \"type\": \"string\",\n \"description\": \"YAML config file with model definitions\"\n },\n \"run\": {\n \"type\": \"string\",\n \"description\": \"Model ID to run from config file (or comma-separated IDs)\"\n },\n \"bigram-off-at\": {\n \"type\": \"integer\",\n \"description\": \"Step at which to detach bigram expert (0=never)\",\n \"default\": 0\n },\n \"router\": {\n \"type\": \"string\",\n \"description\": \"Router type: global, context, gated\",\n \"enum\": [\"global\", \"context\", \"gated\"],\n \"default\": \"global\"\n }\n },\n \"required\": [\"file\"]\n }"

VERSION = "0.1.0"

backend=(backend : Backend)

build_inverted_causal_mask(seq_len : Int32) : Mat

build_mask(seq_len : Int32, k : Int32) : Mat

load_yaml_config(config_path : String, run_id : String) : Hash(String, YAML::Any)

log_result(id : String, params : Int64, steps : Int32, final_loss : Float64, extra : String = "", data_file : String = "")

parse_specs(spec_str : String) : Array(ModelSpec)