class

Memo::Queries

Abstract query interface for database-backend-specific SQL.

Each backend (SQLite, PostgreSQL) implements all query methods with SQL native to that backend. This avoids placeholder translation and keeps SQL explicit per backend.

Constructors

for(db : DB::Database, connection_string : String) : Queries

Source

new(db : DBHandle)

Source

Class methods

register_pg

Source

Instance methods

chunk_content_ids(source_id : Int64) : Array(Int64)

The texts a source's chunks were cut from

Source

chunk_states(source_id : Int64, service_id : Int64) : Array(Tuple(Int64 | Nil, Int64 | Nil, Int64 | Nil, String, Bool))

Per chunk of a source: {content_id, pair_id, parent_id, source_type, whether its hash has an embedding, with a stored vector, for the service}

Source

clear_completed_queue

Source

clear_completed_queue_texts

Drop the text of finished items (from before memo did so itself)

Source

clear_default_service

Source

clear_query_cache(service_id : Int64) : Nil

Source

clear_queue

Source

count_chunks_by_hash(hash : Bytes) : Int64

Source

count_chunks_for_service(service_id : Int64) : Int64

Source

count_embeddings_for_service(service_id : Int64) : Int64

Source

count_files

Source

count_query_cache(service_id : Int64) : Int64

Source

count_service_chunks(service_id : Int64) : Int64

Source

count_service_embeddings(service_id : Int64) : Int64

Source

count_service_sources(service_id : Int64) : Int64

Source

count_services

Source

count_vectors(service_id : Int64) : Int64

Source

count_vocab(service_id : Int64) : Int64

Source

delete_chunks(hash : Bytes, source_id : Int64, source_type : String) : Int32

Source

delete_chunks(hash : Bytes, source_id : Int64) : Int32

Source

delete_chunks_by_hash(hash : Bytes) : Nil

Source

delete_embeddings_by_hash(hash : Bytes) : Nil

Source

delete_embeddings_by_service(service_id : Int64) : Nil

Source

delete_file(source_id : Int64) : Int64

Source

delete_index_data(service_id : Int64) : Nil

Remove a service's vectors, journal entries and journal state

Source

delete_service(id : Int64) : Nil

Source

delete_source_blob(source_type : String, external_id : Bytes) : Int64

Source

delete_source_by_id(internal_id : Int64) : Int64

Source

delete_source_int(source_type : String, external_id : Int64) : Int64

Source

delete_source_text(source_type : String, external_id : String) : Int64

Source

delete_vector(embedding_id : Int64) : Nil

Source

delete_vocab(service_id : Int64) : Nil

Source

embedded_hashes(hashes : Array(Bytes), service_id : Int64) : Set(Bytes)

Which of hashes have an embedding for the service with a stored vector (one without has been lost: embed it again)

Source

embedding_ids_for_hash(hash : Bytes) : Array(Tuple(Int64, Int64))

{embedding id, service id} for every service's embedding of a hash

Source

embedding_ids_without_vectors(service_id : Int64) : Array(Int64)

Source

enqueue(source_id : Int64, text : String, created_at : Int64) : Nil

Source

Source

filter_candidates(service_id : Int64, params : Array(DB::Any), where_clauses : Array(String), text_join : String, fts_join : String, candidate_ids : Array(Int64)) : Set(UInt64)

Which of candidate_ids pass the filter clauses (looked up by id, so the cost depends on the number of candidates, not the collection)

Source

find_content_id(hash : Bytes) : Int64 | Nil

Source

find_service_id(name : String) : Int64 | Nil

Source

find_source_blob(source_type : String, external_id : Bytes) : Int64 | Nil

Source

find_source_blob_any_type(external_id : Bytes) : Int64 | Nil

Source

find_source_int(source_type : String, external_id : Int64) : Int64 | Nil

Source

find_source_int_any_type(external_id : Int64) : Int64 | Nil

Source

find_source_text(source_type : String, external_id : String) : Int64 | Nil

Source

find_source_text_any_type(external_id : String) : Int64 | Nil

Source

for_connection(cnn : DB::Connection) : Queries

Same queries, run on one connection (for use inside a transaction).

Source

get_all_texts

Every source's stored text (a text shared by several, once for each)

Source

get_chunk_hashes(source_id : Int64, source_type : String | Nil) : Array(Bytes)

Source

get_chunk_id(source_id : Int64, offset : Int32 | Nil) : Int64

Source

get_chunks_for_reindex(source_type : String, service_id : Int64) : Array(Tuple(Int64, Int64 | Nil, String | Nil, Int64 | Nil, Int64 | Nil))

Source

get_default_service

Source

get_embedding_hashes_for_service(service_id : Int64) : Array(Bytes)

Source

get_embedding_rowid(hash : Bytes, service_id : Int64) : Int64

Source

get_embedding_rowid?(hash : Bytes, service_id : Int64) : Int64 | Nil

Source

get_existing_words(service_id : Int64, words : Array(String)) : Set(String)

Source

get_file_by_hash(hash : Bytes) : Files::FileRecord | Nil

Source

get_file_by_path(path : String) : Files::FileRecord | Nil

Source

get_file_by_source(source_id : Int64) : Files::FileRecord | Nil

Source

get_index_state(service_id : Int64) : Tuple(Int64, Bool) | Nil

{pruned_through, vectors_backfilled}

Source

get_meta(key : String) : String | Nil

Source

get_pending_queue(limit : Int32) : Array(Tuple(Int64, Int64, String))

Source

get_query_cache(query : String, service_id : Int64) : Tuple(Bytes, Int32) | Nil

Source

get_queue_attempts(id : Int64) : Int32

Source

get_queue_item(source_id : Int64) : Tuple(Int64, String) | Nil

Source

get_recent_query_cache(service_id : Int64, limit : Int32) : Array(Tuple(String, Bytes, Int32))

Source

get_service_by_format_model(format : String, model : String) : Tuple(Int64, String, String | Nil, String, Int32, Int32, Float64) | Nil

Source

get_service_by_name(name : String) : Tuple(Int64, String, String | Nil, String, Int32, Int32, Float64) | Nil

Source

get_service_info(id : Int64) : ServiceProvider::Info | Nil

Source

get_service_info_by_name(name : String) : ServiceProvider::Info | Nil

Source

get_service_tokens_per_byte(service_id : Int64) : Float64 | Nil

Source

get_source_external(internal_id : Int64) : Tuple(String, Int64 | Nil, String | Nil, Bytes | Nil) | Nil

Source

get_source_text(source_id : Int64) : String | Nil

A source's latest text (nil if not stored)

Source

get_texts_for_reindex(source_type : String) : Array(Tuple(Int64, Int64 | Nil, Int64 | Nil, String))

Source

get_tokens_per_byte(service_id : Int64) : Float64 | Nil

Source

get_vector(embedding_id : Int64) : Bytes | Nil

Source

get_vocab(service_id : Int64, &block : DB::ResultSet -> ) : Nil

Source

increment_match_count(chunk_ids : Array(Int64)) : Nil

Source

increment_read_count(chunk_ids : Array(Int64)) : Nil

Source

index_changes_since(service_id : Int64, seq : Int64) : Array(Int64)

Distinct embedding ids logged after seq

Source

insert_chunk_ignore(hash : Bytes, source_id : Int64, source_type : String, content_id : Int64 | Nil, pair_id : Int64 | Nil, parent_id : Int64 | Nil, offset : Int32 | Nil, size : Int32, created_at : Int64) : Int64

Source

insert_embedding_ignore(hash : Bytes, service_id : Int64, token_count : Int32, created_at : Int64) : Int64

Source

insert_service(name : String, format : String, base_url : String | Nil, model : String, dimensions : Int32, max_tokens : Int32, created_at : Int64) : Int64

Source

insert_service_full(name : String, format : String, base_url : String | Nil, model : String, dimensions : Int32, max_tokens : Int32, is_default : Int32, created_at : Int64) : Int64

Insert service with is_default and return ID (for ServiceProvider.create)

Source

insert_source(source_type : String, created_at : Int64) : Int64

Source

insert_source_blob(source_type : String, external_id : Bytes, created_at : Int64) : Int64

Source

insert_source_int(source_type : String, external_id : Int64, created_at : Int64) : Int64

Source

insert_source_text(source_type : String, external_id : String, created_at : Int64) : Int64

Source

legacy_texts_after(source_id : Int64, limit : Int32) : Array(Tuple(Int64, String, Bytes | Nil))

memo_texts rows (databases from before 0.13): {source_id, content, content_hash}

Source

list_files(limit : Int32, offset : Int32) : Array(Files::FileRecord)

Source

list_services

Source

list_services_by_format(format : String) : Array(ServiceProvider::Info)

Source

list_sources(source_type : String, limit : Int32, offset : Int32) : Array(Tuple(Int64, Int64 | Nil, String | Nil, Bytes | Nil))

Source

load_embedding_rowids(service_id : Int64, source_type : String, external_ids : Array(Int64)) : Array(Tuple(Int64, Int64))

Source

log_index_change(service_id : Int64, embedding_id : Int64) : Nil

Source

mark_queue_failed(id : Int64, text : String, error_message : String | Nil, attempts : Int32, processed_at : Int64) : Nil

Source

mark_queue_item_success(id : Int64, text : String, processed_at : Int64, attempts : Int32) : Nil

Source

mark_queue_retry(id : Int64, text : String, attempts : Int32, error_message : String | Nil) : Nil

Source

mark_queue_success(id : Int64, text : String, processed_at : Int64) : Nil

The mark_queue_* methods take the item's text as it was read, and do nothing if it has changed since: the source was queued again, and the newer text still needs embedding.

Source

max_index_log_seq(service_id : Int64) : Int64

Source

prune_index_log(service_id : Int64, through : Int64) : Nil

Source

prune_query_cache(service_id : Int64, count : Int64) : Nil

Source

queue_stats

Source

release_content(content_id : Int64) : Nil

Delete a text unless a source or chunk still points at it

Source

search_filtered_rowids(service_id : Int64, params : Array(DB::Any), where_clauses : Array(String), text_join : String, fts_join : String) : Set(UInt64)

Source

service_exists?(id : Int64) : Bool

Source

set_chunks_content(source_id : Int64, content_id : Int64) : Nil

Source

set_default_service(id : Int64) : Nil

Source

set_index_pruned_through(service_id : Int64, seq : Int64) : Nil

Source

set_meta(key : String, value : String) : Nil

Source

set_source_content(source_id : Int64, content_id : Int64 | Nil) : Nil

Source

set_vectors_backfilled(service_id : Int64) : Nil

Source

source_content_id(source_id : Int64) : Int64 | Nil

Source

update_service(id : Int64, updates : Array(String), params : Array(DB::Any)) : Int64

Source

update_tokens_per_byte(tokens_per_byte : Float64, service_id : Int64) : Nil

Source

update_word_frequency(count : Int32, word : String, service_id : Int64) : Nil

Source

upsert_content(hash : Bytes, content : String | Nil, created_at : Int64) : Int64

The id of the text with this hash, storing it if new. content nil stores the hash alone.

Source

upsert_file(source_id : Int64, path : String, content_hash : Bytes, mtime : Int64, size : Int64, created_at : Int64) : Nil

Source

upsert_query_cache(query : String, service_id : Int64, embedding : Bytes, token_count : Int32, created_at : Int64) : Nil

Source

upsert_vector(embedding_id : Int64, service_id : Int64, vector : Bytes) : Nil

Source

upsert_vocab(word : String, service_id : Int64, embedding : Bytes, frequency : Int32, created_at : Int64) : Nil

Source

vectors_after(service_id : Int64, after_id : Int64, limit : Int32) : Array(Tuple(Int64, Bytes))

Up to limit {embedding id, vector} pairs with ids above after_id, in id order

Source

Constructors

Class methods

Instance methods

Nested types