class

Memo::Queries::SQLite

Inherits Memo::Queries < Reference < Object

Instance methods

chunk_content_ids(source_id : Int64) : Array(Int64)

The texts a source's chunks were cut from

Source
chunk_states(source_id : Int64, service_id : Int64) : Array(Tuple(Int64 | Nil, Int64 | Nil, Int64 | Nil, String, Bool))

Per chunk of a source: {content_id, pair_id, parent_id, source_type, whether its hash has an embedding, with a stored vector, for the service}

Source
clear_completed_queue
Source
clear_completed_queue_texts

Drop the text of finished items (from before memo did so itself)

Source
clear_default_service
Source
clear_query_cache(service_id : Int64) : Nil
Source
clear_queue
Source
count_chunks_by_hash(hash : Bytes) : Int64
Source
count_chunks_for_service(service_id : Int64) : Int64
Source
count_embeddings_for_service(service_id : Int64) : Int64
Source
count_files
Source
count_query_cache(service_id : Int64) : Int64
Source
count_service_chunks(service_id : Int64) : Int64
Source
count_service_embeddings(service_id : Int64) : Int64
Source
count_service_sources(service_id : Int64) : Int64
Source
count_services
Source
count_vectors(service_id : Int64) : Int64
Source
count_vocab(service_id : Int64) : Int64
Source
delete_chunks(hash : Bytes, source_id : Int64, source_type : String) : Int32
Source
delete_chunks(hash : Bytes, source_id : Int64) : Int32
Source
delete_chunks_by_hash(hash : Bytes) : Nil
Source
delete_embeddings_by_hash(hash : Bytes) : Nil
Source
delete_embeddings_by_service(service_id : Int64) : Nil
Source
delete_file(source_id : Int64) : Int64
Source
delete_index_data(service_id : Int64) : Nil

Remove a service's vectors, journal entries and journal state

Source
delete_service(id : Int64) : Nil
Source
delete_source_blob(source_type : String, external_id : Bytes) : Int64
Source
delete_source_by_id(internal_id : Int64) : Int64
Source
delete_source_int(source_type : String, external_id : Int64) : Int64
Source
delete_source_text(source_type : String, external_id : String) : Int64
Source
delete_vector(embedding_id : Int64) : Nil
Source
delete_vocab(service_id : Int64) : Nil
Source
embedded_hashes(hashes : Array(Bytes), service_id : Int64) : Set(Bytes)

Which of hashes have an embedding for the service with a stored vector (one without has been lost: embed it again)

Source
embedding_ids_for_hash(hash : Bytes) : Array(Tuple(Int64, Int64))

{embedding id, service id} for every service's embedding of a hash

Source
embedding_ids_without_vectors(service_id : Int64) : Array(Int64)
Source
enqueue(source_id : Int64, text : String, created_at : Int64) : Nil
Source
fetch_search_results(rowids : Array(Int64), service_id : Int64, include_text : Bool) : Array(Tuple(UInt64, Int64, Bytes, String, Int64, Int64 | Nil, String | Nil, Bytes | Nil, Int64 | Nil, Int64 | Nil, String | Nil, Bytes | Nil, Int64 | Nil, Int64 | Nil, String | Nil, Bytes | Nil, Int32 | Nil, Int32, Int32, Int32, String | Nil))
Source
filter_candidates(service_id : Int64, params : Array(DB::Any), where_clauses : Array(String), text_join : String, fts_join : String, candidate_ids : Array(Int64)) : Set(UInt64)

Which of candidate_ids pass the filter clauses (looked up by id, so the cost depends on the number of candidates, not the collection)

Source
find_content_id(hash : Bytes) : Int64 | Nil
Source
find_service_id(name : String) : Int64 | Nil
Source
find_source_blob(source_type : String, external_id : Bytes) : Int64 | Nil
Source
find_source_blob_any_type(external_id : Bytes) : Int64 | Nil
Source
find_source_int(source_type : String, external_id : Int64) : Int64 | Nil
Source
find_source_int_any_type(external_id : Int64) : Int64 | Nil
Source
find_source_text(source_type : String, external_id : String) : Int64 | Nil
Source
find_source_text_any_type(external_id : String) : Int64 | Nil
Source
for_connection(cnn : DB::Connection) : Queries

Same queries, run on one connection (for use inside a transaction).

Source
get_all_texts

Every source's stored text (a text shared by several, once for each)

Source
get_chunk_hashes(source_id : Int64, source_type : String | Nil) : Array(Bytes)
Source
get_chunk_id(source_id : Int64, offset : Int32 | Nil) : Int64
Source
get_chunks_for_reindex(source_type : String, service_id : Int64) : Array(Tuple(Int64, Int64 | Nil, String | Nil, Int64 | Nil, Int64 | Nil))
Source
get_default_service
Source
get_embedding_hashes_for_service(service_id : Int64) : Array(Bytes)
Source
get_embedding_rowid(hash : Bytes, service_id : Int64) : Int64
Source
get_embedding_rowid?(hash : Bytes, service_id : Int64) : Int64 | Nil
Source
get_existing_words(service_id : Int64, words : Array(String)) : Set(String)
Source
get_file_by_hash(hash : Bytes) : Files::FileRecord | Nil
Source
get_file_by_path(path : String) : Files::FileRecord | Nil
Source
get_file_by_source(source_id : Int64) : Files::FileRecord | Nil
Source
get_index_state(service_id : Int64) : Tuple(Int64, Bool) | Nil

{pruned_through, vectors_backfilled}

Source
get_meta(key : String) : String | Nil
Source
get_pending_queue(limit : Int32) : Array(Tuple(Int64, Int64, String))
Source
get_query_cache(query : String, service_id : Int64) : Tuple(Bytes, Int32) | Nil
Source
get_queue_attempts(id : Int64) : Int32
Source
get_queue_item(source_id : Int64) : Tuple(Int64, String) | Nil
Source
get_recent_query_cache(service_id : Int64, limit : Int32) : Array(Tuple(String, Bytes, Int32))
Source
get_service_by_format_model(format : String, model : String) : Tuple(Int64, String, String | Nil, String, Int32, Int32, Float64) | Nil
Source
get_service_by_name(name : String) : Tuple(Int64, String, String | Nil, String, Int32, Int32, Float64) | Nil
Source
get_service_info(id : Int64) : ServiceProvider::Info | Nil
Source
get_service_info_by_name(name : String) : ServiceProvider::Info | Nil
Source
get_service_tokens_per_byte(service_id : Int64) : Float64 | Nil
Source
get_source_external(internal_id : Int64) : Tuple(String, Int64 | Nil, String | Nil, Bytes | Nil) | Nil
Source
get_source_text(source_id : Int64) : String | Nil

A source's latest text (nil if not stored)

Source
get_texts_for_reindex(source_type : String) : Array(Tuple(Int64, Int64 | Nil, Int64 | Nil, String))
Source
get_tokens_per_byte(service_id : Int64) : Float64 | Nil
Source
get_vector(embedding_id : Int64) : Bytes | Nil
Source
get_vocab(service_id : Int64, &block : DB::ResultSet -> ) : Nil
Source
increment_match_count(chunk_ids : Array(Int64)) : Nil
Source
increment_read_count(chunk_ids : Array(Int64)) : Nil
Source
index_changes_since(service_id : Int64, seq : Int64) : Array(Int64)

Distinct embedding ids logged after seq

Source
insert_chunk_ignore(hash : Bytes, source_id : Int64, source_type : String, content_id : Int64 | Nil, pair_id : Int64 | Nil, parent_id : Int64 | Nil, offset : Int32 | Nil, size : Int32, created_at : Int64) : Int64
Source
insert_embedding_ignore(hash : Bytes, service_id : Int64, token_count : Int32, created_at : Int64) : Int64
Source
insert_service(name : String, format : String, base_url : String | Nil, model : String, dimensions : Int32, max_tokens : Int32, created_at : Int64) : Int64
Source
insert_service_full(name : String, format : String, base_url : String | Nil, model : String, dimensions : Int32, max_tokens : Int32, is_default : Int32, created_at : Int64) : Int64

Insert service with is_default and return ID (for ServiceProvider.create)

Source
insert_source(source_type : String, created_at : Int64) : Int64
Source
insert_source_blob(source_type : String, external_id : Bytes, created_at : Int64) : Int64
Source
insert_source_int(source_type : String, external_id : Int64, created_at : Int64) : Int64
Source
insert_source_text(source_type : String, external_id : String, created_at : Int64) : Int64
Source
legacy_texts_after(source_id : Int64, limit : Int32) : Array(Tuple(Int64, String, Bytes | Nil))

memo_texts rows (databases from before 0.13): {source_id, content, content_hash}

Source
list_files(limit : Int32, offset : Int32) : Array(Files::FileRecord)
Source
list_services
Source
list_services_by_format(format : String) : Array(ServiceProvider::Info)
Source
list_sources(source_type : String, limit : Int32, offset : Int32) : Array(Tuple(Int64, Int64 | Nil, String | Nil, Bytes | Nil))
Source
load_embedding_rowids(service_id : Int64, source_type : String, external_ids : Array(Int64)) : Array(Tuple(Int64, Int64))
Source
log_index_change(service_id : Int64, embedding_id : Int64) : Nil
Source
mark_queue_failed(id : Int64, text : String, error_message : String | Nil, attempts : Int32, processed_at : Int64) : Nil
Source
mark_queue_item_success(id : Int64, text : String, processed_at : Int64, attempts : Int32) : Nil
Source
mark_queue_retry(id : Int64, text : String, attempts : Int32, error_message : String | Nil) : Nil
Source
mark_queue_success(id : Int64, text : String, processed_at : Int64) : Nil

The mark_queue_* methods take the item's text as it was read, and do nothing if it has changed since: the source was queued again, and the newer text still needs embedding.

Source
max_index_log_seq(service_id : Int64) : Int64
Source
prune_index_log(service_id : Int64, through : Int64) : Nil
Source
prune_query_cache(service_id : Int64, count : Int64) : Nil
Source
queue_stats
Source
release_content(content_id : Int64) : Nil

Delete a text unless a source or chunk still points at it

Source
search_filtered_rowids(service_id : Int64, params : Array(DB::Any), where_clauses : Array(String), text_join : String, fts_join : String) : Set(UInt64)
Source
service_exists?(id : Int64) : Bool
Source
set_chunks_content(source_id : Int64, content_id : Int64) : Nil
Source
set_default_service(id : Int64) : Nil
Source
set_index_pruned_through(service_id : Int64, seq : Int64) : Nil
Source
set_meta(key : String, value : String) : Nil
Source
set_source_content(source_id : Int64, content_id : Int64 | Nil) : Nil
Source
set_vectors_backfilled(service_id : Int64) : Nil
Source
source_content_id(source_id : Int64) : Int64 | Nil
Source
update_service(id : Int64, updates : Array(String), params : Array(DB::Any)) : Int64
Source
update_tokens_per_byte(tokens_per_byte : Float64, service_id : Int64) : Nil
Source
update_word_frequency(count : Int32, word : String, service_id : Int64) : Nil
Source
upsert_content(hash : Bytes, content : String | Nil, created_at : Int64) : Int64

The id of the text with this hash, storing it if new. content nil stores the hash alone.

Source
upsert_file(source_id : Int64, path : String, content_hash : Bytes, mtime : Int64, size : Int64, created_at : Int64) : Nil
Source
upsert_query_cache(query : String, service_id : Int64, embedding : Bytes, token_count : Int32, created_at : Int64) : Nil
Source
upsert_vector(embedding_id : Int64, service_id : Int64, vector : Bytes) : Nil
Source
upsert_vocab(word : String, service_id : Int64, embedding : Bytes, frequency : Int32, created_at : Int64) : Nil
Source
vectors_after(service_id : Int64, after_id : Int64, limit : Int32) : Array(Tuple(Int64, Bytes))

Up to limit {embedding id, vector} pairs with ids above after_id, in id order

Source