mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-08-15 15:49:20 +00:00
c8a012d4d2
Some checks failed
CodeQL / Analyze (actions) (push) Has been cancelled
CodeQL / Analyze (javascript-typescript) (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
ci / docker publish / build (amd64) (push) Has been cancelled
ci / docker publish / build (arm64) (push) Has been cancelled
ci / docker publish / merge manifest + tag (push) Has been cancelled
* fix(memory): don't let an unreadable store get overwritten with an empty one
load_all() answered a failed read the same way it answered an empty store:
with []. Every mutation path is a read-modify-write (load the whole file,
change it, save it back), so a failed read became
load_all() -> [] -> [].append(new) -> save([new])
and save() is atomic, so the replacement stuck.
The case that actually destroys data is a store that is READABLE but not
parseable - a truncated file, or one holding {} instead of []. Nothing
obstructs the write, so adding a memory returns HTTP 200 and every memory
already stored is gone. Verified end-to-end against a running instance: on the
current code a truncated memory.json plus one add leaves the file holding only
the new entry. Truncation is reachable - core/database.py rewrites memory.json
during migration with a plain open(.., "w") + json.dump, which is not atomic.
A live exclusive lock is not the dangerous case: it blocks the read and the
os.replace alike, so the save fails too and the store survives. That path
currently 500s and loses nothing.
_read_entries() now returns [] only when the file genuinely does not exist and
raises MemoryStoreUnreadable for every other failure, including a store that
parses but is not a JSON array. load_all() keeps the old lenient behaviour so
display, search and context injection still degrade quietly instead of
breaking chat. The read-modify-write callers switch to load_all_for_update(),
which propagates the error: the memory routes turn it into a 503 and change
nothing, backup import refuses rather than saving only the incoming rows, and
auto-extraction and the audit merge skip the write. The audit merge mattered
most - it rebuilds the whole file from one owner's slice plus everyone else's
rows, so an empty read there dropped every other tenant's memories.
The corrupt-JSON path still gets its one shot at the legacy memory.txt
migration before raising, so that recovery is unchanged.
The two updated fakes gained load_all_for_update because the real class has it;
MagicMock would otherwise hand the import path a Mock instead of the seeded list.
Fixes #5673
* fix(memory): fail closed on the remaining read-modify-write add paths
The strict loader landed with the routes, the backup import and the extractor
converted, but three read-modify-write sinks still called load_all(), which
degrades an unreadable store to []. Two of them are the paths users actually
reach, so the data loss in #5673 stayed reproducible:
- src/ai_interaction.py do_manage_memory, action "add" — reached from ordinary
chat via src/tool_execution.py:793 -> dispatch_ai_tool. "Remember that I
prefer X" against an unreadable store wrote a one-entry file over it and
reported success.
- mcp_servers/memory_server.py, action "add" — the same shape through
_scope_entries(), registered as a built-in in src/builtin_mcp.py.
- src/memory_provider.py NativeMemoryProvider.remember and .delete — wired
into app state in src/app_initializer.py but not consumed outside tests yet,
converted here so the pattern is uniform before it goes live.
The MCP server takes _scope_entries(for_update=True) so list keeps the lenient
read. The edit and delete branches on both tool paths were already fail-closed
by accident — an empty view matches nothing and returns before the save — so
they are left alone.
The three new tests drive the real entry points rather than replaying the
shape, and use a truncated store, which is the case that reads back fine so
nothing stops the save. Each asserts memory.json is byte-identical afterwards;
all three fail on the previous commit with the store overwritten.
457 lines
19 KiB
Python
457 lines
19 KiB
Python
|
|
import json
|
|
import logging
|
|
import os
|
|
import time
|
|
import uuid
|
|
import re
|
|
from typing import List, Dict, Tuple
|
|
from datetime import datetime
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class MemoryStoreUnreadable(RuntimeError):
|
|
"""memory.json exists on disk but could not be read or parsed.
|
|
|
|
"The contents are unknown" is categorically different from "there are no
|
|
memories". A read-modify-write caller that conflates the two appends to an
|
|
empty view and then persists it, destroying the whole store — the writes
|
|
are atomic, so the loss is durable. Raised by
|
|
:meth:`MemoryManager.load_all_for_update` so those callers fail closed.
|
|
"""
|
|
|
|
|
|
def tokenize(text: str) -> List[str]:
|
|
"""Simple tokenizer that splits on whitespace and removes punctuation."""
|
|
return [word.strip('.,!?";') for word in text.split()]
|
|
|
|
def get_text_similarity(text1: str, text2: str) -> float:
|
|
"""Calculate Jaccard similarity between two texts."""
|
|
if not text1 or not text2:
|
|
return 0.0
|
|
|
|
tokens1 = set(tokenize(text1.lower()))
|
|
tokens2 = set(tokenize(text2.lower()))
|
|
|
|
if not tokens1 and not tokens2:
|
|
return 1.0
|
|
if not tokens1 or not tokens2:
|
|
return 0.0
|
|
|
|
intersection = tokens1.intersection(tokens2)
|
|
union = tokens1.union(tokens2)
|
|
|
|
return len(intersection) / len(union)
|
|
|
|
class MemoryManager:
|
|
def __init__(self, data_dir: str):
|
|
self.memory_file = os.path.join(data_dir, "memory.json")
|
|
self.ensure_file_exists()
|
|
|
|
def extract_memory_from_chat(self, chat_history: List[Dict], session_id: str = None) -> List[Dict]:
|
|
"""
|
|
Extract memory entries from chat history as a fallback when LLM fails.
|
|
|
|
Args:
|
|
chat_history: List of chat messages with 'role' and 'content' keys
|
|
session_id: Optional session ID to associate with extracted memories
|
|
|
|
Returns:
|
|
List of memory entries with text, timestamp, and optional session_id
|
|
"""
|
|
memories = []
|
|
|
|
for msg in chat_history:
|
|
if not isinstance(msg, dict):
|
|
continue
|
|
if msg.get("role") == "assistant":
|
|
content = str(msg.get("content", ""))
|
|
lines = content.split('\n')
|
|
|
|
for line in lines:
|
|
line = line.strip()
|
|
# Look for bullet points or numbered lists that might contain memories
|
|
if re.match(r'^[-*•]|\d+\.', line):
|
|
# Extract the text after the bullet/number. Group both
|
|
# markers so the capture applies to either — the previous
|
|
# `^[-*•]|\d+\.\s*(.*)` put the group on the numbered branch
|
|
# only, so a bullet line matched with group(1)=None and
|
|
# crashed on .strip().
|
|
text_match = re.match(r'^(?:[-*•]|\d+\.)\s*(.*)', line)
|
|
if text_match:
|
|
text = text_match.group(1).strip()
|
|
if text:
|
|
memories.append({
|
|
"text": text,
|
|
"timestamp": int(datetime.now().timestamp()),
|
|
"session_id": session_id
|
|
})
|
|
# If we see a heading that suggests memories
|
|
elif re.search(r'memory|fact|note|remember', line, re.I):
|
|
pass
|
|
# If we see a clear separator or end
|
|
elif re.match(r'^={3,}|-{3,}|_{3,}', line):
|
|
pass
|
|
|
|
return memories
|
|
|
|
def process_inline_memory_command(self, message: str) -> Tuple[bool, str]:
|
|
"""
|
|
Check if a message is an inline memory command (e.g. "remember: X").
|
|
|
|
Args:
|
|
message: The user message to check
|
|
|
|
Returns:
|
|
Tuple of (is_command, extracted_text) where is_command is True if
|
|
the message matches the memory command pattern
|
|
"""
|
|
# Pattern for memory commands: "remember: X", "memorize: X", "save: X", etc.
|
|
pattern = r'^(?:remember|memorize|save|note|store)[:\-]?\s+(.+)$'
|
|
match = re.match(pattern, message.strip(), re.IGNORECASE)
|
|
|
|
if match:
|
|
return True, match.group(1).strip()
|
|
else:
|
|
return False, ""
|
|
|
|
def ensure_file_exists(self):
|
|
"""Create memory file if it doesn't exist."""
|
|
if not os.path.exists(self.memory_file):
|
|
with open(self.memory_file, 'w', encoding='utf-8') as f:
|
|
json.dump([], f, ensure_ascii=False, indent=2)
|
|
|
|
def _read_entries(self) -> List[Dict]:
|
|
"""Parse the store, or raise :class:`MemoryStoreUnreadable`.
|
|
|
|
Returns ``[]`` only when the file genuinely does not exist. Every other
|
|
failure mode raises, so callers can tell "no memories" apart from
|
|
"couldn't read the memories".
|
|
"""
|
|
if not os.path.exists(self.memory_file):
|
|
return []
|
|
|
|
try:
|
|
with open(self.memory_file, "r", encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
except OSError as e:
|
|
# PermissionError is an OSError (a scanner holding the file, a
|
|
# permissions problem, bad media).
|
|
raise MemoryStoreUnreadable(
|
|
f"cannot read {self.memory_file}: {e}"
|
|
) from e
|
|
except json.JSONDecodeError as e:
|
|
# This is the branch that actually destroyed stores: the file reads
|
|
# back fine, so nothing stops the save that follows. A truncated
|
|
# memory.json is reachable because core/database.py rewrites it with
|
|
# a plain open(..,"w") + json.dump during migration.
|
|
#
|
|
# Preserved behaviour: a corrupt store still gets one shot at the
|
|
# pre-JSON memory.txt migration. Only raise when that finds nothing,
|
|
# so we never report "empty" for a store we simply failed to parse.
|
|
legacy = self._migrate_from_legacy()
|
|
if legacy:
|
|
return legacy
|
|
raise MemoryStoreUnreadable(
|
|
f"{self.memory_file} is not valid JSON: {e}"
|
|
) from e
|
|
|
|
if not isinstance(data, list):
|
|
raise MemoryStoreUnreadable(
|
|
f"{self.memory_file} is not a JSON array (got {type(data).__name__})"
|
|
)
|
|
return self._validate_entries(data)
|
|
|
|
def load_all(self) -> List[Dict]:
|
|
"""Load all memory entries from JSON file (unfiltered).
|
|
|
|
Lenient by design: this feeds display, search, and context-injection
|
|
paths, so an unreadable store degrades to an empty list rather than
|
|
breaking chat. Never build a value from this that you intend to save
|
|
back — use :meth:`load_all_for_update` for that.
|
|
"""
|
|
try:
|
|
return self._read_entries()
|
|
except MemoryStoreUnreadable as e:
|
|
logger.error("Error loading memory.json: %s", e)
|
|
return []
|
|
|
|
def load_all_for_update(self) -> List[Dict]:
|
|
"""Load for a read-modify-write cycle.
|
|
|
|
Propagates :class:`MemoryStoreUnreadable` instead of degrading to ``[]``
|
|
so a caller can never append to an empty view and persist it over a
|
|
store that was only temporarily unreadable (issue #5673).
|
|
"""
|
|
return self._read_entries()
|
|
|
|
def load(self, owner: str = None) -> List[Dict]:
|
|
"""Load memory entries, optionally filtered by owner."""
|
|
entries = self.load_all()
|
|
if owner is None:
|
|
return entries
|
|
return [e for e in entries if e.get("owner") == owner]
|
|
|
|
def claim_ownerless(self, owner: str):
|
|
"""Assign all ownerless memory entries to the given owner."""
|
|
try:
|
|
entries = self.load_all_for_update()
|
|
except MemoryStoreUnreadable as e:
|
|
# Skip the sweep rather than rewrite the store from an unknown view.
|
|
logger.error("Skipping ownerless claim, memory store unreadable: %s", e)
|
|
return
|
|
changed = False
|
|
claimed = 0
|
|
for entry in entries:
|
|
if not entry.get("owner"):
|
|
entry["owner"] = owner
|
|
changed = True
|
|
claimed += 1
|
|
if changed:
|
|
self.save(entries)
|
|
logger.info("Claimed %d ownerless memories for %s", claimed, owner)
|
|
|
|
def _validate_entries(self, entries: List[Dict]) -> List[Dict]:
|
|
"""Ensure all entries have required fields."""
|
|
validated = []
|
|
for entry in entries:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
if "id" not in entry:
|
|
entry["id"] = str(uuid.uuid4())
|
|
if "timestamp" not in entry:
|
|
entry["timestamp"] = int(time.time())
|
|
if "source" not in entry:
|
|
entry["source"] = "unknown"
|
|
if "category" not in entry:
|
|
entry["category"] = "fact"
|
|
if "uses" not in entry:
|
|
entry["uses"] = 0
|
|
validated.append(entry)
|
|
return validated
|
|
|
|
def _migrate_from_legacy(self) -> List[Dict]:
|
|
"""Migrate from old text format to JSON if needed."""
|
|
legacy_path = os.path.join(os.path.dirname(self.memory_file), "memory.txt")
|
|
if not os.path.exists(legacy_path):
|
|
return []
|
|
|
|
logger.info("Converting legacy memory.txt to new JSON format")
|
|
try:
|
|
with open(legacy_path, "r", encoding="utf-8") as f:
|
|
lines = [ln.strip() for ln in f.readlines() if ln.strip()]
|
|
|
|
entries = []
|
|
for line in lines:
|
|
entries.append({
|
|
"id": str(uuid.uuid4()),
|
|
"text": line,
|
|
"timestamp": int(time.time()),
|
|
"source": "user",
|
|
"category": "fact"
|
|
})
|
|
|
|
self.save(entries)
|
|
return entries
|
|
except Exception as e:
|
|
logger.error("Failed to convert legacy memory: %s", e)
|
|
return []
|
|
|
|
def save(self, entries: List[Dict]):
|
|
"""Save memory entries to JSON file."""
|
|
# Validate entries before saving
|
|
for entry in entries:
|
|
if "id" not in entry:
|
|
entry["id"] = str(uuid.uuid4())
|
|
if "timestamp" not in entry:
|
|
entry["timestamp"] = int(time.time())
|
|
if "source" not in entry:
|
|
entry["source"] = "user"
|
|
if "category" not in entry:
|
|
entry["category"] = "fact"
|
|
|
|
# Use atomic write
|
|
tmp_file = self.memory_file + ".tmp"
|
|
with open(tmp_file, "w", encoding="utf-8") as f:
|
|
json.dump(entries, f, ensure_ascii=False, indent=2)
|
|
os.replace(tmp_file, self.memory_file)
|
|
|
|
def add_entry(self, text: str, source: str = "user", category: str = "fact", owner: str = None) -> Dict:
|
|
"""Add a new memory entry."""
|
|
if not text.strip():
|
|
raise ValueError("Memory text cannot be empty")
|
|
|
|
entry = {
|
|
"id": str(uuid.uuid4()),
|
|
"text": text.strip(),
|
|
"timestamp": int(time.time()),
|
|
"source": source,
|
|
"category": category,
|
|
"uses": 0,
|
|
}
|
|
if owner:
|
|
entry["owner"] = owner
|
|
return entry
|
|
|
|
def increment_uses(self, ids: List[str]) -> None:
|
|
"""Bump the uses counter for each memory id. Called after a memory has
|
|
actually been injected into a chat's context (not just retrieved)."""
|
|
if not ids:
|
|
return
|
|
id_set = set(ids)
|
|
try:
|
|
entries = self.load_all_for_update()
|
|
except MemoryStoreUnreadable as e:
|
|
# Best-effort counter; never worth rewriting the store blind.
|
|
logger.error("Skipping uses bump, memory store unreadable: %s", e)
|
|
return
|
|
changed = False
|
|
for e in entries:
|
|
if e.get("id") in id_set:
|
|
e["uses"] = int(e.get("uses", 0) or 0) + 1
|
|
changed = True
|
|
if changed:
|
|
self.save(entries)
|
|
|
|
def find_duplicates(self, text: str, entries: List[Dict] = None) -> List[Dict]:
|
|
"""Find duplicate memory entries based on text content."""
|
|
if entries is None:
|
|
entries = self.load()
|
|
|
|
text_lower = text.strip().lower()
|
|
return [entry for entry in entries if entry["text"].lower() == text_lower]
|
|
|
|
def categorize_memory_by_relevance(self, message: str, memories: list):
|
|
"""Categorize memories by type and relevance"""
|
|
categories = {
|
|
"contacts": [],
|
|
"preferences": [],
|
|
"facts": [],
|
|
"tasks": []
|
|
}
|
|
|
|
msg_lower = message.lower()
|
|
|
|
for mem in memories:
|
|
text_lower = mem["text"].lower()
|
|
|
|
# Contact info
|
|
if any(word in text_lower for word in ["phone", "email", "address", "lives", "works"]):
|
|
if any(word in msg_lower for word in ["contact", "phone", "address", "email"]):
|
|
categories["contacts"].append(mem)
|
|
|
|
# Personal preferences
|
|
elif any(word in text_lower for word in ["likes", "dislikes", "prefers", "favorite"]):
|
|
if any(word in msg_lower for word in ["like", "prefer", "favorite", "want"]):
|
|
categories["preferences"].append(mem)
|
|
|
|
# Tasks and todos
|
|
elif any(word in text_lower for word in ["todo", "task", "remind", "meeting"]):
|
|
if any(word in msg_lower for word in ["todo", "task", "schedule", "remind"]):
|
|
categories["tasks"].append(mem)
|
|
|
|
# General facts - only if very relevant
|
|
else:
|
|
if get_text_similarity(message, mem["text"]) > 0.4:
|
|
categories["facts"].append(mem)
|
|
|
|
return categories
|
|
|
|
def get_relevant_memories(self, query: str, memories: list, threshold: float = 0.05, max_items: int = 8):
|
|
"""Get memories that are relevant to the query based on text similarity and semantic keyword matching."""
|
|
if not memories or not query.strip():
|
|
return []
|
|
|
|
# Define keyword categories for semantic matching
|
|
identity_words = ["name", "who", "i", "am", "called", "identity", "myself", "me", "my"]
|
|
contact_words = ["phone", "email", "address", "contact", "number", "where", "located", "reach"]
|
|
preference_words = ["like", "prefer", "favorite", "want", "love", "hate", "dislike", "enjoy", "interested"]
|
|
task_words = ["todo", "task", "remind", "meeting", "appointment", "schedule", "deadline"]
|
|
fact_words = ["what", "when", "where", "how", "why", "explain", "describe", "information", "know"]
|
|
|
|
query_lower = query.lower()
|
|
|
|
# Determine query type based on keywords
|
|
query_type = None
|
|
if any(word in query_lower for word in identity_words):
|
|
query_type = "identity"
|
|
elif any(word in query_lower for word in contact_words):
|
|
query_type = "contact"
|
|
elif any(word in query_lower for word in preference_words):
|
|
query_type = "preference"
|
|
elif any(word in query_lower for word in task_words):
|
|
query_type = "task"
|
|
elif any(word in query_lower for word in fact_words):
|
|
query_type = "fact"
|
|
|
|
relevant = []
|
|
identity_memories = []
|
|
other_memories = []
|
|
|
|
# Separate identity memories from others
|
|
for memory in memories:
|
|
memory_text = memory["text"].lower()
|
|
# Check if this is an identity memory (contains name patterns or identity indicators)
|
|
is_identity = any([
|
|
re.search(r'\b[A-Z][a-z]+ [A-Z][a-z]+\b', memory["text"]),
|
|
any(word in memory_text for word in ["name is", "i'm", "i am", "called", "my name", "named", "call me"])
|
|
])
|
|
if is_identity:
|
|
identity_memories.append(memory)
|
|
else:
|
|
other_memories.append(memory)
|
|
|
|
# For identity queries, include all identity memories regardless of similarity
|
|
if query_type == "identity" and identity_memories:
|
|
# Give them high scores to ensure they're included first
|
|
for memory in identity_memories:
|
|
relevant.append((0.9, memory)) # High score for identity memories in identity queries
|
|
|
|
# Process other memories with similarity scoring
|
|
for memory in other_memories:
|
|
memory_text = memory["text"].lower()
|
|
memory_tokens = set(tokenize(memory_text))
|
|
query_tokens = set(tokenize(query_lower))
|
|
|
|
# Calculate base Jaccard similarity
|
|
if not query_tokens or not memory_tokens:
|
|
continue
|
|
|
|
base_similarity = len(query_tokens & memory_tokens) / len(query_tokens | memory_tokens)
|
|
final_score = base_similarity
|
|
|
|
# Apply boosts based on semantic matching
|
|
if query_type == "contact":
|
|
# Boost memories with contact information
|
|
has_contact_info = any(word in memory_text for word in ["@gmail.com", "@", ".com",
|
|
"phone", "number", "address",
|
|
"http", "www", "tel:"])
|
|
if has_contact_info:
|
|
final_score *= 1.4 # 40% boost for contact-related memories
|
|
|
|
elif query_type == "preference":
|
|
# Boost memories with preference indicators
|
|
has_preference = any(word in memory_text for word in ["like", "love", "hate", "dislike",
|
|
"prefer", "favorite", "enjoy", "interested"])
|
|
if has_preference:
|
|
final_score *= 1.3 # 30% boost for preference-related memories
|
|
|
|
elif query_type == "task":
|
|
# Boost memories with task indicators
|
|
has_task = any(word in memory_text for word in ["todo", "task", "remind", "meeting",
|
|
"appointment", "schedule", "deadline", "need to"])
|
|
if has_task:
|
|
final_score *= 1.3 # 30% boost for task-related memories
|
|
|
|
# Always consider exact phrase matches as highly relevant
|
|
if query.lower() in memory["text"].lower():
|
|
final_score = max(final_score, 0.8) # Ensure high relevance for exact matches
|
|
|
|
# Include memory if it meets threshold after boosts
|
|
if final_score >= threshold:
|
|
relevant.append((final_score, memory))
|
|
|
|
# Sort by final score (descending) and return top matches
|
|
relevant.sort(key=lambda x: x[0], reverse=True)
|
|
return [mem for _, mem in relevant[:max_items]]
|