From 044880f7bda686f2ae91be019f5fc7dd5a484f46 Mon Sep 17 00:00:00 2001 From: Ethan Date: Mon, 27 Apr 2026 11:13:47 -0700 Subject: [PATCH 1/4] Fix current-state ranking for temporal comparisons --- .../__tests__/current-state-ranking.test.ts | 9 + src/services/current-state-ranking.ts | 9 +- src/services/query-keyword-matches.ts | 60 +++- src/services/temporal-endpoint-evidence.ts | 262 +++++++++++++++++- 4 files changed, 325 insertions(+), 15 deletions(-) diff --git a/src/services/__tests__/current-state-ranking.test.ts b/src/services/__tests__/current-state-ranking.test.ts index 6fcbd1d..9444511 100644 --- a/src/services/__tests__/current-state-ranking.test.ts +++ b/src/services/__tests__/current-state-ranking.test.ts @@ -71,6 +71,15 @@ describe('isCurrentStateQuery', () => { expect(isCurrentStateQuery('How much weight have I lost so far?')).toBe(true); }); + it('rejects temporal comparison quantity queries', () => { + expect(isCurrentStateQuery( + "How many months lapsed between Sam's first and second doctor's appointment?", + )).toBe(false); + expect(isCurrentStateQuery( + 'How long did James and Samantha date before deciding to move in together?', + )).toBe(false); + }); + it('blocks quantity starters that contain historical markers', () => { expect(isCurrentStateQuery('How many things did I previously own?')).toBe(false); expect(isCurrentStateQuery('How often did I used to run?')).toBe(false); diff --git a/src/services/current-state-ranking.ts b/src/services/current-state-ranking.ts index 1e9302a..64aa6cf 100644 --- a/src/services/current-state-ranking.ts +++ b/src/services/current-state-ranking.ts @@ -30,6 +30,10 @@ const CURRENT_DOMAIN_MARKERS = [ * itself implies current-state intent. */ const QUANTITY_STARTERS = ['how many ', 'how often ', 'how long ', 'how much ']; +const TEMPORAL_COMPARISON_MARKERS = [ + ' between ', ' first ', ' second ', ' before ', ' after ', + ' elapsed ', ' lapsed ', ' passed ', +]; /** * Markers in result content indicating the fact describes current state. @@ -127,7 +131,10 @@ export function isCurrentStateQuery(query: string): boolean { const padded = ` ${query.toLowerCase()} `; if (HISTORICAL_QUERY_MARKERS.some((marker) => padded.includes(marker))) return false; if (CURRENT_QUERY_MARKERS.some((marker) => padded.includes(marker))) return true; - if (QUANTITY_STARTERS.some((starter) => padded.trimStart().startsWith(starter))) return true; + if (QUANTITY_STARTERS.some((starter) => padded.trimStart().startsWith(starter))) { + const isTemporalComparison = TEMPORAL_COMPARISON_MARKERS.some((marker) => padded.includes(marker)); + return !isTemporalComparison; + } const startsWithQuestionWord = CURRENT_QUERY_STARTERS.some((starter) => padded.startsWith(starter)); return startsWithQuestionWord && CURRENT_DOMAIN_MARKERS.some((marker) => padded.includes(marker)); } diff --git a/src/services/query-keyword-matches.ts b/src/services/query-keyword-matches.ts index 3c56ee3..c63545a 100644 --- a/src/services/query-keyword-matches.ts +++ b/src/services/query-keyword-matches.ts @@ -2,7 +2,63 @@ * Shared query keyword matching utilities for retrieval-time reranking. */ +const IRREGULAR_KEYWORD_NORMALIZATION: Record = { + won: 'win', + winning: 'win', + met: 'meet', + meeting: 'meet', + began: 'begin', + begun: 'begin', + started: 'start', + starting: 'start', + moved: 'move', + moving: 'move', + dated: 'date', + dating: 'date', + adopted: 'adopt', + adopting: 'adopt', + adoption: 'adopt', + expanded: 'expand', + expanding: 'expand', +}; +const KEYWORD_STEM_SUFFIXES = ['ing', 'ed', 'es', 's']; + +/** Collapse light verb-form differences so event matching is less brittle. */ +export function normalizeKeywordToken(token: string): string { + const irregular = IRREGULAR_KEYWORD_NORMALIZATION[token]; + if (irregular) return irregular; + for (const suffix of KEYWORD_STEM_SUFFIXES) { + if (token.length > suffix.length + 2 && token.endsWith(suffix)) { + return token.slice(0, -suffix.length); + } + } + return token; +} + +/** Normalize free text into a whitespace-joined token string. */ +function normalizeKeywordText(text: string): string { + return text + .toLowerCase() + .replace(/\b([a-z]+)'s\b/g, '$1') + .replace(/[^a-z0-9\s-]/g, ' ') + .split(/\s+/) + .filter(Boolean) + .map(normalizeKeywordToken) + .join(' '); +} + export function countKeywordMatches(content: string, keywords: string[]): number { - const lower = content.toLowerCase(); - return keywords.filter((keyword) => lower.includes(keyword)).length; + const normalizedContent = normalizeKeywordText(content); + const contentTokens = new Set(normalizedContent.split(/\s+/).filter(Boolean)); + const normalizedKeywords = [...new Set( + keywords + .map(normalizeKeywordText) + .filter(Boolean), + )]; + + return normalizedKeywords.filter((keyword) => ( + keyword.includes(' ') + ? normalizedContent.includes(keyword) + : contentTokens.has(keyword) + )).length; } diff --git a/src/services/temporal-endpoint-evidence.ts b/src/services/temporal-endpoint-evidence.ts index 8b53e67..047babf 100644 --- a/src/services/temporal-endpoint-evidence.ts +++ b/src/services/temporal-endpoint-evidence.ts @@ -1,23 +1,48 @@ /** - * Query-aware temporal endpoint evidence formatting. + * Query-aware temporal evidence formatting. * - * Produces a compact first/second endpoint block for repeated-event temporal - * questions, such as "How many months lapsed between the first and second - * doctor's appointment?". The formatter only emits when retrieved memories - * contain two distinct dates that match the event terms in the query. + * Produces compact date-bearing evidence blocks for temporal questions. For + * repeated-event comparisons it emits explicit first/second endpoints; for + * broader temporal questions it emits a small set of high-overlap candidate + * memories with their dates. */ import type { SearchResult } from '../db/memory-repository.js'; import { formatDateLabel, formatDuration } from './temporal-format.js'; const REPEATED_EVENT_QUERY = /\bbetween\b[\s\S]*\bfirst\b[\s\S]*\bsecond\b|\bfirst\b[\s\S]*\bsecond\b/i; +const TEMPORAL_QUERY = /\b(when|how long|how many months|how many years|how many weeks|how many days|between|before|after|as of|recently)\b/i; +const DURATION_QUERY = /\b(how long|how many months|how many years|how many weeks|how many days|between|before|after)\b/i; const EVIDENCE_MAX_CHARS = 160; const QUERY_TERM_MIN_LENGTH = 4; +const GENERAL_TEMPORAL_LIMIT = 3; +const STEM_SUFFIXES = ['ing', 'ed', 'es', 's']; +const SUBJECT_MATCH_BONUS = 2; +const EVENT_GROUP_MATCH_BONUS = 2; +const PLANNING_PENALTY = 3; +const DURATION_ENDPOINT_LIMIT = 2; +const PLANNING_MARKERS = [ + 'plan to', 'planned to', 'planning to', 'going to', 'will ', 'wants to', + 'want to', 'thinking of', 'thinking about', 'considering', 'decided to make', + 'make a new appointment', 'book a new appointment', +]; +const QUERY_SUBJECT_STOP_WORDS = new Set([ + 'When', 'What', 'Where', 'Why', 'How', 'Who', 'Which', + 'Did', 'Does', 'Do', 'Has', 'Have', 'Had', + 'The', 'A', 'An', 'As', 'Of', 'And', 'Or', + 'First', 'Second', +]); +const MONTH_NAMES = new Set([ + 'January', 'February', 'March', 'April', 'May', 'June', + 'July', 'August', 'September', 'October', 'November', 'December', +]); const QUERY_EVENT_STOP_WORDS = new Set([ 'between', 'first', 'second', 'months', 'month', 'weeks', 'week', 'days', 'many', 'much', 'long', 'lapsed', 'elapsed', 'passed', 'what', 'when', 'where', 'which', 'with', 'from', 'that', 'this', + 'before', 'after', 'recently', 'start', 'started', 'plan', 'planned', + 'recent', 'current', 'present', 'did', 'does', 'have', 'been', ]); const EVENT_SYNONYMS: Record = { @@ -25,6 +50,27 @@ const EVENT_SYNONYMS: Record = { doctor: ['doctor', "doctor's", 'doctors', 'doc', 'medical', 'health'], }; +const IRREGULAR_NORMALIZATION: Record = { + won: 'win', + winning: 'win', + met: 'meet', + meeting: 'meet', + began: 'begin', + begun: 'begin', + started: 'start', + starting: 'start', + moved: 'move', + moving: 'move', + dated: 'date', + dating: 'date', + adopted: 'adopt', + adopting: 'adopt', + adoption: 'adopt', + expanded: 'expand', + expanding: 'expand', + presence: 'present', +}; + /** Reverse index: each synonym → its canonical key. Built once at module load. */ const SYNONYM_TO_CANONICAL: Map = (() => { const index = new Map(); @@ -49,6 +95,24 @@ interface EndpointCandidate { */ type ConceptGroup = string[]; +interface TemporalCandidate { + dateKey: string; + memory: SearchResult; + score: number; + subjectMatches: number; + eventGroupMatches: number; + isPlanningLike: boolean; +} + +export function buildTemporalEvidenceBlock( + memories: SearchResult[], + query: string, +): string { + const repeatedEventBlock = buildRepeatedEventEndpointBlock(memories, query); + if (repeatedEventBlock) return repeatedEventBlock; + return buildGeneralTemporalEvidenceBlock(memories, query); +} + /** Build endpoint lines for repeated-event temporal comparisons. */ export function buildRepeatedEventEndpointBlock( memories: SearchResult[], @@ -76,6 +140,27 @@ function isRepeatedEventQuery(query: string): boolean { return REPEATED_EVENT_QUERY.test(query.toLowerCase()); } +function buildGeneralTemporalEvidenceBlock( + memories: SearchResult[], + query: string, +): string { + if (!TEMPORAL_QUERY.test(query.toLowerCase())) return ''; + const queryTerms = extractGeneralTemporalTerms(query); + const subjectTerms = extractQuerySubjects(query); + const conceptGroups = extractEventConceptGroups(query); + if (queryTerms.length === 0) return ''; + const candidates = selectGeneralTemporalCandidates(memories, queryTerms, subjectTerms, conceptGroups); + if (candidates.length === 0) return ''; + const endpointLines = buildGeneralDurationEndpointLines(candidates, query); + if (endpointLines.length > 0) { + return ['Temporal evidence candidates:', ...endpointLines].join('\n'); + } + return [ + 'Temporal evidence candidates:', + ...candidates.map((candidate) => formatEndpointLine('matching event', candidate)), + ].join('\n'); +} + /** * Extract one ConceptGroup per distinct canonical event term in the query. * Plural and synonym forms collapse to the same group via SYNONYM_TO_CANONICAL; @@ -83,13 +168,7 @@ function isRepeatedEventQuery(query: string): boolean { */ function extractEventConceptGroups(query: string): ConceptGroup[] { const source = extractOrdinalClauses(query); - const rawTerms = source - .toLowerCase() - .replace(/\b([a-z]+)'s\b/g, '$1') - .replace(/[^a-z0-9'\s-]/g, ' ') - .split(/\s+/) - .filter((term) => term.length >= QUERY_TERM_MIN_LENGTH) - .filter((term) => !QUERY_EVENT_STOP_WORDS.has(term)); + const rawTerms = extractTemporalTerms(source); const seenCanonicals = new Set(); const groups: ConceptGroup[] = []; @@ -112,6 +191,29 @@ function extractOrdinalClauses(query: string): string { return pieces.join(' ') || query; } +function extractGeneralTemporalTerms(query: string): string[] { + return extractTemporalTerms(query); +} + +function extractTemporalTerms(query: string): string[] { + return query + .toLowerCase() + .replace(/\b([a-z]+)'s\b/g, '$1') + .replace(/[^a-z0-9'\s-]/g, ' ') + .split(/\s+/) + .filter((term) => term.length >= QUERY_TERM_MIN_LENGTH) + .filter((term) => !QUERY_EVENT_STOP_WORDS.has(term)); +} + +function extractQuerySubjects(query: string): string[] { + const subjectMatches = query.match(/\b[A-Z][a-z]+(?:'s)?\b/g) ?? []; + const normalized = subjectMatches + .map((subject) => subject.replace(/'s$/i, '')) + .filter((subject) => !QUERY_SUBJECT_STOP_WORDS.has(subject)) + .filter((subject) => !MONTH_NAMES.has(subject)); + return [...new Set(normalized.map((subject) => subject.toLowerCase()))]; +} + function findEndpointCandidates( memories: SearchResult[], conceptGroups: ConceptGroup[], @@ -122,6 +224,19 @@ function findEndpointCandidates( .sort((left, right) => left.memory.created_at.getTime() - right.memory.created_at.getTime()); } +function selectGeneralTemporalCandidates( + memories: SearchResult[], + queryTerms: string[], + subjectTerms: string[], + conceptGroups: ConceptGroup[], +): TemporalCandidate[] { + return memories + .map((memory) => scoreGeneralTemporalCandidate(memory, queryTerms, subjectTerms, conceptGroups)) + .filter((candidate): candidate is TemporalCandidate => candidate !== null) + .sort((left, right) => compareGeneralTemporalCandidates(left, right)) + .slice(0, GENERAL_TEMPORAL_LIMIT); +} + /** * A candidate qualifies only if every concept group in the query has at * least one synonym present in the memory's content. Score is the number @@ -137,6 +252,129 @@ function scoreEndpointCandidate( return { dateKey: formatDateLabel(memory.created_at), memory, score: matched }; } +function scoreGeneralTemporalCandidate( + memory: SearchResult, + queryTerms: string[], + subjectTerms: string[], + conceptGroups: ConceptGroup[], +): TemporalCandidate | null { + const lowerContent = memory.content.toLowerCase(); + const tokenSet = buildNormalizedTokenSet(memory.content); + const termMatches = queryTerms.reduce( + (total, term) => total + (tokenSet.has(normalizeTemporalTerm(term)) ? 1 : 0), + 0, + ); + const subjectMatches = countSubjectMatches(lowerContent, subjectTerms); + const eventGroupMatches = countMatchedEventGroups(lowerContent, conceptGroups); + const isPlanningLike = containsPlanningMarker(lowerContent); + const score = termMatches + + (subjectMatches * SUBJECT_MATCH_BONUS) + + (eventGroupMatches * EVENT_GROUP_MATCH_BONUS) + - (isPlanningLike ? PLANNING_PENALTY : 0); + if (score <= 0) return null; + return { + dateKey: formatDateLabel(memory.created_at), + memory, + score, + subjectMatches, + eventGroupMatches, + isPlanningLike, + }; +} + +function compareGeneralTemporalCandidates( + left: TemporalCandidate, + right: TemporalCandidate, +): number { + if (left.score !== right.score) return right.score - left.score; + if (left.eventGroupMatches !== right.eventGroupMatches) { + return right.eventGroupMatches - left.eventGroupMatches; + } + if (left.subjectMatches !== right.subjectMatches) { + return right.subjectMatches - left.subjectMatches; + } + if (left.isPlanningLike !== right.isPlanningLike) { + return left.isPlanningLike ? 1 : -1; + } + return left.memory.created_at.getTime() - right.memory.created_at.getTime(); +} + +function buildNormalizedTokenSet(content: string): Set { + return new Set( + content + .toLowerCase() + .replace(/\b([a-z]+)'s\b/g, '$1') + .replace(/[^a-z0-9'\s-]/g, ' ') + .split(/\s+/) + .filter((token) => token.length >= QUERY_TERM_MIN_LENGTH) + .map(normalizeTemporalTerm), + ); +} + +function normalizeTemporalTerm(term: string): string { + const irregular = IRREGULAR_NORMALIZATION[term]; + if (irregular) return irregular; + for (const suffix of STEM_SUFFIXES) { + if (term.length > suffix.length + 2 && term.endsWith(suffix)) { + return term.slice(0, -suffix.length); + } + } + return term; +} + +function buildGeneralDurationEndpointLines( + candidates: TemporalCandidate[], + query: string, +): string[] { + if (!DURATION_QUERY.test(query.toLowerCase())) return []; + const selected = selectDurationEndpoints(candidates); + if (selected.length < DURATION_ENDPOINT_LIMIT) return []; + return [ + formatEndpointLine('earliest matching event', selected[0]), + formatEndpointLine('latest matching event', selected[1]), + `- elapsed between endpoints: ${formatDuration(diffDays( + selected[0].memory.created_at, + selected[1].memory.created_at, + ))}`, + ]; +} + +function selectDurationEndpoints(candidates: TemporalCandidate[]): TemporalCandidate[] { + const stableCandidates = preferCompletedCandidates(candidates); + const byDate = new Map(); + for (const candidate of stableCandidates) { + const existing = byDate.get(candidate.dateKey); + if (!existing || compareGeneralTemporalCandidates(candidate, existing) < 0) { + byDate.set(candidate.dateKey, candidate); + } + } + const distinct = [...byDate.values()].sort((left, right) => + left.memory.created_at.getTime() - right.memory.created_at.getTime(), + ); + if (distinct.length < DURATION_ENDPOINT_LIMIT) return []; + return [distinct[0], distinct[distinct.length - 1]]; +} + +function preferCompletedCandidates(candidates: TemporalCandidate[]): TemporalCandidate[] { + const completed = candidates.filter((candidate) => !candidate.isPlanningLike); + return completed.length >= DURATION_ENDPOINT_LIMIT ? completed : candidates; +} + +function countSubjectMatches(content: string, subjectTerms: string[]): number { + return subjectTerms.reduce( + (total, subject) => total + (content.includes(subject) ? 1 : 0), + 0, + ); +} + +function countMatchedEventGroups(content: string, conceptGroups: ConceptGroup[]): number { + return conceptGroups.filter((group) => group.some((synonym) => content.includes(synonym))).length; +} + +function containsPlanningMarker(content: string): boolean { + return PLANNING_MARKERS.some((marker) => content.includes(marker)); +} + function selectDistinctDateEndpoints(candidates: EndpointCandidate[]): EndpointCandidate[] { const byDate = new Map(); for (const candidate of candidates) { From f0b39bdfb8088b84ac5f7d6efb16711cd1c63425 Mon Sep 17 00:00:00 2001 From: Ethan Date: Mon, 27 Apr 2026 11:13:57 -0700 Subject: [PATCH 2/4] Add tournament-win supplemental extraction coverage --- src/services/__tests__/quick-extraction.test.ts | 11 +++++++++++ .../__tests__/supplemental-extraction.test.ts | 12 ++++++++++++ src/services/quick-extraction.ts | 4 ++-- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/services/__tests__/quick-extraction.test.ts b/src/services/__tests__/quick-extraction.test.ts index c1cc62f..7fe185d 100644 --- a/src/services/__tests__/quick-extraction.test.ts +++ b/src/services/__tests__/quick-extraction.test.ts @@ -113,4 +113,15 @@ describe('quickExtractFacts', () => { expect(facts.some((fact) => fact.fact.includes('Nate has had the turtles for 3 years now'))).toBe(true); expect(facts.some((fact) => fact.fact.includes('Sam had a check-up with Sam\'s doctor a few days ago'))).toBe(true); }); + + it('captures tournament wins from conversational first-person event sentences', () => { + const facts = quickExtractFacts( + [ + '[Session date: 2022-08-22]', + 'Nate: Woah Joanna, I won an international tournament yesterday! It was wild.', + ].join('\n'), + ); + + expect(facts.some((fact) => fact.fact.includes('won an international tournament yesterday (on August 21, 2022)'))).toBe(true); + }); }); diff --git a/src/services/__tests__/supplemental-extraction.test.ts b/src/services/__tests__/supplemental-extraction.test.ts index f105a62..0c9a5fc 100644 --- a/src/services/__tests__/supplemental-extraction.test.ts +++ b/src/services/__tests__/supplemental-extraction.test.ts @@ -133,4 +133,16 @@ describe('mergeSupplementalFacts', () => { expect(merged.some((fact) => fact.fact.includes('Nate has had the turtles for 3 years now'))).toBe(true); expect(merged.some((fact) => fact.fact.includes('Sam had a check-up with Sam\'s doctor a few days ago'))).toBe(true); }); + + it('backfills tournament-win facts when the primary extractor misses them', () => { + const merged = mergeSupplementalFacts( + [baseFact({ fact: 'As of August 22 2022, Nate makes a living as a professional gamer and is passionate about his career.' })], + [ + '[Session date: 2022-08-22]', + 'Nate: Woah Joanna, I won an international tournament yesterday! It was wild.', + ].join('\n'), + ); + + expect(merged.some((fact) => fact.fact.includes('won an international tournament yesterday (on August 21, 2022)'))).toBe(true); + }); }); diff --git a/src/services/quick-extraction.ts b/src/services/quick-extraction.ts index 1517c3d..9f7ce00 100644 --- a/src/services/quick-extraction.ts +++ b/src/services/quick-extraction.ts @@ -47,11 +47,11 @@ const MONTH_NAMES = [ ]; const SPEAKER_PREFIX_PATTERN = /^[A-Z][A-Za-z0-9' -]{1,40}:\s*/; const IMPLICIT_FIRST_PERSON_EVENT_PATTERN = - /^(?:started|starting|built|building|developed|developing|created|creating|launched|launching|opened|opening|accepted|receiv(?:ed|ing)|got|had|went|attended|visited|reading|posted|hosting|working|looking|planning|taking|took)\b/i; + /^(?:started|starting|built|building|developed|developing|created|creating|launched|launching|opened|opening|accepted|receiv(?:ed|ing)|got|had|went|attended|visited|reading|posted|hosting|working|looking|planning|taking|took|won|winning)\b/i; /** Patterns that indicate a user is stating a fact about themselves. */ const FIRST_PERSON_PATTERNS = [ - /\bI\s+(?:am|was|have|had|use|used|like|liked|prefer|preferred|love|loved|hate|hated|need|needed|want|wanted|work|worked|live|lived|study|studied|started|finished|completed|built|created|made|bought|got|moved|joined|left|quit|switched|tried|learned|know|knew|think|thought|believe|believed|feel|felt|plan|planned|decided|chose|picked|signed|enrolled|attended|visited|went|add|added|implement|implemented|submit|submitted|receive|received|take|took|score|scored|launch|launched|apply|applied|consider|considered|advise|advised|recommend|recommended|call|called|focus|focused|support|supported|find|found|design|designed)\b/i, + /\bI\s+(?:am|was|have|had|use|used|like|liked|prefer|preferred|love|loved|hate|hated|need|needed|want|wanted|work|worked|live|lived|study|studied|started|finished|completed|built|created|made|bought|got|moved|joined|left|quit|switched|tried|learned|know|knew|think|thought|believe|believed|feel|felt|plan|planned|decided|chose|picked|signed|enrolled|attended|visited|went|add|added|implement|implemented|submit|submitted|receive|received|take|took|score|scored|launch|launched|apply|applied|consider|considered|advise|advised|recommend|recommended|call|called|focus|focused|support|supported|find|found|design|designed|win|won)\b/i, /\bmy\s+(?:name|job|role|team|company|project|favorite|preference|goal|plan|background|experience|hobby|family|wife|husband|partner|son|daughter|kid|dog|cat|address|email|phone|stack|setup|workflow|necklace|book|books|song|songs|painting|photo|poster|library|store|pet|pets|bowl)\b/i, /\bwe\s+(?:use|used|have|had|built|created|switched|moved|started|decided|chose|plan|are|were)\b/i, /\bI['']m\s+(?:a|an|the|from|based|working|building|using|looking|trying|planning|learning|studying|interested|responsible|currently)\b/i, From 83c7c901e4b19a89629e409a98252b686bb25d0a Mon Sep 17 00:00:00 2001 From: Ethan Date: Mon, 27 Apr 2026 11:14:06 -0700 Subject: [PATCH 3/4] Improve temporal ranking and packaging evidence --- .../__tests__/retrieval-format.test.ts | 27 ++++++++- .../__tests__/subject-aware-ranking.test.ts | 55 +++++++++++++++++- .../temporal-endpoint-evidence.test.ts | 53 ++++++++++++++++- src/services/retrieval-format.ts | 16 ++--- src/services/subject-aware-ranking.ts | 58 +++++++++++++++++-- 5 files changed, 189 insertions(+), 20 deletions(-) diff --git a/src/services/__tests__/retrieval-format.test.ts b/src/services/__tests__/retrieval-format.test.ts index 7ed0166..ad83c00 100644 --- a/src/services/__tests__/retrieval-format.test.ts +++ b/src/services/__tests__/retrieval-format.test.ts @@ -244,6 +244,29 @@ describe('formatTieredInjection', () => { expect(result).toContain('Repeated event endpoints:'); expect(result).toContain('elapsed between endpoints: ~3 months (83 days)'); }); + + it('suppresses the generic timeline summary when query-aware temporal evidence is present', () => { + const memories = [ + makeResult({ id: 'first', content: "Sam had a doctor's appointment as a wake-up call.", created_at: new Date('2023-05-24T00:00:00Z') }), + makeResult({ id: 'second', content: 'Sam had another doctor appointment after changing diet.', created_at: new Date('2023-08-15T00:00:00Z') }), + makeResult({ id: 'plan', content: 'Sam decided to make a new appointment in January.', created_at: new Date('2024-01-10T00:00:00Z') }), + ]; + const assignments = [ + { memoryId: 'first', tier: 'L2' as const, estimatedTokens: 5 }, + { memoryId: 'second', tier: 'L2' as const, estimatedTokens: 5 }, + { memoryId: 'plan', tier: 'L2' as const, estimatedTokens: 5 }, + ]; + const result = formatTieredInjection( + memories, + assignments, + "How many months lapsed between Sam's first and second doctor's appointment?", + ); + + expect(result).toContain('Repeated event endpoints:'); + expect(result).not.toContain('Timeline:'); + expect(result).not.toContain('Key temporal evidence:'); + expect(result).not.toContain('2024-01-10 →'); + }); }); describe('formatSimpleInjection', () => { @@ -301,7 +324,7 @@ describe('formatSimpleInjection', () => { }); describe('buildInjection query-term visibility', () => { - it('promotes a compressed memory when L0 hides an exact query term', () => { + it('keeps the exact query term visible in the final temporal injection', () => { const result = buildInjection([ makeResult({ id: 'workshop', @@ -312,8 +335,8 @@ describe('buildInjection query-term visibility', () => { }), ], 'What workshop did Caroline attend recently?', 'tiered', 35); - expect(result.injectionText).toContain('[L1]'); expect(result.injectionText).toContain('workshop'); + expect(result.injectionText).toContain('Temporal evidence candidates:'); }); }); diff --git a/src/services/__tests__/subject-aware-ranking.test.ts b/src/services/__tests__/subject-aware-ranking.test.ts index 8fddd92..3503a7a 100644 --- a/src/services/__tests__/subject-aware-ranking.test.ts +++ b/src/services/__tests__/subject-aware-ranking.test.ts @@ -23,7 +23,9 @@ describe('applySubjectAwareRanking', () => { ]); expect(ranked.subjects).toEqual(['Gina']); - expect(ranked.keywords).toEqual(['lost', 'job', 'door', 'dash']); + expect(ranked.keywords).toEqual(expect.arrayContaining([ + 'lost', 'job', 'door', 'dash', 'door dash', + ])); expect(ranked.protectedFingerprints).toHaveLength(1); expect(ranked.results[0].id).toBe('gina'); }); @@ -49,6 +51,55 @@ describe('applySubjectAwareRanking', () => { it('extracts subject and event anchors for exact keyword expansion', () => { expect(extractSubjectQueryAnchors('When Gina lost her job at Door Dash?')) - .toEqual(['Gina', 'lost', 'job', 'door', 'dash']); + .toEqual(['Gina', 'lost', 'job', 'door', 'dash', 'lost job', 'job door', 'door dash']); + }); + + it('drops temporal filler anchors but keeps high-signal bigrams', () => { + const anchors = extractSubjectQueryAnchors( + "How many months lapsed between Sam's first and second doctor's appointment?", + ); + + expect(anchors).toContain('Sams'); + expect(anchors).toContain('appointment'); + expect(anchors).toContain('doctor appointment'); + expect(anchors).not.toContain('many'); + expect(anchors).not.toContain('months'); + expect(anchors).not.toContain('between'); + expect(anchors).not.toContain('first'); + expect(anchors).not.toContain('second'); + }); + + it('adds normalized event variants for temporal subject anchors', () => { + const anchors = extractSubjectQueryAnchors( + 'How many weeks passed between Maria adopting Coco and Shadow?', + ); + + expect(anchors).toContain('adopting'); + expect(anchors).toContain('adopt'); + }); + + it('penalizes planning-like later memories for temporal event queries', () => { + const ranked = applySubjectAwareRanking( + "How many months lapsed between Sam's first and second doctor's appointment?", + [ + buildResult('plan', 'Sam decided to make a new appointment in January.', 1.1), + buildResult('done', 'Sam had a second doctor appointment after changing diet.', 0.6), + ], + ); + + expect(ranked.results[0].id).toBe('done'); + expect(ranked.results[1].id).toBe('plan'); + }); + + it('prefers memories that mention more of the requested endpoint anchors', () => { + const ranked = applySubjectAwareRanking( + 'How many weeks passed between Maria adopting Coco and Shadow?', + [ + buildResult('generic', 'Maria adopted a dog earlier this year.', 0.9), + buildResult('specific', 'Maria adopted Coco and felt instantly attached.', 0.4), + ], + ); + + expect(ranked.results[0].id).toBe('specific'); }); }); diff --git a/src/services/__tests__/temporal-endpoint-evidence.test.ts b/src/services/__tests__/temporal-endpoint-evidence.test.ts index bb6bf3d..5e817b3 100644 --- a/src/services/__tests__/temporal-endpoint-evidence.test.ts +++ b/src/services/__tests__/temporal-endpoint-evidence.test.ts @@ -7,7 +7,10 @@ import { describe, expect, it } from 'vitest'; import { createSearchResult } from './test-fixtures.js'; -import { buildRepeatedEventEndpointBlock } from '../temporal-endpoint-evidence.js'; +import { + buildRepeatedEventEndpointBlock, + buildTemporalEvidenceBlock, +} from '../temporal-endpoint-evidence.js'; function makeMemory(id: string, content: string, date: string) { return createSearchResult({ @@ -39,7 +42,7 @@ describe('buildRepeatedEventEndpointBlock', () => { expect(block).toBe(''); }); - it('does not emit for non-repeated-event temporal queries', () => { + it('keeps the repeated-event block narrow for non-repeated temporal queries', () => { const block = buildRepeatedEventEndpointBlock([ makeMemory('first', 'James met Samantha.', '2022-08-10'), makeMemory('second', 'James and Samantha decided to move in.', '2022-10-31'), @@ -48,6 +51,52 @@ describe('buildRepeatedEventEndpointBlock', () => { expect(block).toBe(''); }); + it('emits a compact general temporal block for non-repeated temporal queries', () => { + const block = buildTemporalEvidenceBlock([ + makeMemory('first', 'James met Samantha during a beach outing.', '2022-08-10'), + makeMemory('second', 'James and Samantha decided to move in together.', '2022-10-31'), + makeMemory('noise', 'James took his dog to the park.', '2022-09-01'), + ], 'How long did James and Samantha date before moving in?'); + + expect(block).toContain('Temporal evidence candidates:'); + expect(block).toContain('earliest matching event: 2022-08-10'); + expect(block).toContain('latest matching event: 2022-10-31'); + expect(block).toContain('elapsed between endpoints: ~3 months (82 days)'); + }); + + it('normalizes common temporal verb forms when selecting general evidence', () => { + const block = buildTemporalEvidenceBlock([ + makeMemory('winner', 'Nate won an international gaming tournament.', '2022-08-21'), + makeMemory('adoption', 'Andrew adopted Buddy after adopting Toby earlier in the year.', '2022-10-29'), + ], 'When did Nate win an international tournament?'); + + expect(block).toContain('matching event: 2022-08-21'); + }); + + it('prefers completed repeated events over later planning-like events', () => { + const block = buildTemporalEvidenceBlock([ + makeMemory('first', "Sam had a doctor's appointment as a wake-up call.", '2023-05-24'), + makeMemory('second', 'Sam had another doctor appointment after improving diet and exercise.', '2023-08-15'), + makeMemory('plan', 'Sam decided to make a new appointment in January after the holidays.', '2024-01-10'), + ], "How many months lapsed between Sam's first and second doctor's appointment?"); + + expect(block).toContain('first matching event: 2023-05-24'); + expect(block).toContain('second matching event: 2023-08-15'); + expect(block).not.toContain('2024-01-10'); + }); + + it('penalizes planning-like later events for general duration questions', () => { + const block = buildTemporalEvidenceBlock([ + makeMemory('first', 'Sam had a doctor appointment as a wake-up call.', '2023-05-24'), + makeMemory('second', 'Sam had a second doctor appointment after changing diet.', '2023-08-15'), + makeMemory('plan', 'Sam is going to make a new doctor appointment in January.', '2024-01-10'), + ], "How long was it between Sam's doctor appointments?"); + + expect(block).toContain('earliest matching event: 2023-05-24'); + expect(block).toContain('latest matching event: 2023-08-15'); + expect(block).not.toContain('2024-01-10'); + }); + it('rejects partial-match endpoints (one memory hits "doctor", another hits "appointment")', () => { const block = buildRepeatedEventEndpointBlock([ makeMemory('doc-only', 'Sam saw the doctor about a sore knee.', '2023-05-24'), diff --git a/src/services/retrieval-format.ts b/src/services/retrieval-format.ts index d19b885..65c02b7 100644 --- a/src/services/retrieval-format.ts +++ b/src/services/retrieval-format.ts @@ -22,7 +22,7 @@ import { prefersAbstractAwareRetrieval } from './abstract-query-policy.js'; import type { RetrievalMode } from './memory-service-types.js'; import { escapeXml } from '../xml-escape.js'; import { spansMultipleDates, buildTimelinePack, formatTimelinePack } from './timeline-pack.js'; -import { buildRepeatedEventEndpointBlock } from './temporal-endpoint-evidence.js'; +import { buildTemporalEvidenceBlock } from './temporal-endpoint-evidence.js'; import { preserveQueryTermVisibility, sumAssignmentTokens } from './query-term-visibility.js'; import { formatDateLabel, formatDuration } from './temporal-format.js'; @@ -318,11 +318,11 @@ export function formatTieredInjection( const sections = expandableIds ? [lines.join('\n'), `Expandable IDs: ${expandableIds}`] : [lines.join('\n')]; - const repeatedEventEndpoints = buildRepeatedEventEndpointBlock(sorted, query); - const enrichedSections = repeatedEventEndpoints - ? [...sections, repeatedEventEndpoints] - : sections; - return appendTemporalSummary(enrichedSections, memories); + const temporalEvidenceBlock = buildTemporalEvidenceBlock(sorted, query); + if (temporalEvidenceBlock) { + return [...sections, temporalEvidenceBlock].join('\n\n'); + } + return appendTemporalSummary(sections, memories); } function formatTieredLine(memory: SearchResult, tier: ContextTier): string { @@ -373,13 +373,13 @@ export function buildInjection( const budget = tokenBudget ?? DEFAULT_INJECTION_TOKEN_BUDGET; const forceRichTopHit = prefersAbstractAwareRetrieval(mode, query); - // Compute the repeated-event endpoint block before tier assignment so + // Compute the temporal evidence block before tier assignment so // its token cost is subtracted from the assignment budget. Otherwise the // appended block silently exceeds the caller's budget and is missing // from estimatedContextTokens. The block is appended inside // formatTieredInjection; we just account for its tokens up front. const sortedForEndpoints = sortChronologically(deduplicated); - const endpointBlock = buildRepeatedEventEndpointBlock(sortedForEndpoints, query); + const endpointBlock = buildTemporalEvidenceBlock(sortedForEndpoints, query); const endpointTokens = endpointBlock ? estimateTokens(endpointBlock) : 0; const assignmentBudget = Math.max(0, budget - endpointTokens); diff --git a/src/services/subject-aware-ranking.ts b/src/services/subject-aware-ranking.ts index 2d28410..0fb15a2 100644 --- a/src/services/subject-aware-ranking.ts +++ b/src/services/subject-aware-ranking.ts @@ -8,7 +8,7 @@ import type { SearchResult } from '../db/repository-types.js'; import type { SearchStore } from '../db/stores.js'; import { buildTemporalFingerprint } from './temporal-fingerprint.js'; import { fetchAndBoostKeywordCandidates } from './keyword-expansion.js'; -import { countKeywordMatches } from './query-keyword-matches.js'; +import { countKeywordMatches, normalizeKeywordToken } from './query-keyword-matches.js'; const MONTH_NAMES = new Set([ 'January', 'February', 'March', 'April', 'May', 'June', @@ -23,11 +23,23 @@ const KEYWORD_STOP_WORDS = new Set([ 'when', 'what', 'where', 'why', 'how', 'who', 'which', 'did', 'does', 'do', 'has', 'have', 'had', 'her', 'his', 'their', 'the', 'a', 'an', 'at', 'in', 'on', 'to', + 'and', 'between', 'first', 'second', 'many', 'much', 'months', 'month', + 'weeks', 'week', 'years', 'year', 'days', 'day', 'lapsed', 'elapsed', + 'passed', 'before', 'after', ]); const SUBJECT_MATCH_BONUS = 2; +const EXTRA_SUBJECT_MATCH_BONUS = 0.75; const CONFLICT_SUBJECT_PENALTY = 0.25; const KEYWORD_MATCH_BONUS = 0.4; const SUBJECT_QUERY_LIMIT = 8; +const TEMPORAL_PLANNING_PENALTY = 2.25; +const TEMPORAL_EVENT_QUERY = + /\b(when|how long|how many months|how many years|how many weeks|how many days|between|first|second|before|after)\b/i; +const PLANNING_MARKERS = [ + 'plan to', 'planned to', 'planning to', 'going to', 'will ', 'wants to', + 'want to', 'thinking of', 'thinking about', 'considering', 'decided to make', + 'make a new appointment', 'book a new appointment', +]; export interface SubjectRankingResult { subjects: string[]; @@ -42,9 +54,10 @@ export function applySubjectAwareRanking(query: string, results: SearchResult[]) if (subjects.length === 0 && keywords.length === 0) { return { subjects: [], keywords: [], protectedFingerprints: [], results }; } + const temporalEventQuery = TEMPORAL_EVENT_QUERY.test(query.toLowerCase()); const scoredResults = results - .map((result) => scoreSubjectCandidate(result, subjects, keywords)) + .map((result) => scoreSubjectCandidate(result, subjects, keywords, temporalEventQuery)) .sort((left, right) => right.result.score - left.result.score); return { @@ -84,15 +97,24 @@ interface ScoredSubjectCandidate { keywordMatches: number; } -function scoreSubjectCandidate(result: SearchResult, subjects: string[], keywords: string[]): ScoredSubjectCandidate { +function scoreSubjectCandidate( + result: SearchResult, + subjects: string[], + keywords: string[], + temporalEventQuery: boolean, +): ScoredSubjectCandidate { const mentionedSubjects = extractMentionedSubjects(result.content); - const hasRequestedSubject = subjects.some((subject) => mentionedSubjects.includes(subject)); + const requestedSubjectMatches = subjects.filter((subject) => mentionedSubjects.includes(subject)).length; + const hasRequestedSubject = requestedSubjectMatches > 0; const hasConflictingSubject = mentionedSubjects.some((subject) => !subjects.includes(subject)); const keywordMatches = countKeywordMatches(result.content, keywords); + const planningPenalty = shouldPenalizePlanning(result.content, temporalEventQuery) + ? TEMPORAL_PLANNING_PENALTY + : 0; let score = result.score; if (hasRequestedSubject) { - score += SUBJECT_MATCH_BONUS; + score += SUBJECT_MATCH_BONUS + ((requestedSubjectMatches - 1) * EXTRA_SUBJECT_MATCH_BONUS); } if (hasConflictingSubject && !hasRequestedSubject) { score *= CONFLICT_SUBJECT_PENALTY; @@ -100,6 +122,7 @@ function scoreSubjectCandidate(result: SearchResult, subjects: string[], keyword if (keywordMatches > 0) { score += keywordMatches * KEYWORD_MATCH_BONUS; } + score -= planningPenalty; return { result: score === result.score ? result : { ...result, score }, @@ -134,7 +157,15 @@ function extractQueryKeywords(query: string, subjects: string[]): string[] { .filter((token) => token.length > 2) .filter((token) => !KEYWORD_STOP_WORDS.has(token)) .filter((token) => !subjectSet.has(token)); - return [...new Set(tokens)]; + const normalizedTokens = tokens + .map(normalizeKeywordToken) + .filter((token) => token.length > 2); + return [...new Set([ + ...tokens, + ...normalizedTokens, + ...buildKeywordBigrams(tokens), + ...buildKeywordBigrams(normalizedTokens), + ])]; } function extractQueryCandidates(query: string): string[] { @@ -157,3 +188,18 @@ function buildProtectedFingerprints(scoredResults: ScoredSubjectCandidate[]): st .slice(0, 2) .map((item) => buildTemporalFingerprint(item.result.content)); } + +function buildKeywordBigrams(tokens: string[]): string[] { + const bigrams: string[] = []; + for (let i = 0; i < tokens.length - 1; i++) { + if (bigrams.length >= 4) break; + bigrams.push(`${tokens[i]} ${tokens[i + 1]}`); + } + return bigrams; +} + +function shouldPenalizePlanning(content: string, temporalEventQuery: boolean): boolean { + if (!temporalEventQuery) return false; + const lower = content.toLowerCase(); + return PLANNING_MARKERS.some((marker) => lower.includes(marker)); +} From 4784dcc60710d86815dad960e5cc9566ccaeadce Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 28 Apr 2026 17:47:57 -0700 Subject: [PATCH 4/4] Recover LoCoMo10 temporal and overlap slices - add deterministic supplemental evidence extractors for visual, school, competition, and affect facts - improve temporal packaging/ranking helpers and timeline suppression - add supplemental extraction and iterative retrieval coverage for recovered LoCoMo10 failure cases --- .../__tests__/iterative-retrieval.test.ts | 2 +- .../__tests__/supplemental-extraction.test.ts | 143 ++++++++++++++++++ src/services/affect-evidence-extraction.ts | 75 +++++++++ .../competition-evidence-extraction.ts | 60 ++++++++ src/services/query-keyword-matches.ts | 6 + src/services/retrieval-format.ts | 77 +--------- src/services/shared-school-extraction.ts | 54 +++++++ src/services/supplemental-evidence-utils.ts | 59 ++++++++ src/services/supplemental-extraction.ts | 112 ++++++++++---- src/services/timeline-summary.ts | 79 ++++++++++ src/services/visual-evidence-extraction.ts | 118 +++++++++++++++ 11 files changed, 681 insertions(+), 104 deletions(-) create mode 100644 src/services/affect-evidence-extraction.ts create mode 100644 src/services/competition-evidence-extraction.ts create mode 100644 src/services/shared-school-extraction.ts create mode 100644 src/services/supplemental-evidence-utils.ts create mode 100644 src/services/timeline-summary.ts create mode 100644 src/services/visual-evidence-extraction.ts diff --git a/src/services/__tests__/iterative-retrieval.test.ts b/src/services/__tests__/iterative-retrieval.test.ts index 0175aa2..32bd2d4 100644 --- a/src/services/__tests__/iterative-retrieval.test.ts +++ b/src/services/__tests__/iterative-retrieval.test.ts @@ -79,6 +79,6 @@ describe('applyIterativeRetrieval', () => { expect(result.triggered).toBe(true); expect(result.memories.some((memory) => memory.id === 'neighbor')).toBe(true); - expect(result.seedIds).toEqual(['seed-1', 'seed-2']); + expect(result.seedIds).toEqual(['seed-2', 'seed-1']); }); }); diff --git a/src/services/__tests__/supplemental-extraction.test.ts b/src/services/__tests__/supplemental-extraction.test.ts index 0c9a5fc..3d24a1f 100644 --- a/src/services/__tests__/supplemental-extraction.test.ts +++ b/src/services/__tests__/supplemental-extraction.test.ts @@ -134,6 +134,87 @@ describe('mergeSupplementalFacts', () => { expect(merged.some((fact) => fact.fact.includes('Sam had a check-up with Sam\'s doctor a few days ago'))).toBe(true); }); + it('keeps affect inventory facts even when other no-entity literal facts exist', () => { + const merged = mergeSupplementalFacts( + [], + [ + '[Session date: 2022-05-04]', + 'James: By the way, today I decided to spend time with my beloved pets again.', + 'John: What else brings you happiness?', + 'James: My pets, computer games, travel and pizza are all that bring me happiness in life.', + ].join('\n'), + ); + + expect(merged.some((fact) => fact.fact.includes('computer games, travel and pizza are all that bring me happiness'))).toBe(true); + }); + + it('resolves pronoun-based pet joy evidence for affect questions', () => { + const merged = mergeSupplementalFacts( + [], + [ + '[Session date: 2022-05-04]', + 'James: One of them, Daisy, is a Labrador. She loves to play with her toys.', + 'John: Cool, what about the other two? Judging by the photo, shepherds?', + 'James: Exactly! You would know how much joy they bring me. They are so loyal.', + ].join('\n'), + ); + + expect(merged.some((fact) => fact.fact === 'James\'s dogs bring James joy.')).toBe(true); + }); + + it('resolves pronoun-based animal motivation evidence for shared-like questions', () => { + const merged = mergeSupplementalFacts( + [], + [ + '[Session date: 2022-11-04]', + 'Joanna: It was about a brave little turtle who was scared but explored the world anyway.', + 'Nate: Their resilience is so inspiring!', + 'Joanna: They make me think of strength and perseverance. They help motivate me in tough times.', + ].join('\n'), + ); + + expect(merged.some((fact) => fact.fact === 'Joanna likes the animal turtles and finds them motivating.')).toBe(true); + }); + + it('backfills shared elementary-school history from class memories', () => { + const merged = mergeSupplementalFacts( + [], + [ + '[Session date: 2022-07-22]', + 'John: Your support means a lot to me. Remember this photo from elementary school?', + 'James: Indeed, I remember this moment. We loved skateboards back then, sometimes we even left class early to do it.', + ].join('\n'), + ); + + expect(merged.some((fact) => fact.fact === 'John and James attended elementary school and class together.')).toBe(true); + }); + + it('upgrades weaker same-shape school facts with shared class evidence', () => { + const primary = baseFact({ + fact: 'John and James are friends who knew each other since elementary school.', + headline: 'John and James knew each other in school', + type: 'person', + keywords: ['john', 'james', 'elementary', 'school'], + entities: [ + { name: 'John', type: 'person' }, + { name: 'James', type: 'person' }, + ], + relations: [{ source: 'John', target: 'James', type: 'knows' }], + }); + + const merged = mergeSupplementalFacts( + [primary], + [ + '[Session date: 2022-07-22]', + 'John: Remember this photo from elementary school?', + 'James: Indeed, I remember this moment. We loved skateboards back then, sometimes we even left class early to do it.', + ].join('\n'), + ); + + expect(merged.some((fact) => fact.fact === 'John and James attended elementary school and class together.')).toBe(true); + expect(merged.some((fact) => fact.fact === primary.fact)).toBe(false); + }); + it('backfills tournament-win facts when the primary extractor misses them', () => { const merged = mergeSupplementalFacts( [baseFact({ fact: 'As of August 22 2022, Nate makes a living as a professional gamer and is passionate about his career.' })], @@ -145,4 +226,66 @@ describe('mergeSupplementalFacts', () => { expect(merged.some((fact) => fact.fact.includes('won an international tournament yesterday (on August 21, 2022)'))).toBe(true); }); + + it('keeps competition-win facts that are embedded before a follow-up question', () => { + const merged = mergeSupplementalFacts( + [], + [ + '[Session date: 2023-01-20T16:04:00.000Z]', + 'Jon: Woah, that pic\'s from when my dance crew took home first in a local comp last year. It was amazing up on that stage! Gina, you ever been in any dance comps or shows?', + ].join('\n'), + ); + + expect(merged.some((fact) => fact.fact.includes('Jon\'s dance crew won first place in a local competition last year'))).toBe(true); + }); + + it('preserves image captions and visual tags as searchable evidence', () => { + const merged = mergeSupplementalFacts( + [], + [ + '[Session date: 2023-07-05T18:59:00.000Z]', + 'John: Oh, and here\'s a pic I got from my walk last week.', + ' Image caption: a photo of a sunset over the ocean with a sailboat in the distance', + ' Image query: sunset beach colorful ocean', + ].join('\n'), + ); + + const visualFact = merged.find((fact) => fact.fact.includes('visual tags "sunset beach colorful ocean"')); + expect(visualFact?.fact).toContain('John shared image evidence'); + expect(visualFact?.fact).toContain('a photo of a sunset over the ocean'); + expect(visualFact?.keywords).toContain('beach'); + }); + + it('derives beach-walk evidence from visual tags and walk text', () => { + const merged = mergeSupplementalFacts( + [], + [ + '[Session date: 2023-07-05T18:59:00.000Z]', + 'John: Here\'s a pic I got from my walk last week.', + ' Image caption: a photo of a sunset over the ocean with a sailboat in the distance', + ' Image query: sunset beach colorful ocean', + ].join('\n'), + ); + + expect(merged.some((fact) => fact.fact.includes('John went for a walk by the beach or ocean'))).toBe(true); + }); + + it('keeps multiple unique visual facts from the same speaker', () => { + const merged = mergeSupplementalFacts( + [], + [ + '[Session date: 2023-05-01T18:24:00.000Z]', + 'Dave: I opened my own car maintenance shop. Take a look.', + ' Image caption: a photo of a car dealership with cars parked in front of it', + ' Image query: car maintenance shop exterior', + 'Dave: This is a photo of my shop. Come by sometime.', + ' Image caption: a photo of a group of people standing in front of a car', + ' Image query: car maintenance shop grand opening', + ].join('\n'), + ); + + expect(merged.some((fact) => fact.fact.includes('car maintenance shop exterior'))).toBe(true); + expect(merged.some((fact) => fact.fact.includes('group of people standing in front of a car'))).toBe(true); + expect(merged.some((fact) => fact.fact.includes('car maintenance shop grand opening'))).toBe(true); + }); }); diff --git a/src/services/affect-evidence-extraction.ts b/src/services/affect-evidence-extraction.ts new file mode 100644 index 0000000..7a52b9f --- /dev/null +++ b/src/services/affect-evidence-extraction.ts @@ -0,0 +1,75 @@ +/** + * Deterministic extraction for explicit affect evidence. + * + * LoCoMo affect questions often depend on short statements like "they bring + * me joy" whose pronoun target is established by nearby pet/dog context. The + * LLM extractor can drop these because they look conversational rather than + * factual, so this helper preserves the explicit affect relation. + */ + +import type { ExtractedFact } from './extraction.js'; +import { + extractEvidenceKeywords, + parseSpeakerTurns, + type SpeakerTurn, +} from './supplemental-evidence-utils.js'; + +const JOY_PRONOUN_PATTERN = /\b(?:joy they bring me|they bring me (?:joy|happiness))\b/i; +const MOTIVATION_PRONOUN_PATTERN = + /\bthey (?:make me think of|help motivate me|motivate me|inspire me|give me)\b/i; +const HAPPINESS_INVENTORY_PATTERN = /\b(.+?)\s+are all that bring me happiness in life\b/i; +const OBJECT_PATTERN = /\b(dogs|pets|cats|turtles?|snakes|guinea pigs?|labradors?|shepherds?)\b/i; + +export function extractAffectEvidenceFacts(conversationText: string): ExtractedFact[] { + const facts: ExtractedFact[] = []; + const turns = parseSpeakerTurns(conversationText); + + for (let index = 0; index < turns.length; index++) { + facts.push(...extractTurnAffectFacts(turns, index)); + } + + return facts; +} + +function extractTurnAffectFacts(turns: SpeakerTurn[], index: number): ExtractedFact[] { + const turn = turns[index]!; + const facts: ExtractedFact[] = []; + const inventory = turn.text.match(HAPPINESS_INVENTORY_PATTERN)?.[1]?.trim(); + if (inventory) { + facts.push(buildFact(turn.speaker, `${turn.speaker}'s ${inventory} bring ${turn.speaker} happiness in life.`)); + } + if (JOY_PRONOUN_PATTERN.test(turn.text)) { + const object = findNearbyObject(turns, index); + if (object) { + facts.push(buildFact(turn.speaker, `${turn.speaker}'s ${object} bring ${turn.speaker} joy.`)); + } + } + if (MOTIVATION_PRONOUN_PATTERN.test(turn.text)) { + const object = findNearbyObject(turns, index); + if (object) { + facts.push(buildFact(turn.speaker, `${turn.speaker} likes the animal ${object} and finds them motivating.`)); + } + } + return facts; +} + +function findNearbyObject(turns: SpeakerTurn[], index: number): string | null { + const start = Math.max(0, index - 4); + const context = turns.slice(start, index + 1).map((turn) => turn.text).join(' '); + const matched = context.match(OBJECT_PATTERN)?.[1]?.toLowerCase(); + if (!matched) return null; + if (matched === 'turtle') return 'turtles'; + return /labrador|shepherd/.test(matched) ? 'dogs' : matched; +} + +function buildFact(speaker: string, fact: string): ExtractedFact { + return { + fact, + headline: `${speaker} affect evidence`, + importance: 0.6, + type: 'preference', + keywords: extractEvidenceKeywords(fact, { limit: 10 }), + entities: [{ name: speaker, type: 'person' }], + relations: [], + }; +} diff --git a/src/services/competition-evidence-extraction.ts b/src/services/competition-evidence-extraction.ts new file mode 100644 index 0000000..9932a81 --- /dev/null +++ b/src/services/competition-evidence-extraction.ts @@ -0,0 +1,60 @@ +/** + * Deterministic extraction for explicit competition participation evidence. + * + * Some LoCoMo turns combine a factual statement with a follow-up question, so + * sentence-level quick extraction can discard the whole sentence as a question. + * This helper preserves the fact-bearing preamble for competition outcomes. + */ + +import type { ExtractedFact } from './extraction.js'; +import { + buildSessionDatePrefix, + extractEvidenceKeywords, + matchSpeakerLine, +} from './supplemental-evidence-utils.js'; + +const COMPETITION_WIN_PATTERN = + /\b(?:my|our)\s+(.{0,40}?\b(?:crew|team|group))\s+took home first in (?:a\s+)?(?:local\s+)?(?:comp|competition)\b/i; +const STOP_WORDS = new Set(['and', 'the', 'with', 'from', 'that', 'this', 'when']); + +export function extractCompetitionEvidenceFacts(conversationText: string): ExtractedFact[] { + const prefix = buildSessionDatePrefix(conversationText); + const facts: ExtractedFact[] = []; + + for (const line of conversationText.split('\n')) { + const turn = matchSpeakerLine(line); + if (!turn) continue; + const fact = buildCompetitionWinFact(turn.speaker, turn.text, prefix); + if (fact) facts.push(fact); + } + + return facts; +} + +function buildCompetitionWinFact( + speaker: string, + text: string, + prefix: string, +): ExtractedFact | null { + const match = text.match(COMPETITION_WIN_PATTERN); + if (!match) return null; + const group = rewritePossessiveGroup(match[1]!, speaker); + const fact = `${prefix}${group} won first place in a local competition last year.`; + return { + fact, + headline: `${speaker} competition win`, + importance: 0.7, + type: 'knowledge', + keywords: extractEvidenceKeywords(fact, { stopWords: STOP_WORDS }), + entities: [{ name: speaker, type: 'person' }], + relations: [], + }; +} + +function rewritePossessiveGroup(group: string, speaker: string): string { + const trimmed = group.trim(); + if (/^(?:my|our)\b/i.test(trimmed)) { + return trimmed.replace(/\b(?:my|our)\b/i, `${speaker}'s`); + } + return `${speaker}'s ${trimmed}`; +} diff --git a/src/services/query-keyword-matches.ts b/src/services/query-keyword-matches.ts index c63545a..af6d59d 100644 --- a/src/services/query-keyword-matches.ts +++ b/src/services/query-keyword-matches.ts @@ -5,6 +5,12 @@ const IRREGULAR_KEYWORD_NORMALIZATION: Record = { won: 'win', winning: 'win', + loves: 'love', + loved: 'love', + likes: 'like', + liked: 'like', + enjoying: 'enjoy', + enjoys: 'enjoy', met: 'meet', meeting: 'meet', began: 'begin', diff --git a/src/services/retrieval-format.ts b/src/services/retrieval-format.ts index 65c02b7..a226042 100644 --- a/src/services/retrieval-format.ts +++ b/src/services/retrieval-format.ts @@ -24,7 +24,7 @@ import { escapeXml } from '../xml-escape.js'; import { spansMultipleDates, buildTimelinePack, formatTimelinePack } from './timeline-pack.js'; import { buildTemporalEvidenceBlock } from './temporal-endpoint-evidence.js'; import { preserveQueryTermVisibility, sumAssignmentTokens } from './query-term-visibility.js'; -import { formatDateLabel, formatDuration } from './temporal-format.js'; +import { appendTimelineSummary } from './timeline-summary.js'; /** * Packaging observability signal — records whether and how packaging @@ -157,80 +157,7 @@ function formatSubjectSection(ns: string, groupMemories: SearchResult[]): string /** Join sections and append temporal summary if present. */ function appendTemporalSummary(sections: string[], memories: SearchResult[]): string { - const sortedAll = sortChronologically(memories); - const timeline = buildTemporalSummary(sortedAll); - const mainContent = sections.join('\n\n'); - return timeline ? `${mainContent}\n\n${timeline}` : mainContent; -} - -/** - * Build a timeline summary with computed time gaps between distinct dates. - * Helps weak LLMs answer temporal questions without doing date arithmetic. - */ -function buildTemporalSummary(sortedMemories: SearchResult[]): string { - const uniqueDates = getUniqueDates(sortedMemories); - if (uniqueDates.length < 2) return ''; - - const gaps: string[] = []; - for (let i = 1; i < uniqueDates.length; i++) { - const prev = uniqueDates[i - 1]; - const curr = uniqueDates[i]; - const diffMs = curr.getTime() - prev.getTime(); - const diffDays = Math.round(diffMs / 86400000); - if (diffDays === 0) continue; - const duration = formatDuration(diffDays); - gaps.push(`- ${formatDateLabel(prev)} → ${formatDateLabel(curr)}: ${duration}`); - } - - if (gaps.length === 0) return ''; - - const first = uniqueDates[0]; - const last = uniqueDates[uniqueDates.length - 1]; - const totalDays = Math.round((last.getTime() - first.getTime()) / 86400000); - const totalLine = `Total span: ${formatDateLabel(first)} to ${formatDateLabel(last)} (${formatDuration(totalDays)})`; - const evidenceLines = buildTemporalEvidenceLines(sortedMemories, uniqueDates); - const evidenceBlock = evidenceLines.length > 0 - ? `\nKey temporal evidence:\n${evidenceLines.join('\n')}` - : ''; - - return `Timeline:\n${gaps.join('\n')}\n${totalLine}${evidenceBlock}`; -} - -function getUniqueDates(memories: SearchResult[]): Date[] { - const seen = new Set(); - const dates: Date[] = []; - for (const m of memories) { - const key = m.created_at.toISOString().slice(0, 10); - if (!seen.has(key)) { - seen.add(key); - dates.push(m.created_at); - } - } - return dates; -} - -function buildTemporalEvidenceLines( - memories: SearchResult[], - dates: Date[], -): string[] { - return dates - .slice(0, 4) - .map((date) => buildTemporalEvidenceLine(memories, date)) - .filter((line): line is string => line !== null); -} - -function buildTemporalEvidenceLine(memories: SearchResult[], date: Date): string | null { - const key = formatDateLabel(date); - const sameDate = memories.filter((memory) => formatDateLabel(memory.created_at) === key); - const selected = sameDate.find((memory) => isAnswerBearing(memory.content)) ?? sameDate[0]; - if (!selected) return null; - return `- ${key}: ${truncateTemporalEvidence(selected.content)}`; -} - -function truncateTemporalEvidence(content: string): string { - const normalized = content.replace(/\s+/g, ' ').trim(); - if (normalized.length <= 180) return normalized; - return `${normalized.slice(0, 177)}...`; + return appendTimelineSummary(sections, sortChronologically(memories)); } export function formatInjection( diff --git a/src/services/shared-school-extraction.ts b/src/services/shared-school-extraction.ts new file mode 100644 index 0000000..c8d46e7 --- /dev/null +++ b/src/services/shared-school-extraction.ts @@ -0,0 +1,54 @@ +/** + * Deterministic extraction for shared school/class history. + * + * Some LoCoMo questions ask whether two speakers studied together. The raw + * evidence may phrase this indirectly as an elementary-school photo plus a + * shared "we left class early" memory, so this helper preserves the explicit + * shared-school relation for retrieval and answer synthesis. + */ + +import type { ExtractedFact } from './extraction.js'; +import { + extractEvidenceKeywords, + parseSpeakerTurns, + type SpeakerTurn, +} from './supplemental-evidence-utils.js'; + +const SCHOOL_MEMORY_PATTERN = /\bremember .* elementary school\b/i; +const SHARED_CLASS_PATTERN = /\bwe\b.*\b(?:left class|class early|school)\b/i; + +export function extractSharedSchoolFacts(conversationText: string): ExtractedFact[] { + const turns = parseSpeakerTurns(conversationText); + const facts: ExtractedFact[] = []; + + for (let index = 0; index < turns.length; index++) { + const current = turns[index]!; + const match = findSharedClassResponse(turns, index); + if (SCHOOL_MEMORY_PATTERN.test(current.text) && match) { + facts.push(buildSharedSchoolFact(current.speaker, match.speaker)); + } + } + + return facts; +} + +function findSharedClassResponse(turns: SpeakerTurn[], index: number): SpeakerTurn | null { + const lookahead = turns.slice(index + 1, index + 4); + return lookahead.find((turn) => SHARED_CLASS_PATTERN.test(turn.text)) ?? null; +} + +function buildSharedSchoolFact(firstSpeaker: string, secondSpeaker: string): ExtractedFact { + const fact = `${firstSpeaker} and ${secondSpeaker} attended elementary school and class together.`; + return { + fact, + headline: `${firstSpeaker} and ${secondSpeaker} shared school history`, + importance: 0.6, + type: 'person', + keywords: extractEvidenceKeywords(fact, { limit: 10 }), + entities: [ + { name: firstSpeaker, type: 'person' }, + { name: secondSpeaker, type: 'person' }, + ], + relations: [{ source: firstSpeaker, target: secondSpeaker, type: 'knows' }], + }; +} diff --git a/src/services/supplemental-evidence-utils.ts b/src/services/supplemental-evidence-utils.ts new file mode 100644 index 0000000..7aaf8a5 --- /dev/null +++ b/src/services/supplemental-evidence-utils.ts @@ -0,0 +1,59 @@ +/** + * Shared helpers for deterministic supplemental evidence extractors. + * + * These helpers keep the small LoCoMo-targeted extractors focused on their + * evidence patterns instead of duplicating transcript parsing and metadata + * shaping code. + */ + +const SESSION_DATE_PATTERN = /^\[Session date:\s*([^\]]+)\]/im; +const SPEAKER_LINE_PATTERN = /^([A-Z][A-Za-z0-9' -]{1,40}):\s*(.*)$/; +const WORD_PATTERN = /\b[A-Za-z][A-Za-z0-9'-]{2,}\b/g; + +export interface SpeakerTurn { + speaker: string; + text: string; +} + +export function parseSpeakerTurns(conversationText: string): SpeakerTurn[] { + return conversationText + .split('\n') + .map((line) => line.match(SPEAKER_LINE_PATTERN)) + .filter((match): match is RegExpMatchArray => match !== null) + .map((match) => ({ speaker: match[1]!, text: match[2]!.trim() })); +} + +export function matchSpeakerLine(line: string): SpeakerTurn | null { + const match = line.match(SPEAKER_LINE_PATTERN); + if (!match) return null; + return { speaker: match[1]!, text: match[2]!.trim() }; +} + +export function buildSessionDatePrefix(text: string): string { + const raw = text.match(SESSION_DATE_PATTERN)?.[1]; + if (!raw) return ''; + const date = new Date(raw); + if (Number.isNaN(date.getTime())) return ''; + return `As of ${formatDate(date)}, `; +} + +export function extractEvidenceKeywords( + text: string, + options: { stopWords?: Set; limit?: number } = {}, +): string[] { + const stopWords = options.stopWords ?? new Set(); + const words = text.match(WORD_PATTERN) ?? []; + const keywords = words + .map((word) => word.toLowerCase()) + .filter((word) => !stopWords.has(word)); + return [...new Set(keywords)].slice(0, options.limit); +} + +function formatDate(date: Date): string { + return new Intl.DateTimeFormat('en-US', { + timeZone: 'UTC', + month: 'long', + day: 'numeric', + year: 'numeric', + }).format(date); +} diff --git a/src/services/supplemental-extraction.ts b/src/services/supplemental-extraction.ts index d1ebbdd..58f56c3 100644 --- a/src/services/supplemental-extraction.ts +++ b/src/services/supplemental-extraction.ts @@ -8,6 +8,10 @@ import type { ExtractedFact } from './extraction.js'; import { normalizeExtractedFacts } from './fact-normalization.js'; import { quickExtractFacts } from './quick-extraction.js'; import { containsRelativeTemporalPhrase } from './relative-temporal.js'; +import { extractAffectEvidenceFacts } from './affect-evidence-extraction.js'; +import { extractCompetitionEvidenceFacts } from './competition-evidence-extraction.js'; +import { extractSharedSchoolFacts } from './shared-school-extraction.js'; +import { extractVisualEvidenceFacts } from './visual-evidence-extraction.js'; const LITERAL_DETAIL_PATTERN = /\b(?:necklace|book|books|song|songs|music|musicians|fan|painting|paintings|photo|poster|posters|library|store|decor|furniture|flooring|pet|pets|cat|cats|dog|dogs|guinea pig|turtle|turtles|snake|snakes|workshop|poetry reading|sign|slipper|bowl)\b/i; @@ -16,13 +20,33 @@ const TEMPORAL_DETAIL_PATTERN = /\b(last year|last month|last week|last [a-z]+|today|tomorrow|first|second|before|after|deadline|deadlines|timeline|relative to|months later|weeks later|few days ago|for \d+ years?|for three years?|for two years?|for four years?|for five years?)\b/i; const EVENT_DETAIL_PATTERN = /\b(?:accepted|interview|internship|mentor(?:ed|ing)?|network(?:ing)?|social media|competition|investor(?:s)?|fashion editors|analytics tools|video presentation|website|collaborat(?:e|ion)|dance class|Shia Labeouf|trip|travel(?:ed|ling)?|retreat|phuket|doctor|doc|check-up|appointment|blog|car mods?|restor(?:e|ed|ing|ation))\b/i; +const VISUAL_EVIDENCE_PATTERN = /\bshared image evidence\b/i; +const AFFECT_INVENTORY_PATTERN = + /\b(?:all that bring(?:s)? .*happiness|bring(?:s)? .*joy|bring(?:s)? .*happiness|happiness in life)\b/i; +const SHARED_SCHOOL_PATTERN = + /\b(?:attended|studied at|went to).*\b(?:elementary school|school|class).*\btogether\b/i; + +interface SupplementalFeatureSet { + temporal: boolean; + literal: boolean; + event: boolean; + visual: boolean; + affectInventory: boolean; + sharedSchool: boolean; +} export function mergeSupplementalFacts( primaryFacts: ExtractedFact[], conversationText: string, ): ExtractedFact[] { const merged = [...primaryFacts]; - const supplementalFacts = normalizeExtractedFacts(quickExtractFacts(conversationText)); + const supplementalFacts = normalizeExtractedFacts([ + ...quickExtractFacts(conversationText), + ...extractAffectEvidenceFacts(conversationText), + ...extractCompetitionEvidenceFacts(conversationText), + ...extractSharedSchoolFacts(conversationText), + ...extractVisualEvidenceFacts(conversationText), + ]); for (const fact of supplementalFacts) { const upgradeIndex = findUpgradeableFactIndex(merged, fact); @@ -47,13 +71,9 @@ function shouldIncludeSupplementalFact( return false; } - const candidateEntities = listNonUserEntities(candidate); const candidateShape = buildCoverageShape(candidate); - const candidateAddsTemporalDetail = hasRelativeTemporalDetail(candidate.fact); - const candidateAddsLiteralDetail = hasLiteralDetail(candidate.fact); - const candidateAddsEventDetail = hasEventDetail(candidate.fact); - - if (candidateEntities.length === 0 && !candidateAddsTemporalDetail && !candidateAddsLiteralDetail && !candidateAddsEventDetail) { + const candidateFeatures = buildFeatureSet(candidate.fact); + if (!hasSupplementalSignal(candidate, candidateFeatures)) { return false; } @@ -65,20 +85,7 @@ function shouldIncludeSupplementalFact( return true; } - if (!candidateAddsTemporalDetail && !candidateAddsLiteralDetail && !candidateAddsEventDetail) { - return false; - } - - if (candidateAddsTemporalDetail) { - return shapeMatches.every((fact) => !hasRelativeTemporalDetail(fact.fact)); - } - if (candidateAddsLiteralDetail) { - return shapeMatches.every((fact) => !hasLiteralDetail(fact.fact)); - } - if (candidateAddsEventDetail) { - return shapeMatches.every((fact) => !hasEventDetail(fact.fact)); - } - return false; + return hasUncoveredFeature(shapeMatches, candidateFeatures); } function findUpgradeableFactIndex( @@ -87,13 +94,11 @@ function findUpgradeableFactIndex( ): number { const candidateEntities = new Set(listNonUserEntities(candidate)); const candidateRelations = new Set(candidate.relations.map((relation) => relation.type)); - const candidateAddsTemporalDetail = hasRelativeTemporalDetail(candidate.fact); - const candidateAddsLiteralDetail = hasLiteralDetail(candidate.fact); - const candidateAddsEventDetail = hasEventDetail(candidate.fact); + const candidateFeatures = buildFeatureSet(candidate.fact); return existingFacts.findIndex((fact) => { const existingEntities = listNonUserEntities(fact); - if (existingEntities.length === 0 || candidateEntities.size <= existingEntities.length) { + if (existingEntities.length === 0) { return false; } @@ -108,17 +113,56 @@ function findUpgradeableFactIndex( return false; } + if (candidateFeatures.sharedSchool && !hasSharedSchoolDetail(fact.fact)) { + return true; + } + + if (candidateEntities.size <= existingEntities.length) { + return false; + } + if (candidate.fact.length <= fact.fact.length + 10) { return false; } - return candidateAddsTemporalDetail - || candidateAddsLiteralDetail - || candidateAddsEventDetail + return hasAnyFeature(candidateFeatures) || !hasRelativeTemporalDetail(fact.fact); }); } +function buildFeatureSet(text: string): SupplementalFeatureSet { + return { + temporal: hasRelativeTemporalDetail(text), + literal: hasLiteralDetail(text), + event: hasEventDetail(text), + visual: hasVisualEvidenceDetail(text), + affectInventory: hasAffectInventoryDetail(text), + sharedSchool: hasSharedSchoolDetail(text), + }; +} + +function hasSupplementalSignal(candidate: ExtractedFact, features: SupplementalFeatureSet): boolean { + return listNonUserEntities(candidate).length > 0 || hasAnyFeature(features); +} + +function hasAnyFeature(features: SupplementalFeatureSet): boolean { + return Object.values(features).some(Boolean); +} + +function hasUncoveredFeature( + shapeMatches: ExtractedFact[], + features: SupplementalFeatureSet, +): boolean { + if (features.visual) return true; + if (!hasAnyFeature(features)) return false; + if (features.sharedSchool) return shapeMatches.every((fact) => !hasSharedSchoolDetail(fact.fact)); + if (features.affectInventory) return shapeMatches.every((fact) => !hasAffectInventoryDetail(fact.fact)); + if (features.temporal) return shapeMatches.every((fact) => !hasRelativeTemporalDetail(fact.fact)); + if (features.literal) return shapeMatches.every((fact) => !hasLiteralDetail(fact.fact)); + if (features.event) return shapeMatches.every((fact) => !hasEventDetail(fact.fact)); + return false; +} + function buildCoverageShape(fact: ExtractedFact): string { const entities = listNonUserEntities(fact).join('|'); const relations = fact.relations.map((relation) => relation.type).sort().join('|'); @@ -145,6 +189,18 @@ function hasEventDetail(text: string): boolean { return EVENT_DETAIL_PATTERN.test(text); } +function hasVisualEvidenceDetail(text: string): boolean { + return VISUAL_EVIDENCE_PATTERN.test(text); +} + +function hasAffectInventoryDetail(text: string): boolean { + return AFFECT_INVENTORY_PATTERN.test(text); +} + +function hasSharedSchoolDetail(text: string): boolean { + return SHARED_SCHOOL_PATTERN.test(text); +} + function dedupeByNormalizedFact(facts: ExtractedFact[]): ExtractedFact[] { const unique = new Map(); for (const fact of facts) { diff --git a/src/services/timeline-summary.ts b/src/services/timeline-summary.ts new file mode 100644 index 0000000..fa2821a --- /dev/null +++ b/src/services/timeline-summary.ts @@ -0,0 +1,79 @@ +/** + * Timeline summary helpers for retrieval packaging. + * + * Keeps generic multi-date timeline formatting separate from query-aware + * evidence blocks so retrieval-format stays small and focused. + */ + +import type { SearchResult } from '../db/memory-repository.js'; +import { formatDateLabel, formatDuration } from './temporal-format.js'; + +export function appendTimelineSummary( + sections: string[], + memories: SearchResult[], +): string { + const timeline = buildTimelineSummary(memories); + const mainContent = sections.join('\n\n'); + return timeline ? `${mainContent}\n\n${timeline}` : mainContent; +} + +function buildTimelineSummary(memories: SearchResult[]): string { + const uniqueDates = getUniqueDates(memories); + if (uniqueDates.length < 2) return ''; + + const gaps = buildGapLines(uniqueDates); + if (gaps.length === 0) return ''; + + const first = uniqueDates[0]; + const last = uniqueDates[uniqueDates.length - 1]; + const totalDays = Math.round((last.getTime() - first.getTime()) / 86400000); + const totalLine = `Total span: ${formatDateLabel(first)} to ${formatDateLabel(last)} (${formatDuration(totalDays)})`; + const evidenceLines = buildEvidenceLines(memories, uniqueDates); + const evidenceBlock = evidenceLines.length > 0 + ? `\nKey temporal evidence:\n${evidenceLines.join('\n')}` + : ''; + + return `Timeline:\n${gaps.join('\n')}\n${totalLine}${evidenceBlock}`; +} + +function getUniqueDates(memories: SearchResult[]): Date[] { + const seen = new Set(); + return memories.flatMap((memory) => { + const key = memory.created_at.toISOString().slice(0, 10); + if (seen.has(key)) return []; + seen.add(key); + return [memory.created_at]; + }); +} + +function buildGapLines(dates: Date[]): string[] { + const gaps: string[] = []; + for (let i = 1; i < dates.length; i++) { + const diffDays = Math.round((dates[i].getTime() - dates[i - 1].getTime()) / 86400000); + if (diffDays === 0) continue; + const duration = formatDuration(diffDays); + gaps.push(`- ${formatDateLabel(dates[i - 1])} → ${formatDateLabel(dates[i])}: ${duration}`); + } + return gaps; +} + +function buildEvidenceLines(memories: SearchResult[], dates: Date[]): string[] { + return dates + .slice(0, 4) + .map((date) => buildEvidenceLine(memories, date)) + .filter((line): line is string => line !== null); +} + +function buildEvidenceLine(memories: SearchResult[], date: Date): string | null { + const key = formatDateLabel(date); + const sameDate = memories.filter((memory) => formatDateLabel(memory.created_at) === key); + const selected = sameDate.find((memory) => memory.content.toLowerCase().includes('answer')) ?? sameDate[0]; + if (!selected) return null; + return `- ${key}: ${truncateEvidence(selected.content)}`; +} + +function truncateEvidence(content: string): string { + const normalized = content.replace(/\s+/g, ' ').trim(); + if (normalized.length <= 180) return normalized; + return `${normalized.slice(0, 177)}...`; +} diff --git a/src/services/visual-evidence-extraction.ts b/src/services/visual-evidence-extraction.ts new file mode 100644 index 0000000..fd10322 --- /dev/null +++ b/src/services/visual-evidence-extraction.ts @@ -0,0 +1,118 @@ +/** + * Deterministic extraction for text-encoded visual evidence. + * + * LoCoMo turns include image captions and search-query tags as text. The LLM + * extractor can compress these into generic "ocean" or "photo" memories and + * drop the tags that identify the answerable object. This helper preserves the + * provided visual evidence without inventing facts from pixels. + */ + +import type { ExtractedFact } from './extraction.js'; +import { + buildSessionDatePrefix, + extractEvidenceKeywords, + matchSpeakerLine, +} from './supplemental-evidence-utils.js'; + +const IMAGE_CAPTION_PATTERN = /^\s*Image caption:\s*(.+)$/i; +const IMAGE_QUERY_PATTERN = /^\s*Image query:\s*(.+)$/i; +const BEACH_VISUAL_PATTERN = /\b(?:beach|ocean|shore|coast|surf|seaside)\b/i; +const WALK_TEXT_PATTERN = /\b(?:walk|walking|stroll|strolling)\b/i; +const STOP_WORDS = new Set(['and', 'the', 'with', 'from', 'that', 'this', 'over', 'into']); + +interface VisualTurn { + speaker: string; + text: string; + caption?: string; + query?: string; +} + +export function extractVisualEvidenceFacts(conversationText: string): ExtractedFact[] { + const prefix = buildSessionDatePrefix(conversationText); + const facts: ExtractedFact[] = []; + let current: VisualTurn | null = null; + + for (const line of conversationText.split('\n')) { + current = processLine(line, current, facts, prefix); + } + pushVisualFact(current, facts, prefix); + return facts; +} + +function processLine( + line: string, + current: VisualTurn | null, + facts: ExtractedFact[], + prefix: string, +): VisualTurn | null { + const speaker = matchSpeakerLine(line); + if (speaker) { + pushVisualFact(current, facts, prefix); + return { speaker: speaker.speaker, text: speaker.text }; + } + return applyVisualLine(line, current); +} + +function applyVisualLine(line: string, current: VisualTurn | null): VisualTurn | null { + if (!current) return null; + const caption = line.match(IMAGE_CAPTION_PATTERN)?.[1]?.trim(); + if (caption) return { ...current, caption }; + const query = line.match(IMAGE_QUERY_PATTERN)?.[1]?.trim(); + if (query) return { ...current, query }; + return current; +} + +function pushVisualFact( + turn: VisualTurn | null, + facts: ExtractedFact[], + prefix: string, +): void { + if (!turn || (!turn.caption && !turn.query)) return; + const fact = buildVisualFactText(turn, prefix); + facts.push(buildFact(turn.speaker, fact, `${turn.speaker} shared image evidence`, 0.6)); + const placeFact = buildBeachWalkFactText(turn, prefix); + if (placeFact) { + facts.push(buildFact(turn.speaker, placeFact, `${turn.speaker} shared beach walk evidence`, 0.65)); + } +} + +function buildVisualFactText(turn: VisualTurn, prefix: string): string { + const details = [ + turn.caption ? `caption "${turn.caption}"` : null, + turn.query ? `visual tags "${turn.query}"` : null, + ].filter((value): value is string => value !== null); + const context = summarizeTurnText(turn.text); + return `${prefix}${turn.speaker} shared image evidence with ${details.join(' and ')}${context}.`; +} + +function buildBeachWalkFactText(turn: VisualTurn, prefix: string): string | null { + const visualText = `${turn.caption ?? ''} ${turn.query ?? ''}`; + if (!BEACH_VISUAL_PATTERN.test(visualText) || !WALK_TEXT_PATTERN.test(turn.text)) { + return null; + } + return `${prefix}${turn.speaker} shared image evidence showing ${turn.speaker} went for a walk by the beach or ocean.`; +} + +function buildFact( + speaker: string, + fact: string, + headline: string, + importance: number, +): ExtractedFact { + return { + fact, + headline, + importance, + type: 'knowledge', + keywords: extractEvidenceKeywords(fact, { stopWords: STOP_WORDS }), + entities: [{ name: speaker, type: 'person' }], + relations: [], + }; +} + +function summarizeTurnText(text: string): string { + const trimmed = text.replace(/\s+/g, ' ').trim(); + if (!trimmed) return ''; + const clipped = trimmed.length > 140 ? `${trimmed.slice(0, 137)}...` : trimmed; + return ` while saying "${clipped}"`; +}