From 94ac1219dc986e797b867d99aa8a5526ea254d7a Mon Sep 17 00:00:00 2001 From: Bernhard Jackiewicz Date: Sun, 4 Oct 2026 14:38:36 +0200 Subject: [PATCH] fix(ui): say why the model sentence was left out The note said the sentence "named something the evidence does not show" also when it was dropped for a claim word such as "output" or "Array" that the source does not show. explanationSentence now returns the reason with the drop, the claim word or the unknown name as the sentence wrote it, and the notes say what was claimed or named and what it was checked against, in English and German. The K12 hand test accepts the named form. Signed-off-by: Bernhard Jackiewicz --- .../src/browser-ai/BrowserChatDock.test.tsx | 10 +-- graph-ui/src/browser-ai/BrowserChatDock.tsx | 14 +-- .../BrowserChatDock.wording.test.tsx | 90 +++++++++++++++++++ graph-ui/src/browser-ai/chat-wording.test.ts | 39 +++++++- .../browser-ai/explanation-response.test.ts | 2 +- .../src/browser-ai/explanation-response.ts | 19 +++- .../browser-ai/grounded-explanation.test.ts | 10 +-- graph-ui/src/browser-ai/strings.ts | 20 ++++- graph-ui/tools/handtest-fixes-chat.mjs | 2 +- 9 files changed, 179 insertions(+), 27 deletions(-) create mode 100644 graph-ui/src/browser-ai/BrowserChatDock.wording.test.tsx diff --git a/graph-ui/src/browser-ai/BrowserChatDock.test.tsx b/graph-ui/src/browser-ai/BrowserChatDock.test.tsx index bf926021d..b734c17a8 100644 --- a/graph-ui/src/browser-ai/BrowserChatDock.test.tsx +++ b/graph-ui/src/browser-ai/BrowserChatDock.test.tsx @@ -1429,8 +1429,8 @@ describe('grounded automatic explanations (K14, K7)', () => { await render({ ...props, proactive: true, proactiveSelection: jsonbAggEvidence(), readSource }); await click('Download & load'); await settle(); expect(runtime.chat).toHaveBeenCalledOnce(); expect(card().textContent).toContain('Selected: JSONBAgg (Class) in django/contrib/postgres/aggregates/general.py:50-54.'); - expect(card().textContent).not.toContain('flake8'); - expect(card().textContent).toContain("The model's sentence named something the evidence does not show and was left out."); + expect(card().textContent).not.toContain('checked by the flake8 linter'); + expect(card().textContent).toContain("The model's sentence named something that is in neither the source nor the facts (here: flake8) and was left out."); }); it('does not show a prompt capacity note on a card whose facts are listed in full', async () => { @@ -1502,8 +1502,8 @@ describe('grounded automatic explanations (K14, K7)', () => { await click('Download & load'); await settle(); await act(async () => [...card().querySelectorAll('button')].find(item => item.textContent === 'Ask the model')!.click()); await settle(); expect(runtime.chat).toHaveBeenCalledOnce(); - expect(card().textContent).not.toContain('flake8'); - expect(card().textContent).toContain("Read from the file. The model's text named something the file does not show and was left out."); + expect(card().textContent).not.toContain('configures flake8'); + expect(card().textContent).toContain("Read from the file. The model's text named something the file does not show (here: flake8) and was left out."); }); it('explains an Architecture area from readable facts without reading source', async () => { @@ -1562,7 +1562,7 @@ describe('general questions, prompts without a question and configuration files await type('was kansnt du mir über den code sagen'); await click('Send ↑'); expect(answer()).toContain('Ausgewählt: JSONBAgg (Class)'); expect(answer()).not.toContain('gibt eine Liste'); - expect(answer()).toContain('Der Satz des Modells nannte etwas, das die Fakten nicht zeigen, und wurde weggelassen.'); + expect(answer()).toContain('Der Satz des Modells behauptete etwas, das der Quelltext nicht zeigt (hier: „gibt … zurück“), und wurde weggelassen.'); }); it('lists only the facts without source and sends detail questions to the model as before (C5)', async () => { diff --git a/graph-ui/src/browser-ai/BrowserChatDock.tsx b/graph-ui/src/browser-ai/BrowserChatDock.tsx index 3b10ff2a0..1b6361da7 100644 --- a/graph-ui/src/browser-ai/BrowserChatDock.tsx +++ b/graph-ui/src/browser-ai/BrowserChatDock.tsx @@ -6,7 +6,7 @@ import { keepAgent, loadProgress, offerAgent, peekAgent, takeAgent, type AgentHa import { buildChatMessages, selectionLocation, snapshotAttachment, snapshotReaderContext, trimChatHistory, type BrowserChatAttachment, type BrowserChatContext, type BrowserChatReaderContext, type BrowserChatSource, type BrowserChatTurn } from './chat-model'; import { explanationInput, EXPLANATION_DELAY_MS, type ExplanationInput } from './proactive-selection'; import { prepareExplanationContext, selectionSummary, type PreparedExplanationContext } from './explanation-context'; -import { AUTO_INPUT_TOKENS, AUTO_OUTPUT_TOKENS, citedInterpretation, explanationMode, explanationSentence, namesNotIn, parseExplanationResponse, explanationMessages, formatExplanationEvidence } from './explanation-response'; +import { AUTO_INPUT_TOKENS, AUTO_OUTPUT_TOKENS, citedInterpretation, explanationMode, explanationSentence, namesNotIn, parseExplanationResponse, explanationMessages, formatExplanationEvidence, type DroppedReason } from './explanation-response'; import { carriedSource, selectionSubject, SYMBOL_SOURCE_LINES, sourceTargetOf, symbolSource, type SymbolSourceReader } from './symbol-source'; import { relationshipAnswer, relationshipSuggestion } from './relationship-answer'; import { clampTokenLimits, tokenLimitBounds, tokenLimitsFor, useAgentPreferences, type TokenLimits } from './agent-preferences'; @@ -73,10 +73,12 @@ const SOURCE_CACHE_SIZE = 32; /** The listed facts first, then the model's sentence if it stood the check, then who wrote what (K7). * The facts come from the indexed graph, or for an open workflow from the file itself (K12). The * answer to a general question says it in the language of that question (C5). */ -function groundedExplanation(summary: readonly string[], sentence: string | undefined, dropped: boolean, from: 'graph' | 'file' = 'graph', language: 'en' | 'de' = 'en'): string { +function groundedExplanation(summary: readonly string[], sentence: string | undefined, dropped: boolean | DroppedReason, from: 'graph' | 'file' = 'graph', language: 'en' | 'de' = 'en'): string { const words = groundedText[language]; - const note = from === 'file' ? sentence ? browserChatText.fileFactsAndSentence : dropped ? browserChatText.fileSentenceDropped : browserChatText.fileFactsOnly - : sentence ? words.factsAndSentence : dropped ? words.sentenceDropped : words.factsOnly; + // The note names what the left out sentence claimed or named, where the check says it (W5). + const reason = typeof dropped === 'object' ? dropped : undefined; + const note = from === 'file' ? sentence ? browserChatText.fileFactsAndSentence : dropped ? browserChatText.fileSentenceDropped(reason) : browserChatText.fileFactsOnly + : sentence ? words.factsAndSentence : dropped ? words.sentenceDropped(reason) : words.factsOnly; return [summary.map(line => `- ${line}`).join('\n'), ...sentence ? [sentence] : [], `_${note}_`].join('\n\n'); } @@ -439,7 +441,7 @@ export default function BrowserChatDock({ proactiveSelection, selectionScope = " // One sentence beside the facts (two for reader code), and none that names what the evidence lacks. const checked = result.status === 'generated' ? explanationSentence(result.markdown, packet, request.map(message => message.content).join('\n')) : {}; if (summary.length || (result.status === 'generated' && checked.dropped)) { - const markdown = summary.length ? groundedExplanation(summary, checked.sentence, checked.dropped !== undefined, from) : `_${browserChatText.explanationDropped}_`; + const markdown = summary.length ? groundedExplanation(summary, checked.sentence, checked.reason ?? checked.dropped !== undefined, from) : `_${browserChatText.explanationDropped(checked.reason)}_`; const complete: Explanation = { key: snapshot.key, label: snapshot.label, answer: markdown, status: 'complete', mode: 'interpretation', packet, grounded: summary.length > 0, ...shortened && checked.sentence ? { shortened, limit: { inputTokens: autoInput, outputTokens: autoOutput } } : {}, ...snapshot.evidence ? { evidence: snapshot.evidence } : {} }; remember(complete); setExplanation(complete); @@ -749,7 +751,7 @@ export default function BrowserChatDock({ proactiveSelection, selectionScope = " // One sentence beside the facts, checked like the sentence of the explanation card. const result = parseExplanationResponse(output, packet!); const checked = !stopRequested.current && result.status === 'generated' ? explanationSentence(result.markdown, packet!, request.map(message => message.content).join('\n')) : {}; - const answer = groundedExplanation(grounded.summary, checked.sentence, checked.dropped !== undefined, 'graph', language); + const answer = groundedExplanation(grounded.summary, checked.sentence, checked.reason ?? checked.dropped !== undefined, 'graph', language); setTurns(previous => previous.map(item => item.id === id ? { ...item, answer, status: stopRequested.current ? 'stopped' : 'complete' } : item)); return; } diff --git a/graph-ui/src/browser-ai/BrowserChatDock.wording.test.tsx b/graph-ui/src/browser-ai/BrowserChatDock.wording.test.tsx new file mode 100644 index 000000000..0a930752d --- /dev/null +++ b/graph-ui/src/browser-ai/BrowserChatDock.wording.test.tsx @@ -0,0 +1,90 @@ +// @vitest-environment jsdom +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import BrowserChatDock, { type BrowserChatDockProps, type BrowserChatReaderContext } from './BrowserChatDock'; +import type { BrowserAiProgress, BrowserChatMessage } from './browser-ai-runtime'; +import type { BrowserChatOptions } from './browser-ai-controller'; +import { jsonbAggEvidence } from './galaxy-evidence.fixture'; + +/* The chat dock shows the wording of the third review (W5 to W10) where it builds it. */ + +let container: HTMLDivElement; +let root: Root; +let renderedProps: BrowserChatDockProps; +beforeEach(() => { + (globalThis as unknown as Record).IS_REACT_ACT_ENVIRONMENT = true; + window.localStorage.clear(); + container = document.createElement('div'); document.body.appendChild(container); root = createRoot(container); +}); +afterEach(async () => { await act(async () => root.unmount()); container.remove(); vi.restoreAllMocks(); vi.useRealTimers(); }); + +const snippet = { source: 'class JSONBAgg(OrderableAggMixin, Aggregate):\n function = "JSONB_AGG"\n allow_distinct = True\n', + file_path: '/abs/django/contrib/postgres/aggregates/general.py', start_line: 50, end_line: 52, source_mode: 'full' }; +function fixture() { + const runtime = { + prepare: vi.fn(async (_progress: (value: BrowserAiProgress) => void, _options?: { cacheOnly?: boolean }) => {}), + explain: vi.fn(async () => 'Legacy'), + countTokens: vi.fn(async (_messages: readonly BrowserChatMessage[]) => 100), + chat: vi.fn(async (_messages: readonly BrowserChatMessage[], _onToken: (chunk: string) => void, _options?: BrowserChatOptions) => 'Adds the two values.'), + stop: vi.fn(), dispose: vi.fn(), + }; + const props = { proactive: false, open: true, onClose: vi.fn(), onAttachmentConsumed: vi.fn(), onAttachmentRemoved: vi.fn(), createRuntime: vi.fn(() => runtime), + removeCache: vi.fn(async () => {}), readSource: vi.fn(async () => snippet) }; + return { runtime, props }; +} +const reader = (text: string, path: string): BrowserChatReaderContext => ({ project: 'sample', path, status: 'ready', + source: { id: `reader-${path}`, text, path, kind: 'file', project: 'sample', startLine: 1, startColumn: 1, endLine: text.split('\n').length, endColumn: 1, sourceVersion: 'sha256:1' } }); +async function render(props: BrowserChatDockProps): Promise { renderedProps = props; await act(async () => root.render()); } +function button(label: string): HTMLButtonElement { + const target = [...document.body.querySelectorAll('button')].find(item => item.textContent === label || item.getAttribute('aria-label') === label); + expect(target, `button ${label}`).toBeDefined(); return target!; +} +async function load(): Promise { + await render({ ...renderedProps, settingsRequest: (renderedProps.settingsRequest ?? 0) + 1 }); + await act(async () => button('Download & load').click()); +} +async function ask(value: string): Promise { + const input = container.querySelector('textarea')!; + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value')!.set!.call(input, value); + input.dispatchEvent(new Event('input', { bubbles: true })); + }); + await act(async () => button('Send ↑').click()); +} +const last = () => [...container.querySelectorAll('.cbm-chat-turn')].at(-1)!; +const answer = () => last().querySelector('.cbm-chat-answer-text')?.textContent ?? ''; +const card = () => container.querySelector('[aria-label="Current selection explanation"]')?.textContent ?? ''; +const settle = async () => { await act(async () => { await vi.advanceTimersByTimeAsync(650); }); }; + +describe('the note under a left out model sentence (W5)', () => { + it('names the claim word of a German general question and that it was checked against the source', async () => { + const { props, runtime } = fixture(); + runtime.chat.mockResolvedValueOnce('JSON-BAgg ist eine Aggregation, die die Zeilen in einen JSON-Array umwandelt.'); + await render({ ...props, proactiveSelection: jsonbAggEvidence() }); await load(); + await ask('was macht diese klasse? sehr kurze antwort'); + expect(answer()).not.toContain('JSON-Array'); + expect(answer()).toContain('Der Satz des Modells behauptete etwas, das der Quelltext nicht zeigt (hier: „Array“), und wurde weggelassen.'); + expect(answer()).not.toContain('das die Fakten nicht zeigen'); + }); + + it('names the claim word of an English one', async () => { + const { props, runtime } = fixture(); + runtime.chat.mockResolvedValueOnce('JSONBAgg aggregates rows into one output value.'); + await render({ ...props, proactiveSelection: jsonbAggEvidence() }); await load(); + await ask('What does this class do?'); + expect(answer()).toContain('The model\'s sentence claimed something the source does not show (here: "output") and was left out.'); + }); + + it('names the unknown name under an automatic explanation and under a configuration file card', async () => { + vi.useFakeTimers(); + const { props, runtime } = fixture(); + runtime.chat.mockResolvedValueOnce('JSONBAgg is checked by the flake8 linter.'); + await render({ ...props, proactive: true, proactiveSelection: jsonbAggEvidence() }); await load(); await settle(); + expect(card()).toContain('The model\'s sentence named something that is in neither the source nor the facts (here: flake8) and was left out.'); + runtime.chat.mockResolvedValueOnce('This file configures `flake8` for the Python sources.'); + await render({ ...props, proactive: true, selectionScope: 'django-demo:explore', readerContext: reader('[metadata]\nname = demo\n', 'setup.cfg') }); await settle(); + await act(async () => button('Ask the model').click()); await settle(); + expect(card()).toContain('Read from the file. The model\'s text named something the file does not show (here: flake8) and was left out.'); + }); +}); diff --git a/graph-ui/src/browser-ai/chat-wording.test.ts b/graph-ui/src/browser-ai/chat-wording.test.ts index 24356eb5a..e84482ec4 100644 --- a/graph-ui/src/browser-ai/chat-wording.test.ts +++ b/graph-ui/src/browser-ai/chat-wording.test.ts @@ -1,8 +1,10 @@ import { describe, expect, it } from 'vitest'; import { prepareExplanationContext, selectionSummary } from './explanation-context'; -import { formatExplanationEvidence } from './explanation-response'; +import { explanationSentence, formatExplanationEvidence } from './explanation-response'; import { jsonbAggEvidence, jsonbAggRenderLimited } from './galaxy-evidence.fixture'; import { relationshipAnswer } from './relationship-answer'; +import { browserChatText, groundedText } from './strings'; +import { sourceTargetOf, symbolSource } from './symbol-source'; /* Wording of the chat texts after the third review of the hand test (2026-10-04, W1 to W10). */ @@ -122,3 +124,38 @@ describe('the heading of a listed caller or callee answer (W3)', () => { expect(answer('Was ruft JSONBAgg auf?', jsonbAggEvidence({ edges: [] })).split('\n')[0]).toBe('`JSONBAgg` ruft in diesem Ausschnitt nichts auf und hat keine anderen ausgehenden Beziehungen.'); }); }); + +describe('why the model sentence was left out (W5)', () => { + const snippet = { source: 'class JSONBAgg(OrderableAggMixin, Aggregate):\n function = "JSONB_AGG"\n allow_distinct = True\n', + file_path: '/abs/django/contrib/postgres/aggregates/general.py', start_line: 50, end_line: 52, source_mode: 'full' }; + const packet = () => prepareExplanationContext(undefined, jsonbAggEvidence(), 3200, symbolSource(sourceTargetOf(jsonbAggEvidence())!, snippet, 'g1')!); + + it('returns the claim word or the unknown name with the dropped sentence, as the sentence wrote it', () => { + expect(explanationSentence('This method tests the aggregation of JSONB data using a predefined model and expected output.', packet())) + .toEqual({ dropped: 'unsupported', reason: { kind: 'claim', text: 'output' } }); + expect(explanationSentence('JSON-BAgg ist eine Aggregation, die die Zeilen in einen JSON-Array umwandelt.', packet())) + .toEqual({ dropped: 'unsupported', reason: { kind: 'claim', text: 'Array' } }); + expect(explanationSentence('JSONBAgg gibt eine Liste von JSON-Daten zurück.', packet())) + .toEqual({ dropped: 'unsupported', reason: { kind: 'claim', text: 'gibt … zurück' } }); + expect(explanationSentence('It wraps `json_agg_helper` around the query.', packet())) + .toEqual({ dropped: 'unsupported', reason: { kind: 'name', text: 'json_agg_helper' } }); + expect(explanationSentence('`JSONBAgg` sets `function` to "JSONB_AGG".', packet())).toEqual({ sentence: '`JSONBAgg` sets `function` to "JSONB_AGG".' }); + }); + + it('says in the note what was claimed or named and what it was checked against, in both languages', () => { + expect(groundedText.en.sentenceDropped({ kind: 'claim', text: 'output' })) + .toBe('Listed from the indexed graph. The model\'s sentence claimed something the source does not show (here: "output") and was left out.'); + expect(groundedText.de.sentenceDropped({ kind: 'claim', text: 'Array' })) + .toBe('Aus dem indizierten Graphen gelistet. Der Satz des Modells behauptete etwas, das der Quelltext nicht zeigt (hier: „Array“), und wurde weggelassen.'); + expect(groundedText.en.sentenceDropped({ kind: 'name', text: 'flake8' })) + .toBe('Listed from the indexed graph. The model\'s sentence named something that is in neither the source nor the facts (here: `flake8`) and was left out.'); + expect(groundedText.de.sentenceDropped({ kind: 'name', text: 'flake8' })) + .toBe('Aus dem indizierten Graphen gelistet. Der Satz des Modells nannte etwas, das weder im Quelltext noch in den Fakten steht (hier: `flake8`), und wurde weggelassen.'); + expect(browserChatText.fileSentenceDropped({ kind: 'name', text: 'flake8' })) + .toBe('Read from the file. The model\'s text named something the file does not show (here: `flake8`) and was left out.'); + expect(browserChatText.explanationDropped({ kind: 'name', text: 'flake8' })) + .toBe('The model\'s explanation named something the source does not show (here: `flake8`) and was left out. Ask a question about the code instead.'); + // Without a known reason the note claims none. + expect(groundedText.de.sentenceDropped()).not.toMatch(/hier:|Fakten nicht zeigen/); + }); +}); diff --git a/graph-ui/src/browser-ai/explanation-response.test.ts b/graph-ui/src/browser-ai/explanation-response.test.ts index 5ef5a7a11..7567f30c1 100644 --- a/graph-ui/src/browser-ai/explanation-response.test.ts +++ b/graph-ui/src/browser-ai/explanation-response.test.ts @@ -105,7 +105,7 @@ describe('names an answer was not given (C3)', () => { it('applies the same comparison to the sentence of an automatic explanation', () => { const packet = { label: 'JSONBAgg', evidence: [{ id: 'source-1', text: source, source: 'code' as const }], limitations: [], fallback: '', characterCount: 0 }; expect(explanationSentence('`JSONBAGG` sets the `distinct` and `ORDER_BY` parts of its template.', packet)).toEqual({ sentence: '`JSONBAGG` sets the `distinct` and `ORDER_BY` parts of its template.' }); - expect(explanationSentence('`Jsonb_agg_distinct_false` checks it.', packet)).toEqual({ dropped: 'unsupported' }); + expect(explanationSentence('`Jsonb_agg_distinct_false` checks it.', packet)).toMatchObject({ dropped: 'unsupported' }); }); }); diff --git a/graph-ui/src/browser-ai/explanation-response.ts b/graph-ui/src/browser-ai/explanation-response.ts index e2e304126..1815006a7 100644 --- a/graph-ui/src/browser-ai/explanation-response.ts +++ b/graph-ui/src/browser-ai/explanation-response.ts @@ -89,9 +89,16 @@ export function namesNotIn(answer: string, given: string): string[] { return [...new Set(named.filter(name => name && !known(name)))]; } +/** Why the model's sentence was left out, as the sentence wrote it: a claim word the source does + * not show ("output", "Array") or a name in neither the source nor the facts. The note under the + * answer says which, instead of blaming a name for every drop (W5). */ +export interface DroppedReason { kind: 'claim' | 'name'; text: string } +/** "gibt eine Liste von JSON-Daten zurück" is shown as "gibt … zurück". */ +const shownClaim = (text: string) => { const words = text.trim().split(/\s+/); return words.length > 2 ? `${words[0]} … ${words.at(-1)}` : words.join(' '); }; + /** The model's part of an automatic explanation: its first sentence (two for reader code), * or nothing when it names what the evidence does not contain. */ -export function explanationSentence(output: string, packet: PreparedExplanationContext, given = ''): { sentence?: string; dropped?: 'unsupported' } { +export function explanationSentence(output: string, packet: PreparedExplanationContext, given = ''): { sentence?: string; dropped?: 'unsupported'; reason?: DroppedReason } { const mode = explanationMode(packet); const text = output.trim().replace(/^```\w*\s*|\s*```$/g, '').replace(/\s+/g, ' ').trim(); if (!/[\p{L}\p{N}]/u.test(text)) return {}; @@ -110,9 +117,13 @@ export function explanationSentence(output: string, packet: PreparedExplanationC const named = [...sentence.matchAll(/`([^`]+)`/g)].map(match => match[1]).concat(sentence.replace(/`[^`]*`/g, ' ').match(IDENTIFIER) ?? []); // Without source every type or input/output claim is a guess; with source only one the code itself shows. const code = packet.evidence.filter(item => item.source === 'code').map(item => item.text).join('\n').toLowerCase(); - const claims = [...sentence.matchAll(new RegExp(UNSUPPORTED_CLAIM.source, 'gi'))].map(match => match[0].toLowerCase().replace(/(?:s|ing)$/, '').split(' ')[0]) - .concat(GERMAN_CLAIMS.flatMap(([pattern, word]) => pattern.test(sentence) ? [word] : [])); - if (named.some(name => !known(name)) || (mode !== 'code' && claims.some(claim => mode === 'graph' || !code.includes(claim)))) return { dropped: 'unsupported' }; + const claims = [...sentence.matchAll(new RegExp(UNSUPPORTED_CLAIM.source, 'gi'))] + .map(match => ({ word: match[0].toLowerCase().replace(/(?:s|ing)$/, '').split(' ')[0], shown: match[0] })) + .concat(GERMAN_CLAIMS.flatMap(([pattern, word]) => { const match = pattern.exec(sentence); return match ? [{ word, shown: shownClaim(match[0]) }] : []; })); + const unknown = named.find(name => !known(name)); + if (unknown) return { dropped: 'unsupported', reason: { kind: 'name', text: unknown } }; + const claim = mode === 'code' ? undefined : claims.find(item => mode === 'graph' || !code.includes(item.word)); + if (claim) return { dropped: 'unsupported', reason: { kind: 'claim', text: claim.shown } }; return { sentence }; } diff --git a/graph-ui/src/browser-ai/grounded-explanation.test.ts b/graph-ui/src/browser-ai/grounded-explanation.test.ts index 2eec8830c..910d5a434 100644 --- a/graph-ui/src/browser-ai/grounded-explanation.test.ts +++ b/graph-ui/src/browser-ai/grounded-explanation.test.ts @@ -95,12 +95,12 @@ describe('the selected symbol source in a Galaxy explanation (K14)', () => { const packet = prepareExplanationContext(undefined, jsonbAggEvidence(), 3200, source); expect(explanationSentence('`JSONBAgg` sets `function` to "JSONB_AGG" and allows distinct values. It is used by many tests.', packet)) .toEqual({ sentence: '`JSONBAgg` sets `function` to "JSONB_AGG" and allows distinct values.' }); - expect(explanationSentence('It is checked by the flake8 linter.', packet)).toEqual({ dropped: 'unsupported' }); - expect(explanationSentence('It wraps `json_agg_helper` around the query.', packet)).toEqual({ dropped: 'unsupported' }); + expect(explanationSentence('It is checked by the flake8 linter.', packet)).toMatchObject({ dropped: 'unsupported' }); + expect(explanationSentence('It wraps `json_agg_helper` around the query.', packet)).toMatchObject({ dropped: 'unsupported' }); // With source, an input or output claim must be one the code shows. - expect(explanationSentence('JSONBAgg is a class that aggregates a list of values.', packet)).toEqual({ dropped: 'unsupported' }); + expect(explanationSentence('JSONBAgg is a class that aggregates a list of values.', packet)).toMatchObject({ dropped: 'unsupported' }); const graphOnly = prepareExplanationContext(undefined, jsonbAggEvidence(), 3200); - expect(explanationSentence('The test_jsonb_agg test calls it on a list of integers and the output is a list of strings.', graphOnly)).toEqual({ dropped: 'unsupported' }); + expect(explanationSentence('The test_jsonb_agg test calls it on a list of integers and the output is a list of strings.', graphOnly)).toMatchObject({ dropped: 'unsupported' }); expect(explanationSentence('JSONBAgg is called by 11 tests.', graphOnly)).toEqual({ sentence: 'JSONBAgg is called by 11 tests.' }); }); @@ -128,7 +128,7 @@ describe('the selected symbol source in a Galaxy explanation (K14)', () => { const packet = prepareExplanationContext(undefined, jsonbAggEvidence(), 3200, source); for (const sentence of ['JSONBAgg gibt eine Liste von Werten zurück.', 'Die Rückgabe ist ein JSON-Objekt.', 'JSONBAgg nimmt Parameter entgegen.', 'Die Argumente werden aggregiert.', 'Die Eingabe sind Zeilen.', 'Die Ausgabe ist JSON.', 'Der Datentyp ist JSONB.', 'Sie wird mit einer Liste von Feldern aufgerufen.']) { - expect(explanationSentence(sentence, packet)).toEqual({ dropped: 'unsupported' }); + expect(explanationSentence(sentence, packet)).toMatchObject({ dropped: 'unsupported' }); } expect(explanationSentence('JSONBAgg setzt `function` auf "JSONB_AGG" und erlaubt distinct.', packet)).toEqual({ sentence: 'JSONBAgg setzt `function` auf "JSONB_AGG" und erlaubt distinct.' }); const german = explanationMessages(packet, { name: 'JSONBAgg', kind: 'Class' }, 'de'); diff --git a/graph-ui/src/browser-ai/strings.ts b/graph-ui/src/browser-ai/strings.ts index f0cc8e1e3..77d74b37d 100644 --- a/graph-ui/src/browser-ai/strings.ts +++ b/graph-ui/src/browser-ai/strings.ts @@ -1,3 +1,5 @@ +import type { DroppedReason } from './explanation-response'; + export const browserAiText = { title: 'Browser AI', subtitle: 'Optional explanations, generated on this device.', @@ -183,6 +185,7 @@ export const relationshipWords: { en: RelationshipWords; de: RelationshipWords } /** Notes of the local chat dock about how an answer was produced or bounded. */ const tokens = (value: number) => value.toLocaleString('en-US'); +const code = (text: string) => `\`${text.replace(/`/g, "'")}\``; export const browserChatText = { shortened: 'Token limit reached: the answer was cut short', /** What the expanded token-limit note says, with the limits the answer ran into. */ @@ -205,12 +208,19 @@ export const browserChatText = { /** Who wrote which part of a grounded automatic explanation (K7). */ factsAndSentence: 'Facts listed from the indexed graph; the last sentence is generated by the model.', factsOnly: 'Listed from the indexed graph; not generated by the model.', - sentenceDropped: "Listed from the indexed graph. The model's sentence named something the evidence does not show and was left out.", - explanationDropped: "The model's explanation named something the source does not show and was left out. Ask a question about the code instead.", + /** What the left out sentence claimed or named, and what it was checked against (W5). */ + sentenceDropped: (reason?: DroppedReason) => `Listed from the indexed graph. The model's sentence ${!reason ? 'was not supported by the source or the facts' + : reason.kind === 'claim' ? `claimed something the source does not show (here: "${reason.text}")` + : `named something that is in neither the source nor the facts (here: ${code(reason.text)})`} and was left out.`, + explanationDropped: (reason?: DroppedReason) => `The model's explanation ${!reason ? 'was not supported by the source' + : `${reason.kind === 'claim' ? 'claimed' : 'named'} something the source does not show (here: ${reason.kind === 'claim' ? `"${reason.text}"` : code(reason.text)})`}` + + ' and was left out. Ask a question about the code instead.', /** The same for facts read from an open workflow file (K12). */ fileFactsAndSentence: 'Facts read from the file; the text after them is generated by the model.', fileFactsOnly: 'Read from the file; not generated by the model.', - fileSentenceDropped: "Read from the file. The model's text named something the file does not show and was left out.", + fileSentenceDropped: (reason?: DroppedReason) => `Read from the file. The model's text ${!reason ? 'was not supported by the file' + : `${reason.kind === 'claim' ? 'claimed' : 'named'} something the file does not show (here: ${reason.kind === 'claim' ? `"${reason.text}"` : code(reason.text)})`}` + + ' and was left out.', writingSentence: 'Reading the source; the model is adding one sentence…', readingFacts: 'Listing the facts…', /** Above the first question about another file or selection (K17). */ @@ -234,7 +244,9 @@ export const groundedText = { de: { factsAndSentence: 'Fakten aus dem indizierten Graphen gelistet; der letzte Satz ist vom Modell erzeugt.', factsOnly: 'Aus dem indizierten Graphen gelistet, nicht vom Modell erzeugt.', - sentenceDropped: 'Aus dem indizierten Graphen gelistet. Der Satz des Modells nannte etwas, das die Fakten nicht zeigen, und wurde weggelassen.', + sentenceDropped: (reason?: DroppedReason) => `Aus dem indizierten Graphen gelistet. Der Satz des Modells ${!reason ? 'wurde weggelassen, weil Quelltext und Fakten ihn nicht stützen' + : reason.kind === 'claim' ? `behauptete etwas, das der Quelltext nicht zeigt (hier: „${reason.text}“), und wurde weggelassen` + : `nannte etwas, das weder im Quelltext noch in den Fakten steht (hier: ${code(reason.text)}), und wurde weggelassen`}.`, writingSentence: 'Der Quelltext wird gelesen; das Modell fügt einen Satz hinzu…', }, }; diff --git a/graph-ui/tools/handtest-fixes-chat.mjs b/graph-ui/tools/handtest-fixes-chat.mjs index bb6eecdd7..152b17d0a 100644 --- a/graph-ui/tools/handtest-fixes-chat.mjs +++ b/graph-ui/tools/handtest-fixes-chat.mjs @@ -709,7 +709,7 @@ async function k12(page) { const factsInCard = /1 job: build \("Hello new contributor", runs on ubuntu-latest, 1 step\)/.test(card) && /Trigger: pull_request_target \(types: opened\)/.test(card); const factsOnly = !automatic && askButton === 1 && /Read from the file; not generated by the model\./.test(card) && card.split('\n').filter(Boolean).every((line) => /^(?:Agent|ⓘ Source|Workflow name|Trigger|1 job|Actions used|Read from the file|Ask the model)/.test(line.trim())); const askedLabelled = Boolean(asked) && /1 job: build/.test(askedCard) - && (/Facts read from the file; the text after them is generated by the model\./.test(askedCard) || /The model's text named something the file does not show and was left out\./.test(askedCard)); + && (/Facts read from the file; the text after them is generated by the model\./.test(askedCard) || /The model's text (?:named|claimed) something the file does not show \(here: .+?\) and was left out\./.test(askedCard)); const othersFactsOnly = others.length >= 2 && others.every((item) => item.modelCalls === 0 && item.factsOnly); const invented = questionsAsked.filter((item) => /flake8|python|\bpip\b|\.py\b/i.test(item.answer)); const counts = questionsAsked.filter((item) => /Wie viele|How many/.test(item.question)).map((item) => item.answer.replace(/^You\s+[^\n]*\n+Agent\s+(?:ⓘ Source\s+)?/, '').replace(/\s+/g, ' ').slice(0, 200));