Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 | 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 31x 31x 31x 31x 31x 31x 31x 31x 31x 31x 31x 31x 31x 31x 31x 31x 31x 31x 30x 30x 30x 28x 28x 28x 2x 2x 2x 21x 2x 2x 2x 2x 2x 21x 2x 2x 2x 2x 21x 4x 2x 2x 2x 2x 2x 2x 2x 2x 2x 4x 4x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 4x 4x 4x 4x 4x 4x 4x 4x 4x 2x 2x 2x 2x 2x 30x 31x 31x 31x 31x 31x 31x 31x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 3x 27x 27x 27x 3x 3x 3x 3x 3x 3x 31x 31x 31x 31x 31x 31x 31x 31x 2x 2x 2x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 1x 2x 2x 2x 31x 31x 31x 31x 31x 31x 31x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 2x 31x | /**
* Pattern Detection
*
* Analyzes run history to detect patterns and generate suggestions.
*
* @packageDocumentation
*/
import type { Store } from '../storage/store.js';
import type { TaskType, Suggestion } from '../types.js';
import { nanoid } from 'nanoid';
/**
* Minimum number of runs required to generate a suggestion.
*/
const MIN_RUNS_FOR_SUGGESTION = 10;
/**
* Minimum confidence threshold for suggestions.
*/
const MIN_CONFIDENCE_THRESHOLD = 0.6;
/**
* Minimum improvement threshold to suggest a change (e.g., 10% better).
*/
const MIN_IMPROVEMENT_THRESHOLD = 0.1;
/**
* Pattern detection engine for learning from outcomes.
*/
export class PatternDetector {
private store: Store;
/**
* Creates a new PatternDetector.
*
* @param store - The storage instance to use
*/
constructor(store: Store) {
this.store = store;
}
/**
* Analyzes a task type and generates suggestions if appropriate.
*
* @param taskType - The task type to analyze
* @returns Array of suggestions
*/
analyzeTaskType(taskType: TaskType): Suggestion[] {
const stats = this.store.getLearningStats(taskType);
if (stats.length < 2) {
// Need at least 2 models to compare
return [];
}
// Get the current routing rule
const currentRule = this.store.getRule(taskType);
if (!currentRule) return [];
const currentModel = currentRule.preferredModel;
// Find the current model's stats
const currentStats = stats.find((s) => s.model === currentModel);
if (!currentStats) return [];
// Find better performing models
const suggestions: Suggestion[] = [];
for (const modelStats of stats) {
if (modelStats.model === currentModel) continue;
if (modelStats.runs < MIN_RUNS_FOR_SUGGESTION) continue;
// Calculate improvement potential
const successImprovement = modelStats.outcomeSuccessRate - currentStats.outcomeSuccessRate;
const latencyImprovement = (currentStats.avgDurationMs - modelStats.avgDurationMs) / currentStats.avgDurationMs;
// Check if this model is significantly better
const isSignificantlyBetter =
successImprovement > MIN_IMPROVEMENT_THRESHOLD ||
(successImprovement >= 0 && latencyImprovement > MIN_IMPROVEMENT_THRESHOLD);
if (!isSignificantlyBetter) continue;
// Calculate confidence based on sample size and improvement magnitude
const sampleConfidence = Math.min(modelStats.runs / 50, 1); // Max confidence at 50 runs
const improvementConfidence = Math.min(
Math.abs(successImprovement) / 0.3 + Math.abs(latencyImprovement) / 0.5,
1
);
const confidence = (sampleConfidence + improvementConfidence) / 2;
if (confidence < MIN_CONFIDENCE_THRESHOLD) continue;
// Generate reason
const reasons: string[] = [];
if (successImprovement > 0) {
reasons.push(`${(successImprovement * 100).toFixed(0)}% higher success rate`);
}
if (latencyImprovement > 0) {
reasons.push(`${(latencyImprovement * 100).toFixed(0)}% faster`);
}
const suggestion: Suggestion = {
id: nanoid(),
taskType,
currentModel,
suggestedModel: modelStats.model,
reason: reasons.join(', '),
confidence,
expectedImprovement: {
successRate: successImprovement > 0 ? successImprovement : undefined,
latency: latencyImprovement > 0 ? latencyImprovement : undefined,
},
sampleCount: modelStats.runs,
createdAt: new Date().toISOString(),
};
suggestions.push(suggestion);
}
// Sort by confidence descending
suggestions.sort((a, b) => b.confidence - a.confidence);
return suggestions;
}
/**
* Analyzes all task types and generates suggestions.
*
* @returns Array of all suggestions across task types
*/
analyzeAll(): Suggestion[] {
const taskTypes: TaskType[] = [
'code_generation',
'code_review',
'summarization',
'analysis',
'creative_writing',
'data_extraction',
'translation',
'question_answering',
'general',
];
const allSuggestions: Suggestion[] = [];
for (const taskType of taskTypes) {
const suggestions = this.analyzeTaskType(taskType);
allSuggestions.push(...suggestions);
}
// Sort by confidence descending
allSuggestions.sort((a, b) => b.confidence - a.confidence);
return allSuggestions;
}
/**
* Stores suggestions in the database.
*
* @param suggestions - The suggestions to store
* @returns Array of suggestion IDs
*/
storeSuggestions(suggestions: Suggestion[]): string[] {
const ids: string[] = [];
for (const suggestion of suggestions) {
const id = this.store.recordSuggestion({
taskType: suggestion.taskType,
currentModel: suggestion.currentModel,
suggestedModel: suggestion.suggestedModel,
reason: suggestion.reason,
confidence: suggestion.confidence,
expectedImprovement: JSON.stringify(suggestion.expectedImprovement),
sampleCount: suggestion.sampleCount,
accepted: null,
});
ids.push(id);
}
return ids;
}
/**
* Generates and stores new suggestions, returning only new ones.
*
* @returns Array of new suggestions
*/
generateSuggestions(): Suggestion[] {
const suggestions = this.analyzeAll();
// Filter out suggestions that already exist (same task type + suggested model)
const pending = this.store.getPendingSuggestions();
const existingKeys = new Set(
pending.map((s) => `${s.taskType}:${s.suggestedModel}`)
);
const newSuggestions = suggestions.filter(
(s) => !existingKeys.has(`${s.taskType}:${s.suggestedModel}`)
);
// Store new suggestions
this.storeSuggestions(newSuggestions);
return newSuggestions;
}
}
|