Wire OptimizationLoop to project metrics and add metrics optimize.
Add from_metrics_store factory, OptimizerStore persistence, metrics optimize CLI, consolidate duplicate optimization agent, and add integration tests.
This commit is contained in:
@@ -5,11 +5,16 @@ This module implements the kaizen loop for measuring, analyzing, and refining
|
||||
agent performance over time.
|
||||
"""
|
||||
|
||||
from typing import Dict, Any, List, Optional
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
import statistics
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .metrics import MetricsStore
|
||||
|
||||
MIN_SAMPLES_FOR_RECOMMENDATIONS = 10
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerformanceMetrics:
|
||||
@@ -35,6 +40,60 @@ class OptimizationLoop:
|
||||
self.metrics_history: List[PerformanceMetrics] = []
|
||||
self.optimization_history: List[Dict[str, Any]] = []
|
||||
|
||||
@classmethod
|
||||
def from_metrics_store(
|
||||
cls,
|
||||
store: "MetricsStore",
|
||||
*,
|
||||
min_samples: int = 1,
|
||||
) -> "OptimizationLoop":
|
||||
"""Build an optimization loop from project-scoped execution records."""
|
||||
loop = cls(store.agent_name)
|
||||
records = store.read_executions()
|
||||
if len(records) < min_samples:
|
||||
return loop
|
||||
for record in records:
|
||||
loop.record_metrics(cls._metrics_from_record(record))
|
||||
return loop
|
||||
|
||||
@staticmethod
|
||||
def _metrics_from_record(record: Dict[str, Any]) -> PerformanceMetrics:
|
||||
timestamp_raw = record.get("timestamp")
|
||||
try:
|
||||
timestamp = datetime.fromisoformat(
|
||||
str(timestamp_raw).replace("Z", "+00:00")
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
timestamp = datetime.now()
|
||||
|
||||
success = bool(record.get("success", False))
|
||||
quality = record.get("quality_score")
|
||||
if quality is None:
|
||||
quality = 1.0 if success else 0.0
|
||||
|
||||
metadata = {
|
||||
k: v
|
||||
for k, v in record.items()
|
||||
if k
|
||||
not in {
|
||||
"timestamp",
|
||||
"agent",
|
||||
"success",
|
||||
"execution_time_s",
|
||||
"quality_score",
|
||||
"primary_metric",
|
||||
}
|
||||
}
|
||||
|
||||
return PerformanceMetrics(
|
||||
timestamp=timestamp,
|
||||
execution_time=float(record.get("execution_time_s") or 0.0),
|
||||
success_rate=1.0 if success else 0.0,
|
||||
quality_score=float(quality),
|
||||
resource_usage={},
|
||||
metadata=metadata or None,
|
||||
)
|
||||
|
||||
def record_metrics(self, metrics: PerformanceMetrics) -> None:
|
||||
"""Record performance metrics for analysis."""
|
||||
self.metrics_history.append(metrics)
|
||||
@@ -160,3 +219,17 @@ class OptimizationLoop:
|
||||
"metrics_count": len(self.metrics_history),
|
||||
"optimization_cycles": len(self.optimization_history),
|
||||
}
|
||||
|
||||
def get_optimization_report_json(self) -> Dict[str, Any]:
|
||||
"""JSON-serializable optimization report."""
|
||||
return _to_json_safe(self.get_optimization_report())
|
||||
|
||||
|
||||
def _to_json_safe(value: Any) -> Any:
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
if isinstance(value, dict):
|
||||
return {k: _to_json_safe(v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_to_json_safe(item) for item in value]
|
||||
return value
|
||||
|
||||
Reference in New Issue
Block a user