27 KiB
27 KiB
Hierarchical Memory Manager
Advanced Memory Architecture for Enhanced BMAD System
The Hierarchical Memory Manager provides sophisticated, multi-tiered memory management with intelligent retention, compression, and retrieval capabilities that scale from individual sessions to enterprise-wide knowledge repositories.
Hierarchical Memory Architecture
Multi-Tier Memory Structure
hierarchical_memory_architecture:
memory_tiers:
immediate_memory:
- working_memory: "Current session active context"
- attention_buffer: "Recently accessed high-priority items"
- rapid_access_cache: "Ultra-fast access for current operations"
- conversation_buffer: "Current conversation context"
short_term_memory:
- session_memory: "Complete session knowledge and context"
- recent_patterns: "Recently identified patterns and insights"
- active_decisions: "Ongoing decision processes"
- current_objectives: "Session goals and progress tracking"
medium_term_memory:
- project_memory: "Project-specific knowledge and history"
- team_memory: "Team collaboration patterns and knowledge"
- sprint_memory: "Development cycle knowledge"
- contextual_memory: "Situational knowledge and adaptations"
long_term_memory:
- organizational_memory: "Enterprise-wide knowledge repository"
- domain_memory: "Technical domain expertise and patterns"
- historical_memory: "Long-term trends and evolution"
- strategic_memory: "High-level strategic decisions and outcomes"
permanent_memory:
- core_knowledge: "Fundamental principles and established facts"
- validated_patterns: "Thoroughly validated successful patterns"
- canonical_solutions: "Proven solution templates and frameworks"
- institutional_knowledge: "Critical organizational knowledge"
memory_characteristics:
retention_policies:
- importance_based: "Retain based on knowledge importance scores"
- access_frequency: "Retain frequently accessed memories"
- recency_weighted: "Weight recent memories higher"
- validation_status: "Prioritize validated knowledge"
compression_strategies:
- semantic_compression: "Compress while preserving meaning"
- pattern_abstraction: "Abstract specific instances to patterns"
- hierarchical_summarization: "Multi-level summary creation"
- lossy_compression: "Remove less important details"
retrieval_optimization:
- predictive_preloading: "Preload likely needed memories"
- contextual_indexing: "Index by multiple context dimensions"
- associative_linking: "Link related memories"
- temporal_organization: "Organize by time relationships"
conflict_resolution:
- confidence_scoring: "Resolve based on confidence levels"
- source_credibility: "Weight by information source reliability"
- consensus_analysis: "Use multiple source agreement"
- temporal_precedence: "Newer information supersedes older"
Advanced Memory Manager Implementation
import asyncio
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.cluster import KMeans
import networkx as nx
from collections import defaultdict, deque
import pickle
import lz4
import zstandard as zstd
from datetime import datetime, timedelta
import heapq
from typing import Dict, List, Any, Optional, Tuple
class HierarchicalMemoryManager:
"""
Advanced hierarchical memory management system with intelligent retention and retrieval
"""
def __init__(self, config=None):
self.config = config or {
'immediate_memory_size': 1000,
'short_term_memory_size': 10000,
'medium_term_memory_size': 100000,
'compression_threshold': 0.8,
'importance_threshold': 0.7,
'retention_period_days': {
'immediate': 1,
'short_term': 7,
'medium_term': 90,
'long_term': 365
}
}
# Initialize memory tiers
self.immediate_memory = ImmediateMemory(self.config)
self.short_term_memory = ShortTermMemory(self.config)
self.medium_term_memory = MediumTermMemory(self.config)
self.long_term_memory = LongTermMemory(self.config)
self.permanent_memory = PermanentMemory(self.config)
# Memory management components
self.importance_scorer = ImportanceScorer()
self.compression_engine = CompressionEngine()
self.retrieval_optimizer = RetrievalOptimizer()
self.conflict_resolver = ConflictResolver()
self.retention_policy = RetentionPolicyManager(self.config)
# Memory analytics
self.memory_analytics = MemoryAnalytics()
self.access_patterns = AccessPatternTracker()
async def store_memory(self, memory_item, context=None):
"""
Store memory item in appropriate tier based on characteristics and importance
"""
storage_session = {
'memory_id': memory_item.get('id', generate_uuid()),
'storage_tier': None,
'importance_score': 0.0,
'compression_applied': False,
'conflicts_resolved': [],
'storage_metadata': {}
}
# Calculate importance score
importance_score = await self.importance_scorer.calculate_importance(
memory_item,
context
)
storage_session['importance_score'] = importance_score
# Determine appropriate storage tier
storage_tier = await self.determine_storage_tier(memory_item, importance_score, context)
storage_session['storage_tier'] = storage_tier
# Check for conflicts with existing memories
conflicts = await self.conflict_resolver.detect_conflicts(memory_item, storage_tier)
if conflicts:
resolution_results = await self.conflict_resolver.resolve_conflicts(
memory_item,
conflicts,
storage_tier
)
storage_session['conflicts_resolved'] = resolution_results
# Apply compression if needed
if await self.should_compress_memory(memory_item, storage_tier):
compressed_item = await self.compression_engine.compress_memory(memory_item)
memory_item = compressed_item
storage_session['compression_applied'] = True
# Store in appropriate tier
if storage_tier == 'immediate':
storage_result = await self.immediate_memory.store(memory_item, context)
elif storage_tier == 'short_term':
storage_result = await self.short_term_memory.store(memory_item, context)
elif storage_tier == 'medium_term':
storage_result = await self.medium_term_memory.store(memory_item, context)
elif storage_tier == 'long_term':
storage_result = await self.long_term_memory.store(memory_item, context)
elif storage_tier == 'permanent':
storage_result = await self.permanent_memory.store(memory_item, context)
storage_session['storage_metadata'] = storage_result
# Update access patterns
await self.access_patterns.record_storage(memory_item, storage_tier, context)
# Trigger memory maintenance if needed
await self.trigger_memory_maintenance_if_needed()
return storage_session
async def retrieve_memory(self, query, context=None, retrieval_config=None):
"""
Intelligent memory retrieval across all tiers with optimization
"""
if retrieval_config is None:
retrieval_config = {
'max_results': 10,
'similarity_threshold': 0.7,
'include_compressed': True,
'cross_tier_search': True,
'temporal_weighting': True
}
retrieval_session = {
'query': query,
'context': context,
'tier_results': {},
'fused_results': [],
'retrieval_metadata': {}
}
# Optimize retrieval strategy based on query and context
retrieval_strategy = await self.retrieval_optimizer.optimize_retrieval_strategy(
query,
context,
retrieval_config
)
# Execute retrieval across tiers based on strategy
retrieval_tasks = []
if retrieval_strategy['search_immediate']:
retrieval_tasks.append(
self.retrieve_from_tier('immediate', query, context, retrieval_config)
)
if retrieval_strategy['search_short_term']:
retrieval_tasks.append(
self.retrieve_from_tier('short_term', query, context, retrieval_config)
)
if retrieval_strategy['search_medium_term']:
retrieval_tasks.append(
self.retrieve_from_tier('medium_term', query, context, retrieval_config)
)
if retrieval_strategy['search_long_term']:
retrieval_tasks.append(
self.retrieve_from_tier('long_term', query, context, retrieval_config)
)
if retrieval_strategy['search_permanent']:
retrieval_tasks.append(
self.retrieve_from_tier('permanent', query, context, retrieval_config)
)
# Execute retrievals in parallel
tier_results = await asyncio.gather(*retrieval_tasks)
# Store tier results
tier_names = ['immediate', 'short_term', 'medium_term', 'long_term', 'permanent']
for i, result in enumerate(tier_results):
if i < len(tier_names):
retrieval_session['tier_results'][tier_names[i]] = result
# Fuse results across tiers
fused_results = await self.fuse_cross_tier_results(
tier_results,
query,
context,
retrieval_config
)
retrieval_session['fused_results'] = fused_results
# Update access patterns
await self.access_patterns.record_retrieval(query, fused_results, context)
# Update memory importance based on access
await self.update_memory_importance_from_access(fused_results)
return retrieval_session
async def determine_storage_tier(self, memory_item, importance_score, context):
"""
Determine the appropriate storage tier for a memory item
"""
# Immediate memory criteria
if (context and context.get('session_active', True) and
importance_score > 0.8 and
memory_item.get('type') in ['current_task', 'active_decision', 'working_context']):
return 'immediate'
# Short-term memory criteria
elif (importance_score > 0.6 and
memory_item.get('age_hours', 0) < 24 and
memory_item.get('type') in ['session_memory', 'recent_pattern', 'active_objective']):
return 'short_term'
# Medium-term memory criteria
elif (importance_score > 0.4 and
memory_item.get('age_days', 0) < 30 and
memory_item.get('type') in ['project_memory', 'team_knowledge', 'sprint_outcome']):
return 'medium_term'
# Long-term memory criteria
elif (importance_score > 0.3 and
memory_item.get('validated', False) and
memory_item.get('type') in ['organizational_knowledge', 'domain_expertise']):
return 'long_term'
# Permanent memory criteria
elif (importance_score > 0.7 and
memory_item.get('validated', False) and
memory_item.get('consensus_score', 0) > 0.8 and
memory_item.get('type') in ['core_principle', 'validated_pattern', 'canonical_solution']):
return 'permanent'
# Default to short-term for new items
else:
return 'short_term'
async def memory_maintenance_cycle(self):
"""
Periodic memory maintenance including compression, migration, and cleanup
"""
maintenance_session = {
'session_id': generate_uuid(),
'start_time': datetime.utcnow(),
'maintenance_actions': [],
'performance_improvements': {},
'space_reclaimed': 0
}
# Immediate memory maintenance
immediate_maintenance = await self.maintain_immediate_memory()
maintenance_session['maintenance_actions'].append(immediate_maintenance)
# Short-term memory maintenance
short_term_maintenance = await self.maintain_short_term_memory()
maintenance_session['maintenance_actions'].append(short_term_maintenance)
# Medium-term memory maintenance
medium_term_maintenance = await self.maintain_medium_term_memory()
maintenance_session['maintenance_actions'].append(medium_term_maintenance)
# Long-term memory optimization
long_term_optimization = await self.optimize_long_term_memory()
maintenance_session['maintenance_actions'].append(long_term_optimization)
# Cross-tier memory migration
migration_results = await self.execute_cross_tier_migration()
maintenance_session['maintenance_actions'].append(migration_results)
# Memory compression optimization
compression_optimization = await self.optimize_memory_compression()
maintenance_session['maintenance_actions'].append(compression_optimization)
# Calculate performance improvements
performance_improvements = await self.calculate_maintenance_improvements(
maintenance_session['maintenance_actions']
)
maintenance_session['performance_improvements'] = performance_improvements
maintenance_session['end_time'] = datetime.utcnow()
maintenance_session['duration'] = (
maintenance_session['end_time'] - maintenance_session['start_time']
).total_seconds()
return maintenance_session
async def maintain_immediate_memory(self):
"""
Maintain immediate memory by promoting important items and evicting stale ones
"""
maintenance_result = {
'memory_tier': 'immediate',
'items_processed': 0,
'items_promoted': 0,
'items_evicted': 0,
'space_reclaimed': 0
}
# Get all items from immediate memory
immediate_items = await self.immediate_memory.get_all_items()
maintenance_result['items_processed'] = len(immediate_items)
# Evaluate each item for promotion or eviction
for item in immediate_items:
# Check if item should be promoted to short-term memory
if await self.should_promote_to_short_term(item):
await self.immediate_memory.remove(item['id'])
await self.short_term_memory.store(item)
maintenance_result['items_promoted'] += 1
# Check if item should be evicted due to age or low importance
elif await self.should_evict_from_immediate(item):
space_before = await self.immediate_memory.get_space_usage()
await self.immediate_memory.remove(item['id'])
space_after = await self.immediate_memory.get_space_usage()
maintenance_result['space_reclaimed'] += space_before - space_after
maintenance_result['items_evicted'] += 1
return maintenance_result
async def execute_cross_tier_migration(self):
"""
Migrate memories between tiers based on access patterns and importance
"""
migration_result = {
'migration_type': 'cross_tier',
'migrations_executed': [],
'total_items_migrated': 0,
'performance_impact': {}
}
# Analyze access patterns to identify migration candidates
migration_candidates = await self.identify_migration_candidates()
for candidate in migration_candidates:
source_tier = candidate['current_tier']
target_tier = candidate['recommended_tier']
item_id = candidate['item_id']
# Execute migration
migration_success = await self.migrate_memory_item(
item_id,
source_tier,
target_tier
)
if migration_success:
migration_result['migrations_executed'].append({
'item_id': item_id,
'source_tier': source_tier,
'target_tier': target_tier,
'migration_reason': candidate['reason'],
'expected_benefit': candidate['expected_benefit']
})
migration_result['total_items_migrated'] += 1
return migration_result
class ImportanceScorer:
"""
Calculate importance scores for memory items based on multiple factors
"""
def __init__(self):
self.scoring_weights = {
'recency': 0.2,
'frequency': 0.25,
'context_relevance': 0.2,
'validation_level': 0.15,
'uniqueness': 0.1,
'user_feedback': 0.1
}
async def calculate_importance(self, memory_item, context=None):
"""
Calculate comprehensive importance score for memory item
"""
importance_components = {
'recency_score': await self.calculate_recency_score(memory_item),
'frequency_score': await self.calculate_frequency_score(memory_item),
'context_relevance_score': await self.calculate_context_relevance(memory_item, context),
'validation_score': await self.calculate_validation_score(memory_item),
'uniqueness_score': await self.calculate_uniqueness_score(memory_item),
'user_feedback_score': await self.calculate_user_feedback_score(memory_item)
}
# Calculate weighted importance score
importance_score = 0.0
for component, weight in self.scoring_weights.items():
component_key = f"{component.replace('_', '_')}_score"
if component_key in importance_components:
importance_score += importance_components[component_key] * weight
# Normalize to 0-1 range
importance_score = max(0.0, min(1.0, importance_score))
return {
'overall_score': importance_score,
'components': importance_components,
'calculation_timestamp': datetime.utcnow()
}
async def calculate_recency_score(self, memory_item):
"""
Calculate recency score based on when memory was created/last accessed
"""
timestamp = memory_item.get('timestamp')
if not timestamp:
return 0.5 # Default for items without timestamp
if isinstance(timestamp, str):
timestamp = datetime.fromisoformat(timestamp)
time_diff = datetime.utcnow() - timestamp
days_old = time_diff.total_seconds() / (24 * 3600)
# Exponential decay: score = e^(-days_old/decay_constant)
decay_constant = 30 # 30 days
recency_score = np.exp(-days_old / decay_constant)
return min(1.0, recency_score)
async def calculate_frequency_score(self, memory_item):
"""
Calculate frequency score based on access patterns
"""
access_count = memory_item.get('access_count', 0)
last_access = memory_item.get('last_access')
if access_count == 0:
return 0.1 # Minimum score for unaccessed items
# Calculate frequency adjusted for recency
if last_access:
if isinstance(last_access, str):
last_access = datetime.fromisoformat(last_access)
days_since_access = (datetime.utcnow() - last_access).days
recency_factor = max(0.1, 1.0 - (days_since_access / 365)) # Decay over a year
else:
recency_factor = 0.5
# Logarithmic scaling for access count
frequency_base = min(1.0, np.log(access_count + 1) / np.log(100)) # Max out at 100 accesses
return frequency_base * recency_factor
class CompressionEngine:
"""
Intelligent memory compression while preserving semantic content
"""
def __init__(self):
self.compression_algorithms = {
'lossless': LosslessCompression(),
'semantic': SemanticCompression(),
'pattern_based': PatternBasedCompression(),
'hierarchical': HierarchicalCompression()
}
self.compression_thresholds = {
'size_threshold_mb': 1.0,
'age_threshold_days': 7,
'access_frequency_threshold': 0.1
}
async def compress_memory(self, memory_item, compression_strategy='auto'):
"""
Compress memory item using appropriate strategy
"""
if compression_strategy == 'auto':
compression_strategy = await self.select_compression_strategy(memory_item)
compression_algorithm = self.compression_algorithms.get(
compression_strategy,
self.compression_algorithms['lossless']
)
compressed_result = await compression_algorithm.compress(memory_item)
return {
**memory_item,
'compressed': True,
'compression_strategy': compression_strategy,
'compression_ratio': compressed_result['compression_ratio'],
'compressed_data': compressed_result['compressed_data'],
'compression_metadata': compressed_result['metadata'],
'original_size': compressed_result['original_size'],
'compressed_size': compressed_result['compressed_size']
}
async def decompress_memory(self, compressed_memory_item):
"""
Decompress memory item to restore original content
"""
compression_strategy = compressed_memory_item.get('compression_strategy', 'lossless')
compression_algorithm = self.compression_algorithms.get(compression_strategy)
if not compression_algorithm:
raise ValueError(f"Unknown compression strategy: {compression_strategy}")
decompressed_result = await compression_algorithm.decompress(compressed_memory_item)
# Restore original memory item structure
decompressed_item = {
**compressed_memory_item,
'compressed': False,
**decompressed_result['restored_data']
}
# Remove compression-specific fields
compression_fields = [
'compression_strategy', 'compression_ratio', 'compressed_data',
'compression_metadata', 'original_size', 'compressed_size'
]
for field in compression_fields:
decompressed_item.pop(field, None)
return decompressed_item
class LosslessCompression:
"""
Lossless compression using advanced algorithms
"""
async def compress(self, memory_item):
"""
Apply lossless compression to memory item
"""
# Serialize memory item
serialized_data = pickle.dumps(memory_item)
original_size = len(serialized_data)
# Apply Zstandard compression for best ratio
compressor = zstd.ZstdCompressor(level=19) # Maximum compression
compressed_data = compressor.compress(serialized_data)
compressed_size = len(compressed_data)
compression_ratio = original_size / compressed_size if compressed_size > 0 else 1.0
return {
'compressed_data': compressed_data,
'compression_ratio': compression_ratio,
'original_size': original_size,
'compressed_size': compressed_size,
'metadata': {
'algorithm': 'zstandard',
'compression_level': 19,
'timestamp': datetime.utcnow().isoformat()
}
}
async def decompress(self, compressed_memory_item):
"""
Decompress losslessly compressed memory item
"""
compressed_data = compressed_memory_item['compressed_data']
# Decompress using Zstandard
decompressor = zstd.ZstdDecompressor()
decompressed_data = decompressor.decompress(compressed_data)
# Deserialize back to original structure
restored_data = pickle.loads(decompressed_data)
return {
'restored_data': restored_data,
'decompression_successful': True
}
Advanced Memory Commands
# Memory tier management
bmad memory status --tiers "all" --usage-statistics
bmad memory migrate --item-id "uuid" --from "short_term" --to "long_term"
bmad memory compress --tier "medium_term" --algorithm "semantic"
# Memory maintenance and optimization
bmad memory maintenance --run-cycle --optimize-performance
bmad memory cleanup --tier "immediate" --age-threshold "24h"
bmad memory defragment --all-tiers --compact-storage
# Memory analytics and insights
bmad memory analyze --access-patterns --time-window "30d"
bmad memory importance --recalculate --update-tiers
bmad memory conflicts --detect --resolve-automatically
# Memory retrieval optimization
bmad memory search --query "authentication patterns" --cross-tier
bmad memory preload --predict-usage --context "current-session"
bmad memory export --tier "permanent" --format "knowledge-graph"
This Hierarchical Memory Manager provides enterprise-grade memory management with intelligent tiering, compression, and optimization capabilities that scale from individual sessions to organizational knowledge repositories.