This commit implements a complete test infrastructure for validating hot-reload stability and robustness across multiple scenarios. ## New Test Infrastructure ### Test Helpers (tests/helpers/) - TestMetrics: FPS, memory, reload time tracking with statistics - TestReporter: Assertion tracking and formatted test reports - SystemUtils: Memory usage monitoring via /proc/self/status - TestAssertions: Macro-based assertion framework ### Test Modules - TankModule: Realistic module with 50 tanks for production testing - ChaosModule: Crash-injection module for robustness validation - StressModule: Lightweight module for long-duration stability tests ## Integration Test Scenarios ### Scenario 1: Production Hot-Reload (test_01_production_hotreload.cpp) ✅ PASSED - End-to-end hot-reload validation - 30 seconds simulation (1800 frames @ 60 FPS) - TankModule with 50 tanks, realistic state - Source modification (v1.0 → v2.0), recompilation, reload - State preservation: positions, velocities, frameCount - Metrics: ~163ms reload time, 0.88MB memory growth ### Scenario 2: Chaos Monkey (test_02_chaos_monkey.cpp) ✅ PASSED - Extreme robustness testing - 150+ random crashes per run (5% crash probability per frame) - 5 crash types: runtime_error, logic_error, out_of_range, domain_error, state corruption - 100% recovery rate via automatic hot-reload - Corrupted state detection and rejection - Random seed for unpredictable crash patterns - Proof of real reload: temporary files in /tmp/grove_module_*.so ### Scenario 3: Stress Test (test_03_stress_test.cpp) ✅ PASSED - Long-duration stability validation - 10 minutes simulation (36000 frames @ 60 FPS) - 120 hot-reloads (every 5 seconds) - 100% reload success rate (120/120) - Memory growth: 2 MB (threshold: 50 MB) - Avg reload time: 160ms (threshold: 500ms) - No memory leaks, no file descriptor leaks ## Core Engine Enhancements ### ModuleLoader (src/ModuleLoader.cpp) - Temporary file copy to /tmp/ for Linux dlopen cache bypass - Robust reload() method: getState() → unload() → load() → setState() - Automatic cleanup of temporary files - Comprehensive error handling and logging ### DebugEngine (src/DebugEngine.cpp) - Automatic recovery in processModuleSystems() - Exception catching → logging → module reload → continue - Module state dump utilities for debugging ### SequentialModuleSystem (src/SequentialModuleSystem.cpp) - extractModule() for safe module extraction - registerModule() for module re-registration - Enhanced processModules() with error handling ## Build System - CMake configuration for test infrastructure - Shared library compilation for test modules (.so) - CTest integration for all scenarios - PIC flag management for spdlog compatibility ## Documentation (planTI/) - Complete test architecture documentation - Detailed scenario specifications with success criteria - Global test plan and validation thresholds ## Validation Results All 3 integration scenarios pass successfully: - Production hot-reload: State preservation validated - Chaos Monkey: 100% recovery from 150+ crashes - Stress Test: Stable over 120 reloads, minimal memory growth 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
133 lines
4.1 KiB
C++
133 lines
4.1 KiB
C++
#pragma once
|
|
|
|
#include <memory>
|
|
#include <string>
|
|
#include <vector>
|
|
#include <chrono>
|
|
#include <thread>
|
|
#include <atomic>
|
|
#include <spdlog/spdlog.h>
|
|
|
|
#include "IEngine.h"
|
|
#include "IModuleSystem.h"
|
|
#include "IIO.h"
|
|
#include "IDataNode.h"
|
|
#include "ModuleLoader.h"
|
|
|
|
namespace grove {
|
|
|
|
/**
|
|
* @brief Debug engine implementation with comprehensive logging
|
|
*
|
|
* DebugEngine provides maximum visibility into engine operations:
|
|
* - Verbose logging of all operations
|
|
* - Step-by-step execution capabilities
|
|
* - Module isolation and debugging
|
|
* - Performance metrics and timing
|
|
* - IIO health monitoring and reporting
|
|
* - Detailed socket management logging
|
|
*/
|
|
class DebugEngine : public IEngine {
|
|
private:
|
|
std::shared_ptr<spdlog::logger> logger;
|
|
std::atomic<bool> running{false};
|
|
std::atomic<bool> debugPaused{false};
|
|
|
|
// Module management
|
|
std::vector<std::unique_ptr<IModuleSystem>> moduleSystems;
|
|
std::vector<std::string> moduleNames;
|
|
std::vector<std::unique_ptr<ModuleLoader>> moduleLoaders;
|
|
|
|
// Socket management
|
|
std::unique_ptr<IIO> coordinatorSocket;
|
|
std::vector<std::unique_ptr<IIO>> clientSockets;
|
|
|
|
// Performance tracking
|
|
std::chrono::high_resolution_clock::time_point lastFrameTime;
|
|
std::chrono::high_resolution_clock::time_point engineStartTime;
|
|
size_t frameCount = 0;
|
|
|
|
// Configuration
|
|
std::unique_ptr<IDataNode> engineConfig;
|
|
|
|
// Helper methods
|
|
void logEngineStart();
|
|
void logEngineShutdown();
|
|
void logFrameStart(float deltaTime);
|
|
void logFrameEnd(float frameTime);
|
|
void logModuleHealth();
|
|
void logSocketHealth();
|
|
void processModuleSystems(float deltaTime);
|
|
void processClientMessages();
|
|
void processCoordinatorMessages();
|
|
float calculateDeltaTime();
|
|
void validateConfiguration();
|
|
|
|
public:
|
|
DebugEngine();
|
|
virtual ~DebugEngine();
|
|
|
|
// IEngine implementation
|
|
void initialize() override;
|
|
void run() override;
|
|
void step(float deltaTime) override;
|
|
void shutdown() override;
|
|
void loadModules(const std::string& configPath) override;
|
|
void registerMainSocket(std::unique_ptr<IIO> coordinatorSocket) override;
|
|
void registerNewClientSocket(std::unique_ptr<IIO> clientSocket) override;
|
|
EngineType getType() const override;
|
|
|
|
// Debug-specific methods
|
|
void pauseExecution();
|
|
void resumeExecution();
|
|
void stepSingleFrame();
|
|
bool isPaused() const;
|
|
std::unique_ptr<IDataNode> getDetailedStatus() const;
|
|
void setLogLevel(spdlog::level::level_enum level);
|
|
|
|
// Hot-reload methods
|
|
/**
|
|
* @brief Register a module from .so file with hot-reload support
|
|
* @param name Module identifier
|
|
* @param modulePath Path to .so file
|
|
* @param strategy Module system strategy (sequential, threaded, etc.)
|
|
*/
|
|
void registerModuleFromFile(const std::string& name, const std::string& modulePath, ModuleSystemType strategy);
|
|
|
|
/**
|
|
* @brief Hot-reload a module by name
|
|
* @param name Module identifier to reload
|
|
*
|
|
* This performs zero-downtime hot-reload:
|
|
* 1. Extract state from current module
|
|
* 2. Unload old .so
|
|
* 3. Load new .so (recompiled version)
|
|
* 4. Restore state to new module
|
|
* 5. Continue execution without stopping engine
|
|
*/
|
|
void reloadModule(const std::string& name);
|
|
|
|
/**
|
|
* @brief Get list of all registered module names
|
|
*/
|
|
std::vector<std::string> getModuleNames() const { return moduleNames; }
|
|
|
|
/**
|
|
* @brief Dump the current state of a specific module to logs
|
|
* @param name Module identifier
|
|
*
|
|
* Retrieves the module's state via getState() and pretty-prints it
|
|
* as formatted JSON in the logs. Useful for debugging and inspection.
|
|
*/
|
|
void dumpModuleState(const std::string& name);
|
|
|
|
/**
|
|
* @brief Dump the state of all registered modules to logs
|
|
*
|
|
* Iterates through all modules and dumps their state.
|
|
* Useful for comprehensive system state snapshots.
|
|
*/
|
|
void dumpAllModulesState();
|
|
};
|
|
|
|
} // namespace grove
|