This commit implements a complete test infrastructure for validating hot-reload stability and robustness across multiple scenarios. ## New Test Infrastructure ### Test Helpers (tests/helpers/) - TestMetrics: FPS, memory, reload time tracking with statistics - TestReporter: Assertion tracking and formatted test reports - SystemUtils: Memory usage monitoring via /proc/self/status - TestAssertions: Macro-based assertion framework ### Test Modules - TankModule: Realistic module with 50 tanks for production testing - ChaosModule: Crash-injection module for robustness validation - StressModule: Lightweight module for long-duration stability tests ## Integration Test Scenarios ### Scenario 1: Production Hot-Reload (test_01_production_hotreload.cpp) ✅ PASSED - End-to-end hot-reload validation - 30 seconds simulation (1800 frames @ 60 FPS) - TankModule with 50 tanks, realistic state - Source modification (v1.0 → v2.0), recompilation, reload - State preservation: positions, velocities, frameCount - Metrics: ~163ms reload time, 0.88MB memory growth ### Scenario 2: Chaos Monkey (test_02_chaos_monkey.cpp) ✅ PASSED - Extreme robustness testing - 150+ random crashes per run (5% crash probability per frame) - 5 crash types: runtime_error, logic_error, out_of_range, domain_error, state corruption - 100% recovery rate via automatic hot-reload - Corrupted state detection and rejection - Random seed for unpredictable crash patterns - Proof of real reload: temporary files in /tmp/grove_module_*.so ### Scenario 3: Stress Test (test_03_stress_test.cpp) ✅ PASSED - Long-duration stability validation - 10 minutes simulation (36000 frames @ 60 FPS) - 120 hot-reloads (every 5 seconds) - 100% reload success rate (120/120) - Memory growth: 2 MB (threshold: 50 MB) - Avg reload time: 160ms (threshold: 500ms) - No memory leaks, no file descriptor leaks ## Core Engine Enhancements ### ModuleLoader (src/ModuleLoader.cpp) - Temporary file copy to /tmp/ for Linux dlopen cache bypass - Robust reload() method: getState() → unload() → load() → setState() - Automatic cleanup of temporary files - Comprehensive error handling and logging ### DebugEngine (src/DebugEngine.cpp) - Automatic recovery in processModuleSystems() - Exception catching → logging → module reload → continue - Module state dump utilities for debugging ### SequentialModuleSystem (src/SequentialModuleSystem.cpp) - extractModule() for safe module extraction - registerModule() for module re-registration - Enhanced processModules() with error handling ## Build System - CMake configuration for test infrastructure - Shared library compilation for test modules (.so) - CTest integration for all scenarios - PIC flag management for spdlog compatibility ## Documentation (planTI/) - Complete test architecture documentation - Detailed scenario specifications with success criteria - Global test plan and validation thresholds ## Validation Results All 3 integration scenarios pass successfully: - Production hot-reload: State preservation validated - Chaos Monkey: 100% recovery from 150+ crashes - Stress Test: Stable over 120 reloads, minimal memory growth 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
174 lines
5.2 KiB
C++
174 lines
5.2 KiB
C++
#include "StressModule.h"
|
|
#include <grove/JsonDataNode.h>
|
|
#include <spdlog/spdlog.h>
|
|
#include <spdlog/sinks/stdout_color_sinks.h>
|
|
|
|
namespace grove {
|
|
|
|
void StressModule::setConfiguration(const IDataNode& configNode, IIO* io, ITaskScheduler* scheduler) {
|
|
// Initialize logger
|
|
logger_ = spdlog::get("StressModule");
|
|
if (!logger_) {
|
|
logger_ = spdlog::stdout_color_mt("StressModule");
|
|
}
|
|
logger_->set_level(spdlog::level::info);
|
|
|
|
// Clone config
|
|
const auto* jsonConfigNode = dynamic_cast<const JsonDataNode*>(&configNode);
|
|
if (jsonConfigNode) {
|
|
config_ = std::make_unique<JsonDataNode>("config", jsonConfigNode->getJsonData());
|
|
} else {
|
|
config_ = std::make_unique<JsonDataNode>("config");
|
|
}
|
|
|
|
logger_->info("Initializing StressModule");
|
|
initializeDummyData();
|
|
}
|
|
|
|
const IDataNode& StressModule::getConfiguration() {
|
|
return *config_;
|
|
}
|
|
|
|
void StressModule::process(const IDataNode& input) {
|
|
isProcessing_ = true;
|
|
frameCount_++;
|
|
|
|
// Lightweight processing - just validate data integrity periodically
|
|
if (frameCount_ % 1000 == 0) {
|
|
if (!validateDummyData()) {
|
|
logger_->error("Data validation failed at frame {}", frameCount_);
|
|
}
|
|
}
|
|
|
|
// Log progress every 60 seconds (3600 frames @ 60 FPS)
|
|
if (frameCount_ % 3600 == 0) {
|
|
logger_->info("Progress: {} frames, {} reloads", frameCount_, reloadCount_);
|
|
}
|
|
|
|
isProcessing_ = false;
|
|
}
|
|
|
|
std::unique_ptr<IDataNode> StressModule::getHealthStatus() {
|
|
nlohmann::json healthJson;
|
|
healthJson["status"] = validateDummyData() ? "healthy" : "corrupted";
|
|
healthJson["frameCount"] = frameCount_;
|
|
healthJson["reloadCount"] = reloadCount_;
|
|
return std::make_unique<JsonDataNode>("health", healthJson);
|
|
}
|
|
|
|
void StressModule::shutdown() {
|
|
logger_->info("Shutting down StressModule");
|
|
logger_->info(" Total frames: {}", frameCount_);
|
|
logger_->info(" Total reloads: {}", reloadCount_);
|
|
}
|
|
|
|
std::string StressModule::getType() const {
|
|
return "stress";
|
|
}
|
|
|
|
std::unique_ptr<IDataNode> StressModule::getState() {
|
|
nlohmann::json json;
|
|
json["frameCount"] = frameCount_;
|
|
json["reloadCount"] = reloadCount_;
|
|
|
|
// Save dummy data as array
|
|
nlohmann::json dataArray = nlohmann::json::array();
|
|
for (size_t i = 0; i < DUMMY_DATA_SIZE; ++i) {
|
|
dataArray.push_back(dummyData_[i]);
|
|
}
|
|
json["dummyData"] = dataArray;
|
|
|
|
logger_->debug("State saved: frame={}, reload={}", frameCount_, reloadCount_);
|
|
|
|
return std::make_unique<JsonDataNode>("state", json);
|
|
}
|
|
|
|
void StressModule::setState(const IDataNode& state) {
|
|
const auto* jsonNode = dynamic_cast<const JsonDataNode*>(&state);
|
|
if (!jsonNode) {
|
|
if (logger_) {
|
|
logger_->error("setState: Invalid state (not JsonDataNode)");
|
|
}
|
|
return;
|
|
}
|
|
|
|
const auto& json = jsonNode->getJsonData();
|
|
|
|
// Ensure logger is initialized (needed after hot-reload)
|
|
if (!logger_) {
|
|
logger_ = spdlog::get("StressModule");
|
|
if (!logger_) {
|
|
logger_ = spdlog::stdout_color_mt("StressModule");
|
|
}
|
|
}
|
|
|
|
// Ensure config is initialized (needed after hot-reload)
|
|
if (!config_) {
|
|
config_ = std::make_unique<JsonDataNode>("config");
|
|
}
|
|
|
|
// Defensive: Initialize dummy data first
|
|
initializeDummyData();
|
|
|
|
// Restore state
|
|
frameCount_ = json.value("frameCount", 0);
|
|
reloadCount_ = json.value("reloadCount", 0);
|
|
|
|
// Increment reload count
|
|
reloadCount_++;
|
|
|
|
// Restore dummy data
|
|
if (json.contains("dummyData") && json["dummyData"].is_array()) {
|
|
const auto& dataArray = json["dummyData"];
|
|
if (dataArray.size() == DUMMY_DATA_SIZE) {
|
|
for (size_t i = 0; i < DUMMY_DATA_SIZE; ++i) {
|
|
dummyData_[i] = dataArray[i].get<int>();
|
|
}
|
|
} else {
|
|
logger_->warn("Dummy data size mismatch: expected {}, got {}",
|
|
DUMMY_DATA_SIZE, dataArray.size());
|
|
}
|
|
}
|
|
|
|
// Validate restored data
|
|
if (!validateDummyData()) {
|
|
logger_->error("Data validation failed after setState");
|
|
initializeDummyData(); // Re-initialize if corrupt
|
|
}
|
|
|
|
logger_->info("State restored: frame={}, reload={}", frameCount_, reloadCount_);
|
|
}
|
|
|
|
void StressModule::initializeDummyData() {
|
|
// Initialize with predictable pattern for validation
|
|
for (size_t i = 0; i < DUMMY_DATA_SIZE; ++i) {
|
|
dummyData_[i] = static_cast<int>(i * 42);
|
|
}
|
|
}
|
|
|
|
bool StressModule::validateDummyData() const {
|
|
for (size_t i = 0; i < DUMMY_DATA_SIZE; ++i) {
|
|
if (dummyData_[i] != static_cast<int>(i * 42)) {
|
|
if (logger_) {
|
|
logger_->error("Data corruption detected at index {}: expected {}, got {}",
|
|
i, i * 42, dummyData_[i]);
|
|
}
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
} // namespace grove
|
|
|
|
// Factory functions
|
|
extern "C" {
|
|
grove::IModule* createModule() {
|
|
return new grove::StressModule();
|
|
}
|
|
|
|
void destroyModule(grove::IModule* module) {
|
|
delete module;
|
|
}
|
|
}
|