StillHammer
4f60de68d6
- Fix BatchProcessor constructor to avoid server blocking during startup - Add comprehensive integration tests for all modular combinations - Enhance CLAUDE.md documentation with new test commands - Update SelectiveLayers configuration for better LLM allocation - Add AutoReporter system for test automation - Include production workflow validation tests 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
566 lines
19 KiB
JavaScript
566 lines
19 KiB
JavaScript
// ========================================
|
|
// AUTO PROCESSOR - REFACTORISÉ
|
|
// Responsabilité: Mode AUTO - Traitement Batch Google Sheets automatique
|
|
// ========================================
|
|
|
|
const { QueueProcessor } = require('../shared/QueueProcessor');
|
|
const { logSh } = require('../ErrorReporting');
|
|
const path = require('path');
|
|
|
|
/**
|
|
* AUTO PROCESSOR
|
|
* Spécialisé pour traitement automatique avec monitoring intégré
|
|
*/
|
|
class AutoProcessor extends QueueProcessor {
|
|
|
|
constructor(options = {}) {
|
|
super({
|
|
name: 'AutoProcessor',
|
|
config: {
|
|
batchSize: options.batchSize || 5,
|
|
delayBetweenItems: options.delayBetweenItems || 2000,
|
|
delayBetweenBatches: options.delayBetweenBatches || 30000,
|
|
maxRetries: options.maxRetries || 3,
|
|
startRow: options.startRow || 2,
|
|
endRow: options.endRow || null,
|
|
selective: 'standardEnhancement', // Config fixe pour AUTO
|
|
adversarial: 'light',
|
|
humanSimulation: 'lightSimulation',
|
|
patternBreaking: 'standardPatternBreaking',
|
|
intensity: 1.0,
|
|
monitoringPort: options.monitoringPort || 3001,
|
|
...options
|
|
}
|
|
});
|
|
|
|
this.monitoringServer = null;
|
|
this.processingInterval = null;
|
|
this.healthInterval = null;
|
|
}
|
|
|
|
// ========================================
|
|
// DÉMARRAGE ET ARRÊT SPÉCIALISÉS
|
|
// ========================================
|
|
|
|
/**
|
|
* Démarrage AutoProcessor complet avec monitoring
|
|
*/
|
|
async start() {
|
|
if (this.isRunning) {
|
|
logSh('⚠️ AutoProcessor déjà en cours d\'exécution', 'WARNING');
|
|
return;
|
|
}
|
|
|
|
logSh('🤖 Démarrage AutoProcessor...', 'INFO');
|
|
|
|
try {
|
|
// 1. Charger la queue depuis Google Sheets
|
|
await this.populateQueueFromSheets();
|
|
|
|
// 2. Serveur de monitoring
|
|
await this.startMonitoringServer();
|
|
|
|
// 3. Démarrer le traitement avec batches
|
|
this.startBatchProcessing();
|
|
|
|
// 4. Monitoring périodique
|
|
this.startHealthMonitoring();
|
|
|
|
this.isRunning = true;
|
|
this.startTime = new Date();
|
|
|
|
logSh(`✅ AutoProcessor démarré: ${this.stats.itemsQueued} éléments en queue`, 'INFO');
|
|
logSh(`📊 Monitoring sur http://localhost:${this.config.monitoringPort}`, 'INFO');
|
|
|
|
} catch (error) {
|
|
logSh(`❌ Erreur démarrage AutoProcessor: ${error.message}`, 'ERROR');
|
|
await this.stop();
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Arrêt AutoProcessor complet
|
|
*/
|
|
async stop() {
|
|
if (!this.isRunning) return;
|
|
|
|
logSh('🛑 Arrêt AutoProcessor...', 'INFO');
|
|
|
|
try {
|
|
this.isRunning = false;
|
|
|
|
// Arrêter la boucle de traitement
|
|
if (this.processingInterval) {
|
|
clearInterval(this.processingInterval);
|
|
this.processingInterval = null;
|
|
}
|
|
|
|
// Attendre la fin du traitement en cours
|
|
if (this.currentRow) {
|
|
logSh('⏳ Attente fin traitement en cours...', 'INFO');
|
|
await this.waitForCurrentProcessing();
|
|
}
|
|
|
|
// Arrêter monitoring
|
|
if (this.healthInterval) {
|
|
clearInterval(this.healthInterval);
|
|
this.healthInterval = null;
|
|
}
|
|
|
|
// Arrêter serveur monitoring
|
|
if (this.monitoringServer) {
|
|
await new Promise((resolve) => {
|
|
this.monitoringServer.close(() => resolve());
|
|
});
|
|
this.monitoringServer = null;
|
|
}
|
|
|
|
// Sauvegarder progression
|
|
await this.saveProgress();
|
|
|
|
logSh('✅ AutoProcessor arrêté', 'INFO');
|
|
|
|
} catch (error) {
|
|
logSh(`⚠️ Erreur arrêt AutoProcessor: ${error.message}`, 'WARNING');
|
|
}
|
|
}
|
|
|
|
// ========================================
|
|
// TRAITEMENT BATCH SPÉCIALISÉ
|
|
// ========================================
|
|
|
|
/**
|
|
* Démarre le traitement par batches
|
|
*/
|
|
startBatchProcessing() {
|
|
if (this.queue.length === 0) {
|
|
logSh('⚠️ Queue vide, pas de traitement à démarrer', 'WARNING');
|
|
return;
|
|
}
|
|
|
|
logSh('🔄 Démarrage traitement par batches...', 'INFO');
|
|
|
|
// Traitement immédiat du premier batch
|
|
setTimeout(() => {
|
|
this.processNextBatch();
|
|
}, 1000);
|
|
|
|
// Puis traitement périodique
|
|
this.processingInterval = setInterval(() => {
|
|
if (!this.isPaused) {
|
|
this.processNextBatch();
|
|
}
|
|
}, this.config.delayBetweenBatches);
|
|
}
|
|
|
|
/**
|
|
* Traite le prochain batch
|
|
*/
|
|
async processNextBatch() {
|
|
if (this.isPaused || !this.isRunning || this.currentRow) {
|
|
return;
|
|
}
|
|
|
|
const pendingItems = this.queue.filter(item => item.status === 'pending');
|
|
if (pendingItems.length === 0) {
|
|
logSh('✅ Tous les éléments ont été traités', 'INFO');
|
|
await this.complete();
|
|
return;
|
|
}
|
|
|
|
const batchItems = pendingItems.slice(0, this.config.batchSize);
|
|
logSh(`🚀 Traitement batch: ${batchItems.length} éléments`, 'INFO');
|
|
|
|
try {
|
|
for (const item of batchItems) {
|
|
if (!this.isRunning) break;
|
|
|
|
await this.processItem(item);
|
|
|
|
if (this.config.delayBetweenItems > 0) {
|
|
await this.sleep(this.config.delayBetweenItems);
|
|
}
|
|
}
|
|
|
|
logSh(`✅ Batch terminé: ${batchItems.length} éléments traités`, 'INFO');
|
|
|
|
} catch (error) {
|
|
logSh(`❌ Erreur traitement batch: ${error.message}`, 'ERROR');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Configuration spécifique AutoProcessor
|
|
*/
|
|
buildRowConfig(rowNumber, data = null) {
|
|
return {
|
|
rowNumber,
|
|
selectiveStack: this.config.selective,
|
|
adversarialMode: this.config.adversarial,
|
|
humanSimulationMode: this.config.humanSimulation,
|
|
patternBreakingMode: this.config.patternBreaking,
|
|
source: `auto_processor_row_${rowNumber}`
|
|
};
|
|
}
|
|
|
|
// ========================================
|
|
// SERVEUR MONITORING
|
|
// ========================================
|
|
|
|
/**
|
|
* Démarre le serveur de monitoring
|
|
*/
|
|
async startMonitoringServer() {
|
|
const express = require('express');
|
|
const app = express();
|
|
|
|
app.use(express.json());
|
|
|
|
// Page de status principale
|
|
app.get('/', (req, res) => {
|
|
res.send(this.generateStatusPage());
|
|
});
|
|
|
|
// API status JSON
|
|
app.get('/api/status', (req, res) => {
|
|
res.json(this.getDetailedStatus());
|
|
});
|
|
|
|
// API stats JSON
|
|
app.get('/api/stats', (req, res) => {
|
|
res.json({
|
|
success: true,
|
|
stats: { ...this.stats },
|
|
queue: {
|
|
total: this.queue.length,
|
|
pending: this.queue.filter(i => i.status === 'pending').length,
|
|
processing: this.queue.filter(i => i.status === 'processing').length,
|
|
completed: this.queue.filter(i => i.status === 'completed').length,
|
|
failed: this.queue.filter(i => i.status === 'failed').length
|
|
},
|
|
timestamp: new Date().toISOString()
|
|
});
|
|
});
|
|
|
|
// Actions de contrôle
|
|
app.post('/api/pause', (req, res) => {
|
|
this.pauseProcessing();
|
|
res.json({ success: true, message: 'Traitement mis en pause' });
|
|
});
|
|
|
|
app.post('/api/resume', (req, res) => {
|
|
this.resumeProcessing();
|
|
res.json({ success: true, message: 'Traitement repris' });
|
|
});
|
|
|
|
// 404 pour autres routes
|
|
app.use('*', (req, res) => {
|
|
res.status(404).json({
|
|
success: false,
|
|
error: 'Route non trouvée',
|
|
mode: 'AUTO',
|
|
message: 'Interface de monitoring en lecture seule'
|
|
});
|
|
});
|
|
|
|
// Démarrage serveur
|
|
return new Promise((resolve, reject) => {
|
|
try {
|
|
this.monitoringServer = app.listen(this.config.monitoringPort, '0.0.0.0', () => {
|
|
logSh(`📊 Serveur monitoring démarré sur http://localhost:${this.config.monitoringPort}`, 'DEBUG');
|
|
resolve();
|
|
});
|
|
|
|
this.monitoringServer.on('error', (error) => {
|
|
reject(error);
|
|
});
|
|
|
|
} catch (error) {
|
|
reject(error);
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Génère la page de status HTML
|
|
*/
|
|
generateStatusPage() {
|
|
const uptime = Math.floor((Date.now() - this.stats.startTime) / 1000);
|
|
const progress = this.stats.itemsQueued > 0 ?
|
|
Math.round((this.stats.itemsProcessed / this.stats.itemsQueued) * 100) : 0;
|
|
|
|
const pendingCount = this.queue.filter(i => i.status === 'pending').length;
|
|
const completedCount = this.queue.filter(i => i.status === 'completed').length;
|
|
const failedCount = this.queue.filter(i => i.status === 'failed').length;
|
|
|
|
return `
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>SEO Generator - Mode AUTO</title>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<style>
|
|
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; margin: 0; padding: 20px; background: linear-gradient(135deg, #4facfe 0%, #00f2fe 100%); min-height: 100vh; }
|
|
.container { max-width: 1200px; margin: 0 auto; background: white; padding: 30px; border-radius: 15px; box-shadow: 0 10px 30px rgba(0,0,0,0.1); }
|
|
.header { text-align: center; margin-bottom: 40px; }
|
|
.header h1 { color: #2d3748; margin-bottom: 10px; }
|
|
.mode-badge { background: #4299e1; color: white; padding: 8px 16px; border-radius: 20px; font-weight: bold; }
|
|
.progress { background: #e2e8f0; height: 20px; border-radius: 10px; margin: 20px 0; overflow: hidden; }
|
|
.progress-bar { background: linear-gradient(90deg, #4facfe, #00f2fe); height: 100%; width: ${progress}%; transition: width 0.3s; }
|
|
.stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px; margin-bottom: 40px; }
|
|
.stat-card { background: #f7fafc; padding: 20px; border-radius: 10px; text-align: center; border: 1px solid #e2e8f0; }
|
|
.stat-card.processing { background: #fef5e7; border-color: #f6ad55; }
|
|
.stat-card.completed { background: #f0fff4; border-color: #48bb78; }
|
|
.stat-card.failed { background: #fed7d7; border-color: #f56565; }
|
|
.stat-number { font-size: 2em; font-weight: bold; color: #2d3748; }
|
|
.stat-label { color: #718096; margin-top: 5px; }
|
|
.section { margin: 30px 0; padding: 25px; border: 1px solid #e2e8f0; border-radius: 10px; background: #f9f9f9; }
|
|
.button { display: inline-block; padding: 12px 24px; margin: 8px; background: linear-gradient(135deg, #4facfe, #00f2fe); color: white; text-decoration: none; border-radius: 8px; border: none; cursor: pointer; font-weight: 500; }
|
|
.alert { padding: 15px; margin: 20px 0; border-radius: 8px; border-left: 4px solid; }
|
|
.alert.info { background: #ebf8ff; border-color: #4299e1; color: #2a4365; }
|
|
.current-item { background: #e6fffa; padding: 15px; border-radius: 8px; border: 1px solid #38b2ac; margin: 15px 0; }
|
|
</style>
|
|
<script>
|
|
function refreshPage() { window.location.reload(); }
|
|
function pauseProcessing() {
|
|
fetch('/api/pause', { method: 'POST' })
|
|
.then(() => setTimeout(refreshPage, 1000));
|
|
}
|
|
function resumeProcessing() {
|
|
fetch('/api/resume', { method: 'POST' })
|
|
.then(() => setTimeout(refreshPage, 1000));
|
|
}
|
|
setInterval(refreshPage, 30000); // Auto-refresh 30s
|
|
</script>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<div class="header">
|
|
<h1>🤖 SEO Generator Server</h1>
|
|
<span class="mode-badge">MODE AUTO - REFACTORISÉ</span>
|
|
<p style="color: #718096; margin-top: 15px;">Traitement Automatique Google Sheets</p>
|
|
</div>
|
|
|
|
<div class="alert info">
|
|
<strong>🤖 Mode AUTO Actif</strong><br>
|
|
Traitement batch des Google Sheets • Interface monitoring lecture seule
|
|
</div>
|
|
|
|
<div class="progress">
|
|
<div class="progress-bar"></div>
|
|
</div>
|
|
<div style="text-align: center; margin-bottom: 20px; color: #4a5568;">
|
|
Progression: ${progress}% (${completedCount}/${this.stats.itemsQueued})
|
|
</div>
|
|
|
|
<div class="stats">
|
|
<div class="stat-card">
|
|
<div class="stat-number">${uptime}s</div>
|
|
<div class="stat-label">Uptime</div>
|
|
</div>
|
|
<div class="stat-card">
|
|
<div class="stat-number">${pendingCount}</div>
|
|
<div class="stat-label">En Attente</div>
|
|
</div>
|
|
<div class="stat-card processing">
|
|
<div class="stat-number">${this.currentRow ? '1' : '0'}</div>
|
|
<div class="stat-label">En Traitement</div>
|
|
</div>
|
|
<div class="stat-card completed">
|
|
<div class="stat-number">${completedCount}</div>
|
|
<div class="stat-label">Terminés</div>
|
|
</div>
|
|
<div class="stat-card failed">
|
|
<div class="stat-number">${failedCount}</div>
|
|
<div class="stat-label">Échecs</div>
|
|
</div>
|
|
<div class="stat-card">
|
|
<div class="stat-number">${this.stats.averageProcessingTime}ms</div>
|
|
<div class="stat-label">Temps Moyen</div>
|
|
</div>
|
|
</div>
|
|
|
|
${this.currentRow ? `
|
|
<div class="current-item">
|
|
<strong>🎯 Traitement en cours:</strong><br>
|
|
Ligne ${this.currentRow}<br>
|
|
</div>
|
|
` : ''}
|
|
|
|
<div class="section">
|
|
<h2>🎛️ Contrôles</h2>
|
|
${this.isPaused ?
|
|
'<button class="button" onclick="resumeProcessing()">▶️ Reprendre</button>' :
|
|
'<button class="button" onclick="pauseProcessing()">⏸️ Pause</button>'
|
|
}
|
|
<button class="button" onclick="refreshPage()">🔄 Actualiser</button>
|
|
<a href="/api/stats" target="_blank" class="button">📊 Stats JSON</a>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>📋 Configuration AUTO</h2>
|
|
<ul style="color: #4a5568; line-height: 1.6;">
|
|
<li><strong>Batch Size:</strong> ${this.config.batchSize} éléments</li>
|
|
<li><strong>Délai entre éléments:</strong> ${this.config.delayBetweenItems}ms</li>
|
|
<li><strong>Délai entre batches:</strong> ${this.config.delayBetweenBatches}ms</li>
|
|
<li><strong>Max Retries:</strong> ${this.config.maxRetries}</li>
|
|
<li><strong>Mode Selective:</strong> ${this.config.selective}</li>
|
|
<li><strong>Mode Adversarial:</strong> ${this.config.adversarial}</li>
|
|
<li><strong>Lignes:</strong> ${this.config.startRow} - ${this.config.endRow || '∞'}</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
`;
|
|
}
|
|
|
|
// ========================================
|
|
// CONTRÔLES SPÉCIFIQUES
|
|
// ========================================
|
|
|
|
/**
|
|
* Met en pause le traitement
|
|
*/
|
|
pauseProcessing() {
|
|
this.isPaused = true;
|
|
logSh('⏸️ Traitement AutoProcessor mis en pause', 'INFO');
|
|
}
|
|
|
|
/**
|
|
* Reprend le traitement
|
|
*/
|
|
resumeProcessing() {
|
|
this.isPaused = false;
|
|
logSh('▶️ Traitement AutoProcessor repris', 'INFO');
|
|
}
|
|
|
|
/**
|
|
* Attendre la fin du traitement actuel
|
|
*/
|
|
async waitForCurrentProcessing(timeout = 30000) {
|
|
const startWait = Date.now();
|
|
|
|
while (this.currentRow && (Date.now() - startWait) < timeout) {
|
|
await this.sleep(1000);
|
|
}
|
|
|
|
if (this.currentRow) {
|
|
logSh('⚠️ Timeout attente fin traitement', 'WARNING');
|
|
}
|
|
}
|
|
|
|
// ========================================
|
|
// MONITORING ET HEALTH
|
|
// ========================================
|
|
|
|
/**
|
|
* Démarre le monitoring de santé
|
|
*/
|
|
startHealthMonitoring() {
|
|
const HEALTH_INTERVAL = 60000; // 1 minute
|
|
|
|
this.healthInterval = setInterval(() => {
|
|
this.performHealthCheck();
|
|
}, HEALTH_INTERVAL);
|
|
|
|
logSh('💓 Health monitoring AutoProcessor démarré', 'DEBUG');
|
|
}
|
|
|
|
/**
|
|
* Health check périodique
|
|
*/
|
|
performHealthCheck() {
|
|
const memUsage = process.memoryUsage();
|
|
const queueStatus = {
|
|
pending: this.queue.filter(i => i.status === 'pending').length,
|
|
completed: this.queue.filter(i => i.status === 'completed').length,
|
|
failed: this.queue.filter(i => i.status === 'failed').length
|
|
};
|
|
|
|
logSh(`💓 AutoProcessor Health - Queue: ${queueStatus.pending}P/${queueStatus.completed}C/${queueStatus.failed}F | RAM: ${Math.round(memUsage.rss / 1024 / 1024)}MB`, 'TRACE');
|
|
|
|
// Alertes
|
|
if (memUsage.rss > 2 * 1024 * 1024 * 1024) { // > 2GB
|
|
logSh('⚠️ Utilisation mémoire très élevée', 'WARNING');
|
|
}
|
|
|
|
if (this.stats.itemsFailed > this.stats.itemsProcessed * 0.5) {
|
|
logSh('⚠️ Taux d\'échec élevé détecté', 'WARNING');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Retourne le status détaillé
|
|
*/
|
|
getDetailedStatus() {
|
|
const baseStatus = this.getStatus();
|
|
return {
|
|
success: true,
|
|
mode: 'AUTO',
|
|
isRunning: this.isRunning,
|
|
state: {
|
|
isRunning: this.isRunning,
|
|
isPaused: this.isPaused,
|
|
currentRow: this.currentRow,
|
|
startTime: this.startTime,
|
|
lastActivity: Date.now()
|
|
},
|
|
stats: {
|
|
...this.stats,
|
|
uptime: Date.now() - this.stats.startTime
|
|
},
|
|
queue: {
|
|
total: this.queue.length,
|
|
pending: this.queue.filter(i => i.status === 'pending').length,
|
|
processing: this.queue.filter(i => i.status === 'processing').length,
|
|
completed: this.queue.filter(i => i.status === 'completed').length,
|
|
failed: this.queue.filter(i => i.status === 'failed').length
|
|
},
|
|
config: { ...this.config },
|
|
currentItem: this.currentRow ? {
|
|
rowNumber: this.currentRow
|
|
} : null,
|
|
urls: {
|
|
monitoring: `http://localhost:${this.config.monitoringPort}`,
|
|
api: `http://localhost:${this.config.monitoringPort}/api/stats`
|
|
},
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// ========================================
|
|
// PERSISTANCE
|
|
// ========================================
|
|
|
|
/**
|
|
* Sauvegarde la progression
|
|
*/
|
|
async saveProgress() {
|
|
try {
|
|
const fs = require('fs').promises;
|
|
const progressFile = path.join(__dirname, '../../auto-processor-progress.json');
|
|
const progress = {
|
|
processedRows: this.processedItems.map(item => item.rowNumber),
|
|
failedRows: this.failedItems.map(item => ({
|
|
rowNumber: item.rowNumber,
|
|
error: item.error,
|
|
attempts: item.attempts
|
|
})),
|
|
stats: { ...this.stats },
|
|
lastSaved: Date.now(),
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
|
|
await fs.writeFile(progressFile, JSON.stringify(progress, null, 2));
|
|
|
|
} catch (error) {
|
|
logSh(`⚠️ Erreur sauvegarde progression: ${error.message}`, 'WARNING');
|
|
}
|
|
}
|
|
}
|
|
|
|
// ============= EXPORTS =============
|
|
module.exports = { AutoProcessor }; |