Trouve Alexis
870cfb0340
- Add intermediate saves (v1.0-v1.4) to Generated_Articles_Versioned - Fix compiled_text pipeline (generatedTexts object structure) - Add /api/workflow-modulaire endpoint with version tracking - Create test-modulaire.html interface with real-time logs - Support parent-child linking via Parent_Article_ID
815 lines
28 KiB
JavaScript
815 lines
28 KiB
JavaScript
// ========================================
|
|
// FICHIER: AutoProcessor.js
|
|
// RESPONSABILITÉ: Mode AUTO - Traitement Batch Google Sheets
|
|
// FONCTIONNALITÉS: Processing queue, scheduling, monitoring
|
|
// ========================================
|
|
|
|
const { logSh } = require('../ErrorReporting');
|
|
const { handleModularWorkflow } = require('../main_modulaire');
|
|
const { readInstructionsData } = require('../BrainConfig');
|
|
|
|
/**
|
|
* PROCESSEUR MODE AUTO
|
|
* Traitement automatique et séquentiel des lignes Google Sheets
|
|
*/
|
|
class AutoProcessor {
|
|
|
|
constructor(options = {}) {
|
|
this.config = {
|
|
batchSize: options.batchSize || 5, // Lignes par batch
|
|
delayBetweenItems: options.delayBetweenItems || 2000, // 2s entre chaque ligne
|
|
delayBetweenBatches: options.delayBetweenBatches || 30000, // 30s entre batches
|
|
maxRetries: options.maxRetries || 3,
|
|
startRow: options.startRow || 2,
|
|
endRow: options.endRow || null, // null = jusqu'à la fin
|
|
autoMode: options.autoMode || 'standardEnhancement', // Config par défaut
|
|
monitoringPort: options.monitoringPort || 3001,
|
|
...options
|
|
};
|
|
|
|
this.processingQueue = [];
|
|
this.processedItems = [];
|
|
this.failedItems = [];
|
|
|
|
this.state = {
|
|
isProcessing: false,
|
|
isPaused: false,
|
|
currentItem: null,
|
|
startTime: null,
|
|
lastActivity: null,
|
|
totalProcessed: 0,
|
|
totalErrors: 0
|
|
};
|
|
|
|
this.stats = {
|
|
itemsQueued: 0,
|
|
itemsProcessed: 0,
|
|
itemsFailed: 0,
|
|
averageProcessingTime: 0,
|
|
totalProcessingTime: 0,
|
|
startTime: Date.now(),
|
|
lastProcessedAt: null
|
|
};
|
|
|
|
this.monitoringServer = null;
|
|
this.processingInterval = null;
|
|
this.isRunning = false;
|
|
}
|
|
|
|
// ========================================
|
|
// DÉMARRAGE ET ARRÊT
|
|
// ========================================
|
|
|
|
/**
|
|
* Démarre le processeur AUTO complet
|
|
*/
|
|
async start() {
|
|
if (this.isRunning) {
|
|
logSh('⚠️ AutoProcessor déjà en cours d\'exécution', 'WARNING');
|
|
return;
|
|
}
|
|
|
|
logSh('🤖 Démarrage AutoProcessor...', 'INFO');
|
|
|
|
try {
|
|
// 1. Charger la queue depuis Google Sheets
|
|
await this.loadProcessingQueue();
|
|
|
|
// 2. Serveur de monitoring (lecture seule)
|
|
await this.startMonitoringServer();
|
|
|
|
// 3. Démarrer le traitement
|
|
this.startProcessingLoop();
|
|
|
|
// 4. Monitoring périodique
|
|
this.startHealthMonitoring();
|
|
|
|
this.isRunning = true;
|
|
this.state.startTime = Date.now();
|
|
|
|
logSh(`✅ AutoProcessor démarré: ${this.stats.itemsQueued} éléments en queue`, 'INFO');
|
|
logSh(`📊 Monitoring sur http://localhost:${this.config.monitoringPort}`, 'INFO');
|
|
|
|
} catch (error) {
|
|
logSh(`❌ Erreur démarrage AutoProcessor: ${error.message}`, 'ERROR');
|
|
await this.stop();
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Arrête le processeur AUTO
|
|
*/
|
|
async stop() {
|
|
if (!this.isRunning) return;
|
|
|
|
logSh('🛑 Arrêt AutoProcessor...', 'INFO');
|
|
|
|
try {
|
|
// Marquer comme en arrêt
|
|
this.isRunning = false;
|
|
|
|
// Arrêter la boucle de traitement
|
|
if (this.processingInterval) {
|
|
clearInterval(this.processingInterval);
|
|
this.processingInterval = null;
|
|
}
|
|
|
|
// Attendre la fin du traitement en cours
|
|
if (this.state.isProcessing) {
|
|
logSh('⏳ Attente fin traitement en cours...', 'INFO');
|
|
await this.waitForCurrentProcessing();
|
|
}
|
|
|
|
// Arrêter monitoring
|
|
if (this.healthInterval) {
|
|
clearInterval(this.healthInterval);
|
|
this.healthInterval = null;
|
|
}
|
|
|
|
// Arrêter serveur monitoring
|
|
if (this.monitoringServer) {
|
|
await new Promise((resolve) => {
|
|
this.monitoringServer.close(() => resolve());
|
|
});
|
|
this.monitoringServer = null;
|
|
}
|
|
|
|
// Sauvegarder progression
|
|
await this.saveProgress();
|
|
|
|
logSh('✅ AutoProcessor arrêté', 'INFO');
|
|
|
|
} catch (error) {
|
|
logSh(`⚠️ Erreur arrêt AutoProcessor: ${error.message}`, 'WARNING');
|
|
}
|
|
}
|
|
|
|
// ========================================
|
|
// CHARGEMENT QUEUE
|
|
// ========================================
|
|
|
|
/**
|
|
* Charge la queue de traitement depuis Google Sheets
|
|
*/
|
|
async loadProcessingQueue() {
|
|
logSh('📋 Chargement queue depuis Google Sheets...', 'INFO');
|
|
|
|
try {
|
|
// Restaurer progression si disponible - TEMPORAIREMENT DÉSACTIVÉ
|
|
// const savedProgress = await this.loadProgress();
|
|
// const processedRows = new Set(savedProgress?.processedRows || []);
|
|
const processedRows = new Set(); // Ignore la progression sauvegardée
|
|
|
|
// Scanner les lignes disponibles
|
|
let currentRow = this.config.startRow;
|
|
let consecutiveEmptyRows = 0;
|
|
const maxEmptyRows = 5; // Arrêt après 5 lignes vides consécutives
|
|
|
|
while (currentRow <= (this.config.endRow || 10)) { // 🔧 LIMITE MAX POUR ÉVITER BOUCLE INFINIE
|
|
// Vérifier limite max si définie
|
|
if (this.config.endRow && currentRow > this.config.endRow) {
|
|
break;
|
|
}
|
|
|
|
try {
|
|
// Tenter de lire la ligne
|
|
const csvData = await readInstructionsData(currentRow);
|
|
|
|
if (!csvData || !csvData.mc0) {
|
|
// Ligne vide ou invalide
|
|
consecutiveEmptyRows++;
|
|
if (consecutiveEmptyRows >= maxEmptyRows) {
|
|
logSh(`🛑 Arrêt scan après ${maxEmptyRows} lignes vides consécutives à partir de la ligne ${currentRow - maxEmptyRows + 1}`, 'INFO');
|
|
break;
|
|
}
|
|
} else {
|
|
// Ligne valide trouvée
|
|
consecutiveEmptyRows = 0;
|
|
|
|
// Ajouter à la queue si pas déjà traitée
|
|
if (!processedRows.has(currentRow)) {
|
|
this.processingQueue.push({
|
|
rowNumber: currentRow,
|
|
data: csvData,
|
|
attempts: 0,
|
|
status: 'pending',
|
|
addedAt: Date.now()
|
|
});
|
|
} else {
|
|
logSh(`⏭️ Ligne ${currentRow} déjà traitée, ignorée`, 'DEBUG');
|
|
}
|
|
}
|
|
|
|
} catch (error) {
|
|
// Erreur de lecture = ligne probablement vide
|
|
consecutiveEmptyRows++;
|
|
if (consecutiveEmptyRows >= maxEmptyRows) {
|
|
break;
|
|
}
|
|
}
|
|
|
|
currentRow++;
|
|
}
|
|
|
|
this.stats.itemsQueued = this.processingQueue.length;
|
|
|
|
logSh(`📊 Queue chargée: ${this.stats.itemsQueued} éléments (lignes ${this.config.startRow}-${currentRow - 1})`, 'INFO');
|
|
|
|
if (this.stats.itemsQueued === 0) {
|
|
logSh('⚠️ Aucun élément à traiter trouvé', 'WARNING');
|
|
}
|
|
|
|
} catch (error) {
|
|
logSh(`❌ Erreur chargement queue: ${error.message}`, 'ERROR');
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
// ========================================
|
|
// BOUCLE DE TRAITEMENT
|
|
// ========================================
|
|
|
|
/**
|
|
* Démarre la boucle principale de traitement
|
|
*/
|
|
startProcessingLoop() {
|
|
if (this.processingQueue.length === 0) {
|
|
logSh('⚠️ Queue vide, pas de traitement à démarrer', 'WARNING');
|
|
return;
|
|
}
|
|
|
|
logSh('🔄 Démarrage boucle de traitement...', 'INFO');
|
|
|
|
// Traitement immédiat du premier batch
|
|
setTimeout(() => {
|
|
this.processNextBatch();
|
|
}, 1000);
|
|
|
|
// Puis traitement périodique
|
|
this.processingInterval = setInterval(() => {
|
|
if (!this.state.isProcessing && !this.state.isPaused) {
|
|
this.processNextBatch();
|
|
}
|
|
}, this.config.delayBetweenBatches);
|
|
}
|
|
|
|
/**
|
|
* Traite le prochain batch d'éléments
|
|
*/
|
|
async processNextBatch() {
|
|
if (this.state.isProcessing || this.state.isPaused || !this.isRunning) {
|
|
return;
|
|
}
|
|
|
|
// Vérifier s'il reste des éléments
|
|
const pendingItems = this.processingQueue.filter(item => item.status === 'pending');
|
|
if (pendingItems.length === 0) {
|
|
logSh('✅ Tous les éléments ont été traités', 'INFO');
|
|
await this.completeProcessing();
|
|
return;
|
|
}
|
|
|
|
// Prendre le prochain batch
|
|
const batchItems = pendingItems.slice(0, this.config.batchSize);
|
|
|
|
logSh(`🚀 Traitement batch: ${batchItems.length} éléments`, 'INFO');
|
|
this.state.isProcessing = true;
|
|
this.state.lastActivity = Date.now();
|
|
|
|
try {
|
|
// Traiter chaque élément du batch séquentiellement
|
|
for (const item of batchItems) {
|
|
if (!this.isRunning) break; // Arrêt demandé
|
|
|
|
await this.processItem(item);
|
|
|
|
// Délai entre éléments
|
|
if (this.config.delayBetweenItems > 0) {
|
|
await this.sleep(this.config.delayBetweenItems);
|
|
}
|
|
}
|
|
|
|
logSh(`✅ Batch terminé: ${batchItems.length} éléments traités`, 'INFO');
|
|
|
|
} catch (error) {
|
|
logSh(`❌ Erreur traitement batch: ${error.message}`, 'ERROR');
|
|
} finally {
|
|
this.state.isProcessing = false;
|
|
this.state.currentItem = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Traite un élément individuel
|
|
*/
|
|
async processItem(item) {
|
|
const startTime = Date.now();
|
|
this.state.currentItem = item;
|
|
|
|
logSh(`🎯 Traitement ligne ${item.rowNumber}: ${item.data.mc0}`, 'INFO');
|
|
|
|
try {
|
|
item.status = 'processing';
|
|
item.attempts++;
|
|
item.startedAt = startTime;
|
|
|
|
// Configuration de traitement automatique
|
|
const processingConfig = {
|
|
rowNumber: item.rowNumber,
|
|
selectiveStack: this.config.autoMode,
|
|
adversarialMode: 'light',
|
|
humanSimulationMode: 'lightSimulation',
|
|
patternBreakingMode: 'standardPatternBreaking',
|
|
source: `auto_processor_row_${item.rowNumber}`
|
|
};
|
|
|
|
// Exécution du workflow modulaire
|
|
const result = await handleModularWorkflow(processingConfig);
|
|
|
|
const duration = Date.now() - startTime;
|
|
|
|
// Succès
|
|
item.status = 'completed';
|
|
item.completedAt = Date.now();
|
|
item.duration = duration;
|
|
item.result = {
|
|
stats: result.stats,
|
|
success: true
|
|
};
|
|
|
|
this.processedItems.push(item);
|
|
this.stats.itemsProcessed++;
|
|
this.stats.totalProcessingTime += duration;
|
|
this.stats.averageProcessingTime = Math.round(this.stats.totalProcessingTime / this.stats.itemsProcessed);
|
|
this.stats.lastProcessedAt = Date.now();
|
|
|
|
logSh(`✅ Ligne ${item.rowNumber} terminée (${duration}ms) - ${result.stats.totalModifications || 0} modifications`, 'INFO');
|
|
|
|
} catch (error) {
|
|
const duration = Date.now() - startTime;
|
|
|
|
// Échec
|
|
item.status = 'failed';
|
|
item.failedAt = Date.now();
|
|
item.duration = duration;
|
|
item.error = error.message;
|
|
|
|
this.stats.totalErrors++;
|
|
|
|
logSh(`❌ Échec ligne ${item.rowNumber} (tentative ${item.attempts}/${this.config.maxRetries}): ${error.message}`, 'ERROR');
|
|
|
|
// Retry si possible
|
|
if (item.attempts < this.config.maxRetries) {
|
|
logSh(`🔄 Retry programmé pour ligne ${item.rowNumber}`, 'INFO');
|
|
item.status = 'pending'; // Remettre en queue
|
|
} else {
|
|
logSh(`💀 Ligne ${item.rowNumber} abandonnée après ${item.attempts} tentatives`, 'WARNING');
|
|
this.failedItems.push(item);
|
|
this.stats.itemsFailed++;
|
|
}
|
|
}
|
|
|
|
// Sauvegarder progression périodiquement
|
|
if (this.stats.itemsProcessed % 5 === 0) {
|
|
await this.saveProgress();
|
|
}
|
|
}
|
|
|
|
// ========================================
|
|
// SERVEUR MONITORING
|
|
// ========================================
|
|
|
|
/**
|
|
* Démarre le serveur de monitoring (lecture seule)
|
|
*/
|
|
async startMonitoringServer() {
|
|
const express = require('express');
|
|
const app = express();
|
|
|
|
app.use(express.json());
|
|
|
|
// Page de status principale
|
|
app.get('/', (req, res) => {
|
|
res.send(this.generateStatusPage());
|
|
});
|
|
|
|
// API status JSON
|
|
app.get('/api/status', (req, res) => {
|
|
res.json(this.getDetailedStatus());
|
|
});
|
|
|
|
// API stats JSON
|
|
app.get('/api/stats', (req, res) => {
|
|
res.json({
|
|
success: true,
|
|
stats: { ...this.stats },
|
|
queue: {
|
|
total: this.processingQueue.length,
|
|
pending: this.processingQueue.filter(i => i.status === 'pending').length,
|
|
processing: this.processingQueue.filter(i => i.status === 'processing').length,
|
|
completed: this.processingQueue.filter(i => i.status === 'completed').length,
|
|
failed: this.processingQueue.filter(i => i.status === 'failed').length
|
|
},
|
|
timestamp: new Date().toISOString()
|
|
});
|
|
});
|
|
|
|
// Actions de contrôle (limitées)
|
|
app.post('/api/pause', (req, res) => {
|
|
this.pauseProcessing();
|
|
res.json({ success: true, message: 'Traitement mis en pause' });
|
|
});
|
|
|
|
app.post('/api/resume', (req, res) => {
|
|
this.resumeProcessing();
|
|
res.json({ success: true, message: 'Traitement repris' });
|
|
});
|
|
|
|
// 404 pour autres routes
|
|
app.use('*', (req, res) => {
|
|
res.status(404).json({
|
|
success: false,
|
|
error: 'Route non trouvée',
|
|
mode: 'AUTO',
|
|
message: 'Interface de monitoring en lecture seule'
|
|
});
|
|
});
|
|
|
|
// Démarrage serveur
|
|
return new Promise((resolve, reject) => {
|
|
try {
|
|
this.monitoringServer = app.listen(this.config.monitoringPort, '0.0.0.0', () => {
|
|
logSh(`📊 Serveur monitoring démarré sur http://localhost:${this.config.monitoringPort}`, 'DEBUG');
|
|
resolve();
|
|
});
|
|
|
|
this.monitoringServer.on('error', (error) => {
|
|
reject(error);
|
|
});
|
|
|
|
} catch (error) {
|
|
reject(error);
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Génère la page de status HTML
|
|
*/
|
|
generateStatusPage() {
|
|
const uptime = Math.floor((Date.now() - this.stats.startTime) / 1000);
|
|
const progress = this.stats.itemsQueued > 0 ?
|
|
Math.round((this.stats.itemsProcessed / this.stats.itemsQueued) * 100) : 0;
|
|
|
|
const pendingCount = this.processingQueue.filter(i => i.status === 'pending').length;
|
|
const completedCount = this.processingQueue.filter(i => i.status === 'completed').length;
|
|
const failedCount = this.processingQueue.filter(i => i.status === 'failed').length;
|
|
|
|
return `
|
|
<!DOCTYPE html>
|
|
<html>
|
|
<head>
|
|
<title>SEO Generator - Mode AUTO</title>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<style>
|
|
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; margin: 0; padding: 20px; background: linear-gradient(135deg, #4facfe 0%, #00f2fe 100%); min-height: 100vh; }
|
|
.container { max-width: 1200px; margin: 0 auto; background: white; padding: 30px; border-radius: 15px; box-shadow: 0 10px 30px rgba(0,0,0,0.1); }
|
|
.header { text-align: center; margin-bottom: 40px; }
|
|
.header h1 { color: #2d3748; margin-bottom: 10px; }
|
|
.mode-badge { background: #4299e1; color: white; padding: 8px 16px; border-radius: 20px; font-weight: bold; }
|
|
.progress { background: #e2e8f0; height: 20px; border-radius: 10px; margin: 20px 0; overflow: hidden; }
|
|
.progress-bar { background: linear-gradient(90deg, #4facfe, #00f2fe); height: 100%; width: ${progress}%; transition: width 0.3s; }
|
|
.stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); gap: 20px; margin-bottom: 40px; }
|
|
.stat-card { background: #f7fafc; padding: 20px; border-radius: 10px; text-align: center; border: 1px solid #e2e8f0; }
|
|
.stat-card.processing { background: #fef5e7; border-color: #f6ad55; }
|
|
.stat-card.completed { background: #f0fff4; border-color: #48bb78; }
|
|
.stat-card.failed { background: #fed7d7; border-color: #f56565; }
|
|
.stat-number { font-size: 2em; font-weight: bold; color: #2d3748; }
|
|
.stat-label { color: #718096; margin-top: 5px; }
|
|
.section { margin: 30px 0; padding: 25px; border: 1px solid #e2e8f0; border-radius: 10px; background: #f9f9f9; }
|
|
.button { display: inline-block; padding: 12px 24px; margin: 8px; background: linear-gradient(135deg, #4facfe, #00f2fe); color: white; text-decoration: none; border-radius: 8px; border: none; cursor: pointer; font-weight: 500; }
|
|
.alert { padding: 15px; margin: 20px 0; border-radius: 8px; border-left: 4px solid; }
|
|
.alert.info { background: #ebf8ff; border-color: #4299e1; color: #2a4365; }
|
|
.alert.warning { background: #fefcbf; border-color: #f6ad55; color: #744210; }
|
|
.current-item { background: #e6fffa; padding: 15px; border-radius: 8px; border: 1px solid #38b2ac; margin: 15px 0; }
|
|
</style>
|
|
<script>
|
|
function refreshPage() { window.location.reload(); }
|
|
function pauseProcessing() {
|
|
fetch('/api/pause', { method: 'POST' })
|
|
.then(() => setTimeout(refreshPage, 1000));
|
|
}
|
|
function resumeProcessing() {
|
|
fetch('/api/resume', { method: 'POST' })
|
|
.then(() => setTimeout(refreshPage, 1000));
|
|
}
|
|
setInterval(refreshPage, 30000); // Auto-refresh 30s
|
|
</script>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<div class="header">
|
|
<h1>🤖 SEO Generator Server</h1>
|
|
<span class="mode-badge">MODE AUTO</span>
|
|
<p style="color: #718096; margin-top: 15px;">Traitement Automatique Google Sheets</p>
|
|
</div>
|
|
|
|
<div class="alert info">
|
|
<strong>🤖 Mode AUTO Actif</strong><br>
|
|
Traitement batch des Google Sheets • Interface monitoring lecture seule
|
|
</div>
|
|
|
|
<div class="progress">
|
|
<div class="progress-bar"></div>
|
|
</div>
|
|
<div style="text-align: center; margin-bottom: 20px; color: #4a5568;">
|
|
Progression: ${progress}% (${completedCount}/${this.stats.itemsQueued})
|
|
</div>
|
|
|
|
<div class="stats">
|
|
<div class="stat-card">
|
|
<div class="stat-number">${uptime}s</div>
|
|
<div class="stat-label">Uptime</div>
|
|
</div>
|
|
<div class="stat-card">
|
|
<div class="stat-number">${pendingCount}</div>
|
|
<div class="stat-label">En Attente</div>
|
|
</div>
|
|
<div class="stat-card processing">
|
|
<div class="stat-number">${this.state.isProcessing ? '1' : '0'}</div>
|
|
<div class="stat-label">En Traitement</div>
|
|
</div>
|
|
<div class="stat-card completed">
|
|
<div class="stat-number">${completedCount}</div>
|
|
<div class="stat-label">Terminés</div>
|
|
</div>
|
|
<div class="stat-card failed">
|
|
<div class="stat-number">${failedCount}</div>
|
|
<div class="stat-label">Échecs</div>
|
|
</div>
|
|
<div class="stat-card">
|
|
<div class="stat-number">${this.stats.averageProcessingTime}ms</div>
|
|
<div class="stat-label">Temps Moyen</div>
|
|
</div>
|
|
</div>
|
|
|
|
${this.state.currentItem ? `
|
|
<div class="current-item">
|
|
<strong>🎯 Traitement en cours:</strong><br>
|
|
Ligne ${this.state.currentItem.rowNumber}: ${this.state.currentItem.data.mc0}<br>
|
|
<small>Tentative ${this.state.currentItem.attempts}/${this.config.maxRetries}</small>
|
|
</div>
|
|
` : ''}
|
|
|
|
<div class="section">
|
|
<h2>🎛️ Contrôles</h2>
|
|
${this.state.isPaused ?
|
|
'<button class="button" onclick="resumeProcessing()">▶️ Reprendre</button>' :
|
|
'<button class="button" onclick="pauseProcessing()">⏸️ Pause</button>'
|
|
}
|
|
<button class="button" onclick="refreshPage()">🔄 Actualiser</button>
|
|
<a href="/api/stats" target="_blank" class="button">📊 Stats JSON</a>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>📋 Configuration</h2>
|
|
<ul style="color: #4a5568; line-height: 1.6;">
|
|
<li><strong>Batch Size:</strong> ${this.config.batchSize} éléments</li>
|
|
<li><strong>Délai entre éléments:</strong> ${this.config.delayBetweenItems}ms</li>
|
|
<li><strong>Délai entre batches:</strong> ${this.config.delayBetweenBatches}ms</li>
|
|
<li><strong>Max Retries:</strong> ${this.config.maxRetries}</li>
|
|
<li><strong>Mode Auto:</strong> ${this.config.autoMode}</li>
|
|
<li><strong>Lignes:</strong> ${this.config.startRow} - ${this.config.endRow || '∞'}</li>
|
|
</ul>
|
|
</div>
|
|
</div>
|
|
</body>
|
|
</html>
|
|
`;
|
|
}
|
|
|
|
// ========================================
|
|
// CONTRÔLES ET ÉTAT
|
|
// ========================================
|
|
|
|
/**
|
|
* Met en pause le traitement
|
|
*/
|
|
pauseProcessing() {
|
|
this.state.isPaused = true;
|
|
logSh('⏸️ Traitement mis en pause', 'INFO');
|
|
}
|
|
|
|
/**
|
|
* Reprend le traitement
|
|
*/
|
|
resumeProcessing() {
|
|
this.state.isPaused = false;
|
|
logSh('▶️ Traitement repris', 'INFO');
|
|
}
|
|
|
|
/**
|
|
* Vérifie si le processeur est en cours de traitement
|
|
*/
|
|
isProcessing() {
|
|
return this.state.isProcessing;
|
|
}
|
|
|
|
/**
|
|
* Attendre la fin du traitement actuel
|
|
*/
|
|
async waitForCurrentProcessing(timeout = 30000) {
|
|
const startWait = Date.now();
|
|
|
|
while (this.state.isProcessing && (Date.now() - startWait) < timeout) {
|
|
await this.sleep(1000);
|
|
}
|
|
|
|
if (this.state.isProcessing) {
|
|
logSh('⚠️ Timeout attente fin traitement', 'WARNING');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Termine le traitement (tous éléments traités)
|
|
*/
|
|
async completeProcessing() {
|
|
logSh('🎉 Traitement terminé - Tous les éléments ont été traités', 'INFO');
|
|
|
|
const summary = {
|
|
totalItems: this.stats.itemsQueued,
|
|
processed: this.stats.itemsProcessed,
|
|
failed: this.stats.itemsFailed,
|
|
totalTime: Date.now() - this.stats.startTime,
|
|
averageTime: this.stats.averageProcessingTime
|
|
};
|
|
|
|
logSh(`📊 Résumé final: ${summary.processed}/${summary.totalItems} traités, ${summary.failed} échecs`, 'INFO');
|
|
logSh(`⏱️ Temps total: ${Math.floor(summary.totalTime / 1000)}s, moyenne: ${summary.averageTime}ms/item`, 'INFO');
|
|
|
|
// Arrêter la boucle
|
|
if (this.processingInterval) {
|
|
clearInterval(this.processingInterval);
|
|
this.processingInterval = null;
|
|
}
|
|
|
|
// Sauvegarder résultats finaux
|
|
await this.saveProgress();
|
|
|
|
this.state.isProcessing = false;
|
|
}
|
|
|
|
// ========================================
|
|
// MONITORING ET HEALTH
|
|
// ========================================
|
|
|
|
/**
|
|
* Démarre le monitoring de santé
|
|
*/
|
|
startHealthMonitoring() {
|
|
const HEALTH_INTERVAL = 60000; // 1 minute
|
|
|
|
this.healthInterval = setInterval(() => {
|
|
this.performHealthCheck();
|
|
}, HEALTH_INTERVAL);
|
|
|
|
logSh('💓 Health monitoring AutoProcessor démarré', 'DEBUG');
|
|
}
|
|
|
|
/**
|
|
* Health check périodique
|
|
*/
|
|
performHealthCheck() {
|
|
const memUsage = process.memoryUsage();
|
|
const uptime = Date.now() - this.stats.startTime;
|
|
const queueStatus = {
|
|
pending: this.processingQueue.filter(i => i.status === 'pending').length,
|
|
completed: this.processingQueue.filter(i => i.status === 'completed').length,
|
|
failed: this.processingQueue.filter(i => i.status === 'failed').length
|
|
};
|
|
|
|
logSh(`💓 AutoProcessor Health - Queue: ${queueStatus.pending}P/${queueStatus.completed}C/${queueStatus.failed}F | RAM: ${Math.round(memUsage.rss / 1024 / 1024)}MB`, 'TRACE');
|
|
|
|
// Alertes
|
|
if (memUsage.rss > 2 * 1024 * 1024 * 1024) { // > 2GB
|
|
logSh('⚠️ Utilisation mémoire très élevée', 'WARNING');
|
|
}
|
|
|
|
if (this.stats.itemsFailed > this.stats.itemsProcessed * 0.5) {
|
|
logSh('⚠️ Taux d\'échec élevé détecté', 'WARNING');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Retourne le status détaillé
|
|
*/
|
|
getDetailedStatus() {
|
|
return {
|
|
success: true,
|
|
mode: 'AUTO',
|
|
isRunning: this.isRunning,
|
|
state: { ...this.state },
|
|
stats: {
|
|
...this.stats,
|
|
uptime: Date.now() - this.stats.startTime
|
|
},
|
|
queue: {
|
|
total: this.processingQueue.length,
|
|
pending: this.processingQueue.filter(i => i.status === 'pending').length,
|
|
processing: this.processingQueue.filter(i => i.status === 'processing').length,
|
|
completed: this.processingQueue.filter(i => i.status === 'completed').length,
|
|
failed: this.processingQueue.filter(i => i.status === 'failed').length
|
|
},
|
|
config: { ...this.config },
|
|
currentItem: this.state.currentItem ? {
|
|
rowNumber: this.state.currentItem.rowNumber,
|
|
data: this.state.currentItem.data.mc0,
|
|
attempts: this.state.currentItem.attempts
|
|
} : null,
|
|
urls: {
|
|
monitoring: `http://localhost:${this.config.monitoringPort}`,
|
|
api: `http://localhost:${this.config.monitoringPort}/api/stats`
|
|
},
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
|
|
// ========================================
|
|
// PERSISTANCE ET RÉCUPÉRATION
|
|
// ========================================
|
|
|
|
/**
|
|
* Sauvegarde la progression
|
|
*/
|
|
async saveProgress() {
|
|
try {
|
|
const fs = require('fs').promises;
|
|
const path = require('path');
|
|
|
|
const progressFile = path.join(__dirname, '../../auto-processor-progress.json');
|
|
const progress = {
|
|
processedRows: this.processedItems.map(item => item.rowNumber),
|
|
failedRows: this.failedItems.map(item => ({
|
|
rowNumber: item.rowNumber,
|
|
error: item.error,
|
|
attempts: item.attempts
|
|
})),
|
|
stats: { ...this.stats },
|
|
lastSaved: Date.now(),
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
|
|
await fs.writeFile(progressFile, JSON.stringify(progress, null, 2));
|
|
|
|
} catch (error) {
|
|
logSh(`⚠️ Erreur sauvegarde progression: ${error.message}`, 'WARNING');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Charge la progression sauvegardée
|
|
*/
|
|
async loadProgress() {
|
|
try {
|
|
const fs = require('fs').promises;
|
|
const path = require('path');
|
|
|
|
const progressFile = path.join(__dirname, '../../auto-processor-progress.json');
|
|
|
|
try {
|
|
const data = await fs.readFile(progressFile, 'utf8');
|
|
const progress = JSON.parse(data);
|
|
|
|
logSh(`📂 Progression restaurée: ${progress.processedRows?.length || 0} éléments déjà traités`, 'INFO');
|
|
|
|
return progress;
|
|
|
|
} catch (readError) {
|
|
if (readError.code !== 'ENOENT') {
|
|
logSh(`⚠️ Erreur lecture progression: ${readError.message}`, 'WARNING');
|
|
}
|
|
return null;
|
|
}
|
|
|
|
} catch (error) {
|
|
logSh(`⚠️ Erreur chargement progression: ${error.message}`, 'WARNING');
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// ========================================
|
|
// UTILITAIRES
|
|
// ========================================
|
|
|
|
/**
|
|
* Pause asynchrone
|
|
*/
|
|
sleep(ms) {
|
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
}
|
|
}
|
|
|
|
// ============= EXPORTS =============
|
|
module.exports = { AutoProcessor }; |