// ======================================== // QUEUE PROCESSOR - CLASSE COMMUNE // Responsabilité: Logique partagée de queue, retry, persistance // ======================================== const { logSh } = require('../ErrorReporting'); const { tracer } = require('../trace'); const { handleModularWorkflow } = require('../Main'); const { readInstructionsData } = require('../BrainConfig'); const fs = require('fs').promises; const path = require('path'); /** * QUEUE PROCESSOR BASE * Classe commune pour la gestion de queue avec retry logic et persistance */ class QueueProcessor { constructor(options = {}) { this.name = options.name || 'QueueProcessor'; this.configPath = options.configPath; this.statusPath = options.statusPath; this.queuePath = options.queuePath; // Configuration par défaut this.config = { selective: 'standardEnhancement', adversarial: 'light', humanSimulation: 'none', patternBreaking: 'none', intensity: 1.0, rowRange: { start: 2, end: 10 }, saveIntermediateSteps: false, maxRetries: 3, delayBetweenItems: 1000, batchSize: 1, ...options.config }; // État du processeur this.isRunning = false; this.isPaused = false; this.currentRow = null; this.queue = []; this.processedItems = []; this.failedItems = []; // Métriques this.startTime = null; this.processedCount = 0; this.errorCount = 0; // Stats détaillées this.stats = { itemsQueued: 0, itemsProcessed: 0, itemsFailed: 0, averageProcessingTime: 0, totalProcessingTime: 0, startTime: Date.now(), lastProcessedAt: null }; // Callbacks optionnels this.onStatusUpdate = null; this.onProgress = null; this.onError = null; this.onComplete = null; this.onItemProcessed = null; } // ======================================== // INITIALISATION // ======================================== /** * Initialise le processeur */ async initialize() { try { await this.loadConfig(); await this.initializeQueue(); logSh(`🎯 ${this.name} initialisé`, 'DEBUG'); } catch (error) { logSh(`❌ Erreur initialisation ${this.name}: ${error.message}`, 'ERROR'); throw error; } } /** * Charge la configuration */ async loadConfig() { if (!this.configPath) return; try { const configData = await fs.readFile(this.configPath, 'utf8'); this.config = { ...this.config, ...JSON.parse(configData) }; logSh(`📋 Configuration ${this.name} chargée`, 'DEBUG'); } catch (error) { logSh(`⚠️ Configuration non trouvée pour ${this.name}, utilisation valeurs par défaut`, 'WARNING'); } } /** * Initialise les fichiers de configuration */ async initializeFiles() { if (!this.configPath) return; try { const configDir = path.dirname(this.configPath); await fs.mkdir(configDir, { recursive: true }); // Créer config par défaut si inexistant try { await fs.access(this.configPath); } catch { await fs.writeFile(this.configPath, JSON.stringify(this.config, null, 2)); logSh(`📝 Configuration ${this.name} par défaut créée`, 'DEBUG'); } // Créer status par défaut si inexistant if (this.statusPath) { const defaultStatus = this.getDefaultStatus(); try { await fs.access(this.statusPath); } catch { await fs.writeFile(this.statusPath, JSON.stringify(defaultStatus, null, 2)); logSh(`📊 Status ${this.name} par défaut créé`, 'DEBUG'); } } } catch (error) { logSh(`❌ Erreur initialisation fichiers ${this.name}: ${error.message}`, 'ERROR'); } } // ======================================== // GESTION QUEUE // ======================================== /** * Initialise la queue */ async initializeQueue() { try { // Essayer de charger la queue existante if (this.queuePath) { try { const queueData = await fs.readFile(this.queuePath, 'utf8'); const savedQueue = JSON.parse(queueData); if (savedQueue.queue && Array.isArray(savedQueue.queue)) { this.queue = savedQueue.queue; this.processedCount = savedQueue.processedCount || 0; logSh(`📊 Queue ${this.name} restaurée: ${this.queue.length} éléments`, 'DEBUG'); } } catch { // Queue n'existe pas, on la créera } } // Si queue vide, la populer if (this.queue.length === 0) { await this.populateQueue(); } } catch (error) { logSh(`❌ Erreur initialisation queue ${this.name}: ${error.message}`, 'ERROR'); } } /** * Popule la queue avec les lignes à traiter */ async populateQueue() { try { this.queue = []; const { start, end } = this.config.rowRange; for (let rowNumber = start; rowNumber <= end; rowNumber++) { this.queue.push({ rowNumber, status: 'pending', attempts: 0, maxAttempts: this.config.maxRetries, error: null, result: null, startTime: null, endTime: null, addedAt: Date.now() }); } await this.saveQueue(); this.stats.itemsQueued = this.queue.length; logSh(`📋 Queue ${this.name} populée: ${this.queue.length} lignes (${start} à ${end})`, 'INFO'); } catch (error) { logSh(`❌ Erreur population queue ${this.name}: ${error.message}`, 'ERROR'); throw error; } } /** * Popule la queue depuis Google Sheets (version avancée) */ async populateQueueFromSheets() { try { this.queue = []; let currentRow = this.config.startRow || 2; let consecutiveEmptyRows = 0; const maxEmptyRows = 5; while (currentRow <= (this.config.endRow || 50)) { if (this.config.endRow && currentRow > this.config.endRow) { break; } try { const csvData = await readInstructionsData(currentRow); if (!csvData || !csvData.mc0) { consecutiveEmptyRows++; if (consecutiveEmptyRows >= maxEmptyRows) { logSh(`🛑 Arrêt scan après ${maxEmptyRows} lignes vides consécutives`, 'INFO'); break; } } else { consecutiveEmptyRows = 0; this.queue.push({ rowNumber: currentRow, data: csvData, status: 'pending', attempts: 0, maxAttempts: this.config.maxRetries, error: null, result: null, startTime: null, endTime: null, addedAt: Date.now() }); } } catch (error) { consecutiveEmptyRows++; if (consecutiveEmptyRows >= maxEmptyRows) { break; } } currentRow++; } await this.saveQueue(); this.stats.itemsQueued = this.queue.length; logSh(`📊 Queue ${this.name} chargée depuis Sheets: ${this.stats.itemsQueued} éléments`, 'INFO'); } catch (error) { logSh(`❌ Erreur chargement queue depuis Sheets: ${error.message}`, 'ERROR'); throw error; } } /** * Sauvegarde la queue */ async saveQueue() { if (!this.queuePath) return; try { const queueData = { queue: this.queue, processedCount: this.processedCount, lastUpdate: new Date().toISOString() }; await fs.writeFile(this.queuePath, JSON.stringify(queueData, null, 2)); } catch (error) { logSh(`❌ Erreur sauvegarde queue ${this.name}: ${error.message}`, 'ERROR'); } } // ======================================== // CONTRÔLES PRINCIPAUX // ======================================== /** * Démarre le traitement */ async start() { return tracer.run(`${this.name}.start`, async () => { if (this.isRunning) { throw new Error(`${this.name} est déjà en cours`); } logSh(`🚀 Démarrage ${this.name}`, 'INFO'); this.isRunning = true; this.isPaused = false; this.startTime = new Date(); this.processedCount = 0; this.errorCount = 0; await this.loadConfig(); if (this.queue.length === 0) { await this.populateQueue(); } await this.updateStatus(); // Démarrer le traitement asynchrone this.processQueue().catch(error => { logSh(`❌ Erreur traitement queue ${this.name}: ${error.message}`, 'ERROR'); this.handleError(error); }); return this.getStatus(); }); } /** * Arrête le traitement */ async stop() { return tracer.run(`${this.name}.stop`, async () => { logSh(`🛑 Arrêt ${this.name}`, 'INFO'); this.isRunning = false; this.isPaused = false; this.currentRow = null; await this.updateStatus(); return this.getStatus(); }); } /** * Met en pause le traitement */ async pause() { return tracer.run(`${this.name}.pause`, async () => { if (!this.isRunning) { throw new Error(`Aucun traitement ${this.name} en cours`); } logSh(`⏸️ Mise en pause ${this.name}`, 'INFO'); this.isPaused = true; await this.updateStatus(); return this.getStatus(); }); } /** * Reprend le traitement */ async resume() { return tracer.run(`${this.name}.resume`, async () => { if (!this.isRunning || !this.isPaused) { throw new Error(`Aucun traitement ${this.name} en pause`); } logSh(`▶️ Reprise ${this.name}`, 'INFO'); this.isPaused = false; await this.updateStatus(); // Reprendre le traitement this.processQueue().catch(error => { logSh(`❌ Erreur reprise traitement ${this.name}: ${error.message}`, 'ERROR'); this.handleError(error); }); return this.getStatus(); }); } // ======================================== // TRAITEMENT QUEUE // ======================================== /** * Traite la queue */ async processQueue() { return tracer.run(`${this.name}.processQueue`, async () => { while (this.isRunning && !this.isPaused) { const nextItem = this.queue.find(item => item.status === 'pending' || (item.status === 'error' && item.attempts < item.maxAttempts)); if (!nextItem) { logSh(`✅ Traitement ${this.name} terminé`, 'INFO'); await this.complete(); break; } await this.processItem(nextItem); if (this.config.delayBetweenItems > 0) { await this.sleep(this.config.delayBetweenItems); } } }); } /** * Traite un élément de la queue */ async processItem(item) { return tracer.run(`${this.name}.processItem`, async () => { logSh(`🔄 Traitement ${this.name} ligne ${item.rowNumber} (tentative ${item.attempts + 1}/${item.maxAttempts})`, 'INFO'); this.currentRow = item.rowNumber; item.status = 'processing'; item.startTime = new Date().toISOString(); item.attempts++; await this.updateStatus(); await this.saveQueue(); try { const result = await this.processRow(item.rowNumber, item.data); // Succès item.status = 'completed'; item.result = result; item.endTime = new Date().toISOString(); item.error = null; this.processedCount++; this.processedItems.push(item); const duration = Date.now() - new Date(item.startTime).getTime(); this.stats.itemsProcessed++; this.stats.totalProcessingTime += duration; this.stats.averageProcessingTime = Math.round(this.stats.totalProcessingTime / this.stats.itemsProcessed); this.stats.lastProcessedAt = Date.now(); logSh(`✅ ${this.name} ligne ${item.rowNumber} traitée avec succès (${duration}ms)`, 'INFO'); if (this.onItemProcessed) { this.onItemProcessed(item, result); } if (this.onProgress) { this.onProgress(item, this.getProgress()); } } catch (error) { item.error = { message: error.message, stack: error.stack, timestamp: new Date().toISOString() }; if (item.attempts >= item.maxAttempts) { item.status = 'failed'; this.errorCount++; this.failedItems.push(item); logSh(`❌ ${this.name} ligne ${item.rowNumber} échouée définitivement après ${item.attempts} tentatives`, 'ERROR'); } else { item.status = 'error'; logSh(`⚠️ ${this.name} ligne ${item.rowNumber} échouée, retry possible`, 'WARNING'); } if (this.onError) { this.onError(item, error); } } this.currentRow = null; await this.updateStatus(); await this.saveQueue(); }); } /** * Traite une ligne spécifique - à surcharger dans les classes enfants */ async processRow(rowNumber, data = null) { const rowConfig = this.buildRowConfig(rowNumber, data); logSh(`🎯 Configuration ${this.name} ligne ${rowNumber}: ${JSON.stringify(rowConfig)}`, 'DEBUG'); const result = await handleModularWorkflow(rowConfig); logSh(`📊 Résultat ${this.name} ligne ${rowNumber}: ${result ? 'SUCCESS' : 'FAILED'}`, 'INFO'); return result; } /** * Construit la configuration pour une ligne - à surcharger si nécessaire */ buildRowConfig(rowNumber, data = null) { return { rowNumber, source: `${this.name.toLowerCase()}_row_${rowNumber}`, selectiveStack: this.config.selective, adversarialMode: this.config.adversarial, humanSimulationMode: this.config.humanSimulation, patternBreakingMode: this.config.patternBreaking, intensity: this.config.intensity, saveIntermediateSteps: this.config.saveIntermediateSteps, data }; } // ======================================== // GESTION ÉTAT // ======================================== /** * Met à jour le status */ async updateStatus() { const status = this.getStatus(); if (this.statusPath) { try { await fs.writeFile(this.statusPath, JSON.stringify(status, null, 2)); } catch (error) { logSh(`❌ Erreur mise à jour status ${this.name}: ${error.message}`, 'ERROR'); } } if (this.onStatusUpdate) { this.onStatusUpdate(status); } } /** * Retourne le status actuel */ getStatus() { const now = new Date(); const completedItems = this.queue.filter(item => item.status === 'completed').length; const failedItems = this.queue.filter(item => item.status === 'failed').length; const totalItems = this.queue.length; const progress = totalItems > 0 ? ((completedItems + failedItems) / totalItems) * 100 : 0; let status = 'idle'; if (this.isRunning && this.isPaused) { status = 'paused'; } else if (this.isRunning) { status = 'running'; } else if (completedItems + failedItems === totalItems && totalItems > 0) { status = 'completed'; } return { status, currentRow: this.currentRow, totalRows: totalItems, completedRows: completedItems, failedRows: failedItems, progress: Math.round(progress), startTime: this.startTime ? this.startTime.toISOString() : null, estimatedEnd: this.estimateCompletionTime(), errors: this.queue.filter(item => item.error).map(item => ({ rowNumber: item.rowNumber, error: item.error, attempts: item.attempts })), lastResult: this.getLastResult(), config: this.config, queue: this.queue, stats: this.stats }; } /** * Retourne la progression détaillée */ getProgress() { // Calcul direct des métriques sans appeler getStatus() pour éviter la récursion const now = new Date(); const elapsed = this.startTime ? now - this.startTime : 0; const completedRows = this.processedItems.length; const failedRows = this.failedItems.length; const totalRows = this.queue.length + completedRows + failedRows; const avgTimePerRow = completedRows > 0 ? elapsed / completedRows : 0; const remainingRows = totalRows - completedRows - failedRows; const estimatedRemaining = avgTimePerRow * remainingRows; return { status: this.status, currentRow: this.currentItem ? this.currentItem.rowNumber : null, totalRows: totalRows, completedRows: completedRows, failedRows: failedRows, progress: totalRows > 0 ? Math.round((completedRows / totalRows) * 100) : 0, startTime: this.startTime ? this.startTime.toISOString() : null, estimatedEnd: null, // Calculé séparément pour éviter récursion errors: this.failedItems.map(item => ({ row: item.rowNumber, error: item.error })), lastResult: this.processedItems.length > 0 ? this.processedItems[this.processedItems.length - 1].result : null, config: this.config, queue: this.queue, stats: { itemsQueued: this.queue.length, itemsProcessed: completedRows, itemsFailed: failedRows, averageProcessingTime: avgTimePerRow, totalProcessingTime: elapsed, startTime: this.startTime ? this.startTime.getTime() : null, lastProcessedAt: this.processedItems.length > 0 ? this.processedItems[this.processedItems.length - 1].endTime : null }, metrics: { elapsedTime: elapsed, avgTimePerRow: avgTimePerRow, estimatedRemaining: estimatedRemaining, completionPercentage: totalRows > 0 ? (completedRows / totalRows) * 100 : 0, throughput: completedRows > 0 && elapsed > 0 ? (completedRows / (elapsed / 1000 / 60)) : 0 } }; } /** * Estime l'heure de fin */ estimateCompletionTime() { if (!this.startTime || !this.isRunning || this.isPaused) { return null; } // Calcul direct sans appeler getProgress() pour éviter la récursion const now = new Date(); const elapsed = now - this.startTime; const completedRows = this.processedItems.length; if (completedRows > 0) { const avgTimePerRow = elapsed / completedRows; const remainingRows = this.queue.length; const estimatedRemaining = avgTimePerRow * remainingRows; if (estimatedRemaining > 0) { const endTime = new Date(Date.now() + estimatedRemaining); return endTime.toISOString(); } } return null; } /** * Retourne le dernier résultat */ getLastResult() { const completedItems = this.queue.filter(item => item.status === 'completed'); if (completedItems.length === 0) return null; const lastItem = completedItems[completedItems.length - 1]; return { rowNumber: lastItem.rowNumber, result: lastItem.result, endTime: lastItem.endTime }; } /** * Status par défaut */ getDefaultStatus() { return { status: 'idle', currentRow: null, totalRows: 0, progress: 0, startTime: null, estimatedEnd: null, errors: [], lastResult: null, config: this.config }; } // ======================================== // GESTION ERREURS // ======================================== /** * Gère les erreurs critiques */ async handleError(error) { logSh(`💥 Erreur critique ${this.name}: ${error.message}`, 'ERROR'); this.isRunning = false; this.isPaused = false; await this.updateStatus(); if (this.onError) { this.onError(null, error); } } /** * Termine le traitement */ async complete() { logSh(`🏁 Traitement ${this.name} terminé`, 'INFO'); this.isRunning = false; this.isPaused = false; this.currentRow = null; await this.updateStatus(); if (this.onComplete) { this.onComplete(this.getStatus()); } } // ======================================== // UTILITAIRES // ======================================== /** * Pause l'exécution */ async sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } /** * Reset la queue */ async resetQueue() { logSh(`🔄 Reset de la queue ${this.name}`, 'INFO'); this.queue = []; this.processedCount = 0; this.errorCount = 0; await this.populateQueue(); await this.updateStatus(); } /** * Configure les callbacks */ setCallbacks({ onStatusUpdate, onProgress, onError, onComplete, onItemProcessed }) { this.onStatusUpdate = onStatusUpdate; this.onProgress = onProgress; this.onError = onError; this.onComplete = onComplete; this.onItemProcessed = onItemProcessed; } } // ============= EXPORTS ============= module.exports = { QueueProcessor };