From a974846efc9aa926a66e14497a36ffa16cf7512e Mon Sep 17 00:00:00 2001 From: StillHammer Date: Tue, 25 Nov 2025 10:49:39 +0800 Subject: [PATCH] Add AI summarization feature with GPT-5.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add summarization service (text, files, YouTube videos) - Add Summarize tab with Text/Files/YouTube Link modes - Add full pipeline endpoint: download + transcribe + summarize - Support for concise, detailed, and bullet point styles - Real-time progress tracking with SSE - Multi-language output support 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- public/app.js | 340 ++++++++++++++++++++++++++++++++++++++ public/index.html | 166 +++++++++++++++++++ public/style.css | 24 +++ src/server.js | 316 ++++++++++++++++++++++++++++++++++- src/services/summarize.js | 193 ++++++++++++++++++++++ src/services/youtube.js | 134 ++++++++++----- 6 files changed, 1124 insertions(+), 49 deletions(-) create mode 100644 src/services/summarize.js diff --git a/public/app.js b/public/app.js index ce1a8ff..8b20402 100644 --- a/public/app.js +++ b/public/app.js @@ -634,3 +634,343 @@ document.getElementById('translate-file-form').addEventListener('submit', async setLoading(button, false); } }); + +// ==================== SUMMARIZE TAB ==================== + +// Mode switching +const summarizeModeBtns = document.querySelectorAll('.summarize-mode-selector .mode-btn'); +const summarizeTextMode = document.getElementById('summarize-text-mode'); +const summarizeFileMode = document.getElementById('summarize-file-mode'); + +const summarizeLinkMode = document.getElementById('summarize-link-mode'); + +summarizeModeBtns.forEach(btn => { + btn.addEventListener('click', () => { + summarizeModeBtns.forEach(b => b.classList.remove('active')); + btn.classList.add('active'); + + summarizeTextMode.style.display = 'none'; + summarizeFileMode.style.display = 'none'; + if (summarizeLinkMode) summarizeLinkMode.style.display = 'none'; + + if (btn.dataset.mode === 'text') { + summarizeTextMode.style.display = 'block'; + } else if (btn.dataset.mode === 'file') { + summarizeFileMode.style.display = 'block'; + } else if (btn.dataset.mode === 'link' && summarizeLinkMode) { + summarizeLinkMode.style.display = 'block'; + } + }); +}); + +// Text summarization form +document.getElementById('summarize-text-form').addEventListener('submit', async (e) => { + e.preventDefault(); + const button = document.getElementById('summarize-text-btn'); + const text = document.getElementById('summarize-input').value; + const style = document.getElementById('summarize-style').value; + const language = document.getElementById('summarize-language').value; + + if (!text.trim()) { + showResult('summarize-text-result', false, '

Error

Please enter text to summarize

'); + return; + } + + setLoading(button, true); + document.getElementById('summarize-text-result').classList.remove('show'); + + try { + const response = await fetch(`${API_URL}/summarize`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text, style, language }) + }); + + const data = await response.json(); + if (!response.ok) throw new Error(data.error || 'Summarization failed'); + + showResult('summarize-text-result', true, ` +

Summary Complete!

+

Model: ${data.model} | Style: ${data.style} | Chunks: ${data.chunks}

+
${data.summary}
+ `); + } catch (error) { + showResult('summarize-text-result', false, `

Error

${error.message}

`); + } finally { + setLoading(button, false); + } +}); + +// File summarization - Drag & Drop +let summarizeSelectedFiles = []; +const summarizeDropZone = document.getElementById('summarize-drop-zone'); +const summarizeFileInput = document.getElementById('summarize-file-input'); +const summarizeSelectedFilesDiv = document.getElementById('summarize-selected-files'); +const summarizeFilesList = document.getElementById('summarize-files-list'); +const summarizeFileBtn = document.getElementById('summarize-file-btn'); +const summarizeClearFilesBtn = document.getElementById('summarize-clear-files'); + +function updateSummarizeFilesList() { + if (!summarizeSelectedFilesDiv || !summarizeFileBtn || !summarizeFilesList) return; + + if (summarizeSelectedFiles.length === 0) { + summarizeSelectedFilesDiv.style.display = 'none'; + summarizeFileBtn.disabled = true; + return; + } + + summarizeSelectedFilesDiv.style.display = 'block'; + summarizeFileBtn.disabled = false; + + summarizeFilesList.innerHTML = summarizeSelectedFiles.map((file, index) => ` +
  • + ${file.name} + ${formatSize(file.size)} + +
    • + `).join(''); + + summarizeFilesList.querySelectorAll('.remove-file').forEach(btn => { + btn.addEventListener('click', () => { + summarizeSelectedFiles.splice(parseInt(btn.dataset.index), 1); + updateSummarizeFilesList(); + }); + }); +} + +function addSummarizeFiles(files) { + console.log('Adding files:', files); + const textFiles = Array.from(files).filter(f => + f.type === 'text/plain' || f.name.endsWith('.txt') + ); + console.log('Filtered text files:', textFiles); + summarizeSelectedFiles = [...summarizeSelectedFiles, ...textFiles]; + updateSummarizeFilesList(); +} + +if (summarizeDropZone) { + summarizeDropZone.addEventListener('click', () => { + console.log('Drop zone clicked'); + summarizeFileInput.click(); + }); + + summarizeDropZone.addEventListener('dragover', (e) => { + e.preventDefault(); + e.stopPropagation(); + summarizeDropZone.classList.add('drag-over'); + }); + + summarizeDropZone.addEventListener('dragleave', (e) => { + e.preventDefault(); + e.stopPropagation(); + summarizeDropZone.classList.remove('drag-over'); + }); + + summarizeDropZone.addEventListener('drop', (e) => { + e.preventDefault(); + e.stopPropagation(); + console.log('Files dropped:', e.dataTransfer.files); + summarizeDropZone.classList.remove('drag-over'); + addSummarizeFiles(e.dataTransfer.files); + }); +} + +if (summarizeFileInput) { + summarizeFileInput.addEventListener('change', () => { + console.log('File input changed:', summarizeFileInput.files); + addSummarizeFiles(summarizeFileInput.files); + summarizeFileInput.value = ''; + }); +} + +if (summarizeClearFilesBtn) { + summarizeClearFilesBtn.addEventListener('click', () => { + summarizeSelectedFiles = []; + updateSummarizeFilesList(); + }); +} + +// File summarization form submit +document.getElementById('summarize-file-form').addEventListener('submit', async (e) => { + e.preventDefault(); + if (summarizeSelectedFiles.length === 0) return; + + const button = summarizeFileBtn; + const style = document.getElementById('summarize-file-style').value; + const language = document.getElementById('summarize-file-language').value; + + setLoading(button, true); + document.getElementById('summarize-file-result').classList.remove('show'); + + const formData = new FormData(); + summarizeSelectedFiles.forEach(file => formData.append('files', file)); + formData.append('style', style); + formData.append('language', language); + + try { + const response = await fetch(`${API_URL}/summarize-file`, { + method: 'POST', + body: formData + }); + + const data = await response.json(); + if (!response.ok) throw new Error(data.error || 'Summarization failed'); + + showResult('summarize-file-result', true, ` +

    Summarization Complete!

    +

    ${data.successCount}/${data.totalFiles} files summarized

    + + ${data.results[0]?.summary ? ` +

    Preview (first file):

    +
    ${data.results[0].summary.substring(0, 1000)}${data.results[0].summary.length > 1000 ? '...' : ''}
    + ` : ''} + `); + + summarizeSelectedFiles = []; + updateSummarizeFilesList(); + + } catch (error) { + showResult('summarize-file-result', false, `

    Error

    ${error.message}

    `); + } finally { + setLoading(button, false); + } +}); + +// ==================== SUMMARIZE LINK MODE (Full Pipeline) ==================== + +const summarizeLinkForm = document.getElementById('summarize-link-form'); +if (summarizeLinkForm) { + const linkProgress = document.getElementById('summarize-link-progress'); + const linkProgressFill = document.getElementById('summarize-link-progress-fill'); + const linkProgressTitle = document.getElementById('summarize-link-progress-title'); + const linkProgressPercent = document.getElementById('summarize-link-progress-percent'); + const linkProgressPhase = document.getElementById('summarize-link-progress-phase'); + const linkProgressSpeed = document.getElementById('summarize-link-progress-speed'); + const linkProgressCurrent = document.getElementById('summarize-link-progress-current'); + const linkProgressEta = document.getElementById('summarize-link-progress-eta'); + + function resetLinkProgress() { + if (linkProgressFill) linkProgressFill.style.width = '0%'; + if (linkProgressPercent) linkProgressPercent.textContent = '0%'; + if (linkProgressTitle) linkProgressTitle.textContent = 'Processing...'; + if (linkProgressPhase) linkProgressPhase.textContent = ''; + if (linkProgressSpeed) linkProgressSpeed.textContent = ''; + if (linkProgressCurrent) linkProgressCurrent.textContent = ''; + if (linkProgressEta) linkProgressEta.textContent = ''; + } + + summarizeLinkForm.addEventListener('submit', async (e) => { + e.preventDefault(); + const button = document.getElementById('summarize-link-btn'); + const url = document.getElementById('summarize-url').value; + const style = document.getElementById('summarize-link-style').value; + const language = document.getElementById('summarize-link-language').value; + const resultDiv = document.getElementById('summarize-link-result'); + + setLoading(button, true); + resetLinkProgress(); + linkProgress.style.display = 'block'; + resultDiv.classList.remove('show'); + + const params = new URLSearchParams({ url, style, language }); + const eventSource = new EventSource(`${API_URL}/summarize-stream?${params}`); + + eventSource.addEventListener('status', (e) => { + const data = JSON.parse(e.data); + linkProgressTitle.textContent = data.message; + if (data.percent !== undefined) { + linkProgressFill.style.width = `${data.percent}%`; + linkProgressPercent.textContent = `${Math.round(data.percent)}%`; + } + }); + + eventSource.addEventListener('info', (e) => { + const data = JSON.parse(e.data); + linkProgressTitle.textContent = data.totalVideos > 1 + ? `Processing playlist: ${data.playlistTitle} (${data.totalVideos} videos)` + : `Processing: ${data.title}`; + }); + + eventSource.addEventListener('progress', (e) => { + const data = JSON.parse(e.data); + linkProgressFill.style.width = `${data.percent}%`; + linkProgressPercent.textContent = `${Math.round(data.percent)}%`; + linkProgressPhase.textContent = data.phaseLabel || ''; + if (data.speed) linkProgressSpeed.textContent = data.speed; + if (data.title) { + linkProgressCurrent.innerHTML = `${data.phaseLabel}: ${data.title}`; + } + if (data.totalVideos > 1) { + linkProgressCurrent.innerHTML += ` (${data.currentVideo}/${data.totalVideos})`; + } + }); + + eventSource.addEventListener('video-complete', (e) => { + const data = JSON.parse(e.data); + linkProgressCurrent.innerHTML = `Downloaded: ${data.title}`; + }); + + eventSource.addEventListener('transcribe-complete', (e) => { + const data = JSON.parse(e.data); + linkProgressCurrent.innerHTML = `Transcribed: ${data.title}`; + }); + + eventSource.addEventListener('summarize-complete', (e) => { + const data = JSON.parse(e.data); + linkProgressCurrent.innerHTML = `Summarized: ${data.title}`; + }); + + eventSource.addEventListener('complete', (e) => { + const data = JSON.parse(e.data); + eventSource.close(); + linkProgressFill.style.width = '100%'; + linkProgressPercent.textContent = '100%'; + linkProgressTitle.textContent = 'Complete!'; + linkProgressPhase.textContent = ''; + linkProgressEta.textContent = `Total: ${formatTime(data.totalTime)}`; + + showResult('summarize-link-result', true, ` +

    Pipeline Complete!

    + ${data.playlistTitle ? `

    Playlist: ${data.playlistTitle}

    ` : ''} +

    Downloaded: ${data.downloadedCount}/${data.totalVideos} | Transcribed: ${data.transcribedCount} | Summarized: ${data.summarizedCount}

    + + ${data.results[0]?.summary ? ` +

    Summary Preview:

    +
    ${data.results[0].summary.substring(0, 2000)}${data.results[0].summary.length > 2000 ? '...' : ''}
    + ` : ''} + `); + setLoading(button, false); + }); + + eventSource.addEventListener('error', (e) => { + let errorMsg = 'Processing failed'; + try { errorMsg = JSON.parse(e.data).message || errorMsg; } catch {} + eventSource.close(); + linkProgress.style.display = 'none'; + showResult('summarize-link-result', false, `

    Error

    ${errorMsg}

    `); + setLoading(button, false); + }); + + eventSource.onerror = () => { + eventSource.close(); + linkProgress.style.display = 'none'; + showResult('summarize-link-result', false, `

    Error

    Connection lost

    `); + setLoading(button, false); + }; + }); +} diff --git a/public/index.html b/public/index.html index 5e4f8a4..fa820a2 100644 --- a/public/index.html +++ b/public/index.html @@ -19,6 +19,7 @@ + @@ -338,6 +339,171 @@
    + + +
    +

    Summarize Text (GPT-5.1)

    + + +
    + + + +
    + + +
    +
    +
    + + +
    +
    +
    + + +
    +
    + + +
    +
    + +
    +
    +
    + + + + + + +
    diff --git a/public/style.css b/public/style.css index 5b688c8..95555e2 100644 --- a/public/style.css +++ b/public/style.css @@ -683,3 +683,27 @@ textarea::placeholder { color: #ccd6f6; line-height: 1.6; } + +/* Summarize Tab */ +.summarize-mode-selector { + display: flex; + gap: 0.5rem; + margin-bottom: 1.5rem; +} + +.summarize-mode { + animation: fadeIn 0.3s ease; +} + +/* Summary Result */ +.summary-output { + margin-top: 1rem; + padding: 1rem; + background: rgba(0, 0, 0, 0.3); + border-radius: 8px; + white-space: pre-wrap; + max-height: 400px; + overflow-y: auto; + color: #ccd6f6; + line-height: 1.6; +} diff --git a/src/server.js b/src/server.js index 804aca8..089e008 100644 --- a/src/server.js +++ b/src/server.js @@ -3,10 +3,12 @@ import cors from 'cors'; import dotenv from 'dotenv'; import path from 'path'; import fs from 'fs'; +import { fileURLToPath } from 'url'; import multer from 'multer'; import { download, getInfo } from './services/youtube.js'; import { transcribeFile, transcribeAndSave, transcribeMultiple } from './services/transcription.js'; import { translateText, translateFile, translateMultiple, getLanguages } from './services/translation.js'; +import { summarizeText, summarizeFile, getSummaryStyles } from './services/summarize.js'; dotenv.config(); @@ -58,8 +60,18 @@ const uploadText = multer({ app.use(cors()); app.use(express.json()); +// Set permissive CSP for development +app.use((req, res, next) => { + res.setHeader( + 'Content-Security-Policy', + "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'" + ); + next(); +}); + // Serve static files (HTML interface) -const __dirname = path.dirname(new URL(import.meta.url).pathname); +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); app.use(express.static(path.join(__dirname, '../public'))); // Serve downloaded files @@ -733,14 +745,302 @@ app.post('/translate-file', uploadText.array('files', 50), async (req, res) => { } }); +/** + * GET /summary-styles + * Get available summary styles + */ +app.get('/summary-styles', (req, res) => { + res.json({ styles: getSummaryStyles() }); +}); + +/** + * POST /summarize + * Summarize text using GPT-5.1 + * Body: { text: string, style?: string, language?: string, model?: string } + */ +app.post('/summarize', async (req, res) => { + try { + const { text, style = 'concise', language = 'same', model = 'gpt-5.1' } = req.body; + + if (!text) { + return res.status(400).json({ error: 'text required in request body' }); + } + + if (!process.env.OPENAI_API_KEY) { + return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); + } + + console.log(`Summarizing text with ${model} (style: ${style})`); + const result = await summarizeText(text, { style, language, model }); + + res.json({ + success: true, + ...result, + }); + } catch (error) { + res.status(500).json({ error: error.message }); + } +}); + +/** + * POST /summarize-file + * Summarize uploaded text files using GPT-5.1 + */ +app.post('/summarize-file', uploadText.array('files', 50), async (req, res) => { + try { + if (!process.env.OPENAI_API_KEY) { + return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); + } + + if (!req.files || req.files.length === 0) { + return res.status(400).json({ error: 'No files uploaded' }); + } + + const { style = 'concise', language = 'same', model = 'gpt-5.1' } = req.body; + const results = []; + + console.log(`Summarizing ${req.files.length} files with ${model}`); + + for (let i = 0; i < req.files.length; i++) { + const file = req.files[i]; + console.log(`[${i + 1}/${req.files.length}] Summarizing: ${file.originalname}`); + + try { + const result = await summarizeFile(file.path, { style, language, model }); + results.push({ + success: true, + fileName: file.originalname, + summaryPath: result.summaryPath, + summaryUrl: `/files/${path.basename(result.summaryPath)}`, + summary: result.summary, + model: result.model, + chunks: result.chunks, + }); + } catch (error) { + console.error(`Failed to summarize ${file.originalname}: ${error.message}`); + results.push({ + success: false, + fileName: file.originalname, + error: error.message, + }); + } + } + + res.json({ + success: true, + totalFiles: req.files.length, + successCount: results.filter(r => r.success).length, + failCount: results.filter(r => !r.success).length, + results, + }); + } catch (error) { + res.status(500).json({ error: error.message }); + } +}); + +/** + * GET /summarize-stream + * Full pipeline: Download -> Transcribe -> Summarize with SSE progress + * Query: url, style?, language?, model? + */ +app.get('/summarize-stream', async (req, res) => { + const { url, style = 'concise', language = 'same', model = 'gpt-4o-transcribe' } = req.query; + + if (!url) { + return res.status(400).json({ error: 'URL parameter required' }); + } + + if (!process.env.OPENAI_API_KEY) { + return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); + } + + // Set up SSE + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + res.setHeader('Access-Control-Allow-Origin', '*'); + + const sendEvent = (event, data) => { + res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); + }; + + const startTime = Date.now(); + let totalVideos = 1; + + try { + // Phase 1: Get info + sendEvent('status', { message: 'Fetching video info...', phase: 'info', percent: 0 }); + const hasPlaylist = url.includes('list='); + const info = await getInfo(url, hasPlaylist); + + totalVideos = info._type === 'playlist' ? (info.entries?.length || 1) : 1; + sendEvent('info', { + title: info.title, + type: info._type || 'video', + totalVideos, + playlistTitle: info._type === 'playlist' ? info.title : null, + }); + + // Phase 2: Download (0-33%) + sendEvent('status', { message: 'Downloading...', phase: 'downloading', percent: 5 }); + console.log(`[Summarize Pipeline] Downloading: ${url}`); + + let videosDownloaded = 0; + const downloadResult = await download(url, { + outputDir: OUTPUT_DIR, + onDownloadProgress: (progress) => { + const videoProgress = progress.percent || 0; + const overallPercent = ((videosDownloaded + (videoProgress / 100)) / totalVideos) * 33; + + sendEvent('progress', { + percent: Math.round(overallPercent * 10) / 10, + phase: 'downloading', + phaseLabel: 'Downloading', + title: progress.title, + speed: progress.speed, + currentVideo: progress.videoIndex || videosDownloaded + 1, + totalVideos, + }); + }, + onVideoComplete: (video) => { + videosDownloaded++; + sendEvent('video-complete', { + title: video.title, + phase: 'downloading', + videosCompleted: videosDownloaded, + totalVideos, + }); + }, + }); + + // Phase 3: Transcribe (33-66%) + sendEvent('status', { message: 'Transcribing...', phase: 'transcribing', percent: 33 }); + console.log(`[Summarize Pipeline] Transcribing ${downloadResult.successCount} files`); + + const successfulDownloads = downloadResult.videos.filter(v => v.success); + const transcribeResults = []; + + for (let i = 0; i < successfulDownloads.length; i++) { + const video = successfulDownloads[i]; + const percent = 33 + ((i / successfulDownloads.length) * 33); + + sendEvent('progress', { + percent: Math.round(percent * 10) / 10, + phase: 'transcribing', + phaseLabel: 'Transcribing', + title: video.title, + currentVideo: i + 1, + totalVideos: successfulDownloads.length, + }); + + try { + const result = await transcribeAndSave(video.filePath, { + responseFormat: 'text', + outputFormat: 'txt', + model, + }); + transcribeResults.push({ ...result, title: video.title, success: true }); + + sendEvent('transcribe-complete', { + title: video.title, + success: true, + videosCompleted: i + 1, + totalVideos: successfulDownloads.length, + }); + } catch (error) { + transcribeResults.push({ title: video.title, success: false, error: error.message }); + sendEvent('transcribe-complete', { + title: video.title, + success: false, + error: error.message, + }); + } + } + + // Phase 4: Summarize (66-100%) + sendEvent('status', { message: 'Summarizing with GPT-5.1...', phase: 'summarizing', percent: 66 }); + console.log(`[Summarize Pipeline] Summarizing ${transcribeResults.filter(t => t.success).length} transcriptions`); + + const summaryResults = []; + const successfulTranscriptions = transcribeResults.filter(t => t.success); + + for (let i = 0; i < successfulTranscriptions.length; i++) { + const transcription = successfulTranscriptions[i]; + const percent = 66 + ((i / successfulTranscriptions.length) * 34); + + sendEvent('progress', { + percent: Math.round(percent * 10) / 10, + phase: 'summarizing', + phaseLabel: 'Summarizing', + title: transcription.title, + currentVideo: i + 1, + totalVideos: successfulTranscriptions.length, + }); + + try { + const result = await summarizeFile(transcription.transcriptionPath, { style, language, model: 'gpt-5.1' }); + summaryResults.push({ + title: transcription.title, + success: true, + summary: result.summary, + summaryPath: result.summaryPath, + summaryUrl: `/files/${path.basename(result.summaryPath)}`, + transcriptionUrl: `/files/${path.basename(transcription.transcriptionPath)}`, + audioUrl: transcription.filePath ? `/files/${path.basename(transcription.filePath)}` : null, + }); + + sendEvent('summarize-complete', { + title: transcription.title, + success: true, + videosCompleted: i + 1, + totalVideos: successfulTranscriptions.length, + }); + } catch (error) { + summaryResults.push({ + title: transcription.title, + success: false, + error: error.message, + transcriptionUrl: `/files/${path.basename(transcription.transcriptionPath)}`, + }); + sendEvent('summarize-complete', { + title: transcription.title, + success: false, + error: error.message, + }); + } + } + + // Final result + sendEvent('complete', { + success: true, + playlistTitle: downloadResult.playlistTitle, + totalVideos: downloadResult.totalVideos, + downloadedCount: downloadResult.successCount, + transcribedCount: transcribeResults.filter(t => t.success).length, + summarizedCount: summaryResults.filter(s => s.success).length, + totalTime: Math.round((Date.now() - startTime) / 1000), + results: summaryResults, + }); + + } catch (error) { + console.error(`[Summarize Pipeline] Error: ${error.message}`); + sendEvent('error', { message: error.message }); + } finally { + res.end(); + } +}); + app.listen(PORT, () => { console.log(`Server running on http://localhost:${PORT}`); console.log('\nEndpoints:'); - console.log(' GET /health - Health check'); - console.log(' GET /info?url= - Get video/playlist info'); - console.log(' POST /download - Download as MP3'); - console.log(' POST /transcribe - Transcribe audio file'); - console.log(' POST /process - Download + transcribe'); - console.log(' GET /files-list - List downloaded files'); - console.log(' GET /files/ - Serve downloaded files'); + console.log(' GET /health - Health check'); + console.log(' GET /info?url= - Get video/playlist info'); + console.log(' POST /download - Download as MP3'); + console.log(' POST /transcribe - Transcribe audio file'); + console.log(' POST /process - Download + transcribe'); + console.log(' POST /summarize - Summarize text (GPT-5.1)'); + console.log(' POST /summarize-file - Summarize files (GPT-5.1)'); + console.log(' GET /summarize-stream - Full pipeline: Download + Transcribe + Summarize'); + console.log(' GET /files-list - List downloaded files'); + console.log(' GET /files/ - Serve downloaded files'); }); diff --git a/src/services/summarize.js b/src/services/summarize.js new file mode 100644 index 0000000..fedc9b1 --- /dev/null +++ b/src/services/summarize.js @@ -0,0 +1,193 @@ +import OpenAI from 'openai'; +import fs from 'fs'; +import path from 'path'; + +let openai = null; + +// Max characters per chunk for summarization +const MAX_CHUNK_CHARS = 30000; + +/** + * Get OpenAI client (lazy initialization) + */ +function getOpenAI() { + if (!openai) { + if (!process.env.OPENAI_API_KEY) { + throw new Error('OPENAI_API_KEY environment variable is not set'); + } + openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); + } + return openai; +} + +/** + * Summarize text using GPT-4o + */ +export async function summarizeText(text, options = {}) { + const { + model = 'gpt-5.1', // GPT-5.1 - latest OpenAI model (Nov 2025) + language = 'same', // 'same' = same as input, or specify language code + style = 'concise', // 'concise', 'detailed', 'bullet' + maxLength = null, // optional max length in words + } = options; + + const client = getOpenAI(); + + let styleInstruction = ''; + switch (style) { + case 'detailed': + styleInstruction = 'Provide a detailed summary that captures all important points and nuances.'; + break; + case 'bullet': + styleInstruction = 'Provide the summary as bullet points, highlighting the key points.'; + break; + case 'concise': + default: + styleInstruction = 'Provide a concise summary that captures the main points.'; + } + + let languageInstruction = ''; + if (language === 'same') { + languageInstruction = 'Write the summary in the same language as the input text.'; + } else { + languageInstruction = `Write the summary in ${language}.`; + } + + let lengthInstruction = ''; + if (maxLength) { + lengthInstruction = `Keep the summary under ${maxLength} words.`; + } + + const systemPrompt = `You are an expert summarizer. ${styleInstruction} ${languageInstruction} ${lengthInstruction} +Focus on the most important information and main ideas. Be accurate and objective.`; + + // Handle long texts by chunking + if (text.length > MAX_CHUNK_CHARS) { + return await summarizeLongText(text, { model, systemPrompt, style }); + } + + const response = await client.chat.completions.create({ + model, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: `Please summarize the following text:\n\n${text}` }, + ], + temperature: 0.3, + }); + + return { + summary: response.choices[0].message.content, + model, + style, + inputLength: text.length, + chunks: 1, + }; +} + +/** + * Summarize long text by chunking and combining summaries + */ +async function summarizeLongText(text, options) { + const { model, systemPrompt, style } = options; + const client = getOpenAI(); + + // Split into chunks + const chunks = []; + let currentChunk = ''; + const sentences = text.split(/(?<=[.!?。！？\n])\s*/); + + for (const sentence of sentences) { + if ((currentChunk + sentence).length > MAX_CHUNK_CHARS && currentChunk) { + chunks.push(currentChunk.trim()); + currentChunk = sentence; + } else { + currentChunk += ' ' + sentence; + } + } + if (currentChunk.trim()) { + chunks.push(currentChunk.trim()); + } + + console.log(`Summarizing ${chunks.length} chunks...`); + + // Summarize each chunk + const chunkSummaries = []; + for (let i = 0; i < chunks.length; i++) { + console.log(`[${i + 1}/${chunks.length}] Summarizing chunk...`); + const response = await client.chat.completions.create({ + model, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: `Please summarize the following text (part ${i + 1} of ${chunks.length}):\n\n${chunks[i]}` }, + ], + temperature: 0.3, + }); + chunkSummaries.push(response.choices[0].message.content); + } + + // Combine summaries if multiple chunks + if (chunkSummaries.length === 1) { + return { + summary: chunkSummaries[0], + model, + style, + inputLength: text.length, + chunks: 1, + }; + } + + // Create final combined summary + const combinedText = chunkSummaries.join('\n\n---\n\n'); + const finalResponse = await client.chat.completions.create({ + model, + messages: [ + { role: 'system', content: `You are an expert summarizer. Combine and synthesize the following partial summaries into a single coherent ${style} summary. Remove redundancy and ensure a smooth flow.` }, + { role: 'user', content: `Please combine these summaries into one:\n\n${combinedText}` }, + ], + temperature: 0.3, + }); + + return { + summary: finalResponse.choices[0].message.content, + model, + style, + inputLength: text.length, + chunks: chunks.length, + }; +} + +/** + * Summarize a text file + */ +export async function summarizeFile(filePath, options = {}) { + if (!fs.existsSync(filePath)) { + throw new Error(`File not found: ${filePath}`); + } + + const text = fs.readFileSync(filePath, 'utf-8'); + const result = await summarizeText(text, options); + + // Save summary to file + const dir = path.dirname(filePath); + const baseName = path.basename(filePath, path.extname(filePath)); + const summaryPath = path.join(dir, `${baseName}_summary.txt`); + + fs.writeFileSync(summaryPath, result.summary, 'utf-8'); + + return { + ...result, + filePath, + summaryPath, + }; +} + +/** + * Get available summary styles + */ +export function getSummaryStyles() { + return { + concise: 'A brief summary capturing main points', + detailed: 'A comprehensive summary with nuances', + bullet: 'Key points as bullet points', + }; +} diff --git a/src/services/youtube.js b/src/services/youtube.js index 4e2021f..55d53a5 100644 --- a/src/services/youtube.js +++ b/src/services/youtube.js @@ -1,6 +1,73 @@ -import youtubedl from 'youtube-dl-exec'; +import { createRequire } from 'module'; import path from 'path'; import fs from 'fs'; +import { spawn } from 'child_process'; + +// Use system yt-dlp binary (check common paths) +const YTDLP_PATH = process.env.YTDLP_PATH || 'yt-dlp'; + +/** + * Execute yt-dlp command and return parsed JSON + */ +async function ytdlp(url, args = []) { + return new Promise((resolve, reject) => { + const proc = spawn(YTDLP_PATH, [...args, url]); + let stdout = ''; + let stderr = ''; + + proc.stdout.on('data', (data) => { stdout += data; }); + proc.stderr.on('data', (data) => { stderr += data; }); + + proc.on('close', (code) => { + if (code === 0) { + try { + resolve(JSON.parse(stdout)); + } catch { + resolve(stdout); + } + } else { + reject(new Error(stderr || `yt-dlp exited with code ${code}`)); + } + }); + }); +} + +/** + * Execute yt-dlp command with progress callback + */ +function ytdlpExec(url, args = [], onProgress) { + return new Promise((resolve, reject) => { + const proc = spawn(YTDLP_PATH, [...args, url]); + let stderr = ''; + + proc.stdout.on('data', (data) => { + const line = data.toString(); + if (onProgress) { + const progressMatch = line.match(/\[download\]\s+(\d+\.?\d*)%/); + const etaMatch = line.match(/ETA\s+(\d+:\d+)/); + const speedMatch = line.match(/at\s+([\d.]+\w+\/s)/); + + if (progressMatch) { + onProgress({ + percent: parseFloat(progressMatch[1]), + eta: etaMatch ? etaMatch[1] : null, + speed: speedMatch ? speedMatch[1] : null, + }); + } + } + }); + + proc.stderr.on('data', (data) => { stderr += data; }); + + proc.on('close', (code) => { + if (code === 0) { + resolve(); + } else { + reject(new Error(stderr || `yt-dlp exited with code ${code}`)); + } + }); + }); +} const OUTPUT_DIR = process.env.OUTPUT_DIR || './output'; @@ -47,12 +114,12 @@ export async function getInfo(url, forcePlaylist = false) { const playlistUrl = extractPlaylistUrl(url); const targetUrl = (forcePlaylist && playlistUrl) ? playlistUrl : url; - const info = await youtubedl(targetUrl, { - dumpSingleJson: true, - noDownload: true, - noWarnings: true, - flatPlaylist: true, - }); + const info = await ytdlp(targetUrl, [ + '--dump-single-json', + '--no-download', + '--no-warnings', + '--flat-playlist', + ]); return info; } catch (error) { throw new Error(`Failed to get info: ${error.message}`); @@ -80,47 +147,32 @@ export async function downloadVideo(url, options = {}) { try { // Get video info first - const info = await youtubedl(url, { - dumpSingleJson: true, - noDownload: true, - noWarnings: true, - }); + const info = await ytdlp(url, [ + '--dump-single-json', + '--no-download', + '--no-warnings', + ]); const title = sanitizeFilename(info.title); const outputPath = path.join(outputDir, `${title}.mp3`); // Download and convert to MP3 with progress - const subprocess = youtubedl.exec(url, { - extractAudio: true, - audioFormat: 'mp3', - audioQuality: 0, - output: outputPath, - noWarnings: true, - newline: true, + await ytdlpExec(url, [ + '--extract-audio', + '--audio-format', 'mp3', + '--audio-quality', '0', + '-o', outputPath, + '--no-warnings', + '--newline', + ], (progress) => { + if (onDownloadProgress) { + onDownloadProgress({ + ...progress, + title: info.title, + }); + } }); - // Parse progress from yt-dlp output - if (onDownloadProgress && subprocess.stdout) { - subprocess.stdout.on('data', (data) => { - const line = data.toString(); - // Parse progress: [download] 45.2% of 10.5MiB at 1.2MiB/s ETA 00:05 - const progressMatch = line.match(/\[download\]\s+(\d+\.?\d*)%/); - const etaMatch = line.match(/ETA\s+(\d+:\d+)/); - const speedMatch = line.match(/at\s+([\d.]+\w+\/s)/); - - if (progressMatch) { - onDownloadProgress({ - percent: parseFloat(progressMatch[1]), - eta: etaMatch ? etaMatch[1] : null, - speed: speedMatch ? speedMatch[1] : null, - title: info.title, - }); - } - }); - } - - await subprocess; - return { success: true, title: info.title,