diff --git a/docs/API.md b/docs/API.md index aaa6df1..2d6d031 100644 --- a/docs/API.md +++ b/docs/API.md @@ -70,9 +70,11 @@ curl -H "Authorization: Bearer your_api_token_here" http://localhost:8888/endpoi - [Health & Info](#health--info) - [Download Endpoints](#download-endpoints) - [Transcription Endpoints](#transcription-endpoints) +- [Conversion Endpoints](#conversion-endpoints) - [Translation Endpoints](#translation-endpoints) - [Summarization Endpoints](#summarization-endpoints) - [File Management](#file-management) +- [Admin Endpoints](#admin-endpoints) - [Security Configuration](#security-configuration) --- @@ -341,6 +343,182 @@ curl -X POST http://localhost:8888/process \ } ``` +### POST /upload-process +**đŸŽ¯ Smart endpoint that auto-detects input and processes accordingly:** +- **Video files** (MP4, AVI, MKV, etc.) → Convert to MP3 → Transcribe +- **Audio files** (MP3, WAV, M4A, etc.) → Transcribe directly +- **URL parameter** → Download from YouTube → Transcribe +- **Mixed input** → Process both uploaded files AND URL + +This endpoint intelligently handles whatever you send it! + +**Form Data:** +- `files`: Video or audio file(s) (optional, multiple files supported, max 50) +- `url`: YouTube URL (optional) +- `language`: Language code for transcription (optional) +- `model`: Transcription model (optional, default: gpt-4o-mini-transcribe) +- `outputPath`: Custom output directory (optional) + +**Note:** You must provide either `files`, `url`, or both. + +**Example 1: Upload video files** +```bash +curl -H "X-API-Key: your_token" \ + -X POST http://localhost:8888/upload-process \ + -F "files=@meeting.mp4" \ + -F "files=@interview.avi" \ + -F "language=en" \ + -F "model=gpt-4o-mini-transcribe" +``` + +**Example 2: Upload audio files** +```bash +curl -H "X-API-Key: your_token" \ + -X POST http://localhost:8888/upload-process \ + -F "files=@podcast.mp3" \ + -F "files=@lecture.wav" \ + -F "language=fr" +``` + +**Example 3: Process YouTube URL** +```bash +curl -H "X-API-Key: your_token" \ + -X POST http://localhost:8888/upload-process \ + -F "url=https://www.youtube.com/watch?v=VIDEO_ID" \ + -F "language=en" +``` + +**Example 4: Mixed - Files AND URL** +```bash +curl -H "X-API-Key: your_token" \ + -X POST http://localhost:8888/upload-process \ + -F "files=@local_video.mp4" \ + -F "url=https://www.youtube.com/watch?v=VIDEO_ID" \ + -F "language=en" +``` + +**Response:** +```json +{ + "success": true, + "totalFiles": 3, + "successCount": 3, + "failCount": 0, + "results": [ + { + "success": true, + "source": "upload", + "sourceType": "video", + "fileName": "meeting.mp4", + "converted": true, + "audioPath": "./output/meeting.mp3", + "audioUrl": "/files/meeting.mp3", + "transcriptionPath": "./output/meeting.txt", + "transcriptionUrl": "/files/meeting.txt", + "text": "Transcribed content..." + }, + { + "success": true, + "source": "upload", + "sourceType": "audio", + "fileName": "podcast.mp3", + "converted": false, + "audioPath": "./output/podcast.mp3", + "audioUrl": "/files/podcast.mp3", + "transcriptionPath": "./output/podcast.txt", + "transcriptionUrl": "/files/podcast.txt", + "text": "Transcribed content..." + }, + { + "success": true, + "source": "url", + "sourceType": "youtube", + "title": "Video Title from YouTube", + "audioPath": "./output/video_title.mp3", + "audioUrl": "/files/video_title.mp3", + "transcriptionPath": "./output/video_title.txt", + "transcriptionUrl": "/files/video_title.txt", + "text": "Transcribed content..." + } + ] +} +``` + +**Supported Video Formats:** +- MP4, AVI, MKV, MOV, WMV, FLV, WebM, M4V + +**Supported Audio Formats:** +- MP3, WAV, M4A, FLAC, OGG, AAC + +--- + +## Conversion Endpoints + +### POST /convert-to-mp3 +Upload video or audio files and convert them to MP3 format. + +**Form Data:** +- `files`: Video or audio file(s) (multiple files supported, max 50) +- `bitrate`: Audio bitrate (optional, default: 192k) +- `quality`: Audio quality 0-9, where 0 is best (optional, default: 2) + +**Example:** +```bash +curl -H "X-API-Key: your_token" \ + -X POST http://localhost:8888/convert-to-mp3 \ + -F "files=@video.mp4" \ + -F "files=@another_video.avi" \ + -F "bitrate=320k" \ + -F "quality=0" +``` + +**Response:** +```json +{ + "success": true, + "totalFiles": 2, + "successCount": 2, + "failCount": 0, + "results": [ + { + "success": true, + "fileName": "video.mp4", + "inputPath": "./output/video.mp4", + "outputPath": "./output/video.mp3", + "outputUrl": "/files/video.mp3", + "size": "5.2 MB" + }, + { + "success": true, + "fileName": "another_video.avi", + "inputPath": "./output/another_video.avi", + "outputPath": "./output/another_video.mp3", + "outputUrl": "/files/another_video.mp3", + "size": "3.8 MB" + } + ] +} +``` + +### GET /supported-formats +Get list of supported video and audio formats for conversion. + +**Example:** +```bash +curl -H "X-API-Key: your_token" \ + http://localhost:8888/supported-formats +``` + +**Response:** +```json +{ + "formats": { + "video": [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".m4v"], + "audio": [".m4a", ".wav", ".flac", ".ogg", ".aac", ".wma", ".opus"] + } +} +``` + --- ## Translation Endpoints @@ -628,6 +806,140 @@ Ensure `OPENAI_API_KEY` is set in your `.env` file for transcription, translatio --- +## Admin Endpoints + +### POST /admin/upload-cookies +Upload YouTube cookies file to enable authentication bypass for bot detection. + +**Purpose**: When YouTube blocks downloads with "Sign in to confirm you're not a bot", this endpoint allows you to upload cookies from your browser to authenticate requests. + +**Authentication**: Required (use your API token) + +**Request:** +- Method: `POST` +- Content-Type: `multipart/form-data` +- Body: File upload with field name `cookies` + +**Example (cURL):** +```bash +# Upload cookies file +curl -X POST \ + -H "X-API-Key: your_api_token" \ + -F "cookies=@youtube-cookies.txt" \ + http://localhost:8888/admin/upload-cookies +``` + +**Example (Using the automation script):** +```bash +# Extract cookies from browser and upload automatically +export API_TOKEN="your_api_token" +export API_URL="http://localhost:8888" +./extract-and-upload-cookies.sh +``` + +**Response (Success - 200):** +```json +{ + "success": true, + "message": "Cookies uploaded successfully", + "paths": { + "local": "/home/user/project/youtube-cookies.txt", + "persistent": "/tmp/share/youtube-cookies.txt" + }, + "note": "Cookies are now active. No restart required." +} +``` + +**Response (Error - 400):** +```json +{ + "error": "No file uploaded", + "message": "Please upload a cookies.txt file", + "help": "Export cookies from your browser using a 'Get cookies.txt' extension" +} +``` + +**Response (Error - 500):** +```json +{ + "error": "Failed to upload cookies", + "message": "Error details..." +} +``` + +### How to Get YouTube Cookies + +**Method 1: Automated Script (Recommended)** + +Use the provided `extract-and-upload-cookies.sh` script: + +```bash +# Set your API credentials +export API_TOKEN="your_api_token" +export API_URL="http://localhost:8888" + +# Run the script - it will auto-detect your browser +./extract-and-upload-cookies.sh +``` + +The script will: +1. Detect installed browsers (Chrome, Firefox, Edge) +2. Extract cookies using yt-dlp +3. Upload them to the API automatically + +**Method 2: Manual Export** + +1. **Install browser extension:** + - Chrome/Edge: [Get cookies.txt LOCALLY](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) + - Firefox: [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) + +2. **Login to YouTube:** + - Visit https://www.youtube.com + - Make sure you're logged into your account + +3. **Export cookies:** + - Click the extension icon + - Click "Export" or "Download" + - Save the file as `youtube-cookies.txt` + +4. **Upload via API:** + ```bash + curl -X POST \ + -H "X-API-Key: your_api_token" \ + -F "cookies=@youtube-cookies.txt" \ + http://localhost:8888/admin/upload-cookies + ``` + +### Cookie Storage + +Cookies are saved to two locations: + +1. **Local project directory**: `/path/to/project/youtube-cookies.txt` + - Used immediately by the API + - Active without restart + +2. **Persistent storage**: `/tmp/share/youtube-cookies.txt` + - Persists across server restarts + - Automatically loaded on startup (via `refresh-cookies.sh`) + +### Cookie Expiration + +- YouTube cookies typically expire after **2-4 weeks** +- When expired, you'll see "YouTube Bot Detection" errors +- Re-upload fresh cookies using the same method + +### Security Notes + +⚠ī¸ **Important Cookie Security:** + +- Cookies = Your YouTube session (treat like a password) +- Never commit `youtube-cookies.txt` to git (already in .gitignore) +- Don't share publicly +- File permissions are automatically set to `600` (owner read/write only) +- Re-export periodically when they expire + +--- + ## Security Configuration ### Environment Variables diff --git a/extract-and-upload-cookies.sh b/extract-and-upload-cookies.sh new file mode 100755 index 0000000..ae7da47 --- /dev/null +++ b/extract-and-upload-cookies.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# Extract and Upload Cookies - All-in-one script +# This script extracts YouTube cookies from your browser and uploads them to the API + +set -e # Exit on error + +# Colors for pretty output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +API_URL="${API_URL:-http://localhost:8888}" +API_TOKEN="${API_TOKEN:-}" +TEMP_COOKIES="/tmp/youtube-cookies-temp.txt" + +echo -e "${BLUE}========================================${NC}" +echo -e "${BLUE}YouTube Cookies Extractor & Uploader${NC}" +echo -e "${BLUE}========================================${NC}" +echo "" + +# Step 1: Check if API token is set +if [ -z "$API_TOKEN" ]; then + echo -e "${YELLOW}⚠ī¸ API_TOKEN not set in environment${NC}" + echo -e "Please enter your API token:" + read -s API_TOKEN + echo "" +fi + +# Step 2: Detect available browsers +echo -e "${BLUE}🔍 Detecting browsers...${NC}" +AVAILABLE_BROWSERS=() + +if command -v google-chrome &> /dev/null || command -v chromium &> /dev/null || [ -d "$HOME/.config/google-chrome" ] || [ -d "$HOME/.config/chromium" ]; then + AVAILABLE_BROWSERS+=("chrome") +fi + +if command -v firefox &> /dev/null || [ -d "$HOME/.mozilla/firefox" ]; then + AVAILABLE_BROWSERS+=("firefox") +fi + +if command -v microsoft-edge &> /dev/null || [ -d "$HOME/.config/microsoft-edge" ]; then + AVAILABLE_BROWSERS+=("edge") +fi + +if [ ${#AVAILABLE_BROWSERS[@]} -eq 0 ]; then + echo -e "${RED}❌ No supported browsers found${NC}" + echo -e "${YELLOW}Supported browsers: Chrome, Firefox, Edge${NC}" + echo "" + echo -e "${YELLOW}Alternative: Manually export cookies${NC}" + echo -e "1. Install browser extension 'Get cookies.txt LOCALLY'" + echo -e "2. Visit youtube.com and login" + echo -e "3. Export cookies to a file" + echo -e "4. Run: curl -X POST -H \"X-API-Key: \$API_TOKEN\" -F \"cookies=@youtube-cookies.txt\" $API_URL/admin/upload-cookies" + exit 1 +fi + +echo -e "${GREEN}✓ Found browsers:${NC} ${AVAILABLE_BROWSERS[*]}" +echo "" + +# Step 3: Let user choose browser +if [ ${#AVAILABLE_BROWSERS[@]} -eq 1 ]; then + BROWSER="${AVAILABLE_BROWSERS[0]}" + echo -e "${GREEN}Using browser: $BROWSER${NC}" +else + echo -e "${YELLOW}Multiple browsers found. Choose one:${NC}" + select BROWSER in "${AVAILABLE_BROWSERS[@]}"; do + if [ -n "$BROWSER" ]; then + break + fi + done +fi + +echo "" + +# Step 4: Check if yt-dlp is installed +if ! command -v yt-dlp &> /dev/null; then + echo -e "${RED}❌ yt-dlp is not installed${NC}" + echo -e "${YELLOW}Install with: pip install yt-dlp${NC}" + echo -e "${YELLOW}Or: sudo apt install yt-dlp${NC}" + exit 1 +fi + +# Step 5: Extract cookies using yt-dlp +echo -e "${BLUE}đŸĒ Extracting cookies from $BROWSER...${NC}" + +if yt-dlp --cookies-from-browser "$BROWSER" --cookies "$TEMP_COOKIES" --no-download "https://www.youtube.com" > /dev/null 2>&1; then + echo -e "${GREEN}✓ Cookies extracted successfully${NC}" +else + echo -e "${RED}❌ Failed to extract cookies${NC}" + echo -e "${YELLOW}Make sure:${NC}" + echo -e " 1. You're logged into YouTube in $BROWSER" + echo -e " 2. $BROWSER is closed (some browsers lock the cookies database)" + echo -e " 3. You have the necessary permissions" + exit 1 +fi + +# Step 6: Upload cookies to API +echo -e "${BLUE}📤 Uploading cookies to API...${NC}" + +UPLOAD_RESPONSE=$(curl -s -w "\n%{http_code}" -X POST \ + -H "X-API-Key: $API_TOKEN" \ + -F "cookies=@$TEMP_COOKIES" \ + "$API_URL/admin/upload-cookies") + +HTTP_CODE=$(echo "$UPLOAD_RESPONSE" | tail -n1) +RESPONSE_BODY=$(echo "$UPLOAD_RESPONSE" | head -n-1) + +if [ "$HTTP_CODE" = "200" ]; then + echo -e "${GREEN}✓ Cookies uploaded successfully!${NC}" + echo "" + echo -e "${GREEN}═══════════════════════════════════════${NC}" + echo -e "${GREEN}SUCCESS! You're all set!${NC}" + echo -e "${GREEN}═══════════════════════════════════════${NC}" + echo "" + echo "$RESPONSE_BODY" | python3 -m json.tool 2>/dev/null || echo "$RESPONSE_BODY" + echo "" + echo -e "${BLUE}ℹī¸ Your cookies are now active${NC}" + echo -e "${BLUE}ℹī¸ YouTube downloads should work without bot detection${NC}" + echo -e "${BLUE}ℹī¸ Cookies expire after a few weeks - run this script again if needed${NC}" +else + echo -e "${RED}❌ Failed to upload cookies${NC}" + echo -e "${RED}HTTP Status: $HTTP_CODE${NC}" + echo "" + echo "$RESPONSE_BODY" + exit 1 +fi + +# Step 7: Cleanup +rm -f "$TEMP_COOKIES" + +echo "" +echo -e "${GREEN}🎉 All done!${NC}" diff --git a/public/app.js b/public/app.js index a9218d5..323d577 100644 --- a/public/app.js +++ b/public/app.js @@ -35,6 +35,67 @@ async function apiFetch(url, options = {}) { return response; } +// Helper: Format YouTube enhanced errors +function formatYouTubeError(errorData) { + // Check if it's an enhanced YouTube bot detection error + if (errorData.error === 'YouTube Bot Detection' || errorData.solution) { + return ` +
+

đŸšĢ ${errorData.error || 'YouTube Error'}

+

${errorData.message}

+ + ${errorData.reason ? `
+ Technical Details + 
${errorData.reason}
+
` : ''} + +
+

💡 Solution

+

${errorData.solution.quick}

+
    + ${errorData.solution.steps.map(step => `
  1. ${step}
    2. `).join('')} +
+ ${errorData.solution.alternative ? ` +

Alternative: ${errorData.solution.alternative}

+ ` : ''} + ${errorData.solution.documentation ? ` +

📚 ${errorData.solution.documentation}

+ ` : ''} +
+ + ${errorData.currentConfig ? ` +
+ Current Configuration + +
+ ` : ''} + +
+ +
+
+ `; + } + + // Generic YouTube error + if (errorData.error && errorData.error.includes('YouTube')) { + return ` +

❌ ${errorData.error}

+

${errorData.message}

+ ${errorData.solution ? `

${errorData.solution}

` : ''} + `; + } + + // Default error + return `

Error

${errorData.message || errorData.error || 'Unknown error'}

`; +} + // API Configuration Panel const configPanel = document.getElementById('api-config-panel'); const configHeader = document.querySelector('.config-header'); @@ -363,11 +424,16 @@ document.getElementById('download-form').addEventListener('submit', async (e) => }); eventSource.addEventListener('error', (e) => { - let errorMsg = 'Download failed'; - try { errorMsg = JSON.parse(e.data).message || errorMsg; } catch {} + let errorHtml = '

Error

Download failed

'; + try { + const errorData = JSON.parse(e.data); + errorHtml = formatYouTubeError(errorData); + } catch { + // Keep default error + } eventSource.close(); progressContainer.style.display = 'none'; - showResult('download-result', false, `

Error

${errorMsg}

`); + showResult('download-result', false, errorHtml); setLoading(button, false); }); @@ -788,11 +854,16 @@ document.getElementById('process-form').addEventListener('submit', async (e) => }); eventSource.addEventListener('error', (e) => { - let errorMsg = 'Processing failed'; - try { errorMsg = JSON.parse(e.data).message || errorMsg; } catch {} + let errorHtml = '

Error

Processing failed

'; + try { + const errorData = JSON.parse(e.data); + errorHtml = formatYouTubeError(errorData); + } catch { + // Keep default error + } eventSource.close(); processProgress.style.display = 'none'; - showResult('process-result', false, `

Error

${errorMsg}

`); + showResult('process-result', false, errorHtml); setLoading(button, false); }); @@ -1328,11 +1399,16 @@ if (summarizeLinkForm) { }); eventSource.addEventListener('error', (e) => { - let errorMsg = 'Processing failed'; - try { errorMsg = JSON.parse(e.data).message || errorMsg; } catch {} + let errorHtml = '

Error

Processing failed

'; + try { + const errorData = JSON.parse(e.data); + errorHtml = formatYouTubeError(errorData); + } catch { + // Keep default error + } eventSource.close(); linkProgress.style.display = 'none'; - showResult('summarize-link-result', false, `

Error

${errorMsg}

`); + showResult('summarize-link-result', false, errorHtml); setLoading(button, false); }); diff --git a/src/server.js b/src/server.js index 90d833e..879e614 100644 --- a/src/server.js +++ b/src/server.js @@ -134,6 +134,14 @@ const authenticate = (req, res, next) => { // Apply authentication to all routes app.use(authenticate); +// Helper function to handle YouTube enhanced errors +function handleYouTubeError(error, res, defaultMessage = 'Operation failed') { + if (error.isEnhanced && error.details) { + return res.status(503).json(error.details); + } + return res.status(500).json({ error: error.message || defaultMessage }); +} + // Serve static files (HTML interface) const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -153,6 +161,7 @@ app.get('/api', (req, res) => { 'POST /download': 'Download as MP3', 'POST /transcribe': 'Transcribe audio file', 'POST /process': 'Download + transcribe', + 'POST /upload-process': 'Smart: Upload video/audio OR URL -> Transcribe', 'GET /files-list': 'List downloaded files', 'GET /files/': 'Serve downloaded files', }, @@ -192,7 +201,7 @@ app.get('/info', async (req, res) => { videoCount: info._type === 'playlist' ? info.entries?.length : 1, }); } catch (error) { - res.status(500).json({ error: error.message }); + handleYouTubeError(error, res); } }); @@ -313,7 +322,12 @@ app.get('/download-stream', async (req, res) => { }); } catch (error) { - sendEvent('error', { message: error.message }); + // Enhanced error for bot detection + if (error.isEnhanced && error.details) { + sendEvent('error', error.details); + } else { + sendEvent('error', { message: error.message }); + } } finally { res.end(); } @@ -351,7 +365,7 @@ app.post('/download', async (req, res) => { })), }); } catch (error) { - res.status(500).json({ error: error.message }); + handleYouTubeError(error, res); } }); @@ -456,6 +470,166 @@ app.post('/upload-transcribe', upload.array('files', 50), async (req, res) => { } }); +/** + * POST /upload-process + * Smart endpoint that auto-detects input type and processes accordingly: + * - Video files (MP4, AVI, etc.) -> Convert to MP3 -> Transcribe + * - Audio files (MP3, WAV, etc.) -> Transcribe directly + * - URL parameter -> Download from YouTube -> Transcribe + * Body: { url?: string, language?: string, model?: string, outputPath?: string } + * Files: files[] (optional, video or audio files) + */ +app.post('/upload-process', uploadVideo.array('files', 50), async (req, res) => { + try { + if (!process.env.OPENAI_API_KEY) { + return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); + } + + const { url, language, model = 'gpt-4o-mini-transcribe', outputPath } = req.body; + const outputDir = outputPath || OUTPUT_DIR; + + // Detect input type + const hasFiles = req.files && req.files.length > 0; + const hasUrl = url && url.trim() !== ''; + + if (!hasFiles && !hasUrl) { + return res.status(400).json({ + error: 'No input provided. Please provide either files to upload or a URL parameter' + }); + } + + const results = []; + let totalFiles = 0; + + // Process URL if provided + if (hasUrl) { + console.log(`[Smart Process] Detected URL: ${url}`); + try { + // Step 1: Download from YouTube + console.log(`[Smart Process] Downloading from YouTube...`); + const downloadResult = await download(url, { outputDir }); + + // Step 2: Transcribe downloaded files + const successfulDownloads = downloadResult.videos.filter(v => v.success); + console.log(`[Smart Process] Transcribing ${successfulDownloads.length} downloaded files...`); + + for (const video of successfulDownloads) { + totalFiles++; + try { + const transcribeResult = await transcribeAndSave(video.filePath, { + language: language || undefined, + responseFormat: 'text', + outputFormat: 'txt', + model, + outputDir, + }); + + results.push({ + success: true, + source: 'url', + sourceType: 'youtube', + title: video.title, + audioPath: video.filePath, + audioUrl: `/files/${path.basename(video.filePath)}`, + transcriptionPath: transcribeResult.transcriptionPath, + transcriptionUrl: `/files/${path.basename(transcribeResult.transcriptionPath)}`, + text: transcribeResult.text, + }); + } catch (error) { + results.push({ + success: false, + source: 'url', + title: video.title, + error: error.message, + }); + } + } + } catch (error) { + results.push({ + success: false, + source: 'url', + error: error.message, + }); + } + } + + // Process uploaded files if provided + if (hasFiles) { + console.log(`[Smart Process] Detected ${req.files.length} uploaded files`); + + for (let i = 0; i < req.files.length; i++) { + const file = req.files[i]; + totalFiles++; + + const isVideo = /\.(mp4|avi|mkv|mov|wmv|flv|webm|m4v)$/i.test(file.originalname); + const isAudio = /\.(mp3|wav|m4a|flac|ogg|aac)$/i.test(file.originalname); + + console.log(`[${i + 1}/${req.files.length}] Processing: ${file.originalname} (${isVideo ? 'video' : 'audio'})`); + + try { + let audioFilePath = file.path; + let conversionResult = null; + + // Step 1: Convert to MP3 if it's a video + if (isVideo) { + console.log(` → Converting video to MP3...`); + conversionResult = await convertToMP3(file.path, { + outputDir, + bitrate: '192k', + quality: '2', + }); + audioFilePath = conversionResult.outputPath; + } + + // Step 2: Transcribe the audio + console.log(` → Transcribing audio...`); + const transcribeResult = await transcribeAndSave(audioFilePath, { + language: language || undefined, + responseFormat: 'text', + outputFormat: 'txt', + model, + outputDir, + }); + + results.push({ + success: true, + source: 'upload', + sourceType: isVideo ? 'video' : 'audio', + fileName: file.originalname, + converted: isVideo, + audioPath: audioFilePath, + audioUrl: `/files/${path.basename(audioFilePath)}`, + transcriptionPath: transcribeResult.transcriptionPath, + transcriptionUrl: `/files/${path.basename(transcribeResult.transcriptionPath)}`, + text: transcribeResult.text, + }); + } catch (error) { + console.error(` ✗ Failed to process ${file.originalname}: ${error.message}`); + results.push({ + success: false, + source: 'upload', + fileName: file.originalname, + error: error.message, + }); + } + } + } + + const successCount = results.filter(r => r.success).length; + const failCount = results.filter(r => !r.success).length; + + res.json({ + success: true, + totalFiles, + successCount, + failCount, + results, + }); + } catch (error) { + res.status(500).json({ error: error.message }); + } +}); + /** * POST /convert-to-mp3 * Upload video/audio files and convert them to MP3 @@ -686,7 +860,12 @@ app.get('/process-stream', async (req, res) => { }); } catch (error) { - sendEvent('error', { message: error.message }); + // Enhanced error for bot detection + if (error.isEnhanced && error.details) { + sendEvent('error', error.details); + } else { + sendEvent('error', { message: error.message }); + } } finally { res.end(); } @@ -757,7 +936,7 @@ app.post('/process', async (req, res) => { results: combinedResults, }); } catch (error) { - res.status(500).json({ error: error.message }); + handleYouTubeError(error, res); } }); @@ -1163,12 +1342,89 @@ app.get('/summarize-stream', async (req, res) => { } catch (error) { console.error(`[Summarize Pipeline] Error: ${error.message}`); - sendEvent('error', { message: error.message }); + // Enhanced error for bot detection + if (error.isEnhanced && error.details) { + sendEvent('error', error.details); + } else { + sendEvent('error', { message: error.message }); + } } finally { res.end(); } }); +/** + * POST /admin/upload-cookies + * Upload YouTube cookies file to persist authentication + */ +const uploadCookies = multer({ + storage: multer.memoryStorage(), + fileFilter: (req, file, cb) => { + if (file.mimetype === 'text/plain' || file.originalname.endsWith('.txt')) { + cb(null, true); + } else { + cb(new Error('Invalid file type. Only .txt files are allowed.')); + } + }, + limits: { + fileSize: 1024 * 1024, // 1MB max + } +}); + +app.post('/admin/upload-cookies', uploadCookies.single('cookies'), async (req, res) => { + try { + if (!req.file) { + return res.status(400).json({ + error: 'No file uploaded', + message: 'Please upload a cookies.txt file', + help: 'Export cookies from your browser using a "Get cookies.txt" extension' + }); + } + + // Paths for storing cookies + const localCookiesPath = path.join(process.cwd(), 'youtube-cookies.txt'); + const shareCookiesPath = '/tmp/share/youtube-cookies.txt'; + + // Write to local directory + fs.writeFileSync(localCookiesPath, req.file.buffer); + fs.chmodSync(localCookiesPath, 0o600); // Secure permissions + + console.log(`✓ Cookies saved to: ${localCookiesPath}`); + + // Also save to /tmp/share if it exists (for persistence across restarts) + try { + if (!fs.existsSync('/tmp/share')) { + fs.mkdirSync('/tmp/share', { recursive: true }); + } + fs.writeFileSync(shareCookiesPath, req.file.buffer); + fs.chmodSync(shareCookiesPath, 0o600); + console.log(`✓ Cookies also saved to: ${shareCookiesPath} (persistent)`); + } catch (err) { + console.warn(`⚠ī¸ Could not save to /tmp/share: ${err.message}`); + } + + // Update environment variable for immediate use + process.env.YOUTUBE_COOKIES_PATH = localCookiesPath; + + res.json({ + success: true, + message: 'Cookies uploaded successfully', + paths: { + local: localCookiesPath, + persistent: fs.existsSync(shareCookiesPath) ? shareCookiesPath : null, + }, + note: 'Cookies are now active. No restart required.' + }); + + } catch (error) { + console.error(`[Upload Cookies] Error: ${error.message}`); + res.status(500).json({ + error: 'Failed to upload cookies', + message: error.message + }); + } +}); + app.listen(PORT, () => { console.log(`Server running on http://localhost:${PORT}`); console.log('\nEndpoints:'); @@ -1177,9 +1433,11 @@ app.listen(PORT, () => { console.log(' POST /download - Download as MP3'); console.log(' POST /transcribe - Transcribe audio file'); console.log(' POST /process - Download + transcribe'); + console.log(' POST /upload-process - Smart: Upload video/audio OR URL -> Transcribe'); console.log(' POST /summarize - Summarize text (GPT-5.1)'); console.log(' POST /summarize-file - Summarize files (GPT-5.1)'); console.log(' GET /summarize-stream - Full pipeline: Download + Transcribe + Summarize'); console.log(' GET /files-list - List downloaded files'); console.log(' GET /files/ - Serve downloaded files'); + console.log(' POST /admin/upload-cookies - Upload YouTube cookies for bot detection bypass'); }); diff --git a/src/services/youtube.js b/src/services/youtube.js index ef9681b..d91926f 100644 --- a/src/services/youtube.js +++ b/src/services/youtube.js @@ -9,14 +9,74 @@ const YTDLP_PATH = process.env.YTDLP_PATH || 'yt-dlp'; // Path to cookies file (optional) const COOKIES_PATH = process.env.YOUTUBE_COOKIES_PATH || null; +// Browser to extract cookies from (chrome, firefox, edge, safari, etc.) +const COOKIES_BROWSER = process.env.YOUTUBE_COOKIES_BROWSER || null; + /** - * Add cookies argument if cookies file exists + * Enhanced error message for YouTube bot detection + */ +function enhanceYouTubeError(error) { + const errorMsg = error.message || error.toString(); + + // Check if it's a bot detection error + if (errorMsg.includes('Sign in to confirm') || + errorMsg.includes('not a bot') || + errorMsg.includes('confirm you\'re not a bot') || + errorMsg.includes('ERROR: Unable to extract')) { + + const cookiesConfigured = COOKIES_BROWSER || COOKIES_PATH; + + return { + error: 'YouTube Bot Detection', + message: 'YouTube is blocking this request. Authentication required.', + reason: errorMsg, + solution: { + quick: 'Upload fresh cookies from your browser', + steps: [ + '1. Install browser extension: "Get cookies.txt LOCALLY"', + '2. Visit youtube.com and log into your account', + '3. Export cookies using the extension', + '4. Upload via API: POST /admin/upload-cookies', + ' Or use the web interface at http://yourserver:8888', + ], + alternative: 'Use extract-and-upload-cookies.sh script for automation', + documentation: 'See COOKIES_QUICK_START.md for detailed instructions' + }, + currentConfig: { + cookiesFile: COOKIES_PATH || 'Not configured', + cookiesBrowser: COOKIES_BROWSER || 'Not configured', + status: cookiesConfigured ? '⚠ī¸ Configured but may be expired' : '❌ Not configured' + } + }; + } + + // Generic YouTube error + return { + error: 'YouTube Download Failed', + message: errorMsg, + solution: 'Check if the URL is valid and the video is available' + }; +} + +/** + * Add cookies argument - prioritizes live browser cookies over file */ function addCookiesArg(args, cookiesPath = null) { + // Option 1: Extract cookies from browser (always fresh) + if (COOKIES_BROWSER) { + console.log(`Using live cookies from ${COOKIES_BROWSER} browser`); + return ['--cookies-from-browser', COOKIES_BROWSER, ...args]; + } + + // Option 2: Use static cookies file (may expire) const cookies = cookiesPath || COOKIES_PATH; if (cookies && fs.existsSync(cookies)) { + console.log(`Using cookies file: ${cookies}`); return ['--cookies', cookies, ...args]; } + + // Option 3: No cookies (may fail on some videos) + console.log('No cookies configured - some videos may fail'); return args; } @@ -140,7 +200,11 @@ export async function getInfo(url, forcePlaylist = false, options = {}) { ], options); return info; } catch (error) { - throw new Error(`Failed to get info: ${error.message}`); + const enhancedError = enhanceYouTubeError(error); + const err = new Error(JSON.stringify(enhancedError)); + err.isEnhanced = true; + err.details = enhancedError; + throw err; } } @@ -199,7 +263,11 @@ export async function downloadVideo(url, options = {}) { url: url, }; } catch (error) { - throw new Error(`Failed to download: ${error.message}`); + const enhancedError = enhanceYouTubeError(error); + const err = new Error(JSON.stringify(enhancedError)); + err.isEnhanced = true; + err.details = enhancedError; + throw err; } } @@ -281,7 +349,11 @@ export async function downloadPlaylist(url, options = {}) { failCount: results.filter(r => !r.success).length, }; } catch (error) { - throw new Error(`Failed to download playlist: ${error.message}`); + const enhancedError = enhanceYouTubeError(error); + const err = new Error(JSON.stringify(enhancedError)); + err.isEnhanced = true; + err.details = enhancedError; + throw err; } }