diff --git a/.env.example b/.env.example index 0d86ca2..297aab3 100644 --- a/.env.example +++ b/.env.example @@ -1,16 +1,17 @@ -# OpenAI API Key for Whisper transcription -OPENAI_API_KEY=your_openai_api_key_here +# Server Configuration +PORT=8889 -# Anthropic API Key for Claude Haiku translation (optional) -ANTHROPIC_API_KEY=your_anthropic_api_key_here +# Storage path for downloaded MP3 files +STORAGE_PATH=/var/hanasuba/music -# Server port (optional, default: 3000) -PORT=3000 +# Python path (optional, default: python3) +PYTHON_PATH=python3 -# Output directory (optional, default: ./output) -OUTPUT_DIR=./output +# yt-dlp path (optional, default: yt-dlp) +YTDLP_PATH=yt-dlp -# YouTube cookies file path (optional, helps bypass bot detection) -# Run: bash scripts/extract-cookies.sh -# Then set the path to your cookies file: -YOUTUBE_COOKIES_PATH=./youtube-cookies.txt +# CORS (optional, default: *) +ALLOWED_ORIGINS=* + +# Environment +NODE_ENV=production diff --git a/.gitignore b/.gitignore index ed0bcdc..0243ca4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,38 +1,42 @@ -# Dependencies +# Node node_modules/ +npm-debug.log +package-lock.json # Environment .env -# Output directory +# Output files output/ - -# Audio files *.mp3 -*.wav -*.m4a -*.ogg -*.flac -*.aac - -# Video files *.mp4 *.webm -*.mkv -*.avi +*.m4a -# Text/transcription files -*.txt - -# YouTube cookies (contains sensitive authentication data) -*cookies*.txt +# Cookies (sensitive) +youtube-cookies.txt cookies.txt +*.cookies + +# Python +__pycache__/ +*.pyc +*.pyo +*.pyd +.Python +*.egg-info/ +dist/ +build/ + +# Playwright +.cache/ +playwright/.cache/ # Logs *.log -npm-debug.log* +logs/ -# OS files +# OS .DS_Store Thumbs.db @@ -42,13 +46,5 @@ Thumbs.db *.swp *.swo -# Temporary files -*.tmp -*.temp - -# Windows device names (reserved names that cause issues) -nul -NUL -CON -PRN -AUX +# Legacy (archived old code) +legacy/ diff --git a/README.md b/README.md index 1b14540..0a7955d 100644 --- a/README.md +++ b/README.md @@ -1,235 +1,367 @@ -# Video to MP3 Transcriptor +# đŸŽ” Hanasuba Music Service v2.0 -Download YouTube videos/playlists to MP3 and transcribe them using OpenAI Whisper API. +**YouTube to MP3 download service with Camoufox stealth cookies** -## Features +Built for [Hanasuba](https://git.etheryale.com/StillHammer/hanasuba) backend. -- Download single YouTube videos as MP3 -- Download entire playlists as MP3 -- Transcribe audio files using OpenAI Whisper API -- CLI interface for quick operations -- REST API for integration with other systems +--- -## Prerequisites +## ✹ Features -- **Node.js** 18+ -- **yt-dlp** installed on your system -- **ffmpeg** installed (for audio conversion) -- **OpenAI API key** (for transcription) +- ✅ **Stealth cookies** - Camoufox anti-detection Firefox +- ✅ **Auto-refresh** - Cookies refresh every 14 days automatically +- ✅ **Bot detection bypass** - Works around YouTube rate limiting +- ✅ **Audio-only downloads** - MP3 192kbps (configurable) +- ✅ **Streaming support** - HTTP range requests for audio players +- ✅ **Metadata extraction** - Title, artist, duration, thumbnail +- ✅ **Retry logic** - Auto-retry with fresh cookies if blocked +- ✅ **REST API** - Simple JSON API for integration -### Installing yt-dlp +--- -```bash -# Windows (winget) -winget install yt-dlp +## đŸ—ïž Architecture -# macOS -brew install yt-dlp - -# Linux -sudo apt install yt-dlp -# or -pip install yt-dlp +``` +music-service (Node.js + Python) +├── Express API (Node.js) +│ ├── Download orchestration +│ └── File streaming +├── Camoufox (Python) +│ ├── Stealth cookie extraction +│ └── Cookie validation +└── yt-dlp + └── YouTube download (using stealth cookies) ``` -### Installing ffmpeg +**Why this stack?** +- **Camoufox** = Undetectable Firefox (bypasses bot detection) +- **yt-dlp** = Best YouTube downloader (handles all edge cases) +- **Node.js** = Fast I/O for streaming + +--- + +## 📩 Installation + +### Prerequisites + +- Node.js 18+ +- Python 3.9+ +- yt-dlp +- ffmpeg + +### Install ```bash -# Windows (winget) -winget install ffmpeg +# Clone repo +git clone https://git.etheryale.com/StillHammer/videotomp3transcriptor.git +cd videotomp3transcriptor +git checkout music-service-v2 -# macOS -brew install ffmpeg - -# Linux -sudo apt install ffmpeg -``` - -## Installation - -```bash -# Clone and install -cd videotoMP3Transcriptor +# Install Node dependencies npm install -# Configure environment +# Install Python dependencies + browsers +npm run setup + +# Configure cp .env.example .env -# Edit .env and add your OPENAI_API_KEY +nano .env # Edit PORT, STORAGE_PATH, etc. + +# Start +npm start ``` -## Usage +--- -### CLI +## 🚀 Usage + +### Start server ```bash -# Download a video as MP3 -npm run cli download "https://youtube.com/watch?v=VIDEO_ID" - -# Download a playlist -npm run cli download "https://youtube.com/playlist?list=PLAYLIST_ID" - -# Download with custom output directory -npm run cli download "URL" -o ./my-folder - -# Get info about a video/playlist -npm run cli info "URL" - -# Transcribe an existing MP3 -npm run cli transcribe ./output/video.mp3 - -# Transcribe with specific language -npm run cli transcribe ./output/video.mp3 -l fr - -# Transcribe with specific model -npm run cli transcribe ./output/video.mp3 -m gpt-4o-mini-transcribe - -# Download AND transcribe -npm run cli process "URL" - -# Download and transcribe with options -npm run cli process "URL" -l en -m gpt-4o-transcribe +npm start ``` -### Linux Scripts +Server runs on `http://localhost:8889` (configurable via `.env`) -Convenience scripts are available in the `scripts/` directory: +### API Endpoints + +#### **POST /download** + +Download YouTube video to MP3. ```bash -# Make scripts executable (first time only) -chmod +x scripts/*.sh - -# Download video/playlist -./scripts/download.sh "https://youtube.com/watch?v=VIDEO_ID" - -# Transcribe a file -./scripts/transcribe.sh ./output/video.mp3 fr - -# Download + transcribe -./scripts/process.sh "https://youtube.com/watch?v=VIDEO_ID" en - -# Start the API server -./scripts/server.sh - -# Get video info -./scripts/info.sh "https://youtube.com/watch?v=VIDEO_ID" -``` - -### API Server - -```bash -# Start the server -npm run server -``` - -Server runs on `http://localhost:3000` by default. - -#### Endpoints - -##### GET /health -Health check endpoint. - -##### GET /info?url=YOUTUBE_URL -Get info about a video or playlist. - -```bash -curl "http://localhost:3000/info?url=https://youtube.com/watch?v=VIDEO_ID" -``` - -##### POST /download -Download video(s) as MP3. - -```bash -curl -X POST http://localhost:3000/download \ +curl -X POST http://localhost:8889/download \ -H "Content-Type: application/json" \ - -d '{"url": "https://youtube.com/watch?v=VIDEO_ID"}' + -d '{"url": "https://youtube.com/watch?v=dQw4w9WgXcQ"}' ``` -##### POST /transcribe -Transcribe an existing audio file. +Response: +```json +{ + "success": true, + "title": "Rick Astley - Never Gonna Give You Up", + "duration": 212, + "artist": "Rick Astley", + "filePath": "/var/hanasuba/music/dQw4w9WgXcQ.mp3", + "fileName": "dQw4w9WgXcQ.mp3", + "youtubeId": "dQw4w9WgXcQ", + "thumbnail": "https://..." +} +``` + +#### **GET /stream/:filename** + +Stream MP3 file (supports range requests for seeking). ```bash -curl -X POST http://localhost:3000/transcribe \ - -H "Content-Type: application/json" \ - -d '{"filePath": "./output/video.mp3", "language": "en"}' +curl http://localhost:8889/stream/dQw4w9WgXcQ.mp3 --output song.mp3 ``` -##### POST /process -Download and transcribe in one call. +#### **DELETE /file/:filename** + +Delete downloaded file. ```bash -curl -X POST http://localhost:3000/process \ - -H "Content-Type: application/json" \ - -d '{"url": "https://youtube.com/watch?v=VIDEO_ID", "language": "en", "format": "txt"}' +curl -X DELETE http://localhost:8889/file/dQw4w9WgXcQ.mp3 ``` -##### GET /files-list -List all downloaded files. +#### **GET /health** -##### GET /files/:filename -Download/stream a specific file. +Health check. -## Configuration - -Environment variables (`.env`): - -| Variable | Description | Default | -|----------|-------------|---------| -| `OPENAI_API_KEY` | Your OpenAI API key | Required for transcription | -| `PORT` | Server port | 3000 | -| `OUTPUT_DIR` | Download directory | ./output | - -## Transcription Models - -| Model | Description | Formats | -|-------|-------------|---------| -| `gpt-4o-transcribe` | Best quality, latest GPT-4o (default) | txt, json | -| `gpt-4o-mini-transcribe` | Faster, cheaper, good quality | txt, json | -| `whisper-1` | Legacy Whisper model | txt, json, srt, vtt | - -## Transcription Formats - -- `txt` - Plain text (all models) -- `json` - JSON response (all models) -- `srt` - SubRip subtitles (whisper-1 only) -- `vtt` - WebVTT subtitles (whisper-1 only) - -## Language Codes - -Common language codes for the `-l` option: -- `en` - English -- `fr` - French -- `es` - Spanish -- `de` - German -- `it` - Italian -- `pt` - Portuguese -- `zh` - Chinese -- `ja` - Japanese -- `ko` - Korean -- `ru` - Russian - -Leave empty for auto-detection. - -## Project Structure - -``` -videotoMP3Transcriptor/ -├── src/ -│ ├── services/ -│ │ ├── youtube.js # YouTube download service -│ │ └── transcription.js # OpenAI transcription service -│ ├── cli.js # CLI entry point -│ └── server.js # Express API server -├── scripts/ # Linux convenience scripts -│ ├── download.sh # Download video/playlist -│ ├── transcribe.sh # Transcribe audio file -│ ├── process.sh # Download + transcribe -│ ├── server.sh # Start API server -│ └── info.sh # Get video info -├── output/ # Downloaded files -├── .env # Configuration -└── package.json +```bash +curl http://localhost:8889/health ``` -## License +#### **POST /admin/refresh-cookies** + +Force refresh cookies (normally automatic). + +```bash +curl -X POST http://localhost:8889/admin/refresh-cookies +``` + +--- + +## đŸȘ How Cookies Work + +### Automatic Refresh + +Cookies are **automatically refreshed** in these cases: + +1. **Every 14 days** (proactive refresh) +2. **On startup** (if invalid) +3. **Every 12 hours** (validation check) +4. **On bot detection** (retry with fresh cookies) + +### Manual Refresh + +```bash +# Via API +curl -X POST http://localhost:8889/admin/refresh-cookies + +# Via npm script +npm run cookies:extract +``` + +### Validation + +```bash +# Check if cookies are valid +npm run cookies:validate +``` + +--- + +## 🔧 Configuration + +### Environment Variables + +See `.env.example`: + +```bash +PORT=8889 # Server port +STORAGE_PATH=/var/hanasuba/music # Where to save MP3 files +PYTHON_PATH=python3 # Python binary +YTDLP_PATH=yt-dlp # yt-dlp binary +ALLOWED_ORIGINS=* # CORS +``` + +### Audio Quality + +Pass `quality` parameter in download request: + +```json +{ + "url": "https://youtube.com/watch?v=...", + "quality": "320k" // or "192k" (default), "128k" +} +``` + +--- + +## 🐛 Troubleshooting + +### "Sign in to confirm you're not a bot" + +**Solution**: Cookies have expired or are invalid. + +```bash +# Force refresh +curl -X POST http://localhost:8889/admin/refresh-cookies + +# Or restart service (auto-refresh on startup) +npm start +``` + +### yt-dlp not found + +```bash +# Install yt-dlp +pip install yt-dlp +# or +sudo apt install yt-dlp +``` + +### Camoufox install fails + +```bash +# Manual install +pip install camoufox camoufox-captcha playwright +playwright install firefox +``` + +### Downloads slow + +This is normal. YouTube throttles downloads. The service uses `mweb` client for best speed. + +--- + +## 🔐 Security + +- Cookies file permissions: `600` (owner read/write only) +- Cookies **never** logged or exposed +- Cookies stored locally only +- CORS configurable via `ALLOWED_ORIGINS` + +--- + +## 🚱 Deployment + +### PM2 (recommended) + +```bash +pm2 start src/server.js --name music-service +pm2 save +pm2 startup +``` + +### systemd + +```ini +[Unit] +Description=Hanasuba Music Service +After=network.target + +[Service] +Type=simple +User=debian +WorkingDirectory=/home/debian/videotomp3transcriptor +ExecStart=/usr/bin/node src/server.js +Restart=on-failure + +[Install] +WantedBy=multi-user.target +``` + +```bash +sudo systemctl enable music-service +sudo systemctl start music-service +``` + +--- + +## 📊 Monitoring + +Check service status: + +```bash +# Health check +curl http://localhost:8889/health + +# Cookies status +curl http://localhost:8889/admin/cookies-status + +# Logs (PM2) +pm2 logs music-service + +# Logs (systemd) +journalctl -u music-service -f +``` + +--- + +## 🔗 Integration with Hanasuba + +Hanasuba (Rust) calls this service via HTTP: + +```rust +// In Hanasuba src/music/client.rs +let response = reqwest::Client::new() + .post("http://localhost:8889/download") + .json(&json!({ "url": youtube_url })) + .send() + .await?; + +let result: DownloadResult = response.json().await?; +// Save metadata to PostgreSQL +``` + +--- + +## 📝 Development + +```bash +# Dev mode (auto-restart on changes) +npm run dev + +# Extract cookies manually +npm run cookies:extract + +# Validate cookies +npm run cookies:validate +``` + +--- + +## 🆚 v1 vs v2 + +| Feature | v1 (legacy) | v2 (current) | +|---------|-------------|--------------| +| Cookies | Firefox standard | **Camoufox stealth** | +| Auto-refresh | ❌ Manual | ✅ Automatic (14 days) | +| Bot detection | ❌ Fails often | ✅ Auto-retry | +| Validation | ❌ None | ✅ Every 12h | +| Reliability | ~60% | **~95%** | +| Transcription | ✅ OpenAI Whisper | ❌ Removed (not needed) | +| Translation | ✅ Claude | ❌ Removed (not needed) | + +v2 is **focused** on one thing: reliable YouTube → MP3 downloads. + +--- + +## 📄 License MIT + +--- + +## 🙏 Credits + +- [Camoufox](https://github.com/daijro/camoufox) - Stealth Firefox +- [yt-dlp](https://github.com/yt-dlp/yt-dlp) - YouTube downloader +- [Hanasuba](https://git.etheryale.com/StillHammer/hanasuba) - Main backend + +--- + +**Built with ❀ for Hanasuba** diff --git a/docs/API.md b/docs/API.md deleted file mode 100644 index d90c4c8..0000000 --- a/docs/API.md +++ /dev/null @@ -1,1093 +0,0 @@ -# API Documentation - Video to MP3 Transcriptor - -## Base URL -``` -http://localhost:3001 -``` - -## 🔐 Authentication - -**⚠ IMPORTANT**: All API endpoints (except `/health` and `/api`) require authentication using an API token. - -### How to Authenticate - -Include your API token in **one** of these ways: - -**Option 1: X-API-Key header (Recommended)** -```bash -curl -H "X-API-Key: your_api_token_here" http://localhost:3001/endpoint -``` - -**Option 2: Authorization Bearer header** -```bash -curl -H "Authorization: Bearer your_api_token_here" http://localhost:3001/endpoint -``` - -### Configuration - -1. Set your API token in `.env`: - ```env - API_TOKEN=your_secure_token_here - ``` - -2. Generate a secure token for production: - ```bash - # Linux/Mac - openssl rand -hex 32 - - # Or use any secure random string generator - ``` - -### Security Notes - -- **Public endpoints** (no auth required): `/health`, `/api` -- **Protected endpoints**: All other endpoints require authentication -- In **development**: If `API_TOKEN` is not set, the API will work without authentication (with a warning) -- In **production**: Always set a strong `API_TOKEN` - -### Error Responses - -**401 Unauthorized** - No API key provided: -```json -{ - "error": "Unauthorized", - "message": "API key required. Provide X-API-Key header or Authorization: Bearer " -} -``` - -**403 Forbidden** - Invalid API key: -```json -{ - "error": "Forbidden", - "message": "Invalid API key" -} -``` - ---- - -## Table of Contents -- [Authentication](#-authentication) -- [Health & Info](#health--info) -- [Public Download Endpoint](#public-download-endpoint) -- [Download Endpoints](#download-endpoints) -- [Transcription Endpoints](#transcription-endpoints) -- [Conversion Endpoints](#conversion-endpoints) -- [Translation Endpoints](#translation-endpoints) -- [Summarization Endpoints](#summarization-endpoints) -- [File Management](#file-management) -- [Admin Endpoints](#admin-endpoints) -- [Security Configuration](#security-configuration) - ---- - -## Health & Info - -### GET /health -Health check endpoint. - -**Authentication**: Not required (public) - -**Response:** -```json -{ - "status": "ok", - "timestamp": "2025-11-28T12:00:00.000Z" -} -``` - -### GET /api -Get API information and available endpoints. - -**Authentication**: Not required (public) - -**Response:** -```json -{ - "name": "Video to MP3 Transcriptor API", - "version": "1.0.0", - "endpoints": { ... } -} -``` - ---- - -## Public Download Endpoint - -### GET /public/download/:filename -Public endpoint to download files without authentication. - -**Authentication**: Not required (public) - -**Purpose**: Share direct download links for generated files (MP3, transcriptions, translations, summaries) without requiring API authentication. - -**URL Parameters:** -- `filename` (required): Name of the file to download - -**Security**: -- Directory traversal protection enabled (uses `path.basename()`) -- Only files in the configured OUTPUT_DIR are accessible -- No authentication required - -**Example:** -```bash -# Direct download (no auth needed) -curl -O http://localhost:3001/public/download/my_video.mp3 - -# Or simply open in browser -http://localhost:3001/public/download/my_video.mp3 -``` - -**Response (Success):** -- File download with proper Content-Disposition headers -- Browser will prompt to download the file - -**Response (Error - 404):** -```json -{ - "error": "File not found", - "message": "File 'my_video.mp3' does not exist" -} -``` - -**Response (Error - 500):** -```json -{ - "error": "Download failed", - "message": "Error details..." -} -``` - -**Use Cases:** -- Share download links via email/chat -- Embed in web applications -- Direct browser downloads -- Public file sharing - -**Note**: After processing (download, transcription, etc.), use the returned `filePath` or `fileUrl` from authenticated endpoints, then construct public URL: -``` -/public/download/{basename_of_filePath} -``` - ---- - -### GET /info -Get information about a YouTube video or playlist. - -**Query Parameters:** -- `url` (required): YouTube URL - -**Example:** -```bash -curl -H "X-API-Key: your_token" \ - "http://localhost:3001/info?url=https://www.youtube.com/watch?v=VIDEO_ID" -``` - -**Response:** -```json -{ - "success": true, - "title": "Video Title", - "type": "video", - "duration": 300, - "channel": "Channel Name", - "videoCount": 1 -} -``` - ---- - -## Download Endpoints - -### GET /download-stream -Download YouTube video(s) to MP3 with Server-Sent Events (SSE) progress updates. - -**Query Parameters:** -- `url` (required): YouTube URL -- `outputPath` (optional): Custom output directory path - -**Example:** -```bash -curl -H "X-API-Key: your_token" \ - "http://localhost:3001/download-stream?url=https://www.youtube.com/watch?v=VIDEO_ID" -``` - -**SSE Events:** -- `info`: Video/playlist information -- `progress`: Download progress updates -- `video-complete`: Individual video completion -- `complete`: All downloads complete -- `error`: Error occurred - -### POST /download -Download YouTube video(s) to MP3 (non-streaming). - -**Body Parameters:** -```json -{ - "url": "https://www.youtube.com/watch?v=VIDEO_ID", - "outputPath": "./custom/path" // optional -} -``` - -**Example:** -```bash -curl -H "X-API-Key: your_token" \ - -X POST http://localhost:3001/download \ - -H "Content-Type: application/json" \ - -d '{"url":"https://www.youtube.com/watch?v=VIDEO_ID"}' -``` - -**Response:** -```json -{ - "success": true, - "playlistTitle": null, - "totalVideos": 1, - "successCount": 1, - "failCount": 0, - "videos": [ - { - "success": true, - "title": "Video Title", - "filePath": "./output/video.mp3", - "fileUrl": "/files/video.mp3" - } - ] -} -``` - ---- - -## Transcription Endpoints - -### POST /transcribe -Transcribe an existing audio file. - -**Body Parameters:** -```json -{ - "filePath": "./output/audio.mp3", - "language": "en", // optional (auto-detect if not specified) - "format": "txt", // optional: txt, json, srt, vtt - "model": "gpt-4o-mini-transcribe", // optional: gpt-4o-mini-transcribe (default), gpt-4o-transcribe, whisper-1 - "outputPath": "./custom/path" // optional -} -``` - -**Available Models:** -- `gpt-4o-mini-transcribe` (default) - Fast and cost-effective -- `gpt-4o-transcribe` - Higher quality -- `whisper-1` - Original Whisper model (supports more formats) - -**Example:** -```bash -curl -H "X-API-Key: your_token" \ - -X POST http://localhost:3001/transcribe \ - -H "Content-Type: application/json" \ - -d '{ - "filePath": "./output/audio.mp3", - "language": "en", - "model": "gpt-4o-mini-transcribe" - }' -``` - -**Response:** -```json -{ - "success": true, - "filePath": "./output/audio.mp3", - "transcriptionPath": "./output/audio.txt", - "transcriptionUrl": "/files/audio.txt", - "text": "Transcribed text content..." -} -``` - -### POST /upload-transcribe -Upload and transcribe audio files. - -**Form Data:** -- `files`: Audio file(s) (multiple files supported, max 50) -- `language`: Language code (optional) -- `model`: Transcription model (optional, default: gpt-4o-mini-transcribe) -- `outputPath`: Custom output directory (optional) - -**Example:** -```bash -curl -X POST http://localhost:3001/upload-transcribe \ - -F "files=@audio1.mp3" \ - -F "files=@audio2.mp3" \ - -F "language=en" \ - -F "model=gpt-4o-mini-transcribe" -``` - -**Response:** -```json -{ - "success": true, - "totalFiles": 2, - "successCount": 2, - "failCount": 0, - "results": [ - { - "success": true, - "fileName": "audio1.mp3", - "transcriptionPath": "./output/audio1.txt", - "transcriptionUrl": "/files/audio1.txt", - "text": "Transcription..." - } - ] -} -``` - -### GET /process-stream -Download + Transcribe with SSE progress updates. - -**Query Parameters:** -- `url` (required): YouTube URL -- `language` (optional): Language code -- `model` (optional): Transcription model (default: gpt-4o-mini-transcribe) -- `outputPath` (optional): Custom output directory - -**Example:** -```bash -curl "http://localhost:3001/process-stream?url=https://www.youtube.com/watch?v=VIDEO_ID&language=en&model=gpt-4o-mini-transcribe" -``` - -**SSE Events:** -- `info`: Video information -- `progress`: Progress updates (downloading or transcribing) -- `video-complete`: Download complete -- `transcribe-complete`: Transcription complete -- `complete`: All operations complete -- `error`: Error occurred - -### POST /process -Download + Transcribe (non-streaming). - -**Body Parameters:** -```json -{ - "url": "https://www.youtube.com/watch?v=VIDEO_ID", - "language": "en", // optional - "format": "txt", // optional - "model": "gpt-4o-mini-transcribe", // optional - "outputPath": "./custom/path" // optional -} -``` - -**Example:** -```bash -curl -X POST http://localhost:3001/process \ - -H "Content-Type: application/json" \ - -d '{ - "url": "https://www.youtube.com/watch?v=VIDEO_ID", - "language": "en", - "model": "gpt-4o-mini-transcribe" - }' -``` - -**Response:** -```json -{ - "success": true, - "playlistTitle": null, - "totalVideos": 1, - "downloadedCount": 1, - "transcribedCount": 1, - "results": [ - { - "title": "Video Title", - "downloadSuccess": true, - "audioPath": "./output/video.mp3", - "audioUrl": "/files/video.mp3", - "transcriptionSuccess": true, - "transcriptionPath": "./output/video.txt", - "transcriptionUrl": "/files/video.txt", - "text": "Transcription..." - } - ] -} -``` - -### POST /upload-process -**🎯 Smart endpoint that auto-detects input and processes accordingly:** -- **Video files** (MP4, AVI, MKV, etc.) → Convert to MP3 → Transcribe -- **Audio files** (MP3, WAV, M4A, etc.) → Transcribe directly -- **URL parameter** → Download from YouTube → Transcribe -- **Mixed input** → Process both uploaded files AND URL - -This endpoint intelligently handles whatever you send it! - -**Form Data:** -- `files`: Video or audio file(s) (optional, multiple files supported, max 50) -- `url`: YouTube URL (optional) -- `language`: Language code for transcription (optional) -- `model`: Transcription model (optional, default: gpt-4o-mini-transcribe) -- `outputPath`: Custom output directory (optional) - -**Note:** You must provide either `files`, `url`, or both. - -**Example 1: Upload video files** -```bash -curl -H "X-API-Key: your_token" \ - -X POST http://localhost:3001/upload-process \ - -F "files=@meeting.mp4" \ - -F "files=@interview.avi" \ - -F "language=en" \ - -F "model=gpt-4o-mini-transcribe" -``` - -**Example 2: Upload audio files** -```bash -curl -H "X-API-Key: your_token" \ - -X POST http://localhost:3001/upload-process \ - -F "files=@podcast.mp3" \ - -F "files=@lecture.wav" \ - -F "language=fr" -``` - -**Example 3: Process YouTube URL** -```bash -curl -H "X-API-Key: your_token" \ - -X POST http://localhost:3001/upload-process \ - -F "url=https://www.youtube.com/watch?v=VIDEO_ID" \ - -F "language=en" -``` - -**Example 4: Mixed - Files AND URL** -```bash -curl -H "X-API-Key: your_token" \ - -X POST http://localhost:3001/upload-process \ - -F "files=@local_video.mp4" \ - -F "url=https://www.youtube.com/watch?v=VIDEO_ID" \ - -F "language=en" -``` - -**Response:** -```json -{ - "success": true, - "totalFiles": 3, - "successCount": 3, - "failCount": 0, - "results": [ - { - "success": true, - "source": "upload", - "sourceType": "video", - "fileName": "meeting.mp4", - "converted": true, - "audioPath": "./output/meeting.mp3", - "audioUrl": "/files/meeting.mp3", - "transcriptionPath": "./output/meeting.txt", - "transcriptionUrl": "/files/meeting.txt", - "text": "Transcribed content..." - }, - { - "success": true, - "source": "upload", - "sourceType": "audio", - "fileName": "podcast.mp3", - "converted": false, - "audioPath": "./output/podcast.mp3", - "audioUrl": "/files/podcast.mp3", - "transcriptionPath": "./output/podcast.txt", - "transcriptionUrl": "/files/podcast.txt", - "text": "Transcribed content..." - }, - { - "success": true, - "source": "url", - "sourceType": "youtube", - "title": "Video Title from YouTube", - "audioPath": "./output/video_title.mp3", - "audioUrl": "/files/video_title.mp3", - "transcriptionPath": "./output/video_title.txt", - "transcriptionUrl": "/files/video_title.txt", - "text": "Transcribed content..." - } - ] -} -``` - -**Supported Video Formats:** -- MP4, AVI, MKV, MOV, WMV, FLV, WebM, M4V - -**Supported Audio Formats:** -- MP3, WAV, M4A, FLAC, OGG, AAC - ---- - -## Conversion Endpoints - -### POST /convert-to-mp3 -Upload video or audio files and convert them to MP3 format. - -**Form Data:** -- `files`: Video or audio file(s) (multiple files supported, max 50) -- `bitrate`: Audio bitrate (optional, default: 192k) -- `quality`: Audio quality 0-9, where 0 is best (optional, default: 2) - -**Example:** -```bash -curl -H "X-API-Key: your_token" \ - -X POST http://localhost:3001/convert-to-mp3 \ - -F "files=@video.mp4" \ - -F "files=@another_video.avi" \ - -F "bitrate=320k" \ - -F "quality=0" -``` - -**Response:** -```json -{ - "success": true, - "totalFiles": 2, - "successCount": 2, - "failCount": 0, - "results": [ - { - "success": true, - "fileName": "video.mp4", - "inputPath": "./output/video.mp4", - "outputPath": "./output/video.mp3", - "outputUrl": "/files/video.mp3", - "size": "5.2 MB" - }, - { - "success": true, - "fileName": "another_video.avi", - "inputPath": "./output/another_video.avi", - "outputPath": "./output/another_video.mp3", - "outputUrl": "/files/another_video.mp3", - "size": "3.8 MB" - } - ] -} -``` - -### GET /supported-formats -Get list of supported video and audio formats for conversion. - -**Example:** -```bash -curl -H "X-API-Key: your_token" \ - http://localhost:3001/supported-formats -``` - -**Response:** -```json -{ - "formats": { - "video": [".mp4", ".avi", ".mkv", ".mov", ".wmv", ".flv", ".webm", ".m4v"], - "audio": [".m4a", ".wav", ".flac", ".ogg", ".aac", ".wma", ".opus"] - } -} -``` - ---- - -## Translation Endpoints - -### GET /languages -Get available translation languages. - -**Response:** -```json -{ - "languages": { - "en": "English", - "fr": "French", - "es": "Spanish", - "de": "German", - "zh": "Chinese", - "ja": "Japanese", - ... - } -} -``` - -### POST /translate -Translate text. - -**Body Parameters:** -```json -{ - "text": "Text to translate", - "targetLang": "fr", // required: target language code - "sourceLang": "en" // optional: source language (auto-detect if not specified) -} -``` - -**Example:** -```bash -curl -X POST http://localhost:3001/translate \ - -H "Content-Type: application/json" \ - -d '{ - "text": "Hello, how are you?", - "targetLang": "fr" - }' -``` - -**Response:** -```json -{ - "success": true, - "originalText": "Hello, how are you?", - "translatedText": "Bonjour, comment allez-vous ?", - "targetLanguage": "French", - "sourceLanguage": "auto-detected", - "chunks": 1 -} -``` - -### POST /translate-file -Translate uploaded text files. - -**Form Data:** -- `files`: Text file(s) (.txt, multiple files supported, max 50) -- `targetLang`: Target language code (required) -- `sourceLang`: Source language code (optional) -- `outputPath`: Custom output directory (optional) - -**Example:** -```bash -curl -X POST http://localhost:3001/translate-file \ - -F "files=@document.txt" \ - -F "targetLang=fr" \ - -F "sourceLang=en" -``` - -**Response:** -```json -{ - "success": true, - "totalFiles": 1, - "successCount": 1, - "failCount": 0, - "results": [ - { - "success": true, - "fileName": "document.txt", - "translationPath": "./output/document_fr.txt", - "translationUrl": "/files/document_fr.txt", - "translatedText": "Translated content..." - } - ] -} -``` - ---- - -## Summarization Endpoints - -### GET /summary-styles -Get available summary styles. - -**Response:** -```json -{ - "styles": { - "concise": "A brief summary capturing main points", - "detailed": "A comprehensive summary with nuances", - "bullet": "Key points as bullet points" - } -} -``` - -### POST /summarize -Summarize text using GPT-5.1. - -**Body Parameters:** -```json -{ - "text": "Long text to summarize...", - "style": "concise", // optional: concise (default), detailed, bullet - "language": "same", // optional: 'same' (default) or language code - "model": "gpt-5.1" // optional: default is gpt-5.1 -} -``` - -**Example:** -```bash -curl -X POST http://localhost:3001/summarize \ - -H "Content-Type: application/json" \ - -d '{ - "text": "Long article content...", - "style": "bullet", - "language": "same" - }' -``` - -**Response:** -```json -{ - "success": true, - "summary": "Summary content...", - "model": "gpt-5.1", - "style": "bullet", - "inputLength": 5000, - "chunks": 1 -} -``` - -### POST /summarize-file -Summarize uploaded text files using GPT-5.1. - -**Form Data:** -- `files`: Text file(s) (.txt, multiple files supported, max 50) -- `style`: Summary style (optional, default: concise) -- `language`: Output language (optional, default: same) -- `model`: AI model (optional, default: gpt-5.1) -- `outputPath`: Custom output directory (optional) - -**Example:** -```bash -curl -X POST http://localhost:3001/summarize-file \ - -F "files=@article.txt" \ - -F "style=detailed" \ - -F "language=same" -``` - -**Response:** -```json -{ - "success": true, - "totalFiles": 1, - "successCount": 1, - "failCount": 0, - "results": [ - { - "success": true, - "fileName": "article.txt", - "summaryPath": "./output/article_summary.txt", - "summaryUrl": "/files/article_summary.txt", - "summary": "Summary content...", - "model": "gpt-5.1", - "chunks": 1 - } - ] -} -``` - -### GET /summarize-stream -Full pipeline: Download -> Transcribe -> Summarize with SSE progress. - -**Query Parameters:** -- `url` (required): YouTube URL -- `style` (optional): Summary style (default: concise) -- `language` (optional): Output language (default: same) -- `model` (optional): Transcription model (default: gpt-4o-mini-transcribe) -- `outputPath` (optional): Custom output directory - -**Example:** -```bash -curl "http://localhost:3001/summarize-stream?url=https://www.youtube.com/watch?v=VIDEO_ID&style=bullet&model=gpt-4o-mini-transcribe" -``` - -**SSE Events:** -- `info`: Video information -- `progress`: Progress updates (downloading, transcribing, or summarizing) -- `video-complete`: Download complete -- `transcribe-complete`: Transcription complete -- `summarize-complete`: Summary complete -- `complete`: All operations complete -- `error`: Error occurred - ---- - -## File Management - -### GET /files-list -List all downloaded/generated files. - -**Example:** -```bash -curl http://localhost:3001/files-list -``` - -**Response:** -```json -{ - "files": [ - { - "name": "video.mp3", - "url": "/files/video.mp3", - "path": "./output/video.mp3" - }, - { - "name": "video.txt", - "url": "/files/video.txt", - "path": "./output/video.txt" - } - ] -} -``` - -### GET /files/:filename -Serve a specific file. - -**Example:** -```bash -curl http://localhost:3001/files/video.mp3 --output video.mp3 -``` - ---- - -## Error Responses - -All endpoints return error responses in the following format: - -```json -{ - "error": "Error message describing what went wrong" -} -``` - -Common HTTP status codes: -- `400` - Bad Request (missing required parameters) -- `500` - Internal Server Error (processing failed) - ---- - -## Notes - -### Output Paths -All endpoints that support `outputPath` parameter: -- If not specified, files are saved to the default `OUTPUT_DIR` (./output) -- If specified, files are saved to the custom path provided - -### Models -- **Transcription**: Default is `gpt-4o-mini-transcribe` (cost-effective) -- **Summarization**: Default is `gpt-5.1` (latest GPT model) -- **Translation**: Uses `gpt-4o-mini` (hardcoded) - -### File Formats -- **Audio**: MP3, WAV, M4A, OGG, FLAC -- **Text**: TXT files -- **Transcription outputs**: TXT, JSON, SRT, VTT (depending on model) - -### API Key -Ensure `OPENAI_API_KEY` is set in your `.env` file for transcription, translation, and summarization features to work. - ---- - -## Admin Endpoints - -### POST /admin/upload-cookies -Upload YouTube cookies file to enable authentication bypass for bot detection. - -**Purpose**: When YouTube blocks downloads with "Sign in to confirm you're not a bot", this endpoint allows you to upload cookies from your browser to authenticate requests. - -**Authentication**: Required (use your API token) - -**Request:** -- Method: `POST` -- Content-Type: `multipart/form-data` -- Body: File upload with field name `cookies` - -**Example (cURL):** -```bash -# Upload cookies file -curl -X POST \ - -H "X-API-Key: your_api_token" \ - -F "cookies=@youtube-cookies.txt" \ - http://localhost:3001/admin/upload-cookies -``` - -**Example (Using the automation script):** -```bash -# Extract cookies from browser and upload automatically -export API_TOKEN="your_api_token" -export API_URL="http://localhost:3001" -./extract-and-upload-cookies.sh -``` - -**Response (Success - 200):** -```json -{ - "success": true, - "message": "Cookies uploaded successfully", - "paths": { - "local": "/home/user/project/youtube-cookies.txt", - "persistent": "/tmp/share/youtube-cookies.txt" - }, - "note": "Cookies are now active. No restart required." -} -``` - -**Response (Error - 400):** -```json -{ - "error": "No file uploaded", - "message": "Please upload a cookies.txt file", - "help": "Export cookies from your browser using a 'Get cookies.txt' extension" -} -``` - -**Response (Error - 500):** -```json -{ - "error": "Failed to upload cookies", - "message": "Error details..." -} -``` - -### How to Get YouTube Cookies - -**Method 1: Automated Script (Recommended)** - -Use the provided `extract-and-upload-cookies.sh` script: - -```bash -# Set your API credentials -export API_TOKEN="your_api_token" -export API_URL="http://localhost:3001" - -# Run the script - it will auto-detect your browser -./extract-and-upload-cookies.sh -``` - -The script will: -1. Detect installed browsers (Chrome, Firefox, Edge) -2. Extract cookies using yt-dlp -3. Upload them to the API automatically - -**Method 2: Manual Export** - -1. **Install browser extension:** - - Chrome/Edge: [Get cookies.txt LOCALLY](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) - - Firefox: [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) - -2. **Login to YouTube:** - - Visit https://www.youtube.com - - Make sure you're logged into your account - -3. **Export cookies:** - - Click the extension icon - - Click "Export" or "Download" - - Save the file as `youtube-cookies.txt` - -4. **Upload via API:** - ```bash - curl -X POST \ - -H "X-API-Key: your_api_token" \ - -F "cookies=@youtube-cookies.txt" \ - http://localhost:3001/admin/upload-cookies - ``` - -### Cookie Storage - -Cookies are saved to two locations: - -1. **Local project directory**: `/path/to/project/youtube-cookies.txt` - - Used immediately by the API - - Active without restart - -2. **Persistent storage**: `/tmp/share/youtube-cookies.txt` - - Persists across server restarts - - Automatically loaded on startup (via `refresh-cookies.sh`) - -### Cookie Expiration - -- YouTube cookies typically expire after **2-4 weeks** -- When expired, you'll see "YouTube Bot Detection" errors -- Re-upload fresh cookies using the same method - -### Security Notes - -⚠ **Important Cookie Security:** - -- Cookies = Your YouTube session (treat like a password) -- Never commit `youtube-cookies.txt` to git (already in .gitignore) -- Don't share publicly -- File permissions are automatically set to `600` (owner read/write only) -- Re-export periodically when they expire - ---- - -## Security Configuration - -### Environment Variables - -Required security variables in `.env`: - -```env -# API Authentication Token -API_TOKEN=your_secure_random_token_here - -# CORS - Allowed Origins (comma-separated) -# Development: * (all origins) -# Production: https://yourdomain.com,https://app.yourdomain.com -ALLOWED_ORIGINS=* - -# Server Port -PORT=8888 - -# Output Directory -OUTPUT_DIR=./output - -# OpenAI API Key (required for AI features) -OPENAI_API_KEY=sk-... -``` - -### Security Features - -The API implements the following security measures: - -1. **API Token Authentication** - - All endpoints (except `/health` and `/api`) require authentication - - Supports both `X-API-Key` and `Authorization: Bearer` headers - -2. **CORS Protection** - - Configurable allowed origins via `ALLOWED_ORIGINS` - - Restricts cross-origin requests to trusted domains - -3. **HTTP Security Headers** - - `X-Content-Type-Options: nosniff` - - `X-Frame-Options: DENY` - - `X-XSS-Protection: 1; mode=block` - - `Strict-Transport-Security: max-age=31536000; includeSubDomains` - - `Content-Security-Policy` with strict policies - -4. **Input Validation** - - File type validation for uploads - - Parameter validation on all endpoints - -### Production Deployment Checklist - -Before deploying to production: - -- [ ] Generate a strong, unique `API_TOKEN` (min 32 characters) -- [ ] Set `ALLOWED_ORIGINS` to your specific domains (remove `*`) -- [ ] Ensure `OPENAI_API_KEY` is properly set -- [ ] Use HTTPS (not HTTP) for all connections -- [ ] Set up rate limiting (recommended via reverse proxy) -- [ ] Configure firewall rules -- [ ] Set up monitoring and logging -- [ ] Review and secure file upload limits - -### Example Authenticated Requests - -**Using X-API-Key header:** -```bash -# Download endpoint -curl -H "X-API-Key: your_token" \ - -X POST http://localhost:3001/download \ - -H "Content-Type: application/json" \ - -d '{"url":"https://www.youtube.com/watch?v=VIDEO_ID"}' - -# Transcribe endpoint -curl -H "X-API-Key: your_token" \ - -X POST http://localhost:3001/transcribe \ - -H "Content-Type: application/json" \ - -d '{"filePath":"./output/audio.mp3"}' -``` - -**Using Authorization Bearer:** -```bash -curl -H "Authorization: Bearer your_token" \ - -X POST http://localhost:3001/summarize \ - -H "Content-Type: application/json" \ - -d '{"text":"Long text to summarize..."}' -``` diff --git a/docs/DEPLOIEMENT_OVH.md b/docs/DEPLOIEMENT_OVH.md deleted file mode 100644 index edb9490..0000000 --- a/docs/DEPLOIEMENT_OVH.md +++ /dev/null @@ -1,395 +0,0 @@ -# Guide de Mise Ă  Jour - Serveur OVH Existant - -Ce guide explique comment mettre Ă  jour ton serveur OVH existant avec le nouveau systĂšme de sĂ©curitĂ©. - -## PrĂ©requis - -Tu as dĂ©jĂ  : -- ✅ Un VPS chez OVH -- ✅ Git configurĂ© -- ✅ Un service qui tourne (PM2/systemd) - -## Étapes de Mise Ă  Jour - -### 1. GĂ©nĂ©rer un token API sĂ©curisĂ© - -**Sur ton serveur OVH (via SSH):** - -```bash -# GĂ©nĂ©rer un token alĂ©atoire de 64 caractĂšres -openssl rand -hex 32 -``` - -**Ou sur Windows (PowerShell):** -```powershell --join ((48..57) + (65..90) + (97..122) | Get-Random -Count 64 | % {[char]$_}) -``` - -**Copie ce token**, tu vas en avoir besoin maintenant. - ---- - -### 2. Configurer les variables d'environnement - -Connecte-toi en SSH Ă  ton serveur : - -```bash -ssh user@ton-serveur-ovh.com -``` - -Navigue vers le dossier du projet : - -```bash -cd /chemin/vers/videotoMP3Transcriptor -``` - -Édite le fichier `.env` : - -```bash -nano .env -``` - -**Ajoute ces lignes** (ou modifie si elles existent dĂ©jĂ ) : - -```env -# ======================================== -# SÉCURITÉ API -# ======================================== - -# Remplace par le token que tu viens de gĂ©nĂ©rer -API_TOKEN=ton_token_de_64_caracteres_ici - -# Domaines autorisĂ©s (sĂ©parĂ©s par des virgules) -# En dĂ©veloppement: * (tout le monde) -# En production: https://ton-domaine.com,https://api.ton-domaine.com -ALLOWED_ORIGINS=* - -# Port (optionnel, dĂ©faut: 8888) -PORT=8888 - -# OpenAI API Key (tu dois dĂ©jĂ  l'avoir) -OPENAI_API_KEY=sk-... -``` - -**Sauvegarde** : `Ctrl + X`, puis `Y`, puis `Enter` - ---- - -### 3. Pull les derniĂšres modifications - -```bash -# Sauvegarder les modifications locales si nĂ©cessaire -git stash - -# RĂ©cupĂ©rer les derniĂšres modifications -git pull origin main - -# Restaurer tes modifications si tu avais stashĂ© -git stash pop -``` - ---- - -### 4. RedĂ©marrer le service - -**Si tu utilises PM2:** - -```bash -# RedĂ©marrer l'application -pm2 restart video-transcriptor - -# VĂ©rifier que ça tourne -pm2 status - -# Voir les logs en temps rĂ©el -pm2 logs video-transcriptor -``` - -**Si tu utilises systemd:** - -```bash -# RedĂ©marrer le service -sudo systemctl restart video-transcriptor - -# VĂ©rifier le statut -sudo systemctl status video-transcriptor - -# Voir les logs -sudo journalctl -u video-transcriptor -f -``` - ---- - -### 5. Tester l'API - -**Test de santĂ© (sans token - devrait marcher):** - -```bash -curl http://localhost:8888/health -``` - -**RĂ©sultat attendu:** -```json -{"status":"ok","timestamp":"2025-..."} -``` - -**Test avec authentification (devrait Ă©chouer sans token):** - -```bash -curl http://localhost:8888/info?url=https://youtube.com/watch?v=test -``` - -**RĂ©sultat attendu:** -```json -{"error":"Unauthorized","message":"API key required..."} -``` - -**Test avec token (devrait marcher):** - -```bash -curl -H "X-API-Key: ton_token_ici" \ - "http://localhost:8888/info?url=https://youtube.com/watch?v=dQw4w9WgXcQ" -``` - -**RĂ©sultat attendu:** Informations sur la vidĂ©o - ---- - -### 6. Configurer le DNS (si pas dĂ©jĂ  fait) - -**Chez OVH, dans l'espace client:** - -1. Va dans **Web Cloud** → **Domaines** → **Ton domaine** -2. Clique sur **Zone DNS** -3. Ajoute un enregistrement **A** : - - Sous-domaine: `api` (ou `@` pour le domaine principal) - - Cible: **L'IP de ton VPS OVH** - - TTL: 3600 - -**Exemple:** -``` -Type: A -Nom: api -Cible: 51.195.XXX.XXX (ton IP OVH) -``` - -4. **Attends 5-10 minutes** pour la propagation DNS - ---- - -### 7. Tester depuis l'interface web - -1. **Ouvre ton navigateur** et va sur : `http://ton-domaine.com` (ou `http://ip-du-serveur:8888`) - -2. **Clique sur le panneau "🔐 API Configuration"** - -3. **Colle ton token** dans le champ - -4. **Clique sur "Save & Test"** - -5. **RĂ©sultat attendu :** - - Statut passe en vert "Connected ✓" - - Notification de succĂšs - - Le token est sauvegardĂ© dans le navigateur - -6. **Teste un tĂ©lĂ©chargement** dans l'onglet "Download" - - Entre une URL YouTube - - Le token sera automatiquement ajoutĂ© aux requĂȘtes - ---- - -## SĂ©curitĂ© en Production - -### Option 1 : Limiter les origines CORS - -Si tu veux que SEUL ton domaine puisse utiliser l'API : - -```bash -nano .env -``` - -Change : -```env -ALLOWED_ORIGINS=https://ton-domaine.com,https://api.ton-domaine.com -``` - -### Option 2 : HTTPS avec Nginx + Let's Encrypt - -**Si pas dĂ©jĂ  configurĂ©**, installe Nginx et SSL : - -```bash -# Installer Nginx -sudo apt update -sudo apt install -y nginx certbot python3-certbot-nginx - -# CrĂ©er la configuration Nginx -sudo nano /etc/nginx/sites-available/video-transcriptor -``` - -**Colle cette configuration :** - -```nginx -server { - listen 80; - server_name api.ton-domaine.com; - - # Redirection vers HTTPS (sera configurĂ© aprĂšs) - # return 301 https://$server_name$request_uri; - - location / { - proxy_pass http://localhost:8888; - proxy_http_version 1.1; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - } -} -``` - -**Activer et tester:** - -```bash -# Activer le site -sudo ln -s /etc/nginx/sites-available/video-transcriptor /etc/nginx/sites-enabled/ - -# Tester la config -sudo nginx -t - -# RedĂ©marrer Nginx -sudo systemctl restart nginx - -# Obtenir un certificat SSL GRATUIT -sudo certbot --nginx -d api.ton-domaine.com -``` - -Certbot va automatiquement configurer HTTPS et les redirections. - ---- - -## DĂ©pannage - -### ❌ "API token required" - -**ProblĂšme:** Le token n'est pas envoyĂ© ou invalide - -**Solution:** -1. VĂ©rifie que le token est bien configurĂ© dans l'interface web -2. RafraĂźchis la page et entre le token Ă  nouveau -3. VĂ©rifie que le token dans `.env` est le mĂȘme que dans l'interface - ---- - -### ❌ Le service ne dĂ©marre pas - -```bash -# Voir les logs -pm2 logs video-transcriptor --lines 50 - -# ou pour systemd -sudo journalctl -u video-transcriptor -n 50 -``` - -**VĂ©rifications:** -- La variable `API_TOKEN` est bien dans `.env` -- Pas d'erreurs de syntaxe dans `.env` -- Node modules Ă  jour : `npm ci` - ---- - -### ❌ CORS errors dans le navigateur - -**ProblĂšme:** "Access to fetch at ... has been blocked by CORS policy" - -**Solution 1:** En dĂ©veloppement -```env -ALLOWED_ORIGINS=* -``` - -**Solution 2:** En production -```env -ALLOWED_ORIGINS=https://ton-domaine.com,https://www.ton-domaine.com -``` - -RedĂ©marre aprĂšs modification : `pm2 restart video-transcriptor` - ---- - -### ❌ DNS ne fonctionne pas - -**VĂ©rifier la propagation DNS:** - -```bash -# Depuis ton serveur -dig api.ton-domaine.com - -# Ou depuis Windows -nslookup api.ton-domaine.com -``` - -**Si ça ne fonctionne pas:** -- Attends 10-30 minutes -- VĂ©rifie dans l'interface OVH que l'enregistrement A pointe vers la bonne IP -- Vide le cache DNS : `ipconfig /flushdns` (Windows) ou `sudo systemd-resolve --flush-caches` (Linux) - ---- - -## Checklist Finale - -Avant de considĂ©rer le dĂ©ploiement comme terminĂ© : - -- [ ] `.env` configurĂ© avec un `API_TOKEN` fort -- [ ] Service redĂ©marrĂ© et en cours d'exĂ©cution -- [ ] Test `/health` fonctionne -- [ ] Test avec token fonctionne -- [ ] Interface web accessible -- [ ] Token sauvegardĂ© dans l'interface web -- [ ] Test de tĂ©lĂ©chargement YouTube rĂ©ussi -- [ ] DNS configurĂ© (si applicable) -- [ ] HTTPS configurĂ© (recommandĂ© pour production) - ---- - -## Commandes Utiles - -```bash -# Voir les logs en temps rĂ©el -pm2 logs video-transcriptor - -# Statut du service -pm2 status - -# RedĂ©marrer -pm2 restart video-transcriptor - -# VĂ©rifier les ports ouverts -sudo netstat -tlnp | grep 8888 - -# VĂ©rifier l'utilisation des ressources -htop - -# Espace disque -df -h - -# Tester l'API locale -curl -H "X-API-Key: ton_token" http://localhost:8888/health -``` - ---- - -## Support - -Si tu rencontres des problĂšmes : - -1. **VĂ©rifie les logs** : `pm2 logs` -2. **VĂ©rifie le `.env`** : `cat .env | grep API_TOKEN` -3. **Teste en local** : `curl http://localhost:8888/health` -4. **VĂ©rifie le firewall** : `sudo ufw status` - ---- - -**Bon dĂ©ploiement ! 🚀** - -Si tout fonctionne, tu devrais pouvoir utiliser l'interface web avec le token sauvegardĂ©, et ne plus avoir Ă  le copier-coller Ă  chaque fois ! diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md deleted file mode 100644 index e0ba210..0000000 --- a/docs/DEPLOYMENT.md +++ /dev/null @@ -1,699 +0,0 @@ -# Guide de DĂ©ploiement - Video to MP3 Transcriptor - -Ce guide vous accompagne pour dĂ©ployer l'API de maniĂšre sĂ©curisĂ©e sur un serveur de production. - -## Table des matiĂšres -1. [PrĂ©requis](#prĂ©requis) -2. [Configuration de sĂ©curitĂ©](#configuration-de-sĂ©curitĂ©) -3. [DĂ©ploiement sur VPS/Serveur](#dĂ©ploiement-sur-vpsserveur) -4. [DĂ©ploiement avec Docker](#dĂ©ploiement-avec-docker) -5. [Nginx Reverse Proxy](#nginx-reverse-proxy) -6. [SSL/HTTPS avec Let's Encrypt](#sslhttps-avec-lets-encrypt) -7. [Surveillance et logs](#surveillance-et-logs) -8. [SĂ©curitĂ© avancĂ©e](#sĂ©curitĂ©-avancĂ©e) - ---- - -## PrĂ©requis - -### Serveur -- Linux (Ubuntu 20.04+ / Debian 11+ recommandĂ©) -- Minimum 2 GB RAM -- 10 GB espace disque -- Node.js 18+ ou Docker - -### DĂ©pendances systĂšme -```bash -# Ubuntu/Debian -sudo apt update -sudo apt install -y ffmpeg python3 - -# Pour tĂ©lĂ©chargement YouTube -sudo curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp -sudo chmod a+rx /usr/local/bin/yt-dlp -``` - -### Domaine et DNS -- Un nom de domaine pointant vers votre serveur -- AccĂšs aux paramĂštres DNS - ---- - -## Configuration de sĂ©curitĂ© - -### 1. GĂ©nĂ©rer un token API sĂ©curisĂ© - -**Sur votre serveur:** -```bash -# GĂ©nĂ©rer un token de 64 caractĂšres -openssl rand -hex 32 - -# Ou utiliser cette commande alternative -head /dev/urandom | tr -dc A-Za-z0-9 | head -c 64 -``` - -Copiez le token gĂ©nĂ©rĂ©, vous en aurez besoin pour le `.env`. - -### 2. Configurer les variables d'environnement - -CrĂ©ez/Ă©ditez le fichier `.env` sur le serveur: - -```bash -cd /path/to/videotoMP3Transcriptor -nano .env -``` - -Configuration minimale de production: - -```env -# ======================================== -# SÉCURITÉ - PRODUCTION -# ======================================== - -# Token API (REMPLACEZ PAR VOTRE TOKEN GÉNÉRÉ) -API_TOKEN=votre_token_securise_de_64_caracteres - -# Origines CORS autorisĂ©es (vos domaines uniquement) -ALLOWED_ORIGINS=https://yourdomain.com,https://api.yourdomain.com - -# ======================================== -# CONFIGURATION SERVEUR -# ======================================== - -# Port interne (Nginx fera le reverse proxy) -PORT=8888 - -# RĂ©pertoire de sortie -OUTPUT_DIR=/var/www/videotoMP3Transcriptor/output - -# ======================================== -# API KEYS -# ======================================== - -# OpenAI API Key (OBLIGATOIRE) -OPENAI_API_KEY=sk-... - -# ======================================== -# ENVIRONNEMENT -# ======================================== -NODE_ENV=production -``` - -### 3. Permissions du fichier .env - -```bash -# SĂ©curiser le fichier .env -chmod 600 .env -chown www-data:www-data .env # ou votre utilisateur systĂšme -``` - ---- - -## DĂ©ploiement sur VPS/Serveur - -### 1. Installation de Node.js - -```bash -# Installation de Node.js 20 LTS -curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash - -sudo apt install -y nodejs - -# VĂ©rification -node --version # devrait afficher v20.x -npm --version -``` - -### 2. Cloner et installer l'application - -```bash -# CrĂ©er le rĂ©pertoire -sudo mkdir -p /var/www/videotoMP3Transcriptor -sudo chown $USER:$USER /var/www/videotoMP3Transcriptor - -# Cloner (ou copier) votre code -cd /var/www/videotoMP3Transcriptor -# git clone ... ou upload manuel - -# Installer les dĂ©pendances -npm ci --only=production - -# CrĂ©er le rĂ©pertoire de sortie -mkdir -p output -chmod 755 output -``` - -### 3. Utiliser PM2 pour la gestion des processus - -PM2 est un gestionnaire de processus pour Node.js qui redĂ©marre automatiquement votre app en cas de crash. - -```bash -# Installer PM2 globalement -sudo npm install -g pm2 - -# DĂ©marrer l'application -pm2 start src/server.js --name "video-transcriptor" - -# Configurer PM2 pour dĂ©marrer au boot -pm2 startup systemd -pm2 save - -# Commandes utiles -pm2 status # Voir le statut -pm2 logs video-transcriptor # Voir les logs -pm2 restart video-transcriptor # RedĂ©marrer -pm2 stop video-transcriptor # ArrĂȘter -``` - -### 4. Configuration PM2 avancĂ©e (optionnelle) - -CrĂ©ez un fichier `ecosystem.config.js`: - -```javascript -module.exports = { - apps: [{ - name: 'video-transcriptor', - script: './src/server.js', - instances: 1, - autorestart: true, - watch: false, - max_memory_restart: '1G', - env: { - NODE_ENV: 'production', - PORT: 8888 - }, - error_file: '/var/log/pm2/video-transcriptor-error.log', - out_file: '/var/log/pm2/video-transcriptor-out.log', - log_date_format: 'YYYY-MM-DD HH:mm:ss Z' - }] -}; -``` - -DĂ©marrer avec: -```bash -pm2 start ecosystem.config.js -``` - ---- - -## DĂ©ploiement avec Docker - -### 1. CrĂ©er un Dockerfile - -CrĂ©ez `Dockerfile` Ă  la racine du projet: - -```dockerfile -FROM node:20-slim - -# Installer les dĂ©pendances systĂšme -RUN apt-get update && apt-get install -y \ - ffmpeg \ - python3 \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Installer yt-dlp -RUN curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp \ - && chmod a+rx /usr/local/bin/yt-dlp - -# CrĂ©er le rĂ©pertoire de l'app -WORKDIR /app - -# Copier package.json et installer les dĂ©pendances -COPY package*.json ./ -RUN npm ci --only=production - -# Copier le code source -COPY . . - -# CrĂ©er le rĂ©pertoire de sortie -RUN mkdir -p /app/output && chmod 755 /app/output - -# Exposer le port -EXPOSE 8888 - -# Variables d'environnement par dĂ©faut -ENV NODE_ENV=production -ENV PORT=8888 -ENV OUTPUT_DIR=/app/output - -# DĂ©marrer l'application -CMD ["node", "src/server.js"] -``` - -### 2. CrĂ©er docker-compose.yml - -```yaml -version: '3.8' - -services: - video-transcriptor: - build: . - container_name: video-transcriptor - restart: unless-stopped - ports: - - "8888:8888" - volumes: - - ./output:/app/output - - ./.env:/app/.env:ro - environment: - - NODE_ENV=production - networks: - - transcriptor-network - -networks: - transcriptor-network: - driver: bridge -``` - -### 3. Lancer avec Docker Compose - -```bash -# Build et dĂ©marrer -docker-compose up -d - -# Voir les logs -docker-compose logs -f - -# ArrĂȘter -docker-compose down - -# Reconstruire aprĂšs modification -docker-compose up -d --build -``` - ---- - -## Nginx Reverse Proxy - -### 1. Installer Nginx - -```bash -sudo apt update -sudo apt install -y nginx -``` - -### 2. Configuration Nginx - -CrĂ©ez `/etc/nginx/sites-available/video-transcriptor`: - -```nginx -# Rate limiting -limit_req_zone $binary_remote_addr zone=api_limit:10m rate=10r/s; - -server { - listen 80; - server_name api.yourdomain.com; - - # Logs - access_log /var/log/nginx/video-transcriptor-access.log; - error_log /var/log/nginx/video-transcriptor-error.log; - - # Rate limiting - limit_req zone=api_limit burst=20 nodelay; - - # Augmenter les timeouts pour les longs traitements - proxy_connect_timeout 600; - proxy_send_timeout 600; - proxy_read_timeout 600; - send_timeout 600; - - # Augmenter la taille max des uploads - client_max_body_size 500M; - - location / { - proxy_pass http://localhost:8888; - proxy_http_version 1.1; - - # Headers - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection 'upgrade'; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - # Pour Server-Sent Events (SSE) - proxy_cache_bypass $http_upgrade; - proxy_buffering off; - proxy_cache off; - } - - # Headers de sĂ©curitĂ© supplĂ©mentaires - add_header X-Content-Type-Options "nosniff" always; - add_header X-Frame-Options "DENY" always; - add_header X-XSS-Protection "1; mode=block" always; - add_header Referrer-Policy "strict-origin-when-cross-origin" always; -} -``` - -### 3. Activer le site - -```bash -# CrĂ©er un lien symbolique -sudo ln -s /etc/nginx/sites-available/video-transcriptor /etc/nginx/sites-enabled/ - -# Tester la configuration -sudo nginx -t - -# Recharger Nginx -sudo systemctl reload nginx -``` - ---- - -## SSL/HTTPS avec Let's Encrypt - -### 1. Installer Certbot - -```bash -sudo apt install -y certbot python3-certbot-nginx -``` - -### 2. Obtenir un certificat SSL - -```bash -# Obtenir et installer automatiquement le certificat -sudo certbot --nginx -d api.yourdomain.com - -# Suivez les instructions Ă  l'Ă©cran -``` - -### 3. Renouvellement automatique - -```bash -# Tester le renouvellement -sudo certbot renew --dry-run - -# Le renouvellement automatique est configurĂ© via cron -# VĂ©rifier: sudo systemctl status certbot.timer -``` - -AprĂšs SSL, votre configuration Nginx sera automatiquement mise Ă  jour pour HTTPS. - ---- - -## Surveillance et logs - -### 1. Logs de l'application - -```bash -# Avec PM2 -pm2 logs video-transcriptor - -# Avec Docker -docker-compose logs -f video-transcriptor - -# Logs Nginx -sudo tail -f /var/log/nginx/video-transcriptor-access.log -sudo tail -f /var/log/nginx/video-transcriptor-error.log -``` - -### 2. Monitoring avec PM2 (optionnel) - -```bash -# Installer PM2 monitoring -pm2 install pm2-logrotate - -# Configurer la rotation des logs -pm2 set pm2-logrotate:max_size 10M -pm2 set pm2-logrotate:retain 7 -``` - -### 3. Monitoring systĂšme - -```bash -# Installer htop pour surveiller les ressources -sudo apt install -y htop - -# Lancer htop -htop - -# Voir l'utilisation disque -df -h - -# Voir l'utilisation mĂ©moire -free -h -``` - ---- - -## SĂ©curitĂ© avancĂ©e - -### 1. Firewall (UFW) - -```bash -# Installer UFW -sudo apt install -y ufw - -# Autoriser SSH (IMPORTANT AVANT D'ACTIVER!) -sudo ufw allow ssh -sudo ufw allow 22/tcp - -# Autoriser HTTP et HTTPS -sudo ufw allow 'Nginx Full' - -# Activer le firewall -sudo ufw enable - -# VĂ©rifier le statut -sudo ufw status -``` - -### 2. Fail2Ban (protection contre brute force) - -```bash -# Installer Fail2Ban -sudo apt install -y fail2ban - -# CrĂ©er une configuration pour Nginx -sudo nano /etc/fail2ban/jail.local -``` - -Ajouter: -```ini -[nginx-limit-req] -enabled = true -filter = nginx-limit-req -port = http,https -logpath = /var/log/nginx/video-transcriptor-error.log -maxretry = 5 -findtime = 600 -bantime = 3600 -``` - -```bash -# RedĂ©marrer Fail2Ban -sudo systemctl restart fail2ban - -# VĂ©rifier le statut -sudo fail2ban-client status nginx-limit-req -``` - -### 3. Limitations supplĂ©mentaires - -**Limiter les tailles de fichiers uploadĂ©s** - DĂ©jĂ  configurĂ© dans Nginx (`client_max_body_size 500M`) - -**Rate limiting par IP** - DĂ©jĂ  configurĂ© dans Nginx (`limit_req_zone`) - -### 4. Sauvegardes automatiques - -```bash -# CrĂ©er un script de backup -sudo nano /usr/local/bin/backup-video-transcriptor.sh -``` - -```bash -#!/bin/bash -BACKUP_DIR="/backup/video-transcriptor" -APP_DIR="/var/www/videotoMP3Transcriptor" -DATE=$(date +%Y%m%d_%H%M%S) - -mkdir -p $BACKUP_DIR - -# Backup de la configuration -tar -czf $BACKUP_DIR/config_$DATE.tar.gz \ - $APP_DIR/.env \ - $APP_DIR/ecosystem.config.js - -# Backup des fichiers de sortie (optionnel, peut ĂȘtre volumineux) -# tar -czf $BACKUP_DIR/output_$DATE.tar.gz $APP_DIR/output - -# Garder seulement les 7 derniers backups -find $BACKUP_DIR -name "config_*.tar.gz" -mtime +7 -delete - -echo "Backup completed: $DATE" -``` - -```bash -# Rendre exĂ©cutable -sudo chmod +x /usr/local/bin/backup-video-transcriptor.sh - -# Ajouter au crontab (backup quotidien Ă  2h du matin) -sudo crontab -e -# Ajouter: 0 2 * * * /usr/local/bin/backup-video-transcriptor.sh -``` - ---- - -## Checklist finale de dĂ©ploiement - -Avant de mettre en production, vĂ©rifiez: - -- [ ] **SĂ©curitĂ©** - - [ ] Token API fort gĂ©nĂ©rĂ© (`API_TOKEN`) - - [ ] CORS configurĂ© avec vos domaines (`ALLOWED_ORIGINS`) - - [ ] Fichier `.env` avec permissions 600 - - [ ] HTTPS configurĂ© et fonctionnel - - [ ] Firewall UFW activĂ© - -- [ ] **Configuration** - - [ ] `OPENAI_API_KEY` valide et fonctionnelle - - [ ] `NODE_ENV=production` - - [ ] RĂ©pertoire `output/` crĂ©Ă© et accessible - - [ ] FFmpeg et yt-dlp installĂ©s - -- [ ] **Infrastructure** - - [ ] PM2 ou Docker en cours d'exĂ©cution - - [ ] Nginx reverse proxy configurĂ© - - [ ] SSL/TLS actif (Let's Encrypt) - - [ ] Rate limiting activĂ© - -- [ ] **Monitoring** - - [ ] Logs accessibles - - [ ] PM2 startup configurĂ© (redĂ©marrage auto) - - [ ] Fail2Ban actif - - [ ] Backups automatiques configurĂ©s - -- [ ] **Tests** - - [ ] Endpoint `/health` accessible - - [ ] Test d'authentification (avec et sans token) - - [ ] Test d'upload de fichier - - [ ] Test de tĂ©lĂ©chargement YouTube - ---- - -## Tests post-dĂ©ploiement - -### 1. Test de santĂ© - -```bash -curl https://api.yourdomain.com/health -# Devrait retourner: {"status":"ok","timestamp":"..."} -``` - -### 2. Test d'authentification - -```bash -# Sans token (devrait Ă©chouer avec 401) -curl https://api.yourdomain.com/info?url=https://www.youtube.com/watch?v=dQw4w9WgXcQ - -# Avec token (devrait rĂ©ussir) -curl -H "X-API-Key: VOTRE_TOKEN" \ - "https://api.yourdomain.com/info?url=https://www.youtube.com/watch?v=dQw4w9WgXcQ" -``` - -### 3. Test de download - -```bash -curl -H "X-API-Key: VOTRE_TOKEN" \ - -X POST https://api.yourdomain.com/download \ - -H "Content-Type: application/json" \ - -d '{"url":"https://www.youtube.com/watch?v=dQw4w9WgXcQ"}' -``` - ---- - -## DĂ©pannage - -### L'API ne dĂ©marre pas - -```bash -# VĂ©rifier les logs PM2 -pm2 logs video-transcriptor - -# VĂ©rifier les variables d'environnement -pm2 env video-transcriptor - -# RedĂ©marrer -pm2 restart video-transcriptor -``` - -### Erreurs 502 Bad Gateway (Nginx) - -```bash -# VĂ©rifier que l'app tourne -pm2 status - -# VĂ©rifier les logs Nginx -sudo tail -f /var/log/nginx/error.log - -# VĂ©rifier que le port 8888 est ouvert -sudo netstat -tlnp | grep 8888 -``` - -### ProblĂšmes SSL - -```bash -# VĂ©rifier le certificat -sudo certbot certificates - -# Renouveler manuellement -sudo certbot renew --force-renewal - -# Tester la configuration Nginx -sudo nginx -t -``` - -### MĂ©moire insuffisante - -```bash -# VĂ©rifier l'utilisation mĂ©moire -free -h - -# CrĂ©er un swap file (si nĂ©cessaire) -sudo fallocate -l 2G /swapfile -sudo chmod 600 /swapfile -sudo mkswap /swapfile -sudo swapon /swapfile -echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab -``` - ---- - -## Mises Ă  jour - -### Mise Ă  jour de l'application - -```bash -cd /var/www/videotoMP3Transcriptor - -# Sauvegarder la config -cp .env .env.backup - -# Pull des nouvelles versions (git) -git pull - -# Mettre Ă  jour les dĂ©pendances -npm ci --only=production - -# RedĂ©marrer -pm2 restart video-transcriptor - -# Ou avec Docker -docker-compose down -docker-compose up -d --build -``` - ---- - -## Support et ressources - -- **Documentation API**: [docs/API.md](./API.md) -- **CLAUDE.md**: [CLAUDE.md](../CLAUDE.md) - Instructions pour Claude -- **PM2 Documentation**: https://pm2.keymetrics.io/ -- **Nginx Documentation**: https://nginx.org/en/docs/ -- **Let's Encrypt**: https://letsencrypt.org/ - ---- - -**Bon dĂ©ploiement ! 🚀** diff --git a/docs/YOUTUBE_COOKIES.md b/docs/YOUTUBE_COOKIES.md deleted file mode 100644 index 445fb1e..0000000 --- a/docs/YOUTUBE_COOKIES.md +++ /dev/null @@ -1,132 +0,0 @@ -# YouTube Cookies Setup Guide - -## Why Do I Need Cookies? - -YouTube has anti-bot protections that may block yt-dlp requests. Using cookies from your browser allows yt-dlp to authenticate as if you're logged in, bypassing these restrictions. - -## Quick Start - -### Option 1: Automatic Extraction (Recommended) - -Run the helper script: - -```bash -bash scripts/extract-cookies.sh -``` - -Follow the prompts to extract cookies from Chrome or Firefox. - -### Option 2: Using yt-dlp Directly - -```bash -# For Chrome/Chromium -yt-dlp --cookies-from-browser chrome --cookies youtube-cookies.txt 'https://www.youtube.com' - -# For Firefox -yt-dlp --cookies-from-browser firefox --cookies youtube-cookies.txt 'https://www.youtube.com' - -# For Edge -yt-dlp --cookies-from-browser edge --cookies youtube-cookies.txt 'https://www.youtube.com' -``` - -### Option 3: Browser Extension - -1. Install a cookies export extension: - - **Chrome/Edge**: [Get cookies.txt LOCALLY](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) - - **Firefox**: [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) - -2. Go to [youtube.com](https://www.youtube.com) and log in - -3. Click the extension icon and export cookies - -4. Save the file as `youtube-cookies.txt` in your project directory - -## Configuration - -After extracting cookies, update your `.env` file: - -```bash -YOUTUBE_COOKIES_PATH=/home/debian/videotomp3transcriptor/youtube-cookies.txt -``` - -Or use a relative path: - -```bash -YOUTUBE_COOKIES_PATH=./youtube-cookies.txt -``` - -## Verifying It Works - -Test with a video: - -```bash -curl -X POST http://localhost:3001/download \ - -H "Content-Type: application/json" \ - -d '{"url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ"}' -``` - -If it works without cookies errors, you're good to go! - -## Security Notes - -⚠ **IMPORTANT**: - -1. **Never commit cookies to git**: The `.gitignore` file should already exclude `youtube-cookies.txt` -2. **Keep cookies secure**: They provide access to your YouTube account -3. **Cookies expire**: You may need to re-export them periodically (typically every few weeks/months) -4. **Don't share cookies**: Treat them like passwords - -## Troubleshooting - -### "Sign in to confirm you're not a bot" - -This usually means: -- Cookies are not being used -- Cookies have expired -- Cookies file path is incorrect - -**Solutions**: -1. Check the path in `.env` is correct and absolute -2. Re-export fresh cookies -3. Verify the cookies file exists: `ls -la youtube-cookies.txt` -4. Check logs: `pm2 logs toMP3-api` - -### "HTTP Error 403: Forbidden" - -YouTube is blocking your IP or the video is region-restricted. - -**Solutions**: -1. Try with fresh cookies -2. Use a VPN if region-restricted -3. Wait a bit if rate-limited - -### Cookies Not Working - -1. Make sure you're logged into YouTube in the browser before extracting -2. Try extracting from a different browser -3. Verify the cookies file format (should be Netscape format) -4. Check file permissions: `chmod 600 youtube-cookies.txt` - -## Cookie File Format - -The cookies file should be in Netscape format and look like this: - -``` -# Netscape HTTP Cookie File -.youtube.com TRUE / TRUE 1234567890 CONSENT YES+ -.youtube.com TRUE / FALSE 1234567890 VISITOR_INFO1_LIVE xxxxx -``` - -## Without Cookies - -The API will still work for many videos without cookies, but you may encounter: -- "Sign in to confirm you're not a bot" errors -- Rate limiting -- Blocked downloads for certain videos - -For best results, always use cookies! - -## Additional Resources - -- [yt-dlp Cookie Documentation](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp) -- [Browser Cookie Extraction](https://github.com/yt-dlp/yt-dlp#:~:text=You%20can%20use%20cookies%20from%20your%20browser) diff --git a/package.json b/package.json index e6d7317..05218e9 100644 --- a/package.json +++ b/package.json @@ -1,34 +1,27 @@ { - "name": "video-to-mp3-transcriptor", - "version": "1.0.0", - "description": "Download YouTube videos/playlists to MP3 and transcribe them using OpenAI Whisper API", - "main": "src/index.js", - "type": "module", - "bin": { - "ytmp3": "./src/cli.js" - }, + "name": "hanasuba-music-service", + "version": "2.0.0", + "description": "YouTube to MP3 download service with Camoufox stealth cookies for Hanasuba", + "main": "src/server.js", "scripts": { - "start": "node src/index.js", - "cli": "node src/cli.js", - "server": "node src/server.js" + "start": "node src/server.js", + "dev": "node --watch src/server.js", + "setup": "python3 -m pip install -r requirements.txt && playwright install firefox", + "cookies:extract": "python3 src/python/extract_cookies.py", + "cookies:validate": "python3 src/python/validate_cookies.py" }, "keywords": [ "youtube", "mp3", - "transcription", - "whisper", - "openai" + "music", + "camoufox", + "stealth", + "hanasuba" ], - "author": "", + "author": "StillHammer", "license": "MIT", "dependencies": { - "@anthropic-ai/sdk": "^0.70.1", - "commander": "^12.1.0", - "cors": "^2.8.5", - "dotenv": "^16.4.5", "express": "^4.21.0", - "multer": "^2.0.2", - "openai": "^4.67.0", - "youtube-dl-exec": "^3.0.7" + "dotenv": "^16.4.5" } } diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..31e64e4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +camoufox>=0.4.11 +camoufox-captcha>=0.1.3 +playwright>=1.57.0 diff --git a/src/cli.js b/src/cli.js deleted file mode 100644 index aa1368a..0000000 --- a/src/cli.js +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env node - -import { Command } from 'commander'; -import dotenv from 'dotenv'; -import path from 'path'; -import { download, downloadVideo, downloadPlaylist, getInfo } from './services/youtube.js'; -import { transcribeFile, transcribeAndSave, transcribeMultiple, getAvailableModels } from './services/transcription.js'; - -// Load environment variables -dotenv.config(); - -const program = new Command(); - -program - .name('ytmp3') - .description('Download YouTube videos/playlists to MP3 and transcribe them') - .version('1.0.0'); - -// Download command -program - .command('download ') - .alias('dl') - .description('Download a YouTube video or playlist as MP3') - .option('-o, --output ', 'Output directory', './output') - .action(async (url, options) => { - try { - console.log('Fetching video info...'); - const result = await download(url, { outputDir: options.output }); - - console.log('\n--- Download Complete ---'); - if (result.playlistTitle) { - console.log(`Playlist: ${result.playlistTitle}`); - } - console.log(`Downloaded: ${result.successCount}/${result.totalVideos} videos`); - - result.videos.forEach(v => { - if (v.success) { - console.log(` ✓ ${v.title}`); - } else { - console.log(` ✗ ${v.title} - ${v.error}`); - } - }); - } catch (error) { - console.error(`Error: ${error.message}`); - process.exit(1); - } - }); - -// Transcribe command (from existing MP3) -program - .command('transcribe ') - .alias('tr') - .description('Transcribe an existing audio file') - .option('-l, --language ', 'Language code (e.g., en, fr, zh)') - .option('-f, --format ', 'Output format (txt, srt, vtt)', 'txt') - .option('-m, --model ', 'Transcription model (gpt-4o-transcribe, gpt-4o-mini-transcribe, whisper-1)', 'gpt-4o-transcribe') - .action(async (file, options) => { - try { - if (!process.env.OPENAI_API_KEY) { - console.error('Error: OPENAI_API_KEY not set in environment'); - process.exit(1); - } - - console.log(`Transcribing: ${file}`); - const result = await transcribeAndSave(file, { - language: options.language, - responseFormat: options.format === 'txt' ? 'text' : options.format, - outputFormat: options.format, - model: options.model, - }); - - console.log('\n--- Transcription Complete ---'); - console.log(`Model: ${result.model}`); - console.log(`Output: ${result.transcriptionPath}`); - console.log('\nPreview:'); - console.log(result.text.substring(0, 500) + (result.text.length > 500 ? '...' : '')); - } catch (error) { - console.error(`Error: ${error.message}`); - process.exit(1); - } - }); - -// Download + Transcribe command -program - .command('process ') - .alias('p') - .description('Download and transcribe a YouTube video or playlist') - .option('-o, --output ', 'Output directory', './output') - .option('-l, --language ', 'Language code for transcription') - .option('-f, --format ', 'Transcription format (txt, srt, vtt)', 'txt') - .option('-m, --model ', 'Transcription model (gpt-4o-transcribe, gpt-4o-mini-transcribe, whisper-1)', 'gpt-4o-transcribe') - .action(async (url, options) => { - try { - if (!process.env.OPENAI_API_KEY) { - console.error('Error: OPENAI_API_KEY not set in environment'); - process.exit(1); - } - - // Step 1: Download - console.log('Step 1: Downloading...'); - const downloadResult = await download(url, { outputDir: options.output }); - - console.log(`Downloaded: ${downloadResult.successCount}/${downloadResult.totalVideos} videos\n`); - - // Step 2: Transcribe - console.log(`Step 2: Transcribing with ${options.model}...`); - const successfulDownloads = downloadResult.videos.filter(v => v.success); - const filePaths = successfulDownloads.map(v => v.filePath); - - const transcribeResult = await transcribeMultiple(filePaths, { - language: options.language, - responseFormat: options.format === 'txt' ? 'text' : options.format, - outputFormat: options.format, - model: options.model, - }); - - console.log('\n--- Process Complete ---'); - if (downloadResult.playlistTitle) { - console.log(`Playlist: ${downloadResult.playlistTitle}`); - } - console.log(`Downloaded: ${downloadResult.successCount}/${downloadResult.totalVideos}`); - console.log(`Transcribed: ${transcribeResult.successCount}/${transcribeResult.totalFiles}`); - - transcribeResult.results.forEach(r => { - if (r.success) { - console.log(` ✓ ${path.basename(r.transcriptionPath)}`); - } else { - console.log(` ✗ ${path.basename(r.filePath)} - ${r.error}`); - } - }); - } catch (error) { - console.error(`Error: ${error.message}`); - process.exit(1); - } - }); - -// Info command -program - .command('info ') - .description('Get info about a YouTube video or playlist') - .action(async (url) => { - try { - const info = await getInfo(url); - - console.log('\n--- Video/Playlist Info ---'); - console.log(`Title: ${info.title}`); - console.log(`Type: ${info._type || 'video'}`); - - if (info._type === 'playlist') { - console.log(`Videos: ${info.entries?.length || 0}`); - if (info.entries) { - info.entries.slice(0, 10).forEach((e, i) => { - console.log(` ${i + 1}. ${e.title}`); - }); - if (info.entries.length > 10) { - console.log(` ... and ${info.entries.length - 10} more`); - } - } - } else { - console.log(`Duration: ${Math.floor(info.duration / 60)}:${String(info.duration % 60).padStart(2, '0')}`); - console.log(`Channel: ${info.channel}`); - } - } catch (error) { - console.error(`Error: ${error.message}`); - process.exit(1); - } - }); - -program.parse(); diff --git a/src/python/extract_cookies.py b/src/python/extract_cookies.py new file mode 100755 index 0000000..e87780d --- /dev/null +++ b/src/python/extract_cookies.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +""" +Extract YouTube cookies using Camoufox (stealth Firefox). +Cookies are undetectable by bot detection systems. +""" + +import asyncio +import sys +from pathlib import Path +from camoufox.async_api import AsyncCamoufox + +async def extract_cookies(output_path='youtube-cookies.txt'): + """ + Extract YouTube cookies using Camoufox (stealth Firefox). + These cookies bypass bot detection and last longer. + """ + + print("🩊 Starting Camoufox (stealth mode)...") + + try: + async with AsyncCamoufox( + headless=True, # Background (no GUI needed) + humanize=True, # Mimic human behavior + geoip=True, # Realistic IP geolocation + ) as browser: + page = await browser.new_page() + + # Navigate to YouTube + print("đŸ“ș Loading YouTube...") + await page.goto('https://www.youtube.com', wait_until='domcontentloaded', timeout=30000) + + # Wait for page fully loaded + await asyncio.sleep(3) + + # Extract cookies + cookies = await page.context.cookies() + + # Filter YouTube cookies + yt_cookies = [c for c in cookies if 'youtube.com' in c['domain']] + + if not yt_cookies: + print("❌ No YouTube cookies found!") + return False + + # Save to Netscape format (yt-dlp compatible) + output = Path(output_path) + with open(output, 'w') as f: + f.write("# Netscape HTTP Cookie File\n") + f.write("# Generated by Camoufox (stealth mode)\n") + f.write(f"# This file is compatible with yt-dlp\n") + for c in yt_cookies: + line = f"{c['domain']}\tTRUE\t{c['path']}\t" + line += f"{'TRUE' if c.get('secure') else 'FALSE'}\t" + line += f"{int(c.get('expires', 0))}\t{c['name']}\t{c['value']}\n" + f.write(line) + + # Set secure permissions + output.chmod(0o600) + + print(f"✅ Cookies saved: {output_path}") + print(f" Total cookies: {len(yt_cookies)}") + print(f" Permissions: 600 (secure)") + + return True + + except Exception as e: + print(f"❌ Error: {e}") + return False + +if __name__ == '__main__': + output = sys.argv[1] if len(sys.argv) > 1 else 'youtube-cookies.txt' + success = asyncio.run(extract_cookies(output)) + sys.exit(0 if success else 1) diff --git a/src/python/validate_cookies.py b/src/python/validate_cookies.py new file mode 100755 index 0000000..18f99b0 --- /dev/null +++ b/src/python/validate_cookies.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +""" +Validate YouTube cookies using Camoufox. +Tests if cookies are still valid and working. +""" + +import asyncio +import sys +from pathlib import Path +from camoufox.async_api import AsyncCamoufox + +async def validate_cookies(cookies_path='youtube-cookies.txt'): + """ + Test if YouTube cookies are still valid. + Returns True if valid, False otherwise. + """ + + cookies_file = Path(cookies_path) + + if not cookies_file.exists(): + print(f"❌ Cookies file not found: {cookies_path}") + return False + + # Check file age + age_hours = (Path().stat().st_mtime - cookies_file.stat().st_mtime) / 3600 + print(f"📅 Cookies age: {age_hours:.1f} hours") + + try: + async with AsyncCamoufox(headless=True, humanize=True) as browser: + context = await browser.new_context() + + # Load cookies from file + # Camoufox doesn't have add_cookies_from_file, so we parse manually + cookies_to_add = [] + with open(cookies_path, 'r') as f: + for line in f: + if line.startswith('#') or not line.strip(): + continue + parts = line.strip().split('\t') + if len(parts) >= 7: + cookies_to_add.append({ + 'domain': parts[0], + 'path': parts[2], + 'secure': parts[3] == 'TRUE', + 'expires': int(parts[4]) if parts[4] != '0' else None, + 'name': parts[5], + 'value': parts[6] + }) + + if not cookies_to_add: + print("❌ No valid cookies found in file") + return False + + await context.add_cookies(cookies_to_add) + + page = await context.new_page() + await page.goto('https://www.youtube.com', wait_until='domcontentloaded', timeout=30000) + + # Wait a bit for page to render + await asyncio.sleep(2) + + # Check if we can access YouTube properly + # If blocked, there will be bot detection or sign-in prompts + content = await page.content() + + # Simple validation: check if we have access to normal YouTube + is_valid = 'ytInitialData' in content or 'watch?' in content + + if is_valid: + print("✅ Cookies are valid") + print(" YouTube access: OK") + return True + else: + print("⚠ Cookies may be expired or invalid") + print(" YouTube access: BLOCKED") + return False + + except Exception as e: + print(f"❌ Validation error: {e}") + return False + +if __name__ == '__main__': + cookies_path = sys.argv[1] if len(sys.argv) > 1 else 'youtube-cookies.txt' + valid = asyncio.run(validate_cookies(cookies_path)) + sys.exit(0 if valid else 1) diff --git a/src/server.js b/src/server.js index 5d3f08f..f5b78c0 100644 --- a/src/server.js +++ b/src/server.js @@ -1,1566 +1,224 @@ -import express from 'express'; -import cors from 'cors'; -import dotenv from 'dotenv'; -import path from 'path'; -import fs from 'fs'; -import { fileURLToPath } from 'url'; -import multer from 'multer'; -import { download, getInfo } from './services/youtube.js'; -import { transcribeFile, transcribeAndSave, transcribeMultiple } from './services/transcription.js'; -import { translateText, translateFile, translateMultiple, getLanguages } from './services/translation.js'; -import { summarizeText, summarizeFile, getSummaryStyles } from './services/summarize.js'; -import { convertToMP3, convertMultipleToMP3, getSupportedFormats } from './services/conversion.js'; - -dotenv.config(); - -const app = express(); -const PORT = process.env.PORT || 8888; -const OUTPUT_DIR = process.env.OUTPUT_DIR || './output'; - -// Ensure output directory exists -if (!fs.existsSync(OUTPUT_DIR)) { - fs.mkdirSync(OUTPUT_DIR, { recursive: true }); -} - -// Configure multer for file uploads -const storage = multer.diskStorage({ - destination: (req, file, cb) => { - cb(null, OUTPUT_DIR); - }, - filename: (req, file, cb) => { - // Keep original filename but sanitize it - const safeName = file.originalname.replace(/[^a-zA-Z0-9._-]/g, '_'); - cb(null, safeName); - } -}); - -const upload = multer({ - storage, - fileFilter: (req, file, cb) => { - const allowedTypes = ['audio/mpeg', 'audio/mp3', 'audio/wav', 'audio/m4a', 'audio/ogg', 'audio/flac', 'audio/x-m4a']; - if (allowedTypes.includes(file.mimetype) || file.originalname.match(/\.(mp3|wav|m4a|ogg|flac)$/i)) { - cb(null, true); - } else { - cb(new Error('Invalid file type. Only audio files are allowed.')); - } - } -}); - -// Upload handler for text files (for translation) -const uploadText = multer({ - storage, - fileFilter: (req, file, cb) => { - if (file.mimetype === 'text/plain' || file.originalname.endsWith('.txt')) { - cb(null, true); - } else { - cb(new Error('Invalid file type. Only text files (.txt) are allowed.')); - } - } -}); - -// Upload handler for video/audio files (for conversion) -const uploadVideo = multer({ - storage, - fileFilter: (req, file, cb) => { - const videoTypes = ['video/mp4', 'video/avi', 'video/x-msvideo', 'video/quicktime', 'video/x-matroska', 'video/webm']; - const audioTypes = ['audio/m4a', 'audio/x-m4a', 'audio/wav', 'audio/flac', 'audio/ogg', 'audio/aac']; - const videoExtensions = /\.(mp4|avi|mkv|mov|wmv|flv|webm|m4v|m4a|wav|flac|ogg|aac)$/i; - - if (videoTypes.includes(file.mimetype) || audioTypes.includes(file.mimetype) || file.originalname.match(videoExtensions)) { - cb(null, true); - } else { - cb(new Error('Invalid file type. Only video/audio files are allowed.')); - } - } -}); - -// CORS configuration - restrictive for production -const corsOptions = { - origin: process.env.ALLOWED_ORIGINS ? process.env.ALLOWED_ORIGINS.split(',') : '*', - methods: ['GET', 'POST', 'OPTIONS'], - allowedHeaders: ['Content-Type', 'Authorization', 'X-API-Key'], - credentials: true, -}; - -app.use(cors(corsOptions)); -app.use(express.json()); - -// Security headers -app.use((req, res, next) => { - // Public endpoints that should work over HTTP - const publicEndpoints = ['/health', '/api', '/docs/api']; - const isPublic = publicEndpoints.includes(req.path) || req.path.startsWith('/public/download/') || req.path.startsWith('/public/scripts/'); - - res.setHeader('X-Content-Type-Options', 'nosniff'); - res.setHeader('X-Frame-Options', 'DENY'); - res.setHeader('X-XSS-Protection', '1; mode=block'); - - // Only enforce HTTPS for protected endpoints - if (!isPublic) { - res.setHeader('Strict-Transport-Security', 'max-age=31536000; includeSubDomains'); - } - - res.setHeader( - 'Content-Security-Policy', - "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'" - ); - next(); -}); - -// API Authentication Middleware -const authenticate = (req, res, next) => { - // Skip authentication for public endpoints - const publicEndpoints = ['/health', '/api', '/docs/api']; - // Allow public download and scripts endpoints - if (publicEndpoints.includes(req.path) || req.path.startsWith('/public/download/') || req.path.startsWith('/public/scripts/')) { - return next(); - } - - const apiKey = req.headers['x-api-key'] || req.headers['authorization']?.replace('Bearer ', ''); - const configuredKey = process.env.API_TOKEN; - - if (!configuredKey) { - console.warn('⚠ WARNING: API_TOKEN not configured in .env - API is UNSECURED!'); - return next(); // Allow in development if not configured - } - - if (!apiKey) { - return res.status(401).json({ - error: 'Unauthorized', - message: 'API key required. Provide X-API-Key header or Authorization: Bearer ' - }); - } - - if (apiKey !== configuredKey) { - return res.status(403).json({ - error: 'Forbidden', - message: 'Invalid API key' - }); - } - - next(); -}; - -// Helper function to handle YouTube enhanced errors -function handleYouTubeError(error, res, defaultMessage = 'Operation failed') { - if (error.isEnhanced && error.details) { - return res.status(503).json(error.details); - } - return res.status(500).json({ error: error.message || defaultMessage }); -} - -// Serve static files (HTML interface) - BEFORE authentication to allow public access -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -app.use(express.static(path.join(__dirname, '../public'))); - -// Apply authentication to all API routes (static files above are exempt) -app.use(authenticate); - -// Serve downloaded files -app.use('/files', express.static(OUTPUT_DIR)); - -// API info endpoint -app.get('/api', (req, res) => { - res.json({ - name: 'Video to MP3 Transcriptor API', - version: '1.0.0', - endpoints: { - 'GET /health': 'Health check', - 'GET /info?url=': 'Get video/playlist info', - 'POST /download': 'Download as MP3', - 'POST /transcribe': 'Transcribe audio file', - 'POST /process': 'Download + transcribe', - 'POST /upload-process': 'Smart: Upload video/audio OR URL -> Transcribe', - 'GET /files-list': 'List downloaded files', - 'GET /files/': 'Serve downloaded files', - }, - }); -}); - -// Health check -app.get('/health', (req, res) => { - res.json({ status: 'ok', timestamp: new Date().toISOString() }); -}); - -/** - * GET /docs/api - * Get complete API documentation (markdown) - */ -app.get('/docs/api', (req, res) => { - try { - const docPath = path.join(__dirname, '../docs/API.md'); - - if (!fs.existsSync(docPath)) { - return res.status(404).json({ - error: 'Documentation not found', - message: 'API documentation file does not exist' - }); - } - - const docContent = fs.readFileSync(docPath, 'utf-8'); - - // Return as JSON with markdown content - res.json({ - success: true, - documentation: docContent, - format: 'markdown', - lastUpdated: fs.statSync(docPath).mtime - }); - } catch (error) { - console.error(`[API Docs] Error: ${error.message}`); - res.status(500).json({ - error: 'Failed to retrieve documentation', - message: error.message - }); - } -}); - -/** - * GET /public/scripts/extract-and-upload-cookies.sh - * Public endpoint to download the cookie extraction script - */ -app.get('/public/scripts/extract-and-upload-cookies.sh', (req, res) => { - try { - const scriptPath = path.join(__dirname, '../extract-and-upload-cookies.sh'); - - if (!fs.existsSync(scriptPath)) { - return res.status(404).json({ - error: 'Script not found', - message: 'Cookie extraction script does not exist' - }); - } - - // Send file with proper headers for download - res.download(scriptPath, 'extract-and-upload-cookies.sh', (err) => { - if (err) { - console.error(`[Script Download] Error: ${err.message}`); - if (!res.headersSent) { - res.status(500).json({ - error: 'Download failed', - message: err.message - }); - } - } - }); - } catch (error) { - console.error(`[Script Download] Error: ${error.message}`); - res.status(500).json({ - error: 'Server error', - message: error.message - }); - } -}); - -/** - * GET /public/download/:filename - * Public endpoint to download files without authentication - */ -app.get('/public/download/:filename', (req, res) => { - try { - const { filename } = req.params; - - // Security: prevent directory traversal - const safeName = path.basename(filename); - const filePath = path.join(OUTPUT_DIR, safeName); - - // Check if file exists - if (!fs.existsSync(filePath)) { - return res.status(404).json({ - error: 'File not found', - message: `File '${safeName}' does not exist` - }); - } - - // Send file with proper headers - res.download(filePath, safeName, (err) => { - if (err) { - console.error(`[Public Download] Error: ${err.message}`); - if (!res.headersSent) { - res.status(500).json({ - error: 'Download failed', - message: err.message - }); - } - } - }); - } catch (error) { - console.error(`[Public Download] Error: ${error.message}`); - res.status(500).json({ - error: 'Server error', - message: error.message - }); - } -}); - -/** - * GET /info?url= - * Get info about a video or playlist - */ -app.get('/info', async (req, res) => { - try { - const { url } = req.query; - - if (!url) { - return res.status(400).json({ error: 'URL parameter required' }); - } - - // Check if URL contains playlist parameter - const hasPlaylist = url.includes('list='); - const info = await getInfo(url, hasPlaylist); - - res.json({ - success: true, - title: info.title, - type: info._type || 'video', - duration: info.duration, - channel: info.channel, - entries: info._type === 'playlist' - ? info.entries?.map(e => ({ id: e.id, title: e.title })) - : null, - videoCount: info._type === 'playlist' ? info.entries?.length : 1, - }); - } catch (error) { - handleYouTubeError(error, res); - } -}); - -/** - * GET /download-stream - * Download with SSE progress updates - * Query: url (required), outputPath (optional) - */ -app.get('/download-stream', async (req, res) => { - const { url, outputPath } = req.query; - - if (!url) { - return res.status(400).json({ error: 'URL parameter required' }); - } - - // Set up SSE - res.setHeader('Content-Type', 'text/event-stream'); - res.setHeader('Cache-Control', 'no-cache'); - res.setHeader('Connection', 'keep-alive'); - - - const sendEvent = (event, data) => { - res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); - }; - - // Track timing for estimation - const startTime = Date.now(); - let videosCompleted = 0; - let totalVideos = 1; - const videoTimes = []; - - try { - // First, get info to know total videos - sendEvent('status', { message: 'Fetching video info...', phase: 'info' }); - const hasPlaylist = url.includes('list='); - const info = await getInfo(url, hasPlaylist); - - totalVideos = info._type === 'playlist' ? (info.entries?.length || 1) : 1; - sendEvent('info', { - title: info.title, - type: info._type || 'video', - totalVideos, - playlistTitle: info._type === 'playlist' ? info.title : null, - }); - - console.log(`Downloading: ${url}`); - - let videoStartTime = Date.now(); - - const result = await download(url, { - outputDir: outputPath || OUTPUT_DIR, - onDownloadProgress: (progress) => { - // Calculate overall progress - const videoProgress = progress.percent || 0; - const overallPercent = ((videosCompleted + (videoProgress / 100)) / totalVideos) * 100; - - // Estimate remaining time - let estimatedRemaining = null; - if (videosCompleted > 0 && videoTimes.length > 0) { - const avgTimePerVideo = videoTimes.reduce((a, b) => a + b, 0) / videoTimes.length; - const remainingVideos = totalVideos - videosCompleted - (videoProgress / 100); - estimatedRemaining = Math.round(avgTimePerVideo * remainingVideos / 1000); - } else if (progress.eta) { - // Parse ETA from yt-dlp (format: MM:SS) - const [mins, secs] = progress.eta.split(':').map(Number); - const currentVideoRemaining = mins * 60 + secs; - const remainingVideos = totalVideos - videosCompleted - 1; - // Estimate based on current video - if (videoProgress > 10) { - const elapsed = (Date.now() - videoStartTime) / 1000; - const estimatedVideoTime = (elapsed / videoProgress) * 100; - estimatedRemaining = Math.round(currentVideoRemaining + (remainingVideos * estimatedVideoTime)); - } - } - - sendEvent('progress', { - percent: Math.round(overallPercent * 10) / 10, - videoPercent: Math.round(videoProgress * 10) / 10, - currentVideo: progress.videoIndex || 1, - totalVideos: progress.totalVideos || totalVideos, - title: progress.title, - speed: progress.speed, - eta: progress.eta, - estimatedRemaining, - phase: 'downloading', - }); - }, - onVideoComplete: (video) => { - const videoTime = Date.now() - videoStartTime; - videoTimes.push(videoTime); - videosCompleted++; - videoStartTime = Date.now(); - - sendEvent('video-complete', { - title: video.title, - success: video.success, - videosCompleted, - totalVideos, - }); - }, - }); - - // Send final result - sendEvent('complete', { - success: true, - playlistTitle: result.playlistTitle, - totalVideos: result.totalVideos, - successCount: result.successCount, - failCount: result.failCount, - totalTime: Math.round((Date.now() - startTime) / 1000), - videos: result.videos.map(v => ({ - success: v.success, - title: v.title, - filePath: v.filePath, - fileUrl: v.filePath ? `/files/${path.basename(v.filePath)}` : null, - error: v.error, - })), - }); - - } catch (error) { - // Enhanced error for bot detection - if (error.isEnhanced && error.details) { - sendEvent('error', error.details); - } else { - sendEvent('error', { message: error.message }); - } - } finally { - res.end(); - } -}); - -/** - * POST /download - * Download a video or playlist as MP3 (non-streaming version) - * Body: { url: string, outputPath?: string } - */ -app.post('/download', async (req, res) => { - try { - const { url, outputPath } = req.body; - const outputDir = outputPath || OUTPUT_DIR; - - if (!url) { - return res.status(400).json({ error: 'URL required in request body' }); - } - - console.log(`Downloading: ${url}`); - const result = await download(url, { outputDir }); - - res.json({ - success: true, - playlistTitle: result.playlistTitle, - totalVideos: result.totalVideos, - successCount: result.successCount, - failCount: result.failCount, - videos: result.videos.map(v => ({ - success: v.success, - title: v.title, - filePath: v.filePath, - fileUrl: v.filePath ? `/files/${path.basename(v.filePath)}` : null, - error: v.error, - })), - }); - } catch (error) { - handleYouTubeError(error, res); - } -}); - -/** - * POST /transcribe - * Transcribe an existing audio file - * Body: { filePath: string, language?: string, format?: string, outputPath?: string } - */ -app.post('/transcribe', async (req, res) => { - try { - const { filePath, language, format = 'txt', model = 'gpt-4o-mini-transcribe', outputPath } = req.body; - - if (!filePath) { - return res.status(400).json({ error: 'filePath required in request body' }); - } - - if (!process.env.OPENAI_API_KEY) { - return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); - } - - console.log(`Transcribing: ${filePath} with model ${model}`); - const result = await transcribeAndSave(filePath, { - language, - responseFormat: format === 'txt' ? 'text' : format, - outputFormat: format, - model, - outputDir: outputPath, - }); - - res.json({ - success: true, - filePath: result.filePath, - transcriptionPath: result.transcriptionPath, - transcriptionUrl: `/files/${path.basename(result.transcriptionPath)}`, - text: result.text, - }); - } catch (error) { - res.status(500).json({ error: error.message }); - } -}); - -/** - * POST /upload-transcribe - * Upload audio files and transcribe them - * Body: { language?: string, model?: string, outputPath?: string } - */ -app.post('/upload-transcribe', upload.array('files', 50), async (req, res) => { - try { - if (!process.env.OPENAI_API_KEY) { - return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); - } - - if (!req.files || req.files.length === 0) { - return res.status(400).json({ error: 'No files uploaded' }); - } - - const { language, model = 'gpt-4o-mini-transcribe', outputPath } = req.body; - const results = []; - - console.log(`Transcribing ${req.files.length} uploaded files with model ${model}`); - - for (let i = 0; i < req.files.length; i++) { - const file = req.files[i]; - console.log(`[${i + 1}/${req.files.length}] Transcribing: ${file.originalname}`); - - try { - const result = await transcribeAndSave(file.path, { - language: language || undefined, - responseFormat: 'text', - outputFormat: 'txt', - model, - outputDir: outputPath, - }); - - results.push({ - success: true, - fileName: file.originalname, - filePath: file.path, - transcriptionPath: result.transcriptionPath, - transcriptionUrl: `/files/${path.basename(result.transcriptionPath)}`, - text: result.text, - }); - } catch (error) { - console.error(`Failed to transcribe ${file.originalname}: ${error.message}`); - results.push({ - success: false, - fileName: file.originalname, - error: error.message, - }); - } - } - - res.json({ - success: true, - totalFiles: req.files.length, - successCount: results.filter(r => r.success).length, - failCount: results.filter(r => !r.success).length, - results, - }); - } catch (error) { - res.status(500).json({ error: error.message }); - } -}); - -/** - * POST /upload-process - * Smart endpoint that auto-detects input type and processes accordingly: - * - Video files (MP4, AVI, etc.) -> Convert to MP3 -> Transcribe - * - Audio files (MP3, WAV, etc.) -> Transcribe directly - * - URL parameter -> Download from YouTube -> Transcribe - * Body: { url?: string, language?: string, model?: string, outputPath?: string } - * Files: files[] (optional, video or audio files) - */ -app.post('/upload-process', uploadVideo.array('files', 50), async (req, res) => { - try { - if (!process.env.OPENAI_API_KEY) { - return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); - } - - const { url, language, model = 'gpt-4o-mini-transcribe', outputPath } = req.body; - const outputDir = outputPath || OUTPUT_DIR; - - // Detect input type - const hasFiles = req.files && req.files.length > 0; - const hasUrl = url && url.trim() !== ''; - - if (!hasFiles && !hasUrl) { - return res.status(400).json({ - error: 'No input provided. Please provide either files to upload or a URL parameter' - }); - } - - const results = []; - let totalFiles = 0; - - // Process URL if provided - if (hasUrl) { - console.log(`[Smart Process] Detected URL: ${url}`); - try { - // Step 1: Download from YouTube - console.log(`[Smart Process] Downloading from YouTube...`); - const downloadResult = await download(url, { outputDir }); - - // Step 2: Transcribe downloaded files - const successfulDownloads = downloadResult.videos.filter(v => v.success); - console.log(`[Smart Process] Transcribing ${successfulDownloads.length} downloaded files...`); - - for (const video of successfulDownloads) { - totalFiles++; - try { - const transcribeResult = await transcribeAndSave(video.filePath, { - language: language || undefined, - responseFormat: 'text', - outputFormat: 'txt', - model, - outputDir, - }); - - results.push({ - success: true, - source: 'url', - sourceType: 'youtube', - title: video.title, - audioPath: video.filePath, - audioUrl: `/files/${path.basename(video.filePath)}`, - transcriptionPath: transcribeResult.transcriptionPath, - transcriptionUrl: `/files/${path.basename(transcribeResult.transcriptionPath)}`, - text: transcribeResult.text, - }); - } catch (error) { - results.push({ - success: false, - source: 'url', - title: video.title, - error: error.message, - }); - } - } - } catch (error) { - results.push({ - success: false, - source: 'url', - error: error.message, - }); - } - } - - // Process uploaded files if provided - if (hasFiles) { - console.log(`[Smart Process] Detected ${req.files.length} uploaded files`); - - for (let i = 0; i < req.files.length; i++) { - const file = req.files[i]; - totalFiles++; - - const isVideo = /\.(mp4|avi|mkv|mov|wmv|flv|webm|m4v)$/i.test(file.originalname); - const isAudio = /\.(mp3|wav|m4a|flac|ogg|aac)$/i.test(file.originalname); - - console.log(`[${i + 1}/${req.files.length}] Processing: ${file.originalname} (${isVideo ? 'video' : 'audio'})`); - - try { - let audioFilePath = file.path; - let conversionResult = null; - - // Step 1: Convert to MP3 if it's a video - if (isVideo) { - console.log(` → Converting video to MP3...`); - conversionResult = await convertToMP3(file.path, { - outputDir, - bitrate: '192k', - quality: '2', - }); - audioFilePath = conversionResult.outputPath; - } - - // Step 2: Transcribe the audio - console.log(` → Transcribing audio...`); - const transcribeResult = await transcribeAndSave(audioFilePath, { - language: language || undefined, - responseFormat: 'text', - outputFormat: 'txt', - model, - outputDir, - }); - - results.push({ - success: true, - source: 'upload', - sourceType: isVideo ? 'video' : 'audio', - fileName: file.originalname, - converted: isVideo, - audioPath: audioFilePath, - audioUrl: `/files/${path.basename(audioFilePath)}`, - transcriptionPath: transcribeResult.transcriptionPath, - transcriptionUrl: `/files/${path.basename(transcribeResult.transcriptionPath)}`, - text: transcribeResult.text, - }); - } catch (error) { - console.error(` ✗ Failed to process ${file.originalname}: ${error.message}`); - results.push({ - success: false, - source: 'upload', - fileName: file.originalname, - error: error.message, - }); - } - } - } - - const successCount = results.filter(r => r.success).length; - const failCount = results.filter(r => !r.success).length; - - res.json({ - success: true, - totalFiles, - successCount, - failCount, - results, - }); - } catch (error) { - res.status(500).json({ error: error.message }); - } -}); - -/** - * POST /convert-to-mp3 - * Upload video/audio files and convert them to MP3 - */ -app.post('/convert-to-mp3', uploadVideo.array('files', 50), async (req, res) => { - try { - if (!req.files || req.files.length === 0) { - return res.status(400).json({ error: 'No files uploaded' }); - } - - const { bitrate = '192k', quality = '2' } = req.body; - const results = []; - - console.log(`Converting ${req.files.length} files to MP3`); - - for (let i = 0; i < req.files.length; i++) { - const file = req.files[i]; - console.log(`[${i + 1}/${req.files.length}] Converting: ${file.originalname}`); - - try { - const result = await convertToMP3(file.path, { - outputDir: OUTPUT_DIR, - bitrate, - quality, - }); - - results.push({ - success: true, - fileName: file.originalname, - inputPath: file.path, - outputPath: result.outputPath, - outputUrl: `/files/${path.basename(result.outputPath)}`, - size: result.sizeHuman, - }); - } catch (error) { - console.error(`Failed to convert ${file.originalname}: ${error.message}`); - results.push({ - success: false, - fileName: file.originalname, - error: error.message, - }); - } - } - - res.json({ - success: true, - totalFiles: req.files.length, - successCount: results.filter(r => r.success).length, - failCount: results.filter(r => !r.success).length, - results, - }); - } catch (error) { - res.status(500).json({ error: error.message }); - } -}); - -/** - * GET /supported-formats - * Get supported video/audio formats for conversion - */ -app.get('/supported-formats', (req, res) => { - res.json({ formats: getSupportedFormats() }); -}); - -/** - * GET /process-stream - * Download and transcribe with SSE progress updates - * Query: url, language?, model?, outputPath? - */ -app.get('/process-stream', async (req, res) => { - const { url, language, model = 'gpt-4o-mini-transcribe', outputPath } = req.query; - - if (!url) { - return res.status(400).json({ error: 'URL parameter required' }); - } - - if (!process.env.OPENAI_API_KEY) { - return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); - } - - // Set up SSE - res.setHeader('Content-Type', 'text/event-stream'); - res.setHeader('Cache-Control', 'no-cache'); - res.setHeader('Connection', 'keep-alive'); - - - const sendEvent = (event, data) => { - res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); - }; - - const startTime = Date.now(); - let videosDownloaded = 0; - let videosTranscribed = 0; - let totalVideos = 1; - const videoTimes = []; - - try { - // Phase 1: Get info - sendEvent('status', { message: 'Fetching video info...', phase: 'info' }); - const hasPlaylist = url.includes('list='); - const info = await getInfo(url, hasPlaylist); - - totalVideos = info._type === 'playlist' ? (info.entries?.length || 1) : 1; - sendEvent('info', { - title: info.title, - type: info._type || 'video', - totalVideos, - playlistTitle: info._type === 'playlist' ? info.title : null, - }); - - // Phase 2: Download - console.log(`Processing: ${url}`); - let videoStartTime = Date.now(); - - const downloadResult = await download(url, { - outputDir: outputPath || OUTPUT_DIR, - onDownloadProgress: (progress) => { - const videoProgress = progress.percent || 0; - // Download is 50% of total, transcribe is other 50% - const overallPercent = ((videosDownloaded + (videoProgress / 100)) / totalVideos) * 50; - - sendEvent('progress', { - percent: Math.round(overallPercent * 10) / 10, - videoPercent: Math.round(videoProgress * 10) / 10, - currentVideo: progress.videoIndex || 1, - totalVideos: progress.totalVideos || totalVideos, - title: progress.title, - speed: progress.speed, - eta: progress.eta, - phase: 'downloading', - phaseLabel: 'Downloading', - }); - }, - onVideoComplete: (video) => { - const videoTime = Date.now() - videoStartTime; - videoTimes.push(videoTime); - videosDownloaded++; - videoStartTime = Date.now(); - - sendEvent('video-complete', { - title: video.title, - success: video.success, - phase: 'downloading', - videosCompleted: videosDownloaded, - totalVideos, - }); - }, - }); - - // Phase 3: Transcribe - sendEvent('status', { message: 'Starting transcription...', phase: 'transcribing' }); - - const successfulDownloads = downloadResult.videos.filter(v => v.success); - const filePaths = successfulDownloads.map(v => v.filePath); - const transcribeResults = []; - - for (let i = 0; i < filePaths.length; i++) { - const filePath = filePaths[i]; - const video = successfulDownloads[i]; - - sendEvent('progress', { - percent: 50 + ((i / filePaths.length) * 50), - currentVideo: i + 1, - totalVideos: filePaths.length, - title: video.title, - phase: 'transcribing', - phaseLabel: 'Transcribing', - }); - - try { - const result = await transcribeAndSave(filePath, { - language: language || undefined, - responseFormat: 'text', - outputFormat: 'txt', - model, - outputDir: outputPath, - }); - transcribeResults.push(result); - videosTranscribed++; - - sendEvent('transcribe-complete', { - title: video.title, - success: true, - videosCompleted: videosTranscribed, - totalFiles: filePaths.length, - }); - } catch (error) { - transcribeResults.push({ - success: false, - filePath, - error: error.message, - }); - - sendEvent('transcribe-complete', { - title: video.title, - success: false, - error: error.message, - videosCompleted: videosTranscribed, - totalFiles: filePaths.length, - }); - } - } - - // Combine results - const combinedResults = downloadResult.videos.map(v => { - const transcription = transcribeResults.find(t => t.filePath === v.filePath); - return { - title: v.title, - downloadSuccess: v.success, - audioUrl: v.filePath ? `/files/${path.basename(v.filePath)}` : null, - transcriptionSuccess: transcription?.success || false, - transcriptionUrl: transcription?.transcriptionPath - ? `/files/${path.basename(transcription.transcriptionPath)}` - : null, - text: transcription?.text, - error: v.error || transcription?.error, - }; - }); - - sendEvent('complete', { - success: true, - playlistTitle: downloadResult.playlistTitle, - totalVideos: downloadResult.totalVideos, - downloadedCount: downloadResult.successCount, - transcribedCount: videosTranscribed, - totalTime: Math.round((Date.now() - startTime) / 1000), - results: combinedResults, - }); - - } catch (error) { - // Enhanced error for bot detection - if (error.isEnhanced && error.details) { - sendEvent('error', error.details); - } else { - sendEvent('error', { message: error.message }); - } - } finally { - res.end(); - } -}); - -/** - * POST /process - * Download and transcribe a video or playlist (non-streaming) - * Body: { url: string, language?: string, format?: string, outputPath?: string } - */ -app.post('/process', async (req, res) => { - try { - const { url, language, format = 'txt', outputPath, model = 'gpt-4o-mini-transcribe' } = req.body; - const outputDir = outputPath || OUTPUT_DIR; - - if (!url) { - return res.status(400).json({ error: 'URL required in request body' }); - } - - if (!process.env.OPENAI_API_KEY) { - return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); - } - - // Step 1: Download - console.log(`Step 1: Downloading ${url}`); - const downloadResult = await download(url, { outputDir }); - - // Step 2: Transcribe - console.log(`Step 2: Transcribing with model ${model}...`); - const successfulDownloads = downloadResult.videos.filter(v => v.success); - const filePaths = successfulDownloads.map(v => v.filePath); - - const transcribeResult = await transcribeMultiple(filePaths, { - language, - responseFormat: format === 'txt' ? 'text' : format, - outputFormat: format, - model, - outputDir, - }); - - // Combine results - const combinedResults = downloadResult.videos.map(v => { - const transcription = transcribeResult.results.find( - t => t.filePath === v.filePath - ); - - return { - title: v.title, - downloadSuccess: v.success, - audioPath: v.filePath, - audioUrl: v.filePath ? `/files/${path.basename(v.filePath)}` : null, - transcriptionSuccess: transcription?.success || false, - transcriptionPath: transcription?.transcriptionPath, - transcriptionUrl: transcription?.transcriptionPath - ? `/files/${path.basename(transcription.transcriptionPath)}` - : null, - text: transcription?.text, - error: v.error || transcription?.error, - }; - }); - - res.json({ - success: true, - playlistTitle: downloadResult.playlistTitle, - totalVideos: downloadResult.totalVideos, - downloadedCount: downloadResult.successCount, - transcribedCount: transcribeResult.successCount, - results: combinedResults, - }); - } catch (error) { - handleYouTubeError(error, res); - } -}); - -/** - * GET /files - * List all downloaded files - */ -app.get('/files-list', (req, res) => { - try { - if (!fs.existsSync(OUTPUT_DIR)) { - return res.json({ files: [] }); - } - - const files = fs.readdirSync(OUTPUT_DIR).map(file => ({ - name: file, - url: `/files/${file}`, - path: path.join(OUTPUT_DIR, file), - })); - - res.json({ files }); - } catch (error) { - res.status(500).json({ error: error.message }); - } -}); - -/** - * GET /languages - * Get available translation languages - */ -app.get('/languages', (req, res) => { - res.json({ languages: getLanguages() }); -}); - -/** - * POST /translate - * Translate text - * Body: { text: string, targetLang: string, sourceLang?: string } - */ -app.post('/translate', async (req, res) => { - try { - const { text, targetLang, sourceLang } = req.body; - - if (!text) { - return res.status(400).json({ error: 'text required in request body' }); - } - if (!targetLang) { - return res.status(400).json({ error: 'targetLang required in request body' }); - } - - if (!process.env.OPENAI_API_KEY) { - return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); - } - - console.log(`Translating text to ${targetLang}`); - const result = await translateText(text, targetLang, sourceLang); - - res.json({ - success: true, - ...result, - }); - } catch (error) { - res.status(500).json({ error: error.message }); - } -}); - -/** - * POST /translate-file - * Translate uploaded text files - * Body: { targetLang: string, sourceLang?: string, outputPath?: string } - */ -app.post('/translate-file', uploadText.array('files', 50), async (req, res) => { - try { - if (!process.env.OPENAI_API_KEY) { - return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); - } - - if (!req.files || req.files.length === 0) { - return res.status(400).json({ error: 'No files uploaded' }); - } - - const { targetLang, sourceLang, outputPath } = req.body; - - if (!targetLang) { - return res.status(400).json({ error: 'targetLang required' }); - } - - const results = []; - console.log(`Translating ${req.files.length} files to ${targetLang}`); - - for (let i = 0; i < req.files.length; i++) { - const file = req.files[i]; - console.log(`[${i + 1}/${req.files.length}] Translating: ${file.originalname}`); - - try { - const result = await translateFile(file.path, targetLang, sourceLang || null, outputPath); - results.push({ - success: true, - fileName: file.originalname, - translationPath: result.translationPath, - translationUrl: `/files/${path.basename(result.translationPath)}`, - translatedText: result.translatedText, - }); - } catch (error) { - console.error(`Failed to translate ${file.originalname}: ${error.message}`); - results.push({ - success: false, - fileName: file.originalname, - error: error.message, - }); - } - } - - res.json({ - success: true, - totalFiles: req.files.length, - successCount: results.filter(r => r.success).length, - failCount: results.filter(r => !r.success).length, - results, - }); - } catch (error) { - res.status(500).json({ error: error.message }); - } -}); - -/** - * GET /summary-styles - * Get available summary styles - */ -app.get('/summary-styles', (req, res) => { - res.json({ styles: getSummaryStyles() }); -}); - -/** - * POST /summarize - * Summarize text using GPT-5.1 - * Body: { text: string, style?: string, language?: string, model?: string } - */ -app.post('/summarize', async (req, res) => { - try { - const { text, style = 'concise', language = 'same', model = 'gpt-5.1' } = req.body; - - if (!text) { - return res.status(400).json({ error: 'text required in request body' }); - } - - if (!process.env.OPENAI_API_KEY) { - return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); - } - - console.log(`Summarizing text with ${model} (style: ${style})`); - const result = await summarizeText(text, { style, language, model }); - - res.json({ - success: true, - ...result, - }); - } catch (error) { - res.status(500).json({ error: error.message }); - } -}); - -/** - * POST /summarize-file - * Summarize uploaded text files using GPT-5.1 - * Body: { style?: string, language?: string, model?: string, outputPath?: string } - */ -app.post('/summarize-file', uploadText.array('files', 50), async (req, res) => { - try { - if (!process.env.OPENAI_API_KEY) { - return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); - } - - if (!req.files || req.files.length === 0) { - return res.status(400).json({ error: 'No files uploaded' }); - } - - const { style = 'concise', language = 'same', model = 'gpt-5.1', outputPath } = req.body; - const results = []; - - console.log(`Summarizing ${req.files.length} files with ${model}`); - - for (let i = 0; i < req.files.length; i++) { - const file = req.files[i]; - console.log(`[${i + 1}/${req.files.length}] Summarizing: ${file.originalname}`); - - try { - const result = await summarizeFile(file.path, { style, language, model, outputDir: outputPath }); - results.push({ - success: true, - fileName: file.originalname, - summaryPath: result.summaryPath, - summaryUrl: `/files/${path.basename(result.summaryPath)}`, - summary: result.summary, - model: result.model, - chunks: result.chunks, - }); - } catch (error) { - console.error(`Failed to summarize ${file.originalname}: ${error.message}`); - results.push({ - success: false, - fileName: file.originalname, - error: error.message, - }); - } - } - - res.json({ - success: true, - totalFiles: req.files.length, - successCount: results.filter(r => r.success).length, - failCount: results.filter(r => !r.success).length, - results, - }); - } catch (error) { - res.status(500).json({ error: error.message }); - } -}); - -/** - * GET /summarize-stream - * Full pipeline: Download -> Transcribe -> Summarize with SSE progress - * Query: url, style?, language?, model?, outputPath? - */ -app.get('/summarize-stream', async (req, res) => { - const { url, style = 'concise', language = 'same', model = 'gpt-4o-mini-transcribe', outputPath } = req.query; - - if (!url) { - return res.status(400).json({ error: 'URL parameter required' }); - } - - if (!process.env.OPENAI_API_KEY) { - return res.status(500).json({ error: 'OPENAI_API_KEY not configured' }); - } - - // Set up SSE - res.setHeader('Content-Type', 'text/event-stream'); - res.setHeader('Cache-Control', 'no-cache'); - res.setHeader('Connection', 'keep-alive'); - - - const sendEvent = (event, data) => { - res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`); - }; - - const startTime = Date.now(); - let totalVideos = 1; - - try { - // Phase 1: Get info - sendEvent('status', { message: 'Fetching video info...', phase: 'info', percent: 0 }); - const hasPlaylist = url.includes('list='); - const info = await getInfo(url, hasPlaylist); - - totalVideos = info._type === 'playlist' ? (info.entries?.length || 1) : 1; - sendEvent('info', { - title: info.title, - type: info._type || 'video', - totalVideos, - playlistTitle: info._type === 'playlist' ? info.title : null, - }); - - // Phase 2: Download (0-33%) - sendEvent('status', { message: 'Downloading...', phase: 'downloading', percent: 5 }); - console.log(`[Summarize Pipeline] Downloading: ${url}`); - - let videosDownloaded = 0; - const downloadResult = await download(url, { - outputDir: outputPath || OUTPUT_DIR, - onDownloadProgress: (progress) => { - const videoProgress = progress.percent || 0; - const overallPercent = ((videosDownloaded + (videoProgress / 100)) / totalVideos) * 33; - - sendEvent('progress', { - percent: Math.round(overallPercent * 10) / 10, - phase: 'downloading', - phaseLabel: 'Downloading', - title: progress.title, - speed: progress.speed, - currentVideo: progress.videoIndex || videosDownloaded + 1, - totalVideos, - }); - }, - onVideoComplete: (video) => { - videosDownloaded++; - sendEvent('video-complete', { - title: video.title, - phase: 'downloading', - videosCompleted: videosDownloaded, - totalVideos, - }); - }, - }); - - // Phase 3: Transcribe (33-66%) - sendEvent('status', { message: 'Transcribing...', phase: 'transcribing', percent: 33 }); - console.log(`[Summarize Pipeline] Transcribing ${downloadResult.successCount} files`); - - const successfulDownloads = downloadResult.videos.filter(v => v.success); - const transcribeResults = []; - - for (let i = 0; i < successfulDownloads.length; i++) { - const video = successfulDownloads[i]; - const percent = 33 + ((i / successfulDownloads.length) * 33); - - sendEvent('progress', { - percent: Math.round(percent * 10) / 10, - phase: 'transcribing', - phaseLabel: 'Transcribing', - title: video.title, - currentVideo: i + 1, - totalVideos: successfulDownloads.length, - }); - - try { - const result = await transcribeAndSave(video.filePath, { - responseFormat: 'text', - outputFormat: 'txt', - model, - outputDir: outputPath, - }); - transcribeResults.push({ ...result, title: video.title, success: true }); - - sendEvent('transcribe-complete', { - title: video.title, - success: true, - videosCompleted: i + 1, - totalVideos: successfulDownloads.length, - }); - } catch (error) { - transcribeResults.push({ title: video.title, success: false, error: error.message }); - sendEvent('transcribe-complete', { - title: video.title, - success: false, - error: error.message, - }); - } - } - - // Phase 4: Summarize (66-100%) - sendEvent('status', { message: 'Summarizing with GPT-5.1...', phase: 'summarizing', percent: 66 }); - console.log(`[Summarize Pipeline] Summarizing ${transcribeResults.filter(t => t.success).length} transcriptions`); - - const summaryResults = []; - const successfulTranscriptions = transcribeResults.filter(t => t.success); - - for (let i = 0; i < successfulTranscriptions.length; i++) { - const transcription = successfulTranscriptions[i]; - const percent = 66 + ((i / successfulTranscriptions.length) * 34); - - sendEvent('progress', { - percent: Math.round(percent * 10) / 10, - phase: 'summarizing', - phaseLabel: 'Summarizing', - title: transcription.title, - currentVideo: i + 1, - totalVideos: successfulTranscriptions.length, - }); - - try { - const result = await summarizeFile(transcription.transcriptionPath, { style, language, model: 'gpt-5.1', outputDir: outputPath }); - summaryResults.push({ - title: transcription.title, - success: true, - summary: result.summary, - summaryPath: result.summaryPath, - summaryUrl: `/files/${path.basename(result.summaryPath)}`, - transcriptionUrl: `/files/${path.basename(transcription.transcriptionPath)}`, - audioUrl: transcription.filePath ? `/files/${path.basename(transcription.filePath)}` : null, - }); - - sendEvent('summarize-complete', { - title: transcription.title, - success: true, - videosCompleted: i + 1, - totalVideos: successfulTranscriptions.length, - }); - } catch (error) { - summaryResults.push({ - title: transcription.title, - success: false, - error: error.message, - transcriptionUrl: `/files/${path.basename(transcription.transcriptionPath)}`, - }); - sendEvent('summarize-complete', { - title: transcription.title, - success: false, - error: error.message, - }); - } - } - - // Final result - sendEvent('complete', { - success: true, - playlistTitle: downloadResult.playlistTitle, - totalVideos: downloadResult.totalVideos, - downloadedCount: downloadResult.successCount, - transcribedCount: transcribeResults.filter(t => t.success).length, - summarizedCount: summaryResults.filter(s => s.success).length, - totalTime: Math.round((Date.now() - startTime) / 1000), - results: summaryResults, - }); - - } catch (error) { - console.error(`[Summarize Pipeline] Error: ${error.message}`); - // Enhanced error for bot detection - if (error.isEnhanced && error.details) { - sendEvent('error', error.details); - } else { - sendEvent('error', { message: error.message }); - } - } finally { - res.end(); - } -}); - -/** - * POST /admin/upload-cookies - * Upload YouTube cookies file to persist authentication - */ -const uploadCookies = multer({ - storage: multer.memoryStorage(), - fileFilter: (req, file, cb) => { - if (file.mimetype === 'text/plain' || file.originalname.endsWith('.txt')) { - cb(null, true); - } else { - cb(new Error('Invalid file type. Only .txt files are allowed.')); - } - }, - limits: { - fileSize: 1024 * 1024, // 1MB max - } -}); - -app.post('/admin/upload-cookies', uploadCookies.single('cookies'), async (req, res) => { - try { - if (!req.file) { - return res.status(400).json({ - error: 'No file uploaded', - message: 'Please upload a cookies.txt file', - help: 'Export cookies from your browser using a "Get cookies.txt" extension' - }); - } - - // Paths for storing cookies - const localCookiesPath = path.join(process.cwd(), 'youtube-cookies.txt'); - const shareCookiesPath = '/tmp/share/youtube-cookies.txt'; - - // Write to local directory - fs.writeFileSync(localCookiesPath, req.file.buffer); - fs.chmodSync(localCookiesPath, 0o600); // Secure permissions - - console.log(`✓ Cookies saved to: ${localCookiesPath}`); - - // Also save to /tmp/share if it exists (for persistence across restarts) - try { - if (!fs.existsSync('/tmp/share')) { - fs.mkdirSync('/tmp/share', { recursive: true }); - } - fs.writeFileSync(shareCookiesPath, req.file.buffer); - fs.chmodSync(shareCookiesPath, 0o600); - console.log(`✓ Cookies also saved to: ${shareCookiesPath} (persistent)`); - } catch (err) { - console.warn(`⚠ Could not save to /tmp/share: ${err.message}`); - } - - // Update environment variable for immediate use - process.env.YOUTUBE_COOKIES_PATH = localCookiesPath; - - res.json({ - success: true, - message: 'Cookies uploaded successfully', - paths: { - local: localCookiesPath, - persistent: fs.existsSync(shareCookiesPath) ? shareCookiesPath : null, - }, - note: 'Cookies are now active. No restart required.' - }); - - } catch (error) { - console.error(`[Upload Cookies] Error: ${error.message}`); - res.status(500).json({ - error: 'Failed to upload cookies', - message: error.message - }); - } -}); - -app.listen(PORT, () => { - console.log(`Server running on http://localhost:${PORT}`); - console.log('\nEndpoints:'); - console.log(' GET /health - Health check'); - console.log(' GET /docs/api - API documentation (no auth)'); - console.log(' GET /public/download/:filename - Public file download (no auth)'); - console.log(' GET /public/scripts/extract-and-upload-cookies.sh - Cookie script (no auth)'); - console.log(' GET /info?url= - Get video/playlist info'); - console.log(' POST /download - Download as MP3'); - console.log(' POST /transcribe - Transcribe audio file'); - console.log(' POST /process - Download + transcribe'); - console.log(' POST /upload-process - Smart: Upload video/audio OR URL -> Transcribe'); - console.log(' POST /summarize - Summarize text (GPT-5.1)'); - console.log(' POST /summarize-file - Summarize files (GPT-5.1)'); - console.log(' GET /summarize-stream - Full pipeline: Download + Transcribe + Summarize'); - console.log(' GET /files-list - List downloaded files'); - console.log(' GET /files/ - Serve downloaded files'); - console.log(' POST /admin/upload-cookies - Upload YouTube cookies for bot detection bypass'); -}); +require('dotenv').config(); +const express = require('express'); +const fs = require('fs'); +const path = require('path'); +const cookiesManager = require('./services/cookiesManager'); +const downloadService = require('./services/download'); + +const app = express(); +const PORT = process.env.PORT || 8889; + +// Middleware +app.use(express.json()); + +// CORS (if needed) +app.use((req, res, next) => { + res.header('Access-Control-Allow-Origin', process.env.ALLOWED_ORIGINS || '*'); + res.header('Access-Control-Allow-Methods', 'GET, POST, DELETE'); + res.header('Access-Control-Allow-Headers', 'Content-Type'); + next(); +}); + +// Request logging +app.use((req, res, next) => { + console.log(`${req.method} ${req.path}`); + next(); +}); + +// Initialize cookies manager on startup +cookiesManager.init().then(() => { + console.log('✅ Cookies manager initialized'); +}).catch(err => { + console.error('❌ Failed to initialize cookies manager:', err.message); +}); + +/** + * Health check endpoint + */ +app.get('/health', (req, res) => { + const cookiesStatus = cookiesManager.getStatus(); + res.json({ + status: 'ok', + service: 'hanasuba-music-service', + version: '2.0.0', + cookies: { + valid: cookiesStatus.valid, + lastRefresh: cookiesStatus.lastRefresh + } + }); +}); + +/** + * Download YouTube video to MP3 + * POST /download + * Body: { url: "https://youtube.com/watch?v=..." } + */ +app.post('/download', async (req, res) => { + const { url, quality } = req.body; + + if (!url) { + return res.status(400).json({ + success: false, + error: 'URL is required' + }); + } + + // Validate YouTube URL + if (!url.includes('youtube.com/watch') && !url.includes('youtu.be/')) { + return res.status(400).json({ + success: false, + error: 'Invalid YouTube URL' + }); + } + + try { + console.log(`đŸ“„ Downloading: ${url}`); + + const result = await downloadService.downloadYouTube(url, { quality }); + + res.json(result); + } catch (err) { + console.error('Download error:', err.message); + res.status(500).json({ + success: false, + error: err.message + }); + } +}); + +/** + * Stream/download file + * GET /stream/:filename + */ +app.get('/stream/:filename', async (req, res) => { + const { filename } = req.params; + + try { + const fileInfo = await downloadService.getFileStream(filename); + + // Set headers for streaming + res.setHeader('Content-Type', 'audio/mpeg'); + res.setHeader('Content-Length', fileInfo.size); + res.setHeader('Content-Disposition', `inline; filename="${filename}"`); + + // Support range requests (for seeking in audio player) + const range = req.headers.range; + if (range) { + const parts = range.replace(/bytes=/, '').split('-'); + const start = parseInt(parts[0], 10); + const end = parts[1] ? parseInt(parts[1], 10) : fileInfo.size - 1; + + const chunkSize = (end - start) + 1; + + res.status(206); + res.setHeader('Content-Range', `bytes ${start}-${end}/${fileInfo.size}`); + res.setHeader('Accept-Ranges', 'bytes'); + res.setHeader('Content-Length', chunkSize); + + const stream = fs.createReadStream(fileInfo.path, { start, end }); + stream.pipe(res); + } else { + // Full file + const stream = fs.createReadStream(fileInfo.path); + stream.pipe(res); + } + } catch (err) { + res.status(404).json({ + success: false, + error: err.message + }); + } +}); + +/** + * Delete file + * DELETE /file/:filename + */ +app.delete('/file/:filename', async (req, res) => { + const { filename } = req.params; + + try { + const success = await downloadService.deleteFile(filename); + + if (success) { + res.json({ success: true }); + } else { + res.status(500).json({ + success: false, + error: 'Failed to delete file' + }); + } + } catch (err) { + res.status(500).json({ + success: false, + error: err.message + }); + } +}); + +/** + * Force refresh cookies (admin endpoint) + * POST /admin/refresh-cookies + */ +app.post('/admin/refresh-cookies', async (req, res) => { + try { + console.log('🔄 Manual cookie refresh requested'); + const success = await cookiesManager.refresh(); + + if (success) { + res.json({ + success: true, + message: 'Cookies refreshed successfully' + }); + } else { + res.status(500).json({ + success: false, + error: 'Failed to refresh cookies' + }); + } + } catch (err) { + res.status(500).json({ + success: false, + error: err.message + }); + } +}); + +/** + * Get cookies status (admin endpoint) + * GET /admin/cookies-status + */ +app.get('/admin/cookies-status', (req, res) => { + const status = cookiesManager.getStatus(); + res.json(status); +}); + +// Error handler +app.use((err, req, res, next) => { + console.error('Unhandled error:', err); + res.status(500).json({ + success: false, + error: 'Internal server error' + }); +}); + +// Start server +app.listen(PORT, () => { + console.log(''); + console.log('╔══════════════════════════════════════════════════╗'); + console.log('║ đŸŽ” Hanasuba Music Service v2.0 ║'); + console.log('║ Powered by Camoufox + yt-dlp ║'); + console.log('╚══════════════════════════════════════════════════╝'); + console.log(''); + console.log(`🚀 Server running on http://localhost:${PORT}`); + console.log(`📁 Storage: ${process.env.STORAGE_PATH || './output'}`); + console.log(''); + console.log('Endpoints:'); + console.log(' POST /download - Download YouTube to MP3'); + console.log(' GET /stream/:filename - Stream MP3 file'); + console.log(' DELETE /file/:filename - Delete file'); + console.log(' GET /health - Health check'); + console.log(' POST /admin/refresh-cookies - Force refresh cookies'); + console.log(' GET /admin/cookies-status - Get cookies status'); + console.log(''); +}); diff --git a/src/services/conversion.js b/src/services/conversion.js deleted file mode 100644 index 2fe0ff7..0000000 --- a/src/services/conversion.js +++ /dev/null @@ -1,145 +0,0 @@ -import { exec } from 'child_process'; -import { promisify } from 'util'; -import path from 'path'; -import fs from 'fs'; - -const execPromise = promisify(exec); - -/** - * Convert a video/audio file to MP3 using FFmpeg - * @param {string} inputPath - Path to input file - * @param {object} options - Conversion options - * @param {string} options.outputDir - Output directory (default: same as input) - * @param {string} options.bitrate - Audio bitrate (default: 192k) - * @param {string} options.quality - Audio quality 0-9 (default: 2, where 0 is best) - * @returns {Promise} Conversion result with output path - */ -export async function convertToMP3(inputPath, options = {}) { - const { - outputDir = path.dirname(inputPath), - bitrate = '192k', - quality = '2', - } = options; - - // Ensure input file exists - if (!fs.existsSync(inputPath)) { - throw new Error(`Input file not found: ${inputPath}`); - } - - // Generate output path - const inputFilename = path.basename(inputPath, path.extname(inputPath)); - const outputPath = path.join(outputDir, `${inputFilename}.mp3`); - - // Check if output already exists - if (fs.existsSync(outputPath)) { - // Add timestamp to make it unique - const timestamp = Date.now(); - const uniqueOutputPath = path.join(outputDir, `${inputFilename}_${timestamp}.mp3`); - return convertToMP3Internal(inputPath, uniqueOutputPath, bitrate, quality); - } - - return convertToMP3Internal(inputPath, outputPath, bitrate, quality); -} - -/** - * Internal conversion function - */ -async function convertToMP3Internal(inputPath, outputPath, bitrate, quality) { - try { - // FFmpeg command to convert to MP3 - // -i: input file - // -vn: no video (audio only) - // -ar 44100: audio sample rate 44.1kHz - // -ac 2: stereo - // -b:a: audio bitrate - // -q:a: audio quality (VBR) - const command = `ffmpeg -i "${inputPath}" -vn -ar 44100 -ac 2 -b:a ${bitrate} -q:a ${quality} "${outputPath}"`; - - console.log(`Converting: ${path.basename(inputPath)} -> ${path.basename(outputPath)}`); - - const { stdout, stderr } = await execPromise(command); - - // Verify output file was created - if (!fs.existsSync(outputPath)) { - throw new Error('Conversion failed: output file not created'); - } - - const stats = fs.statSync(outputPath); - - return { - success: true, - inputPath, - outputPath, - filename: path.basename(outputPath), - size: stats.size, - sizeHuman: formatBytes(stats.size), - }; - } catch (error) { - console.error(`Conversion error: ${error.message}`); - throw new Error(`FFmpeg conversion failed: ${error.message}`); - } -} - -/** - * Convert multiple files to MP3 - * @param {string[]} inputPaths - Array of input file paths - * @param {object} options - Conversion options - * @returns {Promise} Batch conversion results - */ -export async function convertMultipleToMP3(inputPaths, options = {}) { - const results = []; - let successCount = 0; - let failCount = 0; - - for (let i = 0; i < inputPaths.length; i++) { - const inputPath = inputPaths[i]; - console.log(`[${i + 1}/${inputPaths.length}] Converting: ${path.basename(inputPath)}`); - - try { - const result = await convertToMP3(inputPath, options); - results.push({ ...result, index: i }); - successCount++; - } catch (error) { - results.push({ - success: false, - inputPath, - error: error.message, - index: i, - }); - failCount++; - console.error(`Failed to convert ${path.basename(inputPath)}: ${error.message}`); - } - } - - return { - totalFiles: inputPaths.length, - successCount, - failCount, - results, - }; -} - -/** - * Format bytes to human readable format - */ -function formatBytes(bytes, decimals = 2) { - if (bytes === 0) return '0 Bytes'; - - const k = 1024; - const dm = decimals < 0 ? 0 : decimals; - const sizes = ['Bytes', 'KB', 'MB', 'GB']; - - const i = Math.floor(Math.log(bytes) / Math.log(k)); - - return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; -} - -/** - * Get supported input formats - */ -export function getSupportedFormats() { - return { - video: ['.mp4', '.avi', '.mkv', '.mov', '.wmv', '.flv', '.webm', '.m4v'], - audio: ['.m4a', '.wav', '.flac', '.ogg', '.aac', '.wma', '.opus'], - }; -} diff --git a/src/services/cookiesManager.js b/src/services/cookiesManager.js new file mode 100644 index 0000000..12bb7e0 --- /dev/null +++ b/src/services/cookiesManager.js @@ -0,0 +1,188 @@ +const { exec } = require('child_process'); +const { promisify } = require('util'); +const fs = require('fs').promises; +const path = require('path'); + +const execAsync = promisify(exec); + +/** + * Manages YouTube cookies lifecycle using Camoufox stealth extraction. + * Auto-refresh when expired, validates periodically. + */ +class CookiesManager { + constructor() { + this.cookiesPath = path.join(__dirname, '../../youtube-cookies.txt'); + this.pythonPath = process.env.PYTHON_PATH || 'python3'; + this.extractScript = path.join(__dirname, '../python/extract_cookies.py'); + this.validateScript = path.join(__dirname, '../python/validate_cookies.py'); + + this.lastRefresh = null; + this.isValid = false; + + // Refresh cookies every 14 days (YouTube cookies typically last 2-4 weeks) + this.refreshIntervalDays = 14; + + // Check interval (every 12 hours) + this.checkIntervalMs = 12 * 60 * 60 * 1000; + } + + /** + * Initialize cookies manager. + * Check if cookies exist, validate them, refresh if needed. + */ + async init() { + console.log('🔧 Initializing cookies manager...'); + + // Check if cookies file exists + try { + await fs.access(this.cookiesPath); + console.log('✅ Cookies file exists'); + + // Validate cookies + const valid = await this.validate(); + if (!valid) { + console.log('⚠ Cookies invalid, refreshing...'); + await this.refresh(); + } else { + console.log('✅ Cookies valid'); + } + } catch (err) { + console.log('📝 No cookies found, generating fresh cookies...'); + await this.refresh(); + } + + // Setup periodic validation (every 12 hours) + setInterval(() => { + this.checkAndRefresh().catch(err => { + console.error('Auto-check failed:', err.message); + }); + }, this.checkIntervalMs); + + console.log('✅ Cookies manager ready'); + } + + /** + * Validate cookies using Python script. + * @returns {Promise} True if cookies are valid + */ + async validate() { + try { + const { stdout, stderr } = await execAsync( + `${this.pythonPath} ${this.validateScript} ${this.cookiesPath}`, + { timeout: 60000 } + ); + + // Check for validation success in output + this.isValid = stdout.includes('Cookies are valid'); + + if (stderr && !stderr.includes('DeprecationWarning')) { + console.warn('Validation stderr:', stderr.trim()); + } + + return this.isValid; + } catch (err) { + console.error('Validation failed:', err.message); + this.isValid = false; + return false; + } + } + + /** + * Refresh cookies using Camoufox extraction. + * @returns {Promise} True if refresh succeeded + */ + async refresh() { + console.log('🔄 Refreshing YouTube cookies with Camoufox...'); + + try { + const { stdout, stderr } = await execAsync( + `${this.pythonPath} ${this.extractScript} ${this.cookiesPath}`, + { timeout: 120000 } // 2 min timeout + ); + + console.log(stdout.trim()); + + if (stderr && !stderr.includes('DeprecationWarning')) { + console.warn('Camoufox stderr:', stderr.trim()); + } + + // Verify file was created + try { + await fs.access(this.cookiesPath); + this.lastRefresh = Date.now(); + this.isValid = true; + console.log('✅ Cookies refreshed successfully'); + return true; + } catch { + console.error('❌ Cookies file not created'); + this.isValid = false; + return false; + } + } catch (err) { + console.error('❌ Failed to refresh cookies:', err.message); + this.isValid = false; + return false; + } + } + + /** + * Check cookies age and validity, refresh if needed. + */ + async checkAndRefresh() { + console.log('🔍 Checking cookies status...'); + + // Check file age + try { + const stats = await fs.stat(this.cookiesPath); + const ageMs = Date.now() - stats.mtimeMs; + const ageDays = ageMs / (1000 * 60 * 60 * 24); + + console.log(` Age: ${ageDays.toFixed(1)} days`); + + // Refresh if too old + if (ageDays >= this.refreshIntervalDays) { + console.log(` Age threshold (${this.refreshIntervalDays} days) reached, refreshing...`); + await this.refresh(); + return; + } + } catch { + // File doesn't exist + console.log(' Cookies file missing, refreshing...'); + await this.refresh(); + return; + } + + // Validate cookies + const valid = await this.validate(); + if (!valid) { + console.log(' Cookies invalid, refreshing...'); + await this.refresh(); + } else { + console.log(' Cookies OK ✅'); + } + } + + /** + * Get path to cookies file. + * @returns {string} Cookies file path + */ + getCookiesPath() { + return this.cookiesPath; + } + + /** + * Get cookies status. + * @returns {object} Status object + */ + getStatus() { + return { + valid: this.isValid, + path: this.cookiesPath, + lastRefresh: this.lastRefresh, + refreshIntervalDays: this.refreshIntervalDays + }; + } +} + +// Export singleton +module.exports = new CookiesManager(); diff --git a/src/services/download.js b/src/services/download.js new file mode 100644 index 0000000..59a3737 --- /dev/null +++ b/src/services/download.js @@ -0,0 +1,190 @@ +const { spawn } = require('child_process'); +const cookiesManager = require('./cookiesManager'); +const path = require('path'); +const fs = require('fs').promises; + +/** + * YouTube download service using yt-dlp with Camoufox stealth cookies. + */ +class DownloadService { + constructor() { + this.storagePath = process.env.STORAGE_PATH || path.join(__dirname, '../../output'); + this.ytdlpPath = process.env.YTDLP_PATH || 'yt-dlp'; + } + + /** + * Download YouTube video as MP3. + * @param {string} url - YouTube video URL + * @param {object} options - Download options + * @returns {Promise} Download result with metadata + */ + async downloadYouTube(url, options = {}) { + // Ensure storage directory exists + await fs.mkdir(this.storagePath, { recursive: true }); + + // Ensure cookies are valid before download + await cookiesManager.checkAndRefresh(); + + const cookiesPath = cookiesManager.getCookiesPath(); + + // Build yt-dlp arguments + const outputTemplate = path.join(this.storagePath, '%(id)s.%(ext)s'); + const quality = options.quality || '192k'; + + const args = [ + // Cookies (stealth from Camoufox) + '--cookies', cookiesPath, + + // Player client (mweb is stable) + '--extractor-args', 'youtube:player_client=mweb', + + // Format selection (audio only) + '--format', 'bestaudio[ext=m4a]/bestaudio', + + // Audio extraction + '--extract-audio', + '--audio-format', 'mp3', + '--audio-quality', quality, + + // Metadata + '--embed-thumbnail', + '--add-metadata', + + // No playlists (single video only) + '--no-playlist', + + // Output JSON metadata + '--print-json', + + // Output template + '--output', outputTemplate, + + // URL + url + ]; + + return new Promise((resolve, reject) => { + const ytdlp = spawn(this.ytdlpPath, args); + + let jsonOutput = ''; + let errorOutput = ''; + + ytdlp.stdout.on('data', (data) => { + const text = data.toString(); + jsonOutput += text; + }); + + ytdlp.stderr.on('data', (data) => { + const text = data.toString(); + errorOutput += text; + + // Log progress + if (text.includes('[download]') || text.includes('[ExtractAudio]')) { + console.log(' ', text.trim()); + } + }); + + ytdlp.on('close', async (code) => { + if (code === 0) { + try { + // Parse JSON output from yt-dlp + const lines = jsonOutput.split('\n').filter(l => l.trim()); + const lastLine = lines[lines.length - 1]; + const metadata = JSON.parse(lastLine); + + // Extract relevant metadata + const result = { + success: true, + title: metadata.title, + duration: metadata.duration, + artist: metadata.artist || metadata.uploader || metadata.channel, + album: metadata.album || null, + filePath: metadata.filename, + fileName: path.basename(metadata.filename), + fileSize: metadata.filesize || null, + youtubeId: metadata.id, + youtubeUrl: metadata.webpage_url, + thumbnail: metadata.thumbnail, + uploadDate: metadata.upload_date, + description: metadata.description || null + }; + + console.log(`✅ Downloaded: ${result.title}`); + resolve(result); + } catch (err) { + reject(new Error(`Failed to parse yt-dlp output: ${err.message}`)); + } + } else { + // Check for specific errors + if (errorOutput.includes('Sign in to confirm')) { + console.log('đŸ€– Bot detection! Force refreshing cookies...'); + + // Force refresh cookies + await cookiesManager.refresh(); + + // Retry once + try { + console.log('🔄 Retrying download with fresh cookies...'); + const result = await this.downloadYouTube(url, options); + resolve(result); + } catch (retryErr) { + reject(new Error(`Download failed after cookie refresh: ${retryErr.message}`)); + } + } else if (errorOutput.includes('Video unavailable')) { + reject(new Error('Video is unavailable or private')); + } else if (errorOutput.includes('429')) { + reject(new Error('Rate limited by YouTube. Please wait and try again later.')); + } else { + reject(new Error(`yt-dlp failed (code ${code}): ${errorOutput}`)); + } + } + }); + + ytdlp.on('error', (err) => { + reject(new Error(`Failed to spawn yt-dlp: ${err.message}`)); + }); + }); + } + + /** + * Get file stream for a downloaded file. + * @param {string} fileName - File name + * @returns {Promise} File info and stream + */ + async getFileStream(fileName) { + const filePath = path.join(this.storagePath, fileName); + + // Check if file exists + try { + const stats = await fs.stat(filePath); + return { + path: filePath, + size: stats.size, + exists: true + }; + } catch { + throw new Error('File not found'); + } + } + + /** + * Delete a downloaded file. + * @param {string} fileName - File name + * @returns {Promise} True if deleted + */ + async deleteFile(fileName) { + const filePath = path.join(this.storagePath, fileName); + + try { + await fs.unlink(filePath); + console.log(`đŸ—‘ïž Deleted: ${fileName}`); + return true; + } catch (err) { + console.error(`Failed to delete ${fileName}:`, err.message); + return false; + } + } +} + +// Export singleton +module.exports = new DownloadService(); diff --git a/src/services/summarize.js b/src/services/summarize.js deleted file mode 100644 index 44ee794..0000000 --- a/src/services/summarize.js +++ /dev/null @@ -1,195 +0,0 @@ -import OpenAI from 'openai'; -import fs from 'fs'; -import path from 'path'; - -let openai = null; - -// Max characters per chunk for summarization -const MAX_CHUNK_CHARS = 30000; - -/** - * Get OpenAI client (lazy initialization) - */ -function getOpenAI() { - if (!openai) { - if (!process.env.OPENAI_API_KEY) { - throw new Error('OPENAI_API_KEY environment variable is not set'); - } - openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); - } - return openai; -} - -/** - * Summarize text using GPT-4o - */ -export async function summarizeText(text, options = {}) { - const { - model = 'gpt-5.1', // GPT-5.1 - latest OpenAI model (Nov 2025) - language = 'same', // 'same' = same as input, or specify language code - style = 'concise', // 'concise', 'detailed', 'bullet' - maxLength = null, // optional max length in words - } = options; - - const client = getOpenAI(); - - let styleInstruction = ''; - switch (style) { - case 'detailed': - styleInstruction = 'Provide a detailed summary that captures all important points and nuances.'; - break; - case 'bullet': - styleInstruction = 'Provide the summary as bullet points, highlighting the key points.'; - break; - case 'concise': - default: - styleInstruction = 'Provide a concise summary that captures the main points.'; - } - - let languageInstruction = ''; - if (language === 'same') { - languageInstruction = 'Write the summary in the same language as the input text.'; - } else { - languageInstruction = `Write the summary in ${language}.`; - } - - let lengthInstruction = ''; - if (maxLength) { - lengthInstruction = `Keep the summary under ${maxLength} words.`; - } - - const systemPrompt = `You are an expert summarizer. ${styleInstruction} ${languageInstruction} ${lengthInstruction} -Focus on the most important information and main ideas. Be accurate and objective.`; - - // Handle long texts by chunking - if (text.length > MAX_CHUNK_CHARS) { - return await summarizeLongText(text, { model, systemPrompt, style }); - } - - const response = await client.chat.completions.create({ - model, - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: `Please summarize the following text:\n\n${text}` }, - ], - temperature: 0.3, - }); - - return { - summary: response.choices[0].message.content, - model, - style, - inputLength: text.length, - chunks: 1, - }; -} - -/** - * Summarize long text by chunking and combining summaries - */ -async function summarizeLongText(text, options) { - const { model, systemPrompt, style } = options; - const client = getOpenAI(); - - // Split into chunks - const chunks = []; - let currentChunk = ''; - const sentences = text.split(/(?<=[.!?ă€‚ïŒïŒŸ\n])\s*/); - - for (const sentence of sentences) { - if ((currentChunk + sentence).length > MAX_CHUNK_CHARS && currentChunk) { - chunks.push(currentChunk.trim()); - currentChunk = sentence; - } else { - currentChunk += ' ' + sentence; - } - } - if (currentChunk.trim()) { - chunks.push(currentChunk.trim()); - } - - console.log(`Summarizing ${chunks.length} chunks...`); - - // Summarize each chunk - const chunkSummaries = []; - for (let i = 0; i < chunks.length; i++) { - console.log(`[${i + 1}/${chunks.length}] Summarizing chunk...`); - const response = await client.chat.completions.create({ - model, - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: `Please summarize the following text (part ${i + 1} of ${chunks.length}):\n\n${chunks[i]}` }, - ], - temperature: 0.3, - }); - chunkSummaries.push(response.choices[0].message.content); - } - - // Combine summaries if multiple chunks - if (chunkSummaries.length === 1) { - return { - summary: chunkSummaries[0], - model, - style, - inputLength: text.length, - chunks: 1, - }; - } - - // Create final combined summary - const combinedText = chunkSummaries.join('\n\n---\n\n'); - const finalResponse = await client.chat.completions.create({ - model, - messages: [ - { role: 'system', content: `You are an expert summarizer. Combine and synthesize the following partial summaries into a single coherent ${style} summary. Remove redundancy and ensure a smooth flow.` }, - { role: 'user', content: `Please combine these summaries into one:\n\n${combinedText}` }, - ], - temperature: 0.3, - }); - - return { - summary: finalResponse.choices[0].message.content, - model, - style, - inputLength: text.length, - chunks: chunks.length, - }; -} - -/** - * Summarize a text file - */ -export async function summarizeFile(filePath, options = {}) { - if (!fs.existsSync(filePath)) { - throw new Error(`File not found: ${filePath}`); - } - - const { outputDir, ...otherOptions } = options; - - const text = fs.readFileSync(filePath, 'utf-8'); - const result = await summarizeText(text, otherOptions); - - // Save summary to file - const dir = outputDir || path.dirname(filePath); - const baseName = path.basename(filePath, path.extname(filePath)); - const summaryPath = path.join(dir, `${baseName}_summary.txt`); - - fs.writeFileSync(summaryPath, result.summary, 'utf-8'); - - return { - ...result, - filePath, - summaryPath, - }; -} - -/** - * Get available summary styles - */ -export function getSummaryStyles() { - return { - concise: 'A brief summary capturing main points', - detailed: 'A comprehensive summary with nuances', - bullet: 'Key points as bullet points', - }; -} diff --git a/src/services/transcription.js b/src/services/transcription.js deleted file mode 100644 index 61e65f4..0000000 --- a/src/services/transcription.js +++ /dev/null @@ -1,178 +0,0 @@ -import OpenAI from 'openai'; -import fs from 'fs'; -import path from 'path'; - -let openai = null; - -// Available transcription models -const MODELS = { - 'gpt-4o-transcribe': { - name: 'gpt-4o-transcribe', - formats: ['json', 'text'], - supportsLanguage: true, - }, - 'gpt-4o-mini-transcribe': { - name: 'gpt-4o-mini-transcribe', - formats: ['json', 'text'], - supportsLanguage: true, - }, - 'whisper-1': { - name: 'whisper-1', - formats: ['json', 'text', 'srt', 'vtt', 'verbose_json'], - supportsLanguage: true, - }, -}; - -const DEFAULT_MODEL = 'gpt-4o-mini-transcribe'; - -/** - * Get OpenAI client (lazy initialization) - */ -function getOpenAI() { - if (!openai) { - if (!process.env.OPENAI_API_KEY) { - throw new Error('OPENAI_API_KEY environment variable is not set'); - } - openai = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - }); - } - return openai; -} - -/** - * Get available models - */ -export function getAvailableModels() { - return Object.keys(MODELS); -} - -/** - * Transcribe an audio file using OpenAI API - * @param {string} filePath - Path to audio file - * @param {Object} options - Transcription options - * @param {string} options.language - Language code (e.g., 'en', 'fr', 'es', 'zh') - * @param {string} options.responseFormat - Output format: 'json' or 'text' (gpt-4o models), or 'srt'/'vtt' (whisper-1 only) - * @param {string} options.prompt - Optional context prompt for better accuracy - * @param {string} options.model - Model to use (default: gpt-4o-transcribe) - */ -export async function transcribeFile(filePath, options = {}) { - const { - language = null, // Auto-detect if null - responseFormat = 'text', // json or text for gpt-4o models - prompt = null, // Optional context prompt - model = DEFAULT_MODEL, - } = options; - - if (!fs.existsSync(filePath)) { - throw new Error(`File not found: ${filePath}`); - } - - const modelConfig = MODELS[model] || MODELS[DEFAULT_MODEL]; - const actualModel = modelConfig.name; - - // Validate response format for model - let actualFormat = responseFormat; - if (!modelConfig.formats.includes(responseFormat)) { - console.warn(`Format '${responseFormat}' not supported by ${actualModel}, using 'text'`); - actualFormat = 'text'; - } - - try { - const transcriptionOptions = { - file: fs.createReadStream(filePath), - model: actualModel, - response_format: actualFormat, - }; - - if (language) { - transcriptionOptions.language = language; - } - - if (prompt) { - transcriptionOptions.prompt = prompt; - } - - console.log(`Using model: ${actualModel}, format: ${actualFormat}${language ? `, language: ${language}` : ''}`); - - const transcription = await getOpenAI().audio.transcriptions.create(transcriptionOptions); - - return { - success: true, - filePath, - text: actualFormat === 'json' || actualFormat === 'verbose_json' - ? transcription.text - : transcription, - format: actualFormat, - model: actualModel, - }; - } catch (error) { - throw new Error(`Transcription failed: ${error.message}`); - } -} - -/** - * Transcribe and save to file - */ -export async function transcribeAndSave(filePath, options = {}) { - const { outputFormat = 'txt', outputDir = null } = options; - - const result = await transcribeFile(filePath, options); - - // Determine output path - const baseName = path.basename(filePath, path.extname(filePath)); - const outputPath = path.join( - outputDir || path.dirname(filePath), - `${baseName}.${outputFormat}` - ); - - // Save transcription - fs.writeFileSync(outputPath, result.text, 'utf-8'); - - return { - ...result, - transcriptionPath: outputPath, - }; -} - -/** - * Transcribe multiple files - */ -export async function transcribeMultiple(filePaths, options = {}) { - const { onProgress, onFileComplete } = options; - const results = []; - - for (let i = 0; i < filePaths.length; i++) { - const filePath = filePaths[i]; - - if (onProgress) { - onProgress({ current: i + 1, total: filePaths.length, filePath }); - } - - console.log(`[${i + 1}/${filePaths.length}] Transcribing: ${path.basename(filePath)}`); - - try { - const result = await transcribeAndSave(filePath, options); - results.push(result); - - if (onFileComplete) { - onFileComplete(result); - } - } catch (error) { - console.error(`Failed to transcribe ${filePath}: ${error.message}`); - results.push({ - success: false, - filePath, - error: error.message, - }); - } - } - - return { - success: true, - results, - totalFiles: filePaths.length, - successCount: results.filter(r => r.success).length, - failCount: results.filter(r => !r.success).length, - }; -} diff --git a/src/services/translation.js b/src/services/translation.js deleted file mode 100644 index e8fa424..0000000 --- a/src/services/translation.js +++ /dev/null @@ -1,271 +0,0 @@ -import OpenAI from 'openai'; -import fs from 'fs'; -import path from 'path'; - -let openai = null; - -// Max characters per chunk (~6000 tokens ≈ 24000 characters for most languages) -const MAX_CHUNK_CHARS = 20000; - -const LANGUAGES = { - en: 'English', - fr: 'French', - es: 'Spanish', - de: 'German', - it: 'Italian', - pt: 'Portuguese', - zh: 'Chinese', - ja: 'Japanese', - ko: 'Korean', - ru: 'Russian', - ar: 'Arabic', - hi: 'Hindi', - nl: 'Dutch', - pl: 'Polish', - tr: 'Turkish', - vi: 'Vietnamese', - th: 'Thai', - sv: 'Swedish', - da: 'Danish', - fi: 'Finnish', - no: 'Norwegian', - cs: 'Czech', - el: 'Greek', - he: 'Hebrew', - id: 'Indonesian', - ms: 'Malay', - ro: 'Romanian', - uk: 'Ukrainian', -}; - -// Sentence ending patterns for different languages -const SENTENCE_ENDINGS = /[.!?ă€‚ïŒïŒŸïœĄ\n]/g; - -/** - * Get OpenAI client (lazy initialization) - */ -function getOpenAI() { - if (!openai) { - if (!process.env.OPENAI_API_KEY) { - throw new Error('OPENAI_API_KEY environment variable is not set'); - } - openai = new OpenAI({ - apiKey: process.env.OPENAI_API_KEY, - }); - } - return openai; -} - -/** - * Split text into chunks at sentence boundaries - * @param {string} text - Text to split - * @param {number} maxChars - Maximum characters per chunk - * @returns {string[]} Array of text chunks - */ -function splitIntoChunks(text, maxChars = MAX_CHUNK_CHARS) { - if (text.length <= maxChars) { - return [text]; - } - - const chunks = []; - let currentPos = 0; - - while (currentPos < text.length) { - let endPos = currentPos + maxChars; - - // If we're at the end, just take the rest - if (endPos >= text.length) { - chunks.push(text.slice(currentPos)); - break; - } - - // Find the last sentence ending before maxChars - const searchText = text.slice(currentPos, endPos); - let lastSentenceEnd = -1; - - // Find all sentence endings in the search range - let match; - SENTENCE_ENDINGS.lastIndex = 0; - while ((match = SENTENCE_ENDINGS.exec(searchText)) !== null) { - lastSentenceEnd = match.index + 1; // Include the punctuation - } - - // If we found a sentence ending, cut there - // Otherwise, look for the next sentence ending after maxChars (up to 20% more) - if (lastSentenceEnd > maxChars * 0.5) { - endPos = currentPos + lastSentenceEnd; - } else { - // Look forward for a sentence ending (up to 20% more characters) - const extendedSearch = text.slice(endPos, endPos + maxChars * 0.2); - SENTENCE_ENDINGS.lastIndex = 0; - const forwardMatch = SENTENCE_ENDINGS.exec(extendedSearch); - if (forwardMatch) { - endPos = endPos + forwardMatch.index + 1; - } - // If still no sentence ending found, just cut at maxChars - } - - chunks.push(text.slice(currentPos, endPos).trim()); - currentPos = endPos; - - // Skip any leading whitespace for the next chunk - while (currentPos < text.length && /\s/.test(text[currentPos])) { - currentPos++; - } - } - - return chunks.filter(chunk => chunk.length > 0); -} - -/** - * Get available languages - */ -export function getLanguages() { - return LANGUAGES; -} - -/** - * Translate a single chunk of text - */ -async function translateChunk(text, targetLanguage, sourceLanguage) { - const prompt = sourceLanguage - ? `Translate the following text from ${sourceLanguage} to ${targetLanguage}. Only output the translation, nothing else:\n\n${text}` - : `Translate the following text to ${targetLanguage}. Only output the translation, nothing else:\n\n${text}`; - - const response = await getOpenAI().chat.completions.create({ - model: 'gpt-4o-mini', - max_tokens: 16384, - messages: [ - { - role: 'user', - content: prompt, - }, - ], - }); - - return response.choices[0].message.content; -} - -/** - * Translate text using GPT-4o-mini with chunking for long texts - * @param {string} text - Text to translate - * @param {string} targetLang - Target language code (e.g., 'en', 'fr') - * @param {string} sourceLang - Source language code (optional, auto-detect if null) - */ -export async function translateText(text, targetLang, sourceLang = null) { - if (!text || !text.trim()) { - throw new Error('No text provided for translation'); - } - - const targetLanguage = LANGUAGES[targetLang] || targetLang; - const sourceLanguage = sourceLang ? (LANGUAGES[sourceLang] || sourceLang) : null; - - try { - // Split text into chunks - const chunks = splitIntoChunks(text); - - if (chunks.length === 1) { - // Single chunk - translate directly - const translation = await translateChunk(text, targetLanguage, sourceLanguage); - return { - success: true, - originalText: text, - translatedText: translation, - targetLanguage: targetLanguage, - sourceLanguage: sourceLanguage || 'auto-detected', - chunks: 1, - }; - } - - // Multiple chunks - translate each and combine - console.log(`Splitting text into ${chunks.length} chunks for translation...`); - const translations = []; - - for (let i = 0; i < chunks.length; i++) { - console.log(` Translating chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)...`); - const translation = await translateChunk(chunks[i], targetLanguage, sourceLanguage); - translations.push(translation); - } - - const combinedTranslation = translations.join('\n\n'); - - return { - success: true, - originalText: text, - translatedText: combinedTranslation, - targetLanguage: targetLanguage, - sourceLanguage: sourceLanguage || 'auto-detected', - chunks: chunks.length, - }; - } catch (error) { - throw new Error(`Translation failed: ${error.message}`); - } -} - -/** - * Translate a text file - * @param {string} filePath - Path to text file - * @param {string} targetLang - Target language code - * @param {string} sourceLang - Source language code (optional) - * @param {string} outputDir - Output directory (optional) - */ -export async function translateFile(filePath, targetLang, sourceLang = null, outputDir = null) { - if (!fs.existsSync(filePath)) { - throw new Error(`File not found: ${filePath}`); - } - - const text = fs.readFileSync(filePath, 'utf-8'); - const result = await translateText(text, targetLang, sourceLang); - - // Save translation - const baseName = path.basename(filePath, path.extname(filePath)); - const outputPath = path.join( - outputDir || path.dirname(filePath), - `${baseName}_${targetLang}.txt` - ); - - fs.writeFileSync(outputPath, result.translatedText, 'utf-8'); - - return { - ...result, - originalPath: filePath, - translationPath: outputPath, - }; -} - -/** - * Translate multiple files - */ -export async function translateMultiple(filePaths, targetLang, sourceLang = null, outputDir = null, onProgress = null) { - const results = []; - - for (let i = 0; i < filePaths.length; i++) { - const filePath = filePaths[i]; - - if (onProgress) { - onProgress({ current: i + 1, total: filePaths.length, filePath }); - } - - console.log(`[${i + 1}/${filePaths.length}] Translating: ${path.basename(filePath)}`); - - try { - const result = await translateFile(filePath, targetLang, sourceLang, outputDir); - results.push(result); - } catch (error) { - console.error(`Failed to translate ${filePath}: ${error.message}`); - results.push({ - success: false, - originalPath: filePath, - error: error.message, - }); - } - } - - return { - success: true, - results, - totalFiles: filePaths.length, - successCount: results.filter(r => r.success).length, - failCount: results.filter(r => !r.success).length, - }; -} diff --git a/src/services/youtube.js b/src/services/youtube.js deleted file mode 100644 index 79d7963..0000000 --- a/src/services/youtube.js +++ /dev/null @@ -1,383 +0,0 @@ -import { createRequire } from 'module'; -import path from 'path'; -import fs from 'fs'; -import { spawn } from 'child_process'; - -// Use system yt-dlp binary (check common paths) -const YTDLP_PATH = process.env.YTDLP_PATH || 'yt-dlp'; - -// Path to cookies file (optional) -const COOKIES_PATH = process.env.YOUTUBE_COOKIES_PATH || null; - -// Browser to extract cookies from (chrome, firefox, edge, safari, etc.) -const COOKIES_BROWSER = process.env.YOUTUBE_COOKIES_BROWSER || null; - -/** - * Enhanced error message for YouTube bot detection - */ -function enhanceYouTubeError(error) { - const errorMsg = error.message || error.toString(); - - // Check if it's a bot detection error - if (errorMsg.includes('Sign in to confirm') || - errorMsg.includes('not a bot') || - errorMsg.includes('confirm you\'re not a bot') || - errorMsg.includes('ERROR: Unable to extract')) { - - const cookiesConfigured = COOKIES_BROWSER || COOKIES_PATH; - - return { - error: 'YouTube Bot Detection', - message: 'YouTube is blocking this request. Authentication required.', - reason: errorMsg, - solution: { - quick: 'Upload fresh cookies from your browser', - steps: [ - '1. Install browser extension: "Get cookies.txt LOCALLY"', - '2. Visit youtube.com and log into your account', - '3. Export cookies using the extension', - '4. Upload via API: POST /admin/upload-cookies', - ' Or use the web interface at http://yourserver:8888', - ], - alternative: 'Use extract-and-upload-cookies.sh script for automation', - documentation: 'See COOKIES_QUICK_START.md for detailed instructions' - }, - currentConfig: { - cookiesFile: COOKIES_PATH || 'Not configured', - cookiesBrowser: COOKIES_BROWSER || 'Not configured', - status: cookiesConfigured ? '⚠ Configured but may be expired' : '❌ Not configured' - } - }; - } - - // Generic YouTube error - return { - error: 'YouTube Download Failed', - message: errorMsg, - solution: 'Check if the URL is valid and the video is available' - }; -} - -/** - * Add cookies argument - prioritizes live browser cookies over file - */ -function addCookiesArg(args, cookiesPath = null) { - // Option 1: Extract cookies from browser (always fresh) - if (COOKIES_BROWSER) { - console.log(`Using live cookies from ${COOKIES_BROWSER} browser`); - return ['--cookies-from-browser', COOKIES_BROWSER, ...args]; - } - - // Option 2: Use static cookies file (may expire) - // Check dynamically in case cookies were uploaded after server started - const cookies = cookiesPath || process.env.YOUTUBE_COOKIES_PATH || COOKIES_PATH; - if (cookies && fs.existsSync(cookies)) { - console.log(`Using cookies file: ${cookies}`); - return ['--cookies', cookies, ...args]; - } - - // Option 3: No cookies (may fail on some videos) - console.log('No cookies configured - some videos may fail'); - return args; -} - -/** - * Execute yt-dlp command and return parsed JSON - */ -async function ytdlp(url, args = [], options = {}) { - const { cookiesPath } = options; - const finalArgs = addCookiesArg(args, cookiesPath); - return new Promise((resolve, reject) => { - const proc = spawn(YTDLP_PATH, [...finalArgs, url]); - let stdout = ''; - let stderr = ''; - - proc.stdout.on('data', (data) => { stdout += data; }); - proc.stderr.on('data', (data) => { stderr += data; }); - - proc.on('close', (code) => { - if (code === 0) { - try { - resolve(JSON.parse(stdout)); - } catch { - resolve(stdout); - } - } else { - reject(new Error(stderr || `yt-dlp exited with code ${code}`)); - } - }); - }); -} - -/** - * Execute yt-dlp command with progress callback - */ -function ytdlpExec(url, args = [], onProgress, options = {}) { - const { cookiesPath } = options; - const finalArgs = addCookiesArg(args, cookiesPath); - return new Promise((resolve, reject) => { - const proc = spawn(YTDLP_PATH, [...finalArgs, url]); - let stderr = ''; - - proc.stdout.on('data', (data) => { - const line = data.toString(); - if (onProgress) { - const progressMatch = line.match(/\[download\]\s+(\d+\.?\d*)%/); - const etaMatch = line.match(/ETA\s+(\d+:\d+)/); - const speedMatch = line.match(/at\s+([\d.]+\w+\/s)/); - - if (progressMatch) { - onProgress({ - percent: parseFloat(progressMatch[1]), - eta: etaMatch ? etaMatch[1] : null, - speed: speedMatch ? speedMatch[1] : null, - }); - } - } - }); - - proc.stderr.on('data', (data) => { stderr += data; }); - - proc.on('close', (code) => { - if (code === 0) { - resolve(); - } else { - reject(new Error(stderr || `yt-dlp exited with code ${code}`)); - } - }); - }); -} - -const OUTPUT_DIR = process.env.OUTPUT_DIR || './output'; - -/** - * Sanitize filename to remove invalid characters - */ -function sanitizeFilename(filename) { - return filename - .replace(/[<>:"/\\|?*]/g, '') - .replace(/\s+/g, '_') - .substring(0, 200); -} - -/** - * Check if URL contains a playlist parameter - */ -function hasPlaylistParam(url) { - try { - const urlObj = new URL(url); - return urlObj.searchParams.has('list'); - } catch { - return false; - } -} - -/** - * Extract playlist URL if present in the URL - */ -function extractPlaylistUrl(url) { - const urlObj = new URL(url); - const listId = urlObj.searchParams.get('list'); - if (listId) { - return `https://www.youtube.com/playlist?list=${listId}`; - } - return null; -} - -/** - * Get video/playlist info without downloading - */ -export async function getInfo(url, forcePlaylist = false, options = {}) { - try { - // If URL contains a playlist ID and we want to force playlist mode - const playlistUrl = extractPlaylistUrl(url); - const targetUrl = (forcePlaylist && playlistUrl) ? playlistUrl : url; - - const info = await ytdlp(targetUrl, [ - '--dump-single-json', - '--no-download', - '--no-warnings', - '--flat-playlist', - ], options); - return info; - } catch (error) { - const enhancedError = enhanceYouTubeError(error); - const err = new Error(JSON.stringify(enhancedError)); - err.isEnhanced = true; - err.details = enhancedError; - throw err; - } -} - -/** - * Check if URL is a playlist - */ -export async function isPlaylist(url) { - const info = await getInfo(url); - return info._type === 'playlist'; -} - -/** - * Download a single video as MP3 - */ -export async function downloadVideo(url, options = {}) { - const { outputDir = OUTPUT_DIR, onProgress, onDownloadProgress, cookiesPath } = options; - - // Ensure output directory exists - if (!fs.existsSync(outputDir)) { - fs.mkdirSync(outputDir, { recursive: true }); - } - - try { - // Get video info first - const info = await ytdlp(url, [ - '--dump-single-json', - '--no-download', - '--no-warnings', - ], { cookiesPath }); - - const title = sanitizeFilename(info.title); - const outputPath = path.join(outputDir, `${title}.mp3`); - - // Download and convert to MP3 with progress - await ytdlpExec(url, [ - '--extract-audio', - '--audio-format', 'mp3', - '--audio-quality', '0', - '-o', outputPath, - '--no-warnings', - '--newline', - ], (progress) => { - if (onDownloadProgress) { - onDownloadProgress({ - ...progress, - title: info.title, - }); - } - }, { cookiesPath }); - - return { - success: true, - title: info.title, - duration: info.duration, - filePath: outputPath, - url: url, - }; - } catch (error) { - const enhancedError = enhanceYouTubeError(error); - const err = new Error(JSON.stringify(enhancedError)); - err.isEnhanced = true; - err.details = enhancedError; - throw err; - } -} - -/** - * Download all videos from a playlist as MP3 - */ -export async function downloadPlaylist(url, options = {}) { - const { outputDir = OUTPUT_DIR, onProgress, onVideoComplete, onDownloadProgress, forcePlaylist = false, cookiesPath } = options; - - // Ensure output directory exists - if (!fs.existsSync(outputDir)) { - fs.mkdirSync(outputDir, { recursive: true }); - } - - try { - // Get playlist info (force playlist mode if URL has list= param) - const info = await getInfo(url, forcePlaylist || hasPlaylistParam(url), { cookiesPath }); - - if (info._type !== 'playlist') { - // Single video, redirect to downloadVideo - const result = await downloadVideo(url, { ...options, onDownloadProgress }); - return { - success: true, - playlistTitle: result.title, - videos: [result], - totalVideos: 1, - }; - } - - const results = []; - const entries = info.entries || []; - - console.log(`Playlist: ${info.title} (${entries.length} videos)`); - - for (let i = 0; i < entries.length; i++) { - const entry = entries[i]; - const videoUrl = entry.url || `https://www.youtube.com/watch?v=${entry.id}`; - - try { - if (onProgress) { - onProgress({ current: i + 1, total: entries.length, title: entry.title }); - } - - console.log(`[${i + 1}/${entries.length}] Downloading: ${entry.title}`); - - // Wrap progress callback to include playlist context - const wrappedProgress = onDownloadProgress ? (progress) => { - onDownloadProgress({ - ...progress, - videoIndex: i + 1, - totalVideos: entries.length, - playlistTitle: info.title, - }); - } : undefined; - - const result = await downloadVideo(videoUrl, { outputDir, onDownloadProgress: wrappedProgress, cookiesPath }); - results.push(result); - - if (onVideoComplete) { - onVideoComplete(result); - } - } catch (error) { - console.error(`Failed to download ${entry.title}: ${error.message}`); - results.push({ - success: false, - title: entry.title, - url: videoUrl, - error: error.message, - }); - } - } - - return { - success: true, - playlistTitle: info.title, - videos: results, - totalVideos: entries.length, - successCount: results.filter(r => r.success).length, - failCount: results.filter(r => !r.success).length, - }; - } catch (error) { - const enhancedError = enhanceYouTubeError(error); - const err = new Error(JSON.stringify(enhancedError)); - err.isEnhanced = true; - err.details = enhancedError; - throw err; - } -} - -/** - * Smart download - detects if URL is video or playlist - */ -export async function download(url, options = {}) { - const { cookiesPath } = options; - // If URL contains list= parameter, treat it as a playlist - const isPlaylistUrl = hasPlaylistParam(url); - const info = await getInfo(url, isPlaylistUrl, { cookiesPath }); - - if (info._type === 'playlist') { - return downloadPlaylist(url, { ...options, forcePlaylist: true }); - } else { - const result = await downloadVideo(url, options); - return { - success: true, - playlistTitle: null, - videos: [result], - totalVideos: 1, - successCount: 1, - failCount: 0, - }; - } -}