From ea0caa9349d5bac1afec67ab601cb09b1273228a Mon Sep 17 00:00:00 2001 From: Debian User Date: Fri, 5 Dec 2025 13:32:35 +0000 Subject: [PATCH] Add YouTube cookies support to bypass bot detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add cookies support in youtube.js service - Create extract-cookies.sh script for automatic cookie extraction - Add YOUTUBE_COOKIES_PATH environment variable - Update .env.example with cookies configuration - Add comprehensive documentation (YOUTUBE_COOKIES.md, COOKIES_QUICK_START.md) - Update .gitignore to exclude cookies files for security - Pass cookiesPath option through all download functions This fixes Sign in to confirm you're not a bot errors from YouTube. đŸ€– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .env.example | 5 ++ .gitignore | 4 ++ COOKIES_QUICK_START.md | 109 ++++++++++++++++++++++++++++++ FILE_SHARING.md | 94 ++++++++++++++++++++++++++ docs/YOUTUBE_COOKIES.md | 132 +++++++++++++++++++++++++++++++++++++ scripts/extract-cookies.sh | 124 ++++++++++++++++++++++++++++++++++ src/services/youtube.js | 45 +++++++++---- 7 files changed, 500 insertions(+), 13 deletions(-) create mode 100644 COOKIES_QUICK_START.md create mode 100644 FILE_SHARING.md create mode 100644 docs/YOUTUBE_COOKIES.md create mode 100755 scripts/extract-cookies.sh diff --git a/.env.example b/.env.example index 7da55bc..0d86ca2 100644 --- a/.env.example +++ b/.env.example @@ -9,3 +9,8 @@ PORT=3000 # Output directory (optional, default: ./output) OUTPUT_DIR=./output + +# YouTube cookies file path (optional, helps bypass bot detection) +# Run: bash scripts/extract-cookies.sh +# Then set the path to your cookies file: +YOUTUBE_COOKIES_PATH=./youtube-cookies.txt diff --git a/.gitignore b/.gitignore index 879b489..ed0bcdc 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,10 @@ output/ # Text/transcription files *.txt +# YouTube cookies (contains sensitive authentication data) +*cookies*.txt +cookies.txt + # Logs *.log npm-debug.log* diff --git a/COOKIES_QUICK_START.md b/COOKIES_QUICK_START.md new file mode 100644 index 0000000..1464410 --- /dev/null +++ b/COOKIES_QUICK_START.md @@ -0,0 +1,109 @@ +# Quick Start: Fix YouTube "Sign in to confirm you're not a bot" + +## The Problem + +YouTube blocks automated downloads with this error: +``` +Sign in to confirm you're not a bot +``` + +## The Solution + +Use cookies from your browser to authenticate! + +## Step-by-Step Instructions + +### 1. Extract Cookies (Choose ONE method) + +#### Method A: Automatic (Chrome/Firefox on same machine) + +```bash +cd /home/debian/videotomp3transcriptor +bash scripts/extract-cookies.sh +``` + +Follow the prompts and choose your browser. + +#### Method B: Manual (Any browser, any machine) + +1. **Install browser extension:** + - Chrome/Edge: [Get cookies.txt LOCALLY](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) + - Firefox: [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) + +2. **Go to YouTube and log in:** + - Visit https://www.youtube.com + - Make sure you're logged into your account + +3. **Export cookies:** + - Click the extension icon + - Click "Export" or "Download" + - Save the file as `youtube-cookies.txt` + +4. **Upload to server:** + ```bash + # On your local machine + scp youtube-cookies.txt debian@toMP3.etheryale.com:/home/debian/videotomp3transcriptor/ + ``` + +### 2. Configure the API + +Edit `.env` file: + +```bash +nano /home/debian/videotomp3transcriptor/.env +``` + +Add or uncomment this line: +```bash +YOUTUBE_COOKIES_PATH=/home/debian/videotomp3transcriptor/youtube-cookies.txt +``` + +Save and exit (Ctrl+X, then Y, then Enter) + +### 3. Restart the API + +```bash +pm2 restart toMP3-api +``` + +### 4. Test It + +```bash +curl -X POST http://localhost:3001/download \ + -H "Content-Type: application/json" \ + -d '{"url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ"}' +``` + +If it works, you'll see download progress! 🎉 + +## Common Issues + +### "Permission denied" when reading cookies + +```bash +chmod 600 /home/debian/videotomp3transcriptor/youtube-cookies.txt +``` + +### Cookies expired + +Re-export fresh cookies from your browser. Cookies typically last a few weeks. + +### Still getting "not a bot" error + +1. Make sure you logged into YouTube before exporting cookies +2. Try a different browser +3. Check the path in `.env` is correct +4. Verify the file exists: `ls -la /home/debian/videotomp3transcriptor/youtube-cookies.txt` + +## Security Warning ⚠ + +**Cookies = Your YouTube Session** + +- Keep cookies file secure (like a password) +- Never commit to git (already in .gitignore) +- Don't share publicly +- Re-export periodically when they expire + +## Need More Help? + +See full documentation: [docs/YOUTUBE_COOKIES.md](docs/YOUTUBE_COOKIES.md) diff --git a/FILE_SHARING.md b/FILE_SHARING.md new file mode 100644 index 0000000..24c9218 --- /dev/null +++ b/FILE_SHARING.md @@ -0,0 +1,94 @@ +# Serveur de partage de fichiers temporaire + +## DĂ©marrage rapide + +Pour hĂ©berger des fichiers temporairement et les partager via un lien : + +```bash +# CrĂ©er le dossier et dĂ©marrer le serveur +mkdir -p /tmp/share && cd /tmp/share && python3 -m http.server 8000 +``` + +Ou en arriĂšre-plan : +```bash +mkdir -p /tmp/share +cd /tmp/share +python3 -m http.server 8000 & +``` + +## Utilisation + +### 1. Ajouter des fichiers Ă  partager +```bash +cp votre-fichier.ext /tmp/share/ +``` + +### 2. Partager le lien +Le fichier sera accessible Ă  : +``` +http://localhost:8000/votre-fichier.ext +``` + +Pour un accĂšs externe (si le serveur est exposĂ©) : +``` +http://VOTRE_IP:8000/votre-fichier.ext +``` + +### 3. Lister les fichiers disponibles +AccĂ©dez simplement Ă  : +``` +http://localhost:8000/ +``` + +### 4. Supprimer un fichier aprĂšs tĂ©lĂ©chargement +```bash +rm /tmp/share/nom-du-fichier.ext +``` + +## ArrĂȘter le serveur + +Si lancĂ© en arriĂšre-plan, trouver le processus et l'arrĂȘter : +```bash +# Trouver le processus +ps aux | grep "http.server" + +# Tuer le processus (remplacer PID par le numĂ©ro du processus) +kill PID +``` + +Ou plus simplement : +```bash +pkill -f "http.server" +``` + +## Nettoyer complĂštement + +```bash +# ArrĂȘter le serveur +pkill -f "http.server" + +# Supprimer tous les fichiers +rm -rf /tmp/share +``` + +## SĂ©curitĂ© + +⚠ **Attention** : Ce serveur est ultra simple et ne doit ĂȘtre utilisĂ© que pour du partage temporaire : +- Pas d'authentification +- Pas de HTTPS +- Accessible Ă  tous ceux qui ont l'URL +- À utiliser uniquement sur des rĂ©seaux de confiance +- Supprimer les fichiers aprĂšs tĂ©lĂ©chargement + +## Alternative : Utiliser le serveur du projet + +Le serveur principal du projet peut aussi servir des fichiers : +- Endpoint : `GET /files/` +- Port : 3001 (configurĂ© dans .env) +- Les fichiers doivent ĂȘtre dans le dossier `./output/` + +Exemple : +```bash +cp fichier.mp3 ./output/ +# Accessible Ă  : http://localhost:3001/files/fichier.mp3 +``` diff --git a/docs/YOUTUBE_COOKIES.md b/docs/YOUTUBE_COOKIES.md new file mode 100644 index 0000000..445fb1e --- /dev/null +++ b/docs/YOUTUBE_COOKIES.md @@ -0,0 +1,132 @@ +# YouTube Cookies Setup Guide + +## Why Do I Need Cookies? + +YouTube has anti-bot protections that may block yt-dlp requests. Using cookies from your browser allows yt-dlp to authenticate as if you're logged in, bypassing these restrictions. + +## Quick Start + +### Option 1: Automatic Extraction (Recommended) + +Run the helper script: + +```bash +bash scripts/extract-cookies.sh +``` + +Follow the prompts to extract cookies from Chrome or Firefox. + +### Option 2: Using yt-dlp Directly + +```bash +# For Chrome/Chromium +yt-dlp --cookies-from-browser chrome --cookies youtube-cookies.txt 'https://www.youtube.com' + +# For Firefox +yt-dlp --cookies-from-browser firefox --cookies youtube-cookies.txt 'https://www.youtube.com' + +# For Edge +yt-dlp --cookies-from-browser edge --cookies youtube-cookies.txt 'https://www.youtube.com' +``` + +### Option 3: Browser Extension + +1. Install a cookies export extension: + - **Chrome/Edge**: [Get cookies.txt LOCALLY](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) + - **Firefox**: [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) + +2. Go to [youtube.com](https://www.youtube.com) and log in + +3. Click the extension icon and export cookies + +4. Save the file as `youtube-cookies.txt` in your project directory + +## Configuration + +After extracting cookies, update your `.env` file: + +```bash +YOUTUBE_COOKIES_PATH=/home/debian/videotomp3transcriptor/youtube-cookies.txt +``` + +Or use a relative path: + +```bash +YOUTUBE_COOKIES_PATH=./youtube-cookies.txt +``` + +## Verifying It Works + +Test with a video: + +```bash +curl -X POST http://localhost:3001/download \ + -H "Content-Type: application/json" \ + -d '{"url": "https://www.youtube.com/watch?v=dQw4w9WgXcQ"}' +``` + +If it works without cookies errors, you're good to go! + +## Security Notes + +⚠ **IMPORTANT**: + +1. **Never commit cookies to git**: The `.gitignore` file should already exclude `youtube-cookies.txt` +2. **Keep cookies secure**: They provide access to your YouTube account +3. **Cookies expire**: You may need to re-export them periodically (typically every few weeks/months) +4. **Don't share cookies**: Treat them like passwords + +## Troubleshooting + +### "Sign in to confirm you're not a bot" + +This usually means: +- Cookies are not being used +- Cookies have expired +- Cookies file path is incorrect + +**Solutions**: +1. Check the path in `.env` is correct and absolute +2. Re-export fresh cookies +3. Verify the cookies file exists: `ls -la youtube-cookies.txt` +4. Check logs: `pm2 logs toMP3-api` + +### "HTTP Error 403: Forbidden" + +YouTube is blocking your IP or the video is region-restricted. + +**Solutions**: +1. Try with fresh cookies +2. Use a VPN if region-restricted +3. Wait a bit if rate-limited + +### Cookies Not Working + +1. Make sure you're logged into YouTube in the browser before extracting +2. Try extracting from a different browser +3. Verify the cookies file format (should be Netscape format) +4. Check file permissions: `chmod 600 youtube-cookies.txt` + +## Cookie File Format + +The cookies file should be in Netscape format and look like this: + +``` +# Netscape HTTP Cookie File +.youtube.com TRUE / TRUE 1234567890 CONSENT YES+ +.youtube.com TRUE / FALSE 1234567890 VISITOR_INFO1_LIVE xxxxx +``` + +## Without Cookies + +The API will still work for many videos without cookies, but you may encounter: +- "Sign in to confirm you're not a bot" errors +- Rate limiting +- Blocked downloads for certain videos + +For best results, always use cookies! + +## Additional Resources + +- [yt-dlp Cookie Documentation](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp) +- [Browser Cookie Extraction](https://github.com/yt-dlp/yt-dlp#:~:text=You%20can%20use%20cookies%20from%20your%20browser) diff --git a/scripts/extract-cookies.sh b/scripts/extract-cookies.sh new file mode 100755 index 0000000..bda272e --- /dev/null +++ b/scripts/extract-cookies.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +# YouTube Cookies Extractor Script +# This script helps you extract cookies from your browser for yt-dlp + +echo "==========================================" +echo "YouTube Cookies Extractor" +echo "==========================================" +echo "" +echo "This script will help you extract cookies from your browser" +echo "to bypass YouTube's bot detection." +echo "" + +COOKIES_FILE="./youtube-cookies.txt" + +echo "Options:" +echo "" +echo "1. Extract from Chrome/Chromium" +echo "2. Extract from Firefox" +echo "3. Manual instructions (export from browser extension)" +echo "" +read -p "Choose an option (1-3): " choice + +case $choice in + 1) + echo "" + echo "Extracting cookies from Chrome/Chromium..." + echo "" + + # Check if yt-dlp can extract cookies + if command -v yt-dlp &> /dev/null; then + echo "Using yt-dlp to extract cookies from Chrome..." + yt-dlp --cookies-from-browser chrome --cookies "$COOKIES_FILE" --print-to-file webpage_url - "https://www.youtube.com" 2>/dev/null + + if [ -f "$COOKIES_FILE" ]; then + echo "✓ Cookies extracted successfully to: $COOKIES_FILE" + echo "" + echo "Add this to your .env file:" + echo "YOUTUBE_COOKIES_PATH=$(realpath $COOKIES_FILE)" + else + echo "✗ Failed to extract cookies automatically." + echo "You may need to export cookies manually (see option 3)" + fi + else + echo "✗ yt-dlp not found. Please install it first:" + echo " pip install yt-dlp" + fi + ;; + + 2) + echo "" + echo "Extracting cookies from Firefox..." + echo "" + + if command -v yt-dlp &> /dev/null; then + echo "Using yt-dlp to extract cookies from Firefox..." + yt-dlp --cookies-from-browser firefox --cookies "$COOKIES_FILE" --print-to-file webpage_url - "https://www.youtube.com" 2>/dev/null + + if [ -f "$COOKIES_FILE" ]; then + echo "✓ Cookies extracted successfully to: $COOKIES_FILE" + echo "" + echo "Add this to your .env file:" + echo "YOUTUBE_COOKIES_PATH=$(realpath $COOKIES_FILE)" + else + echo "✗ Failed to extract cookies automatically." + echo "You may need to export cookies manually (see option 3)" + fi + else + echo "✗ yt-dlp not found. Please install it first:" + echo " pip install yt-dlp" + fi + ;; + + 3) + echo "" + echo "==========================================" + echo "Manual Cookie Export Instructions" + echo "==========================================" + echo "" + echo "Method 1: Using a browser extension (recommended)" + echo " 1. Install one of these browser extensions:" + echo " - Chrome: 'Get cookies.txt LOCALLY' or 'cookies.txt'" + echo " - Firefox: 'cookies.txt'" + echo "" + echo " 2. Go to https://www.youtube.com and log in" + echo "" + echo " 3. Click on the extension icon and export cookies as 'youtube-cookies.txt'" + echo "" + echo " 4. Save the file to: $(realpath .)/youtube-cookies.txt" + echo "" + echo "Method 2: Using yt-dlp directly" + echo " Run this command:" + echo " yt-dlp --cookies-from-browser chrome --cookies youtube-cookies.txt 'https://www.youtube.com'" + echo "" + echo " (Replace 'chrome' with 'firefox', 'edge', 'safari', etc. if needed)" + echo "" + echo "Method 3: Using browser DevTools (advanced)" + echo " 1. Go to https://www.youtube.com and log in" + echo " 2. Open DevTools (F12)" + echo " 3. Go to Application/Storage tab" + echo " 4. Find Cookies > https://www.youtube.com" + echo " 5. Export all cookies manually to Netscape format" + echo "" + echo "After creating the file, add this to your .env:" + echo "YOUTUBE_COOKIES_PATH=$(realpath .)/youtube-cookies.txt" + echo "" + ;; + + *) + echo "Invalid option" + exit 1 + ;; +esac + +echo "" +echo "==========================================" +echo "" +echo "IMPORTANT: Keep your cookies file secure!" +echo "Do NOT share it or commit it to git." +echo "The cookies give access to your YouTube account." +echo "" +echo "The cookies will expire eventually, so you may need" +echo "to re-export them periodically." +echo "" diff --git a/src/services/youtube.js b/src/services/youtube.js index 5948d9a..ef9681b 100644 --- a/src/services/youtube.js +++ b/src/services/youtube.js @@ -6,12 +6,28 @@ import { spawn } from 'child_process'; // Use system yt-dlp binary (check common paths) const YTDLP_PATH = process.env.YTDLP_PATH || 'yt-dlp'; +// Path to cookies file (optional) +const COOKIES_PATH = process.env.YOUTUBE_COOKIES_PATH || null; + +/** + * Add cookies argument if cookies file exists + */ +function addCookiesArg(args, cookiesPath = null) { + const cookies = cookiesPath || COOKIES_PATH; + if (cookies && fs.existsSync(cookies)) { + return ['--cookies', cookies, ...args]; + } + return args; +} + /** * Execute yt-dlp command and return parsed JSON */ -async function ytdlp(url, args = []) { +async function ytdlp(url, args = [], options = {}) { + const { cookiesPath } = options; + const finalArgs = addCookiesArg(args, cookiesPath); return new Promise((resolve, reject) => { - const proc = spawn(YTDLP_PATH, [...args, url]); + const proc = spawn(YTDLP_PATH, [...finalArgs, url]); let stdout = ''; let stderr = ''; @@ -35,9 +51,11 @@ async function ytdlp(url, args = []) { /** * Execute yt-dlp command with progress callback */ -function ytdlpExec(url, args = [], onProgress) { +function ytdlpExec(url, args = [], onProgress, options = {}) { + const { cookiesPath } = options; + const finalArgs = addCookiesArg(args, cookiesPath); return new Promise((resolve, reject) => { - const proc = spawn(YTDLP_PATH, [...args, url]); + const proc = spawn(YTDLP_PATH, [...finalArgs, url]); let stderr = ''; proc.stdout.on('data', (data) => { @@ -108,7 +126,7 @@ function extractPlaylistUrl(url) { /** * Get video/playlist info without downloading */ -export async function getInfo(url, forcePlaylist = false) { +export async function getInfo(url, forcePlaylist = false, options = {}) { try { // If URL contains a playlist ID and we want to force playlist mode const playlistUrl = extractPlaylistUrl(url); @@ -119,7 +137,7 @@ export async function getInfo(url, forcePlaylist = false) { '--no-download', '--no-warnings', '--flat-playlist', - ]); + ], options); return info; } catch (error) { throw new Error(`Failed to get info: ${error.message}`); @@ -138,7 +156,7 @@ export async function isPlaylist(url) { * Download a single video as MP3 */ export async function downloadVideo(url, options = {}) { - const { outputDir = OUTPUT_DIR, onProgress, onDownloadProgress } = options; + const { outputDir = OUTPUT_DIR, onProgress, onDownloadProgress, cookiesPath } = options; // Ensure output directory exists if (!fs.existsSync(outputDir)) { @@ -151,7 +169,7 @@ export async function downloadVideo(url, options = {}) { '--dump-single-json', '--no-download', '--no-warnings', - ]); + ], { cookiesPath }); const title = sanitizeFilename(info.title); const outputPath = path.join(outputDir, `${title}.mp3`); @@ -171,7 +189,7 @@ export async function downloadVideo(url, options = {}) { title: info.title, }); } - }); + }, { cookiesPath }); return { success: true, @@ -189,7 +207,7 @@ export async function downloadVideo(url, options = {}) { * Download all videos from a playlist as MP3 */ export async function downloadPlaylist(url, options = {}) { - const { outputDir = OUTPUT_DIR, onProgress, onVideoComplete, onDownloadProgress, forcePlaylist = false } = options; + const { outputDir = OUTPUT_DIR, onProgress, onVideoComplete, onDownloadProgress, forcePlaylist = false, cookiesPath } = options; // Ensure output directory exists if (!fs.existsSync(outputDir)) { @@ -198,7 +216,7 @@ export async function downloadPlaylist(url, options = {}) { try { // Get playlist info (force playlist mode if URL has list= param) - const info = await getInfo(url, forcePlaylist || hasPlaylistParam(url)); + const info = await getInfo(url, forcePlaylist || hasPlaylistParam(url), { cookiesPath }); if (info._type !== 'playlist') { // Single video, redirect to downloadVideo @@ -237,7 +255,7 @@ export async function downloadPlaylist(url, options = {}) { }); } : undefined; - const result = await downloadVideo(videoUrl, { outputDir, onDownloadProgress: wrappedProgress }); + const result = await downloadVideo(videoUrl, { outputDir, onDownloadProgress: wrappedProgress, cookiesPath }); results.push(result); if (onVideoComplete) { @@ -271,9 +289,10 @@ export async function downloadPlaylist(url, options = {}) { * Smart download - detects if URL is video or playlist */ export async function download(url, options = {}) { + const { cookiesPath } = options; // If URL contains list= parameter, treat it as a playlist const isPlaylistUrl = hasPlaylistParam(url); - const info = await getInfo(url, isPlaylistUrl); + const info = await getInfo(url, isPlaylistUrl, { cookiesPath }); if (info._type === 'playlist') { return downloadPlaylist(url, { ...options, forcePlaylist: true });