dockify
This commit is contained in:
64
.dockerignore
Normal file
64
.dockerignore
Normal file
@@ -0,0 +1,64 @@
|
||||
# Dependencies
|
||||
node_modules
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
# Build outputs
|
||||
dist
|
||||
.next
|
||||
out
|
||||
build
|
||||
|
||||
# Development files
|
||||
.env
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
*.log
|
||||
logs
|
||||
server.log
|
||||
server.pid
|
||||
|
||||
# Version control
|
||||
.git
|
||||
.gitignore
|
||||
|
||||
# IDE
|
||||
.vscode
|
||||
.idea
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS generated files
|
||||
.DS_Store
|
||||
.DS_Store?
|
||||
._*
|
||||
.Spotlight-V100
|
||||
.Trashes
|
||||
ehthumbs.db
|
||||
Thumbs.db
|
||||
|
||||
# Testing
|
||||
coverage
|
||||
.nyc_output
|
||||
|
||||
# Temporary files
|
||||
tmp
|
||||
temp
|
||||
.tmp
|
||||
|
||||
# Documentation
|
||||
*.md
|
||||
docs
|
||||
|
||||
# Docker files
|
||||
Dockerfile*
|
||||
docker-compose*
|
||||
.dockerignore
|
||||
|
||||
# Misc
|
||||
.cache
|
||||
.parcel-cache
|
||||
13
.env.docker
Normal file
13
.env.docker
Normal file
@@ -0,0 +1,13 @@
|
||||
# Domain configuration
|
||||
DOMAIN=localhost
|
||||
|
||||
# SSL/TLS configuration for Let's Encrypt
|
||||
ACME_EMAIL=admin@example.com
|
||||
|
||||
# Traefik dashboard authentication (admin:password)
|
||||
# Generated with: htpasswd -nb admin password
|
||||
TRAEFIK_AUTH=admin:$$2y$$10$$8qCUOc.FKLB8o4X8ZGVb7OU4xrslBUjOdBPtRz9wM7YJ9.XsGVzui
|
||||
|
||||
# Application environment
|
||||
NODE_ENV=production
|
||||
PORT=3000
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -44,3 +44,4 @@ Thumbs.db
|
||||
dist/
|
||||
build/
|
||||
!public
|
||||
!.serena/**/*
|
||||
|
||||
Binary file not shown.
53
.serena/memories/alternative_music_sources_analysis.md
Normal file
53
.serena/memories/alternative_music_sources_analysis.md
Normal file
@@ -0,0 +1,53 @@
|
||||
# Alternative Music Sources Analysis for Telegram MiniApp
|
||||
|
||||
## YouTube Status: BLOCKED
|
||||
- All anonymous methods blocked by bot detection
|
||||
- Requires authentication/cookies (not suitable for MiniApp)
|
||||
|
||||
## SoundCloud Status: LIMITED/BROKEN
|
||||
- User mentioned "больше не работает"
|
||||
- SoundCloud has restricted API access
|
||||
- Most third-party tools broken in 2024
|
||||
|
||||
## Viable Alternatives Analysis:
|
||||
|
||||
### 1. Spotify (via spotify-web-api-node)
|
||||
- **Pros**: Huge library, stable API, good search
|
||||
- **Cons**: 30-second previews only, no full tracks
|
||||
- **Use case**: Preview/discovery only
|
||||
- **API Status**: Active, requires client credentials
|
||||
|
||||
### 2. Bandcamp
|
||||
- **Pros**: Artists upload full tracks, no restrictions on many songs
|
||||
- **Cons**: Smaller library, mostly indie artists
|
||||
- **Use case**: Full downloads of free/pay-what-you-want tracks
|
||||
- **API Status**: Limited but functional
|
||||
|
||||
### 3. Archive.org (Internet Archive)
|
||||
- **Pros**: Large collection, no restrictions, full downloads
|
||||
- **Cons**: Older/obscure content, variable quality
|
||||
- **Use case**: Classic/rare music, full downloads
|
||||
- **API Status**: Stable, no authentication needed
|
||||
|
||||
### 4. Jamendo
|
||||
- **Pros**: Creative Commons music, full downloads
|
||||
- **Cons**: Smaller library, mostly unknown artists
|
||||
- **Use case**: Background music, royalty-free content
|
||||
- **API Status**: Active, simple REST API
|
||||
|
||||
### 5. Free Music Archive (FMA)
|
||||
- **Pros**: Curated free music, good quality
|
||||
- **Cons**: Limited library
|
||||
- **Use case**: Legal free music downloads
|
||||
- **API Status**: Available but limited
|
||||
|
||||
## Recommendation for MiniApp:
|
||||
1. **Primary**: Archive.org - largest free collection
|
||||
2. **Secondary**: Jamendo - for newer CC music
|
||||
3. **Tertiary**: Bandcamp - for indie discoveries
|
||||
4. **Fallback**: Show YouTube links with instructions
|
||||
|
||||
## Implementation Priority:
|
||||
1. Start with Archive.org (easiest, largest)
|
||||
2. Add graceful error handling for YouTube
|
||||
3. Consider Jamendo as second source
|
||||
47
.serena/memories/code_style_and_conventions.md
Normal file
47
.serena/memories/code_style_and_conventions.md
Normal file
@@ -0,0 +1,47 @@
|
||||
# Code Style and Conventions
|
||||
|
||||
## General Style
|
||||
- **Language**: JavaScript (Node.js backend, vanilla frontend)
|
||||
- **Module System**: CommonJS (require/module.exports)
|
||||
- **Indentation**: 4 spaces (based on observed code)
|
||||
- **Semicolons**: Used consistently
|
||||
- **Quotes**: Single quotes for strings
|
||||
- **Line Endings**: LF (Unix-style)
|
||||
|
||||
## Naming Conventions
|
||||
- **Variables**: camelCase (e.g., `downloadsDir`, `YouTubeService`)
|
||||
- **Constants**: camelCase (no SCREAMING_SNAKE_CASE observed)
|
||||
- **Classes**: PascalCase (e.g., `Database`, `YouTubeService`)
|
||||
- **Files**: kebab-case for some, camelCase for others (mixed)
|
||||
- **Directories**: lowercase
|
||||
|
||||
## Code Organization
|
||||
- **Classes**: Each service has its own class (Database, YouTubeService)
|
||||
- **Modules**: One class per file
|
||||
- **Error Handling**: Try-catch blocks for async operations
|
||||
- **Async/Await**: Preferred over promises .then()
|
||||
- **Express Routes**: Inline callback functions
|
||||
|
||||
## File Structure Patterns
|
||||
- **Imports**: All requires at the top of file
|
||||
- **Constants**: Defined after imports
|
||||
- **Class Instantiation**: After middleware setup
|
||||
- **Routes**: Defined after setup and initialization
|
||||
|
||||
## Database Conventions
|
||||
- **SQLite**: Used for data persistence
|
||||
- **Tables**: snake_case naming (users, search_history, downloads)
|
||||
- **Class Wrapper**: Database operations wrapped in Database class
|
||||
|
||||
## API Conventions
|
||||
- **RESTful**: Standard HTTP methods and status codes
|
||||
- **JSON**: All API responses in JSON format
|
||||
- **Error Responses**: Consistent error object structure
|
||||
- **Async Handlers**: All route handlers are async functions
|
||||
|
||||
## No Testing/Linting Framework
|
||||
The project currently has no configured:
|
||||
- Testing framework (no test scripts)
|
||||
- ESLint or other linting tools
|
||||
- Prettier or other formatting tools
|
||||
- TypeScript (pure JavaScript)
|
||||
20
.serena/memories/progress_note_youtube_fix.md
Normal file
20
.serena/memories/progress_note_youtube_fix.md
Normal file
@@ -0,0 +1,20 @@
|
||||
# YouTube Bot Detection Fix - Progress Note
|
||||
|
||||
## Problem Identified
|
||||
YouTube was blocking requests with "Sign in to confirm you're not a bot" error when trying to download audio streams.
|
||||
|
||||
## Solutions Implemented
|
||||
|
||||
1. **Installed yt-dlp system dependency** - More reliable YouTube downloader that better handles bot detection
|
||||
2. **Updated YouTubeService** - Added fallback mechanism:
|
||||
- First tries play-dl (existing method)
|
||||
- If that fails, falls back to yt-dlp system command
|
||||
- yt-dlp gets direct audio URL then creates stream via axios
|
||||
|
||||
## Code Changes
|
||||
- Added `spawn` import for child process execution
|
||||
- Modified `getAudioStream()` to try both methods
|
||||
- Added new `getAudioStreamWithYtDlp()` method as fallback
|
||||
|
||||
## Next Steps
|
||||
User should restart server to test the fix. If still issues, they can set up YouTube cookies as documented in YOUTUBE_SETUP.md.
|
||||
36
.serena/memories/project_overview.md
Normal file
36
.serena/memories/project_overview.md
Normal file
@@ -0,0 +1,36 @@
|
||||
# Quixotic Project Overview
|
||||
|
||||
## Purpose
|
||||
Quixotic is a Telegram miniapp for YouTube music search and MP3 conversion. It allows users to search for videos on YouTube and convert them to MP3 format through a Telegram Web App interface.
|
||||
|
||||
## Key Features
|
||||
- 🔍 YouTube video search
|
||||
- 🎵 MP3 conversion using FFmpeg
|
||||
- 📱 Telegram Web App interface
|
||||
- 💾 SQLite database for storage
|
||||
- 📊 Search history tracking
|
||||
- 🤖 Telegram Bot integration
|
||||
|
||||
## Tech Stack
|
||||
- **Backend**: Node.js with Express.js server
|
||||
- **Database**: SQLite with sqlite3 package
|
||||
- **YouTube API**: play-dl package for YouTube video handling
|
||||
- **Audio Processing**: fluent-ffmpeg for MP3 conversion
|
||||
- **Telegram**: node-telegram-bot-api for bot functionality
|
||||
- **HTTP Client**: axios for API requests
|
||||
- **Frontend**: Vanilla HTML/CSS/JavaScript (no framework)
|
||||
|
||||
## Main Dependencies
|
||||
- express: ^4.18.2
|
||||
- sqlite3: ^5.1.6
|
||||
- fluent-ffmpeg: ^2.1.2
|
||||
- play-dl: ^1.9.7
|
||||
- node-telegram-bot-api: ^0.64.0
|
||||
- axios: ^1.6.2
|
||||
|
||||
## Development Dependencies
|
||||
- nodemon: ^3.0.2 (for development server)
|
||||
|
||||
## Node.js Requirements
|
||||
- Node.js >= 16.0.0
|
||||
- Package manager: Yarn 1.22.19
|
||||
48
.serena/memories/project_structure.md
Normal file
48
.serena/memories/project_structure.md
Normal file
@@ -0,0 +1,48 @@
|
||||
# Quixotic Project Structure
|
||||
|
||||
## Directory Layout
|
||||
```
|
||||
quixotic/
|
||||
├── src/ # Backend source code
|
||||
│ ├── server.js # Main Express server (entry point)
|
||||
│ ├── bot.js # Telegram bot logic
|
||||
│ ├── youtube.js # YouTube service (YouTubeService class)
|
||||
│ └── database.js # SQLite database wrapper (Database class)
|
||||
├── public/ # Static frontend assets
|
||||
│ ├── index.html # Main Web App interface
|
||||
│ ├── style.css # Styles for the Web App
|
||||
│ └── script.js # Client-side JavaScript
|
||||
├── database/ # SQLite database files
|
||||
├── downloads/ # Generated MP3 files (created at runtime)
|
||||
├── .serena/ # Serena configuration
|
||||
├── .claude/ # Claude Code configuration
|
||||
├── package.json # Project configuration and dependencies
|
||||
├── yarn.lock # Yarn lockfile
|
||||
├── .env.example # Environment variables template
|
||||
├── .gitignore # Git ignore rules
|
||||
├── README.md # Project documentation
|
||||
├── WORKLOG.md # Development log
|
||||
├── YOUTUBE_SETUP.md # YouTube setup instructions
|
||||
└── .mcp.json # MCP configuration
|
||||
```
|
||||
|
||||
## Key Files and Their Roles
|
||||
|
||||
### Backend (src/)
|
||||
- **server.js**: Main Express server with API endpoints:
|
||||
- `POST /api/search` - Search YouTube videos
|
||||
- `POST /api/convert` - Convert video to MP3
|
||||
- `GET /downloads/:filename` - Serve MP3 files
|
||||
- `GET /health` - Health check endpoint
|
||||
|
||||
- **youtube.js**: YouTube service handling video search and metadata
|
||||
- **database.js**: SQLite database wrapper with tables for users, search_history, downloads
|
||||
- **bot.js**: Telegram bot integration
|
||||
|
||||
### Frontend (public/)
|
||||
- **index.html**: Telegram Web App interface
|
||||
- **script.js**: Client-side logic for search and conversion
|
||||
- **style.css**: Web App styling
|
||||
|
||||
## API Architecture
|
||||
RESTful API with Express.js serving both the Web App and API endpoints. The app uses SQLite for persistent storage and FFmpeg for audio processing.
|
||||
46
.serena/memories/soundcloud_failure_final_summary.md
Normal file
46
.serena/memories/soundcloud_failure_final_summary.md
Normal file
@@ -0,0 +1,46 @@
|
||||
# SoundCloud Integration Failure - Final Summary
|
||||
|
||||
## Date: August 27, 2025
|
||||
|
||||
## Problem
|
||||
SoundCloud integration also failed - both for track resolution and download attempts.
|
||||
|
||||
## Error Details
|
||||
- 404 Not Found errors from SoundCloud API
|
||||
- Client ID appears to be working but tracks not found
|
||||
- Both direct ID and URL-based approaches failed
|
||||
- Error message: "could not find the song... it may be private - check the URL"
|
||||
|
||||
## Root Issue
|
||||
YouTube video ID `4JkIs37a2JE` is not a SoundCloud track ID. The frontend is still passing YouTube video IDs to the backend, but the backend now expects SoundCloud track IDs.
|
||||
|
||||
## What Was Attempted
|
||||
1. **YouTube Integration** - Completely abandoned due to bot detection
|
||||
- Removed: play-dl, ytdl-core, youtube-dl-exec
|
||||
- All anonymous methods blocked by YouTube
|
||||
|
||||
2. **SoundCloud Integration** - Failed due to ID mismatch
|
||||
- Installed: soundcloud-downloader
|
||||
- Created: SoundCloudService class
|
||||
- Updated: server.js to use SoundCloud
|
||||
|
||||
## Critical Realization
|
||||
**The fundamental problem**: Frontend still searches YouTube and sends YouTube IDs, but backend expects SoundCloud IDs. This is an architectural mismatch.
|
||||
|
||||
## Required Next Steps (Not Implemented)
|
||||
1. **Frontend Update Required**: Update search to use SoundCloud instead of YouTube
|
||||
2. **API Consistency**: Ensure frontend and backend use same service
|
||||
3. **Alternative Approach**: Consider hybrid approach or different strategy
|
||||
|
||||
## Current State
|
||||
- Backend: SoundCloud-ready but receives wrong IDs
|
||||
- Frontend: Still YouTube-based
|
||||
- System: Completely broken due to service mismatch
|
||||
|
||||
## Final Recommendation
|
||||
Either:
|
||||
1. Update frontend to search SoundCloud, or
|
||||
2. Revert to YouTube with better bot evasion techniques, or
|
||||
3. Consider completely different approach (local uploads, different platforms, etc.)
|
||||
|
||||
The project needs architectural decision before continuing.
|
||||
52
.serena/memories/suggested_commands.md
Normal file
52
.serena/memories/suggested_commands.md
Normal file
@@ -0,0 +1,52 @@
|
||||
# Suggested Commands for Quixotic Development
|
||||
|
||||
## Development Commands
|
||||
- `yarn install` - Install project dependencies
|
||||
- `yarn dev` - Start development server with nodemon (auto-restart)
|
||||
- `yarn start` - Start production server
|
||||
|
||||
## System Requirements
|
||||
- **FFmpeg Installation**:
|
||||
- macOS: `brew install ffmpeg`
|
||||
- Ubuntu/Debian: `sudo apt install ffmpeg`
|
||||
- Windows: Download from ffmpeg.org
|
||||
|
||||
## Environment Setup
|
||||
- `cp .env.example .env` - Create environment configuration file
|
||||
- Edit `.env` file with required values:
|
||||
- `TELEGRAM_BOT_TOKEN=your_bot_token_here`
|
||||
- `WEB_APP_URL=https://your-domain.com`
|
||||
- `PORT=3000`
|
||||
|
||||
## Git Commands (Darwin System)
|
||||
- `git status` - Check repository status
|
||||
- `git add .` - Stage all changes
|
||||
- `git commit -m "message"` - Commit changes
|
||||
- `git push` - Push to remote repository
|
||||
|
||||
## File System Commands (macOS/Darwin)
|
||||
- `ls -la` - List files with details
|
||||
- `find . -name "*.js"` - Find JavaScript files
|
||||
- `grep -r "text" .` - Search for text in files
|
||||
- `cd directory` - Change directory
|
||||
- `mkdir directory` - Create directory
|
||||
- `rm -rf directory` - Remove directory recursively
|
||||
|
||||
## Database Management
|
||||
- SQLite database files are in `database/` directory
|
||||
- No dedicated database management commands configured
|
||||
- Database is automatically initialized on first run
|
||||
|
||||
## Process Management (Production)
|
||||
- `pm2 start src/server.js --name quixotic` - Start with PM2
|
||||
- `pm2 logs quixotic` - View logs
|
||||
- `pm2 restart quixotic` - Restart application
|
||||
- `pm2 stop quixotic` - Stop application
|
||||
|
||||
## Health Check
|
||||
- Visit `http://localhost:3000/health` - Check if server is running
|
||||
- Check server logs for errors
|
||||
|
||||
## Port Information
|
||||
- Default port: 3000 (configurable via PORT environment variable)
|
||||
- Development server runs on same port as production
|
||||
57
.serena/memories/task_completion_checklist.md
Normal file
57
.serena/memories/task_completion_checklist.md
Normal file
@@ -0,0 +1,57 @@
|
||||
# Task Completion Checklist
|
||||
|
||||
## When a Development Task is Completed
|
||||
|
||||
### Code Quality Checks
|
||||
⚠️ **Note**: This project currently has NO configured linting, formatting, or testing tools.
|
||||
|
||||
**Recommended Actions**:
|
||||
1. **Manual Code Review**: Carefully review code changes for:
|
||||
- Syntax errors
|
||||
- Logic errors
|
||||
- Consistent code style
|
||||
- Proper error handling
|
||||
- Security considerations (no hardcoded secrets)
|
||||
|
||||
2. **Manual Testing**:
|
||||
- Start the development server: `yarn dev`
|
||||
- Test the affected functionality manually
|
||||
- Check API endpoints with tools like Postman or curl
|
||||
- Verify Web App functionality in browser
|
||||
- Test Telegram bot integration if applicable
|
||||
|
||||
### Environment Verification
|
||||
1. **Dependencies**: Ensure all required packages are installed (`yarn install`)
|
||||
2. **Environment Variables**: Verify `.env` file is properly configured
|
||||
3. **External Dependencies**: Ensure FFmpeg is installed and accessible
|
||||
4. **Database**: Check that SQLite database initializes correctly
|
||||
|
||||
### Testing Checklist
|
||||
Since no automated testing exists, manually verify:
|
||||
- [ ] Server starts without errors
|
||||
- [ ] API endpoints respond correctly
|
||||
- [ ] YouTube search functionality works
|
||||
- [ ] MP3 conversion process completes
|
||||
- [ ] File downloads work properly
|
||||
- [ ] Database operations succeed
|
||||
- [ ] Telegram Web App loads correctly
|
||||
|
||||
### Deployment Preparation
|
||||
- [ ] Environment variables are set correctly
|
||||
- [ ] All dependencies are in package.json
|
||||
- [ ] No hardcoded secrets in code
|
||||
- [ ] Downloads directory is created properly
|
||||
- [ ] FFmpeg is available in production environment
|
||||
|
||||
### Git Workflow
|
||||
- [ ] Review all changes before committing
|
||||
- [ ] Use meaningful commit messages
|
||||
- [ ] Ensure no sensitive data is committed
|
||||
- [ ] Check .gitignore includes necessary exclusions
|
||||
|
||||
### Future Improvements
|
||||
Consider adding in future iterations:
|
||||
- ESLint for code linting
|
||||
- Prettier for code formatting
|
||||
- Jest or Mocha for testing
|
||||
- TypeScript for type safety
|
||||
18
.serena/memories/telegram_app_constraints.md
Normal file
18
.serena/memories/telegram_app_constraints.md
Normal file
@@ -0,0 +1,18 @@
|
||||
# Telegram MiniApp Development Constraints
|
||||
|
||||
## Important: No Cookies or User Authentication
|
||||
- This is a Telegram MiniApp, not a personal application
|
||||
- NEVER use cookies-based authentication with external services like YouTube
|
||||
- Users should not be required to login or authenticate with external services
|
||||
- Keep all functionality anonymous and public
|
||||
|
||||
## YouTube Integration Constraints
|
||||
- Use only anonymous/public access methods
|
||||
- Do not require users to provide their own credentials
|
||||
- Focus on simple, cookie-free download methods
|
||||
- Prioritize methods that don't need user authentication
|
||||
|
||||
## General MiniApp Principles
|
||||
- Keep it simple and accessible
|
||||
- No complex setup or configuration required
|
||||
- Should work for any Telegram user without additional steps
|
||||
36
.serena/memories/traefik-docker-setup.md
Normal file
36
.serena/memories/traefik-docker-setup.md
Normal file
@@ -0,0 +1,36 @@
|
||||
# Traefik Docker Setup for Quixotic
|
||||
|
||||
## Files Created
|
||||
- `docker-compose.yml` - Main orchestration with Traefik reverse proxy
|
||||
- `Dockerfile` - Multi-stage build for Node.js app with ffmpeg
|
||||
- `.env.docker` - Environment variables for production
|
||||
- `traefik.yml` - Static Traefik configuration
|
||||
- `.dockerignore` - Docker build exclusions
|
||||
|
||||
## Key Features
|
||||
- Traefik v3.0 reverse proxy with automatic HTTPS (Let's Encrypt)
|
||||
- HTTP to HTTPS redirect
|
||||
- Traefik dashboard on port 8080 with basic auth
|
||||
- Persistent volumes for SSL certs and downloads
|
||||
- Health checks for the application
|
||||
- Multi-stage Docker build for optimization
|
||||
|
||||
## Commands Added to package.json
|
||||
- `docker:up` - Start with Traefik
|
||||
- `docker:down` - Stop containers
|
||||
- `docker:logs` - View logs
|
||||
- `docker:rebuild` - Full rebuild
|
||||
|
||||
## Current Issues
|
||||
- ffmpeg installation in Alpine causing exit code 255
|
||||
- Fixed by using `apk add --no-cache ffmpeg` with version check
|
||||
- May need to debug ffmpeg stream handling in container environment
|
||||
|
||||
## Usage
|
||||
```bash
|
||||
docker-compose --env-file .env.docker up -d
|
||||
```
|
||||
|
||||
Access:
|
||||
- App: https://localhost (or configured domain)
|
||||
- Traefik dashboard: http://localhost:8080
|
||||
54
.serena/memories/typescript_migration_complete.md
Normal file
54
.serena/memories/typescript_migration_complete.md
Normal file
@@ -0,0 +1,54 @@
|
||||
# TypeScript Migration Complete
|
||||
|
||||
## Summary
|
||||
Successfully migrated the Quixotic project from JavaScript to TypeScript.
|
||||
|
||||
## Changes Made
|
||||
|
||||
### 1. Dependencies Added
|
||||
- `typescript` - TypeScript compiler
|
||||
- `@types/node` - Node.js type definitions
|
||||
- `@types/express` - Express.js type definitions
|
||||
- `@types/fluent-ffmpeg` - FFmpeg wrapper type definitions
|
||||
- `@types/node-telegram-bot-api` - Telegram Bot API type definitions
|
||||
- `ts-node` - TypeScript execution engine
|
||||
|
||||
### 2. Configuration Files
|
||||
- `tsconfig.json` - Backend TypeScript configuration
|
||||
- `tsconfig.frontend.json` - Frontend TypeScript configuration
|
||||
|
||||
### 3. Files Converted
|
||||
|
||||
#### Backend (src/)
|
||||
- `server.js` → `server.ts` - Main Express server with proper typing
|
||||
- `database.js` → `database.ts` - Database class with interfaces
|
||||
- `soundcloud.js` → `soundcloud.ts` - SoundCloud service with types
|
||||
- `bot.js` → `bot.ts` - Telegram bot with proper interfaces
|
||||
|
||||
#### Frontend (public/)
|
||||
- `script.js` → `script.ts` - Frontend app with TypeScript interfaces
|
||||
- Updated `index.html` to reference compiled JS file
|
||||
|
||||
### 4. Package.json Updates
|
||||
- Updated main entry point to `dist/server.js`
|
||||
- Added build scripts for both backend and frontend
|
||||
- Updated dev script to use `ts-node`
|
||||
- Modified validation scripts
|
||||
|
||||
### 5. Type Safety Improvements
|
||||
- Added interfaces for API responses
|
||||
- Typed Telegram WebApp integration
|
||||
- Proper error handling with typed exceptions
|
||||
- Database query result typing
|
||||
|
||||
## Build Process
|
||||
- `npm run build` - Compiles both backend and frontend
|
||||
- `npm run dev` - Runs development server with ts-node
|
||||
- `npm run start` - Runs compiled JavaScript in production
|
||||
|
||||
## Files Removed
|
||||
All original JavaScript files were removed after successful conversion:
|
||||
- src/server.js, database.js, soundcloud.js, bot.js
|
||||
- public/script.js
|
||||
|
||||
The project now has full TypeScript support with proper type checking and IntelliSense.
|
||||
28
.serena/memories/youtube_abandonment_decision.md
Normal file
28
.serena/memories/youtube_abandonment_decision.md
Normal file
@@ -0,0 +1,28 @@
|
||||
# YouTube Abandonment Decision - August 27, 2025
|
||||
|
||||
## Decision Made
|
||||
**Abandon YouTube integration completely** and switch to SoundCloud for the Quixotic Telegram MiniApp.
|
||||
|
||||
## Reasoning
|
||||
1. **YouTube bot detection too strong** - all anonymous methods blocked
|
||||
2. **Cookie-based auth inappropriate** for Telegram MiniApp (should be anonymous)
|
||||
3. **User needs mainstream music** like "Virtual Insanity" - Archive.org won't have it
|
||||
4. **SoundCloud still viable option** for popular tracks
|
||||
|
||||
## Actions Taken
|
||||
- Remove all YouTube-related code from src/youtube.js
|
||||
- Remove YouTube dependencies (play-dl, ytdl-core, youtube-dl-exec)
|
||||
- Replace with SoundCloud integration
|
||||
- Keep similar API interface for frontend compatibility
|
||||
|
||||
## SoundCloud Implementation Plan
|
||||
1. Use scdl-core or soundcloud-downloader npm packages
|
||||
2. Search functionality via SoundCloud API
|
||||
3. Download functionality for available tracks
|
||||
4. Keep same UI/UX, just change backend source
|
||||
|
||||
## Fallback Strategy
|
||||
If SoundCloud also fails, consider:
|
||||
1. Multiple source aggregation
|
||||
2. User-provided links approach
|
||||
3. Different content strategy (playlists, podcasts, etc.)
|
||||
35
.serena/memories/youtube_bot_detection_analysis.md
Normal file
35
.serena/memories/youtube_bot_detection_analysis.md
Normal file
@@ -0,0 +1,35 @@
|
||||
# YouTube Bot Detection Analysis
|
||||
|
||||
## Current Situation
|
||||
YouTube has significantly strengthened bot detection in 2024-2025:
|
||||
- All anonymous methods are being blocked
|
||||
- Status code 410 indicates content unavailable/blocked
|
||||
- Even different libraries (play-dl, ytdl-core, yt-dlp) are detected
|
||||
|
||||
## Possible Solutions
|
||||
|
||||
### Option 1: Use proxy/VPN rotation
|
||||
- Rotate IP addresses
|
||||
- Use residential proxies
|
||||
- Complex but may work temporarily
|
||||
|
||||
### Option 2: Alternative video sources
|
||||
- Switch to other video platforms
|
||||
- Use YouTube alternatives
|
||||
- Different API endpoints
|
||||
|
||||
### Option 3: Web scraping approach
|
||||
- Use headless browser (Puppeteer)
|
||||
- Simulate real user behavior
|
||||
- More complex but harder to detect
|
||||
|
||||
### Option 4: Accept limitation
|
||||
- Show error message to users
|
||||
- Suggest users download manually
|
||||
- Focus on other features
|
||||
|
||||
## Recommendation for Telegram MiniApp
|
||||
For a Telegram MiniApp, the most viable approaches are:
|
||||
1. Alternative video sources (SoundCloud, etc.)
|
||||
2. Graceful error handling with user instructions
|
||||
3. Consider pivot to different content source
|
||||
59
Dockerfile
Normal file
59
Dockerfile
Normal file
@@ -0,0 +1,59 @@
|
||||
# Build stage
|
||||
FROM node:18-alpine AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy package files
|
||||
COPY package*.json ./
|
||||
COPY yarn.lock* ./
|
||||
|
||||
# Install all dependencies (including dev for build)
|
||||
RUN npm install && npm cache clean --force
|
||||
|
||||
# Copy source code
|
||||
COPY . .
|
||||
|
||||
# Build the application
|
||||
RUN npm run build
|
||||
|
||||
# Clean dev dependencies
|
||||
RUN npm prune --production
|
||||
|
||||
# Production stage
|
||||
FROM node:18-alpine AS production
|
||||
|
||||
# Install ffmpeg from Alpine packages (architecture-aware)
|
||||
RUN apk update && apk add --no-cache ffmpeg
|
||||
|
||||
# Set ffmpeg paths
|
||||
ENV FFMPEG_PATH=/usr/bin/ffmpeg
|
||||
ENV FFPROBE_PATH=/usr/bin/ffprobe
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Copy built application and dependencies
|
||||
COPY --from=builder /app/dist ./dist
|
||||
COPY --from=builder /app/node_modules ./node_modules
|
||||
COPY --from=builder /app/public ./public
|
||||
COPY --from=builder /app/package*.json ./
|
||||
|
||||
# Create necessary directories
|
||||
RUN mkdir -p downloads database
|
||||
|
||||
# Create non-root user
|
||||
RUN addgroup -g 1001 -S nodejs
|
||||
RUN adduser -S quixotic -u 1001
|
||||
|
||||
# Change ownership of app directory
|
||||
RUN chown -R quixotic:nodejs /app
|
||||
USER quixotic
|
||||
|
||||
# Expose port
|
||||
EXPOSE 3000
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD node -e "require('http').get('http://localhost:3000/health', (res) => { process.exit(res.statusCode === 200 ? 0 : 1) })"
|
||||
|
||||
# Start the application
|
||||
CMD ["node", "dist/server.js"]
|
||||
72
docker-compose.yml
Normal file
72
docker-compose.yml
Normal file
@@ -0,0 +1,72 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
# Traefik reverse proxy
|
||||
traefik:
|
||||
image: traefik:v3.0
|
||||
container_name: quixotic-traefik
|
||||
restart: unless-stopped
|
||||
command:
|
||||
- --api.dashboard=true
|
||||
- --api.insecure=true
|
||||
- --providers.docker=true
|
||||
- --providers.docker.exposedbydefault=false
|
||||
- --entrypoints.web.address=:80
|
||||
- --entrypoints.websecure.address=:443
|
||||
- --certificatesresolvers.myresolver.acme.tlschallenge=true
|
||||
- --certificatesresolvers.myresolver.acme.email=${ACME_EMAIL:-admin@example.com}
|
||||
- --certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json
|
||||
- --log.level=INFO
|
||||
ports:
|
||||
- "80:80"
|
||||
- "443:443"
|
||||
- "8080:8080" # Traefik dashboard
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
||||
- traefik-ssl-certs:/letsencrypt
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.http.routers.traefik.rule=Host(`traefik.${DOMAIN:-localhost}`)"
|
||||
- "traefik.http.routers.traefik.service=api@internal"
|
||||
- "traefik.http.routers.traefik.middlewares=auth"
|
||||
- "traefik.http.middlewares.auth.basicauth.users=${TRAEFIK_AUTH:-admin:$$2y$$10$$8qCUOc.FKLB8o4X8ZGVb7OU4xrslBUjOdBPtRz9wM7YJ9.XsGVzui}" # admin:password
|
||||
networks:
|
||||
- quixotic
|
||||
|
||||
# Main application
|
||||
quixotic-app:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: quixotic-app
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
- PORT=3000
|
||||
volumes:
|
||||
- downloads:/app/downloads
|
||||
- ./database:/app/database
|
||||
labels:
|
||||
- "traefik.enable=true"
|
||||
- "traefik.http.routers.quixotic.rule=Host(`${DOMAIN:-localhost}`)"
|
||||
- "traefik.http.routers.quixotic.entrypoints=websecure"
|
||||
- "traefik.http.routers.quixotic.tls.certresolver=myresolver"
|
||||
- "traefik.http.routers.quixotic.service=quixotic"
|
||||
- "traefik.http.services.quixotic.loadbalancer.server.port=3000"
|
||||
# HTTP to HTTPS redirect
|
||||
- "traefik.http.routers.quixotic-http.rule=Host(`${DOMAIN:-localhost}`)"
|
||||
- "traefik.http.routers.quixotic-http.entrypoints=web"
|
||||
- "traefik.http.routers.quixotic-http.middlewares=redirect-to-https"
|
||||
- "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"
|
||||
depends_on:
|
||||
- traefik
|
||||
networks:
|
||||
- quixotic
|
||||
|
||||
volumes:
|
||||
traefik-ssl-certs:
|
||||
downloads:
|
||||
|
||||
networks:
|
||||
quixotic:
|
||||
driver: bridge
|
||||
@@ -16,7 +16,14 @@
|
||||
"lint": "eslint src/ public/ --ext .ts,.js",
|
||||
"lint:fix": "eslint src/ public/ --ext .ts,.js --fix",
|
||||
"validate": "npm run lint && npm run build && echo '✅ All checks passed!'",
|
||||
"pretest": "npm run validate"
|
||||
"pretest": "npm run validate",
|
||||
"docker:build": "docker-compose build",
|
||||
"docker:up": "docker-compose --env-file .env.docker up -d",
|
||||
"docker:down": "docker-compose down",
|
||||
"docker:logs": "docker-compose logs -f",
|
||||
"docker:restart": "docker-compose restart",
|
||||
"docker:rebuild": "docker-compose down && docker-compose build --no-cache && docker-compose --env-file .env.docker up -d",
|
||||
"docker:dev": "docker-compose --env-file .env.docker up --build"
|
||||
},
|
||||
"packageManager": "yarn@1.22.19",
|
||||
"dependencies": {
|
||||
|
||||
@@ -2,6 +2,10 @@ import express, { Request, Response, NextFunction } from 'express';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import ffmpeg from 'fluent-ffmpeg';
|
||||
|
||||
// Configure ffmpeg paths
|
||||
ffmpeg.setFfmpegPath('/usr/bin/ffmpeg');
|
||||
ffmpeg.setFfprobePath('/usr/bin/ffprobe');
|
||||
import { Database } from './database';
|
||||
import { SoundCloudService } from './soundcloud';
|
||||
|
||||
@@ -87,9 +91,36 @@ app.post('/api/convert', async (req: Request, res: Response) => {
|
||||
const audioStream = await soundcloud.getAudioStream(videoId, url);
|
||||
console.log('Audio stream obtained, starting FFmpeg conversion...');
|
||||
|
||||
// Convert to MP3 using ffmpeg
|
||||
// Download to temporary file first, then convert
|
||||
const tempInputPath = path.join(downloadsDir, `temp_${videoId}.tmp`);
|
||||
|
||||
// Save stream to temporary file
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const conversion = ffmpeg(audioStream)
|
||||
const writeStream = fs.createWriteStream(tempInputPath);
|
||||
audioStream.pipe(writeStream);
|
||||
audioStream.on('end', resolve);
|
||||
audioStream.on('error', reject);
|
||||
writeStream.on('error', reject);
|
||||
});
|
||||
|
||||
console.log('Temporary file saved, starting FFmpeg conversion...');
|
||||
|
||||
// Debug: check temp file
|
||||
const stats = fs.statSync(tempInputPath);
|
||||
console.log(`Temp file size: ${stats.size} bytes`);
|
||||
|
||||
// Test ffmpeg with simple command first
|
||||
try {
|
||||
const { execSync } = require('child_process');
|
||||
const result = execSync(`ffmpeg -i "${tempInputPath}" -t 1 -f null -`, { encoding: 'utf8', stdio: 'pipe' });
|
||||
console.log('FFmpeg file test passed');
|
||||
} catch (e: any) {
|
||||
console.error('FFmpeg file test failed:', e.stderr || e.message);
|
||||
}
|
||||
|
||||
// Convert temporary file to MP3 using ffmpeg
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const conversion = ffmpeg(tempInputPath)
|
||||
.audioCodec('libmp3lame')
|
||||
.audioBitrate('192k')
|
||||
.audioChannels(2)
|
||||
@@ -106,10 +137,16 @@ app.post('/api/convert', async (req: Request, res: Response) => {
|
||||
})
|
||||
.on('end', () => {
|
||||
console.log('MP3 conversion completed successfully');
|
||||
// Clean up temporary file
|
||||
fs.unlink(tempInputPath, (err) => {
|
||||
if (err) console.error('Failed to delete temp file:', err);
|
||||
});
|
||||
resolve();
|
||||
})
|
||||
.on('error', (err: Error) => {
|
||||
console.error('FFmpeg error:', err.message);
|
||||
// Clean up temporary file on error
|
||||
fs.unlink(tempInputPath, () => {});
|
||||
reject(err);
|
||||
});
|
||||
|
||||
|
||||
@@ -49,9 +49,6 @@ export class SoundCloudService {
|
||||
resourceType: 'tracks'
|
||||
}) as any;
|
||||
|
||||
console.log('Search result type:', typeof searchResult);
|
||||
console.log('Search result:', searchResult);
|
||||
|
||||
// Handle different response formats
|
||||
let tracks: any[] = [];
|
||||
|
||||
@@ -172,4 +169,4 @@ export class SoundCloudService {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
51
traefik.yml
Normal file
51
traefik.yml
Normal file
@@ -0,0 +1,51 @@
|
||||
# Static configuration file for Traefik
|
||||
global:
|
||||
checkNewVersion: false
|
||||
sendAnonymousUsage: false
|
||||
|
||||
# Entry points configuration
|
||||
entryPoints:
|
||||
web:
|
||||
address: ":80"
|
||||
http:
|
||||
redirections:
|
||||
entryPoint:
|
||||
to: websecure
|
||||
scheme: https
|
||||
permanent: true
|
||||
websecure:
|
||||
address: ":443"
|
||||
|
||||
# API and dashboard configuration
|
||||
api:
|
||||
dashboard: true
|
||||
insecure: true
|
||||
|
||||
# Providers configuration
|
||||
providers:
|
||||
docker:
|
||||
endpoint: "unix:///var/run/docker.sock"
|
||||
exposedByDefault: false
|
||||
network: quixotic
|
||||
|
||||
# Certificate resolvers
|
||||
certificatesResolvers:
|
||||
letsencrypt:
|
||||
acme:
|
||||
email: admin@example.com
|
||||
storage: /letsencrypt/acme.json
|
||||
tlsChallenge: {}
|
||||
|
||||
# Logging
|
||||
log:
|
||||
level: INFO
|
||||
filePath: "/var/log/traefik/traefik.log"
|
||||
|
||||
accessLog:
|
||||
filePath: "/var/log/traefik/access.log"
|
||||
|
||||
# Metrics
|
||||
metrics:
|
||||
prometheus:
|
||||
addEntryPointsLabels: true
|
||||
addServicesLabels: true
|
||||
Reference in New Issue
Block a user